# # This file and its contents are supplied under the terms of the # Common Development and Distribution License ("CDDL"), version 1.0. # You may only use this file in accordance with the terms of version # 1.0 of the CDDL. # # A full copy of the text of the CDDL should have accompanied this # source. A copy of the CDDL is also available via the Internet at # http://www.illumos.org/license/CDDL. # # # Copyright 2024 OmniOS Community Edition (OmniOSce) Association. # include $(SRC)/cmd/Makefile.cmd SUBDIRS = $(MACH64) test all : TARGET = all clean : TARGET = clean clobber : TARGET = clobber install : TARGET = install .KEEP_STATE: all clean clobber install: $(SUBDIRS) $(SUBDIRS): FRC @cd $@; pwd; $(MAKE) $(TARGET) FRC: # # This file and its contents are supplied under the terms of the # Common Development and Distribution License ("CDDL"), version 1.0. # You may only use this file in accordance with the terms of version # 1.0 of the CDDL. # # A full copy of the text of the CDDL should have accompanied this # source. A copy of the CDDL is also available via the Internet at # http://www.illumos.org/license/CDDL. # # # Copyright 2014 Pluribus Networks Inc. # Copyright 2020 Joyent, Inc. # Copyright 2020 Oxide Computer Company # Copyright 2024 OmniOS Community Edition (OmniOSce) Association. # PROG = bhyve include $(SRC)/cmd/Makefile.cmd include $(SRC)/cmd/Makefile.cmd.64 include $(SRC)/cmd/Makefile.ctf COMMON_OBJS = \ acpi.o \ acpi_device.o \ basl.o \ bhyvegc.o \ bhyverun.o \ block_if.o \ bootrom.o \ config.o \ console.o \ crc16.o \ gdb.o \ hexdump.o \ ilstr.o \ iov.o \ mem.o \ mevent.o \ net_backend_dlpi.o \ net_backends.o \ net_utils.o \ pci_ahci.o \ pci_e82545.o \ pci_emul.o \ pci_hostbridge.o \ pci_irq.o \ pci_nvme.o \ pci_uart.o \ pci_virtio_9p.o \ pci_virtio_block.o \ pci_virtio_console.o \ pci_virtio_net.o \ pci_virtio_rnd.o \ pci_virtio_scsi.o \ pci_virtio_scsi_uscsi.o \ pci_virtio_viona.o \ pci_xhci.o \ privileges.o \ qemu_fwcfg.o \ qemu_loader.o \ smbiostbl.o \ sockstream.o \ tpm_device.o \ tpm_emul_passthru.o \ tpm_emul_swtpm.o \ tpm_intf_crb.o \ tpm_ppi_qemu.o \ uart_backend.o \ uart_emul.o \ usb_emul.o \ usb_mouse.o \ virtio.o \ vmgenc.o \ bhyve_sol_glue.o CFLAGS += $(CCVERBOSE) CFLAGS += -Wimplicit-function-declaration CPPFLAGS = -I../common \ -I$(COMPAT)/bhyve -I$(CONTRIB)/bhyve \ -I$(COMPAT)/bhyve/amd64 -I$(CONTRIB)/bhyve/amd64 \ -I$(CONTRIB)/bhyve/dev/usb/controller \ -I$(CONTRIB)/bhyve/dev/mii \ -I$(SRC)/lib/lib9p/common \ -I$(SRC)/uts/common/io/e1000api \ $(CPPFLAGS.master) \ -I$(SRC)/uts/intel/io/vmm \ -I$(SRC)/uts/common \ -I$(SRC)/uts/intel \ -DWITHOUT_CAPSICUM \ -DOPENSSL_API_COMPAT=10101 SMOFF += all_func_returns rfb.o : SMOFF= CSTD= $(CSTD_GNU99) $(PROG) : LDLIBS += \ -l9p \ -lcmdutils \ -lcrypto \ -ldladm \ -ldlpi \ -lidspace \ -lmd \ -lnsl \ -lnvpair \ -lscsi \ -lsocket \ -lumem \ -luuid \ -lvmmapi \ -lz # Hammerhead: Disable PIE - bhyve objects are not compiled with -fPIC # and GNU ld rejects R_X86_64_32 relocations in PIE executables. # GCC 14 defaults to -pie; ZASLR only controlled illumos ld's -zaslr flag. $(PROG) : LDFLAGS += -no-pie # libscsi.so lives at /usr/lib/scsi/ on Hammerhead (flattened, no $(MACH64) # subdir). Bhyve links against -lscsi via pci_nvme; the default -L search # path doesn't include this subdir, so point both the link-time -L and the # runtime -R there. Upstream illumos uses /usr/lib/scsi/$(MACH64). $(PROG) : LDFLAGS += -Wl,-L$(ROOT)/usr/lib/scsi $(PROG) : LDFLAGS += -Wl,-R/usr/lib/scsi OBJS = $(BHYVE_OBJS:%=pics/%) CLEANFILES = $(OBJS) CLOBBERFILES = $(PROG) all: pics $(PROG) clean: $(RM) $(CLEANFILES) clobber: clean $(RM) $(CLOBBERFILES) pics: FRC $(MKDIR) -p $@ pics/%.o: ../common/%.c $(COMPILE.c) $< -o $@ $(POST_PROCESS_O) pics/%.o: %.c $(COMPILE.c) $< -o $@ $(POST_PROCESS_O) pics/%.o: $(SRC)/common/hexdump/%.c $(COMPILE.c) $< -o $@ $(POST_PROCESS_O) pics/%.o: $(SRC)/common/ilstr/%.c $(COMPILE.c) $< -o $@ $(POST_PROCESS_O) $(PROG): pics $(OBJS) $(LINK.c) -o $@ $(OBJS) $(LDFLAGS) $(LDLIBS) $(POST_PROCESS) FRC: Having been ported from FreeBSD, bhyve bears the BSD license. Subsequent changes made to bhyve in illumos are dual-licensed under both the BSD license and the CDDL. Use or redistribution of those subsequent changes may be done under either license. The CDDL license header added to bhyve source files is meant to cover only those dual-licensed modifications, not the entire file. Git commit hashes in this file refer to the official FreeBSD distributed public Git repository at https://git.freebsd.org/src.git The bhyve userland code in this directory, and its associated libraries and parts of the kernel module have been updated to the latest upstream FreeBSD sources as of: commit d70230783ae27594a77560582093dd86426d3360 Author: Mark Johnston Date: Thu Nov 7 20:07:39 2024 +0000 vmm: Postpone vmm module initialization to after SI_SUB_DEVFS Divergence Notes: The kernel component of bhyve has diverged from the FreeBSD sources and will continue to evolve away from the initial compatibility interfaces towards native illumos ones. The vmmapi library abstracts those interfaces wherever possible so that the bhyve userspace can remain close to upstream. The source code for the following components is present and included in syncs from FreeBSD, but currently disabled. See the comments in 'Makefile' for more details: kernemu_dev virtio-scsi audio backend virtio_input The draft Save/Restore functionality, added in FreeBSD commit 483d953a86a2507355f8287c5107dc827a0ff516, has not been synced into illumos bhyve yet. It is not built by default in FreeBSD, so we're not interested in taking it until it successfully endures more in-depth testing. The stub usr/src/compat/bhyve/stdatomic.h file only includes enough glue to satisfy the use of in usr/src/cmd/bhyve/rfb.c, and in particular assumes that atomic variables are sized as an int. If other bhyve pieces start using stdatomic.h, this will need enhancing. SPDX-License-Identifier: BSD-2-Clause Copyright (c) 1992-2020 The FreeBSD Project. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. Bhyve hypervisor # # This file and its contents are supplied under the terms of the # Common Development and Distribution License ("CDDL"), version 1.0. # You may only use this file in accordance with the terms of version # 1.0 of the CDDL. # # A full copy of the text of the CDDL should have accompanied this # source. A copy of the CDDL is also available via the Internet at # http://www.illumos.org/license/CDDL. # # # Copyright 2024 OmniOS Community Edition (OmniOSce) Association. # BHYVE_OBJS = $(COMMON_OBJS) \ atkbdc.o \ bhyverun_machdep.o \ e820.o \ inout.o \ ioapic.o \ fwctl.o \ mptbl.o \ pci_fbuf.o \ pci_lpc.o \ pci_passthru.o \ pctestdev.o \ post.o \ pm.o \ ps2kbd.o \ ps2mouse.o \ rfb.o \ rtc.o \ spinup_ap.o \ task_switch.o \ vga.o \ vmexit.o \ xmsr.o include ../Makefile.com CPPFLAGS += -I. install: all $(ROOTUSRSBINPROG) /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2014 Tycho Nightingale * Copyright (c) 2015 Nahanni Systems Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include #include #include #include #include #include #include #include #include #include #include #include #include "acpi.h" #include "atkbdc.h" #include "inout.h" #include "pci_emul.h" #include "pci_irq.h" #include "pci_lpc.h" #include "ps2kbd.h" #include "ps2mouse.h" #define KBD_DATA_PORT 0x60 #define KBD_STS_CTL_PORT 0x64 #define KBDC_RESET 0xfe #define KBD_DEV_IRQ 1 #define AUX_DEV_IRQ 12 /* controller commands */ #define KBDC_SET_COMMAND_BYTE 0x60 #define KBDC_GET_COMMAND_BYTE 0x20 #define KBDC_DISABLE_AUX_PORT 0xa7 #define KBDC_ENABLE_AUX_PORT 0xa8 #define KBDC_TEST_AUX_PORT 0xa9 #define KBDC_TEST_CTRL 0xaa #define KBDC_TEST_KBD_PORT 0xab #define KBDC_DISABLE_KBD_PORT 0xad #define KBDC_ENABLE_KBD_PORT 0xae #define KBDC_READ_INPORT 0xc0 #define KBDC_READ_OUTPORT 0xd0 #define KBDC_WRITE_OUTPORT 0xd1 #define KBDC_WRITE_KBD_OUTBUF 0xd2 #define KBDC_WRITE_AUX_OUTBUF 0xd3 #define KBDC_WRITE_TO_AUX 0xd4 /* controller command byte (set by KBDC_SET_COMMAND_BYTE) */ #define KBD_TRANSLATION 0x40 #define KBD_SYS_FLAG_BIT 0x04 #define KBD_DISABLE_KBD_PORT 0x10 #define KBD_DISABLE_AUX_PORT 0x20 #define KBD_ENABLE_AUX_INT 0x02 #define KBD_ENABLE_KBD_INT 0x01 #define KBD_KBD_CONTROL_BITS (KBD_DISABLE_KBD_PORT | KBD_ENABLE_KBD_INT) #define KBD_AUX_CONTROL_BITS (KBD_DISABLE_AUX_PORT | KBD_ENABLE_AUX_INT) /* controller status bits */ #define KBDS_KBD_BUFFER_FULL 0x01 #define KBDS_SYS_FLAG 0x04 #define KBDS_CTRL_FLAG 0x08 #define KBDS_AUX_BUFFER_FULL 0x20 /* controller output port */ #define KBDO_KBD_OUTFULL 0x10 #define KBDO_AUX_OUTFULL 0x20 #define RAMSZ 32 #define FIFOSZ 15 #define CTRL_CMD_FLAG 0x8000 struct kbd_dev { bool irq_active; int irq; uint8_t buffer[FIFOSZ]; int brd, bwr; int bcnt; }; struct aux_dev { bool irq_active; int irq; }; struct atkbdc_softc { struct vmctx *ctx; pthread_mutex_t mtx; struct ps2kbd_softc *ps2kbd_sc; struct ps2mouse_softc *ps2mouse_sc; uint8_t status; /* status register */ uint8_t outport; /* controller output port */ uint8_t ram[RAMSZ]; /* byte0 = controller config */ uint32_t curcmd; /* current command for next byte */ uint32_t ctrlbyte; struct kbd_dev kbd; struct aux_dev aux; }; static void atkbdc_assert_kbd_intr(struct atkbdc_softc *sc) { if ((sc->ram[0] & KBD_ENABLE_KBD_INT) != 0) { sc->kbd.irq_active = true; vm_isa_pulse_irq(sc->ctx, sc->kbd.irq, sc->kbd.irq); } } static void atkbdc_assert_aux_intr(struct atkbdc_softc *sc) { if ((sc->ram[0] & KBD_ENABLE_AUX_INT) != 0) { sc->aux.irq_active = true; vm_isa_pulse_irq(sc->ctx, sc->aux.irq, sc->aux.irq); } } static int atkbdc_kbd_queue_data(struct atkbdc_softc *sc, uint8_t val) { assert(pthread_mutex_isowned_np(&sc->mtx)); if (sc->kbd.bcnt < FIFOSZ) { sc->kbd.buffer[sc->kbd.bwr] = val; sc->kbd.bwr = (sc->kbd.bwr + 1) % FIFOSZ; sc->kbd.bcnt++; sc->status |= KBDS_KBD_BUFFER_FULL; sc->outport |= KBDO_KBD_OUTFULL; } else { printf("atkbd data buffer full\n"); } return (sc->kbd.bcnt < FIFOSZ); } static void atkbdc_kbd_read(struct atkbdc_softc *sc) { const uint8_t translation[256] = { 0xff, 0x43, 0x41, 0x3f, 0x3d, 0x3b, 0x3c, 0x58, 0x64, 0x44, 0x42, 0x40, 0x3e, 0x0f, 0x29, 0x59, 0x65, 0x38, 0x2a, 0x70, 0x1d, 0x10, 0x02, 0x5a, 0x66, 0x71, 0x2c, 0x1f, 0x1e, 0x11, 0x03, 0x5b, 0x67, 0x2e, 0x2d, 0x20, 0x12, 0x05, 0x04, 0x5c, 0x68, 0x39, 0x2f, 0x21, 0x14, 0x13, 0x06, 0x5d, 0x69, 0x31, 0x30, 0x23, 0x22, 0x15, 0x07, 0x5e, 0x6a, 0x72, 0x32, 0x24, 0x16, 0x08, 0x09, 0x5f, 0x6b, 0x33, 0x25, 0x17, 0x18, 0x0b, 0x0a, 0x60, 0x6c, 0x34, 0x35, 0x26, 0x27, 0x19, 0x0c, 0x61, 0x6d, 0x73, 0x28, 0x74, 0x1a, 0x0d, 0x62, 0x6e, 0x3a, 0x36, 0x1c, 0x1b, 0x75, 0x2b, 0x63, 0x76, 0x55, 0x56, 0x77, 0x78, 0x79, 0x7a, 0x0e, 0x7b, 0x7c, 0x4f, 0x7d, 0x4b, 0x47, 0x7e, 0x7f, 0x6f, 0x52, 0x53, 0x50, 0x4c, 0x4d, 0x48, 0x01, 0x45, 0x57, 0x4e, 0x51, 0x4a, 0x37, 0x49, 0x46, 0x54, 0x80, 0x81, 0x82, 0x41, 0x54, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff }; uint8_t val; uint8_t release = 0; assert(pthread_mutex_isowned_np(&sc->mtx)); if (sc->ram[0] & KBD_TRANSLATION) { while (ps2kbd_read(sc->ps2kbd_sc, &val) != -1) { if (val == 0xf0) { release = 0x80; continue; } else { val = translation[val] | release; } atkbdc_kbd_queue_data(sc, val); break; } } else { while (sc->kbd.bcnt < FIFOSZ) { if (ps2kbd_read(sc->ps2kbd_sc, &val) != -1) atkbdc_kbd_queue_data(sc, val); else break; } } if (((sc->ram[0] & KBD_DISABLE_AUX_PORT) || ps2mouse_fifocnt(sc->ps2mouse_sc) == 0) && sc->kbd.bcnt > 0) atkbdc_assert_kbd_intr(sc); } static void atkbdc_aux_poll(struct atkbdc_softc *sc) { if (ps2mouse_fifocnt(sc->ps2mouse_sc) > 0) { sc->status |= KBDS_AUX_BUFFER_FULL | KBDS_KBD_BUFFER_FULL; sc->outport |= KBDO_AUX_OUTFULL; atkbdc_assert_aux_intr(sc); } } static void atkbdc_kbd_poll(struct atkbdc_softc *sc) { assert(pthread_mutex_isowned_np(&sc->mtx)); atkbdc_kbd_read(sc); } static void atkbdc_poll(struct atkbdc_softc *sc) { atkbdc_aux_poll(sc); atkbdc_kbd_poll(sc); } static void atkbdc_dequeue_data(struct atkbdc_softc *sc, uint8_t *buf) { assert(pthread_mutex_isowned_np(&sc->mtx)); if (ps2mouse_read(sc->ps2mouse_sc, buf) == 0) { if (ps2mouse_fifocnt(sc->ps2mouse_sc) == 0) { if (sc->kbd.bcnt == 0) sc->status &= ~(KBDS_AUX_BUFFER_FULL | KBDS_KBD_BUFFER_FULL); else sc->status &= ~(KBDS_AUX_BUFFER_FULL); sc->outport &= ~KBDO_AUX_OUTFULL; } atkbdc_poll(sc); return; } if (sc->kbd.bcnt > 0) { *buf = sc->kbd.buffer[sc->kbd.brd]; sc->kbd.brd = (sc->kbd.brd + 1) % FIFOSZ; sc->kbd.bcnt--; if (sc->kbd.bcnt == 0) { sc->status &= ~KBDS_KBD_BUFFER_FULL; sc->outport &= ~KBDO_KBD_OUTFULL; } atkbdc_poll(sc); } if (ps2mouse_fifocnt(sc->ps2mouse_sc) == 0 && sc->kbd.bcnt == 0) { sc->status &= ~(KBDS_AUX_BUFFER_FULL | KBDS_KBD_BUFFER_FULL); } } static int atkbdc_data_handler(struct vmctx *ctx __unused, int in, int port __unused, int bytes, uint32_t *eax, void *arg) { struct atkbdc_softc *sc; uint8_t buf; int retval; if (bytes != 1) return (-1); sc = arg; retval = 0; pthread_mutex_lock(&sc->mtx); if (in) { sc->curcmd = 0; if (sc->ctrlbyte != 0) { *eax = sc->ctrlbyte & 0xff; sc->ctrlbyte = 0; } else { /* read device buffer; includes kbd cmd responses */ atkbdc_dequeue_data(sc, &buf); *eax = buf; } sc->status &= ~KBDS_CTRL_FLAG; pthread_mutex_unlock(&sc->mtx); return (retval); } if (sc->status & KBDS_CTRL_FLAG) { /* * Command byte for the controller. */ switch (sc->curcmd) { case KBDC_SET_COMMAND_BYTE: sc->ram[0] = *eax; if (sc->ram[0] & KBD_SYS_FLAG_BIT) sc->status |= KBDS_SYS_FLAG; else sc->status &= ~KBDS_SYS_FLAG; break; case KBDC_WRITE_OUTPORT: sc->outport = *eax; break; case KBDC_WRITE_TO_AUX: ps2mouse_write(sc->ps2mouse_sc, *eax, 0); atkbdc_poll(sc); break; case KBDC_WRITE_KBD_OUTBUF: atkbdc_kbd_queue_data(sc, *eax); break; case KBDC_WRITE_AUX_OUTBUF: ps2mouse_write(sc->ps2mouse_sc, *eax, 1); sc->status |= (KBDS_AUX_BUFFER_FULL | KBDS_KBD_BUFFER_FULL); atkbdc_aux_poll(sc); break; default: /* write to particular RAM byte */ if (sc->curcmd >= 0x61 && sc->curcmd <= 0x7f) { int byten; byten = (sc->curcmd - 0x60) & 0x1f; sc->ram[byten] = *eax & 0xff; } break; } sc->curcmd = 0; sc->status &= ~KBDS_CTRL_FLAG; pthread_mutex_unlock(&sc->mtx); return (retval); } /* * Data byte for the device. */ ps2kbd_write(sc->ps2kbd_sc, *eax); atkbdc_poll(sc); pthread_mutex_unlock(&sc->mtx); return (retval); } static int atkbdc_sts_ctl_handler(struct vmctx *ctx, int in, int port __unused, int bytes, uint32_t *eax, void *arg) { struct atkbdc_softc *sc; int error, retval; if (bytes != 1) return (-1); sc = arg; retval = 0; pthread_mutex_lock(&sc->mtx); if (in) { /* read status register */ *eax = sc->status; pthread_mutex_unlock(&sc->mtx); return (retval); } sc->curcmd = 0; sc->status |= KBDS_CTRL_FLAG; sc->ctrlbyte = 0; switch (*eax) { case KBDC_GET_COMMAND_BYTE: sc->ctrlbyte = CTRL_CMD_FLAG | sc->ram[0]; break; case KBDC_TEST_CTRL: sc->ctrlbyte = CTRL_CMD_FLAG | 0x55; break; case KBDC_TEST_AUX_PORT: case KBDC_TEST_KBD_PORT: sc->ctrlbyte = CTRL_CMD_FLAG | 0; break; case KBDC_READ_INPORT: sc->ctrlbyte = CTRL_CMD_FLAG | 0; break; case KBDC_READ_OUTPORT: sc->ctrlbyte = CTRL_CMD_FLAG | sc->outport; break; case KBDC_SET_COMMAND_BYTE: case KBDC_WRITE_OUTPORT: case KBDC_WRITE_KBD_OUTBUF: case KBDC_WRITE_AUX_OUTBUF: sc->curcmd = *eax; break; case KBDC_DISABLE_KBD_PORT: sc->ram[0] |= KBD_DISABLE_KBD_PORT; break; case KBDC_ENABLE_KBD_PORT: sc->ram[0] &= ~KBD_DISABLE_KBD_PORT; if (sc->kbd.bcnt > 0) sc->status |= KBDS_KBD_BUFFER_FULL; atkbdc_poll(sc); break; case KBDC_WRITE_TO_AUX: sc->curcmd = *eax; break; case KBDC_DISABLE_AUX_PORT: sc->ram[0] |= KBD_DISABLE_AUX_PORT; ps2mouse_toggle(sc->ps2mouse_sc, 0); sc->status &= ~(KBDS_AUX_BUFFER_FULL | KBDS_KBD_BUFFER_FULL); sc->outport &= ~KBDS_AUX_BUFFER_FULL; break; case KBDC_ENABLE_AUX_PORT: sc->ram[0] &= ~KBD_DISABLE_AUX_PORT; ps2mouse_toggle(sc->ps2mouse_sc, 1); if (ps2mouse_fifocnt(sc->ps2mouse_sc) > 0) sc->status |= KBDS_AUX_BUFFER_FULL | KBDS_KBD_BUFFER_FULL; break; case KBDC_RESET: /* Pulse "reset" line */ error = vm_suspend(ctx, VM_SUSPEND_RESET); assert(error == 0 || errno == EALREADY); break; default: if (*eax >= 0x21 && *eax <= 0x3f) { /* read "byte N" from RAM */ int byten; byten = (*eax - 0x20) & 0x1f; sc->ctrlbyte = CTRL_CMD_FLAG | sc->ram[byten]; } break; } pthread_mutex_unlock(&sc->mtx); if (sc->ctrlbyte != 0) { sc->status |= KBDS_KBD_BUFFER_FULL; sc->status &= ~KBDS_AUX_BUFFER_FULL; atkbdc_assert_kbd_intr(sc); } else if (ps2mouse_fifocnt(sc->ps2mouse_sc) > 0 && (sc->ram[0] & KBD_DISABLE_AUX_PORT) == 0) { sc->status |= KBDS_AUX_BUFFER_FULL | KBDS_KBD_BUFFER_FULL; atkbdc_assert_aux_intr(sc); } else if (sc->kbd.bcnt > 0 && (sc->ram[0] & KBD_DISABLE_KBD_PORT) == 0) { sc->status |= KBDS_KBD_BUFFER_FULL; atkbdc_assert_kbd_intr(sc); } return (retval); } void atkbdc_event(struct atkbdc_softc *sc, int iskbd) { pthread_mutex_lock(&sc->mtx); if (iskbd) atkbdc_kbd_poll(sc); else atkbdc_aux_poll(sc); pthread_mutex_unlock(&sc->mtx); } void atkbdc_init(struct vmctx *ctx) { struct inout_port iop; struct atkbdc_softc *sc; int error; sc = calloc(1, sizeof(struct atkbdc_softc)); sc->ctx = ctx; pthread_mutex_init(&sc->mtx, NULL); bzero(&iop, sizeof(struct inout_port)); iop.name = "atkdbc"; iop.port = KBD_STS_CTL_PORT; iop.size = 1; iop.flags = IOPORT_F_INOUT; iop.handler = atkbdc_sts_ctl_handler; iop.arg = sc; error = register_inout(&iop); assert(error == 0); bzero(&iop, sizeof(struct inout_port)); iop.name = "atkdbc"; iop.port = KBD_DATA_PORT; iop.size = 1; iop.flags = IOPORT_F_INOUT; iop.handler = atkbdc_data_handler; iop.arg = sc; error = register_inout(&iop); assert(error == 0); pci_irq_reserve(KBD_DEV_IRQ); sc->kbd.irq = KBD_DEV_IRQ; pci_irq_reserve(AUX_DEV_IRQ); sc->aux.irq = AUX_DEV_IRQ; sc->ps2kbd_sc = ps2kbd_init(sc); sc->ps2mouse_sc = ps2mouse_init(sc); } static void atkbdc_dsdt(void) { dsdt_line(""); dsdt_line("Device (KBD)"); dsdt_line("{"); dsdt_line(" Name (_HID, EisaId (\"PNP0303\"))"); dsdt_line(" Name (_CRS, ResourceTemplate ()"); dsdt_line(" {"); dsdt_indent(2); dsdt_fixed_ioport(KBD_DATA_PORT, 1); dsdt_fixed_ioport(KBD_STS_CTL_PORT, 1); dsdt_fixed_irq(1); dsdt_unindent(2); dsdt_line(" })"); dsdt_line("}"); dsdt_line(""); dsdt_line("Device (MOU)"); dsdt_line("{"); dsdt_line(" Name (_HID, EisaId (\"PNP0F13\"))"); dsdt_line(" Name (_CRS, ResourceTemplate ()"); dsdt_line(" {"); dsdt_indent(2); dsdt_fixed_ioport(KBD_DATA_PORT, 1); dsdt_fixed_ioport(KBD_STS_CTL_PORT, 1); dsdt_fixed_irq(12); dsdt_unindent(2); dsdt_line(" })"); dsdt_line("}"); } LPC_DSDT(atkbdc_dsdt); /*- * Copyright (c) 2015 Tycho Nightingale * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef _ATKBDC_H_ #define _ATKBDC_H_ struct atkbdc_softc; struct vmctx; void atkbdc_init(struct vmctx *ctx); void atkbdc_event(struct atkbdc_softc *sc, int iskbd); #endif /* _ATKBDC_H_ */ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2011 NetApp, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * This file and its contents are supplied under the terms of the * Common Development and Distribution License ("CDDL"), version 1.0. * You may only use this file in accordance with the terms of version * 1.0 of the CDDL. * * A full copy of the text of the CDDL should have accompanied this * source. A copy of the CDDL is also available via the Internet at * http://www.illumos.org/license/CDDL. * * Copyright 2015 Pluribus Networks Inc. * Copyright 2018 Joyent, Inc. * Copyright 2025 Oxide Computer Company * Copyright 2022 OmniOS Community Edition (OmniOSce) Association. */ #include #include #include #include #include #include #include #include #include #include #include "bhyverun.h" #include "bootrom.h" #include "acpi.h" #include "atkbdc.h" #include "config.h" #include "debug.h" #include "e820.h" #include "fwctl.h" #include "ioapic.h" #include "inout.h" #ifndef __FreeBSD__ #include "kernemu_dev.h" #endif #include "mptbl.h" #include "pci_emul.h" #include "pci_irq.h" #include "spinup_ap.h" #include "pci_lpc.h" #include "rtc.h" #include "smbiostbl.h" #include "xmsr.h" void bhyve_usage(int code) { const char *progname = getprogname(); fprintf(stderr, #ifdef __FreeBSD__ "Usage: %s [-AaCDeHhPSuWwxY]\n" #else "Usage: %s [-aCDdeHhPSuWwxY]\n" #endif " %2$.*3$s [-c [[cpus=]numcpus][,sockets=n][,cores=n][,threads=n]]\n" #ifdef __FreeBSD__ " %2$.*3$s [-G port] [-k config_file] [-l lpc] [-m mem] [-o var=value]\n" " %2$.*3$s [-p vcpu:hostcpu] [-r file] [-s pci] [-U uuid] vmname\n" " -A: create ACPI tables\n" #else " %2$.*3$s [-k ] [-l ] [-m mem] [-o =]\n" " %2$.*3$s [-s ] [-U uuid] vmname\n" #endif " -a: local apic is in xAPIC mode (deprecated)\n" #ifndef __FreeBSD__ " -B type,key=value,...: set SMBIOS information\n" #endif " -C: include guest memory in core file\n" " -c: number of CPUs and/or topology specification\n" " -D: destroy on power-off\n" #ifndef __FreeBSD__ " -d: suspend cpu at boot\n" #endif " -e: exit on unhandled I/O access\n" #ifdef __FreeBSD__ " -G: start a debug server\n" #endif " -H: vmexit from the guest on HLT\n" " -h: help\n" " -k: key=value flat config file\n" " -K: PS2 keyboard layout\n" " -l: LPC device configuration\n" " -m: memory size\n" " -o: set config 'var' to 'value'\n" " -P: vmexit from the guest on pause\n" #ifdef __FreeBSD__ " -p: pin 'vcpu' to 'hostcpu'\n" #endif " -S: guest memory cannot be swapped\n" " -s: PCI slot config\n" " -U: UUID\n" " -u: RTC keeps UTC time\n" " -W: force virtio to use single-vector MSI\n" " -w: ignore unimplemented MSRs\n" " -x: local APIC is in x2APIC mode\n" " -Y: disable MPtable generation\n", progname, "", (int)strlen(progname)); exit(code); } void bhyve_optparse(int argc, char **argv) { const char *optstr; int c; #ifdef __FreeBSD__ optstr = "aehuwxACDHIPSWYk:f:o:p:G:c:s:m:l:K:U:"; #else /* +d, +B, -p */ optstr = "adehuwxACDHIPSWYk:f:o:G:c:s:m:l:B:K:U:"; #endif while ((c = getopt(argc, argv, optstr)) != -1) { switch (c) { case 'a': set_config_bool("x86.x2apic", false); break; case 'A': set_config_bool("acpi_tables", true); break; case 'D': set_config_bool("destroy_on_poweroff", true); break; #ifndef __FreeBSD__ case 'B': if (smbios_parse(optarg) != 0) { errx(EX_USAGE, "invalid SMBIOS " "configuration '%s'", optarg); } break; case 'd': set_config_bool("suspend_at_boot", true); break; #endif #ifdef __FreeBSD__ case 'p': if (pincpu_parse(optarg) != 0) { errx(EX_USAGE, "invalid vcpu pinning " "configuration '%s'", optarg); } break; #endif case 'c': if (bhyve_topology_parse(optarg) != 0) { errx(EX_USAGE, "invalid cpu topology " "'%s'", optarg); } break; case 'C': set_config_bool("memory.guest_in_core", true); break; case 'f': if (qemu_fwcfg_parse_cmdline_arg(optarg) != 0) { errx(EX_USAGE, "invalid fwcfg item '%s'", optarg); } break; case 'G': bhyve_parse_gdb_options(optarg); break; case 'k': bhyve_parse_simple_config_file(optarg); break; case 'K': set_config_value("keyboard.layout", optarg); break; case 'l': if (strncmp(optarg, "help", strlen(optarg)) == 0) { lpc_print_supported_devices(); exit(0); } else if (lpc_device_parse(optarg) != 0) { errx(EX_USAGE, "invalid lpc device " "configuration '%s'", optarg); } break; case 's': if (strncmp(optarg, "help", strlen(optarg)) == 0) { pci_print_supported_devices(); exit(0); } else if (pci_parse_slot(optarg) != 0) exit(4); else break; case 'S': set_config_bool("memory.wired", true); break; case 'm': set_config_value("memory.size", optarg); break; case 'o': if (!bhyve_parse_config_option(optarg)) errx(EX_USAGE, "invalid configuration option '%s'", optarg); break; case 'H': set_config_bool("x86.vmexit_on_hlt", true); break; case 'I': /* * The "-I" option was used to add an ioapic to the * virtual machine. * * An ioapic is now provided unconditionally for each * virtual machine and this option is now deprecated. */ break; case 'P': set_config_bool("x86.vmexit_on_pause", true); break; case 'e': set_config_bool("x86.strictio", true); break; case 'u': set_config_bool("rtc.use_localtime", false); break; case 'U': set_config_value("uuid", optarg); break; case 'w': set_config_bool("x86.strictmsr", false); break; case 'W': set_config_bool("virtio.msix", false); break; case 'x': set_config_bool("x86.x2apic", true); break; case 'Y': set_config_bool("x86.mptable", false); break; case 'h': bhyve_usage(0); default: bhyve_usage(1); } } /* Handle backwards compatibility aliases in config options. */ if (get_config_value("lpc.bootrom") != NULL && get_config_value("bootrom") == NULL) { warnx("lpc.bootrom is deprecated, use '-o bootrom' instead"); set_config_value("bootrom", get_config_value("lpc.bootrom")); } if (get_config_value("lpc.bootvars") != NULL && get_config_value("bootvars") == NULL) { warnx("lpc.bootvars is deprecated, use '-o bootvars' instead"); set_config_value("bootvars", get_config_value("lpc.bootvars")); } } void bhyve_init_config(void) { init_config(); /* Set default values prior to option parsing. */ set_config_bool("acpi_tables", false); set_config_bool("acpi_tables_in_memory", true); set_config_value("memory.size", "256M"); set_config_bool("x86.strictmsr", true); set_config_value("lpc.fwcfg", "bhyve"); } void bhyve_init_vcpu(struct vcpu *vcpu) { int err, tmp; #ifdef __FreeBSD__ if (get_config_bool_default("x86.vmexit_on_hlt", false)) { err = vm_get_capability(vcpu, VM_CAP_HALT_EXIT, &tmp); if (err < 0) { EPRINTLN("VM exit on HLT not supported"); exit(4); } vm_set_capability(vcpu, VM_CAP_HALT_EXIT, 1); } #else /* * We insist that vmexit-on-hlt is available on the host CPU, and enable * it by default. Configuration of that feature is done with both of * those facts in mind. */ tmp = (int)get_config_bool_default("x86.vmexit_on_hlt", true); err = vm_set_capability(vcpu, VM_CAP_HALT_EXIT, tmp); if (err < 0) { fprintf(stderr, "VM exit on HLT not supported\n"); exit(4); } #endif /* __FreeBSD__ */ if (get_config_bool_default("x86.vmexit_on_pause", false)) { /* * pause exit support required for this mode */ err = vm_get_capability(vcpu, VM_CAP_PAUSE_EXIT, &tmp); if (err < 0) { EPRINTLN("SMP mux requested, no pause support"); exit(4); } vm_set_capability(vcpu, VM_CAP_PAUSE_EXIT, 1); } if (get_config_bool_default("x86.x2apic", false)) err = vm_set_x2apic_state(vcpu, X2APIC_ENABLED); else err = vm_set_x2apic_state(vcpu, X2APIC_DISABLED); if (err) { EPRINTLN("Unable to set x2apic state (%d)", err); exit(4); } #ifdef __FreeBSD__ vm_set_capability(vcpu, VM_CAP_ENABLE_INVPCID, 1); err = vm_set_capability(vcpu, VM_CAP_IPI_EXIT, 1); assert(err == 0); #endif } void bhyve_start_vcpu(struct vcpu *vcpu, bool bsp, bool suspend) { int error; if (!bsp) { #ifndef __FreeBSD__ /* * On illumos, all APs are spun up halted and run-state * transitions (INIT, SIPI, etc) are handled in-kernel. */ spinup_ap(vcpu, 0); #endif bhyve_init_vcpu(vcpu); #ifdef __FreeBSD__ /* * Enable the 'unrestricted guest' mode for APs. * * APs startup in power-on 16-bit mode. */ error = vm_set_capability(vcpu, VM_CAP_UNRESTRICTED_GUEST, 1); assert(error == 0); #endif } #ifndef __FreeBSD__ /* * The value of 'suspend' for the BSP depends on whether the -d * (suspend_at_boot) flag was given to bhyve. Regardless of that * value we always want to set the BSP to VRS_RUN and all others to * VRS_HALT. */ error = vm_set_run_state(vcpu, bsp ? VRS_RUN : VRS_HALT, 0); assert(error == 0); #endif fbsdrun_addcpu(vcpu_id(vcpu), suspend); } int bhyve_init_platform(struct vmctx *ctx, struct vcpu *bsp __unused) { int error; error = init_msr(); if (error != 0) return (error); init_inout(); #ifdef __FreeBSD__ kernemu_dev_init(); #endif atkbdc_init(ctx); pci_irq_init(ctx); ioapic_init(ctx); rtc_init(ctx); sci_init(ctx); #ifndef __FreeBSD__ pmtmr_init(ctx); #endif error = e820_init(ctx); if (error != 0) return (error); error = bootrom_loadrom(ctx); if (error != 0) return (error); #ifndef __FreeBSD__ if (get_config_bool_default("e820.debug", false)) e820_dump_table(); #endif return (0); } int bhyve_init_platform_late(struct vmctx *ctx, struct vcpu *bsp __unused) { int error; if (get_config_bool_default("x86.mptable", true)) { error = mptable_build(ctx, guest_ncpus); if (error != 0) return (error); } error = smbios_build(ctx); if (error != 0) return (error); error = e820_finalize(); if (error != 0) return (error); if (bootrom_boot() && strcmp(lpc_fwcfg(), "bhyve") == 0) fwctl_init(); if (get_config_bool("acpi_tables")) { error = acpi_build(ctx, guest_ncpus); assert(error == 0); } return (0); } /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2021 Beckhoff Automation GmbH & Co. KG * Author: Corvin Köhne */ #include #include #include #include #include #include #include #include #include #include "debug.h" #include "e820.h" #include "qemu_fwcfg.h" /* * E820 always uses 64 bit entries. Emulation code will use vm_paddr_t since it * works on physical addresses. If vm_paddr_t is larger than uint64_t E820 can't * hold all possible physical addresses and we can get into trouble. */ static_assert(sizeof(vm_paddr_t) <= sizeof(uint64_t), "Unable to represent physical memory by E820 table"); #define E820_FWCFG_FILE_NAME "etc/e820" #define KB (1024UL) #define MB (1024 * KB) #define GB (1024 * MB) /* * Fix E820 memory holes: * [ A0000, C0000) VGA * [ C0000, 100000) ROM */ #define E820_VGA_MEM_BASE 0xA0000 #define E820_VGA_MEM_END 0xC0000 #define E820_ROM_MEM_BASE 0xC0000 #define E820_ROM_MEM_END 0x100000 struct e820_element { TAILQ_ENTRY(e820_element) chain; uint64_t base; uint64_t end; enum e820_memory_type type; }; static TAILQ_HEAD(e820_table, e820_element) e820_table = TAILQ_HEAD_INITIALIZER( e820_table); static struct e820_element * e820_element_alloc(uint64_t base, uint64_t end, enum e820_memory_type type) { struct e820_element *element; element = calloc(1, sizeof(*element)); if (element == NULL) { return (NULL); } element->base = base; element->end = end; element->type = type; return (element); } static const char * e820_get_type_name(const enum e820_memory_type type) { switch (type) { case E820_TYPE_MEMORY: return ("RAM"); case E820_TYPE_RESERVED: return ("Reserved"); case E820_TYPE_ACPI: return ("ACPI"); case E820_TYPE_NVS: return ("NVS"); default: return ("Unknown"); } } void e820_dump_table(void) { struct e820_element *element; uint64_t i; EPRINTLN("E820 map:"); i = 0; TAILQ_FOREACH(element, &e820_table, chain) { EPRINTLN(" (%4lu) [%16lx, %16lx] %s", i, element->base, element->end, e820_get_type_name(element->type)); ++i; } } static struct qemu_fwcfg_item * e820_get_fwcfg_item(void) { struct qemu_fwcfg_item *fwcfg_item; struct e820_element *element; struct e820_entry *entries; int count, i; count = 0; TAILQ_FOREACH(element, &e820_table, chain) { ++count; } if (count == 0) { warnx("%s: E820 table empty", __func__); return (NULL); } fwcfg_item = calloc(1, sizeof(struct qemu_fwcfg_item)); if (fwcfg_item == NULL) { return (NULL); } fwcfg_item->size = count * sizeof(struct e820_entry); fwcfg_item->data = calloc(count, sizeof(struct e820_entry)); if (fwcfg_item->data == NULL) { free(fwcfg_item); return (NULL); } i = 0; entries = (struct e820_entry *)fwcfg_item->data; TAILQ_FOREACH(element, &e820_table, chain) { struct e820_entry *entry = &entries[i]; entry->base = element->base; entry->length = element->end - element->base; entry->type = element->type; ++i; } return (fwcfg_item); } static int e820_add_entry(const uint64_t base, const uint64_t end, const enum e820_memory_type type) { struct e820_element *new_element; struct e820_element *element; struct e820_element *sib_element; struct e820_element *ram_element; assert(end >= base); new_element = e820_element_alloc(base, end, type); if (new_element == NULL) { return (ENOMEM); } /* * E820 table should always be sorted in ascending order. Therefore, * search for a range whose end is larger than the base parameter. */ TAILQ_FOREACH(element, &e820_table, chain) { if (element->end > base) { break; } } /* * System memory requires special handling. */ if (type == E820_TYPE_MEMORY) { /* * base is larger than of any existing element. Add new system * memory at the end of the table. */ if (element == NULL) { TAILQ_INSERT_TAIL(&e820_table, new_element, chain); return (0); } /* * System memory shouldn't overlap with any existing element. */ assert(end >= element->base); TAILQ_INSERT_BEFORE(element, new_element, chain); return (0); } assert(element != NULL); /* Non system memory should be allocated inside system memory. */ assert(element->type == E820_TYPE_MEMORY); /* New element should fit into existing system memory element. */ assert(base >= element->base && end <= element->end); if (base == element->base && end == element->end) { /* * The new entry replaces an existing one. * * Old table: * [ 0x1000, 0x4000] RAM <-- element * New table: * [ 0x1000, 0x4000] Reserved */ TAILQ_INSERT_BEFORE(element, new_element, chain); TAILQ_REMOVE(&e820_table, element, chain); free(element); } else if (base == element->base) { /* * New element at system memory base boundary. Add new * element before current and adjust the base of the old * element. * * Old table: * [ 0x1000, 0x4000] RAM <-- element * New table: * [ 0x1000, 0x2000] Reserved * [ 0x2000, 0x4000] RAM <-- element */ TAILQ_INSERT_BEFORE(element, new_element, chain); element->base = end; } else if (end == element->end) { /* * New element at system memory end boundary. Add new * element after current and adjust the end of the * current element. * * Old table: * [ 0x1000, 0x4000] RAM <-- element * New table: * [ 0x1000, 0x3000] RAM <-- element * [ 0x3000, 0x4000] Reserved */ TAILQ_INSERT_AFTER(&e820_table, element, new_element, chain); element->end = base; } else { /* * New element inside system memory entry. Split it by * adding a system memory element and the new element * before current. * * Old table: * [ 0x1000, 0x4000] RAM <-- element * New table: * [ 0x1000, 0x2000] RAM * [ 0x2000, 0x3000] Reserved * [ 0x3000, 0x4000] RAM <-- element */ ram_element = e820_element_alloc(element->base, base, E820_TYPE_MEMORY); if (ram_element == NULL) { return (ENOMEM); } TAILQ_INSERT_BEFORE(element, ram_element, chain); TAILQ_INSERT_BEFORE(element, new_element, chain); element->base = end; } /* * If the previous element has the same type and ends at our base * boundary, we can merge both entries. */ sib_element = TAILQ_PREV(new_element, e820_table, chain); if (sib_element != NULL && sib_element->type == new_element->type && sib_element->end == new_element->base) { new_element->base = sib_element->base; TAILQ_REMOVE(&e820_table, sib_element, chain); free(sib_element); } /* * If the next element has the same type and starts at our end * boundary, we can merge both entries. */ sib_element = TAILQ_NEXT(new_element, chain); if (sib_element != NULL && sib_element->type == new_element->type && sib_element->base == new_element->end) { /* Merge new element into subsequent one. */ new_element->end = sib_element->end; TAILQ_REMOVE(&e820_table, sib_element, chain); free(sib_element); } return (0); } static int e820_add_memory_hole(const uint64_t base, const uint64_t end) { struct e820_element *element; struct e820_element *ram_element; assert(end >= base); /* * E820 table should be always sorted in ascending order. Therefore, * search for an element which end is larger than the base parameter. */ TAILQ_FOREACH(element, &e820_table, chain) { if (element->end > base) { break; } } if (element == NULL || end <= element->base) { /* Nothing to do. Hole already exists */ return (0); } /* Memory holes are only allowed in system memory */ assert(element->type == E820_TYPE_MEMORY); if (base == element->base) { /* * New hole at system memory base boundary. * * Old table: * [ 0x1000, 0x4000] RAM * New table: * [ 0x2000, 0x4000] RAM */ element->base = end; } else if (end == element->end) { /* * New hole at system memory end boundary. * * Old table: * [ 0x1000, 0x4000] RAM * New table: * [ 0x1000, 0x3000] RAM */ element->end = base; } else { /* * New hole inside system memory entry. Split the system memory. * * Old table: * [ 0x1000, 0x4000] RAM <-- element * New table: * [ 0x1000, 0x2000] RAM * [ 0x3000, 0x4000] RAM <-- element */ ram_element = e820_element_alloc(element->base, base, E820_TYPE_MEMORY); if (ram_element == NULL) { return (ENOMEM); } TAILQ_INSERT_BEFORE(element, ram_element, chain); element->base = end; } return (0); } static uint64_t e820_alloc_highest(const uint64_t max_address, const uint64_t length, const uint64_t alignment, const enum e820_memory_type type) { struct e820_element *element; TAILQ_FOREACH_REVERSE(element, &e820_table, e820_table, chain) { uint64_t address, base, end; end = MIN(max_address, element->end); base = roundup2(element->base, alignment); /* * If end - length == 0, we would allocate memory at address 0. This * address is mostly unusable and we should avoid allocating it. * Therefore, search for another block in that case. */ if (element->type != E820_TYPE_MEMORY || end < base || end - base < length || end - length == 0) { continue; } address = rounddown2(end - length, alignment); if (e820_add_entry(address, address + length, type) != 0) { return (0); } return (address); } return (0); } static uint64_t e820_alloc_lowest(const uint64_t min_address, const uint64_t length, const uint64_t alignment, const enum e820_memory_type type) { struct e820_element *element; TAILQ_FOREACH(element, &e820_table, chain) { uint64_t base, end; end = element->end; base = MAX(min_address, roundup2(element->base, alignment)); /* * If base == 0, we would allocate memory at address 0. This * address is mostly unusable and we should avoid allocating it. * Therefore, search for another block in that case. */ if (element->type != E820_TYPE_MEMORY || end < base || end - base < length || base == 0) { continue; } if (e820_add_entry(base, base + length, type) != 0) { return (0); } return (base); } return (0); } uint64_t e820_alloc(const uint64_t address, const uint64_t length, const uint64_t alignment, const enum e820_memory_type type, const enum e820_allocation_strategy strategy) { assert(powerof2(alignment)); assert((address & (alignment - 1)) == 0); switch (strategy) { case E820_ALLOCATE_ANY: /* * Allocate any address. Therefore, ignore the address parameter * and reuse the code path for allocating the lowest address. */ return (e820_alloc_lowest(0, length, alignment, type)); case E820_ALLOCATE_LOWEST: return (e820_alloc_lowest(address, length, alignment, type)); case E820_ALLOCATE_HIGHEST: return (e820_alloc_highest(address, length, alignment, type)); case E820_ALLOCATE_SPECIFIC: if (e820_add_entry(address, address + length, type) != 0) { return (0); } return (address); } return (0); } int e820_init(struct vmctx *const ctx) { uint64_t lowmem_size, highmem_size; int error; TAILQ_INIT(&e820_table); lowmem_size = vm_get_lowmem_size(ctx); error = e820_add_entry(0, lowmem_size, E820_TYPE_MEMORY); if (error) { warnx("%s: Could not add lowmem", __func__); return (error); } highmem_size = vm_get_highmem_size(ctx); if (highmem_size != 0) { error = e820_add_entry(4 * GB, 4 * GB + highmem_size, E820_TYPE_MEMORY); if (error) { warnx("%s: Could not add highmem", __func__); return (error); } } error = e820_add_memory_hole(E820_VGA_MEM_BASE, E820_VGA_MEM_END); if (error) { warnx("%s: Could not add VGA memory", __func__); return (error); } error = e820_add_memory_hole(E820_ROM_MEM_BASE, E820_ROM_MEM_END); if (error) { warnx("%s: Could not add ROM area", __func__); return (error); } return (0); } int e820_finalize(void) { struct qemu_fwcfg_item *e820_fwcfg_item; int error; e820_fwcfg_item = e820_get_fwcfg_item(); if (e820_fwcfg_item == NULL) { warnx("invalid e820 table"); return (ENOMEM); } error = qemu_fwcfg_add_file("etc/e820", e820_fwcfg_item->size, e820_fwcfg_item->data); if (error != 0) { warnx("could not add qemu fwcfg etc/e820"); free(e820_fwcfg_item->data); free(e820_fwcfg_item); return (error); } free(e820_fwcfg_item); return (0); } /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2021 Beckhoff Automation GmbH & Co. KG * Author: Corvin Köhne */ #pragma once #include #include "qemu_fwcfg.h" enum e820_memory_type { E820_TYPE_MEMORY = 1, E820_TYPE_RESERVED = 2, E820_TYPE_ACPI = 3, E820_TYPE_NVS = 4 }; enum e820_allocation_strategy { /* allocate any address */ E820_ALLOCATE_ANY, /* allocate lowest address larger than address */ E820_ALLOCATE_LOWEST, /* allocate highest address lower than address */ E820_ALLOCATE_HIGHEST, /* allocate a specific address */ E820_ALLOCATE_SPECIFIC }; struct e820_entry { uint64_t base; uint64_t length; uint32_t type; } __packed; #define E820_ALIGNMENT_NONE 1 uint64_t e820_alloc(const uint64_t address, const uint64_t length, const uint64_t alignment, const enum e820_memory_type type, const enum e820_allocation_strategy strategy); void e820_dump_table(void); int e820_init(struct vmctx *const ctx); int e820_finalize(void); /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2015 Peter Grehan * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * Guest firmware interface. Uses i/o ports x510/x511 as Qemu does, * but with a request/response messaging protocol. */ #include #include #include #include #include #include #include #include #include "bhyverun.h" #include "inout.h" #include "fwctl.h" /* * Messaging protocol base operations */ #define OP_NULL 1 #define OP_ECHO 2 #define OP_GET 3 #define OP_GET_LEN 4 #define OP_SET 5 #define OP_MAX OP_SET /* I/O ports */ #define FWCTL_OUT 0x510 #define FWCTL_IN 0x511 /* * Back-end state-machine */ static enum state { IDENT, REQ, RESP } be_state; static uint8_t sig[] = { 'B', 'H', 'Y', 'V' }; static u_int ident_idx; struct op_info { int op; int (*op_start)(uint32_t len); void (*op_data)(uint32_t data); int (*op_result)(struct iovec **data); void (*op_done)(struct iovec *data); }; static struct op_info *ops[OP_MAX+1]; /* Return 0-padded uint32_t */ static uint32_t fwctl_send_rest(uint8_t *data, size_t len) { union { uint8_t c[4]; uint32_t w; } u; size_t i; u.w = 0; for (i = 0; i < len; i++) u.c[i] = *data++; return (u.w); } /* * error op dummy proto - drop all data sent and return an error */ static int errop_code; static void errop_set(int err) { errop_code = err; } static int errop_start(uint32_t len __unused) { errop_code = ENOENT; /* accept any length */ return (errop_code); } static void errop_data(uint32_t data __unused) { /* ignore */ } static int errop_result(struct iovec **data) { /* no data to send back; always successful */ *data = NULL; return (errop_code); } static void errop_done(struct iovec *data __unused) { /* assert data is NULL */ } static struct op_info errop_info = { .op_start = errop_start, .op_data = errop_data, .op_result = errop_result, .op_done = errop_done }; /* OID search */ SET_DECLARE(ctl_set, struct ctl); CTL_NODE("hw.ncpu", &guest_ncpus, sizeof(guest_ncpus)); static struct ctl * ctl_locate(const char *str, int maxlen) { struct ctl *cp, **cpp; SET_FOREACH(cpp, ctl_set) { cp = *cpp; if (!strncmp(str, cp->c_oid, maxlen)) return (cp); } return (NULL); } /* uefi-sysctl get-len */ #define FGET_STRSZ 80 static struct iovec fget_biov[2]; static char fget_str[FGET_STRSZ]; static struct { size_t f_sz; uint32_t f_data[1024]; } fget_buf; static int fget_cnt; static size_t fget_size; static int fget_start(uint32_t len) { if (len > FGET_STRSZ) return(E2BIG); fget_cnt = 0; return (0); } static void fget_data(uint32_t data) { assert(fget_cnt + sizeof(uint32_t) <= sizeof(fget_str)); memcpy(&fget_str[fget_cnt], &data, sizeof(data)); fget_cnt += sizeof(uint32_t); } static int fget_result(struct iovec **data, int val) { struct ctl *cp; int err; err = 0; /* Locate the OID */ cp = ctl_locate(fget_str, fget_cnt); if (cp == NULL) { *data = NULL; err = ENOENT; } else { if (val) { /* For now, copy the len/data into a buffer */ memset(&fget_buf, 0, sizeof(fget_buf)); fget_buf.f_sz = cp->c_len; memcpy(fget_buf.f_data, cp->c_data, cp->c_len); fget_biov[0].iov_base = (char *)&fget_buf; fget_biov[0].iov_len = sizeof(fget_buf.f_sz) + cp->c_len; } else { fget_size = cp->c_len; fget_biov[0].iov_base = (char *)&fget_size; fget_biov[0].iov_len = sizeof(fget_size); } fget_biov[1].iov_base = NULL; fget_biov[1].iov_len = 0; *data = fget_biov; } return (err); } static void fget_done(struct iovec *data __unused) { /* nothing needs to be freed */ } static int fget_len_result(struct iovec **data) { return (fget_result(data, 0)); } static int fget_val_result(struct iovec **data) { return (fget_result(data, 1)); } static struct op_info fgetlen_info = { .op_start = fget_start, .op_data = fget_data, .op_result = fget_len_result, .op_done = fget_done }; static struct op_info fgetval_info = { .op_start = fget_start, .op_data = fget_data, .op_result = fget_val_result, .op_done = fget_done }; static struct req_info { int req_error; u_int req_count; uint32_t req_size; uint32_t req_type; uint32_t req_txid; struct op_info *req_op; int resp_error; int resp_count; size_t resp_size; size_t resp_off; struct iovec *resp_biov; } rinfo; static void fwctl_response_done(void) { (*rinfo.req_op->op_done)(rinfo.resp_biov); /* reinit the req data struct */ memset(&rinfo, 0, sizeof(rinfo)); } static void fwctl_request_done(void) { rinfo.resp_error = (*rinfo.req_op->op_result)(&rinfo.resp_biov); /* XXX only a single vector supported at the moment */ rinfo.resp_off = 0; if (rinfo.resp_biov == NULL) { rinfo.resp_size = 0; } else { rinfo.resp_size = rinfo.resp_biov[0].iov_len; } } static int fwctl_request_start(void) { int err; /* Data size doesn't include header */ rinfo.req_size -= 12; rinfo.req_op = &errop_info; if (rinfo.req_type <= OP_MAX && ops[rinfo.req_type] != NULL) rinfo.req_op = ops[rinfo.req_type]; err = (*rinfo.req_op->op_start)(rinfo.req_size); if (err) { errop_set(err); rinfo.req_op = &errop_info; } /* Catch case of zero-length message here */ if (rinfo.req_size == 0) { fwctl_request_done(); return (1); } return (0); } static int fwctl_request_data(uint32_t value) { /* Make sure remaining size is > 0 */ assert(rinfo.req_size > 0); if (rinfo.req_size <= sizeof(uint32_t)) rinfo.req_size = 0; else rinfo.req_size -= sizeof(uint32_t); (*rinfo.req_op->op_data)(value); if (rinfo.req_size < sizeof(uint32_t)) { fwctl_request_done(); return (1); } return (0); } static int fwctl_request(uint32_t value) { int ret; ret = 0; switch (rinfo.req_count) { case 0: /* Verify size */ if (value < 12) { printf("msg size error"); exit(4); } rinfo.req_size = value; rinfo.req_count = 1; break; case 1: rinfo.req_type = value; rinfo.req_count++; break; case 2: rinfo.req_txid = value; rinfo.req_count++; ret = fwctl_request_start(); break; default: ret = fwctl_request_data(value); break; } return (ret); } static int fwctl_response(uint32_t *retval) { uint8_t *dp; ssize_t remlen; switch(rinfo.resp_count) { case 0: /* 4 x u32 header len + data */ *retval = 4*sizeof(uint32_t) + roundup(rinfo.resp_size, sizeof(uint32_t)); rinfo.resp_count++; break; case 1: *retval = rinfo.req_type; rinfo.resp_count++; break; case 2: *retval = rinfo.req_txid; rinfo.resp_count++; break; case 3: *retval = rinfo.resp_error; rinfo.resp_count++; break; default: remlen = rinfo.resp_size - rinfo.resp_off; dp = (uint8_t *)rinfo.resp_biov->iov_base + rinfo.resp_off; if (remlen >= (ssize_t)sizeof(uint32_t)) { memcpy(retval, dp, sizeof(uint32_t)); } else if (remlen > 0) { *retval = fwctl_send_rest(dp, remlen); } rinfo.resp_off += sizeof(uint32_t); break; } if (rinfo.resp_count > 3 && rinfo.resp_off >= rinfo.resp_size) { fwctl_response_done(); return (1); } return (0); } static void fwctl_reset(void) { switch (be_state) { case RESP: /* If a response was generated but not fully read, discard it. */ fwctl_response_done(); break; case REQ: /* Discard partially-received request. */ memset(&rinfo, 0, sizeof(rinfo)); break; case IDENT: break; } be_state = IDENT; ident_idx = 0; } /* * i/o port handling. */ static uint8_t fwctl_inb(void) { uint8_t retval; retval = 0xff; switch (be_state) { case IDENT: retval = sig[ident_idx++]; if (ident_idx >= sizeof(sig)) be_state = REQ; break; default: break; } return (retval); } static void fwctl_outw(uint16_t val) { if (val == 0) { /* * The guest wants to read the signature. It's possible that the * guest is unaware of the fwctl state at this moment. For that * reason, reset the state machine unconditionally. */ fwctl_reset(); } } static uint32_t fwctl_inl(void) { uint32_t retval; switch (be_state) { case RESP: if (fwctl_response(&retval)) be_state = REQ; break; default: retval = 0xffffffff; break; } return (retval); } static void fwctl_outl(uint32_t val) { switch (be_state) { case REQ: if (fwctl_request(val)) be_state = RESP; default: break; } } static int fwctl_handler(struct vmctx *ctx __unused, int in, int port __unused, int bytes, uint32_t *eax, void *arg __unused) { if (in) { if (bytes == 1) *eax = fwctl_inb(); else if (bytes == 4) *eax = fwctl_inl(); else *eax = 0xffff; } else { if (bytes == 2) fwctl_outw(*eax); else if (bytes == 4) fwctl_outl(*eax); } return (0); } void fwctl_init(void) { struct inout_port iop; int error; bzero(&iop, sizeof(iop)); iop.name = "fwctl_wreg"; iop.port = FWCTL_OUT; iop.size = 1; iop.flags = IOPORT_F_INOUT; iop.handler = fwctl_handler; error = register_inout(&iop); assert(error == 0); bzero(&iop, sizeof(iop)); iop.name = "fwctl_rreg"; iop.port = FWCTL_IN; iop.size = 1; iop.flags = IOPORT_F_IN; iop.handler = fwctl_handler; error = register_inout(&iop); assert(error == 0); ops[OP_GET_LEN] = &fgetlen_info; ops[OP_GET] = &fgetval_info; be_state = IDENT; } /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2015 Peter Grehan * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef _FWCTL_H_ #define _FWCTL_H_ #include /* * Linker set api for export of information to guest firmware via * a sysctl-like OID interface */ struct ctl { const char *c_oid; const void *c_data; const int c_len; }; #define CTL_NODE(oid, data, len) \ static struct ctl __CONCAT(__ctl, __LINE__) = { \ oid, \ (data), \ (len), \ }; \ DATA_SET(ctl_set, __CONCAT(__ctl, __LINE__)) void fwctl_init(void); #endif /* _FWCTL_H_ */ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2011 NetApp, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * This file and its contents are supplied under the terms of the * Common Development and Distribution License ("CDDL"), version 1.0. * You may only use this file in accordance with the terms of version * 1.0 of the CDDL. * * A full copy of the text of the CDDL should have accompanied this * source. A copy of the CDDL is also available via the Internet at * http://www.illumos.org/license/CDDL. * * Copyright 2020 Oxide Computer Company */ #include #include #include #include #include #include #include #include #include #include #include #include "bhyverun.h" #include "config.h" #include "inout.h" SET_DECLARE(inout_port_set, struct inout_port); #define MAX_IOPORTS (1 << 16) #define VERIFY_IOPORT(port, size) \ assert((port) >= 0 && (size) > 0 && ((port) + (size)) <= MAX_IOPORTS) struct inout_handler { const char *name; int flags; inout_func_t handler; void *arg; }; static struct inout_handler inout_handlers[MAX_IOPORTS]; static int default_inout(struct vmctx *ctx __unused, int in, int port __unused, int bytes, uint32_t *eax, void *arg __unused) { if (in) { switch (bytes) { case 4: *eax = 0xffffffff; break; case 2: *eax = 0xffff; break; case 1: *eax = 0xff; break; } } return (0); } static void register_default_iohandler(int start, int size) { struct inout_port iop; VERIFY_IOPORT(start, size); bzero(&iop, sizeof(iop)); iop.name = "default"; iop.port = start; iop.size = size; iop.flags = IOPORT_F_INOUT | IOPORT_F_DEFAULT; iop.handler = default_inout; register_inout(&iop); } int emulate_inout(struct vmctx *ctx, struct vcpu *vcpu, struct vm_inout *inout) { struct inout_handler handler; inout_func_t hfunc; void *harg; int error; uint8_t bytes; bool in; bytes = inout->bytes; in = (inout->flags & INOUT_IN) != 0; assert(bytes == 1 || bytes == 2 || bytes == 4); handler = inout_handlers[inout->port]; hfunc = handler.handler; harg = handler.arg; if (hfunc == default_inout && get_config_bool_default("x86.strictio", false)) return (-1); if (in) { if (!(handler.flags & IOPORT_F_IN)) return (-1); } else { if (!(handler.flags & IOPORT_F_OUT)) return (-1); } error = hfunc(ctx, in, inout->port, bytes, &inout->eax, harg); return (error); } void init_inout(void) { struct inout_port **iopp, *iop; /* * Set up the default handler for all ports */ register_default_iohandler(0, MAX_IOPORTS); /* * Overwrite with specified handlers */ SET_FOREACH(iopp, inout_port_set) { iop = *iopp; assert(iop->port < MAX_IOPORTS); inout_handlers[iop->port].name = iop->name; inout_handlers[iop->port].flags = iop->flags; inout_handlers[iop->port].handler = iop->handler; inout_handlers[iop->port].arg = NULL; } } int register_inout(struct inout_port *iop) { int i; VERIFY_IOPORT(iop->port, iop->size); /* * Verify that the new registration is not overwriting an already * allocated i/o range. */ if ((iop->flags & IOPORT_F_DEFAULT) == 0) { for (i = iop->port; i < iop->port + iop->size; i++) { if ((inout_handlers[i].flags & IOPORT_F_DEFAULT) == 0) return (-1); } } for (i = iop->port; i < iop->port + iop->size; i++) { inout_handlers[i].name = iop->name; inout_handlers[i].flags = iop->flags; inout_handlers[i].handler = iop->handler; inout_handlers[i].arg = iop->arg; } return (0); } int unregister_inout(struct inout_port *iop) { VERIFY_IOPORT(iop->port, iop->size); assert(inout_handlers[iop->port].name == iop->name); register_default_iohandler(iop->port, iop->size); return (0); } /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2011 NetApp, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * This file and its contents are supplied under the terms of the * Common Development and Distribution License ("CDDL"), version 1.0. * You may only use this file in accordance with the terms of version * 1.0 of the CDDL. * * A full copy of the text of the CDDL should have accompanied this * source. A copy of the CDDL is also available via the Internet at * http://www.illumos.org/license/CDDL. * * Copyright 2014 Pluribus Networks Inc. */ #ifndef _INOUT_H_ #define _INOUT_H_ #include struct vcpu; struct vmctx; struct vm_exit; #ifndef __FreeBSD__ struct vm_inout; #endif /* * inout emulation handlers return 0 on success and -1 on failure. */ typedef int (*inout_func_t)(struct vmctx *ctx, int in, int port, int bytes, uint32_t *eax, void *arg); struct inout_port { const char *name; int port; int size; int flags; inout_func_t handler; void *arg; }; #define IOPORT_F_IN 0x1 #define IOPORT_F_OUT 0x2 #define IOPORT_F_INOUT (IOPORT_F_IN | IOPORT_F_OUT) /* * The following flags are used internally and must not be used by * device models. */ #define IOPORT_F_DEFAULT 0x80000000 /* claimed by default handler */ #define INOUT_PORT(name, port, flags, handler) \ static struct inout_port __CONCAT(__inout_port, __LINE__) = { \ #name, \ (port), \ 1, \ (flags), \ (handler), \ 0 \ }; \ DATA_SET(inout_port_set, __CONCAT(__inout_port, __LINE__)) void init_inout(void); #ifdef __FreeBSD__ int emulate_inout(struct vmctx *, struct vcpu *vcpu, struct vm_exit *vmexit); #else int emulate_inout(struct vmctx *, struct vcpu *vcpu, struct vm_inout *inout); #endif int register_inout(struct inout_port *iop); int unregister_inout(struct inout_port *iop); #endif /* _INOUT_H_ */ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2014 Hudson River Trading LLC * Written by: John H. Baldwin * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include #include #include #include #include "bootrom.h" #include "ioapic.h" #include "pci_emul.h" #include "pci_lpc.h" /* * Assign PCI INTx interrupts to I/O APIC pins in a round-robin * fashion. Note that we have no idea what the HPET is using, but the * HPET is also programmable whereas this is intended for hardwired * PCI interrupts. * * This assumes a single I/O APIC where pins >= 16 are permitted for * PCI devices. */ static int pci_pins; void ioapic_init(struct vmctx *ctx) { if (vm_ioapic_pincount(ctx, &pci_pins) < 0) { pci_pins = 0; return; } /* Ignore the first 16 pins. */ if (pci_pins <= 16) { pci_pins = 0; return; } pci_pins -= 16; } int ioapic_pci_alloc_irq(struct pci_devinst *pi) { static int last_pin; if (pci_pins == 0) return (-1); if (bootrom_boot()) { /* For external bootrom use fixed mapping. */ return (16 + (4 + pi->pi_slot + pi->pi_lintr.pin) % 8); } return (16 + (last_pin++ % pci_pins)); } /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2014 Hudson River Trading LLC * Written by: John H. Baldwin * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef _IOAPIC_H_ #define _IOAPIC_H_ struct pci_devinst; /* * Allocate a PCI IRQ from the I/O APIC. */ void ioapic_init(struct vmctx *ctx); int ioapic_pci_alloc_irq(struct pci_devinst *pi); #endif /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright 2020 Conrad Meyer . All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include #include #include #include #include struct vm; struct vm_hpet_cap; #include #include #include #include #include #include "kernemu_dev.h" #include "mem.h" static int apic_handler(struct vcpu *vcpu, int dir, uint64_t addr, int size, uint64_t *val, void *arg1 __unused, long arg2 __unused) { if (vm_readwrite_kernemu_device(vcpu, addr, (dir == MEM_F_WRITE), size, val) != 0) return (errno); return (0); } static struct mem_range lapic_mmio = { .name = "kern-lapic-mmio", .base = DEFAULT_APIC_BASE, .size = PAGE_SIZE, .flags = MEM_F_RW | MEM_F_IMMUTABLE, .handler = apic_handler, }; static struct mem_range ioapic_mmio = { .name = "kern-ioapic-mmio", .base = VIOAPIC_BASE, .size = VIOAPIC_SIZE, .flags = MEM_F_RW | MEM_F_IMMUTABLE, .handler = apic_handler, }; static struct mem_range hpet_mmio = { .name = "kern-hpet-mmio", .base = VHPET_BASE, .size = VHPET_SIZE, .flags = MEM_F_RW | MEM_F_IMMUTABLE, .handler = apic_handler, }; void kernemu_dev_init(void) { int rc; rc = register_mem(&lapic_mmio); if (rc != 0) errc(4, rc, "register_mem: LAPIC (0x%08x)", (unsigned)lapic_mmio.base); rc = register_mem(&ioapic_mmio); if (rc != 0) errc(4, rc, "register_mem: IOAPIC (0x%08x)", (unsigned)ioapic_mmio.base); rc = register_mem(&hpet_mmio); if (rc != 0) errc(4, rc, "register_mem: HPET (0x%08x)", (unsigned)hpet_mmio.base); } /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright 2020 Conrad Meyer . All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #pragma once void kernemu_dev_init(void); /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2012 NetApp, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include #include #include #include #include #include "acpi.h" #include "debug.h" #include "bhyverun.h" #include "mptbl.h" #include "pci_emul.h" #define MPTABLE_BASE 0xE0000 /* floating pointer length + maximum length of configuration table */ #define MPTABLE_MAX_LENGTH (65536 + 16) #define LAPIC_PADDR 0xFEE00000 #define LAPIC_VERSION 16 #define IOAPIC_PADDR 0xFEC00000 #define IOAPIC_VERSION 0x11 #define MP_SPECREV 4 #define MPFP_SIG "_MP_" /* Configuration header defines */ #define MPCH_SIG "PCMP" #define MPCH_OEMID "BHyVe " #define MPCH_OEMID_LEN 8 #define MPCH_PRODID "Hypervisor " #define MPCH_PRODID_LEN 12 /* Processor entry defines */ #define MPEP_SIG_FAMILY 6 /* XXX bhyve should supply this */ #define MPEP_SIG_MODEL 26 #define MPEP_SIG_STEPPING 5 #define MPEP_SIG \ ((MPEP_SIG_FAMILY << 8) | \ (MPEP_SIG_MODEL << 4) | \ (MPEP_SIG_STEPPING)) #define MPEP_FEATURES (0xBFEBFBFF) /* XXX Intel i7 */ /* Number of local intr entries */ #define MPEII_NUM_LOCAL_IRQ 2 /* Bus entry defines */ #define MPE_NUM_BUSES 2 #define MPE_BUSNAME_LEN 6 #define MPE_BUSNAME_ISA "ISA " #define MPE_BUSNAME_PCI "PCI " static void *oem_tbl_start; static int oem_tbl_size; static uint8_t mpt_compute_checksum(void *base, size_t len) { uint8_t *bytes; uint8_t sum; for(bytes = base, sum = 0; len > 0; len--) { sum += *bytes++; } return (256 - sum); } static void mpt_build_mpfp(mpfps_t mpfp, vm_paddr_t gpa) { memset(mpfp, 0, sizeof(*mpfp)); memcpy(mpfp->signature, MPFP_SIG, 4); mpfp->pap = gpa + sizeof(*mpfp); mpfp->length = 1; mpfp->spec_rev = MP_SPECREV; mpfp->checksum = mpt_compute_checksum(mpfp, sizeof(*mpfp)); } static void mpt_build_mpch(mpcth_t mpch) { memset(mpch, 0, sizeof(*mpch)); memcpy(mpch->signature, MPCH_SIG, 4); mpch->spec_rev = MP_SPECREV; memcpy(mpch->oem_id, MPCH_OEMID, MPCH_OEMID_LEN); memcpy(mpch->product_id, MPCH_PRODID, MPCH_PRODID_LEN); mpch->apic_address = LAPIC_PADDR; } static void mpt_build_proc_entries(proc_entry_ptr mpep, int ncpu) { int i; for (i = 0; i < ncpu; i++) { memset(mpep, 0, sizeof(*mpep)); mpep->type = MPCT_ENTRY_PROCESSOR; mpep->apic_id = i; // XXX mpep->apic_version = LAPIC_VERSION; mpep->cpu_flags = PROCENTRY_FLAG_EN; if (i == 0) mpep->cpu_flags |= PROCENTRY_FLAG_BP; mpep->cpu_signature = MPEP_SIG; mpep->feature_flags = MPEP_FEATURES; mpep++; } } static void mpt_build_localint_entries(int_entry_ptr mpie) { /* Hardcode LINT0 as ExtINT on all CPUs. */ memset(mpie, 0, sizeof(*mpie)); mpie->type = MPCT_ENTRY_LOCAL_INT; mpie->int_type = INTENTRY_TYPE_EXTINT; mpie->int_flags = INTENTRY_FLAGS_POLARITY_CONFORM | INTENTRY_FLAGS_TRIGGER_CONFORM; mpie->dst_apic_id = 0xff; mpie->dst_apic_int = 0; mpie++; /* Hardcode LINT1 as NMI on all CPUs. */ memset(mpie, 0, sizeof(*mpie)); mpie->type = MPCT_ENTRY_LOCAL_INT; mpie->int_type = INTENTRY_TYPE_NMI; mpie->int_flags = INTENTRY_FLAGS_POLARITY_CONFORM | INTENTRY_FLAGS_TRIGGER_CONFORM; mpie->dst_apic_id = 0xff; mpie->dst_apic_int = 1; } static void mpt_build_bus_entries(bus_entry_ptr mpeb) { memset(mpeb, 0, sizeof(*mpeb)); mpeb->type = MPCT_ENTRY_BUS; mpeb->bus_id = 0; memcpy(mpeb->bus_type, MPE_BUSNAME_PCI, MPE_BUSNAME_LEN); mpeb++; memset(mpeb, 0, sizeof(*mpeb)); mpeb->type = MPCT_ENTRY_BUS; mpeb->bus_id = 1; memcpy(mpeb->bus_type, MPE_BUSNAME_ISA, MPE_BUSNAME_LEN); } static void mpt_build_ioapic_entries(io_apic_entry_ptr mpei, int id) { memset(mpei, 0, sizeof(*mpei)); mpei->type = MPCT_ENTRY_IOAPIC; mpei->apic_id = id; mpei->apic_version = IOAPIC_VERSION; mpei->apic_flags = IOAPICENTRY_FLAG_EN; mpei->apic_address = IOAPIC_PADDR; } static int mpt_count_ioint_entries(void) { int bus, count; count = 0; for (bus = 0; bus <= PCI_BUSMAX; bus++) count += pci_count_lintr(bus); /* * Always include entries for the first 16 pins along with a entry * for each active PCI INTx pin. */ return (16 + count); } static void mpt_generate_pci_int(int bus, int slot, int pin, int pirq_pin __unused, int ioapic_irq, void *arg) { int_entry_ptr *mpiep, mpie; mpiep = arg; mpie = *mpiep; memset(mpie, 0, sizeof(*mpie)); /* * This is always after another I/O interrupt entry, so cheat * and fetch the I/O APIC ID from the prior entry. */ mpie->type = MPCT_ENTRY_INT; mpie->int_type = INTENTRY_TYPE_INT; mpie->src_bus_id = bus; mpie->src_bus_irq = slot << 2 | (pin - 1); mpie->dst_apic_id = mpie[-1].dst_apic_id; mpie->dst_apic_int = ioapic_irq; *mpiep = mpie + 1; } static void mpt_build_ioint_entries(int_entry_ptr mpie, int id) { int pin, bus; /* * The following config is taken from kernel mptable.c * mptable_parse_default_config_ints(...), for now * just use the default config, tweek later if needed. */ /* First, generate the first 16 pins. */ for (pin = 0; pin < 16; pin++) { memset(mpie, 0, sizeof(*mpie)); mpie->type = MPCT_ENTRY_INT; mpie->src_bus_id = 1; mpie->dst_apic_id = id; /* * All default configs route IRQs from bus 0 to the first 16 * pins of the first I/O APIC with an APIC ID of 2. */ mpie->dst_apic_int = pin; switch (pin) { case 0: /* Pin 0 is an ExtINT pin. */ mpie->int_type = INTENTRY_TYPE_EXTINT; break; case 2: /* IRQ 0 is routed to pin 2. */ mpie->int_type = INTENTRY_TYPE_INT; mpie->src_bus_irq = 0; break; case SCI_INT: /* ACPI SCI is level triggered and active-lo. */ mpie->int_flags = INTENTRY_FLAGS_POLARITY_ACTIVELO | INTENTRY_FLAGS_TRIGGER_LEVEL; mpie->int_type = INTENTRY_TYPE_INT; mpie->src_bus_irq = SCI_INT; break; default: /* All other pins are identity mapped. */ mpie->int_type = INTENTRY_TYPE_INT; mpie->src_bus_irq = pin; break; } mpie++; } /* Next, generate entries for any PCI INTx interrupts. */ for (bus = 0; bus <= PCI_BUSMAX; bus++) pci_walk_lintr(bus, mpt_generate_pci_int, &mpie); } void mptable_add_oemtbl(void *tbl, int tblsz) { oem_tbl_start = tbl; oem_tbl_size = tblsz; } int mptable_build(struct vmctx *ctx, int ncpu) { mpcth_t mpch; bus_entry_ptr mpeb; io_apic_entry_ptr mpei; proc_entry_ptr mpep; mpfps_t mpfp; int_entry_ptr mpie; int ioints, bus; char *curraddr; char *startaddr; startaddr = paddr_guest2host(ctx, MPTABLE_BASE, MPTABLE_MAX_LENGTH); if (startaddr == NULL) { EPRINTLN("mptable requires mapped mem"); return (ENOMEM); } /* * There is no way to advertise multiple PCI hierarchies via MPtable * so require that there is no PCI hierarchy with a non-zero bus * number. */ for (bus = 1; bus <= PCI_BUSMAX; bus++) { if (pci_bus_configured(bus)) { EPRINTLN("MPtable is incompatible with " "multiple PCI hierarchies."); EPRINTLN("MPtable generation can be disabled " "by passing the -Y option to bhyve(8)."); return (EINVAL); } } curraddr = startaddr; mpfp = (mpfps_t)curraddr; mpt_build_mpfp(mpfp, MPTABLE_BASE); curraddr += sizeof(*mpfp); mpch = (mpcth_t)curraddr; mpt_build_mpch(mpch); curraddr += sizeof(*mpch); mpep = (proc_entry_ptr)curraddr; mpt_build_proc_entries(mpep, ncpu); curraddr += sizeof(*mpep) * ncpu; mpch->entry_count += ncpu; mpeb = (bus_entry_ptr) curraddr; mpt_build_bus_entries(mpeb); curraddr += sizeof(*mpeb) * MPE_NUM_BUSES; mpch->entry_count += MPE_NUM_BUSES; mpei = (io_apic_entry_ptr)curraddr; mpt_build_ioapic_entries(mpei, 0); curraddr += sizeof(*mpei); mpch->entry_count++; mpie = (int_entry_ptr) curraddr; ioints = mpt_count_ioint_entries(); mpt_build_ioint_entries(mpie, 0); curraddr += sizeof(*mpie) * ioints; mpch->entry_count += ioints; mpie = (int_entry_ptr)curraddr; mpt_build_localint_entries(mpie); curraddr += sizeof(*mpie) * MPEII_NUM_LOCAL_IRQ; mpch->entry_count += MPEII_NUM_LOCAL_IRQ; if (oem_tbl_start) { mpch->oem_table_pointer = curraddr - startaddr + MPTABLE_BASE; mpch->oem_table_size = oem_tbl_size; memcpy(curraddr, oem_tbl_start, oem_tbl_size); } mpch->base_table_length = curraddr - (char *)mpch; mpch->checksum = mpt_compute_checksum(mpch, mpch->base_table_length); return (0); } /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2012 NetApp, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef _MPTBL_H_ #define _MPTBL_H_ int mptable_build(struct vmctx *ctx, int ncpu); void mptable_add_oemtbl(void *tbl, int tblsz); #endif /* _MPTBL_H_ */ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2013 Neel Natu * Copyright (c) 2013 Tycho Nightingale * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * Copyright 2018 Joyent, Inc. */ #include #include #include #include #include #include #include #include "acpi.h" #include "debug.h" #include "bootrom.h" #include "config.h" #include "inout.h" #include "pci_emul.h" #include "pci_irq.h" #include "pci_lpc.h" #include "pci_passthru.h" #include "pctestdev.h" #include "tpm_device.h" #include "uart_emul.h" #define IO_ICU1 0x20 #define IO_ICU2 0xA0 SET_DECLARE(lpc_dsdt_set, struct lpc_dsdt); SET_DECLARE(lpc_sysres_set, struct lpc_sysres); #define ELCR_PORT 0x4d0 SYSRES_IO(ELCR_PORT, 2); #define IO_TIMER1_PORT 0x40 #define NMISC_PORT 0x61 SYSRES_IO(NMISC_PORT, 1); static struct pci_devinst *lpc_bridge; #define LPC_UART_NUM 4 static struct lpc_uart_softc { struct uart_ns16550_softc *uart_softc; int iobase; int irq; int enabled; } lpc_uart_softc[LPC_UART_NUM]; static const char *lpc_uart_names[LPC_UART_NUM] = { "com1", "com2", "com3", "com4" }; static const char *lpc_uart_acpi_names[LPC_UART_NUM] = { "COM1", "COM2", "COM3", "COM4" }; /* * LPC device configuration is in the following form: * [,] * For e.g. "com1,stdio" or "bootrom,/var/romfile" */ int lpc_device_parse(const char *opts) { int unit, error; char *str, *cpy, *lpcdev, *node_name; const char *romfile, *varfile, *tpm_type, *tpm_path; error = -1; str = cpy = strdup(opts); lpcdev = strsep(&str, ","); if (lpcdev != NULL) { if (strcasecmp(lpcdev, "bootrom") == 0) { romfile = strsep(&str, ","); if (romfile == NULL) { errx(4, "invalid bootrom option \"%s\"", opts); } set_config_value("bootrom", romfile); varfile = strsep(&str, ","); if (varfile == NULL) { error = 0; goto done; } if (strchr(varfile, '=') == NULL) { set_config_value("bootvars", varfile); } else { /* varfile doesn't exist, it's another config * option */ pci_parse_legacy_config(find_config_node("lpc"), varfile); } pci_parse_legacy_config(find_config_node("lpc"), str); error = 0; goto done; } if (strcasecmp(lpcdev, "tpm") == 0) { nvlist_t *nvl = create_config_node("tpm"); tpm_type = strsep(&str, ","); if (tpm_type == NULL) { errx(4, "invalid tpm type \"%s\"", opts); } set_config_value_node(nvl, "type", tpm_type); tpm_path = strsep(&str, ","); if (tpm_path == NULL) { errx(4, "invalid tpm path \"%s\"", opts); } set_config_value_node(nvl, "path", tpm_path); pci_parse_legacy_config(find_config_node("tpm"), str); set_config_value_node_if_unset(nvl, "version", "2.0"); error = 0; goto done; } for (unit = 0; unit < LPC_UART_NUM; unit++) { if (strcasecmp(lpcdev, lpc_uart_names[unit]) == 0) { asprintf(&node_name, "lpc.%s.path", lpc_uart_names[unit]); set_config_value(node_name, str); free(node_name); error = 0; goto done; } } if (strcasecmp(lpcdev, pctestdev_getname()) == 0) { asprintf(&node_name, "lpc.%s", pctestdev_getname()); set_config_bool(node_name, true); free(node_name); error = 0; goto done; } } done: free(cpy); return (error); } void lpc_print_supported_devices(void) { size_t i; printf("bootrom\n"); for (i = 0; i < LPC_UART_NUM; i++) printf("%s\n", lpc_uart_names[i]); printf("tpm\n"); printf("%s\n", pctestdev_getname()); } const char * lpc_fwcfg(void) { return (get_config_value("lpc.fwcfg")); } static void lpc_uart_intr_assert(void *arg) { struct lpc_uart_softc *sc = arg; assert(sc->irq >= 0); vm_isa_pulse_irq(lpc_bridge->pi_vmctx, sc->irq, sc->irq); } static void lpc_uart_intr_deassert(void *arg __unused) { /* * The COM devices on the LPC bus generate edge triggered interrupts, * so nothing more to do here. */ } static int lpc_uart_io_handler(struct vmctx *ctx __unused, int in, int port, int bytes, uint32_t *eax, void *arg) { int offset; struct lpc_uart_softc *sc = arg; offset = port - sc->iobase; switch (bytes) { case 1: if (in) *eax = uart_ns16550_read(sc->uart_softc, offset); else uart_ns16550_write(sc->uart_softc, offset, *eax); break; case 2: if (in) { *eax = uart_ns16550_read(sc->uart_softc, offset); *eax |= uart_ns16550_read(sc->uart_softc, offset + 1) << 8; } else { uart_ns16550_write(sc->uart_softc, offset, *eax); uart_ns16550_write(sc->uart_softc, offset + 1, *eax >> 8); } break; #ifndef __FreeBSD__ case 4: if (in) { *eax = uart_ns16550_read(sc->uart_softc, offset); *eax |= uart_ns16550_read(sc->uart_softc, offset + 1) << 8; *eax |= uart_ns16550_read(sc->uart_softc, offset + 2) << 16; *eax |= uart_ns16550_read(sc->uart_softc, offset + 3) << 24; } else { uart_ns16550_write(sc->uart_softc, offset, *eax); uart_ns16550_write(sc->uart_softc, offset + 1, *eax >> 8); uart_ns16550_write(sc->uart_softc, offset + 2, *eax >> 16); uart_ns16550_write(sc->uart_softc, offset + 3, *eax >> 24); } break; #endif default: return (-1); } return (0); } static int lpc_init(struct vmctx *ctx) { struct lpc_uart_softc *sc; struct inout_port iop; const char *backend, *name; char *node_name; int unit, error; /* COM1 and COM2 */ for (unit = 0; unit < LPC_UART_NUM; unit++) { sc = &lpc_uart_softc[unit]; name = lpc_uart_names[unit]; if (uart_legacy_alloc(unit, &sc->iobase, &sc->irq) != 0) { EPRINTLN("Unable to allocate resources for " "LPC device %s", name); return (-1); } pci_irq_reserve(sc->irq); sc->uart_softc = uart_ns16550_init(lpc_uart_intr_assert, lpc_uart_intr_deassert, sc); asprintf(&node_name, "lpc.%s.path", name); backend = get_config_value(node_name); free(node_name); if (backend != NULL && uart_ns16550_tty_open(sc->uart_softc, backend) != 0) { EPRINTLN("Unable to initialize backend '%s' " "for LPC device %s", backend, name); return (-1); } bzero(&iop, sizeof(struct inout_port)); iop.name = name; iop.port = sc->iobase; iop.size = UART_NS16550_IO_BAR_SIZE; iop.flags = IOPORT_F_INOUT; iop.handler = lpc_uart_io_handler; iop.arg = sc; error = register_inout(&iop); assert(error == 0); sc->enabled = 1; } /* pc-testdev */ asprintf(&node_name, "lpc.%s", pctestdev_getname()); if (get_config_bool_default(node_name, false)) { error = pctestdev_init(ctx); if (error) return (error); } free(node_name); return (0); } static void pci_lpc_write_dsdt(struct pci_devinst *pi) { struct lpc_dsdt **ldpp, *ldp; dsdt_line(""); dsdt_line("Device (ISA)"); dsdt_line("{"); dsdt_line(" Name (_ADR, 0x%04X%04X)", pi->pi_slot, pi->pi_func); dsdt_line(" OperationRegion (LPCR, PCI_Config, 0x00, 0x100)"); dsdt_line(" Field (LPCR, AnyAcc, NoLock, Preserve)"); dsdt_line(" {"); dsdt_line(" Offset (0x60),"); dsdt_line(" PIRA, 8,"); dsdt_line(" PIRB, 8,"); dsdt_line(" PIRC, 8,"); dsdt_line(" PIRD, 8,"); dsdt_line(" Offset (0x68),"); dsdt_line(" PIRE, 8,"); dsdt_line(" PIRF, 8,"); dsdt_line(" PIRG, 8,"); dsdt_line(" PIRH, 8"); dsdt_line(" }"); dsdt_line(""); dsdt_indent(1); SET_FOREACH(ldpp, lpc_dsdt_set) { ldp = *ldpp; ldp->handler(); } dsdt_line(""); dsdt_line("Device (PIC)"); dsdt_line("{"); dsdt_line(" Name (_HID, EisaId (\"PNP0000\"))"); dsdt_line(" Name (_CRS, ResourceTemplate ()"); dsdt_line(" {"); dsdt_indent(2); dsdt_fixed_ioport(IO_ICU1, 2); dsdt_fixed_ioport(IO_ICU2, 2); dsdt_fixed_irq(2); dsdt_unindent(2); dsdt_line(" })"); dsdt_line("}"); dsdt_line(""); dsdt_line("Device (TIMR)"); dsdt_line("{"); dsdt_line(" Name (_HID, EisaId (\"PNP0100\"))"); dsdt_line(" Name (_CRS, ResourceTemplate ()"); dsdt_line(" {"); dsdt_indent(2); dsdt_fixed_ioport(IO_TIMER1_PORT, 4); dsdt_fixed_irq(0); dsdt_unindent(2); dsdt_line(" })"); dsdt_line("}"); dsdt_unindent(1); dsdt_line("}"); } static void pci_lpc_sysres_dsdt(void) { struct lpc_sysres **lspp, *lsp; dsdt_line(""); dsdt_line("Device (SIO)"); dsdt_line("{"); dsdt_line(" Name (_HID, EisaId (\"PNP0C02\"))"); dsdt_line(" Name (_CRS, ResourceTemplate ()"); dsdt_line(" {"); dsdt_indent(2); SET_FOREACH(lspp, lpc_sysres_set) { lsp = *lspp; switch (lsp->type) { case LPC_SYSRES_IO: dsdt_fixed_ioport(lsp->base, lsp->length); break; case LPC_SYSRES_MEM: dsdt_fixed_mem32(lsp->base, lsp->length); break; } } dsdt_unindent(2); dsdt_line(" })"); dsdt_line("}"); } LPC_DSDT(pci_lpc_sysres_dsdt); static void pci_lpc_uart_dsdt(void) { struct lpc_uart_softc *sc; int unit; for (unit = 0; unit < LPC_UART_NUM; unit++) { sc = &lpc_uart_softc[unit]; if (!sc->enabled) continue; dsdt_line(""); dsdt_line("Device (%s)", lpc_uart_acpi_names[unit]); dsdt_line("{"); dsdt_line(" Name (_HID, EisaId (\"PNP0501\"))"); dsdt_line(" Name (_UID, %d)", unit + 1); dsdt_line(" Name (_CRS, ResourceTemplate ()"); dsdt_line(" {"); dsdt_indent(2); dsdt_fixed_ioport(sc->iobase, UART_NS16550_IO_BAR_SIZE); dsdt_fixed_irq(sc->irq); dsdt_unindent(2); dsdt_line(" })"); dsdt_line("}"); } } LPC_DSDT(pci_lpc_uart_dsdt); static int pci_lpc_cfgwrite(struct pci_devinst *pi, int coff, int bytes, uint32_t val) { int pirq_pin; if (bytes == 1) { pirq_pin = 0; if (coff >= 0x60 && coff <= 0x63) pirq_pin = coff - 0x60 + 1; if (coff >= 0x68 && coff <= 0x6b) pirq_pin = coff - 0x68 + 5; if (pirq_pin != 0) { pirq_write(pi->pi_vmctx, pirq_pin, val); pci_set_cfgdata8(pi, coff, pirq_read(pirq_pin)); return (0); } } return (-1); } static void pci_lpc_write(struct pci_devinst *pi __unused, int baridx __unused, uint64_t offset __unused, int size __unused, uint64_t value __unused) { } static uint64_t pci_lpc_read(struct pci_devinst *pi __unused, int baridx __unused, uint64_t offset __unused, int size __unused) { return (0); } #define LPC_DEV 0x7000 #define LPC_VENDOR 0x8086 #define LPC_REVID 0x00 #define LPC_SUBVEND_0 0x0000 #define LPC_SUBDEV_0 0x0000 #ifdef __FreeBSD__ static int pci_lpc_get_sel(struct pcisel *const sel) { assert(sel != NULL); memset(sel, 0, sizeof(*sel)); for (uint8_t slot = 0; slot <= PCI_SLOTMAX; ++slot) { uint8_t max_func = 0; sel->pc_dev = slot; sel->pc_func = 0; if (pci_host_read_config(sel, PCIR_HDRTYPE, 1) & PCIM_MFDEV) max_func = PCI_FUNCMAX; for (uint8_t func = 0; func <= max_func; ++func) { sel->pc_func = func; if (pci_host_read_config(sel, PCIR_CLASS, 1) == PCIC_BRIDGE && pci_host_read_config(sel, PCIR_SUBCLASS, 1) == PCIS_BRIDGE_ISA) { return (0); } } } warnx("%s: Unable to find host selector of LPC bridge.", __func__); return (-1); } #else /* * This function is used to find the PCI selector for the host's LPC so that * its various IDs can be used to configure the guest LPC with the same values * when the `host` keyword is used in the configuration. * On illumos we always just report that we cannot find the host LPC. This is * likely to be true in the case that we're running in a zone anyway. */ static int pci_lpc_get_sel(struct pcisel *const sel __unused) { return (-1); } #endif static int pci_lpc_init(struct pci_devinst *pi, nvlist_t *nvl) { struct pcisel sel = { 0 }; struct pcisel *selp = NULL; uint16_t device, subdevice, subvendor, vendor; uint8_t revid; /* * Do not allow more than one LPC bridge to be configured. */ if (lpc_bridge != NULL) { EPRINTLN("Only one LPC bridge is allowed."); return (-1); } /* * Enforce that the LPC can only be configured on bus 0. This * simplifies the ACPI DSDT because it can provide a decode for * all legacy i/o ports behind bus 0. */ if (pi->pi_bus != 0) { EPRINTLN("LPC bridge can be present only on bus 0."); return (-1); } if (lpc_init(pi->pi_vmctx) != 0) return (-1); if (pci_lpc_get_sel(&sel) == 0) selp = &sel; vendor = pci_config_read_reg(selp, nvl, PCIR_VENDOR, 2, LPC_VENDOR); device = pci_config_read_reg(selp, nvl, PCIR_DEVICE, 2, LPC_DEV); revid = pci_config_read_reg(selp, nvl, PCIR_REVID, 1, LPC_REVID); subvendor = pci_config_read_reg(selp, nvl, PCIR_SUBVEND_0, 2, LPC_SUBVEND_0); subdevice = pci_config_read_reg(selp, nvl, PCIR_SUBDEV_0, 2, LPC_SUBDEV_0); /* initialize config space */ pci_set_cfgdata16(pi, PCIR_VENDOR, vendor); pci_set_cfgdata16(pi, PCIR_DEVICE, device); pci_set_cfgdata8(pi, PCIR_CLASS, PCIC_BRIDGE); pci_set_cfgdata8(pi, PCIR_SUBCLASS, PCIS_BRIDGE_ISA); pci_set_cfgdata8(pi, PCIR_REVID, revid); pci_set_cfgdata16(pi, PCIR_SUBVEND_0, subvendor); pci_set_cfgdata16(pi, PCIR_SUBDEV_0, subdevice); lpc_bridge = pi; return (0); } char * lpc_pirq_name(int pin) { char *name; if (lpc_bridge == NULL) return (NULL); asprintf(&name, "\\_SB.PC00.ISA.LNK%c,", 'A' + pin - 1); return (name); } void lpc_pirq_routed(void) { int pin; if (lpc_bridge == NULL) return; for (pin = 0; pin < 4; pin++) pci_set_cfgdata8(lpc_bridge, 0x60 + pin, pirq_read(pin + 1)); for (pin = 0; pin < 4; pin++) pci_set_cfgdata8(lpc_bridge, 0x68 + pin, pirq_read(pin + 5)); } static const struct pci_devemu pci_de_lpc = { .pe_emu = "lpc", .pe_init = pci_lpc_init, .pe_write_dsdt = pci_lpc_write_dsdt, .pe_cfgwrite = pci_lpc_cfgwrite, .pe_barwrite = pci_lpc_write, .pe_barread = pci_lpc_read }; PCI_EMUL_SET(pci_de_lpc); /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2013 Neel Natu * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef _LPC_H_ #define _LPC_H_ #include typedef void (*lpc_write_dsdt_t)(void); struct lpc_dsdt { lpc_write_dsdt_t handler; }; #define LPC_DSDT(handler) \ static struct lpc_dsdt __CONCAT(__lpc_dsdt, __LINE__) = { \ (handler), \ }; \ DATA_SET(lpc_dsdt_set, __CONCAT(__lpc_dsdt, __LINE__)) enum lpc_sysres_type { LPC_SYSRES_IO, LPC_SYSRES_MEM }; struct lpc_sysres { enum lpc_sysres_type type; uint32_t base; uint32_t length; }; #define LPC_SYSRES(type, base, length) \ static struct lpc_sysres __CONCAT(__lpc_sysres, __LINE__) = { \ (type), \ (base), \ (length) \ }; \ DATA_SET(lpc_sysres_set, __CONCAT(__lpc_sysres, __LINE__)) #define SYSRES_IO(base, length) LPC_SYSRES(LPC_SYSRES_IO, base, length) #define SYSRES_MEM(base, length) LPC_SYSRES(LPC_SYSRES_MEM, base, length) int lpc_device_parse(const char *opt); void lpc_print_supported_devices(void); char *lpc_pirq_name(int pin); void lpc_pirq_routed(void); const char *lpc_fwcfg(void); #endif /* _LPC_H_ */ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2013 Hudson River Trading LLC * Written by: John H. Baldwin * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * Copyright 2018 Joyent, Inc. * Copyright 2020 Oxide Computer Company */ #include #include #include #include #include #ifndef __FreeBSD__ #include #endif #include #include #include "acpi.h" #include "inout.h" #ifdef __FreeBSD__ #include "mevent.h" #endif #include "pci_irq.h" #include "pci_lpc.h" static pthread_mutex_t pm_lock = PTHREAD_MUTEX_INITIALIZER; #ifdef __FreeBSD__ static struct mevent *power_button; static sig_t old_power_handler; #else struct vmctx *pwr_ctx; #endif static unsigned gpe0_active; static unsigned gpe0_enabled; static const unsigned gpe0_valid = (1u << GPE_VMGENC); /* * Reset Control register at I/O port 0xcf9. Bit 2 forces a system * reset when it transitions from 0 to 1. Bit 1 selects the type of * reset to attempt: 0 selects a "soft" reset, and 1 selects a "hard" * reset. */ static int reset_handler(struct vmctx *ctx __unused, int in, int port __unused, int bytes, uint32_t *eax, void *arg __unused) { int error; static uint8_t reset_control; if (bytes != 1) return (-1); if (in) *eax = reset_control; else { reset_control = *eax; /* Treat hard and soft resets the same. */ if (reset_control & 0x4) { error = vm_suspend(ctx, VM_SUSPEND_RESET); assert(error == 0 || errno == EALREADY); } } return (0); } INOUT_PORT(reset_reg, 0xCF9, IOPORT_F_INOUT, reset_handler); /* * ACPI's SCI is a level-triggered interrupt. */ static int sci_active; static void sci_assert(struct vmctx *ctx) { if (sci_active) return; vm_isa_assert_irq(ctx, SCI_INT, SCI_INT); sci_active = 1; } static void sci_deassert(struct vmctx *ctx) { if (!sci_active) return; vm_isa_deassert_irq(ctx, SCI_INT, SCI_INT); sci_active = 0; } /* * Power Management 1 Event Registers * * The only power management event supported is a power button upon * receiving SIGTERM. */ static uint16_t pm1_enable, pm1_status; #define PM1_TMR_STS 0x0001 #define PM1_BM_STS 0x0010 #define PM1_GBL_STS 0x0020 #define PM1_PWRBTN_STS 0x0100 #define PM1_SLPBTN_STS 0x0200 #define PM1_RTC_STS 0x0400 #define PM1_WAK_STS 0x8000 #define PM1_TMR_EN 0x0001 #define PM1_GBL_EN 0x0020 #define PM1_PWRBTN_EN 0x0100 #define PM1_SLPBTN_EN 0x0200 #define PM1_RTC_EN 0x0400 static void sci_update(struct vmctx *ctx) { int need_sci; /* See if the SCI should be active or not. */ need_sci = 0; if ((pm1_enable & PM1_TMR_EN) && (pm1_status & PM1_TMR_STS)) need_sci = 1; if ((pm1_enable & PM1_GBL_EN) && (pm1_status & PM1_GBL_STS)) need_sci = 1; if ((pm1_enable & PM1_PWRBTN_EN) && (pm1_status & PM1_PWRBTN_STS)) need_sci = 1; if ((pm1_enable & PM1_SLPBTN_EN) && (pm1_status & PM1_SLPBTN_STS)) need_sci = 1; if ((pm1_enable & PM1_RTC_EN) && (pm1_status & PM1_RTC_STS)) need_sci = 1; if ((gpe0_enabled & gpe0_active) != 0) need_sci = 1; if (need_sci) sci_assert(ctx); else sci_deassert(ctx); } static int pm1_status_handler(struct vmctx *ctx, int in, int port __unused, int bytes, uint32_t *eax, void *arg __unused) { if (bytes != 2) return (-1); pthread_mutex_lock(&pm_lock); if (in) *eax = pm1_status; else { /* * Writes are only permitted to clear certain bits by * writing 1 to those flags. */ pm1_status &= ~(*eax & (PM1_WAK_STS | PM1_RTC_STS | PM1_SLPBTN_STS | PM1_PWRBTN_STS | PM1_BM_STS)); sci_update(ctx); } pthread_mutex_unlock(&pm_lock); return (0); } static int pm1_enable_handler(struct vmctx *ctx, int in, int port __unused, int bytes, uint32_t *eax, void *arg __unused) { if (bytes != 2) return (-1); pthread_mutex_lock(&pm_lock); if (in) *eax = pm1_enable; else { /* * Only permit certain bits to be set. We never use * the global lock, but ACPI-CA whines profusely if it * can't set GBL_EN. */ pm1_enable = *eax & (PM1_RTC_EN | PM1_PWRBTN_EN | PM1_GBL_EN); sci_update(ctx); } pthread_mutex_unlock(&pm_lock); return (0); } INOUT_PORT(pm1_status, PM1A_EVT_ADDR, IOPORT_F_INOUT, pm1_status_handler); INOUT_PORT(pm1_enable, PM1A_EVT_ADDR + 2, IOPORT_F_INOUT, pm1_enable_handler); #ifdef __FreeBSD__ static void power_button_handler(int signal __unused, enum ev_type type __unused, void *arg) { struct vmctx *ctx; ctx = arg; pthread_mutex_lock(&pm_lock); if (!(pm1_status & PM1_PWRBTN_STS)) { pm1_status |= PM1_PWRBTN_STS; sci_update(ctx); } pthread_mutex_unlock(&pm_lock); } #else /* * Initiate graceful power off. */ /*ARGSUSED*/ static void power_button_handler(int signal, siginfo_t *type, void *cp) { /* * In theory, taking the 'pm_lock' mutex from within this signal * handler could lead to deadlock if the main thread already held this * mutex. In reality, this mutex is local to this file and all of the * other usage in this file only occurs in functions which are FreeBSD * specific (and thus currently not used). Thus, for consistency with * the other code in this file, we take the mutex, but in the future, * if these other functions are ever enabled for use on non-FreeBSD * systems and these functions could be called directly by a thread * (which would then hold the mutex), then we need to revisit the use * of this mutex in this signal handler. */ pthread_mutex_lock(&pm_lock); if (!(pm1_status & PM1_PWRBTN_STS)) { pm1_status |= PM1_PWRBTN_STS; sci_update(pwr_ctx); } pthread_mutex_unlock(&pm_lock); } #endif /* * Power Management 1 Control Register * * This is mostly unimplemented except that we wish to handle writes that * set SPL_EN to handle S5 (soft power off). */ static uint16_t pm1_control; #define PM1_SCI_EN 0x0001 #define PM1_SLP_TYP 0x1c00 #define PM1_SLP_EN 0x2000 #define PM1_ALWAYS_ZERO 0xc003 static int pm1_control_handler(struct vmctx *ctx, int in, int port __unused, int bytes, uint32_t *eax, void *arg __unused) { int error; if (bytes != 2) return (-1); if (in) *eax = pm1_control; else { /* * Various bits are write-only or reserved, so force them * to zero in pm1_control. Always preserve SCI_EN as OSPM * can never change it. */ pm1_control = (pm1_control & PM1_SCI_EN) | (*eax & ~(PM1_SLP_EN | PM1_ALWAYS_ZERO)); /* * If SLP_EN is set, check for S5. Bhyve's _S5_ method * says that '5' should be stored in SLP_TYP for S5. */ if (*eax & PM1_SLP_EN) { if ((pm1_control & PM1_SLP_TYP) >> 10 == 5) { error = vm_suspend(ctx, VM_SUSPEND_POWEROFF); assert(error == 0 || errno == EALREADY); } } } return (0); } INOUT_PORT(pm1_control, PM1A_CNT_ADDR, IOPORT_F_INOUT, pm1_control_handler); #ifdef __FreeBSD__ SYSRES_IO(PM1A_EVT_ADDR, 8); #endif void acpi_raise_gpe(struct vmctx *ctx, unsigned bit) { unsigned mask; assert(bit < (IO_GPE0_LEN * (8 / 2))); mask = (1u << bit); assert((mask & ~gpe0_valid) == 0); pthread_mutex_lock(&pm_lock); gpe0_active |= mask; sci_update(ctx); pthread_mutex_unlock(&pm_lock); } static int gpe0_sts(struct vmctx *ctx, int in, int port __unused, int bytes, uint32_t *eax, void *arg __unused) { /* * ACPI 6.2 specifies the GPE register blocks are accessed * byte-at-a-time. */ if (bytes != 1) return (-1); pthread_mutex_lock(&pm_lock); if (in) *eax = gpe0_active; else { /* W1C */ gpe0_active &= ~(*eax & gpe0_valid); sci_update(ctx); } pthread_mutex_unlock(&pm_lock); return (0); } INOUT_PORT(gpe0_sts, IO_GPE0_STS, IOPORT_F_INOUT, gpe0_sts); static int gpe0_en(struct vmctx *ctx, int in, int port __unused, int bytes, uint32_t *eax, void *arg __unused) { if (bytes != 1) return (-1); pthread_mutex_lock(&pm_lock); if (in) *eax = gpe0_enabled; else { gpe0_enabled = (*eax & gpe0_valid); sci_update(ctx); } pthread_mutex_unlock(&pm_lock); return (0); } INOUT_PORT(gpe0_en, IO_GPE0_EN, IOPORT_F_INOUT, gpe0_en); /* * ACPI SMI Command Register * * This write-only register is used to enable and disable ACPI. */ static int smi_cmd_handler(struct vmctx *ctx, int in, int port __unused, int bytes, uint32_t *eax, void *arg __unused) { assert(!in); if (bytes != 1) return (-1); pthread_mutex_lock(&pm_lock); switch (*eax) { case BHYVE_ACPI_ENABLE: pm1_control |= PM1_SCI_EN; #ifdef __FreeBSD__ if (power_button == NULL) { power_button = mevent_add(SIGTERM, EVF_SIGNAL, power_button_handler, ctx); old_power_handler = signal(SIGTERM, SIG_IGN); } #endif break; case BHYVE_ACPI_DISABLE: pm1_control &= ~PM1_SCI_EN; #ifdef __FreeBSD__ if (power_button != NULL) { mevent_delete(power_button); power_button = NULL; signal(SIGTERM, old_power_handler); } #endif break; } pthread_mutex_unlock(&pm_lock); return (0); } INOUT_PORT(smi_cmd, SMI_CMD, IOPORT_F_OUT, smi_cmd_handler); #ifdef __FreeBSD__ SYSRES_IO(SMI_CMD, 1); #endif void sci_init(struct vmctx *ctx) { /* * Mark ACPI's SCI as level trigger and bump its use count * in the PIRQ router. */ pci_irq_use(SCI_INT); vm_isa_set_irq_trigger(ctx, SCI_INT, LEVEL_TRIGGER); #ifndef __FreeBSD__ { /* * Install SIGTERM signal handler for graceful power off. */ struct sigaction act; pwr_ctx = ctx; act.sa_flags = 0; act.sa_sigaction = power_button_handler; (void) sigaction(SIGTERM, &act, NULL); } #endif } #ifndef __FreeBSD__ void pmtmr_init(struct vmctx *ctx) { int err; /* Attach in-kernel PM timer emulation to correct IO port */ err = vm_pmtmr_set_location(ctx, IO_PMTMR); assert(err == 0); } #endif /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2011 NetApp, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include #include #include "inout.h" #include "pci_lpc.h" static int post_data_handler(struct vmctx *ctx __unused, int in, int port __unused, int bytes, uint32_t *eax, void *arg __unused) { assert(in == 1); if (bytes != 1) return (-1); *eax = 0xff; /* return some garbage */ return (0); } INOUT_PORT(post, 0x84, IOPORT_F_IN, post_data_handler); SYSRES_IO(0x84, 1); /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2015 Tycho Nightingale * Copyright (c) 2015 Nahanni Systems Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include #include #include #include #include #include #include #include #include #include #include #include "atkbdc.h" #include "bhyverun.h" #include "config.h" #include "console.h" #include "debug.h" #include "ps2kbd.h" /* keyboard device commands */ #define PS2KC_RESET_DEV 0xff #define PS2KC_SET_DEFAULTS 0xf6 #define PS2KC_DISABLE 0xf5 #define PS2KC_ENABLE 0xf4 #define PS2KC_SET_TYPEMATIC 0xf3 #define PS2KC_SEND_DEV_ID 0xf2 #define PS2KC_SET_SCANCODE_SET 0xf0 #define PS2KC_ECHO 0xee #define PS2KC_SET_LEDS 0xed #define PS2KC_BAT_SUCCESS 0xaa #define PS2KC_ACK 0xfa #define PS2KBD_FIFOSZ 16 #define PS2KBD_LAYOUT_BASEDIR "/usr/share/bhyve/kbdlayout/" #define MAX_PATHNAME 256 struct fifo { uint8_t buf[PS2KBD_FIFOSZ]; int rindex; /* index to read from */ int windex; /* index to write to */ int num; /* number of bytes in the fifo */ int size; /* size of the fifo */ }; struct ps2kbd_softc { struct atkbdc_softc *atkbdc_sc; pthread_mutex_t mtx; bool enabled; struct fifo fifo; uint8_t curcmd; /* current command for next byte */ }; #define SCANCODE_E0_PREFIX 1 struct extended_translation { uint32_t keysym; uint8_t scancode; int flags; }; /* * FIXME: Pause/break and Print Screen/SysRq require special handling. */ static struct extended_translation extended_translations[128] = { {0xff08, 0x66, 0}, /* Back space */ {0xff09, 0x0d, 0}, /* Tab */ {0xff0d, 0x5a, 0}, /* Return */ {0xff1b, 0x76, 0}, /* Escape */ {0xff50, 0x6c, SCANCODE_E0_PREFIX}, /* Home */ {0xff51, 0x6b, SCANCODE_E0_PREFIX}, /* Left arrow */ {0xff52, 0x75, SCANCODE_E0_PREFIX}, /* Up arrow */ {0xff53, 0x74, SCANCODE_E0_PREFIX}, /* Right arrow */ {0xff54, 0x72, SCANCODE_E0_PREFIX}, /* Down arrow */ {0xff55, 0x7d, SCANCODE_E0_PREFIX}, /* PgUp */ {0xff56, 0x7a, SCANCODE_E0_PREFIX}, /* PgDown */ {0xff57, 0x69, SCANCODE_E0_PREFIX}, /* End */ {0xff63, 0x70, SCANCODE_E0_PREFIX}, /* Ins */ {0xff8d, 0x5a, SCANCODE_E0_PREFIX}, /* Keypad Enter */ {0xffe1, 0x12, 0}, /* Left shift */ {0xffe2, 0x59, 0}, /* Right shift */ {0xffe3, 0x14, 0}, /* Left control */ {0xffe4, 0x14, SCANCODE_E0_PREFIX}, /* Right control */ /* {0xffe7, XXX}, Left meta */ /* {0xffe8, XXX}, Right meta */ {0xffe9, 0x11, 0}, /* Left alt */ {0xfe03, 0x11, SCANCODE_E0_PREFIX}, /* AltGr */ {0xffea, 0x11, SCANCODE_E0_PREFIX}, /* Right alt */ {0xffeb, 0x1f, SCANCODE_E0_PREFIX}, /* Left Windows */ {0xffec, 0x27, SCANCODE_E0_PREFIX}, /* Right Windows */ {0xffbe, 0x05, 0}, /* F1 */ {0xffbf, 0x06, 0}, /* F2 */ {0xffc0, 0x04, 0}, /* F3 */ {0xffc1, 0x0c, 0}, /* F4 */ {0xffc2, 0x03, 0}, /* F5 */ {0xffc3, 0x0b, 0}, /* F6 */ {0xffc4, 0x83, 0}, /* F7 */ {0xffc5, 0x0a, 0}, /* F8 */ {0xffc6, 0x01, 0}, /* F9 */ {0xffc7, 0x09, 0}, /* F10 */ {0xffc8, 0x78, 0}, /* F11 */ {0xffc9, 0x07, 0}, /* F12 */ {0xffff, 0x71, SCANCODE_E0_PREFIX}, /* Del */ {0xff14, 0x7e, 0}, /* ScrollLock */ /* NumLock and Keypads*/ {0xff7f, 0x77, 0}, /* NumLock */ {0xffaf, 0x4a, SCANCODE_E0_PREFIX}, /* Keypad slash */ {0xffaa, 0x7c, 0}, /* Keypad asterisk */ {0xffad, 0x7b, 0}, /* Keypad minus */ {0xffab, 0x79, 0}, /* Keypad plus */ {0xffb7, 0x6c, 0}, /* Keypad 7 */ {0xff95, 0x6c, 0}, /* Keypad home */ {0xffb8, 0x75, 0}, /* Keypad 8 */ {0xff97, 0x75, 0}, /* Keypad up arrow */ {0xffb9, 0x7d, 0}, /* Keypad 9 */ {0xff9a, 0x7d, 0}, /* Keypad PgUp */ {0xffb4, 0x6b, 0}, /* Keypad 4 */ {0xff96, 0x6b, 0}, /* Keypad left arrow */ {0xffb5, 0x73, 0}, /* Keypad 5 */ {0xff9d, 0x73, 0}, /* Keypad empty */ {0xffb6, 0x74, 0}, /* Keypad 6 */ {0xff98, 0x74, 0}, /* Keypad right arrow */ {0xffb1, 0x69, 0}, /* Keypad 1 */ {0xff9c, 0x69, 0}, /* Keypad end */ {0xffb2, 0x72, 0}, /* Keypad 2 */ {0xff99, 0x72, 0}, /* Keypad down arrow */ {0xffb3, 0x7a, 0}, /* Keypad 3 */ {0xff9b, 0x7a, 0}, /* Keypad PgDown */ {0xffb0, 0x70, 0}, /* Keypad 0 */ {0xff9e, 0x70, 0}, /* Keypad ins */ {0xffae, 0x71, 0}, /* Keypad . */ {0xff9f, 0x71, 0}, /* Keypad del */ {0, 0, 0} /* Terminator */ }; /* ASCII to type 2 scancode lookup table */ static uint8_t ascii_translations[128] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x29, 0x16, 0x52, 0x26, 0x25, 0x2e, 0x3d, 0x52, 0x46, 0x45, 0x3e, 0x55, 0x41, 0x4e, 0x49, 0x4a, 0x45, 0x16, 0x1e, 0x26, 0x25, 0x2e, 0x36, 0x3d, 0x3e, 0x46, 0x4c, 0x4c, 0x41, 0x55, 0x49, 0x4a, 0x1e, 0x1c, 0x32, 0x21, 0x23, 0x24, 0x2b, 0x34, 0x33, 0x43, 0x3b, 0x42, 0x4b, 0x3a, 0x31, 0x44, 0x4d, 0x15, 0x2d, 0x1b, 0x2c, 0x3c, 0x2a, 0x1d, 0x22, 0x35, 0x1a, 0x54, 0x5d, 0x5b, 0x36, 0x4e, 0x0e, 0x1c, 0x32, 0x21, 0x23, 0x24, 0x2b, 0x34, 0x33, 0x43, 0x3b, 0x42, 0x4b, 0x3a, 0x31, 0x44, 0x4d, 0x15, 0x2d, 0x1b, 0x2c, 0x3c, 0x2a, 0x1d, 0x22, 0x35, 0x1a, 0x54, 0x5d, 0x5b, 0x0e, 0x00, }; /* ScanCode set1 to set2 lookup table */ static const uint8_t keyset1to2_translations[128] = { 0, 0x76, 0x16, 0x1E, 0x26, 0x25, 0x2e, 0x36, 0x3d, 0x3e, 0x46, 0x45, 0x4e, 0x55, 0x66, 0x0d, 0x15, 0x1d, 0x24, 0x2d, 0x2c, 0x35, 0x3c, 0x43, 0x44, 0x4d, 0x54, 0x5b, 0x5a, 0x14, 0x1c, 0x1b, 0x23, 0x2b, 0x34, 0x33, 0x3b, 0x42, 0x4b, 0x4c, 0x52, 0x0e, 0x12, 0x5d, 0x1a, 0x22, 0x21, 0x2a, 0x32, 0x31, 0x3a, 0x41, 0x49, 0x4a, 0x59, 0x7c, 0x11, 0x29, 0x58, 0x05, 0x06, 0x04, 0x0c, 0x03, 0x0b, 0x83, 0x0a, 0x01, 0x09, 0x77, 0x7e, 0x6c, 0x75, 0x7d, 0x7b, 0x6b, 0x73, 0x74, 0x79, 0x69, 0x72, 0x7a, 0x70, 0x71, 0x84, 0x60, 0x61, 0x78, 0x07, 0x0f, 0x17, 0x1f, 0x27, 0x2f, 0x37, 0x3f, 0x47, 0x4f, 0x56, 0x5e, 0x08, 0x10, 0x18, 0x20, 0x28, 0x30, 0x38, 0x40, 0x48, 0x50, 0x57, 0x6f, 0x13, 0x19, 0x39, 0x51, 0x53, 0x5c, 0x5f, 0x62, 0x63, 0x64, 0x65, 0x67, 0x68, 0x6a, 0x6d, 0x6e, }; static void fifo_init(struct ps2kbd_softc *sc) { struct fifo *fifo; fifo = &sc->fifo; fifo->size = sizeof(((struct fifo *)0)->buf); } static void fifo_reset(struct ps2kbd_softc *sc) { struct fifo *fifo; fifo = &sc->fifo; bzero(fifo, sizeof(struct fifo)); fifo->size = sizeof(((struct fifo *)0)->buf); } static void fifo_put(struct ps2kbd_softc *sc, uint8_t val) { struct fifo *fifo; fifo = &sc->fifo; if (fifo->num < fifo->size) { fifo->buf[fifo->windex] = val; fifo->windex = (fifo->windex + 1) % fifo->size; fifo->num++; } } static int fifo_get(struct ps2kbd_softc *sc, uint8_t *val) { struct fifo *fifo; fifo = &sc->fifo; if (fifo->num > 0) { *val = fifo->buf[fifo->rindex]; fifo->rindex = (fifo->rindex + 1) % fifo->size; fifo->num--; return (0); } return (-1); } int ps2kbd_read(struct ps2kbd_softc *sc, uint8_t *val) { int retval; pthread_mutex_lock(&sc->mtx); retval = fifo_get(sc, val); pthread_mutex_unlock(&sc->mtx); return (retval); } void ps2kbd_write(struct ps2kbd_softc *sc, uint8_t val) { pthread_mutex_lock(&sc->mtx); if (sc->curcmd) { switch (sc->curcmd) { case PS2KC_SET_TYPEMATIC: fifo_put(sc, PS2KC_ACK); break; case PS2KC_SET_SCANCODE_SET: fifo_put(sc, PS2KC_ACK); break; case PS2KC_SET_LEDS: fifo_put(sc, PS2KC_ACK); break; default: EPRINTLN("Unhandled ps2 keyboard current " "command byte 0x%02x", val); break; } sc->curcmd = 0; } else { switch (val) { case 0x00: fifo_put(sc, PS2KC_ACK); break; case PS2KC_RESET_DEV: fifo_reset(sc); fifo_put(sc, PS2KC_ACK); fifo_put(sc, PS2KC_BAT_SUCCESS); break; case PS2KC_SET_DEFAULTS: fifo_reset(sc); fifo_put(sc, PS2KC_ACK); break; case PS2KC_DISABLE: sc->enabled = false; fifo_put(sc, PS2KC_ACK); break; case PS2KC_ENABLE: sc->enabled = true; fifo_reset(sc); fifo_put(sc, PS2KC_ACK); break; case PS2KC_SET_TYPEMATIC: sc->curcmd = val; fifo_put(sc, PS2KC_ACK); break; case PS2KC_SEND_DEV_ID: fifo_put(sc, PS2KC_ACK); fifo_put(sc, 0xab); fifo_put(sc, 0x83); break; case PS2KC_SET_SCANCODE_SET: sc->curcmd = val; fifo_put(sc, PS2KC_ACK); break; case PS2KC_ECHO: fifo_put(sc, PS2KC_ECHO); break; case PS2KC_SET_LEDS: sc->curcmd = val; fifo_put(sc, PS2KC_ACK); break; default: EPRINTLN("Unhandled ps2 keyboard command " "0x%02x", val); break; } } pthread_mutex_unlock(&sc->mtx); } /* * Translate keysym to type 2 scancode and insert into keyboard buffer. */ static void ps2kbd_keysym_queue(struct ps2kbd_softc *sc, int down, uint32_t keysym, uint32_t keycode) { const struct extended_translation *trans; int e0_prefix, found; uint8_t code; assert(pthread_mutex_isowned_np(&sc->mtx)); if (keycode) { code = keyset1to2_translations[(uint8_t)(keycode & 0x7f)]; e0_prefix = ((keycode & 0x80) ? SCANCODE_E0_PREFIX : 0); found = 1; } else { found = 0; if (keysym < 0x80) { code = ascii_translations[keysym]; e0_prefix = 0; found = 1; } else { for (trans = &extended_translations[0]; trans->keysym != 0; trans++) { if (keysym == trans->keysym) { code = trans->scancode; e0_prefix = trans->flags & SCANCODE_E0_PREFIX; found = 1; break; } } } } if (!found) { EPRINTLN("Unhandled ps2 keyboard keysym 0x%x", keysym); return; } if (e0_prefix) fifo_put(sc, 0xe0); if (!down) fifo_put(sc, 0xf0); fifo_put(sc, code); } static void ps2kbd_event(int down, uint32_t keysym, uint32_t keycode, void *arg) { struct ps2kbd_softc *sc = arg; int fifo_full; pthread_mutex_lock(&sc->mtx); if (!sc->enabled) { pthread_mutex_unlock(&sc->mtx); return; } fifo_full = sc->fifo.num == PS2KBD_FIFOSZ; ps2kbd_keysym_queue(sc, down, keysym, keycode); pthread_mutex_unlock(&sc->mtx); if (!fifo_full) atkbdc_event(sc->atkbdc_sc, 1); } static void ps2kbd_update_extended_translation(uint32_t keycode, uint32_t scancode, uint32_t prefix) { int i = 0; do { if (extended_translations[i].keysym == keycode) break; } while (extended_translations[++i].keysym); if (i == (sizeof(extended_translations) / sizeof(struct extended_translation) - 1)) return; if (!extended_translations[i].keysym) { extended_translations[i].keysym = keycode; extended_translations[i+1].keysym = 0; extended_translations[i+1].scancode = 0; extended_translations[i+1].flags = 0; } extended_translations[i].scancode = (uint8_t)(scancode & 0xff); extended_translations[i].flags = (prefix ? SCANCODE_E0_PREFIX : 0); } static void ps2kbd_setkbdlayout(void) { int err; int fd; char path[MAX_PATHNAME]; char *buf, *next, *line; struct stat sb; ssize_t sz; uint8_t ascii; uint32_t keycode, scancode, prefix; snprintf(path, MAX_PATHNAME, PS2KBD_LAYOUT_BASEDIR"%s", get_config_value("keyboard.layout") ); err = stat(path, &sb); if (err) return; buf = (char *)malloc(sizeof(char) * sb.st_size); if (buf == NULL) return; fd = open(path, O_RDONLY); if (fd == -1) goto out; sz = read(fd, buf, sb.st_size); close(fd); if (sz < 0 || sz != sb.st_size) goto out; next = buf; while ((line = strsep(&next, "\n")) != NULL) { if (sscanf(line, "'%c',%x;", &ascii, &scancode) == 2) { if (ascii < 0x80) ascii_translations[ascii] = (uint8_t)(scancode & 0xff); } else if (sscanf(line, "%x,%x,%x;", &keycode, &scancode, &prefix) == 3 ) { ps2kbd_update_extended_translation(keycode, scancode, prefix); } else if (sscanf(line, "%x,%x;", &keycode, &scancode) == 2) { if (keycode < 0x80) ascii_translations[(uint8_t)(keycode & 0xff)] = (uint8_t)(scancode & 0xff); else ps2kbd_update_extended_translation(keycode, scancode, 0); } } out: free(buf); } struct ps2kbd_softc * ps2kbd_init(struct atkbdc_softc *atkbdc_sc) { struct ps2kbd_softc *sc; if (get_config_value("keyboard.layout") != NULL) ps2kbd_setkbdlayout(); sc = calloc(1, sizeof (struct ps2kbd_softc)); pthread_mutex_init(&sc->mtx, NULL); fifo_init(sc); sc->atkbdc_sc = atkbdc_sc; console_kbd_register(ps2kbd_event, sc, 1); return (sc); } /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2015 Tycho Nightingale * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef _PS2KBD_H_ #define _PS2KBD_H_ struct atkbdc_softc; struct ps2kbd_softc *ps2kbd_init(struct atkbdc_softc *sc); int ps2kbd_read(struct ps2kbd_softc *sc, uint8_t *val); void ps2kbd_write(struct ps2kbd_softc *sc, uint8_t val); #endif /* _PS2KBD_H_ */ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2015 Tycho Nightingale * Copyright (c) 2015 Nahanni Systems Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include #include #include #include #include #include #include #include #include "atkbdc.h" #include "console.h" #include "debug.h" #include "ps2mouse.h" /* mouse device commands */ #define PS2MC_RESET_DEV 0xff #define PS2MC_SET_DEFAULTS 0xf6 #define PS2MC_DISABLE 0xf5 #define PS2MC_ENABLE 0xf4 #define PS2MC_SET_SAMPLING_RATE 0xf3 #define PS2MC_SEND_DEV_ID 0xf2 #define PS2MC_SET_REMOTE_MODE 0xf0 #define PS2MC_SEND_DEV_DATA 0xeb #define PS2MC_SET_STREAM_MODE 0xea #define PS2MC_SEND_DEV_STATUS 0xe9 #define PS2MC_SET_RESOLUTION 0xe8 #define PS2MC_SET_SCALING1 0xe7 #define PS2MC_SET_SCALING2 0xe6 #define PS2MC_BAT_SUCCESS 0xaa #define PS2MC_ACK 0xfa /* mouse device id */ #define PS2MOUSE_DEV_ID 0x0 /* mouse data bits */ #define PS2M_DATA_Y_OFLOW 0x80 #define PS2M_DATA_X_OFLOW 0x40 #define PS2M_DATA_Y_SIGN 0x20 #define PS2M_DATA_X_SIGN 0x10 #define PS2M_DATA_AONE 0x08 #define PS2M_DATA_MID_BUTTON 0x04 #define PS2M_DATA_RIGHT_BUTTON 0x02 #define PS2M_DATA_LEFT_BUTTON 0x01 /* mouse status bits */ #define PS2M_STS_REMOTE_MODE 0x40 #define PS2M_STS_ENABLE_DEV 0x20 #define PS2M_STS_SCALING_21 0x10 #define PS2M_STS_MID_BUTTON 0x04 #define PS2M_STS_RIGHT_BUTTON 0x02 #define PS2M_STS_LEFT_BUTTON 0x01 #define PS2MOUSE_FIFOSZ 16 struct fifo { uint8_t buf[PS2MOUSE_FIFOSZ]; int rindex; /* index to read from */ int windex; /* index to write to */ int num; /* number of bytes in the fifo */ int size; /* size of the fifo */ }; struct ps2mouse_softc { struct atkbdc_softc *atkbdc_sc; pthread_mutex_t mtx; uint8_t status; uint8_t resolution; uint8_t sampling_rate; int ctrlenable; struct fifo fifo; uint8_t curcmd; /* current command for next byte */ int cur_x, cur_y; int delta_x, delta_y; }; static void fifo_init(struct ps2mouse_softc *sc) { struct fifo *fifo; fifo = &sc->fifo; fifo->size = sizeof(((struct fifo *)0)->buf); } static void fifo_reset(struct ps2mouse_softc *sc) { struct fifo *fifo; fifo = &sc->fifo; bzero(fifo, sizeof(struct fifo)); fifo->size = sizeof(((struct fifo *)0)->buf); } static void fifo_put(struct ps2mouse_softc *sc, uint8_t val) { struct fifo *fifo; fifo = &sc->fifo; if (fifo->num < fifo->size) { fifo->buf[fifo->windex] = val; fifo->windex = (fifo->windex + 1) % fifo->size; fifo->num++; } } static int fifo_get(struct ps2mouse_softc *sc, uint8_t *val) { struct fifo *fifo; fifo = &sc->fifo; if (fifo->num > 0) { *val = fifo->buf[fifo->rindex]; fifo->rindex = (fifo->rindex + 1) % fifo->size; fifo->num--; return (0); } return (-1); } static void movement_reset(struct ps2mouse_softc *sc) { assert(pthread_mutex_isowned_np(&sc->mtx)); sc->delta_x = 0; sc->delta_y = 0; } static void movement_update(struct ps2mouse_softc *sc, int x, int y) { sc->delta_x += x - sc->cur_x; sc->delta_y += sc->cur_y - y; sc->cur_x = x; sc->cur_y = y; } static void movement_get(struct ps2mouse_softc *sc) { uint8_t val0, val1, val2; assert(pthread_mutex_isowned_np(&sc->mtx)); val0 = PS2M_DATA_AONE; val0 |= sc->status & (PS2M_DATA_LEFT_BUTTON | PS2M_DATA_RIGHT_BUTTON | PS2M_DATA_MID_BUTTON); if (sc->delta_x >= 0) { if (sc->delta_x > 255) { val0 |= PS2M_DATA_X_OFLOW; val1 = 255; } else val1 = sc->delta_x; } else { val0 |= PS2M_DATA_X_SIGN; if (sc->delta_x < -255) { val0 |= PS2M_DATA_X_OFLOW; val1 = 255; } else val1 = sc->delta_x; } sc->delta_x = 0; if (sc->delta_y >= 0) { if (sc->delta_y > 255) { val0 |= PS2M_DATA_Y_OFLOW; val2 = 255; } else val2 = sc->delta_y; } else { val0 |= PS2M_DATA_Y_SIGN; if (sc->delta_y < -255) { val0 |= PS2M_DATA_Y_OFLOW; val2 = 255; } else val2 = sc->delta_y; } sc->delta_y = 0; if (sc->fifo.num < (sc->fifo.size - 3)) { fifo_put(sc, val0); fifo_put(sc, val1); fifo_put(sc, val2); } } static void ps2mouse_reset(struct ps2mouse_softc *sc) { assert(pthread_mutex_isowned_np(&sc->mtx)); fifo_reset(sc); movement_reset(sc); sc->status = PS2M_STS_ENABLE_DEV; sc->resolution = 4; sc->sampling_rate = 100; sc->cur_x = 0; sc->cur_y = 0; sc->delta_x = 0; sc->delta_y = 0; } int ps2mouse_read(struct ps2mouse_softc *sc, uint8_t *val) { int retval; pthread_mutex_lock(&sc->mtx); retval = fifo_get(sc, val); pthread_mutex_unlock(&sc->mtx); return (retval); } int ps2mouse_fifocnt(struct ps2mouse_softc *sc) { return (sc->fifo.num); } void ps2mouse_toggle(struct ps2mouse_softc *sc, int enable) { pthread_mutex_lock(&sc->mtx); if (enable) sc->ctrlenable = 1; else { sc->ctrlenable = 0; sc->fifo.rindex = 0; sc->fifo.windex = 0; sc->fifo.num = 0; } pthread_mutex_unlock(&sc->mtx); } void ps2mouse_write(struct ps2mouse_softc *sc, uint8_t val, int insert) { pthread_mutex_lock(&sc->mtx); fifo_reset(sc); if (sc->curcmd) { switch (sc->curcmd) { case PS2MC_SET_SAMPLING_RATE: sc->sampling_rate = val; fifo_put(sc, PS2MC_ACK); break; case PS2MC_SET_RESOLUTION: sc->resolution = val; fifo_put(sc, PS2MC_ACK); break; default: EPRINTLN("Unhandled ps2 mouse current " "command byte 0x%02x", val); break; } sc->curcmd = 0; } else if (insert) { fifo_put(sc, val); } else { switch (val) { case 0x00: fifo_put(sc, PS2MC_ACK); break; case PS2MC_RESET_DEV: ps2mouse_reset(sc); fifo_put(sc, PS2MC_ACK); fifo_put(sc, PS2MC_BAT_SUCCESS); fifo_put(sc, PS2MOUSE_DEV_ID); break; case PS2MC_SET_DEFAULTS: ps2mouse_reset(sc); fifo_put(sc, PS2MC_ACK); break; case PS2MC_DISABLE: fifo_reset(sc); sc->status &= ~PS2M_STS_ENABLE_DEV; fifo_put(sc, PS2MC_ACK); break; case PS2MC_ENABLE: fifo_reset(sc); sc->status |= PS2M_STS_ENABLE_DEV; fifo_put(sc, PS2MC_ACK); break; case PS2MC_SET_SAMPLING_RATE: sc->curcmd = val; fifo_put(sc, PS2MC_ACK); break; case PS2MC_SEND_DEV_ID: fifo_put(sc, PS2MC_ACK); fifo_put(sc, PS2MOUSE_DEV_ID); break; case PS2MC_SET_REMOTE_MODE: sc->status |= PS2M_STS_REMOTE_MODE; fifo_put(sc, PS2MC_ACK); break; case PS2MC_SEND_DEV_DATA: fifo_put(sc, PS2MC_ACK); movement_get(sc); break; case PS2MC_SET_STREAM_MODE: sc->status &= ~PS2M_STS_REMOTE_MODE; fifo_put(sc, PS2MC_ACK); break; case PS2MC_SEND_DEV_STATUS: fifo_put(sc, PS2MC_ACK); fifo_put(sc, sc->status); fifo_put(sc, sc->resolution); fifo_put(sc, sc->sampling_rate); break; case PS2MC_SET_RESOLUTION: sc->curcmd = val; fifo_put(sc, PS2MC_ACK); break; case PS2MC_SET_SCALING1: case PS2MC_SET_SCALING2: fifo_put(sc, PS2MC_ACK); break; default: fifo_put(sc, PS2MC_ACK); EPRINTLN("Unhandled ps2 mouse command " "0x%02x", val); break; } } pthread_mutex_unlock(&sc->mtx); } static void ps2mouse_event(uint8_t button, int x, int y, void *arg) { struct ps2mouse_softc *sc = arg; pthread_mutex_lock(&sc->mtx); movement_update(sc, x, y); sc->status &= ~(PS2M_STS_LEFT_BUTTON | PS2M_STS_RIGHT_BUTTON | PS2M_STS_MID_BUTTON); if (button & (1 << 0)) sc->status |= PS2M_STS_LEFT_BUTTON; if (button & (1 << 1)) sc->status |= PS2M_STS_MID_BUTTON; if (button & (1 << 2)) sc->status |= PS2M_STS_RIGHT_BUTTON; if ((sc->status & PS2M_STS_ENABLE_DEV) == 0 || !sc->ctrlenable) { /* no data reporting */ pthread_mutex_unlock(&sc->mtx); return; } movement_get(sc); pthread_mutex_unlock(&sc->mtx); if (sc->fifo.num > 0) atkbdc_event(sc->atkbdc_sc, 0); } struct ps2mouse_softc * ps2mouse_init(struct atkbdc_softc *atkbdc_sc) { struct ps2mouse_softc *sc; sc = calloc(1, sizeof (struct ps2mouse_softc)); pthread_mutex_init(&sc->mtx, NULL); fifo_init(sc); sc->atkbdc_sc = atkbdc_sc; pthread_mutex_lock(&sc->mtx); ps2mouse_reset(sc); pthread_mutex_unlock(&sc->mtx); console_ptr_register(ps2mouse_event, sc, 1); return (sc); } /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2015 Tycho Nightingale * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef _PS2MOUSE_H_ #define _PS2MOUSE_H_ struct atkbdc_softc; struct ps2mouse_softc *ps2mouse_init(struct atkbdc_softc *sc); int ps2mouse_read(struct ps2mouse_softc *sc, uint8_t *val); void ps2mouse_write(struct ps2mouse_softc *sc, uint8_t val, int insert); void ps2mouse_toggle(struct ps2mouse_softc *sc, int enable); int ps2mouse_fifocnt(struct ps2mouse_softc *sc); #endif /* _PS2MOUSE_H_ */ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2011 NetApp, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include #include #include #include #include #include "acpi.h" #include "config.h" #include "pci_lpc.h" #include "rtc.h" #define IO_RTC 0x70 #define RTC_LMEM_LSB 0x34 #define RTC_LMEM_MSB 0x35 #define RTC_HMEM_LSB 0x5b #define RTC_HMEM_SB 0x5c #define RTC_HMEM_MSB 0x5d #define m_64KB (64*1024) #define m_16MB (16*1024*1024) #define m_4GB (4ULL*1024*1024*1024) /* * Returns the current RTC time as number of seconds since 00:00:00 Jan 1, 1970 */ #ifdef __FreeBSD__ static time_t rtc_time(void) { struct tm tm; time_t t; time(&t); if (get_config_bool_default("rtc.use_localtime", true)) { localtime_r(&t, &tm); t = timegm(&tm); } return (t); } #else /* __FreeBSD__ */ static void rtc_time(timespec_t *ts) { (void) clock_gettime(CLOCK_REALTIME, ts); if (get_config_bool_default("rtc.use_localtime", true)) { struct tm tm; localtime_r(&ts->tv_sec, &tm); ts->tv_sec = timegm(&tm); } } #endif /* __FreeBSD__ */ void rtc_init(struct vmctx *ctx) { size_t himem; size_t lomem; int err; /* XXX init diag/reset code/equipment/checksum ? */ /* * Report guest memory size in nvram cells as required by UEFI. * Little-endian encoding. * 0x34/0x35 - 64KB chunks above 16MB, below 4GB * 0x5b/0x5c/0x5d - 64KB chunks above 4GB */ lomem = (vm_get_lowmem_size(ctx) - m_16MB) / m_64KB; err = vm_rtc_write(ctx, RTC_LMEM_LSB, lomem); assert(err == 0); err = vm_rtc_write(ctx, RTC_LMEM_MSB, lomem >> 8); assert(err == 0); himem = vm_get_highmem_size(ctx) / m_64KB; err = vm_rtc_write(ctx, RTC_HMEM_LSB, himem); assert(err == 0); err = vm_rtc_write(ctx, RTC_HMEM_SB, himem >> 8); assert(err == 0); err = vm_rtc_write(ctx, RTC_HMEM_MSB, himem >> 16); assert(err == 0); #ifdef __FreeBSD__ err = vm_rtc_settime(ctx, rtc_time()); #else timespec_t ts; rtc_time(&ts); err = vm_rtc_settime(ctx, &ts); #endif assert(err == 0); } static void rtc_dsdt(void) { dsdt_line(""); dsdt_line("Device (RTC)"); dsdt_line("{"); dsdt_line(" Name (_HID, EisaId (\"PNP0B00\"))"); dsdt_line(" Name (_CRS, ResourceTemplate ()"); dsdt_line(" {"); dsdt_indent(2); dsdt_fixed_ioport(IO_RTC, 2); dsdt_fixed_irq(8); dsdt_unindent(2); dsdt_line(" })"); dsdt_line("}"); } LPC_DSDT(rtc_dsdt); /* * Reserve the extended RTC I/O ports although they are not emulated at this * time. */ SYSRES_IO(0x72, 6); /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2013 Peter Grehan * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef _RTC_H_ #define _RTC_H_ void rtc_init(struct vmctx *ctx); #endif /* _RTC_H_ */ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2012 NetApp, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * This file and its contents are supplied under the terms of the * Common Development and Distribution License ("CDDL"), version 1.0. * You may only use this file in accordance with the terms of version * 1.0 of the CDDL. * * A full copy of the text of the CDDL should have accompanied this * source. A copy of the CDDL is also available via the Internet at * http://www.illumos.org/license/CDDL. * * Copyright 2020 Oxide Computer Company */ #include #include #include #include #include #include #include #include "bhyverun.h" #include "spinup_ap.h" #ifdef __FreeBSD__ static void spinup_ap_realmode(struct vcpu *newcpu, uint64_t rip) { int vector, error; uint16_t cs; uint64_t desc_base; uint32_t desc_limit, desc_access; vector = rip >> PAGE_SHIFT; /* * Update the %cs and %rip of the guest so that it starts * executing real mode code at 'vector << 12'. */ error = vm_set_register(newcpu, VM_REG_GUEST_RIP, 0); assert(error == 0); error = vm_get_desc(newcpu, VM_REG_GUEST_CS, &desc_base, &desc_limit, &desc_access); assert(error == 0); desc_base = vector << PAGE_SHIFT; error = vm_set_desc(newcpu, VM_REG_GUEST_CS, desc_base, desc_limit, desc_access); assert(error == 0); cs = (vector << PAGE_SHIFT) >> 4; error = vm_set_register(newcpu, VM_REG_GUEST_CS, cs); assert(error == 0); } #endif /* __FreeBSD__ */ void spinup_ap(struct vcpu *newcpu, uint64_t rip) { int error; error = vcpu_reset(newcpu); assert(error == 0); #ifdef __FreeBSD__ spinup_ap_realmode(newcpu, rip); vm_resume_cpu(newcpu); #endif } /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2012 NetApp, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef _SPINUP_AP_H_ #define _SPINUP_AP_H_ void spinup_ap(struct vcpu *newcpu, uint64_t rip); #endif /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2014 Neel Natu * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * This file and its contents are supplied under the terms of the * Common Development and Distribution License ("CDDL"), version 1.0. * You may only use this file in accordance with the terms of version * 1.0 of the CDDL. * * A full copy of the text of the CDDL should have accompanied this * source. A copy of the CDDL is also available via the Internet at * http://www.illumos.org/license/CDDL. * * Copyright 2020 Oxide Computer Company */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include "bhyverun.h" #include "debug.h" /* * Using 'struct i386tss' is tempting but causes myriad sign extension * issues because all of its fields are defined as signed integers. */ struct tss32 { uint16_t tss_link; uint16_t rsvd1; uint32_t tss_esp0; uint16_t tss_ss0; uint16_t rsvd2; uint32_t tss_esp1; uint16_t tss_ss1; uint16_t rsvd3; uint32_t tss_esp2; uint16_t tss_ss2; uint16_t rsvd4; uint32_t tss_cr3; uint32_t tss_eip; uint32_t tss_eflags; uint32_t tss_eax; uint32_t tss_ecx; uint32_t tss_edx; uint32_t tss_ebx; uint32_t tss_esp; uint32_t tss_ebp; uint32_t tss_esi; uint32_t tss_edi; uint16_t tss_es; uint16_t rsvd5; uint16_t tss_cs; uint16_t rsvd6; uint16_t tss_ss; uint16_t rsvd7; uint16_t tss_ds; uint16_t rsvd8; uint16_t tss_fs; uint16_t rsvd9; uint16_t tss_gs; uint16_t rsvd10; uint16_t tss_ldt; uint16_t rsvd11; uint16_t tss_trap; uint16_t tss_iomap; }; static_assert(sizeof(struct tss32) == 104, "compile-time assertion failed"); #define SEL_START(sel) (((sel) & ~0x7)) #define SEL_LIMIT(sel) (((sel) | 0x7)) #define TSS_BUSY(type) (((type) & 0x2) != 0) static uint64_t GETREG(struct vcpu *vcpu, int reg) { uint64_t val; int error; error = vm_get_register(vcpu, reg, &val); assert(error == 0); return (val); } static void SETREG(struct vcpu *vcpu, int reg, uint64_t val) { int error; error = vm_set_register(vcpu, reg, val); assert(error == 0); } static struct seg_desc usd_to_seg_desc(struct user_segment_descriptor *usd) { struct seg_desc seg_desc; seg_desc.base = (u_int)USD_GETBASE(usd); if (usd->sd_gran) seg_desc.limit = (u_int)(USD_GETLIMIT(usd) << 12) | 0xfff; else seg_desc.limit = (u_int)USD_GETLIMIT(usd); seg_desc.access = usd->sd_type | usd->sd_dpl << 5 | usd->sd_p << 7; seg_desc.access |= usd->sd_xx << 12; seg_desc.access |= usd->sd_def32 << 14; seg_desc.access |= usd->sd_gran << 15; return (seg_desc); } /* * Inject an exception with an error code that is a segment selector. * The format of the error code is described in section 6.13, "Error Code", * Intel SDM volume 3. * * Bit 0 (EXT) denotes whether the exception occurred during delivery * of an external event like an interrupt. * * Bit 1 (IDT) indicates whether the selector points to a gate descriptor * in the IDT. * * Bit 2(GDT/LDT) has the usual interpretation of Table Indicator (TI). */ static void sel_exception(struct vcpu *vcpu, int vector, uint16_t sel, int ext) { /* * Bit 2 from the selector is retained as-is in the error code. * * Bit 1 can be safely cleared because none of the selectors * encountered during task switch emulation refer to a task * gate in the IDT. * * Bit 0 is set depending on the value of 'ext'. */ sel &= ~0x3; if (ext) sel |= 0x1; vm_inject_fault(vcpu, vector, 1, sel); } /* * Return 0 if the selector 'sel' in within the limits of the GDT/LDT * and non-zero otherwise. */ static int desc_table_limit_check(struct vcpu *vcpu, uint16_t sel) { uint64_t base; uint32_t limit, access; int error, reg; reg = ISLDT(sel) ? VM_REG_GUEST_LDTR : VM_REG_GUEST_GDTR; error = vm_get_desc(vcpu, reg, &base, &limit, &access); assert(error == 0); if (reg == VM_REG_GUEST_LDTR) { if (SEG_DESC_UNUSABLE(access) || !SEG_DESC_PRESENT(access)) return (-1); } if (limit < SEL_LIMIT(sel)) return (-1); else return (0); } /* * Read/write the segment descriptor 'desc' into the GDT/LDT slot referenced * by the selector 'sel'. * * Returns 0 on success. * Returns 1 if an exception was injected into the guest. * Returns -1 otherwise. */ static int desc_table_rw(struct vcpu *vcpu, struct vm_guest_paging *paging, uint16_t sel, struct user_segment_descriptor *desc, bool doread, int *faultptr) { struct iovec iov[2]; uint64_t base; uint32_t limit, access; int error, reg; reg = ISLDT(sel) ? VM_REG_GUEST_LDTR : VM_REG_GUEST_GDTR; error = vm_get_desc(vcpu, reg, &base, &limit, &access); assert(error == 0); assert(limit >= SEL_LIMIT(sel)); error = vm_copy_setup(vcpu, paging, base + SEL_START(sel), sizeof(*desc), doread ? PROT_READ : PROT_WRITE, iov, nitems(iov), faultptr); if (error || *faultptr) return (error); if (doread) vm_copyin(iov, desc, sizeof(*desc)); else vm_copyout(desc, iov, sizeof(*desc)); return (0); } static int desc_table_read(struct vcpu *vcpu, struct vm_guest_paging *paging, uint16_t sel, struct user_segment_descriptor *desc, int *faultptr) { return (desc_table_rw(vcpu, paging, sel, desc, true, faultptr)); } static int desc_table_write(struct vcpu *vcpu, struct vm_guest_paging *paging, uint16_t sel, struct user_segment_descriptor *desc, int *faultptr) { return (desc_table_rw(vcpu, paging, sel, desc, false, faultptr)); } /* * Read the TSS descriptor referenced by 'sel' into 'desc'. * * Returns 0 on success. * Returns 1 if an exception was injected into the guest. * Returns -1 otherwise. */ static int read_tss_descriptor(struct vcpu *vcpu, struct vm_task_switch *ts, uint16_t sel, struct user_segment_descriptor *desc, int *faultptr) { struct vm_guest_paging sup_paging; int error; assert(!ISLDT(sel)); assert(IDXSEL(sel) != 0); /* Fetch the new TSS descriptor */ if (desc_table_limit_check(vcpu, sel)) { if (ts->reason == TSR_IRET) sel_exception(vcpu, IDT_TS, sel, ts->ext); else sel_exception(vcpu, IDT_GP, sel, ts->ext); return (1); } sup_paging = ts->paging; sup_paging.cpl = 0; /* implicit supervisor mode */ error = desc_table_read(vcpu, &sup_paging, sel, desc, faultptr); return (error); } static bool code_desc(int sd_type) { /* code descriptor */ return ((sd_type & 0x18) == 0x18); } static bool stack_desc(int sd_type) { /* writable data descriptor */ return ((sd_type & 0x1A) == 0x12); } static bool data_desc(int sd_type) { /* data descriptor or a readable code descriptor */ return ((sd_type & 0x18) == 0x10 || (sd_type & 0x1A) == 0x1A); } static bool ldt_desc(int sd_type) { return (sd_type == SDT_SYSLDT); } /* * Validate the descriptor 'seg_desc' associated with 'segment'. */ static int validate_seg_desc(struct vcpu *vcpu, struct vm_task_switch *ts, int segment, struct seg_desc *seg_desc, int *faultptr) { struct vm_guest_paging sup_paging; struct user_segment_descriptor usd; int error, idtvec; int cpl, dpl, rpl; uint16_t sel, cs; bool ldtseg, codeseg, stackseg, dataseg, conforming; ldtseg = codeseg = stackseg = dataseg = false; switch (segment) { case VM_REG_GUEST_LDTR: ldtseg = true; break; case VM_REG_GUEST_CS: codeseg = true; break; case VM_REG_GUEST_SS: stackseg = true; break; case VM_REG_GUEST_DS: case VM_REG_GUEST_ES: case VM_REG_GUEST_FS: case VM_REG_GUEST_GS: dataseg = true; break; default: assert(0); } /* Get the segment selector */ sel = GETREG(vcpu, segment); /* LDT selector must point into the GDT */ if (ldtseg && ISLDT(sel)) { sel_exception(vcpu, IDT_TS, sel, ts->ext); return (1); } /* Descriptor table limit check */ if (desc_table_limit_check(vcpu, sel)) { sel_exception(vcpu, IDT_TS, sel, ts->ext); return (1); } /* NULL selector */ if (IDXSEL(sel) == 0) { /* Code and stack segment selectors cannot be NULL */ if (codeseg || stackseg) { sel_exception(vcpu, IDT_TS, sel, ts->ext); return (1); } seg_desc->base = 0; seg_desc->limit = 0; seg_desc->access = 0x10000; /* unusable */ return (0); } /* Read the descriptor from the GDT/LDT */ sup_paging = ts->paging; sup_paging.cpl = 0; /* implicit supervisor mode */ error = desc_table_read(vcpu, &sup_paging, sel, &usd, faultptr); if (error || *faultptr) return (error); /* Verify that the descriptor type is compatible with the segment */ if ((ldtseg && !ldt_desc(usd.sd_type)) || (codeseg && !code_desc(usd.sd_type)) || (dataseg && !data_desc(usd.sd_type)) || (stackseg && !stack_desc(usd.sd_type))) { sel_exception(vcpu, IDT_TS, sel, ts->ext); return (1); } /* Segment must be marked present */ if (!usd.sd_p) { if (ldtseg) idtvec = IDT_TS; else if (stackseg) idtvec = IDT_SS; else idtvec = IDT_NP; sel_exception(vcpu, idtvec, sel, ts->ext); return (1); } cs = GETREG(vcpu, VM_REG_GUEST_CS); cpl = cs & SEL_RPL_MASK; rpl = sel & SEL_RPL_MASK; dpl = usd.sd_dpl; if (stackseg && (rpl != cpl || dpl != cpl)) { sel_exception(vcpu, IDT_TS, sel, ts->ext); return (1); } if (codeseg) { conforming = (usd.sd_type & 0x4) ? true : false; if ((conforming && (cpl < dpl)) || (!conforming && (cpl != dpl))) { sel_exception(vcpu, IDT_TS, sel, ts->ext); return (1); } } if (dataseg) { /* * A data segment is always non-conforming except when it's * descriptor is a readable, conforming code segment. */ if (code_desc(usd.sd_type) && (usd.sd_type & 0x4) != 0) conforming = true; else conforming = false; if (!conforming && (rpl > dpl || cpl > dpl)) { sel_exception(vcpu, IDT_TS, sel, ts->ext); return (1); } } *seg_desc = usd_to_seg_desc(&usd); return (0); } static void tss32_save(struct vcpu *vcpu, struct vm_task_switch *task_switch, uint32_t eip, struct tss32 *tss, struct iovec *iov) { /* General purpose registers */ tss->tss_eax = GETREG(vcpu, VM_REG_GUEST_RAX); tss->tss_ecx = GETREG(vcpu, VM_REG_GUEST_RCX); tss->tss_edx = GETREG(vcpu, VM_REG_GUEST_RDX); tss->tss_ebx = GETREG(vcpu, VM_REG_GUEST_RBX); tss->tss_esp = GETREG(vcpu, VM_REG_GUEST_RSP); tss->tss_ebp = GETREG(vcpu, VM_REG_GUEST_RBP); tss->tss_esi = GETREG(vcpu, VM_REG_GUEST_RSI); tss->tss_edi = GETREG(vcpu, VM_REG_GUEST_RDI); /* Segment selectors */ tss->tss_es = GETREG(vcpu, VM_REG_GUEST_ES); tss->tss_cs = GETREG(vcpu, VM_REG_GUEST_CS); tss->tss_ss = GETREG(vcpu, VM_REG_GUEST_SS); tss->tss_ds = GETREG(vcpu, VM_REG_GUEST_DS); tss->tss_fs = GETREG(vcpu, VM_REG_GUEST_FS); tss->tss_gs = GETREG(vcpu, VM_REG_GUEST_GS); /* eflags and eip */ tss->tss_eflags = GETREG(vcpu, VM_REG_GUEST_RFLAGS); if (task_switch->reason == TSR_IRET) tss->tss_eflags &= ~PSL_NT; tss->tss_eip = eip; /* Copy updated old TSS into guest memory */ vm_copyout(tss, iov, sizeof(struct tss32)); } static void update_seg_desc(struct vcpu *vcpu, int reg, struct seg_desc *sd) { int error; error = vm_set_desc(vcpu, reg, sd->base, sd->limit, sd->access); assert(error == 0); } /* * Update the vcpu registers to reflect the state of the new task. */ static int tss32_restore(struct vmctx *ctx, struct vcpu *vcpu, struct vm_task_switch *ts, uint16_t ot_sel, struct tss32 *tss, struct iovec *iov, int *faultptr) { struct seg_desc seg_desc, seg_desc2; uint64_t *pdpte, maxphyaddr, reserved; uint32_t eflags; int error, i; bool nested; nested = false; if (ts->reason != TSR_IRET && ts->reason != TSR_JMP) { tss->tss_link = ot_sel; nested = true; } eflags = tss->tss_eflags; if (nested) eflags |= PSL_NT; /* LDTR */ SETREG(vcpu, VM_REG_GUEST_LDTR, tss->tss_ldt); /* PBDR */ if (ts->paging.paging_mode != PAGING_MODE_FLAT) { if (ts->paging.paging_mode == PAGING_MODE_PAE) { /* * XXX Assuming 36-bit MAXPHYADDR. */ maxphyaddr = (1UL << 36) - 1; pdpte = paddr_guest2host(ctx, tss->tss_cr3 & ~0x1f, 32); for (i = 0; i < 4; i++) { /* Check reserved bits if the PDPTE is valid */ if (!(pdpte[i] & 0x1)) continue; /* * Bits 2:1, 8:5 and bits above the processor's * maximum physical address are reserved. */ reserved = ~maxphyaddr | 0x1E6; if (pdpte[i] & reserved) { vm_inject_gp(vcpu); return (1); } } SETREG(vcpu, VM_REG_GUEST_PDPTE0, pdpte[0]); SETREG(vcpu, VM_REG_GUEST_PDPTE1, pdpte[1]); SETREG(vcpu, VM_REG_GUEST_PDPTE2, pdpte[2]); SETREG(vcpu, VM_REG_GUEST_PDPTE3, pdpte[3]); } SETREG(vcpu, VM_REG_GUEST_CR3, tss->tss_cr3); ts->paging.cr3 = tss->tss_cr3; } /* eflags and eip */ SETREG(vcpu, VM_REG_GUEST_RFLAGS, eflags); SETREG(vcpu, VM_REG_GUEST_RIP, tss->tss_eip); /* General purpose registers */ SETREG(vcpu, VM_REG_GUEST_RAX, tss->tss_eax); SETREG(vcpu, VM_REG_GUEST_RCX, tss->tss_ecx); SETREG(vcpu, VM_REG_GUEST_RDX, tss->tss_edx); SETREG(vcpu, VM_REG_GUEST_RBX, tss->tss_ebx); SETREG(vcpu, VM_REG_GUEST_RSP, tss->tss_esp); SETREG(vcpu, VM_REG_GUEST_RBP, tss->tss_ebp); SETREG(vcpu, VM_REG_GUEST_RSI, tss->tss_esi); SETREG(vcpu, VM_REG_GUEST_RDI, tss->tss_edi); /* Segment selectors */ SETREG(vcpu, VM_REG_GUEST_ES, tss->tss_es); SETREG(vcpu, VM_REG_GUEST_CS, tss->tss_cs); SETREG(vcpu, VM_REG_GUEST_SS, tss->tss_ss); SETREG(vcpu, VM_REG_GUEST_DS, tss->tss_ds); SETREG(vcpu, VM_REG_GUEST_FS, tss->tss_fs); SETREG(vcpu, VM_REG_GUEST_GS, tss->tss_gs); /* * If this is a nested task then write out the new TSS to update * the previous link field. */ if (nested) vm_copyout(tss, iov, sizeof(*tss)); /* Validate segment descriptors */ error = validate_seg_desc(vcpu, ts, VM_REG_GUEST_LDTR, &seg_desc, faultptr); if (error || *faultptr) return (error); update_seg_desc(vcpu, VM_REG_GUEST_LDTR, &seg_desc); /* * Section "Checks on Guest Segment Registers", Intel SDM, Vol 3. * * The SS and CS attribute checks on VM-entry are inter-dependent so * we need to make sure that both segments are valid before updating * either of them. This ensures that the VMCS state can pass the * VM-entry checks so the guest can handle any exception injected * during task switch emulation. */ error = validate_seg_desc(vcpu, ts, VM_REG_GUEST_CS, &seg_desc, faultptr); if (error || *faultptr) return (error); error = validate_seg_desc(vcpu, ts, VM_REG_GUEST_SS, &seg_desc2, faultptr); if (error || *faultptr) return (error); update_seg_desc(vcpu, VM_REG_GUEST_CS, &seg_desc); update_seg_desc(vcpu, VM_REG_GUEST_SS, &seg_desc2); ts->paging.cpl = tss->tss_cs & SEL_RPL_MASK; error = validate_seg_desc(vcpu, ts, VM_REG_GUEST_DS, &seg_desc, faultptr); if (error || *faultptr) return (error); update_seg_desc(vcpu, VM_REG_GUEST_DS, &seg_desc); error = validate_seg_desc(vcpu, ts, VM_REG_GUEST_ES, &seg_desc, faultptr); if (error || *faultptr) return (error); update_seg_desc(vcpu, VM_REG_GUEST_ES, &seg_desc); error = validate_seg_desc(vcpu, ts, VM_REG_GUEST_FS, &seg_desc, faultptr); if (error || *faultptr) return (error); update_seg_desc(vcpu, VM_REG_GUEST_FS, &seg_desc); error = validate_seg_desc(vcpu, ts, VM_REG_GUEST_GS, &seg_desc, faultptr); if (error || *faultptr) return (error); update_seg_desc(vcpu, VM_REG_GUEST_GS, &seg_desc); return (0); } /* * Copy of vie_alignment_check() from vmm_instruction_emul.c */ static int alignment_check(int cpl, int size, uint64_t cr0, uint64_t rf, uint64_t gla) { assert(size == 1 || size == 2 || size == 4 || size == 8); assert(cpl >= 0 && cpl <= 3); if (cpl != 3 || (cr0 & CR0_AM) == 0 || (rf & PSL_AC) == 0) return (0); return ((gla & (size - 1)) ? 1 : 0); } /* * Copy of vie_size2mask() from vmm_instruction_emul.c */ static uint64_t size2mask(int size) { switch (size) { case 1: return (0xff); case 2: return (0xffff); case 4: return (0xffffffff); case 8: return (0xffffffffffffffff); default: assert(0); /* not reached */ return (0); } } /* * Copy of vie_calculate_gla() from vmm_instruction_emul.c */ static int calculate_gla(enum vm_cpu_mode cpu_mode, enum vm_reg_name seg, struct seg_desc *desc, uint64_t offset, int length, int addrsize, int prot, uint64_t *gla) { uint64_t firstoff, low_limit, high_limit, segbase; int glasize, type; assert(seg >= VM_REG_GUEST_ES && seg <= VM_REG_GUEST_GS); assert((length == 1 || length == 2 || length == 4 || length == 8)); assert((prot & ~(PROT_READ | PROT_WRITE)) == 0); firstoff = offset; if (cpu_mode == CPU_MODE_64BIT) { assert(addrsize == 4 || addrsize == 8); glasize = 8; } else { assert(addrsize == 2 || addrsize == 4); glasize = 4; /* * If the segment selector is loaded with a NULL selector * then the descriptor is unusable and attempting to use * it results in a #GP(0). */ if (SEG_DESC_UNUSABLE(desc->access)) return (-1); /* * The processor generates a #NP exception when a segment * register is loaded with a selector that points to a * descriptor that is not present. If this was the case then * it would have been checked before the VM-exit. */ assert(SEG_DESC_PRESENT(desc->access)); /* * The descriptor type must indicate a code/data segment. */ type = SEG_DESC_TYPE(desc->access); assert(type >= 16 && type <= 31); if (prot & PROT_READ) { /* #GP on a read access to a exec-only code segment */ if ((type & 0xA) == 0x8) return (-1); } if (prot & PROT_WRITE) { /* * #GP on a write access to a code segment or a * read-only data segment. */ if (type & 0x8) /* code segment */ return (-1); if ((type & 0xA) == 0) /* read-only data seg */ return (-1); } /* * 'desc->limit' is fully expanded taking granularity into * account. */ if ((type & 0xC) == 0x4) { /* expand-down data segment */ low_limit = desc->limit + 1; high_limit = SEG_DESC_DEF32(desc->access) ? 0xffffffff : 0xffff; } else { /* code segment or expand-up data segment */ low_limit = 0; high_limit = desc->limit; } while (length > 0) { offset &= size2mask(addrsize); if (offset < low_limit || offset > high_limit) return (-1); offset++; length--; } } /* * In 64-bit mode all segments except %fs and %gs have a segment * base address of 0. */ if (cpu_mode == CPU_MODE_64BIT && seg != VM_REG_GUEST_FS && seg != VM_REG_GUEST_GS) { segbase = 0; } else { segbase = desc->base; } /* * Truncate 'firstoff' to the effective address size before adding * it to the segment base. */ firstoff &= size2mask(addrsize); *gla = (segbase + firstoff) & size2mask(glasize); return (0); } /* * Push an error code on the stack of the new task. This is needed if the * task switch was triggered by a hardware exception that causes an error * code to be saved (e.g. #PF). */ static int push_errcode(struct vcpu *vcpu, struct vm_guest_paging *paging, int task_type, uint32_t errcode, int *faultptr) { struct iovec iov[2]; struct seg_desc seg_desc; int stacksize, bytes, error; uint64_t gla, cr0, rflags; uint32_t esp; uint16_t stacksel; *faultptr = 0; cr0 = GETREG(vcpu, VM_REG_GUEST_CR0); rflags = GETREG(vcpu, VM_REG_GUEST_RFLAGS); stacksel = GETREG(vcpu, VM_REG_GUEST_SS); error = vm_get_desc(vcpu, VM_REG_GUEST_SS, &seg_desc.base, &seg_desc.limit, &seg_desc.access); assert(error == 0); /* * Section "Error Code" in the Intel SDM vol 3: the error code is * pushed on the stack as a doubleword or word (depending on the * default interrupt, trap or task gate size). */ if (task_type == SDT_SYS386BSY || task_type == SDT_SYS386TSS) bytes = 4; else bytes = 2; /* * PUSH instruction from Intel SDM vol 2: the 'B' flag in the * stack-segment descriptor determines the size of the stack * pointer outside of 64-bit mode. */ if (SEG_DESC_DEF32(seg_desc.access)) stacksize = 4; else stacksize = 2; esp = GETREG(vcpu, VM_REG_GUEST_RSP); esp -= bytes; if (calculate_gla(paging->cpu_mode, VM_REG_GUEST_SS, &seg_desc, esp, bytes, stacksize, PROT_WRITE, &gla)) { sel_exception(vcpu, IDT_SS, stacksel, 1); *faultptr = 1; return (0); } if (alignment_check(paging->cpl, bytes, cr0, rflags, gla)) { vm_inject_ac(vcpu, 1); *faultptr = 1; return (0); } error = vm_copy_setup(vcpu, paging, gla, bytes, PROT_WRITE, iov, nitems(iov), faultptr); if (error || *faultptr) return (error); vm_copyout(&errcode, iov, bytes); SETREG(vcpu, VM_REG_GUEST_RSP, esp); return (0); } /* * Evaluate return value from helper functions and potentially return to * the VM run loop. */ #define CHKERR(error,fault) \ do { \ assert((error == 0) || (error == EFAULT)); \ if (error) \ return (VMEXIT_ABORT); \ else if (fault) \ return (VMEXIT_CONTINUE); \ } while (0) int vmexit_task_switch(struct vmctx *ctx, struct vcpu *vcpu, struct vm_exit *vmexit) { struct seg_desc nt; struct tss32 oldtss, newtss; struct vm_task_switch *task_switch; struct vm_guest_paging *paging, sup_paging; struct user_segment_descriptor nt_desc, ot_desc; struct iovec nt_iov[2], ot_iov[2]; uint64_t cr0, ot_base; uint32_t eip, ot_lim, access; int error, ext, fault, minlimit, nt_type, ot_type; enum task_switch_reason reason; uint16_t nt_sel, ot_sel; task_switch = &vmexit->u.task_switch; nt_sel = task_switch->tsssel; ext = vmexit->u.task_switch.ext; reason = vmexit->u.task_switch.reason; paging = &vmexit->u.task_switch.paging; assert(paging->cpu_mode == CPU_MODE_PROTECTED); /* * Calculate the instruction pointer to store in the old TSS. */ eip = vmexit->rip + vmexit->inst_length; /* * Section 4.6, "Access Rights" in Intel SDM Vol 3. * The following page table accesses are implicitly supervisor mode: * - accesses to GDT or LDT to load segment descriptors * - accesses to the task state segment during task switch */ sup_paging = *paging; sup_paging.cpl = 0; /* implicit supervisor mode */ /* Fetch the new TSS descriptor */ error = read_tss_descriptor(vcpu, task_switch, nt_sel, &nt_desc, &fault); CHKERR(error, fault); nt = usd_to_seg_desc(&nt_desc); /* Verify the type of the new TSS */ nt_type = SEG_DESC_TYPE(nt.access); if (nt_type != SDT_SYS386BSY && nt_type != SDT_SYS386TSS && nt_type != SDT_SYS286BSY && nt_type != SDT_SYS286TSS) { sel_exception(vcpu, IDT_TS, nt_sel, ext); goto done; } /* TSS descriptor must have present bit set */ if (!SEG_DESC_PRESENT(nt.access)) { sel_exception(vcpu, IDT_NP, nt_sel, ext); goto done; } /* * TSS must have a minimum length of 104 bytes for a 32-bit TSS and * 44 bytes for a 16-bit TSS. */ if (nt_type == SDT_SYS386BSY || nt_type == SDT_SYS386TSS) minlimit = 104 - 1; else if (nt_type == SDT_SYS286BSY || nt_type == SDT_SYS286TSS) minlimit = 44 - 1; else minlimit = 0; assert(minlimit > 0); if (nt.limit < (unsigned int)minlimit) { sel_exception(vcpu, IDT_TS, nt_sel, ext); goto done; } /* TSS must be busy if task switch is due to IRET */ if (reason == TSR_IRET && !TSS_BUSY(nt_type)) { sel_exception(vcpu, IDT_TS, nt_sel, ext); goto done; } /* * TSS must be available (not busy) if task switch reason is * CALL, JMP, exception or interrupt. */ if (reason != TSR_IRET && TSS_BUSY(nt_type)) { sel_exception(vcpu, IDT_GP, nt_sel, ext); goto done; } /* Fetch the new TSS */ error = vm_copy_setup(vcpu, &sup_paging, nt.base, minlimit + 1, PROT_READ | PROT_WRITE, nt_iov, nitems(nt_iov), &fault); CHKERR(error, fault); vm_copyin(nt_iov, &newtss, minlimit + 1); /* Get the old TSS selector from the guest's task register */ ot_sel = GETREG(vcpu, VM_REG_GUEST_TR); if (ISLDT(ot_sel) || IDXSEL(ot_sel) == 0) { /* * This might happen if a task switch was attempted without * ever loading the task register with LTR. In this case the * TR would contain the values from power-on: * (sel = 0, base = 0, limit = 0xffff). */ sel_exception(vcpu, IDT_TS, ot_sel, task_switch->ext); goto done; } /* Get the old TSS base and limit from the guest's task register */ error = vm_get_desc(vcpu, VM_REG_GUEST_TR, &ot_base, &ot_lim, &access); assert(error == 0); assert(!SEG_DESC_UNUSABLE(access) && SEG_DESC_PRESENT(access)); ot_type = SEG_DESC_TYPE(access); assert(ot_type == SDT_SYS386BSY || ot_type == SDT_SYS286BSY); /* Fetch the old TSS descriptor */ error = read_tss_descriptor(vcpu, task_switch, ot_sel, &ot_desc, &fault); CHKERR(error, fault); /* Get the old TSS */ error = vm_copy_setup(vcpu, &sup_paging, ot_base, minlimit + 1, PROT_READ | PROT_WRITE, ot_iov, nitems(ot_iov), &fault); CHKERR(error, fault); vm_copyin(ot_iov, &oldtss, minlimit + 1); /* * Clear the busy bit in the old TSS descriptor if the task switch * due to an IRET or JMP instruction. */ if (reason == TSR_IRET || reason == TSR_JMP) { ot_desc.sd_type &= ~0x2; error = desc_table_write(vcpu, &sup_paging, ot_sel, &ot_desc, &fault); CHKERR(error, fault); } if (nt_type == SDT_SYS286BSY || nt_type == SDT_SYS286TSS) { EPRINTLN("Task switch to 16-bit TSS not supported"); return (VMEXIT_ABORT); } /* Save processor state in old TSS */ tss32_save(vcpu, task_switch, eip, &oldtss, ot_iov); /* * If the task switch was triggered for any reason other than IRET * then set the busy bit in the new TSS descriptor. */ if (reason != TSR_IRET) { nt_desc.sd_type |= 0x2; error = desc_table_write(vcpu, &sup_paging, nt_sel, &nt_desc, &fault); CHKERR(error, fault); } /* Update task register to point at the new TSS */ SETREG(vcpu, VM_REG_GUEST_TR, nt_sel); /* Update the hidden descriptor state of the task register */ nt = usd_to_seg_desc(&nt_desc); update_seg_desc(vcpu, VM_REG_GUEST_TR, &nt); /* Set CR0.TS */ cr0 = GETREG(vcpu, VM_REG_GUEST_CR0); SETREG(vcpu, VM_REG_GUEST_CR0, cr0 | CR0_TS); /* * We are now committed to the task switch. Any exceptions encountered * after this point will be handled in the context of the new task and * the saved instruction pointer will belong to the new task. */ error = vm_set_register(vcpu, VM_REG_GUEST_RIP, newtss.tss_eip); assert(error == 0); /* Load processor state from new TSS */ error = tss32_restore(ctx, vcpu, task_switch, ot_sel, &newtss, nt_iov, &fault); CHKERR(error, fault); /* * Section "Interrupt Tasks" in Intel SDM, Vol 3: if an exception * caused an error code to be generated, this error code is copied * to the stack of the new task. */ if (task_switch->errcode_valid) { assert(task_switch->ext); assert(task_switch->reason == TSR_IDT_GATE); error = push_errcode(vcpu, &task_switch->paging, nt_type, task_switch->errcode, &fault); CHKERR(error, fault); } /* * Treatment of virtual-NMI blocking if NMI is delivered through * a task gate. * * Section "Architectural State Before A VM Exit", Intel SDM, Vol3: * If the virtual NMIs VM-execution control is 1, VM entry injects * an NMI, and delivery of the NMI causes a task switch that causes * a VM exit, virtual-NMI blocking is in effect before the VM exit * commences. * * Thus, virtual-NMI blocking is in effect at the time of the task * switch VM exit. */ /* * Treatment of virtual-NMI unblocking on IRET from NMI handler task. * * Section "Changes to Instruction Behavior in VMX Non-Root Operation" * If "virtual NMIs" control is 1 IRET removes any virtual-NMI blocking. * This unblocking of virtual-NMI occurs even if IRET causes a fault. * * Thus, virtual-NMI blocking is cleared at the time of the task switch * VM exit. */ /* * If the task switch was triggered by an event delivered through * the IDT then extinguish the pending event from the vcpu's * exitintinfo. */ if (task_switch->reason == TSR_IDT_GATE) { error = vm_set_intinfo(vcpu, 0); assert(error == 0); } /* * XXX should inject debug exception if 'T' bit is 1 */ done: return (VMEXIT_CONTINUE); } /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2015 Tycho Nightingale * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * Copyright 2018 Joyent, Inc. */ #include #include #include #include #include #include #include #include #include "bhyvegc.h" #include "console.h" #include "inout.h" #include "mem.h" #include "vga.h" #define KB (1024UL) #define MB (1024 * 1024UL) struct vga_softc { struct mem_range mr; struct bhyvegc *gc; int gc_width; int gc_height; struct bhyvegc_image *gc_image; uint8_t *vga_ram; /* * General registers */ uint8_t vga_misc; uint8_t vga_sts1; /* * Sequencer */ struct { int seq_index; uint8_t seq_reset; uint8_t seq_clock_mode; int seq_cm_dots; uint8_t seq_map_mask; uint8_t seq_cmap_sel; int seq_cmap_pri_off; int seq_cmap_sec_off; uint8_t seq_mm; } vga_seq; /* * CRT Controller */ struct { int crtc_index; uint8_t crtc_mode_ctrl; uint8_t crtc_horiz_total; uint8_t crtc_horiz_disp_end; uint8_t crtc_start_horiz_blank; uint8_t crtc_end_horiz_blank; uint8_t crtc_start_horiz_retrace; uint8_t crtc_end_horiz_retrace; uint8_t crtc_vert_total; uint8_t crtc_overflow; uint8_t crtc_present_row_scan; uint8_t crtc_max_scan_line; uint8_t crtc_cursor_start; uint8_t crtc_cursor_on; uint8_t crtc_cursor_end; uint8_t crtc_start_addr_high; uint8_t crtc_start_addr_low; uint16_t crtc_start_addr; uint8_t crtc_cursor_loc_low; uint8_t crtc_cursor_loc_high; uint16_t crtc_cursor_loc; uint8_t crtc_vert_retrace_start; uint8_t crtc_vert_retrace_end; uint8_t crtc_vert_disp_end; uint8_t crtc_offset; uint8_t crtc_underline_loc; uint8_t crtc_start_vert_blank; uint8_t crtc_end_vert_blank; uint8_t crtc_line_compare; } vga_crtc; /* * Graphics Controller */ struct { int gc_index; uint8_t gc_set_reset; uint8_t gc_enb_set_reset; uint8_t gc_color_compare; uint8_t gc_rotate; uint8_t gc_op; uint8_t gc_read_map_sel; uint8_t gc_mode; bool gc_mode_c4; /* chain 4 */ bool gc_mode_oe; /* odd/even */ uint8_t gc_mode_rm; /* read mode */ uint8_t gc_mode_wm; /* write mode */ uint8_t gc_misc; uint8_t gc_misc_gm; /* graphics mode */ uint8_t gc_misc_mm; /* memory map */ uint8_t gc_color_dont_care; uint8_t gc_bit_mask; uint8_t gc_latch0; uint8_t gc_latch1; uint8_t gc_latch2; uint8_t gc_latch3; } vga_gc; /* * Attribute Controller */ struct { int atc_flipflop; int atc_index; uint8_t atc_palette[16]; uint8_t atc_mode; uint8_t atc_overscan_color; uint8_t atc_color_plane_enb; uint8_t atc_horiz_pixel_panning; uint8_t atc_color_select; uint8_t atc_color_select_45; uint8_t atc_color_select_67; } vga_atc; /* * DAC */ struct { uint8_t dac_state; uint8_t dac_rd_index; uint8_t dac_rd_subindex; uint8_t dac_wr_index; uint8_t dac_wr_subindex; uint8_t dac_palette[3 * 256]; uint32_t dac_palette_rgb[256]; } vga_dac; }; static bool vga_in_reset(struct vga_softc *sc) { return (((sc->vga_seq.seq_clock_mode & SEQ_CM_SO) != 0) || ((sc->vga_seq.seq_reset & SEQ_RESET_ASYNC) == 0) || ((sc->vga_seq.seq_reset & SEQ_RESET_SYNC) == 0) || ((sc->vga_crtc.crtc_mode_ctrl & CRTC_MC_TE) == 0)); } static void vga_check_size(struct bhyvegc *gc, struct vga_softc *sc) { int old_width, old_height; if (vga_in_reset(sc)) return; //old_width = sc->gc_width; //old_height = sc->gc_height; old_width = sc->gc_image->width; old_height = sc->gc_image->height; /* * Horizontal Display End: For text modes this is the number * of characters. For graphics modes this is the number of * pixels per scanlines divided by the number of pixels per * character clock. */ sc->gc_width = (sc->vga_crtc.crtc_horiz_disp_end + 1) * sc->vga_seq.seq_cm_dots; sc->gc_height = (sc->vga_crtc.crtc_vert_disp_end | (((sc->vga_crtc.crtc_overflow & CRTC_OF_VDE8) >> CRTC_OF_VDE8_SHIFT) << 8) | (((sc->vga_crtc.crtc_overflow & CRTC_OF_VDE9) >> CRTC_OF_VDE9_SHIFT) << 9)) + 1; if (old_width != sc->gc_width || old_height != sc->gc_height) bhyvegc_resize(gc, sc->gc_width, sc->gc_height); } static uint32_t vga_get_pixel(struct vga_softc *sc, int x, int y) { int offset; int bit; uint8_t data; uint8_t idx; offset = (y * sc->gc_width / 8) + (x / 8); bit = 7 - (x % 8); data = (((sc->vga_ram[offset + 0 * 64*KB] >> bit) & 0x1) << 0) | (((sc->vga_ram[offset + 1 * 64*KB] >> bit) & 0x1) << 1) | (((sc->vga_ram[offset + 2 * 64*KB] >> bit) & 0x1) << 2) | (((sc->vga_ram[offset + 3 * 64*KB] >> bit) & 0x1) << 3); data &= sc->vga_atc.atc_color_plane_enb; if (sc->vga_atc.atc_mode & ATC_MC_IPS) { idx = sc->vga_atc.atc_palette[data] & 0x0f; idx |= sc->vga_atc.atc_color_select_45; } else { idx = sc->vga_atc.atc_palette[data]; } idx |= sc->vga_atc.atc_color_select_67; return (sc->vga_dac.dac_palette_rgb[idx]); } static void vga_render_graphics(struct vga_softc *sc) { int x, y; for (y = 0; y < sc->gc_height; y++) { for (x = 0; x < sc->gc_width; x++) { int offset; offset = y * sc->gc_width + x; sc->gc_image->data[offset] = vga_get_pixel(sc, x, y); } } } static uint32_t vga_get_text_pixel(struct vga_softc *sc, int x, int y) { int dots, offset, bit, font_offset; uint8_t ch, attr, font; uint8_t idx; dots = sc->vga_seq.seq_cm_dots; offset = 2 * sc->vga_crtc.crtc_start_addr; offset += (y / 16 * sc->gc_width / dots) * 2 + (x / dots) * 2; bit = 7 - (x % dots > 7 ? 7 : x % dots); ch = sc->vga_ram[offset + 0 * 64*KB]; attr = sc->vga_ram[offset + 1 * 64*KB]; if (sc->vga_crtc.crtc_cursor_on && (offset == (sc->vga_crtc.crtc_cursor_loc * 2)) && ((y % 16) >= (sc->vga_crtc.crtc_cursor_start & CRTC_CS_CS)) && ((y % 16) <= (sc->vga_crtc.crtc_cursor_end & CRTC_CE_CE))) { idx = sc->vga_atc.atc_palette[attr & 0xf]; return (sc->vga_dac.dac_palette_rgb[idx]); } if ((sc->vga_seq.seq_mm & SEQ_MM_EM) && sc->vga_seq.seq_cmap_pri_off != sc->vga_seq.seq_cmap_sec_off) { if (attr & 0x8) font_offset = sc->vga_seq.seq_cmap_pri_off + (ch << 5) + y % 16; else font_offset = sc->vga_seq.seq_cmap_sec_off + (ch << 5) + y % 16; attr &= ~0x8; } else { font_offset = (ch << 5) + y % 16; } font = sc->vga_ram[font_offset + 2 * 64*KB]; if (font & (1 << bit)) idx = sc->vga_atc.atc_palette[attr & 0xf]; else idx = sc->vga_atc.atc_palette[attr >> 4]; return (sc->vga_dac.dac_palette_rgb[idx]); } static void vga_render_text(struct vga_softc *sc) { int x, y; for (y = 0; y < sc->gc_height; y++) { for (x = 0; x < sc->gc_width; x++) { int offset; offset = y * sc->gc_width + x; sc->gc_image->data[offset] = vga_get_text_pixel(sc, x, y); } } } void vga_render(struct bhyvegc *gc, void *arg) { struct vga_softc *sc = arg; vga_check_size(gc, sc); if (vga_in_reset(sc)) { memset(sc->gc_image->data, 0, sc->gc_image->width * sc->gc_image->height * sizeof (uint32_t)); return; } if (sc->vga_gc.gc_misc_gm && (sc->vga_atc.atc_mode & ATC_MC_GA)) vga_render_graphics(sc); else vga_render_text(sc); } static uint64_t vga_mem_rd_handler(uint64_t addr, void *arg1) { struct vga_softc *sc = arg1; uint8_t map_sel; int offset; offset = addr; switch (sc->vga_gc.gc_misc_mm) { case 0x0: /* * extended mode: base 0xa0000 size 128k */ offset -=0xa0000; offset &= (128 * KB - 1); break; case 0x1: /* * EGA/VGA mode: base 0xa0000 size 64k */ offset -=0xa0000; offset &= (64 * KB - 1); break; case 0x2: /* * monochrome text mode: base 0xb0000 size 32kb */ #ifdef __FreeBSD__ assert(0); #else abort(); #endif case 0x3: /* * color text mode and CGA: base 0xb8000 size 32kb */ offset -=0xb8000; offset &= (32 * KB - 1); break; } /* Fill latches. */ sc->vga_gc.gc_latch0 = sc->vga_ram[offset + 0*64*KB]; sc->vga_gc.gc_latch1 = sc->vga_ram[offset + 1*64*KB]; sc->vga_gc.gc_latch2 = sc->vga_ram[offset + 2*64*KB]; sc->vga_gc.gc_latch3 = sc->vga_ram[offset + 3*64*KB]; if (sc->vga_gc.gc_mode_rm) { /* read mode 1 */ assert(0); } map_sel = sc->vga_gc.gc_read_map_sel; if (sc->vga_gc.gc_mode_oe) { map_sel |= (offset & 1); offset &= ~1; } /* read mode 0: return the byte from the selected plane. */ offset += map_sel * 64*KB; return (sc->vga_ram[offset]); } static void vga_mem_wr_handler(uint64_t addr, uint8_t val, void *arg1) { struct vga_softc *sc = arg1; uint8_t c0, c1, c2, c3; uint8_t m0, m1, m2, m3; uint8_t set_reset; uint8_t enb_set_reset; uint8_t mask; int offset; offset = addr; switch (sc->vga_gc.gc_misc_mm) { case 0x0: /* * extended mode: base 0xa0000 size 128kb */ offset -=0xa0000; offset &= (128 * KB - 1); break; case 0x1: /* * EGA/VGA mode: base 0xa0000 size 64kb */ offset -=0xa0000; offset &= (64 * KB - 1); break; case 0x2: /* * monochrome text mode: base 0xb0000 size 32kb */ #ifdef __FreeBSD__ assert(0); #else abort(); #endif case 0x3: /* * color text mode and CGA: base 0xb8000 size 32kb */ offset -=0xb8000; offset &= (32 * KB - 1); break; } set_reset = sc->vga_gc.gc_set_reset; enb_set_reset = sc->vga_gc.gc_enb_set_reset; c0 = sc->vga_gc.gc_latch0; c1 = sc->vga_gc.gc_latch1; c2 = sc->vga_gc.gc_latch2; c3 = sc->vga_gc.gc_latch3; switch (sc->vga_gc.gc_mode_wm) { case 0: /* write mode 0 */ mask = sc->vga_gc.gc_bit_mask; val = (val >> sc->vga_gc.gc_rotate) | (val << (8 - sc->vga_gc.gc_rotate)); switch (sc->vga_gc.gc_op) { case 0x00: /* replace */ m0 = (set_reset & 1) ? mask : 0x00; m1 = (set_reset & 2) ? mask : 0x00; m2 = (set_reset & 4) ? mask : 0x00; m3 = (set_reset & 8) ? mask : 0x00; c0 = (enb_set_reset & 1) ? (c0 & ~mask) : (val & mask); c1 = (enb_set_reset & 2) ? (c1 & ~mask) : (val & mask); c2 = (enb_set_reset & 4) ? (c2 & ~mask) : (val & mask); c3 = (enb_set_reset & 8) ? (c3 & ~mask) : (val & mask); c0 |= m0; c1 |= m1; c2 |= m2; c3 |= m3; break; case 0x08: /* AND */ m0 = set_reset & 1 ? 0xff : ~mask; m1 = set_reset & 2 ? 0xff : ~mask; m2 = set_reset & 4 ? 0xff : ~mask; m3 = set_reset & 8 ? 0xff : ~mask; c0 = enb_set_reset & 1 ? c0 & m0 : val & m0; c1 = enb_set_reset & 2 ? c1 & m1 : val & m1; c2 = enb_set_reset & 4 ? c2 & m2 : val & m2; c3 = enb_set_reset & 8 ? c3 & m3 : val & m3; break; case 0x10: /* OR */ m0 = set_reset & 1 ? mask : 0x00; m1 = set_reset & 2 ? mask : 0x00; m2 = set_reset & 4 ? mask : 0x00; m3 = set_reset & 8 ? mask : 0x00; c0 = enb_set_reset & 1 ? c0 | m0 : val | m0; c1 = enb_set_reset & 2 ? c1 | m1 : val | m1; c2 = enb_set_reset & 4 ? c2 | m2 : val | m2; c3 = enb_set_reset & 8 ? c3 | m3 : val | m3; break; case 0x18: /* XOR */ m0 = set_reset & 1 ? mask : 0x00; m1 = set_reset & 2 ? mask : 0x00; m2 = set_reset & 4 ? mask : 0x00; m3 = set_reset & 8 ? mask : 0x00; c0 = enb_set_reset & 1 ? c0 ^ m0 : val ^ m0; c1 = enb_set_reset & 2 ? c1 ^ m1 : val ^ m1; c2 = enb_set_reset & 4 ? c2 ^ m2 : val ^ m2; c3 = enb_set_reset & 8 ? c3 ^ m3 : val ^ m3; break; } break; case 1: /* write mode 1 */ break; case 2: /* write mode 2 */ mask = sc->vga_gc.gc_bit_mask; switch (sc->vga_gc.gc_op) { case 0x00: /* replace */ m0 = (val & 1 ? 0xff : 0x00) & mask; m1 = (val & 2 ? 0xff : 0x00) & mask; m2 = (val & 4 ? 0xff : 0x00) & mask; m3 = (val & 8 ? 0xff : 0x00) & mask; c0 &= ~mask; c1 &= ~mask; c2 &= ~mask; c3 &= ~mask; c0 |= m0; c1 |= m1; c2 |= m2; c3 |= m3; break; case 0x08: /* AND */ m0 = (val & 1 ? 0xff : 0x00) | ~mask; m1 = (val & 2 ? 0xff : 0x00) | ~mask; m2 = (val & 4 ? 0xff : 0x00) | ~mask; m3 = (val & 8 ? 0xff : 0x00) | ~mask; c0 &= m0; c1 &= m1; c2 &= m2; c3 &= m3; break; case 0x10: /* OR */ m0 = (val & 1 ? 0xff : 0x00) & mask; m1 = (val & 2 ? 0xff : 0x00) & mask; m2 = (val & 4 ? 0xff : 0x00) & mask; m3 = (val & 8 ? 0xff : 0x00) & mask; c0 |= m0; c1 |= m1; c2 |= m2; c3 |= m3; break; case 0x18: /* XOR */ m0 = (val & 1 ? 0xff : 0x00) & mask; m1 = (val & 2 ? 0xff : 0x00) & mask; m2 = (val & 4 ? 0xff : 0x00) & mask; m3 = (val & 8 ? 0xff : 0x00) & mask; c0 ^= m0; c1 ^= m1; c2 ^= m2; c3 ^= m3; break; } break; case 3: /* write mode 3 */ mask = sc->vga_gc.gc_bit_mask & val; val = (val >> sc->vga_gc.gc_rotate) | (val << (8 - sc->vga_gc.gc_rotate)); switch (sc->vga_gc.gc_op) { case 0x00: /* replace */ m0 = (set_reset & 1 ? 0xff : 0x00) & mask; m1 = (set_reset & 2 ? 0xff : 0x00) & mask; m2 = (set_reset & 4 ? 0xff : 0x00) & mask; m3 = (set_reset & 8 ? 0xff : 0x00) & mask; c0 &= ~mask; c1 &= ~mask; c2 &= ~mask; c3 &= ~mask; c0 |= m0; c1 |= m1; c2 |= m2; c3 |= m3; break; case 0x08: /* AND */ m0 = (set_reset & 1 ? 0xff : 0x00) | ~mask; m1 = (set_reset & 2 ? 0xff : 0x00) | ~mask; m2 = (set_reset & 4 ? 0xff : 0x00) | ~mask; m3 = (set_reset & 8 ? 0xff : 0x00) | ~mask; c0 &= m0; c1 &= m1; c2 &= m2; c3 &= m3; break; case 0x10: /* OR */ m0 = (set_reset & 1 ? 0xff : 0x00) & mask; m1 = (set_reset & 2 ? 0xff : 0x00) & mask; m2 = (set_reset & 4 ? 0xff : 0x00) & mask; m3 = (set_reset & 8 ? 0xff : 0x00) & mask; c0 |= m0; c1 |= m1; c2 |= m2; c3 |= m3; break; case 0x18: /* XOR */ m0 = (set_reset & 1 ? 0xff : 0x00) & mask; m1 = (set_reset & 2 ? 0xff : 0x00) & mask; m2 = (set_reset & 4 ? 0xff : 0x00) & mask; m3 = (set_reset & 8 ? 0xff : 0x00) & mask; c0 ^= m0; c1 ^= m1; c2 ^= m2; c3 ^= m3; break; } break; } if (sc->vga_gc.gc_mode_oe) { if (offset & 1) { offset &= ~1; if (sc->vga_seq.seq_map_mask & 2) sc->vga_ram[offset + 1*64*KB] = c1; if (sc->vga_seq.seq_map_mask & 8) sc->vga_ram[offset + 3*64*KB] = c3; } else { if (sc->vga_seq.seq_map_mask & 1) sc->vga_ram[offset + 0*64*KB] = c0; if (sc->vga_seq.seq_map_mask & 4) sc->vga_ram[offset + 2*64*KB] = c2; } } else { if (sc->vga_seq.seq_map_mask & 1) sc->vga_ram[offset + 0*64*KB] = c0; if (sc->vga_seq.seq_map_mask & 2) sc->vga_ram[offset + 1*64*KB] = c1; if (sc->vga_seq.seq_map_mask & 4) sc->vga_ram[offset + 2*64*KB] = c2; if (sc->vga_seq.seq_map_mask & 8) sc->vga_ram[offset + 3*64*KB] = c3; } } static int vga_mem_handler(struct vcpu *vcpu __unused, int dir, uint64_t addr, int size, uint64_t *val, void *arg1, long arg2 __unused) { if (dir == MEM_F_WRITE) { switch (size) { case 1: vga_mem_wr_handler(addr, *val, arg1); break; case 2: vga_mem_wr_handler(addr, *val, arg1); vga_mem_wr_handler(addr + 1, *val >> 8, arg1); break; case 4: vga_mem_wr_handler(addr, *val, arg1); vga_mem_wr_handler(addr + 1, *val >> 8, arg1); vga_mem_wr_handler(addr + 2, *val >> 16, arg1); vga_mem_wr_handler(addr + 3, *val >> 24, arg1); break; case 8: vga_mem_wr_handler(addr, *val, arg1); vga_mem_wr_handler(addr + 1, *val >> 8, arg1); vga_mem_wr_handler(addr + 2, *val >> 16, arg1); vga_mem_wr_handler(addr + 3, *val >> 24, arg1); vga_mem_wr_handler(addr + 4, *val >> 32, arg1); vga_mem_wr_handler(addr + 5, *val >> 40, arg1); vga_mem_wr_handler(addr + 6, *val >> 48, arg1); vga_mem_wr_handler(addr + 7, *val >> 56, arg1); break; } } else { switch (size) { case 1: *val = vga_mem_rd_handler(addr, arg1); break; case 2: *val = vga_mem_rd_handler(addr, arg1); *val |= vga_mem_rd_handler(addr + 1, arg1) << 8; break; case 4: *val = vga_mem_rd_handler(addr, arg1); *val |= vga_mem_rd_handler(addr + 1, arg1) << 8; *val |= vga_mem_rd_handler(addr + 2, arg1) << 16; *val |= vga_mem_rd_handler(addr + 3, arg1) << 24; break; case 8: *val = vga_mem_rd_handler(addr, arg1); *val |= vga_mem_rd_handler(addr + 1, arg1) << 8; *val |= vga_mem_rd_handler(addr + 2, arg1) << 16; *val |= vga_mem_rd_handler(addr + 3, arg1) << 24; *val |= vga_mem_rd_handler(addr + 4, arg1) << 32; *val |= vga_mem_rd_handler(addr + 5, arg1) << 40; *val |= vga_mem_rd_handler(addr + 6, arg1) << 48; *val |= vga_mem_rd_handler(addr + 7, arg1) << 56; break; } } return (0); } static int vga_port_in_handler(struct vmctx *ctx, int in, int port, int bytes, uint8_t *val, void *arg) { struct vga_softc *sc = arg; switch (port) { case CRTC_IDX_MONO_PORT: case CRTC_IDX_COLOR_PORT: *val = sc->vga_crtc.crtc_index; break; case CRTC_DATA_MONO_PORT: case CRTC_DATA_COLOR_PORT: switch (sc->vga_crtc.crtc_index) { case CRTC_HORIZ_TOTAL: *val = sc->vga_crtc.crtc_horiz_total; break; case CRTC_HORIZ_DISP_END: *val = sc->vga_crtc.crtc_horiz_disp_end; break; case CRTC_START_HORIZ_BLANK: *val = sc->vga_crtc.crtc_start_horiz_blank; break; case CRTC_END_HORIZ_BLANK: *val = sc->vga_crtc.crtc_end_horiz_blank; break; case CRTC_START_HORIZ_RETRACE: *val = sc->vga_crtc.crtc_start_horiz_retrace; break; case CRTC_END_HORIZ_RETRACE: *val = sc->vga_crtc.crtc_end_horiz_retrace; break; case CRTC_VERT_TOTAL: *val = sc->vga_crtc.crtc_vert_total; break; case CRTC_OVERFLOW: *val = sc->vga_crtc.crtc_overflow; break; case CRTC_PRESET_ROW_SCAN: *val = sc->vga_crtc.crtc_present_row_scan; break; case CRTC_MAX_SCAN_LINE: *val = sc->vga_crtc.crtc_max_scan_line; break; case CRTC_CURSOR_START: *val = sc->vga_crtc.crtc_cursor_start; break; case CRTC_CURSOR_END: *val = sc->vga_crtc.crtc_cursor_end; break; case CRTC_START_ADDR_HIGH: *val = sc->vga_crtc.crtc_start_addr_high; break; case CRTC_START_ADDR_LOW: *val = sc->vga_crtc.crtc_start_addr_low; break; case CRTC_CURSOR_LOC_HIGH: *val = sc->vga_crtc.crtc_cursor_loc_high; break; case CRTC_CURSOR_LOC_LOW: *val = sc->vga_crtc.crtc_cursor_loc_low; break; case CRTC_VERT_RETRACE_START: *val = sc->vga_crtc.crtc_vert_retrace_start; break; case CRTC_VERT_RETRACE_END: *val = sc->vga_crtc.crtc_vert_retrace_end; break; case CRTC_VERT_DISP_END: *val = sc->vga_crtc.crtc_vert_disp_end; break; case CRTC_OFFSET: *val = sc->vga_crtc.crtc_offset; break; case CRTC_UNDERLINE_LOC: *val = sc->vga_crtc.crtc_underline_loc; break; case CRTC_START_VERT_BLANK: *val = sc->vga_crtc.crtc_start_vert_blank; break; case CRTC_END_VERT_BLANK: *val = sc->vga_crtc.crtc_end_vert_blank; break; case CRTC_MODE_CONTROL: *val = sc->vga_crtc.crtc_mode_ctrl; break; case CRTC_LINE_COMPARE: *val = sc->vga_crtc.crtc_line_compare; break; default: //printf("XXX VGA CRTC: inb 0x%04x at index %d\n", port, sc->vga_crtc.crtc_index); assert(0); break; } break; case ATC_IDX_PORT: *val = sc->vga_atc.atc_index; break; case ATC_DATA_PORT: switch (sc->vga_atc.atc_index) { case ATC_PALETTE0 ... ATC_PALETTE15: *val = sc->vga_atc.atc_palette[sc->vga_atc.atc_index]; break; case ATC_MODE_CONTROL: *val = sc->vga_atc.atc_mode; break; case ATC_OVERSCAN_COLOR: *val = sc->vga_atc.atc_overscan_color; break; case ATC_COLOR_PLANE_ENABLE: *val = sc->vga_atc.atc_color_plane_enb; break; case ATC_HORIZ_PIXEL_PANNING: *val = sc->vga_atc.atc_horiz_pixel_panning; break; case ATC_COLOR_SELECT: *val = sc->vga_atc.atc_color_select; break; default: //printf("XXX VGA ATC inb 0x%04x at index %d\n", port , sc->vga_atc.atc_index); assert(0); break; } break; case SEQ_IDX_PORT: *val = sc->vga_seq.seq_index; break; case SEQ_DATA_PORT: switch (sc->vga_seq.seq_index) { case SEQ_RESET: *val = sc->vga_seq.seq_reset; break; case SEQ_CLOCKING_MODE: *val = sc->vga_seq.seq_clock_mode; break; case SEQ_MAP_MASK: *val = sc->vga_seq.seq_map_mask; break; case SEQ_CHAR_MAP_SELECT: *val = sc->vga_seq.seq_cmap_sel; break; case SEQ_MEMORY_MODE: *val = sc->vga_seq.seq_mm; break; default: //printf("XXX VGA SEQ: inb 0x%04x at index %d\n", port, sc->vga_seq.seq_index); assert(0); break; } break; case DAC_DATA_PORT: *val = sc->vga_dac.dac_palette[3 * sc->vga_dac.dac_rd_index + sc->vga_dac.dac_rd_subindex]; sc->vga_dac.dac_rd_subindex++; if (sc->vga_dac.dac_rd_subindex == 3) { sc->vga_dac.dac_rd_index++; sc->vga_dac.dac_rd_subindex = 0; } break; case GC_IDX_PORT: *val = sc->vga_gc.gc_index; break; case GC_DATA_PORT: switch (sc->vga_gc.gc_index) { case GC_SET_RESET: *val = sc->vga_gc.gc_set_reset; break; case GC_ENABLE_SET_RESET: *val = sc->vga_gc.gc_enb_set_reset; break; case GC_COLOR_COMPARE: *val = sc->vga_gc.gc_color_compare; break; case GC_DATA_ROTATE: *val = sc->vga_gc.gc_rotate; break; case GC_READ_MAP_SELECT: *val = sc->vga_gc.gc_read_map_sel; break; case GC_MODE: *val = sc->vga_gc.gc_mode; break; case GC_MISCELLANEOUS: *val = sc->vga_gc.gc_misc; break; case GC_COLOR_DONT_CARE: *val = sc->vga_gc.gc_color_dont_care; break; case GC_BIT_MASK: *val = sc->vga_gc.gc_bit_mask; break; default: //printf("XXX VGA GC: inb 0x%04x at index %d\n", port, sc->vga_crtc.crtc_index); assert(0); break; } break; case GEN_MISC_OUTPUT_PORT: *val = sc->vga_misc; break; case GEN_INPUT_STS0_PORT: assert(0); break; case GEN_INPUT_STS1_MONO_PORT: case GEN_INPUT_STS1_COLOR_PORT: sc->vga_atc.atc_flipflop = 0; #ifdef __FreeBSD__ sc->vga_sts1 = GEN_IS1_VR | GEN_IS1_DE; //sc->vga_sts1 ^= (GEN_IS1_VR | GEN_IS1_DE); #else /* * During the bhyve bring-up process, a guest image was failing * to successfully boot. It appeared to be spinning, waiting * for this value to be toggled. Until it can be ruled out * that this is unnecessary (and documentation seems to * indicate that it should be present), the toggle should * remain. */ sc->vga_sts1 ^= (GEN_IS1_VR | GEN_IS1_DE); #endif *val = sc->vga_sts1; break; case GEN_FEATURE_CTRL_PORT: // OpenBSD calls this with bytes = 1 //assert(0); *val = 0; break; case 0x3c3: *val = 0; break; default: printf("XXX vga_port_in_handler() unhandled port 0x%x\n", port); //assert(0); return (-1); } return (0); } static int vga_port_out_handler(struct vmctx *ctx, int in, int port, int bytes, uint8_t val, void *arg) { struct vga_softc *sc = arg; switch (port) { case CRTC_IDX_MONO_PORT: case CRTC_IDX_COLOR_PORT: sc->vga_crtc.crtc_index = val; break; case CRTC_DATA_MONO_PORT: case CRTC_DATA_COLOR_PORT: switch (sc->vga_crtc.crtc_index) { case CRTC_HORIZ_TOTAL: sc->vga_crtc.crtc_horiz_total = val; break; case CRTC_HORIZ_DISP_END: sc->vga_crtc.crtc_horiz_disp_end = val; break; case CRTC_START_HORIZ_BLANK: sc->vga_crtc.crtc_start_horiz_blank = val; break; case CRTC_END_HORIZ_BLANK: sc->vga_crtc.crtc_end_horiz_blank = val; break; case CRTC_START_HORIZ_RETRACE: sc->vga_crtc.crtc_start_horiz_retrace = val; break; case CRTC_END_HORIZ_RETRACE: sc->vga_crtc.crtc_end_horiz_retrace = val; break; case CRTC_VERT_TOTAL: sc->vga_crtc.crtc_vert_total = val; break; case CRTC_OVERFLOW: sc->vga_crtc.crtc_overflow = val; break; case CRTC_PRESET_ROW_SCAN: sc->vga_crtc.crtc_present_row_scan = val; break; case CRTC_MAX_SCAN_LINE: sc->vga_crtc.crtc_max_scan_line = val; break; case CRTC_CURSOR_START: sc->vga_crtc.crtc_cursor_start = val; sc->vga_crtc.crtc_cursor_on = (val & CRTC_CS_CO) == 0; break; case CRTC_CURSOR_END: sc->vga_crtc.crtc_cursor_end = val; break; case CRTC_START_ADDR_HIGH: sc->vga_crtc.crtc_start_addr_high = val; sc->vga_crtc.crtc_start_addr &= 0x00ff; sc->vga_crtc.crtc_start_addr |= (val << 8); break; case CRTC_START_ADDR_LOW: sc->vga_crtc.crtc_start_addr_low = val; sc->vga_crtc.crtc_start_addr &= 0xff00; sc->vga_crtc.crtc_start_addr |= (val & 0xff); break; case CRTC_CURSOR_LOC_HIGH: sc->vga_crtc.crtc_cursor_loc_high = val; sc->vga_crtc.crtc_cursor_loc &= 0x00ff; sc->vga_crtc.crtc_cursor_loc |= (val << 8); break; case CRTC_CURSOR_LOC_LOW: sc->vga_crtc.crtc_cursor_loc_low = val; sc->vga_crtc.crtc_cursor_loc &= 0xff00; sc->vga_crtc.crtc_cursor_loc |= (val & 0xff); break; case CRTC_VERT_RETRACE_START: sc->vga_crtc.crtc_vert_retrace_start = val; break; case CRTC_VERT_RETRACE_END: sc->vga_crtc.crtc_vert_retrace_end = val; break; case CRTC_VERT_DISP_END: sc->vga_crtc.crtc_vert_disp_end = val; break; case CRTC_OFFSET: sc->vga_crtc.crtc_offset = val; break; case CRTC_UNDERLINE_LOC: sc->vga_crtc.crtc_underline_loc = val; break; case CRTC_START_VERT_BLANK: sc->vga_crtc.crtc_start_vert_blank = val; break; case CRTC_END_VERT_BLANK: sc->vga_crtc.crtc_end_vert_blank = val; break; case CRTC_MODE_CONTROL: sc->vga_crtc.crtc_mode_ctrl = val; break; case CRTC_LINE_COMPARE: sc->vga_crtc.crtc_line_compare = val; break; default: //printf("XXX VGA CRTC: outb 0x%04x, 0x%02x at index %d\n", port, val, sc->vga_crtc.crtc_index); assert(0); break; } break; case ATC_IDX_PORT: if (sc->vga_atc.atc_flipflop == 0) { if (sc->vga_atc.atc_index & 0x20) assert(0); sc->vga_atc.atc_index = val & ATC_IDX_MASK; } else { switch (sc->vga_atc.atc_index) { case ATC_PALETTE0 ... ATC_PALETTE15: sc->vga_atc.atc_palette[sc->vga_atc.atc_index] = val & 0x3f; break; case ATC_MODE_CONTROL: sc->vga_atc.atc_mode = val; break; case ATC_OVERSCAN_COLOR: sc->vga_atc.atc_overscan_color = val; break; case ATC_COLOR_PLANE_ENABLE: sc->vga_atc.atc_color_plane_enb = val; break; case ATC_HORIZ_PIXEL_PANNING: sc->vga_atc.atc_horiz_pixel_panning = val; break; case ATC_COLOR_SELECT: sc->vga_atc.atc_color_select = val; sc->vga_atc.atc_color_select_45 = (val & ATC_CS_C45) << 4; sc->vga_atc.atc_color_select_67 = ((val & ATC_CS_C67) >> 2) << 6; break; default: //printf("XXX VGA ATC: outb 0x%04x, 0x%02x at index %d\n", port, val, sc->vga_atc.atc_index); assert(0); break; } } sc->vga_atc.atc_flipflop ^= 1; break; case ATC_DATA_PORT: break; case SEQ_IDX_PORT: sc->vga_seq.seq_index = val & 0x1f; break; case SEQ_DATA_PORT: switch (sc->vga_seq.seq_index) { case SEQ_RESET: sc->vga_seq.seq_reset = val; break; case SEQ_CLOCKING_MODE: sc->vga_seq.seq_clock_mode = val; sc->vga_seq.seq_cm_dots = (val & SEQ_CM_89) ? 8 : 9; break; case SEQ_MAP_MASK: sc->vga_seq.seq_map_mask = val; break; case SEQ_CHAR_MAP_SELECT: sc->vga_seq.seq_cmap_sel = val; sc->vga_seq.seq_cmap_pri_off = ((((val & SEQ_CMS_SA) >> SEQ_CMS_SA_SHIFT) * 2) + ((val & SEQ_CMS_SAH) >> SEQ_CMS_SAH_SHIFT)) * 8 * KB; sc->vga_seq.seq_cmap_sec_off = ((((val & SEQ_CMS_SB) >> SEQ_CMS_SB_SHIFT) * 2) + ((val & SEQ_CMS_SBH) >> SEQ_CMS_SBH_SHIFT)) * 8 * KB; break; case SEQ_MEMORY_MODE: sc->vga_seq.seq_mm = val; /* Windows queries Chain4 */ //assert((sc->vga_seq.seq_mm & SEQ_MM_C4) == 0); break; default: //printf("XXX VGA SEQ: outb 0x%04x, 0x%02x at index %d\n", port, val, sc->vga_seq.seq_index); assert(0); break; } break; case DAC_MASK: break; case DAC_IDX_RD_PORT: sc->vga_dac.dac_rd_index = val; sc->vga_dac.dac_rd_subindex = 0; break; case DAC_IDX_WR_PORT: sc->vga_dac.dac_wr_index = val; sc->vga_dac.dac_wr_subindex = 0; break; case DAC_DATA_PORT: sc->vga_dac.dac_palette[3 * sc->vga_dac.dac_wr_index + sc->vga_dac.dac_wr_subindex] = val; sc->vga_dac.dac_wr_subindex++; if (sc->vga_dac.dac_wr_subindex == 3) { sc->vga_dac.dac_palette_rgb[sc->vga_dac.dac_wr_index] = ((((sc->vga_dac.dac_palette[3*sc->vga_dac.dac_wr_index + 0] << 2) | ((sc->vga_dac.dac_palette[3*sc->vga_dac.dac_wr_index + 0] & 0x1) << 1) | (sc->vga_dac.dac_palette[3*sc->vga_dac.dac_wr_index + 0] & 0x1)) << 16) | (((sc->vga_dac.dac_palette[3*sc->vga_dac.dac_wr_index + 1] << 2) | ((sc->vga_dac.dac_palette[3*sc->vga_dac.dac_wr_index + 1] & 0x1) << 1) | (sc->vga_dac.dac_palette[3*sc->vga_dac.dac_wr_index + 1] & 0x1)) << 8) | (((sc->vga_dac.dac_palette[3*sc->vga_dac.dac_wr_index + 2] << 2) | ((sc->vga_dac.dac_palette[3*sc->vga_dac.dac_wr_index + 2] & 0x1) << 1) | (sc->vga_dac.dac_palette[3*sc->vga_dac.dac_wr_index + 2] & 0x1)) << 0)); sc->vga_dac.dac_wr_index++; sc->vga_dac.dac_wr_subindex = 0; } break; case GC_IDX_PORT: sc->vga_gc.gc_index = val; break; case GC_DATA_PORT: switch (sc->vga_gc.gc_index) { case GC_SET_RESET: sc->vga_gc.gc_set_reset = val; break; case GC_ENABLE_SET_RESET: sc->vga_gc.gc_enb_set_reset = val; break; case GC_COLOR_COMPARE: sc->vga_gc.gc_color_compare = val; break; case GC_DATA_ROTATE: sc->vga_gc.gc_rotate = val; sc->vga_gc.gc_op = (val >> 3) & 0x3; break; case GC_READ_MAP_SELECT: sc->vga_gc.gc_read_map_sel = val; break; case GC_MODE: sc->vga_gc.gc_mode = val; sc->vga_gc.gc_mode_c4 = (val & GC_MODE_C4) != 0; assert(!sc->vga_gc.gc_mode_c4); sc->vga_gc.gc_mode_oe = (val & GC_MODE_OE) != 0; sc->vga_gc.gc_mode_rm = (val >> 3) & 0x1; sc->vga_gc.gc_mode_wm = val & 0x3; if (sc->gc_image) sc->gc_image->vgamode = 1; break; case GC_MISCELLANEOUS: sc->vga_gc.gc_misc = val; sc->vga_gc.gc_misc_gm = val & GC_MISC_GM; sc->vga_gc.gc_misc_mm = (val & GC_MISC_MM) >> GC_MISC_MM_SHIFT; break; case GC_COLOR_DONT_CARE: sc->vga_gc.gc_color_dont_care = val; break; case GC_BIT_MASK: sc->vga_gc.gc_bit_mask = val; break; default: //printf("XXX VGA GC: outb 0x%04x, 0x%02x at index %d\n", port, val, sc->vga_gc.gc_index); assert(0); break; } break; case GEN_INPUT_STS0_PORT: /* write to Miscellaneous Output Register */ sc->vga_misc = val; break; case GEN_INPUT_STS1_MONO_PORT: case GEN_INPUT_STS1_COLOR_PORT: /* write to Feature Control Register */ break; // case 0x3c3: // break; default: printf("XXX vga_port_out_handler() unhandled port 0x%x, val 0x%x\n", port, val); //assert(0); return (-1); } return (0); } static int vga_port_handler(struct vmctx *ctx, int in, int port, int bytes, uint32_t *eax, void *arg) { uint8_t val; int error; switch (bytes) { case 1: if (in) { *eax &= ~0xff; error = vga_port_in_handler(ctx, in, port, 1, &val, arg); if (!error) { *eax |= val & 0xff; } } else { val = *eax & 0xff; error = vga_port_out_handler(ctx, in, port, 1, val, arg); } break; case 2: if (in) { *eax &= ~0xffff; error = vga_port_in_handler(ctx, in, port, 1, &val, arg); if (!error) { *eax |= val & 0xff; } error = vga_port_in_handler(ctx, in, port + 1, 1, &val, arg); if (!error) { *eax |= (val & 0xff) << 8; } } else { val = *eax & 0xff; error = vga_port_out_handler(ctx, in, port, 1, val, arg); val = (*eax >> 8) & 0xff; error =vga_port_out_handler(ctx, in, port + 1, 1, val, arg); } break; default: assert(0); return (-1); } return (error); } void * vga_init(int io_only) { struct inout_port iop; struct vga_softc *sc; int port, error; sc = calloc(1, sizeof(struct vga_softc)); bzero(&iop, sizeof(struct inout_port)); iop.name = "VGA"; for (port = VGA_IOPORT_START; port <= VGA_IOPORT_END; port++) { iop.port = port; iop.size = 1; iop.flags = IOPORT_F_INOUT; iop.handler = vga_port_handler; iop.arg = sc; error = register_inout(&iop); assert(error == 0); } sc->gc_image = console_get_image(); /* only handle io ports; vga graphics is disabled */ if (io_only) return(sc); sc->mr.name = "VGA memory"; sc->mr.flags = MEM_F_RW; sc->mr.base = 640 * KB; sc->mr.size = 128 * KB; sc->mr.handler = vga_mem_handler; sc->mr.arg1 = sc; error = register_mem_fallback(&sc->mr); assert(error == 0); sc->vga_ram = malloc(256 * KB); memset(sc->vga_ram, 0, 256 * KB); { static uint8_t palette[] = { 0x00,0x00,0x00, 0x00,0x00,0x2a, 0x00,0x2a,0x00, 0x00,0x2a,0x2a, 0x2a,0x00,0x00, 0x2a,0x00,0x2a, 0x2a,0x2a,0x00, 0x2a,0x2a,0x2a, 0x00,0x00,0x15, 0x00,0x00,0x3f, 0x00,0x2a,0x15, 0x00,0x2a,0x3f, 0x2a,0x00,0x15, 0x2a,0x00,0x3f, 0x2a,0x2a,0x15, 0x2a,0x2a,0x3f, }; int i; memcpy(sc->vga_dac.dac_palette, palette, 16 * 3 * sizeof (uint8_t)); for (i = 0; i < 16; i++) { sc->vga_dac.dac_palette_rgb[i] = ((((sc->vga_dac.dac_palette[3*i + 0] << 2) | ((sc->vga_dac.dac_palette[3*i + 0] & 0x1) << 1) | (sc->vga_dac.dac_palette[3*i + 0] & 0x1)) << 16) | (((sc->vga_dac.dac_palette[3*i + 1] << 2) | ((sc->vga_dac.dac_palette[3*i + 1] & 0x1) << 1) | (sc->vga_dac.dac_palette[3*i + 1] & 0x1)) << 8) | (((sc->vga_dac.dac_palette[3*i + 2] << 2) | ((sc->vga_dac.dac_palette[3*i + 2] & 0x1) << 1) | (sc->vga_dac.dac_palette[3*i + 2] & 0x1)) << 0)); } } return (sc); } /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2015 Tycho Nightingale * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef _VGA_H_ #define _VGA_H_ #define VGA_IOPORT_START 0x3c0 #define VGA_IOPORT_END 0x3df /* General registers */ #define GEN_INPUT_STS0_PORT 0x3c2 #define GEN_FEATURE_CTRL_PORT 0x3ca #define GEN_MISC_OUTPUT_PORT 0x3cc #define GEN_INPUT_STS1_MONO_PORT 0x3ba #define GEN_INPUT_STS1_COLOR_PORT 0x3da #define GEN_IS1_VR 0x08 /* Vertical retrace */ #define GEN_IS1_DE 0x01 /* Display enable not */ /* Attribute controller registers. */ #define ATC_IDX_PORT 0x3c0 #define ATC_DATA_PORT 0x3c1 #define ATC_IDX_MASK 0x1f #define ATC_PALETTE0 0 #define ATC_PALETTE15 15 #define ATC_MODE_CONTROL 16 #define ATC_MC_IPS 0x80 /* Internal palette size */ #define ATC_MC_GA 0x01 /* Graphics/alphanumeric */ #define ATC_OVERSCAN_COLOR 17 #define ATC_COLOR_PLANE_ENABLE 18 #define ATC_HORIZ_PIXEL_PANNING 19 #define ATC_COLOR_SELECT 20 #define ATC_CS_C67 0x0c /* Color select bits 6+7 */ #define ATC_CS_C45 0x03 /* Color select bits 4+5 */ /* Sequencer registers. */ #define SEQ_IDX_PORT 0x3c4 #define SEQ_DATA_PORT 0x3c5 #define SEQ_RESET 0 #define SEQ_RESET_ASYNC 0x1 #define SEQ_RESET_SYNC 0x2 #define SEQ_CLOCKING_MODE 1 #define SEQ_CM_SO 0x20 /* Screen off */ #define SEQ_CM_89 0x01 /* 8/9 dot clock */ #define SEQ_MAP_MASK 2 #define SEQ_CHAR_MAP_SELECT 3 #define SEQ_CMS_SAH 0x20 /* Char map A bit 2 */ #define SEQ_CMS_SAH_SHIFT 5 #define SEQ_CMS_SA 0x0c /* Char map A bits 0+1 */ #define SEQ_CMS_SA_SHIFT 2 #define SEQ_CMS_SBH 0x10 /* Char map B bit 2 */ #define SEQ_CMS_SBH_SHIFT 4 #define SEQ_CMS_SB 0x03 /* Char map B bits 0+1 */ #define SEQ_CMS_SB_SHIFT 0 #define SEQ_MEMORY_MODE 4 #define SEQ_MM_C4 0x08 /* Chain 4 */ #define SEQ_MM_OE 0x04 /* Odd/even */ #define SEQ_MM_EM 0x02 /* Extended memory */ /* Graphics controller registers. */ #define GC_IDX_PORT 0x3ce #define GC_DATA_PORT 0x3cf #define GC_SET_RESET 0 #define GC_ENABLE_SET_RESET 1 #define GC_COLOR_COMPARE 2 #define GC_DATA_ROTATE 3 #define GC_READ_MAP_SELECT 4 #define GC_MODE 5 #define GC_MODE_OE 0x10 /* Odd/even */ #define GC_MODE_C4 0x04 /* Chain 4 */ #define GC_MISCELLANEOUS 6 #define GC_MISC_GM 0x01 /* Graphics/alphanumeric */ #define GC_MISC_MM 0x0c /* memory map */ #define GC_MISC_MM_SHIFT 2 #define GC_COLOR_DONT_CARE 7 #define GC_BIT_MASK 8 /* CRT controller registers. */ #define CRTC_IDX_MONO_PORT 0x3b4 #define CRTC_DATA_MONO_PORT 0x3b5 #define CRTC_IDX_COLOR_PORT 0x3d4 #define CRTC_DATA_COLOR_PORT 0x3d5 #define CRTC_HORIZ_TOTAL 0 #define CRTC_HORIZ_DISP_END 1 #define CRTC_START_HORIZ_BLANK 2 #define CRTC_END_HORIZ_BLANK 3 #define CRTC_START_HORIZ_RETRACE 4 #define CRTC_END_HORIZ_RETRACE 5 #define CRTC_VERT_TOTAL 6 #define CRTC_OVERFLOW 7 #define CRTC_OF_VRS9 0x80 /* VRS bit 9 */ #define CRTC_OF_VRS9_SHIFT 7 #define CRTC_OF_VDE9 0x40 /* VDE bit 9 */ #define CRTC_OF_VDE9_SHIFT 6 #define CRTC_OF_VRS8 0x04 /* VRS bit 8 */ #define CRTC_OF_VRS8_SHIFT 2 #define CRTC_OF_VDE8 0x02 /* VDE bit 8 */ #define CRTC_OF_VDE8_SHIFT 1 #define CRTC_PRESET_ROW_SCAN 8 #define CRTC_MAX_SCAN_LINE 9 #define CRTC_MSL_MSL 0x1f #define CRTC_CURSOR_START 10 #define CRTC_CS_CO 0x20 /* Cursor off */ #define CRTC_CS_CS 0x1f /* Cursor start */ #define CRTC_CURSOR_END 11 #define CRTC_CE_CE 0x1f /* Cursor end */ #define CRTC_START_ADDR_HIGH 12 #define CRTC_START_ADDR_LOW 13 #define CRTC_CURSOR_LOC_HIGH 14 #define CRTC_CURSOR_LOC_LOW 15 #define CRTC_VERT_RETRACE_START 16 #define CRTC_VERT_RETRACE_END 17 #define CRTC_VRE_MASK 0xf #define CRTC_VERT_DISP_END 18 #define CRTC_OFFSET 19 #define CRTC_UNDERLINE_LOC 20 #define CRTC_START_VERT_BLANK 21 #define CRTC_END_VERT_BLANK 22 #define CRTC_MODE_CONTROL 23 #define CRTC_MC_TE 0x80 /* Timing enable */ #define CRTC_LINE_COMPARE 24 /* DAC registers */ #define DAC_MASK 0x3c6 #define DAC_IDX_RD_PORT 0x3c7 #define DAC_IDX_WR_PORT 0x3c8 #define DAC_DATA_PORT 0x3c9 void *vga_init(int io_only); #endif /* _VGA_H_ */ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2011 NetApp, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * This file and its contents are supplied under the terms of the * Common Development and Distribution License ("CDDL"), version 1.0. * You may only use this file in accordance with the terms of version * 1.0 of the CDDL. * * A full copy of the text of the CDDL should have accompanied this * source. A copy of the CDDL is also available via the Internet at * http://www.illumos.org/license/CDDL. * * Copyright 2015 Pluribus Networks Inc. * Copyright 2018 Joyent, Inc. * Copyright 2022 Oxide Computer Company * Copyright 2022 OmniOS Community Edition (OmniOSce) Association. */ #include #ifndef __FreeBSD__ #include #include #endif #include #ifndef WITHOUT_CAPSICUM #include #endif #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "bhyverun.h" #include "config.h" #include "debug.h" #include "gdb.h" #include "inout.h" #include "mem.h" #include "spinup_ap.h" #include "vmexit.h" #include "xmsr.h" #ifndef __FreeBSD__ static struct vm_entry *vmentry; int vmentry_init(int ncpus) { vmentry = calloc(ncpus, sizeof(*vmentry)); return (vmentry == NULL ? -1 : 0); } struct vm_entry * vmentry_vcpu(int vcpuid) { return (&vmentry[vcpuid]); } static void vmentry_mmio_read(struct vcpu *vcpu, uint64_t gpa, uint8_t bytes, uint64_t data) { struct vm_entry *entry = &vmentry[vcpu_id(vcpu)]; struct vm_mmio *mmio = &entry->u.mmio; assert(entry->cmd == VEC_DEFAULT); entry->cmd = VEC_FULFILL_MMIO; mmio->bytes = bytes; mmio->read = 1; mmio->gpa = gpa; mmio->data = data; } static void vmentry_mmio_write(struct vcpu *vcpu, uint64_t gpa, uint8_t bytes) { struct vm_entry *entry = &vmentry[vcpu_id(vcpu)]; struct vm_mmio *mmio = &entry->u.mmio; assert(entry->cmd == VEC_DEFAULT); entry->cmd = VEC_FULFILL_MMIO; mmio->bytes = bytes; mmio->read = 0; mmio->gpa = gpa; mmio->data = 0; } static void vmentry_inout_read(struct vcpu *vcpu, uint16_t port, uint8_t bytes, uint32_t data) { struct vm_entry *entry = &vmentry[vcpu_id(vcpu)]; struct vm_inout *inout = &entry->u.inout; assert(entry->cmd == VEC_DEFAULT); entry->cmd = VEC_FULFILL_INOUT; inout->bytes = bytes; inout->flags = INOUT_IN; inout->port = port; inout->eax = data; } static void vmentry_inout_write(struct vcpu *vcpu, uint16_t port, uint8_t bytes) { struct vm_entry *entry = &vmentry[vcpu_id(vcpu)]; struct vm_inout *inout = &entry->u.inout; assert(entry->cmd == VEC_DEFAULT); entry->cmd = VEC_FULFILL_INOUT; inout->bytes = bytes; inout->flags = 0; inout->port = port; inout->eax = 0; } #endif #ifdef __FreeBSD__ void vm_inject_fault(struct vcpu *vcpu, int vector, int errcode_valid, int errcode) { int error, restart_instruction; restart_instruction = 1; error = vm_inject_exception(vcpu, vector, errcode_valid, errcode, restart_instruction); assert(error == 0); } #endif static int vmexit_inout(struct vmctx *ctx, struct vcpu *vcpu, struct vm_exit *vme) { int error; struct vm_inout inout; bool in; uint8_t bytes; inout = vme->u.inout; in = (inout.flags & INOUT_IN) != 0; bytes = inout.bytes; error = emulate_inout(ctx, vcpu, &inout); if (error) { EPRINTLN("Unhandled %s%c 0x%04x at 0x%lx", in ? "in" : "out", bytes == 1 ? 'b' : (bytes == 2 ? 'w' : 'l'), inout.port, vme->rip); return (VMEXIT_ABORT); } else { /* * Communicate the status of the inout operation back to the * in-kernel instruction emulation. */ if (in) { vmentry_inout_read(vcpu, inout.port, bytes, inout.eax); } else { vmentry_inout_write(vcpu, inout.port, bytes); } return (VMEXIT_CONTINUE); } } static int vmexit_rdmsr(struct vmctx *ctx __unused, struct vcpu *vcpu, struct vm_exit *vme) { uint64_t val; uint32_t eax, edx; int error; val = 0; error = emulate_rdmsr(vcpu, vme->u.msr.code, &val); if (error != 0) { EPRINTLN("rdmsr to register %#x on vcpu %d", vme->u.msr.code, vcpu_id(vcpu)); if (get_config_bool("x86.strictmsr")) { vm_inject_gp(vcpu); return (VMEXIT_CONTINUE); } } eax = val; error = vm_set_register(vcpu, VM_REG_GUEST_RAX, eax); assert(error == 0); edx = val >> 32; error = vm_set_register(vcpu, VM_REG_GUEST_RDX, edx); assert(error == 0); return (VMEXIT_CONTINUE); } static int vmexit_wrmsr(struct vmctx *ctx __unused, struct vcpu *vcpu, struct vm_exit *vme) { int error; error = emulate_wrmsr(vcpu, vme->u.msr.code, vme->u.msr.wval); if (error != 0) { EPRINTLN("wrmsr to register %#x(%#lx) on vcpu %d", vme->u.msr.code, vme->u.msr.wval, vcpu_id(vcpu)); if (get_config_bool("x86.strictmsr")) { vm_inject_gp(vcpu); return (VMEXIT_CONTINUE); } } return (VMEXIT_CONTINUE); } static const char * const vmx_exit_reason_desc[] = { [EXIT_REASON_EXCEPTION] = "Exception or non-maskable interrupt (NMI)", [EXIT_REASON_EXT_INTR] = "External interrupt", [EXIT_REASON_TRIPLE_FAULT] = "Triple fault", [EXIT_REASON_INIT] = "INIT signal", [EXIT_REASON_SIPI] = "Start-up IPI (SIPI)", [EXIT_REASON_IO_SMI] = "I/O system-management interrupt (SMI)", [EXIT_REASON_SMI] = "Other SMI", [EXIT_REASON_INTR_WINDOW] = "Interrupt window", [EXIT_REASON_NMI_WINDOW] = "NMI window", [EXIT_REASON_TASK_SWITCH] = "Task switch", [EXIT_REASON_CPUID] = "CPUID", [EXIT_REASON_GETSEC] = "GETSEC", [EXIT_REASON_HLT] = "HLT", [EXIT_REASON_INVD] = "INVD", [EXIT_REASON_INVLPG] = "INVLPG", [EXIT_REASON_RDPMC] = "RDPMC", [EXIT_REASON_RDTSC] = "RDTSC", [EXIT_REASON_RSM] = "RSM", [EXIT_REASON_VMCALL] = "VMCALL", [EXIT_REASON_VMCLEAR] = "VMCLEAR", [EXIT_REASON_VMLAUNCH] = "VMLAUNCH", [EXIT_REASON_VMPTRLD] = "VMPTRLD", [EXIT_REASON_VMPTRST] = "VMPTRST", [EXIT_REASON_VMREAD] = "VMREAD", [EXIT_REASON_VMRESUME] = "VMRESUME", [EXIT_REASON_VMWRITE] = "VMWRITE", [EXIT_REASON_VMXOFF] = "VMXOFF", [EXIT_REASON_VMXON] = "VMXON", [EXIT_REASON_CR_ACCESS] = "Control-register accesses", [EXIT_REASON_DR_ACCESS] = "MOV DR", [EXIT_REASON_INOUT] = "I/O instruction", [EXIT_REASON_RDMSR] = "RDMSR", [EXIT_REASON_WRMSR] = "WRMSR", [EXIT_REASON_INVAL_VMCS] = "VM-entry failure due to invalid guest state", [EXIT_REASON_INVAL_MSR] = "VM-entry failure due to MSR loading", [EXIT_REASON_MWAIT] = "MWAIT", [EXIT_REASON_MTF] = "Monitor trap flag", [EXIT_REASON_MONITOR] = "MONITOR", [EXIT_REASON_PAUSE] = "PAUSE", [EXIT_REASON_MCE_DURING_ENTRY] = "VM-entry failure due to machine-check event", [EXIT_REASON_TPR] = "TPR below threshold", [EXIT_REASON_APIC_ACCESS] = "APIC access", [EXIT_REASON_VIRTUALIZED_EOI] = "Virtualized EOI", [EXIT_REASON_GDTR_IDTR] = "Access to GDTR or IDTR", [EXIT_REASON_LDTR_TR] = "Access to LDTR or TR", [EXIT_REASON_EPT_FAULT] = "EPT violation", [EXIT_REASON_EPT_MISCONFIG] = "EPT misconfiguration", [EXIT_REASON_INVEPT] = "INVEPT", [EXIT_REASON_RDTSCP] = "RDTSCP", [EXIT_REASON_VMX_PREEMPT] = "VMX-preemption timer expired", [EXIT_REASON_INVVPID] = "INVVPID", [EXIT_REASON_WBINVD] = "WBINVD", [EXIT_REASON_XSETBV] = "XSETBV", [EXIT_REASON_APIC_WRITE] = "APIC write", [EXIT_REASON_RDRAND] = "RDRAND", [EXIT_REASON_INVPCID] = "INVPCID", [EXIT_REASON_VMFUNC] = "VMFUNC", [EXIT_REASON_ENCLS] = "ENCLS", [EXIT_REASON_RDSEED] = "RDSEED", [EXIT_REASON_PM_LOG_FULL] = "Page-modification log full", [EXIT_REASON_XSAVES] = "XSAVES", [EXIT_REASON_XRSTORS] = "XRSTORS" }; #ifndef __FreeBSD__ static int vmexit_run_state(struct vmctx *ctx __unused, struct vcpu *vcpu __unused, struct vm_exit *vme __unused) { /* * Run-state transitions (INIT, SIPI, etc) are handled in-kernel, so an * exit to userspace with that code is not expected. */ fprintf(stderr, "unexpected run-state VM exit"); return (VMEXIT_ABORT); } static int vmexit_paging(struct vmctx *ctx __unused, struct vcpu *vcpu, struct vm_exit *vme) { fprintf(stderr, "vm exit[%d]\n", vcpu_id(vcpu)); fprintf(stderr, "\treason\t\tPAGING\n"); fprintf(stderr, "\trip\t\t0x%016lx\n", vme->rip); fprintf(stderr, "\tgpa\t\t0x%016lx\n", vme->u.paging.gpa); fprintf(stderr, "\tfault_type\t\t%d\n", vme->u.paging.fault_type); return (VMEXIT_ABORT); } #endif /* __FreeBSD__ */ #ifdef __FreeBSD__ #define DEBUG_EPT_MISCONFIG #else /* EPT misconfig debugging not possible now that raw VMCS access is gone */ #endif #ifdef DEBUG_EPT_MISCONFIG #define VMCS_GUEST_PHYSICAL_ADDRESS 0x00002400 static uint64_t ept_misconfig_gpa, ept_misconfig_pte[4]; static int ept_misconfig_ptenum; #endif static const char * vmexit_vmx_desc(uint32_t exit_reason) { if (exit_reason >= nitems(vmx_exit_reason_desc) || vmx_exit_reason_desc[exit_reason] == NULL) return ("Unknown"); return (vmx_exit_reason_desc[exit_reason]); } static int vmexit_vmx(struct vmctx *ctx, struct vcpu *vcpu, struct vm_exit *vme) { EPRINTLN("vm exit[%d]", vcpu_id(vcpu)); EPRINTLN("\treason\t\tVMX"); EPRINTLN("\trip\t\t0x%016lx", vme->rip); EPRINTLN("\tinst_length\t%d", vme->inst_length); EPRINTLN("\tstatus\t\t%d", vme->u.vmx.status); EPRINTLN("\texit_reason\t%u (%s)", vme->u.vmx.exit_reason, vmexit_vmx_desc(vme->u.vmx.exit_reason)); EPRINTLN("\tqualification\t0x%016lx", vme->u.vmx.exit_qualification); EPRINTLN("\tinst_type\t\t%d", vme->u.vmx.inst_type); EPRINTLN("\tinst_error\t\t%d", vme->u.vmx.inst_error); #ifdef DEBUG_EPT_MISCONFIG if (vme->u.vmx.exit_reason == EXIT_REASON_EPT_MISCONFIG) { vm_get_register(vcpu, VMCS_IDENT(VMCS_GUEST_PHYSICAL_ADDRESS), &ept_misconfig_gpa); vm_get_gpa_pmap(ctx, ept_misconfig_gpa, ept_misconfig_pte, &ept_misconfig_ptenum); EPRINTLN("\tEPT misconfiguration:"); EPRINTLN("\t\tGPA: %#lx", ept_misconfig_gpa); EPRINTLN("\t\tPTE(%d): %#lx %#lx %#lx %#lx", ept_misconfig_ptenum, ept_misconfig_pte[0], ept_misconfig_pte[1], ept_misconfig_pte[2], ept_misconfig_pte[3]); } #endif /* DEBUG_EPT_MISCONFIG */ return (VMEXIT_ABORT); } static int vmexit_svm(struct vmctx *ctx __unused, struct vcpu *vcpu, struct vm_exit *vme) { EPRINTLN("vm exit[%d]", vcpu_id(vcpu)); EPRINTLN("\treason\t\tSVM"); EPRINTLN("\trip\t\t0x%016lx", vme->rip); EPRINTLN("\tinst_length\t%d", vme->inst_length); EPRINTLN("\texitcode\t%#lx", vme->u.svm.exitcode); EPRINTLN("\texitinfo1\t%#lx", vme->u.svm.exitinfo1); EPRINTLN("\texitinfo2\t%#lx", vme->u.svm.exitinfo2); return (VMEXIT_ABORT); } static int vmexit_bogus(struct vmctx *ctx __unused, struct vcpu *vcpu __unused, struct vm_exit *vme) { assert(vme->inst_length == 0); return (VMEXIT_CONTINUE); } static int vmexit_hlt(struct vmctx *ctx __unused, struct vcpu *vcpu __unused, struct vm_exit *vme __unused) { /* * Just continue execution with the next instruction. We use * the HLT VM exit as a way to be friendly with the host * scheduler. */ return (VMEXIT_CONTINUE); } static int vmexit_pause(struct vmctx *ctx __unused, struct vcpu *vcpu __unused, struct vm_exit *vme __unused) { return (VMEXIT_CONTINUE); } static int vmexit_mtrap(struct vmctx *ctx __unused, struct vcpu *vcpu, struct vm_exit *vme) { assert(vme->inst_length == 0); gdb_cpu_mtrap(vcpu); return (VMEXIT_CONTINUE); } static int vmexit_inst_emul(struct vmctx *ctx __unused, struct vcpu *vcpu, struct vm_exit *vme) { uint8_t i, valid; fprintf(stderr, "Failed to emulate instruction sequence "); valid = vme->u.inst_emul.num_valid; if (valid != 0) { assert(valid <= sizeof (vme->u.inst_emul.inst)); fprintf(stderr, "["); for (i = 0; i < valid; i++) { if (i == 0) { fprintf(stderr, "%02x", vme->u.inst_emul.inst[i]); } else { fprintf(stderr, ", %02x", vme->u.inst_emul.inst[i]); } } fprintf(stderr, "] "); } fprintf(stderr, "@ %rip = %x\n", vme->rip); return (VMEXIT_ABORT); } #ifndef __FreeBSD__ static int vmexit_mmio(struct vmctx *ctx __unused, struct vcpu *vcpu, struct vm_exit *vme) { int err; struct vm_mmio mmio; bool is_read; mmio = vme->u.mmio; is_read = (mmio.read != 0); err = emulate_mem(vcpu, &mmio); if (err == ESRCH) { fprintf(stderr, "Unhandled memory access to 0x%lx\n", mmio.gpa); /* * Access to non-existent physical addresses is not likely to * result in fatal errors on hardware machines, but rather reads * of all-ones or discarded-but-acknowledged writes. */ mmio.data = ~0UL; err = 0; } if (err == 0) { if (is_read) { vmentry_mmio_read(vcpu, mmio.gpa, mmio.bytes, mmio.data); } else { vmentry_mmio_write(vcpu, mmio.gpa, mmio.bytes); } return (VMEXIT_CONTINUE); } fprintf(stderr, "Unhandled mmio error to 0x%lx: %d\n", mmio.gpa, err); return (VMEXIT_ABORT); } #endif /* !__FreeBSD__ */ static int vmexit_suspend(struct vmctx *ctx, struct vcpu *vcpu, struct vm_exit *vme) { enum vm_suspend_how how; int vcpuid = vcpu_id(vcpu); how = vme->u.suspended.how; fbsdrun_deletecpu(vcpuid); switch (how) { case VM_SUSPEND_RESET: exit(0); case VM_SUSPEND_POWEROFF: if (get_config_bool_default("destroy_on_poweroff", false)) vm_destroy(ctx); exit(1); case VM_SUSPEND_HALT: exit(2); case VM_SUSPEND_TRIPLEFAULT: exit(3); default: EPRINTLN("vmexit_suspend: invalid reason %d", how); exit(100); } return (0); /* NOTREACHED */ } static int vmexit_debug(struct vmctx *ctx __unused, struct vcpu *vcpu, struct vm_exit *vme __unused) { gdb_cpu_suspend(vcpu); /* * Sleep for a short period to avoid chewing up the CPU in the * window between activation of the vCPU thread and the STARTUP IPI. */ usleep(1000); return (VMEXIT_CONTINUE); } static int vmexit_breakpoint(struct vmctx *ctx __unused, struct vcpu *vcpu, struct vm_exit *vme) { gdb_cpu_breakpoint(vcpu, vme); return (VMEXIT_CONTINUE); } #ifdef __FreeBSD__ static int vmexit_ipi(struct vmctx *ctx __unused, struct vcpu *vcpu __unused, struct vm_exit *vme) { int error = -1; int i; switch (vme->u.ipi.mode) { case APIC_DELMODE_INIT: CPU_FOREACH_ISSET(i, &vme->u.ipi.dmask) { error = vm_suspend_cpu(vcpu_info[i].vcpu); if (error) { warnx("%s: failed to suspend cpu %d\n", __func__, i); break; } } break; case APIC_DELMODE_STARTUP: CPU_FOREACH_ISSET(i, &vme->u.ipi.dmask) { spinup_ap(vcpu_info[i].vcpu, vme->u.ipi.vector << PAGE_SHIFT); } error = 0; break; default: break; } return (error); } #endif const vmexit_handler_t vmexit_handlers[VM_EXITCODE_MAX] = { [VM_EXITCODE_INOUT] = vmexit_inout, #ifndef __FreeBSD__ [VM_EXITCODE_MMIO] = vmexit_mmio, #endif [VM_EXITCODE_VMX] = vmexit_vmx, [VM_EXITCODE_SVM] = vmexit_svm, [VM_EXITCODE_BOGUS] = vmexit_bogus, [VM_EXITCODE_RDMSR] = vmexit_rdmsr, [VM_EXITCODE_WRMSR] = vmexit_wrmsr, [VM_EXITCODE_MTRAP] = vmexit_mtrap, [VM_EXITCODE_INST_EMUL] = vmexit_inst_emul, #ifndef __FreeBSD__ [VM_EXITCODE_RUN_STATE] = vmexit_run_state, [VM_EXITCODE_PAGING] = vmexit_paging, #endif [VM_EXITCODE_SUSPENDED] = vmexit_suspend, [VM_EXITCODE_TASK_SWITCH] = vmexit_task_switch, [VM_EXITCODE_DEBUG] = vmexit_debug, [VM_EXITCODE_BPT] = vmexit_breakpoint, #ifdef __FreeBSD__ [VM_EXITCODE_IPI] = vmexit_ipi, #endif [VM_EXITCODE_HLT] = vmexit_hlt, [VM_EXITCODE_PAUSE] = vmexit_pause, }; /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2011 NetApp, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include #include #include #include #include #include #include #include #include #include "debug.h" #include "xmsr.h" static int cpu_vendor_intel, cpu_vendor_amd, cpu_vendor_hygon; int emulate_wrmsr(struct vcpu *vcpu __unused, uint32_t num, uint64_t val __unused) { if (cpu_vendor_intel) { switch (num) { #ifndef __FreeBSD__ case MSR_PERFCTR0: case MSR_PERFCTR1: case MSR_EVNTSEL0: case MSR_EVNTSEL1: return (0); #endif case 0xd04: /* Sandy Bridge uncore PMCs */ case 0xc24: return (0); case MSR_BIOS_UPDT_TRIG: return (0); case MSR_BIOS_SIGN: return (0); default: break; } } else if (cpu_vendor_amd || cpu_vendor_hygon) { switch (num) { case MSR_HWCR: /* * Ignore writes to hardware configuration MSR. */ return (0); case MSR_NB_CFG1: case MSR_LS_CFG: case MSR_IC_CFG: return (0); /* Ignore writes */ case MSR_PERFEVSEL0: case MSR_PERFEVSEL1: case MSR_PERFEVSEL2: case MSR_PERFEVSEL3: /* Ignore writes to the PerfEvtSel MSRs */ return (0); case MSR_K7_PERFCTR0: case MSR_K7_PERFCTR1: case MSR_K7_PERFCTR2: case MSR_K7_PERFCTR3: /* Ignore writes to the PerfCtr MSRs */ return (0); case MSR_P_STATE_CONTROL: /* Ignore write to change the P-state */ return (0); default: break; } } return (-1); } int emulate_rdmsr(struct vcpu *vcpu __unused, uint32_t num, uint64_t *val) { int error = 0; if (cpu_vendor_intel) { switch (num) { case MSR_BIOS_SIGN: case MSR_IA32_PLATFORM_ID: case MSR_PKG_ENERGY_STATUS: case MSR_PP0_ENERGY_STATUS: case MSR_PP1_ENERGY_STATUS: case MSR_DRAM_ENERGY_STATUS: case MSR_MISC_FEATURE_ENABLES: *val = 0; break; case MSR_RAPL_POWER_UNIT: /* * Use the default value documented in section * "RAPL Interfaces" in Intel SDM vol3. */ *val = 0x000a1003; break; case MSR_IA32_FEATURE_CONTROL: /* * Windows guests check this MSR. * Set the lock bit to avoid writes * to this MSR. */ *val = IA32_FEATURE_CONTROL_LOCK; break; default: error = -1; break; } } else if (cpu_vendor_amd || cpu_vendor_hygon) { switch (num) { case MSR_BIOS_SIGN: *val = 0; break; case MSR_HWCR: /* * Bios and Kernel Developer's Guides for AMD Families * 12H, 14H, 15H and 16H. */ *val = 0x01000010; /* Reset value */ *val |= 1 << 9; /* MONITOR/MWAIT disable */ break; case MSR_NB_CFG1: case MSR_LS_CFG: case MSR_IC_CFG: /* * The reset value is processor family dependent so * just return 0. */ *val = 0; break; case MSR_PERFEVSEL0: case MSR_PERFEVSEL1: case MSR_PERFEVSEL2: case MSR_PERFEVSEL3: /* * PerfEvtSel MSRs are not properly virtualized so just * return zero. */ *val = 0; break; case MSR_K7_PERFCTR0: case MSR_K7_PERFCTR1: case MSR_K7_PERFCTR2: case MSR_K7_PERFCTR3: /* * PerfCtr MSRs are not properly virtualized so just * return zero. */ *val = 0; break; case MSR_SMM_ADDR: case MSR_SMM_MASK: /* * Return the reset value defined in the AMD Bios and * Kernel Developer's Guide. */ *val = 0; break; case MSR_P_STATE_LIMIT: case MSR_P_STATE_CONTROL: case MSR_P_STATE_STATUS: case MSR_P_STATE_CONFIG(0): /* P0 configuration */ *val = 0; break; /* * OpenBSD guests test bit 0 of this MSR to detect if the * workaround for erratum 721 is already applied. * https://support.amd.com/TechDocs/41322_10h_Rev_Gd.pdf */ case 0xC0011029: *val = 1; break; #ifndef __FreeBSD__ case MSR_VM_CR: /* * We currently don't support nested virt. * Windows seems to ignore the cpuid bits and reads this * MSR anyways. */ *val = VM_CR_SVMDIS; break; #endif default: error = -1; break; } } else { error = -1; } return (error); } int init_msr(void) { int error; u_int regs[4]; char cpu_vendor[13]; do_cpuid(0, regs); ((u_int *)&cpu_vendor)[0] = regs[1]; ((u_int *)&cpu_vendor)[1] = regs[3]; ((u_int *)&cpu_vendor)[2] = regs[2]; cpu_vendor[12] = '\0'; error = 0; if (strcmp(cpu_vendor, "AuthenticAMD") == 0) { cpu_vendor_amd = 1; } else if (strcmp(cpu_vendor, "HygonGenuine") == 0) { cpu_vendor_hygon = 1; } else if (strcmp(cpu_vendor, "GenuineIntel") == 0) { cpu_vendor_intel = 1; } else { EPRINTLN("Unknown cpu vendor \"%s\"", cpu_vendor); error = ENOENT; } return (error); } /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2011 NetApp, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef _XMSR_H_ #define _XMSR_H_ int init_msr(void); int emulate_wrmsr(struct vcpu *vcpu, uint32_t code, uint64_t val); int emulate_rdmsr(struct vcpu *vcpu, uint32_t code, uint64_t *val); #endif /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2012 NetApp, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * bhyve ACPI table generator. * * Create the minimal set of ACPI tables required to boot FreeBSD (and * hopefully other o/s's). * * The tables are placed in the guest's ROM area just below 1MB physical, * above the MPTable. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "bhyverun.h" #include "acpi.h" #include "basl.h" #include "pci_emul.h" #include "vmgenc.h" #define BHYVE_ASL_TEMPLATE "bhyve.XXXXXXX" #define BHYVE_ASL_SUFFIX ".aml" #define BHYVE_ASL_COMPILER "/usr/sbin/iasl" #define BHYVE_ADDRESS_IOAPIC 0xFEC00000 #define BHYVE_ADDRESS_HPET 0xFED00000 #define BHYVE_ADDRESS_LAPIC 0xFEE00000 static int basl_keep_temps; static int basl_verbose_iasl; static int basl_ncpu; static uint32_t hpet_capabilities; /* * Contains the full pathname of the template to be passed * to mkstemp/mkstemps(3C) */ static char basl_template[MAXPATHLEN]; static char basl_stemplate[MAXPATHLEN]; /* * State for dsdt_line(), dsdt_indent(), and dsdt_unindent(). */ static FILE *dsdt_fp; static int dsdt_indent_level; static int dsdt_error; struct basl_fio { int fd; FILE *fp; char f_name[MAXPATHLEN]; }; #define EFPRINTF(...) \ if (fprintf(__VA_ARGS__) < 0) goto err_exit #define EFFLUSH(x) \ if (fflush(x) != 0) goto err_exit /* * A list for additional ACPI devices like a TPM. */ struct acpi_device_list_entry { SLIST_ENTRY(acpi_device_list_entry) chain; const struct acpi_device *dev; }; static SLIST_HEAD(acpi_device_list, acpi_device_list_entry) acpi_devices = SLIST_HEAD_INITIALIZER(acpi_devices); int acpi_tables_add_device(const struct acpi_device *const dev) { struct acpi_device_list_entry *const entry = calloc(1, sizeof(*entry)); if (entry == NULL) { return (ENOMEM); } entry->dev = dev; SLIST_INSERT_HEAD(&acpi_devices, entry, chain); return (0); } /* * Helper routines for writing to the DSDT from other modules. */ void dsdt_line(const char *fmt, ...) { va_list ap; if (dsdt_error != 0) return; if (strcmp(fmt, "") != 0) { if (dsdt_indent_level != 0) EFPRINTF(dsdt_fp, "%*c", dsdt_indent_level * 2, ' '); va_start(ap, fmt); if (vfprintf(dsdt_fp, fmt, ap) < 0) { va_end(ap); goto err_exit; } va_end(ap); } EFPRINTF(dsdt_fp, "\n"); return; err_exit: dsdt_error = errno; } void dsdt_indent(int levels) { dsdt_indent_level += levels; assert(dsdt_indent_level >= 0); } void dsdt_unindent(int levels) { assert(dsdt_indent_level >= levels); dsdt_indent_level -= levels; } void dsdt_fixed_ioport(uint16_t iobase, uint16_t length) { dsdt_line("IO (Decode16,"); dsdt_line(" 0x%04X, // Range Minimum", iobase); dsdt_line(" 0x%04X, // Range Maximum", iobase); dsdt_line(" 0x01, // Alignment"); dsdt_line(" 0x%02X, // Length", length); dsdt_line(" )"); } void dsdt_fixed_irq(uint8_t irq) { dsdt_line("IRQNoFlags ()"); dsdt_line(" {%d}", irq); } void dsdt_fixed_mem32(uint32_t base, uint32_t length) { dsdt_line("Memory32Fixed (ReadWrite,"); dsdt_line(" 0x%08X, // Address Base", base); dsdt_line(" 0x%08X, // Address Length", length); dsdt_line(" )"); } static int basl_fwrite_dsdt(FILE *fp) { dsdt_fp = fp; dsdt_error = 0; dsdt_indent_level = 0; dsdt_line("/*"); dsdt_line(" * bhyve DSDT template"); dsdt_line(" */"); dsdt_line("DefinitionBlock (\"bhyve_dsdt.aml\", \"DSDT\", 2," "\"BHYVE \", \"BVDSDT \", 0x00000001)"); dsdt_line("{"); dsdt_line(" Name (_S5, Package ()"); dsdt_line(" {"); dsdt_line(" 0x05,"); dsdt_line(" Zero,"); dsdt_line(" })"); pci_write_dsdt(); dsdt_line(""); dsdt_line(" Scope (_SB.PC00)"); dsdt_line(" {"); dsdt_line(" Device (HPET)"); dsdt_line(" {"); dsdt_line(" Name (_HID, EISAID(\"PNP0103\"))"); dsdt_line(" Name (_UID, 0)"); dsdt_line(" Name (_CRS, ResourceTemplate ()"); dsdt_line(" {"); dsdt_indent(4); dsdt_fixed_mem32(0xFED00000, 0x400); dsdt_unindent(4); dsdt_line(" })"); dsdt_line(" }"); dsdt_line(" }"); vmgenc_write_dsdt(); const struct acpi_device_list_entry *entry; SLIST_FOREACH(entry, &acpi_devices, chain) { BASL_EXEC(acpi_device_write_dsdt(entry->dev)); } dsdt_line("}"); if (dsdt_error != 0) return (dsdt_error); EFFLUSH(fp); return (0); err_exit: return (errno); } static int basl_open(struct basl_fio *bf, int suffix) { int err; err = 0; if (suffix) { strlcpy(bf->f_name, basl_stemplate, MAXPATHLEN); bf->fd = mkstemps(bf->f_name, strlen(BHYVE_ASL_SUFFIX)); } else { strlcpy(bf->f_name, basl_template, MAXPATHLEN); bf->fd = mkstemp(bf->f_name); } if (bf->fd > 0) { bf->fp = fdopen(bf->fd, "w+"); if (bf->fp == NULL) { unlink(bf->f_name); close(bf->fd); } } else { err = 1; } return (err); } static void basl_close(struct basl_fio *bf) { if (!basl_keep_temps) unlink(bf->f_name); fclose(bf->fp); } static int basl_start(struct basl_fio *in, struct basl_fio *out) { int err; err = basl_open(in, 0); if (!err) { err = basl_open(out, 1); if (err) { basl_close(in); } } return (err); } static void basl_end(struct basl_fio *in, struct basl_fio *out) { basl_close(in); basl_close(out); } static int basl_load(struct vmctx *ctx, int fd) { struct stat sb; void *addr; if (fstat(fd, &sb) < 0) return (errno); addr = calloc(1, sb.st_size); if (addr == NULL) return (ENOMEM); if (read(fd, addr, sb.st_size) < 0) return (errno); struct basl_table *table; uint8_t name[ACPI_NAMESEG_SIZE + 1] = { 0 }; memcpy(name, addr, sizeof(name) - 1 /* last char is '\0' */); BASL_EXEC(basl_table_create(&table, ctx, name, BASL_TABLE_ALIGNMENT)); BASL_EXEC(basl_table_append_bytes(table, addr, sb.st_size)); free(addr); return (0); } static int basl_compile(struct vmctx *ctx, int (*fwrite_section)(FILE *)) { struct basl_fio io[2]; static char iaslbuf[3*MAXPATHLEN + 10]; const char *fmt; int err; err = basl_start(&io[0], &io[1]); if (!err) { err = (*fwrite_section)(io[0].fp); if (!err) { /* * iasl sends the results of the compilation to * stdout. Shut this down by using the shell to * redirect stdout to /dev/null, unless the user * has requested verbose output for debugging * purposes */ fmt = basl_verbose_iasl ? "%s -p %s %s" : "/bin/sh -c \"%s -p %s %s\" 1> /dev/null"; snprintf(iaslbuf, sizeof(iaslbuf), fmt, BHYVE_ASL_COMPILER, io[1].f_name, io[0].f_name); err = system(iaslbuf); if (!err) { /* * Copy the aml output file into guest * memory at the specified location */ err = basl_load(ctx, io[1].fd); } } basl_end(&io[0], &io[1]); } return (err); } static int basl_make_templates(void) { const char *tmpdir; int err; int len; err = 0; /* * */ if ((tmpdir = getenv("BHYVE_TMPDIR")) == NULL || *tmpdir == '\0' || (tmpdir = getenv("TMPDIR")) == NULL || *tmpdir == '\0') { tmpdir = _PATH_TMP; } len = strlen(tmpdir); if ((len + sizeof(BHYVE_ASL_TEMPLATE) + 1) < MAXPATHLEN) { strcpy(basl_template, tmpdir); while (len > 0 && basl_template[len - 1] == '/') len--; basl_template[len] = '/'; strcpy(&basl_template[len + 1], BHYVE_ASL_TEMPLATE); } else err = E2BIG; if (!err) { /* * len has been initialized (and maybe adjusted) above */ if ((len + sizeof(BHYVE_ASL_TEMPLATE) + 1 + sizeof(BHYVE_ASL_SUFFIX)) < MAXPATHLEN) { strcpy(basl_stemplate, tmpdir); basl_stemplate[len] = '/'; strcpy(&basl_stemplate[len + 1], BHYVE_ASL_TEMPLATE); len = strlen(basl_stemplate); strcpy(&basl_stemplate[len], BHYVE_ASL_SUFFIX); } else err = E2BIG; } return (err); } static int build_dsdt(struct vmctx *const ctx) { BASL_EXEC(basl_compile(ctx, basl_fwrite_dsdt)); return (0); } static int build_facs(struct vmctx *const ctx) { ACPI_TABLE_FACS facs; struct basl_table *table; BASL_EXEC(basl_table_create(&table, ctx, ACPI_SIG_FACS, BASL_TABLE_ALIGNMENT_FACS)); memset(&facs, 0, sizeof(facs)); memcpy(facs.Signature, ACPI_SIG_FACS, ACPI_NAMESEG_SIZE); facs.Length = sizeof(facs); facs.Version = htole32(2); BASL_EXEC(basl_table_append_bytes(table, &facs, sizeof(facs))); return (0); } static int build_fadt(struct vmctx *const ctx) { ACPI_TABLE_FADT fadt; struct basl_table *table; BASL_EXEC(basl_table_create(&table, ctx, ACPI_SIG_FADT, BASL_TABLE_ALIGNMENT)); memset(&fadt, 0, sizeof(fadt)); BASL_EXEC(basl_table_append_header(table, ACPI_SIG_FADT, 5, 1)); fadt.Facs = htole32(0); /* patched by basl */ fadt.Dsdt = htole32(0); /* patched by basl */ fadt.SciInterrupt = htole16(SCI_INT); fadt.SmiCommand = htole32(SMI_CMD); fadt.AcpiEnable = BHYVE_ACPI_ENABLE; fadt.AcpiDisable = BHYVE_ACPI_DISABLE; fadt.Pm1aEventBlock = htole32(PM1A_EVT_ADDR); fadt.Pm1aControlBlock = htole32(PM1A_CNT_ADDR); fadt.PmTimerBlock = htole32(IO_PMTMR); fadt.Gpe0Block = htole32(IO_GPE0_BLK); fadt.Pm1EventLength = 4; fadt.Pm1ControlLength = 2; fadt.PmTimerLength = 4; fadt.Gpe0BlockLength = IO_GPE0_LEN; fadt.Century = 0x32; fadt.BootFlags = htole16(ACPI_FADT_NO_VGA | ACPI_FADT_NO_ASPM); fadt.Flags = htole32(ACPI_FADT_WBINVD | ACPI_FADT_C1_SUPPORTED | ACPI_FADT_SLEEP_BUTTON | ACPI_FADT_32BIT_TIMER | ACPI_FADT_RESET_REGISTER | ACPI_FADT_HEADLESS | ACPI_FADT_APIC_PHYSICAL); basl_fill_gas(&fadt.ResetRegister, ACPI_ADR_SPACE_SYSTEM_IO, 8, 0, ACPI_GAS_ACCESS_WIDTH_BYTE, 0xCF9); fadt.ResetValue = 6; fadt.MinorRevision = 1; fadt.XFacs = htole64(0); /* patched by basl */ fadt.XDsdt = htole64(0); /* patched by basl */ basl_fill_gas(&fadt.XPm1aEventBlock, ACPI_ADR_SPACE_SYSTEM_IO, 0x20, 0, ACPI_GAS_ACCESS_WIDTH_WORD, PM1A_EVT_ADDR); basl_fill_gas(&fadt.XPm1bEventBlock, ACPI_ADR_SPACE_SYSTEM_IO, 0, 0, ACPI_GAS_ACCESS_WIDTH_UNDEFINED, 0); basl_fill_gas(&fadt.XPm1aControlBlock, ACPI_ADR_SPACE_SYSTEM_IO, 0x10, 0, ACPI_GAS_ACCESS_WIDTH_WORD, PM1A_CNT_ADDR); basl_fill_gas(&fadt.XPm1bControlBlock, ACPI_ADR_SPACE_SYSTEM_IO, 0, 0, ACPI_GAS_ACCESS_WIDTH_UNDEFINED, 0); basl_fill_gas(&fadt.XPm2ControlBlock, ACPI_ADR_SPACE_SYSTEM_IO, 8, 0, ACPI_GAS_ACCESS_WIDTH_UNDEFINED, 0); basl_fill_gas(&fadt.XPmTimerBlock, ACPI_ADR_SPACE_SYSTEM_IO, 0x20, 0, ACPI_GAS_ACCESS_WIDTH_DWORD, IO_PMTMR); basl_fill_gas(&fadt.XGpe0Block, ACPI_ADR_SPACE_SYSTEM_IO, IO_GPE0_LEN * 8, 0, ACPI_GAS_ACCESS_WIDTH_BYTE, IO_GPE0_BLK); basl_fill_gas(&fadt.XGpe1Block, ACPI_ADR_SPACE_SYSTEM_IO, 0, 0, ACPI_GAS_ACCESS_WIDTH_UNDEFINED, 0); basl_fill_gas(&fadt.SleepControl, ACPI_ADR_SPACE_SYSTEM_IO, 8, 0, ACPI_GAS_ACCESS_WIDTH_BYTE, 0); basl_fill_gas(&fadt.SleepStatus, ACPI_ADR_SPACE_SYSTEM_IO, 8, 0, ACPI_GAS_ACCESS_WIDTH_BYTE, 0); BASL_EXEC(basl_table_append_content(table, &fadt, sizeof(fadt))); BASL_EXEC(basl_table_add_pointer(table, ACPI_SIG_FACS, offsetof(ACPI_TABLE_FADT, Facs), sizeof(fadt.Facs))); BASL_EXEC(basl_table_add_pointer(table, ACPI_SIG_DSDT, offsetof(ACPI_TABLE_FADT, Dsdt), sizeof(fadt.Dsdt))); BASL_EXEC(basl_table_add_pointer(table, ACPI_SIG_FACS, offsetof(ACPI_TABLE_FADT, XFacs), sizeof(fadt.XFacs))); BASL_EXEC(basl_table_add_pointer(table, ACPI_SIG_DSDT, offsetof(ACPI_TABLE_FADT, XDsdt), sizeof(fadt.XDsdt))); BASL_EXEC(basl_table_register_to_rsdt(table)); return (0); } static int build_hpet(struct vmctx *const ctx) { ACPI_TABLE_HPET hpet; struct basl_table *table; BASL_EXEC(basl_table_create(&table, ctx, ACPI_SIG_HPET, BASL_TABLE_ALIGNMENT)); memset(&hpet, 0, sizeof(hpet)); BASL_EXEC(basl_table_append_header(table, ACPI_SIG_HPET, 1, 1)); hpet.Id = htole32(hpet_capabilities); basl_fill_gas(&hpet.Address, ACPI_ADR_SPACE_SYSTEM_MEMORY, 0, 0, ACPI_GAS_ACCESS_WIDTH_LEGACY, BHYVE_ADDRESS_HPET); hpet.Flags = ACPI_HPET_PAGE_PROTECT4; BASL_EXEC(basl_table_append_content(table, &hpet, sizeof(hpet))); BASL_EXEC(basl_table_register_to_rsdt(table)); return (0); } static int build_madt(struct vmctx *const ctx) { ACPI_TABLE_MADT madt; ACPI_MADT_LOCAL_APIC madt_lapic; ACPI_MADT_IO_APIC madt_ioapic; ACPI_MADT_INTERRUPT_OVERRIDE madt_irq_override; ACPI_MADT_LOCAL_APIC_NMI madt_lapic_nmi; struct basl_table *table; BASL_EXEC(basl_table_create(&table, ctx, ACPI_SIG_MADT, BASL_TABLE_ALIGNMENT)); memset(&madt, 0, sizeof(madt)); BASL_EXEC(basl_table_append_header(table, ACPI_SIG_MADT, 1, 1)); madt.Address = htole32(BHYVE_ADDRESS_LAPIC); madt.Flags = htole32(ACPI_MADT_PCAT_COMPAT); BASL_EXEC(basl_table_append_content(table, &madt, sizeof(madt))); /* Local APIC for each CPU */ for (int i = 0; i < basl_ncpu; ++i) { memset(&madt_lapic, 0, sizeof(madt_lapic)); madt_lapic.Header.Type = ACPI_MADT_TYPE_LOCAL_APIC; madt_lapic.Header.Length = sizeof(madt_lapic); madt_lapic.ProcessorId = i; madt_lapic.Id = i; madt_lapic.LapicFlags = htole32(ACPI_MADT_ENABLED); BASL_EXEC(basl_table_append_bytes(table, &madt_lapic, sizeof(madt_lapic))); } /* I/O APIC */ memset(&madt_ioapic, 0, sizeof(madt_ioapic)); madt_ioapic.Header.Type = ACPI_MADT_TYPE_IO_APIC; madt_ioapic.Header.Length = sizeof(madt_ioapic); madt_ioapic.Address = htole32(BHYVE_ADDRESS_IOAPIC); BASL_EXEC( basl_table_append_bytes(table, &madt_ioapic, sizeof(madt_ioapic))); /* Legacy IRQ0 is connected to pin 2 of the I/O APIC */ memset(&madt_irq_override, 0, sizeof(madt_irq_override)); madt_irq_override.Header.Type = ACPI_MADT_TYPE_INTERRUPT_OVERRIDE; madt_irq_override.Header.Length = sizeof(madt_irq_override); madt_irq_override.GlobalIrq = htole32(2); madt_irq_override.IntiFlags = htole16( ACPI_MADT_POLARITY_ACTIVE_HIGH | ACPI_MADT_TRIGGER_EDGE); BASL_EXEC(basl_table_append_bytes(table, &madt_irq_override, sizeof(madt_irq_override))); memset(&madt_irq_override, 0, sizeof(madt_irq_override)); madt_irq_override.Header.Type = ACPI_MADT_TYPE_INTERRUPT_OVERRIDE; madt_irq_override.Header.Length = sizeof(madt_irq_override); madt_irq_override.SourceIrq = SCI_INT; madt_irq_override.GlobalIrq = htole32(SCI_INT); madt_irq_override.IntiFlags = htole16( ACPI_MADT_POLARITY_ACTIVE_LOW | ACPI_MADT_TRIGGER_LEVEL); BASL_EXEC(basl_table_append_bytes(table, &madt_irq_override, sizeof(madt_irq_override))); /* Local APIC NMI is conntected to LINT 1 on all CPUs */ memset(&madt_lapic_nmi, 0, sizeof(madt_lapic_nmi)); madt_lapic_nmi.Header.Type = ACPI_MADT_TYPE_LOCAL_APIC_NMI; madt_lapic_nmi.Header.Length = sizeof(madt_lapic_nmi); madt_lapic_nmi.ProcessorId = 0xFF; madt_lapic_nmi.IntiFlags = htole16( ACPI_MADT_POLARITY_ACTIVE_HIGH | ACPI_MADT_TRIGGER_EDGE); madt_lapic_nmi.Lint = 1; BASL_EXEC(basl_table_append_bytes(table, &madt_lapic_nmi, sizeof(madt_lapic_nmi))); BASL_EXEC(basl_table_register_to_rsdt(table)); return (0); } static int build_mcfg(struct vmctx *const ctx) { ACPI_TABLE_MCFG mcfg; ACPI_MCFG_ALLOCATION mcfg_allocation; struct basl_table *table; BASL_EXEC(basl_table_create(&table, ctx, ACPI_SIG_MCFG, BASL_TABLE_ALIGNMENT)); memset(&mcfg, 0, sizeof(mcfg)); BASL_EXEC(basl_table_append_header(table, ACPI_SIG_MCFG, 1, 1)); BASL_EXEC(basl_table_append_content(table, &mcfg, sizeof(mcfg))); memset(&mcfg_allocation, 0, sizeof(mcfg_allocation)); mcfg_allocation.Address = htole64(pci_ecfg_base()); mcfg_allocation.EndBusNumber = 0xFF; BASL_EXEC(basl_table_append_bytes(table, &mcfg_allocation, sizeof(mcfg_allocation))); BASL_EXEC(basl_table_register_to_rsdt(table)); return (0); } static int build_rsdp(struct vmctx *const ctx) { ACPI_TABLE_RSDP rsdp; struct basl_table *table; BASL_EXEC(basl_table_create(&table, ctx, ACPI_RSDP_NAME, BASL_TABLE_ALIGNMENT)); memset(&rsdp, 0, sizeof(rsdp)); memcpy(rsdp.Signature, ACPI_SIG_RSDP, 8); rsdp.Checksum = 0; /* patched by basl */ memcpy(rsdp.OemId, "BHYVE ", ACPI_OEM_ID_SIZE); rsdp.Revision = 2; rsdp.RsdtPhysicalAddress = htole32(0); /* patched by basl */ rsdp.Length = htole32(0); /* patched by basl */ rsdp.XsdtPhysicalAddress = htole64(0); /* patched by basl */ rsdp.ExtendedChecksum = 0; /* patched by basl */ BASL_EXEC(basl_table_append_bytes(table, &rsdp, sizeof(rsdp))); BASL_EXEC(basl_table_add_checksum(table, offsetof(ACPI_TABLE_RSDP, Checksum), 0, 20)); BASL_EXEC(basl_table_add_pointer(table, ACPI_SIG_RSDT, offsetof(ACPI_TABLE_RSDP, RsdtPhysicalAddress), sizeof(rsdp.RsdtPhysicalAddress))); BASL_EXEC(basl_table_add_length(table, offsetof(ACPI_TABLE_RSDP, Length), sizeof(rsdp.Length))); BASL_EXEC(basl_table_add_pointer(table, ACPI_SIG_XSDT, offsetof(ACPI_TABLE_RSDP, XsdtPhysicalAddress), sizeof(rsdp.XsdtPhysicalAddress))); BASL_EXEC(basl_table_add_checksum(table, offsetof(ACPI_TABLE_RSDP, ExtendedChecksum), 0, BASL_TABLE_CHECKSUM_LEN_FULL_TABLE)); return (0); } static int build_spcr(struct vmctx *const ctx) { ACPI_TABLE_SPCR spcr; struct basl_table *table; BASL_EXEC(basl_table_create(&table, ctx, ACPI_SIG_SPCR, BASL_TABLE_ALIGNMENT)); memset(&spcr, 0, sizeof(spcr)); BASL_EXEC(basl_table_append_header(table, ACPI_SIG_SPCR, 1, 1)); spcr.InterfaceType = ACPI_DBG2_16550_COMPATIBLE; basl_fill_gas(&spcr.SerialPort, ACPI_ADR_SPACE_SYSTEM_IO, 8, 0, ACPI_GAS_ACCESS_WIDTH_LEGACY, 0x3F8); spcr.InterruptType = ACPI_SPCR_INTERRUPT_TYPE_8259; spcr.PcInterrupt = 4; spcr.BaudRate = ACPI_SPCR_BAUD_RATE_115200; spcr.Parity = ACPI_SPCR_PARITY_NO_PARITY; spcr.StopBits = ACPI_SPCR_STOP_BITS_1; spcr.FlowControl = 3; /* RTS/CTS | DCD */ spcr.TerminalType = ACPI_SPCR_TERMINAL_TYPE_VT_UTF8; BASL_EXEC(basl_table_append_content(table, &spcr, sizeof(spcr))); BASL_EXEC(basl_table_register_to_rsdt(table)); return (0); } int acpi_build(struct vmctx *ctx, int ncpu) { int err; basl_ncpu = ncpu; err = vm_get_hpet_capabilities(ctx, &hpet_capabilities); if (err != 0) return (err); /* * For debug, allow the user to have iasl compiler output sent * to stdout rather than /dev/null */ if (getenv("BHYVE_ACPI_VERBOSE_IASL")) basl_verbose_iasl = 1; /* * Allow the user to keep the generated ASL files for debugging * instead of deleting them following use */ if (getenv("BHYVE_ACPI_KEEPTMPS")) basl_keep_temps = 1; BASL_EXEC(basl_init(ctx)); BASL_EXEC(basl_make_templates()); /* * Generate ACPI tables and copy them into guest memory. * * According to UEFI Specification v6.3 chapter 5.1 the FADT should be * the first table pointed to by XSDT. For that reason, build it as the * first table after XSDT. */ BASL_EXEC(build_rsdp(ctx)); BASL_EXEC(build_fadt(ctx)); BASL_EXEC(build_madt(ctx)); BASL_EXEC(build_hpet(ctx)); BASL_EXEC(build_mcfg(ctx)); BASL_EXEC(build_facs(ctx)); BASL_EXEC(build_spcr(ctx)); /* Build ACPI device-specific tables such as a TPM2 table. */ const struct acpi_device_list_entry *entry; SLIST_FOREACH(entry, &acpi_devices, chain) { BASL_EXEC(acpi_device_build_table(entry->dev)); } BASL_EXEC(build_dsdt(ctx)); BASL_EXEC(basl_finish()); return (0); } /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2012 NetApp, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef _ACPI_H_ #define _ACPI_H_ #include "acpi_device.h" #define SCI_INT 9 #define SMI_CMD 0xb2 #define BHYVE_ACPI_ENABLE 0xa0 #define BHYVE_ACPI_DISABLE 0xa1 #define PM1A_EVT_ADDR 0x400 #define PM1A_CNT_ADDR 0x404 #define IO_PMTMR 0x408 /* 4-byte i/o port for the timer */ #define IO_GPE0_BLK 0x40c /* 2x 1-byte IO port for GPE0_STS/EN */ #define IO_GPE0_LEN 0x2 #define IO_GPE0_STS IO_GPE0_BLK #define IO_GPE0_EN (IO_GPE0_BLK + (IO_GPE0_LEN / 2)) /* Allocated GPE bits. */ #define GPE_VMGENC 0 struct vmctx; int acpi_build(struct vmctx *ctx, int ncpu); void acpi_raise_gpe(struct vmctx *ctx, unsigned bit); int acpi_tables_add_device(const struct acpi_device *const dev); void dsdt_line(const char *fmt, ...); void dsdt_fixed_ioport(uint16_t iobase, uint16_t length); void dsdt_fixed_irq(uint8_t irq); void dsdt_fixed_mem32(uint32_t base, uint32_t length); void dsdt_indent(int levels); void dsdt_unindent(int levels); void sci_init(struct vmctx *ctx); #ifndef __FreeBSD__ void pmtmr_init(struct vmctx *ctx); #endif #endif /* _ACPI_H_ */ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2021 Beckhoff Automation GmbH & Co. KG * Author: Corvin Köhne */ #include #include #include #include #include #include #include #include "acpi.h" #include "acpi_device.h" #include "basl.h" /** * List entry to enumerate all resources used by an ACPI device. * * @param chain Used to chain multiple elements together. * @param type Type of the ACPI resource. * @param data Data of the ACPI resource. */ struct acpi_resource_list_entry { SLIST_ENTRY(acpi_resource_list_entry) chain; UINT32 type; ACPI_RESOURCE_DATA data; }; /** * Holds information about an ACPI device. * * @param vm_ctx VM context the ACPI device was created in. * @param softc A pointer to the software context of the ACPI device. * @param emul Device emulation struct. It contains some information like the name of the ACPI device and some device specific functions. * @param crs Current resources used by the ACPI device. */ struct acpi_device { struct vmctx *vm_ctx; void *softc; const struct acpi_device_emul *emul; SLIST_HEAD(acpi_resource_list, acpi_resource_list_entry) crs; }; int acpi_device_create(struct acpi_device **const new_dev, void *const softc, struct vmctx *const vm_ctx, const struct acpi_device_emul *const emul) { assert(new_dev != NULL); assert(vm_ctx != NULL); assert(emul != NULL); struct acpi_device *const dev = calloc(1, sizeof(*dev)); if (dev == NULL) { return (ENOMEM); } dev->vm_ctx = vm_ctx; dev->softc = softc; dev->emul = emul; SLIST_INIT(&dev->crs); const int error = acpi_tables_add_device(dev); if (error) { acpi_device_destroy(dev); return (error); } *new_dev = dev; return (0); } void acpi_device_destroy(struct acpi_device *const dev) { if (dev == NULL) { return; } struct acpi_resource_list_entry *res; while (!SLIST_EMPTY(&dev->crs)) { res = SLIST_FIRST(&dev->crs); SLIST_REMOVE_HEAD(&dev->crs, chain); free(res); } free(dev); } int acpi_device_add_res_fixed_ioport(struct acpi_device *const dev, const UINT16 port, const UINT8 length) { if (dev == NULL) { return (EINVAL); } struct acpi_resource_list_entry *const res = calloc(1, sizeof(*res)); if (res == NULL) { return (ENOMEM); } res->type = ACPI_RESOURCE_TYPE_FIXED_IO; res->data.FixedIo.Address = port; res->data.FixedIo.AddressLength = length; SLIST_INSERT_HEAD(&dev->crs, res, chain); return (0); } int acpi_device_add_res_fixed_memory32(struct acpi_device *const dev, const UINT8 write_protected, const UINT32 address, const UINT32 length) { if (dev == NULL) { return (EINVAL); } struct acpi_resource_list_entry *const res = calloc(1, sizeof(*res)); if (res == NULL) { return (ENOMEM); } res->type = ACPI_RESOURCE_TYPE_FIXED_MEMORY32; res->data.FixedMemory32.WriteProtect = write_protected; res->data.FixedMemory32.Address = address; res->data.FixedMemory32.AddressLength = length; SLIST_INSERT_HEAD(&dev->crs, res, chain); return (0); } void * acpi_device_get_softc(const struct acpi_device *const dev) { assert(dev != NULL); return (dev->softc); } int acpi_device_build_table(const struct acpi_device *const dev) { assert(dev != NULL); assert(dev->emul != NULL); if (dev->emul->build_table != NULL) { return (dev->emul->build_table(dev)); } return (0); } static int acpi_device_write_dsdt_crs(const struct acpi_device *const dev) { const struct acpi_resource_list_entry *res; SLIST_FOREACH(res, &dev->crs, chain) { switch (res->type) { case ACPI_RESOURCE_TYPE_FIXED_IO: dsdt_fixed_ioport(res->data.FixedIo.Address, res->data.FixedIo.AddressLength); break; case ACPI_RESOURCE_TYPE_FIXED_MEMORY32: dsdt_fixed_mem32(res->data.FixedMemory32.Address, res->data.FixedMemory32.AddressLength); break; default: assert(0); break; } } return (0); } int acpi_device_write_dsdt(const struct acpi_device *const dev) { assert(dev != NULL); dsdt_line(""); dsdt_line(" Scope (\\_SB)"); dsdt_line(" {"); dsdt_line(" Device (%s)", dev->emul->name); dsdt_line(" {"); dsdt_line(" Name (_HID, \"%s\")", dev->emul->hid); dsdt_line(" Name (_STA, 0x0F)"); dsdt_line(" Name (_CRS, ResourceTemplate ()"); dsdt_line(" {"); dsdt_indent(4); BASL_EXEC(acpi_device_write_dsdt_crs(dev)); dsdt_unindent(4); dsdt_line(" })"); if (dev->emul->write_dsdt != NULL) { dsdt_indent(3); BASL_EXEC(dev->emul->write_dsdt(dev)); dsdt_unindent(3); } dsdt_line(" }"); dsdt_line(" }"); return (0); } /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2021 Beckhoff Automation GmbH & Co. KG * Author: Corvin Köhne */ #pragma once #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wunused-parameter" #include #pragma GCC diagnostic pop struct vmctx; struct acpi_device; /** * Device specific information and emulation. * * @param name Used as device name in the DSDT. * @param hid Used as _HID in the DSDT. * @param build_table Called to build a device specific ACPI table like the TPM2 * table. * @param write_dsdt Called to append the DSDT with device specific * information. */ struct acpi_device_emul { const char *name; const char *hid; int (*build_table)(const struct acpi_device *dev); int (*write_dsdt)(const struct acpi_device *dev); }; /** * Creates an ACPI device. * * @param[out] new_dev Returns the newly create ACPI device. * @param[in] softc Pointer to the software context of the ACPI device. * @param[in] vm_ctx VM context the ACPI device is created in. * @param[in] emul Device emulation struct. It contains some information * like the name of the ACPI device and some device specific * functions. */ int acpi_device_create(struct acpi_device **new_dev, void *softc, struct vmctx *vm_ctx, const struct acpi_device_emul *emul); void acpi_device_destroy(struct acpi_device *dev); int acpi_device_add_res_fixed_ioport(struct acpi_device *dev, UINT16 port, UINT8 length); int acpi_device_add_res_fixed_memory32(struct acpi_device *dev, UINT8 write_protected, UINT32 address, UINT32 length); void *acpi_device_get_softc(const struct acpi_device *dev); int acpi_device_build_table(const struct acpi_device *dev); int acpi_device_write_dsdt(const struct acpi_device *dev); /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 1998 - 2008 Søren Schmidt * Copyright (c) 2009-2012 Alexander Motin * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer, * without modification, immediately at the beginning of the file. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef _AHCI_H_ #define _AHCI_H_ /* ATA register defines */ #define ATA_DATA 0 /* (RW) data */ #define ATA_FEATURE 1 /* (W) feature */ #define ATA_F_DMA 0x01 /* enable DMA */ #define ATA_F_OVL 0x02 /* enable overlap */ #define ATA_COUNT 2 /* (W) sector count */ #define ATA_SECTOR 3 /* (RW) sector # */ #define ATA_CYL_LSB 4 /* (RW) cylinder# LSB */ #define ATA_CYL_MSB 5 /* (RW) cylinder# MSB */ #define ATA_DRIVE 6 /* (W) Sector/Drive/Head */ #define ATA_D_LBA 0x40 /* use LBA addressing */ #define ATA_D_IBM 0xa0 /* 512 byte sectors, ECC */ #define ATA_COMMAND 7 /* (W) command */ #define ATA_ERROR 8 /* (R) error */ #define ATA_E_ILI 0x01 /* illegal length */ #define ATA_E_NM 0x02 /* no media */ #define ATA_E_ABORT 0x04 /* command aborted */ #define ATA_E_MCR 0x08 /* media change request */ #define ATA_E_IDNF 0x10 /* ID not found */ #define ATA_E_MC 0x20 /* media changed */ #define ATA_E_UNC 0x40 /* uncorrectable data */ #define ATA_E_ICRC 0x80 /* UDMA crc error */ #define ATA_E_ATAPI_SENSE_MASK 0xf0 /* ATAPI sense key mask */ #define ATA_IREASON 9 /* (R) interrupt reason */ #define ATA_I_CMD 0x01 /* cmd (1) | data (0) */ #define ATA_I_IN 0x02 /* read (1) | write (0) */ #define ATA_I_RELEASE 0x04 /* released bus (1) */ #define ATA_I_TAGMASK 0xf8 /* tag mask */ #define ATA_STATUS 10 /* (R) status */ #define ATA_ALTSTAT 11 /* (R) alternate status */ #define ATA_S_ERROR 0x01 /* error */ #define ATA_S_INDEX 0x02 /* index */ #define ATA_S_CORR 0x04 /* data corrected */ #define ATA_S_DRQ 0x08 /* data request */ #define ATA_S_DSC 0x10 /* drive seek completed */ #define ATA_S_SERVICE 0x10 /* drive needs service */ #define ATA_S_DWF 0x20 /* drive write fault */ #define ATA_S_DMA 0x20 /* DMA ready */ #define ATA_S_READY 0x40 /* drive ready */ #define ATA_S_BUSY 0x80 /* busy */ #define ATA_CONTROL 12 /* (W) control */ #define ATA_A_IDS 0x02 /* disable interrupts */ #define ATA_A_RESET 0x04 /* RESET controller */ #define ATA_A_4BIT 0x08 /* 4 head bits */ #define ATA_A_HOB 0x80 /* High Order Byte enable */ /* SATA register defines */ #define ATA_SSTATUS 13 #define ATA_SS_DET_MASK 0x0000000f #define ATA_SS_DET_NO_DEVICE 0x00000000 #define ATA_SS_DET_DEV_PRESENT 0x00000001 #define ATA_SS_DET_PHY_ONLINE 0x00000003 #define ATA_SS_DET_PHY_OFFLINE 0x00000004 #define ATA_SS_SPD_MASK 0x000000f0 #define ATA_SS_SPD_NO_SPEED 0x00000000 #define ATA_SS_SPD_GEN1 0x00000010 #define ATA_SS_SPD_GEN2 0x00000020 #define ATA_SS_SPD_GEN3 0x00000030 #define ATA_SS_IPM_MASK 0x00000f00 #define ATA_SS_IPM_NO_DEVICE 0x00000000 #define ATA_SS_IPM_ACTIVE 0x00000100 #define ATA_SS_IPM_PARTIAL 0x00000200 #define ATA_SS_IPM_SLUMBER 0x00000600 #define ATA_SS_IPM_DEVSLEEP 0x00000800 #define ATA_SERROR 14 #define ATA_SE_DATA_CORRECTED 0x00000001 #define ATA_SE_COMM_CORRECTED 0x00000002 #define ATA_SE_DATA_ERR 0x00000100 #define ATA_SE_COMM_ERR 0x00000200 #define ATA_SE_PROT_ERR 0x00000400 #define ATA_SE_HOST_ERR 0x00000800 #define ATA_SE_PHY_CHANGED 0x00010000 #define ATA_SE_PHY_IERROR 0x00020000 #define ATA_SE_COMM_WAKE 0x00040000 #define ATA_SE_DECODE_ERR 0x00080000 #define ATA_SE_PARITY_ERR 0x00100000 #define ATA_SE_CRC_ERR 0x00200000 #define ATA_SE_HANDSHAKE_ERR 0x00400000 #define ATA_SE_LINKSEQ_ERR 0x00800000 #define ATA_SE_TRANSPORT_ERR 0x01000000 #define ATA_SE_UNKNOWN_FIS 0x02000000 #define ATA_SE_EXCHANGED 0x04000000 #define ATA_SCONTROL 15 #define ATA_SC_DET_MASK 0x0000000f #define ATA_SC_DET_IDLE 0x00000000 #define ATA_SC_DET_RESET 0x00000001 #define ATA_SC_DET_DISABLE 0x00000004 #define ATA_SC_SPD_MASK 0x000000f0 #define ATA_SC_SPD_NO_SPEED 0x00000000 #define ATA_SC_SPD_SPEED_GEN1 0x00000010 #define ATA_SC_SPD_SPEED_GEN2 0x00000020 #define ATA_SC_SPD_SPEED_GEN3 0x00000030 #define ATA_SC_IPM_MASK 0x00000f00 #define ATA_SC_IPM_NONE 0x00000000 #define ATA_SC_IPM_DIS_PARTIAL 0x00000100 #define ATA_SC_IPM_DIS_SLUMBER 0x00000200 #define ATA_SC_IPM_DIS_DEVSLEEP 0x00000400 #define ATA_SACTIVE 16 #define AHCI_MAX_PORTS 32 #define AHCI_MAX_SLOTS 32 #define AHCI_MAX_IRQS 16 /* SATA AHCI v1.0 register defines */ #define AHCI_CAP 0x00 #define AHCI_CAP_NPMASK 0x0000001f #define AHCI_CAP_SXS 0x00000020 #define AHCI_CAP_EMS 0x00000040 #define AHCI_CAP_CCCS 0x00000080 #define AHCI_CAP_NCS 0x00001F00 #define AHCI_CAP_NCS_SHIFT 8 #define AHCI_CAP_PSC 0x00002000 #define AHCI_CAP_SSC 0x00004000 #define AHCI_CAP_PMD 0x00008000 #define AHCI_CAP_FBSS 0x00010000 #define AHCI_CAP_SPM 0x00020000 #define AHCI_CAP_SAM 0x00080000 #define AHCI_CAP_ISS 0x00F00000 #define AHCI_CAP_ISS_SHIFT 20 #define AHCI_CAP_SCLO 0x01000000 #define AHCI_CAP_SAL 0x02000000 #define AHCI_CAP_SALP 0x04000000 #define AHCI_CAP_SSS 0x08000000 #define AHCI_CAP_SMPS 0x10000000 #define AHCI_CAP_SSNTF 0x20000000 #define AHCI_CAP_SNCQ 0x40000000 #define AHCI_CAP_64BIT 0x80000000 #define AHCI_GHC 0x04 #define AHCI_GHC_AE 0x80000000 #define AHCI_GHC_MRSM 0x00000004 #define AHCI_GHC_IE 0x00000002 #define AHCI_GHC_HR 0x00000001 #define AHCI_IS 0x08 #define AHCI_PI 0x0c #define AHCI_VS 0x10 #define AHCI_CCCC 0x14 #define AHCI_CCCC_TV_MASK 0xffff0000 #define AHCI_CCCC_TV_SHIFT 16 #define AHCI_CCCC_CC_MASK 0x0000ff00 #define AHCI_CCCC_CC_SHIFT 8 #define AHCI_CCCC_INT_MASK 0x000000f8 #define AHCI_CCCC_INT_SHIFT 3 #define AHCI_CCCC_EN 0x00000001 #define AHCI_CCCP 0x18 #define AHCI_EM_LOC 0x1C #define AHCI_EM_CTL 0x20 #define AHCI_EM_MR 0x00000001 #define AHCI_EM_TM 0x00000100 #define AHCI_EM_RST 0x00000200 #define AHCI_EM_LED 0x00010000 #define AHCI_EM_SAFTE 0x00020000 #define AHCI_EM_SES2 0x00040000 #define AHCI_EM_SGPIO 0x00080000 #define AHCI_EM_SMB 0x01000000 #define AHCI_EM_XMT 0x02000000 #define AHCI_EM_ALHD 0x04000000 #define AHCI_EM_PM 0x08000000 #define AHCI_CAP2 0x24 #define AHCI_CAP2_BOH 0x00000001 #define AHCI_CAP2_NVMP 0x00000002 #define AHCI_CAP2_APST 0x00000004 #define AHCI_CAP2_SDS 0x00000008 #define AHCI_CAP2_SADM 0x00000010 #define AHCI_CAP2_DESO 0x00000020 #define AHCI_OFFSET 0x100 #define AHCI_STEP 0x80 #define AHCI_P_CLB 0x00 #define AHCI_P_CLBU 0x04 #define AHCI_P_FB 0x08 #define AHCI_P_FBU 0x0c #define AHCI_P_IS 0x10 #define AHCI_P_IE 0x14 #define AHCI_P_IX_DHR 0x00000001 #define AHCI_P_IX_PS 0x00000002 #define AHCI_P_IX_DS 0x00000004 #define AHCI_P_IX_SDB 0x00000008 #define AHCI_P_IX_UF 0x00000010 #define AHCI_P_IX_DP 0x00000020 #define AHCI_P_IX_PC 0x00000040 #define AHCI_P_IX_MP 0x00000080 #define AHCI_P_IX_PRC 0x00400000 #define AHCI_P_IX_IPM 0x00800000 #define AHCI_P_IX_OF 0x01000000 #define AHCI_P_IX_INF 0x04000000 #define AHCI_P_IX_IF 0x08000000 #define AHCI_P_IX_HBD 0x10000000 #define AHCI_P_IX_HBF 0x20000000 #define AHCI_P_IX_TFE 0x40000000 #define AHCI_P_IX_CPD 0x80000000 #define AHCI_P_CMD 0x18 #define AHCI_P_CMD_ST 0x00000001 #define AHCI_P_CMD_SUD 0x00000002 #define AHCI_P_CMD_POD 0x00000004 #define AHCI_P_CMD_CLO 0x00000008 #define AHCI_P_CMD_FRE 0x00000010 #define AHCI_P_CMD_CCS_MASK 0x00001f00 #define AHCI_P_CMD_CCS_SHIFT 8 #define AHCI_P_CMD_ISS 0x00002000 #define AHCI_P_CMD_FR 0x00004000 #define AHCI_P_CMD_CR 0x00008000 #define AHCI_P_CMD_CPS 0x00010000 #define AHCI_P_CMD_PMA 0x00020000 #define AHCI_P_CMD_HPCP 0x00040000 #define AHCI_P_CMD_MPSP 0x00080000 #define AHCI_P_CMD_CPD 0x00100000 #define AHCI_P_CMD_ESP 0x00200000 #define AHCI_P_CMD_FBSCP 0x00400000 #define AHCI_P_CMD_APSTE 0x00800000 #define AHCI_P_CMD_ATAPI 0x01000000 #define AHCI_P_CMD_DLAE 0x02000000 #define AHCI_P_CMD_ALPE 0x04000000 #define AHCI_P_CMD_ASP 0x08000000 #define AHCI_P_CMD_ICC_MASK 0xf0000000 #define AHCI_P_CMD_NOOP 0x00000000 #define AHCI_P_CMD_ACTIVE 0x10000000 #define AHCI_P_CMD_PARTIAL 0x20000000 #define AHCI_P_CMD_SLUMBER 0x60000000 #define AHCI_P_CMD_DEVSLEEP 0x80000000 #define AHCI_P_TFD 0x20 #define AHCI_P_SIG 0x24 #define AHCI_P_SSTS 0x28 #define AHCI_P_SCTL 0x2c #define AHCI_P_SERR 0x30 #define AHCI_P_SACT 0x34 #define AHCI_P_CI 0x38 #define AHCI_P_SNTF 0x3C #define AHCI_P_FBS 0x40 #define AHCI_P_FBS_EN 0x00000001 #define AHCI_P_FBS_DEC 0x00000002 #define AHCI_P_FBS_SDE 0x00000004 #define AHCI_P_FBS_DEV 0x00000f00 #define AHCI_P_FBS_DEV_SHIFT 8 #define AHCI_P_FBS_ADO 0x0000f000 #define AHCI_P_FBS_ADO_SHIFT 12 #define AHCI_P_FBS_DWE 0x000f0000 #define AHCI_P_FBS_DWE_SHIFT 16 #define AHCI_P_DEVSLP 0x44 #define AHCI_P_DEVSLP_ADSE 0x00000001 #define AHCI_P_DEVSLP_DSP 0x00000002 #define AHCI_P_DEVSLP_DETO 0x000003fc #define AHCI_P_DEVSLP_DETO_SHIFT 2 #define AHCI_P_DEVSLP_MDAT 0x00007c00 #define AHCI_P_DEVSLP_MDAT_SHIFT 10 #define AHCI_P_DEVSLP_DITO 0x01ff8000 #define AHCI_P_DEVSLP_DITO_SHIFT 15 #define AHCI_P_DEVSLP_DM 0x0e000000 #define AHCI_P_DEVSLP_DM_SHIFT 25 /* Just to be sure, if building as module. */ #if MAXPHYS < 512 * 1024 #undef MAXPHYS #define MAXPHYS 512 * 1024 #endif /* Pessimistic prognosis on number of required S/G entries */ #define AHCI_SG_ENTRIES (roundup(btoc(MAXPHYS) + 1, 8)) /* Command list. 32 commands. First, 1Kbyte aligned. */ #define AHCI_CL_OFFSET 0 #define AHCI_CL_SIZE 32 /* Command tables. Up to 32 commands, Each, 128byte aligned. */ #define AHCI_CT_OFFSET (AHCI_CL_OFFSET + AHCI_CL_SIZE * AHCI_MAX_SLOTS) #define AHCI_CT_SIZE (128 + AHCI_SG_ENTRIES * 16) /* Total main work area. */ #define AHCI_WORK_SIZE (AHCI_CT_OFFSET + AHCI_CT_SIZE * ch->numslots) #endif /* _AHCI_H_ */ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2016 Alex Teaca * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * */ #ifndef WITHOUT_CAPSICUM #include #include #endif #include #include #include #include #include #include #include #include #include #include "audio.h" #include "pci_hda.h" /* * Audio Player internal data structures */ struct audio { int fd; uint8_t dir; uint8_t inited; char dev_name[64]; }; /* * Audio Player module function definitions */ /* * audio_init - initialize an instance of audio player * @dev_name - the backend sound device used to play / capture * @dir - dir = 1 for write mode, dir = 0 for read mode */ struct audio * audio_init(const char *dev_name, uint8_t dir) { struct audio *aud = NULL; #ifndef WITHOUT_CAPSICUM cap_rights_t rights; cap_ioctl_t cmds[] = { SNDCTL_DSP_RESET, SNDCTL_DSP_SETFMT, SNDCTL_DSP_CHANNELS, SNDCTL_DSP_SPEED, #ifdef DEBUG_HDA SNDCTL_DSP_GETOSPACE, SNDCTL_DSP_GETISPACE, #endif }; #endif assert(dev_name); aud = calloc(1, sizeof(*aud)); if (!aud) return NULL; if (strlen(dev_name) < sizeof(aud->dev_name)) memcpy(aud->dev_name, dev_name, strlen(dev_name) + 1); else { DPRINTF("dev_name too big"); free(aud); return NULL; } aud->dir = dir; aud->fd = open(aud->dev_name, aud->dir ? O_WRONLY : O_RDONLY, 0); if (aud->fd == -1) { DPRINTF("Failed to open dev: %s, errno: %d", aud->dev_name, errno); free(aud); return (NULL); } #ifndef WITHOUT_CAPSICUM cap_rights_init(&rights, CAP_IOCTL, CAP_READ, CAP_WRITE); if (caph_rights_limit(aud->fd, &rights) == -1) errx(EX_OSERR, "Unable to apply rights for sandbox"); if (caph_ioctls_limit(aud->fd, cmds, nitems(cmds)) == -1) errx(EX_OSERR, "Unable to limit ioctl rights for sandbox"); #endif return aud; } /* * audio_set_params - reset the sound device and set the audio params * @aud - the audio player to be configured * @params - the audio parameters to be set */ int audio_set_params(struct audio *aud, struct audio_params *params) { int audio_fd; int format, channels, rate; int err; #if DEBUG_HDA == 1 audio_buf_info info; #endif assert(aud); assert(params); if ((audio_fd = aud->fd) < 0) { DPRINTF("Incorrect audio device descriptor for %s", aud->dev_name); return (-1); } /* Reset the device if it was previously opened */ if (aud->inited) { err = ioctl(audio_fd, SNDCTL_DSP_RESET, NULL); if (err == -1) { DPRINTF("Failed to reset fd: %d, errno: %d", aud->fd, errno); return (-1); } } else aud->inited = 1; /* Set the Format (Bits per Sample) */ format = params->format; err = ioctl(audio_fd, SNDCTL_DSP_SETFMT, &format); if (err == -1) { DPRINTF("Fail to set fmt: 0x%x errno: %d", params->format, errno); return -1; } /* The device does not support the requested audio format */ if (format != params->format) { DPRINTF("Mismatch format: 0x%x params->format: 0x%x", format, params->format); return -1; } /* Set the Number of Channels */ channels = params->channels; err = ioctl(audio_fd, SNDCTL_DSP_CHANNELS, &channels); if (err == -1) { DPRINTF("Fail to set channels: %d errno: %d", params->channels, errno); return -1; } /* The device does not support the requested no. of channels */ if (channels != params->channels) { DPRINTF("Mismatch channels: %d params->channels: %d", channels, params->channels); return -1; } /* Set the Sample Rate / Speed */ rate = params->rate; err = ioctl(audio_fd, SNDCTL_DSP_SPEED, &rate); if (err == -1) { DPRINTF("Fail to set speed: %d errno: %d", params->rate, errno); return -1; } /* The device does not support the requested rate / speed */ if (rate != params->rate) { DPRINTF("Mismatch rate: %d params->rate: %d", rate, params->rate); return -1; } #if DEBUG_HDA == 1 err = ioctl(audio_fd, aud->dir ? SNDCTL_DSP_GETOSPACE : SNDCTL_DSP_GETISPACE, &info); if (err == -1) { DPRINTF("Fail to get audio buf info errno: %d", errno); return -1; } DPRINTF("fragstotal: 0x%x fragsize: 0x%x", info.fragstotal, info.fragsize); #endif return 0; } /* * audio_playback - plays samples to the sound device using blocking operations * @aud - the audio player used to play the samples * @buf - the buffer containing the samples * @count - the number of bytes in buffer */ int audio_playback(struct audio *aud, const uint8_t *buf, size_t count) { ssize_t len; size_t total; int audio_fd; assert(aud); assert(aud->dir); assert(buf); audio_fd = aud->fd; assert(audio_fd != -1); for (total = 0; total < count; total += len) { len = write(audio_fd, buf + total, count - total); if (len < 0) { DPRINTF("Fail to write to fd: %d, errno: %d", audio_fd, errno); return -1; } } return 0; } /* * audio_record - records samples from the sound device using * blocking operations. * @aud - the audio player used to capture the samples * @buf - the buffer to receive the samples * @count - the number of bytes to capture in buffer * Returns -1 on error and 0 on success */ int audio_record(struct audio *aud, uint8_t *buf, size_t count) { ssize_t len; size_t total; int audio_fd; assert(aud); assert(!aud->dir); assert(buf); audio_fd = aud->fd; assert(audio_fd != -1); for (total = 0; total < count; total += len) { len = read(audio_fd, buf + total, count - total); if (len < 0) { DPRINTF("Fail to write to fd: %d, errno: %d", audio_fd, errno); return -1; } } return 0; } /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2016 Alex Teaca * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef _AUDIO_EMUL_H_ #define _AUDIO_EMUL_H_ #include #include /* * Audio Player data structures */ struct audio; struct audio_params { int channels; int format; int rate; }; /* * Audio Player API */ /* * audio_init - initialize an instance of audio player * @dev_name - the backend sound device used to play / capture * @dir - dir = 1 for write mode, dir = 0 for read mode * Returns NULL on error and the address of the audio player instance */ struct audio *audio_init(const char *dev_name, uint8_t dir); /* * audio_set_params - reset the sound device and set the audio params * @aud - the audio player to be configured * @params - the audio parameters to be set * Returns -1 on error and 0 on success */ int audio_set_params(struct audio *aud, struct audio_params *params); /* * audio_playback - plays samples to the sound device using blocking operations * @aud - the audio player used to play the samples * @buf - the buffer containing the samples * @count - the number of bytes in buffer * Returns -1 on error and 0 on success */ int audio_playback(struct audio *aud, const uint8_t *buf, size_t count); /* * audio_record - records samples from the sound device using blocking * operations. * @aud - the audio player used to capture the samples * @buf - the buffer to receive the samples * @count - the number of bytes to capture in buffer * Returns -1 on error and 0 on success */ int audio_record(struct audio *aud, uint8_t *buf, size_t count); #endif /* _AUDIO_EMUL_H_ */ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2022 Beckhoff Automation GmbH & Co. KG */ #include #include #include #include #include #include #include #include #include #include #include #include #ifndef __FreeBSD__ #include #endif #include "basl.h" #include "config.h" #include "qemu_loader.h" struct basl_table_checksum { STAILQ_ENTRY(basl_table_checksum) chain; uint32_t off; uint32_t start; uint32_t len; }; struct basl_table_length { STAILQ_ENTRY(basl_table_length) chain; uint32_t off; uint8_t size; }; struct basl_table_pointer { STAILQ_ENTRY(basl_table_pointer) chain; uint8_t src_signature[ACPI_NAMESEG_SIZE]; uint32_t off; uint8_t size; }; struct basl_table { STAILQ_ENTRY(basl_table) chain; struct vmctx *ctx; uint8_t fwcfg_name[QEMU_FWCFG_MAX_NAME]; void *data; uint32_t len; uint32_t off; uint32_t alignment; STAILQ_HEAD(basl_table_checksum_list, basl_table_checksum) checksums; STAILQ_HEAD(basl_table_length_list, basl_table_length) lengths; STAILQ_HEAD(basl_table_pointer_list, basl_table_pointer) pointers; }; static STAILQ_HEAD(basl_table_list, basl_table) basl_tables = STAILQ_HEAD_INITIALIZER( basl_tables); static struct qemu_loader *basl_loader; static struct basl_table *rsdt; static struct basl_table *xsdt; static bool load_into_memory; static __inline uint64_t basl_le_dec(void *pp, size_t len) { assert(len <= 8); switch (len) { case 1: return ((uint8_t *)pp)[0]; case 2: return le16dec(pp); case 4: return le32dec(pp); case 8: return le64dec(pp); } return 0; } static __inline void basl_le_enc(void *pp, uint64_t val, size_t len) { char buf[8]; assert(len <= 8); le64enc(buf, val); memcpy(pp, buf, len); } static int basl_dump_table(const struct basl_table *const table, const bool mem) { const ACPI_TABLE_HEADER *const header = table->data; const uint8_t *data; if (!mem) { data = table->data; } else { data = vm_map_gpa(table->ctx, BHYVE_ACPI_BASE + table->off, table->len); if (data == NULL) { return (ENOMEM); } } #ifdef __FreeBSD__ printf("%.4s @ %8x (%s)\n", header->Signature, BHYVE_ACPI_BASE + table->off, mem ? "Memory" : "FwCfg"); hexdump(data, table->len, NULL, 0); #else (void) printf("%.4s @ %8x (%s)\n", header->Signature, BHYVE_ACPI_BASE + table->off, mem ? "Memory" : "FwCfg"); hexdump_t h; hexdump_init(&h); hexdump_set_addr(&h, BHYVE_ACPI_BASE + table->off); (void) hexdump_fileh(&h, data, table->len, HDF_DEFAULT, stdout); hexdump_fini(&h); (void) printf("\n"); #endif return (0); } static int basl_dump(const bool mem) { struct basl_table *table; STAILQ_FOREACH(table, &basl_tables, chain) { BASL_EXEC(basl_dump_table(table, mem)); } return (0); } void basl_fill_gas(ACPI_GENERIC_ADDRESS *const gas, const uint8_t space_id, const uint8_t bit_width, const uint8_t bit_offset, const uint8_t access_width, const uint64_t address) { assert(gas != NULL); gas->SpaceId = space_id; gas->BitWidth = bit_width; gas->BitOffset = bit_offset; gas->AccessWidth = access_width; gas->Address = htole64(address); } static int basl_finish_install_guest_tables(struct basl_table *const table, uint32_t *const off) { void *gva; table->off = roundup2(*off, table->alignment); *off = table->off + table->len; if (*off <= table->off) { warnx("%s: invalid table length 0x%8x @ offset 0x%8x", __func__, table->len, table->off); return (EFAULT); } /* Cause guest BIOS to copy the ACPI table into guest memory. */ #ifdef __FreeBSD__ BASL_EXEC( qemu_fwcfg_add_file(table->fwcfg_name, table->len, table->data)); #else BASL_EXEC( qemu_fwcfg_add_file((const char *)table->fwcfg_name, table->len, table->data)); #endif BASL_EXEC(qemu_loader_alloc(basl_loader, table->fwcfg_name, table->alignment, QEMU_LOADER_ALLOC_HIGH)); if (!load_into_memory) { return (0); } /* * Install ACPI tables directly in guest memory for use by guests which * do not boot via EFI. EFI ROMs provide a pointer to the firmware * generated ACPI tables instead, but it doesn't hurt to install the * tables always. */ gva = vm_map_gpa(table->ctx, BHYVE_ACPI_BASE + table->off, table->len); if (gva == NULL) { warnx("%s: could not map gpa [ 0x%16lx, 0x%16lx ]", __func__, (uint64_t)BHYVE_ACPI_BASE + table->off, (uint64_t)BHYVE_ACPI_BASE + table->off + table->len); return (ENOMEM); } memcpy(gva, table->data, table->len); return (0); } static int basl_finish_patch_checksums(struct basl_table *const table) { struct basl_table_checksum *checksum; STAILQ_FOREACH(checksum, &table->checksums, chain) { uint8_t *gva, *checksum_gva; uint64_t gpa; uint32_t len; uint8_t sum; len = checksum->len; if (len == BASL_TABLE_CHECKSUM_LEN_FULL_TABLE) { len = table->len; } assert(checksum->off < table->len); assert(checksum->start < table->len); assert(checksum->start + len <= table->len); /* Cause guest BIOS to patch the checksum. */ BASL_EXEC(qemu_loader_add_checksum(basl_loader, table->fwcfg_name, checksum->off, checksum->start, len)); if (!load_into_memory) { continue; } /* * Install ACPI tables directly in guest memory for use by * guests which do not boot via EFI. EFI ROMs provide a pointer * to the firmware generated ACPI tables instead, but it doesn't * hurt to install the tables always. */ gpa = BHYVE_ACPI_BASE + table->off + checksum->start; if ((gpa < BHYVE_ACPI_BASE) || (gpa < BHYVE_ACPI_BASE + table->off)) { warnx("%s: invalid gpa (off 0x%8x start 0x%8x)", __func__, table->off, checksum->start); return (EFAULT); } gva = vm_map_gpa(table->ctx, gpa, len); if (gva == NULL) { warnx("%s: could not map gpa [ 0x%16lx, 0x%16lx ]", __func__, gpa, gpa + len); return (ENOMEM); } checksum_gva = gva + checksum->off; if (checksum_gva < gva) { warnx("%s: invalid checksum offset 0x%8x", __func__, checksum->off); return (EFAULT); } sum = 0; for (uint32_t i = 0; i < len; ++i) { sum += *(gva + i); } *checksum_gva = -sum; } return (0); } static struct basl_table * basl_get_table_by_signature(const uint8_t signature[ACPI_NAMESEG_SIZE]) { struct basl_table *table; STAILQ_FOREACH(table, &basl_tables, chain) { const ACPI_TABLE_HEADER *const header = (const ACPI_TABLE_HEADER *)table->data; #ifdef __FreeBSD__ if (strncmp(header->Signature, signature, sizeof(header->Signature)) == 0) { return (table); #else if (strncmp(header->Signature, (char *)signature, sizeof(header->Signature)) == 0) { return (table); #endif } } warnx("%s: %.4s not found", __func__, signature); return (NULL); } static int basl_finish_patch_pointers(struct basl_table *const table) { struct basl_table_pointer *pointer; STAILQ_FOREACH(pointer, &table->pointers, chain) { const struct basl_table *src_table; uint8_t *gva; uint64_t gpa, val; assert(pointer->off < table->len); assert(pointer->off + pointer->size <= table->len); src_table = basl_get_table_by_signature(pointer->src_signature); if (src_table == NULL) { warnx("%s: could not find ACPI table %.4s", __func__, pointer->src_signature); return (EFAULT); } /* Cause guest BIOS to patch the pointer. */ BASL_EXEC( qemu_loader_add_pointer(basl_loader, table->fwcfg_name, src_table->fwcfg_name, pointer->off, pointer->size)); if (!load_into_memory) { continue; } /* * Install ACPI tables directly in guest memory for use by * guests which do not boot via EFI. EFI ROMs provide a pointer * to the firmware generated ACPI tables instead, but it doesn't * hurt to install the tables always. */ gpa = BHYVE_ACPI_BASE + table->off; if (gpa < BHYVE_ACPI_BASE) { warnx("%s: table offset of 0x%8x is too large", __func__, table->off); return (EFAULT); } gva = vm_map_gpa(table->ctx, gpa, table->len); if (gva == NULL) { warnx("%s: could not map gpa [ 0x%16lx, 0x%16lx ]", __func__, gpa, gpa + table->len); return (ENOMEM); } val = basl_le_dec(gva + pointer->off, pointer->size); val += BHYVE_ACPI_BASE + src_table->off; basl_le_enc(gva + pointer->off, val, pointer->size); } return (0); } static int basl_finish_set_length(struct basl_table *const table) { struct basl_table_length *length; STAILQ_FOREACH(length, &table->lengths, chain) { assert(length->off < table->len); assert(length->off + length->size <= table->len); basl_le_enc((uint8_t *)table->data + length->off, table->len, length->size); } return (0); } int basl_finish(void) { struct basl_table *table; uint32_t off = 0; if (STAILQ_EMPTY(&basl_tables)) { warnx("%s: no ACPI tables found", __func__); return (EINVAL); } /* * If we install ACPI tables by FwCfg and by memory, Windows will use * the tables from memory. This can cause issues when using advanced * features like a TPM log because we aren't able to patch the memory * tables accordingly. */ load_into_memory = get_config_bool_default("acpi_tables_in_memory", true); #ifndef __FreeBSD__ if (get_config_bool_default("basl.debug", false)) basl_dump(false); #endif /* * We have to install all tables before we can patch them. Therefore, * use two loops. The first one installs all tables and the second one * patches them. */ STAILQ_FOREACH(table, &basl_tables, chain) { BASL_EXEC(basl_finish_set_length(table)); BASL_EXEC(basl_finish_install_guest_tables(table, &off)); } STAILQ_FOREACH(table, &basl_tables, chain) { BASL_EXEC(basl_finish_patch_pointers(table)); /* * Calculate the checksum as last step! */ BASL_EXEC(basl_finish_patch_checksums(table)); } BASL_EXEC(qemu_loader_finish(basl_loader)); #ifndef __FreeBSD__ if (get_config_bool_default("basl.debug", false)) basl_dump(true); #endif return (0); } static int basl_init_rsdt(struct vmctx *const ctx) { BASL_EXEC( basl_table_create(&rsdt, ctx, ACPI_SIG_RSDT, BASL_TABLE_ALIGNMENT)); /* Header */ BASL_EXEC(basl_table_append_header(rsdt, ACPI_SIG_RSDT, 1, 1)); /* Pointers (added by basl_table_register_to_rsdt) */ return (0); } static int basl_init_xsdt(struct vmctx *const ctx) { BASL_EXEC( basl_table_create(&xsdt, ctx, ACPI_SIG_XSDT, BASL_TABLE_ALIGNMENT)); /* Header */ BASL_EXEC(basl_table_append_header(xsdt, ACPI_SIG_XSDT, 1, 1)); /* Pointers (added by basl_table_register_to_rsdt) */ return (0); } int basl_init(struct vmctx *const ctx) { BASL_EXEC(basl_init_rsdt(ctx)); BASL_EXEC(basl_init_xsdt(ctx)); #ifdef __FreeBSD__ BASL_EXEC( qemu_loader_create(&basl_loader, QEMU_FWCFG_FILE_TABLE_LOADER)); #else BASL_EXEC( qemu_loader_create(&basl_loader, (uint8_t *)QEMU_FWCFG_FILE_TABLE_LOADER)); #endif return (0); } int basl_table_add_checksum(struct basl_table *const table, const uint32_t off, const uint32_t start, const uint32_t len) { struct basl_table_checksum *checksum; assert(table != NULL); checksum = calloc(1, sizeof(struct basl_table_checksum)); if (checksum == NULL) { warnx("%s: failed to allocate checksum", __func__); return (ENOMEM); } checksum->off = off; checksum->start = start; checksum->len = len; STAILQ_INSERT_TAIL(&table->checksums, checksum, chain); return (0); } int basl_table_add_length(struct basl_table *const table, const uint32_t off, const uint8_t size) { struct basl_table_length *length; assert(table != NULL); assert(size == 4 || size == 8); length = calloc(1, sizeof(struct basl_table_length)); if (length == NULL) { warnx("%s: failed to allocate length", __func__); return (ENOMEM); } length->off = off; length->size = size; STAILQ_INSERT_TAIL(&table->lengths, length, chain); return (0); } int basl_table_add_pointer(struct basl_table *const table, const uint8_t src_signature[ACPI_NAMESEG_SIZE], const uint32_t off, const uint8_t size) { struct basl_table_pointer *pointer; assert(table != NULL); assert(size == 4 || size == 8); pointer = calloc(1, sizeof(struct basl_table_pointer)); if (pointer == NULL) { warnx("%s: failed to allocate pointer", __func__); return (ENOMEM); } memcpy(pointer->src_signature, src_signature, sizeof(pointer->src_signature)); pointer->off = off; pointer->size = size; STAILQ_INSERT_TAIL(&table->pointers, pointer, chain); return (0); } int basl_table_append_bytes(struct basl_table *const table, const void *const bytes, const uint32_t len) { void *end; assert(table != NULL); assert(bytes != NULL); if (table->len + len <= table->len) { warnx("%s: table too large (table->len 0x%8x len 0x%8x)", __func__, table->len, len); return (EFAULT); } table->data = reallocf(table->data, table->len + len); if (table->data == NULL) { warnx("%s: failed to realloc table to length 0x%8x", __func__, table->len + len); table->len = 0; return (ENOMEM); } end = (uint8_t *)table->data + table->len; table->len += len; memcpy(end, bytes, len); return (0); } int basl_table_append_checksum(struct basl_table *const table, const uint32_t start, const uint32_t len) { assert(table != NULL); BASL_EXEC(basl_table_add_checksum(table, table->len, start, len)); BASL_EXEC(basl_table_append_int(table, 0, 1)); return (0); } int basl_table_append_content(struct basl_table *table, void *data, uint32_t len) { assert(data != NULL); assert(len >= sizeof(ACPI_TABLE_HEADER)); return (basl_table_append_bytes(table, (void *)((uintptr_t)(data) + sizeof(ACPI_TABLE_HEADER)), len - sizeof(ACPI_TABLE_HEADER))); } int basl_table_append_fwcfg(struct basl_table *const table, const uint8_t *fwcfg_name, const uint32_t alignment, const uint8_t size) { assert(table != NULL); assert(fwcfg_name != NULL); assert(size <= sizeof(uint64_t)); BASL_EXEC(qemu_loader_alloc(basl_loader, fwcfg_name, alignment, QEMU_LOADER_ALLOC_HIGH)); BASL_EXEC(qemu_loader_add_pointer(basl_loader, table->fwcfg_name, fwcfg_name, table->len, size)); BASL_EXEC(basl_table_append_int(table, 0, size)); return (0); } int basl_table_append_gas(struct basl_table *const table, const uint8_t space_id, const uint8_t bit_width, const uint8_t bit_offset, const uint8_t access_width, const uint64_t address) { ACPI_GENERIC_ADDRESS gas_le = { .SpaceId = space_id, .BitWidth = bit_width, .BitOffset = bit_offset, .AccessWidth = access_width, .Address = htole64(address), }; return (basl_table_append_bytes(table, &gas_le, sizeof(gas_le))); } int basl_table_append_header(struct basl_table *const table, const uint8_t signature[ACPI_NAMESEG_SIZE], const uint8_t revision, const uint32_t oem_revision) { ACPI_TABLE_HEADER header_le; /* + 1 is required for the null terminator */ char oem_table_id[ACPI_OEM_TABLE_ID_SIZE + 1]; assert(table != NULL); assert(table->len == 0); memcpy(header_le.Signature, signature, ACPI_NAMESEG_SIZE); header_le.Length = 0; /* patched by basl_finish */ header_le.Revision = revision; header_le.Checksum = 0; /* patched by basl_finish */ memcpy(header_le.OemId, "BHYVE ", ACPI_OEM_ID_SIZE); snprintf(oem_table_id, ACPI_OEM_TABLE_ID_SIZE, "BV%.4s ", signature); memcpy(header_le.OemTableId, oem_table_id, sizeof(header_le.OemTableId)); header_le.OemRevision = htole32(oem_revision); memcpy(header_le.AslCompilerId, "BASL", ACPI_NAMESEG_SIZE); header_le.AslCompilerRevision = htole32(0x20220504); BASL_EXEC( basl_table_append_bytes(table, &header_le, sizeof(header_le))); BASL_EXEC(basl_table_add_length(table, offsetof(ACPI_TABLE_HEADER, Length), sizeof(header_le.Length))); BASL_EXEC(basl_table_add_checksum(table, offsetof(ACPI_TABLE_HEADER, Checksum), 0, BASL_TABLE_CHECKSUM_LEN_FULL_TABLE)); return (0); } int basl_table_append_int(struct basl_table *const table, const uint64_t val, const uint8_t size) { char buf[8]; assert(size <= sizeof(val)); basl_le_enc(buf, val, size); return (basl_table_append_bytes(table, buf, size)); } int basl_table_append_length(struct basl_table *const table, const uint8_t size) { assert(table != NULL); assert(size <= sizeof(table->len)); BASL_EXEC(basl_table_add_length(table, table->len, size)); BASL_EXEC(basl_table_append_int(table, 0, size)); return (0); } int basl_table_append_pointer(struct basl_table *const table, const uint8_t src_signature[ACPI_NAMESEG_SIZE], const uint8_t size) { assert(table != NULL); assert(size == 4 || size == 8); BASL_EXEC(basl_table_add_pointer(table, src_signature, table->len, size)); BASL_EXEC(basl_table_append_int(table, 0, size)); return (0); } int basl_table_create(struct basl_table **const table, struct vmctx *ctx, const uint8_t *const name, const uint32_t alignment) { struct basl_table *new_table; assert(table != NULL); new_table = calloc(1, sizeof(struct basl_table)); if (new_table == NULL) { warnx("%s: failed to allocate table", __func__); return (ENOMEM); } new_table->ctx = ctx; #ifdef __FreeBSD__ snprintf(new_table->fwcfg_name, sizeof(new_table->fwcfg_name), "etc/acpi/%s", name); #else snprintf((char *)new_table->fwcfg_name, sizeof (new_table->fwcfg_name), "etc/acpi/%s", name); #endif new_table->alignment = alignment; STAILQ_INIT(&new_table->checksums); STAILQ_INIT(&new_table->lengths); STAILQ_INIT(&new_table->pointers); STAILQ_INSERT_TAIL(&basl_tables, new_table, chain); *table = new_table; return (0); } int basl_table_register_to_rsdt(struct basl_table *table) { const ACPI_TABLE_HEADER *header; assert(table != NULL); header = (const ACPI_TABLE_HEADER *)table->data; #ifdef __FreeBSD__ BASL_EXEC(basl_table_append_pointer(rsdt, header->Signature, ACPI_RSDT_ENTRY_SIZE)); BASL_EXEC(basl_table_append_pointer(xsdt, header->Signature, ACPI_XSDT_ENTRY_SIZE)); #else BASL_EXEC(basl_table_append_pointer(rsdt, (uint8_t *)header->Signature, ACPI_RSDT_ENTRY_SIZE)); BASL_EXEC(basl_table_append_pointer(xsdt, (uint8_t *)header->Signature, ACPI_XSDT_ENTRY_SIZE)); #endif return (0); } /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2022 Beckhoff Automation GmbH & Co. KG */ #pragma once #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wunused-parameter" #include #pragma GCC diagnostic pop #ifndef __FreeBSD__ /* Until the in-gate ACPI is updated, map the new name to the old. */ #define ACPI_NAMESEG_SIZE ACPI_NAME_SIZE /* * Also to avoid type conflicts without having to pepper the code with ifdefs, * redefine these constants to be uint8_t * */ #undef ACPI_SIG_FACS #undef ACPI_SIG_MCFG #undef ACPI_SIG_HPET #undef ACPI_SIG_MADT #undef ACPI_SIG_DSDT #undef ACPI_SIG_FADT #undef ACPI_SIG_XSDT #undef ACPI_SIG_RSDT #undef ACPI_RSDP_NAME #undef ACPI_SIG_SPCR #undef ACPI_SIG_TPM2 #define ACPI_SIG_FACS (const uint8_t *)"FACS" #define ACPI_SIG_MCFG (const uint8_t *)"MCFG" #define ACPI_SIG_HPET (const uint8_t *)"HPET" #define ACPI_SIG_MADT (const uint8_t *)"APIC" #define ACPI_SIG_DSDT (const uint8_t *)"DSDT" #define ACPI_SIG_FADT (const uint8_t *)"FACP" #define ACPI_SIG_XSDT (const uint8_t *)"XSDT" #define ACPI_SIG_RSDT (const uint8_t *)"RSDT" #define ACPI_RSDP_NAME (const uint8_t *)"RSDP" #define ACPI_SIG_SPCR (const uint8_t *)"SPCR" #define ACPI_SIG_TPM2 (const uint8_t *)"TPM2" #endif /* !__FreeBSD__ */ #include "qemu_fwcfg.h" #define ACPI_GAS_ACCESS_WIDTH_LEGACY 0 #define ACPI_GAS_ACCESS_WIDTH_UNDEFINED 0 #define ACPI_GAS_ACCESS_WIDTH_BYTE 1 #define ACPI_GAS_ACCESS_WIDTH_WORD 2 #define ACPI_GAS_ACCESS_WIDTH_DWORD 3 #define ACPI_GAS_ACCESS_WIDTH_QWORD 4 #define ACPI_SPCR_INTERRUPT_TYPE_8259 0x1 #define ACPI_SPCR_INTERRUPT_TYPE_APIC 0x2 #define ACPI_SPCR_INTERRUPT_TYPE_SAPIC 0x4 #define ACPI_SPCR_INTERRUPT_TYPE_GIC 0x8 #define ACPI_SPCR_BAUD_RATE_9600 3 #define ACPI_SPCR_BAUD_RATE_19200 4 #define ACPI_SPCR_BAUD_RATE_57600 6 #define ACPI_SPCR_BAUD_RATE_115200 7 #define ACPI_SPCR_PARITY_NO_PARITY 0 #define ACPI_SPCR_STOP_BITS_1 1 #define ACPI_SPCR_FLOW_CONTROL_DCD 0x1 #define ACPI_SPCR_FLOW_CONTROL_RTS_CTS 0x2 #define ACPI_SPCR_FLOW_CONTROL_XON_XOFF 0x4 #define ACPI_SPCR_TERMINAL_TYPE_VT100 0 #define ACPI_SPCR_TERMINAL_TYPE_VT100_PLUS 1 #define ACPI_SPCR_TERMINAL_TYPE_VT_UTF8 2 #define ACPI_SPCR_TERMINAL_TYPE_ANSI 3 #define BHYVE_ACPI_BASE 0xf2400 #define BASL_TABLE_ALIGNMENT 0x10 #define BASL_TABLE_ALIGNMENT_FACS 0x40 #define BASL_TABLE_CHECKSUM_LEN_FULL_TABLE (-1U) #define BASL_EXEC(x) \ do { \ const int error = (x); \ if (error) { \ warnc(error, \ "BASL failed @ %s:%d\n Failed to execute %s", \ __func__, __LINE__, #x); \ return (error); \ } \ } while (0) struct basl_table; void basl_fill_gas(ACPI_GENERIC_ADDRESS *gas, uint8_t space_id, uint8_t bit_width, uint8_t bit_offset, uint8_t access_width, uint64_t address); int basl_finish(void); int basl_init(struct vmctx *ctx); int basl_table_add_checksum(struct basl_table *const table, const uint32_t off, const uint32_t start, const uint32_t len); int basl_table_add_length(struct basl_table *const table, const uint32_t off, const uint8_t size); int basl_table_add_pointer(struct basl_table *const table, const uint8_t src_signature[ACPI_NAMESEG_SIZE], const uint32_t off, const uint8_t size); int basl_table_append_bytes(struct basl_table *table, const void *bytes, uint32_t len); int basl_table_append_checksum(struct basl_table *table, uint32_t start, uint32_t len); /* Add an ACPI_TABLE_* to basl without its header. */ int basl_table_append_content(struct basl_table *table, void *data, uint32_t len); int basl_table_append_fwcfg(struct basl_table *table, const uint8_t *fwcfg_name, uint32_t alignment, uint8_t size); int basl_table_append_gas(struct basl_table *table, uint8_t space_id, uint8_t bit_width, uint8_t bit_offset, uint8_t access_width, uint64_t address); int basl_table_append_header(struct basl_table *table, const uint8_t signature[ACPI_NAMESEG_SIZE], uint8_t revision, uint32_t oem_revision); int basl_table_append_int(struct basl_table *table, uint64_t val, uint8_t size); int basl_table_append_length(struct basl_table *table, uint8_t size); int basl_table_append_pointer(struct basl_table *table, const uint8_t src_signature[ACPI_NAMESEG_SIZE], uint8_t size); int basl_table_create(struct basl_table **table, struct vmctx *ctx, const uint8_t *name, uint32_t alignment); /* Adds the table to RSDT and XSDT */ int basl_table_register_to_rsdt(struct basl_table *table); /* * This file and its contents are supplied under the terms of the * Common Development and Distribution License ("CDDL"), version 1.0. * You may only use this file in accordance with the terms of version * 1.0 of the CDDL. * * A full copy of the text of the CDDL should have accompanied this * source. A copy of the CDDL is also available via the Internet at * http://www.illumos.org/license/CDDL. */ /* This file is dual-licensed; see usr/src/contrib/bhyve/LICENSE */ /* * Copyright 2013 Pluribus Networks Inc. * Copyright 2017 Joyent, Inc. */ #include #include #include /* * Make a pre-existing termios structure into "raw" mode: character-at-a-time * mode with no characters interpreted, 8-bit data path. */ void cfmakeraw(struct termios *t) { t->c_iflag &= ~(IMAXBEL|IXOFF|INPCK|BRKINT|PARMRK|ISTRIP|INLCR|IGNCR| ICRNL|IXON|IGNPAR); t->c_iflag |= IGNBRK; t->c_oflag &= ~OPOST; t->c_lflag &= ~(ECHO|ECHOE|ECHOK|ECHONL|ICANON|ISIG|IEXTEN|NOFLSH| TOSTOP|PENDIN); t->c_cflag &= ~(CSIZE|PARENB); t->c_cflag |= CS8|CREAD; t->c_cc[VMIN] = 1; t->c_cc[VTIME] = 0; } /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2015 Tycho Nightingale * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include #include #include #include #ifndef __FreeBSD__ #include #endif #include "bhyvegc.h" struct bhyvegc { struct bhyvegc_image *gc_image; int raw; }; struct bhyvegc * bhyvegc_init(int width, int height, void *fbaddr) { struct bhyvegc *gc; struct bhyvegc_image *gc_image; gc = calloc(1, sizeof (struct bhyvegc)); gc_image = calloc(1, sizeof(struct bhyvegc_image)); gc_image->width = width; gc_image->height = height; if (fbaddr) { gc_image->data = fbaddr; gc->raw = 1; } else { gc_image->data = calloc(width * height, sizeof (uint32_t)); gc->raw = 0; } gc->gc_image = gc_image; #ifndef __FreeBSD__ pthread_mutex_init(&gc_image->mtx, NULL); #endif return (gc); } void bhyvegc_set_fbaddr(struct bhyvegc *gc, void *fbaddr) { #ifndef __FreeBSD__ pthread_mutex_lock(&gc->gc_image->mtx); #endif gc->raw = 1; if (gc->gc_image->data && gc->gc_image->data != fbaddr) free(gc->gc_image->data); gc->gc_image->data = fbaddr; #ifndef __FreeBSD__ pthread_mutex_unlock(&gc->gc_image->mtx); #endif } void bhyvegc_resize(struct bhyvegc *gc, int width, int height) { struct bhyvegc_image *gc_image; #ifndef __FreeBSD__ pthread_mutex_lock(&gc->gc_image->mtx); #endif gc_image = gc->gc_image; gc_image->width = width; gc_image->height = height; if (!gc->raw) { gc_image->data = reallocarray(gc_image->data, width * height, sizeof (uint32_t)); if (gc_image->data != NULL) memset(gc_image->data, 0, width * height * sizeof (uint32_t)); } #ifndef __FreeBSD__ pthread_mutex_unlock(&gc->gc_image->mtx); #endif } struct bhyvegc_image * bhyvegc_get_image(struct bhyvegc *gc) { if (gc == NULL) return (NULL); return (gc->gc_image); } /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2015 Tycho Nightingale * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef _BHYVEGC_H_ #define _BHYVEGC_H_ #ifndef __FreeBSD__ #include #endif struct bhyvegc; struct bhyvegc_image { int vgamode; int width; int height; uint32_t *data; #ifndef __FreeBSD__ pthread_mutex_t mtx; #endif }; struct bhyvegc *bhyvegc_init(int width, int height, void *fbaddr); void bhyvegc_set_fbaddr(struct bhyvegc *gc, void *fbaddr); void bhyvegc_resize(struct bhyvegc *gc, int width, int height); struct bhyvegc_image *bhyvegc_get_image(struct bhyvegc *gc); #endif /* _BHYVEGC_H_ */ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2011 NetApp, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * This file and its contents are supplied under the terms of the * Common Development and Distribution License ("CDDL"), version 1.0. * You may only use this file in accordance with the terms of version * 1.0 of the CDDL. * * A full copy of the text of the CDDL should have accompanied this * source. A copy of the CDDL is also available via the Internet at * http://www.illumos.org/license/CDDL. * * Copyright 2015 Pluribus Networks Inc. * Copyright 2018 Joyent, Inc. * Copyright 2025 Oxide Computer Company * Copyright 2026 OmniOS Community Edition (OmniOSce) Association. */ #include #ifndef WITHOUT_CAPSICUM #include #endif #include #include #ifdef __FreeBSD__ #include #else #include #include #endif #include #ifndef WITHOUT_CAPSICUM #include #endif #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifndef WITHOUT_CAPSICUM #include #endif #ifdef __FreeBSD__ #include #endif #include #include "acpi.h" #include "bhyverun.h" #include "bootrom.h" #include "config.h" #include "debug.h" #include "gdb.h" #include "ioapic.h" #include "mem.h" #include "mevent.h" #include "pci_emul.h" #include "pci_lpc.h" #include "qemu_fwcfg.h" #include "tpm_device.h" #include "spinup_ap.h" #include "vmgenc.h" #include "vmexit.h" #ifndef __FreeBSD__ #include "smbiostbl.h" #include "privileges.h" #endif #define MB (1024UL * 1024) #define GB (1024UL * MB) int guest_ncpus; uint16_t cpu_cores, cpu_sockets, cpu_threads; int raw_stdio = 0; static const int BSP = 0; static cpuset_t cpumask; static void vm_loop(struct vmctx *ctx, struct vcpu *vcpu); static struct vcpu_info { struct vmctx *ctx; struct vcpu *vcpu; int vcpuid; } *vcpu_info; #ifdef __FreeBSD__ static cpuset_t **vcpumap; #endif /* * XXX This parser is known to have the following issues: * 1. It accepts null key=value tokens ",," as setting "cpus" to an * empty string. * * The acceptance of a null specification ('-c ""') is by design to match the * manual page syntax specification, this results in a topology of 1 vCPU. */ int bhyve_topology_parse(const char *opt) { char *cp, *str, *tofree; if (*opt == '\0') { set_config_value("sockets", "1"); set_config_value("cores", "1"); set_config_value("threads", "1"); set_config_value("cpus", "1"); return (0); } tofree = str = strdup(opt); if (str == NULL) errx(4, "Failed to allocate memory"); while ((cp = strsep(&str, ",")) != NULL) { if (strncmp(cp, "cpus=", strlen("cpus=")) == 0) set_config_value("cpus", cp + strlen("cpus=")); else if (strncmp(cp, "sockets=", strlen("sockets=")) == 0) set_config_value("sockets", cp + strlen("sockets=")); else if (strncmp(cp, "cores=", strlen("cores=")) == 0) set_config_value("cores", cp + strlen("cores=")); else if (strncmp(cp, "threads=", strlen("threads=")) == 0) set_config_value("threads", cp + strlen("threads=")); else if (strchr(cp, '=') != NULL) goto out; else set_config_value("cpus", cp); } free(tofree); return (0); out: free(tofree); return (-1); } static int parse_int_value(const char *key, const char *value, int minval, int maxval) { char *cp; long lval; errno = 0; lval = strtol(value, &cp, 0); if (errno != 0 || *cp != '\0' || cp == value || lval < minval || lval > maxval) errx(4, "Invalid value for %s: '%s'", key, value); return (lval); } /* * Set the sockets, cores, threads, and guest_cpus variables based on * the configured topology. * * The limits of UINT16_MAX are due to the types passed to * vm_set_topology(). vmm.ko may enforce tighter limits. */ static void calc_topology(void) { const char *value; bool explicit_cpus; uint64_t ncpus; value = get_config_value("cpus"); if (value != NULL) { guest_ncpus = parse_int_value("cpus", value, 1, UINT16_MAX); explicit_cpus = true; } else { guest_ncpus = 1; explicit_cpus = false; } value = get_config_value("cores"); if (value != NULL) cpu_cores = parse_int_value("cores", value, 1, UINT16_MAX); else cpu_cores = 1; value = get_config_value("threads"); if (value != NULL) cpu_threads = parse_int_value("threads", value, 1, UINT16_MAX); else cpu_threads = 1; value = get_config_value("sockets"); if (value != NULL) cpu_sockets = parse_int_value("sockets", value, 1, UINT16_MAX); else cpu_sockets = guest_ncpus; /* * Compute sockets * cores * threads avoiding overflow. The * range check above insures these are 16 bit values. */ ncpus = (uint64_t)cpu_sockets * cpu_cores * cpu_threads; if (ncpus > UINT16_MAX) errx(4, "Computed number of vCPUs too high: %ju", (uintmax_t)ncpus); if (explicit_cpus) { if (guest_ncpus != (int)ncpus) errx(4, "Topology (%d sockets, %d cores, %d threads) " "does not match %d vCPUs", cpu_sockets, cpu_cores, cpu_threads, guest_ncpus); } else guest_ncpus = ncpus; } #ifdef __FreeBSD__ int bhyve_pincpu_parse(const char *opt) { int vcpu, pcpu; const char *value; char *newval; char key[16]; if (sscanf(opt, "%d:%d", &vcpu, &pcpu) != 2) { fprintf(stderr, "invalid format: %s\n", opt); return (-1); } if (vcpu < 0) { fprintf(stderr, "invalid vcpu '%d'\n", vcpu); return (-1); } if (pcpu < 0 || pcpu >= CPU_SETSIZE) { fprintf(stderr, "hostcpu '%d' outside valid range from " "0 to %d\n", pcpu, CPU_SETSIZE - 1); return (-1); } snprintf(key, sizeof(key), "vcpu.%d.cpuset", vcpu); value = get_config_value(key); if (asprintf(&newval, "%s%s%d", value != NULL ? value : "", value != NULL ? "," : "", pcpu) == -1) { perror("failed to build new cpuset string"); return (-1); } set_config_value(key, newval); free(newval); return (0); } static void parse_cpuset(int vcpu, const char *list, cpuset_t *set) { char *cp, *token; int pcpu, start; CPU_ZERO(set); start = -1; token = __DECONST(char *, list); for (;;) { pcpu = strtoul(token, &cp, 0); if (cp == token) errx(4, "invalid cpuset for vcpu %d: '%s'", vcpu, list); if (pcpu < 0 || pcpu >= CPU_SETSIZE) errx(4, "hostcpu '%d' outside valid range from 0 to %d", pcpu, CPU_SETSIZE - 1); switch (*cp) { case ',': case '\0': if (start >= 0) { if (start > pcpu) errx(4, "Invalid hostcpu range %d-%d", start, pcpu); while (start < pcpu) { CPU_SET(start, set); start++; } start = -1; } CPU_SET(pcpu, set); break; case '-': if (start >= 0) errx(4, "invalid cpuset for vcpu %d: '%s'", vcpu, list); start = pcpu; break; default: errx(4, "invalid cpuset for vcpu %d: '%s'", vcpu, list); } if (*cp == '\0') break; token = cp + 1; } } static void build_vcpumaps(void) { char key[16]; const char *value; int vcpu; vcpumap = calloc(guest_ncpus, sizeof(*vcpumap)); for (vcpu = 0; vcpu < guest_ncpus; vcpu++) { snprintf(key, sizeof(key), "vcpu.%d.cpuset", vcpu); value = get_config_value(key); if (value == NULL) continue; vcpumap[vcpu] = malloc(sizeof(cpuset_t)); if (vcpumap[vcpu] == NULL) err(4, "Failed to allocate cpuset for vcpu %d", vcpu); parse_cpuset(vcpu, value, vcpumap[vcpu]); } } #endif /* __FreeBSD__ */ void * paddr_guest2host(struct vmctx *ctx, uintptr_t gaddr, size_t len) { return (vm_map_gpa(ctx, gaddr, len)); } bool fbsdrun_virtio_msix(void) { /* * The configuration key used to be the non-namespaced `virtio_msix`. * Check for that too in case anyone is using the old name in a * configuration file. If either of these exists and is set to false * we'll disable MSI-X for Virtio. */ return (get_config_bool_default("virtio.msix", true) && get_config_bool_default("virtio_msix", true)); } struct vcpu * fbsdrun_vcpu(int vcpuid) { return (vcpu_info[vcpuid].vcpu); } static void * fbsdrun_start_thread(void *param) { char tname[MAXCOMLEN + 1]; struct vcpu_info *vi = param; #ifdef __FreeBSD__ int error; #endif snprintf(tname, sizeof(tname), "vcpu %d", vi->vcpuid); pthread_set_name_np(pthread_self(), tname); #ifdef __FreeBSD__ if (vcpumap[vi->vcpuid] != NULL) { error = pthread_setaffinity_np(pthread_self(), sizeof(cpuset_t), vcpumap[vi->vcpuid]); assert(error == 0); } #endif gdb_cpu_add(vi->vcpu); vm_loop(vi->ctx, vi->vcpu); /* not reached */ exit(1); return (NULL); } void fbsdrun_addcpu(int vcpuid, bool suspend) { struct vcpu_info *vi; pthread_t thr; int error; vi = &vcpu_info[vcpuid]; error = vm_activate_cpu(vi->vcpu); if (error != 0) err(EX_OSERR, "could not activate CPU %d", vi->vcpuid); CPU_SET_ATOMIC(vcpuid, &cpumask); if (suspend) { error = vm_suspend_cpu(vi->vcpu); assert(error == 0); } error = pthread_create(&thr, NULL, fbsdrun_start_thread, vi); assert(error == 0); } void fbsdrun_deletecpu(int vcpu) { static pthread_mutex_t resetcpu_mtx = PTHREAD_MUTEX_INITIALIZER; static pthread_cond_t resetcpu_cond = PTHREAD_COND_INITIALIZER; pthread_mutex_lock(&resetcpu_mtx); if (!CPU_ISSET(vcpu, &cpumask)) { EPRINTLN("Attempting to delete unknown cpu %d", vcpu); exit(4); } CPU_CLR(vcpu, &cpumask); if (vcpu != BSP) { pthread_cond_signal(&resetcpu_cond); pthread_mutex_unlock(&resetcpu_mtx); pthread_exit(NULL); /* NOTREACHED */ } while (!CPU_EMPTY(&cpumask)) { pthread_cond_wait(&resetcpu_cond, &resetcpu_mtx); } pthread_mutex_unlock(&resetcpu_mtx); } static void vm_loop(struct vmctx *ctx, struct vcpu *vcpu) { struct vm_exit vme; int error, rc; enum vm_exitcode exitcode; cpuset_t active_cpus; struct vm_entry *ventry; error = vm_active_cpus(ctx, &active_cpus); assert(CPU_ISSET(vcpu_id(vcpu), &active_cpus)); ventry = vmentry_vcpu(vcpu_id(vcpu)); while (1) { error = vm_run(vcpu, ventry, &vme); if (error != 0) break; if (ventry->cmd != VEC_DEFAULT) { /* * Discard any lingering entry state after it has been * submitted via vm_run(). */ bzero(ventry, sizeof (*ventry)); } exitcode = vme.exitcode; if (exitcode >= VM_EXITCODE_MAX || vmexit_handlers[exitcode] == NULL) { fprintf(stderr, "vm_loop: unexpected exitcode 0x%x\n", exitcode); exit(4); } rc = (*vmexit_handlers[exitcode])(ctx, vcpu, &vme); switch (rc) { case VMEXIT_CONTINUE: break; case VMEXIT_ABORT: abort(); default: exit(4); } } EPRINTLN("vm_run error %d, errno %d", error, errno); } static int num_vcpus_allowed(struct vmctx *ctx, struct vcpu *vcpu) { uint16_t sockets, cores, threads, maxcpus; #ifdef __FreeBSD__ int tmp, error; /* * The guest is allowed to spinup more than one processor only if the * UNRESTRICTED_GUEST capability is available. */ error = vm_get_capability(vcpu, VM_CAP_UNRESTRICTED_GUEST, &tmp); if (error != 0) return (1); #else int error; /* Unrestricted Guest is always enabled on illumos */ #endif /* __FreeBSD__ */ error = vm_get_topology(ctx, &sockets, &cores, &threads, &maxcpus); if (error == 0) return (maxcpus); else return (1); } static struct vmctx * do_open(const char *vmname) { struct vmctx *ctx; int error; bool reinit, romboot; reinit = romboot = false; romboot = bootrom_boot(); #ifndef __FreeBSD__ uint64_t create_flags = 0; if (get_config_bool_default("memory.use_reservoir", false)) { create_flags |= VCF_RESERVOIR_MEM; } error = vm_create(vmname, create_flags); #else error = vm_create(vmname); #endif /* __FreeBSD__ */ if (error) { if (errno == EEXIST) { if (romboot) { reinit = true; } else { /* * The virtual machine has been setup by the * userspace bootloader. */ } } else { perror("vm_create"); exit(4); } } else { if (!romboot) { /* * If the virtual machine was just created then a * bootrom must be configured to boot it. */ fprintf(stderr, "virtual machine cannot be booted\n"); exit(4); } } ctx = vm_open(vmname); if (ctx == NULL) { perror("vm_open"); #ifndef __FreeBSD__ if (errno == ENOENT) { fprintf(stderr, "Does the VM already exist in the global or " "another zone?\n"); } #endif exit(4); } #ifndef WITHOUT_CAPSICUM if (vm_limit_rights(ctx) != 0) err(EX_OSERR, "vm_limit_rights"); #endif if (reinit) { #ifndef __FreeBSD__ error = vm_reinit(ctx, 0); #else error = vm_reinit(ctx); #endif if (error) { perror("vm_reinit"); exit(4); } } error = vm_set_topology(ctx, cpu_sockets, cpu_cores, cpu_threads, 0); if (error) errx(EX_OSERR, "vm_set_topology"); return (ctx); } bool bhyve_parse_config_option(const char *option) { const char *value; char *path; value = strchr(option, '='); if (value == NULL || value[1] == '\0') return (false); path = strndup(option, value - option); if (path == NULL) err(4, "Failed to allocate memory"); set_config_value(path, value + 1); free(path); return (true); } void bhyve_parse_simple_config_file(const char *path) { FILE *fp; char *line, *cp; size_t linecap; unsigned int lineno; fp = fopen(path, "r"); if (fp == NULL) err(4, "Failed to open configuration file %s", path); line = NULL; linecap = 0; lineno = 1; for (lineno = 1; getline(&line, &linecap, fp) > 0; lineno++) { if (*line == '#' || *line == '\n') continue; cp = strchr(line, '\n'); if (cp != NULL) *cp = '\0'; if (!bhyve_parse_config_option(line)) errx(4, "%s line %u: invalid config option '%s'", path, lineno, line); } free(line); fclose(fp); } void bhyve_parse_gdb_options(const char *opt) { const char *sport; char *colon; if (opt[0] == 'w') { set_config_bool("gdb.wait", true); opt++; } colon = strrchr(opt, ':'); if (colon == NULL) { sport = opt; } else { *colon = '\0'; colon++; sport = colon; set_config_value("gdb.address", opt); } set_config_value("gdb.port", sport); } int main(int argc, char *argv[]) { int error; int max_vcpus, memflags; struct vcpu *bsp; struct vmctx *ctx; size_t memsize; const char *value, *vmname; bhyve_init_config(); bhyve_optparse(argc, argv); argc -= optind; argv += optind; if (argc > 1) bhyve_usage(1); if (argc == 1) set_config_value("name", argv[0]); vmname = get_config_value("name"); if (vmname == NULL) bhyve_usage(1); if (get_config_bool_default("config.dump", false)) { dump_config(); exit(1); } #ifndef __FreeBSD__ illumos_priv_init(); #endif calc_topology(); #ifdef __FreeBSD__ build_vcpumaps(); #endif value = get_config_value("memory.size"); error = vm_parse_memsize(value, &memsize); if (error) errx(EX_USAGE, "invalid memsize '%s'", value); ctx = do_open(vmname); bsp = vm_vcpu_open(ctx, BSP); max_vcpus = num_vcpus_allowed(ctx, bsp); if (guest_ncpus > max_vcpus) { fprintf(stderr, "%d vCPUs requested but only %d available\n", guest_ncpus, max_vcpus); exit(4); } bhyve_init_vcpu(bsp); /* Allocate per-VCPU resources. */ vcpu_info = calloc(guest_ncpus, sizeof(*vcpu_info)); for (int vcpuid = 0; vcpuid < guest_ncpus; vcpuid++) { vcpu_info[vcpuid].ctx = ctx; vcpu_info[vcpuid].vcpuid = vcpuid; if (vcpuid == BSP) vcpu_info[vcpuid].vcpu = bsp; else vcpu_info[vcpuid].vcpu = vm_vcpu_open(ctx, vcpuid); } memflags = 0; if (get_config_bool_default("memory.wired", false)) memflags |= VM_MEM_F_WIRED; if (get_config_bool_default("memory.guest_in_core", false)) memflags |= VM_MEM_F_INCORE; vm_set_memflags(ctx, memflags); #ifdef __FreeBSD__ error = vm_setup_memory(ctx, memsize, VM_MMAP_ALL); #else int _errno; do { errno = 0; error = vm_setup_memory(ctx, memsize, VM_MMAP_ALL); _errno = errno; if (error != 0 && _errno == ENOMEM) { (void) fprintf(stderr, "Unable to allocate memory " "(%llu), retrying in 1 second\n", memsize); sleep(1); } } while (_errno == ENOMEM); #endif if (error) { fprintf(stderr, "Unable to set up memory (%d)\n", errno); exit(4); } init_mem(guest_ncpus); init_bootrom(ctx); if (bhyve_init_platform(ctx, bsp) != 0) exit(4); if (qemu_fwcfg_init(ctx) != 0) { fprintf(stderr, "qemu fwcfg initialization error\n"); exit(4); } if (qemu_fwcfg_add_file("opt/bhyve/hw.ncpu", sizeof(guest_ncpus), &guest_ncpus) != 0) { fprintf(stderr, "Could not add qemu fwcfg opt/bhyve/hw.ncpu\n"); exit(4); } /* * Exit if a device emulation finds an error in its initialization */ if (init_pci(ctx) != 0) { EPRINTLN("Device emulation initialization error: %s", strerror(errno)); exit(4); } if (init_tpm(ctx) != 0) { EPRINTLN("Failed to init TPM device"); exit(4); } /* * Initialize after PCI, to allow a bootrom file to reserve the high * region. */ if (get_config_bool("acpi_tables")) vmgenc_init(ctx); #ifdef __FreeBSD__ init_gdb(ctx); #else if (value != NULL) { int port = atoi(value); if (port < 0) init_mdb(ctx); else init_gdb(ctx); } #endif if (bootrom_boot()) { #ifdef __FreeBSD__ if (vm_set_capability(bsp, VM_CAP_UNRESTRICTED_GUEST, 1)) { fprintf(stderr, "ROM boot failed: unrestricted guest " "capability not available\n"); exit(4); } #else /* Unrestricted Guest is always enabled on illumos */ #endif error = vcpu_reset(bsp); assert(error == 0); } if (bhyve_init_platform_late(ctx, bsp) != 0) exit(4); /* * Change the proc title to include the VM name. */ setproctitle("%s", vmname); #ifndef WITHOUT_CAPSICUM caph_cache_catpages(); if (caph_limit_stdout() == -1 || caph_limit_stderr() == -1) errx(EX_OSERR, "Unable to apply rights for sandbox"); if (caph_enter() == -1) errx(EX_OSERR, "cap_enter() failed"); #endif #ifndef __FreeBSD__ illumos_priv_lock(); #endif #ifndef __FreeBSD__ if (vmentry_init(guest_ncpus) != 0) err(EX_OSERR, "vmentry_init() failed"); #endif /* * Add all vCPUs. */ for (int vcpuid = 0; vcpuid < guest_ncpus; vcpuid++) { #ifdef __FreeBSD__ bool suspend = (vcpuid != BSP); #else bool suspend = vcpuid == BSP && get_config_bool_default("suspend_at_boot", false); #endif bhyve_start_vcpu(vcpu_info[vcpuid].vcpu, vcpuid == BSP, suspend); } /* * Head off to the main event dispatch loop */ mevent_dispatch(); exit(4); } /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2011 NetApp, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * This file and its contents are supplied under the terms of the * Common Development and Distribution License ("CDDL"), version 1.0. * You may only use this file in accordance with the terms of version * 1.0 of the CDDL. * * A full copy of the text of the CDDL should have accompanied this * source. A copy of the CDDL is also available via the Internet at * http://www.illumos.org/license/CDDL. * * Copyright 2015 Pluribus Networks Inc. * Copyright 2025 Oxide Computer Company */ #ifndef _BHYVERUN_H_ #define _BHYVERUN_H_ #define VMEXIT_CONTINUE (0) #define VMEXIT_ABORT (-1) #include extern int guest_ncpus; extern uint16_t cpu_cores, cpu_sockets, cpu_threads; struct vcpu; struct vmctx; struct vm_exit; extern void *paddr_guest2host(struct vmctx *ctx, uintptr_t addr, size_t len); struct vcpu *fbsdrun_vcpu(int vcpuid); void fbsdrun_addcpu(int vcpuid, bool); void fbsdrun_deletecpu(int vcpuid); bool fbsdrun_virtio_msix(void); typedef int (*vmexit_handler_t)(struct vmctx *, struct vcpu *, struct vm_exit *); extern int vmexit_task_switch(struct vmctx *, struct vcpu *, struct vm_exit *); /* Interfaces implemented by machine-dependent code. */ void bhyve_init_config(void); void bhyve_init_vcpu(struct vcpu *vcpu); void bhyve_start_vcpu(struct vcpu *vcpu, bool bsp, bool suspend); int bhyve_init_platform(struct vmctx *ctx, struct vcpu *bsp); int bhyve_init_platform_late(struct vmctx *ctx, struct vcpu *bsp); void bhyve_optparse(int argc, char **argv); void bhyve_usage(int code); /* Interfaces used by command-line option-parsing code. */ bool bhyve_parse_config_option(const char *option); void bhyve_parse_simple_config_file(const char *path); void bhyve_parse_gdb_options(const char *opt); #ifdef __FreeBSD__ int bhyve_pincpu_parse(const char *opt); #endif int bhyve_topology_parse(const char *opt); #endif /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2013 Peter Grehan * All rights reserved. * Copyright 2020 Joyent, Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * Copyright 2020 Joyent, Inc. */ #include #ifndef WITHOUT_CAPSICUM #include #endif #include #include #include #include #include #ifndef __FreeBSD__ #include #include #include #endif #include #ifndef WITHOUT_CAPSICUM #include #endif #include #include #include #include #include #include #include #include #include #include #include #include "bhyverun.h" #include "config.h" #include "debug.h" #include "mevent.h" #include "pci_emul.h" #include "block_if.h" #define BLOCKIF_SIG 0xb109b109 #ifdef __FreeBSD__ #define BLOCKIF_NUMTHR 8 #else /* Enlarge to keep pace with the virtio-block ring size */ #define BLOCKIF_NUMTHR 16 #endif #define BLOCKIF_MAXREQ (BLOCKIF_RING_MAX + BLOCKIF_NUMTHR) enum blockop { BOP_READ, BOP_WRITE, #ifndef __FreeBSD__ BOP_WRITE_SYNC, #endif BOP_FLUSH, BOP_DELETE }; enum blockstat { BST_FREE, BST_BLOCK, BST_PEND, BST_BUSY, BST_DONE }; struct blockif_elem { TAILQ_ENTRY(blockif_elem) be_link; struct blockif_req *be_req; enum blockop be_op; enum blockstat be_status; pthread_t be_tid; off_t be_block; }; #ifndef __FreeBSD__ enum blockif_wce { WCE_NONE = 0, WCE_IOCTL, WCE_FCNTL }; #endif struct blockif_ctxt { unsigned int bc_magic; int bc_fd; int bc_ischr; int bc_isgeom; int bc_candelete; #ifndef __FreeBSD__ enum blockif_wce bc_wce; #endif int bc_rdonly; off_t bc_size; int bc_sectsz; int bc_psectsz; int bc_psectoff; int bc_closing; pthread_t bc_btid[BLOCKIF_NUMTHR]; pthread_mutex_t bc_mtx; pthread_cond_t bc_cond; blockif_resize_cb *bc_resize_cb; void *bc_resize_cb_arg; struct mevent *bc_resize_event; /* Request elements and free/pending/busy queues */ TAILQ_HEAD(, blockif_elem) bc_freeq; TAILQ_HEAD(, blockif_elem) bc_pendq; TAILQ_HEAD(, blockif_elem) bc_busyq; struct blockif_elem bc_reqs[BLOCKIF_MAXREQ]; int bc_bootindex; }; static pthread_once_t blockif_once = PTHREAD_ONCE_INIT; struct blockif_sig_elem { pthread_mutex_t bse_mtx; pthread_cond_t bse_cond; int bse_pending; struct blockif_sig_elem *bse_next; }; static struct blockif_sig_elem *blockif_bse_head; static int blockif_enqueue(struct blockif_ctxt *bc, struct blockif_req *breq, enum blockop op) { struct blockif_elem *be, *tbe; off_t off; int i; be = TAILQ_FIRST(&bc->bc_freeq); assert(be != NULL); assert(be->be_status == BST_FREE); TAILQ_REMOVE(&bc->bc_freeq, be, be_link); be->be_req = breq; be->be_op = op; switch (op) { case BOP_READ: case BOP_WRITE: #ifndef __FreeBSD__ case BOP_WRITE_SYNC: #endif case BOP_DELETE: off = breq->br_offset; for (i = 0; i < breq->br_iovcnt; i++) off += breq->br_iov[i].iov_len; break; default: off = OFF_MAX; } be->be_block = off; TAILQ_FOREACH(tbe, &bc->bc_pendq, be_link) { if (tbe->be_block == breq->br_offset) break; } if (tbe == NULL) { TAILQ_FOREACH(tbe, &bc->bc_busyq, be_link) { if (tbe->be_block == breq->br_offset) break; } } if (tbe == NULL) be->be_status = BST_PEND; else be->be_status = BST_BLOCK; TAILQ_INSERT_TAIL(&bc->bc_pendq, be, be_link); return (be->be_status == BST_PEND); } static int blockif_dequeue(struct blockif_ctxt *bc, pthread_t t, struct blockif_elem **bep) { struct blockif_elem *be; TAILQ_FOREACH(be, &bc->bc_pendq, be_link) { if (be->be_status == BST_PEND) break; assert(be->be_status == BST_BLOCK); } if (be == NULL) return (0); TAILQ_REMOVE(&bc->bc_pendq, be, be_link); be->be_status = BST_BUSY; be->be_tid = t; TAILQ_INSERT_TAIL(&bc->bc_busyq, be, be_link); *bep = be; return (1); } static void blockif_complete(struct blockif_ctxt *bc, struct blockif_elem *be) { struct blockif_elem *tbe; if (be->be_status == BST_DONE || be->be_status == BST_BUSY) TAILQ_REMOVE(&bc->bc_busyq, be, be_link); else TAILQ_REMOVE(&bc->bc_pendq, be, be_link); TAILQ_FOREACH(tbe, &bc->bc_pendq, be_link) { if (tbe->be_req->br_offset == be->be_block) tbe->be_status = BST_PEND; } be->be_tid = 0; be->be_status = BST_FREE; be->be_req = NULL; TAILQ_INSERT_TAIL(&bc->bc_freeq, be, be_link); } static int blockif_flush_bc(struct blockif_ctxt *bc) { #ifdef __FreeBSD__ if (bc->bc_ischr) { if (ioctl(bc->bc_fd, DIOCGFLUSH)) return (errno); } else if (fsync(bc->bc_fd)) return (errno); #else /* * This fsync() should be adequate to flush the cache of a file * or device. In VFS, the VOP_SYNC operation is converted to * the appropriate ioctl in both sdev (for real devices) and * zfs (for zvols). */ if (fsync(bc->bc_fd)) return (errno); #endif return (0); } static void blockif_proc(struct blockif_ctxt *bc, struct blockif_elem *be, uint8_t *buf) { #ifdef __FreeBSD__ struct spacectl_range range; #endif struct blockif_req *br; #ifdef __FreeBSD__ off_t arg[2]; #endif ssize_t n; size_t clen, len, off, boff, voff; int i, err; br = be->be_req; assert(br->br_resid >= 0); if (br->br_iovcnt <= 1) buf = NULL; err = 0; switch (be->be_op) { case BOP_READ: if (buf == NULL) { if ((n = preadv(bc->bc_fd, br->br_iov, br->br_iovcnt, br->br_offset)) < 0) err = errno; else br->br_resid -= n; break; } i = 0; off = voff = 0; while (br->br_resid > 0) { len = MIN(br->br_resid, MAXPHYS); n = pread(bc->bc_fd, buf, len, br->br_offset + off); if (n < 0) { err = errno; break; } len = (size_t)n; boff = 0; do { clen = MIN(len - boff, br->br_iov[i].iov_len - voff); memcpy((uint8_t *)br->br_iov[i].iov_base + voff, buf + boff, clen); if (clen < br->br_iov[i].iov_len - voff) voff += clen; else { i++; voff = 0; } boff += clen; } while (boff < len); off += len; br->br_resid -= len; } break; case BOP_WRITE: if (bc->bc_rdonly) { err = EROFS; break; } if (buf == NULL) { if ((n = pwritev(bc->bc_fd, br->br_iov, br->br_iovcnt, br->br_offset)) < 0) err = errno; else br->br_resid -= n; break; } i = 0; off = voff = 0; while (br->br_resid > 0) { len = MIN(br->br_resid, MAXPHYS); boff = 0; do { clen = MIN(len - boff, br->br_iov[i].iov_len - voff); memcpy(buf + boff, (uint8_t *)br->br_iov[i].iov_base + voff, clen); if (clen < br->br_iov[i].iov_len - voff) voff += clen; else { i++; voff = 0; } boff += clen; } while (boff < len); n = pwrite(bc->bc_fd, buf, len, br->br_offset + off); if (n < 0) { err = errno; break; } off += n; br->br_resid -= n; } break; case BOP_FLUSH: err = blockif_flush_bc(bc); break; case BOP_DELETE: if (!bc->bc_candelete) err = EOPNOTSUPP; else if (bc->bc_rdonly) err = EROFS; #ifdef __FreeBSD__ else if (bc->bc_ischr) { arg[0] = br->br_offset; arg[1] = br->br_resid; if (ioctl(bc->bc_fd, DIOCGDELETE, arg)) err = errno; else br->br_resid = 0; } else { range.r_offset = br->br_offset; range.r_len = br->br_resid; while (range.r_len > 0) { if (fspacectl(bc->bc_fd, SPACECTL_DEALLOC, &range, 0, &range) != 0) { err = errno; break; } } if (err == 0) br->br_resid = 0; } #else else if (bc->bc_ischr) { dkioc_free_list_t dfl = { .dfl_num_exts = 1, .dfl_offset = 0, .dfl_flags = 0, .dfl_exts = { { .dfle_start = br->br_offset, .dfle_length = br->br_resid } } }; if (ioctl(bc->bc_fd, DKIOCFREE, &dfl)) err = errno; else br->br_resid = 0; } else { struct flock fl = { .l_whence = 0, .l_type = F_WRLCK, .l_start = br->br_offset, .l_len = br->br_resid }; if (fcntl(bc->bc_fd, F_FREESP, &fl)) err = errno; else br->br_resid = 0; } #endif break; default: err = EINVAL; break; } be->be_status = BST_DONE; (*br->br_callback)(br, err); } static inline bool blockif_empty(const struct blockif_ctxt *bc) { return (TAILQ_EMPTY(&bc->bc_pendq) && TAILQ_EMPTY(&bc->bc_busyq)); } static void * blockif_thr(void *arg) { struct blockif_ctxt *bc; struct blockif_elem *be; pthread_t t; uint8_t *buf; bc = arg; if (bc->bc_isgeom) buf = malloc(MAXPHYS); else buf = NULL; t = pthread_self(); pthread_mutex_lock(&bc->bc_mtx); for (;;) { while (blockif_dequeue(bc, t, &be)) { pthread_mutex_unlock(&bc->bc_mtx); blockif_proc(bc, be, buf); pthread_mutex_lock(&bc->bc_mtx); blockif_complete(bc, be); } /* Check ctxt status here to see if exit requested */ if (bc->bc_closing) break; pthread_cond_wait(&bc->bc_cond, &bc->bc_mtx); } pthread_mutex_unlock(&bc->bc_mtx); if (buf) free(buf); pthread_exit(NULL); return (NULL); } #ifdef __FreeBSD__ static void blockif_sigcont_handler(int signal __unused, enum ev_type type __unused, void *arg __unused) #else static void blockif_sigcont_handler(int signal __unused) #endif { struct blockif_sig_elem *bse; for (;;) { /* * Process the entire list even if not intended for * this thread. */ do { bse = blockif_bse_head; if (bse == NULL) return; } while (!atomic_cmpset_ptr((uintptr_t *)&blockif_bse_head, (uintptr_t)bse, (uintptr_t)bse->bse_next)); pthread_mutex_lock(&bse->bse_mtx); bse->bse_pending = 0; pthread_cond_signal(&bse->bse_cond); pthread_mutex_unlock(&bse->bse_mtx); } } static void blockif_init(void) { #ifdef __FreeBSD__ mevent_add(SIGCONT, EVF_SIGNAL, blockif_sigcont_handler, NULL); (void) signal(SIGCONT, SIG_IGN); #else (void) sigset(SIGCONT, blockif_sigcont_handler); #endif } int blockif_legacy_config(nvlist_t *nvl, const char *opts) { char *cp, *path; if (opts == NULL) return (0); cp = strchr(opts, ','); if (cp == NULL) { set_config_value_node(nvl, "path", opts); return (0); } path = strndup(opts, cp - opts); set_config_value_node(nvl, "path", path); free(path); return (pci_parse_legacy_config(nvl, cp + 1)); } int blockif_add_boot_device(struct pci_devinst *const pi, struct blockif_ctxt *const bc) { if (bc->bc_bootindex < 0) return (0); return (pci_emul_add_boot_device(pi, bc->bc_bootindex)); } struct blockif_ctxt * blockif_open(nvlist_t *nvl, const char *ident) { char tname[MAXCOMLEN + 1]; #ifdef __FreeBSD__ char name[MAXPATHLEN]; #endif const char *path, *pssval, *ssval, *bootindex_val; char *cp; struct blockif_ctxt *bc; struct stat sbuf; #ifdef __FreeBSD__ struct diocgattr_arg arg; #else enum blockif_wce wce = WCE_NONE; #endif off_t size, psectsz, psectoff; int extra, fd, i, sectsz; int ro, candelete, geom, ssopt, pssopt; int nodelete; int bootindex; #ifndef WITHOUT_CAPSICUM cap_rights_t rights; cap_ioctl_t cmds[] = { DIOCGFLUSH, DIOCGDELETE, DIOCGMEDIASIZE }; #endif pthread_once(&blockif_once, blockif_init); fd = -1; extra = 0; ssopt = 0; #ifndef __FreeBSD__ pssopt = 0; #endif ro = 0; nodelete = 0; bootindex = -1; if (get_config_bool_node_default(nvl, "nocache", false)) extra |= O_DIRECT; if (get_config_bool_node_default(nvl, "nodelete", false)) nodelete = 1; if (get_config_bool_node_default(nvl, "sync", false) || get_config_bool_node_default(nvl, "direct", false)) extra |= O_SYNC; if (get_config_bool_node_default(nvl, "ro", false)) ro = 1; ssval = get_config_value_node(nvl, "sectorsize"); if (ssval != NULL) { ssopt = strtol(ssval, &cp, 10); if (cp == ssval) { EPRINTLN("Invalid sector size \"%s\"", ssval); goto err; } if (*cp == '\0') { pssopt = ssopt; } else if (*cp == '/') { pssval = cp + 1; pssopt = strtol(pssval, &cp, 10); if (cp == pssval || *cp != '\0') { EPRINTLN("Invalid sector size \"%s\"", ssval); goto err; } } else { EPRINTLN("Invalid sector size \"%s\"", ssval); goto err; } } bootindex_val = get_config_value_node(nvl, "bootindex"); if (bootindex_val != NULL) { bootindex = atoi(bootindex_val); } path = get_config_value_node(nvl, "path"); if (path == NULL) { EPRINTLN("Missing \"path\" for block device."); goto err; } fd = open(path, (ro ? O_RDONLY : O_RDWR) | extra); if (fd < 0 && !ro) { /* Attempt a r/w fail with a r/o open */ fd = open(path, O_RDONLY | extra); ro = 1; } if (fd < 0) { warn("Could not open backing file: %s", path); goto err; } if (fstat(fd, &sbuf) < 0) { warn("Could not stat backing file %s", path); goto err; } #ifndef WITHOUT_CAPSICUM cap_rights_init(&rights, CAP_FSYNC, CAP_IOCTL, CAP_READ, CAP_SEEK, CAP_WRITE, CAP_FSTAT, CAP_EVENT, CAP_FPATHCONF); if (ro) cap_rights_clear(&rights, CAP_FSYNC, CAP_WRITE); if (caph_rights_limit(fd, &rights) == -1) errx(EX_OSERR, "Unable to apply rights for sandbox"); #endif /* * Deal with raw devices */ size = sbuf.st_size; sectsz = DEV_BSIZE; psectsz = psectoff = 0; candelete = geom = 0; #ifdef __FreeBSD__ if (S_ISCHR(sbuf.st_mode)) { if (ioctl(fd, DIOCGMEDIASIZE, &size) < 0 || ioctl(fd, DIOCGSECTORSIZE, §sz)) { perror("Could not fetch dev blk/sector size"); goto err; } assert(size != 0); assert(sectsz != 0); if (ioctl(fd, DIOCGSTRIPESIZE, &psectsz) == 0 && psectsz > 0) ioctl(fd, DIOCGSTRIPEOFFSET, &psectoff); strlcpy(arg.name, "GEOM::candelete", sizeof(arg.name)); arg.len = sizeof(arg.value.i); if (nodelete == 0 && ioctl(fd, DIOCGATTR, &arg) == 0) candelete = arg.value.i; if (ioctl(fd, DIOCGPROVIDERNAME, name) == 0) geom = 1; } else { psectsz = sbuf.st_blksize; /* Avoid fallback implementation */ candelete = fpathconf(fd, _PC_DEALLOC_PRESENT) == 1; } #else psectsz = sbuf.st_blksize; if (S_ISCHR(sbuf.st_mode)) { struct dk_minfo_ext dkmext; int wce_val; /* Look for a more accurate physical block/media size */ if (ioctl(fd, DKIOCGMEDIAINFOEXT, &dkmext) == 0) { psectsz = dkmext.dki_pbsize; size = dkmext.dki_lbsize * dkmext.dki_capacity; } /* See if a configurable write cache is present and working */ if (ioctl(fd, DKIOCGETWCE, &wce_val) == 0) { /* * If WCE is already active, disable it until the * specific device driver calls for its return. If it * is not active, toggle it on and off to verify that * such actions are possible. */ if (wce_val != 0) { wce_val = 0; /* * Inability to disable the cache is a threat * to data durability. */ assert(ioctl(fd, DKIOCSETWCE, &wce_val) == 0); wce = WCE_IOCTL; } else { int r1, r2; wce_val = 1; r1 = ioctl(fd, DKIOCSETWCE, &wce_val); wce_val = 0; r2 = ioctl(fd, DKIOCSETWCE, &wce_val); if (r1 == 0 && r2 == 0) { wce = WCE_IOCTL; } else { /* * If the cache cache toggle was not * successful, ensure that the cache * was not left enabled. */ assert(r1 != 0); } } } if (nodelete == 0 && ioctl(fd, DKIOC_CANFREE, &candelete)) candelete = 0; } else { int flags; if ((flags = fcntl(fd, F_GETFL)) >= 0) { flags |= O_DSYNC; if (fcntl(fd, F_SETFL, flags) != -1) { wce = WCE_FCNTL; } } /* * We don't have a way to discover if a file supports the * FREESP fcntl cmd (other than trying it). However, * zfs, ufs, tmpfs, and udfs all support the FREESP fcntl cmd. * Nfsv4 and nfsv4 also forward the FREESP request * to the server, so we always enable it for file based * volumes. Anyone trying to run volumes on an unsupported * configuration is on their own, and should be prepared * for the requests to fail. */ if (nodelete == 0) candelete = 1; } #endif #ifndef WITHOUT_CAPSICUM if (caph_ioctls_limit(fd, cmds, nitems(cmds)) == -1) errx(EX_OSERR, "Unable to apply rights for sandbox"); #endif if (ssopt != 0) { if (!powerof2(ssopt) || !powerof2(pssopt) || ssopt < 512 || ssopt > pssopt) { EPRINTLN("Invalid sector size %d/%d", ssopt, pssopt); goto err; } /* * Some backend drivers (e.g. cd0, ada0) require that the I/O * size be a multiple of the device's sector size. * * Validate that the emulated sector size complies with this * requirement. */ if (S_ISCHR(sbuf.st_mode)) { if (ssopt < sectsz || (ssopt % sectsz) != 0) { EPRINTLN("Sector size %d incompatible " "with underlying device sector size %d", ssopt, sectsz); goto err; } } sectsz = ssopt; psectsz = pssopt; psectoff = 0; } bc = calloc(1, sizeof(struct blockif_ctxt)); if (bc == NULL) { perror("calloc"); goto err; } bc->bc_magic = BLOCKIF_SIG; bc->bc_fd = fd; bc->bc_ischr = S_ISCHR(sbuf.st_mode); bc->bc_isgeom = geom; bc->bc_candelete = candelete; #ifndef __FreeBSD__ bc->bc_wce = wce; #endif bc->bc_rdonly = ro; bc->bc_size = size; bc->bc_sectsz = sectsz; bc->bc_psectsz = psectsz; bc->bc_psectoff = psectoff; pthread_mutex_init(&bc->bc_mtx, NULL); pthread_cond_init(&bc->bc_cond, NULL); TAILQ_INIT(&bc->bc_freeq); TAILQ_INIT(&bc->bc_pendq); TAILQ_INIT(&bc->bc_busyq); bc->bc_bootindex = bootindex; for (i = 0; i < BLOCKIF_MAXREQ; i++) { bc->bc_reqs[i].be_status = BST_FREE; TAILQ_INSERT_HEAD(&bc->bc_freeq, &bc->bc_reqs[i], be_link); } for (i = 0; i < BLOCKIF_NUMTHR; i++) { pthread_create(&bc->bc_btid[i], NULL, blockif_thr, bc); snprintf(tname, sizeof(tname), "blk-%s-%d", ident, i); pthread_set_name_np(bc->bc_btid[i], tname); } return (bc); err: if (fd >= 0) close(fd); return (NULL); } static void blockif_resized(int fd, enum ev_type type __unused, void *arg) { struct blockif_ctxt *bc; struct stat sb; off_t mediasize; if (fstat(fd, &sb) != 0) return; #ifdef __FreeBSD__ if (S_ISCHR(sb.st_mode)) { if (ioctl(fd, DIOCGMEDIASIZE, &mediasize) < 0) { EPRINTLN("blockif_resized: get mediasize failed: %s", strerror(errno)); return; } } else mediasize = sb.st_size; #else mediasize = sb.st_size; if (S_ISCHR(sb.st_mode)) { struct dk_minfo dkm; if (ioctl(fd, DKIOCGMEDIAINFO, &dkm) == 0) mediasize = dkm.dki_lbsize * dkm.dki_capacity; } #endif bc = arg; pthread_mutex_lock(&bc->bc_mtx); if (mediasize != bc->bc_size) { bc->bc_size = mediasize; bc->bc_resize_cb(bc, bc->bc_resize_cb_arg, bc->bc_size); } pthread_mutex_unlock(&bc->bc_mtx); } int blockif_register_resize_callback(struct blockif_ctxt *bc, blockif_resize_cb *cb, void *cb_arg) { struct stat sb; int err; if (cb == NULL) return (EINVAL); err = 0; pthread_mutex_lock(&bc->bc_mtx); if (bc->bc_resize_cb != NULL) { err = EBUSY; goto out; } assert(bc->bc_closing == 0); if (fstat(bc->bc_fd, &sb) != 0) { err = errno; goto out; } bc->bc_resize_event = mevent_add_flags(bc->bc_fd, EVF_VNODE, EVFF_ATTRIB, blockif_resized, bc); if (bc->bc_resize_event == NULL) { err = ENXIO; goto out; } bc->bc_resize_cb = cb; bc->bc_resize_cb_arg = cb_arg; out: pthread_mutex_unlock(&bc->bc_mtx); return (err); } static int blockif_request(struct blockif_ctxt *bc, struct blockif_req *breq, enum blockop op) { int err; err = 0; pthread_mutex_lock(&bc->bc_mtx); if (!TAILQ_EMPTY(&bc->bc_freeq)) { /* * Enqueue and inform the block i/o thread * that there is work available */ if (blockif_enqueue(bc, breq, op)) pthread_cond_signal(&bc->bc_cond); } else { /* * Callers are not allowed to enqueue more than * the specified blockif queue limit. Return an * error to indicate that the queue length has been * exceeded. */ err = E2BIG; } pthread_mutex_unlock(&bc->bc_mtx); return (err); } int blockif_read(struct blockif_ctxt *bc, struct blockif_req *breq) { assert(bc->bc_magic == BLOCKIF_SIG); return (blockif_request(bc, breq, BOP_READ)); } int blockif_write(struct blockif_ctxt *bc, struct blockif_req *breq) { assert(bc->bc_magic == BLOCKIF_SIG); return (blockif_request(bc, breq, BOP_WRITE)); } int blockif_flush(struct blockif_ctxt *bc, struct blockif_req *breq) { assert(bc->bc_magic == BLOCKIF_SIG); return (blockif_request(bc, breq, BOP_FLUSH)); } int blockif_delete(struct blockif_ctxt *bc, struct blockif_req *breq) { assert(bc->bc_magic == BLOCKIF_SIG); return (blockif_request(bc, breq, BOP_DELETE)); } int blockif_cancel(struct blockif_ctxt *bc, struct blockif_req *breq) { struct blockif_elem *be; assert(bc->bc_magic == BLOCKIF_SIG); pthread_mutex_lock(&bc->bc_mtx); /* * Check pending requests. */ TAILQ_FOREACH(be, &bc->bc_pendq, be_link) { if (be->be_req == breq) break; } if (be != NULL) { /* * Found it. */ blockif_complete(bc, be); pthread_mutex_unlock(&bc->bc_mtx); return (0); } /* * Check in-flight requests. */ TAILQ_FOREACH(be, &bc->bc_busyq, be_link) { if (be->be_req == breq) break; } if (be == NULL) { /* * Didn't find it. */ pthread_mutex_unlock(&bc->bc_mtx); return (EINVAL); } /* * Interrupt the processing thread to force it return * prematurely via it's normal callback path. */ while (be->be_status == BST_BUSY) { struct blockif_sig_elem bse, *old_head; pthread_mutex_init(&bse.bse_mtx, NULL); pthread_cond_init(&bse.bse_cond, NULL); bse.bse_pending = 1; do { old_head = blockif_bse_head; bse.bse_next = old_head; } while (!atomic_cmpset_ptr((uintptr_t *)&blockif_bse_head, (uintptr_t)old_head, (uintptr_t)&bse)); pthread_kill(be->be_tid, SIGCONT); pthread_mutex_lock(&bse.bse_mtx); while (bse.bse_pending) pthread_cond_wait(&bse.bse_cond, &bse.bse_mtx); pthread_mutex_unlock(&bse.bse_mtx); } pthread_mutex_unlock(&bc->bc_mtx); /* * The processing thread has been interrupted. Since it's not * clear if the callback has been invoked yet, return EBUSY. */ return (EBUSY); } int blockif_close(struct blockif_ctxt *bc) { void *jval; int i; assert(bc->bc_magic == BLOCKIF_SIG); /* * Stop the block i/o thread */ pthread_mutex_lock(&bc->bc_mtx); bc->bc_closing = 1; if (bc->bc_resize_event != NULL) mevent_disable(bc->bc_resize_event); pthread_mutex_unlock(&bc->bc_mtx); pthread_cond_broadcast(&bc->bc_cond); for (i = 0; i < BLOCKIF_NUMTHR; i++) pthread_join(bc->bc_btid[i], &jval); /* XXX Cancel queued i/o's ??? */ /* * Release resources */ bc->bc_magic = 0; close(bc->bc_fd); free(bc); return (0); } /* * Return virtual C/H/S values for a given block. Use the algorithm * outlined in the VHD specification to calculate values. */ void blockif_chs(struct blockif_ctxt *bc, uint16_t *c, uint8_t *h, uint8_t *s) { off_t sectors; /* total sectors of the block dev */ off_t hcyl; /* cylinders times heads */ uint16_t secpt; /* sectors per track */ uint8_t heads; assert(bc->bc_magic == BLOCKIF_SIG); sectors = bc->bc_size / bc->bc_sectsz; /* Clamp the size to the largest possible with CHS */ if (sectors > 65535L * 16 * 255) sectors = 65535L * 16 * 255; if (sectors >= 65536L * 16 * 63) { secpt = 255; heads = 16; hcyl = sectors / secpt; } else { secpt = 17; hcyl = sectors / secpt; heads = (hcyl + 1023) / 1024; if (heads < 4) heads = 4; if (hcyl >= (heads * 1024) || heads > 16) { secpt = 31; heads = 16; hcyl = sectors / secpt; } if (hcyl >= (heads * 1024)) { secpt = 63; heads = 16; hcyl = sectors / secpt; } } *c = hcyl / heads; *h = heads; *s = secpt; } /* * Accessors */ off_t blockif_size(struct blockif_ctxt *bc) { assert(bc->bc_magic == BLOCKIF_SIG); return (bc->bc_size); } int blockif_sectsz(struct blockif_ctxt *bc) { assert(bc->bc_magic == BLOCKIF_SIG); return (bc->bc_sectsz); } void blockif_psectsz(struct blockif_ctxt *bc, int *size, int *off) { assert(bc->bc_magic == BLOCKIF_SIG); *size = bc->bc_psectsz; *off = bc->bc_psectoff; } int blockif_queuesz(struct blockif_ctxt *bc) { assert(bc->bc_magic == BLOCKIF_SIG); return (BLOCKIF_MAXREQ - 1); } int blockif_is_ro(struct blockif_ctxt *bc) { assert(bc->bc_magic == BLOCKIF_SIG); return (bc->bc_rdonly); } int blockif_candelete(struct blockif_ctxt *bc) { assert(bc->bc_magic == BLOCKIF_SIG); return (bc->bc_candelete); } #ifndef __FreeBSD__ int blockif_set_wce(struct blockif_ctxt *bc, int wc_enable) { int res = 0, flags; int clean_val = (wc_enable != 0) ? 1 : 0; (void) pthread_mutex_lock(&bc->bc_mtx); switch (bc->bc_wce) { case WCE_IOCTL: res = ioctl(bc->bc_fd, DKIOCSETWCE, &clean_val); break; case WCE_FCNTL: if ((flags = fcntl(bc->bc_fd, F_GETFL)) >= 0) { if (wc_enable == 0) { flags |= O_DSYNC; } else { flags &= ~O_DSYNC; } if (fcntl(bc->bc_fd, F_SETFL, flags) == -1) { res = -1; } } else { res = -1; } break; default: break; } /* * After a successful disable of the write cache, ensure that any * lingering data in the cache is synced out. */ if (res == 0 && wc_enable == 0) { res = fsync(bc->bc_fd); } (void) pthread_mutex_unlock(&bc->bc_mtx); return (res); } #endif /* __FreeBSD__ */ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2013 Peter Grehan * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * The block API to be used by bhyve block-device emulations. The routines * are thread safe, with no assumptions about the context of the completion * callback - it may occur in the caller's context, or asynchronously in * another thread. */ #ifndef _BLOCK_IF_H_ #define _BLOCK_IF_H_ #include #include #include /* * BLOCKIF_IOV_MAX is the maximum number of scatter/gather entries in * a single request. BLOCKIF_RING_MAX is the maxmimum number of * pending requests that can be queued. */ #define BLOCKIF_IOV_MAX 128 /* not practical to be IOV_MAX */ #define BLOCKIF_RING_MAX 128 struct blockif_req { int br_iovcnt; off_t br_offset; ssize_t br_resid; void (*br_callback)(struct blockif_req *req, int err); void *br_param; struct iovec br_iov[BLOCKIF_IOV_MAX]; }; struct pci_devinst; struct blockif_ctxt; typedef void blockif_resize_cb(struct blockif_ctxt *, void *, size_t); int blockif_legacy_config(nvlist_t *nvl, const char *opts); int blockif_add_boot_device(struct pci_devinst *const pi, struct blockif_ctxt *const bc); struct blockif_ctxt *blockif_open(nvlist_t *nvl, const char *ident); int blockif_register_resize_callback(struct blockif_ctxt *bc, blockif_resize_cb *cb, void *cb_arg); off_t blockif_size(struct blockif_ctxt *bc); void blockif_chs(struct blockif_ctxt *bc, uint16_t *c, uint8_t *h, uint8_t *s); int blockif_sectsz(struct blockif_ctxt *bc); void blockif_psectsz(struct blockif_ctxt *bc, int *size, int *off); int blockif_queuesz(struct blockif_ctxt *bc); int blockif_is_ro(struct blockif_ctxt *bc); int blockif_candelete(struct blockif_ctxt *bc); #ifndef __FreeBSD__ int blockif_set_wce(struct blockif_ctxt *bc, int enable); #endif int blockif_read(struct blockif_ctxt *bc, struct blockif_req *breq); int blockif_write(struct blockif_ctxt *bc, struct blockif_req *breq); int blockif_flush(struct blockif_ctxt *bc, struct blockif_req *breq); int blockif_delete(struct blockif_ctxt *bc, struct blockif_req *breq); int blockif_cancel(struct blockif_ctxt *bc, struct blockif_req *breq); int blockif_close(struct blockif_ctxt *bc); #endif /* _BLOCK_IF_H_ */ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2015 Neel Natu * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "bhyverun.h" #include "bootrom.h" #include "debug.h" #include "mem.h" #define BOOTROM_SIZE (16 * 1024 * 1024) /* 16 MB */ /* * ROM region is 16 MB at the top of 4GB ("low") memory. * * The size is limited so it doesn't encroach into reserved MMIO space (e.g., * APIC, HPET, MSI). * * It is allocated in page-multiple blocks on a first-come first-serve basis, * from high to low, during initialization, and does not change at runtime. */ static char *romptr; /* Pointer to userspace-mapped bootrom region. */ static vm_paddr_t gpa_base; /* GPA of low end of region. */ static vm_paddr_t gpa_allocbot; /* Low GPA of free region. */ static vm_paddr_t gpa_alloctop; /* High GPA, minus 1, of free region. */ #define CFI_BCS_WRITE_BYTE 0x10 #define CFI_BCS_CLEAR_STATUS 0x50 #define CFI_BCS_READ_STATUS 0x70 #define CFI_BCS_READ_ARRAY 0xff static struct bootrom_var_state { uint8_t *mmap; uint64_t gpa; off_t size; uint8_t cmd; } var = { NULL, 0, 0, CFI_BCS_READ_ARRAY }; /* * Emulate just those CFI basic commands that will convince EDK II * that the Firmware Volume area is writable and persistent. */ static int bootrom_var_mem_handler(struct vcpu *vcpu __unused, int dir, uint64_t addr, int size, uint64_t *val, void *arg1 __unused, long arg2 __unused) { off_t offset; offset = addr - var.gpa; if (offset + size > var.size || offset < 0 || offset + size <= offset) return (EINVAL); if (dir == MEM_F_WRITE) { switch (var.cmd) { case CFI_BCS_WRITE_BYTE: memcpy(var.mmap + offset, val, size); var.cmd = CFI_BCS_READ_ARRAY; break; default: var.cmd = *(uint8_t *)val; } } else { switch (var.cmd) { case CFI_BCS_CLEAR_STATUS: case CFI_BCS_READ_STATUS: memset(val, 0, size); var.cmd = CFI_BCS_READ_ARRAY; break; default: memcpy(val, var.mmap + offset, size); break; } } return (0); } void init_bootrom(struct vmctx *ctx) { vm_paddr_t highmem; romptr = vm_create_devmem(ctx, VM_BOOTROM, "bootrom", BOOTROM_SIZE); if (romptr == MAP_FAILED) err(4, "%s: vm_create_devmem", __func__); highmem = vm_get_highmem_base(ctx); gpa_base = highmem - BOOTROM_SIZE; gpa_allocbot = gpa_base; gpa_alloctop = highmem - 1; } int bootrom_alloc(struct vmctx *ctx, size_t len, int prot, int flags, char **region_out, uint64_t *gpa_out) { static const int bootrom_valid_flags = BOOTROM_ALLOC_TOP; vm_paddr_t gpa; vm_ooffset_t segoff; if (flags & ~bootrom_valid_flags) { warnx("%s: Invalid flags: %x", __func__, flags & ~bootrom_valid_flags); return (EINVAL); } if (prot & ~_PROT_ALL) { warnx("%s: Invalid protection: %x", __func__, prot & ~_PROT_ALL); return (EINVAL); } if (len == 0 || len > BOOTROM_SIZE) { warnx("ROM size %zu is invalid", len); return (EINVAL); } if (len & PAGE_MASK) { warnx("ROM size %zu is not a multiple of the page size", len); return (EINVAL); } if (flags & BOOTROM_ALLOC_TOP) { gpa = (gpa_alloctop - len) + 1; if (gpa < gpa_allocbot) { warnx("No room for %zu ROM in bootrom region", len); return (ENOMEM); } } else { gpa = gpa_allocbot; if (gpa > (gpa_alloctop - len) + 1) { warnx("No room for %zu ROM in bootrom region", len); return (ENOMEM); } } segoff = gpa - gpa_base; if (vm_mmap_memseg(ctx, gpa, VM_BOOTROM, segoff, len, prot) != 0) { int serrno = errno; warn("%s: vm_mmap_mapseg", __func__); return (serrno); } if (flags & BOOTROM_ALLOC_TOP) gpa_alloctop = gpa - 1; else gpa_allocbot = gpa + len; *region_out = romptr + segoff; if (gpa_out != NULL) *gpa_out = gpa; return (0); } int bootrom_loadrom(struct vmctx *ctx) { struct stat sbuf; ssize_t rlen; off_t rom_size, var_size, total_size; char *ptr, *romfile; int fd, varfd, i, rv; const char *bootrom, *varfile; rv = -1; varfd = -1; bootrom = get_config_value("bootrom"); if (bootrom == NULL) { return (0); } /* * get_config_value_node may use a thread local buffer to return * variables. So, when we query the second variable, the first variable * might get overwritten. For that reason, the bootrom should be * duplicated. */ romfile = strdup(bootrom); if (romfile == NULL) { return (-1); } fd = open(romfile, O_RDONLY); if (fd < 0) { EPRINTLN("Error opening bootrom \"%s\": %s", romfile, strerror(errno)); goto done; } if (fstat(fd, &sbuf) < 0) { EPRINTLN("Could not fstat bootrom file \"%s\": %s", romfile, strerror(errno)); goto done; } rom_size = sbuf.st_size; varfile = get_config_value("bootvars"); var_size = 0; if (varfile != NULL) { varfd = open(varfile, O_RDWR); if (varfd < 0) { EPRINTLN("Error opening bootrom variable file " "\"%s\": %s", varfile, strerror(errno)); goto done; } if (fstat(varfd, &sbuf) < 0) { EPRINTLN( "Could not fstat bootrom variable file \"%s\": %s", varfile, strerror(errno)); goto done; } var_size = sbuf.st_size; } if (var_size > BOOTROM_SIZE || (var_size != 0 && var_size < PAGE_SIZE)) { EPRINTLN("Invalid bootrom variable size %ld", var_size); goto done; } total_size = rom_size + var_size; if (total_size > BOOTROM_SIZE) { EPRINTLN("Invalid bootrom and variable aggregate size %ld", total_size); goto done; } /* Map the bootrom into the guest address space */ if (bootrom_alloc(ctx, rom_size, PROT_READ | PROT_EXEC, BOOTROM_ALLOC_TOP, &ptr, NULL) != 0) { goto done; } /* Read 'romfile' into the guest address space */ for (i = 0; i < rom_size / PAGE_SIZE; i++) { rlen = read(fd, ptr + i * PAGE_SIZE, PAGE_SIZE); if (rlen != PAGE_SIZE) { EPRINTLN("Incomplete read of page %d of bootrom " "file %s: %ld bytes", i, romfile, rlen); goto done; } } if (varfd >= 0) { #ifdef __FreeBSD__ var.mmap = mmap(NULL, var_size, PROT_READ | PROT_WRITE, MAP_SHARED, varfd, 0); #else var.mmap = (uint8_t *)mmap(NULL, var_size, PROT_READ | PROT_WRITE, MAP_SHARED, varfd, 0); #endif if (var.mmap == MAP_FAILED) goto done; var.size = var_size; var.gpa = (gpa_alloctop - var_size) + 1; gpa_alloctop = var.gpa - 1; rv = register_mem(&(struct mem_range){ .name = "bootrom variable", .flags = MEM_F_RW, .handler = bootrom_var_mem_handler, .base = var.gpa, .size = var.size, }); if (rv != 0) goto done; } rv = 0; done: if (varfd >= 0) close(varfd); if (fd >= 0) close(fd); free(romfile); return (rv); } /* * Are we relying on a bootrom to initialize the guest's CPU context? */ bool bootrom_boot(void) { return (get_config_value("bootrom") != NULL); } /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2015 Neel Natu * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef _BOOTROM_H_ #define _BOOTROM_H_ #include #include #include #include #include "config.h" struct vmctx; void init_bootrom(struct vmctx *ctx); enum { BOOTROM_ALLOC_TOP = 0x80, _FORCE_INT = INT_MIN, }; int bootrom_alloc(struct vmctx *ctx, size_t len, int prot, int flags, char **region_out, uint64_t *gpa_out); bool bootrom_boot(void); int bootrom_loadrom(struct vmctx *ctx); #endif /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2021 John H. Baldwin * Copyright 2026 Hans Rosenfeld * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include #include #include #include #include #ifndef __FreeBSD__ #include #endif #include "config.h" static nvlist_t *config_root; void init_config(void) { config_root = nvlist_create(0); if (config_root == NULL) err(4, "Failed to create configuration root nvlist"); } static nvlist_t * _lookup_config_node(nvlist_t *parent, const char *path, bool create) { char *copy, *name, *tofree; nvlist_t *nvl, *new_nvl; copy = strdup(path); if (copy == NULL) errx(4, "Failed to allocate memory"); tofree = copy; nvl = parent; while ((name = strsep(©, ".")) != NULL) { if (*name == '\0') { warnx("Invalid configuration node: %s", path); nvl = NULL; break; } if (nvlist_exists_nvlist(nvl, name)) /* * XXX-MJ it is incorrect to cast away the const * qualifier like this since the contract with nvlist * says that values are immutable, and some consumers * will indeed add nodes to the returned nvlist. In * practice, however, it appears to be harmless with the * current nvlist implementation, so we just live with * it until the implementation is reworked. */ nvl = __DECONST(nvlist_t *, nvlist_get_nvlist(nvl, name)); else if (nvlist_exists(nvl, name)) { for (copy = tofree; copy < name; copy++) if (*copy == '\0') *copy = '.'; warnx( "Configuration node %s is a child of existing variable %s", path, tofree); nvl = NULL; break; } else if (create) { /* * XXX-MJ as with the case above, "new_nvl" shouldn't be * mutated after its ownership is given to "nvl". */ new_nvl = nvlist_create(0); if (new_nvl == NULL) errx(4, "Failed to allocate memory"); #ifdef __FreeBSD__ nvlist_move_nvlist(nvl, name, new_nvl); #else if (nvlist_add_nvlist(nvl, name, new_nvl) != 0) errx(4, "Failed to allocate memory"); (void) nvlist_free(new_nvl); if (nvlist_lookup_nvlist(nvl, name, &new_nvl) != 0) errx(4, "Failed to retrieve new nvlist"); #endif nvl = new_nvl; } else { nvl = NULL; break; } } free(tofree); return (nvl); } nvlist_t * create_config_node(const char *path) { return (_lookup_config_node(config_root, path, true)); } nvlist_t * find_config_node(const char *path) { return (_lookup_config_node(config_root, path, false)); } nvlist_t * create_relative_config_node(nvlist_t *parent, const char *path) { return (_lookup_config_node(parent, path, true)); } nvlist_t * find_relative_config_node(nvlist_t *parent, const char *path) { return (_lookup_config_node(parent, path, false)); } void set_config_value_node(nvlist_t *parent, const char *name, const char *value) { if (strchr(name, '.') != NULL) errx(4, "Invalid config node name %s", name); if (parent == NULL) parent = config_root; if (nvlist_exists_string(parent, name)) nvlist_free_string(parent, name); else if (nvlist_exists(parent, name)) errx(4, "Attempting to add value %s to existing node %s of list %p", value, name, parent); nvlist_add_string(parent, name, value); } void set_config_value_node_if_unset(nvlist_t *const parent, const char *const name, const char *const value) { if (get_config_value_node(parent, name) != NULL) { return; } set_config_value_node(parent, name, value); } void set_config_value(const char *path, const char *value) { const char *name; char *node_name; nvlist_t *nvl; /* Look for last separator. */ name = strrchr(path, '.'); if (name == NULL) { nvl = config_root; name = path; } else { node_name = strndup(path, name - path); if (node_name == NULL) errx(4, "Failed to allocate memory"); nvl = create_config_node(node_name); if (nvl == NULL) errx(4, "Failed to create configuration node %s", node_name); free(node_name); /* Skip over '.'. */ name++; } if (nvlist_exists_nvlist(nvl, name)) errx(4, "Attempting to add value %s to existing node %s", value, path); set_config_value_node(nvl, name, value); } void set_config_value_if_unset(const char *const path, const char *const value) { if (get_config_value(path) != NULL) { return; } set_config_value(path, value); } static const char * get_raw_config_value(const char *path) { const char *name; char *node_name; nvlist_t *nvl; /* Look for last separator. */ name = strrchr(path, '.'); if (name == NULL) { nvl = config_root; name = path; } else { node_name = strndup(path, name - path); if (node_name == NULL) errx(4, "Failed to allocate memory"); nvl = find_config_node(node_name); free(node_name); if (nvl == NULL) return (NULL); /* Skip over '.'. */ name++; } if (nvlist_exists_string(nvl, name)) return (nvlist_get_string(nvl, name)); if (nvlist_exists_nvlist(nvl, name)) warnx("Attempting to fetch value of node %s", path); return (NULL); } static char * _expand_config_value(const char *value, int depth) { FILE *valfp; const char *cp, *vp; char *nestedval, *path, *valbuf; size_t valsize; valfp = open_memstream(&valbuf, &valsize); if (valfp == NULL) errx(4, "Failed to allocate memory"); vp = value; while (*vp != '\0') { switch (*vp) { case '%': if (depth > 15) { warnx( "Too many recursive references in configuration value"); fputc('%', valfp); vp++; break; } if (vp[1] != '(' || vp[2] == '\0') cp = NULL; else cp = strchr(vp + 2, ')'); if (cp == NULL) { warnx( "Invalid reference in configuration value \"%s\"", value); fputc('%', valfp); vp++; break; } vp += 2; if (cp == vp) { warnx( "Empty reference in configuration value \"%s\"", value); vp++; break; } /* Allocate a C string holding the path. */ path = strndup(vp, cp - vp); if (path == NULL) errx(4, "Failed to allocate memory"); /* Advance 'vp' past the reference. */ vp = cp + 1; /* Fetch the referenced value. */ cp = get_raw_config_value(path); if (cp == NULL) warnx( "Failed to fetch referenced configuration variable %s", path); else { nestedval = _expand_config_value(cp, depth + 1); fputs(nestedval, valfp); free(nestedval); } free(path); break; case '\\': vp++; if (*vp == '\0') { warnx( "Trailing \\ in configuration value \"%s\"", value); break; } /* FALLTHROUGH */ default: fputc(*vp, valfp); vp++; break; } } fclose(valfp); return (valbuf); } static const char * expand_config_value(const char *value) { static char *valbuf; if (strchr(value, '%') == NULL) return (value); free(valbuf); valbuf = _expand_config_value(value, 0); return (valbuf); } const char * get_config_value(const char *path) { const char *value; value = get_raw_config_value(path); if (value == NULL) return (NULL); return (expand_config_value(value)); } const char * get_config_value_node(const nvlist_t *parent, const char *name) { if (strchr(name, '.') != NULL) errx(4, "Invalid config node name %s", name); if (parent == NULL) parent = config_root; if (nvlist_exists_nvlist(parent, name)) warnx("Attempt to fetch value of node %s of list %p", name, parent); if (!nvlist_exists_string(parent, name)) return (NULL); return (expand_config_value(nvlist_get_string(parent, name))); } static bool _bool_value(const char *name, const char *value) { if (strcasecmp(value, "true") == 0 || strcasecmp(value, "on") == 0 || strcasecmp(value, "yes") == 0 || strcmp(value, "1") == 0) return (true); if (strcasecmp(value, "false") == 0 || strcasecmp(value, "off") == 0 || strcasecmp(value, "no") == 0 || strcmp(value, "0") == 0) return (false); err(4, "Invalid value %s for boolean variable %s", value, name); } bool get_config_bool(const char *path) { const char *value; value = get_config_value(path); if (value == NULL) err(4, "Failed to fetch boolean variable %s", path); return (_bool_value(path, value)); } bool get_config_bool_default(const char *path, bool def) { const char *value; value = get_config_value(path); if (value == NULL) return (def); return (_bool_value(path, value)); } bool get_config_bool_node(const nvlist_t *parent, const char *name) { const char *value; value = get_config_value_node(parent, name); if (value == NULL) err(4, "Failed to fetch boolean variable %s", name); return (_bool_value(name, value)); } bool get_config_bool_node_default(const nvlist_t *parent, const char *name, bool def) { const char *value; value = get_config_value_node(parent, name); if (value == NULL) return (def); return (_bool_value(name, value)); } void set_config_bool(const char *path, bool value) { set_config_value(path, value ? "true" : "false"); } void set_config_bool_node(nvlist_t *parent, const char *name, bool value) { set_config_value_node(parent, name, value ? "true" : "false"); } int walk_config_nodes(const char *prefix, const nvlist_t *parent, void *arg, int (*cb)(const char *, const nvlist_t *, const char *, int, void *)) { void *cookie = NULL; const char *name; int type; while ((name = nvlist_next(parent, &type, &cookie)) != NULL) { int ret; ret = cb(prefix, parent, name, type, arg); if (ret != 0) return (ret); } return (0); } static int dump_node_cb(const char *prefix, const nvlist_t *parent, const char *name, int type, void *arg) { if (type == NV_TYPE_NVLIST) { char *new_prefix; int ret; asprintf(&new_prefix, "%s%s.", prefix, name); ret = walk_config_nodes(new_prefix, nvlist_get_nvlist(parent, name), arg, dump_node_cb); free(new_prefix); return (ret); } assert(type == NV_TYPE_STRING); printf("%s%s=%s\n", prefix, name, nvlist_get_string(parent, name)); return (0); } void dump_config(void) { (void) walk_config_nodes("", config_root, NULL, dump_node_cb); } /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2021 John H. Baldwin * Copyright 2026 Hans Rosenfeld * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef __CONFIG_H__ #define __CONFIG_H__ #include /*- * Manages a configuration database backed by an nv(9) list. * * The database only stores string values. Callers should parse * values into other types if needed. String values can reference * other configuration variables using a '%(name)' syntax. In this * case, the name must be the full path of the configuration * variable. The % character can be escaped with a preceding \ to * avoid expansion. Any \ characters must be escaped. * * Configuration variables are stored in a tree. The full path of a * variable is specified as a dot-separated name similar to sysctl(8) * OIDs. */ /* * Walk the nodes under a parent nvlist. For each node found, call the given * callback function passing the current prefix, nvlist, node name and type, * and the given argument. */ int walk_config_nodes(const char *, const nvlist_t *, void *, int (*cb)(const char *, const nvlist_t *, const char *, int, void *)); /* * Fetches the value of a configuration variable. If the "raw" value * contains references to other configuration variables, this function * expands those references and returns a pointer to the parsed * string. The string's storage is only stable until the next call to * this function. * * If no node is found, returns NULL. * * If 'parent' is NULL, 'name' is assumed to be a top-level variable. */ const char *get_config_value_node(const nvlist_t *parent, const char *name); /* * Similar to get_config_value_node but expects a full path to the * leaf node. */ const char *get_config_value(const char *path); /* Initializes the tree to an empty state. */ void init_config(void); /* * Creates an existing configuration node via a dot-separated OID * path. Will fail if the path names an existing leaf configuration * variable. If the node already exists, this returns a pointer to * the existing node. */ nvlist_t *create_config_node(const char *path); /* * Looks for an existing configuration node via a dot-separated OID * path. Will fail if the path names an existing leaf configuration * variable. */ nvlist_t *find_config_node(const char *path); /* * Similar to the above, but treats the path relative to an existing * 'parent' node rather than as an absolute path. */ nvlist_t *create_relative_config_node(nvlist_t *parent, const char *path); nvlist_t *find_relative_config_node(nvlist_t *parent, const char *path); /* * Adds or replaces the value of the specified variable. * * If 'parent' is NULL, 'name' is assumed to be a top-level variable. */ void set_config_value_node(nvlist_t *parent, const char *name, const char *value); /* * Similar to set_config_value_node but only sets value if it's unset yet. */ void set_config_value_node_if_unset(nvlist_t *const parent, const char *const name, const char *const value); /* * Similar to set_config_value_node but expects a full path to the * leaf node. */ void set_config_value(const char *path, const char *value); /* * Similar to set_config_value but only sets the value if it's unset yet. */ void set_config_value_if_unset(const char *const path, const char *const value); /* Convenience wrappers for boolean variables. */ bool get_config_bool(const char *path); bool get_config_bool_node(const nvlist_t *parent, const char *name); bool get_config_bool_default(const char *path, bool def); bool get_config_bool_node_default(const nvlist_t *parent, const char *name, bool def); void set_config_bool(const char *path, bool value); void set_config_bool_node(nvlist_t *parent, const char *name, bool value); void dump_config(void); #endif /* !__CONFIG_H__ */ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2015 Tycho Nightingale * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include #include "bhyvegc.h" #include "console.h" static struct { struct bhyvegc *gc; fb_render_func_t fb_render_cb; void *fb_arg; kbd_event_func_t kbd_event_cb; void *kbd_arg; int kbd_priority; ptr_event_func_t ptr_event_cb; void *ptr_arg; int ptr_priority; } console; void console_init(int w, int h, void *fbaddr) { console.gc = bhyvegc_init(w, h, fbaddr); } void console_set_fbaddr(void *fbaddr) { bhyvegc_set_fbaddr(console.gc, fbaddr); } struct bhyvegc_image * console_get_image(void) { struct bhyvegc_image *bhyvegc_image; bhyvegc_image = bhyvegc_get_image(console.gc); return (bhyvegc_image); } void console_fb_register(fb_render_func_t render_cb, void *arg) { console.fb_render_cb = render_cb; console.fb_arg = arg; } void console_refresh(void) { if (console.fb_render_cb) (*console.fb_render_cb)(console.gc, console.fb_arg); } void console_kbd_register(kbd_event_func_t event_cb, void *arg, int pri) { if (pri > console.kbd_priority) { console.kbd_event_cb = event_cb; console.kbd_arg = arg; console.kbd_priority = pri; } } void console_ptr_register(ptr_event_func_t event_cb, void *arg, int pri) { if (pri > console.ptr_priority) { console.ptr_event_cb = event_cb; console.ptr_arg = arg; console.ptr_priority = pri; } } void console_key_event(int down, uint32_t keysym, uint32_t keycode) { if (console.kbd_event_cb) (*console.kbd_event_cb)(down, keysym, keycode, console.kbd_arg); } void console_ptr_event(uint8_t button, int x, int y) { if (console.ptr_event_cb) (*console.ptr_event_cb)(button, x, y, console.ptr_arg); } /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2015 Tycho Nightingale * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef _CONSOLE_H_ #define _CONSOLE_H_ struct bhyvegc; typedef void (*fb_render_func_t)(struct bhyvegc *gc, void *arg); typedef void (*kbd_event_func_t)(int down, uint32_t keysym, uint32_t keycode, void *arg); typedef void (*ptr_event_func_t)(uint8_t mask, int x, int y, void *arg); void console_init(int w, int h, void *fbaddr); void console_set_fbaddr(void *fbaddr); struct bhyvegc_image *console_get_image(void); void console_fb_register(fb_render_func_t render_cb, void *arg); void console_refresh(void); void console_kbd_register(kbd_event_func_t event_cb, void *arg, int pri); void console_key_event(int down, uint32_t keysym, uint32_t keycode); void console_ptr_register(ptr_event_func_t event_cb, void *arg, int pri); void console_ptr_event(uint8_t button, int x, int y); #endif /* _CONSOLE_H_ */ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2017, Fedor Uporov * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include #include "crc16.h" /* CRC table for the CRC-16. The poly is 0x8005 (x16 + x15 + x2 + 1). */ uint16_t const crc16_table[256] = { 0x0000, 0xC0C1, 0xC181, 0x0140, 0xC301, 0x03C0, 0x0280, 0xC241, 0xC601, 0x06C0, 0x0780, 0xC741, 0x0500, 0xC5C1, 0xC481, 0x0440, 0xCC01, 0x0CC0, 0x0D80, 0xCD41, 0x0F00, 0xCFC1, 0xCE81, 0x0E40, 0x0A00, 0xCAC1, 0xCB81, 0x0B40, 0xC901, 0x09C0, 0x0880, 0xC841, 0xD801, 0x18C0, 0x1980, 0xD941, 0x1B00, 0xDBC1, 0xDA81, 0x1A40, 0x1E00, 0xDEC1, 0xDF81, 0x1F40, 0xDD01, 0x1DC0, 0x1C80, 0xDC41, 0x1400, 0xD4C1, 0xD581, 0x1540, 0xD701, 0x17C0, 0x1680, 0xD641, 0xD201, 0x12C0, 0x1380, 0xD341, 0x1100, 0xD1C1, 0xD081, 0x1040, 0xF001, 0x30C0, 0x3180, 0xF141, 0x3300, 0xF3C1, 0xF281, 0x3240, 0x3600, 0xF6C1, 0xF781, 0x3740, 0xF501, 0x35C0, 0x3480, 0xF441, 0x3C00, 0xFCC1, 0xFD81, 0x3D40, 0xFF01, 0x3FC0, 0x3E80, 0xFE41, 0xFA01, 0x3AC0, 0x3B80, 0xFB41, 0x3900, 0xF9C1, 0xF881, 0x3840, 0x2800, 0xE8C1, 0xE981, 0x2940, 0xEB01, 0x2BC0, 0x2A80, 0xEA41, 0xEE01, 0x2EC0, 0x2F80, 0xEF41, 0x2D00, 0xEDC1, 0xEC81, 0x2C40, 0xE401, 0x24C0, 0x2580, 0xE541, 0x2700, 0xE7C1, 0xE681, 0x2640, 0x2200, 0xE2C1, 0xE381, 0x2340, 0xE101, 0x21C0, 0x2080, 0xE041, 0xA001, 0x60C0, 0x6180, 0xA141, 0x6300, 0xA3C1, 0xA281, 0x6240, 0x6600, 0xA6C1, 0xA781, 0x6740, 0xA501, 0x65C0, 0x6480, 0xA441, 0x6C00, 0xACC1, 0xAD81, 0x6D40, 0xAF01, 0x6FC0, 0x6E80, 0xAE41, 0xAA01, 0x6AC0, 0x6B80, 0xAB41, 0x6900, 0xA9C1, 0xA881, 0x6840, 0x7800, 0xB8C1, 0xB981, 0x7940, 0xBB01, 0x7BC0, 0x7A80, 0xBA41, 0xBE01, 0x7EC0, 0x7F80, 0xBF41, 0x7D00, 0xBDC1, 0xBC81, 0x7C40, 0xB401, 0x74C0, 0x7580, 0xB541, 0x7700, 0xB7C1, 0xB681, 0x7640, 0x7200, 0xB2C1, 0xB381, 0x7340, 0xB101, 0x71C0, 0x7080, 0xB041, 0x5000, 0x90C1, 0x9181, 0x5140, 0x9301, 0x53C0, 0x5280, 0x9241, 0x9601, 0x56C0, 0x5780, 0x9741, 0x5500, 0x95C1, 0x9481, 0x5440, 0x9C01, 0x5CC0, 0x5D80, 0x9D41, 0x5F00, 0x9FC1, 0x9E81, 0x5E40, 0x5A00, 0x9AC1, 0x9B81, 0x5B40, 0x9901, 0x59C0, 0x5880, 0x9841, 0x8801, 0x48C0, 0x4980, 0x8941, 0x4B00, 0x8BC1, 0x8A81, 0x4A40, 0x4E00, 0x8EC1, 0x8F81, 0x4F40, 0x8D01, 0x4DC0, 0x4C80, 0x8C41, 0x4400, 0x84C1, 0x8581, 0x4540, 0x8701, 0x47C0, 0x4680, 0x8641, 0x8201, 0x42C0, 0x4380, 0x8341, 0x4100, 0x81C1, 0x8081, 0x4040 }; /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2017, Fedor Uporov * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include #ifndef _CRC16_H_ #define _CRC16_H_ extern uint16_t const crc16_table[256]; static inline uint16_t crc16(uint16_t crc, const void *buffer, unsigned int len) { const unsigned char *cp = buffer; while (len--) crc = (((crc >> 8) & 0xffU) ^ crc16_table[(crc ^ *cp++) & 0xffU]) & 0x0000ffffU; return crc; } #endif /* !_CRC16_H_ */ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2019 Vincenzo Maffione * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef _DEBUG_H_ #define _DEBUG_H_ extern int raw_stdio; #define FPRINTLN(filep, fmt, arg...) \ do { \ if (raw_stdio) \ fprintf(filep, fmt "\r\n", ##arg); \ else \ fprintf(filep, fmt "\n", ##arg); \ } while (0) #define PRINTLN(fmt, arg...) FPRINTLN(stdout, fmt, ##arg) #define EPRINTLN(fmt, arg...) FPRINTLN(stderr, fmt, ##arg) #endif /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2017-2018 John H. Baldwin * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include #ifndef WITHOUT_CAPSICUM #include #endif #ifdef __FreeBSD__ #include #else #include #endif #include #include #include #include #include #include #include #include #include #ifndef WITHOUT_CAPSICUM #include #endif #include #include #include #include #include #include #include #include #include #include #include #include #include #include "bhyverun.h" #include "config.h" #include "debug.h" #include "gdb.h" #include "mem.h" #include "mevent.h" /* * GDB_SIGNAL_* numbers are part of the GDB remote protocol. Most stops * use SIGTRAP. */ #define GDB_SIGNAL_TRAP 5 #define GDB_BP_SIZE 1 #define GDB_BP_INSTR (uint8_t []){0xcc} #define GDB_PC_REGNAME VM_REG_GUEST_RIP _Static_assert(sizeof(GDB_BP_INSTR) == GDB_BP_SIZE, "GDB_BP_INSTR has wrong size"); static void gdb_resume_vcpus(void); static void check_command(int fd); static struct mevent *read_event, *write_event; static cpuset_t vcpus_active, vcpus_suspended, vcpus_waiting; static pthread_mutex_t gdb_lock; static pthread_cond_t idle_vcpus; static bool first_stop, report_next_stop, swbreak_enabled; /* * An I/O buffer contains 'capacity' bytes of room at 'data'. For a * read buffer, 'start' is unused and 'len' contains the number of * valid bytes in the buffer. For a write buffer, 'start' is set to * the index of the next byte in 'data' to send, and 'len' contains * the remaining number of valid bytes to send. */ struct io_buffer { uint8_t *data; size_t capacity; size_t start; size_t len; }; struct breakpoint { uint64_t gpa; uint8_t shadow_inst[GDB_BP_SIZE]; TAILQ_ENTRY(breakpoint) link; }; /* * When a vCPU stops to due to an event that should be reported to the * debugger, information about the event is stored in this structure. * The vCPU thread then sets 'stopped_vcpu' if it is not already set * and stops other vCPUs so the event can be reported. The * report_stop() function reports the event for the 'stopped_vcpu' * vCPU. When the debugger resumes execution via continue or step, * the event for 'stopped_vcpu' is cleared. vCPUs will loop in their * event handlers until the associated event is reported or disabled. * * An idle vCPU will have all of the boolean fields set to false. * * When a vCPU is stepped, 'stepping' is set to true when the vCPU is * released to execute the stepped instruction. When the vCPU reports * the stepping trap, 'stepped' is set. * * When a vCPU hits a breakpoint set by the debug server, * 'hit_swbreak' is set to true. */ struct vcpu_state { bool stepping; bool stepped; bool hit_swbreak; }; static struct io_buffer cur_comm, cur_resp; static uint8_t cur_csum; static struct vmctx *ctx; static int cur_fd = -1; static TAILQ_HEAD(, breakpoint) breakpoints; static struct vcpu_state *vcpu_state; static struct vcpu **vcpus; static int cur_vcpu, stopped_vcpu; static bool gdb_active = false; struct gdb_reg { enum vm_reg_name id; int size; }; static const struct gdb_reg gdb_regset[] = { { .id = VM_REG_GUEST_RAX, .size = 8 }, { .id = VM_REG_GUEST_RBX, .size = 8 }, { .id = VM_REG_GUEST_RCX, .size = 8 }, { .id = VM_REG_GUEST_RDX, .size = 8 }, { .id = VM_REG_GUEST_RSI, .size = 8 }, { .id = VM_REG_GUEST_RDI, .size = 8 }, { .id = VM_REG_GUEST_RBP, .size = 8 }, { .id = VM_REG_GUEST_RSP, .size = 8 }, { .id = VM_REG_GUEST_R8, .size = 8 }, { .id = VM_REG_GUEST_R9, .size = 8 }, { .id = VM_REG_GUEST_R10, .size = 8 }, { .id = VM_REG_GUEST_R11, .size = 8 }, { .id = VM_REG_GUEST_R12, .size = 8 }, { .id = VM_REG_GUEST_R13, .size = 8 }, { .id = VM_REG_GUEST_R14, .size = 8 }, { .id = VM_REG_GUEST_R15, .size = 8 }, { .id = VM_REG_GUEST_RIP, .size = 8 }, { .id = VM_REG_GUEST_RFLAGS, .size = 4 }, { .id = VM_REG_GUEST_CS, .size = 4 }, { .id = VM_REG_GUEST_SS, .size = 4 }, { .id = VM_REG_GUEST_DS, .size = 4 }, { .id = VM_REG_GUEST_ES, .size = 4 }, { .id = VM_REG_GUEST_FS, .size = 4 }, { .id = VM_REG_GUEST_GS, .size = 4 }, }; #ifdef GDB_LOG #include #include static void __printflike(1, 2) debug(const char *fmt, ...) { static FILE *logfile; va_list ap; if (logfile == NULL) { logfile = fopen("/tmp/bhyve_gdb.log", "w"); if (logfile == NULL) return; #ifndef WITHOUT_CAPSICUM if (caph_limit_stream(fileno(logfile), CAPH_WRITE) == -1) { fclose(logfile); logfile = NULL; return; } #endif setlinebuf(logfile); } va_start(ap, fmt); vfprintf(logfile, fmt, ap); va_end(ap); } #else #ifndef __FreeBSD__ /* * A totally empty debug() makes the compiler grumpy due to how its used with * some control flow here. */ #define debug(...) do { } while (0) #else #define debug(...) #endif #endif static void remove_all_sw_breakpoints(void); static int guest_paging_info(struct vcpu *vcpu, struct vm_guest_paging *paging) { uint64_t regs[4]; const int regset[4] = { VM_REG_GUEST_CR0, VM_REG_GUEST_CR3, VM_REG_GUEST_CR4, VM_REG_GUEST_EFER }; if (vm_get_register_set(vcpu, nitems(regset), regset, regs) == -1) return (-1); /* * For the debugger, always pretend to be the kernel (CPL 0), * and if long-mode is enabled, always parse addresses as if * in 64-bit mode. */ paging->cr3 = regs[1]; paging->cpl = 0; if (regs[3] & EFER_LMA) paging->cpu_mode = CPU_MODE_64BIT; else if (regs[0] & CR0_PE) paging->cpu_mode = CPU_MODE_PROTECTED; else paging->cpu_mode = CPU_MODE_REAL; if (!(regs[0] & CR0_PG)) paging->paging_mode = PAGING_MODE_FLAT; else if (!(regs[2] & CR4_PAE)) paging->paging_mode = PAGING_MODE_32; else if (regs[3] & EFER_LME) paging->paging_mode = PAGING_MODE_64; else paging->paging_mode = PAGING_MODE_PAE; return (0); } /* * Map a guest virtual address to a physical address (for a given vcpu). * If a guest virtual address is valid, return 1. If the address is * not valid, return 0. If an error occurs obtaining the mapping, * return -1. */ static int guest_vaddr2paddr(struct vcpu *vcpu, uint64_t vaddr, uint64_t *paddr) { struct vm_guest_paging paging; int fault; if (guest_paging_info(vcpu, &paging) == -1) return (-1); /* * Always use PROT_READ. We really care if the VA is * accessible, not if the current vCPU can write. */ if (vm_gla2gpa_nofault(vcpu, &paging, vaddr, PROT_READ, paddr, &fault) == -1) return (-1); if (fault) return (0); return (1); } static uint64_t guest_pc(struct vm_exit *vme) { return (vme->rip); } static void io_buffer_reset(struct io_buffer *io) { io->start = 0; io->len = 0; } /* Available room for adding data. */ static size_t io_buffer_avail(struct io_buffer *io) { return (io->capacity - (io->start + io->len)); } static uint8_t * io_buffer_head(struct io_buffer *io) { return (io->data + io->start); } static uint8_t * io_buffer_tail(struct io_buffer *io) { return (io->data + io->start + io->len); } static void io_buffer_advance(struct io_buffer *io, size_t amount) { assert(amount <= io->len); io->start += amount; io->len -= amount; } static void io_buffer_consume(struct io_buffer *io, size_t amount) { io_buffer_advance(io, amount); if (io->len == 0) { io->start = 0; return; } /* * XXX: Consider making this move optional and compacting on a * future read() before realloc(). */ memmove(io->data, io_buffer_head(io), io->len); io->start = 0; } static void io_buffer_grow(struct io_buffer *io, size_t newsize) { uint8_t *new_data; size_t avail, new_cap; avail = io_buffer_avail(io); if (newsize <= avail) return; new_cap = io->capacity + (newsize - avail); new_data = realloc(io->data, new_cap); if (new_data == NULL) err(1, "Failed to grow GDB I/O buffer"); io->data = new_data; io->capacity = new_cap; } static bool response_pending(void) { if (cur_resp.start == 0 && cur_resp.len == 0) return (false); if (cur_resp.start + cur_resp.len == 1 && cur_resp.data[0] == '+') return (false); return (true); } static void close_connection(void) { /* * XXX: This triggers a warning because mevent does the close * before the EV_DELETE. */ pthread_mutex_lock(&gdb_lock); mevent_delete(write_event); mevent_delete_close(read_event); write_event = NULL; read_event = NULL; io_buffer_reset(&cur_comm); io_buffer_reset(&cur_resp); cur_fd = -1; remove_all_sw_breakpoints(); /* Clear any pending events. */ memset(vcpu_state, 0, guest_ncpus * sizeof(*vcpu_state)); /* Resume any stopped vCPUs. */ gdb_resume_vcpus(); pthread_mutex_unlock(&gdb_lock); } static uint8_t hex_digit(uint8_t nibble) { if (nibble <= 9) return (nibble + '0'); else return (nibble + 'a' - 10); } static uint8_t parse_digit(uint8_t v) { if (v >= '0' && v <= '9') return (v - '0'); if (v >= 'a' && v <= 'f') return (v - 'a' + 10); if (v >= 'A' && v <= 'F') return (v - 'A' + 10); return (0xF); } /* Parses big-endian hexadecimal. */ static uintmax_t parse_integer(const uint8_t *p, size_t len) { uintmax_t v; v = 0; while (len > 0) { v <<= 4; v |= parse_digit(*p); p++; len--; } return (v); } static uint8_t parse_byte(const uint8_t *p) { return (parse_digit(p[0]) << 4 | parse_digit(p[1])); } static void send_pending_data(int fd) { ssize_t nwritten; if (cur_resp.len == 0) { mevent_disable(write_event); return; } nwritten = write(fd, io_buffer_head(&cur_resp), cur_resp.len); if (nwritten == -1) { warn("Write to GDB socket failed"); close_connection(); } else { io_buffer_advance(&cur_resp, nwritten); if (cur_resp.len == 0) mevent_disable(write_event); else mevent_enable(write_event); } } /* Append a single character to the output buffer. */ static void send_char(uint8_t data) { io_buffer_grow(&cur_resp, 1); *io_buffer_tail(&cur_resp) = data; cur_resp.len++; } /* Append an array of bytes to the output buffer. */ static void send_data(const uint8_t *data, size_t len) { io_buffer_grow(&cur_resp, len); memcpy(io_buffer_tail(&cur_resp), data, len); cur_resp.len += len; } static void format_byte(uint8_t v, uint8_t *buf) { buf[0] = hex_digit(v >> 4); buf[1] = hex_digit(v & 0xf); } /* * Append a single byte (formatted as two hex characters) to the * output buffer. */ static void send_byte(uint8_t v) { uint8_t buf[2]; format_byte(v, buf); send_data(buf, sizeof(buf)); } static void start_packet(void) { send_char('$'); cur_csum = 0; } static void finish_packet(void) { send_char('#'); send_byte(cur_csum); debug("-> %.*s\n", (int)cur_resp.len, io_buffer_head(&cur_resp)); } /* * Append a single character (for the packet payload) and update the * checksum. */ static void append_char(uint8_t v) { send_char(v); cur_csum += v; } /* * Append an array of bytes (for the packet payload) and update the * checksum. */ static void append_packet_data(const uint8_t *data, size_t len) { send_data(data, len); while (len > 0) { cur_csum += *data; data++; len--; } } static void append_string(const char *str) { #ifdef __FreeBSD__ append_packet_data(str, strlen(str)); #else append_packet_data((const uint8_t *)str, strlen(str)); #endif } static void append_byte(uint8_t v) { uint8_t buf[2]; format_byte(v, buf); append_packet_data(buf, sizeof(buf)); } static void append_unsigned_native(uintmax_t value, size_t len) { size_t i; for (i = 0; i < len; i++) { append_byte(value); value >>= 8; } } static void append_unsigned_be(uintmax_t value, size_t len) { char buf[len * 2]; size_t i; for (i = 0; i < len; i++) { #ifdef __FreeBSD__ format_byte(value, buf + (len - i - 1) * 2); #else format_byte(value, (uint8_t *)(buf + (len - i - 1) * 2)); #endif value >>= 8; } #ifdef __FreeBSD__ append_packet_data(buf, sizeof(buf)); #else append_packet_data((const uint8_t *)buf, sizeof(buf)); #endif } static void append_integer(unsigned int value) { if (value == 0) append_char('0'); else append_unsigned_be(value, (fls(value) + 7) / 8); } static void append_asciihex(const char *str) { while (*str != '\0') { append_byte(*str); str++; } } static void send_empty_response(void) { start_packet(); finish_packet(); } static void send_error(int error) { start_packet(); append_char('E'); append_byte(error); finish_packet(); } static void send_ok(void) { start_packet(); append_string("OK"); finish_packet(); } static int parse_threadid(const uint8_t *data, size_t len) { if (len == 1 && *data == '0') return (0); if (len == 2 && memcmp(data, "-1", 2) == 0) return (-1); if (len == 0) return (-2); return (parse_integer(data, len)); } /* * Report the current stop event to the debugger. If the stop is due * to an event triggered on a specific vCPU such as a breakpoint or * stepping trap, stopped_vcpu will be set to the vCPU triggering the * stop. If 'set_cur_vcpu' is true, then cur_vcpu will be updated to * the reporting vCPU for vCPU events. */ static void report_stop(bool set_cur_vcpu) { struct vcpu_state *vs; start_packet(); if (stopped_vcpu == -1) { append_char('S'); append_byte(GDB_SIGNAL_TRAP); } else { vs = &vcpu_state[stopped_vcpu]; if (set_cur_vcpu) cur_vcpu = stopped_vcpu; append_char('T'); append_byte(GDB_SIGNAL_TRAP); append_string("thread:"); append_integer(stopped_vcpu + 1); append_char(';'); if (vs->hit_swbreak) { debug("$vCPU %d reporting swbreak\n", stopped_vcpu); if (swbreak_enabled) append_string("swbreak:;"); } else if (vs->stepped) debug("$vCPU %d reporting step\n", stopped_vcpu); else debug("$vCPU %d reporting ???\n", stopped_vcpu); } finish_packet(); report_next_stop = false; } /* * If this stop is due to a vCPU event, clear that event to mark it as * acknowledged. */ static void discard_stop(void) { struct vcpu_state *vs; if (stopped_vcpu != -1) { vs = &vcpu_state[stopped_vcpu]; vs->hit_swbreak = false; vs->stepped = false; stopped_vcpu = -1; } report_next_stop = true; } static void gdb_finish_suspend_vcpus(void) { if (first_stop) { first_stop = false; stopped_vcpu = -1; } else if (report_next_stop) { assert(!response_pending()); report_stop(true); send_pending_data(cur_fd); } } /* * vCPU threads invoke this function whenever the vCPU enters the * debug server to pause or report an event. vCPU threads wait here * as long as the debug server keeps them suspended. */ static void _gdb_cpu_suspend(struct vcpu *vcpu, bool report_stop) { int vcpuid = vcpu_id(vcpu); debug("$vCPU %d suspending\n", vcpuid); CPU_SET(vcpuid, &vcpus_waiting); if (report_stop && CPU_CMP(&vcpus_waiting, &vcpus_suspended) == 0) gdb_finish_suspend_vcpus(); while (CPU_ISSET(vcpuid, &vcpus_suspended)) pthread_cond_wait(&idle_vcpus, &gdb_lock); CPU_CLR(vcpuid, &vcpus_waiting); debug("$vCPU %d resuming\n", vcpuid); } /* * Invoked at the start of a vCPU thread's execution to inform the * debug server about the new thread. */ void gdb_cpu_add(struct vcpu *vcpu) { int vcpuid; if (!gdb_active) return; vcpuid = vcpu_id(vcpu); debug("$vCPU %d starting\n", vcpuid); pthread_mutex_lock(&gdb_lock); assert(vcpuid < guest_ncpus); assert(vcpus[vcpuid] == NULL); vcpus[vcpuid] = vcpu; CPU_SET(vcpuid, &vcpus_active); if (!TAILQ_EMPTY(&breakpoints)) { vm_set_capability(vcpu, VM_CAP_BPT_EXIT, 1); debug("$vCPU %d enabled breakpoint exits\n", vcpuid); } /* * If a vcpu is added while vcpus are stopped, suspend the new * vcpu so that it will pop back out with a debug exit before * executing the first instruction. */ if (!CPU_EMPTY(&vcpus_suspended)) { CPU_SET(vcpuid, &vcpus_suspended); _gdb_cpu_suspend(vcpu, false); } pthread_mutex_unlock(&gdb_lock); } /* * Invoked by vCPU before resuming execution. This enables stepping * if the vCPU is marked as stepping. */ static void gdb_cpu_resume(struct vcpu *vcpu) { struct vcpu_state *vs; int error; vs = &vcpu_state[vcpu_id(vcpu)]; /* * Any pending event should already be reported before * resuming. */ assert(vs->hit_swbreak == false); assert(vs->stepped == false); if (vs->stepping) { error = vm_set_capability(vcpu, VM_CAP_MTRAP_EXIT, 1); assert(error == 0); } } /* * Handler for VM_EXITCODE_DEBUG used to suspend a vCPU when the guest * has been suspended due to an event on different vCPU or in response * to a guest-wide suspend such as Ctrl-C or the stop on attach. */ void gdb_cpu_suspend(struct vcpu *vcpu) { if (!gdb_active) return; pthread_mutex_lock(&gdb_lock); _gdb_cpu_suspend(vcpu, true); gdb_cpu_resume(vcpu); pthread_mutex_unlock(&gdb_lock); } static void gdb_suspend_vcpus(void) { assert(pthread_mutex_isowned_np(&gdb_lock)); debug("suspending all CPUs\n"); vcpus_suspended = vcpus_active; vm_suspend_all_cpus(ctx); if (CPU_CMP(&vcpus_waiting, &vcpus_suspended) == 0) gdb_finish_suspend_vcpus(); } /* * Handler for VM_EXITCODE_MTRAP reported when a vCPU single-steps via * the VT-x-specific MTRAP exit. */ void gdb_cpu_mtrap(struct vcpu *vcpu) { struct vcpu_state *vs; int vcpuid; if (!gdb_active) return; vcpuid = vcpu_id(vcpu); debug("$vCPU %d MTRAP\n", vcpuid); pthread_mutex_lock(&gdb_lock); vs = &vcpu_state[vcpuid]; if (vs->stepping) { vs->stepping = false; vs->stepped = true; vm_set_capability(vcpu, VM_CAP_MTRAP_EXIT, 0); while (vs->stepped) { if (stopped_vcpu == -1) { debug("$vCPU %d reporting step\n", vcpuid); stopped_vcpu = vcpuid; gdb_suspend_vcpus(); } _gdb_cpu_suspend(vcpu, true); } gdb_cpu_resume(vcpu); } pthread_mutex_unlock(&gdb_lock); } static struct breakpoint * find_breakpoint(uint64_t gpa) { struct breakpoint *bp; TAILQ_FOREACH(bp, &breakpoints, link) { if (bp->gpa == gpa) return (bp); } return (NULL); } void gdb_cpu_breakpoint(struct vcpu *vcpu, struct vm_exit *vmexit) { struct breakpoint *bp; struct vcpu_state *vs; uint64_t gpa; int error, vcpuid; if (!gdb_active) { EPRINTLN("vm_loop: unexpected VMEXIT_DEBUG"); exit(4); } vcpuid = vcpu_id(vcpu); pthread_mutex_lock(&gdb_lock); error = guest_vaddr2paddr(vcpu, guest_pc(vmexit), &gpa); assert(error == 1); bp = find_breakpoint(gpa); if (bp != NULL) { vs = &vcpu_state[vcpuid]; assert(vs->stepping == false); assert(vs->stepped == false); assert(vs->hit_swbreak == false); vs->hit_swbreak = true; vm_set_register(vcpu, GDB_PC_REGNAME, guest_pc(vmexit)); for (;;) { if (stopped_vcpu == -1) { debug("$vCPU %d reporting breakpoint at rip %#lx\n", vcpuid, guest_pc(vmexit)); stopped_vcpu = vcpuid; gdb_suspend_vcpus(); } _gdb_cpu_suspend(vcpu, true); if (!vs->hit_swbreak) { /* Breakpoint reported. */ break; } bp = find_breakpoint(gpa); if (bp == NULL) { /* Breakpoint was removed. */ vs->hit_swbreak = false; break; } } gdb_cpu_resume(vcpu); } else { debug("$vCPU %d injecting breakpoint at rip %#lx\n", vcpuid, guest_pc(vmexit)); error = vm_set_register(vcpu, VM_REG_GUEST_ENTRY_INST_LENGTH, vmexit->u.bpt.inst_length); assert(error == 0); error = vm_inject_exception(vcpu, IDT_BP, 0, 0, 0); assert(error == 0); } pthread_mutex_unlock(&gdb_lock); } static bool gdb_step_vcpu(struct vcpu *vcpu) { int error, val, vcpuid; vcpuid = vcpu_id(vcpu); debug("$vCPU %d step\n", vcpuid); error = vm_get_capability(vcpu, VM_CAP_MTRAP_EXIT, &val); if (error < 0) return (false); discard_stop(); vcpu_state[vcpuid].stepping = true; vm_resume_cpu(vcpu); CPU_CLR(vcpuid, &vcpus_suspended); pthread_cond_broadcast(&idle_vcpus); return (true); } static void gdb_resume_vcpus(void) { assert(pthread_mutex_isowned_np(&gdb_lock)); vm_resume_all_cpus(ctx); debug("resuming all CPUs\n"); CPU_ZERO(&vcpus_suspended); pthread_cond_broadcast(&idle_vcpus); } static void gdb_read_regs(void) { uint64_t regvals[nitems(gdb_regset)]; int regnums[nitems(gdb_regset)]; for (size_t i = 0; i < nitems(gdb_regset); i++) regnums[i] = gdb_regset[i].id; if (vm_get_register_set(vcpus[cur_vcpu], nitems(gdb_regset), regnums, regvals) == -1) { send_error(errno); return; } start_packet(); for (size_t i = 0; i < nitems(gdb_regset); i++) append_unsigned_native(regvals[i], gdb_regset[i].size); finish_packet(); } static void gdb_read_one_reg(const uint8_t *data, size_t len) { uint64_t regval; uintmax_t reg; reg = parse_integer(data, len); if (reg >= nitems(gdb_regset)) { send_error(EINVAL); return; } if (vm_get_register(vcpus[cur_vcpu], gdb_regset[reg].id, ®val) == -1) { send_error(errno); return; } start_packet(); append_unsigned_native(regval, gdb_regset[reg].size); finish_packet(); } static void gdb_read_mem(const uint8_t *data, size_t len) { uint64_t gpa, gva, val; uint8_t *cp; size_t resid, todo, bytes; bool started; int error; assert(len >= 1); /* Skip 'm' */ data += 1; len -= 1; /* Parse and consume address. */ cp = memchr(data, ',', len); if (cp == NULL || cp == data) { send_error(EINVAL); return; } gva = parse_integer(data, cp - data); len -= (cp - data) + 1; data += (cp - data) + 1; /* Parse length. */ resid = parse_integer(data, len); started = false; while (resid > 0) { error = guest_vaddr2paddr(vcpus[cur_vcpu], gva, &gpa); if (error == -1) { if (started) finish_packet(); else send_error(errno); return; } if (error == 0) { if (started) finish_packet(); else send_error(EFAULT); return; } /* Read bytes from current page. */ todo = getpagesize() - gpa % getpagesize(); if (todo > resid) todo = resid; cp = paddr_guest2host(ctx, gpa, todo); if (cp != NULL) { /* * If this page is guest RAM, read it a byte * at a time. */ if (!started) { start_packet(); started = true; } while (todo > 0) { append_byte(*cp); cp++; gpa++; gva++; resid--; todo--; } } else { /* * If this page isn't guest RAM, try to handle * it via MMIO. For MMIO requests, use * aligned reads of words when possible. */ while (todo > 0) { if (gpa & 1 || todo == 1) bytes = 1; else if (gpa & 2 || todo == 2) bytes = 2; else bytes = 4; error = read_mem(vcpus[cur_vcpu], gpa, &val, bytes); if (error == 0) { if (!started) { start_packet(); started = true; } gpa += bytes; gva += bytes; resid -= bytes; todo -= bytes; while (bytes > 0) { append_byte(val); val >>= 8; bytes--; } } else { if (started) finish_packet(); else send_error(EFAULT); return; } } } assert(resid == 0 || gpa % getpagesize() == 0); } if (!started) start_packet(); finish_packet(); } static void gdb_write_mem(const uint8_t *data, size_t len) { uint64_t gpa, gva, val; uint8_t *cp; size_t resid, todo, bytes; int error; assert(len >= 1); /* Skip 'M' */ data += 1; len -= 1; /* Parse and consume address. */ cp = memchr(data, ',', len); if (cp == NULL || cp == data) { send_error(EINVAL); return; } gva = parse_integer(data, cp - data); len -= (cp - data) + 1; data += (cp - data) + 1; /* Parse and consume length. */ cp = memchr(data, ':', len); if (cp == NULL || cp == data) { send_error(EINVAL); return; } resid = parse_integer(data, cp - data); len -= (cp - data) + 1; data += (cp - data) + 1; /* Verify the available bytes match the length. */ if (len != resid * 2) { send_error(EINVAL); return; } while (resid > 0) { error = guest_vaddr2paddr(vcpus[cur_vcpu], gva, &gpa); if (error == -1) { send_error(errno); return; } if (error == 0) { send_error(EFAULT); return; } /* Write bytes to current page. */ todo = getpagesize() - gpa % getpagesize(); if (todo > resid) todo = resid; cp = paddr_guest2host(ctx, gpa, todo); if (cp != NULL) { /* * If this page is guest RAM, write it a byte * at a time. */ while (todo > 0) { assert(len >= 2); *cp = parse_byte(data); data += 2; len -= 2; cp++; gpa++; gva++; resid--; todo--; } } else { /* * If this page isn't guest RAM, try to handle * it via MMIO. For MMIO requests, use * aligned writes of words when possible. */ while (todo > 0) { if (gpa & 1 || todo == 1) { bytes = 1; val = parse_byte(data); } else if (gpa & 2 || todo == 2) { bytes = 2; val = be16toh(parse_integer(data, 4)); } else { bytes = 4; val = be32toh(parse_integer(data, 8)); } error = write_mem(vcpus[cur_vcpu], gpa, val, bytes); if (error == 0) { gpa += bytes; gva += bytes; resid -= bytes; todo -= bytes; data += 2 * bytes; len -= 2 * bytes; } else { send_error(EFAULT); return; } } } assert(resid == 0 || gpa % getpagesize() == 0); } assert(len == 0); send_ok(); } static bool set_breakpoint_caps(bool enable) { cpuset_t mask; int vcpu; mask = vcpus_active; while (!CPU_EMPTY(&mask)) { vcpu = CPU_FFS(&mask) - 1; CPU_CLR(vcpu, &mask); if (vm_set_capability(vcpus[vcpu], VM_CAP_BPT_EXIT, enable ? 1 : 0) < 0) return (false); debug("$vCPU %d %sabled breakpoint exits\n", vcpu, enable ? "en" : "dis"); } return (true); } static void remove_all_sw_breakpoints(void) { struct breakpoint *bp, *nbp; uint8_t *cp; if (TAILQ_EMPTY(&breakpoints)) return; TAILQ_FOREACH_SAFE(bp, &breakpoints, link, nbp) { debug("remove breakpoint at %#lx\n", bp->gpa); cp = paddr_guest2host(ctx, bp->gpa, sizeof(bp->shadow_inst)); memcpy(cp, bp->shadow_inst, sizeof(bp->shadow_inst)); TAILQ_REMOVE(&breakpoints, bp, link); free(bp); } TAILQ_INIT(&breakpoints); set_breakpoint_caps(false); } static void update_sw_breakpoint(uint64_t gva, int kind, bool insert) { struct breakpoint *bp; uint64_t gpa; uint8_t *cp; int error; if (kind != GDB_BP_SIZE) { send_error(EINVAL); return; } error = guest_vaddr2paddr(vcpus[cur_vcpu], gva, &gpa); if (error == -1) { send_error(errno); return; } if (error == 0) { send_error(EFAULT); return; } cp = paddr_guest2host(ctx, gpa, sizeof(bp->shadow_inst)); /* Only permit breakpoints in guest RAM. */ if (cp == NULL) { send_error(EFAULT); return; } /* Find any existing breakpoint. */ bp = find_breakpoint(gpa); /* * Silently ignore duplicate commands since the protocol * requires these packets to be idempotent. */ if (insert) { if (bp == NULL) { if (TAILQ_EMPTY(&breakpoints) && !set_breakpoint_caps(true)) { send_empty_response(); return; } bp = malloc(sizeof(*bp)); bp->gpa = gpa; memcpy(bp->shadow_inst, cp, sizeof(bp->shadow_inst)); memcpy(cp, GDB_BP_INSTR, sizeof(bp->shadow_inst)); TAILQ_INSERT_TAIL(&breakpoints, bp, link); debug("new breakpoint at %#lx\n", gpa); } } else { if (bp != NULL) { debug("remove breakpoint at %#lx\n", gpa); memcpy(cp, bp->shadow_inst, sizeof(bp->shadow_inst)); TAILQ_REMOVE(&breakpoints, bp, link); free(bp); if (TAILQ_EMPTY(&breakpoints)) set_breakpoint_caps(false); } } send_ok(); } static void parse_breakpoint(const uint8_t *data, size_t len) { uint64_t gva; uint8_t *cp; bool insert; int kind, type; insert = data[0] == 'Z'; /* Skip 'Z/z' */ data += 1; len -= 1; /* Parse and consume type. */ cp = memchr(data, ',', len); if (cp == NULL || cp == data) { send_error(EINVAL); return; } type = parse_integer(data, cp - data); len -= (cp - data) + 1; data += (cp - data) + 1; /* Parse and consume address. */ cp = memchr(data, ',', len); if (cp == NULL || cp == data) { send_error(EINVAL); return; } gva = parse_integer(data, cp - data); len -= (cp - data) + 1; data += (cp - data) + 1; /* Parse and consume kind. */ cp = memchr(data, ';', len); if (cp == data) { send_error(EINVAL); return; } if (cp != NULL) { /* * We do not advertise support for either the * ConditionalBreakpoints or BreakpointCommands * features, so we should not be getting conditions or * commands from the remote end. */ send_empty_response(); return; } kind = parse_integer(data, len); data += len; len = 0; switch (type) { case 0: update_sw_breakpoint(gva, kind, insert); break; default: send_empty_response(); break; } } static bool command_equals(const uint8_t *data, size_t len, const char *cmd) { if (strlen(cmd) > len) return (false); return (memcmp(data, cmd, strlen(cmd)) == 0); } static void check_features(const uint8_t *data, size_t len) { char *feature, *next_feature, *str, *value; bool supported; str = malloc(len + 1); memcpy(str, data, len); str[len] = '\0'; next_feature = str; while ((feature = strsep(&next_feature, ";")) != NULL) { /* * Null features shouldn't exist, but skip if they * do. */ if (strcmp(feature, "") == 0) continue; /* * Look for the value or supported / not supported * flag. */ value = strchr(feature, '='); if (value != NULL) { *value = '\0'; value++; supported = true; } else { value = feature + strlen(feature) - 1; switch (*value) { case '+': supported = true; break; case '-': supported = false; break; default: /* * This is really a protocol error, * but we just ignore malformed * features for ease of * implementation. */ continue; } value = NULL; } if (strcmp(feature, "swbreak") == 0) swbreak_enabled = supported; #ifndef __FreeBSD__ /* * The compiler dislikes 'supported' being set but never used. * Make it happy here. */ if (supported) { debug("feature '%s' supported\n", feature); } #endif /* __FreeBSD__ */ } free(str); start_packet(); /* This is an arbitrary limit. */ append_string("PacketSize=4096"); append_string(";swbreak+"); finish_packet(); } static void gdb_query(const uint8_t *data, size_t len) { /* * TODO: * - qSearch */ if (command_equals(data, len, "qAttached")) { start_packet(); append_char('1'); finish_packet(); } else if (command_equals(data, len, "qC")) { start_packet(); append_string("QC"); append_integer(cur_vcpu + 1); finish_packet(); } else if (command_equals(data, len, "qfThreadInfo")) { cpuset_t mask; bool first; int vcpu; if (CPU_EMPTY(&vcpus_active)) { send_error(EINVAL); return; } mask = vcpus_active; start_packet(); append_char('m'); first = true; while (!CPU_EMPTY(&mask)) { vcpu = CPU_FFS(&mask) - 1; CPU_CLR(vcpu, &mask); if (first) first = false; else append_char(','); append_integer(vcpu + 1); } finish_packet(); } else if (command_equals(data, len, "qsThreadInfo")) { start_packet(); append_char('l'); finish_packet(); } else if (command_equals(data, len, "qSupported")) { data += strlen("qSupported"); len -= strlen("qSupported"); check_features(data, len); } else if (command_equals(data, len, "qThreadExtraInfo")) { char buf[16]; int tid; data += strlen("qThreadExtraInfo"); len -= strlen("qThreadExtraInfo"); if (len == 0 || *data != ',') { send_error(EINVAL); return; } tid = parse_threadid(data + 1, len - 1); if (tid <= 0 || !CPU_ISSET(tid - 1, &vcpus_active)) { send_error(EINVAL); return; } snprintf(buf, sizeof(buf), "vCPU %d", tid - 1); start_packet(); append_asciihex(buf); finish_packet(); } else send_empty_response(); } static void handle_command(const uint8_t *data, size_t len) { /* Reject packets with a sequence-id. */ if (len >= 3 && data[0] >= '0' && data[0] <= '9' && data[0] >= '0' && data[0] <= '9' && data[2] == ':') { send_empty_response(); return; } switch (*data) { case 'c': if (len != 1) { send_error(EINVAL); break; } discard_stop(); gdb_resume_vcpus(); break; case 'D': send_ok(); /* TODO: Resume any stopped CPUs. */ break; case 'g': gdb_read_regs(); break; case 'p': gdb_read_one_reg(data + 1, len - 1); break; case 'H': { int tid; if (len < 2 || (data[1] != 'g' && data[1] != 'c')) { send_error(EINVAL); break; } tid = parse_threadid(data + 2, len - 2); if (tid == -2) { send_error(EINVAL); break; } if (CPU_EMPTY(&vcpus_active)) { send_error(EINVAL); break; } if (tid == -1 || tid == 0) cur_vcpu = CPU_FFS(&vcpus_active) - 1; else if (CPU_ISSET(tid - 1, &vcpus_active)) cur_vcpu = tid - 1; else { send_error(EINVAL); break; } send_ok(); break; } case 'm': gdb_read_mem(data, len); break; case 'M': gdb_write_mem(data, len); break; case 'T': { int tid; tid = parse_threadid(data + 1, len - 1); if (tid <= 0 || !CPU_ISSET(tid - 1, &vcpus_active)) { send_error(EINVAL); return; } send_ok(); break; } case 'q': gdb_query(data, len); break; case 's': if (len != 1) { send_error(EINVAL); break; } /* Don't send a reply until a stop occurs. */ if (!gdb_step_vcpu(vcpus[cur_vcpu])) { send_error(EOPNOTSUPP); break; } break; case 'z': case 'Z': parse_breakpoint(data, len); break; case '?': report_stop(false); break; case 'G': /* TODO */ case 'v': /* Handle 'vCont' */ /* 'vCtrlC' */ case 'P': /* TODO */ case 'Q': /* TODO */ case 't': /* TODO */ case 'X': /* TODO */ default: send_empty_response(); } } /* Check for a valid packet in the command buffer. */ static void check_command(int fd) { uint8_t *head, *hash, *p, sum; size_t avail, plen; for (;;) { avail = cur_comm.len; if (avail == 0) return; head = io_buffer_head(&cur_comm); switch (*head) { case 0x03: debug("<- Ctrl-C\n"); io_buffer_consume(&cur_comm, 1); gdb_suspend_vcpus(); break; case '+': /* ACK of previous response. */ debug("<- +\n"); if (response_pending()) io_buffer_reset(&cur_resp); io_buffer_consume(&cur_comm, 1); if (stopped_vcpu != -1 && report_next_stop) { report_stop(true); send_pending_data(fd); } break; case '-': /* NACK of previous response. */ debug("<- -\n"); if (response_pending()) { cur_resp.len += cur_resp.start; cur_resp.start = 0; if (cur_resp.data[0] == '+') io_buffer_advance(&cur_resp, 1); debug("-> %.*s\n", (int)cur_resp.len, io_buffer_head(&cur_resp)); } io_buffer_consume(&cur_comm, 1); send_pending_data(fd); break; case '$': /* Packet. */ if (response_pending()) { warnx("New GDB command while response in " "progress"); io_buffer_reset(&cur_resp); } /* Is packet complete? */ hash = memchr(head, '#', avail); if (hash == NULL) return; plen = (hash - head + 1) + 2; if (avail < plen) return; debug("<- %.*s\n", (int)plen, head); /* Verify checksum. */ for (sum = 0, p = head + 1; p < hash; p++) sum += *p; if (sum != parse_byte(hash + 1)) { io_buffer_consume(&cur_comm, plen); debug("-> -\n"); send_char('-'); send_pending_data(fd); break; } send_char('+'); handle_command(head + 1, hash - (head + 1)); io_buffer_consume(&cur_comm, plen); if (!response_pending()) { debug("-> +\n"); } send_pending_data(fd); break; default: /* XXX: Possibly drop connection instead. */ debug("-> %02x\n", *head); io_buffer_consume(&cur_comm, 1); break; } } } static void gdb_readable(int fd, enum ev_type event __unused, void *arg __unused) { size_t pending; ssize_t nread; int n; if (ioctl(fd, FIONREAD, &n) == -1) { warn("FIONREAD on GDB socket"); return; } assert(n >= 0); pending = n; /* * 'pending' might be zero due to EOF. We need to call read * with a non-zero length to detect EOF. */ if (pending == 0) pending = 1; /* Ensure there is room in the command buffer. */ io_buffer_grow(&cur_comm, pending); assert(io_buffer_avail(&cur_comm) >= pending); nread = read(fd, io_buffer_tail(&cur_comm), io_buffer_avail(&cur_comm)); if (nread == 0) { close_connection(); } else if (nread == -1) { if (errno == EAGAIN) return; warn("Read from GDB socket"); close_connection(); } else { cur_comm.len += nread; pthread_mutex_lock(&gdb_lock); check_command(fd); pthread_mutex_unlock(&gdb_lock); } } static void gdb_writable(int fd, enum ev_type event __unused, void *arg __unused) { send_pending_data(fd); } static void new_connection(int fd, enum ev_type event __unused, void *arg) { int optval, s; s = accept4(fd, NULL, NULL, SOCK_NONBLOCK); if (s == -1) { if (arg != NULL) err(1, "Failed accepting initial GDB connection"); /* Silently ignore errors post-startup. */ return; } optval = 1; if (setsockopt(s, SOL_SOCKET, SO_NOSIGPIPE, &optval, sizeof(optval)) == -1) { warn("Failed to disable SIGPIPE for GDB connection"); close(s); return; } pthread_mutex_lock(&gdb_lock); if (cur_fd != -1) { close(s); warnx("Ignoring additional GDB connection."); } read_event = mevent_add(s, EVF_READ, gdb_readable, NULL); if (read_event == NULL) { if (arg != NULL) err(1, "Failed to setup initial GDB connection"); pthread_mutex_unlock(&gdb_lock); return; } write_event = mevent_add(s, EVF_WRITE, gdb_writable, NULL); if (write_event == NULL) { if (arg != NULL) err(1, "Failed to setup initial GDB connection"); mevent_delete_close(read_event); read_event = NULL; } cur_fd = s; cur_vcpu = 0; stopped_vcpu = -1; /* Break on attach. */ first_stop = true; report_next_stop = false; gdb_suspend_vcpus(); pthread_mutex_unlock(&gdb_lock); } #ifndef WITHOUT_CAPSICUM static void limit_gdb_socket(int s) { cap_rights_t rights; unsigned long ioctls[] = { FIONREAD }; cap_rights_init(&rights, CAP_ACCEPT, CAP_EVENT, CAP_READ, CAP_WRITE, CAP_SETSOCKOPT, CAP_IOCTL); if (caph_rights_limit(s, &rights) == -1) errx(EX_OSERR, "Unable to apply rights for sandbox"); if (caph_ioctls_limit(s, ioctls, nitems(ioctls)) == -1) errx(EX_OSERR, "Unable to apply rights for sandbox"); } #endif #ifndef __FreeBSD__ /* * Equivalent to init_gdb() below, but without configuring the listening socket. * This will allow the bhyve process to tolerate mdb attaching/detaching from * the instance while it is running. */ void init_mdb(struct vmctx *_ctx) { int error; bool wait; wait = get_config_bool_default("gdb.wait", false); error = pthread_mutex_init(&gdb_lock, NULL); if (error != 0) errc(1, error, "gdb mutex init"); error = pthread_cond_init(&idle_vcpus, NULL); if (error != 0) errc(1, error, "gdb cv init"); ctx = _ctx; stopped_vcpu = -1; TAILQ_INIT(&breakpoints); vcpu_state = calloc(guest_ncpus, sizeof(*vcpu_state)); if (wait) { /* * Set vcpu 0 in vcpus_suspended. This will trigger the * logic in gdb_cpu_add() to suspend the first vcpu before * it starts execution. The vcpu will remain suspended * until a debugger connects. */ CPU_SET(0, &vcpus_suspended); stopped_vcpu = 0; } } #endif void init_gdb(struct vmctx *_ctx) { int error, flags, optval, s; struct addrinfo hints; struct addrinfo *gdbaddr; const char *saddr, *value; char *sport; bool wait; value = get_config_value("gdb.port"); if (value == NULL) return; sport = strdup(value); if (sport == NULL) errx(4, "Failed to allocate memory"); wait = get_config_bool_default("gdb.wait", false); saddr = get_config_value("gdb.address"); if (saddr == NULL) { saddr = "localhost"; } debug("==> starting on %s:%s, %swaiting\n", saddr, sport, wait ? "" : "not "); error = pthread_mutex_init(&gdb_lock, NULL); if (error != 0) errc(1, error, "gdb mutex init"); error = pthread_cond_init(&idle_vcpus, NULL); if (error != 0) errc(1, error, "gdb cv init"); memset(&hints, 0, sizeof(hints)); hints.ai_family = AF_UNSPEC; hints.ai_socktype = SOCK_STREAM; hints.ai_flags = AI_NUMERICSERV | AI_PASSIVE; error = getaddrinfo(saddr, sport, &hints, &gdbaddr); if (error != 0) errx(1, "gdb address resolution: %s", gai_strerror(error)); ctx = _ctx; s = socket(gdbaddr->ai_family, gdbaddr->ai_socktype, 0); if (s < 0) err(1, "gdb socket create"); optval = 1; (void)setsockopt(s, SOL_SOCKET, SO_REUSEADDR, &optval, sizeof(optval)); if (bind(s, gdbaddr->ai_addr, gdbaddr->ai_addrlen) < 0) err(1, "gdb socket bind"); if (listen(s, 1) < 0) err(1, "gdb socket listen"); stopped_vcpu = -1; TAILQ_INIT(&breakpoints); vcpus = calloc(guest_ncpus, sizeof(*vcpus)); vcpu_state = calloc(guest_ncpus, sizeof(*vcpu_state)); if (wait) { /* * Set vcpu 0 in vcpus_suspended. This will trigger the * logic in gdb_cpu_add() to suspend the first vcpu before * it starts execution. The vcpu will remain suspended * until a debugger connects. */ CPU_SET(0, &vcpus_suspended); stopped_vcpu = 0; } flags = fcntl(s, F_GETFL); if (fcntl(s, F_SETFL, flags | O_NONBLOCK) == -1) err(1, "Failed to mark gdb socket non-blocking"); #ifndef WITHOUT_CAPSICUM limit_gdb_socket(s); #endif mevent_add(s, EVF_READ, new_connection, NULL); gdb_active = true; freeaddrinfo(gdbaddr); free(sport); } /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2017 John H. Baldwin * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef __GDB_H__ #define __GDB_H__ void gdb_cpu_add(struct vcpu *vcpu); void gdb_cpu_breakpoint(struct vcpu *vcpu, struct vm_exit *vmexit); void gdb_cpu_mtrap(struct vcpu *vcpu); void gdb_cpu_suspend(struct vcpu *vcpu); void init_gdb(struct vmctx *ctx); #ifndef __FreeBSD__ void init_mdb(struct vmctx *ctx); #endif #endif /* !__GDB_H__ */ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2016 Alex Teaca * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * */ #include #include #include #include "pci_hda.h" #include "audio.h" /* * HDA Codec defines */ #define INTEL_VENDORID 0x8086 #define HDA_CODEC_SUBSYSTEM_ID ((INTEL_VENDORID << 16) | 0x01) #define HDA_CODEC_ROOT_NID 0x00 #define HDA_CODEC_FG_NID 0x01 #define HDA_CODEC_AUDIO_OUTPUT_NID 0x02 #define HDA_CODEC_PIN_OUTPUT_NID 0x03 #define HDA_CODEC_AUDIO_INPUT_NID 0x04 #define HDA_CODEC_PIN_INPUT_NID 0x05 #define HDA_CODEC_STREAMS_COUNT 0x02 #define HDA_CODEC_STREAM_OUTPUT 0x00 #define HDA_CODEC_STREAM_INPUT 0x01 #define HDA_CODEC_PARAMS_COUNT 0x14 #define HDA_CODEC_CONN_LIST_COUNT 0x01 #define HDA_CODEC_RESPONSE_EX_UNSOL 0x10 #define HDA_CODEC_RESPONSE_EX_SOL 0x00 #define HDA_CODEC_AMP_NUMSTEPS 0x4a #define HDA_CODEC_SUPP_STREAM_FORMATS_PCM \ (1 << HDA_PARAM_SUPP_STREAM_FORMATS_PCM_SHIFT) #define HDA_CODEC_FMT_BASE_MASK (0x01 << 14) #define HDA_CODEC_FMT_MULT_MASK (0x07 << 11) #define HDA_CODEC_FMT_MULT_2 (0x01 << 11) #define HDA_CODEC_FMT_MULT_3 (0x02 << 11) #define HDA_CODEC_FMT_MULT_4 (0x03 << 11) #define HDA_CODEC_FMT_DIV_MASK 0x07 #define HDA_CODEC_FMT_DIV_SHIFT 8 #define HDA_CODEC_FMT_BITS_MASK (0x07 << 4) #define HDA_CODEC_FMT_BITS_8 (0x00 << 4) #define HDA_CODEC_FMT_BITS_16 (0x01 << 4) #define HDA_CODEC_FMT_BITS_24 (0x03 << 4) #define HDA_CODEC_FMT_BITS_32 (0x04 << 4) #define HDA_CODEC_FMT_CHAN_MASK (0x0f << 0) #define HDA_CODEC_AUDIO_WCAP_OUTPUT \ (0x00 << HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_SHIFT) #define HDA_CODEC_AUDIO_WCAP_INPUT \ (0x01 << HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_SHIFT) #define HDA_CODEC_AUDIO_WCAP_PIN \ (0x04 << HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_SHIFT) #define HDA_CODEC_AUDIO_WCAP_CONN_LIST \ (1 << HDA_PARAM_AUDIO_WIDGET_CAP_CONN_LIST_SHIFT) #define HDA_CODEC_AUDIO_WCAP_FORMAT_OVR \ (1 << HDA_PARAM_AUDIO_WIDGET_CAP_FORMAT_OVR_SHIFT) #define HDA_CODEC_AUDIO_WCAP_AMP_OVR \ (1 << HDA_PARAM_AUDIO_WIDGET_CAP_AMP_OVR_SHIFT) #define HDA_CODEC_AUDIO_WCAP_OUT_AMP \ (1 << HDA_PARAM_AUDIO_WIDGET_CAP_OUT_AMP_SHIFT) #define HDA_CODEC_AUDIO_WCAP_IN_AMP \ (1 << HDA_PARAM_AUDIO_WIDGET_CAP_IN_AMP_SHIFT) #define HDA_CODEC_AUDIO_WCAP_STEREO \ (1 << HDA_PARAM_AUDIO_WIDGET_CAP_STEREO_SHIFT) #define HDA_CODEC_PIN_CAP_OUTPUT \ (1 << HDA_PARAM_PIN_CAP_OUTPUT_CAP_SHIFT) #define HDA_CODEC_PIN_CAP_INPUT \ (1 << HDA_PARAM_PIN_CAP_INPUT_CAP_SHIFT) #define HDA_CODEC_PIN_CAP_PRESENCE_DETECT \ (1 << HDA_PARAM_PIN_CAP_PRESENCE_DETECT_CAP_SHIFT) #define HDA_CODEC_OUTPUT_AMP_CAP_MUTE_CAP \ (1 << HDA_PARAM_OUTPUT_AMP_CAP_MUTE_CAP_SHIFT) #define HDA_CODEC_OUTPUT_AMP_CAP_STEPSIZE \ (0x03 << HDA_PARAM_OUTPUT_AMP_CAP_STEPSIZE_SHIFT) #define HDA_CODEC_OUTPUT_AMP_CAP_NUMSTEPS \ (HDA_CODEC_AMP_NUMSTEPS << HDA_PARAM_OUTPUT_AMP_CAP_NUMSTEPS_SHIFT) #define HDA_CODEC_OUTPUT_AMP_CAP_OFFSET \ (HDA_CODEC_AMP_NUMSTEPS << HDA_PARAM_OUTPUT_AMP_CAP_OFFSET_SHIFT) #define HDA_CODEC_SET_AMP_GAIN_MUTE_MUTE 0x80 #define HDA_CODEC_SET_AMP_GAIN_MUTE_GAIN_MASK 0x7f #define HDA_CODEC_PIN_SENSE_PRESENCE_PLUGGED (1 << 31) #define HDA_CODEC_PIN_WIDGET_CTRL_OUT_ENABLE \ (1 << HDA_CMD_GET_PIN_WIDGET_CTRL_OUT_ENABLE_SHIFT) #define HDA_CODEC_PIN_WIDGET_CTRL_IN_ENABLE \ (1 << HDA_CMD_GET_PIN_WIDGET_CTRL_IN_ENABLE_SHIFT) #define HDA_CONFIG_DEFAULTCONF_COLOR_BLACK \ (0x01 << HDA_CONFIG_DEFAULTCONF_COLOR_SHIFT) #define HDA_CONFIG_DEFAULTCONF_COLOR_RED \ (0x05 << HDA_CONFIG_DEFAULTCONF_COLOR_SHIFT) #define HDA_CODEC_BUF_SIZE HDA_FIFO_SIZE #define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0])) /* * HDA Audio Context data structures */ typedef void (*transfer_func_t)(void *arg); typedef int (*setup_func_t)(void *arg); struct hda_audio_ctxt { char name[64]; uint8_t run; uint8_t started; void *priv; pthread_t tid; pthread_mutex_t mtx; pthread_cond_t cond; setup_func_t do_setup; transfer_func_t do_transfer; }; /* * HDA Audio Context module function declarations */ static void *hda_audio_ctxt_thr(void *arg); static int hda_audio_ctxt_init(struct hda_audio_ctxt *actx, const char *tname, transfer_func_t do_transfer, setup_func_t do_setup, void *priv); static int hda_audio_ctxt_start(struct hda_audio_ctxt *actx); static int hda_audio_ctxt_stop(struct hda_audio_ctxt *actx); /* * HDA Codec data structures */ struct hda_codec_softc; typedef uint32_t (*verb_func_t)(struct hda_codec_softc *sc, uint16_t verb, uint16_t payload); struct hda_codec_stream { uint8_t buf[HDA_CODEC_BUF_SIZE]; uint8_t channel; uint16_t fmt; uint8_t stream; uint8_t left_gain; uint8_t right_gain; uint8_t left_mute; uint8_t right_mute; struct audio *aud; struct hda_audio_ctxt actx; }; struct hda_codec_softc { uint32_t no_nodes; uint32_t subsystem_id; const uint32_t (*get_parameters)[HDA_CODEC_PARAMS_COUNT]; const uint8_t (*conn_list)[HDA_CODEC_CONN_LIST_COUNT]; const uint32_t *conf_default; const uint8_t *pin_ctrl_default; const verb_func_t *verb_handlers; struct hda_codec_inst *hci; struct hda_codec_stream streams[HDA_CODEC_STREAMS_COUNT]; }; /* * HDA Codec module function declarations */ static int hda_codec_init(struct hda_codec_inst *hci, const char *play, const char *rec); static int hda_codec_reset(struct hda_codec_inst *hci); static int hda_codec_command(struct hda_codec_inst *hci, uint32_t cmd_data); static int hda_codec_notify(struct hda_codec_inst *hci, uint8_t run, uint8_t stream, uint8_t dir); static int hda_codec_parse_format(uint16_t fmt, struct audio_params *params); static uint32_t hda_codec_audio_output_nid(struct hda_codec_softc *sc, uint16_t verb, uint16_t payload); static void hda_codec_audio_output_do_transfer(void *arg); static int hda_codec_audio_output_do_setup(void *arg); static uint32_t hda_codec_audio_input_nid(struct hda_codec_softc *sc, uint16_t verb, uint16_t payload); static void hda_codec_audio_input_do_transfer(void *arg); static int hda_codec_audio_input_do_setup(void *arg); static uint32_t hda_codec_audio_inout_nid(struct hda_codec_stream *st, uint16_t verb, uint16_t payload); /* * HDA Codec global data */ #define HDA_CODEC_ROOT_DESC \ [HDA_CODEC_ROOT_NID] = { \ [HDA_PARAM_VENDOR_ID] = INTEL_VENDORID, \ [HDA_PARAM_REVISION_ID] = 0xffff, \ /* 1 Subnode, StartNid = 1 */ \ [HDA_PARAM_SUB_NODE_COUNT] = 0x00010001, \ }, \ #define HDA_CODEC_FG_COMMON_DESC \ [HDA_PARAM_FCT_GRP_TYPE] = HDA_PARAM_FCT_GRP_TYPE_NODE_TYPE_AUDIO,\ /* B8 - B32, 8.0 - 192.0kHz */ \ [HDA_PARAM_SUPP_PCM_SIZE_RATE] = (0x1f << 16) | 0x7ff, \ [HDA_PARAM_SUPP_STREAM_FORMATS] = HDA_CODEC_SUPP_STREAM_FORMATS_PCM,\ [HDA_PARAM_INPUT_AMP_CAP] = 0x00, /* None */ \ [HDA_PARAM_OUTPUT_AMP_CAP] = 0x00, /* None */ \ [HDA_PARAM_GPIO_COUNT] = 0x00, \ #define HDA_CODEC_FG_OUTPUT_DESC \ [HDA_CODEC_FG_NID] = { \ /* 2 Subnodes, StartNid = 2 */ \ [HDA_PARAM_SUB_NODE_COUNT] = 0x00020002, \ HDA_CODEC_FG_COMMON_DESC \ }, \ #define HDA_CODEC_FG_INPUT_DESC \ [HDA_CODEC_FG_NID] = { \ /* 2 Subnodes, StartNid = 4 */ \ [HDA_PARAM_SUB_NODE_COUNT] = 0x00040002, \ HDA_CODEC_FG_COMMON_DESC \ }, \ #define HDA_CODEC_FG_DUPLEX_DESC \ [HDA_CODEC_FG_NID] = { \ /* 4 Subnodes, StartNid = 2 */ \ [HDA_PARAM_SUB_NODE_COUNT] = 0x00020004, \ HDA_CODEC_FG_COMMON_DESC \ }, \ #define HDA_CODEC_OUTPUT_DESC \ [HDA_CODEC_AUDIO_OUTPUT_NID] = { \ [HDA_PARAM_AUDIO_WIDGET_CAP] = \ HDA_CODEC_AUDIO_WCAP_OUTPUT | \ HDA_CODEC_AUDIO_WCAP_FORMAT_OVR | \ HDA_CODEC_AUDIO_WCAP_AMP_OVR | \ HDA_CODEC_AUDIO_WCAP_OUT_AMP | \ HDA_CODEC_AUDIO_WCAP_STEREO, \ /* B16, 16.0 - 192.0kHz */ \ [HDA_PARAM_SUPP_PCM_SIZE_RATE] = (0x02 << 16) | 0x7fc, \ [HDA_PARAM_SUPP_STREAM_FORMATS] = \ HDA_CODEC_SUPP_STREAM_FORMATS_PCM, \ [HDA_PARAM_INPUT_AMP_CAP] = 0x00, /* None */ \ [HDA_PARAM_CONN_LIST_LENGTH] = 0x00, \ [HDA_PARAM_OUTPUT_AMP_CAP] = \ HDA_CODEC_OUTPUT_AMP_CAP_MUTE_CAP | \ HDA_CODEC_OUTPUT_AMP_CAP_STEPSIZE | \ HDA_CODEC_OUTPUT_AMP_CAP_NUMSTEPS | \ HDA_CODEC_OUTPUT_AMP_CAP_OFFSET, \ }, \ [HDA_CODEC_PIN_OUTPUT_NID] = { \ [HDA_PARAM_AUDIO_WIDGET_CAP] = \ HDA_CODEC_AUDIO_WCAP_PIN | \ HDA_CODEC_AUDIO_WCAP_CONN_LIST | \ HDA_CODEC_AUDIO_WCAP_STEREO, \ [HDA_PARAM_PIN_CAP] = HDA_CODEC_PIN_CAP_OUTPUT | \ HDA_CODEC_PIN_CAP_PRESENCE_DETECT,\ [HDA_PARAM_INPUT_AMP_CAP] = 0x00, /* None */ \ [HDA_PARAM_CONN_LIST_LENGTH] = 0x01, \ [HDA_PARAM_OUTPUT_AMP_CAP] = 0x00, /* None */ \ }, \ #define HDA_CODEC_INPUT_DESC \ [HDA_CODEC_AUDIO_INPUT_NID] = { \ [HDA_PARAM_AUDIO_WIDGET_CAP] = \ HDA_CODEC_AUDIO_WCAP_INPUT | \ HDA_CODEC_AUDIO_WCAP_CONN_LIST | \ HDA_CODEC_AUDIO_WCAP_FORMAT_OVR | \ HDA_CODEC_AUDIO_WCAP_AMP_OVR | \ HDA_CODEC_AUDIO_WCAP_IN_AMP | \ HDA_CODEC_AUDIO_WCAP_STEREO, \ /* B16, 16.0 - 192.0kHz */ \ [HDA_PARAM_SUPP_PCM_SIZE_RATE] = (0x02 << 16) | 0x7fc, \ [HDA_PARAM_SUPP_STREAM_FORMATS] = \ HDA_CODEC_SUPP_STREAM_FORMATS_PCM, \ [HDA_PARAM_OUTPUT_AMP_CAP] = 0x00, /* None */ \ [HDA_PARAM_CONN_LIST_LENGTH] = 0x01, \ [HDA_PARAM_INPUT_AMP_CAP] = \ HDA_CODEC_OUTPUT_AMP_CAP_MUTE_CAP | \ HDA_CODEC_OUTPUT_AMP_CAP_STEPSIZE | \ HDA_CODEC_OUTPUT_AMP_CAP_NUMSTEPS | \ HDA_CODEC_OUTPUT_AMP_CAP_OFFSET, \ }, \ [HDA_CODEC_PIN_INPUT_NID] = { \ [HDA_PARAM_AUDIO_WIDGET_CAP] = \ HDA_CODEC_AUDIO_WCAP_PIN | \ HDA_CODEC_AUDIO_WCAP_STEREO, \ [HDA_PARAM_PIN_CAP] = HDA_CODEC_PIN_CAP_INPUT | \ HDA_CODEC_PIN_CAP_PRESENCE_DETECT, \ [HDA_PARAM_INPUT_AMP_CAP] = 0x00, /* None */ \ [HDA_PARAM_OUTPUT_AMP_CAP] = 0x00, /* None */ \ }, \ static const uint32_t hda_codec_output_parameters[][HDA_CODEC_PARAMS_COUNT] = { HDA_CODEC_ROOT_DESC HDA_CODEC_FG_OUTPUT_DESC HDA_CODEC_OUTPUT_DESC }; static const uint32_t hda_codec_input_parameters[][HDA_CODEC_PARAMS_COUNT] = { HDA_CODEC_ROOT_DESC HDA_CODEC_FG_INPUT_DESC HDA_CODEC_INPUT_DESC }; static const uint32_t hda_codec_duplex_parameters[][HDA_CODEC_PARAMS_COUNT] = { HDA_CODEC_ROOT_DESC HDA_CODEC_FG_DUPLEX_DESC HDA_CODEC_OUTPUT_DESC HDA_CODEC_INPUT_DESC }; #define HDA_CODEC_NODES_COUNT (ARRAY_SIZE(hda_codec_duplex_parameters)) static const uint8_t hda_codec_conn_list[HDA_CODEC_NODES_COUNT][HDA_CODEC_CONN_LIST_COUNT] = { [HDA_CODEC_PIN_OUTPUT_NID] = {HDA_CODEC_AUDIO_OUTPUT_NID}, [HDA_CODEC_AUDIO_INPUT_NID] = {HDA_CODEC_PIN_INPUT_NID}, }; static const uint32_t hda_codec_conf_default[HDA_CODEC_NODES_COUNT] = { [HDA_CODEC_PIN_OUTPUT_NID] = \ HDA_CONFIG_DEFAULTCONF_CONNECTIVITY_JACK | HDA_CONFIG_DEFAULTCONF_DEVICE_LINE_OUT | HDA_CONFIG_DEFAULTCONF_COLOR_BLACK | (0x01 << HDA_CONFIG_DEFAULTCONF_ASSOCIATION_SHIFT), [HDA_CODEC_PIN_INPUT_NID] = HDA_CONFIG_DEFAULTCONF_CONNECTIVITY_JACK | HDA_CONFIG_DEFAULTCONF_DEVICE_LINE_IN | HDA_CONFIG_DEFAULTCONF_COLOR_RED | (0x02 << HDA_CONFIG_DEFAULTCONF_ASSOCIATION_SHIFT), }; static const uint8_t hda_codec_pin_ctrl_default[HDA_CODEC_NODES_COUNT] = { [HDA_CODEC_PIN_OUTPUT_NID] = HDA_CODEC_PIN_WIDGET_CTRL_OUT_ENABLE, [HDA_CODEC_PIN_INPUT_NID] = HDA_CODEC_PIN_WIDGET_CTRL_IN_ENABLE, }; static const verb_func_t hda_codec_verb_handlers[HDA_CODEC_NODES_COUNT] = { [HDA_CODEC_AUDIO_OUTPUT_NID] = hda_codec_audio_output_nid, [HDA_CODEC_AUDIO_INPUT_NID] = hda_codec_audio_input_nid, }; /* * HDA Codec module function definitions */ static int hda_codec_init(struct hda_codec_inst *hci, const char *play, const char *rec) { struct hda_codec_softc *sc = NULL; struct hda_codec_stream *st = NULL; int err; if (!(play || rec)) return (-1); sc = calloc(1, sizeof(*sc)); if (!sc) return (-1); if (play && rec) sc->get_parameters = hda_codec_duplex_parameters; else { if (play) sc->get_parameters = hda_codec_output_parameters; else sc->get_parameters = hda_codec_input_parameters; } sc->subsystem_id = HDA_CODEC_SUBSYSTEM_ID; sc->no_nodes = HDA_CODEC_NODES_COUNT; sc->conn_list = hda_codec_conn_list; sc->conf_default = hda_codec_conf_default; sc->pin_ctrl_default = hda_codec_pin_ctrl_default; sc->verb_handlers = hda_codec_verb_handlers; DPRINTF("HDA Codec nodes: %d", sc->no_nodes); /* * Initialize the Audio Output stream */ if (play) { st = &sc->streams[HDA_CODEC_STREAM_OUTPUT]; err = hda_audio_ctxt_init(&st->actx, "hda-audio-output", hda_codec_audio_output_do_transfer, hda_codec_audio_output_do_setup, sc); assert(!err); st->aud = audio_init(play, 1); if (!st->aud) { DPRINTF("Fail to init the output audio player"); return (-1); } } /* * Initialize the Audio Input stream */ if (rec) { st = &sc->streams[HDA_CODEC_STREAM_INPUT]; err = hda_audio_ctxt_init(&st->actx, "hda-audio-input", hda_codec_audio_input_do_transfer, hda_codec_audio_input_do_setup, sc); assert(!err); st->aud = audio_init(rec, 0); if (!st->aud) { DPRINTF("Fail to init the input audio player"); return (-1); } } sc->hci = hci; hci->priv = sc; return (0); } static int hda_codec_reset(struct hda_codec_inst *hci) { const struct hda_ops *hops = NULL; struct hda_codec_softc *sc = NULL; struct hda_codec_stream *st = NULL; int i; assert(hci); hops = hci->hops; assert(hops); sc = (struct hda_codec_softc *)hci->priv; assert(sc); for (i = 0; i < HDA_CODEC_STREAMS_COUNT; i++) { st = &sc->streams[i]; st->left_gain = HDA_CODEC_AMP_NUMSTEPS; st->right_gain = HDA_CODEC_AMP_NUMSTEPS; st->left_mute = HDA_CODEC_SET_AMP_GAIN_MUTE_MUTE; st->right_mute = HDA_CODEC_SET_AMP_GAIN_MUTE_MUTE; } DPRINTF("cad: 0x%x", hci->cad); if (!hops->signal) { DPRINTF("The controller ops does not implement \ the signal function"); return (-1); } return (hops->signal(hci)); } static int hda_codec_command(struct hda_codec_inst *hci, uint32_t cmd_data) { const struct hda_ops *hops = NULL; struct hda_codec_softc *sc = NULL; uint8_t cad = 0, nid = 0; uint16_t verb = 0, payload = 0; uint32_t res = 0; /* 4 bits */ cad = (cmd_data >> HDA_CMD_CAD_SHIFT) & 0x0f; /* 8 bits */ nid = (cmd_data >> HDA_CMD_NID_SHIFT) & 0xff; if ((cmd_data & 0x70000) == 0x70000) { /* 12 bits */ verb = (cmd_data >> HDA_CMD_VERB_12BIT_SHIFT) & 0x0fff; /* 8 bits */ payload = cmd_data & 0xff; } else { /* 4 bits */ verb = (cmd_data >> HDA_CMD_VERB_4BIT_SHIFT) & 0x0f; /* 16 bits */ payload = cmd_data & 0xffff; } assert(hci); hops = hci->hops; assert(hops); sc = (struct hda_codec_softc *)hci->priv; assert(sc); if (cad != hci->cad || nid >= sc->no_nodes) { DPRINTF("Invalid command data"); return (-1); } if (!hops->response) { DPRINTF("The controller ops does not implement \ the response function"); return (-1); } switch (verb) { case HDA_CMD_VERB_GET_PARAMETER: if (payload < HDA_CODEC_PARAMS_COUNT) res = sc->get_parameters[nid][payload]; break; case HDA_CMD_VERB_GET_CONN_LIST_ENTRY: res = sc->conn_list[nid][0]; break; case HDA_CMD_VERB_GET_PIN_WIDGET_CTRL: res = sc->pin_ctrl_default[nid]; break; case HDA_CMD_VERB_GET_PIN_SENSE: res = HDA_CODEC_PIN_SENSE_PRESENCE_PLUGGED; break; case HDA_CMD_VERB_GET_CONFIGURATION_DEFAULT: res = sc->conf_default[nid]; break; case HDA_CMD_VERB_GET_SUBSYSTEM_ID: res = sc->subsystem_id; break; default: assert(sc->verb_handlers); if (sc->verb_handlers[nid]) res = sc->verb_handlers[nid](sc, verb, payload); else DPRINTF("Unknown VERB: 0x%x", verb); break; } DPRINTF("cad: 0x%x nid: 0x%x verb: 0x%x payload: 0x%x response: 0x%x", cad, nid, verb, payload, res); return (hops->response(hci, res, HDA_CODEC_RESPONSE_EX_SOL)); } static int hda_codec_notify(struct hda_codec_inst *hci, uint8_t run, uint8_t stream, uint8_t dir) { struct hda_codec_softc *sc = NULL; struct hda_codec_stream *st = NULL; struct hda_audio_ctxt *actx = NULL; int i; int err; assert(hci); assert(stream); sc = (struct hda_codec_softc *)hci->priv; assert(sc); i = dir ? HDA_CODEC_STREAM_OUTPUT : HDA_CODEC_STREAM_INPUT; st = &sc->streams[i]; DPRINTF("run: %d, stream: 0x%x, st->stream: 0x%x dir: %d", run, stream, st->stream, dir); if (stream != st->stream) { DPRINTF("Stream not found"); return (0); } actx = &st->actx; if (run) err = hda_audio_ctxt_start(actx); else err = hda_audio_ctxt_stop(actx); return (err); } static int hda_codec_parse_format(uint16_t fmt, struct audio_params *params) { uint8_t div = 0; assert(params); /* Compute the Sample Rate */ params->rate = (fmt & HDA_CODEC_FMT_BASE_MASK) ? 44100 : 48000; switch (fmt & HDA_CODEC_FMT_MULT_MASK) { case HDA_CODEC_FMT_MULT_2: params->rate *= 2; break; case HDA_CODEC_FMT_MULT_3: params->rate *= 3; break; case HDA_CODEC_FMT_MULT_4: params->rate *= 4; break; } div = (fmt >> HDA_CODEC_FMT_DIV_SHIFT) & HDA_CODEC_FMT_DIV_MASK; params->rate /= (div + 1); /* Compute the Bits per Sample */ switch (fmt & HDA_CODEC_FMT_BITS_MASK) { case HDA_CODEC_FMT_BITS_8: params->format = AFMT_U8; break; case HDA_CODEC_FMT_BITS_16: params->format = AFMT_S16_LE; break; case HDA_CODEC_FMT_BITS_24: params->format = AFMT_S24_LE; break; case HDA_CODEC_FMT_BITS_32: params->format = AFMT_S32_LE; break; default: DPRINTF("Unknown format bits: 0x%x", fmt & HDA_CODEC_FMT_BITS_MASK); return (-1); } /* Compute the Number of Channels */ params->channels = (fmt & HDA_CODEC_FMT_CHAN_MASK) + 1; return (0); } static uint32_t hda_codec_audio_output_nid(struct hda_codec_softc *sc, uint16_t verb, uint16_t payload) { struct hda_codec_stream *st = &sc->streams[HDA_CODEC_STREAM_OUTPUT]; int res; res = hda_codec_audio_inout_nid(st, verb, payload); return (res); } static void hda_codec_audio_output_do_transfer(void *arg) { const struct hda_ops *hops = NULL; struct hda_codec_softc *sc = (struct hda_codec_softc *)arg; struct hda_codec_inst *hci = NULL; struct hda_codec_stream *st = NULL; struct audio *aud = NULL; int err; hci = sc->hci; assert(hci); hops = hci->hops; assert(hops); st = &sc->streams[HDA_CODEC_STREAM_OUTPUT]; aud = st->aud; err = hops->transfer(hci, st->stream, 1, st->buf, sizeof(st->buf)); if (err) return; err = audio_playback(aud, st->buf, sizeof(st->buf)); assert(!err); } static int hda_codec_audio_output_do_setup(void *arg) { struct hda_codec_softc *sc = (struct hda_codec_softc *)arg; struct hda_codec_stream *st = NULL; struct audio *aud = NULL; struct audio_params params; int err; st = &sc->streams[HDA_CODEC_STREAM_OUTPUT]; aud = st->aud; err = hda_codec_parse_format(st->fmt, ¶ms); if (err) return (-1); DPRINTF("rate: %d, channels: %d, format: 0x%x", params.rate, params.channels, params.format); return (audio_set_params(aud, ¶ms)); } static uint32_t hda_codec_audio_input_nid(struct hda_codec_softc *sc, uint16_t verb, uint16_t payload) { struct hda_codec_stream *st = &sc->streams[HDA_CODEC_STREAM_INPUT]; int res; res = hda_codec_audio_inout_nid(st, verb, payload); return (res); } static void hda_codec_audio_input_do_transfer(void *arg) { const struct hda_ops *hops = NULL; struct hda_codec_softc *sc = (struct hda_codec_softc *)arg; struct hda_codec_inst *hci = NULL; struct hda_codec_stream *st = NULL; struct audio *aud = NULL; int err; hci = sc->hci; assert(hci); hops = hci->hops; assert(hops); st = &sc->streams[HDA_CODEC_STREAM_INPUT]; aud = st->aud; err = audio_record(aud, st->buf, sizeof(st->buf)); assert(!err); hops->transfer(hci, st->stream, 0, st->buf, sizeof(st->buf)); } static int hda_codec_audio_input_do_setup(void *arg) { struct hda_codec_softc *sc = (struct hda_codec_softc *)arg; struct hda_codec_stream *st = NULL; struct audio *aud = NULL; struct audio_params params; int err; st = &sc->streams[HDA_CODEC_STREAM_INPUT]; aud = st->aud; err = hda_codec_parse_format(st->fmt, ¶ms); if (err) return (-1); DPRINTF("rate: %d, channels: %d, format: 0x%x", params.rate, params.channels, params.format); return (audio_set_params(aud, ¶ms)); } static uint32_t hda_codec_audio_inout_nid(struct hda_codec_stream *st, uint16_t verb, uint16_t payload) { uint32_t res = 0; uint8_t mute = 0; uint8_t gain = 0; DPRINTF("%s verb: 0x%x, payload, 0x%x", st->actx.name, verb, payload); switch (verb) { case HDA_CMD_VERB_GET_CONV_FMT: res = st->fmt; break; case HDA_CMD_VERB_SET_CONV_FMT: st->fmt = payload; break; case HDA_CMD_VERB_GET_AMP_GAIN_MUTE: if (payload & HDA_CMD_GET_AMP_GAIN_MUTE_LEFT) { res = st->left_gain | st->left_mute; DPRINTF("GET_AMP_GAIN_MUTE_LEFT: 0x%x", res); } else { res = st->right_gain | st->right_mute; DPRINTF("GET_AMP_GAIN_MUTE_RIGHT: 0x%x", res); } break; case HDA_CMD_VERB_SET_AMP_GAIN_MUTE: mute = payload & HDA_CODEC_SET_AMP_GAIN_MUTE_MUTE; gain = payload & HDA_CODEC_SET_AMP_GAIN_MUTE_GAIN_MASK; if (payload & HDA_CMD_SET_AMP_GAIN_MUTE_LEFT) { st->left_mute = mute; st->left_gain = gain; DPRINTF("SET_AMP_GAIN_MUTE_LEFT: \ mute: 0x%x gain: 0x%x", mute, gain); } if (payload & HDA_CMD_SET_AMP_GAIN_MUTE_RIGHT) { st->right_mute = mute; st->right_gain = gain; DPRINTF("SET_AMP_GAIN_MUTE_RIGHT: \ mute: 0x%x gain: 0x%x", mute, gain); } break; case HDA_CMD_VERB_GET_CONV_STREAM_CHAN: res = (st->stream << 4) | st->channel; break; case HDA_CMD_VERB_SET_CONV_STREAM_CHAN: st->channel = payload & 0x0f; st->stream = (payload >> 4) & 0x0f; DPRINTF("st->channel: 0x%x st->stream: 0x%x", st->channel, st->stream); if (!st->stream) hda_audio_ctxt_stop(&st->actx); break; default: DPRINTF("Unknown VERB: 0x%x", verb); break; } return (res); } static const struct hda_codec_class hda_codec = { .name = "hda_codec", .init = hda_codec_init, .reset = hda_codec_reset, .command = hda_codec_command, .notify = hda_codec_notify, }; HDA_EMUL_SET(hda_codec); /* * HDA Audio Context module function definitions */ static void * hda_audio_ctxt_thr(void *arg) { struct hda_audio_ctxt *actx = arg; DPRINTF("Start Thread: %s", actx->name); pthread_mutex_lock(&actx->mtx); while (1) { while (!actx->run) pthread_cond_wait(&actx->cond, &actx->mtx); actx->do_transfer(actx->priv); } pthread_mutex_unlock(&actx->mtx); pthread_exit(NULL); return (NULL); } static int hda_audio_ctxt_init(struct hda_audio_ctxt *actx, const char *tname, transfer_func_t do_transfer, setup_func_t do_setup, void *priv) { int err; assert(actx); assert(tname); assert(do_transfer); assert(do_setup); assert(priv); memset(actx, 0, sizeof(*actx)); actx->run = 0; actx->do_transfer = do_transfer; actx->do_setup = do_setup; actx->priv = priv; if (strlen(tname) < sizeof(actx->name)) memcpy(actx->name, tname, strlen(tname) + 1); else strcpy(actx->name, "unknown"); err = pthread_mutex_init(&actx->mtx, NULL); assert(!err); err = pthread_cond_init(&actx->cond, NULL); assert(!err); err = pthread_create(&actx->tid, NULL, hda_audio_ctxt_thr, actx); assert(!err); pthread_set_name_np(actx->tid, tname); actx->started = 1; return (0); } static int hda_audio_ctxt_start(struct hda_audio_ctxt *actx) { int err = 0; assert(actx); assert(actx->started); /* The stream is supposed to be stopped */ if (actx->run) return (-1); pthread_mutex_lock(&actx->mtx); err = (* actx->do_setup)(actx->priv); if (!err) { actx->run = 1; pthread_cond_signal(&actx->cond); } pthread_mutex_unlock(&actx->mtx); return (err); } static int hda_audio_ctxt_stop(struct hda_audio_ctxt *actx) { actx->run = 0; return (0); } /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2006 Stephane E. Potvin * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef _HDA_REG_H_ #define _HDA_REG_H_ /**************************************************************************** * HDA Device Verbs ****************************************************************************/ /* HDA Command */ #define HDA_CMD_VERB_MASK 0x000fffff #define HDA_CMD_VERB_SHIFT 0 #define HDA_CMD_NID_MASK 0x0ff00000 #define HDA_CMD_NID_SHIFT 20 #define HDA_CMD_CAD_MASK 0xf0000000 #define HDA_CMD_CAD_SHIFT 28 #define HDA_CMD_VERB_4BIT_SHIFT 16 #define HDA_CMD_VERB_12BIT_SHIFT 8 #define HDA_CMD_VERB_4BIT(verb, payload) \ (((verb) << HDA_CMD_VERB_4BIT_SHIFT) | (payload)) #define HDA_CMD_4BIT(cad, nid, verb, payload) \ (((cad) << HDA_CMD_CAD_SHIFT) | \ ((nid) << HDA_CMD_NID_SHIFT) | \ (HDA_CMD_VERB_4BIT((verb), (payload)))) #define HDA_CMD_VERB_12BIT(verb, payload) \ (((verb) << HDA_CMD_VERB_12BIT_SHIFT) | (payload)) #define HDA_CMD_12BIT(cad, nid, verb, payload) \ (((cad) << HDA_CMD_CAD_SHIFT) | \ ((nid) << HDA_CMD_NID_SHIFT) | \ (HDA_CMD_VERB_12BIT((verb), (payload)))) /* Get Parameter */ #define HDA_CMD_VERB_GET_PARAMETER 0xf00 #define HDA_CMD_GET_PARAMETER(cad, nid, payload) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_GET_PARAMETER, (payload))) /* Connection Select Control */ #define HDA_CMD_VERB_GET_CONN_SELECT_CONTROL 0xf01 #define HDA_CMD_VERB_SET_CONN_SELECT_CONTROL 0x701 #define HDA_CMD_GET_CONN_SELECT_CONTROL(cad, nid) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_GET_CONN_SELECT_CONTROL, 0x0)) #define HDA_CMD_SET_CONNECTION_SELECT_CONTROL(cad, nid, payload) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_SET_CONN_SELECT_CONTROL, (payload))) /* Connection List Entry */ #define HDA_CMD_VERB_GET_CONN_LIST_ENTRY 0xf02 #define HDA_CMD_GET_CONN_LIST_ENTRY(cad, nid, payload) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_GET_CONN_LIST_ENTRY, (payload))) #define HDA_CMD_GET_CONN_LIST_ENTRY_SIZE_SHORT 1 #define HDA_CMD_GET_CONN_LIST_ENTRY_SIZE_LONG 2 /* Processing State */ #define HDA_CMD_VERB_GET_PROCESSING_STATE 0xf03 #define HDA_CMD_VERB_SET_PROCESSING_STATE 0x703 #define HDA_CMD_GET_PROCESSING_STATE(cad, nid) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_GET_PROCESSING_STATE, 0x0)) #define HDA_CMD_SET_PROCESSING_STATE(cad, nid, payload) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_SET_PROCESSING_STATE, (payload))) #define HDA_CMD_GET_PROCESSING_STATE_STATE_OFF 0x00 #define HDA_CMD_GET_PROCESSING_STATE_STATE_ON 0x01 #define HDA_CMD_GET_PROCESSING_STATE_STATE_BENIGN 0x02 /* Coefficient Index */ #define HDA_CMD_VERB_GET_COEFF_INDEX 0xd #define HDA_CMD_VERB_SET_COEFF_INDEX 0x5 #define HDA_CMD_GET_COEFF_INDEX(cad, nid) \ (HDA_CMD_4BIT((cad), (nid), \ HDA_CMD_VERB_GET_COEFF_INDEX, 0x0)) #define HDA_CMD_SET_COEFF_INDEX(cad, nid, payload) \ (HDA_CMD_4BIT((cad), (nid), \ HDA_CMD_VERB_SET_COEFF_INDEX, (payload))) /* Processing Coefficient */ #define HDA_CMD_VERB_GET_PROCESSING_COEFF 0xc #define HDA_CMD_VERB_SET_PROCESSING_COEFF 0x4 #define HDA_CMD_GET_PROCESSING_COEFF(cad, nid) \ (HDA_CMD_4BIT((cad), (nid), \ HDA_CMD_VERB_GET_PROCESSING_COEFF, 0x0)) #define HDA_CMD_SET_PROCESSING_COEFF(cad, nid, payload) \ (HDA_CMD_4BIT((cad), (nid), \ HDA_CMD_VERB_SET_PROCESSING_COEFF, (payload))) /* Amplifier Gain/Mute */ #define HDA_CMD_VERB_GET_AMP_GAIN_MUTE 0xb #define HDA_CMD_VERB_SET_AMP_GAIN_MUTE 0x3 #define HDA_CMD_GET_AMP_GAIN_MUTE(cad, nid, payload) \ (HDA_CMD_4BIT((cad), (nid), \ HDA_CMD_VERB_GET_AMP_GAIN_MUTE, (payload))) #define HDA_CMD_SET_AMP_GAIN_MUTE(cad, nid, payload) \ (HDA_CMD_4BIT((cad), (nid), \ HDA_CMD_VERB_SET_AMP_GAIN_MUTE, (payload))) #define HDA_CMD_GET_AMP_GAIN_MUTE_INPUT 0x0000 #define HDA_CMD_GET_AMP_GAIN_MUTE_OUTPUT 0x8000 #define HDA_CMD_GET_AMP_GAIN_MUTE_RIGHT 0x0000 #define HDA_CMD_GET_AMP_GAIN_MUTE_LEFT 0x2000 #define HDA_CMD_GET_AMP_GAIN_MUTE_MUTE_MASK 0x00000008 #define HDA_CMD_GET_AMP_GAIN_MUTE_MUTE_SHIFT 7 #define HDA_CMD_GET_AMP_GAIN_MUTE_GAIN_MASK 0x00000007 #define HDA_CMD_GET_AMP_GAIN_MUTE_GAIN_SHIFT 0 #define HDA_CMD_GET_AMP_GAIN_MUTE_MUTE(rsp) \ (((rsp) & HDA_CMD_GET_AMP_GAIN_MUTE_MUTE_MASK) >> \ HDA_CMD_GET_AMP_GAIN_MUTE_MUTE_SHIFT) #define HDA_CMD_GET_AMP_GAIN_MUTE_GAIN(rsp) \ (((rsp) & HDA_CMD_GET_AMP_GAIN_MUTE_GAIN_MASK) >> \ HDA_CMD_GET_AMP_GAIN_MUTE_GAIN_SHIFT) #define HDA_CMD_SET_AMP_GAIN_MUTE_OUTPUT 0x8000 #define HDA_CMD_SET_AMP_GAIN_MUTE_INPUT 0x4000 #define HDA_CMD_SET_AMP_GAIN_MUTE_LEFT 0x2000 #define HDA_CMD_SET_AMP_GAIN_MUTE_RIGHT 0x1000 #define HDA_CMD_SET_AMP_GAIN_MUTE_INDEX_MASK 0x0f00 #define HDA_CMD_SET_AMP_GAIN_MUTE_INDEX_SHIFT 8 #define HDA_CMD_SET_AMP_GAIN_MUTE_MUTE 0x0080 #define HDA_CMD_SET_AMP_GAIN_MUTE_GAIN_MASK 0x0007 #define HDA_CMD_SET_AMP_GAIN_MUTE_GAIN_SHIFT 0 #define HDA_CMD_SET_AMP_GAIN_MUTE_INDEX(index) \ (((index) << HDA_CMD_SET_AMP_GAIN_MUTE_INDEX_SHIFT) & \ HDA_CMD_SET_AMP_GAIN_MUTE_INDEX_MASK) #define HDA_CMD_SET_AMP_GAIN_MUTE_GAIN(index) \ (((index) << HDA_CMD_SET_AMP_GAIN_MUTE_GAIN_SHIFT) & \ HDA_CMD_SET_AMP_GAIN_MUTE_GAIN_MASK) /* Converter format */ #define HDA_CMD_VERB_GET_CONV_FMT 0xa #define HDA_CMD_VERB_SET_CONV_FMT 0x2 #define HDA_CMD_GET_CONV_FMT(cad, nid) \ (HDA_CMD_4BIT((cad), (nid), \ HDA_CMD_VERB_GET_CONV_FMT, 0x0)) #define HDA_CMD_SET_CONV_FMT(cad, nid, payload) \ (HDA_CMD_4BIT((cad), (nid), \ HDA_CMD_VERB_SET_CONV_FMT, (payload))) /* Digital Converter Control */ #define HDA_CMD_VERB_GET_DIGITAL_CONV_FMT1 0xf0d #define HDA_CMD_VERB_GET_DIGITAL_CONV_FMT2 0xf0e #define HDA_CMD_VERB_SET_DIGITAL_CONV_FMT1 0x70d #define HDA_CMD_VERB_SET_DIGITAL_CONV_FMT2 0x70e #define HDA_CMD_GET_DIGITAL_CONV_FMT(cad, nid) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_GET_DIGITAL_CONV_FMT1, 0x0)) #define HDA_CMD_SET_DIGITAL_CONV_FMT1(cad, nid, payload) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_SET_DIGITAL_CONV_FMT1, (payload))) #define HDA_CMD_SET_DIGITAL_CONV_FMT2(cad, nid, payload) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_SET_DIGITAL_CONV_FMT2, (payload))) #define HDA_CMD_GET_DIGITAL_CONV_FMT_CC_MASK 0x7f00 #define HDA_CMD_GET_DIGITAL_CONV_FMT_CC_SHIFT 8 #define HDA_CMD_GET_DIGITAL_CONV_FMT_L_MASK 0x0080 #define HDA_CMD_GET_DIGITAL_CONV_FMT_L_SHIFT 7 #define HDA_CMD_GET_DIGITAL_CONV_FMT_PRO_MASK 0x0040 #define HDA_CMD_GET_DIGITAL_CONV_FMT_PRO_SHIFT 6 #define HDA_CMD_GET_DIGITAL_CONV_FMT_NAUDIO_MASK 0x0020 #define HDA_CMD_GET_DIGITAL_CONV_FMT_NAUDIO_SHIFT 5 #define HDA_CMD_GET_DIGITAL_CONV_FMT_COPY_MASK 0x0010 #define HDA_CMD_GET_DIGITAL_CONV_FMT_COPY_SHIFT 4 #define HDA_CMD_GET_DIGITAL_CONV_FMT_PRE_MASK 0x0008 #define HDA_CMD_GET_DIGITAL_CONV_FMT_PRE_SHIFT 3 #define HDA_CMD_GET_DIGITAL_CONV_FMT_VCFG_MASK 0x0004 #define HDA_CMD_GET_DIGITAL_CONV_FMT_VCFG_SHIFT 2 #define HDA_CMD_GET_DIGITAL_CONV_FMT_V_MASK 0x0002 #define HDA_CMD_GET_DIGITAL_CONV_FMT_V_SHIFT 1 #define HDA_CMD_GET_DIGITAL_CONV_FMT_DIGEN_MASK 0x0001 #define HDA_CMD_GET_DIGITAL_CONV_FMT_DIGEN_SHIFT 0 #define HDA_CMD_GET_DIGITAL_CONV_FMT_CC(rsp) \ (((rsp) & HDA_CMD_GET_DIGITAL_CONV_FMT_CC_MASK) >> \ HDA_CMD_GET_DIGITAL_CONV_FMT_CC_SHIFT) #define HDA_CMD_GET_DIGITAL_CONV_FMT_L(rsp) \ (((rsp) & HDA_CMD_GET_DIGITAL_CONV_FMT_L_MASK) >> \ HDA_CMD_GET_DIGITAL_CONV_FMT_L_SHIFT) #define HDA_CMD_GET_DIGITAL_CONV_FMT_PRO(rsp) \ (((rsp) & HDA_CMD_GET_DIGITAL_CONV_FMT_PRO_MASK) >> \ HDA_CMD_GET_DIGITAL_CONV_FMT_PRO_SHIFT) #define HDA_CMD_GET_DIGITAL_CONV_FMT_NAUDIO(rsp) \ (((rsp) & HDA_CMD_GET_DIGITAL_CONV_FMT_NAUDIO_MASK) >> \ HDA_CMD_GET_DIGITAL_CONV_FMT_NAUDIO_SHIFT) #define HDA_CMD_GET_DIGITAL_CONV_FMT_COPY(rsp) \ (((rsp) & HDA_CMD_GET_DIGITAL_CONV_FMT_COPY_MASK) >> \ HDA_CMD_GET_DIGITAL_CONV_FMT_COPY_SHIFT) #define HDA_CMD_GET_DIGITAL_CONV_FMT_PRE(rsp) \ (((rsp) & HDA_CMD_GET_DIGITAL_CONV_FMT_PRE_MASK) >> \ HDA_CMD_GET_DIGITAL_CONV_FMT_PRE_SHIFT) #define HDA_CMD_GET_DIGITAL_CONV_FMT_VCFG(rsp) \ (((rsp) & HDA_CMD_GET_DIGITAL_CONV_FMT_VCFG_MASK) >> \ HDA_CMD_GET_DIGITAL_CONV_FMT_VCFG_SHIFT) #define HDA_CMD_GET_DIGITAL_CONV_FMT_V(rsp) \ (((rsp) & HDA_CMD_GET_DIGITAL_CONV_FMT_V_MASK) >> \ HDA_CMD_GET_DIGITAL_CONV_FMT_V_SHIFT) #define HDA_CMD_GET_DIGITAL_CONV_FMT_DIGEN(rsp) \ (((rsp) & HDA_CMD_GET_DIGITAL_CONV_FMT_DIGEN_MASK) >> \ HDA_CMD_GET_DIGITAL_CONV_FMT_DIGEN_SHIFT) #define HDA_CMD_SET_DIGITAL_CONV_FMT1_L 0x80 #define HDA_CMD_SET_DIGITAL_CONV_FMT1_PRO 0x40 #define HDA_CMD_SET_DIGITAL_CONV_FMT1_NAUDIO 0x20 #define HDA_CMD_SET_DIGITAL_CONV_FMT1_COPY 0x10 #define HDA_CMD_SET_DIGITAL_CONV_FMT1_PRE 0x08 #define HDA_CMD_SET_DIGITAL_CONV_FMT1_VCFG 0x04 #define HDA_CMD_SET_DIGITAL_CONV_FMT1_V 0x02 #define HDA_CMD_SET_DIGITAL_CONV_FMT1_DIGEN 0x01 /* Power State */ #define HDA_CMD_VERB_GET_POWER_STATE 0xf05 #define HDA_CMD_VERB_SET_POWER_STATE 0x705 #define HDA_CMD_GET_POWER_STATE(cad, nid) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_GET_POWER_STATE, 0x0)) #define HDA_CMD_SET_POWER_STATE(cad, nid, payload) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_SET_POWER_STATE, (payload))) #define HDA_CMD_POWER_STATE_D0 0x00 #define HDA_CMD_POWER_STATE_D1 0x01 #define HDA_CMD_POWER_STATE_D2 0x02 #define HDA_CMD_POWER_STATE_D3 0x03 #define HDA_CMD_POWER_STATE_ACT_MASK 0x000000f0 #define HDA_CMD_POWER_STATE_ACT_SHIFT 4 #define HDA_CMD_POWER_STATE_SET_MASK 0x0000000f #define HDA_CMD_POWER_STATE_SET_SHIFT 0 #define HDA_CMD_GET_POWER_STATE_ACT(rsp) \ (((rsp) & HDA_CMD_POWER_STATE_ACT_MASK) >> \ HDA_CMD_POWER_STATE_ACT_SHIFT) #define HDA_CMD_GET_POWER_STATE_SET(rsp) \ (((rsp) & HDA_CMD_POWER_STATE_SET_MASK) >> \ HDA_CMD_POWER_STATE_SET_SHIFT) #define HDA_CMD_SET_POWER_STATE_ACT(ps) \ (((ps) << HDA_CMD_POWER_STATE_ACT_SHIFT) & \ HDA_CMD_POWER_STATE_ACT_MASK) #define HDA_CMD_SET_POWER_STATE_SET(ps) \ (((ps) << HDA_CMD_POWER_STATE_SET_SHIFT) & \ HDA_CMD_POWER_STATE_ACT_MASK) /* Converter Stream, Channel */ #define HDA_CMD_VERB_GET_CONV_STREAM_CHAN 0xf06 #define HDA_CMD_VERB_SET_CONV_STREAM_CHAN 0x706 #define HDA_CMD_GET_CONV_STREAM_CHAN(cad, nid) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_GET_CONV_STREAM_CHAN, 0x0)) #define HDA_CMD_SET_CONV_STREAM_CHAN(cad, nid, payload) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_SET_CONV_STREAM_CHAN, (payload))) #define HDA_CMD_CONV_STREAM_CHAN_STREAM_MASK 0x000000f0 #define HDA_CMD_CONV_STREAM_CHAN_STREAM_SHIFT 4 #define HDA_CMD_CONV_STREAM_CHAN_CHAN_MASK 0x0000000f #define HDA_CMD_CONV_STREAM_CHAN_CHAN_SHIFT 0 #define HDA_CMD_GET_CONV_STREAM_CHAN_STREAM(rsp) \ (((rsp) & HDA_CMD_CONV_STREAM_CHAN_STREAM_MASK) >> \ HDA_CMD_CONV_STREAM_CHAN_STREAM_SHIFT) #define HDA_CMD_GET_CONV_STREAM_CHAN_CHAN(rsp) \ (((rsp) & HDA_CMD_CONV_STREAM_CHAN_CHAN_MASK) >> \ HDA_CMD_CONV_STREAM_CHAN_CHAN_SHIFT) #define HDA_CMD_SET_CONV_STREAM_CHAN_STREAM(param) \ (((param) << HDA_CMD_CONV_STREAM_CHAN_STREAM_SHIFT) & \ HDA_CMD_CONV_STREAM_CHAN_STREAM_MASK) #define HDA_CMD_SET_CONV_STREAM_CHAN_CHAN(param) \ (((param) << HDA_CMD_CONV_STREAM_CHAN_CHAN_SHIFT) & \ HDA_CMD_CONV_STREAM_CHAN_CHAN_MASK) /* Input Converter SDI Select */ #define HDA_CMD_VERB_GET_INPUT_CONVERTER_SDI_SELECT 0xf04 #define HDA_CMD_VERB_SET_INPUT_CONVERTER_SDI_SELECT 0x704 #define HDA_CMD_GET_INPUT_CONVERTER_SDI_SELECT(cad, nid) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_GET_INPUT_CONVERTER_SDI_SELECT, 0x0)) #define HDA_CMD_SET_INPUT_CONVERTER_SDI_SELECT(cad, nid, payload) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_SET_INPUT_CONVERTER_SDI_SELECT, (payload))) /* Pin Widget Control */ #define HDA_CMD_VERB_GET_PIN_WIDGET_CTRL 0xf07 #define HDA_CMD_VERB_SET_PIN_WIDGET_CTRL 0x707 #define HDA_CMD_GET_PIN_WIDGET_CTRL(cad, nid) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_GET_PIN_WIDGET_CTRL, 0x0)) #define HDA_CMD_SET_PIN_WIDGET_CTRL(cad, nid, payload) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_SET_PIN_WIDGET_CTRL, (payload))) #define HDA_CMD_GET_PIN_WIDGET_CTRL_HPHN_ENABLE_MASK 0x00000080 #define HDA_CMD_GET_PIN_WIDGET_CTRL_HPHN_ENABLE_SHIFT 7 #define HDA_CMD_GET_PIN_WIDGET_CTRL_OUT_ENABLE_MASK 0x00000040 #define HDA_CMD_GET_PIN_WIDGET_CTRL_OUT_ENABLE_SHIFT 6 #define HDA_CMD_GET_PIN_WIDGET_CTRL_IN_ENABLE_MASK 0x00000020 #define HDA_CMD_GET_PIN_WIDGET_CTRL_IN_ENABLE_SHIFT 5 #define HDA_CMD_GET_PIN_WIDGET_CTRL_VREF_ENABLE_MASK 0x00000007 #define HDA_CMD_GET_PIN_WIDGET_CTRL_VREF_ENABLE_SHIFT 0 #define HDA_CMD_GET_PIN_WIDGET_CTRL_HPHN_ENABLE(rsp) \ (((rsp) & HDA_CMD_GET_PIN_WIDGET_CTRL_HPHN_ENABLE_MASK) >> \ HDA_CMD_GET_PIN_WIDGET_CTRL_HPHN_ENABLE_SHIFT) #define HDA_CMD_GET_PIN_WIDGET_CTRL_OUT_ENABLE(rsp) \ (((rsp) & HDA_CMD_GET_PIN_WIDGET_CTRL_OUT_ENABLE_MASK) >> \ HDA_GET_CMD_PIN_WIDGET_CTRL_OUT_ENABLE_SHIFT) #define HDA_CMD_GET_PIN_WIDGET_CTRL_IN_ENABLE(rsp) \ (((rsp) & HDA_CMD_GET_PIN_WIDGET_CTRL_IN_ENABLE_MASK) >> \ HDA_CMD_GET_PIN_WIDGET_CTRL_IN_ENABLE_SHIFT) #define HDA_CMD_GET_PIN_WIDGET_CTRL_VREF_ENABLE(rsp) \ (((rsp) & HDA_CMD_GET_PIN_WIDGET_CTRL_VREF_ENABLE_MASK) >> \ HDA_CMD_GET_PIN_WIDGET_CTRL_VREF_ENABLE_SHIFT) #define HDA_CMD_SET_PIN_WIDGET_CTRL_HPHN_ENABLE 0x80 #define HDA_CMD_SET_PIN_WIDGET_CTRL_OUT_ENABLE 0x40 #define HDA_CMD_SET_PIN_WIDGET_CTRL_IN_ENABLE 0x20 #define HDA_CMD_SET_PIN_WIDGET_CTRL_VREF_ENABLE_MASK 0x07 #define HDA_CMD_SET_PIN_WIDGET_CTRL_VREF_ENABLE_SHIFT 0 #define HDA_CMD_SET_PIN_WIDGET_CTRL_VREF_ENABLE(param) \ (((param) << HDA_CMD_SET_PIN_WIDGET_CTRL_VREF_ENABLE_SHIFT) & \ HDA_CMD_SET_PIN_WIDGET_CTRL_VREF_ENABLE_MASK) #define HDA_CMD_PIN_WIDGET_CTRL_VREF_ENABLE_HIZ 0 #define HDA_CMD_PIN_WIDGET_CTRL_VREF_ENABLE_50 1 #define HDA_CMD_PIN_WIDGET_CTRL_VREF_ENABLE_GROUND 2 #define HDA_CMD_PIN_WIDGET_CTRL_VREF_ENABLE_80 4 #define HDA_CMD_PIN_WIDGET_CTRL_VREF_ENABLE_100 5 /* Unsolicited Response */ #define HDA_CMD_VERB_GET_UNSOLICITED_RESPONSE 0xf08 #define HDA_CMD_VERB_SET_UNSOLICITED_RESPONSE 0x708 #define HDA_CMD_GET_UNSOLICITED_RESPONSE(cad, nid) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_GET_UNSOLICITED_RESPONSE, 0x0)) #define HDA_CMD_SET_UNSOLICITED_RESPONSE(cad, nid, payload) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_SET_UNSOLICITED_RESPONSE, (payload))) #define HDA_CMD_GET_UNSOLICITED_RESPONSE_ENABLE_MASK 0x00000080 #define HDA_CMD_GET_UNSOLICITED_RESPONSE_ENABLE_SHIFT 7 #define HDA_CMD_GET_UNSOLICITED_RESPONSE_TAG_MASK 0x0000001f #define HDA_CMD_GET_UNSOLICITED_RESPONSE_TAG_SHIFT 0 #define HDA_CMD_GET_UNSOLICITED_RESPONSE_ENABLE(rsp) \ (((rsp) & HDA_CMD_GET_UNSOLICITED_RESPONSE_ENABLE_MASK) >> \ HDA_CMD_GET_UNSOLICITED_RESPONSE_ENABLE_SHIFT) #define HDA_CMD_GET_UNSOLICITED_RESPONSE_TAG(rsp) \ (((rsp) & HDA_CMD_GET_UNSOLICITED_RESPONSE_TAG_MASK) >> \ HDA_CMD_GET_UNSOLICITED_RESPONSE_TAG_SHIFT) #define HDA_CMD_SET_UNSOLICITED_RESPONSE_ENABLE 0x80 #define HDA_CMD_SET_UNSOLICITED_RESPONSE_TAG_MASK 0x3f #define HDA_CMD_SET_UNSOLICITED_RESPONSE_TAG_SHIFT 0 #define HDA_CMD_SET_UNSOLICITED_RESPONSE_TAG(param) \ (((param) << HDA_CMD_SET_UNSOLICITED_RESPONSE_TAG_SHIFT) & \ HDA_CMD_SET_UNSOLICITED_RESPONSE_TAG_MASK) /* Pin Sense */ #define HDA_CMD_VERB_GET_PIN_SENSE 0xf09 #define HDA_CMD_VERB_SET_PIN_SENSE 0x709 #define HDA_CMD_GET_PIN_SENSE(cad, nid) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_GET_PIN_SENSE, 0x0)) #define HDA_CMD_SET_PIN_SENSE(cad, nid, payload) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_SET_PIN_SENSE, (payload))) #define HDA_CMD_GET_PIN_SENSE_PRESENCE_DETECT 0x80000000 #define HDA_CMD_GET_PIN_SENSE_ELD_VALID 0x40000000 #define HDA_CMD_GET_PIN_SENSE_IMP_SENSE_MASK 0x7fffffff #define HDA_CMD_GET_PIN_SENSE_IMP_SENSE_SHIFT 0 #define HDA_CMD_GET_PIN_SENSE_IMP_SENSE(rsp) \ (((rsp) & HDA_CMD_GET_PIN_SENSE_IMP_SENSE_MASK) >> \ HDA_CMD_GET_PIN_SENSE_IMP_SENSE_SHIFT) #define HDA_CMD_GET_PIN_SENSE_IMP_SENSE_INVALID 0x7fffffff #define HDA_CMD_SET_PIN_SENSE_LEFT_CHANNEL 0x00 #define HDA_CMD_SET_PIN_SENSE_RIGHT_CHANNEL 0x01 /* EAPD/BTL Enable */ #define HDA_CMD_VERB_GET_EAPD_BTL_ENABLE 0xf0c #define HDA_CMD_VERB_SET_EAPD_BTL_ENABLE 0x70c #define HDA_CMD_GET_EAPD_BTL_ENABLE(cad, nid) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_GET_EAPD_BTL_ENABLE, 0x0)) #define HDA_CMD_SET_EAPD_BTL_ENABLE(cad, nid, payload) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_SET_EAPD_BTL_ENABLE, (payload))) #define HDA_CMD_GET_EAPD_BTL_ENABLE_LR_SWAP_MASK 0x00000004 #define HDA_CMD_GET_EAPD_BTL_ENABLE_LR_SWAP_SHIFT 2 #define HDA_CMD_GET_EAPD_BTL_ENABLE_EAPD_MASK 0x00000002 #define HDA_CMD_GET_EAPD_BTL_ENABLE_EAPD_SHIFT 1 #define HDA_CMD_GET_EAPD_BTL_ENABLE_BTL_MASK 0x00000001 #define HDA_CMD_GET_EAPD_BTL_ENABLE_BTL_SHIFT 0 #define HDA_CMD_GET_EAPD_BTL_ENABLE_LR_SWAP(rsp) \ (((rsp) & HDA_CMD_GET_EAPD_BTL_ENABLE_LR_SWAP_MASK) >> \ HDA_CMD_GET_EAPD_BTL_ENABLE_LR_SWAP_SHIFT) #define HDA_CMD_GET_EAPD_BTL_ENABLE_EAPD(rsp) \ (((rsp) & HDA_CMD_GET_EAPD_BTL_ENABLE_EAPD_MASK) >> \ HDA_CMD_GET_EAPD_BTL_ENABLE_EAPD_SHIFT) #define HDA_CMD_GET_EAPD_BTL_ENABLE_BTL(rsp) \ (((rsp) & HDA_CMD_GET_EAPD_BTL_ENABLE_BTL_MASK) >> \ HDA_CMD_GET_EAPD_BTL_ENABLE_BTL_SHIFT) #define HDA_CMD_SET_EAPD_BTL_ENABLE_LR_SWAP 0x04 #define HDA_CMD_SET_EAPD_BTL_ENABLE_EAPD 0x02 #define HDA_CMD_SET_EAPD_BTL_ENABLE_BTL 0x01 /* GPI Data */ #define HDA_CMD_VERB_GET_GPI_DATA 0xf10 #define HDA_CMD_VERB_SET_GPI_DATA 0x710 #define HDA_CMD_GET_GPI_DATA(cad, nid) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_GET_GPI_DATA, 0x0)) #define HDA_CMD_SET_GPI_DATA(cad, nid) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_SET_GPI_DATA, (payload))) /* GPI Wake Enable Mask */ #define HDA_CMD_VERB_GET_GPI_WAKE_ENABLE_MASK 0xf11 #define HDA_CMD_VERB_SET_GPI_WAKE_ENABLE_MASK 0x711 #define HDA_CMD_GET_GPI_WAKE_ENABLE_MASK(cad, nid) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_GET_GPI_WAKE_ENABLE_MASK, 0x0)) #define HDA_CMD_SET_GPI_WAKE_ENABLE_MASK(cad, nid, payload) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_SET_GPI_WAKE_ENABLE_MASK, (payload))) /* GPI Unsolicited Enable Mask */ #define HDA_CMD_VERB_GET_GPI_UNSOLICITED_ENABLE_MASK 0xf12 #define HDA_CMD_VERB_SET_GPI_UNSOLICITED_ENABLE_MASK 0x712 #define HDA_CMD_GET_GPI_UNSOLICITED_ENABLE_MASK(cad, nid) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_GET_GPI_UNSOLICITED_ENABLE_MASK, 0x0)) #define HDA_CMD_SET_GPI_UNSOLICITED_ENABLE_MASK(cad, nid, payload) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_SET_GPI_UNSOLICITED_ENABLE_MASK, (payload))) /* GPI Sticky Mask */ #define HDA_CMD_VERB_GET_GPI_STICKY_MASK 0xf13 #define HDA_CMD_VERB_SET_GPI_STICKY_MASK 0x713 #define HDA_CMD_GET_GPI_STICKY_MASK(cad, nid) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_GET_GPI_STICKY_MASK, 0x0)) #define HDA_CMD_SET_GPI_STICKY_MASK(cad, nid, payload) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_SET_GPI_STICKY_MASK, (payload))) /* GPO Data */ #define HDA_CMD_VERB_GET_GPO_DATA 0xf14 #define HDA_CMD_VERB_SET_GPO_DATA 0x714 #define HDA_CMD_GET_GPO_DATA(cad, nid) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_GET_GPO_DATA, 0x0)) #define HDA_CMD_SET_GPO_DATA(cad, nid, payload) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_SET_GPO_DATA, (payload))) /* GPIO Data */ #define HDA_CMD_VERB_GET_GPIO_DATA 0xf15 #define HDA_CMD_VERB_SET_GPIO_DATA 0x715 #define HDA_CMD_GET_GPIO_DATA(cad, nid) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_GET_GPIO_DATA, 0x0)) #define HDA_CMD_SET_GPIO_DATA(cad, nid, payload) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_SET_GPIO_DATA, (payload))) /* GPIO Enable Mask */ #define HDA_CMD_VERB_GET_GPIO_ENABLE_MASK 0xf16 #define HDA_CMD_VERB_SET_GPIO_ENABLE_MASK 0x716 #define HDA_CMD_GET_GPIO_ENABLE_MASK(cad, nid) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_GET_GPIO_ENABLE_MASK, 0x0)) #define HDA_CMD_SET_GPIO_ENABLE_MASK(cad, nid, payload) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_SET_GPIO_ENABLE_MASK, (payload))) /* GPIO Direction */ #define HDA_CMD_VERB_GET_GPIO_DIRECTION 0xf17 #define HDA_CMD_VERB_SET_GPIO_DIRECTION 0x717 #define HDA_CMD_GET_GPIO_DIRECTION(cad, nid) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_GET_GPIO_DIRECTION, 0x0)) #define HDA_CMD_SET_GPIO_DIRECTION(cad, nid, payload) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_SET_GPIO_DIRECTION, (payload))) /* GPIO Wake Enable Mask */ #define HDA_CMD_VERB_GET_GPIO_WAKE_ENABLE_MASK 0xf18 #define HDA_CMD_VERB_SET_GPIO_WAKE_ENABLE_MASK 0x718 #define HDA_CMD_GET_GPIO_WAKE_ENABLE_MASK(cad, nid) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_GET_GPIO_WAKE_ENABLE_MASK, 0x0)) #define HDA_CMD_SET_GPIO_WAKE_ENABLE_MASK(cad, nid, payload) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_SET_GPIO_WAKE_ENABLE_MASK, (payload))) /* GPIO Unsolicited Enable Mask */ #define HDA_CMD_VERB_GET_GPIO_UNSOLICITED_ENABLE_MASK 0xf19 #define HDA_CMD_VERB_SET_GPIO_UNSOLICITED_ENABLE_MASK 0x719 #define HDA_CMD_GET_GPIO_UNSOLICITED_ENABLE_MASK(cad, nid) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_GET_GPIO_UNSOLICITED_ENABLE_MASK, 0x0)) #define HDA_CMD_SET_GPIO_UNSOLICITED_ENABLE_MASK(cad, nid, payload) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_SET_GPIO_UNSOLICITED_ENABLE_MASK, (payload))) /* GPIO_STICKY_MASK */ #define HDA_CMD_VERB_GET_GPIO_STICKY_MASK 0xf1a #define HDA_CMD_VERB_SET_GPIO_STICKY_MASK 0x71a #define HDA_CMD_GET_GPIO_STICKY_MASK(cad, nid) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_GET_GPIO_STICKY_MASK, 0x0)) #define HDA_CMD_SET_GPIO_STICKY_MASK(cad, nid, payload) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_SET_GPIO_STICKY_MASK, (payload))) /* Beep Generation */ #define HDA_CMD_VERB_GET_BEEP_GENERATION 0xf0a #define HDA_CMD_VERB_SET_BEEP_GENERATION 0x70a #define HDA_CMD_GET_BEEP_GENERATION(cad, nid) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_GET_BEEP_GENERATION, 0x0)) #define HDA_CMD_SET_BEEP_GENERATION(cad, nid, payload) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_SET_BEEP_GENERATION, (payload))) /* Volume Knob */ #define HDA_CMD_VERB_GET_VOLUME_KNOB 0xf0f #define HDA_CMD_VERB_SET_VOLUME_KNOB 0x70f #define HDA_CMD_GET_VOLUME_KNOB(cad, nid) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_GET_VOLUME_KNOB, 0x0)) #define HDA_CMD_SET_VOLUME_KNOB(cad, nid, payload) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_SET_VOLUME_KNOB, (payload))) /* Subsystem ID */ #define HDA_CMD_VERB_GET_SUBSYSTEM_ID 0xf20 #define HDA_CMD_VERB_SET_SUSBYSTEM_ID1 0x720 #define HDA_CMD_VERB_SET_SUBSYSTEM_ID2 0x721 #define HDA_CMD_VERB_SET_SUBSYSTEM_ID3 0x722 #define HDA_CMD_VERB_SET_SUBSYSTEM_ID4 0x723 #define HDA_CMD_GET_SUBSYSTEM_ID(cad, nid) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_GET_SUBSYSTEM_ID, 0x0)) #define HDA_CMD_SET_SUBSYSTEM_ID1(cad, nid, payload) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_SET_SUSBYSTEM_ID1, (payload))) #define HDA_CMD_SET_SUBSYSTEM_ID2(cad, nid, payload) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_SET_SUSBYSTEM_ID2, (payload))) #define HDA_CMD_SET_SUBSYSTEM_ID3(cad, nid, payload) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_SET_SUSBYSTEM_ID3, (payload))) #define HDA_CMD_SET_SUBSYSTEM_ID4(cad, nid, payload) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_SET_SUSBYSTEM_ID4, (payload))) /* Configuration Default */ #define HDA_CMD_VERB_GET_CONFIGURATION_DEFAULT 0xf1c #define HDA_CMD_VERB_SET_CONFIGURATION_DEFAULT1 0x71c #define HDA_CMD_VERB_SET_CONFIGURATION_DEFAULT2 0x71d #define HDA_CMD_VERB_SET_CONFIGURATION_DEFAULT3 0x71e #define HDA_CMD_VERB_SET_CONFIGURATION_DEFAULT4 0x71f #define HDA_CMD_GET_CONFIGURATION_DEFAULT(cad, nid) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_GET_CONFIGURATION_DEFAULT, 0x0)) #define HDA_CMD_SET_CONFIGURATION_DEFAULT1(cad, nid, payload) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_SET_CONFIGURATION_DEFAULT1, (payload))) #define HDA_CMD_SET_CONFIGURATION_DEFAULT2(cad, nid, payload) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_SET_CONFIGURATION_DEFAULT2, (payload))) #define HDA_CMD_SET_CONFIGURATION_DEFAULT3(cad, nid, payload) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_SET_CONFIGURATION_DEFAULT3, (payload))) #define HDA_CMD_SET_CONFIGURATION_DEFAULT4(cad, nid, payload) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_SET_CONFIGURATION_DEFAULT4, (payload))) /* Stripe Control */ #define HDA_CMD_VERB_GET_STRIPE_CONTROL 0xf24 #define HDA_CMD_VERB_SET_STRIPE_CONTROL 0x724 #define HDA_CMD_GET_STRIPE_CONTROL(cad, nid) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_GET_STRIPE_CONTROL, 0x0)) #define HDA_CMD_SET_STRIPE_CONTROL(cad, nid, payload) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_SET_STRIPE_CONTROL, (payload))) /* Channel Count Control */ #define HDA_CMD_VERB_GET_CONV_CHAN_COUNT 0xf2d #define HDA_CMD_VERB_SET_CONV_CHAN_COUNT 0x72d #define HDA_CMD_GET_CONV_CHAN_COUNT(cad, nid) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_GET_CONV_CHAN_COUNT, 0x0)) #define HDA_CMD_SET_CONV_CHAN_COUNT(cad, nid, payload) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_SET_CONV_CHAN_COUNT, (payload))) #define HDA_CMD_VERB_GET_HDMI_DIP_SIZE 0xf2e #define HDA_CMD_GET_HDMI_DIP_SIZE(cad, nid, arg) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_GET_HDMI_DIP_SIZE, (arg))) #define HDA_CMD_VERB_GET_HDMI_ELDD 0xf2f #define HDA_CMD_GET_HDMI_ELDD(cad, nid, off) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_GET_HDMI_ELDD, (off))) #define HDA_CMD_VERB_GET_HDMI_DIP_INDEX 0xf30 #define HDA_CMD_VERB_SET_HDMI_DIP_INDEX 0x730 #define HDA_CMD_GET_HDMI_DIP_INDEX(cad, nid) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_GET_HDMI_DIP_INDEX, 0x0)) #define HDA_CMD_SET_HDMI_DIP_INDEX(cad, nid, payload) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_SET_HDMI_DIP_INDEX, (payload))) #define HDA_CMD_VERB_GET_HDMI_DIP_DATA 0xf31 #define HDA_CMD_VERB_SET_HDMI_DIP_DATA 0x731 #define HDA_CMD_GET_HDMI_DIP_DATA(cad, nid) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_GET_HDMI_DIP_DATA, 0x0)) #define HDA_CMD_SET_HDMI_DIP_DATA(cad, nid, payload) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_SET_HDMI_DIP_DATA, (payload))) #define HDA_CMD_VERB_GET_HDMI_DIP_XMIT 0xf32 #define HDA_CMD_VERB_SET_HDMI_DIP_XMIT 0x732 #define HDA_CMD_GET_HDMI_DIP_XMIT(cad, nid) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_GET_HDMI_DIP_XMIT, 0x0)) #define HDA_CMD_SET_HDMI_DIP_XMIT(cad, nid, payload) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_SET_HDMI_DIP_XMIT, (payload))) #define HDA_CMD_VERB_GET_HDMI_CP_CTRL 0xf33 #define HDA_CMD_VERB_SET_HDMI_CP_CTRL 0x733 #define HDA_CMD_VERB_GET_HDMI_CHAN_SLOT 0xf34 #define HDA_CMD_VERB_SET_HDMI_CHAN_SLOT 0x734 #define HDA_CMD_GET_HDMI_CHAN_SLOT(cad, nid) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_GET_HDMI_CHAN_SLOT, 0x0)) #define HDA_CMD_SET_HDMI_CHAN_SLOT(cad, nid, payload) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_SET_HDMI_CHAN_SLOT, (payload))) #define HDA_HDMI_CODING_TYPE_REF_STREAM_HEADER 0 #define HDA_HDMI_CODING_TYPE_LPCM 1 #define HDA_HDMI_CODING_TYPE_AC3 2 #define HDA_HDMI_CODING_TYPE_MPEG1 3 #define HDA_HDMI_CODING_TYPE_MP3 4 #define HDA_HDMI_CODING_TYPE_MPEG2 5 #define HDA_HDMI_CODING_TYPE_AACLC 6 #define HDA_HDMI_CODING_TYPE_DTS 7 #define HDA_HDMI_CODING_TYPE_ATRAC 8 #define HDA_HDMI_CODING_TYPE_SACD 9 #define HDA_HDMI_CODING_TYPE_EAC3 10 #define HDA_HDMI_CODING_TYPE_DTS_HD 11 #define HDA_HDMI_CODING_TYPE_MLP 12 #define HDA_HDMI_CODING_TYPE_DST 13 #define HDA_HDMI_CODING_TYPE_WMAPRO 14 #define HDA_HDMI_CODING_TYPE_REF_CTX 15 /* Function Reset */ #define HDA_CMD_VERB_FUNCTION_RESET 0x7ff #define HDA_CMD_FUNCTION_RESET(cad, nid) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_FUNCTION_RESET, 0x0)) /**************************************************************************** * HDA Device Parameters ****************************************************************************/ /* Vendor ID */ #define HDA_PARAM_VENDOR_ID 0x00 #define HDA_PARAM_VENDOR_ID_VENDOR_ID_MASK 0xffff0000 #define HDA_PARAM_VENDOR_ID_VENDOR_ID_SHIFT 16 #define HDA_PARAM_VENDOR_ID_DEVICE_ID_MASK 0x0000ffff #define HDA_PARAM_VENDOR_ID_DEVICE_ID_SHIFT 0 #define HDA_PARAM_VENDOR_ID_VENDOR_ID(param) \ (((param) & HDA_PARAM_VENDOR_ID_VENDOR_ID_MASK) >> \ HDA_PARAM_VENDOR_ID_VENDOR_ID_SHIFT) #define HDA_PARAM_VENDOR_ID_DEVICE_ID(param) \ (((param) & HDA_PARAM_VENDOR_ID_DEVICE_ID_MASK) >> \ HDA_PARAM_VENDOR_ID_DEVICE_ID_SHIFT) /* Revision ID */ #define HDA_PARAM_REVISION_ID 0x02 #define HDA_PARAM_REVISION_ID_MAJREV_MASK 0x00f00000 #define HDA_PARAM_REVISION_ID_MAJREV_SHIFT 20 #define HDA_PARAM_REVISION_ID_MINREV_MASK 0x000f0000 #define HDA_PARAM_REVISION_ID_MINREV_SHIFT 16 #define HDA_PARAM_REVISION_ID_REVISION_ID_MASK 0x0000ff00 #define HDA_PARAM_REVISION_ID_REVISION_ID_SHIFT 8 #define HDA_PARAM_REVISION_ID_STEPPING_ID_MASK 0x000000ff #define HDA_PARAM_REVISION_ID_STEPPING_ID_SHIFT 0 #define HDA_PARAM_REVISION_ID_MAJREV(param) \ (((param) & HDA_PARAM_REVISION_ID_MAJREV_MASK) >> \ HDA_PARAM_REVISION_ID_MAJREV_SHIFT) #define HDA_PARAM_REVISION_ID_MINREV(param) \ (((param) & HDA_PARAM_REVISION_ID_MINREV_MASK) >> \ HDA_PARAM_REVISION_ID_MINREV_SHIFT) #define HDA_PARAM_REVISION_ID_REVISION_ID(param) \ (((param) & HDA_PARAM_REVISION_ID_REVISION_ID_MASK) >> \ HDA_PARAM_REVISION_ID_REVISION_ID_SHIFT) #define HDA_PARAM_REVISION_ID_STEPPING_ID(param) \ (((param) & HDA_PARAM_REVISION_ID_STEPPING_ID_MASK) >> \ HDA_PARAM_REVISION_ID_STEPPING_ID_SHIFT) /* Subordinate Node Cound */ #define HDA_PARAM_SUB_NODE_COUNT 0x04 #define HDA_PARAM_SUB_NODE_COUNT_START_MASK 0x00ff0000 #define HDA_PARAM_SUB_NODE_COUNT_START_SHIFT 16 #define HDA_PARAM_SUB_NODE_COUNT_TOTAL_MASK 0x000000ff #define HDA_PARAM_SUB_NODE_COUNT_TOTAL_SHIFT 0 #define HDA_PARAM_SUB_NODE_COUNT_START(param) \ (((param) & HDA_PARAM_SUB_NODE_COUNT_START_MASK) >> \ HDA_PARAM_SUB_NODE_COUNT_START_SHIFT) #define HDA_PARAM_SUB_NODE_COUNT_TOTAL(param) \ (((param) & HDA_PARAM_SUB_NODE_COUNT_TOTAL_MASK) >> \ HDA_PARAM_SUB_NODE_COUNT_TOTAL_SHIFT) /* Function Group Type */ #define HDA_PARAM_FCT_GRP_TYPE 0x05 #define HDA_PARAM_FCT_GRP_TYPE_UNSOL_MASK 0x00000100 #define HDA_PARAM_FCT_GRP_TYPE_UNSOL_SHIFT 8 #define HDA_PARAM_FCT_GRP_TYPE_NODE_TYPE_MASK 0x000000ff #define HDA_PARAM_FCT_GRP_TYPE_NODE_TYPE_SHIFT 0 #define HDA_PARAM_FCT_GRP_TYPE_UNSOL(param) \ (((param) & HDA_PARAM_FCT_GRP_TYPE_UNSOL_MASK) >> \ HDA_PARAM_FCT_GROUP_TYPE_UNSOL_SHIFT) #define HDA_PARAM_FCT_GRP_TYPE_NODE_TYPE(param) \ (((param) & HDA_PARAM_FCT_GRP_TYPE_NODE_TYPE_MASK) >> \ HDA_PARAM_FCT_GRP_TYPE_NODE_TYPE_SHIFT) #define HDA_PARAM_FCT_GRP_TYPE_NODE_TYPE_AUDIO 0x01 #define HDA_PARAM_FCT_GRP_TYPE_NODE_TYPE_MODEM 0x02 /* Audio Function Group Capabilities */ #define HDA_PARAM_AUDIO_FCT_GRP_CAP 0x08 #define HDA_PARAM_AUDIO_FCT_GRP_CAP_BEEP_GEN_MASK 0x00010000 #define HDA_PARAM_AUDIO_FCT_GRP_CAP_BEEP_GEN_SHIFT 16 #define HDA_PARAM_AUDIO_FCT_GRP_CAP_INPUT_DELAY_MASK 0x00000f00 #define HDA_PARAM_AUDIO_FCT_GRP_CAP_INPUT_DELAY_SHIFT 8 #define HDA_PARAM_AUDIO_FCT_GRP_CAP_OUTPUT_DELAY_MASK 0x0000000f #define HDA_PARAM_AUDIO_FCT_GRP_CAP_OUTPUT_DELAY_SHIFT 0 #define HDA_PARAM_AUDIO_FCT_GRP_CAP_BEEP_GEN(param) \ (((param) & HDA_PARAM_AUDIO_FCT_GRP_CAP_BEEP_GEN_MASK) >> \ HDA_PARAM_AUDIO_FCT_GRP_CAP_BEEP_GEN_SHIFT) #define HDA_PARAM_AUDIO_FCT_GRP_CAP_INPUT_DELAY(param) \ (((param) & HDA_PARAM_AUDIO_FCT_GRP_CAP_INPUT_DELAY_MASK) >> \ HDA_PARAM_AUDIO_FCT_GRP_CAP_INPUT_DELAY_SHIFT) #define HDA_PARAM_AUDIO_FCT_GRP_CAP_OUTPUT_DELAY(param) \ (((param) & HDA_PARAM_AUDIO_FCT_GRP_CAP_OUTPUT_DELAY_MASK) >> \ HDA_PARAM_AUDIO_FCT_GRP_CAP_OUTPUT_DELAY_SHIFT) /* Audio Widget Capabilities */ #define HDA_PARAM_AUDIO_WIDGET_CAP 0x09 #define HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_MASK 0x00f00000 #define HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_SHIFT 20 #define HDA_PARAM_AUDIO_WIDGET_CAP_DELAY_MASK 0x000f0000 #define HDA_PARAM_AUDIO_WIDGET_CAP_DELAY_SHIFT 16 #define HDA_PARAM_AUDIO_WIDGET_CAP_CC_EXT_MASK 0x0000e000 #define HDA_PARAM_AUDIO_WIDGET_CAP_CC_EXT_SHIFT 13 #define HDA_PARAM_AUDIO_WIDGET_CAP_CP_MASK 0x00001000 #define HDA_PARAM_AUDIO_WIDGET_CAP_CP_SHIFT 12 #define HDA_PARAM_AUDIO_WIDGET_CAP_LR_SWAP_MASK 0x00000800 #define HDA_PARAM_AUDIO_WIDGET_CAP_LR_SWAP_SHIFT 11 #define HDA_PARAM_AUDIO_WIDGET_CAP_POWER_CTRL_MASK 0x00000400 #define HDA_PARAM_AUDIO_WIDGET_CAP_POWER_CTRL_SHIFT 10 #define HDA_PARAM_AUDIO_WIDGET_CAP_DIGITAL_MASK 0x00000200 #define HDA_PARAM_AUDIO_WIDGET_CAP_DIGITAL_SHIFT 9 #define HDA_PARAM_AUDIO_WIDGET_CAP_CONN_LIST_MASK 0x00000100 #define HDA_PARAM_AUDIO_WIDGET_CAP_CONN_LIST_SHIFT 8 #define HDA_PARAM_AUDIO_WIDGET_CAP_UNSOL_CAP_MASK 0x00000080 #define HDA_PARAM_AUDIO_WIDGET_CAP_UNSOL_CAP_SHIFT 7 #define HDA_PARAM_AUDIO_WIDGET_CAP_PROC_WIDGET_MASK 0x00000040 #define HDA_PARAM_AUDIO_WIDGET_CAP_PROC_WIDGET_SHIFT 6 #define HDA_PARAM_AUDIO_WIDGET_CAP_STRIPE_MASK 0x00000020 #define HDA_PARAM_AUDIO_WIDGET_CAP_STRIPE_SHIFT 5 #define HDA_PARAM_AUDIO_WIDGET_CAP_FORMAT_OVR_MASK 0x00000010 #define HDA_PARAM_AUDIO_WIDGET_CAP_FORMAT_OVR_SHIFT 4 #define HDA_PARAM_AUDIO_WIDGET_CAP_AMP_OVR_MASK 0x00000008 #define HDA_PARAM_AUDIO_WIDGET_CAP_AMP_OVR_SHIFT 3 #define HDA_PARAM_AUDIO_WIDGET_CAP_OUT_AMP_MASK 0x00000004 #define HDA_PARAM_AUDIO_WIDGET_CAP_OUT_AMP_SHIFT 2 #define HDA_PARAM_AUDIO_WIDGET_CAP_IN_AMP_MASK 0x00000002 #define HDA_PARAM_AUDIO_WIDGET_CAP_IN_AMP_SHIFT 1 #define HDA_PARAM_AUDIO_WIDGET_CAP_STEREO_MASK 0x00000001 #define HDA_PARAM_AUDIO_WIDGET_CAP_STEREO_SHIFT 0 #define HDA_PARAM_AUDIO_WIDGET_CAP_TYPE(param) \ (((param) & HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_MASK) >> \ HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_SHIFT) #define HDA_PARAM_AUDIO_WIDGET_CAP_DELAY(param) \ (((param) & HDA_PARAM_AUDIO_WIDGET_CAP_DELAY_MASK) >> \ HDA_PARAM_AUDIO_WIDGET_CAP_DELAY_SHIFT) #define HDA_PARAM_AUDIO_WIDGET_CAP_CC(param) \ ((((param) & HDA_PARAM_AUDIO_WIDGET_CAP_CC_EXT_MASK) >> \ (HDA_PARAM_AUDIO_WIDGET_CAP_CC_EXT_SHIFT - 1)) | \ (((param) & HDA_PARAM_AUDIO_WIDGET_CAP_STEREO_MASK) >> \ HDA_PARAM_AUDIO_WIDGET_CAP_STEREO_SHIFT)) #define HDA_PARAM_AUDIO_WIDGET_CAP_CP(param) \ (((param) & HDA_PARAM_AUDIO_WIDGET_CAP_CP_MASK) >> \ HDA_PARAM_AUDIO_WIDGET_CAP_CP_SHIFT) #define HDA_PARAM_AUDIO_WIDGET_CAP_LR_SWAP(param) \ (((param) & HDA_PARAM_AUDIO_WIDGET_CAP_LR_SWAP_MASK) >> \ HDA_PARAM_AUDIO_WIDGET_CAP_LR_SWAP_SHIFT) #define HDA_PARAM_AUDIO_WIDGET_CAP_POWER_CTRL(param) \ (((param) & HDA_PARAM_AUDIO_WIDGET_CAP_POWER_CTRL_MASK) >> \ HDA_PARAM_AUDIO_WIDGET_CAP_POWER_CTRL_SHIFT) #define HDA_PARAM_AUDIO_WIDGET_CAP_DIGITAL(param) \ (((param) & HDA_PARAM_AUDIO_WIDGET_CAP_DIGITAL_MASK) >> \ HDA_PARAM_AUDIO_WIDGET_CAP_DIGITAL_SHIFT) #define HDA_PARAM_AUDIO_WIDGET_CAP_CONN_LIST(param) \ (((param) & HDA_PARAM_AUDIO_WIDGET_CAP_CONN_LIST_MASK) >> \ HDA_PARAM_AUDIO_WIDGET_CAP_CONN_LIST_SHIFT) #define HDA_PARAM_AUDIO_WIDGET_CAP_UNSOL_CAP(param) \ (((param) & HDA_PARAM_AUDIO_WIDGET_CAP_UNSOL_CAP_MASK) >> \ HDA_PARAM_AUDIO_WIDGET_CAP_UNSOL_CAP_SHIFT) #define HDA_PARAM_AUDIO_WIDGET_CAP_PROC_WIDGET(param) \ (((param) & HDA_PARAM_AUDIO_WIDGET_CAP_PROC_WIDGET_MASK) >> \ HDA_PARAM_AUDIO_WIDGET_CAP_PROC_WIDGET_SHIFT) #define HDA_PARAM_AUDIO_WIDGET_CAP_STRIPE(param) \ (((param) & HDA_PARAM_AUDIO_WIDGET_CAP_STRIPE_MASK) >> \ HDA_PARAM_AUDIO_WIDGET_CAP_STRIPE_SHIFT) #define HDA_PARAM_AUDIO_WIDGET_CAP_FORMAT_OVR(param) \ (((param) & HDA_PARAM_AUDIO_WIDGET_CAP_FORMAT_OVR_MASK) >> \ HDA_PARAM_AUDIO_WIDGET_CAP_FORMAT_OVR_SHIFT) #define HDA_PARAM_AUDIO_WIDGET_CAP_AMP_OVR(param) \ (((param) & HDA_PARAM_AUDIO_WIDGET_CAP_AMP_OVR_MASK) >> \ HDA_PARAM_AUDIO_WIDGET_CAP_AMP_OVR_SHIFT) #define HDA_PARAM_AUDIO_WIDGET_CAP_OUT_AMP(param) \ (((param) & HDA_PARAM_AUDIO_WIDGET_CAP_OUT_AMP_MASK) >> \ HDA_PARAM_AUDIO_WIDGET_CAP_OUT_AMP_SHIFT) #define HDA_PARAM_AUDIO_WIDGET_CAP_IN_AMP(param) \ (((param) & HDA_PARAM_AUDIO_WIDGET_CAP_IN_AMP_MASK) >> \ HDA_PARAM_AUDIO_WIDGET_CAP_IN_AMP_SHIFT) #define HDA_PARAM_AUDIO_WIDGET_CAP_STEREO(param) \ (((param) & HDA_PARAM_AUDIO_WIDGET_CAP_STEREO_MASK) >> \ HDA_PARAM_AUDIO_WIDGET_CAP_STEREO_SHIFT) #define HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_AUDIO_OUTPUT 0x0 #define HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_AUDIO_INPUT 0x1 #define HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_AUDIO_MIXER 0x2 #define HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_AUDIO_SELECTOR 0x3 #define HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_PIN_COMPLEX 0x4 #define HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_POWER_WIDGET 0x5 #define HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_VOLUME_WIDGET 0x6 #define HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_BEEP_WIDGET 0x7 #define HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_VENDOR_WIDGET 0xf /* Supported PCM Size, Rates */ #define HDA_PARAM_SUPP_PCM_SIZE_RATE 0x0a #define HDA_PARAM_SUPP_PCM_SIZE_RATE_32BIT_MASK 0x00100000 #define HDA_PARAM_SUPP_PCM_SIZE_RATE_32BIT_SHIFT 20 #define HDA_PARAM_SUPP_PCM_SIZE_RATE_24BIT_MASK 0x00080000 #define HDA_PARAM_SUPP_PCM_SIZE_RATE_24BIT_SHIFT 19 #define HDA_PARAM_SUPP_PCM_SIZE_RATE_20BIT_MASK 0x00040000 #define HDA_PARAM_SUPP_PCM_SIZE_RATE_20BIT_SHIFT 18 #define HDA_PARAM_SUPP_PCM_SIZE_RATE_16BIT_MASK 0x00020000 #define HDA_PARAM_SUPP_PCM_SIZE_RATE_16BIT_SHIFT 17 #define HDA_PARAM_SUPP_PCM_SIZE_RATE_8BIT_MASK 0x00010000 #define HDA_PARAM_SUPP_PCM_SIZE_RATE_8BIT_SHIFT 16 #define HDA_PARAM_SUPP_PCM_SIZE_RATE_8KHZ_MASK 0x00000001 #define HDA_PARAM_SUPP_PCM_SIZE_RATE_8KHZ_SHIFT 0 #define HDA_PARAM_SUPP_PCM_SIZE_RATE_11KHZ_MASK 0x00000002 #define HDA_PARAM_SUPP_PCM_SIZE_RATE_11KHZ_SHIFT 1 #define HDA_PARAM_SUPP_PCM_SIZE_RATE_16KHZ_MASK 0x00000004 #define HDA_PARAM_SUPP_PCM_SIZE_RATE_16KHZ_SHIFT 2 #define HDA_PARAM_SUPP_PCM_SIZE_RATE_22KHZ_MASK 0x00000008 #define HDA_PARAM_SUPP_PCM_SIZE_RATE_22KHZ_SHIFT 3 #define HDA_PARAM_SUPP_PCM_SIZE_RATE_32KHZ_MASK 0x00000010 #define HDA_PARAM_SUPP_PCM_SIZE_RATE_32KHZ_SHIFT 4 #define HDA_PARAM_SUPP_PCM_SIZE_RATE_44KHZ_MASK 0x00000020 #define HDA_PARAM_SUPP_PCM_SIZE_RATE_44KHZ_SHIFT 5 #define HDA_PARAM_SUPP_PCM_SIZE_RATE_48KHZ_MASK 0x00000040 #define HDA_PARAM_SUPP_PCM_SIZE_RATE_48KHZ_SHIFT 6 #define HDA_PARAM_SUPP_PCM_SIZE_RATE_88KHZ_MASK 0x00000080 #define HDA_PARAM_SUPP_PCM_SIZE_RATE_88KHZ_SHIFT 7 #define HDA_PARAM_SUPP_PCM_SIZE_RATE_96KHZ_MASK 0x00000100 #define HDA_PARAM_SUPP_PCM_SIZE_RATE_96KHZ_SHIFT 8 #define HDA_PARAM_SUPP_PCM_SIZE_RATE_176KHZ_MASK 0x00000200 #define HDA_PARAM_SUPP_PCM_SIZE_RATE_176KHZ_SHIFT 9 #define HDA_PARAM_SUPP_PCM_SIZE_RATE_192KHZ_MASK 0x00000400 #define HDA_PARAM_SUPP_PCM_SIZE_RATE_192KHZ_SHIFT 10 #define HDA_PARAM_SUPP_PCM_SIZE_RATE_384KHZ_MASK 0x00000800 #define HDA_PARAM_SUPP_PCM_SIZE_RATE_384KHZ_SHIFT 11 #define HDA_PARAM_SUPP_PCM_SIZE_RATE_32BIT(param) \ (((param) & HDA_PARAM_SUPP_PCM_SIZE_RATE_32BIT_MASK) >> \ HDA_PARAM_SUPP_PCM_SIZE_RATE_32BIT_SHIFT) #define HDA_PARAM_SUPP_PCM_SIZE_RATE_24BIT(param) \ (((param) & HDA_PARAM_SUPP_PCM_SIZE_RATE_24BIT_MASK) >> \ HDA_PARAM_SUPP_PCM_SIZE_RATE_24BIT_SHIFT) #define HDA_PARAM_SUPP_PCM_SIZE_RATE_20BIT(param) \ (((param) & HDA_PARAM_SUPP_PCM_SIZE_RATE_20BIT_MASK) >> \ HDA_PARAM_SUPP_PCM_SIZE_RATE_20BIT_SHIFT) #define HDA_PARAM_SUPP_PCM_SIZE_RATE_16BIT(param) \ (((param) & HDA_PARAM_SUPP_PCM_SIZE_RATE_16BIT_MASK) >> \ HDA_PARAM_SUPP_PCM_SIZE_RATE_16BIT_SHIFT) #define HDA_PARAM_SUPP_PCM_SIZE_RATE_8BIT(param) \ (((param) & HDA_PARAM_SUPP_PCM_SIZE_RATE_8BIT_MASK) >> \ HDA_PARAM_SUPP_PCM_SIZE_RATE_8BIT_SHIFT) #define HDA_PARAM_SUPP_PCM_SIZE_RATE_8KHZ(param) \ (((param) & HDA_PARAM_SUPP_PCM_SIZE_RATE_8KHZ_MASK) >> \ HDA_PARAM_SUPP_PCM_SIZE_RATE_8KHZ_SHIFT) #define HDA_PARAM_SUPP_PCM_SIZE_RATE_11KHZ(param) \ (((param) & HDA_PARAM_SUPP_PCM_SIZE_RATE_11KHZ_MASK) >> \ HDA_PARAM_SUPP_PCM_SIZE_RATE_11KHZ_SHIFT) #define HDA_PARAM_SUPP_PCM_SIZE_RATE_16KHZ(param) \ (((param) & HDA_PARAM_SUPP_PCM_SIZE_RATE_16KHZ_MASK) >> \ HDA_PARAM_SUPP_PCM_SIZE_RATE_16KHZ_SHIFT) #define HDA_PARAM_SUPP_PCM_SIZE_RATE_22KHZ(param) \ (((param) & HDA_PARAM_SUPP_PCM_SIZE_RATE_22KHZ_MASK) >> \ HDA_PARAM_SUPP_PCM_SIZE_RATE_22KHZ_SHIFT) #define HDA_PARAM_SUPP_PCM_SIZE_RATE_32KHZ(param) \ (((param) & HDA_PARAM_SUPP_PCM_SIZE_RATE_32KHZ_MASK) >> \ HDA_PARAM_SUPP_PCM_SIZE_RATE_32KHZ_SHIFT) #define HDA_PARAM_SUPP_PCM_SIZE_RATE_44KHZ(param) \ (((param) & HDA_PARAM_SUPP_PCM_SIZE_RATE_44KHZ_MASK) >> \ HDA_PARAM_SUPP_PCM_SIZE_RATE_44KHZ_SHIFT) #define HDA_PARAM_SUPP_PCM_SIZE_RATE_48KHZ(param) \ (((param) & HDA_PARAM_SUPP_PCM_SIZE_RATE_48KHZ_MASK) >> \ HDA_PARAM_SUPP_PCM_SIZE_RATE_48KHZ_SHIFT) #define HDA_PARAM_SUPP_PCM_SIZE_RATE_88KHZ(param) \ (((param) & HDA_PARAM_SUPP_PCM_SIZE_RATE_88KHZ_MASK) >> \ HDA_PARAM_SUPP_PCM_SIZE_RATE_88KHZ_SHIFT) #define HDA_PARAM_SUPP_PCM_SIZE_RATE_96KHZ(param) \ (((param) & HDA_PARAM_SUPP_PCM_SIZE_RATE_96KHZ_MASK) >> \ HDA_PARAM_SUPP_PCM_SIZE_RATE_96KHZ_SHIFT) #define HDA_PARAM_SUPP_PCM_SIZE_RATE_176KHZ(param) \ (((param) & HDA_PARAM_SUPP_PCM_SIZE_RATE_176KHZ_MASK) >> \ HDA_PARAM_SUPP_PCM_SIZE_RATE_176KHZ_SHIFT) #define HDA_PARAM_SUPP_PCM_SIZE_RATE_192KHZ(param) \ (((param) & HDA_PARAM_SUPP_PCM_SIZE_RATE_192KHZ_MASK) >> \ HDA_PARAM_SUPP_PCM_SIZE_RATE_192KHZ_SHIFT) #define HDA_PARAM_SUPP_PCM_SIZE_RATE_384KHZ(param) \ (((param) & HDA_PARAM_SUPP_PCM_SIZE_RATE_384KHZ_MASK) >> \ HDA_PARAM_SUPP_PCM_SIZE_RATE_384KHZ_SHIFT) /* Supported Stream Formats */ #define HDA_PARAM_SUPP_STREAM_FORMATS 0x0b #define HDA_PARAM_SUPP_STREAM_FORMATS_AC3_MASK 0x00000004 #define HDA_PARAM_SUPP_STREAM_FORMATS_AC3_SHIFT 2 #define HDA_PARAM_SUPP_STREAM_FORMATS_FLOAT32_MASK 0x00000002 #define HDA_PARAM_SUPP_STREAM_FORMATS_FLOAT32_SHIFT 1 #define HDA_PARAM_SUPP_STREAM_FORMATS_PCM_MASK 0x00000001 #define HDA_PARAM_SUPP_STREAM_FORMATS_PCM_SHIFT 0 #define HDA_PARAM_SUPP_STREAM_FORMATS_AC3(param) \ (((param) & HDA_PARAM_SUPP_STREAM_FORMATS_AC3_MASK) >> \ HDA_PARAM_SUPP_STREAM_FORMATS_AC3_SHIFT) #define HDA_PARAM_SUPP_STREAM_FORMATS_FLOAT32(param) \ (((param) & HDA_PARAM_SUPP_STREAM_FORMATS_FLOAT32_MASK) >> \ HDA_PARAM_SUPP_STREAM_FORMATS_FLOAT32_SHIFT) #define HDA_PARAM_SUPP_STREAM_FORMATS_PCM(param) \ (((param) & HDA_PARAM_SUPP_STREAM_FORMATS_PCM_MASK) >> \ HDA_PARAM_SUPP_STREAM_FORMATS_PCM_SHIFT) /* Pin Capabilities */ #define HDA_PARAM_PIN_CAP 0x0c #define HDA_PARAM_PIN_CAP_HBR_MASK 0x08000000 #define HDA_PARAM_PIN_CAP_HBR_SHIFT 27 #define HDA_PARAM_PIN_CAP_DP_MASK 0x01000000 #define HDA_PARAM_PIN_CAP_DP_SHIFT 24 #define HDA_PARAM_PIN_CAP_EAPD_CAP_MASK 0x00010000 #define HDA_PARAM_PIN_CAP_EAPD_CAP_SHIFT 16 #define HDA_PARAM_PIN_CAP_VREF_CTRL_MASK 0x0000ff00 #define HDA_PARAM_PIN_CAP_VREF_CTRL_SHIFT 8 #define HDA_PARAM_PIN_CAP_VREF_CTRL_100_MASK 0x00002000 #define HDA_PARAM_PIN_CAP_VREF_CTRL_100_SHIFT 13 #define HDA_PARAM_PIN_CAP_VREF_CTRL_80_MASK 0x00001000 #define HDA_PARAM_PIN_CAP_VREF_CTRL_80_SHIFT 12 #define HDA_PARAM_PIN_CAP_VREF_CTRL_GROUND_MASK 0x00000400 #define HDA_PARAM_PIN_CAP_VREF_CTRL_GROUND_SHIFT 10 #define HDA_PARAM_PIN_CAP_VREF_CTRL_50_MASK 0x00000200 #define HDA_PARAM_PIN_CAP_VREF_CTRL_50_SHIFT 9 #define HDA_PARAM_PIN_CAP_VREF_CTRL_HIZ_MASK 0x00000100 #define HDA_PARAM_PIN_CAP_VREF_CTRL_HIZ_SHIFT 8 #define HDA_PARAM_PIN_CAP_HDMI_MASK 0x00000080 #define HDA_PARAM_PIN_CAP_HDMI_SHIFT 7 #define HDA_PARAM_PIN_CAP_BALANCED_IO_PINS_MASK 0x00000040 #define HDA_PARAM_PIN_CAP_BALANCED_IO_PINS_SHIFT 6 #define HDA_PARAM_PIN_CAP_INPUT_CAP_MASK 0x00000020 #define HDA_PARAM_PIN_CAP_INPUT_CAP_SHIFT 5 #define HDA_PARAM_PIN_CAP_OUTPUT_CAP_MASK 0x00000010 #define HDA_PARAM_PIN_CAP_OUTPUT_CAP_SHIFT 4 #define HDA_PARAM_PIN_CAP_HEADPHONE_CAP_MASK 0x00000008 #define HDA_PARAM_PIN_CAP_HEADPHONE_CAP_SHIFT 3 #define HDA_PARAM_PIN_CAP_PRESENCE_DETECT_CAP_MASK 0x00000004 #define HDA_PARAM_PIN_CAP_PRESENCE_DETECT_CAP_SHIFT 2 #define HDA_PARAM_PIN_CAP_TRIGGER_REQD_MASK 0x00000002 #define HDA_PARAM_PIN_CAP_TRIGGER_REQD_SHIFT 1 #define HDA_PARAM_PIN_CAP_IMP_SENSE_CAP_MASK 0x00000001 #define HDA_PARAM_PIN_CAP_IMP_SENSE_CAP_SHIFT 0 #define HDA_PARAM_PIN_CAP_HBR(param) \ (((param) & HDA_PARAM_PIN_CAP_HBR_MASK) >> \ HDA_PARAM_PIN_CAP_HBR_SHIFT) #define HDA_PARAM_PIN_CAP_DP(param) \ (((param) & HDA_PARAM_PIN_CAP_DP_MASK) >> \ HDA_PARAM_PIN_CAP_DP_SHIFT) #define HDA_PARAM_PIN_CAP_EAPD_CAP(param) \ (((param) & HDA_PARAM_PIN_CAP_EAPD_CAP_MASK) >> \ HDA_PARAM_PIN_CAP_EAPD_CAP_SHIFT) #define HDA_PARAM_PIN_CAP_VREF_CTRL(param) \ (((param) & HDA_PARAM_PIN_CAP_VREF_CTRL_MASK) >> \ HDA_PARAM_PIN_CAP_VREF_CTRL_SHIFT) #define HDA_PARAM_PIN_CAP_VREF_CTRL_100(param) \ (((param) & HDA_PARAM_PIN_CAP_VREF_CTRL_100_MASK) >> \ HDA_PARAM_PIN_CAP_VREF_CTRL_100_SHIFT) #define HDA_PARAM_PIN_CAP_VREF_CTRL_80(param) \ (((param) & HDA_PARAM_PIN_CAP_VREF_CTRL_80_MASK) >> \ HDA_PARAM_PIN_CAP_VREF_CTRL_80_SHIFT) #define HDA_PARAM_PIN_CAP_VREF_CTRL_GROUND(param) \ (((param) & HDA_PARAM_PIN_CAP_VREF_CTRL_GROUND_MASK) >> \ HDA_PARAM_PIN_CAP_VREF_CTRL_GROUND_SHIFT) #define HDA_PARAM_PIN_CAP_VREF_CTRL_50(param) \ (((param) & HDA_PARAM_PIN_CAP_VREF_CTRL_50_MASK) >> \ HDA_PARAM_PIN_CAP_VREF_CTRL_50_SHIFT) #define HDA_PARAM_PIN_CAP_VREF_CTRL_HIZ(param) \ (((param) & HDA_PARAM_PIN_CAP_VREF_CTRL_HIZ_MASK) >> \ HDA_PARAM_PIN_CAP_VREF_CTRL_HIZ_SHIFT) #define HDA_PARAM_PIN_CAP_HDMI(param) \ (((param) & HDA_PARAM_PIN_CAP_HDMI_MASK) >> \ HDA_PARAM_PIN_CAP_HDMI_SHIFT) #define HDA_PARAM_PIN_CAP_BALANCED_IO_PINS(param) \ (((param) & HDA_PARAM_PIN_CAP_BALANCED_IO_PINS_MASK) >> \ HDA_PARAM_PIN_CAP_BALANCED_IO_PINS_SHIFT) #define HDA_PARAM_PIN_CAP_INPUT_CAP(param) \ (((param) & HDA_PARAM_PIN_CAP_INPUT_CAP_MASK) >> \ HDA_PARAM_PIN_CAP_INPUT_CAP_SHIFT) #define HDA_PARAM_PIN_CAP_OUTPUT_CAP(param) \ (((param) & HDA_PARAM_PIN_CAP_OUTPUT_CAP_MASK) >> \ HDA_PARAM_PIN_CAP_OUTPUT_CAP_SHIFT) #define HDA_PARAM_PIN_CAP_HEADPHONE_CAP(param) \ (((param) & HDA_PARAM_PIN_CAP_HEADPHONE_CAP_MASK) >> \ HDA_PARAM_PIN_CAP_HEADPHONE_CAP_SHIFT) #define HDA_PARAM_PIN_CAP_PRESENCE_DETECT_CAP(param) \ (((param) & HDA_PARAM_PIN_CAP_PRESENCE_DETECT_CAP_MASK) >> \ HDA_PARAM_PIN_CAP_PRESENCE_DETECT_CAP_SHIFT) #define HDA_PARAM_PIN_CAP_TRIGGER_REQD(param) \ (((param) & HDA_PARAM_PIN_CAP_TRIGGER_REQD_MASK) >> \ HDA_PARAM_PIN_CAP_TRIGGER_REQD_SHIFT) #define HDA_PARAM_PIN_CAP_IMP_SENSE_CAP(param) \ (((param) & HDA_PARAM_PIN_CAP_IMP_SENSE_CAP_MASK) >> \ HDA_PARAM_PIN_CAP_IMP_SENSE_CAP_SHIFT) /* Input Amplifier Capabilities */ #define HDA_PARAM_INPUT_AMP_CAP 0x0d #define HDA_PARAM_INPUT_AMP_CAP_MUTE_CAP_MASK 0x80000000 #define HDA_PARAM_INPUT_AMP_CAP_MUTE_CAP_SHIFT 31 #define HDA_PARAM_INPUT_AMP_CAP_STEPSIZE_MASK 0x007f0000 #define HDA_PARAM_INPUT_AMP_CAP_STEPSIZE_SHIFT 16 #define HDA_PARAM_INPUT_AMP_CAP_NUMSTEPS_MASK 0x00007f00 #define HDA_PARAM_INPUT_AMP_CAP_NUMSTEPS_SHIFT 8 #define HDA_PARAM_INPUT_AMP_CAP_OFFSET_MASK 0x0000007f #define HDA_PARAM_INPUT_AMP_CAP_OFFSET_SHIFT 0 #define HDA_PARAM_INPUT_AMP_CAP_MUTE_CAP(param) \ (((param) & HDA_PARAM_INPUT_AMP_CAP_MUTE_CAP_MASK) >> \ HDA_PARAM_INPUT_AMP_CAP_MUTE_CAP_SHIFT) #define HDA_PARAM_INPUT_AMP_CAP_STEPSIZE(param) \ (((param) & HDA_PARAM_INPUT_AMP_CAP_STEPSIZE_MASK) >> \ HDA_PARAM_INPUT_AMP_CAP_STEPSIZE_SHIFT) #define HDA_PARAM_INPUT_AMP_CAP_NUMSTEPS(param) \ (((param) & HDA_PARAM_INPUT_AMP_CAP_NUMSTEPS_MASK) >> \ HDA_PARAM_INPUT_AMP_CAP_NUMSTEPS_SHIFT) #define HDA_PARAM_INPUT_AMP_CAP_OFFSET(param) \ (((param) & HDA_PARAM_INPUT_AMP_CAP_OFFSET_MASK) >> \ HDA_PARAM_INPUT_AMP_CAP_OFFSET_SHIFT) /* Output Amplifier Capabilities */ #define HDA_PARAM_OUTPUT_AMP_CAP 0x12 #define HDA_PARAM_OUTPUT_AMP_CAP_MUTE_CAP_MASK 0x80000000 #define HDA_PARAM_OUTPUT_AMP_CAP_MUTE_CAP_SHIFT 31 #define HDA_PARAM_OUTPUT_AMP_CAP_STEPSIZE_MASK 0x007f0000 #define HDA_PARAM_OUTPUT_AMP_CAP_STEPSIZE_SHIFT 16 #define HDA_PARAM_OUTPUT_AMP_CAP_NUMSTEPS_MASK 0x00007f00 #define HDA_PARAM_OUTPUT_AMP_CAP_NUMSTEPS_SHIFT 8 #define HDA_PARAM_OUTPUT_AMP_CAP_OFFSET_MASK 0x0000007f #define HDA_PARAM_OUTPUT_AMP_CAP_OFFSET_SHIFT 0 #define HDA_PARAM_OUTPUT_AMP_CAP_MUTE_CAP(param) \ (((param) & HDA_PARAM_OUTPUT_AMP_CAP_MUTE_CAP_MASK) >> \ HDA_PARAM_OUTPUT_AMP_CAP_MUTE_CAP_SHIFT) #define HDA_PARAM_OUTPUT_AMP_CAP_STEPSIZE(param) \ (((param) & HDA_PARAM_OUTPUT_AMP_CAP_STEPSIZE_MASK) >> \ HDA_PARAM_OUTPUT_AMP_CAP_STEPSIZE_SHIFT) #define HDA_PARAM_OUTPUT_AMP_CAP_NUMSTEPS(param) \ (((param) & HDA_PARAM_OUTPUT_AMP_CAP_NUMSTEPS_MASK) >> \ HDA_PARAM_OUTPUT_AMP_CAP_NUMSTEPS_SHIFT) #define HDA_PARAM_OUTPUT_AMP_CAP_OFFSET(param) \ (((param) & HDA_PARAM_OUTPUT_AMP_CAP_OFFSET_MASK) >> \ HDA_PARAM_OUTPUT_AMP_CAP_OFFSET_SHIFT) /* Connection List Length */ #define HDA_PARAM_CONN_LIST_LENGTH 0x0e #define HDA_PARAM_CONN_LIST_LENGTH_LONG_FORM_MASK 0x00000080 #define HDA_PARAM_CONN_LIST_LENGTH_LONG_FORM_SHIFT 7 #define HDA_PARAM_CONN_LIST_LENGTH_LIST_LENGTH_MASK 0x0000007f #define HDA_PARAM_CONN_LIST_LENGTH_LIST_LENGTH_SHIFT 0 #define HDA_PARAM_CONN_LIST_LENGTH_LONG_FORM(param) \ (((param) & HDA_PARAM_CONN_LIST_LENGTH_LONG_FORM_MASK) >> \ HDA_PARAM_CONN_LIST_LENGTH_LONG_FORM_SHIFT) #define HDA_PARAM_CONN_LIST_LENGTH_LIST_LENGTH(param) \ (((param) & HDA_PARAM_CONN_LIST_LENGTH_LIST_LENGTH_MASK) >> \ HDA_PARAM_CONN_LIST_LENGTH_LIST_LENGTH_SHIFT) /* Supported Power States */ #define HDA_PARAM_SUPP_POWER_STATES 0x0f #define HDA_PARAM_SUPP_POWER_STATES_D3_MASK 0x00000008 #define HDA_PARAM_SUPP_POWER_STATES_D3_SHIFT 3 #define HDA_PARAM_SUPP_POWER_STATES_D2_MASK 0x00000004 #define HDA_PARAM_SUPP_POWER_STATES_D2_SHIFT 2 #define HDA_PARAM_SUPP_POWER_STATES_D1_MASK 0x00000002 #define HDA_PARAM_SUPP_POWER_STATES_D1_SHIFT 1 #define HDA_PARAM_SUPP_POWER_STATES_D0_MASK 0x00000001 #define HDA_PARAM_SUPP_POWER_STATES_D0_SHIFT 0 #define HDA_PARAM_SUPP_POWER_STATES_D3(param) \ (((param) & HDA_PARAM_SUPP_POWER_STATES_D3_MASK) >> \ HDA_PARAM_SUPP_POWER_STATES_D3_SHIFT) #define HDA_PARAM_SUPP_POWER_STATES_D2(param) \ (((param) & HDA_PARAM_SUPP_POWER_STATES_D2_MASK) >> \ HDA_PARAM_SUPP_POWER_STATES_D2_SHIFT) #define HDA_PARAM_SUPP_POWER_STATES_D1(param) \ (((param) & HDA_PARAM_SUPP_POWER_STATES_D1_MASK) >> \ HDA_PARAM_SUPP_POWER_STATES_D1_SHIFT) #define HDA_PARAM_SUPP_POWER_STATES_D0(param) \ (((param) & HDA_PARAM_SUPP_POWER_STATES_D0_MASK) >> \ HDA_PARAM_SUPP_POWER_STATES_D0_SHIFT) /* Processing Capabilities */ #define HDA_PARAM_PROCESSING_CAP 0x10 #define HDA_PARAM_PROCESSING_CAP_NUMCOEFF_MASK 0x0000ff00 #define HDA_PARAM_PROCESSING_CAP_NUMCOEFF_SHIFT 8 #define HDA_PARAM_PROCESSING_CAP_BENIGN_MASK 0x00000001 #define HDA_PARAM_PROCESSING_CAP_BENIGN_SHIFT 0 #define HDA_PARAM_PROCESSING_CAP_NUMCOEFF(param) \ (((param) & HDA_PARAM_PROCESSING_CAP_NUMCOEFF_MASK) >> \ HDA_PARAM_PROCESSING_CAP_NUMCOEFF_SHIFT) #define HDA_PARAM_PROCESSING_CAP_BENIGN(param) \ (((param) & HDA_PARAM_PROCESSING_CAP_BENIGN_MASK) >> \ HDA_PARAM_PROCESSING_CAP_BENIGN_SHIFT) /* GPIO Count */ #define HDA_PARAM_GPIO_COUNT 0x11 #define HDA_PARAM_GPIO_COUNT_GPI_WAKE_MASK 0x80000000 #define HDA_PARAM_GPIO_COUNT_GPI_WAKE_SHIFT 31 #define HDA_PARAM_GPIO_COUNT_GPI_UNSOL_MASK 0x40000000 #define HDA_PARAM_GPIO_COUNT_GPI_UNSOL_SHIFT 30 #define HDA_PARAM_GPIO_COUNT_NUM_GPI_MASK 0x00ff0000 #define HDA_PARAM_GPIO_COUNT_NUM_GPI_SHIFT 16 #define HDA_PARAM_GPIO_COUNT_NUM_GPO_MASK 0x0000ff00 #define HDA_PARAM_GPIO_COUNT_NUM_GPO_SHIFT 8 #define HDA_PARAM_GPIO_COUNT_NUM_GPIO_MASK 0x000000ff #define HDA_PARAM_GPIO_COUNT_NUM_GPIO_SHIFT 0 #define HDA_PARAM_GPIO_COUNT_GPI_WAKE(param) \ (((param) & HDA_PARAM_GPIO_COUNT_GPI_WAKE_MASK) >> \ HDA_PARAM_GPIO_COUNT_GPI_WAKE_SHIFT) #define HDA_PARAM_GPIO_COUNT_GPI_UNSOL(param) \ (((param) & HDA_PARAM_GPIO_COUNT_GPI_UNSOL_MASK) >> \ HDA_PARAM_GPIO_COUNT_GPI_UNSOL_SHIFT) #define HDA_PARAM_GPIO_COUNT_NUM_GPI(param) \ (((param) & HDA_PARAM_GPIO_COUNT_NUM_GPI_MASK) >> \ HDA_PARAM_GPIO_COUNT_NUM_GPI_SHIFT) #define HDA_PARAM_GPIO_COUNT_NUM_GPO(param) \ (((param) & HDA_PARAM_GPIO_COUNT_NUM_GPO_MASK) >> \ HDA_PARAM_GPIO_COUNT_NUM_GPO_SHIFT) #define HDA_PARAM_GPIO_COUNT_NUM_GPIO(param) \ (((param) & HDA_PARAM_GPIO_COUNT_NUM_GPIO_MASK) >> \ HDA_PARAM_GPIO_COUNT_NUM_GPIO_SHIFT) /* Volume Knob Capabilities */ #define HDA_PARAM_VOLUME_KNOB_CAP 0x13 #define HDA_PARAM_VOLUME_KNOB_CAP_DELTA_MASK 0x00000080 #define HDA_PARAM_VOLUME_KNOB_CAP_DELTA_SHIFT 7 #define HDA_PARAM_VOLUME_KNOB_CAP_NUM_STEPS_MASK 0x0000007f #define HDA_PARAM_VOLUME_KNOB_CAP_NUM_STEPS_SHIFT 0 #define HDA_PARAM_VOLUME_KNOB_CAP_DELTA(param) \ (((param) & HDA_PARAM_VOLUME_KNOB_CAP_DELTA_MASK) >> \ HDA_PARAM_VOLUME_KNOB_CAP_DELTA_SHIFT) #define HDA_PARAM_VOLUME_KNOB_CAP_NUM_STEPS(param) \ (((param) & HDA_PARAM_VOLUME_KNOB_CAP_NUM_STEPS_MASK) >> \ HDA_PARAM_VOLUME_KNOB_CAP_NUM_STEPS_SHIFT) #define HDA_CONFIG_DEFAULTCONF_SEQUENCE_MASK 0x0000000f #define HDA_CONFIG_DEFAULTCONF_SEQUENCE_SHIFT 0 #define HDA_CONFIG_DEFAULTCONF_ASSOCIATION_MASK 0x000000f0 #define HDA_CONFIG_DEFAULTCONF_ASSOCIATION_SHIFT 4 #define HDA_CONFIG_DEFAULTCONF_MISC_MASK 0x00000f00 #define HDA_CONFIG_DEFAULTCONF_MISC_SHIFT 8 #define HDA_CONFIG_DEFAULTCONF_COLOR_MASK 0x0000f000 #define HDA_CONFIG_DEFAULTCONF_COLOR_SHIFT 12 #define HDA_CONFIG_DEFAULTCONF_CONNECTION_TYPE_MASK 0x000f0000 #define HDA_CONFIG_DEFAULTCONF_CONNECTION_TYPE_SHIFT 16 #define HDA_CONFIG_DEFAULTCONF_DEVICE_MASK 0x00f00000 #define HDA_CONFIG_DEFAULTCONF_DEVICE_SHIFT 20 #define HDA_CONFIG_DEFAULTCONF_LOCATION_MASK 0x3f000000 #define HDA_CONFIG_DEFAULTCONF_LOCATION_SHIFT 24 #define HDA_CONFIG_DEFAULTCONF_CONNECTIVITY_MASK 0xc0000000 #define HDA_CONFIG_DEFAULTCONF_CONNECTIVITY_SHIFT 30 #define HDA_CONFIG_DEFAULTCONF_SEQUENCE(conf) \ (((conf) & HDA_CONFIG_DEFAULTCONF_SEQUENCE_MASK) >> \ HDA_CONFIG_DEFAULTCONF_SEQUENCE_SHIFT) #define HDA_CONFIG_DEFAULTCONF_ASSOCIATION(conf) \ (((conf) & HDA_CONFIG_DEFAULTCONF_ASSOCIATION_MASK) >> \ HDA_CONFIG_DEFAULTCONF_ASSOCIATION_SHIFT) #define HDA_CONFIG_DEFAULTCONF_MISC(conf) \ (((conf) & HDA_CONFIG_DEFAULTCONF_MISC_MASK) >> \ HDA_CONFIG_DEFAULTCONF_MISC_SHIFT) #define HDA_CONFIG_DEFAULTCONF_COLOR(conf) \ (((conf) & HDA_CONFIG_DEFAULTCONF_COLOR_MASK) >> \ HDA_CONFIG_DEFAULTCONF_COLOR_SHIFT) #define HDA_CONFIG_DEFAULTCONF_CONNECTION_TYPE(conf) \ (((conf) & HDA_CONFIG_DEFAULTCONF_CONNECTION_TYPE_MASK) >> \ HDA_CONFIG_DEFAULTCONF_CONNECTION_TYPE_SHIFT) #define HDA_CONFIG_DEFAULTCONF_DEVICE(conf) \ (((conf) & HDA_CONFIG_DEFAULTCONF_DEVICE_MASK) >> \ HDA_CONFIG_DEFAULTCONF_DEVICE_SHIFT) #define HDA_CONFIG_DEFAULTCONF_LOCATION(conf) \ (((conf) & HDA_CONFIG_DEFAULTCONF_LOCATION_MASK) >> \ HDA_CONFIG_DEFAULTCONF_LOCATION_SHIFT) #define HDA_CONFIG_DEFAULTCONF_CONNECTIVITY(conf) \ (((conf) & HDA_CONFIG_DEFAULTCONF_CONNECTIVITY_MASK) >> \ HDA_CONFIG_DEFAULTCONF_CONNECTIVITY_SHIFT) #define HDA_CONFIG_DEFAULTCONF_CONNECTIVITY_JACK (0<<30) #define HDA_CONFIG_DEFAULTCONF_CONNECTIVITY_NONE (1<<30) #define HDA_CONFIG_DEFAULTCONF_CONNECTIVITY_FIXED (2<<30) #define HDA_CONFIG_DEFAULTCONF_CONNECTIVITY_BOTH (3<<30) #define HDA_CONFIG_DEFAULTCONF_DEVICE_LINE_OUT (0<<20) #define HDA_CONFIG_DEFAULTCONF_DEVICE_SPEAKER (1<<20) #define HDA_CONFIG_DEFAULTCONF_DEVICE_HP_OUT (2<<20) #define HDA_CONFIG_DEFAULTCONF_DEVICE_CD (3<<20) #define HDA_CONFIG_DEFAULTCONF_DEVICE_SPDIF_OUT (4<<20) #define HDA_CONFIG_DEFAULTCONF_DEVICE_DIGITAL_OTHER_OUT (5<<20) #define HDA_CONFIG_DEFAULTCONF_DEVICE_MODEM_LINE (6<<20) #define HDA_CONFIG_DEFAULTCONF_DEVICE_MODEM_HANDSET (7<<20) #define HDA_CONFIG_DEFAULTCONF_DEVICE_LINE_IN (8<<20) #define HDA_CONFIG_DEFAULTCONF_DEVICE_AUX (9<<20) #define HDA_CONFIG_DEFAULTCONF_DEVICE_MIC_IN (10<<20) #define HDA_CONFIG_DEFAULTCONF_DEVICE_TELEPHONY (11<<20) #define HDA_CONFIG_DEFAULTCONF_DEVICE_SPDIF_IN (12<<20) #define HDA_CONFIG_DEFAULTCONF_DEVICE_DIGITAL_OTHER_IN (13<<20) #define HDA_CONFIG_DEFAULTCONF_DEVICE_OTHER (15<<20) #endif /* _HDA_REG_H_ */ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2006 Stephane E. Potvin * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef _HDAC_REG_H_ #define _HDAC_REG_H_ /**************************************************************************** * HDA Controller Register Set ****************************************************************************/ #define HDAC_GCAP 0x00 /* 2 - Global Capabilities*/ #define HDAC_VMIN 0x02 /* 1 - Minor Version */ #define HDAC_VMAJ 0x03 /* 1 - Major Version */ #define HDAC_OUTPAY 0x04 /* 2 - Output Payload Capability */ #define HDAC_INPAY 0x06 /* 2 - Input Payload Capability */ #define HDAC_GCTL 0x08 /* 4 - Global Control */ #define HDAC_WAKEEN 0x0c /* 2 - Wake Enable */ #define HDAC_STATESTS 0x0e /* 2 - State Change Status */ #define HDAC_GSTS 0x10 /* 2 - Global Status */ #define HDAC_OUTSTRMPAY 0x18 /* 2 - Output Stream Payload Capability */ #define HDAC_INSTRMPAY 0x1a /* 2 - Input Stream Payload Capability */ #define HDAC_INTCTL 0x20 /* 4 - Interrupt Control */ #define HDAC_INTSTS 0x24 /* 4 - Interrupt Status */ #define HDAC_WALCLK 0x30 /* 4 - Wall Clock Counter */ #define HDAC_SSYNC 0x38 /* 4 - Stream Synchronization */ #define HDAC_CORBLBASE 0x40 /* 4 - CORB Lower Base Address */ #define HDAC_CORBUBASE 0x44 /* 4 - CORB Upper Base Address */ #define HDAC_CORBWP 0x48 /* 2 - CORB Write Pointer */ #define HDAC_CORBRP 0x4a /* 2 - CORB Read Pointer */ #define HDAC_CORBCTL 0x4c /* 1 - CORB Control */ #define HDAC_CORBSTS 0x4d /* 1 - CORB Status */ #define HDAC_CORBSIZE 0x4e /* 1 - CORB Size */ #define HDAC_RIRBLBASE 0x50 /* 4 - RIRB Lower Base Address */ #define HDAC_RIRBUBASE 0x54 /* 4 - RIRB Upper Base Address */ #define HDAC_RIRBWP 0x58 /* 2 - RIRB Write Pointer */ #define HDAC_RINTCNT 0x5a /* 2 - Response Interrupt Count */ #define HDAC_RIRBCTL 0x5c /* 1 - RIRB Control */ #define HDAC_RIRBSTS 0x5d /* 1 - RIRB Status */ #define HDAC_RIRBSIZE 0x5e /* 1 - RIRB Size */ #define HDAC_ICOI 0x60 /* 4 - Immediate Command Output Interface */ #define HDAC_ICII 0x64 /* 4 - Immediate Command Input Interface */ #define HDAC_ICIS 0x68 /* 2 - Immediate Command Status */ #define HDAC_DPIBLBASE 0x70 /* 4 - DMA Position Buffer Lower Base */ #define HDAC_DPIBUBASE 0x74 /* 4 - DMA Position Buffer Upper Base */ #define HDAC_SDCTL0 0x80 /* 3 - Stream Descriptor Control */ #define HDAC_SDCTL1 0x81 /* 3 - Stream Descriptor Control */ #define HDAC_SDCTL2 0x82 /* 3 - Stream Descriptor Control */ #define HDAC_SDSTS 0x83 /* 1 - Stream Descriptor Status */ #define HDAC_SDLPIB 0x84 /* 4 - Link Position in Buffer */ #define HDAC_SDCBL 0x88 /* 4 - Cyclic Buffer Length */ #define HDAC_SDLVI 0x8C /* 2 - Last Valid Index */ #define HDAC_SDFIFOS 0x90 /* 2 - FIFOS */ #define HDAC_SDFMT 0x92 /* 2 - fmt */ #define HDAC_SDBDPL 0x98 /* 4 - Buffer Descriptor Pointer Lower Base */ #define HDAC_SDBDPU 0x9C /* 4 - Buffer Descriptor Pointer Upper Base */ #define _HDAC_ISDOFFSET(n, iss, oss) (0x80 + ((n) * 0x20)) #define _HDAC_ISDCTL(n, iss, oss) (0x00 + _HDAC_ISDOFFSET(n, iss, oss)) #define _HDAC_ISDSTS(n, iss, oss) (0x03 + _HDAC_ISDOFFSET(n, iss, oss)) #define _HDAC_ISDPICB(n, iss, oss) (0x04 + _HDAC_ISDOFFSET(n, iss, oss)) #define _HDAC_ISDCBL(n, iss, oss) (0x08 + _HDAC_ISDOFFSET(n, iss, oss)) #define _HDAC_ISDLVI(n, iss, oss) (0x0c + _HDAC_ISDOFFSET(n, iss, oss)) #define _HDAC_ISDFIFOD(n, iss, oss) (0x10 + _HDAC_ISDOFFSET(n, iss, oss)) #define _HDAC_ISDFMT(n, iss, oss) (0x12 + _HDAC_ISDOFFSET(n, iss, oss)) #define _HDAC_ISDBDPL(n, iss, oss) (0x18 + _HDAC_ISDOFFSET(n, iss, oss)) #define _HDAC_ISDBDPU(n, iss, oss) (0x1c + _HDAC_ISDOFFSET(n, iss, oss)) #define _HDAC_OSDOFFSET(n, iss, oss) (0x80 + ((iss) * 0x20) + ((n) * 0x20)) #define _HDAC_OSDCTL(n, iss, oss) (0x00 + _HDAC_OSDOFFSET(n, iss, oss)) #define _HDAC_OSDSTS(n, iss, oss) (0x03 + _HDAC_OSDOFFSET(n, iss, oss)) #define _HDAC_OSDPICB(n, iss, oss) (0x04 + _HDAC_OSDOFFSET(n, iss, oss)) #define _HDAC_OSDCBL(n, iss, oss) (0x08 + _HDAC_OSDOFFSET(n, iss, oss)) #define _HDAC_OSDLVI(n, iss, oss) (0x0c + _HDAC_OSDOFFSET(n, iss, oss)) #define _HDAC_OSDFIFOD(n, iss, oss) (0x10 + _HDAC_OSDOFFSET(n, iss, oss)) #define _HDAC_OSDFMT(n, iss, oss) (0x12 + _HDAC_OSDOFFSET(n, iss, oss)) #define _HDAC_OSDBDPL(n, iss, oss) (0x18 + _HDAC_OSDOFFSET(n, iss, oss)) #define _HDAC_OSDBDPU(n, iss, oss) (0x1c + _HDAC_OSDOFFSET(n, iss, oss)) #define _HDAC_BSDOFFSET(n, iss, oss) \ (0x80 + ((iss) * 0x20) + ((oss) * 0x20) + ((n) * 0x20)) #define _HDAC_BSDCTL(n, iss, oss) (0x00 + _HDAC_BSDOFFSET(n, iss, oss)) #define _HDAC_BSDSTS(n, iss, oss) (0x03 + _HDAC_BSDOFFSET(n, iss, oss)) #define _HDAC_BSDPICB(n, iss, oss) (0x04 + _HDAC_BSDOFFSET(n, iss, oss)) #define _HDAC_BSDCBL(n, iss, oss) (0x08 + _HDAC_BSDOFFSET(n, iss, oss)) #define _HDAC_BSDLVI(n, iss, oss) (0x0c + _HDAC_BSDOFFSET(n, iss, oss)) #define _HDAC_BSDFIFOD(n, iss, oss) (0x10 + _HDAC_BSDOFFSET(n, iss, oss)) #define _HDAC_BSDFMT(n, iss, oss) (0x12 + _HDAC_BSDOFFSET(n, iss, oss)) #define _HDAC_BSDBDPL(n, iss, oss) (0x18 + _HDAC_BSDOFFSET(n, iss, oss)) #define _HDAC_BSDBDBU(n, iss, oss) (0x1c + _HDAC_BSDOFFSET(n, iss, oss)) /**************************************************************************** * HDA Controller Register Fields ****************************************************************************/ /* GCAP - Global Capabilities */ #define HDAC_GCAP_64OK 0x0001 #define HDAC_GCAP_NSDO_MASK 0x0006 #define HDAC_GCAP_NSDO_SHIFT 1 #define HDAC_GCAP_BSS_MASK 0x00f8 #define HDAC_GCAP_BSS_SHIFT 3 #define HDAC_GCAP_ISS_MASK 0x0f00 #define HDAC_GCAP_ISS_SHIFT 8 #define HDAC_GCAP_OSS_MASK 0xf000 #define HDAC_GCAP_OSS_SHIFT 12 #define HDAC_GCAP_NSDO_1SDO 0x00 #define HDAC_GCAP_NSDO_2SDO 0x02 #define HDAC_GCAP_NSDO_4SDO 0x04 #define HDAC_GCAP_BSS(gcap) \ (((gcap) & HDAC_GCAP_BSS_MASK) >> HDAC_GCAP_BSS_SHIFT) #define HDAC_GCAP_ISS(gcap) \ (((gcap) & HDAC_GCAP_ISS_MASK) >> HDAC_GCAP_ISS_SHIFT) #define HDAC_GCAP_OSS(gcap) \ (((gcap) & HDAC_GCAP_OSS_MASK) >> HDAC_GCAP_OSS_SHIFT) #define HDAC_GCAP_NSDO(gcap) \ (((gcap) & HDAC_GCAP_NSDO_MASK) >> HDAC_GCAP_NSDO_SHIFT) /* GCTL - Global Control */ #define HDAC_GCTL_CRST 0x00000001 #define HDAC_GCTL_FCNTRL 0x00000002 #define HDAC_GCTL_UNSOL 0x00000100 /* WAKEEN - Wake Enable */ #define HDAC_WAKEEN_SDIWEN_MASK 0x7fff #define HDAC_WAKEEN_SDIWEN_SHIFT 0 /* STATESTS - State Change Status */ #define HDAC_STATESTS_SDIWAKE_MASK 0x7fff #define HDAC_STATESTS_SDIWAKE_SHIFT 0 #define HDAC_STATESTS_SDIWAKE(statests, n) \ (((((statests) & HDAC_STATESTS_SDIWAKE_MASK) >> \ HDAC_STATESTS_SDIWAKE_SHIFT) >> (n)) & 0x0001) /* GSTS - Global Status */ #define HDAC_GSTS_FSTS 0x0002 /* INTCTL - Interrut Control */ #define HDAC_INTCTL_SIE_MASK 0x3fffffff #define HDAC_INTCTL_SIE_SHIFT 0 #define HDAC_INTCTL_CIE 0x40000000 #define HDAC_INTCTL_GIE 0x80000000 /* INTSTS - Interrupt Status */ #define HDAC_INTSTS_SIS_MASK 0x3fffffff #define HDAC_INTSTS_SIS_SHIFT 0 #define HDAC_INTSTS_CIS 0x40000000 #define HDAC_INTSTS_GIS 0x80000000 /* SSYNC - Stream Synchronization */ #define HDAC_SSYNC_SSYNC_MASK 0x3fffffff #define HDAC_SSYNC_SSYNC_SHIFT 0 /* CORBWP - CORB Write Pointer */ #define HDAC_CORBWP_CORBWP_MASK 0x00ff #define HDAC_CORBWP_CORBWP_SHIFT 0 /* CORBRP - CORB Read Pointer */ #define HDAC_CORBRP_CORBRP_MASK 0x00ff #define HDAC_CORBRP_CORBRP_SHIFT 0 #define HDAC_CORBRP_CORBRPRST 0x8000 /* CORBCTL - CORB Control */ #define HDAC_CORBCTL_CMEIE 0x01 #define HDAC_CORBCTL_CORBRUN 0x02 /* CORBSTS - CORB Status */ #define HDAC_CORBSTS_CMEI 0x01 /* CORBSIZE - CORB Size */ #define HDAC_CORBSIZE_CORBSIZE_MASK 0x03 #define HDAC_CORBSIZE_CORBSIZE_SHIFT 0 #define HDAC_CORBSIZE_CORBSZCAP_MASK 0xf0 #define HDAC_CORBSIZE_CORBSZCAP_SHIFT 4 #define HDAC_CORBSIZE_CORBSIZE_2 0x00 #define HDAC_CORBSIZE_CORBSIZE_16 0x01 #define HDAC_CORBSIZE_CORBSIZE_256 0x02 #define HDAC_CORBSIZE_CORBSZCAP_2 0x10 #define HDAC_CORBSIZE_CORBSZCAP_16 0x20 #define HDAC_CORBSIZE_CORBSZCAP_256 0x40 #define HDAC_CORBSIZE_CORBSIZE(corbsize) \ (((corbsize) & HDAC_CORBSIZE_CORBSIZE_MASK) >> HDAC_CORBSIZE_CORBSIZE_SHIFT) /* RIRBWP - RIRB Write Pointer */ #define HDAC_RIRBWP_RIRBWP_MASK 0x00ff #define HDAC_RIRBWP_RIRBWP_SHIFT 0 #define HDAC_RIRBWP_RIRBWPRST 0x8000 /* RINTCTN - Response Interrupt Count */ #define HDAC_RINTCNT_MASK 0x00ff #define HDAC_RINTCNT_SHIFT 0 /* RIRBCTL - RIRB Control */ #define HDAC_RIRBCTL_RINTCTL 0x01 #define HDAC_RIRBCTL_RIRBDMAEN 0x02 #define HDAC_RIRBCTL_RIRBOIC 0x04 /* RIRBSTS - RIRB Status */ #define HDAC_RIRBSTS_RINTFL 0x01 #define HDAC_RIRBSTS_RIRBOIS 0x04 /* RIRBSIZE - RIRB Size */ #define HDAC_RIRBSIZE_RIRBSIZE_MASK 0x03 #define HDAC_RIRBSIZE_RIRBSIZE_SHIFT 0 #define HDAC_RIRBSIZE_RIRBSZCAP_MASK 0xf0 #define HDAC_RIRBSIZE_RIRBSZCAP_SHIFT 4 #define HDAC_RIRBSIZE_RIRBSIZE_2 0x00 #define HDAC_RIRBSIZE_RIRBSIZE_16 0x01 #define HDAC_RIRBSIZE_RIRBSIZE_256 0x02 #define HDAC_RIRBSIZE_RIRBSZCAP_2 0x10 #define HDAC_RIRBSIZE_RIRBSZCAP_16 0x20 #define HDAC_RIRBSIZE_RIRBSZCAP_256 0x40 #define HDAC_RIRBSIZE_RIRBSIZE(rirbsize) \ (((rirbsize) & HDAC_RIRBSIZE_RIRBSIZE_MASK) >> HDAC_RIRBSIZE_RIRBSIZE_SHIFT) /* DPLBASE - DMA Position Lower Base Address */ #define HDAC_DPLBASE_DPLBASE_MASK 0xffffff80 #define HDAC_DPLBASE_DPLBASE_SHIFT 7 #define HDAC_DPLBASE_DPLBASE_DMAPBE 0x00000001 /* SDCTL - Stream Descriptor Control */ #define HDAC_SDCTL_SRST 0x000001 #define HDAC_SDCTL_RUN 0x000002 #define HDAC_SDCTL_IOCE 0x000004 #define HDAC_SDCTL_FEIE 0x000008 #define HDAC_SDCTL_DEIE 0x000010 #define HDAC_SDCTL2_STRIPE_MASK 0x03 #define HDAC_SDCTL2_STRIPE_SHIFT 0 #define HDAC_SDCTL2_TP 0x04 #define HDAC_SDCTL2_DIR 0x08 #define HDAC_SDCTL2_STRM_MASK 0xf0 #define HDAC_SDCTL2_STRM_SHIFT 4 #define HDAC_SDSTS_DESE (1 << 4) #define HDAC_SDSTS_FIFOE (1 << 3) #define HDAC_SDSTS_BCIS (1 << 2) #endif /* _HDAC_REG_H_ */ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2016 Jakub Klama . * Copyright (c) 2018 Alexander Motin * Copyright (c) 2026 Hans Rosenfeld * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer * in this position and unchanged. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * This file and its contents are supplied under the terms of the * Common Development and Distribution License ("CDDL"), version 1.0. * You may only use this file in accordance with the terms of version * 1.0 of the CDDL. * * A full copy of the text of the CDDL should have accompanied this * source. A copy of the CDDL is also available via the Internet at * http://www.illumos.org/license/CDDL. */ /* This file is dual-licensed; see usr/src/contrib/bhyve/LICENSE */ /* * Copyright 2025 Oxide Computer Company */ #include #include #include #include #include #include #include #include #include #include #include #include "iov.h" /* * Given an array of iovecs iov, the number of valid iovecs niov, and an * offset, truncate iov at offset. If necessary, split the final iovec, * moving the remaining iovecs up by one in iov. Return a pointer to the * iovec beginning at offset, and the total number of remaining iovecs. * * The caller must take care that iov contains enough space for at least * niov+1 iovecs so the remainder of the iovec array may be moved up by one. */ struct iovec * split_iov(struct iovec *iov, size_t *niov, size_t offset, size_t *niov_rem) { size_t remainder = 0; struct iovec *iov_rem; size_t i; /* * Handle the special case of offset == 0: Return the whole iovec array * as the remainder. */ if (offset == 0) { *niov_rem = *niov; *niov = 0; return (iov); } /* Seek to the requested offset and truncate the final iovec. */ for (i = 0; i < *niov && offset > iov[i].iov_len; i++) { /* * We're seeking past this iovec. Adjust the offset and move on. */ offset -= iov[i].iov_len; } /* We've reached the end of the array without reaching the offset. */ if (i == *niov) { *niov_rem = 0; return (NULL); } /* * We found the iovec covering offset. Calculate the remainder and * truncate at offset. */ remainder = iov[i].iov_len - offset; iov[i].iov_len = offset; *niov_rem = *niov - i - 1; *niov = i + 1; iov_rem = &iov[*niov]; /* * If there's no remainder in this iovec, we're done. Return the * pointer to the next iovec after the offset, or NULL if there * are no more iovecs beyond offset. */ if (remainder == 0) { if (*niov_rem == 0) iov_rem = NULL; return (iov_rem); } /* * In the (unlikely, ideally) case where there is a remainder from the * final iovec before the split, make room for a new iovec covering the * remainder by moving all following iovecs up. It is the caller's * responsibility that there is enough spare space for this extra iovec. */ for (struct iovec *tmp = &iov_rem[*niov_rem]; tmp != iov_rem; tmp[0] = tmp[-1], tmp--) { ; } /* * Fill in the new first iovec, covering the remainder from the split. */ iov_rem[0].iov_len = remainder; iov_rem[0].iov_base = (char *)iov[i].iov_base + offset; (*niov_rem)++; return (iov_rem); } size_t count_iov(const struct iovec *iov, size_t niov) { size_t total = 0; size_t i; for (i = 0; i < niov; i++) total += iov[i].iov_len; return (total); } bool check_iov_len(const struct iovec *iov, size_t niov, size_t len) { size_t total = 0; size_t i; for (i = 0; i < niov; i++) { total += iov[i].iov_len; if (total >= len) return (true); } return (false); } ssize_t iov_to_buf(const struct iovec *iov, size_t niov, void **buf) { size_t ptr, total; size_t i; total = count_iov(iov, niov); *buf = reallocf(*buf, total); if (*buf == NULL) return (-1); for (i = 0, ptr = 0; i < niov; i++) { memcpy((uint8_t *)*buf + ptr, iov[i].iov_base, iov[i].iov_len); ptr += iov[i].iov_len; } return (total); } ssize_t buf_to_iov(const void *buf, size_t buflen, const struct iovec *iov, size_t niov) { size_t off = 0, len; size_t i; for (i = 0; i < niov && off < buflen; i++) { len = MIN(iov[i].iov_len, buflen - off); memcpy(iov[i].iov_base, (const uint8_t *)buf + off, len); off += len; } return ((ssize_t)off); } size_t iov_bunch_init(iov_bunch_t *iob, struct iovec *iov, int niov) { bzero(iob, sizeof (*iob)); iob->ib_iov = iov; iob->ib_remain = count_iov(iov, niov); return (iob->ib_remain); } /* * Copy `sz` bytes from iovecs contained in `iob` to `dst`. * * Returns `true` if copy was successful (implying adequate data was remaining * in the iov_bunch_t). */ bool iov_bunch_copy(iov_bunch_t *iob, void *dst, size_t sz) { if (sz > iob->ib_remain) return (false); if (sz == 0) return (true); caddr_t dest = dst; do { struct iovec *iov = iob->ib_iov; ASSERT3U(iov->iov_len, !=, 0); /* ib_offset is the offset within the current head of ib_iov */ const size_t iov_avail = iov->iov_len - iob->ib_offset; const size_t to_copy = MIN(sz, iov_avail); if (to_copy != 0 && dest != NULL) { bcopy((caddr_t)iov->iov_base + iob->ib_offset, dest, to_copy); dest += to_copy; } sz -= to_copy; iob->ib_remain -= to_copy; iob->ib_offset += to_copy; ASSERT3U(iob->ib_offset, <=, iov->iov_len); if (iob->ib_offset == iov->iov_len) { iob->ib_iov++; iob->ib_offset = 0; } } while (sz > 0); return (true); } /* * Skip `sz` bytes from iovecs contained in `iob`. * * Returns `true` if the skip was successful (implying adequate data was * remaining in the iov_bunch_t). */ bool iov_bunch_skip(iov_bunch_t *iob, size_t sz) { return (iov_bunch_copy(iob, NULL, sz)); } /* * Get the data pointer and length of the current head iovec, less any * offsetting from prior copy operations. This will advance the iov_bunch_t as * if the caller had performed a copy of that chunk length. * * Returns `true` if the iov_bunch_t had at least one iovec (unconsumed bytes) * remaining, setting `chunk` and `chunk_sz` to the chunk pointer and size, * respectively. */ bool iov_bunch_next_chunk(iov_bunch_t *iob, caddr_t *chunk, size_t *chunk_sz) { if (iob->ib_remain == 0) { *chunk = NULL; *chunk_sz = 0; return (false); } *chunk_sz = iob->ib_iov->iov_len - iob->ib_offset; *chunk = (caddr_t)iob->ib_iov->iov_base + iob->ib_offset; iob->ib_remain -= *chunk_sz; iob->ib_iov++; iob->ib_offset = 0; return (true); } /* * Extract the remaining data in an iov_bunch_t into a new iovec. */ void iov_bunch_to_iov(iov_bunch_t *iob, struct iovec *iov, int *niov, uint_t size) { *niov = 0; while (size-- > 0) { caddr_t chunk; size_t sz; if (!iov_bunch_next_chunk(iob, &chunk, &sz)) break; iov->iov_base = chunk; iov->iov_len = sz; iov++; (*niov)++; } } /* * Extract the remaining data in an iov_bunch_t into a buffer, reallocating it * to the required size. If there are no bytes remaining in the iov_bunch_t any * supplied buffer will be freed and this function will return 0. */ ssize_t iov_bunch_to_buf(iov_bunch_t *iob, void **buf) { size_t total = iob->ib_remain; if (total == 0) { free(*buf); *buf = NULL; return (0); } *buf = reallocf(*buf, total); if (*buf == NULL) return (-1); if (!iov_bunch_copy(iob, buf, total)) return (-1); if (total > SSIZE_MAX) { errno = EOVERFLOW; return (-1); } return (total); } /* * Copy data from a buffer into an iob_bunch_t. Returns true if there was * sufficient space for the data, false otherwise. */ bool buf_to_iov_bunch(iov_bunch_t *iob, const void *buf, size_t len) { const char *src = buf; if (iob->ib_remain < len) return (false); do { struct iovec *iov = iob->ib_iov; /* ib_offset is the offset within the current head of ib_iov */ const size_t iov_avail = iov->iov_len - iob->ib_offset; const size_t to_copy = MIN(len, iov_avail); if (to_copy != 0) { bcopy(src, (caddr_t)iov->iov_base + iob->ib_offset, to_copy); } src += to_copy; len -= to_copy; iob->ib_remain -= to_copy; iob->ib_offset += to_copy; ASSERT3U(iob->ib_offset, <=, iov->iov_len); if (iob->ib_offset == iov->iov_len) { iob->ib_iov++; iob->ib_offset = 0; } } while (len > 0); return (true); } /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2016 Jakub Klama . * Copyright (c) 2018 Alexander Motin * Copyright (c) 2026 Hans Rosenfeld * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer * in this position and unchanged. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * This file and its contents are supplied under the terms of the * Common Development and Distribution License ("CDDL"), version 1.0. * You may only use this file in accordance with the terms of version * 1.0 of the CDDL. * * A full copy of the text of the CDDL should have accompanied this * source. A copy of the CDDL is also available via the Internet at * http://www.illumos.org/license/CDDL. */ /* This file is dual-licensed; see usr/src/contrib/bhyve/LICENSE */ /* * Copyright 2025 Oxide Computer Company */ #ifndef _IOV_H_ #define _IOV_H_ #include /* Number of additional iovecs required for split_iov() */ #define SPLIT_IOV_ADDL_IOV 2 struct iovec *split_iov(struct iovec *, size_t *, size_t, size_t *); size_t count_iov(const struct iovec *, size_t); bool check_iov_len(const struct iovec *, size_t, size_t); ssize_t iov_to_buf(const struct iovec *, size_t, void **); ssize_t buf_to_iov(const void *, size_t, const struct iovec *, size_t); /* * Helpers for performing operations on an array of iovec entries. */ typedef struct iov_bunch { /* * The current head of an array of iovec entries, which have an iov_len * sum covering ib_remain bytes. This moves as the bunch is traversed. */ struct iovec *ib_iov; /* * Byte offset in current ib_iov entry. */ size_t ib_offset; /* * Bytes remaining in entries covered by ib_iov entries, not including * the offset specified by ib_offset */ size_t ib_remain; } iov_bunch_t; extern size_t iov_bunch_init(iov_bunch_t *, struct iovec *, int); extern bool iov_bunch_copy(iov_bunch_t *, void *, size_t); extern bool iov_bunch_skip(iov_bunch_t *, size_t); extern bool iov_bunch_next_chunk(iov_bunch_t *, caddr_t *, size_t *); extern void iov_bunch_to_iov(iov_bunch_t *, struct iovec *, int *, uint_t); extern ssize_t iov_bunch_to_buf(iov_bunch_t *iob, void **); extern bool buf_to_iov_bunch(iov_bunch_t *, const void *, size_t); #endif /* _IOV_H_ */ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2012 NetApp, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * This file and its contents are supplied under the terms of the * Common Development and Distribution License ("CDDL"), version 1.0. * You may only use this file in accordance with the terms of version * 1.0 of the CDDL. * * A full copy of the text of the CDDL should have accompanied this * source. A copy of the CDDL is also available via the Internet at * http://www.illumos.org/license/CDDL. * * Copyright 2020 Oxide Computer Company * Copyright 2026 OmniOS Community Edition (OmniOSce) Association. */ /* * Memory ranges are represented with an RB tree. On insertion, the range * is checked for overlaps. On lookup, the key has the same base and limit * so it can be searched within the range. */ #include #include #include #include #include #include #include #include #include #include #include "mem.h" #include "debug.h" struct mmio_rb_range { RB_ENTRY(mmio_rb_range) mr_link; /* RB tree links */ struct mem_range mr_param; uint64_t mr_base; uint64_t mr_end; }; struct mmio_rb_tree; RB_PROTOTYPE(mmio_rb_tree, mmio_rb_range, mr_link, mmio_rb_range_compare); static RB_HEAD(mmio_rb_tree, mmio_rb_range) mmio_rb_root, mmio_rb_fallback; /* * Per-vCPU cache. Since most accesses from a vCPU will be to * consecutive addresses in a range, it makes sense to cache the * result of a lookup. */ static struct mmio_rb_range **mmio_hint; static int mmio_ncpu; static pthread_rwlock_t mmio_rwlock; static int mmio_rb_range_compare(struct mmio_rb_range *a, struct mmio_rb_range *b) { if (a->mr_end < b->mr_base) return (-1); else if (a->mr_base > b->mr_end) return (1); return (0); } static int mmio_rb_lookup(struct mmio_rb_tree *rbt, uint64_t addr, struct mmio_rb_range **entry) { struct mmio_rb_range find, *res; find.mr_base = find.mr_end = addr; res = RB_FIND(mmio_rb_tree, rbt, &find); if (res != NULL) { *entry = res; return (0); } return (ENOENT); } static void mmio_rb_dump(struct mmio_rb_tree *rbt) { int perror; struct mmio_rb_range *np; pthread_rwlock_rdlock(&mmio_rwlock); RB_FOREACH(np, mmio_rb_tree, rbt) { PRINTLN(" %lx:%lx, %s", np->mr_base, np->mr_end, np->mr_param.name); } perror = pthread_rwlock_unlock(&mmio_rwlock); assert(perror == 0); } static int mmio_rb_add(struct mmio_rb_tree *rbt, struct mmio_rb_range *new) { struct mmio_rb_range *overlap; overlap = RB_INSERT(mmio_rb_tree, rbt, new); if (overlap != NULL) { EPRINTLN("overlap detected: new %lx:%lx, tree %lx:%lx, '%s' " "claims region already claimed for '%s'", new->mr_base, new->mr_end, overlap->mr_base, overlap->mr_end, new->mr_param.name, overlap->mr_param.name); return (EEXIST); } return (0); } RB_GENERATE(mmio_rb_tree, mmio_rb_range, mr_link, mmio_rb_range_compare); typedef int (mem_cb_t)(struct vcpu *vcpu, uint64_t gpa, struct mem_range *mr, void *arg); static int mem_read(struct vcpu *vcpu, uint64_t gpa, uint64_t *rval, int size, void *arg) { int error; struct mem_range *mr = arg; error = (*mr->handler)(vcpu, MEM_F_READ, gpa, size, rval, mr->arg1, mr->arg2); return (error); } static int mem_write(struct vcpu *vcpu, uint64_t gpa, uint64_t wval, int size, void *arg) { int error; struct mem_range *mr = arg; error = (*mr->handler)(vcpu, MEM_F_WRITE, gpa, size, &wval, mr->arg1, mr->arg2); return (error); } static int access_memory(struct vcpu *vcpu, uint64_t paddr, mem_cb_t *cb, void *arg) { struct mmio_rb_range *entry; int err, perror, immutable, vcpuid; vcpuid = vcpu_id(vcpu); pthread_rwlock_rdlock(&mmio_rwlock); /* * First check the per-vCPU cache */ if (mmio_hint[vcpuid] && paddr >= mmio_hint[vcpuid]->mr_base && paddr <= mmio_hint[vcpuid]->mr_end) { entry = mmio_hint[vcpuid]; } else entry = NULL; if (entry == NULL) { if (mmio_rb_lookup(&mmio_rb_root, paddr, &entry) == 0) { /* Update the per-vCPU cache */ mmio_hint[vcpuid] = entry; } else if (mmio_rb_lookup(&mmio_rb_fallback, paddr, &entry)) { perror = pthread_rwlock_unlock(&mmio_rwlock); assert(perror == 0); return (ESRCH); } } assert(entry != NULL); /* * An 'immutable' memory range is guaranteed to be never removed * so there is no need to hold 'mmio_rwlock' while calling the * handler. * * XXX writes to the PCIR_COMMAND register can cause register_mem() * to be called. If the guest is using PCI extended config space * to modify the PCIR_COMMAND register then register_mem() can * deadlock on 'mmio_rwlock'. However by registering the extended * config space window as 'immutable' the deadlock can be avoided. */ immutable = (entry->mr_param.flags & MEM_F_IMMUTABLE); if (immutable) { perror = pthread_rwlock_unlock(&mmio_rwlock); assert(perror == 0); } err = cb(vcpu, paddr, &entry->mr_param, arg); if (!immutable) { perror = pthread_rwlock_unlock(&mmio_rwlock); assert(perror == 0); } return (err); } static int emulate_mem_cb(struct vcpu *vcpu, uint64_t paddr, struct mem_range *mr, void *arg) { struct vm_mmio *mmio; int err = 0; mmio = arg; if (mmio->read != 0) { err = mem_read(vcpu, paddr, &mmio->data, mmio->bytes, mr); } else { err = mem_write(vcpu, paddr, mmio->data, mmio->bytes, mr); } return (err); } int emulate_mem(struct vcpu *vcpu, struct vm_mmio *mmio) { return (access_memory(vcpu, mmio->gpa, emulate_mem_cb, mmio)); } struct rw_mem_args { uint64_t *val; int size; int operation; }; static int rw_mem_cb(struct vcpu *vcpu, uint64_t paddr, struct mem_range *mr, void *arg) { struct rw_mem_args *rma; rma = arg; return (mr->handler(vcpu, rma->operation, paddr, rma->size, rma->val, mr->arg1, mr->arg2)); } int read_mem(struct vcpu *vcpu, uint64_t gpa, uint64_t *rval, int size) { struct rw_mem_args rma; rma.val = rval; rma.size = size; rma.operation = MEM_F_READ; return (access_memory(vcpu, gpa, rw_mem_cb, &rma)); } int write_mem(struct vcpu *vcpu, uint64_t gpa, uint64_t wval, int size) { struct rw_mem_args rma; rma.val = &wval; rma.size = size; rma.operation = MEM_F_WRITE; return (access_memory(vcpu, gpa, rw_mem_cb, &rma)); } static int register_mem_int(struct mmio_rb_tree *rbt, struct mem_range *memp) { struct mmio_rb_range *entry, *mrp; int err, perror; err = 0; mrp = malloc(sizeof(struct mmio_rb_range)); if (mrp == NULL) { warn("%s: couldn't allocate memory for mrp\n", __func__); err = ENOMEM; } else { mrp->mr_param = *memp; mrp->mr_base = memp->base; mrp->mr_end = memp->base + memp->size - 1; pthread_rwlock_wrlock(&mmio_rwlock); if (mmio_rb_lookup(rbt, memp->base, &entry) != 0) err = mmio_rb_add(rbt, mrp); #ifndef __FreeBSD__ else /* smatch warn: possible memory leak of 'mrp' */ free(mrp); #endif perror = pthread_rwlock_unlock(&mmio_rwlock); #ifdef __FreeBSD__ assert(perror == 0); #else if (perror != 0) { mmio_rb_dump(rbt); exit(4); } #endif if (err) free(mrp); } return (err); } int register_mem(struct mem_range *memp) { return (register_mem_int(&mmio_rb_root, memp)); } int register_mem_fallback(struct mem_range *memp) { return (register_mem_int(&mmio_rb_fallback, memp)); } int unregister_mem(struct mem_range *memp) { struct mem_range *mr; struct mmio_rb_range *entry = NULL; int err, perror, i; pthread_rwlock_wrlock(&mmio_rwlock); err = mmio_rb_lookup(&mmio_rb_root, memp->base, &entry); if (err == 0) { mr = &entry->mr_param; assert(mr->name == memp->name); assert(mr->base == memp->base && mr->size == memp->size); assert((mr->flags & MEM_F_IMMUTABLE) == 0); RB_REMOVE(mmio_rb_tree, &mmio_rb_root, entry); /* flush Per-vCPU cache */ for (i = 0; i < mmio_ncpu; i++) { if (mmio_hint[i] == entry) mmio_hint[i] = NULL; } } perror = pthread_rwlock_unlock(&mmio_rwlock); assert(perror == 0); if (entry) free(entry); return (err); } void init_mem(int ncpu) { mmio_ncpu = ncpu; mmio_hint = calloc(ncpu, sizeof(*mmio_hint)); RB_INIT(&mmio_rb_root); RB_INIT(&mmio_rb_fallback); pthread_rwlock_init(&mmio_rwlock, NULL); } /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2012 NetApp, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef _MEM_H_ #define _MEM_H_ #include struct vcpu; typedef int (*mem_func_t)(struct vcpu *vcpu, int dir, uint64_t addr, int size, uint64_t *val, void *arg1, long arg2); struct mem_range { const char *name; int flags; mem_func_t handler; void *arg1; long arg2; uint64_t base; uint64_t size; }; #define MEM_F_READ 0x1 #define MEM_F_WRITE 0x2 #define MEM_F_RW 0x3 #define MEM_F_IMMUTABLE 0x4 /* mem_range cannot be unregistered */ void init_mem(int ncpu); int emulate_mem(struct vcpu *vcpu, struct vm_mmio *mmio); int read_mem(struct vcpu *vpu, uint64_t gpa, uint64_t *rval, int size); int register_mem(struct mem_range *memp); int register_mem_fallback(struct mem_range *memp); int unregister_mem(struct mem_range *memp); int write_mem(struct vcpu *vcpu, uint64_t gpa, uint64_t wval, int size); #endif /* _MEM_H_ */ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2011 NetApp, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * Copyright 2018 Joyent, Inc. * Copyright 2022 OmniOS Community Edition (OmniOSce) Association. */ /* * Micro event library for FreeBSD, designed for a single i/o thread * using kqueue, and having events be persistent by default. */ #include #ifndef WITHOUT_CAPSICUM #include #endif #include #include #include #include #include #include #include #include #include #ifndef WITHOUT_CAPSICUM #include #endif #ifdef __FreeBSD__ #include #else #include #include #include #include #include #include #endif #include #include #include #include "mevent.h" #define MEVENT_MAX 64 #ifndef __FreeBSD__ #define EV_ENABLE 0x01 #define EV_ADD EV_ENABLE #define EV_DISABLE 0x02 #define EV_DELETE 0x04 static int mevent_file_poll_interval_ms = 5000; #endif static pthread_t mevent_tid; static pthread_once_t mevent_once = PTHREAD_ONCE_INIT; #ifdef __FreeBSD__ static int mevent_timid = 43; #endif static int mevent_pipefd[2]; static int mfd; static pthread_mutex_t mevent_lmutex = PTHREAD_MUTEX_INITIALIZER; struct mevent { void (*me_func)(int, enum ev_type, void *); #define me_msecs me_fd int me_fd; #ifdef __FreeBSD__ int me_timid; #else timer_t me_timid; #endif enum ev_type me_type; void *me_param; int me_cq; int me_state; /* Desired kevent flags. */ int me_closefd; int me_fflags; #ifndef __FreeBSD__ port_notify_t me_notify; struct sigevent me_sigev; boolean_t me_auto_requeue; struct { int mp_fd; off_t mp_size; void (*mp_func)(int, enum ev_type, void *); void *mp_param; } me_poll; #endif LIST_ENTRY(mevent) me_list; }; static LIST_HEAD(listhead, mevent) global_head, change_head; static void mevent_qlock(void) { pthread_mutex_lock(&mevent_lmutex); } static void mevent_qunlock(void) { pthread_mutex_unlock(&mevent_lmutex); } static void mevent_pipe_read(int fd, enum ev_type type __unused, void *param __unused) { char buf[MEVENT_MAX]; int status; /* * Drain the pipe read side. The fd is non-blocking so this is * safe to do. */ do { status = read(fd, buf, sizeof(buf)); } while (status == MEVENT_MAX); } static void mevent_notify(void) { char c = '\0'; /* * If calling from outside the i/o thread, write a byte on the * pipe to force the i/o thread to exit the blocking kevent call. */ if (mevent_pipefd[1] != 0 && pthread_self() != mevent_tid) { write(mevent_pipefd[1], &c, 1); } } static void mevent_init(void) { #ifndef WITHOUT_CAPSICUM cap_rights_t rights; #endif #ifdef __FreeBSD__ mfd = kqueue(); #else mfd = port_create(); #endif assert(mfd > 0); #ifndef WITHOUT_CAPSICUM cap_rights_init(&rights, CAP_KQUEUE); if (caph_rights_limit(mfd, &rights) == -1) errx(EX_OSERR, "Unable to apply rights for sandbox"); #endif LIST_INIT(&change_head); LIST_INIT(&global_head); } #ifdef __FreeBSD__ static int mevent_kq_filter(struct mevent *mevp) { int retval; retval = 0; if (mevp->me_type == EVF_READ) retval = EVFILT_READ; if (mevp->me_type == EVF_WRITE) retval = EVFILT_WRITE; if (mevp->me_type == EVF_TIMER) retval = EVFILT_TIMER; if (mevp->me_type == EVF_SIGNAL) retval = EVFILT_SIGNAL; if (mevp->me_type == EVF_VNODE) retval = EVFILT_VNODE; return (retval); } static int mevent_kq_flags(struct mevent *mevp) { int retval; retval = mevp->me_state; if (mevp->me_type == EVF_VNODE) retval |= EV_CLEAR; return (retval); } static int mevent_kq_fflags(struct mevent *mevp) { int retval; retval = 0; switch (mevp->me_type) { case EVF_VNODE: if ((mevp->me_fflags & EVFF_ATTRIB) != 0) retval |= NOTE_ATTRIB; break; case EVF_READ: case EVF_WRITE: case EVF_TIMER: case EVF_SIGNAL: break; } return (retval); } static void mevent_populate(struct mevent *mevp, struct kevent *kev) { if (mevp->me_type == EVF_TIMER) { kev->ident = mevp->me_timid; kev->data = mevp->me_msecs; } else { kev->ident = mevp->me_fd; kev->data = 0; } kev->filter = mevent_kq_filter(mevp); kev->flags = mevent_kq_flags(mevp); kev->fflags = mevent_kq_fflags(mevp); kev->udata = mevp; } static int mevent_build(struct kevent *kev) { struct mevent *mevp, *tmpp; int i; i = 0; mevent_qlock(); LIST_FOREACH_SAFE(mevp, &change_head, me_list, tmpp) { if (mevp->me_closefd) { /* * A close of the file descriptor will remove the * event */ close(mevp->me_fd); } else { assert((mevp->me_state & EV_ADD) == 0); mevent_populate(mevp, &kev[i]); i++; } mevp->me_cq = 0; LIST_REMOVE(mevp, me_list); if (mevp->me_state & EV_DELETE) { free(mevp); } else { LIST_INSERT_HEAD(&global_head, mevp, me_list); } assert(i < MEVENT_MAX); } mevent_qunlock(); return (i); } static void mevent_handle(struct kevent *kev, int numev) { struct mevent *mevp; int i; for (i = 0; i < numev; i++) { mevp = kev[i].udata; /* XXX check for EV_ERROR ? */ (*mevp->me_func)(mevp->me_fd, mevp->me_type, mevp->me_param); } } #else /* __FreeBSD__ */ static boolean_t mevent_clarify_state(struct mevent *mevp) { const int state = mevp->me_state; if ((state & EV_DELETE) != 0) { /* All other intents are overriden by delete. */ mevp->me_state = EV_DELETE; return (B_TRUE); } /* * Without a distinction between EV_ADD and EV_ENABLE in our emulation, * handling the add-disabled case means eliding the portfs operation * when both flags are present. * * This is not a concern for subsequent enable/disable operations, as * mevent_update() toggles the flags properly so they are not left in * conflict. */ if (state == (EV_ENABLE|EV_DISABLE)) { mevp->me_state = EV_DISABLE; return (B_FALSE); } return (B_TRUE); } static void mevent_poll_file_attrib(int fd, enum ev_type type, void *param) { struct mevent *mevp = param; struct stat st; if (fstat(mevp->me_poll.mp_fd, &st) != 0) { (void) fprintf(stderr, "%s: fstat(%d) failed: %s\n", __func__, fd, strerror(errno)); return; } /* * The only current consumer of file attribute monitoring is * blockif, which wants to know about size changes. */ if (mevp->me_poll.mp_size != st.st_size) { mevp->me_poll.mp_size = st.st_size; (*mevp->me_poll.mp_func)(mevp->me_poll.mp_fd, EVF_VNODE, mevp->me_poll.mp_param); } } static void mevent_update_one_readwrite(struct mevent *mevp) { int portfd = mevp->me_notify.portnfy_port; mevp->me_auto_requeue = B_FALSE; switch (mevp->me_state) { case EV_ENABLE: { const int events = (mevp->me_type == EVF_READ) ? POLLIN : POLLOUT; if (port_associate(portfd, PORT_SOURCE_FD, mevp->me_fd, events, mevp) != 0) { (void) fprintf(stderr, "port_associate fd %d %p failed: %s\n", mevp->me_fd, mevp, strerror(errno)); } return; } case EV_DISABLE: case EV_DELETE: /* * A disable that comes in while an event is being * handled will result in an ENOENT. */ if (port_dissociate(portfd, PORT_SOURCE_FD, mevp->me_fd) != 0 && errno != ENOENT) { (void) fprintf(stderr, "port_dissociate " "portfd %d fd %d mevp %p failed: %s\n", portfd, mevp->me_fd, mevp, strerror(errno)); } return; default: (void) fprintf(stderr, "%s: unhandled state %d\n", __func__, mevp->me_state); abort(); } } static void mevent_update_one_timer(struct mevent *mevp) { mevp->me_auto_requeue = B_TRUE; switch (mevp->me_state) { case EV_ENABLE: { struct itimerspec it = { 0 }; mevp->me_sigev.sigev_notify = SIGEV_PORT; mevp->me_sigev.sigev_value.sival_ptr = &mevp->me_notify; if (timer_create(CLOCK_REALTIME, &mevp->me_sigev, &mevp->me_timid) != 0) { (void) fprintf(stderr, "timer_create failed: %s", strerror(errno)); return; } /* The first timeout */ it.it_value.tv_sec = mevp->me_msecs / MILLISEC; it.it_value.tv_nsec = MSEC2NSEC(mevp->me_msecs % MILLISEC); /* Repeat at the same interval */ it.it_interval = it.it_value; if (timer_settime(mevp->me_timid, 0, &it, NULL) != 0) { (void) fprintf(stderr, "timer_settime failed: %s", strerror(errno)); } return; } case EV_DISABLE: case EV_DELETE: if (timer_delete(mevp->me_timid) != 0) { (void) fprintf(stderr, "timer_delete failed: %s", strerror(errno)); } mevp->me_timid = -1; return; default: (void) fprintf(stderr, "%s: unhandled state %d\n", __func__, mevp->me_state); abort(); } } static void mevent_update_one_vnode(struct mevent *mevp) { switch (mevp->me_state) { case EV_ENABLE: { struct stat st; int events = 0; if ((mevp->me_fflags & EVFF_ATTRIB) != 0) events |= FILE_ATTRIB; assert(events != 0); /* * It is tempting to use the PORT_SOURCE_FILE type for this in * conjunction with the FILE_ATTRIB event type. Unfortunately * this event type triggers on any change to the file's * ctime, and therefore for every write as well as attribute * changes. It also does not work for ZVOLs. * * Convert this to a timer event and poll for the file * attribute changes that we care about. */ if (fstat(mevp->me_fd, &st) != 0) { (void) fprintf(stderr, "fstat(%d) failed: %s\n", mevp->me_fd, strerror(errno)); return; } mevp->me_poll.mp_fd = mevp->me_fd; mevp->me_poll.mp_size = st.st_size; mevp->me_poll.mp_func = mevp->me_func; mevp->me_poll.mp_param = mevp->me_param; mevp->me_func = mevent_poll_file_attrib; mevp->me_param = mevp; mevp->me_type = EVF_TIMER; mevp->me_timid = -1; mevp->me_msecs = mevent_file_poll_interval_ms; mevent_update_one_timer(mevp); return; } case EV_DISABLE: case EV_DELETE: /* * These events do not really exist as they are converted to * timers; fall through to abort. */ default: (void) fprintf(stderr, "%s: unhandled state %d\n", __func__, mevp->me_state); abort(); } } static void mevent_update_one(struct mevent *mevp) { switch (mevp->me_type) { case EVF_READ: case EVF_WRITE: mevent_update_one_readwrite(mevp); break; case EVF_TIMER: mevent_update_one_timer(mevp); break; case EVF_VNODE: mevent_update_one_vnode(mevp); break; case EVF_SIGNAL: /* EVF_SIGNAL not yet implemented. */ default: (void) fprintf(stderr, "%s: unhandled event type %d\n", __func__, mevp->me_type); abort(); } } static void mevent_populate(struct mevent *mevp) { mevp->me_notify.portnfy_port = mfd; mevp->me_notify.portnfy_user = mevp; } static void mevent_update_pending() { struct mevent *mevp, *tmpp; mevent_qlock(); LIST_FOREACH_SAFE(mevp, &change_head, me_list, tmpp) { mevent_populate(mevp); if (mevp->me_closefd) { /* * A close of the file descriptor will remove the * event */ (void) close(mevp->me_fd); mevp->me_fd = -1; } else { if (mevent_clarify_state(mevp)) { mevent_update_one(mevp); } } mevp->me_cq = 0; LIST_REMOVE(mevp, me_list); if (mevp->me_state & EV_DELETE) { free(mevp); } else { LIST_INSERT_HEAD(&global_head, mevp, me_list); } } mevent_qunlock(); } static void mevent_handle_pe(port_event_t *pe) { struct mevent *mevp = pe->portev_user; (*mevp->me_func)(mevp->me_fd, mevp->me_type, mevp->me_param); mevent_qlock(); if (!mevp->me_cq && !mevp->me_auto_requeue) { mevent_update_one(mevp); } mevent_qunlock(); } #endif static struct mevent * mevent_add_state(int tfd, enum ev_type type, void (*func)(int, enum ev_type, void *), void *param, int state, int fflags) { #ifdef __FreeBSD__ struct kevent kev; #endif struct mevent *lp, *mevp; #ifdef __FreeBSD__ int ret; #endif if (tfd < 0 || func == NULL) { return (NULL); } mevp = NULL; pthread_once(&mevent_once, mevent_init); mevent_qlock(); /* * Verify that the fd/type tuple is not present in any list */ LIST_FOREACH(lp, &global_head, me_list) { if (type != EVF_TIMER && lp->me_fd == tfd && lp->me_type == type) { goto exit; } } LIST_FOREACH(lp, &change_head, me_list) { if (type != EVF_TIMER && lp->me_fd == tfd && lp->me_type == type) { goto exit; } } /* * Allocate an entry and populate it. */ mevp = calloc(1, sizeof(struct mevent)); if (mevp == NULL) { goto exit; } if (type == EVF_TIMER) { mevp->me_msecs = tfd; #ifdef __FreeBSD__ mevp->me_timid = mevent_timid++; #else mevp->me_timid = -1; #endif } else mevp->me_fd = tfd; mevp->me_type = type; mevp->me_func = func; mevp->me_param = param; mevp->me_state = state; mevp->me_fflags = fflags; /* * Try to add the event. If this fails, report the failure to * the caller. */ #ifdef __FreeBSD__ mevent_populate(mevp, &kev); ret = kevent(mfd, &kev, 1, NULL, 0, NULL); if (ret == -1) { free(mevp); mevp = NULL; goto exit; } mevp->me_state &= ~EV_ADD; #else mevent_populate(mevp); if (mevent_clarify_state(mevp)) mevent_update_one(mevp); #endif LIST_INSERT_HEAD(&global_head, mevp, me_list); exit: mevent_qunlock(); return (mevp); } struct mevent * mevent_add(int tfd, enum ev_type type, void (*func)(int, enum ev_type, void *), void *param) { return (mevent_add_state(tfd, type, func, param, EV_ADD, 0)); } struct mevent * mevent_add_flags(int tfd, enum ev_type type, int fflags, void (*func)(int, enum ev_type, void *), void *param) { return (mevent_add_state(tfd, type, func, param, EV_ADD, fflags)); } struct mevent * mevent_add_disabled(int tfd, enum ev_type type, void (*func)(int, enum ev_type, void *), void *param) { return (mevent_add_state(tfd, type, func, param, EV_ADD | EV_DISABLE, 0)); } static int mevent_update(struct mevent *evp, bool enable) { int newstate; mevent_qlock(); /* * It's not possible to enable/disable a deleted event */ assert((evp->me_state & EV_DELETE) == 0); newstate = evp->me_state; if (enable) { newstate |= EV_ENABLE; newstate &= ~EV_DISABLE; } else { newstate |= EV_DISABLE; newstate &= ~EV_ENABLE; } /* * No update needed if state isn't changing */ if (evp->me_state != newstate) { evp->me_state = newstate; /* * Place the entry onto the changed list if not * already there. */ if (evp->me_cq == 0) { evp->me_cq = 1; LIST_REMOVE(evp, me_list); LIST_INSERT_HEAD(&change_head, evp, me_list); mevent_notify(); } } mevent_qunlock(); return (0); } int mevent_enable(struct mevent *evp) { return (mevent_update(evp, true)); } int mevent_disable(struct mevent *evp) { return (mevent_update(evp, false)); } static int mevent_delete_event(struct mevent *evp, int closefd) { mevent_qlock(); /* * Place the entry onto the changed list if not already there, and * mark as to be deleted. */ if (evp->me_cq == 0) { evp->me_cq = 1; LIST_REMOVE(evp, me_list); LIST_INSERT_HEAD(&change_head, evp, me_list); mevent_notify(); } evp->me_state = EV_DELETE; if (closefd) evp->me_closefd = 1; mevent_qunlock(); return (0); } int mevent_delete(struct mevent *evp) { return (mevent_delete_event(evp, 0)); } int mevent_delete_close(struct mevent *evp) { return (mevent_delete_event(evp, 1)); } static void mevent_set_name(void) { pthread_set_name_np(mevent_tid, "mevent"); } void mevent_dispatch(void) { #ifdef __FreeBSD__ struct kevent changelist[MEVENT_MAX]; struct kevent eventlist[MEVENT_MAX]; struct mevent *pipev; int numev; #else struct mevent *pipev; #endif int ret; #ifndef WITHOUT_CAPSICUM cap_rights_t rights; #endif mevent_tid = pthread_self(); mevent_set_name(); pthread_once(&mevent_once, mevent_init); /* * Open the pipe that will be used for other threads to force * the blocking kqueue call to exit by writing to it. Set the * descriptor to non-blocking. */ ret = pipe(mevent_pipefd); if (ret < 0) { perror("pipe"); exit(0); } #ifndef WITHOUT_CAPSICUM cap_rights_init(&rights, CAP_EVENT, CAP_READ, CAP_WRITE); if (caph_rights_limit(mevent_pipefd[0], &rights) == -1) errx(EX_OSERR, "Unable to apply rights for sandbox"); if (caph_rights_limit(mevent_pipefd[1], &rights) == -1) errx(EX_OSERR, "Unable to apply rights for sandbox"); #endif /* * Add internal event handler for the pipe write fd */ pipev = mevent_add(mevent_pipefd[0], EVF_READ, mevent_pipe_read, NULL); assert(pipev != NULL); for (;;) { #ifdef __FreeBSD__ /* * Build changelist if required. * XXX the changelist can be put into the blocking call * to eliminate the extra syscall. Currently better for * debug. */ numev = mevent_build(changelist); if (numev) { ret = kevent(mfd, changelist, numev, NULL, 0, NULL); if (ret == -1) { perror("Error return from kevent change"); } } /* * Block awaiting events */ ret = kevent(mfd, NULL, 0, eventlist, MEVENT_MAX, NULL); if (ret == -1 && errno != EINTR) { perror("Error return from kevent monitor"); } /* * Handle reported events */ mevent_handle(eventlist, ret); #else /* __FreeBSD__ */ port_event_t pev; /* Handle any pending updates */ mevent_update_pending(); /* Block awaiting events */ ret = port_get(mfd, &pev, NULL); if (ret != 0) { if (errno != EINTR) perror("Error return from port_get"); continue; } /* Handle reported event */ mevent_handle_pe(&pev); #endif /* __FreeBSD__ */ } } /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2011 NetApp, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef _MEVENT_H_ #define _MEVENT_H_ enum ev_type { EVF_READ, EVF_WRITE, EVF_TIMER, EVF_SIGNAL, EVF_VNODE, }; /* Filter flags for EVF_VNODE */ #define EVFF_ATTRIB 0x0001 struct mevent; struct mevent *mevent_add(int fd, enum ev_type type, void (*func)(int, enum ev_type, void *), void *param); struct mevent *mevent_add_flags(int fd, enum ev_type type, int fflags, void (*func)(int, enum ev_type, void *), void *param); struct mevent *mevent_add_disabled(int fd, enum ev_type type, void (*func)(int, enum ev_type, void *), void *param); int mevent_enable(struct mevent *evp); int mevent_disable(struct mevent *evp); int mevent_delete(struct mevent *evp); int mevent_delete_close(struct mevent *evp); void mevent_dispatch(void); #endif /* _MEVENT_H_ */ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright 2024 OmniOS Community Edition (OmniOSce) Association. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, * OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ /* * The illumos dlpi backend */ #include #include #include #include #include #include #include #include "config.h" #include "debug.h" #include "iov.h" #include "mevent.h" #include "net_backends.h" #include "net_backends_priv.h" /* * The size of the bounce buffer used to implement the peek callback. * This value should be big enough to accommodate the largest of all possible * frontend packet lengths. The value here matches the definition of * VTNET_MAX_PKT_LEN in pci_virtio_net.c */ #define DLPI_BBUF_SIZE (65536 + 64) typedef struct be_dlpi_priv { dlpi_handle_t bdp_dhp; struct mevent *bdp_mevp; /* * A bounce buffer that allows us to implement the peek_recvlen * callback. Each structure is only used by a single thread so * one is enough. */ uint8_t bdp_bbuf[DLPI_BBUF_SIZE]; ssize_t bdp_bbuflen; } be_dlpi_priv_t; static void be_dlpi_cleanup(net_backend_t *be) { be_dlpi_priv_t *priv = NET_BE_PRIV(be); if (priv->bdp_dhp != NULL) dlpi_close(priv->bdp_dhp); priv->bdp_dhp = NULL; if (priv->bdp_mevp != NULL) mevent_delete(priv->bdp_mevp); priv->bdp_mevp = NULL; priv->bdp_bbuflen = 0; be->fd = -1; } static void be_dlpi_err(int ret, const char *dev, char *msg) { EPRINTLN("%s: %s (%s)", dev, msg, dlpi_strerror(ret)); } static int be_dlpi_init(net_backend_t *be, const char *devname __unused, nvlist_t *nvl, net_be_rxeof_t cb, void *param) { be_dlpi_priv_t *priv = NET_BE_PRIV(be); const char *vnic; int ret; if (cb == NULL) { EPRINTLN("dlpi backend requires non-NULL callback"); return (-1); } vnic = get_config_value_node(nvl, "vnic"); if (vnic == NULL) { EPRINTLN("dlpi backend requires a VNIC"); return (-1); } priv->bdp_bbuflen = 0; ret = dlpi_open(vnic, &priv->bdp_dhp, DLPI_RAW); if (ret != DLPI_SUCCESS) { be_dlpi_err(ret, vnic, "open failed"); goto error; } if ((ret = dlpi_bind(priv->bdp_dhp, DLPI_ANY_SAP, NULL)) != DLPI_SUCCESS) { be_dlpi_err(ret, vnic, "bind failed"); goto error; } if (get_config_bool_node_default(nvl, "promiscrxonly", true)) { if ((ret = dlpi_promiscon(priv->bdp_dhp, DL_PROMISC_RX_ONLY)) != DLPI_SUCCESS) { be_dlpi_err(ret, vnic, "enable promiscuous mode(rxonly) failed"); goto error; } } if (get_config_bool_node_default(nvl, "promiscphys", false)) { if ((ret = dlpi_promiscon(priv->bdp_dhp, DL_PROMISC_PHYS)) != DLPI_SUCCESS) { be_dlpi_err(ret, vnic, "enable promiscuous mode(physical) failed"); goto error; } } if (get_config_bool_node_default(nvl, "promiscsap", true)) { if ((ret = dlpi_promiscon(priv->bdp_dhp, DL_PROMISC_SAP)) != DLPI_SUCCESS) { be_dlpi_err(ret, vnic, "enable promiscuous mode(SAP) failed"); goto error; } } if (get_config_bool_node_default(nvl, "promiscmulti", true)) { if ((ret = dlpi_promiscon(priv->bdp_dhp, DL_PROMISC_MULTI)) != DLPI_SUCCESS) { be_dlpi_err(ret, vnic, "enable promiscuous mode(muticast) failed"); goto error; } } be->fd = dlpi_fd(priv->bdp_dhp); if (fcntl(be->fd, F_SETFL, O_NONBLOCK) < 0) { EPRINTLN("%s: enable O_NONBLOCK failed", vnic); goto error; } priv->bdp_mevp = mevent_add_disabled(be->fd, EVF_READ, cb, param); if (priv->bdp_mevp == NULL) { EPRINTLN("Could not register event"); goto error; } return (0); error: be_dlpi_cleanup(be); return (-1); } /* * Called to send a buffer chain out to the dlpi device */ static ssize_t be_dlpi_send(net_backend_t *be, const struct iovec *iov, int iovcnt) { be_dlpi_priv_t *priv = NET_BE_PRIV(be); ssize_t len = 0; int ret; if (iovcnt == 1) { len = iov[0].iov_len; ret = dlpi_send(priv->bdp_dhp, NULL, 0, iov[0].iov_base, len, NULL); } else { void *buf = NULL; len = iov_to_buf(iov, iovcnt, &buf); if (len <= 0 || buf == NULL) return (-1); ret = dlpi_send(priv->bdp_dhp, NULL, 0, buf, len, NULL); free(buf); } if (ret != DLPI_SUCCESS) return (-1); return (len); } static ssize_t be_dlpi_peek_recvlen(net_backend_t *be) { be_dlpi_priv_t *priv = NET_BE_PRIV(be); dlpi_recvinfo_t recv; size_t len; int ret; /* * We already have a packet in the bounce buffer. * Just return its length. */ if (priv->bdp_bbuflen > 0) return (priv->bdp_bbuflen); /* * Read the next packet (if any) into the bounce buffer, so * that we get to know its length and we can return that * to the caller. */ len = sizeof (priv->bdp_bbuf); ret = dlpi_recv(priv->bdp_dhp, NULL, NULL, priv->bdp_bbuf, &len, 0, &recv); if (ret == DL_SYSERR) { if (errno == EWOULDBLOCK) return (0); return (-1); } else if (ret == DLPI_ETIMEDOUT) { return (0); } else if (ret != DLPI_SUCCESS) { return (-1); } if (recv.dri_totmsglen > sizeof (priv->bdp_bbuf)) { EPRINTLN("DLPI bounce buffer was too small! - needed %x bytes", recv.dri_totmsglen); } priv->bdp_bbuflen = len; return (len); } static ssize_t be_dlpi_recv(net_backend_t *be, const struct iovec *iov, int iovcnt) { be_dlpi_priv_t *priv = NET_BE_PRIV(be); size_t len; int ret; if (priv->bdp_bbuflen > 0) { /* * A packet is available in the bounce buffer, so * we read it from there. */ len = buf_to_iov(priv->bdp_bbuf, priv->bdp_bbuflen, iov, iovcnt); /* Mark the bounce buffer as empty. */ priv->bdp_bbuflen = 0; if (len == 0) return (-1); return (len); } len = iov[0].iov_len; ret = dlpi_recv(priv->bdp_dhp, NULL, NULL, (uint8_t *)iov[0].iov_base, &len, 0, NULL); if (ret == DL_SYSERR) { if (errno == EWOULDBLOCK) return (0); return (-1); } else if (ret == DLPI_ETIMEDOUT) { return (0); } else if (ret != DLPI_SUCCESS) { return (-1); } return (len); } static void be_dlpi_recv_enable(net_backend_t *be) { be_dlpi_priv_t *priv = NET_BE_PRIV(be); mevent_enable(priv->bdp_mevp); } static void be_dlpi_recv_disable(net_backend_t *be) { be_dlpi_priv_t *priv = NET_BE_PRIV(be); mevent_disable(priv->bdp_mevp); } static uint64_t be_dlpi_get_cap(net_backend_t *be) { return (0); /* no capabilities for now */ } static int be_dlpi_set_cap(net_backend_t *be, uint64_t features, unsigned vnet_hdr_len) { return ((features || vnet_hdr_len) ? -1 : 0); } static int be_dlpi_get_mac(net_backend_t *be, void *buf, size_t *buflen) { be_dlpi_priv_t *priv = NET_BE_PRIV(be); uchar_t physaddr[DLPI_PHYSADDR_MAX]; size_t physaddrlen = DLPI_PHYSADDR_MAX; int ret; if ((ret = dlpi_get_physaddr(priv->bdp_dhp, DL_CURR_PHYS_ADDR, physaddr, &physaddrlen)) != DLPI_SUCCESS) { be_dlpi_err(ret, dlpi_linkname(priv->bdp_dhp), "read MAC address failed"); return (EINVAL); } if (physaddrlen != ETHERADDRL) { EPRINTLN("%s: bad MAC address len %d", dlpi_linkname(priv->bdp_dhp), physaddrlen); return (EINVAL); } if (physaddrlen > *buflen) { EPRINTLN("%s: MAC address too long (%d bytes required)", dlpi_linkname(priv->bdp_dhp), physaddrlen); return (ENOMEM); } *buflen = physaddrlen; memcpy(buf, physaddr, *buflen); return (0); } static struct net_backend dlpi_backend = { .prefix = "dlpi", .priv_size = sizeof(struct be_dlpi_priv), .init = be_dlpi_init, .cleanup = be_dlpi_cleanup, .send = be_dlpi_send, .peek_recvlen = be_dlpi_peek_recvlen, .recv = be_dlpi_recv, .recv_enable = be_dlpi_recv_enable, .recv_disable = be_dlpi_recv_disable, .get_cap = be_dlpi_get_cap, .set_cap = be_dlpi_set_cap, .get_mac = be_dlpi_get_mac, }; DATA_SET(net_backend_set, dlpi_backend); /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2019 Vincenzo Maffione * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, * OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ /* * This file implements multiple network backends (tap, netmap, ...), * to be used by network frontends such as virtio-net and e1000. * The API to access the backend (e.g. send/receive packets, negotiate * features) is exported by net_backends.h. */ #include #ifndef WITHOUT_CAPSICUM #include #endif #include #include #include #include #ifdef __FreeBSD__ #include #endif #include #ifndef WITHOUT_CAPSICUM #include #endif #include #include #include #include #include #include #include #include #include #include #include #include #include #include "config.h" #include "debug.h" #include "iov.h" #include "mevent.h" #include "net_backends.h" #include "net_backends_priv.h" #include "pci_emul.h" #define NET_BE_SIZE(be) (sizeof(*be) + (be)->priv_size) #ifdef __FreeBSD__ void tap_cleanup(struct net_backend *be) { struct tap_priv *priv = NET_BE_PRIV(be); if (priv->mevp) { mevent_delete(priv->mevp); } if (be->fd != -1) { close(be->fd); be->fd = -1; } } static int tap_init(struct net_backend *be, const char *devname, nvlist_t *nvl __unused, net_be_rxeof_t cb, void *param) { struct tap_priv *priv = NET_BE_PRIV(be); char tbuf[80]; int opt = 1, up = IFF_UP; #ifndef WITHOUT_CAPSICUM cap_rights_t rights; #endif if (cb == NULL) { EPRINTLN("TAP backend requires non-NULL callback"); return (-1); } strcpy(tbuf, "/dev/"); strlcat(tbuf, devname, sizeof(tbuf)); be->fd = open(tbuf, O_RDWR); if (be->fd == -1) { EPRINTLN("open of tap device %s failed", tbuf); goto error; } /* * Set non-blocking and register for read * notifications with the event loop */ if (ioctl(be->fd, FIONBIO, &opt) < 0) { EPRINTLN("tap device O_NONBLOCK failed"); goto error; } if (ioctl(be->fd, VMIO_SIOCSIFFLAGS, up)) { EPRINTLN("tap device link up failed"); goto error; } #ifndef WITHOUT_CAPSICUM cap_rights_init(&rights, CAP_EVENT, CAP_READ, CAP_WRITE); if (caph_rights_limit(be->fd, &rights) == -1) errx(EX_OSERR, "Unable to apply rights for sandbox"); #endif memset(priv->bbuf, 0, sizeof(priv->bbuf)); priv->bbuflen = 0; priv->mevp = mevent_add_disabled(be->fd, EVF_READ, cb, param); if (priv->mevp == NULL) { EPRINTLN("Could not register event"); goto error; } return (0); error: tap_cleanup(be); return (-1); } /* * Called to send a buffer chain out to the tap device */ ssize_t tap_send(struct net_backend *be, const struct iovec *iov, int iovcnt) { return (writev(be->fd, iov, iovcnt)); } ssize_t tap_peek_recvlen(struct net_backend *be) { struct tap_priv *priv = NET_BE_PRIV(be); ssize_t ret; if (priv->bbuflen > 0) { /* * We already have a packet in the bounce buffer. * Just return its length. */ return priv->bbuflen; } /* * Read the next packet (if any) into the bounce buffer, so * that we get to know its length and we can return that * to the caller. */ ret = read(be->fd, priv->bbuf, sizeof(priv->bbuf)); if (ret < 0 && errno == EWOULDBLOCK) { return (0); } if (ret > 0) priv->bbuflen = ret; return (ret); } ssize_t tap_recv(struct net_backend *be, const struct iovec *iov, int iovcnt) { struct tap_priv *priv = NET_BE_PRIV(be); ssize_t ret; if (priv->bbuflen > 0) { /* * A packet is available in the bounce buffer, so * we read it from there. */ ret = buf_to_iov(priv->bbuf, priv->bbuflen, iov, iovcnt); /* Mark the bounce buffer as empty. */ priv->bbuflen = 0; return (ret); } ret = readv(be->fd, iov, iovcnt); if (ret < 0 && errno == EWOULDBLOCK) { return (0); } return (ret); } void tap_recv_enable(struct net_backend *be) { struct tap_priv *priv = NET_BE_PRIV(be); mevent_enable(priv->mevp); } void tap_recv_disable(struct net_backend *be) { struct tap_priv *priv = NET_BE_PRIV(be); mevent_disable(priv->mevp); } uint64_t tap_get_cap(struct net_backend *be __unused) { return (0); /* no capabilities for now */ } int tap_set_cap(struct net_backend *be __unused, uint64_t features, unsigned vnet_hdr_len) { return ((features || vnet_hdr_len) ? -1 : 0); } static struct net_backend tap_backend = { .prefix = "tap", .priv_size = sizeof(struct tap_priv), .init = tap_init, .cleanup = tap_cleanup, .send = tap_send, .peek_recvlen = tap_peek_recvlen, .recv = tap_recv, .recv_enable = tap_recv_enable, .recv_disable = tap_recv_disable, .get_cap = tap_get_cap, .set_cap = tap_set_cap, }; /* A clone of the tap backend, with a different prefix. */ static struct net_backend vmnet_backend = { .prefix = "vmnet", .priv_size = sizeof(struct tap_priv), .init = tap_init, .cleanup = tap_cleanup, .send = tap_send, .peek_recvlen = tap_peek_recvlen, .recv = tap_recv, .recv_enable = tap_recv_enable, .recv_disable = tap_recv_disable, .get_cap = tap_get_cap, .set_cap = tap_set_cap, }; DATA_SET(net_backend_set, tap_backend); DATA_SET(net_backend_set, vmnet_backend); #endif /* __FreeBSD__ */ #ifdef __FreeBSD__ int netbe_legacy_config(nvlist_t *nvl, const char *opts) { char *backend, *cp; if (opts == NULL) return (0); cp = strchr(opts, ','); if (cp == NULL) { set_config_value_node(nvl, "backend", opts); return (0); } backend = strndup(opts, cp - opts); set_config_value_node(nvl, "backend", backend); free(backend); return (pci_parse_legacy_config(nvl, cp + 1)); } #else int netbe_legacy_config(nvlist_t *nvl, const char *opts) { char *config, *name, *tofree, *value; if (opts == NULL) return (0); /* Default to the 'dlpi' backend - can still be overridden by opts */ set_config_value_node(nvl, "backend", "dlpi"); set_config_value_node(nvl, "type", "dlpi"); config = tofree = strdup(opts); if (config == NULL) err(4, "netbe_legacy_config strdup()"); while ((name = strsep(&config, ",")) != NULL) { value = strchr(name, '='); if (value != NULL) { *value++ = '\0'; set_config_value_node(nvl, name, value); } else { set_config_value_node(nvl, "vnic", name); } } free(tofree); return (0); } #endif /* * Initialize a backend and attach to the frontend. * This is called during frontend initialization. * @ret is a pointer to the backend to be initialized * @devname is the backend-name as supplied on the command line, * e.g. -s 2:0,frontend-name,backend-name[,other-args] * @cb is the receive callback supplied by the frontend, * and it is invoked in the event loop when a receive * event is generated in the hypervisor, * @param is a pointer to the frontend, and normally used as * the argument for the callback. */ int netbe_init(struct net_backend **ret, nvlist_t *nvl, net_be_rxeof_t cb, void *param) { struct net_backend **pbe, *nbe, *tbe = NULL; const char *value, *type; char *devname; int err; value = get_config_value_node(nvl, "backend"); if (value == NULL) { return (-1); } devname = strdup(value); /* * Use the type given by configuration if exists; otherwise * use the prefix of the backend as the type. */ type = get_config_value_node(nvl, "type"); if (type == NULL) type = devname; /* * Find the network backend that matches the user-provided * device name. net_backend_set is built using a linker set. */ SET_FOREACH(pbe, net_backend_set) { if (strncmp(type, (*pbe)->prefix, strlen((*pbe)->prefix)) == 0) { tbe = *pbe; assert(tbe->init != NULL); assert(tbe->cleanup != NULL); assert(tbe->send != NULL); assert(tbe->recv != NULL); assert(tbe->get_cap != NULL); assert(tbe->set_cap != NULL); break; } } *ret = NULL; if (tbe == NULL) { free(devname); return (EINVAL); } nbe = calloc(1, NET_BE_SIZE(tbe)); *nbe = *tbe; /* copy the template */ nbe->fd = -1; nbe->sc = param; nbe->be_vnet_hdr_len = 0; nbe->fe_vnet_hdr_len = 0; /* Initialize the backend. */ err = nbe->init(nbe, devname, nvl, cb, param); if (err) { free(devname); free(nbe); return (err); } *ret = nbe; free(devname); return (0); } void netbe_cleanup(struct net_backend *be) { if (be != NULL) { be->cleanup(be); free(be); } } uint64_t netbe_get_cap(struct net_backend *be) { assert(be != NULL); return (be->get_cap(be)); } int netbe_set_cap(struct net_backend *be, uint64_t features, unsigned vnet_hdr_len) { int ret; assert(be != NULL); /* There are only three valid lengths, i.e., 0, 10 and 12. */ if (vnet_hdr_len && vnet_hdr_len != VNET_HDR_LEN && vnet_hdr_len != (VNET_HDR_LEN - sizeof(uint16_t))) return (-1); be->fe_vnet_hdr_len = vnet_hdr_len; ret = be->set_cap(be, features, vnet_hdr_len); assert(be->be_vnet_hdr_len == 0 || be->be_vnet_hdr_len == be->fe_vnet_hdr_len); return (ret); } ssize_t netbe_send(struct net_backend *be, const struct iovec *iov, int iovcnt) { return (be->send(be, iov, iovcnt)); } ssize_t netbe_peek_recvlen(struct net_backend *be) { return (be->peek_recvlen(be)); } /* * Try to read a packet from the backend, without blocking. * If no packets are available, return 0. In case of success, return * the length of the packet just read. Return -1 in case of errors. */ ssize_t netbe_recv(struct net_backend *be, const struct iovec *iov, int iovcnt) { return (be->recv(be, iov, iovcnt)); } /* * Read a packet from the backend and discard it. * Returns the size of the discarded packet or zero if no packet was available. * A negative error code is returned in case of read error. */ ssize_t netbe_rx_discard(struct net_backend *be) { /* * MP note: the dummybuf is only used to discard frames, * so there is no need for it to be per-vtnet or locked. * We only make it large enough for TSO-sized segment. */ static uint8_t dummybuf[65536 + 64]; struct iovec iov; #ifdef __FreeBSD__ iov.iov_base = dummybuf; #else iov.iov_base = (caddr_t)dummybuf; #endif iov.iov_len = sizeof(dummybuf); return netbe_recv(be, &iov, 1); } void netbe_rx_disable(struct net_backend *be) { return be->recv_disable(be); } void netbe_rx_enable(struct net_backend *be) { return be->recv_enable(be); } size_t netbe_get_vnet_hdr_len(struct net_backend *be) { return (be->be_vnet_hdr_len); } #ifndef __FreeBSD__ int netbe_get_mac(net_backend_t *be, void *buf, size_t *buflen) { if (be->get_mac == NULL) return (ENOTSUP); return (be->get_mac(be, buf, buflen)); } #endif /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2019 Vincenzo Maffione * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, * OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ /* * This file and its contents are supplied under the terms of the * Common Development and Distribution License ("CDDL"), version 1.0. * You may only use this file in accordance with the terms of version * 1.0 of the CDDL. * * A full copy of the text of the CDDL should have accompanied this * source. A copy of the CDDL is also available via the Internet at * http://www.illumos.org/license/CDDL. */ /* This file is dual-licensed; see usr/src/contrib/bhyve/LICENSE */ /* * Copyright 2025 Oxide Computer Company */ #ifndef __NET_BACKENDS_H__ #define __NET_BACKENDS_H__ #include /* Opaque type representing a network backend. */ typedef struct net_backend net_backend_t; /* Interface between network frontends and the network backends. */ typedef void (*net_be_rxeof_t)(int, enum ev_type, void *param); int netbe_init(net_backend_t **be, nvlist_t *nvl, net_be_rxeof_t cb, void *param); int netbe_legacy_config(nvlist_t *nvl, const char *opts); void netbe_cleanup(net_backend_t *be); uint64_t netbe_get_cap(net_backend_t *be); int netbe_set_cap(net_backend_t *be, uint64_t cap, unsigned vnet_hdr_len); size_t netbe_get_vnet_hdr_len(net_backend_t *be); ssize_t netbe_send(net_backend_t *be, const struct iovec *iov, int iovcnt); ssize_t netbe_peek_recvlen(net_backend_t *be); ssize_t netbe_recv(net_backend_t *be, const struct iovec *iov, int iovcnt); ssize_t netbe_rx_discard(net_backend_t *be); void netbe_rx_disable(net_backend_t *be); void netbe_rx_enable(net_backend_t *be); #ifndef __FreeBSD__ int netbe_get_mac(net_backend_t *, void *, size_t *); #endif /* * Network device capabilities taken from the VirtIO standard. * Despite the name, these capabilities can be used by different frontends * (virtio-net, ptnet) and supported by different backends (netmap, tap, ...). */ #define VIRTIO_NET_F_CSUM (1ULL << 0) /* host handles partial cksum */ #define VIRTIO_NET_F_GUEST_CSUM (1ULL << 1) /* guest handles partial cksum */ #define VIRTIO_NET_F_MTU (1ULL << 3) /* initial MTU advice */ #define VIRTIO_NET_F_MAC (1ULL << 5) /* host supplies MAC */ #define VIRTIO_NET_F_GSO_DEPREC (1ULL << 6) /* deprecated: host handles GSO */ #define VIRTIO_NET_F_GUEST_TSO4 (1ULL << 7) /* guest can rcv TSOv4 */ #define VIRTIO_NET_F_GUEST_TSO6 (1ULL << 8) /* guest can rcv TSOv6 */ #define VIRTIO_NET_F_GUEST_ECN (1ULL << 9) /* guest can rcv TSO with ECN */ #define VIRTIO_NET_F_GUEST_UFO (1ULL << 10) /* guest can rcv UFO */ #define VIRTIO_NET_F_HOST_TSO4 (1ULL << 11) /* host can rcv TSOv4 */ #define VIRTIO_NET_F_HOST_TSO6 (1ULL << 12) /* host can rcv TSOv6 */ #define VIRTIO_NET_F_HOST_ECN (1ULL << 13) /* host can rcv TSO with ECN */ #define VIRTIO_NET_F_HOST_UFO (1ULL << 14) /* host can rcv UFO */ #define VIRTIO_NET_F_MRG_RXBUF (1ULL << 15) /* host can merge RX buffers */ #define VIRTIO_NET_F_STATUS (1ULL << 16) /* config status field available */ #define VIRTIO_NET_F_CTRL_VQ (1ULL << 17) /* control chan available */ #define VIRTIO_NET_F_CTRL_RX (1ULL << 18) /* control chan RX mode support */ #define VIRTIO_NET_F_CTRL_VLAN (1ULL << 19) /* control chan VLAN filtering */ #define VIRTIO_NET_F_GUEST_ANNOUNCE \ (1ULL << 21) /* guest can send gratuit. pkts */ #define VIRTIO_NET_F_MQ (1ULL << 22) /* host supp multiple VQ pairs */ #define VIRTIO_F_CTRL_MAC_ADDR (1ULL << 23) /* set MAC address through ctrlq */ #define VIRTIO_NET_F_SPEED_DUPLEX \ (1ULL << 63) /* speed/duplex fields avail */ /* * PCI config-space "registers"; device configuration fields. */ struct virtio_net_config { uint8_t vnc_macaddr[6]; uint16_t vnc_status; uint16_t vnc_max_qpair; uint16_t vnc_mtu; uint32_t vnc_speed; /* Specified in Mb/s */ uint8_t vnc_duplex; } __packed; #define VIRTIO_NET_S_LINK_UP 1 #define VIRTIO_NET_S_ANNOUNCE 2 #define VIRTIO_NET_SPEED_UNKNOWN UINT32_MAX #define VIRTIO_NET_DUPLEX_HALF 0 #define VIRTIO_NET_DUPLEX_FULL 1 #define VIRTIO_NET_DUPLEX_UNK 0xff /* * Fixed network header size */ struct virtio_net_rxhdr { uint8_t vrh_flags; uint8_t vrh_gso_type; uint16_t vrh_hdr_len; uint16_t vrh_gso_size; uint16_t vrh_csum_start; uint16_t vrh_csum_offset; uint16_t vrh_bufs; } __packed; #endif /* __NET_BACKENDS_H__ */ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2019 Vincenzo Maffione * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, * OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef __NET_BACKENDS_PRIV_H__ #define __NET_BACKENDS_PRIV_H__ #include /* * Each network backend registers a set of function pointers that are * used to implement the net backends API. * This might need to be exposed if we implement backends in separate files. */ struct net_backend { const char *prefix; /* prefix matching this backend */ /* * Routines used to initialize and cleanup the resources needed * by a backend. The cleanup function is used internally, * and should not be called by the frontend. */ int (*init)(struct net_backend *be, const char *devname, nvlist_t *nvl, net_be_rxeof_t cb, void *param); void (*cleanup)(struct net_backend *be); /* * Called to serve a guest transmit request. The scatter-gather * vector provided by the caller has 'iovcnt' elements and contains * the packet to send. */ ssize_t (*send)(struct net_backend *be, const struct iovec *iov, int iovcnt); /* * Get the length of the next packet that can be received from * the backend. If no packets are currently available, this * function returns 0. */ ssize_t (*peek_recvlen)(struct net_backend *be); /* * Called to receive a packet from the backend. When the function * returns a positive value 'len', the scatter-gather vector * provided by the caller contains a packet with such length. * The function returns 0 if the backend doesn't have a new packet to * receive. */ ssize_t (*recv)(struct net_backend *be, const struct iovec *iov, int iovcnt); /* * Ask the backend to enable or disable receive operation in the * backend. On return from a disable operation, it is guaranteed * that the receive callback won't be called until receive is * enabled again. Note however that it is up to the caller to make * sure that netbe_recv() is not currently being executed by another * thread. */ void (*recv_enable)(struct net_backend *be); void (*recv_disable)(struct net_backend *be); /* * Ask the backend for the virtio-net features it is able to * support. Possible features are TSO, UFO and checksum offloading * in both rx and tx direction and for both IPv4 and IPv6. */ uint64_t (*get_cap)(struct net_backend *be); /* * Tell the backend to enable/disable the specified virtio-net * features (capabilities). */ int (*set_cap)(struct net_backend *be, uint64_t features, unsigned int vnet_hdr_len); #ifndef __FreeBSD__ int (*get_mac)(struct net_backend *be, void *, size_t *); #endif struct pci_vtnet_softc *sc; int fd; /* * Length of the virtio-net header used by the backend and the * frontend, respectively. A zero value means that the header * is not used. */ unsigned int be_vnet_hdr_len; unsigned int fe_vnet_hdr_len; /* Size of backend-specific private data. */ size_t priv_size; /* Backend-specific private data follows. */ }; #define NET_BE_PRIV(be) ((void *)((be) + 1)) SET_DECLARE(net_backend_set, struct net_backend); #define VNET_HDR_LEN sizeof(struct virtio_net_rxhdr) #ifdef __FreeBSD__ /* * Export the tap backend routines for the benefit of other backends which have * a similar interface to the kernel, i.e., they send and receive data using * standard I/O system calls with a single file descriptor. */ struct tap_priv { struct mevent *mevp; /* * A bounce buffer that allows us to implement the peek_recvlen * callback. In the future we may get the same information from * the kevent data. */ char bbuf[1 << 16]; ssize_t bbuflen; }; void tap_cleanup(struct net_backend *be); ssize_t tap_send(struct net_backend *be, const struct iovec *iov, int io vcnt); ssize_t tap_recv(struct net_backend *be, const struct iovec *iov, int io vcnt); ssize_t tap_peek_recvlen(struct net_backend *be); void tap_recv_enable(struct net_backend *be); ssize_t tap_recv(struct net_backend *be, const struct iovec *iov, int io vcnt); void tap_recv_disable(struct net_backend *be); uint64_t tap_get_cap(struct net_backend *be); int tap_set_cap(struct net_backend *be, uint64_t features, unsigned vnet_hdr_len); #endif /* __FreeBSD__ */ #endif /* !__NET_BACKENDS_PRIV_H__ */ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2011 NetApp, Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, * OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include #include #include #include #include #include #include #include #include #include "bhyverun.h" #include "config.h" #include "debug.h" #include "net_utils.h" int net_parsemac(const char *mac_str, uint8_t *mac_addr) { struct ether_addr *ea; char zero_addr[ETHER_ADDR_LEN] = { 0, 0, 0, 0, 0, 0 }; if (mac_str == NULL) return (EINVAL); ea = ether_aton(mac_str); if (ea == NULL || ETHER_IS_MULTICAST(ea->octet) || memcmp(ea->octet, zero_addr, ETHER_ADDR_LEN) == 0) { EPRINTLN("Invalid MAC %s", mac_str); return (EINVAL); } else memcpy(mac_addr, ea->octet, ETHER_ADDR_LEN); return (0); } int net_parsemtu(const char *mtu_str, unsigned long *mtu) { char *end; unsigned long val; assert(mtu_str != NULL); if (*mtu_str == '-') goto err; val = strtoul(mtu_str, &end, 0); if (*end != '\0') goto err; if (val == ULONG_MAX) return (ERANGE); if (val == 0 && errno == EINVAL) return (EINVAL); *mtu = val; return (0); err: errno = EINVAL; return (EINVAL); } void net_genmac(struct pci_devinst *pi, uint8_t *macaddr) { /* * The default MAC address is the standard NetApp OUI of 00-a0-98, * followed by an MD5 of the PCI slot/func number and dev name */ MD5_CTX mdctx; unsigned char digest[16]; char nstr[80]; snprintf(nstr, sizeof(nstr), "%d-%d-%s", pi->pi_slot, pi->pi_func, get_config_value("name")); MD5Init(&mdctx); MD5Update(&mdctx, nstr, (unsigned int)strlen(nstr)); MD5Final(digest, &mdctx); macaddr[0] = 0x00; macaddr[1] = 0xa0; macaddr[2] = 0x98; macaddr[3] = digest[0]; macaddr[4] = digest[1]; macaddr[5] = digest[2]; } /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2019 Vincenzo Maffione * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, * OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef _NET_UTILS_H_ #define _NET_UTILS_H_ #include #include "pci_emul.h" void net_genmac(struct pci_devinst *pi, uint8_t *macaddr); int net_parsemac(const char *mac_str, uint8_t *mac_addr); int net_parsemtu(const char *mtu_str, unsigned long *mtu); #endif /* _NET_UTILS_H_ */ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2013 Zhixiang Yu * Copyright (c) 2015-2016 Alexander Motin * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include #include #include #include #include #include #include #include #ifndef __FreeBSD__ #include #endif #include #include #include #include #include #include #include #include #include #include #include #include #include #include "bhyverun.h" #include "config.h" #include "debug.h" #include "pci_emul.h" #include "ahci.h" #include "block_if.h" #define DEF_PORTS 6 /* Intel ICH8 AHCI supports 6 ports */ #define MAX_PORTS 32 /* AHCI supports 32 ports */ #define PxSIG_ATA 0x00000101 /* ATA drive */ #define PxSIG_ATAPI 0xeb140101 /* ATAPI drive */ enum sata_fis_type { FIS_TYPE_REGH2D = 0x27, /* Register FIS - host to device */ FIS_TYPE_REGD2H = 0x34, /* Register FIS - device to host */ FIS_TYPE_DMAACT = 0x39, /* DMA activate FIS - device to host */ FIS_TYPE_DMASETUP = 0x41, /* DMA setup FIS - bidirectional */ FIS_TYPE_DATA = 0x46, /* Data FIS - bidirectional */ FIS_TYPE_BIST = 0x58, /* BIST activate FIS - bidirectional */ FIS_TYPE_PIOSETUP = 0x5F, /* PIO setup FIS - device to host */ FIS_TYPE_SETDEVBITS = 0xA1, /* Set dev bits FIS - device to host */ }; /* * SCSI opcodes */ #define TEST_UNIT_READY 0x00 #define REQUEST_SENSE 0x03 #define INQUIRY 0x12 #define START_STOP_UNIT 0x1B #define PREVENT_ALLOW 0x1E #define READ_CAPACITY 0x25 #define READ_10 0x28 #define POSITION_TO_ELEMENT 0x2B #define READ_TOC 0x43 #define GET_EVENT_STATUS_NOTIFICATION 0x4A #define MODE_SENSE_10 0x5A #define REPORT_LUNS 0xA0 #define READ_12 0xA8 #define READ_CD 0xBE /* * SCSI mode page codes */ #define MODEPAGE_RW_ERROR_RECOVERY 0x01 #define MODEPAGE_CD_CAPABILITIES 0x2A /* * ATA commands */ #define ATA_SF_ENAB_SATA_SF 0x10 #define ATA_SATA_SF_AN 0x05 #define ATA_SF_DIS_SATA_SF 0x90 /* * Debug printf */ #ifdef AHCI_DEBUG static FILE *dbg; #define DPRINTF(format, arg...) do{fprintf(dbg, format, ##arg);fflush(dbg);}while(0) #else #define DPRINTF(format, arg...) #endif #define AHCI_PORT_IDENT 20 + 1 struct ahci_ioreq { struct blockif_req io_req; struct ahci_port *io_pr; STAILQ_ENTRY(ahci_ioreq) io_flist; TAILQ_ENTRY(ahci_ioreq) io_blist; uint8_t *cfis; uint8_t *dsm; uint32_t len; uint32_t done; int slot; int more; }; struct ahci_port { struct blockif_ctxt *bctx; struct pci_ahci_softc *pr_sc; struct ata_params ata_ident; uint8_t *cmd_lst; uint8_t *rfis; int port; int atapi; int reset; int waitforclear; int mult_sectors; uint8_t xfermode; uint8_t err_cfis[20]; uint8_t sense_key; uint8_t asc; u_int ccs; uint32_t pending; uint32_t clb; uint32_t clbu; uint32_t fb; uint32_t fbu; uint32_t is; uint32_t ie; uint32_t cmd; uint32_t unused0; uint32_t tfd; uint32_t sig; uint32_t ssts; uint32_t sctl; uint32_t serr; uint32_t sact; uint32_t ci; uint32_t sntf; uint32_t fbs; /* * i/o request info */ struct ahci_ioreq *ioreq; int ioqsz; STAILQ_HEAD(ahci_fhead, ahci_ioreq) iofhd; TAILQ_HEAD(ahci_bhead, ahci_ioreq) iobhd; }; struct ahci_cmd_hdr { uint16_t flags; uint16_t prdtl; uint32_t prdbc; uint64_t ctba; uint32_t reserved[4]; }; struct ahci_prdt_entry { uint64_t dba; uint32_t reserved; #define DBCMASK 0x3fffff uint32_t dbc; }; struct pci_ahci_softc { struct pci_devinst *asc_pi; pthread_mutex_t mtx; int ports; uint32_t cap; uint32_t ghc; uint32_t is; uint32_t pi; uint32_t vs; uint32_t ccc_ctl; uint32_t ccc_pts; uint32_t em_loc; uint32_t em_ctl; uint32_t cap2; uint32_t bohc; uint32_t lintr; struct ahci_port port[MAX_PORTS]; }; #define ahci_ctx(sc) ((sc)->asc_pi->pi_vmctx) static void ahci_handle_next_trim(struct ahci_port *p, int slot, uint8_t *cfis, uint8_t *buf, uint32_t len, uint32_t done); static void ahci_handle_port(struct ahci_port *p); static inline void lba_to_msf(uint8_t *buf, int lba) { lba += 150; buf[0] = (lba / 75) / 60; buf[1] = (lba / 75) % 60; buf[2] = lba % 75; } /* * Generate HBA interrupts on global IS register write. */ static void ahci_generate_intr(struct pci_ahci_softc *sc, uint32_t mask) { struct pci_devinst *pi = sc->asc_pi; struct ahci_port *p; int i, nmsg; uint32_t mmask; /* Update global IS from PxIS/PxIE. */ for (i = 0; i < sc->ports; i++) { p = &sc->port[i]; if (p->is & p->ie) sc->is |= (1 << i); } DPRINTF("%s(%08x) %08x", __func__, mask, sc->is); /* If there is nothing enabled -- clear legacy interrupt and exit. */ if (sc->is == 0 || (sc->ghc & AHCI_GHC_IE) == 0) { if (sc->lintr) { pci_lintr_deassert(pi); sc->lintr = 0; } return; } /* If there is anything and no MSI -- assert legacy interrupt. */ nmsg = pci_msi_maxmsgnum(pi); if (nmsg == 0) { if (!sc->lintr) { sc->lintr = 1; pci_lintr_assert(pi); } return; } /* Assert respective MSIs for ports that were touched. */ for (i = 0; i < nmsg; i++) { if (sc->ports <= nmsg || i < nmsg - 1) mmask = 1 << i; else mmask = 0xffffffff << i; if (sc->is & mask && mmask & mask) pci_generate_msi(pi, i); } } /* * Generate HBA interrupt on specific port event. */ static void ahci_port_intr(struct ahci_port *p) { struct pci_ahci_softc *sc = p->pr_sc; struct pci_devinst *pi = sc->asc_pi; int nmsg; DPRINTF("%s(%d) %08x/%08x %08x", __func__, p->port, p->is, p->ie, sc->is); /* If there is nothing enabled -- we are done. */ if ((p->is & p->ie) == 0) return; /* In case of non-shared MSI always generate interrupt. */ nmsg = pci_msi_maxmsgnum(pi); if (sc->ports <= nmsg || p->port < nmsg - 1) { sc->is |= (1 << p->port); if ((sc->ghc & AHCI_GHC_IE) == 0) return; pci_generate_msi(pi, p->port); return; } /* If IS for this port is already set -- do nothing. */ if (sc->is & (1 << p->port)) return; sc->is |= (1 << p->port); /* If interrupts are enabled -- generate one. */ if ((sc->ghc & AHCI_GHC_IE) == 0) return; if (nmsg > 0) { pci_generate_msi(pi, nmsg - 1); } else if (!sc->lintr) { sc->lintr = 1; pci_lintr_assert(pi); } } static void ahci_write_fis(struct ahci_port *p, enum sata_fis_type ft, uint8_t *fis) { int offset, len, irq; if (p->rfis == NULL || !(p->cmd & AHCI_P_CMD_FRE)) return; switch (ft) { case FIS_TYPE_REGD2H: offset = 0x40; len = 20; irq = (fis[1] & (1 << 6)) ? AHCI_P_IX_DHR : 0; break; case FIS_TYPE_SETDEVBITS: offset = 0x58; len = 8; irq = (fis[1] & (1 << 6)) ? AHCI_P_IX_SDB : 0; break; case FIS_TYPE_PIOSETUP: offset = 0x20; len = 20; irq = (fis[1] & (1 << 6)) ? AHCI_P_IX_PS : 0; break; default: EPRINTLN("unsupported fis type %d", ft); return; } if (fis[2] & ATA_S_ERROR) { p->waitforclear = 1; irq |= AHCI_P_IX_TFE; } memcpy(p->rfis + offset, fis, len); if (irq) { if (~p->is & irq) { p->is |= irq; ahci_port_intr(p); } } } static void ahci_write_fis_piosetup(struct ahci_port *p) { uint8_t fis[20]; memset(fis, 0, sizeof(fis)); fis[0] = FIS_TYPE_PIOSETUP; ahci_write_fis(p, FIS_TYPE_PIOSETUP, fis); } static void ahci_write_fis_sdb(struct ahci_port *p, int slot, uint8_t *cfis, uint32_t tfd) { uint8_t fis[8]; uint8_t error; error = (tfd >> 8) & 0xff; tfd &= 0x77; memset(fis, 0, sizeof(fis)); fis[0] = FIS_TYPE_SETDEVBITS; fis[1] = (1 << 6); fis[2] = tfd; fis[3] = error; if (fis[2] & ATA_S_ERROR) { p->err_cfis[0] = slot; p->err_cfis[2] = tfd; p->err_cfis[3] = error; memcpy(&p->err_cfis[4], cfis + 4, 16); } else { *(uint32_t *)(fis + 4) = (1 << slot); p->sact &= ~(1 << slot); } p->tfd &= ~0x77; p->tfd |= tfd; ahci_write_fis(p, FIS_TYPE_SETDEVBITS, fis); } static void ahci_write_fis_d2h(struct ahci_port *p, int slot, uint8_t *cfis, uint32_t tfd) { uint8_t fis[20]; uint8_t error; error = (tfd >> 8) & 0xff; memset(fis, 0, sizeof(fis)); fis[0] = FIS_TYPE_REGD2H; fis[1] = (1 << 6); fis[2] = tfd & 0xff; fis[3] = error; fis[4] = cfis[4]; fis[5] = cfis[5]; fis[6] = cfis[6]; fis[7] = cfis[7]; fis[8] = cfis[8]; fis[9] = cfis[9]; fis[10] = cfis[10]; fis[11] = cfis[11]; fis[12] = cfis[12]; fis[13] = cfis[13]; if (fis[2] & ATA_S_ERROR) { p->err_cfis[0] = 0x80; p->err_cfis[2] = tfd & 0xff; p->err_cfis[3] = error; memcpy(&p->err_cfis[4], cfis + 4, 16); } else p->ci &= ~(1 << slot); p->tfd = tfd; ahci_write_fis(p, FIS_TYPE_REGD2H, fis); } static void ahci_write_fis_d2h_ncq(struct ahci_port *p, int slot) { uint8_t fis[20]; p->tfd = ATA_S_READY | ATA_S_DSC; memset(fis, 0, sizeof(fis)); fis[0] = FIS_TYPE_REGD2H; fis[1] = 0; /* No interrupt */ fis[2] = p->tfd; /* Status */ fis[3] = 0; /* No error */ p->ci &= ~(1 << slot); ahci_write_fis(p, FIS_TYPE_REGD2H, fis); } static void ahci_write_reset_fis_d2h(struct ahci_port *p) { uint8_t fis[20]; memset(fis, 0, sizeof(fis)); fis[0] = FIS_TYPE_REGD2H; fis[3] = 1; fis[4] = 1; if (p->atapi) { fis[5] = 0x14; fis[6] = 0xeb; } fis[12] = 1; ahci_write_fis(p, FIS_TYPE_REGD2H, fis); } static void ahci_check_stopped(struct ahci_port *p) { /* * If we are no longer processing the command list and nothing * is in-flight, clear the running bit, the current command * slot, the command issue and active bits. */ if (!(p->cmd & AHCI_P_CMD_ST)) { if (p->pending == 0) { p->ccs = 0; p->cmd &= ~(AHCI_P_CMD_CR | AHCI_P_CMD_CCS_MASK); p->ci = 0; p->sact = 0; p->waitforclear = 0; } } } static void ahci_port_stop(struct ahci_port *p) { struct ahci_ioreq *aior; uint8_t *cfis; int slot; int error; assert(pthread_mutex_isowned_np(&p->pr_sc->mtx)); TAILQ_FOREACH(aior, &p->iobhd, io_blist) { /* * Try to cancel the outstanding blockif request. */ error = blockif_cancel(p->bctx, &aior->io_req); if (error != 0) continue; slot = aior->slot; cfis = aior->cfis; if (cfis[2] == ATA_WRITE_FPDMA_QUEUED || cfis[2] == ATA_READ_FPDMA_QUEUED || cfis[2] == ATA_SEND_FPDMA_QUEUED) p->sact &= ~(1 << slot); /* NCQ */ else p->ci &= ~(1 << slot); /* * This command is now done. */ p->pending &= ~(1 << slot); /* * Delete the blockif request from the busy list */ TAILQ_REMOVE(&p->iobhd, aior, io_blist); /* * Move the blockif request back to the free list */ STAILQ_INSERT_TAIL(&p->iofhd, aior, io_flist); } ahci_check_stopped(p); } static void ahci_port_reset(struct ahci_port *pr) { pr->serr = 0; pr->sact = 0; pr->xfermode = ATA_UDMA6; pr->mult_sectors = 128; if (!pr->bctx) { pr->ssts = ATA_SS_DET_NO_DEVICE; pr->sig = 0xFFFFFFFF; pr->tfd = 0x7F; return; } pr->ssts = ATA_SS_DET_PHY_ONLINE | ATA_SS_IPM_ACTIVE; if (pr->sctl & ATA_SC_SPD_MASK) pr->ssts |= (pr->sctl & ATA_SC_SPD_MASK); else pr->ssts |= ATA_SS_SPD_GEN3; pr->tfd = (1 << 8) | ATA_S_DSC | ATA_S_DMA; if (!pr->atapi) { pr->sig = PxSIG_ATA; pr->tfd |= ATA_S_READY; } else pr->sig = PxSIG_ATAPI; ahci_write_reset_fis_d2h(pr); } static void ahci_reset(struct pci_ahci_softc *sc) { int i; sc->ghc = AHCI_GHC_AE; sc->is = 0; if (sc->lintr) { pci_lintr_deassert(sc->asc_pi); sc->lintr = 0; } for (i = 0; i < sc->ports; i++) { sc->port[i].ie = 0; sc->port[i].is = 0; sc->port[i].cmd = (AHCI_P_CMD_SUD | AHCI_P_CMD_POD); if (sc->port[i].bctx) sc->port[i].cmd |= AHCI_P_CMD_CPS; sc->port[i].sctl = 0; ahci_port_reset(&sc->port[i]); } } static void ata_string(uint8_t *dest, const char *src, int len) { int i; for (i = 0; i < len; i++) { if (*src) dest[i ^ 1] = *src++; else dest[i ^ 1] = ' '; } } static void atapi_string(uint8_t *dest, const char *src, int len) { int i; for (i = 0; i < len; i++) { if (*src) dest[i] = *src++; else dest[i] = ' '; } } /* * Build up the iovec based on the PRDT, 'done' and 'len'. */ static void ahci_build_iov(struct ahci_port *p, struct ahci_ioreq *aior, struct ahci_prdt_entry *prdt, uint16_t prdtl) { struct blockif_req *breq = &aior->io_req; uint32_t dbcsz, extra, left, skip, todo; int i, j; assert(aior->len >= aior->done); /* Copy part of PRDT between 'done' and 'len' bytes into the iov. */ skip = aior->done; left = aior->len - aior->done; todo = 0; for (i = 0, j = 0; i < prdtl && j < BLOCKIF_IOV_MAX && left > 0; i++, prdt++) { dbcsz = (prdt->dbc & DBCMASK) + 1; /* Skip already done part of the PRDT */ if (dbcsz <= skip) { skip -= dbcsz; continue; } dbcsz -= skip; if (dbcsz > left) dbcsz = left; breq->br_iov[j].iov_base = paddr_guest2host(ahci_ctx(p->pr_sc), prdt->dba + skip, dbcsz); breq->br_iov[j].iov_len = dbcsz; todo += dbcsz; left -= dbcsz; skip = 0; j++; } /* If we got limited by IOV length, round I/O down to sector size. */ if (j == BLOCKIF_IOV_MAX) { extra = todo % blockif_sectsz(p->bctx); todo -= extra; assert(todo > 0); while (extra > 0) { if (breq->br_iov[j - 1].iov_len > extra) { breq->br_iov[j - 1].iov_len -= extra; break; } extra -= breq->br_iov[j - 1].iov_len; j--; } } breq->br_iovcnt = j; breq->br_resid = todo; aior->done += todo; aior->more = (aior->done < aior->len && i < prdtl); } static void ahci_handle_rw(struct ahci_port *p, int slot, uint8_t *cfis, uint32_t done) { struct ahci_ioreq *aior; struct blockif_req *breq; struct ahci_prdt_entry *prdt; struct ahci_cmd_hdr *hdr; uint64_t lba; uint32_t len; int err, first, ncq, readop; prdt = (struct ahci_prdt_entry *)(cfis + 0x80); hdr = (struct ahci_cmd_hdr *)(p->cmd_lst + slot * AHCI_CL_SIZE); ncq = 0; readop = 1; first = (done == 0); if (cfis[2] == ATA_WRITE || cfis[2] == ATA_WRITE48 || cfis[2] == ATA_WRITE_MUL || cfis[2] == ATA_WRITE_MUL48 || cfis[2] == ATA_WRITE_DMA || cfis[2] == ATA_WRITE_DMA48 || cfis[2] == ATA_WRITE_FPDMA_QUEUED) readop = 0; if (cfis[2] == ATA_WRITE_FPDMA_QUEUED || cfis[2] == ATA_READ_FPDMA_QUEUED) { lba = ((uint64_t)cfis[10] << 40) | ((uint64_t)cfis[9] << 32) | ((uint64_t)cfis[8] << 24) | ((uint64_t)cfis[6] << 16) | ((uint64_t)cfis[5] << 8) | cfis[4]; len = cfis[11] << 8 | cfis[3]; if (!len) len = 65536; ncq = 1; } else if (cfis[2] == ATA_READ48 || cfis[2] == ATA_WRITE48 || cfis[2] == ATA_READ_MUL48 || cfis[2] == ATA_WRITE_MUL48 || cfis[2] == ATA_READ_DMA48 || cfis[2] == ATA_WRITE_DMA48) { lba = ((uint64_t)cfis[10] << 40) | ((uint64_t)cfis[9] << 32) | ((uint64_t)cfis[8] << 24) | ((uint64_t)cfis[6] << 16) | ((uint64_t)cfis[5] << 8) | cfis[4]; len = cfis[13] << 8 | cfis[12]; if (!len) len = 65536; } else { lba = ((cfis[7] & 0xf) << 24) | (cfis[6] << 16) | (cfis[5] << 8) | cfis[4]; len = cfis[12]; if (!len) len = 256; } lba *= blockif_sectsz(p->bctx); len *= blockif_sectsz(p->bctx); /* Pull request off free list */ aior = STAILQ_FIRST(&p->iofhd); assert(aior != NULL); STAILQ_REMOVE_HEAD(&p->iofhd, io_flist); aior->cfis = cfis; aior->slot = slot; aior->len = len; aior->done = done; breq = &aior->io_req; breq->br_offset = lba + done; ahci_build_iov(p, aior, prdt, hdr->prdtl); /* Mark this command in-flight. */ p->pending |= 1 << slot; /* Stuff request onto busy list. */ TAILQ_INSERT_HEAD(&p->iobhd, aior, io_blist); if (ncq && first) ahci_write_fis_d2h_ncq(p, slot); if (readop) err = blockif_read(p->bctx, breq); else err = blockif_write(p->bctx, breq); assert(err == 0); } static void ahci_handle_flush(struct ahci_port *p, int slot, uint8_t *cfis) { struct ahci_ioreq *aior; struct blockif_req *breq; int err; /* * Pull request off free list */ aior = STAILQ_FIRST(&p->iofhd); assert(aior != NULL); STAILQ_REMOVE_HEAD(&p->iofhd, io_flist); aior->cfis = cfis; aior->slot = slot; aior->len = 0; aior->done = 0; aior->more = 0; breq = &aior->io_req; /* * Mark this command in-flight. */ p->pending |= 1 << slot; /* * Stuff request onto busy list */ TAILQ_INSERT_HEAD(&p->iobhd, aior, io_blist); err = blockif_flush(p->bctx, breq); assert(err == 0); } static inline unsigned int read_prdt(struct ahci_port *p, int slot, uint8_t *cfis, void *buf, unsigned int size) { struct ahci_cmd_hdr *hdr; struct ahci_prdt_entry *prdt; uint8_t *to; unsigned int len; int i; hdr = (struct ahci_cmd_hdr *)(p->cmd_lst + slot * AHCI_CL_SIZE); len = size; to = buf; prdt = (struct ahci_prdt_entry *)(cfis + 0x80); for (i = 0; i < hdr->prdtl && len; i++) { uint8_t *ptr; uint32_t dbcsz; unsigned int sublen; dbcsz = (prdt->dbc & DBCMASK) + 1; ptr = paddr_guest2host(ahci_ctx(p->pr_sc), prdt->dba, dbcsz); sublen = MIN(len, dbcsz); memcpy(to, ptr, sublen); len -= sublen; to += sublen; prdt++; } #ifndef __FreeBSD__ VERIFY3U(size, >=, len); #endif return (size - len); } static void ahci_handle_dsm_trim(struct ahci_port *p, int slot, uint8_t *cfis) { uint32_t len; int ncq; uint8_t *buf; unsigned int nread; buf = NULL; if (cfis[2] == ATA_DATA_SET_MANAGEMENT) { len = (uint16_t)cfis[13] << 8 | cfis[12]; len *= 512; ncq = 0; } else { /* ATA_SEND_FPDMA_QUEUED */ len = (uint16_t)cfis[11] << 8 | cfis[3]; len *= 512; ncq = 1; } /* Support for only a single block is advertised via IDENTIFY. */ if (len > 512) { goto invalid_command; } buf = malloc(len); nread = read_prdt(p, slot, cfis, buf, len); if (nread != len) { goto invalid_command; } ahci_handle_next_trim(p, slot, cfis, buf, len, 0); return; invalid_command: free(buf); if (ncq) { ahci_write_fis_d2h_ncq(p, slot); ahci_write_fis_sdb(p, slot, cfis, (ATA_E_ABORT << 8) | ATA_S_READY | ATA_S_ERROR); } else { ahci_write_fis_d2h(p, slot, cfis, (ATA_E_ABORT << 8) | ATA_S_READY | ATA_S_ERROR); } } static void ahci_handle_next_trim(struct ahci_port *p, int slot, uint8_t *cfis, uint8_t *buf, uint32_t len, uint32_t done) { struct ahci_ioreq *aior; struct blockif_req *breq; uint8_t *entry; uint64_t elba; uint32_t elen; int err; bool first, ncq; #ifndef __FreeBSD__ elba = 0; #endif first = (done == 0); if (cfis[2] == ATA_DATA_SET_MANAGEMENT) { ncq = false; } else { /* ATA_SEND_FPDMA_QUEUED */ ncq = true; } /* Find the next range to TRIM. */ while (done < len) { entry = &buf[done]; elba = ((uint64_t)entry[5] << 40) | ((uint64_t)entry[4] << 32) | ((uint64_t)entry[3] << 24) | ((uint64_t)entry[2] << 16) | ((uint64_t)entry[1] << 8) | entry[0]; elen = (uint16_t)entry[7] << 8 | entry[6]; done += 8; if (elen != 0) break; } /* All remaining ranges were empty. */ if (done == len) { free(buf); if (ncq) { if (first) ahci_write_fis_d2h_ncq(p, slot); ahci_write_fis_sdb(p, slot, cfis, ATA_S_READY | ATA_S_DSC); } else { ahci_write_fis_d2h(p, slot, cfis, ATA_S_READY | ATA_S_DSC); } if (!first) { p->pending &= ~(1 << slot); ahci_check_stopped(p); ahci_handle_port(p); } return; } /* * Pull request off free list */ aior = STAILQ_FIRST(&p->iofhd); assert(aior != NULL); STAILQ_REMOVE_HEAD(&p->iofhd, io_flist); aior->cfis = cfis; aior->slot = slot; aior->len = len; aior->done = done; aior->dsm = buf; aior->more = (len != done); breq = &aior->io_req; breq->br_offset = elba * blockif_sectsz(p->bctx); breq->br_resid = elen * blockif_sectsz(p->bctx); /* * Mark this command in-flight. */ p->pending |= 1 << slot; /* * Stuff request onto busy list */ TAILQ_INSERT_HEAD(&p->iobhd, aior, io_blist); if (ncq && first) ahci_write_fis_d2h_ncq(p, slot); err = blockif_delete(p->bctx, breq); assert(err == 0); } static inline void write_prdt(struct ahci_port *p, int slot, uint8_t *cfis, void *buf, unsigned int size) { struct ahci_cmd_hdr *hdr; struct ahci_prdt_entry *prdt; uint8_t *from; unsigned int len; int i; hdr = (struct ahci_cmd_hdr *)(p->cmd_lst + slot * AHCI_CL_SIZE); len = size; from = buf; prdt = (struct ahci_prdt_entry *)(cfis + 0x80); for (i = 0; i < hdr->prdtl && len; i++) { uint8_t *ptr; uint32_t dbcsz; int sublen; dbcsz = (prdt->dbc & DBCMASK) + 1; ptr = paddr_guest2host(ahci_ctx(p->pr_sc), prdt->dba, dbcsz); sublen = MIN(len, dbcsz); memcpy(ptr, from, sublen); len -= sublen; from += sublen; prdt++; } hdr->prdbc = size - len; } static void ahci_checksum(uint8_t *buf, int size) { int i; uint8_t sum = 0; for (i = 0; i < size - 1; i++) sum += buf[i]; buf[size - 1] = 0x100 - sum; } static void ahci_handle_read_log(struct ahci_port *p, int slot, uint8_t *cfis) { struct ahci_cmd_hdr *hdr; uint32_t buf[128]; uint8_t *buf8 = (uint8_t *)buf; uint16_t *buf16 = (uint16_t *)buf; hdr = (struct ahci_cmd_hdr *)(p->cmd_lst + slot * AHCI_CL_SIZE); if (p->atapi || hdr->prdtl == 0 || cfis[5] != 0 || cfis[9] != 0 || cfis[12] != 1 || cfis[13] != 0) { ahci_write_fis_d2h(p, slot, cfis, (ATA_E_ABORT << 8) | ATA_S_READY | ATA_S_ERROR); return; } memset(buf, 0, sizeof(buf)); if (cfis[4] == 0x00) { /* Log directory */ buf16[0x00] = 1; /* Version -- 1 */ buf16[0x10] = 1; /* NCQ Command Error Log -- 1 page */ buf16[0x13] = 1; /* SATA NCQ Send and Receive Log -- 1 page */ } else if (cfis[4] == 0x10) { /* NCQ Command Error Log */ memcpy(buf8, p->err_cfis, sizeof(p->err_cfis)); ahci_checksum(buf8, sizeof(buf)); } else if (cfis[4] == 0x13) { /* SATA NCQ Send and Receive Log */ if (blockif_candelete(p->bctx) && !blockif_is_ro(p->bctx)) { buf[0x00] = 1; /* SFQ DSM supported */ buf[0x01] = 1; /* SFQ DSM TRIM supported */ } } else { ahci_write_fis_d2h(p, slot, cfis, (ATA_E_ABORT << 8) | ATA_S_READY | ATA_S_ERROR); return; } if (cfis[2] == ATA_READ_LOG_EXT) ahci_write_fis_piosetup(p); write_prdt(p, slot, cfis, (void *)buf, sizeof(buf)); ahci_write_fis_d2h(p, slot, cfis, ATA_S_DSC | ATA_S_READY); } static void handle_identify(struct ahci_port *p, int slot, uint8_t *cfis) { struct ahci_cmd_hdr *hdr; hdr = (struct ahci_cmd_hdr *)(p->cmd_lst + slot * AHCI_CL_SIZE); if (p->atapi || hdr->prdtl == 0) { ahci_write_fis_d2h(p, slot, cfis, (ATA_E_ABORT << 8) | ATA_S_READY | ATA_S_ERROR); } else { ahci_write_fis_piosetup(p); write_prdt(p, slot, cfis, (void*)&p->ata_ident, sizeof(struct ata_params)); ahci_write_fis_d2h(p, slot, cfis, ATA_S_DSC | ATA_S_READY); } } static void ata_identify_init(struct ahci_port* p, int atapi) { struct ata_params* ata_ident = &p->ata_ident; if (atapi) { ata_ident->config = ATA_PROTO_ATAPI | ATA_ATAPI_TYPE_CDROM | ATA_ATAPI_REMOVABLE | ATA_DRQ_FAST; ata_ident->capabilities1 = ATA_SUPPORT_LBA | ATA_SUPPORT_DMA; ata_ident->capabilities2 = (1 << 14 | 1); ata_ident->atavalid = ATA_FLAG_64_70 | ATA_FLAG_88; ata_ident->obsolete62 = 0x3f; ata_ident->mwdmamodes = 7; if (p->xfermode & ATA_WDMA0) ata_ident->mwdmamodes |= (1 << ((p->xfermode & 7) + 8)); ata_ident->apiomodes = 3; ata_ident->mwdmamin = 0x0078; ata_ident->mwdmarec = 0x0078; ata_ident->pioblind = 0x0078; ata_ident->pioiordy = 0x0078; ata_ident->satacapabilities = (ATA_SATA_GEN1 | ATA_SATA_GEN2 | ATA_SATA_GEN3); ata_ident->satacapabilities2 = ((p->ssts & ATA_SS_SPD_MASK) >> 3); ata_ident->satasupport = ATA_SUPPORT_NCQ_STREAM; ata_ident->version_major = 0x3f0; ata_ident->support.command1 = (ATA_SUPPORT_POWERMGT | ATA_SUPPORT_PACKET | ATA_SUPPORT_RESET | ATA_SUPPORT_NOP); ata_ident->support.command2 = (1 << 14); ata_ident->support.extension = (1 << 14); ata_ident->enabled.command1 = (ATA_SUPPORT_POWERMGT | ATA_SUPPORT_PACKET | ATA_SUPPORT_RESET | ATA_SUPPORT_NOP); ata_ident->enabled.extension = (1 << 14); ata_ident->udmamodes = 0x7f; if (p->xfermode & ATA_UDMA0) ata_ident->udmamodes |= (1 << ((p->xfermode & 7) + 8)); ata_ident->transport_major = 0x1020; ata_ident->integrity = 0x00a5; } else { uint64_t sectors; int sectsz, psectsz, psectoff, candelete, ro; uint16_t cyl; uint8_t sech, heads; ro = blockif_is_ro(p->bctx); candelete = blockif_candelete(p->bctx); sectsz = blockif_sectsz(p->bctx); sectors = blockif_size(p->bctx) / sectsz; blockif_chs(p->bctx, &cyl, &heads, &sech); blockif_psectsz(p->bctx, &psectsz, &psectoff); ata_ident->config = ATA_DRQ_FAST; ata_ident->cylinders = cyl; ata_ident->heads = heads; ata_ident->sectors = sech; ata_ident->sectors_intr = (0x8000 | 128); ata_ident->tcg = 0; ata_ident->capabilities1 = ATA_SUPPORT_DMA | ATA_SUPPORT_LBA | ATA_SUPPORT_IORDY; ata_ident->capabilities2 = (1 << 14); ata_ident->atavalid = ATA_FLAG_64_70 | ATA_FLAG_88; if (p->mult_sectors) ata_ident->multi = (ATA_MULTI_VALID | p->mult_sectors); if (sectors <= 0x0fffffff) { ata_ident->lba_size_1 = sectors; ata_ident->lba_size_2 = (sectors >> 16); } else { ata_ident->lba_size_1 = 0xffff; ata_ident->lba_size_2 = 0x0fff; } ata_ident->mwdmamodes = 0x7; if (p->xfermode & ATA_WDMA0) ata_ident->mwdmamodes |= (1 << ((p->xfermode & 7) + 8)); ata_ident->apiomodes = 0x3; ata_ident->mwdmamin = 0x0078; ata_ident->mwdmarec = 0x0078; ata_ident->pioblind = 0x0078; ata_ident->pioiordy = 0x0078; ata_ident->support3 = 0; ata_ident->queue = 31; ata_ident->satacapabilities = (ATA_SATA_GEN1 | ATA_SATA_GEN2 | ATA_SATA_GEN3 | ATA_SUPPORT_NCQ); ata_ident->satacapabilities2 = (ATA_SUPPORT_RCVSND_FPDMA_QUEUED | (p->ssts & ATA_SS_SPD_MASK) >> 3); ata_ident->version_major = 0x3f0; ata_ident->version_minor = 0x28; ata_ident->support.command1 = (ATA_SUPPORT_POWERMGT | ATA_SUPPORT_WRITECACHE | ATA_SUPPORT_LOOKAHEAD | ATA_SUPPORT_NOP); ata_ident->support.command2 = (ATA_SUPPORT_ADDRESS48 | ATA_SUPPORT_FLUSHCACHE | ATA_SUPPORT_FLUSHCACHE48 | 1 << 14); ata_ident->support.extension = (1 << 14); ata_ident->enabled.command1 = (ATA_SUPPORT_POWERMGT | ATA_SUPPORT_WRITECACHE | ATA_SUPPORT_LOOKAHEAD | ATA_SUPPORT_NOP); ata_ident->enabled.command2 = (ATA_SUPPORT_ADDRESS48 | ATA_SUPPORT_FLUSHCACHE | ATA_SUPPORT_FLUSHCACHE48 | 1 << 15); ata_ident->enabled.extension = (1 << 14); ata_ident->udmamodes = 0x7f; if (p->xfermode & ATA_UDMA0) ata_ident->udmamodes |= (1 << ((p->xfermode & 7) + 8)); ata_ident->lba_size48_1 = sectors; ata_ident->lba_size48_2 = (sectors >> 16); ata_ident->lba_size48_3 = (sectors >> 32); ata_ident->lba_size48_4 = (sectors >> 48); if (candelete && !ro) { ata_ident->support3 |= ATA_SUPPORT_RZAT | ATA_SUPPORT_DRAT; ata_ident->max_dsm_blocks = 1; ata_ident->support_dsm = ATA_SUPPORT_DSM_TRIM; } ata_ident->pss = ATA_PSS_VALID_VALUE; ata_ident->lsalign = 0x4000; if (psectsz > sectsz) { ata_ident->pss |= ATA_PSS_MULTLS; ata_ident->pss |= ffsl(psectsz / sectsz) - 1; ata_ident->lsalign |= (psectoff / sectsz); } if (sectsz > 512) { ata_ident->pss |= ATA_PSS_LSSABOVE512; ata_ident->lss_1 = sectsz / 2; ata_ident->lss_2 = ((sectsz / 2) >> 16); } ata_ident->support2 = (ATA_SUPPORT_RWLOGDMAEXT | 1 << 14); ata_ident->enabled2 = (ATA_SUPPORT_RWLOGDMAEXT | 1 << 14); ata_ident->transport_major = 0x1020; ata_ident->integrity = 0x00a5; } ahci_checksum((uint8_t*)ata_ident, sizeof(struct ata_params)); } static void handle_atapi_identify(struct ahci_port *p, int slot, uint8_t *cfis) { if (!p->atapi) { ahci_write_fis_d2h(p, slot, cfis, (ATA_E_ABORT << 8) | ATA_S_READY | ATA_S_ERROR); } else { ahci_write_fis_piosetup(p); write_prdt(p, slot, cfis, (void *)&p->ata_ident, sizeof(struct ata_params)); ahci_write_fis_d2h(p, slot, cfis, ATA_S_DSC | ATA_S_READY); } } static void atapi_inquiry(struct ahci_port *p, int slot, uint8_t *cfis) { uint8_t buf[36]; uint8_t *acmd; unsigned int len; uint32_t tfd; acmd = cfis + 0x40; if (acmd[1] & 1) { /* VPD */ if (acmd[2] == 0) { /* Supported VPD pages */ buf[0] = 0x05; buf[1] = 0; buf[2] = 0; buf[3] = 1; buf[4] = 0; len = 4 + buf[3]; } else { p->sense_key = ATA_SENSE_ILLEGAL_REQUEST; p->asc = 0x24; tfd = (p->sense_key << 12) | ATA_S_READY | ATA_S_ERROR; cfis[4] = (cfis[4] & ~7) | ATA_I_CMD | ATA_I_IN; ahci_write_fis_d2h(p, slot, cfis, tfd); return; } } else { buf[0] = 0x05; buf[1] = 0x80; buf[2] = 0x00; buf[3] = 0x21; buf[4] = 31; buf[5] = 0; buf[6] = 0; buf[7] = 0; atapi_string(buf + 8, "BHYVE", 8); atapi_string(buf + 16, "BHYVE DVD-ROM", 16); atapi_string(buf + 32, "001", 4); len = sizeof(buf); } if (len > acmd[4]) len = acmd[4]; cfis[4] = (cfis[4] & ~7) | ATA_I_CMD | ATA_I_IN; write_prdt(p, slot, cfis, buf, len); ahci_write_fis_d2h(p, slot, cfis, ATA_S_READY | ATA_S_DSC); } static void atapi_read_capacity(struct ahci_port *p, int slot, uint8_t *cfis) { uint8_t buf[8]; uint64_t sectors; sectors = blockif_size(p->bctx) / 2048; be32enc(buf, sectors - 1); be32enc(buf + 4, 2048); cfis[4] = (cfis[4] & ~7) | ATA_I_CMD | ATA_I_IN; write_prdt(p, slot, cfis, buf, sizeof(buf)); ahci_write_fis_d2h(p, slot, cfis, ATA_S_READY | ATA_S_DSC); } static void atapi_read_toc(struct ahci_port *p, int slot, uint8_t *cfis) { uint8_t *acmd; uint8_t format; unsigned int len; acmd = cfis + 0x40; len = be16dec(acmd + 7); format = acmd[9] >> 6; switch (format) { case 0: { size_t size; int msf; uint64_t sectors; uint8_t start_track, buf[20], *bp; msf = (acmd[1] >> 1) & 1; start_track = acmd[6]; if (start_track > 1 && start_track != 0xaa) { uint32_t tfd; p->sense_key = ATA_SENSE_ILLEGAL_REQUEST; p->asc = 0x24; tfd = (p->sense_key << 12) | ATA_S_READY | ATA_S_ERROR; cfis[4] = (cfis[4] & ~7) | ATA_I_CMD | ATA_I_IN; ahci_write_fis_d2h(p, slot, cfis, tfd); return; } bp = buf + 2; *bp++ = 1; *bp++ = 1; if (start_track <= 1) { *bp++ = 0; *bp++ = 0x14; *bp++ = 1; *bp++ = 0; if (msf) { *bp++ = 0; lba_to_msf(bp, 0); bp += 3; } else { *bp++ = 0; *bp++ = 0; *bp++ = 0; *bp++ = 0; } } *bp++ = 0; *bp++ = 0x14; *bp++ = 0xaa; *bp++ = 0; sectors = blockif_size(p->bctx) / blockif_sectsz(p->bctx); sectors >>= 2; if (msf) { *bp++ = 0; lba_to_msf(bp, sectors); bp += 3; } else { be32enc(bp, sectors); bp += 4; } size = bp - buf; be16enc(buf, size - 2); if (len > size) len = size; write_prdt(p, slot, cfis, buf, len); cfis[4] = (cfis[4] & ~7) | ATA_I_CMD | ATA_I_IN; ahci_write_fis_d2h(p, slot, cfis, ATA_S_READY | ATA_S_DSC); break; } case 1: { uint8_t buf[12]; memset(buf, 0, sizeof(buf)); buf[1] = 0xa; buf[2] = 0x1; buf[3] = 0x1; if (len > sizeof(buf)) len = sizeof(buf); write_prdt(p, slot, cfis, buf, len); cfis[4] = (cfis[4] & ~7) | ATA_I_CMD | ATA_I_IN; ahci_write_fis_d2h(p, slot, cfis, ATA_S_READY | ATA_S_DSC); break; } case 2: { size_t size; int msf; uint64_t sectors; uint8_t *bp, buf[50]; msf = (acmd[1] >> 1) & 1; bp = buf + 2; *bp++ = 1; *bp++ = 1; *bp++ = 1; *bp++ = 0x14; *bp++ = 0; *bp++ = 0xa0; *bp++ = 0; *bp++ = 0; *bp++ = 0; *bp++ = 0; *bp++ = 1; *bp++ = 0; *bp++ = 0; *bp++ = 1; *bp++ = 0x14; *bp++ = 0; *bp++ = 0xa1; *bp++ = 0; *bp++ = 0; *bp++ = 0; *bp++ = 0; *bp++ = 1; *bp++ = 0; *bp++ = 0; *bp++ = 1; *bp++ = 0x14; *bp++ = 0; *bp++ = 0xa2; *bp++ = 0; *bp++ = 0; *bp++ = 0; sectors = blockif_size(p->bctx) / blockif_sectsz(p->bctx); sectors >>= 2; if (msf) { *bp++ = 0; lba_to_msf(bp, sectors); bp += 3; } else { be32enc(bp, sectors); bp += 4; } *bp++ = 1; *bp++ = 0x14; *bp++ = 0; *bp++ = 1; *bp++ = 0; *bp++ = 0; *bp++ = 0; if (msf) { *bp++ = 0; lba_to_msf(bp, 0); bp += 3; } else { *bp++ = 0; *bp++ = 0; *bp++ = 0; *bp++ = 0; } size = bp - buf; be16enc(buf, size - 2); if (len > size) len = size; write_prdt(p, slot, cfis, buf, len); cfis[4] = (cfis[4] & ~7) | ATA_I_CMD | ATA_I_IN; ahci_write_fis_d2h(p, slot, cfis, ATA_S_READY | ATA_S_DSC); break; } default: { uint32_t tfd; p->sense_key = ATA_SENSE_ILLEGAL_REQUEST; p->asc = 0x24; tfd = (p->sense_key << 12) | ATA_S_READY | ATA_S_ERROR; cfis[4] = (cfis[4] & ~7) | ATA_I_CMD | ATA_I_IN; ahci_write_fis_d2h(p, slot, cfis, tfd); break; } } } static void atapi_report_luns(struct ahci_port *p, int slot, uint8_t *cfis) { uint8_t buf[16]; memset(buf, 0, sizeof(buf)); buf[3] = 8; cfis[4] = (cfis[4] & ~7) | ATA_I_CMD | ATA_I_IN; write_prdt(p, slot, cfis, buf, sizeof(buf)); ahci_write_fis_d2h(p, slot, cfis, ATA_S_READY | ATA_S_DSC); } static void atapi_read(struct ahci_port *p, int slot, uint8_t *cfis, uint32_t done) { struct ahci_ioreq *aior; struct ahci_cmd_hdr *hdr; struct ahci_prdt_entry *prdt; struct blockif_req *breq; uint8_t *acmd; uint64_t lba; uint32_t len; int err; acmd = cfis + 0x40; hdr = (struct ahci_cmd_hdr *)(p->cmd_lst + slot * AHCI_CL_SIZE); prdt = (struct ahci_prdt_entry *)(cfis + 0x80); lba = be32dec(acmd + 2); if (acmd[0] == READ_10) len = be16dec(acmd + 7); else len = be32dec(acmd + 6); if (len == 0) { cfis[4] = (cfis[4] & ~7) | ATA_I_CMD | ATA_I_IN; ahci_write_fis_d2h(p, slot, cfis, ATA_S_READY | ATA_S_DSC); } lba *= 2048; len *= 2048; /* * Pull request off free list */ aior = STAILQ_FIRST(&p->iofhd); assert(aior != NULL); STAILQ_REMOVE_HEAD(&p->iofhd, io_flist); aior->cfis = cfis; aior->slot = slot; aior->len = len; aior->done = done; breq = &aior->io_req; breq->br_offset = lba + done; ahci_build_iov(p, aior, prdt, hdr->prdtl); /* Mark this command in-flight. */ p->pending |= 1 << slot; /* Stuff request onto busy list. */ TAILQ_INSERT_HEAD(&p->iobhd, aior, io_blist); err = blockif_read(p->bctx, breq); assert(err == 0); } static void atapi_request_sense(struct ahci_port *p, int slot, uint8_t *cfis) { uint8_t buf[64]; uint8_t *acmd; unsigned int len; acmd = cfis + 0x40; len = acmd[4]; if (len > sizeof(buf)) len = sizeof(buf); memset(buf, 0, len); buf[0] = 0x70 | (1 << 7); buf[2] = p->sense_key; buf[7] = 10; buf[12] = p->asc; write_prdt(p, slot, cfis, buf, len); cfis[4] = (cfis[4] & ~7) | ATA_I_CMD | ATA_I_IN; ahci_write_fis_d2h(p, slot, cfis, ATA_S_READY | ATA_S_DSC); } static void atapi_start_stop_unit(struct ahci_port *p, int slot, uint8_t *cfis) { uint8_t *acmd = cfis + 0x40; uint32_t tfd; switch (acmd[4] & 3) { case 0: case 1: case 3: cfis[4] = (cfis[4] & ~7) | ATA_I_CMD | ATA_I_IN; tfd = ATA_S_READY | ATA_S_DSC; break; case 2: /* TODO eject media */ cfis[4] = (cfis[4] & ~7) | ATA_I_CMD | ATA_I_IN; p->sense_key = ATA_SENSE_ILLEGAL_REQUEST; p->asc = 0x53; tfd = (p->sense_key << 12) | ATA_S_READY | ATA_S_ERROR; break; } ahci_write_fis_d2h(p, slot, cfis, tfd); } static void atapi_mode_sense(struct ahci_port *p, int slot, uint8_t *cfis) { uint8_t *acmd; uint32_t tfd = 0; uint8_t pc, code; unsigned int len; acmd = cfis + 0x40; len = be16dec(acmd + 7); pc = acmd[2] >> 6; code = acmd[2] & 0x3f; switch (pc) { case 0: switch (code) { case MODEPAGE_RW_ERROR_RECOVERY: { uint8_t buf[16]; if (len > sizeof(buf)) len = sizeof(buf); memset(buf, 0, sizeof(buf)); be16enc(buf, 16 - 2); buf[2] = 0x70; buf[8] = 0x01; buf[9] = 16 - 10; buf[11] = 0x05; write_prdt(p, slot, cfis, buf, len); tfd = ATA_S_READY | ATA_S_DSC; break; } case MODEPAGE_CD_CAPABILITIES: { uint8_t buf[30]; if (len > sizeof(buf)) len = sizeof(buf); memset(buf, 0, sizeof(buf)); be16enc(buf, 30 - 2); buf[2] = 0x70; buf[8] = 0x2A; buf[9] = 30 - 10; buf[10] = 0x08; buf[12] = 0x71; be16enc(&buf[18], 2); be16enc(&buf[20], 512); write_prdt(p, slot, cfis, buf, len); tfd = ATA_S_READY | ATA_S_DSC; break; } default: goto error; break; } break; case 3: p->sense_key = ATA_SENSE_ILLEGAL_REQUEST; p->asc = 0x39; tfd = (p->sense_key << 12) | ATA_S_READY | ATA_S_ERROR; break; error: case 1: case 2: p->sense_key = ATA_SENSE_ILLEGAL_REQUEST; p->asc = 0x24; tfd = (p->sense_key << 12) | ATA_S_READY | ATA_S_ERROR; break; } cfis[4] = (cfis[4] & ~7) | ATA_I_CMD | ATA_I_IN; ahci_write_fis_d2h(p, slot, cfis, tfd); } static void atapi_get_event_status_notification(struct ahci_port *p, int slot, uint8_t *cfis) { uint8_t *acmd; uint32_t tfd; acmd = cfis + 0x40; /* we don't support asynchronous operation */ if (!(acmd[1] & 1)) { p->sense_key = ATA_SENSE_ILLEGAL_REQUEST; p->asc = 0x24; tfd = (p->sense_key << 12) | ATA_S_READY | ATA_S_ERROR; } else { uint8_t buf[8]; unsigned int len; len = be16dec(acmd + 7); if (len > sizeof(buf)) len = sizeof(buf); memset(buf, 0, sizeof(buf)); be16enc(buf, 8 - 2); buf[2] = 0x04; buf[3] = 0x10; buf[5] = 0x02; write_prdt(p, slot, cfis, buf, len); tfd = ATA_S_READY | ATA_S_DSC; } cfis[4] = (cfis[4] & ~7) | ATA_I_CMD | ATA_I_IN; ahci_write_fis_d2h(p, slot, cfis, tfd); } static void handle_packet_cmd(struct ahci_port *p, int slot, uint8_t *cfis) { uint8_t *acmd; acmd = cfis + 0x40; #ifdef AHCI_DEBUG { int i; DPRINTF("ACMD:"); for (i = 0; i < 16; i++) DPRINTF("%02x ", acmd[i]); DPRINTF(""); } #endif switch (acmd[0]) { case TEST_UNIT_READY: cfis[4] = (cfis[4] & ~7) | ATA_I_CMD | ATA_I_IN; ahci_write_fis_d2h(p, slot, cfis, ATA_S_READY | ATA_S_DSC); break; case INQUIRY: atapi_inquiry(p, slot, cfis); break; case READ_CAPACITY: atapi_read_capacity(p, slot, cfis); break; case PREVENT_ALLOW: /* TODO */ cfis[4] = (cfis[4] & ~7) | ATA_I_CMD | ATA_I_IN; ahci_write_fis_d2h(p, slot, cfis, ATA_S_READY | ATA_S_DSC); break; case READ_TOC: atapi_read_toc(p, slot, cfis); break; case REPORT_LUNS: atapi_report_luns(p, slot, cfis); break; case READ_10: case READ_12: atapi_read(p, slot, cfis, 0); break; case REQUEST_SENSE: atapi_request_sense(p, slot, cfis); break; case START_STOP_UNIT: atapi_start_stop_unit(p, slot, cfis); break; case MODE_SENSE_10: atapi_mode_sense(p, slot, cfis); break; case GET_EVENT_STATUS_NOTIFICATION: atapi_get_event_status_notification(p, slot, cfis); break; default: cfis[4] = (cfis[4] & ~7) | ATA_I_CMD | ATA_I_IN; p->sense_key = ATA_SENSE_ILLEGAL_REQUEST; p->asc = 0x20; ahci_write_fis_d2h(p, slot, cfis, (p->sense_key << 12) | ATA_S_READY | ATA_S_ERROR); break; } } static void ahci_handle_cmd(struct ahci_port *p, int slot, uint8_t *cfis) { p->tfd |= ATA_S_BUSY; switch (cfis[2]) { case ATA_ATA_IDENTIFY: handle_identify(p, slot, cfis); break; case ATA_SETFEATURES: { switch (cfis[3]) { case ATA_SF_ENAB_SATA_SF: switch (cfis[12]) { case ATA_SATA_SF_AN: p->tfd = ATA_S_DSC | ATA_S_READY; break; default: p->tfd = ATA_S_ERROR | ATA_S_READY; p->tfd |= (ATA_ERROR_ABORT << 8); break; } break; case ATA_SF_ENAB_WCACHE: case ATA_SF_DIS_WCACHE: case ATA_SF_ENAB_RCACHE: case ATA_SF_DIS_RCACHE: p->tfd = ATA_S_DSC | ATA_S_READY; break; case ATA_SF_SETXFER: { switch (cfis[12] & 0xf8) { case ATA_PIO: case ATA_PIO0: break; case ATA_WDMA0: case ATA_UDMA0: p->xfermode = (cfis[12] & 0x7); break; } p->tfd = ATA_S_DSC | ATA_S_READY; break; } default: p->tfd = ATA_S_ERROR | ATA_S_READY; p->tfd |= (ATA_ERROR_ABORT << 8); break; } ahci_write_fis_d2h(p, slot, cfis, p->tfd); break; } case ATA_SET_MULTI: if (cfis[12] != 0 && (cfis[12] > 128 || (cfis[12] & (cfis[12] - 1)))) { p->tfd = ATA_S_ERROR | ATA_S_READY; p->tfd |= (ATA_ERROR_ABORT << 8); } else { p->mult_sectors = cfis[12]; p->tfd = ATA_S_DSC | ATA_S_READY; } ahci_write_fis_d2h(p, slot, cfis, p->tfd); break; case ATA_READ: case ATA_WRITE: case ATA_READ48: case ATA_WRITE48: case ATA_READ_MUL: case ATA_WRITE_MUL: case ATA_READ_MUL48: case ATA_WRITE_MUL48: case ATA_READ_DMA: case ATA_WRITE_DMA: case ATA_READ_DMA48: case ATA_WRITE_DMA48: case ATA_READ_FPDMA_QUEUED: case ATA_WRITE_FPDMA_QUEUED: ahci_handle_rw(p, slot, cfis, 0); break; case ATA_FLUSHCACHE: case ATA_FLUSHCACHE48: ahci_handle_flush(p, slot, cfis); break; case ATA_DATA_SET_MANAGEMENT: if (cfis[11] == 0 && cfis[3] == ATA_DSM_TRIM && cfis[13] == 0 && cfis[12] == 1) { ahci_handle_dsm_trim(p, slot, cfis); break; } ahci_write_fis_d2h(p, slot, cfis, (ATA_E_ABORT << 8) | ATA_S_READY | ATA_S_ERROR); break; case ATA_SEND_FPDMA_QUEUED: if ((cfis[13] & 0x1f) == ATA_SFPDMA_DSM && cfis[17] == 0 && cfis[16] == ATA_DSM_TRIM && cfis[11] == 0 && cfis[3] == 1) { ahci_handle_dsm_trim(p, slot, cfis); break; } ahci_write_fis_d2h(p, slot, cfis, (ATA_E_ABORT << 8) | ATA_S_READY | ATA_S_ERROR); break; case ATA_READ_LOG_EXT: case ATA_READ_LOG_DMA_EXT: ahci_handle_read_log(p, slot, cfis); break; case ATA_SECURITY_FREEZE_LOCK: case ATA_SMART_CMD: case ATA_NOP: ahci_write_fis_d2h(p, slot, cfis, (ATA_E_ABORT << 8) | ATA_S_READY | ATA_S_ERROR); break; case ATA_CHECK_POWER_MODE: cfis[12] = 0xff; /* always on */ ahci_write_fis_d2h(p, slot, cfis, ATA_S_READY | ATA_S_DSC); break; case ATA_STANDBY_CMD: case ATA_STANDBY_IMMEDIATE: case ATA_IDLE_CMD: case ATA_IDLE_IMMEDIATE: case ATA_SLEEP: case ATA_READ_VERIFY: case ATA_READ_VERIFY48: ahci_write_fis_d2h(p, slot, cfis, ATA_S_READY | ATA_S_DSC); break; case ATA_ATAPI_IDENTIFY: handle_atapi_identify(p, slot, cfis); break; case ATA_PACKET_CMD: if (!p->atapi) { ahci_write_fis_d2h(p, slot, cfis, (ATA_E_ABORT << 8) | ATA_S_READY | ATA_S_ERROR); } else handle_packet_cmd(p, slot, cfis); break; default: EPRINTLN("Unsupported cmd:%02x", cfis[2]); ahci_write_fis_d2h(p, slot, cfis, (ATA_E_ABORT << 8) | ATA_S_READY | ATA_S_ERROR); break; } } static void ahci_handle_slot(struct ahci_port *p, int slot) { struct ahci_cmd_hdr *hdr; #ifdef AHCI_DEBUG struct ahci_prdt_entry *prdt; #endif struct pci_ahci_softc *sc; uint8_t *cfis; #ifdef AHCI_DEBUG int cfl, i; #endif sc = p->pr_sc; hdr = (struct ahci_cmd_hdr *)(p->cmd_lst + slot * AHCI_CL_SIZE); #ifdef AHCI_DEBUG cfl = (hdr->flags & 0x1f) * 4; #endif cfis = paddr_guest2host(ahci_ctx(sc), hdr->ctba, 0x80 + hdr->prdtl * sizeof(struct ahci_prdt_entry)); #ifdef AHCI_DEBUG prdt = (struct ahci_prdt_entry *)(cfis + 0x80); DPRINTF("cfis:"); for (i = 0; i < cfl; i++) { if (i % 10 == 0) DPRINTF(""); DPRINTF("%02x ", cfis[i]); } DPRINTF(""); for (i = 0; i < hdr->prdtl; i++) { DPRINTF("%d@%08"PRIx64"", prdt->dbc & 0x3fffff, prdt->dba); prdt++; } #endif if (cfis[0] != FIS_TYPE_REGH2D) { EPRINTLN("Not a H2D FIS:%02x", cfis[0]); return; } if (cfis[1] & 0x80) { ahci_handle_cmd(p, slot, cfis); } else { if (cfis[15] & (1 << 2)) p->reset = 1; else if (p->reset) { p->reset = 0; ahci_port_reset(p); } p->ci &= ~(1 << slot); } } static void ahci_handle_port(struct ahci_port *p) { if (!(p->cmd & AHCI_P_CMD_ST)) return; /* * Search for any new commands to issue ignoring those that * are already in-flight. Stop if device is busy or in error. */ for (; (p->ci & ~p->pending) != 0; p->ccs = ((p->ccs + 1) & 31)) { if ((p->tfd & (ATA_S_BUSY | ATA_S_DRQ)) != 0) break; if (p->waitforclear) break; if ((p->ci & ~p->pending & (1 << p->ccs)) != 0) { p->cmd &= ~AHCI_P_CMD_CCS_MASK; p->cmd |= p->ccs << AHCI_P_CMD_CCS_SHIFT; ahci_handle_slot(p, p->ccs); } } } /* * blockif callback routine - this runs in the context of the blockif * i/o thread, so the mutex needs to be acquired. */ static void ata_ioreq_cb(struct blockif_req *br, int err) { struct ahci_cmd_hdr *hdr; struct ahci_ioreq *aior; struct ahci_port *p; struct pci_ahci_softc *sc; uint32_t tfd; uint8_t *cfis, *dsm; int slot, ncq; DPRINTF("%s %d", __func__, err); ncq = 0; aior = br->br_param; p = aior->io_pr; cfis = aior->cfis; slot = aior->slot; sc = p->pr_sc; hdr = (struct ahci_cmd_hdr *)(p->cmd_lst + slot * AHCI_CL_SIZE); if (cfis[2] == ATA_WRITE_FPDMA_QUEUED || cfis[2] == ATA_READ_FPDMA_QUEUED || cfis[2] == ATA_SEND_FPDMA_QUEUED) ncq = 1; dsm = aior->dsm; aior->dsm = NULL; pthread_mutex_lock(&sc->mtx); /* * Delete the blockif request from the busy list */ TAILQ_REMOVE(&p->iobhd, aior, io_blist); /* * Move the blockif request back to the free list */ STAILQ_INSERT_TAIL(&p->iofhd, aior, io_flist); if (!err) hdr->prdbc = aior->done; if (!err && aior->more) { if (dsm != NULL) ahci_handle_next_trim(p, slot, cfis, dsm, aior->len, aior->done); else ahci_handle_rw(p, slot, cfis, aior->done); goto out; } if (!err) tfd = ATA_S_READY | ATA_S_DSC; else tfd = (ATA_E_ABORT << 8) | ATA_S_READY | ATA_S_ERROR; if (ncq) ahci_write_fis_sdb(p, slot, cfis, tfd); else ahci_write_fis_d2h(p, slot, cfis, tfd); /* * This command is now complete. */ p->pending &= ~(1 << slot); ahci_check_stopped(p); ahci_handle_port(p); free(dsm); out: pthread_mutex_unlock(&sc->mtx); DPRINTF("%s exit", __func__); } static void atapi_ioreq_cb(struct blockif_req *br, int err) { struct ahci_cmd_hdr *hdr; struct ahci_ioreq *aior; struct ahci_port *p; struct pci_ahci_softc *sc; uint8_t *cfis; uint32_t tfd; int slot; DPRINTF("%s %d", __func__, err); aior = br->br_param; p = aior->io_pr; cfis = aior->cfis; slot = aior->slot; sc = p->pr_sc; hdr = (struct ahci_cmd_hdr *)(p->cmd_lst + aior->slot * AHCI_CL_SIZE); pthread_mutex_lock(&sc->mtx); /* * Delete the blockif request from the busy list */ TAILQ_REMOVE(&p->iobhd, aior, io_blist); /* * Move the blockif request back to the free list */ STAILQ_INSERT_TAIL(&p->iofhd, aior, io_flist); if (!err) hdr->prdbc = aior->done; if (!err && aior->more) { atapi_read(p, slot, cfis, aior->done); goto out; } if (!err) { tfd = ATA_S_READY | ATA_S_DSC; } else { p->sense_key = ATA_SENSE_ILLEGAL_REQUEST; p->asc = 0x21; tfd = (p->sense_key << 12) | ATA_S_READY | ATA_S_ERROR; } cfis[4] = (cfis[4] & ~7) | ATA_I_CMD | ATA_I_IN; ahci_write_fis_d2h(p, slot, cfis, tfd); /* * This command is now complete. */ p->pending &= ~(1 << slot); ahci_check_stopped(p); ahci_handle_port(p); out: pthread_mutex_unlock(&sc->mtx); DPRINTF("%s exit", __func__); } static void pci_ahci_ioreq_init(struct ahci_port *pr) { struct ahci_ioreq *vr; int i; pr->ioqsz = blockif_queuesz(pr->bctx); pr->ioreq = calloc(pr->ioqsz, sizeof(struct ahci_ioreq)); STAILQ_INIT(&pr->iofhd); /* * Add all i/o request entries to the free queue */ for (i = 0; i < pr->ioqsz; i++) { vr = &pr->ioreq[i]; vr->io_pr = pr; if (!pr->atapi) vr->io_req.br_callback = ata_ioreq_cb; else vr->io_req.br_callback = atapi_ioreq_cb; vr->io_req.br_param = vr; STAILQ_INSERT_TAIL(&pr->iofhd, vr, io_flist); } TAILQ_INIT(&pr->iobhd); } static void pci_ahci_port_write(struct pci_ahci_softc *sc, uint64_t offset, uint64_t value) { int port = (offset - AHCI_OFFSET) / AHCI_STEP; offset = (offset - AHCI_OFFSET) % AHCI_STEP; struct ahci_port *p = &sc->port[port]; DPRINTF("pci_ahci_port %d: write offset 0x%"PRIx64" value 0x%"PRIx64"", port, offset, value); switch (offset) { case AHCI_P_CLB: p->clb = value; break; case AHCI_P_CLBU: p->clbu = value; break; case AHCI_P_FB: p->fb = value; break; case AHCI_P_FBU: p->fbu = value; break; case AHCI_P_IS: p->is &= ~value; ahci_port_intr(p); break; case AHCI_P_IE: p->ie = value & 0xFDC000FF; ahci_port_intr(p); break; case AHCI_P_CMD: { p->cmd &= ~(AHCI_P_CMD_ST | AHCI_P_CMD_SUD | AHCI_P_CMD_POD | AHCI_P_CMD_CLO | AHCI_P_CMD_FRE | AHCI_P_CMD_APSTE | AHCI_P_CMD_ATAPI | AHCI_P_CMD_DLAE | AHCI_P_CMD_ALPE | AHCI_P_CMD_ASP | AHCI_P_CMD_ICC_MASK); p->cmd |= (AHCI_P_CMD_ST | AHCI_P_CMD_SUD | AHCI_P_CMD_POD | AHCI_P_CMD_CLO | AHCI_P_CMD_FRE | AHCI_P_CMD_APSTE | AHCI_P_CMD_ATAPI | AHCI_P_CMD_DLAE | AHCI_P_CMD_ALPE | AHCI_P_CMD_ASP | AHCI_P_CMD_ICC_MASK) & value; if (!(value & AHCI_P_CMD_ST)) { ahci_port_stop(p); } else { uint64_t clb; p->cmd |= AHCI_P_CMD_CR; clb = (uint64_t)p->clbu << 32 | p->clb; p->cmd_lst = paddr_guest2host(ahci_ctx(sc), clb, AHCI_CL_SIZE * AHCI_MAX_SLOTS); } if (value & AHCI_P_CMD_FRE) { uint64_t fb; p->cmd |= AHCI_P_CMD_FR; fb = (uint64_t)p->fbu << 32 | p->fb; /* we don't support FBSCP, so rfis size is 256Bytes */ p->rfis = paddr_guest2host(ahci_ctx(sc), fb, 256); } else { p->cmd &= ~AHCI_P_CMD_FR; } if (value & AHCI_P_CMD_CLO) { p->tfd &= ~(ATA_S_BUSY | ATA_S_DRQ); p->cmd &= ~AHCI_P_CMD_CLO; } if (value & AHCI_P_CMD_ICC_MASK) { p->cmd &= ~AHCI_P_CMD_ICC_MASK; } ahci_handle_port(p); break; } case AHCI_P_TFD: case AHCI_P_SIG: case AHCI_P_SSTS: EPRINTLN("pci_ahci_port: read only registers 0x%"PRIx64"", offset); break; case AHCI_P_SCTL: p->sctl = value; if (!(p->cmd & AHCI_P_CMD_ST)) { if (value & ATA_SC_DET_RESET) ahci_port_reset(p); } break; case AHCI_P_SERR: p->serr &= ~value; break; case AHCI_P_SACT: p->sact |= value; break; case AHCI_P_CI: p->ci |= value; ahci_handle_port(p); break; case AHCI_P_SNTF: case AHCI_P_FBS: default: break; } } static void pci_ahci_host_write(struct pci_ahci_softc *sc, uint64_t offset, uint64_t value) { DPRINTF("pci_ahci_host: write offset 0x%"PRIx64" value 0x%"PRIx64"", offset, value); switch (offset) { case AHCI_CAP: case AHCI_PI: case AHCI_VS: case AHCI_CAP2: DPRINTF("pci_ahci_host: read only registers 0x%"PRIx64"", offset); break; case AHCI_GHC: if (value & AHCI_GHC_HR) { ahci_reset(sc); break; } if (value & AHCI_GHC_IE) sc->ghc |= AHCI_GHC_IE; else sc->ghc &= ~AHCI_GHC_IE; ahci_generate_intr(sc, 0xffffffff); break; case AHCI_IS: sc->is &= ~value; ahci_generate_intr(sc, value); break; default: break; } } static void pci_ahci_write(struct pci_devinst *pi, int baridx, uint64_t offset, int size, uint64_t value) { struct pci_ahci_softc *sc = pi->pi_arg; assert(baridx == 5); assert((offset % 4) == 0 && size == 4); pthread_mutex_lock(&sc->mtx); if (offset < AHCI_OFFSET) pci_ahci_host_write(sc, offset, value); else if (offset < (uint64_t)AHCI_OFFSET + sc->ports * AHCI_STEP) pci_ahci_port_write(sc, offset, value); else EPRINTLN("pci_ahci: unknown i/o write offset 0x%"PRIx64"", offset); pthread_mutex_unlock(&sc->mtx); } static uint64_t pci_ahci_host_read(struct pci_ahci_softc *sc, uint64_t offset) { uint32_t value; switch (offset) { case AHCI_CAP: case AHCI_GHC: case AHCI_IS: case AHCI_PI: case AHCI_VS: case AHCI_CCCC: case AHCI_CCCP: case AHCI_EM_LOC: case AHCI_EM_CTL: case AHCI_CAP2: { uint32_t *p = &sc->cap; p += (offset - AHCI_CAP) / sizeof(uint32_t); value = *p; break; } default: value = 0; break; } DPRINTF("pci_ahci_host: read offset 0x%"PRIx64" value 0x%x", offset, value); return (value); } static uint64_t pci_ahci_port_read(struct pci_ahci_softc *sc, uint64_t offset) { uint32_t value; int port = (offset - AHCI_OFFSET) / AHCI_STEP; offset = (offset - AHCI_OFFSET) % AHCI_STEP; switch (offset) { case AHCI_P_CLB: case AHCI_P_CLBU: case AHCI_P_FB: case AHCI_P_FBU: case AHCI_P_IS: case AHCI_P_IE: case AHCI_P_CMD: case AHCI_P_TFD: case AHCI_P_SIG: case AHCI_P_SSTS: case AHCI_P_SCTL: case AHCI_P_SERR: case AHCI_P_SACT: case AHCI_P_CI: case AHCI_P_SNTF: case AHCI_P_FBS: { uint32_t *p= &sc->port[port].clb; p += (offset - AHCI_P_CLB) / sizeof(uint32_t); value = *p; break; } default: value = 0; break; } DPRINTF("pci_ahci_port %d: read offset 0x%"PRIx64" value 0x%x", port, offset, value); return value; } static uint64_t pci_ahci_read(struct pci_devinst *pi, int baridx, uint64_t regoff, int size) { struct pci_ahci_softc *sc = pi->pi_arg; uint64_t offset; uint32_t value; assert(baridx == 5); assert(size == 1 || size == 2 || size == 4); assert((regoff & (size - 1)) == 0); pthread_mutex_lock(&sc->mtx); offset = regoff & ~0x3; /* round down to a multiple of 4 bytes */ if (offset < AHCI_OFFSET) value = pci_ahci_host_read(sc, offset); else if (offset < (uint64_t)AHCI_OFFSET + sc->ports * AHCI_STEP) value = pci_ahci_port_read(sc, offset); else { value = 0; EPRINTLN("pci_ahci: unknown i/o read offset 0x%"PRIx64"", regoff); } value >>= 8 * (regoff & 0x3); pthread_mutex_unlock(&sc->mtx); return (value); } /* * Each AHCI controller has a "port" node which contains nodes for * each port named after the decimal number of the port (no leading * zeroes). Port nodes contain a "type" ("hd" or "cd"), as well as * options for blockif. For example: * * pci.0.1.0 * .device="ahci" * .port * .0 * .type="hd" * .path="/path/to/image" */ static int pci_ahci_legacy_config_port(nvlist_t *nvl, int port, const char *type, const char *opts) { char node_name[sizeof("XX")]; nvlist_t *port_nvl; snprintf(node_name, sizeof(node_name), "%d", port); port_nvl = create_relative_config_node(nvl, node_name); set_config_value_node(port_nvl, "type", type); return (blockif_legacy_config(port_nvl, opts)); } static int pci_ahci_legacy_config(nvlist_t *nvl, const char *opts) { nvlist_t *ports_nvl; const char *type; char *next, *next2, *str, *tofree; int p, ret; if (opts == NULL) return (0); ports_nvl = create_relative_config_node(nvl, "port"); ret = 1; tofree = str = strdup(opts); for (p = 0; p < MAX_PORTS && str != NULL; p++, str = next) { /* Identify and cut off type of present port. */ if (strncmp(str, "hd:", 3) == 0) { type = "hd"; str += 3; } else if (strncmp(str, "cd:", 3) == 0) { type = "cd"; str += 3; } else type = NULL; /* Find and cut off the next port options. */ next = strstr(str, ",hd:"); next2 = strstr(str, ",cd:"); if (next == NULL || (next2 != NULL && next2 < next)) next = next2; if (next != NULL) { next[0] = 0; next++; } if (str[0] == 0) continue; if (type == NULL) { EPRINTLN("Missing or invalid type for port %d: \"%s\"", p, str); goto out; } if (pci_ahci_legacy_config_port(ports_nvl, p, type, str) != 0) goto out; } ret = 0; out: free(tofree); return (ret); } static int pci_ahci_cd_legacy_config(nvlist_t *nvl, const char *opts) { nvlist_t *ports_nvl; ports_nvl = create_relative_config_node(nvl, "port"); return (pci_ahci_legacy_config_port(ports_nvl, 0, "cd", opts)); } static int pci_ahci_hd_legacy_config(nvlist_t *nvl, const char *opts) { nvlist_t *ports_nvl; ports_nvl = create_relative_config_node(nvl, "port"); return (pci_ahci_legacy_config_port(ports_nvl, 0, "hd", opts)); } static int pci_ahci_init(struct pci_devinst *pi, nvlist_t *nvl) { char bident[sizeof("XXX:XXX:XXX")]; char node_name[sizeof("XX")]; struct blockif_ctxt *bctxt; struct pci_ahci_softc *sc; int atapi, ret, slots, p; MD5_CTX mdctx; u_char digest[16]; const char *path, *type, *value; nvlist_t *ports_nvl, *port_nvl; ret = 0; #ifdef AHCI_DEBUG dbg = fopen("/tmp/log", "w+"); #endif sc = calloc(1, sizeof(struct pci_ahci_softc)); pi->pi_arg = sc; sc->asc_pi = pi; pthread_mutex_init(&sc->mtx, NULL); sc->ports = 0; sc->pi = 0; slots = 32; ports_nvl = find_relative_config_node(nvl, "port"); for (p = 0; ports_nvl != NULL && p < MAX_PORTS; p++) { struct ata_params *ata_ident = &sc->port[p].ata_ident; char ident[AHCI_PORT_IDENT]; snprintf(node_name, sizeof(node_name), "%d", p); port_nvl = find_relative_config_node(ports_nvl, node_name); if (port_nvl == NULL) continue; type = get_config_value_node(port_nvl, "type"); if (type == NULL) continue; if (strcmp(type, "hd") == 0) atapi = 0; else atapi = 1; /* * Attempt to open the backing image. Use the PCI slot/func * and the port number for the identifier string. */ snprintf(bident, sizeof(bident), "%u:%u:%u", pi->pi_slot, pi->pi_func, p); bctxt = blockif_open(port_nvl, bident); if (bctxt == NULL) { sc->ports = p; ret = 1; goto open_fail; } ret = blockif_add_boot_device(pi, bctxt); if (ret) { sc->ports = p; goto open_fail; } sc->port[p].bctx = bctxt; sc->port[p].pr_sc = sc; sc->port[p].port = p; sc->port[p].atapi = atapi; /* * Create an identifier for the backing file. * Use parts of the md5 sum of the filename */ path = get_config_value_node(port_nvl, "path"); MD5Init(&mdctx); MD5Update(&mdctx, path, strlen(path)); MD5Final(digest, &mdctx); snprintf(ident, AHCI_PORT_IDENT, "BHYVE-%02X%02X-%02X%02X-%02X%02X", digest[0], digest[1], digest[2], digest[3], digest[4], digest[5]); memset(ata_ident, 0, sizeof(struct ata_params)); ata_string((uint8_t*)&ata_ident->serial, ident, 20); ata_string((uint8_t*)&ata_ident->revision, "001", 8); if (atapi) ata_string((uint8_t*)&ata_ident->model, "BHYVE SATA DVD ROM", 40); else ata_string((uint8_t*)&ata_ident->model, "BHYVE SATA DISK", 40); value = get_config_value_node(port_nvl, "nmrr"); if (value != NULL) ata_ident->media_rotation_rate = atoi(value); value = get_config_value_node(port_nvl, "ser"); if (value != NULL) ata_string((uint8_t*)(&ata_ident->serial), value, 20); value = get_config_value_node(port_nvl, "rev"); if (value != NULL) ata_string((uint8_t*)(&ata_ident->revision), value, 8); value = get_config_value_node(port_nvl, "model"); if (value != NULL) ata_string((uint8_t*)(&ata_ident->model), value, 40); ata_identify_init(&sc->port[p], atapi); #ifndef __FreeBSD__ /* * Attempt to enable the write cache for this device, as the * guest will issue FLUSH commands when it requires durability. * * Failure here is fine, since an always-sync device will not * have an impact on correctness. */ (void) blockif_set_wce(bctxt, 1); #endif /* * Allocate blockif request structures and add them * to the free list */ pci_ahci_ioreq_init(&sc->port[p]); sc->pi |= (1 << p); if (sc->port[p].ioqsz < slots) slots = sc->port[p].ioqsz; } sc->ports = p; /* Intel ICH8 AHCI */ --slots; if (sc->ports < DEF_PORTS) sc->ports = DEF_PORTS; sc->cap = AHCI_CAP_64BIT | AHCI_CAP_SNCQ | AHCI_CAP_SSNTF | AHCI_CAP_SMPS | AHCI_CAP_SSS | AHCI_CAP_SALP | AHCI_CAP_SAL | AHCI_CAP_SCLO | (0x3 << AHCI_CAP_ISS_SHIFT)| AHCI_CAP_PMD | AHCI_CAP_SSC | AHCI_CAP_PSC | (slots << AHCI_CAP_NCS_SHIFT) | AHCI_CAP_SXS | (sc->ports - 1); sc->vs = 0x10300; sc->cap2 = AHCI_CAP2_APST; ahci_reset(sc); pci_set_cfgdata16(pi, PCIR_DEVICE, 0x2821); pci_set_cfgdata16(pi, PCIR_VENDOR, 0x8086); pci_set_cfgdata8(pi, PCIR_CLASS, PCIC_STORAGE); pci_set_cfgdata8(pi, PCIR_SUBCLASS, PCIS_STORAGE_SATA); pci_set_cfgdata8(pi, PCIR_PROGIF, PCIP_STORAGE_SATA_AHCI_1_0); p = MIN(sc->ports, 16); p = flsl(p) - ((p & (p - 1)) ? 0 : 1); pci_emul_add_msicap(pi, 1 << p); pci_emul_alloc_bar(pi, 5, PCIBAR_MEM32, AHCI_OFFSET + sc->ports * AHCI_STEP); pci_lintr_request(pi); open_fail: if (ret) { for (p = 0; p < sc->ports; p++) { if (sc->port[p].bctx != NULL) blockif_close(sc->port[p].bctx); } free(sc); } return (ret); } /* * Use separate emulation names to distinguish drive and atapi devices */ static const struct pci_devemu pci_de_ahci = { .pe_emu = "ahci", .pe_init = pci_ahci_init, .pe_legacy_config = pci_ahci_legacy_config, .pe_barwrite = pci_ahci_write, .pe_barread = pci_ahci_read, }; PCI_EMUL_SET(pci_de_ahci); static const struct pci_devemu pci_de_ahci_hd = { .pe_emu = "ahci-hd", .pe_legacy_config = pci_ahci_hd_legacy_config, .pe_alias = "ahci", }; PCI_EMUL_SET(pci_de_ahci_hd); static const struct pci_devemu pci_de_ahci_cd = { .pe_emu = "ahci-cd", .pe_legacy_config = pci_ahci_cd_legacy_config, .pe_alias = "ahci", }; PCI_EMUL_SET(pci_de_ahci_cd); /* * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2016 Alexander Motin * Copyright (c) 2015 Peter Grehan * Copyright (c) 2013 Jeremiah Lott, Avere Systems * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer * in this position and unchanged. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include #ifndef WITHOUT_CAPSICUM #include #endif #include #include #include #include #include #include #ifndef WITHOUT_CAPSICUM #include #endif #include #include #include #include #include #include #include #include #include #include #include #include "e1000_regs.h" #include "e1000_defines.h" #include "mii.h" #include "bhyverun.h" #include "config.h" #include "debug.h" #include "pci_emul.h" #include "mevent.h" #include "net_utils.h" #include "net_backends.h" /* Hardware/register definitions XXX: move some to common code. */ #define E82545_VENDOR_ID_INTEL 0x8086 #define E82545_DEV_ID_82545EM_COPPER 0x100F #define E82545_SUBDEV_ID 0x1008 #define E82545_REVISION_4 4 #define E82545_MDIC_DATA_MASK 0x0000FFFF #define E82545_MDIC_OP_MASK 0x0c000000 #define E82545_MDIC_IE 0x20000000 #define E82545_EECD_FWE_DIS 0x00000010 /* Flash writes disabled */ #define E82545_EECD_FWE_EN 0x00000020 /* Flash writes enabled */ #define E82545_EECD_FWE_MASK 0x00000030 /* Flash writes mask */ #define E82545_BAR_REGISTER 0 #define E82545_BAR_REGISTER_LEN (128*1024) #define E82545_BAR_FLASH 1 #define E82545_BAR_FLASH_LEN (64*1024) #define E82545_BAR_IO 2 #define E82545_BAR_IO_LEN 8 #define E82545_IOADDR 0x00000000 #define E82545_IODATA 0x00000004 #define E82545_IO_REGISTER_MAX 0x0001FFFF #define E82545_IO_FLASH_BASE 0x00080000 #define E82545_IO_FLASH_MAX 0x000FFFFF #define E82545_ARRAY_ENTRY(reg, offset) (reg + (offset<<2)) #define E82545_RAR_MAX 15 #define E82545_MTA_MAX 127 #define E82545_VFTA_MAX 127 /* Slightly modified from the driver versions, hardcoded for 3 opcode bits, * followed by 6 address bits. * TODO: make opcode bits and addr bits configurable? * NVM Commands - Microwire */ #define E82545_NVM_OPCODE_BITS 3 #define E82545_NVM_ADDR_BITS 6 #define E82545_NVM_DATA_BITS 16 #define E82545_NVM_OPADDR_BITS (E82545_NVM_OPCODE_BITS + E82545_NVM_ADDR_BITS) #define E82545_NVM_ADDR_MASK ((1 << E82545_NVM_ADDR_BITS)-1) #define E82545_NVM_OPCODE_MASK \ (((1 << E82545_NVM_OPCODE_BITS) - 1) << E82545_NVM_ADDR_BITS) #define E82545_NVM_OPCODE_READ (0x6 << E82545_NVM_ADDR_BITS) /* read */ #define E82545_NVM_OPCODE_WRITE (0x5 << E82545_NVM_ADDR_BITS) /* write */ #define E82545_NVM_OPCODE_ERASE (0x7 << E82545_NVM_ADDR_BITS) /* erase */ #define E82545_NVM_OPCODE_EWEN (0x4 << E82545_NVM_ADDR_BITS) /* wr-enable */ #define E82545_NVM_EEPROM_SIZE 64 /* 64 * 16-bit values == 128K */ #define E1000_ICR_SRPD 0x00010000 /* This is an arbitrary number. There is no hard limit on the chip. */ #define I82545_MAX_TXSEGS 64 /* Legacy receive descriptor */ struct e1000_rx_desc { uint64_t buffer_addr; /* Address of the descriptor's data buffer */ uint16_t length; /* Length of data DMAed into data buffer */ uint16_t csum; /* Packet checksum */ uint8_t status; /* Descriptor status */ uint8_t errors; /* Descriptor Errors */ uint16_t special; }; /* Transmit descriptor types */ #define E1000_TXD_MASK (E1000_TXD_CMD_DEXT | 0x00F00000) #define E1000_TXD_TYP_L (0) #define E1000_TXD_TYP_C (E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_C) #define E1000_TXD_TYP_D (E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D) /* Legacy transmit descriptor */ struct e1000_tx_desc { uint64_t buffer_addr; /* Address of the descriptor's data buffer */ union { uint32_t data; struct { uint16_t length; /* Data buffer length */ uint8_t cso; /* Checksum offset */ uint8_t cmd; /* Descriptor control */ } flags; } lower; union { uint32_t data; struct { uint8_t status; /* Descriptor status */ uint8_t css; /* Checksum start */ uint16_t special; } fields; } upper; }; /* Context descriptor */ struct e1000_context_desc { union { uint32_t ip_config; struct { uint8_t ipcss; /* IP checksum start */ uint8_t ipcso; /* IP checksum offset */ uint16_t ipcse; /* IP checksum end */ } ip_fields; } lower_setup; union { uint32_t tcp_config; struct { uint8_t tucss; /* TCP checksum start */ uint8_t tucso; /* TCP checksum offset */ uint16_t tucse; /* TCP checksum end */ } tcp_fields; } upper_setup; uint32_t cmd_and_length; union { uint32_t data; struct { uint8_t status; /* Descriptor status */ uint8_t hdr_len; /* Header length */ uint16_t mss; /* Maximum segment size */ } fields; } tcp_seg_setup; }; /* Data descriptor */ struct e1000_data_desc { uint64_t buffer_addr; /* Address of the descriptor's buffer address */ union { uint32_t data; struct { uint16_t length; /* Data buffer length */ uint8_t typ_len_ext; uint8_t cmd; } flags; } lower; union { uint32_t data; struct { uint8_t status; /* Descriptor status */ uint8_t popts; /* Packet Options */ uint16_t special; } fields; } upper; }; union e1000_tx_udesc { struct e1000_tx_desc td; struct e1000_context_desc cd; struct e1000_data_desc dd; }; /* Tx checksum info for a packet. */ struct ck_info { int ck_valid; /* ck_info is valid */ uint8_t ck_start; /* start byte of cksum calcuation */ uint8_t ck_off; /* offset of cksum insertion */ uint16_t ck_len; /* length of cksum calc: 0 is to packet-end */ }; /* * Debug printf */ static int e82545_debug = 0; #define WPRINTF(msg,params...) PRINTLN("e82545: " msg, ##params) #define DPRINTF(msg,params...) if (e82545_debug) WPRINTF(msg, params) #define MIN(a,b) (((a)<(b))?(a):(b)) #define MAX(a,b) (((a)>(b))?(a):(b)) /* s/w representation of the RAL/RAH regs */ struct eth_uni { int eu_valid; int eu_addrsel; struct ether_addr eu_eth; }; struct e82545_softc { struct pci_devinst *esc_pi; struct vmctx *esc_ctx; struct mevent *esc_mevpitr; pthread_mutex_t esc_mtx; struct ether_addr esc_mac; net_backend_t *esc_be; /* General */ uint32_t esc_CTRL; /* x0000 device ctl */ uint32_t esc_FCAL; /* x0028 flow ctl addr lo */ uint32_t esc_FCAH; /* x002C flow ctl addr hi */ uint32_t esc_FCT; /* x0030 flow ctl type */ uint32_t esc_VET; /* x0038 VLAN eth type */ uint32_t esc_FCTTV; /* x0170 flow ctl tx timer */ uint32_t esc_LEDCTL; /* x0E00 LED control */ uint32_t esc_PBA; /* x1000 pkt buffer allocation */ /* Interrupt control */ int esc_irq_asserted; uint32_t esc_ICR; /* x00C0 cause read/clear */ uint32_t esc_ITR; /* x00C4 intr throttling */ uint32_t esc_ICS; /* x00C8 cause set */ uint32_t esc_IMS; /* x00D0 mask set/read */ uint32_t esc_IMC; /* x00D8 mask clear */ /* Transmit */ union e1000_tx_udesc *esc_txdesc; struct e1000_context_desc esc_txctx; pthread_t esc_tx_tid; pthread_cond_t esc_tx_cond; int esc_tx_enabled; int esc_tx_active; uint32_t esc_TXCW; /* x0178 transmit config */ uint32_t esc_TCTL; /* x0400 transmit ctl */ uint32_t esc_TIPG; /* x0410 inter-packet gap */ uint16_t esc_AIT; /* x0458 Adaptive Interframe Throttle */ uint64_t esc_tdba; /* verified 64-bit desc table addr */ uint32_t esc_TDBAL; /* x3800 desc table addr, low bits */ uint32_t esc_TDBAH; /* x3804 desc table addr, hi 32-bits */ uint32_t esc_TDLEN; /* x3808 # descriptors in bytes */ uint16_t esc_TDH; /* x3810 desc table head idx */ uint16_t esc_TDHr; /* internal read version of TDH */ uint16_t esc_TDT; /* x3818 desc table tail idx */ uint32_t esc_TIDV; /* x3820 intr delay */ uint32_t esc_TXDCTL; /* x3828 desc control */ uint32_t esc_TADV; /* x382C intr absolute delay */ /* L2 frame acceptance */ struct eth_uni esc_uni[16]; /* 16 x unicast MAC addresses */ uint32_t esc_fmcast[128]; /* Multicast filter bit-match */ uint32_t esc_fvlan[128]; /* VLAN 4096-bit filter */ /* Receive */ struct e1000_rx_desc *esc_rxdesc; pthread_cond_t esc_rx_cond; int esc_rx_enabled; int esc_rx_active; int esc_rx_loopback; uint32_t esc_RCTL; /* x0100 receive ctl */ uint32_t esc_FCRTL; /* x2160 flow cntl thresh, low */ uint32_t esc_FCRTH; /* x2168 flow cntl thresh, hi */ uint64_t esc_rdba; /* verified 64-bit desc table addr */ uint32_t esc_RDBAL; /* x2800 desc table addr, low bits */ uint32_t esc_RDBAH; /* x2804 desc table addr, hi 32-bits*/ uint32_t esc_RDLEN; /* x2808 #descriptors */ uint16_t esc_RDH; /* x2810 desc table head idx */ uint16_t esc_RDT; /* x2818 desc table tail idx */ uint32_t esc_RDTR; /* x2820 intr delay */ uint32_t esc_RXDCTL; /* x2828 desc control */ uint32_t esc_RADV; /* x282C intr absolute delay */ uint32_t esc_RSRPD; /* x2C00 recv small packet detect */ uint32_t esc_RXCSUM; /* x5000 receive cksum ctl */ /* IO Port register access */ uint32_t io_addr; /* Shadow copy of MDIC */ uint32_t mdi_control; /* Shadow copy of EECD */ uint32_t eeprom_control; /* Latest NVM in/out */ uint16_t nvm_data; uint16_t nvm_opaddr; /* stats */ uint32_t missed_pkt_count; /* dropped for no room in rx queue */ uint32_t pkt_rx_by_size[6]; uint32_t pkt_tx_by_size[6]; uint32_t good_pkt_rx_count; uint32_t bcast_pkt_rx_count; uint32_t mcast_pkt_rx_count; uint32_t good_pkt_tx_count; uint32_t bcast_pkt_tx_count; uint32_t mcast_pkt_tx_count; uint32_t oversize_rx_count; uint32_t tso_tx_count; uint64_t good_octets_rx; uint64_t good_octets_tx; uint64_t missed_octets; /* counts missed and oversized */ uint8_t nvm_bits:6; /* number of bits remaining in/out */ uint8_t nvm_mode:2; #define E82545_NVM_MODE_OPADDR 0x0 #define E82545_NVM_MODE_DATAIN 0x1 #define E82545_NVM_MODE_DATAOUT 0x2 /* EEPROM data */ uint16_t eeprom_data[E82545_NVM_EEPROM_SIZE]; }; static void e82545_reset(struct e82545_softc *sc, int dev); static void e82545_rx_enable(struct e82545_softc *sc); static void e82545_rx_disable(struct e82545_softc *sc); static void e82545_rx_callback(int fd, enum ev_type type, void *param); static void e82545_tx_start(struct e82545_softc *sc); static void e82545_tx_enable(struct e82545_softc *sc); static void e82545_tx_disable(struct e82545_softc *sc); static inline int __unused e82545_size_stat_index(uint32_t size) { if (size <= 64) { return 0; } else if (size >= 1024) { return 5; } else { /* should be 1-4 */ return (ffs(size) - 6); } } static void e82545_init_eeprom(struct e82545_softc *sc) { uint16_t checksum, i; /* mac addr */ sc->eeprom_data[NVM_MAC_ADDR] = ((uint16_t)sc->esc_mac.octet[0]) | (((uint16_t)sc->esc_mac.octet[1]) << 8); sc->eeprom_data[NVM_MAC_ADDR+1] = ((uint16_t)sc->esc_mac.octet[2]) | (((uint16_t)sc->esc_mac.octet[3]) << 8); sc->eeprom_data[NVM_MAC_ADDR+2] = ((uint16_t)sc->esc_mac.octet[4]) | (((uint16_t)sc->esc_mac.octet[5]) << 8); /* pci ids */ sc->eeprom_data[NVM_SUB_DEV_ID] = E82545_SUBDEV_ID; sc->eeprom_data[NVM_SUB_VEN_ID] = E82545_VENDOR_ID_INTEL; sc->eeprom_data[NVM_DEV_ID] = E82545_DEV_ID_82545EM_COPPER; sc->eeprom_data[NVM_VEN_ID] = E82545_VENDOR_ID_INTEL; /* fill in the checksum */ checksum = 0; for (i = 0; i < NVM_CHECKSUM_REG; i++) { checksum += sc->eeprom_data[i]; } checksum = NVM_SUM - checksum; sc->eeprom_data[NVM_CHECKSUM_REG] = checksum; DPRINTF("eeprom checksum: 0x%x", checksum); } static void e82545_write_mdi(struct e82545_softc *sc __unused, uint8_t reg_addr, uint8_t phy_addr, uint32_t data) { DPRINTF("Write mdi reg:0x%x phy:0x%x data: 0x%x", reg_addr, phy_addr, data); } static uint32_t e82545_read_mdi(struct e82545_softc *sc __unused, uint8_t reg_addr, uint8_t phy_addr) { //DPRINTF("Read mdi reg:0x%x phy:0x%x", reg_addr, phy_addr); switch (reg_addr) { case PHY_STATUS: return (MII_SR_LINK_STATUS | MII_SR_AUTONEG_CAPS | MII_SR_AUTONEG_COMPLETE); case PHY_AUTONEG_ADV: return NWAY_AR_SELECTOR_FIELD; case PHY_LP_ABILITY: return 0; case PHY_1000T_STATUS: return (SR_1000T_LP_FD_CAPS | SR_1000T_REMOTE_RX_STATUS | SR_1000T_LOCAL_RX_STATUS); case PHY_ID1: return (M88E1011_I_PHY_ID >> 16) & 0xFFFF; case PHY_ID2: return (M88E1011_I_PHY_ID | E82545_REVISION_4) & 0xFFFF; default: DPRINTF("Unknown mdi read reg:0x%x phy:0x%x", reg_addr, phy_addr); return 0; } /* not reached */ } static void e82545_eecd_strobe(struct e82545_softc *sc) { /* Microwire state machine */ /* DPRINTF("eeprom state machine srtobe " "0x%x 0x%x 0x%x 0x%x", sc->nvm_mode, sc->nvm_bits, sc->nvm_opaddr, sc->nvm_data);*/ if (sc->nvm_bits == 0) { DPRINTF("eeprom state machine not expecting data! " "0x%x 0x%x 0x%x 0x%x", sc->nvm_mode, sc->nvm_bits, sc->nvm_opaddr, sc->nvm_data); return; } sc->nvm_bits--; if (sc->nvm_mode == E82545_NVM_MODE_DATAOUT) { /* shifting out */ if (sc->nvm_data & 0x8000) { sc->eeprom_control |= E1000_EECD_DO; } else { sc->eeprom_control &= ~E1000_EECD_DO; } sc->nvm_data <<= 1; if (sc->nvm_bits == 0) { /* read done, back to opcode mode. */ sc->nvm_opaddr = 0; sc->nvm_mode = E82545_NVM_MODE_OPADDR; sc->nvm_bits = E82545_NVM_OPADDR_BITS; } } else if (sc->nvm_mode == E82545_NVM_MODE_DATAIN) { /* shifting in */ sc->nvm_data <<= 1; if (sc->eeprom_control & E1000_EECD_DI) { sc->nvm_data |= 1; } if (sc->nvm_bits == 0) { /* eeprom write */ uint16_t op = sc->nvm_opaddr & E82545_NVM_OPCODE_MASK; uint16_t addr = sc->nvm_opaddr & E82545_NVM_ADDR_MASK; if (op != E82545_NVM_OPCODE_WRITE) { DPRINTF("Illegal eeprom write op 0x%x", sc->nvm_opaddr); } else if (addr >= E82545_NVM_EEPROM_SIZE) { DPRINTF("Illegal eeprom write addr 0x%x", sc->nvm_opaddr); } else { DPRINTF("eeprom write eeprom[0x%x] = 0x%x", addr, sc->nvm_data); sc->eeprom_data[addr] = sc->nvm_data; } /* back to opcode mode */ sc->nvm_opaddr = 0; sc->nvm_mode = E82545_NVM_MODE_OPADDR; sc->nvm_bits = E82545_NVM_OPADDR_BITS; } } else if (sc->nvm_mode == E82545_NVM_MODE_OPADDR) { sc->nvm_opaddr <<= 1; if (sc->eeprom_control & E1000_EECD_DI) { sc->nvm_opaddr |= 1; } if (sc->nvm_bits == 0) { uint16_t op = sc->nvm_opaddr & E82545_NVM_OPCODE_MASK; switch (op) { case E82545_NVM_OPCODE_EWEN: DPRINTF("eeprom write enable: 0x%x", sc->nvm_opaddr); /* back to opcode mode */ sc->nvm_opaddr = 0; sc->nvm_mode = E82545_NVM_MODE_OPADDR; sc->nvm_bits = E82545_NVM_OPADDR_BITS; break; case E82545_NVM_OPCODE_READ: { uint16_t addr = sc->nvm_opaddr & E82545_NVM_ADDR_MASK; sc->nvm_mode = E82545_NVM_MODE_DATAOUT; sc->nvm_bits = E82545_NVM_DATA_BITS; if (addr < E82545_NVM_EEPROM_SIZE) { sc->nvm_data = sc->eeprom_data[addr]; DPRINTF("eeprom read: eeprom[0x%x] = 0x%x", addr, sc->nvm_data); } else { DPRINTF("eeprom illegal read: 0x%x", sc->nvm_opaddr); sc->nvm_data = 0; } break; } case E82545_NVM_OPCODE_WRITE: sc->nvm_mode = E82545_NVM_MODE_DATAIN; sc->nvm_bits = E82545_NVM_DATA_BITS; sc->nvm_data = 0; break; default: DPRINTF("eeprom unknown op: 0x%x", sc->nvm_opaddr); /* back to opcode mode */ sc->nvm_opaddr = 0; sc->nvm_mode = E82545_NVM_MODE_OPADDR; sc->nvm_bits = E82545_NVM_OPADDR_BITS; } } } else { DPRINTF("eeprom state machine wrong state! " "0x%x 0x%x 0x%x 0x%x", sc->nvm_mode, sc->nvm_bits, sc->nvm_opaddr, sc->nvm_data); } } static void e82545_itr_callback(int fd __unused, enum ev_type type __unused, void *param) { uint32_t new; struct e82545_softc *sc = param; pthread_mutex_lock(&sc->esc_mtx); new = sc->esc_ICR & sc->esc_IMS; if (new && !sc->esc_irq_asserted) { DPRINTF("itr callback: lintr assert %x", new); sc->esc_irq_asserted = 1; pci_lintr_assert(sc->esc_pi); } else { mevent_delete(sc->esc_mevpitr); sc->esc_mevpitr = NULL; } pthread_mutex_unlock(&sc->esc_mtx); } static void e82545_icr_assert(struct e82545_softc *sc, uint32_t bits) { uint32_t new; DPRINTF("icr assert: 0x%x", bits); /* * An interrupt is only generated if bits are set that * aren't already in the ICR, these bits are unmasked, * and there isn't an interrupt already pending. */ new = bits & ~sc->esc_ICR & sc->esc_IMS; sc->esc_ICR |= bits; if (new == 0) { DPRINTF("icr assert: masked %x, ims %x", new, sc->esc_IMS); } else if (sc->esc_mevpitr != NULL) { DPRINTF("icr assert: throttled %x, ims %x", new, sc->esc_IMS); } else if (!sc->esc_irq_asserted) { DPRINTF("icr assert: lintr assert %x", new); sc->esc_irq_asserted = 1; pci_lintr_assert(sc->esc_pi); if (sc->esc_ITR != 0) { sc->esc_mevpitr = mevent_add( (sc->esc_ITR + 3905) / 3906, /* 256ns -> 1ms */ EVF_TIMER, e82545_itr_callback, sc); } } } static void e82545_ims_change(struct e82545_softc *sc, uint32_t bits) { uint32_t new; /* * Changing the mask may allow previously asserted * but masked interrupt requests to generate an interrupt. */ new = bits & sc->esc_ICR & ~sc->esc_IMS; sc->esc_IMS |= bits; if (new == 0) { DPRINTF("ims change: masked %x, ims %x", new, sc->esc_IMS); } else if (sc->esc_mevpitr != NULL) { DPRINTF("ims change: throttled %x, ims %x", new, sc->esc_IMS); } else if (!sc->esc_irq_asserted) { DPRINTF("ims change: lintr assert %x", new); sc->esc_irq_asserted = 1; pci_lintr_assert(sc->esc_pi); if (sc->esc_ITR != 0) { sc->esc_mevpitr = mevent_add( (sc->esc_ITR + 3905) / 3906, /* 256ns -> 1ms */ EVF_TIMER, e82545_itr_callback, sc); } } } static void e82545_icr_deassert(struct e82545_softc *sc, uint32_t bits) { DPRINTF("icr deassert: 0x%x", bits); sc->esc_ICR &= ~bits; /* * If there are no longer any interrupt sources and there * was an asserted interrupt, clear it */ if (sc->esc_irq_asserted && !(sc->esc_ICR & sc->esc_IMS)) { DPRINTF("icr deassert: lintr deassert %x", bits); pci_lintr_deassert(sc->esc_pi); sc->esc_irq_asserted = 0; } } static void e82545_intr_write(struct e82545_softc *sc, uint32_t offset, uint32_t value) { DPRINTF("intr_write: off %x, val %x", offset, value); switch (offset) { case E1000_ICR: e82545_icr_deassert(sc, value); break; case E1000_ITR: sc->esc_ITR = value; break; case E1000_ICS: sc->esc_ICS = value; /* not used: store for debug */ e82545_icr_assert(sc, value); break; case E1000_IMS: e82545_ims_change(sc, value); break; case E1000_IMC: sc->esc_IMC = value; /* for debug */ sc->esc_IMS &= ~value; // XXX clear interrupts if all ICR bits now masked // and interrupt was pending ? break; default: break; } } static uint32_t e82545_intr_read(struct e82545_softc *sc, uint32_t offset) { uint32_t retval; retval = 0; DPRINTF("intr_read: off %x", offset); switch (offset) { case E1000_ICR: retval = sc->esc_ICR; sc->esc_ICR = 0; e82545_icr_deassert(sc, ~0); break; case E1000_ITR: retval = sc->esc_ITR; break; case E1000_ICS: /* write-only register */ break; case E1000_IMS: retval = sc->esc_IMS; break; case E1000_IMC: /* write-only register */ break; default: break; } return (retval); } static void e82545_devctl(struct e82545_softc *sc, uint32_t val) { sc->esc_CTRL = val & ~E1000_CTRL_RST; if (val & E1000_CTRL_RST) { DPRINTF("e1k: s/w reset, ctl %x", val); e82545_reset(sc, 1); } /* XXX check for phy reset ? */ } static void e82545_rx_update_rdba(struct e82545_softc *sc) { /* XXX verify desc base/len within phys mem range */ sc->esc_rdba = (uint64_t)sc->esc_RDBAH << 32 | sc->esc_RDBAL; /* Cache host mapping of guest descriptor array */ sc->esc_rxdesc = paddr_guest2host(sc->esc_ctx, sc->esc_rdba, sc->esc_RDLEN); } static void e82545_rx_ctl(struct e82545_softc *sc, uint32_t val) { int on; on = ((val & E1000_RCTL_EN) == E1000_RCTL_EN); /* Save RCTL after stripping reserved bits 31:27,24,21,14,11:10,0 */ sc->esc_RCTL = val & ~0xF9204c01; DPRINTF("rx_ctl - %s RCTL %x, val %x", on ? "on" : "off", sc->esc_RCTL, val); /* state change requested */ if (on != sc->esc_rx_enabled) { if (on) { /* Catch disallowed/unimplemented settings */ //assert(!(val & E1000_RCTL_LBM_TCVR)); if (sc->esc_RCTL & E1000_RCTL_LBM_TCVR) { sc->esc_rx_loopback = 1; } else { sc->esc_rx_loopback = 0; } e82545_rx_update_rdba(sc); e82545_rx_enable(sc); } else { e82545_rx_disable(sc); sc->esc_rx_loopback = 0; sc->esc_rdba = 0; sc->esc_rxdesc = NULL; } } } static void e82545_tx_update_tdba(struct e82545_softc *sc) { /* XXX verify desc base/len within phys mem range */ sc->esc_tdba = (uint64_t)sc->esc_TDBAH << 32 | sc->esc_TDBAL; /* Cache host mapping of guest descriptor array */ sc->esc_txdesc = paddr_guest2host(sc->esc_ctx, sc->esc_tdba, sc->esc_TDLEN); } static void e82545_tx_ctl(struct e82545_softc *sc, uint32_t val) { int on; on = ((val & E1000_TCTL_EN) == E1000_TCTL_EN); /* ignore TCTL_EN settings that don't change state */ if (on == sc->esc_tx_enabled) return; if (on) { e82545_tx_update_tdba(sc); e82545_tx_enable(sc); } else { e82545_tx_disable(sc); sc->esc_tdba = 0; sc->esc_txdesc = NULL; } /* Save TCTL value after stripping reserved bits 31:25,23,2,0 */ sc->esc_TCTL = val & ~0xFE800005; } static int e82545_bufsz(uint32_t rctl) { switch (rctl & (E1000_RCTL_BSEX | E1000_RCTL_SZ_256)) { case (E1000_RCTL_SZ_2048): return (2048); case (E1000_RCTL_SZ_1024): return (1024); case (E1000_RCTL_SZ_512): return (512); case (E1000_RCTL_SZ_256): return (256); case (E1000_RCTL_BSEX|E1000_RCTL_SZ_16384): return (16384); case (E1000_RCTL_BSEX|E1000_RCTL_SZ_8192): return (8192); case (E1000_RCTL_BSEX|E1000_RCTL_SZ_4096): return (4096); } return (256); /* Forbidden value. */ } /* XXX one packet at a time until this is debugged */ static void e82545_rx_callback(int fd __unused, enum ev_type type __unused, void *param) { struct e82545_softc *sc = param; struct e1000_rx_desc *rxd; struct iovec vec[64]; ssize_t len; int left, lim, maxpktsz, maxpktdesc, bufsz, i, n, size; uint32_t cause = 0; uint16_t *tp, tag, head; pthread_mutex_lock(&sc->esc_mtx); DPRINTF("rx_run: head %x, tail %x", sc->esc_RDH, sc->esc_RDT); if (!sc->esc_rx_enabled || sc->esc_rx_loopback) { DPRINTF("rx disabled (!%d || %d) -- packet(s) dropped", sc->esc_rx_enabled, sc->esc_rx_loopback); while (netbe_rx_discard(sc->esc_be) > 0) { } goto done1; } bufsz = e82545_bufsz(sc->esc_RCTL); maxpktsz = (sc->esc_RCTL & E1000_RCTL_LPE) ? 16384 : 1522; maxpktdesc = (maxpktsz + bufsz - 1) / bufsz; size = sc->esc_RDLEN / 16; head = sc->esc_RDH; left = (size + sc->esc_RDT - head) % size; if (left < maxpktdesc) { DPRINTF("rx overflow (%d < %d) -- packet(s) dropped", left, maxpktdesc); while (netbe_rx_discard(sc->esc_be) > 0) { } goto done1; } sc->esc_rx_active = 1; pthread_mutex_unlock(&sc->esc_mtx); for (lim = size / 4; lim > 0 && left >= maxpktdesc; lim -= n) { /* Grab rx descriptor pointed to by the head pointer */ for (i = 0; i < maxpktdesc; i++) { rxd = &sc->esc_rxdesc[(head + i) % size]; vec[i].iov_base = paddr_guest2host(sc->esc_ctx, rxd->buffer_addr, bufsz); vec[i].iov_len = bufsz; } len = netbe_recv(sc->esc_be, vec, maxpktdesc); if (len <= 0) { DPRINTF("netbe_recv() returned %zd", len); goto done; } /* * Adjust the packet length based on whether the CRC needs * to be stripped or if the packet is less than the minimum * eth packet size. */ if (len < ETHER_MIN_LEN - ETHER_CRC_LEN) len = ETHER_MIN_LEN - ETHER_CRC_LEN; if (!(sc->esc_RCTL & E1000_RCTL_SECRC)) len += ETHER_CRC_LEN; n = (len + bufsz - 1) / bufsz; DPRINTF("packet read %zd bytes, %d segs, head %d", len, n, head); /* Apply VLAN filter. */ tp = (uint16_t *)vec[0].iov_base + 6; if ((sc->esc_RCTL & E1000_RCTL_VFE) && (ntohs(tp[0]) == sc->esc_VET)) { tag = ntohs(tp[1]) & 0x0fff; if ((sc->esc_fvlan[tag >> 5] & (1 << (tag & 0x1f))) != 0) { DPRINTF("known VLAN %d", tag); } else { DPRINTF("unknown VLAN %d", tag); n = 0; continue; } } /* Update all consumed descriptors. */ for (i = 0; i < n - 1; i++) { rxd = &sc->esc_rxdesc[(head + i) % size]; rxd->length = bufsz; rxd->csum = 0; rxd->errors = 0; rxd->special = 0; rxd->status = E1000_RXD_STAT_DD; } rxd = &sc->esc_rxdesc[(head + i) % size]; rxd->length = len % bufsz; rxd->csum = 0; rxd->errors = 0; rxd->special = 0; /* XXX signal no checksum for now */ rxd->status = E1000_RXD_STAT_PIF | E1000_RXD_STAT_IXSM | E1000_RXD_STAT_EOP | E1000_RXD_STAT_DD; /* Schedule receive interrupts. */ if ((uint32_t)len <= sc->esc_RSRPD) { cause |= E1000_ICR_SRPD | E1000_ICR_RXT0; } else { /* XXX: RDRT and RADV timers should be here. */ cause |= E1000_ICR_RXT0; } head = (head + n) % size; left -= n; } done: pthread_mutex_lock(&sc->esc_mtx); sc->esc_rx_active = 0; if (sc->esc_rx_enabled == 0) pthread_cond_signal(&sc->esc_rx_cond); sc->esc_RDH = head; /* Respect E1000_RCTL_RDMTS */ left = (size + sc->esc_RDT - head) % size; if (left < (size >> (((sc->esc_RCTL >> 8) & 3) + 1))) cause |= E1000_ICR_RXDMT0; /* Assert all accumulated interrupts. */ if (cause != 0) e82545_icr_assert(sc, cause); done1: DPRINTF("rx_run done: head %x, tail %x", sc->esc_RDH, sc->esc_RDT); pthread_mutex_unlock(&sc->esc_mtx); } static uint16_t e82545_carry(uint32_t sum) { sum = (sum & 0xFFFF) + (sum >> 16); if (sum > 0xFFFF) sum -= 0xFFFF; return (sum); } static uint16_t e82545_buf_checksum(uint8_t *buf, int len) { int i; uint32_t sum = 0; /* Checksum all the pairs of bytes first... */ for (i = 0; i < (len & ~1); i += 2) sum += *((u_int16_t *)(buf + i)); /* * If there's a single byte left over, checksum it, too. * Network byte order is big-endian, so the remaining byte is * the high byte. */ if (i < len) sum += htons(buf[i] << 8); return (e82545_carry(sum)); } static uint16_t e82545_iov_checksum(struct iovec *iov, int iovcnt, unsigned int off, unsigned int len) { unsigned int now, odd; uint32_t sum = 0, s; /* Skip completely unneeded vectors. */ while (iovcnt > 0 && iov->iov_len <= off && off > 0) { off -= iov->iov_len; iov++; iovcnt--; } /* Calculate checksum of requested range. */ odd = 0; while (len > 0 && iovcnt > 0) { now = MIN(len, iov->iov_len - off); s = e82545_buf_checksum((uint8_t *)iov->iov_base + off, now); sum += odd ? (s << 8) : s; odd ^= (now & 1); len -= now; off = 0; iov++; iovcnt--; } return (e82545_carry(sum)); } /* * Return the transmit descriptor type. */ static int e82545_txdesc_type(uint32_t lower) { int type; type = 0; if (lower & E1000_TXD_CMD_DEXT) type = lower & E1000_TXD_MASK; return (type); } static void e82545_transmit_checksum(struct iovec *iov, int iovcnt, struct ck_info *ck) { uint16_t cksum; unsigned int cklen; DPRINTF("tx cksum: iovcnt/s/off/len %d/%d/%d/%d", iovcnt, ck->ck_start, ck->ck_off, ck->ck_len); cklen = ck->ck_len ? ck->ck_len - ck->ck_start + 1U : UINT_MAX; cksum = e82545_iov_checksum(iov, iovcnt, ck->ck_start, cklen); *(uint16_t *)((uint8_t *)iov[0].iov_base + ck->ck_off) = ~cksum; } static void e82545_transmit_backend(struct e82545_softc *sc, struct iovec *iov, int iovcnt) { if (sc->esc_be == NULL) return; (void) netbe_send(sc->esc_be, iov, iovcnt); } static void e82545_transmit_done(struct e82545_softc *sc, uint16_t head, uint16_t tail, uint16_t dsize, int *tdwb) { union e1000_tx_udesc *dsc; for ( ; head != tail; head = (head + 1) % dsize) { dsc = &sc->esc_txdesc[head]; if (dsc->td.lower.data & E1000_TXD_CMD_RS) { dsc->td.upper.data |= E1000_TXD_STAT_DD; *tdwb = 1; } } } static int e82545_transmit(struct e82545_softc *sc, uint16_t head, uint16_t tail, uint16_t dsize, uint16_t *rhead, int *tdwb) { uint8_t *hdr, *hdrp; struct iovec iovb[I82545_MAX_TXSEGS + 2]; struct iovec tiov[I82545_MAX_TXSEGS + 2]; struct e1000_context_desc *cd; struct ck_info ckinfo[2]; struct iovec *iov; union e1000_tx_udesc *dsc; int desc, dtype, ntype, iovcnt, tcp, tso, paylen, seg, tiovcnt, pv; unsigned hdrlen, vlen, pktlen, len, left, mss, now, nnow, nleft, pvoff; uint32_t tcpsum, tcpseq; uint16_t ipcs, tcpcs, ipid, ohead; bool invalid; ckinfo[0].ck_valid = ckinfo[1].ck_valid = 0; iovcnt = 0; ntype = 0; tso = 0; pktlen = 0; ohead = head; invalid = false; /* iovb[0/1] may be used for writable copy of headers. */ iov = &iovb[2]; for (desc = 0; ; desc++, head = (head + 1) % dsize) { if (head == tail) { *rhead = head; return (0); } dsc = &sc->esc_txdesc[head]; dtype = e82545_txdesc_type(dsc->td.lower.data); if (desc == 0) { switch (dtype) { case E1000_TXD_TYP_C: DPRINTF("tx ctxt desc idx %d: %016jx " "%08x%08x", head, dsc->td.buffer_addr, dsc->td.upper.data, dsc->td.lower.data); /* Save context and return */ sc->esc_txctx = dsc->cd; goto done; case E1000_TXD_TYP_L: DPRINTF("tx legacy desc idx %d: %08x%08x", head, dsc->td.upper.data, dsc->td.lower.data); /* * legacy cksum start valid in first descriptor */ ntype = dtype; ckinfo[0].ck_start = dsc->td.upper.fields.css; break; case E1000_TXD_TYP_D: DPRINTF("tx data desc idx %d: %08x%08x", head, dsc->td.upper.data, dsc->td.lower.data); ntype = dtype; break; default: break; } } else { /* Descriptor type must be consistent */ assert(dtype == ntype); DPRINTF("tx next desc idx %d: %08x%08x", head, dsc->td.upper.data, dsc->td.lower.data); } len = (dtype == E1000_TXD_TYP_L) ? dsc->td.lower.flags.length : dsc->dd.lower.data & 0xFFFFF; /* Strip checksum supplied by guest. */ if ((dsc->td.lower.data & E1000_TXD_CMD_EOP) != 0 && (dsc->td.lower.data & E1000_TXD_CMD_IFCS) == 0) { if (len <= 2) { WPRINTF("final descriptor too short (%d) -- dropped", len); invalid = true; } else len -= 2; } if (len > 0 && iovcnt < I82545_MAX_TXSEGS) { iov[iovcnt].iov_base = paddr_guest2host(sc->esc_ctx, dsc->td.buffer_addr, len); iov[iovcnt].iov_len = len; iovcnt++; pktlen += len; } /* * Pull out info that is valid in the final descriptor * and exit descriptor loop. */ if (dsc->td.lower.data & E1000_TXD_CMD_EOP) { if (dtype == E1000_TXD_TYP_L) { if (dsc->td.lower.data & E1000_TXD_CMD_IC) { ckinfo[0].ck_valid = 1; ckinfo[0].ck_off = dsc->td.lower.flags.cso; ckinfo[0].ck_len = 0; } } else { cd = &sc->esc_txctx; if (dsc->dd.lower.data & E1000_TXD_CMD_TSE) tso = 1; if (dsc->dd.upper.fields.popts & E1000_TXD_POPTS_IXSM) ckinfo[0].ck_valid = 1; if (dsc->dd.upper.fields.popts & E1000_TXD_POPTS_IXSM || tso) { ckinfo[0].ck_start = cd->lower_setup.ip_fields.ipcss; ckinfo[0].ck_off = cd->lower_setup.ip_fields.ipcso; ckinfo[0].ck_len = cd->lower_setup.ip_fields.ipcse; } if (dsc->dd.upper.fields.popts & E1000_TXD_POPTS_TXSM) ckinfo[1].ck_valid = 1; if (dsc->dd.upper.fields.popts & E1000_TXD_POPTS_TXSM || tso) { ckinfo[1].ck_start = cd->upper_setup.tcp_fields.tucss; ckinfo[1].ck_off = cd->upper_setup.tcp_fields.tucso; ckinfo[1].ck_len = cd->upper_setup.tcp_fields.tucse; } } break; } } if (invalid) goto done; if (iovcnt > I82545_MAX_TXSEGS) { WPRINTF("tx too many descriptors (%d > %d) -- dropped", iovcnt, I82545_MAX_TXSEGS); goto done; } hdrlen = vlen = 0; /* Estimate writable space for VLAN header insertion. */ if ((sc->esc_CTRL & E1000_CTRL_VME) && (dsc->td.lower.data & E1000_TXD_CMD_VLE)) { hdrlen = ETHER_ADDR_LEN*2; vlen = ETHER_VLAN_ENCAP_LEN; } if (!tso) { /* Estimate required writable space for checksums. */ if (ckinfo[0].ck_valid) hdrlen = MAX(hdrlen, ckinfo[0].ck_off + 2U); if (ckinfo[1].ck_valid) hdrlen = MAX(hdrlen, ckinfo[1].ck_off + 2U); /* Round up writable space to the first vector. */ if (hdrlen != 0 && iov[0].iov_len > hdrlen && iov[0].iov_len < hdrlen + 100) hdrlen = iov[0].iov_len; } else { /* In case of TSO header length provided by software. */ hdrlen = sc->esc_txctx.tcp_seg_setup.fields.hdr_len; /* * Cap the header length at 240 based on 7.2.4.5 of * the Intel 82576EB (Rev 2.63) datasheet. */ if (hdrlen > 240) { WPRINTF("TSO hdrlen too large: %d", hdrlen); goto done; } /* * If VLAN insertion is requested, ensure the header * at least holds the amount of data copied during * VLAN insertion below. * * XXX: Realistic packets will include a full Ethernet * header before the IP header at ckinfo[0].ck_start, * but this check is sufficient to prevent * out-of-bounds access below. */ if (vlen != 0 && hdrlen < ETHER_ADDR_LEN*2) { WPRINTF("TSO hdrlen too small for vlan insertion " "(%d vs %d) -- dropped", hdrlen, ETHER_ADDR_LEN*2); goto done; } /* * Ensure that the header length covers the used fields * in the IP and TCP headers as well as the IP and TCP * checksums. The following fields are accessed below: * * Header | Field | Offset | Length * -------+-------+--------+------- * IPv4 | len | 2 | 2 * IPv4 | ID | 4 | 2 * IPv6 | len | 4 | 2 * TCP | seq # | 4 | 4 * TCP | flags | 13 | 1 * UDP | len | 4 | 4 */ if (hdrlen < ckinfo[0].ck_start + 6U || hdrlen < ckinfo[0].ck_off + 2U) { WPRINTF("TSO hdrlen too small for IP fields (%d) " "-- dropped", hdrlen); goto done; } if (sc->esc_txctx.cmd_and_length & E1000_TXD_CMD_TCP) { if (hdrlen < ckinfo[1].ck_start + 14U) { WPRINTF("TSO hdrlen too small for TCP fields " "(%d) -- dropped", hdrlen); goto done; } } else { if (hdrlen < ckinfo[1].ck_start + 8U) { WPRINTF("TSO hdrlen too small for UDP fields " "(%d) -- dropped", hdrlen); goto done; } } if (ckinfo[1].ck_valid && hdrlen < ckinfo[1].ck_off + 2U) { WPRINTF("TSO hdrlen too small for TCP/UDP fields " "(%d) -- dropped", hdrlen); goto done; } if (ckinfo[1].ck_valid && hdrlen < ckinfo[1].ck_off + 2) { WPRINTF("TSO hdrlen too small for TCP/UDP fields " "(%d) -- dropped", hdrlen); goto done; } } if (pktlen < hdrlen + vlen) { WPRINTF("packet too small for writable header"); goto done; } /* Allocate, fill and prepend writable header vector. */ if (hdrlen + vlen != 0) { hdr = __builtin_alloca(hdrlen + vlen); hdr += vlen; for (left = hdrlen, hdrp = hdr; left > 0; left -= now, hdrp += now) { now = MIN(left, iov->iov_len); memcpy(hdrp, iov->iov_base, now); #ifdef __FreeBSD__ iov->iov_base = (uint8_t *)iov->iov_base + now; #else /* * The type of iov_base changed in SUS (XPG4v2) from * caddr_t (char * - note signed) to 'void *'. On * illumos, bhyve is not currently compiled with XPG4v2 * or higher, and so we can't cast the RHS to unsigned. * error: pointer targets in assignment differ in * signedness * This also means that we need to apply some casts to * (caddr_t) below. */ iov->iov_base += now; #endif iov->iov_len -= now; if (iov->iov_len == 0) { iov++; iovcnt--; } } iov--; iovcnt++; #ifdef __FreeBSD__ iov->iov_base = hdr; #else iov->iov_base = (caddr_t)hdr; #endif iov->iov_len = hdrlen; } else hdr = NULL; /* Insert VLAN tag. */ if (vlen != 0) { hdr -= ETHER_VLAN_ENCAP_LEN; memmove(hdr, hdr + ETHER_VLAN_ENCAP_LEN, ETHER_ADDR_LEN*2); hdrlen += ETHER_VLAN_ENCAP_LEN; hdr[ETHER_ADDR_LEN*2 + 0] = sc->esc_VET >> 8; hdr[ETHER_ADDR_LEN*2 + 1] = sc->esc_VET & 0xff; hdr[ETHER_ADDR_LEN*2 + 2] = dsc->td.upper.fields.special >> 8; hdr[ETHER_ADDR_LEN*2 + 3] = dsc->td.upper.fields.special & 0xff; #ifdef __FreeBSD__ iov->iov_base = hdr; #else iov->iov_base = (caddr_t)hdr; #endif iov->iov_len += ETHER_VLAN_ENCAP_LEN; /* Correct checksum offsets after VLAN tag insertion. */ ckinfo[0].ck_start += ETHER_VLAN_ENCAP_LEN; ckinfo[0].ck_off += ETHER_VLAN_ENCAP_LEN; if (ckinfo[0].ck_len != 0) ckinfo[0].ck_len += ETHER_VLAN_ENCAP_LEN; ckinfo[1].ck_start += ETHER_VLAN_ENCAP_LEN; ckinfo[1].ck_off += ETHER_VLAN_ENCAP_LEN; if (ckinfo[1].ck_len != 0) ckinfo[1].ck_len += ETHER_VLAN_ENCAP_LEN; } /* Simple non-TSO case. */ if (!tso) { /* Calculate checksums and transmit. */ if (ckinfo[0].ck_valid) e82545_transmit_checksum(iov, iovcnt, &ckinfo[0]); if (ckinfo[1].ck_valid) e82545_transmit_checksum(iov, iovcnt, &ckinfo[1]); e82545_transmit_backend(sc, iov, iovcnt); goto done; } /* Doing TSO. */ tcp = (sc->esc_txctx.cmd_and_length & E1000_TXD_CMD_TCP) != 0; mss = sc->esc_txctx.tcp_seg_setup.fields.mss; paylen = (sc->esc_txctx.cmd_and_length & 0x000fffff); DPRINTF("tx %s segmentation offload %d+%d/%u bytes %d iovs", tcp ? "TCP" : "UDP", hdrlen, paylen, mss, iovcnt); ipid = ntohs(*(uint16_t *)&hdr[ckinfo[0].ck_start + 4]); tcpseq = 0; if (tcp) tcpseq = ntohl(*(uint32_t *)&hdr[ckinfo[1].ck_start + 4]); ipcs = *(uint16_t *)&hdr[ckinfo[0].ck_off]; tcpcs = 0; if (ckinfo[1].ck_valid) /* Save partial pseudo-header checksum. */ tcpcs = *(uint16_t *)&hdr[ckinfo[1].ck_off]; pv = 1; pvoff = 0; for (seg = 0, left = paylen; left > 0; seg++, left -= now) { now = MIN(left, mss); /* Construct IOVs for the segment. */ /* Include whole original header. */ #ifdef __FreeBSD__ tiov[0].iov_base = hdr; #else tiov[0].iov_base = (caddr_t)hdr; #endif tiov[0].iov_len = hdrlen; tiovcnt = 1; /* Include respective part of payload IOV. */ for (nleft = now; pv < iovcnt && nleft > 0; nleft -= nnow) { nnow = MIN(nleft, iov[pv].iov_len - pvoff); #ifdef __FreeBSD__ tiov[tiovcnt].iov_base = (uint8_t *)iov[pv].iov_base + pvoff; #else tiov[tiovcnt].iov_base = iov[pv].iov_base + pvoff; #endif tiov[tiovcnt++].iov_len = nnow; if (pvoff + nnow == iov[pv].iov_len) { pv++; pvoff = 0; } else pvoff += nnow; } DPRINTF("tx segment %d %d+%d bytes %d iovs", seg, hdrlen, now, tiovcnt); /* Update IP header. */ if (sc->esc_txctx.cmd_and_length & E1000_TXD_CMD_IP) { /* IPv4 -- set length and ID */ *(uint16_t *)&hdr[ckinfo[0].ck_start + 2] = htons(hdrlen - ckinfo[0].ck_start + now); *(uint16_t *)&hdr[ckinfo[0].ck_start + 4] = htons(ipid + seg); } else { /* IPv6 -- set length */ *(uint16_t *)&hdr[ckinfo[0].ck_start + 4] = htons(hdrlen - ckinfo[0].ck_start - 40 + now); } /* Update pseudo-header checksum. */ tcpsum = tcpcs; tcpsum += htons(hdrlen - ckinfo[1].ck_start + now); /* Update TCP/UDP headers. */ if (tcp) { /* Update sequence number and FIN/PUSH flags. */ *(uint32_t *)&hdr[ckinfo[1].ck_start + 4] = htonl(tcpseq + paylen - left); if (now < left) { hdr[ckinfo[1].ck_start + 13] &= ~(TH_FIN | TH_PUSH); } } else { /* Update payload length. */ *(uint32_t *)&hdr[ckinfo[1].ck_start + 4] = hdrlen - ckinfo[1].ck_start + now; } /* Calculate checksums and transmit. */ if (ckinfo[0].ck_valid) { *(uint16_t *)&hdr[ckinfo[0].ck_off] = ipcs; e82545_transmit_checksum(tiov, tiovcnt, &ckinfo[0]); } if (ckinfo[1].ck_valid) { *(uint16_t *)&hdr[ckinfo[1].ck_off] = e82545_carry(tcpsum); e82545_transmit_checksum(tiov, tiovcnt, &ckinfo[1]); } e82545_transmit_backend(sc, tiov, tiovcnt); } done: head = (head + 1) % dsize; e82545_transmit_done(sc, ohead, head, dsize, tdwb); *rhead = head; return (desc + 1); } static void e82545_tx_run(struct e82545_softc *sc) { uint32_t cause; uint16_t head, rhead, tail, size; int lim, tdwb, sent; size = sc->esc_TDLEN / 16; if (size == 0) return; head = sc->esc_TDH % size; tail = sc->esc_TDT % size; DPRINTF("tx_run: head %x, rhead %x, tail %x", sc->esc_TDH, sc->esc_TDHr, sc->esc_TDT); pthread_mutex_unlock(&sc->esc_mtx); rhead = head; tdwb = 0; for (lim = size / 4; sc->esc_tx_enabled && lim > 0; lim -= sent) { sent = e82545_transmit(sc, head, tail, size, &rhead, &tdwb); if (sent == 0) break; head = rhead; } pthread_mutex_lock(&sc->esc_mtx); sc->esc_TDH = head; sc->esc_TDHr = rhead; cause = 0; if (tdwb) cause |= E1000_ICR_TXDW; if (lim != size / 4 && sc->esc_TDH == sc->esc_TDT) cause |= E1000_ICR_TXQE; if (cause) e82545_icr_assert(sc, cause); DPRINTF("tx_run done: head %x, rhead %x, tail %x", sc->esc_TDH, sc->esc_TDHr, sc->esc_TDT); } static _Noreturn void * e82545_tx_thread(void *param) { struct e82545_softc *sc = param; pthread_mutex_lock(&sc->esc_mtx); for (;;) { while (!sc->esc_tx_enabled || sc->esc_TDHr == sc->esc_TDT) { if (sc->esc_tx_enabled && sc->esc_TDHr != sc->esc_TDT) break; sc->esc_tx_active = 0; if (sc->esc_tx_enabled == 0) pthread_cond_signal(&sc->esc_tx_cond); pthread_cond_wait(&sc->esc_tx_cond, &sc->esc_mtx); } sc->esc_tx_active = 1; /* Process some tx descriptors. Lock dropped inside. */ e82545_tx_run(sc); } } static void e82545_tx_start(struct e82545_softc *sc) { if (sc->esc_tx_active == 0) pthread_cond_signal(&sc->esc_tx_cond); } static void e82545_tx_enable(struct e82545_softc *sc) { sc->esc_tx_enabled = 1; } static void e82545_tx_disable(struct e82545_softc *sc) { sc->esc_tx_enabled = 0; while (sc->esc_tx_active) pthread_cond_wait(&sc->esc_tx_cond, &sc->esc_mtx); } static void e82545_rx_enable(struct e82545_softc *sc) { sc->esc_rx_enabled = 1; } static void e82545_rx_disable(struct e82545_softc *sc) { sc->esc_rx_enabled = 0; while (sc->esc_rx_active) pthread_cond_wait(&sc->esc_rx_cond, &sc->esc_mtx); } static void e82545_write_ra(struct e82545_softc *sc, int reg, uint32_t wval) { struct eth_uni *eu; int idx; idx = reg >> 1; assert(idx < 15); eu = &sc->esc_uni[idx]; if (reg & 0x1) { /* RAH */ eu->eu_valid = ((wval & E1000_RAH_AV) == E1000_RAH_AV); eu->eu_addrsel = (wval >> 16) & 0x3; eu->eu_eth.octet[5] = wval >> 8; eu->eu_eth.octet[4] = wval; } else { /* RAL */ eu->eu_eth.octet[3] = wval >> 24; eu->eu_eth.octet[2] = wval >> 16; eu->eu_eth.octet[1] = wval >> 8; eu->eu_eth.octet[0] = wval; } } static uint32_t e82545_read_ra(struct e82545_softc *sc, int reg) { struct eth_uni *eu; uint32_t retval; int idx; idx = reg >> 1; assert(idx < 15); eu = &sc->esc_uni[idx]; if (reg & 0x1) { /* RAH */ retval = (eu->eu_valid << 31) | (eu->eu_addrsel << 16) | (eu->eu_eth.octet[5] << 8) | eu->eu_eth.octet[4]; } else { /* RAL */ retval = (eu->eu_eth.octet[3] << 24) | (eu->eu_eth.octet[2] << 16) | (eu->eu_eth.octet[1] << 8) | eu->eu_eth.octet[0]; } return (retval); } static void e82545_write_register(struct e82545_softc *sc, uint32_t offset, uint32_t value) { int ridx; if (offset & 0x3) { DPRINTF("Unaligned register write offset:0x%x value:0x%x", offset, value); return; } DPRINTF("Register write: 0x%x value: 0x%x", offset, value); switch (offset) { case E1000_CTRL: case E1000_CTRL_DUP: e82545_devctl(sc, value); break; case E1000_FCAL: sc->esc_FCAL = value; break; case E1000_FCAH: sc->esc_FCAH = value & ~0xFFFF0000; break; case E1000_FCT: sc->esc_FCT = value & ~0xFFFF0000; break; case E1000_VET: sc->esc_VET = value & ~0xFFFF0000; break; case E1000_FCTTV: sc->esc_FCTTV = value & ~0xFFFF0000; break; case E1000_LEDCTL: sc->esc_LEDCTL = value & ~0x30303000; break; case E1000_PBA: sc->esc_PBA = value & 0x0000FF80; break; case E1000_ICR: case E1000_ITR: case E1000_ICS: case E1000_IMS: case E1000_IMC: e82545_intr_write(sc, offset, value); break; case E1000_RCTL: e82545_rx_ctl(sc, value); break; case E1000_FCRTL: sc->esc_FCRTL = value & ~0xFFFF0007; break; case E1000_FCRTH: sc->esc_FCRTH = value & ~0xFFFF0007; break; case E1000_RDBAL(0): sc->esc_RDBAL = value & ~0xF; if (sc->esc_rx_enabled) { /* Apparently legal: update cached address */ e82545_rx_update_rdba(sc); } break; case E1000_RDBAH(0): assert(!sc->esc_rx_enabled); sc->esc_RDBAH = value; break; case E1000_RDLEN(0): assert(!sc->esc_rx_enabled); sc->esc_RDLEN = value & ~0xFFF0007F; break; case E1000_RDH(0): /* XXX should only ever be zero ? Range check ? */ sc->esc_RDH = value; break; case E1000_RDT(0): /* XXX if this opens up the rx ring, do something ? */ sc->esc_RDT = value; break; case E1000_RDTR: /* ignore FPD bit 31 */ sc->esc_RDTR = value & ~0xFFFF0000; break; case E1000_RXDCTL(0): sc->esc_RXDCTL = value & ~0xFEC0C0C0; break; case E1000_RADV: sc->esc_RADV = value & ~0xFFFF0000; break; case E1000_RSRPD: sc->esc_RSRPD = value & ~0xFFFFF000; break; case E1000_RXCSUM: sc->esc_RXCSUM = value & ~0xFFFFF800; break; case E1000_TXCW: sc->esc_TXCW = value & ~0x3FFF0000; break; case E1000_TCTL: e82545_tx_ctl(sc, value); break; case E1000_TIPG: sc->esc_TIPG = value; break; case E1000_AIT: sc->esc_AIT = value; break; case E1000_TDBAL(0): sc->esc_TDBAL = value & ~0xF; if (sc->esc_tx_enabled) e82545_tx_update_tdba(sc); break; case E1000_TDBAH(0): sc->esc_TDBAH = value; if (sc->esc_tx_enabled) e82545_tx_update_tdba(sc); break; case E1000_TDLEN(0): sc->esc_TDLEN = value & ~0xFFF0007F; if (sc->esc_tx_enabled) e82545_tx_update_tdba(sc); break; case E1000_TDH(0): if (sc->esc_tx_enabled) { WPRINTF("ignoring write to TDH while transmit enabled"); break; } if (value != 0) { WPRINTF("ignoring non-zero value written to TDH"); break; } sc->esc_TDHr = sc->esc_TDH = value; break; case E1000_TDT(0): sc->esc_TDT = value; if (sc->esc_tx_enabled) e82545_tx_start(sc); break; case E1000_TIDV: sc->esc_TIDV = value & ~0xFFFF0000; break; case E1000_TXDCTL(0): //assert(!sc->esc_tx_enabled); sc->esc_TXDCTL = value & ~0xC0C0C0; break; case E1000_TADV: sc->esc_TADV = value & ~0xFFFF0000; break; case E1000_RAL(0) ... E1000_RAH(15): /* convert to u32 offset */ ridx = (offset - E1000_RAL(0)) >> 2; e82545_write_ra(sc, ridx, value); break; case E1000_MTA ... (E1000_MTA + (127*4)): sc->esc_fmcast[(offset - E1000_MTA) >> 2] = value; break; case E1000_VFTA ... (E1000_VFTA + (127*4)): sc->esc_fvlan[(offset - E1000_VFTA) >> 2] = value; break; case E1000_EECD: { //DPRINTF("EECD write 0x%x -> 0x%x", sc->eeprom_control, value); /* edge triggered low->high */ uint32_t eecd_strobe = ((sc->eeprom_control & E1000_EECD_SK) ? 0 : (value & E1000_EECD_SK)); uint32_t eecd_mask = (E1000_EECD_SK|E1000_EECD_CS| E1000_EECD_DI|E1000_EECD_REQ); sc->eeprom_control &= ~eecd_mask; sc->eeprom_control |= (value & eecd_mask); /* grant/revoke immediately */ if (value & E1000_EECD_REQ) { sc->eeprom_control |= E1000_EECD_GNT; } else { sc->eeprom_control &= ~E1000_EECD_GNT; } if (eecd_strobe && (sc->eeprom_control & E1000_EECD_CS)) { e82545_eecd_strobe(sc); } return; } case E1000_MDIC: { uint8_t reg_addr = (uint8_t)((value & E1000_MDIC_REG_MASK) >> E1000_MDIC_REG_SHIFT); uint8_t phy_addr = (uint8_t)((value & E1000_MDIC_PHY_MASK) >> E1000_MDIC_PHY_SHIFT); sc->mdi_control = (value & ~(E1000_MDIC_ERROR|E1000_MDIC_DEST)); if ((value & E1000_MDIC_READY) != 0) { DPRINTF("Incorrect MDIC ready bit: 0x%x", value); return; } switch (value & E82545_MDIC_OP_MASK) { case E1000_MDIC_OP_READ: sc->mdi_control &= ~E82545_MDIC_DATA_MASK; sc->mdi_control |= e82545_read_mdi(sc, reg_addr, phy_addr); break; case E1000_MDIC_OP_WRITE: e82545_write_mdi(sc, reg_addr, phy_addr, value & E82545_MDIC_DATA_MASK); break; default: DPRINTF("Unknown MDIC op: 0x%x", value); return; } /* TODO: barrier? */ sc->mdi_control |= E1000_MDIC_READY; if (value & E82545_MDIC_IE) { // TODO: generate interrupt } return; } case E1000_MANC: case E1000_STATUS: return; default: DPRINTF("Unknown write register: 0x%x value:%x", offset, value); return; } } static uint32_t e82545_read_register(struct e82545_softc *sc, uint32_t offset) { uint32_t retval; int ridx; if (offset & 0x3) { DPRINTF("Unaligned register read offset:0x%x", offset); return 0; } DPRINTF("Register read: 0x%x", offset); switch (offset) { case E1000_CTRL: retval = sc->esc_CTRL; break; case E1000_STATUS: retval = E1000_STATUS_FD | E1000_STATUS_LU | E1000_STATUS_SPEED_1000; break; case E1000_FCAL: retval = sc->esc_FCAL; break; case E1000_FCAH: retval = sc->esc_FCAH; break; case E1000_FCT: retval = sc->esc_FCT; break; case E1000_VET: retval = sc->esc_VET; break; case E1000_FCTTV: retval = sc->esc_FCTTV; break; case E1000_LEDCTL: retval = sc->esc_LEDCTL; break; case E1000_PBA: retval = sc->esc_PBA; break; case E1000_ICR: case E1000_ITR: case E1000_ICS: case E1000_IMS: case E1000_IMC: retval = e82545_intr_read(sc, offset); break; case E1000_RCTL: retval = sc->esc_RCTL; break; case E1000_FCRTL: retval = sc->esc_FCRTL; break; case E1000_FCRTH: retval = sc->esc_FCRTH; break; case E1000_RDBAL(0): retval = sc->esc_RDBAL; break; case E1000_RDBAH(0): retval = sc->esc_RDBAH; break; case E1000_RDLEN(0): retval = sc->esc_RDLEN; break; case E1000_RDH(0): retval = sc->esc_RDH; break; case E1000_RDT(0): retval = sc->esc_RDT; break; case E1000_RDTR: retval = sc->esc_RDTR; break; case E1000_RXDCTL(0): retval = sc->esc_RXDCTL; break; case E1000_RADV: retval = sc->esc_RADV; break; case E1000_RSRPD: retval = sc->esc_RSRPD; break; case E1000_RXCSUM: retval = sc->esc_RXCSUM; break; case E1000_TXCW: retval = sc->esc_TXCW; break; case E1000_TCTL: retval = sc->esc_TCTL; break; case E1000_TIPG: retval = sc->esc_TIPG; break; case E1000_AIT: retval = sc->esc_AIT; break; case E1000_TDBAL(0): retval = sc->esc_TDBAL; break; case E1000_TDBAH(0): retval = sc->esc_TDBAH; break; case E1000_TDLEN(0): retval = sc->esc_TDLEN; break; case E1000_TDH(0): retval = sc->esc_TDH; break; case E1000_TDT(0): retval = sc->esc_TDT; break; case E1000_TIDV: retval = sc->esc_TIDV; break; case E1000_TXDCTL(0): retval = sc->esc_TXDCTL; break; case E1000_TADV: retval = sc->esc_TADV; break; case E1000_RAL(0) ... E1000_RAH(15): /* convert to u32 offset */ ridx = (offset - E1000_RAL(0)) >> 2; retval = e82545_read_ra(sc, ridx); break; case E1000_MTA ... (E1000_MTA + (127*4)): retval = sc->esc_fmcast[(offset - E1000_MTA) >> 2]; break; case E1000_VFTA ... (E1000_VFTA + (127*4)): retval = sc->esc_fvlan[(offset - E1000_VFTA) >> 2]; break; case E1000_EECD: //DPRINTF("EECD read %x", sc->eeprom_control); retval = sc->eeprom_control; break; case E1000_MDIC: retval = sc->mdi_control; break; case E1000_MANC: retval = 0; break; /* stats that we emulate. */ case E1000_MPC: retval = sc->missed_pkt_count; break; case E1000_PRC64: retval = sc->pkt_rx_by_size[0]; break; case E1000_PRC127: retval = sc->pkt_rx_by_size[1]; break; case E1000_PRC255: retval = sc->pkt_rx_by_size[2]; break; case E1000_PRC511: retval = sc->pkt_rx_by_size[3]; break; case E1000_PRC1023: retval = sc->pkt_rx_by_size[4]; break; case E1000_PRC1522: retval = sc->pkt_rx_by_size[5]; break; case E1000_GPRC: retval = sc->good_pkt_rx_count; break; case E1000_BPRC: retval = sc->bcast_pkt_rx_count; break; case E1000_MPRC: retval = sc->mcast_pkt_rx_count; break; case E1000_GPTC: case E1000_TPT: retval = sc->good_pkt_tx_count; break; case E1000_GORCL: retval = (uint32_t)sc->good_octets_rx; break; case E1000_GORCH: retval = (uint32_t)(sc->good_octets_rx >> 32); break; case E1000_TOTL: case E1000_GOTCL: retval = (uint32_t)sc->good_octets_tx; break; case E1000_TOTH: case E1000_GOTCH: retval = (uint32_t)(sc->good_octets_tx >> 32); break; case E1000_ROC: retval = sc->oversize_rx_count; break; case E1000_TORL: retval = (uint32_t)(sc->good_octets_rx + sc->missed_octets); break; case E1000_TORH: retval = (uint32_t)((sc->good_octets_rx + sc->missed_octets) >> 32); break; case E1000_TPR: retval = sc->good_pkt_rx_count + sc->missed_pkt_count + sc->oversize_rx_count; break; case E1000_PTC64: retval = sc->pkt_tx_by_size[0]; break; case E1000_PTC127: retval = sc->pkt_tx_by_size[1]; break; case E1000_PTC255: retval = sc->pkt_tx_by_size[2]; break; case E1000_PTC511: retval = sc->pkt_tx_by_size[3]; break; case E1000_PTC1023: retval = sc->pkt_tx_by_size[4]; break; case E1000_PTC1522: retval = sc->pkt_tx_by_size[5]; break; case E1000_MPTC: retval = sc->mcast_pkt_tx_count; break; case E1000_BPTC: retval = sc->bcast_pkt_tx_count; break; case E1000_TSCTC: retval = sc->tso_tx_count; break; /* stats that are always 0. */ case E1000_CRCERRS: case E1000_ALGNERRC: case E1000_SYMERRS: case E1000_RXERRC: case E1000_SCC: case E1000_ECOL: case E1000_MCC: case E1000_LATECOL: case E1000_COLC: case E1000_DC: case E1000_TNCRS: case E1000_SEC: case E1000_CEXTERR: case E1000_RLEC: case E1000_XONRXC: case E1000_XONTXC: case E1000_XOFFRXC: case E1000_XOFFTXC: case E1000_FCRUC: case E1000_RNBC: case E1000_RUC: case E1000_RFC: case E1000_RJC: case E1000_MGTPRC: case E1000_MGTPDC: case E1000_MGTPTC: case E1000_TSCTFC: retval = 0; break; default: DPRINTF("Unknown read register: 0x%x", offset); retval = 0; break; } return (retval); } static void e82545_write(struct pci_devinst *pi, int baridx, uint64_t offset, int size, uint64_t value) { struct e82545_softc *sc; //DPRINTF("Write bar:%d offset:0x%lx value:0x%lx size:%d", baridx, offset, value, size); sc = pi->pi_arg; pthread_mutex_lock(&sc->esc_mtx); switch (baridx) { case E82545_BAR_IO: switch (offset) { case E82545_IOADDR: if (size != 4) { DPRINTF("Wrong io addr write sz:%d value:0x%lx", size, value); } else sc->io_addr = (uint32_t)value; break; case E82545_IODATA: if (size != 4) { DPRINTF("Wrong io data write size:%d value:0x%lx", size, value); } else if (sc->io_addr > E82545_IO_REGISTER_MAX) { DPRINTF("Non-register io write addr:0x%x value:0x%lx", sc->io_addr, value); } else e82545_write_register(sc, sc->io_addr, (uint32_t)value); break; default: DPRINTF("Unknown io bar write offset:0x%lx value:0x%lx size:%d", offset, value, size); break; } break; case E82545_BAR_REGISTER: if (size != 4) { DPRINTF("Wrong register write size:%d offset:0x%lx value:0x%lx", size, offset, value); } else e82545_write_register(sc, (uint32_t)offset, (uint32_t)value); break; default: DPRINTF("Unknown write bar:%d off:0x%lx val:0x%lx size:%d", baridx, offset, value, size); } pthread_mutex_unlock(&sc->esc_mtx); } static uint64_t e82545_read(struct pci_devinst *pi, int baridx, uint64_t offset, int size) { struct e82545_softc *sc; uint64_t retval; //DPRINTF("Read bar:%d offset:0x%lx size:%d", baridx, offset, size); sc = pi->pi_arg; retval = 0; pthread_mutex_lock(&sc->esc_mtx); switch (baridx) { case E82545_BAR_IO: switch (offset) { case E82545_IOADDR: if (size != 4) { DPRINTF("Wrong io addr read sz:%d", size); } else retval = sc->io_addr; break; case E82545_IODATA: if (size != 4) { DPRINTF("Wrong io data read sz:%d", size); } if (sc->io_addr > E82545_IO_REGISTER_MAX) { DPRINTF("Non-register io read addr:0x%x", sc->io_addr); } else retval = e82545_read_register(sc, sc->io_addr); break; default: DPRINTF("Unknown io bar read offset:0x%lx size:%d", offset, size); break; } break; case E82545_BAR_REGISTER: if (size != 4) { DPRINTF("Wrong register read size:%d offset:0x%lx", size, offset); } else retval = e82545_read_register(sc, (uint32_t)offset); break; default: DPRINTF("Unknown read bar:%d offset:0x%lx size:%d", baridx, offset, size); break; } pthread_mutex_unlock(&sc->esc_mtx); return (retval); } static void e82545_reset(struct e82545_softc *sc, int drvr) { int i; e82545_rx_disable(sc); e82545_tx_disable(sc); /* clear outstanding interrupts */ if (sc->esc_irq_asserted) pci_lintr_deassert(sc->esc_pi); /* misc */ if (!drvr) { sc->esc_FCAL = 0; sc->esc_FCAH = 0; sc->esc_FCT = 0; sc->esc_VET = 0; sc->esc_FCTTV = 0; } sc->esc_LEDCTL = 0x07061302; sc->esc_PBA = 0x00100030; /* start nvm in opcode mode. */ sc->nvm_opaddr = 0; sc->nvm_mode = E82545_NVM_MODE_OPADDR; sc->nvm_bits = E82545_NVM_OPADDR_BITS; sc->eeprom_control = E1000_EECD_PRES | E82545_EECD_FWE_EN; e82545_init_eeprom(sc); /* interrupt */ sc->esc_ICR = 0; sc->esc_ITR = 250; sc->esc_ICS = 0; sc->esc_IMS = 0; sc->esc_IMC = 0; /* L2 filters */ if (!drvr) { memset(sc->esc_fvlan, 0, sizeof(sc->esc_fvlan)); memset(sc->esc_fmcast, 0, sizeof(sc->esc_fmcast)); memset(sc->esc_uni, 0, sizeof(sc->esc_uni)); /* XXX not necessary on 82545 ?? */ sc->esc_uni[0].eu_valid = 1; memcpy(sc->esc_uni[0].eu_eth.octet, sc->esc_mac.octet, ETHER_ADDR_LEN); } else { /* Clear RAH valid bits */ for (i = 0; i < 16; i++) sc->esc_uni[i].eu_valid = 0; } /* receive */ if (!drvr) { sc->esc_RDBAL = 0; sc->esc_RDBAH = 0; } sc->esc_RCTL = 0; sc->esc_FCRTL = 0; sc->esc_FCRTH = 0; sc->esc_RDLEN = 0; sc->esc_RDH = 0; sc->esc_RDT = 0; sc->esc_RDTR = 0; sc->esc_RXDCTL = (1 << 24) | (1 << 16); /* default GRAN/WTHRESH */ sc->esc_RADV = 0; sc->esc_RXCSUM = 0; /* transmit */ if (!drvr) { sc->esc_TDBAL = 0; sc->esc_TDBAH = 0; sc->esc_TIPG = 0; sc->esc_AIT = 0; sc->esc_TIDV = 0; sc->esc_TADV = 0; } sc->esc_tdba = 0; sc->esc_txdesc = NULL; sc->esc_TXCW = 0; sc->esc_TCTL = 0; sc->esc_TDLEN = 0; sc->esc_TDT = 0; sc->esc_TDHr = sc->esc_TDH = 0; sc->esc_TXDCTL = 0; } static int e82545_init(struct pci_devinst *pi, nvlist_t *nvl) { char nstr[80]; struct e82545_softc *sc; const char *mac; int err; /* Setup our softc */ sc = calloc(1, sizeof(*sc)); pi->pi_arg = sc; sc->esc_pi = pi; sc->esc_ctx = pi->pi_vmctx; pthread_mutex_init(&sc->esc_mtx, NULL); pthread_cond_init(&sc->esc_rx_cond, NULL); pthread_cond_init(&sc->esc_tx_cond, NULL); pthread_create(&sc->esc_tx_tid, NULL, e82545_tx_thread, sc); snprintf(nstr, sizeof(nstr), "e82545-%d:%d tx", pi->pi_slot, pi->pi_func); pthread_set_name_np(sc->esc_tx_tid, nstr); pci_set_cfgdata16(pi, PCIR_DEVICE, E82545_DEV_ID_82545EM_COPPER); pci_set_cfgdata16(pi, PCIR_VENDOR, E82545_VENDOR_ID_INTEL); pci_set_cfgdata8(pi, PCIR_CLASS, PCIC_NETWORK); pci_set_cfgdata8(pi, PCIR_SUBCLASS, PCIS_NETWORK_ETHERNET); pci_set_cfgdata16(pi, PCIR_SUBDEV_0, E82545_SUBDEV_ID); pci_set_cfgdata16(pi, PCIR_SUBVEND_0, E82545_VENDOR_ID_INTEL); pci_set_cfgdata8(pi, PCIR_HDRTYPE, PCIM_HDRTYPE_NORMAL); pci_set_cfgdata8(pi, PCIR_INTPIN, 0x1); /* TODO: this card also supports msi, but the freebsd driver for it * does not, so I have not implemented it. */ pci_lintr_request(pi); pci_emul_alloc_bar(pi, E82545_BAR_REGISTER, PCIBAR_MEM32, E82545_BAR_REGISTER_LEN); pci_emul_alloc_bar(pi, E82545_BAR_FLASH, PCIBAR_MEM32, E82545_BAR_FLASH_LEN); pci_emul_alloc_bar(pi, E82545_BAR_IO, PCIBAR_IO, E82545_BAR_IO_LEN); mac = get_config_value_node(nvl, "mac"); if (mac != NULL) { err = net_parsemac(mac, sc->esc_mac.octet); if (err) { free(sc); return (err); } } else net_genmac(pi, sc->esc_mac.octet); err = netbe_init(&sc->esc_be, nvl, e82545_rx_callback, sc); if (err) { free(sc); return (err); } #ifndef __FreeBSD__ size_t buflen = sizeof (sc->esc_mac.octet); err = netbe_get_mac(sc->esc_be, sc->esc_mac.octet, &buflen); if (err != 0) { free(sc); return (err); } #endif netbe_rx_enable(sc->esc_be); /* H/w initiated reset */ e82545_reset(sc, 0); return (0); } static const struct pci_devemu pci_de_e82545 = { .pe_emu = "e1000", .pe_init = e82545_init, .pe_legacy_config = netbe_legacy_config, .pe_barwrite = e82545_write, .pe_barread = e82545_read, }; PCI_EMUL_SET(pci_de_e82545); /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2011 NetApp, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * This file and its contents are supplied under the terms of the * Common Development and Distribution License ("CDDL"), version 1.0. * You may only use this file in accordance with the terms of version * 1.0 of the CDDL. * * A full copy of the text of the CDDL should have accompanied this * source. A copy of the CDDL is also available via the Internet at * http://www.illumos.org/license/CDDL. * * Copyright 2014 Pluribus Networks Inc. * Copyright 2018 Joyent, Inc. * Copyright 2025 Oxide Computer Company */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "acpi.h" #include "bhyverun.h" #include "bootrom.h" #include "config.h" #include "debug.h" #include "inout.h" #include "ioapic.h" #include "mem.h" #include "pci_emul.h" #include "pci_irq.h" #include "pci_lpc.h" #include "pci_passthru.h" #include "qemu_fwcfg.h" #define CONF1_ADDR_PORT 0x0cf8 #define CONF1_DATA_PORT 0x0cfc #define CONF1_ENABLE 0x80000000ul #define MAXBUSES (PCI_BUSMAX + 1) #define MAXSLOTS (PCI_SLOTMAX + 1) #define MAXFUNCS (PCI_FUNCMAX + 1) #define GB (1024 * 1024 * 1024UL) struct funcinfo { nvlist_t *fi_config; struct pci_devemu *fi_pde; struct pci_devinst *fi_devi; }; struct intxinfo { int ii_count; int ii_pirq_pin; int ii_ioapic_irq; }; struct slotinfo { struct intxinfo si_intpins[4]; struct funcinfo si_funcs[MAXFUNCS]; }; struct businfo { uint16_t iobase, iolimit; /* I/O window */ uint32_t membase32, memlimit32; /* mmio window below 4GB */ uint64_t membase64, memlimit64; /* mmio window above 4GB */ struct slotinfo slotinfo[MAXSLOTS]; }; static struct businfo *pci_businfo[MAXBUSES]; SET_DECLARE(pci_devemu_set, struct pci_devemu); static uint64_t pci_emul_iobase; static uint8_t *pci_emul_rombase; static uint64_t pci_emul_romoffset; static uint8_t *pci_emul_romlim; static uint64_t pci_emul_membase32; static uint64_t pci_emul_membase64; static uint64_t pci_emul_memlim64; struct pci_bar_allocation { TAILQ_ENTRY(pci_bar_allocation) chain; struct pci_devinst *pdi; int idx; enum pcibar_type type; uint64_t size; }; static TAILQ_HEAD(pci_bar_list, pci_bar_allocation) pci_bars = TAILQ_HEAD_INITIALIZER(pci_bars); struct boot_device { TAILQ_ENTRY(boot_device) boot_device_chain; struct pci_devinst *pdi; int bootindex; }; static TAILQ_HEAD(boot_list, boot_device) boot_devices = TAILQ_HEAD_INITIALIZER( boot_devices); #define PCI_EMUL_IOBASE 0x2000 #define PCI_EMUL_IOLIMIT 0x10000 #define PCI_EMUL_ROMSIZE 0x10000000 #define PCI_EMUL_ECFG_BASE 0xE0000000 /* 3.5GB */ #define PCI_EMUL_ECFG_SIZE (MAXBUSES * 1024 * 1024) /* 1MB per bus */ SYSRES_MEM(PCI_EMUL_ECFG_BASE, PCI_EMUL_ECFG_SIZE); /* * OVMF always uses 0xC0000000 as base address for 32 bit PCI MMIO. Don't * change this address without changing it in OVMF. */ #define PCI_EMUL_MEMBASE32 0xC0000000 #define PCI_EMUL_MEMLIMIT32 PCI_EMUL_ECFG_BASE #define PCI_EMUL_MEMSIZE64 (32*GB) static struct pci_devemu *pci_emul_finddev(const char *name); static void pci_lintr_route(struct pci_devinst *pi); static void pci_lintr_update(struct pci_devinst *pi); static void pci_cfgrw(int in, int bus, int slot, int func, int coff, int bytes, uint32_t *val); static __inline void CFGWRITE(struct pci_devinst *pi, int coff, uint32_t val, int bytes) { if (bytes == 1) pci_set_cfgdata8(pi, coff, val); else if (bytes == 2) pci_set_cfgdata16(pi, coff, val); else pci_set_cfgdata32(pi, coff, val); } static __inline uint32_t CFGREAD(struct pci_devinst *pi, int coff, int bytes) { if (bytes == 1) return (pci_get_cfgdata8(pi, coff)); else if (bytes == 2) return (pci_get_cfgdata16(pi, coff)); else return (pci_get_cfgdata32(pi, coff)); } static int is_pcir_bar(int coff) { return (coff >= PCIR_BAR(0) && coff < PCIR_BAR(PCI_BARMAX + 1)); } static int is_pcir_bios(int coff) { return (coff >= PCIR_BIOS && coff < PCIR_BIOS + 4); } /* * I/O access */ /* * Slot options are in the form: * * ::,[,] * [:],[,] * * slot is 0..31 * func is 0..7 * emul is a string describing the type of PCI device e.g. virtio-net * config is an optional string, depending on the device, that can be * used for configuration. * Examples are: * 1,virtio-net,tap0 * 3:0,dummy */ static void pci_parse_slot_usage(char *aopt) { EPRINTLN("Invalid PCI slot info field \"%s\"", aopt); } /* * Helper function to parse a list of comma-separated options where * each option is formatted as "name[=value]". If no value is * provided, the option is treated as a boolean and is given a value * of true. */ int pci_parse_legacy_config(nvlist_t *nvl, const char *opt) { char *config, *name, *tofree, *value; if (opt == NULL) return (0); config = tofree = strdup(opt); while ((name = strsep(&config, ",")) != NULL) { value = strchr(name, '='); if (value != NULL) { *value = '\0'; value++; set_config_value_node(nvl, name, value); } else set_config_bool_node(nvl, name, true); } free(tofree); return (0); } /* * PCI device configuration is stored in MIBs that encode the device's * location: * * pci... * * Where "bus", "slot", and "func" are all decimal values without * leading zeroes. Each valid device must have a "device" node which * identifies the driver model of the device. * * Device backends can provide a parser for the "config" string. If * a custom parser is not provided, pci_parse_legacy_config() is used * to parse the string. */ int pci_parse_slot(char *opt) { char node_name[sizeof("pci.XXX.XX.X")]; struct pci_devemu *pde; char *emul, *config, *str, *cp; int error, bnum, snum, fnum; nvlist_t *nvl; error = -1; str = strdup(opt); emul = config = NULL; if ((cp = strchr(str, ',')) != NULL) { *cp = '\0'; emul = cp + 1; if ((cp = strchr(emul, ',')) != NULL) { *cp = '\0'; config = cp + 1; } } else { pci_parse_slot_usage(opt); goto done; } /* :: */ if (sscanf(str, "%d:%d:%d", &bnum, &snum, &fnum) != 3) { bnum = 0; /* : */ if (sscanf(str, "%d:%d", &snum, &fnum) != 2) { fnum = 0; /* */ if (sscanf(str, "%d", &snum) != 1) { snum = -1; } } } if (bnum < 0 || bnum >= MAXBUSES || snum < 0 || snum >= MAXSLOTS || fnum < 0 || fnum >= MAXFUNCS) { pci_parse_slot_usage(opt); goto done; } pde = pci_emul_finddev(emul); if (pde == NULL) { EPRINTLN("pci slot %d:%d:%d: unknown device \"%s\"", bnum, snum, fnum, emul); goto done; } snprintf(node_name, sizeof(node_name), "pci.%d.%d.%d", bnum, snum, fnum); nvl = find_config_node(node_name); if (nvl != NULL) { EPRINTLN("pci slot %d:%d:%d already occupied!", bnum, snum, fnum); goto done; } nvl = create_config_node(node_name); if (pde->pe_alias != NULL) set_config_value_node(nvl, "device", pde->pe_alias); else set_config_value_node(nvl, "device", pde->pe_emu); if (pde->pe_legacy_config != NULL) error = pde->pe_legacy_config(nvl, config); else error = pci_parse_legacy_config(nvl, config); done: free(str); return (error); } void pci_print_supported_devices(void) { struct pci_devemu **pdpp, *pdp; SET_FOREACH(pdpp, pci_devemu_set) { pdp = *pdpp; printf("%s\n", pdp->pe_emu); } } uint32_t pci_config_read_reg(const struct pcisel *const host_sel, nvlist_t *nvl, const uint32_t reg, const uint8_t size, const uint32_t def) { const char *config; const nvlist_t *pci_regs; assert(size == 1 || size == 2 || size == 4); pci_regs = find_relative_config_node(nvl, "pcireg"); if (pci_regs == NULL) { return def; } switch (reg) { case PCIR_DEVICE: config = get_config_value_node(pci_regs, "device"); break; case PCIR_VENDOR: config = get_config_value_node(pci_regs, "vendor"); break; case PCIR_REVID: config = get_config_value_node(pci_regs, "revid"); break; case PCIR_SUBVEND_0: config = get_config_value_node(pci_regs, "subvendor"); break; case PCIR_SUBDEV_0: config = get_config_value_node(pci_regs, "subdevice"); break; default: return (-1); } if (config == NULL) { return def; } else if (host_sel != NULL && strcmp(config, "host") == 0) { return pci_host_read_config(host_sel, reg, size); } else { return strtol(config, NULL, 16); } } static int pci_valid_pba_offset(struct pci_devinst *pi, uint64_t offset) { if (offset < pi->pi_msix.pba_offset) return (0); if (offset >= pi->pi_msix.pba_offset + pi->pi_msix.pba_size) { return (0); } return (1); } int pci_emul_msix_twrite(struct pci_devinst *pi, uint64_t offset, int size, uint64_t value) { int msix_entry_offset; int tab_index; char *dest; /* support only 4 or 8 byte writes */ if (size != 4 && size != 8) return (-1); /* * Return if table index is beyond what device supports */ tab_index = offset / MSIX_TABLE_ENTRY_SIZE; if (tab_index >= pi->pi_msix.table_count) return (-1); msix_entry_offset = offset % MSIX_TABLE_ENTRY_SIZE; /* support only aligned writes */ if ((msix_entry_offset % size) != 0) return (-1); dest = (char *)(pi->pi_msix.table + tab_index); dest += msix_entry_offset; if (size == 4) *((uint32_t *)dest) = value; else *((uint64_t *)dest) = value; return (0); } uint64_t pci_emul_msix_tread(struct pci_devinst *pi, uint64_t offset, int size) { char *dest; int msix_entry_offset; int tab_index; uint64_t retval = ~0; /* * The PCI standard only allows 4 and 8 byte accesses to the MSI-X * table but we also allow 1 byte access to accommodate reads from * ddb. */ if (size != 1 && size != 4 && size != 8) return (retval); msix_entry_offset = offset % MSIX_TABLE_ENTRY_SIZE; /* support only aligned reads */ if ((msix_entry_offset % size) != 0) { return (retval); } tab_index = offset / MSIX_TABLE_ENTRY_SIZE; if (tab_index < pi->pi_msix.table_count) { /* valid MSI-X Table access */ dest = (char *)(pi->pi_msix.table + tab_index); dest += msix_entry_offset; if (size == 1) retval = *((uint8_t *)dest); else if (size == 4) retval = *((uint32_t *)dest); else retval = *((uint64_t *)dest); } else if (pci_valid_pba_offset(pi, offset)) { /* return 0 for PBA access */ retval = 0; } return (retval); } int pci_msix_table_bar(struct pci_devinst *pi) { if (pi->pi_msix.table != NULL) return (pi->pi_msix.table_bar); else return (-1); } int pci_msix_pba_bar(struct pci_devinst *pi) { if (pi->pi_msix.table != NULL) return (pi->pi_msix.pba_bar); else return (-1); } static int pci_emul_io_handler(struct vmctx *ctx __unused, int in, int port, int bytes, uint32_t *eax, void *arg) { struct pci_devinst *pdi = arg; struct pci_devemu *pe = pdi->pi_d; uint64_t offset; int i; assert(port >= 0); for (i = 0; i <= PCI_BARMAX; i++) { if (pdi->pi_bar[i].type == PCIBAR_IO && (uint64_t)port >= pdi->pi_bar[i].addr && (uint64_t)port + bytes <= pdi->pi_bar[i].addr + pdi->pi_bar[i].size) { offset = port - pdi->pi_bar[i].addr; if (in) *eax = (*pe->pe_barread)(pdi, i, offset, bytes); else (*pe->pe_barwrite)(pdi, i, offset, bytes, *eax); return (0); } } return (-1); } static int pci_emul_mem_handler(struct vcpu *vcpu __unused, int dir, uint64_t addr, int size, uint64_t *val, void *arg1, long arg2) { struct pci_devinst *pdi = arg1; struct pci_devemu *pe = pdi->pi_d; uint64_t offset; int bidx = (int)arg2; assert(bidx <= PCI_BARMAX); assert(pdi->pi_bar[bidx].type == PCIBAR_MEM32 || pdi->pi_bar[bidx].type == PCIBAR_MEM64); assert(addr >= pdi->pi_bar[bidx].addr && addr + size <= pdi->pi_bar[bidx].addr + pdi->pi_bar[bidx].size); offset = addr - pdi->pi_bar[bidx].addr; if (dir == MEM_F_WRITE) { if (size == 8) { (*pe->pe_barwrite)(pdi, bidx, offset, 4, *val & 0xffffffff); (*pe->pe_barwrite)(pdi, bidx, offset + 4, 4, *val >> 32); } else { (*pe->pe_barwrite)(pdi, bidx, offset, size, *val); } } else { if (size == 8) { *val = (*pe->pe_barread)(pdi, bidx, offset, 4); *val |= (*pe->pe_barread)(pdi, bidx, offset + 4, 4) << 32; } else { *val = (*pe->pe_barread)(pdi, bidx, offset, size); } } return (0); } static int pci_emul_alloc_resource(uint64_t *baseptr, uint64_t limit, uint64_t size, uint64_t *addr) { uint64_t base; assert((size & (size - 1)) == 0); /* must be a power of 2 */ base = roundup2(*baseptr, size); if (base + size <= limit) { *addr = base; *baseptr = base + size; return (0); } else return (-1); } /* * Register (or unregister) the MMIO or I/O region associated with the BAR * register 'idx' of an emulated pci device. */ static void modify_bar_registration(struct pci_devinst *pi, int idx, int registration) { struct pci_devemu *pe; int error; struct inout_port iop; struct mem_range mr; pe = pi->pi_d; switch (pi->pi_bar[idx].type) { case PCIBAR_IO: bzero(&iop, sizeof(struct inout_port)); iop.name = pi->pi_name; iop.port = pi->pi_bar[idx].addr; iop.size = pi->pi_bar[idx].size; if (registration) { iop.flags = IOPORT_F_INOUT; iop.handler = pci_emul_io_handler; iop.arg = pi; error = register_inout(&iop); } else error = unregister_inout(&iop); break; case PCIBAR_MEM32: case PCIBAR_MEM64: bzero(&mr, sizeof(struct mem_range)); mr.name = pi->pi_name; mr.base = pi->pi_bar[idx].addr; mr.size = pi->pi_bar[idx].size; if (registration) { mr.flags = MEM_F_RW; mr.handler = pci_emul_mem_handler; mr.arg1 = pi; mr.arg2 = idx; error = register_mem(&mr); } else error = unregister_mem(&mr); break; case PCIBAR_ROM: error = 0; break; default: error = EINVAL; break; } assert(error == 0); if (pe->pe_baraddr != NULL) (*pe->pe_baraddr)(pi, idx, registration, pi->pi_bar[idx].addr); } static void unregister_bar(struct pci_devinst *pi, int idx) { modify_bar_registration(pi, idx, 0); } static void register_bar(struct pci_devinst *pi, int idx) { modify_bar_registration(pi, idx, 1); } /* Is the ROM enabled for the emulated pci device? */ static int romen(struct pci_devinst *pi) { return (pi->pi_bar[PCI_ROM_IDX].lobits & PCIM_BIOS_ENABLE) == PCIM_BIOS_ENABLE; } /* Are we decoding i/o port accesses for the emulated pci device? */ static int porten(struct pci_devinst *pi) { uint16_t cmd; cmd = pci_get_cfgdata16(pi, PCIR_COMMAND); return (cmd & PCIM_CMD_PORTEN); } /* Are we decoding memory accesses for the emulated pci device? */ static int memen(struct pci_devinst *pi) { uint16_t cmd; cmd = pci_get_cfgdata16(pi, PCIR_COMMAND); return (cmd & PCIM_CMD_MEMEN); } /* * Update the MMIO or I/O address that is decoded by the BAR register. * * If the pci device has enabled the address space decoding then intercept * the address range decoded by the BAR register. */ static void update_bar_address(struct pci_devinst *pi, uint64_t addr, int idx, int type) { int decode; if (pi->pi_bar[idx].type == PCIBAR_IO) decode = porten(pi); else decode = memen(pi); if (decode) unregister_bar(pi, idx); switch (type) { case PCIBAR_IO: case PCIBAR_MEM32: pi->pi_bar[idx].addr = addr; break; case PCIBAR_MEM64: pi->pi_bar[idx].addr &= ~0xffffffffUL; pi->pi_bar[idx].addr |= addr; break; case PCIBAR_MEMHI64: pi->pi_bar[idx].addr &= 0xffffffff; pi->pi_bar[idx].addr |= addr; break; default: assert(0); } if (decode) register_bar(pi, idx); } int pci_emul_alloc_bar(struct pci_devinst *pdi, int idx, enum pcibar_type type, uint64_t size) { assert((type == PCIBAR_ROM) || (idx >= 0 && idx <= PCI_BARMAX)); assert((type != PCIBAR_ROM) || (idx == PCI_ROM_IDX)); if ((size & (size - 1)) != 0) size = 1UL << flsl(size); /* round up to a power of 2 */ /* Enforce minimum BAR sizes required by the PCI standard */ if (type == PCIBAR_IO) { if (size < 4) size = 4; } else if (type == PCIBAR_ROM) { if (size < ~PCIM_BIOS_ADDR_MASK + 1) size = ~PCIM_BIOS_ADDR_MASK + 1; } else { if (size < 16) size = 16; } /* * To reduce fragmentation of the MMIO space, we allocate the BARs by * size. Therefore, don't allocate the BAR yet. We create a list of all * BAR allocation which is sorted by BAR size. When all PCI devices are * initialized, we will assign an address to the BARs. */ /* create a new list entry */ struct pci_bar_allocation *const new_bar = malloc(sizeof(*new_bar)); memset(new_bar, 0, sizeof(*new_bar)); new_bar->pdi = pdi; new_bar->idx = idx; new_bar->type = type; new_bar->size = size; /* * Search for a BAR which size is lower than the size of our newly * allocated BAR. */ struct pci_bar_allocation *bar = NULL; TAILQ_FOREACH(bar, &pci_bars, chain) { if (bar->size < size) { break; } } if (bar == NULL) { /* * Either the list is empty or new BAR is the smallest BAR of * the list. Append it to the end of our list. */ TAILQ_INSERT_TAIL(&pci_bars, new_bar, chain); } else { /* * The found BAR is smaller than our new BAR. For that reason, * insert our new BAR before the found BAR. */ TAILQ_INSERT_BEFORE(bar, new_bar, chain); } #ifdef __FreeBSD__ /* * Enable PCI BARs only if we don't have a boot ROM, i.e., bhyveload was * used to load the initial guest image. Otherwise, we rely on the boot * ROM to handle this. */ if (!get_config_bool_default("pci.enable_bars", !bootrom_boot())) return (0); #else /* * Enable PCI BARs unless specifically requested not to. Bootroms * generally used in illumos do not perform PCI BAR enumeration * themselves and so need the BARs enabling here. */ if (!get_config_bool_default("pci.enable_bars", true)) return (0); #endif /* * pci_passthru devices synchronize their physical and virtual command * register on init. For that reason, the virtual cmd reg should be * updated as early as possible. */ uint16_t enbit = 0; switch (type) { case PCIBAR_IO: enbit = PCIM_CMD_PORTEN; break; case PCIBAR_MEM64: case PCIBAR_MEM32: enbit = PCIM_CMD_MEMEN; break; default: enbit = 0; break; } const uint16_t cmd = pci_get_cfgdata16(pdi, PCIR_COMMAND); pci_set_cfgdata16(pdi, PCIR_COMMAND, cmd | enbit); return (0); } static int pci_emul_assign_bar(struct pci_devinst *const pdi, const int idx, const enum pcibar_type type, const uint64_t size) { int error; uint64_t *baseptr, limit, addr, mask, lobits, bar; switch (type) { case PCIBAR_NONE: baseptr = NULL; addr = mask = lobits = 0; break; case PCIBAR_IO: baseptr = &pci_emul_iobase; limit = PCI_EMUL_IOLIMIT; mask = PCIM_BAR_IO_BASE; lobits = PCIM_BAR_IO_SPACE; break; case PCIBAR_MEM64: /* * XXX * Some drivers do not work well if the 64-bit BAR is allocated * above 4GB. Allow for this by allocating small requests under * 4GB unless then allocation size is larger than some arbitrary * number (128MB currently). */ if (size > 128 * 1024 * 1024) { baseptr = &pci_emul_membase64; limit = pci_emul_memlim64; mask = PCIM_BAR_MEM_BASE; lobits = PCIM_BAR_MEM_SPACE | PCIM_BAR_MEM_64 | PCIM_BAR_MEM_PREFETCH; } else { baseptr = &pci_emul_membase32; limit = PCI_EMUL_MEMLIMIT32; mask = PCIM_BAR_MEM_BASE; lobits = PCIM_BAR_MEM_SPACE | PCIM_BAR_MEM_64; } break; case PCIBAR_MEM32: baseptr = &pci_emul_membase32; limit = PCI_EMUL_MEMLIMIT32; mask = PCIM_BAR_MEM_BASE; lobits = PCIM_BAR_MEM_SPACE | PCIM_BAR_MEM_32; break; case PCIBAR_ROM: /* do not claim memory for ROM. OVMF will do it for us. */ baseptr = NULL; limit = 0; mask = PCIM_BIOS_ADDR_MASK; lobits = 0; break; default: printf("pci_emul_alloc_base: invalid bar type %d\n", type); #ifdef FreeBSD assert(0); #else abort(); #endif } if (baseptr != NULL) { error = pci_emul_alloc_resource(baseptr, limit, size, &addr); if (error != 0) return (error); } else { addr = 0; } pdi->pi_bar[idx].type = type; pdi->pi_bar[idx].addr = addr; pdi->pi_bar[idx].size = size; /* * passthru devices are using same lobits as physical device they set * this property */ if (pdi->pi_bar[idx].lobits != 0) { lobits = pdi->pi_bar[idx].lobits; } else { pdi->pi_bar[idx].lobits = lobits; } /* Initialize the BAR register in config space */ bar = (addr & mask) | lobits; pci_set_cfgdata32(pdi, PCIR_BAR(idx), bar); if (type == PCIBAR_MEM64) { assert(idx + 1 <= PCI_BARMAX); pdi->pi_bar[idx + 1].type = PCIBAR_MEMHI64; pci_set_cfgdata32(pdi, PCIR_BAR(idx + 1), bar >> 32); } switch (type) { case PCIBAR_IO: if (porten(pdi)) register_bar(pdi, idx); break; case PCIBAR_MEM32: case PCIBAR_MEM64: case PCIBAR_MEMHI64: if (memen(pdi)) register_bar(pdi, idx); break; default: break; } return (0); } int pci_emul_alloc_rom(struct pci_devinst *const pdi, const uint64_t size, void **const addr) { /* allocate ROM space once on first call */ if (pci_emul_rombase == 0) { pci_emul_rombase = vm_create_devmem(pdi->pi_vmctx, VM_PCIROM, "pcirom", PCI_EMUL_ROMSIZE); if (pci_emul_rombase == MAP_FAILED) { warnx("%s: failed to create rom segment", __func__); return (-1); } pci_emul_romlim = pci_emul_rombase + PCI_EMUL_ROMSIZE; pci_emul_romoffset = 0; } /* ROM size should be a power of 2 and greater than 2 KB */ const uint64_t rom_size = MAX(1UL << flsl(size), ~PCIM_BIOS_ADDR_MASK + 1); /* check if ROM fits into ROM space */ if (pci_emul_romoffset + rom_size > PCI_EMUL_ROMSIZE) { warnx("%s: no space left in rom segment:", __func__); warnx("%16lu bytes left", PCI_EMUL_ROMSIZE - pci_emul_romoffset); warnx("%16lu bytes required by %d/%d/%d", rom_size, pdi->pi_bus, pdi->pi_slot, pdi->pi_func); return (-1); } /* allocate ROM BAR */ const int error = pci_emul_alloc_bar(pdi, PCI_ROM_IDX, PCIBAR_ROM, rom_size); if (error) return error; /* return address */ *addr = pci_emul_rombase + pci_emul_romoffset; /* save offset into ROM Space */ pdi->pi_romoffset = pci_emul_romoffset; /* increase offset for next ROM */ pci_emul_romoffset += rom_size; return (0); } int pci_emul_add_boot_device(struct pci_devinst *pi, int bootindex) { struct boot_device *new_device, *device; /* don't permit a negative bootindex */ if (bootindex < 0) { errx(4, "Invalid bootindex %d for %s", bootindex, pi->pi_name); } /* alloc new boot device */ new_device = calloc(1, sizeof(struct boot_device)); if (new_device == NULL) { return (ENOMEM); } new_device->pdi = pi; new_device->bootindex = bootindex; /* search for boot device with higher boot index */ TAILQ_FOREACH(device, &boot_devices, boot_device_chain) { if (device->bootindex == bootindex) { errx(4, "Could not set bootindex %d for %s. Bootindex already occupied by %s", bootindex, pi->pi_name, device->pdi->pi_name); } else if (device->bootindex > bootindex) { break; } } /* add boot device to queue */ if (device == NULL) { TAILQ_INSERT_TAIL(&boot_devices, new_device, boot_device_chain); } else { TAILQ_INSERT_BEFORE(device, new_device, boot_device_chain); } return (0); } #define CAP_START_OFFSET 0x40 int pci_emul_add_capability(struct pci_devinst *pi, u_char *capdata, int caplen, int *capoffp) { int i, capoff, reallen; uint16_t sts; assert(caplen > 0); reallen = roundup2(caplen, 4); /* dword aligned */ sts = pci_get_cfgdata16(pi, PCIR_STATUS); if ((sts & PCIM_STATUS_CAPPRESENT) == 0) capoff = CAP_START_OFFSET; else capoff = pi->pi_capend + 1; /* Check if we have enough space */ if (capoff + reallen > PCI_REGMAX + 1) return (-1); /* Set the previous capability pointer */ if ((sts & PCIM_STATUS_CAPPRESENT) == 0) { pci_set_cfgdata8(pi, PCIR_CAP_PTR, capoff); pci_set_cfgdata16(pi, PCIR_STATUS, sts|PCIM_STATUS_CAPPRESENT); } else pci_set_cfgdata8(pi, pi->pi_prevcap + 1, capoff); /* Copy the capability */ for (i = 0; i < caplen; i++) pci_set_cfgdata8(pi, capoff + i, capdata[i]); /* Set the next capability pointer */ pci_set_cfgdata8(pi, capoff + 1, 0); pi->pi_prevcap = capoff; pi->pi_capend = capoff + reallen - 1; if (capoffp != NULL) *capoffp = capoff; return (0); } static struct pci_devemu * pci_emul_finddev(const char *name) { struct pci_devemu **pdpp, *pdp; SET_FOREACH(pdpp, pci_devemu_set) { pdp = *pdpp; if (!strcmp(pdp->pe_emu, name)) { return (pdp); } } return (NULL); } static int pci_emul_init(struct vmctx *ctx, struct pci_devemu *pde, int bus, int slot, int func, struct funcinfo *fi) { struct pci_devinst *pdi; int err; pdi = calloc(1, sizeof(struct pci_devinst)); pdi->pi_vmctx = ctx; pdi->pi_bus = bus; pdi->pi_slot = slot; pdi->pi_func = func; pthread_mutex_init(&pdi->pi_lintr.lock, NULL); pdi->pi_lintr.pin = 0; pdi->pi_lintr.state = IDLE; pdi->pi_lintr.pirq_pin = 0; pdi->pi_lintr.ioapic_irq = 0; pdi->pi_d = pde; snprintf(pdi->pi_name, PI_NAMESZ, "%s@pci.%d.%d.%d", pde->pe_emu, bus, slot, func); /* Disable legacy interrupts */ pci_set_cfgdata8(pdi, PCIR_INTLINE, 255); pci_set_cfgdata8(pdi, PCIR_INTPIN, 0); #ifdef __FreeBSD__ if (get_config_bool_default("pci.enable_bars", !bootrom_boot())) pci_set_cfgdata8(pdi, PCIR_COMMAND, PCIM_CMD_BUSMASTEREN); #else if (get_config_bool_default("pci.enable_bars", true)) pci_set_cfgdata8(pdi, PCIR_COMMAND, PCIM_CMD_BUSMASTEREN); #endif err = (*pde->pe_init)(pdi, fi->fi_config); if (err == 0) fi->fi_devi = pdi; else free(pdi); return (err); } void pci_populate_msicap(struct msicap *msicap, int msgnum, int nextptr) { int mmc; /* Number of msi messages must be a power of 2 between 1 and 32 */ assert((msgnum & (msgnum - 1)) == 0 && msgnum >= 1 && msgnum <= 32); mmc = ffs(msgnum) - 1; bzero(msicap, sizeof(struct msicap)); msicap->capid = PCIY_MSI; msicap->nextptr = nextptr; msicap->msgctrl = PCIM_MSICTRL_64BIT | (mmc << 1); } int pci_emul_add_msicap(struct pci_devinst *pi, int msgnum) { struct msicap msicap; pci_populate_msicap(&msicap, msgnum, 0); return (pci_emul_add_capability(pi, (u_char *)&msicap, sizeof(msicap), NULL)); } static void pci_populate_msixcap(struct msixcap *msixcap, int msgnum, int barnum, uint32_t msix_tab_size) { assert(msix_tab_size % 4096 == 0); bzero(msixcap, sizeof(struct msixcap)); msixcap->capid = PCIY_MSIX; /* * Message Control Register, all fields set to * zero except for the Table Size. * Note: Table size N is encoded as N-1 */ msixcap->msgctrl = msgnum - 1; /* * MSI-X BAR setup: * - MSI-X table start at offset 0 * - PBA table starts at a 4K aligned offset after the MSI-X table */ msixcap->table_info = barnum & PCIM_MSIX_BIR_MASK; msixcap->pba_info = msix_tab_size | (barnum & PCIM_MSIX_BIR_MASK); } static void pci_msix_table_init(struct pci_devinst *pi, int table_entries) { int i, table_size; assert(table_entries > 0); assert(table_entries <= MAX_MSIX_TABLE_ENTRIES); table_size = table_entries * MSIX_TABLE_ENTRY_SIZE; pi->pi_msix.table = calloc(1, table_size); /* set mask bit of vector control register */ for (i = 0; i < table_entries; i++) pi->pi_msix.table[i].vector_control |= PCIM_MSIX_VCTRL_MASK; } int pci_emul_add_msixcap(struct pci_devinst *pi, int msgnum, int barnum) { uint32_t tab_size; struct msixcap msixcap; assert(msgnum >= 1 && msgnum <= MAX_MSIX_TABLE_ENTRIES); assert(barnum >= 0 && barnum <= PCIR_MAX_BAR_0); tab_size = msgnum * MSIX_TABLE_ENTRY_SIZE; /* Align table size to nearest 4K */ tab_size = roundup2(tab_size, 4096); pi->pi_msix.table_bar = barnum; pi->pi_msix.pba_bar = barnum; pi->pi_msix.table_offset = 0; pi->pi_msix.table_count = msgnum; pi->pi_msix.pba_offset = tab_size; pi->pi_msix.pba_size = PBA_SIZE(msgnum); pci_msix_table_init(pi, msgnum); pci_populate_msixcap(&msixcap, msgnum, barnum, tab_size); /* allocate memory for MSI-X Table and PBA */ pci_emul_alloc_bar(pi, barnum, PCIBAR_MEM32, tab_size + pi->pi_msix.pba_size); return (pci_emul_add_capability(pi, (u_char *)&msixcap, sizeof(msixcap), NULL)); } static void msixcap_cfgwrite(struct pci_devinst *pi, int capoff, int offset, int bytes, uint32_t val) { uint16_t msgctrl, rwmask; int off; off = offset - capoff; /* Message Control Register */ if (off == 2 && bytes == 2) { rwmask = PCIM_MSIXCTRL_MSIX_ENABLE | PCIM_MSIXCTRL_FUNCTION_MASK; msgctrl = pci_get_cfgdata16(pi, offset); msgctrl &= ~rwmask; msgctrl |= val & rwmask; val = msgctrl; pi->pi_msix.enabled = val & PCIM_MSIXCTRL_MSIX_ENABLE; pi->pi_msix.function_mask = val & PCIM_MSIXCTRL_FUNCTION_MASK; pci_lintr_update(pi); } CFGWRITE(pi, offset, val, bytes); } static void msicap_cfgwrite(struct pci_devinst *pi, int capoff, int offset, int bytes, uint32_t val) { uint16_t msgctrl, rwmask, msgdata, mme; uint32_t addrlo; /* * If guest is writing to the message control register make sure * we do not overwrite read-only fields. */ if ((offset - capoff) == 2 && bytes == 2) { rwmask = PCIM_MSICTRL_MME_MASK | PCIM_MSICTRL_MSI_ENABLE; msgctrl = pci_get_cfgdata16(pi, offset); msgctrl &= ~rwmask; msgctrl |= val & rwmask; val = msgctrl; } CFGWRITE(pi, offset, val, bytes); msgctrl = pci_get_cfgdata16(pi, capoff + 2); addrlo = pci_get_cfgdata32(pi, capoff + 4); if (msgctrl & PCIM_MSICTRL_64BIT) msgdata = pci_get_cfgdata16(pi, capoff + 12); else msgdata = pci_get_cfgdata16(pi, capoff + 8); mme = msgctrl & PCIM_MSICTRL_MME_MASK; pi->pi_msi.enabled = msgctrl & PCIM_MSICTRL_MSI_ENABLE ? 1 : 0; if (pi->pi_msi.enabled) { pi->pi_msi.addr = addrlo; pi->pi_msi.msg_data = msgdata; pi->pi_msi.maxmsgnum = 1 << (mme >> 4); } else { pi->pi_msi.maxmsgnum = 0; } pci_lintr_update(pi); } static void pciecap_cfgwrite(struct pci_devinst *pi, int capoff __unused, int offset, int bytes, uint32_t val) { /* XXX don't write to the readonly parts */ CFGWRITE(pi, offset, val, bytes); } #define PCIECAP_VERSION 0x2 int pci_emul_add_pciecap(struct pci_devinst *pi, int type) { int err; struct pciecap pciecap; bzero(&pciecap, sizeof(pciecap)); /* * Use the integrated endpoint type for endpoints on a root complex bus. * * NB: bhyve currently only supports a single PCI bus that is the root * complex bus, so all endpoints are integrated. */ if ((type == PCIEM_TYPE_ENDPOINT) && (pi->pi_bus == 0)) type = PCIEM_TYPE_ROOT_INT_EP; pciecap.capid = PCIY_EXPRESS; pciecap.pcie_capabilities = PCIECAP_VERSION | type; if (type != PCIEM_TYPE_ROOT_INT_EP) { pciecap.link_capabilities = 0x411; /* gen1, x1 */ pciecap.link_status = 0x11; /* gen1, x1 */ } err = pci_emul_add_capability(pi, (u_char *)&pciecap, sizeof(pciecap), NULL); return (err); } /* * This function assumes that 'coff' is in the capabilities region of the * config space. A capoff parameter of zero will force a search for the * offset and type. */ void pci_emul_capwrite(struct pci_devinst *pi, int offset, int bytes, uint32_t val, uint8_t capoff, int capid) { uint8_t nextoff; /* Do not allow un-aligned writes */ if ((offset & (bytes - 1)) != 0) return; if (capoff == 0) { /* Find the capability that we want to update */ capoff = CAP_START_OFFSET; while (1) { nextoff = pci_get_cfgdata8(pi, capoff + 1); if (nextoff == 0) break; if (offset >= capoff && offset < nextoff) break; capoff = nextoff; } assert(offset >= capoff); capid = pci_get_cfgdata8(pi, capoff); } /* * Capability ID and Next Capability Pointer are readonly. * However, some o/s's do 4-byte writes that include these. * For this case, trim the write back to 2 bytes and adjust * the data. */ if (offset == capoff || offset == capoff + 1) { if (offset == capoff && bytes == 4) { bytes = 2; offset += 2; val >>= 16; } else return; } switch (capid) { case PCIY_MSI: msicap_cfgwrite(pi, capoff, offset, bytes, val); break; case PCIY_MSIX: msixcap_cfgwrite(pi, capoff, offset, bytes, val); break; case PCIY_EXPRESS: pciecap_cfgwrite(pi, capoff, offset, bytes, val); break; default: break; } } static int pci_emul_iscap(struct pci_devinst *pi, int offset) { uint16_t sts; sts = pci_get_cfgdata16(pi, PCIR_STATUS); if ((sts & PCIM_STATUS_CAPPRESENT) != 0) { if (offset >= CAP_START_OFFSET && offset <= pi->pi_capend) return (1); } return (0); } static int pci_emul_fallback_handler(struct vcpu *vcpu __unused, int dir, uint64_t addr __unused, int size __unused, uint64_t *val, void *arg1 __unused, long arg2 __unused) { /* * Ignore writes; return 0xff's for reads. The mem read code * will take care of truncating to the correct size. */ if (dir == MEM_F_READ) { *val = 0xffffffffffffffff; } return (0); } static int pci_emul_ecfg_handler(struct vcpu *vcpu __unused, int dir, uint64_t addr, int bytes, uint64_t *val, void *arg1 __unused, long arg2 __unused) { int bus, slot, func, coff, in; coff = addr & 0xfff; func = (addr >> 12) & 0x7; slot = (addr >> 15) & 0x1f; bus = (addr >> 20) & 0xff; in = (dir == MEM_F_READ); if (in) *val = ~0UL; pci_cfgrw(in, bus, slot, func, coff, bytes, (uint32_t *)val); return (0); } uint64_t pci_ecfg_base(void) { return (PCI_EMUL_ECFG_BASE); } static int init_bootorder(void) { struct boot_device *device; FILE *fp; char *bootorder; size_t bootorder_len; if (TAILQ_EMPTY(&boot_devices)) return (0); fp = open_memstream(&bootorder, &bootorder_len); TAILQ_FOREACH(device, &boot_devices, boot_device_chain) { fprintf(fp, "/pci@i0cf8/pci@%d,%d\n", device->pdi->pi_slot, device->pdi->pi_func); } fclose(fp); return (qemu_fwcfg_add_file("bootorder", bootorder_len, bootorder)); } #define BUSIO_ROUNDUP 32 #define BUSMEM32_ROUNDUP (1024 * 1024) #define BUSMEM64_ROUNDUP (512 * 1024 * 1024) int init_pci(struct vmctx *ctx) { char node_name[sizeof("pci.XXX.XX.X")]; struct mem_range mr; struct pci_devemu *pde; struct businfo *bi; struct slotinfo *si; struct funcinfo *fi; nvlist_t *nvl; const char *emul; size_t lowmem; int bus, slot, func; int error; if (vm_get_lowmem_limit(ctx) > PCI_EMUL_MEMBASE32) errx(EX_OSERR, "Invalid lowmem limit"); pci_emul_iobase = PCI_EMUL_IOBASE; pci_emul_membase32 = PCI_EMUL_MEMBASE32; pci_emul_membase64 = vm_get_highmem_base(ctx) + vm_get_highmem_size(ctx); pci_emul_membase64 = roundup2(pci_emul_membase64, PCI_EMUL_MEMSIZE64); pci_emul_memlim64 = pci_emul_membase64 + PCI_EMUL_MEMSIZE64; TAILQ_INIT(&boot_devices); for (bus = 0; bus < MAXBUSES; bus++) { snprintf(node_name, sizeof(node_name), "pci.%d", bus); nvl = find_config_node(node_name); if (nvl == NULL) continue; pci_businfo[bus] = calloc(1, sizeof(struct businfo)); bi = pci_businfo[bus]; /* * Keep track of the i/o and memory resources allocated to * this bus. */ bi->iobase = pci_emul_iobase; bi->membase32 = pci_emul_membase32; bi->membase64 = pci_emul_membase64; /* first run: init devices */ for (slot = 0; slot < MAXSLOTS; slot++) { si = &bi->slotinfo[slot]; for (func = 0; func < MAXFUNCS; func++) { fi = &si->si_funcs[func]; snprintf(node_name, sizeof(node_name), "pci.%d.%d.%d", bus, slot, func); nvl = find_config_node(node_name); if (nvl == NULL) continue; fi->fi_config = nvl; emul = get_config_value_node(nvl, "device"); if (emul == NULL) { EPRINTLN("pci slot %d:%d:%d: missing " "\"device\" value", bus, slot, func); return (EINVAL); } pde = pci_emul_finddev(emul); if (pde == NULL) { EPRINTLN("pci slot %d:%d:%d: unknown " "device \"%s\"", bus, slot, func, emul); return (EINVAL); } if (pde->pe_alias != NULL) { EPRINTLN("pci slot %d:%d:%d: legacy " "device \"%s\", use \"%s\" instead", bus, slot, func, emul, pde->pe_alias); return (EINVAL); } fi->fi_pde = pde; error = pci_emul_init(ctx, pde, bus, slot, func, fi); if (error) return (error); } } /* second run: assign BARs and free list */ struct pci_bar_allocation *bar; struct pci_bar_allocation *bar_tmp; TAILQ_FOREACH_SAFE(bar, &pci_bars, chain, bar_tmp) { pci_emul_assign_bar(bar->pdi, bar->idx, bar->type, bar->size); free(bar); } TAILQ_INIT(&pci_bars); /* * Add some slop to the I/O and memory resources decoded by * this bus to give a guest some flexibility if it wants to * reprogram the BARs. */ pci_emul_iobase += BUSIO_ROUNDUP; pci_emul_iobase = roundup2(pci_emul_iobase, BUSIO_ROUNDUP); bi->iolimit = pci_emul_iobase; pci_emul_membase32 += BUSMEM32_ROUNDUP; pci_emul_membase32 = roundup2(pci_emul_membase32, BUSMEM32_ROUNDUP); bi->memlimit32 = pci_emul_membase32; pci_emul_membase64 += BUSMEM64_ROUNDUP; pci_emul_membase64 = roundup2(pci_emul_membase64, BUSMEM64_ROUNDUP); bi->memlimit64 = pci_emul_membase64; } /* * PCI backends are initialized before routing INTx interrupts * so that LPC devices are able to reserve ISA IRQs before * routing PIRQ pins. */ for (bus = 0; bus < MAXBUSES; bus++) { if ((bi = pci_businfo[bus]) == NULL) continue; for (slot = 0; slot < MAXSLOTS; slot++) { si = &bi->slotinfo[slot]; for (func = 0; func < MAXFUNCS; func++) { fi = &si->si_funcs[func]; if (fi->fi_devi == NULL) continue; pci_lintr_route(fi->fi_devi); } } } lpc_pirq_routed(); if ((error = init_bootorder()) != 0) { warnx("%s: Unable to init bootorder", __func__); return (error); } /* * The guest physical memory map looks like the following: * [0, lowmem) guest system memory * [lowmem, 0xC0000000) memory hole (may be absent) * [0xC0000000, 0xE0000000) PCI hole (32-bit BAR allocation) * [0xE0000000, 0xF0000000) PCI extended config window * [0xF0000000, 4GB) LAPIC, IOAPIC, HPET, firmware * [4GB, 4GB + highmem) */ /* * Accesses to memory addresses that are not allocated to system * memory or PCI devices return 0xff's. */ lowmem = vm_get_lowmem_size(ctx); bzero(&mr, sizeof(struct mem_range)); mr.name = "PCI hole"; mr.flags = MEM_F_RW | MEM_F_IMMUTABLE; mr.base = lowmem; mr.size = (4ULL * 1024 * 1024 * 1024) - lowmem; mr.handler = pci_emul_fallback_handler; error = register_mem_fallback(&mr); assert(error == 0); /* PCI extended config space */ bzero(&mr, sizeof(struct mem_range)); mr.name = "PCI ECFG"; mr.flags = MEM_F_RW | MEM_F_IMMUTABLE; mr.base = PCI_EMUL_ECFG_BASE; mr.size = PCI_EMUL_ECFG_SIZE; mr.handler = pci_emul_ecfg_handler; error = register_mem(&mr); assert(error == 0); return (0); } static void pci_apic_prt_entry(int bus __unused, int slot, int pin, int pirq_pin __unused, int ioapic_irq, void *arg __unused) { dsdt_line(" Package ()"); dsdt_line(" {"); dsdt_line(" 0x%X,", slot << 16 | 0xffff); dsdt_line(" 0x%02X,", pin - 1); dsdt_line(" Zero,"); dsdt_line(" 0x%X", ioapic_irq); dsdt_line(" },"); } static void pci_pirq_prt_entry(int bus __unused, int slot, int pin, int pirq_pin, int ioapic_irq __unused, void *arg __unused) { char *name; name = lpc_pirq_name(pirq_pin); if (name == NULL) return; dsdt_line(" Package ()"); dsdt_line(" {"); dsdt_line(" 0x%X,", slot << 16 | 0xffff); dsdt_line(" 0x%02X,", pin - 1); dsdt_line(" %s,", name); dsdt_line(" 0x00"); dsdt_line(" },"); free(name); } /* * A bhyve virtual machine has a flat PCI hierarchy with a root port * corresponding to each PCI bus. */ static void pci_bus_write_dsdt(int bus) { struct businfo *bi; struct slotinfo *si; struct pci_devinst *pi; int count, func, slot; /* * If there are no devices on this 'bus' then just return. */ if ((bi = pci_businfo[bus]) == NULL) { /* * Bus 0 is special because it decodes the I/O ports used * for PCI config space access even if there are no devices * on it. */ if (bus != 0) return; } dsdt_line(" Device (PC%02X)", bus); dsdt_line(" {"); dsdt_line(" Name (_HID, EisaId (\"PNP0A03\"))"); dsdt_line(" Method (_BBN, 0, NotSerialized)"); dsdt_line(" {"); dsdt_line(" Return (0x%08X)", bus); dsdt_line(" }"); dsdt_line(" Name (_CRS, ResourceTemplate ()"); dsdt_line(" {"); dsdt_line(" WordBusNumber (ResourceProducer, MinFixed, " "MaxFixed, PosDecode,"); dsdt_line(" 0x0000, // Granularity"); dsdt_line(" 0x%04X, // Range Minimum", bus); dsdt_line(" 0x%04X, // Range Maximum", bus); dsdt_line(" 0x0000, // Translation Offset"); dsdt_line(" 0x0001, // Length"); dsdt_line(" ,, )"); if (bus == 0) { dsdt_indent(3); dsdt_fixed_ioport(0xCF8, 8); dsdt_unindent(3); dsdt_line(" WordIO (ResourceProducer, MinFixed, MaxFixed, " "PosDecode, EntireRange,"); dsdt_line(" 0x0000, // Granularity"); dsdt_line(" 0x0000, // Range Minimum"); dsdt_line(" 0x0CF7, // Range Maximum"); dsdt_line(" 0x0000, // Translation Offset"); dsdt_line(" 0x0CF8, // Length"); dsdt_line(" ,, , TypeStatic)"); dsdt_line(" WordIO (ResourceProducer, MinFixed, MaxFixed, " "PosDecode, EntireRange,"); dsdt_line(" 0x0000, // Granularity"); dsdt_line(" 0x0D00, // Range Minimum"); dsdt_line(" 0x%04X, // Range Maximum", PCI_EMUL_IOBASE - 1); dsdt_line(" 0x0000, // Translation Offset"); dsdt_line(" 0x%04X, // Length", PCI_EMUL_IOBASE - 0x0D00); dsdt_line(" ,, , TypeStatic)"); if (bi == NULL) { dsdt_line(" })"); goto done; } } assert(bi != NULL); /* i/o window */ dsdt_line(" WordIO (ResourceProducer, MinFixed, MaxFixed, " "PosDecode, EntireRange,"); dsdt_line(" 0x0000, // Granularity"); dsdt_line(" 0x%04X, // Range Minimum", bi->iobase); dsdt_line(" 0x%04X, // Range Maximum", bi->iolimit - 1); dsdt_line(" 0x0000, // Translation Offset"); dsdt_line(" 0x%04X, // Length", bi->iolimit - bi->iobase); dsdt_line(" ,, , TypeStatic)"); /* mmio window (32-bit) */ dsdt_line(" DWordMemory (ResourceProducer, PosDecode, " "MinFixed, MaxFixed, NonCacheable, ReadWrite,"); dsdt_line(" 0x00000000, // Granularity"); dsdt_line(" 0x%08X, // Range Minimum\n", bi->membase32); dsdt_line(" 0x%08X, // Range Maximum\n", bi->memlimit32 - 1); dsdt_line(" 0x00000000, // Translation Offset"); dsdt_line(" 0x%08X, // Length\n", bi->memlimit32 - bi->membase32); dsdt_line(" ,, , AddressRangeMemory, TypeStatic)"); /* mmio window (64-bit) */ dsdt_line(" QWordMemory (ResourceProducer, PosDecode, " "MinFixed, MaxFixed, NonCacheable, ReadWrite,"); dsdt_line(" 0x0000000000000000, // Granularity"); dsdt_line(" 0x%016lX, // Range Minimum\n", bi->membase64); dsdt_line(" 0x%016lX, // Range Maximum\n", bi->memlimit64 - 1); dsdt_line(" 0x0000000000000000, // Translation Offset"); dsdt_line(" 0x%016lX, // Length\n", bi->memlimit64 - bi->membase64); dsdt_line(" ,, , AddressRangeMemory, TypeStatic)"); dsdt_line(" })"); count = pci_count_lintr(bus); if (count != 0) { dsdt_indent(2); dsdt_line("Name (PPRT, Package ()"); dsdt_line("{"); pci_walk_lintr(bus, pci_pirq_prt_entry, NULL); dsdt_line("})"); dsdt_line("Name (APRT, Package ()"); dsdt_line("{"); pci_walk_lintr(bus, pci_apic_prt_entry, NULL); dsdt_line("})"); dsdt_line("Method (_PRT, 0, NotSerialized)"); dsdt_line("{"); dsdt_line(" If (PICM)"); dsdt_line(" {"); dsdt_line(" Return (APRT)"); dsdt_line(" }"); dsdt_line(" Else"); dsdt_line(" {"); dsdt_line(" Return (PPRT)"); dsdt_line(" }"); dsdt_line("}"); dsdt_unindent(2); } dsdt_indent(2); for (slot = 0; slot < MAXSLOTS; slot++) { si = &bi->slotinfo[slot]; for (func = 0; func < MAXFUNCS; func++) { pi = si->si_funcs[func].fi_devi; if (pi != NULL && pi->pi_d->pe_write_dsdt != NULL) pi->pi_d->pe_write_dsdt(pi); } } dsdt_unindent(2); done: dsdt_line(" }"); } void pci_write_dsdt(void) { int bus; dsdt_indent(1); dsdt_line("Name (PICM, 0x00)"); dsdt_line("Method (_PIC, 1, NotSerialized)"); dsdt_line("{"); dsdt_line(" Store (Arg0, PICM)"); dsdt_line("}"); dsdt_line(""); dsdt_line("Scope (_SB)"); dsdt_line("{"); for (bus = 0; bus < MAXBUSES; bus++) pci_bus_write_dsdt(bus); dsdt_line("}"); dsdt_unindent(1); } int pci_bus_configured(int bus) { assert(bus >= 0 && bus < MAXBUSES); return (pci_businfo[bus] != NULL); } int pci_msi_enabled(struct pci_devinst *pi) { return (pi->pi_msi.enabled); } int pci_msi_maxmsgnum(struct pci_devinst *pi) { if (pi->pi_msi.enabled) return (pi->pi_msi.maxmsgnum); else return (0); } int pci_msix_enabled(struct pci_devinst *pi) { return (pi->pi_msix.enabled && !pi->pi_msi.enabled); } void pci_generate_msix(struct pci_devinst *pi, int index) { struct msix_table_entry *mte; if (!pci_msix_enabled(pi)) return; if (pi->pi_msix.function_mask) return; if (index >= pi->pi_msix.table_count) return; mte = &pi->pi_msix.table[index]; if ((mte->vector_control & PCIM_MSIX_VCTRL_MASK) == 0) { /* XXX Set PBA bit if interrupt is disabled */ vm_lapic_msi(pi->pi_vmctx, mte->addr, mte->msg_data); } } void pci_generate_msi(struct pci_devinst *pi, int index) { if (pci_msi_enabled(pi) && index < pci_msi_maxmsgnum(pi)) { vm_lapic_msi(pi->pi_vmctx, pi->pi_msi.addr, pi->pi_msi.msg_data + index); } } static bool pci_lintr_permitted(struct pci_devinst *pi) { uint16_t cmd; cmd = pci_get_cfgdata16(pi, PCIR_COMMAND); return (!(pi->pi_msi.enabled || pi->pi_msix.enabled || (cmd & PCIM_CMD_INTxDIS))); } void pci_lintr_request(struct pci_devinst *pi) { struct businfo *bi; struct slotinfo *si; int bestpin, bestcount, pin; bi = pci_businfo[pi->pi_bus]; assert(bi != NULL); /* * Just allocate a pin from our slot. The pin will be * assigned IRQs later when interrupts are routed. */ si = &bi->slotinfo[pi->pi_slot]; bestpin = 0; bestcount = si->si_intpins[0].ii_count; for (pin = 1; pin < 4; pin++) { if (si->si_intpins[pin].ii_count < bestcount) { bestpin = pin; bestcount = si->si_intpins[pin].ii_count; } } si->si_intpins[bestpin].ii_count++; pi->pi_lintr.pin = bestpin + 1; pci_set_cfgdata8(pi, PCIR_INTPIN, bestpin + 1); } static void pci_lintr_route(struct pci_devinst *pi) { struct businfo *bi; struct intxinfo *ii; if (pi->pi_lintr.pin == 0) return; bi = pci_businfo[pi->pi_bus]; assert(bi != NULL); ii = &bi->slotinfo[pi->pi_slot].si_intpins[pi->pi_lintr.pin - 1]; /* * Attempt to allocate an I/O APIC pin for this intpin if one * is not yet assigned. */ if (ii->ii_ioapic_irq == 0) ii->ii_ioapic_irq = ioapic_pci_alloc_irq(pi); assert(ii->ii_ioapic_irq > 0); /* * Attempt to allocate a PIRQ pin for this intpin if one is * not yet assigned. */ if (ii->ii_pirq_pin == 0) ii->ii_pirq_pin = pirq_alloc_pin(pi); assert(ii->ii_pirq_pin > 0); pi->pi_lintr.ioapic_irq = ii->ii_ioapic_irq; pi->pi_lintr.pirq_pin = ii->ii_pirq_pin; pci_set_cfgdata8(pi, PCIR_INTLINE, pirq_irq(ii->ii_pirq_pin)); } void pci_lintr_assert(struct pci_devinst *pi) { assert(pi->pi_lintr.pin > 0); pthread_mutex_lock(&pi->pi_lintr.lock); if (pi->pi_lintr.state == IDLE) { if (pci_lintr_permitted(pi)) { pi->pi_lintr.state = ASSERTED; pci_irq_assert(pi); } else pi->pi_lintr.state = PENDING; } pthread_mutex_unlock(&pi->pi_lintr.lock); } void pci_lintr_deassert(struct pci_devinst *pi) { assert(pi->pi_lintr.pin > 0); pthread_mutex_lock(&pi->pi_lintr.lock); if (pi->pi_lintr.state == ASSERTED) { pi->pi_lintr.state = IDLE; pci_irq_deassert(pi); } else if (pi->pi_lintr.state == PENDING) pi->pi_lintr.state = IDLE; pthread_mutex_unlock(&pi->pi_lintr.lock); } static void pci_lintr_update(struct pci_devinst *pi) { pthread_mutex_lock(&pi->pi_lintr.lock); if (pi->pi_lintr.state == ASSERTED && !pci_lintr_permitted(pi)) { pci_irq_deassert(pi); pi->pi_lintr.state = PENDING; } else if (pi->pi_lintr.state == PENDING && pci_lintr_permitted(pi)) { pi->pi_lintr.state = ASSERTED; pci_irq_assert(pi); } pthread_mutex_unlock(&pi->pi_lintr.lock); #ifndef __FreeBSD__ if (pi->pi_d->pe_lintrupdate != NULL) { pi->pi_d->pe_lintrupdate(pi); } #endif /* __FreeBSD__ */ } int pci_count_lintr(int bus) { int count, slot, pin; struct slotinfo *slotinfo; count = 0; if (pci_businfo[bus] != NULL) { for (slot = 0; slot < MAXSLOTS; slot++) { slotinfo = &pci_businfo[bus]->slotinfo[slot]; for (pin = 0; pin < 4; pin++) { if (slotinfo->si_intpins[pin].ii_count != 0) count++; } } } return (count); } void pci_walk_lintr(int bus, pci_lintr_cb cb, void *arg) { struct businfo *bi; struct slotinfo *si; struct intxinfo *ii; int slot, pin; if ((bi = pci_businfo[bus]) == NULL) return; for (slot = 0; slot < MAXSLOTS; slot++) { si = &bi->slotinfo[slot]; for (pin = 0; pin < 4; pin++) { ii = &si->si_intpins[pin]; if (ii->ii_count != 0) cb(bus, slot, pin + 1, ii->ii_pirq_pin, ii->ii_ioapic_irq, arg); } } } /* * Return 1 if the emulated device in 'slot' is a multi-function device. * Return 0 otherwise. */ static int pci_emul_is_mfdev(int bus, int slot) { struct businfo *bi; struct slotinfo *si; int f, numfuncs; numfuncs = 0; if ((bi = pci_businfo[bus]) != NULL) { si = &bi->slotinfo[slot]; for (f = 0; f < MAXFUNCS; f++) { if (si->si_funcs[f].fi_devi != NULL) { numfuncs++; } } } return (numfuncs > 1); } /* * Ensure that the PCIM_MFDEV bit is properly set (or unset) depending on * whether or not is a multi-function being emulated in the pci 'slot'. */ static void pci_emul_hdrtype_fixup(int bus, int slot, int off, int bytes, uint32_t *rv) { int mfdev; if (off <= PCIR_HDRTYPE && off + bytes > PCIR_HDRTYPE) { mfdev = pci_emul_is_mfdev(bus, slot); switch (bytes) { case 1: case 2: *rv &= ~PCIM_MFDEV; if (mfdev) { *rv |= PCIM_MFDEV; } break; case 4: *rv &= ~(PCIM_MFDEV << 16); if (mfdev) { *rv |= (PCIM_MFDEV << 16); } break; } } } /* * Update device state in response to changes to the PCI command * register. */ void pci_emul_cmd_changed(struct pci_devinst *pi, uint16_t old) { int i; uint16_t changed, new; new = pci_get_cfgdata16(pi, PCIR_COMMAND); changed = old ^ new; /* * If the MMIO or I/O address space decoding has changed then * register/unregister all BARs that decode that address space. */ for (i = 0; i <= PCI_BARMAX_WITH_ROM; i++) { switch (pi->pi_bar[i].type) { case PCIBAR_NONE: case PCIBAR_MEMHI64: break; case PCIBAR_IO: /* I/O address space decoding changed? */ if (changed & PCIM_CMD_PORTEN) { if (new & PCIM_CMD_PORTEN) register_bar(pi, i); else unregister_bar(pi, i); } break; case PCIBAR_ROM: /* skip (un-)register of ROM if it disabled */ if (!romen(pi)) break; /* fallthrough */ case PCIBAR_MEM32: case PCIBAR_MEM64: /* MMIO address space decoding changed? */ if (changed & PCIM_CMD_MEMEN) { if (new & PCIM_CMD_MEMEN) register_bar(pi, i); else unregister_bar(pi, i); } break; default: assert(0); } } /* * If INTx has been unmasked and is pending, assert the * interrupt. */ pci_lintr_update(pi); } static void pci_emul_cmdsts_write(struct pci_devinst *pi, int coff, uint32_t new, int bytes) { int rshift; uint32_t cmd, old, readonly; cmd = pci_get_cfgdata16(pi, PCIR_COMMAND); /* stash old value */ /* * From PCI Local Bus Specification 3.0 sections 6.2.2 and 6.2.3. * * XXX Bits 8, 11, 12, 13, 14 and 15 in the status register are * 'write 1 to clear'. However these bits are not set to '1' by * any device emulation so it is simpler to treat them as readonly. */ rshift = (coff & 0x3) * 8; readonly = 0xFFFFF880 >> rshift; old = CFGREAD(pi, coff, bytes); new &= ~readonly; new |= (old & readonly); CFGWRITE(pi, coff, new, bytes); /* update config */ pci_emul_cmd_changed(pi, cmd); } static void pci_cfgrw(int in, int bus, int slot, int func, int coff, int bytes, uint32_t *valp) { struct businfo *bi; struct slotinfo *si; struct pci_devinst *pi; struct pci_devemu *pe; int idx, needcfg; uint64_t addr, mask; uint64_t bar = 0; if ((bi = pci_businfo[bus]) != NULL) { si = &bi->slotinfo[slot]; pi = si->si_funcs[func].fi_devi; } else pi = NULL; /* * Just return if there is no device at this slot:func or if the * guest is doing an un-aligned access. */ if (pi == NULL || (bytes != 1 && bytes != 2 && bytes != 4) || (coff & (bytes - 1)) != 0) { if (in) *valp = 0xffffffff; return; } /* * Ignore all writes beyond the standard config space and return all * ones on reads. */ if (coff >= PCI_REGMAX + 1) { if (in) { *valp = 0xffffffff; /* * Extended capabilities begin at offset 256 in config * space. Absence of extended capabilities is signaled * with all 0s in the extended capability header at * offset 256. */ if (coff <= PCI_REGMAX + 4) *valp = 0x00000000; } return; } pe = pi->pi_d; /* * Config read */ if (in) { /* Let the device emulation override the default handler */ needcfg = PE_CFGRW_DEFAULT; if (pe->pe_cfgread != NULL) needcfg = pe->pe_cfgread(pi, coff, bytes, valp); if (needcfg != PE_CFGRW_DROP) *valp = CFGREAD(pi, coff, bytes); pci_emul_hdrtype_fixup(bus, slot, coff, bytes, valp); } else { /* Let the device emulation override the default handler */ if (pe->pe_cfgwrite != NULL && pe->pe_cfgwrite(pi, coff, bytes, *valp) == PE_CFGRW_DROP) { return; } /* * Special handling for write to BAR and ROM registers */ if (is_pcir_bar(coff) || is_pcir_bios(coff)) { /* * Ignore writes to BAR registers that are not * 4-byte aligned. */ if (bytes != 4 || (coff & 0x3) != 0) return; if (is_pcir_bar(coff)) { idx = (coff - PCIR_BAR(0)) / 4; } else if (is_pcir_bios(coff)) { idx = PCI_ROM_IDX; } else { errx(4, "%s: invalid BAR offset %d", __func__, coff); } mask = ~(pi->pi_bar[idx].size - 1); switch (pi->pi_bar[idx].type) { case PCIBAR_NONE: pi->pi_bar[idx].addr = bar = 0; break; case PCIBAR_IO: addr = *valp & mask; addr &= 0xffff; bar = addr | pi->pi_bar[idx].lobits; /* * Register the new BAR value for interception */ if (addr != pi->pi_bar[idx].addr) { update_bar_address(pi, addr, idx, PCIBAR_IO); } break; case PCIBAR_MEM32: addr = bar = *valp & mask; bar |= pi->pi_bar[idx].lobits; if (addr != pi->pi_bar[idx].addr) { update_bar_address(pi, addr, idx, PCIBAR_MEM32); } break; case PCIBAR_MEM64: addr = bar = *valp & mask; bar |= pi->pi_bar[idx].lobits; if (addr != (uint32_t)pi->pi_bar[idx].addr) { update_bar_address(pi, addr, idx, PCIBAR_MEM64); } break; case PCIBAR_MEMHI64: mask = ~(pi->pi_bar[idx - 1].size - 1); addr = ((uint64_t)*valp << 32) & mask; bar = addr >> 32; if (bar != pi->pi_bar[idx - 1].addr >> 32) { update_bar_address(pi, addr, idx - 1, PCIBAR_MEMHI64); } break; case PCIBAR_ROM: addr = bar = *valp & mask; if (memen(pi) && romen(pi)) { unregister_bar(pi, idx); } pi->pi_bar[idx].addr = addr; pi->pi_bar[idx].lobits = *valp & PCIM_BIOS_ENABLE; /* romen could have changed it value */ if (memen(pi) && romen(pi)) { register_bar(pi, idx); } bar |= pi->pi_bar[idx].lobits; break; default: assert(0); } pci_set_cfgdata32(pi, coff, bar); } else if (pci_emul_iscap(pi, coff)) { pci_emul_capwrite(pi, coff, bytes, *valp, 0, 0); } else if (coff >= PCIR_COMMAND && coff < PCIR_REVID) { pci_emul_cmdsts_write(pi, coff, *valp, bytes); } else { CFGWRITE(pi, coff, *valp, bytes); } } } static int cfgenable, cfgbus, cfgslot, cfgfunc, cfgoff; static int pci_emul_cfgaddr(struct vmctx *ctx __unused, int in, int port __unused, int bytes, uint32_t *eax, void *arg __unused) { uint32_t x; if (bytes != 4) { if (in) *eax = (bytes == 2) ? 0xffff : 0xff; return (0); } if (in) { x = (cfgbus << 16) | (cfgslot << 11) | (cfgfunc << 8) | cfgoff; if (cfgenable) x |= CONF1_ENABLE; *eax = x; } else { x = *eax; cfgenable = (x & CONF1_ENABLE) == CONF1_ENABLE; cfgoff = (x & PCI_REGMAX) & ~0x03; cfgfunc = (x >> 8) & PCI_FUNCMAX; cfgslot = (x >> 11) & PCI_SLOTMAX; cfgbus = (x >> 16) & PCI_BUSMAX; } return (0); } INOUT_PORT(pci_cfgaddr, CONF1_ADDR_PORT, IOPORT_F_INOUT, pci_emul_cfgaddr); static int pci_emul_cfgdata(struct vmctx *ctx __unused, int in, int port, int bytes, uint32_t *eax, void *arg __unused) { int coff; assert(bytes == 1 || bytes == 2 || bytes == 4); coff = cfgoff + (port - CONF1_DATA_PORT); if (cfgenable) { pci_cfgrw(in, cfgbus, cfgslot, cfgfunc, coff, bytes, eax); } else { /* Ignore accesses to cfgdata if not enabled by cfgaddr */ if (in) *eax = 0xffffffff; } return (0); } INOUT_PORT(pci_cfgdata, CONF1_DATA_PORT+0, IOPORT_F_INOUT, pci_emul_cfgdata); INOUT_PORT(pci_cfgdata, CONF1_DATA_PORT+1, IOPORT_F_INOUT, pci_emul_cfgdata); INOUT_PORT(pci_cfgdata, CONF1_DATA_PORT+2, IOPORT_F_INOUT, pci_emul_cfgdata); INOUT_PORT(pci_cfgdata, CONF1_DATA_PORT+3, IOPORT_F_INOUT, pci_emul_cfgdata); #define PCI_EMUL_TEST #ifdef PCI_EMUL_TEST /* * Define a dummy test device */ #define DIOSZ 8 #define DMEMSZ 4096 struct pci_emul_dsoftc { uint8_t ioregs[DIOSZ]; uint8_t memregs[2][DMEMSZ]; }; #define PCI_EMUL_MSI_MSGS 4 #define PCI_EMUL_MSIX_MSGS 16 static int pci_emul_dinit(struct pci_devinst *pi, nvlist_t *nvl __unused) { int error; struct pci_emul_dsoftc *sc; sc = calloc(1, sizeof(struct pci_emul_dsoftc)); pi->pi_arg = sc; pci_set_cfgdata16(pi, PCIR_DEVICE, 0x0001); pci_set_cfgdata16(pi, PCIR_VENDOR, 0x10DD); pci_set_cfgdata8(pi, PCIR_CLASS, 0x02); error = pci_emul_add_msicap(pi, PCI_EMUL_MSI_MSGS); assert(error == 0); error = pci_emul_alloc_bar(pi, 0, PCIBAR_IO, DIOSZ); assert(error == 0); error = pci_emul_alloc_bar(pi, 1, PCIBAR_MEM32, DMEMSZ); assert(error == 0); error = pci_emul_alloc_bar(pi, 2, PCIBAR_MEM32, DMEMSZ); assert(error == 0); return (0); } static void pci_emul_diow(struct pci_devinst *pi, int baridx, uint64_t offset, int size, uint64_t value) { int i; struct pci_emul_dsoftc *sc = pi->pi_arg; if (baridx == 0) { if (offset + size > DIOSZ) { printf("diow: iow too large, offset %ld size %d\n", offset, size); return; } if (size == 1) { sc->ioregs[offset] = value & 0xff; } else if (size == 2) { *(uint16_t *)&sc->ioregs[offset] = value & 0xffff; } else if (size == 4) { *(uint32_t *)&sc->ioregs[offset] = value; } else { printf("diow: iow unknown size %d\n", size); } /* * Special magic value to generate an interrupt */ if (offset == 4 && size == 4 && pci_msi_enabled(pi)) pci_generate_msi(pi, value % pci_msi_maxmsgnum(pi)); if (value == 0xabcdef) { for (i = 0; i < pci_msi_maxmsgnum(pi); i++) pci_generate_msi(pi, i); } } if (baridx == 1 || baridx == 2) { if (offset + size > DMEMSZ) { printf("diow: memw too large, offset %ld size %d\n", offset, size); return; } i = baridx - 1; /* 'memregs' index */ if (size == 1) { sc->memregs[i][offset] = value; } else if (size == 2) { *(uint16_t *)&sc->memregs[i][offset] = value; } else if (size == 4) { *(uint32_t *)&sc->memregs[i][offset] = value; } else if (size == 8) { *(uint64_t *)&sc->memregs[i][offset] = value; } else { printf("diow: memw unknown size %d\n", size); } /* * magic interrupt ?? */ } if (baridx > 2 || baridx < 0) { printf("diow: unknown bar idx %d\n", baridx); } } static uint64_t pci_emul_dior(struct pci_devinst *pi, int baridx, uint64_t offset, int size) { struct pci_emul_dsoftc *sc = pi->pi_arg; uint32_t value; int i; value = 0; if (baridx == 0) { if (offset + size > DIOSZ) { printf("dior: ior too large, offset %ld size %d\n", offset, size); return (0); } value = 0; if (size == 1) { value = sc->ioregs[offset]; } else if (size == 2) { value = *(uint16_t *) &sc->ioregs[offset]; } else if (size == 4) { value = *(uint32_t *) &sc->ioregs[offset]; } else { printf("dior: ior unknown size %d\n", size); } } if (baridx == 1 || baridx == 2) { if (offset + size > DMEMSZ) { printf("dior: memr too large, offset %ld size %d\n", offset, size); return (0); } i = baridx - 1; /* 'memregs' index */ if (size == 1) { value = sc->memregs[i][offset]; } else if (size == 2) { value = *(uint16_t *) &sc->memregs[i][offset]; } else if (size == 4) { value = *(uint32_t *) &sc->memregs[i][offset]; } else if (size == 8) { value = *(uint64_t *) &sc->memregs[i][offset]; } else { printf("dior: ior unknown size %d\n", size); } } if (baridx > 2 || baridx < 0) { printf("dior: unknown bar idx %d\n", baridx); return (0); } return (value); } static const struct pci_devemu pci_dummy = { .pe_emu = "dummy", .pe_init = pci_emul_dinit, .pe_barwrite = pci_emul_diow, .pe_barread = pci_emul_dior, }; PCI_EMUL_SET(pci_dummy); #endif /* PCI_EMUL_TEST */ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2011 NetApp, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * Copyright 2018 Joyent, Inc. * Copyright 2025 Oxide Computer Company */ #ifndef _PCI_EMUL_H_ #define _PCI_EMUL_H_ #include #include #include #include #include #include #include #include #define PCI_BARMAX PCIR_MAX_BAR_0 /* BAR registers in a Type 0 header */ #define PCI_BARMAX_WITH_ROM (PCI_BARMAX + 1) #define PCI_ROM_IDX (PCI_BARMAX + 1) struct vmctx; struct pci_devinst; struct memory_region; struct pci_devemu { const char *pe_emu; /* Name of device emulation */ /* instance creation */ int (*pe_init)(struct pci_devinst *, nvlist_t *); int (*pe_legacy_config)(nvlist_t *, const char *); const char *pe_alias; /* ACPI DSDT enumeration */ void (*pe_write_dsdt)(struct pci_devinst *); /* config space read/write callbacks */ int (*pe_cfgwrite)(struct pci_devinst *pi, int offset, int bytes, uint32_t val); int (*pe_cfgread)(struct pci_devinst *pi, int offset, int bytes, uint32_t *retval); /* BAR read/write callbacks */ void (*pe_barwrite)(struct pci_devinst *pi, int baridx, uint64_t offset, int size, uint64_t value); uint64_t (*pe_barread)(struct pci_devinst *pi, int baridx, uint64_t offset, int size); void (*pe_baraddr)(struct pci_devinst *pi, int baridx, int enabled, uint64_t address); #ifndef __FreeBSD__ void (*pe_lintrupdate)(struct pci_devinst *pi); #endif /* __FreeBSD__ */ }; #define PCI_EMUL_SET(x) DATA_SET(pci_devemu_set, x) /* * These values are returned by the config space read/write callbacks * (pe_cfgwrite and pe_cfgread in the above structure). * * A return value of PE_CFGRW_DEFAULT will cause the PCI emulation framework to * continue on and access the configuration space as if the callback did not * exist, whereas PE_CFGRW_DROP will not. */ #define PE_CFGRW_DROP (0) #define PE_CFGRW_DEFAULT (1) enum pcibar_type { PCIBAR_NONE, PCIBAR_IO, PCIBAR_MEM32, PCIBAR_MEM64, PCIBAR_MEMHI64, PCIBAR_ROM, }; struct pcibar { enum pcibar_type type; /* io or memory */ uint64_t size; uint64_t addr; uint8_t lobits; }; #define PI_NAMESZ 40 struct msix_table_entry { uint64_t addr; uint32_t msg_data; uint32_t vector_control; } __packed; /* * In case the structure is modified to hold extra information, use a define * for the size that should be emulated. */ #define MSIX_TABLE_ENTRY_SIZE 16 #define MAX_MSIX_TABLE_ENTRIES 2048 #define PBA_SIZE(msgnum) (roundup2((msgnum), 64) / 8) enum lintr_stat { IDLE, ASSERTED, PENDING }; struct pci_devinst { struct pci_devemu *pi_d; struct vmctx *pi_vmctx; uint8_t pi_bus, pi_slot, pi_func; char pi_name[PI_NAMESZ]; int pi_bar_getsize; int pi_prevcap; int pi_capend; struct { int8_t pin; enum lintr_stat state; int pirq_pin; int ioapic_irq; pthread_mutex_t lock; } pi_lintr; struct { int enabled; uint64_t addr; uint64_t msg_data; int maxmsgnum; } pi_msi; struct { int enabled; int table_bar; int pba_bar; uint32_t table_offset; int table_count; uint32_t pba_offset; int pba_size; int function_mask; struct msix_table_entry *table; /* allocated at runtime */ void *pba_page; int pba_page_offset; uint8_t *mapped_addr; size_t mapped_size; } pi_msix; void *pi_arg; /* devemu-private data */ u_char pi_cfgdata[PCI_REGMAX + 1]; /* ROM is handled like a BAR */ struct pcibar pi_bar[PCI_BARMAX_WITH_ROM + 1]; uint64_t pi_romoffset; }; struct msicap { uint8_t capid; uint8_t nextptr; uint16_t msgctrl; uint32_t addrlo; uint32_t addrhi; uint16_t msgdata; } __packed; static_assert(sizeof(struct msicap) == 14, "compile-time assertion failed"); struct msixcap { uint8_t capid; uint8_t nextptr; uint16_t msgctrl; uint32_t table_info; /* bar index and offset within it */ uint32_t pba_info; /* bar index and offset within it */ } __packed; static_assert(sizeof(struct msixcap) == 12, "compile-time assertion failed"); struct pciecap { uint8_t capid; uint8_t nextptr; uint16_t pcie_capabilities; uint32_t dev_capabilities; /* all devices */ uint16_t dev_control; uint16_t dev_status; uint32_t link_capabilities; /* devices with links */ uint16_t link_control; uint16_t link_status; uint32_t slot_capabilities; /* ports with slots */ uint16_t slot_control; uint16_t slot_status; uint16_t root_control; /* root ports */ uint16_t root_capabilities; uint32_t root_status; uint32_t dev_capabilities2; /* all devices */ uint16_t dev_control2; uint16_t dev_status2; uint32_t link_capabilities2; /* devices with links */ uint16_t link_control2; uint16_t link_status2; uint32_t slot_capabilities2; /* ports with slots */ uint16_t slot_control2; uint16_t slot_status2; } __packed; static_assert(sizeof(struct pciecap) == 60, "compile-time assertion failed"); typedef void (*pci_lintr_cb)(int b, int s, int pin, int pirq_pin, int ioapic_irq, void *arg); int init_pci(struct vmctx *ctx); void pci_callback(void); uint32_t pci_config_read_reg(const struct pcisel *host_sel, nvlist_t *nvl, uint32_t reg, uint8_t size, uint32_t def); int pci_emul_alloc_bar(struct pci_devinst *pdi, int idx, enum pcibar_type type, uint64_t size); int pci_emul_alloc_rom(struct pci_devinst *const pdi, const uint64_t size, void **const addr); int pci_emul_add_boot_device(struct pci_devinst *const pi, const int bootindex); int pci_emul_add_capability(struct pci_devinst *pi, u_char *capdata, int caplen, int *capoffp); int pci_emul_add_msicap(struct pci_devinst *pi, int msgnum); int pci_emul_add_pciecap(struct pci_devinst *pi, int pcie_device_type); void pci_emul_capwrite(struct pci_devinst *pi, int offset, int bytes, uint32_t val, uint8_t capoff, int capid); void pci_emul_cmd_changed(struct pci_devinst *pi, uint16_t old); void pci_generate_msi(struct pci_devinst *pi, int msgnum); void pci_generate_msix(struct pci_devinst *pi, int msgnum); void pci_lintr_assert(struct pci_devinst *pi); void pci_lintr_deassert(struct pci_devinst *pi); void pci_lintr_request(struct pci_devinst *pi); int pci_msi_enabled(struct pci_devinst *pi); int pci_msix_enabled(struct pci_devinst *pi); int pci_msix_table_bar(struct pci_devinst *pi); int pci_msix_pba_bar(struct pci_devinst *pi); int pci_msi_maxmsgnum(struct pci_devinst *pi); int pci_parse_legacy_config(nvlist_t *nvl, const char *opt); int pci_parse_slot(char *opt); void pci_print_supported_devices(void); void pci_populate_msicap(struct msicap *cap, int msgs, int nextptr); int pci_emul_add_msixcap(struct pci_devinst *pi, int msgnum, int barnum); int pci_emul_msix_twrite(struct pci_devinst *pi, uint64_t offset, int size, uint64_t value); uint64_t pci_emul_msix_tread(struct pci_devinst *pi, uint64_t offset, int size); int pci_count_lintr(int bus); void pci_walk_lintr(int bus, pci_lintr_cb cb, void *arg); void pci_write_dsdt(void); uint64_t pci_ecfg_base(void); int pci_bus_configured(int bus); static __inline void pci_set_cfgdata8(struct pci_devinst *pi, int offset, uint8_t val) { assert(offset <= PCI_REGMAX); *(uint8_t *)(pi->pi_cfgdata + offset) = val; } static __inline void pci_set_cfgdata16(struct pci_devinst *pi, int offset, uint16_t val) { assert(offset <= (PCI_REGMAX - 1) && (offset & 1) == 0); *(uint16_t *)(pi->pi_cfgdata + offset) = val; } static __inline void pci_set_cfgdata32(struct pci_devinst *pi, int offset, uint32_t val) { assert(offset <= (PCI_REGMAX - 3) && (offset & 3) == 0); *(uint32_t *)(pi->pi_cfgdata + offset) = val; } static __inline uint8_t pci_get_cfgdata8(struct pci_devinst *pi, int offset) { assert(offset <= PCI_REGMAX); return (*(uint8_t *)(pi->pi_cfgdata + offset)); } static __inline uint16_t pci_get_cfgdata16(struct pci_devinst *pi, int offset) { assert(offset <= (PCI_REGMAX - 1) && (offset & 1) == 0); return (*(uint16_t *)(pi->pi_cfgdata + offset)); } static __inline uint32_t pci_get_cfgdata32(struct pci_devinst *pi, int offset) { assert(offset <= (PCI_REGMAX - 3) && (offset & 3) == 0); return (*(uint32_t *)(pi->pi_cfgdata + offset)); } #endif /* _PCI_EMUL_H_ */ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2015 Nahanni Systems, Inc. * Copyright 2018 Joyent, Inc. * Copyright 2021 OmniOS Community Edition (OmniOSce) Association. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include #include #include #include #include #include #include #include #include #include "bhyvegc.h" #include "bhyverun.h" #include "config.h" #include "debug.h" #include "console.h" #include "inout.h" #include "pci_emul.h" #include "rfb.h" #include "vga.h" /* * bhyve Framebuffer device emulation. * BAR0 points to the current mode information. * BAR1 is the 32-bit framebuffer address. * * -s ,fbuf,wait,vga=on|io|off,rfb=:port,w=width,h=height */ static int fbuf_debug = 1; #define DEBUG_INFO 1 #define DEBUG_VERBOSE 4 #define DPRINTF(level, params) if (level <= fbuf_debug) PRINTLN params #define KB (1024UL) #define MB (1024 * 1024UL) #define DMEMSZ 128 #define FB_SIZE (32*MB) #define COLS_MAX 3840 #define ROWS_MAX 2160 #define COLS_DEFAULT 1024 #define ROWS_DEFAULT 768 #define COLS_MIN 640 #define ROWS_MIN 480 struct pci_fbuf_softc { struct pci_devinst *fsc_pi; struct { uint32_t fbsize; uint16_t width; uint16_t height; uint16_t depth; uint16_t refreshrate; uint8_t reserved[116]; } __packed memregs; /* rfb server */ char *rfb_host; char *rfb_password; int rfb_port; #ifndef __FreeBSD__ const char *rfb_unix; #endif int rfb_wait; int vga_enabled; int vga_full; uint32_t fbaddr; char *fb_base; uint16_t gc_width; uint16_t gc_height; void *vgasc; struct bhyvegc_image *gc_image; }; static struct pci_fbuf_softc *fbuf_sc; #define PCI_FBUF_MSI_MSGS 4 static void pci_fbuf_write(struct pci_devinst *pi, int baridx, uint64_t offset, int size, uint64_t value) { struct pci_fbuf_softc *sc; uint8_t *p; assert(baridx == 0); sc = pi->pi_arg; DPRINTF(DEBUG_VERBOSE, ("fbuf wr: offset 0x%lx, size: %d, value: 0x%lx", offset, size, value)); if (offset + size > DMEMSZ) { printf("fbuf: write too large, offset %ld size %d\n", offset, size); return; } p = (uint8_t *)&sc->memregs + offset; switch (size) { case 1: *p = value; break; case 2: *(uint16_t *)p = value; break; case 4: *(uint32_t *)p = value; break; case 8: *(uint64_t *)p = value; break; default: printf("fbuf: write unknown size %d\n", size); break; } if (!sc->gc_image->vgamode && sc->memregs.width == 0 && sc->memregs.height == 0) { DPRINTF(DEBUG_INFO, ("switching to VGA mode")); sc->gc_image->vgamode = 1; sc->gc_width = 0; sc->gc_height = 0; } else if (sc->gc_image->vgamode && sc->memregs.width != 0 && sc->memregs.height != 0) { DPRINTF(DEBUG_INFO, ("switching to VESA mode")); sc->gc_image->vgamode = 0; } } static uint64_t pci_fbuf_read(struct pci_devinst *pi, int baridx, uint64_t offset, int size) { struct pci_fbuf_softc *sc; uint8_t *p; uint64_t value; assert(baridx == 0); sc = pi->pi_arg; if (offset + size > DMEMSZ) { printf("fbuf: read too large, offset %ld size %d\n", offset, size); return (0); } p = (uint8_t *)&sc->memregs + offset; value = 0; switch (size) { case 1: value = *p; break; case 2: value = *(uint16_t *)p; break; case 4: value = *(uint32_t *)p; break; case 8: value = *(uint64_t *)p; break; default: printf("fbuf: read unknown size %d\n", size); break; } DPRINTF(DEBUG_VERBOSE, ("fbuf rd: offset 0x%lx, size: %d, value: 0x%lx", offset, size, value)); return (value); } static void pci_fbuf_baraddr(struct pci_devinst *pi, int baridx, int enabled, uint64_t address) { struct pci_fbuf_softc *sc; int prot; if (baridx != 1) return; sc = pi->pi_arg; if (!enabled) { if (vm_munmap_memseg(pi->pi_vmctx, sc->fbaddr, FB_SIZE) != 0) EPRINTLN("pci_fbuf: munmap_memseg failed"); sc->fbaddr = 0; } else { prot = PROT_READ | PROT_WRITE; if (vm_mmap_memseg(pi->pi_vmctx, address, VM_FRAMEBUFFER, 0, FB_SIZE, prot) != 0) EPRINTLN("pci_fbuf: mmap_memseg failed"); else sc->fbaddr = address; } } static int pci_fbuf_parse_config(struct pci_fbuf_softc *sc, nvlist_t *nvl) { const char *value; char *cp; sc->rfb_wait = get_config_bool_node_default(nvl, "wait", false); /* Prefer "rfb" to "tcp". */ value = get_config_value_node(nvl, "rfb"); if (value == NULL) value = get_config_value_node(nvl, "tcp"); if (value != NULL) { /* * IPv4 -- host-ip:port * IPv6 -- [host-ip%zone]:port * XXX for now port is mandatory for IPv4. */ if (value[0] == '[') { cp = strchr(value + 1, ']'); if (cp == NULL || cp == value + 1) { EPRINTLN("fbuf: Invalid IPv6 address: \"%s\"", value); return (-1); } sc->rfb_host = strndup(value + 1, cp - (value + 1)); cp++; if (*cp == ':') { cp++; if (*cp == '\0') { EPRINTLN( "fbuf: Missing port number: \"%s\"", value); return (-1); } sc->rfb_port = atoi(cp); } else if (*cp != '\0') { EPRINTLN("fbuf: Invalid IPv6 address: \"%s\"", value); return (-1); } } else { cp = strchr(value, ':'); if (cp == NULL) { sc->rfb_port = atoi(value); } else { sc->rfb_host = strndup(value, cp - value); cp++; if (*cp == '\0') { EPRINTLN( "fbuf: Missing port number: \"%s\"", value); return (-1); } sc->rfb_port = atoi(cp); } } } #ifndef __FreeBSD__ sc->rfb_unix = get_config_value_node(nvl, "unix"); #endif value = get_config_value_node(nvl, "vga"); if (value != NULL) { if (strcmp(value, "off") == 0) { sc->vga_enabled = 0; } else if (strcmp(value, "io") == 0) { sc->vga_enabled = 1; sc->vga_full = 0; } else if (strcmp(value, "on") == 0) { sc->vga_enabled = 1; sc->vga_full = 1; } else { EPRINTLN("fbuf: Invalid vga setting: \"%s\"", value); return (-1); } } value = get_config_value_node(nvl, "w"); if (value != NULL) sc->memregs.width = strtol(value, NULL, 10); value = get_config_value_node(nvl, "h"); if (value != NULL) sc->memregs.height = strtol(value, NULL, 10); if (sc->memregs.width > COLS_MAX || sc->memregs.height > ROWS_MAX) { EPRINTLN("fbuf: max resolution is %ux%u", COLS_MAX, ROWS_MAX); return (-1); } if (sc->memregs.width < COLS_MIN || sc->memregs.height < ROWS_MIN) { EPRINTLN("fbuf: minimum resolution is %ux%u", COLS_MIN, ROWS_MIN); return (-1); } value = get_config_value_node(nvl, "password"); if (value != NULL) sc->rfb_password = strdup(value); return (0); } extern void vga_render(struct bhyvegc *gc, void *arg); static void pci_fbuf_render(struct bhyvegc *gc, void *arg) { struct pci_fbuf_softc *sc; sc = arg; if (sc->vga_full && sc->gc_image->vgamode) { /* TODO: mode switching to vga and vesa should use the special * EFI-bhyve protocol port. */ vga_render(gc, sc->vgasc); return; } if (sc->gc_width != sc->memregs.width || sc->gc_height != sc->memregs.height) { bhyvegc_resize(gc, sc->memregs.width, sc->memregs.height); sc->gc_width = sc->memregs.width; sc->gc_height = sc->memregs.height; } } static int pci_fbuf_init(struct pci_devinst *pi, nvlist_t *nvl) { int error; struct pci_fbuf_softc *sc; if (fbuf_sc != NULL) { EPRINTLN("Only one frame buffer device is allowed."); return (-1); } sc = calloc(1, sizeof(struct pci_fbuf_softc)); pi->pi_arg = sc; /* initialize config space */ pci_set_cfgdata16(pi, PCIR_DEVICE, 0x40FB); pci_set_cfgdata16(pi, PCIR_VENDOR, 0xFB5D); pci_set_cfgdata8(pi, PCIR_CLASS, PCIC_DISPLAY); pci_set_cfgdata8(pi, PCIR_SUBCLASS, PCIS_DISPLAY_VGA); sc->fb_base = vm_create_devmem(pi->pi_vmctx, VM_FRAMEBUFFER, "framebuffer", FB_SIZE); if (sc->fb_base == MAP_FAILED) { error = -1; goto done; } error = pci_emul_alloc_bar(pi, 0, PCIBAR_MEM32, DMEMSZ); assert(error == 0); error = pci_emul_alloc_bar(pi, 1, PCIBAR_MEM32, FB_SIZE); assert(error == 0); error = pci_emul_add_msicap(pi, PCI_FBUF_MSI_MSGS); assert(error == 0); sc->memregs.fbsize = FB_SIZE; sc->memregs.width = COLS_DEFAULT; sc->memregs.height = ROWS_DEFAULT; sc->memregs.depth = 32; sc->vga_enabled = 1; sc->vga_full = 0; sc->fsc_pi = pi; error = pci_fbuf_parse_config(sc, nvl); if (error != 0) goto done; /* XXX until VGA rendering is enabled */ if (sc->vga_full != 0) { EPRINTLN("pci_fbuf: VGA rendering not enabled"); #ifndef __FreeBSD__ errno = ENOTSUP; error = -1; #endif goto done; } DPRINTF(DEBUG_INFO, ("fbuf frame buffer base: %p [sz %lu]", sc->fb_base, FB_SIZE)); console_init(sc->memregs.width, sc->memregs.height, sc->fb_base); console_fb_register(pci_fbuf_render, sc); if (sc->vga_enabled) sc->vgasc = vga_init(!sc->vga_full); sc->gc_image = console_get_image(); fbuf_sc = sc; memset((void *)sc->fb_base, 0, FB_SIZE); #ifdef __FreeBSD__ error = rfb_init(sc->rfb_host, sc->rfb_port, sc->rfb_wait, sc->rfb_password); #else char *name; (void) asprintf(&name, "%s (bhyve)", get_config_value("name")); if (sc->rfb_unix != NULL) { error = rfb_init((char *)sc->rfb_unix, -1, sc->rfb_wait, sc->rfb_password, name != NULL ? name : "bhyve"); } else { error = rfb_init(sc->rfb_host, sc->rfb_port, sc->rfb_wait, sc->rfb_password, name != NULL ? name : "bhyve"); } if (error != 0) free(name); #endif done: if (error) free(sc); return (error); } static const struct pci_devemu pci_fbuf = { .pe_emu = "fbuf", .pe_init = pci_fbuf_init, .pe_barwrite = pci_fbuf_write, .pe_barread = pci_fbuf_read, .pe_baraddr = pci_fbuf_baraddr, }; PCI_EMUL_SET(pci_fbuf); /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2016 Alex Teaca * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * */ #include #include #include "pci_hda.h" #include "bhyverun.h" #include "config.h" #include "pci_emul.h" #include "hdac_reg.h" /* * HDA defines */ #define PCIR_HDCTL 0x40 #define INTEL_VENDORID 0x8086 #define HDA_INTEL_82801G 0x27d8 #define HDA_IOSS_NO 0x08 #define HDA_OSS_NO 0x04 #define HDA_ISS_NO 0x04 #define HDA_CODEC_MAX 0x0f #define HDA_LAST_OFFSET \ (0x2084 + ((HDA_ISS_NO) * 0x20) + ((HDA_OSS_NO) * 0x20)) #define HDA_CORB_ENTRY_LEN 0x04 #define HDA_RIRB_ENTRY_LEN 0x08 #define HDA_BDL_ENTRY_LEN 0x10 #define HDA_DMA_PIB_ENTRY_LEN 0x08 #define HDA_STREAM_TAGS_CNT 0x10 #define HDA_STREAM_REGS_BASE 0x80 #define HDA_STREAM_REGS_LEN 0x20 #define HDA_DMA_ACCESS_LEN (sizeof(uint32_t)) #define HDA_BDL_MAX_LEN 0x0100 #define HDAC_SDSTS_FIFORDY (1 << 5) #define HDA_RIRBSTS_IRQ_MASK (HDAC_RIRBSTS_RINTFL | HDAC_RIRBSTS_RIRBOIS) #define HDA_STATESTS_IRQ_MASK ((1 << HDA_CODEC_MAX) - 1) #define HDA_SDSTS_IRQ_MASK \ (HDAC_SDSTS_DESE | HDAC_SDSTS_FIFOE | HDAC_SDSTS_BCIS) /* * HDA data structures */ struct hda_softc; typedef void (*hda_set_reg_handler)(struct hda_softc *sc, uint32_t offset, uint32_t old); struct hda_bdle { uint32_t addrl; uint32_t addrh; uint32_t len; uint32_t ioc; } __packed; struct hda_bdle_desc { void *addr; uint8_t ioc; uint32_t len; }; struct hda_codec_cmd_ctl { const char *name; void *dma_vaddr; uint8_t run; uint16_t rp; uint16_t size; uint16_t wp; }; struct hda_stream_desc { uint8_t dir; uint8_t run; uint8_t stream; /* bp is the no. of bytes transferred in the current bdle */ uint32_t bp; /* be is the no. of bdles transferred in the bdl */ uint32_t be; uint32_t bdl_cnt; struct hda_bdle_desc bdl[HDA_BDL_MAX_LEN]; }; struct hda_softc { struct pci_devinst *pci_dev; uint32_t regs[HDA_LAST_OFFSET]; uint8_t lintr; uint8_t rirb_cnt; uint64_t wall_clock_start; struct hda_codec_cmd_ctl corb; struct hda_codec_cmd_ctl rirb; uint8_t codecs_no; struct hda_codec_inst *codecs[HDA_CODEC_MAX]; /* Base Address of the DMA Position Buffer */ void *dma_pib_vaddr; struct hda_stream_desc streams[HDA_IOSS_NO]; /* 2 tables for output and input */ uint8_t stream_map[2][HDA_STREAM_TAGS_CNT]; }; /* * HDA module function declarations */ static inline void hda_set_reg_by_offset(struct hda_softc *sc, uint32_t offset, uint32_t value); static inline uint32_t hda_get_reg_by_offset(struct hda_softc *sc, uint32_t offset); static inline void hda_set_field_by_offset(struct hda_softc *sc, uint32_t offset, uint32_t mask, uint32_t value); static struct hda_softc *hda_init(nvlist_t *nvl); static void hda_update_intr(struct hda_softc *sc); static void hda_response_interrupt(struct hda_softc *sc); static int hda_codec_constructor(struct hda_softc *sc, struct hda_codec_class *codec, const char *play, const char *rec); static struct hda_codec_class *hda_find_codec_class(const char *name); static int hda_send_command(struct hda_softc *sc, uint32_t verb); static int hda_notify_codecs(struct hda_softc *sc, uint8_t run, uint8_t stream, uint8_t dir); static void hda_reset(struct hda_softc *sc); static void hda_reset_regs(struct hda_softc *sc); static void hda_stream_reset(struct hda_softc *sc, uint8_t stream_ind); static int hda_stream_start(struct hda_softc *sc, uint8_t stream_ind); static int hda_stream_stop(struct hda_softc *sc, uint8_t stream_ind); static uint32_t hda_read(struct hda_softc *sc, uint32_t offset); static int hda_write(struct hda_softc *sc, uint32_t offset, uint8_t size, uint32_t value); static inline void hda_print_cmd_ctl_data(struct hda_codec_cmd_ctl *p); static int hda_corb_start(struct hda_softc *sc); static int hda_corb_run(struct hda_softc *sc); static int hda_rirb_start(struct hda_softc *sc); static void *hda_dma_get_vaddr(struct hda_softc *sc, uint64_t dma_paddr, size_t len); static void hda_dma_st_dword(void *dma_vaddr, uint32_t data); static uint32_t hda_dma_ld_dword(void *dma_vaddr); static inline uint8_t hda_get_stream_by_offsets(uint32_t offset, uint8_t reg_offset); static inline uint32_t hda_get_offset_stream(uint8_t stream_ind); static void hda_set_gctl(struct hda_softc *sc, uint32_t offset, uint32_t old); static void hda_set_statests(struct hda_softc *sc, uint32_t offset, uint32_t old); static void hda_set_corbwp(struct hda_softc *sc, uint32_t offset, uint32_t old); static void hda_set_corbctl(struct hda_softc *sc, uint32_t offset, uint32_t old); static void hda_set_rirbctl(struct hda_softc *sc, uint32_t offset, uint32_t old); static void hda_set_rirbsts(struct hda_softc *sc, uint32_t offset, uint32_t old); static void hda_set_dpiblbase(struct hda_softc *sc, uint32_t offset, uint32_t old); static void hda_set_sdctl(struct hda_softc *sc, uint32_t offset, uint32_t old); static void hda_set_sdctl2(struct hda_softc *sc, uint32_t offset, uint32_t old); static void hda_set_sdsts(struct hda_softc *sc, uint32_t offset, uint32_t old); static int hda_signal_state_change(struct hda_codec_inst *hci); static int hda_response(struct hda_codec_inst *hci, uint32_t response, uint8_t unsol); static int hda_transfer(struct hda_codec_inst *hci, uint8_t stream, uint8_t dir, uint8_t *buf, size_t count); static void hda_set_pib(struct hda_softc *sc, uint8_t stream_ind, uint32_t pib); static uint64_t hda_get_clock_ns(void); /* * PCI HDA function declarations */ static int pci_hda_init(struct pci_devinst *pi, nvlist_t *nvl); static void pci_hda_write(struct pci_devinst *pi, int baridx, uint64_t offset, int size, uint64_t value); static uint64_t pci_hda_read(struct pci_devinst *pi, int baridx, uint64_t offset, int size); /* * HDA global data */ static const hda_set_reg_handler hda_set_reg_table[] = { [HDAC_GCTL] = hda_set_gctl, [HDAC_STATESTS] = hda_set_statests, [HDAC_CORBWP] = hda_set_corbwp, [HDAC_CORBCTL] = hda_set_corbctl, [HDAC_RIRBCTL] = hda_set_rirbctl, [HDAC_RIRBSTS] = hda_set_rirbsts, [HDAC_DPIBLBASE] = hda_set_dpiblbase, #define HDAC_ISTREAM(n, iss, oss) \ [_HDAC_ISDCTL(n, iss, oss)] = hda_set_sdctl, \ [_HDAC_ISDCTL(n, iss, oss) + 2] = hda_set_sdctl2, \ [_HDAC_ISDSTS(n, iss, oss)] = hda_set_sdsts, \ #define HDAC_OSTREAM(n, iss, oss) \ [_HDAC_OSDCTL(n, iss, oss)] = hda_set_sdctl, \ [_HDAC_OSDCTL(n, iss, oss) + 2] = hda_set_sdctl2, \ [_HDAC_OSDSTS(n, iss, oss)] = hda_set_sdsts, \ HDAC_ISTREAM(0, HDA_ISS_NO, HDA_OSS_NO) HDAC_ISTREAM(1, HDA_ISS_NO, HDA_OSS_NO) HDAC_ISTREAM(2, HDA_ISS_NO, HDA_OSS_NO) HDAC_ISTREAM(3, HDA_ISS_NO, HDA_OSS_NO) HDAC_OSTREAM(0, HDA_ISS_NO, HDA_OSS_NO) HDAC_OSTREAM(1, HDA_ISS_NO, HDA_OSS_NO) HDAC_OSTREAM(2, HDA_ISS_NO, HDA_OSS_NO) HDAC_OSTREAM(3, HDA_ISS_NO, HDA_OSS_NO) }; static const uint16_t hda_corb_sizes[] = { [HDAC_CORBSIZE_CORBSIZE_2] = 2, [HDAC_CORBSIZE_CORBSIZE_16] = 16, [HDAC_CORBSIZE_CORBSIZE_256] = 256, [HDAC_CORBSIZE_CORBSIZE_MASK] = 0, }; static const uint16_t hda_rirb_sizes[] = { [HDAC_RIRBSIZE_RIRBSIZE_2] = 2, [HDAC_RIRBSIZE_RIRBSIZE_16] = 16, [HDAC_RIRBSIZE_RIRBSIZE_256] = 256, [HDAC_RIRBSIZE_RIRBSIZE_MASK] = 0, }; static const struct hda_ops hops = { .signal = hda_signal_state_change, .response = hda_response, .transfer = hda_transfer, }; static const struct pci_devemu pci_de_hda = { .pe_emu = "hda", .pe_init = pci_hda_init, .pe_barwrite = pci_hda_write, .pe_barread = pci_hda_read }; PCI_EMUL_SET(pci_de_hda); SET_DECLARE(hda_codec_class_set, struct hda_codec_class); #if DEBUG_HDA == 1 FILE *dbg; #endif /* * HDA module function definitions */ static inline void hda_set_reg_by_offset(struct hda_softc *sc, uint32_t offset, uint32_t value) { assert(offset < HDA_LAST_OFFSET); sc->regs[offset] = value; } static inline uint32_t hda_get_reg_by_offset(struct hda_softc *sc, uint32_t offset) { assert(offset < HDA_LAST_OFFSET); return sc->regs[offset]; } static inline void hda_set_field_by_offset(struct hda_softc *sc, uint32_t offset, uint32_t mask, uint32_t value) { uint32_t reg_value = 0; reg_value = hda_get_reg_by_offset(sc, offset); reg_value &= ~mask; reg_value |= (value & mask); hda_set_reg_by_offset(sc, offset, reg_value); } static struct hda_softc * hda_init(nvlist_t *nvl) { struct hda_softc *sc = NULL; struct hda_codec_class *codec = NULL; const char *value; char *play; char *rec; int err; #if DEBUG_HDA == 1 dbg = fopen(DEBUG_HDA_FILE, "w+"); #endif sc = calloc(1, sizeof(*sc)); if (!sc) return (NULL); hda_reset_regs(sc); /* * TODO search all configured codecs * For now we play with one single codec */ codec = hda_find_codec_class("hda_codec"); if (codec) { value = get_config_value_node(nvl, "play"); if (value == NULL) play = NULL; else play = strdup(value); value = get_config_value_node(nvl, "rec"); if (value == NULL) rec = NULL; else rec = strdup(value); DPRINTF("play: %s rec: %s", play, rec); if (play != NULL || rec != NULL) { err = hda_codec_constructor(sc, codec, play, rec); assert(!err); } free(play); free(rec); } return (sc); } static void hda_update_intr(struct hda_softc *sc) { struct pci_devinst *pi = sc->pci_dev; uint32_t intctl = hda_get_reg_by_offset(sc, HDAC_INTCTL); uint32_t intsts = 0; uint32_t sdsts = 0; uint32_t rirbsts = 0; uint32_t wakeen = 0; uint32_t statests = 0; uint32_t off = 0; int i; /* update the CIS bits */ rirbsts = hda_get_reg_by_offset(sc, HDAC_RIRBSTS); if (rirbsts & (HDAC_RIRBSTS_RINTFL | HDAC_RIRBSTS_RIRBOIS)) intsts |= HDAC_INTSTS_CIS; wakeen = hda_get_reg_by_offset(sc, HDAC_WAKEEN); statests = hda_get_reg_by_offset(sc, HDAC_STATESTS); if (statests & wakeen) intsts |= HDAC_INTSTS_CIS; /* update the SIS bits */ for (i = 0; i < HDA_IOSS_NO; i++) { off = hda_get_offset_stream(i); sdsts = hda_get_reg_by_offset(sc, off + HDAC_SDSTS); if (sdsts & HDAC_SDSTS_BCIS) intsts |= (1 << i); } /* update the GIS bit */ if (intsts) intsts |= HDAC_INTSTS_GIS; hda_set_reg_by_offset(sc, HDAC_INTSTS, intsts); if ((intctl & HDAC_INTCTL_GIE) && ((intsts & \ ~HDAC_INTSTS_GIS) & intctl)) { if (!sc->lintr) { pci_lintr_assert(pi); sc->lintr = 1; } } else { if (sc->lintr) { pci_lintr_deassert(pi); sc->lintr = 0; } } } static void hda_response_interrupt(struct hda_softc *sc) { uint8_t rirbctl = hda_get_reg_by_offset(sc, HDAC_RIRBCTL); if ((rirbctl & HDAC_RIRBCTL_RINTCTL) && sc->rirb_cnt) { sc->rirb_cnt = 0; hda_set_field_by_offset(sc, HDAC_RIRBSTS, HDAC_RIRBSTS_RINTFL, HDAC_RIRBSTS_RINTFL); hda_update_intr(sc); } } static int hda_codec_constructor(struct hda_softc *sc, struct hda_codec_class *codec, const char *play, const char *rec) { struct hda_codec_inst *hci = NULL; if (sc->codecs_no >= HDA_CODEC_MAX) return (-1); hci = calloc(1, sizeof(struct hda_codec_inst)); if (!hci) return (-1); hci->hda = sc; hci->hops = &hops; hci->cad = sc->codecs_no; hci->codec = codec; sc->codecs[sc->codecs_no++] = hci; if (!codec->init) { DPRINTF("This codec does not implement the init function"); return (-1); } return (codec->init(hci, play, rec)); } static struct hda_codec_class * hda_find_codec_class(const char *name) { struct hda_codec_class **pdpp = NULL, *pdp = NULL; SET_FOREACH(pdpp, hda_codec_class_set) { pdp = *pdpp; if (!strcmp(pdp->name, name)) { return (pdp); } } return (NULL); } static int hda_send_command(struct hda_softc *sc, uint32_t verb) { struct hda_codec_inst *hci = NULL; struct hda_codec_class *codec = NULL; uint8_t cad = (verb >> HDA_CMD_CAD_SHIFT) & 0x0f; if (cad >= sc->codecs_no) return (-1); DPRINTF("cad: 0x%x verb: 0x%x", cad, verb); hci = sc->codecs[cad]; assert(hci); codec = hci->codec; assert(codec); if (!codec->command) { DPRINTF("This codec does not implement the command function"); return (-1); } return (codec->command(hci, verb)); } static int hda_notify_codecs(struct hda_softc *sc, uint8_t run, uint8_t stream, uint8_t dir) { struct hda_codec_inst *hci = NULL; struct hda_codec_class *codec = NULL; int err; int i; /* Notify each codec */ for (i = 0; i < sc->codecs_no; i++) { hci = sc->codecs[i]; assert(hci); codec = hci->codec; assert(codec); if (codec->notify) { err = codec->notify(hci, run, stream, dir); if (!err) break; } } return (i == sc->codecs_no ? (-1) : 0); } static void hda_reset(struct hda_softc *sc) { int i; struct hda_codec_inst *hci = NULL; struct hda_codec_class *codec = NULL; hda_reset_regs(sc); /* Reset each codec */ for (i = 0; i < sc->codecs_no; i++) { hci = sc->codecs[i]; assert(hci); codec = hci->codec; assert(codec); if (codec->reset) codec->reset(hci); } sc->wall_clock_start = hda_get_clock_ns(); } static void hda_reset_regs(struct hda_softc *sc) { uint32_t off = 0; uint8_t i; DPRINTF("Reset the HDA controller registers ..."); memset(sc->regs, 0, sizeof(sc->regs)); hda_set_reg_by_offset(sc, HDAC_GCAP, HDAC_GCAP_64OK | (HDA_ISS_NO << HDAC_GCAP_ISS_SHIFT) | (HDA_OSS_NO << HDAC_GCAP_OSS_SHIFT)); hda_set_reg_by_offset(sc, HDAC_VMAJ, 0x01); hda_set_reg_by_offset(sc, HDAC_OUTPAY, 0x3c); hda_set_reg_by_offset(sc, HDAC_INPAY, 0x1d); hda_set_reg_by_offset(sc, HDAC_CORBSIZE, HDAC_CORBSIZE_CORBSZCAP_256 | HDAC_CORBSIZE_CORBSIZE_256); hda_set_reg_by_offset(sc, HDAC_RIRBSIZE, HDAC_RIRBSIZE_RIRBSZCAP_256 | HDAC_RIRBSIZE_RIRBSIZE_256); for (i = 0; i < HDA_IOSS_NO; i++) { off = hda_get_offset_stream(i); hda_set_reg_by_offset(sc, off + HDAC_SDFIFOS, HDA_FIFO_SIZE); } } static void hda_stream_reset(struct hda_softc *sc, uint8_t stream_ind) { struct hda_stream_desc *st = &sc->streams[stream_ind]; uint32_t off = hda_get_offset_stream(stream_ind); DPRINTF("Reset the HDA stream: 0x%x", stream_ind); /* Reset the Stream Descriptor registers */ memset(sc->regs + HDA_STREAM_REGS_BASE + off, 0, HDA_STREAM_REGS_LEN); /* Reset the Stream Descriptor */ memset(st, 0, sizeof(*st)); hda_set_field_by_offset(sc, off + HDAC_SDSTS, HDAC_SDSTS_FIFORDY, HDAC_SDSTS_FIFORDY); hda_set_field_by_offset(sc, off + HDAC_SDCTL0, HDAC_SDCTL_SRST, HDAC_SDCTL_SRST); } static int hda_stream_start(struct hda_softc *sc, uint8_t stream_ind) { struct hda_stream_desc *st = &sc->streams[stream_ind]; struct hda_bdle_desc *bdle_desc = NULL; struct hda_bdle *bdle = NULL; uint32_t lvi = 0; uint32_t bdl_cnt = 0; uint64_t bdpl = 0; uint64_t bdpu = 0; uint64_t bdl_paddr = 0; void *bdl_vaddr = NULL; uint32_t bdle_sz = 0; uint64_t bdle_addrl = 0; uint64_t bdle_addrh = 0; uint64_t bdle_paddr = 0; void *bdle_vaddr = NULL; uint32_t off = hda_get_offset_stream(stream_ind); uint32_t sdctl = 0; uint8_t strm = 0; uint8_t dir = 0; assert(!st->run); lvi = hda_get_reg_by_offset(sc, off + HDAC_SDLVI); bdpl = hda_get_reg_by_offset(sc, off + HDAC_SDBDPL); bdpu = hda_get_reg_by_offset(sc, off + HDAC_SDBDPU); bdl_cnt = lvi + 1; assert(bdl_cnt <= HDA_BDL_MAX_LEN); bdl_paddr = bdpl | (bdpu << 32); bdl_vaddr = hda_dma_get_vaddr(sc, bdl_paddr, HDA_BDL_ENTRY_LEN * bdl_cnt); if (!bdl_vaddr) { DPRINTF("Fail to get the guest virtual address"); return (-1); } DPRINTF("stream: 0x%x bdl_cnt: 0x%x bdl_paddr: 0x%lx", stream_ind, bdl_cnt, bdl_paddr); st->bdl_cnt = bdl_cnt; bdle = (struct hda_bdle *)bdl_vaddr; for (size_t i = 0; i < bdl_cnt; i++, bdle++) { bdle_sz = bdle->len; assert(!(bdle_sz % HDA_DMA_ACCESS_LEN)); bdle_addrl = bdle->addrl; bdle_addrh = bdle->addrh; bdle_paddr = bdle_addrl | (bdle_addrh << 32); bdle_vaddr = hda_dma_get_vaddr(sc, bdle_paddr, bdle_sz); if (!bdle_vaddr) { DPRINTF("Fail to get the guest virtual address"); return (-1); } bdle_desc = &st->bdl[i]; bdle_desc->addr = bdle_vaddr; bdle_desc->len = bdle_sz; bdle_desc->ioc = bdle->ioc; DPRINTF("bdle: 0x%zx bdle_sz: 0x%x", i, bdle_sz); } sdctl = hda_get_reg_by_offset(sc, off + HDAC_SDCTL0); strm = (sdctl >> 20) & 0x0f; dir = stream_ind >= HDA_ISS_NO; DPRINTF("strm: 0x%x, dir: 0x%x", strm, dir); sc->stream_map[dir][strm] = stream_ind; st->stream = strm; st->dir = dir; st->bp = 0; st->be = 0; hda_set_pib(sc, stream_ind, 0); st->run = 1; hda_notify_codecs(sc, 1, strm, dir); return (0); } static int hda_stream_stop(struct hda_softc *sc, uint8_t stream_ind) { struct hda_stream_desc *st = &sc->streams[stream_ind]; uint8_t strm = st->stream; uint8_t dir = st->dir; DPRINTF("stream: 0x%x, strm: 0x%x, dir: 0x%x", stream_ind, strm, dir); st->run = 0; hda_notify_codecs(sc, 0, strm, dir); return (0); } static uint32_t hda_read(struct hda_softc *sc, uint32_t offset) { if (offset == HDAC_WALCLK) return (24 * (hda_get_clock_ns() - \ sc->wall_clock_start) / 1000); return (hda_get_reg_by_offset(sc, offset)); } static int hda_write(struct hda_softc *sc, uint32_t offset, uint8_t size, uint32_t value) { uint32_t old = hda_get_reg_by_offset(sc, offset); uint32_t masks[] = {0x00000000, 0x000000ff, 0x0000ffff, 0x00ffffff, 0xffffffff}; hda_set_reg_handler set_reg_handler = NULL; if (offset < nitems(hda_set_reg_table)) set_reg_handler = hda_set_reg_table[offset]; hda_set_field_by_offset(sc, offset, masks[size], value); if (set_reg_handler) set_reg_handler(sc, offset, old); return (0); } #if DEBUG_HDA == 1 static inline void hda_print_cmd_ctl_data(struct hda_codec_cmd_ctl *p) { DPRINTF("%s size: %d", p->name, p->size); DPRINTF("%s dma_vaddr: %p", p->name, p->dma_vaddr); DPRINTF("%s wp: 0x%x", p->name, p->wp); DPRINTF("%s rp: 0x%x", p->name, p->rp); } #else static inline void hda_print_cmd_ctl_data(struct hda_codec_cmd_ctl *p __unused) {} #endif static int hda_corb_start(struct hda_softc *sc) { struct hda_codec_cmd_ctl *corb = &sc->corb; uint8_t corbsize = 0; uint64_t corblbase = 0; uint64_t corbubase = 0; uint64_t corbpaddr = 0; corb->name = "CORB"; corbsize = hda_get_reg_by_offset(sc, HDAC_CORBSIZE) & \ HDAC_CORBSIZE_CORBSIZE_MASK; corb->size = hda_corb_sizes[corbsize]; if (!corb->size) { DPRINTF("Invalid corb size"); return (-1); } corblbase = hda_get_reg_by_offset(sc, HDAC_CORBLBASE); corbubase = hda_get_reg_by_offset(sc, HDAC_CORBUBASE); corbpaddr = corblbase | (corbubase << 32); DPRINTF("CORB dma_paddr: %p", (void *)corbpaddr); corb->dma_vaddr = hda_dma_get_vaddr(sc, corbpaddr, HDA_CORB_ENTRY_LEN * corb->size); if (!corb->dma_vaddr) { DPRINTF("Fail to get the guest virtual address"); return (-1); } corb->wp = hda_get_reg_by_offset(sc, HDAC_CORBWP); corb->rp = hda_get_reg_by_offset(sc, HDAC_CORBRP); corb->run = 1; hda_print_cmd_ctl_data(corb); return (0); } static int hda_corb_run(struct hda_softc *sc) { struct hda_codec_cmd_ctl *corb = &sc->corb; uint32_t verb = 0; int err; corb->wp = hda_get_reg_by_offset(sc, HDAC_CORBWP); if (corb->wp >= corb->size) { DPRINTF("Invalid HDAC_CORBWP %u >= size %u", corb->wp, corb->size); return (-1); } while (corb->rp != corb->wp && corb->run) { corb->rp++; corb->rp %= corb->size; verb = hda_dma_ld_dword((uint8_t *)corb->dma_vaddr + HDA_CORB_ENTRY_LEN * corb->rp); err = hda_send_command(sc, verb); assert(!err); } hda_set_reg_by_offset(sc, HDAC_CORBRP, corb->rp); if (corb->run) hda_response_interrupt(sc); return (0); } static int hda_rirb_start(struct hda_softc *sc) { struct hda_codec_cmd_ctl *rirb = &sc->rirb; uint8_t rirbsize = 0; uint64_t rirblbase = 0; uint64_t rirbubase = 0; uint64_t rirbpaddr = 0; rirb->name = "RIRB"; rirbsize = hda_get_reg_by_offset(sc, HDAC_RIRBSIZE) & \ HDAC_RIRBSIZE_RIRBSIZE_MASK; rirb->size = hda_rirb_sizes[rirbsize]; if (!rirb->size) { DPRINTF("Invalid rirb size"); return (-1); } rirblbase = hda_get_reg_by_offset(sc, HDAC_RIRBLBASE); rirbubase = hda_get_reg_by_offset(sc, HDAC_RIRBUBASE); rirbpaddr = rirblbase | (rirbubase << 32); DPRINTF("RIRB dma_paddr: %p", (void *)rirbpaddr); rirb->dma_vaddr = hda_dma_get_vaddr(sc, rirbpaddr, HDA_RIRB_ENTRY_LEN * rirb->size); if (!rirb->dma_vaddr) { DPRINTF("Fail to get the guest virtual address"); return (-1); } rirb->wp = hda_get_reg_by_offset(sc, HDAC_RIRBWP); rirb->rp = 0x0000; rirb->run = 1; hda_print_cmd_ctl_data(rirb); return (0); } static void * hda_dma_get_vaddr(struct hda_softc *sc, uint64_t dma_paddr, size_t len) { struct pci_devinst *pi = sc->pci_dev; assert(pi); return (paddr_guest2host(pi->pi_vmctx, (uintptr_t)dma_paddr, len)); } static void hda_dma_st_dword(void *dma_vaddr, uint32_t data) { *(uint32_t*)dma_vaddr = data; } static uint32_t hda_dma_ld_dword(void *dma_vaddr) { return (*(uint32_t*)dma_vaddr); } static inline uint8_t hda_get_stream_by_offsets(uint32_t offset, uint8_t reg_offset) { uint8_t stream_ind = (offset - reg_offset) >> 5; assert(stream_ind < HDA_IOSS_NO); return (stream_ind); } static inline uint32_t hda_get_offset_stream(uint8_t stream_ind) { return (stream_ind << 5); } static void hda_set_gctl(struct hda_softc *sc, uint32_t offset, uint32_t old __unused) { uint32_t value = hda_get_reg_by_offset(sc, offset); if (!(value & HDAC_GCTL_CRST)) { hda_reset(sc); } } static void hda_set_statests(struct hda_softc *sc, uint32_t offset, uint32_t old) { uint32_t value = hda_get_reg_by_offset(sc, offset); hda_set_reg_by_offset(sc, offset, old); /* clear the corresponding bits written by the software (guest) */ hda_set_field_by_offset(sc, offset, value & HDA_STATESTS_IRQ_MASK, 0); hda_update_intr(sc); } static void hda_set_corbwp(struct hda_softc *sc, uint32_t offset __unused, uint32_t old __unused) { hda_corb_run(sc); } static void hda_set_corbctl(struct hda_softc *sc, uint32_t offset, uint32_t old) { uint32_t value = hda_get_reg_by_offset(sc, offset); int err; struct hda_codec_cmd_ctl *corb = NULL; if (value & HDAC_CORBCTL_CORBRUN) { if (!(old & HDAC_CORBCTL_CORBRUN)) { err = hda_corb_start(sc); assert(!err); } } else { corb = &sc->corb; memset(corb, 0, sizeof(*corb)); } hda_corb_run(sc); } static void hda_set_rirbctl(struct hda_softc *sc, uint32_t offset, uint32_t old __unused) { uint32_t value = hda_get_reg_by_offset(sc, offset); int err; struct hda_codec_cmd_ctl *rirb = NULL; if (value & HDAC_RIRBCTL_RIRBDMAEN) { err = hda_rirb_start(sc); assert(!err); } else { rirb = &sc->rirb; memset(rirb, 0, sizeof(*rirb)); } } static void hda_set_rirbsts(struct hda_softc *sc, uint32_t offset, uint32_t old) { uint32_t value = hda_get_reg_by_offset(sc, offset); hda_set_reg_by_offset(sc, offset, old); /* clear the corresponding bits written by the software (guest) */ hda_set_field_by_offset(sc, offset, value & HDA_RIRBSTS_IRQ_MASK, 0); hda_update_intr(sc); } static void hda_set_dpiblbase(struct hda_softc *sc, uint32_t offset, uint32_t old) { uint32_t value = hda_get_reg_by_offset(sc, offset); uint64_t dpiblbase = 0; uint64_t dpibubase = 0; uint64_t dpibpaddr = 0; if ((value & HDAC_DPLBASE_DPLBASE_DMAPBE) != (old & \ HDAC_DPLBASE_DPLBASE_DMAPBE)) { if (value & HDAC_DPLBASE_DPLBASE_DMAPBE) { dpiblbase = value & HDAC_DPLBASE_DPLBASE_MASK; dpibubase = hda_get_reg_by_offset(sc, HDAC_DPIBUBASE); dpibpaddr = dpiblbase | (dpibubase << 32); DPRINTF("DMA Position In Buffer dma_paddr: %p", (void *)dpibpaddr); sc->dma_pib_vaddr = hda_dma_get_vaddr(sc, dpibpaddr, HDA_DMA_PIB_ENTRY_LEN * HDA_IOSS_NO); if (!sc->dma_pib_vaddr) { DPRINTF("Fail to get the guest \ virtual address"); assert(0); } } else { DPRINTF("DMA Position In Buffer Reset"); sc->dma_pib_vaddr = NULL; } } } static void hda_set_sdctl(struct hda_softc *sc, uint32_t offset, uint32_t old) { uint8_t stream_ind = hda_get_stream_by_offsets(offset, HDAC_SDCTL0); uint32_t value = hda_get_reg_by_offset(sc, offset); int err; DPRINTF("stream_ind: 0x%x old: 0x%x value: 0x%x", stream_ind, old, value); if (value & HDAC_SDCTL_SRST) { hda_stream_reset(sc, stream_ind); } if ((value & HDAC_SDCTL_RUN) != (old & HDAC_SDCTL_RUN)) { if (value & HDAC_SDCTL_RUN) { err = hda_stream_start(sc, stream_ind); assert(!err); } else { err = hda_stream_stop(sc, stream_ind); assert(!err); } } } static void hda_set_sdctl2(struct hda_softc *sc, uint32_t offset, uint32_t old __unused) { uint32_t value = hda_get_reg_by_offset(sc, offset); hda_set_field_by_offset(sc, offset - 2, 0x00ff0000, value << 16); } static void hda_set_sdsts(struct hda_softc *sc, uint32_t offset, uint32_t old) { uint32_t value = hda_get_reg_by_offset(sc, offset); hda_set_reg_by_offset(sc, offset, old); /* clear the corresponding bits written by the software (guest) */ hda_set_field_by_offset(sc, offset, value & HDA_SDSTS_IRQ_MASK, 0); hda_update_intr(sc); } static int hda_signal_state_change(struct hda_codec_inst *hci) { struct hda_softc *sc = NULL; uint32_t sdiwake = 0; assert(hci); assert(hci->hda); DPRINTF("cad: 0x%x", hci->cad); sc = hci->hda; sdiwake = 1 << hci->cad; hda_set_field_by_offset(sc, HDAC_STATESTS, sdiwake, sdiwake); hda_update_intr(sc); return (0); } static int hda_response(struct hda_codec_inst *hci, uint32_t response, uint8_t unsol) { struct hda_softc *sc = NULL; struct hda_codec_cmd_ctl *rirb = NULL; uint32_t response_ex = 0; uint8_t rintcnt = 0; assert(hci); assert(hci->cad <= HDA_CODEC_MAX); response_ex = hci->cad | unsol; sc = hci->hda; assert(sc); rirb = &sc->rirb; if (rirb->run) { rirb->wp++; rirb->wp %= rirb->size; hda_dma_st_dword((uint8_t *)rirb->dma_vaddr + HDA_RIRB_ENTRY_LEN * rirb->wp, response); hda_dma_st_dword((uint8_t *)rirb->dma_vaddr + HDA_RIRB_ENTRY_LEN * rirb->wp + 0x04, response_ex); hda_set_reg_by_offset(sc, HDAC_RIRBWP, rirb->wp); sc->rirb_cnt++; } rintcnt = hda_get_reg_by_offset(sc, HDAC_RINTCNT); if (sc->rirb_cnt == rintcnt) hda_response_interrupt(sc); return (0); } static int hda_transfer(struct hda_codec_inst *hci, uint8_t stream, uint8_t dir, uint8_t *buf, size_t count) { struct hda_softc *sc = NULL; struct hda_stream_desc *st = NULL; struct hda_bdle_desc *bdl = NULL; struct hda_bdle_desc *bdle_desc = NULL; uint8_t stream_ind = 0; uint32_t lpib = 0; uint32_t off = 0; size_t left = 0; uint8_t irq = 0; assert(hci); assert(hci->hda); assert(buf); assert(!(count % HDA_DMA_ACCESS_LEN)); if (!stream) { DPRINTF("Invalid stream"); return (-1); } sc = hci->hda; assert(stream < HDA_STREAM_TAGS_CNT); stream_ind = sc->stream_map[dir][stream]; if (!dir) assert(stream_ind < HDA_ISS_NO); else assert(stream_ind >= HDA_ISS_NO && stream_ind < HDA_IOSS_NO); st = &sc->streams[stream_ind]; if (!st->run) { DPRINTF("Stream 0x%x stopped", stream); return (-1); } assert(st->stream == stream); off = hda_get_offset_stream(stream_ind); lpib = hda_get_reg_by_offset(sc, off + HDAC_SDLPIB); bdl = st->bdl; assert(st->be < st->bdl_cnt); assert(st->bp < bdl[st->be].len); left = count; while (left) { bdle_desc = &bdl[st->be]; if (dir) *(uint32_t *)buf = hda_dma_ld_dword( (uint8_t *)bdle_desc->addr + st->bp); else hda_dma_st_dword((uint8_t *)bdle_desc->addr + st->bp, *(uint32_t *)buf); buf += HDA_DMA_ACCESS_LEN; st->bp += HDA_DMA_ACCESS_LEN; lpib += HDA_DMA_ACCESS_LEN; left -= HDA_DMA_ACCESS_LEN; if (st->bp == bdle_desc->len) { st->bp = 0; if (bdle_desc->ioc) irq = 1; st->be++; if (st->be == st->bdl_cnt) { st->be = 0; lpib = 0; } bdle_desc = &bdl[st->be]; } } hda_set_pib(sc, stream_ind, lpib); if (irq) { hda_set_field_by_offset(sc, off + HDAC_SDSTS, HDAC_SDSTS_BCIS, HDAC_SDSTS_BCIS); hda_update_intr(sc); } return (0); } static void hda_set_pib(struct hda_softc *sc, uint8_t stream_ind, uint32_t pib) { uint32_t off = hda_get_offset_stream(stream_ind); hda_set_reg_by_offset(sc, off + HDAC_SDLPIB, pib); /* LPIB Alias */ hda_set_reg_by_offset(sc, 0x2000 + off + HDAC_SDLPIB, pib); if (sc->dma_pib_vaddr) *(uint32_t *)((uint8_t *)sc->dma_pib_vaddr + stream_ind * HDA_DMA_PIB_ENTRY_LEN) = pib; } static uint64_t hda_get_clock_ns(void) { struct timespec ts; int err; err = clock_gettime(CLOCK_MONOTONIC, &ts); assert(!err); return (ts.tv_sec * 1000000000LL + ts.tv_nsec); } /* * PCI HDA function definitions */ static int pci_hda_init(struct pci_devinst *pi, nvlist_t *nvl) { struct hda_softc *sc = NULL; assert(pi != NULL); pci_set_cfgdata16(pi, PCIR_VENDOR, INTEL_VENDORID); pci_set_cfgdata16(pi, PCIR_DEVICE, HDA_INTEL_82801G); pci_set_cfgdata8(pi, PCIR_SUBCLASS, PCIS_MULTIMEDIA_HDA); pci_set_cfgdata8(pi, PCIR_CLASS, PCIC_MULTIMEDIA); /* select the Intel HDA mode */ pci_set_cfgdata8(pi, PCIR_HDCTL, 0x01); /* allocate one BAR register for the Memory address offsets */ pci_emul_alloc_bar(pi, 0, PCIBAR_MEM32, HDA_LAST_OFFSET); /* allocate an IRQ pin for our slot */ pci_lintr_request(pi); sc = hda_init(nvl); if (!sc) return (-1); sc->pci_dev = pi; pi->pi_arg = sc; return (0); } static void pci_hda_write(struct pci_devinst *pi, int baridx, uint64_t offset, int size, uint64_t value) { struct hda_softc *sc = pi->pi_arg; int err; assert(sc); assert(baridx == 0); assert(size <= 4); DPRINTF("offset: 0x%lx value: 0x%lx", offset, value); err = hda_write(sc, offset, size, value); assert(!err); } static uint64_t pci_hda_read(struct pci_devinst *pi, int baridx, uint64_t offset, int size) { struct hda_softc *sc = pi->pi_arg; uint64_t value = 0; assert(sc); assert(baridx == 0); assert(size <= 4); value = hda_read(sc, offset); DPRINTF("offset: 0x%lx value: 0x%lx", offset, value); return (value); } /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2016 Alex Teaca * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef _HDA_EMUL_H_ #define _HDA_EMUL_H_ #include #include #include #include #include #include #include #include #include "hda_reg.h" /* * HDA Debug Log */ #if DEBUG_HDA == 1 extern FILE *dbg; #define DPRINTF(fmt, arg...) \ do {fprintf(dbg, "%s-%d: " fmt "\n", __func__, __LINE__, ##arg); \ fflush(dbg); } while (0) #ifndef DEBUG_HDA_FILE #define DEBUG_HDA_FILE "/tmp/bhyve_hda.log" #endif #else #define DPRINTF(fmt, arg...) #endif #define HDA_FIFO_SIZE 0x100 struct hda_softc; struct hda_codec_class; struct hda_codec_inst { uint8_t cad; struct hda_codec_class *codec; struct hda_softc *hda; const struct hda_ops *hops; void *priv; }; struct hda_codec_class { const char *name; int (*init)(struct hda_codec_inst *hci, const char *play, const char *rec); int (*reset)(struct hda_codec_inst *hci); int (*command)(struct hda_codec_inst *hci, uint32_t cmd_data); int (*notify)(struct hda_codec_inst *hci, uint8_t run, uint8_t stream, uint8_t dir); }; struct hda_ops { int (*signal)(struct hda_codec_inst *hci); int (*response)(struct hda_codec_inst *hci, uint32_t response, uint8_t unsol); int (*transfer)(struct hda_codec_inst *hci, uint8_t stream, uint8_t dir, uint8_t *buf, size_t count); }; #define HDA_EMUL_SET(x) DATA_SET(hda_codec_class_set, x) #endif /* _HDA_EMUL_H_ */ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2011 NetApp, Inc. * Copyright (c) 2018 Joyent, Inc. * Copyright 2021 Oxide Computer Company * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef __FreeBSD__ #include #include #include #include #include #endif #include #include "config.h" #include "pci_emul.h" #ifndef __FreeBSD__ #include "bhyverun.h" #endif #ifndef __FreeBSD__ static struct pci_hostbridge_model { const char *phm_model; uint16_t phm_vendor; uint16_t phm_device; } pci_hb_models[] = { { "amd", 0x1022, 0x7432 }, /* AMD/made-up */ { "netapp", 0x1275, 0x1275 }, /* NetApp/NetApp */ { "i440fx", 0x8086, 0x1237 }, /* Intel/82441 */ { "q35", 0x8086, 0x29b0 }, /* Intel/Q35 HB */ }; #define NUM_HB_MODELS (sizeof (pci_hb_models) / sizeof (pci_hb_models[0])) #endif static int pci_hostbridge_init(struct pci_devinst *pi, nvlist_t *nvl) { const char *value; u_int vendor, device; #ifdef __FreeBSD__ vendor = 0x1275; /* NetApp */ device = 0x1275; /* NetApp */ #else vendor = device = 0; #endif value = get_config_value_node(nvl, "vendor"); if (value != NULL) vendor = strtol(value, NULL, 0); else vendor = pci_config_read_reg(NULL, nvl, PCIR_VENDOR, 2, vendor); value = get_config_value_node(nvl, "devid"); if (value != NULL) device = strtol(value, NULL, 0); else device = pci_config_read_reg(NULL, nvl, PCIR_DEVICE, 2, device); #ifndef __FreeBSD__ const char *model = get_config_value_node(nvl, "model"); if (model != NULL && (vendor != 0 || device != 0)) { warnx("pci_hostbridge: cannot specify model and vendor/device"); return (-1); } else if ((vendor != 0 && device == 0) || (vendor == 0 && device != 0)) { warnx("pci_hostbridge: must specify both vendor and " "device for custom hostbridge"); return (-1); } if (model == NULL && vendor == 0 && device == 0) model = "netapp"; if (model != NULL) { for (uint_t i = 0; i < NUM_HB_MODELS; i++) { if (strcmp(model, pci_hb_models[i].phm_model) != 0) continue; /* found a model match */ vendor = pci_hb_models[i].phm_vendor; device = pci_hb_models[i].phm_device; break; } if (vendor == 0) { warnx("pci_hostbridge: invalid model '%s'", model); return (-1); } } /* Both i440fx and Q35 chipsets feature the concept of Programmable * Address Memory (PAM), where certain physical address ranges can be * configured to direct reads/writes to either DRAM, or to the PCI MMIO * space instead. At boot, they default to bypassing DRAM, so in order * to cheaply paper over our lack of emulation, the memory in PAM0 * (0xf0000-0xfffff, the System BIOS segment) should be zeroed. * * If this emulation is expanded in the future to truly support PAM * behavior, this hack can be removed. */ if (vendor == 0x8086 && (device == 0x1237 || device == 0x29b0)) { const uintptr_t start = 0xf0000; const size_t len = 0x10000; void *system_bios_region = paddr_guest2host(pi->pi_vmctx, start, len); assert(system_bios_region != NULL); bzero(system_bios_region, len); } #endif /* !__FreeBSD__ */ /* config space */ pci_set_cfgdata16(pi, PCIR_VENDOR, vendor); pci_set_cfgdata16(pi, PCIR_DEVICE, device); pci_set_cfgdata8(pi, PCIR_HDRTYPE, PCIM_HDRTYPE_NORMAL); pci_set_cfgdata8(pi, PCIR_CLASS, PCIC_BRIDGE); pci_set_cfgdata8(pi, PCIR_SUBCLASS, PCIS_BRIDGE_HOST); pci_emul_add_pciecap(pi, PCIEM_TYPE_ROOT_PORT); return (0); } static int pci_amd_hostbridge_legacy_config(nvlist_t *nvl, const char *opts __unused) { nvlist_t *pci_regs; pci_regs = create_relative_config_node(nvl, "pcireg"); if (pci_regs == NULL) { warnx("amd_hostbridge: failed to create pciregs node"); return (-1); } set_config_value_node(pci_regs, "vendor", "0x1022"); /* AMD */ set_config_value_node(pci_regs, "device", "0x7432"); /* made up */ return (0); } static const struct pci_devemu pci_de_amd_hostbridge = { .pe_emu = "amd_hostbridge", .pe_legacy_config = pci_amd_hostbridge_legacy_config, .pe_alias = "hostbridge", }; PCI_EMUL_SET(pci_de_amd_hostbridge); static const struct pci_devemu pci_de_hostbridge = { .pe_emu = "hostbridge", .pe_init = pci_hostbridge_init, }; PCI_EMUL_SET(pci_de_hostbridge); /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2014 Hudson River Trading LLC * Written by: John H. Baldwin * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include #include #include #include #include #include #include #include #include "acpi.h" #include "bootrom.h" #include "inout.h" #include "pci_emul.h" #include "pci_irq.h" #include "pci_lpc.h" /* * Implement an 8 pin PCI interrupt router compatible with the router * present on Intel's ICH10 chip. */ /* Fields in each PIRQ register. */ #define PIRQ_DIS 0x80 #define PIRQ_IRQ 0x0f /* Only IRQs 3-7, 9-12, and 14-15 are permitted. */ #define PERMITTED_IRQS 0xdef8 #define IRQ_PERMITTED(irq) (((1U << (irq)) & PERMITTED_IRQS) != 0) /* IRQ count to disable an IRQ. */ #define IRQ_DISABLED 0xff #define NPIRQS 8 static struct pirq { uint8_t reg; int use_count; int active_count; pthread_mutex_t lock; } pirqs[NPIRQS]; #define NIRQ_COUNTS 16 static u_char irq_counts[NIRQ_COUNTS]; static int pirq_cold = 1; /* * Returns true if this pin is enabled with a valid IRQ. Setting the * register to a reserved IRQ causes interrupts to not be asserted as * if the pin was disabled. */ static bool pirq_valid_irq(int reg) { if (reg & PIRQ_DIS) return (false); return (IRQ_PERMITTED(reg & PIRQ_IRQ)); } uint8_t pirq_read(int pin) { assert(pin > 0 && pin <= NPIRQS); return (pirqs[pin - 1].reg); } void pirq_write(struct vmctx *ctx, int pin, uint8_t val) { struct pirq *pirq; assert(pin > 0 && pin <= NPIRQS); pirq = &pirqs[pin - 1]; pthread_mutex_lock(&pirq->lock); if (pirq->reg != (val & (PIRQ_DIS | PIRQ_IRQ))) { if (pirq->active_count != 0 && pirq_valid_irq(pirq->reg)) vm_isa_deassert_irq(ctx, pirq->reg & PIRQ_IRQ, -1); pirq->reg = val & (PIRQ_DIS | PIRQ_IRQ); if (pirq->active_count != 0 && pirq_valid_irq(pirq->reg)) vm_isa_assert_irq(ctx, pirq->reg & PIRQ_IRQ, -1); } pthread_mutex_unlock(&pirq->lock); } void pci_irq_reserve(int irq) { assert(irq >= 0 && irq < NIRQ_COUNTS); assert(pirq_cold); assert(irq_counts[irq] == 0 || irq_counts[irq] == IRQ_DISABLED); irq_counts[irq] = IRQ_DISABLED; } void pci_irq_use(int irq) { assert(irq >= 0 && irq < NIRQ_COUNTS); assert(pirq_cold); assert(irq_counts[irq] != IRQ_DISABLED); irq_counts[irq]++; } void pci_irq_init(struct vmctx *ctx __unused) { int i; for (i = 0; i < NPIRQS; i++) { pirqs[i].reg = PIRQ_DIS; pirqs[i].use_count = 0; pirqs[i].active_count = 0; pthread_mutex_init(&pirqs[i].lock, NULL); } for (i = 0; i < NIRQ_COUNTS; i++) { if (IRQ_PERMITTED(i)) irq_counts[i] = 0; else irq_counts[i] = IRQ_DISABLED; } } void pci_irq_assert(struct pci_devinst *pi) { struct pirq *pirq; int pin; pin = pi->pi_lintr.pirq_pin; if (pin > 0) { assert(pin <= NPIRQS); pirq = &pirqs[pin - 1]; pthread_mutex_lock(&pirq->lock); pirq->active_count++; if (pirq->active_count == 1 && pirq_valid_irq(pirq->reg)) { vm_isa_assert_irq(pi->pi_vmctx, pirq->reg & PIRQ_IRQ, pi->pi_lintr.ioapic_irq); pthread_mutex_unlock(&pirq->lock); return; } pthread_mutex_unlock(&pirq->lock); } vm_ioapic_assert_irq(pi->pi_vmctx, pi->pi_lintr.ioapic_irq); } void pci_irq_deassert(struct pci_devinst *pi) { struct pirq *pirq; int pin; pin = pi->pi_lintr.pirq_pin; if (pin > 0) { assert(pin <= NPIRQS); pirq = &pirqs[pin - 1]; pthread_mutex_lock(&pirq->lock); pirq->active_count--; if (pirq->active_count == 0 && pirq_valid_irq(pirq->reg)) { vm_isa_deassert_irq(pi->pi_vmctx, pirq->reg & PIRQ_IRQ, pi->pi_lintr.ioapic_irq); pthread_mutex_unlock(&pirq->lock); return; } pthread_mutex_unlock(&pirq->lock); } vm_ioapic_deassert_irq(pi->pi_vmctx, pi->pi_lintr.ioapic_irq); } int pirq_alloc_pin(struct pci_devinst *pi) { struct vmctx *ctx = pi->pi_vmctx; int best_count, best_irq, best_pin, irq, pin; pirq_cold = 0; if (bootrom_boot()) { /* For external bootrom use fixed mapping. */ best_pin = (4 + pi->pi_slot + pi->pi_lintr.pin) % 8; } else { /* Find the least-used PIRQ pin. */ best_pin = 0; best_count = pirqs[0].use_count; for (pin = 1; pin < NPIRQS; pin++) { if (pirqs[pin].use_count < best_count) { best_pin = pin; best_count = pirqs[pin].use_count; } } } pirqs[best_pin].use_count++; /* Second, route this pin to an IRQ. */ if (pirqs[best_pin].reg == PIRQ_DIS) { best_irq = -1; best_count = 0; for (irq = 0; irq < NIRQ_COUNTS; irq++) { if (irq_counts[irq] == IRQ_DISABLED) continue; if (best_irq == -1 || irq_counts[irq] < best_count) { best_irq = irq; best_count = irq_counts[irq]; } } assert(best_irq >= 0); irq_counts[best_irq]++; pirqs[best_pin].reg = best_irq; vm_isa_set_irq_trigger(ctx, best_irq, LEVEL_TRIGGER); } return (best_pin + 1); } int pirq_irq(int pin) { assert(pin > 0 && pin <= NPIRQS); return (pirqs[pin - 1].reg & PIRQ_IRQ); } /* XXX: Generate $PIR table. */ static void pirq_dsdt(void) { char *irq_prs, *old; int irq, pin; irq_prs = NULL; for (irq = 0; irq < NIRQ_COUNTS; irq++) { if (!IRQ_PERMITTED(irq)) continue; if (irq_prs == NULL) asprintf(&irq_prs, "%d", irq); else { old = irq_prs; asprintf(&irq_prs, "%s,%d", old, irq); free(old); } } /* * A helper method to validate a link register's value. This * duplicates pirq_valid_irq(). */ dsdt_line(""); dsdt_line("Method (PIRV, 1, NotSerialized)"); dsdt_line("{"); dsdt_line(" If (And (Arg0, 0x%02X))", PIRQ_DIS); dsdt_line(" {"); dsdt_line(" Return (0x00)"); dsdt_line(" }"); dsdt_line(" And (Arg0, 0x%02X, Local0)", PIRQ_IRQ); dsdt_line(" If (LLess (Local0, 0x03))"); dsdt_line(" {"); dsdt_line(" Return (0x00)"); dsdt_line(" }"); dsdt_line(" If (LEqual (Local0, 0x08))"); dsdt_line(" {"); dsdt_line(" Return (0x00)"); dsdt_line(" }"); dsdt_line(" If (LEqual (Local0, 0x0D))"); dsdt_line(" {"); dsdt_line(" Return (0x00)"); dsdt_line(" }"); dsdt_line(" Return (0x01)"); dsdt_line("}"); for (pin = 0; pin < NPIRQS; pin++) { dsdt_line(""); dsdt_line("Device (LNK%c)", 'A' + pin); dsdt_line("{"); dsdt_line(" Name (_HID, EisaId (\"PNP0C0F\"))"); dsdt_line(" Name (_UID, 0x%02X)", pin + 1); dsdt_line(" Method (_STA, 0, NotSerialized)"); dsdt_line(" {"); dsdt_line(" If (PIRV (PIR%c))", 'A' + pin); dsdt_line(" {"); dsdt_line(" Return (0x0B)"); dsdt_line(" }"); dsdt_line(" Else"); dsdt_line(" {"); dsdt_line(" Return (0x09)"); dsdt_line(" }"); dsdt_line(" }"); dsdt_line(" Name (_PRS, ResourceTemplate ()"); dsdt_line(" {"); dsdt_line(" IRQ (Level, ActiveLow, Shared, )"); dsdt_line(" {%s}", irq_prs); dsdt_line(" })"); dsdt_line(" Name (CB%02X, ResourceTemplate ()", pin + 1); dsdt_line(" {"); dsdt_line(" IRQ (Level, ActiveLow, Shared, )"); dsdt_line(" {}"); dsdt_line(" })"); dsdt_line(" CreateWordField (CB%02X, 0x01, CIR%c)", pin + 1, 'A' + pin); dsdt_line(" Method (_CRS, 0, NotSerialized)"); dsdt_line(" {"); dsdt_line(" And (PIR%c, 0x%02X, Local0)", 'A' + pin, PIRQ_DIS | PIRQ_IRQ); dsdt_line(" If (PIRV (Local0))"); dsdt_line(" {"); dsdt_line(" ShiftLeft (0x01, Local0, CIR%c)", 'A' + pin); dsdt_line(" }"); dsdt_line(" Else"); dsdt_line(" {"); dsdt_line(" Store (0x00, CIR%c)", 'A' + pin); dsdt_line(" }"); dsdt_line(" Return (CB%02X)", pin + 1); dsdt_line(" }"); dsdt_line(" Method (_DIS, 0, NotSerialized)"); dsdt_line(" {"); dsdt_line(" Store (0x80, PIR%c)", 'A' + pin); dsdt_line(" }"); dsdt_line(" Method (_SRS, 1, NotSerialized)"); dsdt_line(" {"); dsdt_line(" CreateWordField (Arg0, 0x01, SIR%c)", 'A' + pin); dsdt_line(" FindSetRightBit (SIR%c, Local0)", 'A' + pin); dsdt_line(" Store (Decrement (Local0), PIR%c)", 'A' + pin); dsdt_line(" }"); dsdt_line("}"); } free(irq_prs); } LPC_DSDT(pirq_dsdt); /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2014 Hudson River Trading LLC * Written by: John H. Baldwin * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef __PCI_IRQ_H__ #define __PCI_IRQ_H__ struct pci_devinst; void pci_irq_assert(struct pci_devinst *pi); void pci_irq_deassert(struct pci_devinst *pi); void pci_irq_init(struct vmctx *ctx); void pci_irq_reserve(int irq); void pci_irq_use(int irq); int pirq_alloc_pin(struct pci_devinst *pi); int pirq_irq(int pin); uint8_t pirq_read(int pin); void pirq_write(struct vmctx *ctx, int pin, uint8_t val); #endif /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2017 Shunsuke Mie * Copyright (c) 2018 Leon Dang * Copyright (c) 2020 Chuck Tuffli * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * bhyve PCIe-NVMe device emulation. * * options: * -s ,nvme,devpath,maxq=#,qsz=#,ioslots=#,sectsz=#,ser=A-Z,eui64=#,dsm= * * accepted devpath: * /dev/blockdev * /path/to/image * ram=size_in_MiB * * maxq = max number of queues * qsz = max elements in each queue * ioslots = max number of concurrent io requests * sectsz = sector size (defaults to blockif sector size) * ser = serial number (20-chars max) * eui64 = IEEE Extended Unique Identifier (8 byte value) * dsm = DataSet Management support. Option is one of auto, enable,disable * */ /* TODO: - create async event for smart and log - intr coalesce */ #include #include #ifdef __FreeBSD__ #include #else #include "crc16.h" #endif #include #ifndef __FreeBSD__ #include #endif #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "bhyverun.h" #include "block_if.h" #include "config.h" #include "debug.h" #include "pci_emul.h" static int nvme_debug = 0; #define DPRINTF(fmt, args...) if (nvme_debug) PRINTLN(fmt, ##args) #define WPRINTF(fmt, args...) PRINTLN(fmt, ##args) /* defaults; can be overridden */ #define NVME_MSIX_BAR 4 #define NVME_IOSLOTS 8 /* The NVMe spec defines bits 13:4 in BAR0 as reserved */ #define NVME_MMIO_SPACE_MIN (1 << 14) #define NVME_QUEUES 16 #define NVME_MAX_QENTRIES 2048 /* Memory Page size Minimum reported in CAP register */ #define NVME_MPSMIN 0 /* MPSMIN converted to bytes */ #define NVME_MPSMIN_BYTES (1 << (12 + NVME_MPSMIN)) #define NVME_PRP2_ITEMS (PAGE_SIZE/sizeof(uint64_t)) #define NVME_MDTS 9 /* Note the + 1 allows for the initial descriptor to not be page aligned */ #define NVME_MAX_IOVEC ((1 << NVME_MDTS) + 1) #define NVME_MAX_DATA_SIZE ((1 << NVME_MDTS) * NVME_MPSMIN_BYTES) /* This is a synthetic status code to indicate there is no status */ #define NVME_NO_STATUS 0xffff #define NVME_COMPLETION_VALID(c) ((c).status != NVME_NO_STATUS) /* Reported temperature in Kelvin (i.e. room temperature) */ #define NVME_TEMPERATURE 296 /* helpers */ /* Convert a zero-based value into a one-based value */ #define ONE_BASED(zero) ((zero) + 1) /* Convert a one-based value into a zero-based value */ #define ZERO_BASED(one) ((one) - 1) /* Encode number of SQ's and CQ's for Set/Get Features */ #define NVME_FEATURE_NUM_QUEUES(sc) \ (ZERO_BASED((sc)->num_squeues) & 0xffff) | \ (ZERO_BASED((sc)->num_cqueues) & 0xffff) << 16 #define NVME_DOORBELL_OFFSET offsetof(struct nvme_registers, doorbell) enum nvme_controller_register_offsets { NVME_CR_CAP_LOW = 0x00, NVME_CR_CAP_HI = 0x04, NVME_CR_VS = 0x08, NVME_CR_INTMS = 0x0c, NVME_CR_INTMC = 0x10, NVME_CR_CC = 0x14, NVME_CR_CSTS = 0x1c, NVME_CR_NSSR = 0x20, NVME_CR_AQA = 0x24, NVME_CR_ASQ_LOW = 0x28, NVME_CR_ASQ_HI = 0x2c, NVME_CR_ACQ_LOW = 0x30, NVME_CR_ACQ_HI = 0x34, }; enum nvme_cmd_cdw11 { NVME_CMD_CDW11_PC = 0x0001, NVME_CMD_CDW11_IEN = 0x0002, NVME_CMD_CDW11_IV = 0xFFFF0000, }; enum nvme_copy_dir { NVME_COPY_TO_PRP, NVME_COPY_FROM_PRP, }; #define NVME_CQ_INTEN 0x01 #define NVME_CQ_INTCOAL 0x02 struct nvme_completion_queue { struct nvme_completion *qbase; pthread_mutex_t mtx; uint32_t size; uint16_t tail; /* nvme progress */ uint16_t head; /* guest progress */ uint16_t intr_vec; uint32_t intr_en; }; struct nvme_submission_queue { struct nvme_command *qbase; pthread_mutex_t mtx; uint32_t size; uint16_t head; /* nvme progress */ uint16_t tail; /* guest progress */ uint16_t cqid; /* completion queue id */ int qpriority; }; enum nvme_storage_type { NVME_STOR_BLOCKIF = 0, NVME_STOR_RAM = 1, }; struct pci_nvme_blockstore { enum nvme_storage_type type; void *ctx; uint64_t size; uint32_t sectsz; uint32_t sectsz_bits; uint64_t eui64; uint32_t deallocate:1; }; /* * Calculate the number of additional page descriptors for guest IO requests * based on the advertised Max Data Transfer (MDTS) and given the number of * default iovec's in a struct blockif_req. */ #define MDTS_PAD_SIZE \ ( NVME_MAX_IOVEC > BLOCKIF_IOV_MAX ? \ NVME_MAX_IOVEC - BLOCKIF_IOV_MAX : \ 0 ) struct pci_nvme_ioreq { struct pci_nvme_softc *sc; STAILQ_ENTRY(pci_nvme_ioreq) link; struct nvme_submission_queue *nvme_sq; uint16_t sqid; /* command information */ uint16_t opc; uint16_t cid; uint32_t nsid; uint64_t prev_gpaddr; size_t prev_size; size_t bytes; struct blockif_req io_req; struct iovec iovpadding[MDTS_PAD_SIZE]; }; enum nvme_dsm_type { /* Dataset Management bit in ONCS reflects backing storage capability */ NVME_DATASET_MANAGEMENT_AUTO, /* Unconditionally set Dataset Management bit in ONCS */ NVME_DATASET_MANAGEMENT_ENABLE, /* Unconditionally clear Dataset Management bit in ONCS */ NVME_DATASET_MANAGEMENT_DISABLE, }; struct pci_nvme_softc; struct nvme_feature_obj; typedef void (*nvme_feature_cb)(struct pci_nvme_softc *, struct nvme_feature_obj *, struct nvme_command *, struct nvme_completion *); struct nvme_feature_obj { uint32_t cdw11; nvme_feature_cb set; nvme_feature_cb get; bool namespace_specific; }; #define NVME_FID_MAX (NVME_FEAT_ENDURANCE_GROUP_EVENT_CONFIGURATION + 1) typedef enum { PCI_NVME_AE_TYPE_ERROR = 0, PCI_NVME_AE_TYPE_SMART, PCI_NVME_AE_TYPE_NOTICE, PCI_NVME_AE_TYPE_IO_CMD = 6, PCI_NVME_AE_TYPE_VENDOR = 7, PCI_NVME_AE_TYPE_MAX /* Must be last */ } pci_nvme_async_type; /* Asynchronous Event Requests */ struct pci_nvme_aer { STAILQ_ENTRY(pci_nvme_aer) link; uint16_t cid; /* Command ID of the submitted AER */ }; /** Asynchronous Event Information - Error */ typedef enum { PCI_NVME_AEI_ERROR_INVALID_DB, PCI_NVME_AEI_ERROR_INVALID_DB_VALUE, PCI_NVME_AEI_ERROR_DIAG_FAILURE, PCI_NVME_AEI_ERROR_PERSISTANT_ERR, PCI_NVME_AEI_ERROR_TRANSIENT_ERR, PCI_NVME_AEI_ERROR_FIRMWARE_LOAD_ERR, PCI_NVME_AEI_ERROR_MAX, } pci_nvme_async_event_info_error; /** Asynchronous Event Information - Notice */ typedef enum { PCI_NVME_AEI_NOTICE_NS_ATTR_CHANGED = 0, PCI_NVME_AEI_NOTICE_FW_ACTIVATION, PCI_NVME_AEI_NOTICE_TELEMETRY_CHANGE, PCI_NVME_AEI_NOTICE_ANA_CHANGE, PCI_NVME_AEI_NOTICE_PREDICT_LATENCY_CHANGE, PCI_NVME_AEI_NOTICE_LBA_STATUS_ALERT, PCI_NVME_AEI_NOTICE_ENDURANCE_GROUP_CHANGE, PCI_NVME_AEI_NOTICE_MAX, } pci_nvme_async_event_info_notice; #define PCI_NVME_AEI_NOTICE_SHIFT 8 #define PCI_NVME_AEI_NOTICE_MASK(event) (1 << (event + PCI_NVME_AEI_NOTICE_SHIFT)) /* Asynchronous Event Notifications */ struct pci_nvme_aen { pci_nvme_async_type atype; uint32_t event_data; bool posted; }; /* * By default, enable all Asynchrnous Event Notifications: * SMART / Health Critical Warnings * Namespace Attribute Notices */ #define PCI_NVME_AEN_DEFAULT_MASK 0x11f typedef enum { NVME_CNTRLTYPE_IO = 1, NVME_CNTRLTYPE_DISCOVERY = 2, NVME_CNTRLTYPE_ADMIN = 3, } pci_nvme_cntrl_type; struct pci_nvme_softc { struct pci_devinst *nsc_pi; pthread_mutex_t mtx; struct nvme_registers regs; struct nvme_namespace_data nsdata; struct nvme_controller_data ctrldata; struct nvme_error_information_entry err_log; struct nvme_health_information_page health_log; struct nvme_firmware_page fw_log; struct nvme_ns_list ns_log; struct pci_nvme_blockstore nvstore; uint16_t max_qentries; /* max entries per queue */ uint32_t max_queues; /* max number of IO SQ's or CQ's */ uint32_t num_cqueues; uint32_t num_squeues; bool num_q_is_set; /* Has host set Number of Queues */ struct pci_nvme_ioreq *ioreqs; STAILQ_HEAD(, pci_nvme_ioreq) ioreqs_free; /* free list of ioreqs */ uint32_t pending_ios; uint32_t ioslots; sem_t iosemlock; /* * Memory mapped Submission and Completion queues * Each array includes both Admin and IO queues */ struct nvme_completion_queue *compl_queues; struct nvme_submission_queue *submit_queues; struct nvme_feature_obj feat[NVME_FID_MAX]; enum nvme_dsm_type dataset_management; /* Accounting for SMART data */ __uint128_t read_data_units; __uint128_t write_data_units; __uint128_t read_commands; __uint128_t write_commands; uint32_t read_dunits_remainder; uint32_t write_dunits_remainder; STAILQ_HEAD(, pci_nvme_aer) aer_list; pthread_mutex_t aer_mtx; uint32_t aer_count; struct pci_nvme_aen aen[PCI_NVME_AE_TYPE_MAX]; pthread_t aen_tid; pthread_mutex_t aen_mtx; pthread_cond_t aen_cond; }; static void pci_nvme_cq_update(struct pci_nvme_softc *sc, struct nvme_completion_queue *cq, uint32_t cdw0, uint16_t cid, uint16_t sqid, uint16_t status); static struct pci_nvme_ioreq *pci_nvme_get_ioreq(struct pci_nvme_softc *); static void pci_nvme_release_ioreq(struct pci_nvme_softc *, struct pci_nvme_ioreq *); static void pci_nvme_io_done(struct blockif_req *, int); /* Controller Configuration utils */ #define NVME_CC_GET_EN(cc) \ NVMEV(NVME_CC_REG_EN, cc) #define NVME_CC_GET_CSS(cc) \ NVMEV(NVME_CC_REG_CSS, cc) #define NVME_CC_GET_SHN(cc) \ NVMEV(NVME_CC_REG_SHN, cc) #define NVME_CC_GET_IOSQES(cc) \ NVMEV(NVME_CC_REG_IOSQES, cc) #define NVME_CC_GET_IOCQES(cc) \ NVMEV(NVME_CC_REG_IOCQES, cc) #define NVME_CC_WRITE_MASK \ (NVMEM(NVME_CC_REG_EN) | \ NVMEM(NVME_CC_REG_IOSQES) | \ NVMEM(NVME_CC_REG_IOCQES)) #define NVME_CC_NEN_WRITE_MASK \ (NVMEM(NVME_CC_REG_CSS) | \ NVMEM(NVME_CC_REG_MPS) | \ NVMEM(NVME_CC_REG_AMS)) /* Controller Status utils */ #define NVME_CSTS_GET_RDY(sts) \ NVMEV(NVME_CSTS_REG_RDY, sts) #define NVME_CSTS_RDY (NVMEF(NVME_CSTS_REG_RDY, 1)) #define NVME_CSTS_CFS (NVMEF(NVME_CSTS_REG_CFS, 1)) /* Completion Queue status word utils */ #define NVME_STATUS_P (NVMEF(NVME_STATUS_P, 1)) #define NVME_STATUS_MASK \ (NVMEM(NVME_STATUS_SCT) | \ NVMEM(NVME_STATUS_SC)) #define NVME_ONCS_DSM NVMEM(NVME_CTRLR_DATA_ONCS_DSM) static void nvme_feature_invalid_cb(struct pci_nvme_softc *, struct nvme_feature_obj *, struct nvme_command *, struct nvme_completion *); static void nvme_feature_temperature(struct pci_nvme_softc *, struct nvme_feature_obj *, struct nvme_command *, struct nvme_completion *); static void nvme_feature_num_queues(struct pci_nvme_softc *, struct nvme_feature_obj *, struct nvme_command *, struct nvme_completion *); static void nvme_feature_iv_config(struct pci_nvme_softc *, struct nvme_feature_obj *, struct nvme_command *, struct nvme_completion *); static void nvme_feature_async_event(struct pci_nvme_softc *, struct nvme_feature_obj *, struct nvme_command *, struct nvme_completion *); static void *aen_thr(void *arg); static __inline void cpywithpad(char *dst, size_t dst_size, const char *src, char pad) { size_t len; len = strnlen(src, dst_size); memset(dst, pad, dst_size); memcpy(dst, src, len); } static __inline void pci_nvme_status_tc(uint16_t *status, uint16_t type, uint16_t code) { *status &= ~NVME_STATUS_MASK; *status |= NVMEF(NVME_STATUS_SCT, type) | NVMEF(NVME_STATUS_SC, code); } static __inline void pci_nvme_status_genc(uint16_t *status, uint16_t code) { pci_nvme_status_tc(status, NVME_SCT_GENERIC, code); } /* * Initialize the requested number or IO Submission and Completion Queues. * Admin queues are allocated implicitly. */ static void pci_nvme_init_queues(struct pci_nvme_softc *sc, uint32_t nsq, uint32_t ncq) { uint32_t i; /* * Allocate and initialize the Submission Queues */ if (nsq > NVME_QUEUES) { WPRINTF("%s: clamping number of SQ from %u to %u", __func__, nsq, NVME_QUEUES); nsq = NVME_QUEUES; } sc->num_squeues = nsq; sc->submit_queues = calloc(sc->num_squeues + 1, sizeof(struct nvme_submission_queue)); if (sc->submit_queues == NULL) { WPRINTF("%s: SQ allocation failed", __func__); sc->num_squeues = 0; } else { struct nvme_submission_queue *sq = sc->submit_queues; for (i = 0; i < sc->num_squeues + 1; i++) pthread_mutex_init(&sq[i].mtx, NULL); } /* * Allocate and initialize the Completion Queues */ if (ncq > NVME_QUEUES) { WPRINTF("%s: clamping number of CQ from %u to %u", __func__, ncq, NVME_QUEUES); ncq = NVME_QUEUES; } sc->num_cqueues = ncq; sc->compl_queues = calloc(sc->num_cqueues + 1, sizeof(struct nvme_completion_queue)); if (sc->compl_queues == NULL) { WPRINTF("%s: CQ allocation failed", __func__); sc->num_cqueues = 0; } else { struct nvme_completion_queue *cq = sc->compl_queues; for (i = 0; i < sc->num_cqueues + 1; i++) pthread_mutex_init(&cq[i].mtx, NULL); } } static void pci_nvme_init_ctrldata(struct pci_nvme_softc *sc) { struct nvme_controller_data *cd = &sc->ctrldata; int ret; cd->vid = 0xFB5D; cd->ssvid = 0x0000; cpywithpad((char *)cd->mn, sizeof(cd->mn), "bhyve-NVMe", ' '); cpywithpad((char *)cd->fr, sizeof(cd->fr), "1.0", ' '); /* Num of submission commands that we can handle at a time (2^rab) */ cd->rab = 4; /* FreeBSD OUI */ cd->ieee[0] = 0xfc; cd->ieee[1] = 0x9c; cd->ieee[2] = 0x58; cd->mic = 0; cd->mdts = NVME_MDTS; /* max data transfer size (2^mdts * CAP.MPSMIN) */ cd->ver = NVME_REV(1,4); cd->cntrltype = NVME_CNTRLTYPE_IO; cd->oacs = NVMEF(NVME_CTRLR_DATA_OACS_FORMAT, 1); cd->oaes = NVMEM(NVME_CTRLR_DATA_OAES_NS_ATTR); cd->acl = 2; cd->aerl = 4; /* Advertise 1, Read-only firmware slot */ cd->frmw = NVMEM(NVME_CTRLR_DATA_FRMW_SLOT1_RO) | NVMEF(NVME_CTRLR_DATA_FRMW_NUM_SLOTS, 1); cd->lpa = 0; /* TODO: support some simple things like SMART */ cd->elpe = 0; /* max error log page entries */ /* * Report a single power state (zero-based value) * power_state[] values are left as zero to indicate "Not reported" */ cd->npss = 0; /* Warning Composite Temperature Threshold */ cd->wctemp = 0x0157; cd->cctemp = 0x0157; /* SANICAP must not be 0 for Revision 1.4 and later NVMe Controllers */ cd->sanicap = NVMEF(NVME_CTRLR_DATA_SANICAP_NODMMAS, NVME_CTRLR_DATA_SANICAP_NODMMAS_NO); cd->sqes = NVMEF(NVME_CTRLR_DATA_SQES_MAX, 6) | NVMEF(NVME_CTRLR_DATA_SQES_MIN, 6); cd->cqes = NVMEF(NVME_CTRLR_DATA_CQES_MAX, 4) | NVMEF(NVME_CTRLR_DATA_CQES_MIN, 4); cd->nn = 1; /* number of namespaces */ cd->oncs = 0; switch (sc->dataset_management) { case NVME_DATASET_MANAGEMENT_AUTO: if (sc->nvstore.deallocate) cd->oncs |= NVME_ONCS_DSM; break; case NVME_DATASET_MANAGEMENT_ENABLE: cd->oncs |= NVME_ONCS_DSM; break; default: break; } cd->fna = NVMEM(NVME_CTRLR_DATA_FNA_FORMAT_ALL); cd->vwc = NVMEF(NVME_CTRLR_DATA_VWC_ALL, NVME_CTRLR_DATA_VWC_ALL_NO); #ifdef __FreeBSD__ ret = snprintf(cd->subnqn, sizeof(cd->subnqn), "nqn.2013-12.org.freebsd:bhyve-%s-%u-%u-%u", get_config_value("name"), sc->nsc_pi->pi_bus, sc->nsc_pi->pi_slot, sc->nsc_pi->pi_func); #else ret = snprintf((char *)cd->subnqn, sizeof (cd->subnqn), "nqn.2013-12.org.illumos:bhyve-%s-%u-%u-%u", get_config_value("name"), sc->nsc_pi->pi_bus, sc->nsc_pi->pi_slot, sc->nsc_pi->pi_func); #endif if ((ret < 0) || ((unsigned)ret > sizeof(cd->subnqn))) EPRINTLN("%s: error setting subnqn (%d)", __func__, ret); } static void pci_nvme_init_nsdata_size(struct pci_nvme_blockstore *nvstore, struct nvme_namespace_data *nd) { /* Get capacity and block size information from backing store */ nd->nsze = nvstore->size / nvstore->sectsz; nd->ncap = nd->nsze; nd->nuse = nd->nsze; } static void pci_nvme_init_nsdata(struct pci_nvme_softc *sc, struct nvme_namespace_data *nd, uint32_t nsid, struct pci_nvme_blockstore *nvstore) { pci_nvme_init_nsdata_size(nvstore, nd); if (nvstore->type == NVME_STOR_BLOCKIF) nvstore->deallocate = blockif_candelete(nvstore->ctx); nd->nlbaf = 0; /* NLBAF is a 0's based value (i.e. 1 LBA Format) */ nd->flbas = 0; /* Create an EUI-64 if user did not provide one */ if (nvstore->eui64 == 0) { char *data = NULL; uint64_t eui64 = nvstore->eui64; asprintf(&data, "%s%u%u%u", get_config_value("name"), sc->nsc_pi->pi_bus, sc->nsc_pi->pi_slot, sc->nsc_pi->pi_func); if (data != NULL) { eui64 = OUI_FREEBSD_NVME_LOW | crc16(0, data, strlen(data)); free(data); } nvstore->eui64 = (eui64 << 16) | (nsid & 0xffff); } be64enc(nd->eui64, nvstore->eui64); /* LBA data-sz = 2^lbads */ nd->lbaf[0] = NVMEF(NVME_NS_DATA_LBAF_LBADS, nvstore->sectsz_bits); } static void pci_nvme_init_logpages(struct pci_nvme_softc *sc) { __uint128_t power_cycles = 1; memset(&sc->err_log, 0, sizeof(sc->err_log)); memset(&sc->health_log, 0, sizeof(sc->health_log)); memset(&sc->fw_log, 0, sizeof(sc->fw_log)); memset(&sc->ns_log, 0, sizeof(sc->ns_log)); /* Set read/write remainder to round up according to spec */ sc->read_dunits_remainder = 999; sc->write_dunits_remainder = 999; /* Set nominal Health values checked by implementations */ sc->health_log.temperature = NVME_TEMPERATURE; sc->health_log.available_spare = 100; sc->health_log.available_spare_threshold = 10; /* Set Active Firmware Info to slot 1 */ sc->fw_log.afi = NVMEF(NVME_FIRMWARE_PAGE_AFI_SLOT, 1); memcpy(&sc->fw_log.revision[0], sc->ctrldata.fr, sizeof(sc->fw_log.revision[0])); memcpy(&sc->health_log.power_cycles, &power_cycles, sizeof(sc->health_log.power_cycles)); } static void pci_nvme_init_features(struct pci_nvme_softc *sc) { enum nvme_feature fid; for (fid = 0; fid < NVME_FID_MAX; fid++) { switch (fid) { case NVME_FEAT_ARBITRATION: case NVME_FEAT_POWER_MANAGEMENT: case NVME_FEAT_INTERRUPT_COALESCING: //XXX case NVME_FEAT_WRITE_ATOMICITY: /* Mandatory but no special handling required */ //XXX hang - case NVME_FEAT_PREDICTABLE_LATENCY_MODE_CONFIG: //XXX hang - case NVME_FEAT_HOST_BEHAVIOR_SUPPORT: // this returns a data buffer break; case NVME_FEAT_TEMPERATURE_THRESHOLD: sc->feat[fid].set = nvme_feature_temperature; break; case NVME_FEAT_ERROR_RECOVERY: sc->feat[fid].namespace_specific = true; break; case NVME_FEAT_NUMBER_OF_QUEUES: sc->feat[fid].set = nvme_feature_num_queues; break; case NVME_FEAT_INTERRUPT_VECTOR_CONFIGURATION: sc->feat[fid].set = nvme_feature_iv_config; break; case NVME_FEAT_ASYNC_EVENT_CONFIGURATION: sc->feat[fid].set = nvme_feature_async_event; /* Enable all AENs by default */ sc->feat[fid].cdw11 = PCI_NVME_AEN_DEFAULT_MASK; break; default: sc->feat[fid].set = nvme_feature_invalid_cb; sc->feat[fid].get = nvme_feature_invalid_cb; } } } static void pci_nvme_aer_reset(struct pci_nvme_softc *sc) { STAILQ_INIT(&sc->aer_list); sc->aer_count = 0; } static void pci_nvme_aer_init(struct pci_nvme_softc *sc) { pthread_mutex_init(&sc->aer_mtx, NULL); pci_nvme_aer_reset(sc); } static void pci_nvme_aer_destroy(struct pci_nvme_softc *sc) { struct pci_nvme_aer *aer = NULL; pthread_mutex_lock(&sc->aer_mtx); while (!STAILQ_EMPTY(&sc->aer_list)) { aer = STAILQ_FIRST(&sc->aer_list); STAILQ_REMOVE_HEAD(&sc->aer_list, link); free(aer); } pthread_mutex_unlock(&sc->aer_mtx); pci_nvme_aer_reset(sc); } static bool pci_nvme_aer_available(struct pci_nvme_softc *sc) { return (sc->aer_count != 0); } static bool pci_nvme_aer_limit_reached(struct pci_nvme_softc *sc) { struct nvme_controller_data *cd = &sc->ctrldata; /* AERL is a zero based value while aer_count is one's based */ return (sc->aer_count == (cd->aerl + 1U)); } /* * Add an Async Event Request * * Stores an AER to be returned later if the Controller needs to notify the * host of an event. * Note that while the NVMe spec doesn't require Controllers to return AER's * in order, this implementation does preserve the order. */ static int pci_nvme_aer_add(struct pci_nvme_softc *sc, uint16_t cid) { struct pci_nvme_aer *aer = NULL; aer = calloc(1, sizeof(struct pci_nvme_aer)); if (aer == NULL) return (-1); /* Save the Command ID for use in the completion message */ aer->cid = cid; pthread_mutex_lock(&sc->aer_mtx); sc->aer_count++; STAILQ_INSERT_TAIL(&sc->aer_list, aer, link); pthread_mutex_unlock(&sc->aer_mtx); return (0); } /* * Get an Async Event Request structure * * Returns a pointer to an AER previously submitted by the host or NULL if * no AER's exist. Caller is responsible for freeing the returned struct. */ static struct pci_nvme_aer * pci_nvme_aer_get(struct pci_nvme_softc *sc) { struct pci_nvme_aer *aer = NULL; pthread_mutex_lock(&sc->aer_mtx); aer = STAILQ_FIRST(&sc->aer_list); if (aer != NULL) { STAILQ_REMOVE_HEAD(&sc->aer_list, link); sc->aer_count--; } pthread_mutex_unlock(&sc->aer_mtx); return (aer); } static void pci_nvme_aen_reset(struct pci_nvme_softc *sc) { uint32_t atype; memset(sc->aen, 0, PCI_NVME_AE_TYPE_MAX * sizeof(struct pci_nvme_aen)); for (atype = 0; atype < PCI_NVME_AE_TYPE_MAX; atype++) { sc->aen[atype].atype = atype; } } static void pci_nvme_aen_init(struct pci_nvme_softc *sc) { char nstr[80]; pci_nvme_aen_reset(sc); pthread_mutex_init(&sc->aen_mtx, NULL); pthread_create(&sc->aen_tid, NULL, aen_thr, sc); snprintf(nstr, sizeof(nstr), "nvme-aen-%d:%d", sc->nsc_pi->pi_slot, sc->nsc_pi->pi_func); pthread_set_name_np(sc->aen_tid, nstr); } static void pci_nvme_aen_destroy(struct pci_nvme_softc *sc) { pci_nvme_aen_reset(sc); } /* Notify the AEN thread of pending work */ static void pci_nvme_aen_notify(struct pci_nvme_softc *sc) { pthread_cond_signal(&sc->aen_cond); } /* * Post an Asynchronous Event Notification */ static int32_t pci_nvme_aen_post(struct pci_nvme_softc *sc, pci_nvme_async_type atype, uint32_t event_data) { struct pci_nvme_aen *aen; if (atype >= PCI_NVME_AE_TYPE_MAX) { return(EINVAL); } pthread_mutex_lock(&sc->aen_mtx); aen = &sc->aen[atype]; /* Has the controller already posted an event of this type? */ if (aen->posted) { pthread_mutex_unlock(&sc->aen_mtx); return(EALREADY); } aen->event_data = event_data; aen->posted = true; pthread_mutex_unlock(&sc->aen_mtx); pci_nvme_aen_notify(sc); return(0); } static void pci_nvme_aen_process(struct pci_nvme_softc *sc) { struct pci_nvme_aer *aer; struct pci_nvme_aen *aen; pci_nvme_async_type atype; uint32_t mask; uint16_t status; uint8_t lid; #ifndef __FreeBSD__ lid = 0; #endif assert(pthread_mutex_isowned_np(&sc->aen_mtx)); for (atype = 0; atype < PCI_NVME_AE_TYPE_MAX; atype++) { aen = &sc->aen[atype]; /* Previous iterations may have depleted the available AER's */ if (!pci_nvme_aer_available(sc)) { DPRINTF("%s: no AER", __func__); break; } if (!aen->posted) { DPRINTF("%s: no AEN posted for atype=%#x", __func__, atype); continue; } status = NVME_SC_SUCCESS; /* Is the event masked? */ mask = sc->feat[NVME_FEAT_ASYNC_EVENT_CONFIGURATION].cdw11; DPRINTF("%s: atype=%#x mask=%#x event_data=%#x", __func__, atype, mask, aen->event_data); switch (atype) { case PCI_NVME_AE_TYPE_ERROR: lid = NVME_LOG_ERROR; break; case PCI_NVME_AE_TYPE_SMART: mask &= 0xff; if ((mask & aen->event_data) == 0) continue; lid = NVME_LOG_HEALTH_INFORMATION; break; case PCI_NVME_AE_TYPE_NOTICE: if (aen->event_data >= PCI_NVME_AEI_NOTICE_MAX) { EPRINTLN("%s unknown AEN notice type %u", __func__, aen->event_data); status = NVME_SC_INTERNAL_DEVICE_ERROR; lid = 0; break; } if ((PCI_NVME_AEI_NOTICE_MASK(aen->event_data) & mask) == 0) continue; switch (aen->event_data) { case PCI_NVME_AEI_NOTICE_NS_ATTR_CHANGED: lid = NVME_LOG_CHANGED_NAMESPACE; break; case PCI_NVME_AEI_NOTICE_FW_ACTIVATION: lid = NVME_LOG_FIRMWARE_SLOT; break; case PCI_NVME_AEI_NOTICE_TELEMETRY_CHANGE: lid = NVME_LOG_TELEMETRY_CONTROLLER_INITIATED; break; case PCI_NVME_AEI_NOTICE_ANA_CHANGE: lid = NVME_LOG_ASYMMETRIC_NAMESPACE_ACCESS; break; case PCI_NVME_AEI_NOTICE_PREDICT_LATENCY_CHANGE: lid = NVME_LOG_PREDICTABLE_LATENCY_EVENT_AGGREGATE; break; case PCI_NVME_AEI_NOTICE_LBA_STATUS_ALERT: lid = NVME_LOG_LBA_STATUS_INFORMATION; break; case PCI_NVME_AEI_NOTICE_ENDURANCE_GROUP_CHANGE: lid = NVME_LOG_ENDURANCE_GROUP_EVENT_AGGREGATE; break; default: lid = 0; } break; default: /* bad type?!? */ EPRINTLN("%s unknown AEN type %u", __func__, atype); status = NVME_SC_INTERNAL_DEVICE_ERROR; lid = 0; break; } aer = pci_nvme_aer_get(sc); assert(aer != NULL); DPRINTF("%s: CID=%#x CDW0=%#x", __func__, aer->cid, (lid << 16) | (aen->event_data << 8) | atype); pci_nvme_cq_update(sc, &sc->compl_queues[0], (lid << 16) | (aen->event_data << 8) | atype, /* cdw0 */ aer->cid, 0, /* SQID */ status); aen->event_data = 0; aen->posted = false; pci_generate_msix(sc->nsc_pi, 0); } } static void * aen_thr(void *arg) { struct pci_nvme_softc *sc; sc = arg; pthread_mutex_lock(&sc->aen_mtx); for (;;) { pci_nvme_aen_process(sc); pthread_cond_wait(&sc->aen_cond, &sc->aen_mtx); } #ifdef __FreeBSD__ /* Smatch spots unreachable code */ pthread_mutex_unlock(&sc->aen_mtx); pthread_exit(NULL); #endif return (NULL); } static void pci_nvme_reset_locked(struct pci_nvme_softc *sc) { uint32_t i; DPRINTF("%s", __func__); sc->regs.cap_lo = (ZERO_BASED(sc->max_qentries) & NVME_CAP_LO_REG_MQES_MASK) | NVMEF(NVME_CAP_LO_REG_CQR, 1) | NVMEF(NVME_CAP_LO_REG_TO, 60); sc->regs.cap_hi = NVMEF(NVME_CAP_HI_REG_CSS_NVM, 1); sc->regs.vs = NVME_REV(1,4); /* NVMe v1.4 */ sc->regs.cc = 0; assert(sc->submit_queues != NULL); for (i = 0; i < sc->num_squeues + 1; i++) { sc->submit_queues[i].qbase = NULL; sc->submit_queues[i].size = 0; sc->submit_queues[i].cqid = 0; sc->submit_queues[i].tail = 0; sc->submit_queues[i].head = 0; } assert(sc->compl_queues != NULL); for (i = 0; i < sc->num_cqueues + 1; i++) { sc->compl_queues[i].qbase = NULL; sc->compl_queues[i].size = 0; sc->compl_queues[i].tail = 0; sc->compl_queues[i].head = 0; } sc->num_q_is_set = false; pci_nvme_aer_destroy(sc); pci_nvme_aen_destroy(sc); /* * Clear CSTS.RDY last to prevent the host from enabling Controller * before cleanup completes */ sc->regs.csts = 0; } static void pci_nvme_reset(struct pci_nvme_softc *sc) { pthread_mutex_lock(&sc->mtx); pci_nvme_reset_locked(sc); pthread_mutex_unlock(&sc->mtx); } static int pci_nvme_init_controller(struct pci_nvme_softc *sc) { uint16_t acqs, asqs; DPRINTF("%s", __func__); /* * NVMe 2.0 states that "enabling a controller while this field is * cleared to 0h produces undefined results" for both ACQS and * ASQS. If zero, set CFS and do not become ready. */ asqs = ONE_BASED(NVMEV(NVME_AQA_REG_ASQS, sc->regs.aqa)); if (asqs < 2) { EPRINTLN("%s: illegal ASQS value %#x (aqa=%#x)", __func__, asqs - 1, sc->regs.aqa); sc->regs.csts |= NVME_CSTS_CFS; return (-1); } sc->submit_queues[0].size = asqs; sc->submit_queues[0].qbase = vm_map_gpa(sc->nsc_pi->pi_vmctx, sc->regs.asq, sizeof(struct nvme_command) * asqs); if (sc->submit_queues[0].qbase == NULL) { EPRINTLN("%s: ASQ vm_map_gpa(%lx) failed", __func__, sc->regs.asq); sc->regs.csts |= NVME_CSTS_CFS; return (-1); } DPRINTF("%s mapping Admin-SQ guest 0x%lx, host: %p", __func__, sc->regs.asq, sc->submit_queues[0].qbase); acqs = ONE_BASED(NVMEV(NVME_AQA_REG_ACQS, sc->regs.aqa)); if (acqs < 2) { EPRINTLN("%s: illegal ACQS value %#x (aqa=%#x)", __func__, acqs - 1, sc->regs.aqa); sc->regs.csts |= NVME_CSTS_CFS; return (-1); } sc->compl_queues[0].size = acqs; sc->compl_queues[0].qbase = vm_map_gpa(sc->nsc_pi->pi_vmctx, sc->regs.acq, sizeof(struct nvme_completion) * acqs); if (sc->compl_queues[0].qbase == NULL) { EPRINTLN("%s: ACQ vm_map_gpa(%lx) failed", __func__, sc->regs.acq); sc->regs.csts |= NVME_CSTS_CFS; return (-1); } sc->compl_queues[0].intr_en = NVME_CQ_INTEN; DPRINTF("%s mapping Admin-CQ guest 0x%lx, host: %p", __func__, sc->regs.acq, sc->compl_queues[0].qbase); return (0); } static int nvme_prp_memcpy(struct vmctx *ctx, uint64_t prp1, uint64_t prp2, uint8_t *b, size_t len, enum nvme_copy_dir dir) { uint8_t *p; size_t bytes; if (len > (8 * 1024)) { return (-1); } /* Copy from the start of prp1 to the end of the physical page */ bytes = PAGE_SIZE - (prp1 & PAGE_MASK); bytes = MIN(bytes, len); p = vm_map_gpa(ctx, prp1, bytes); if (p == NULL) { return (-1); } if (dir == NVME_COPY_TO_PRP) memcpy(p, b, bytes); else memcpy(b, p, bytes); b += bytes; len -= bytes; if (len == 0) { return (0); } len = MIN(len, PAGE_SIZE); p = vm_map_gpa(ctx, prp2, len); if (p == NULL) { return (-1); } if (dir == NVME_COPY_TO_PRP) memcpy(p, b, len); else memcpy(b, p, len); return (0); } /* * Write a Completion Queue Entry update * * Write the completion and update the doorbell value */ static void pci_nvme_cq_update(struct pci_nvme_softc *sc, struct nvme_completion_queue *cq, uint32_t cdw0, uint16_t cid, uint16_t sqid, uint16_t status) { struct nvme_submission_queue *sq = &sc->submit_queues[sqid]; struct nvme_completion *cqe; assert(cq->qbase != NULL); pthread_mutex_lock(&cq->mtx); cqe = &cq->qbase[cq->tail]; /* Flip the phase bit */ status |= (cqe->status ^ NVME_STATUS_P) & NVME_STATUS_P_MASK; cqe->cdw0 = cdw0; cqe->sqhd = sq->head; cqe->sqid = sqid; cqe->cid = cid; cqe->status = status; cq->tail++; if (cq->tail >= cq->size) { cq->tail = 0; } pthread_mutex_unlock(&cq->mtx); } static int nvme_opc_delete_io_sq(struct pci_nvme_softc* sc, struct nvme_command* command, struct nvme_completion* compl) { uint16_t qid = command->cdw10 & 0xffff; DPRINTF("%s DELETE_IO_SQ %u", __func__, qid); if (qid == 0 || qid > sc->num_squeues || (sc->submit_queues[qid].qbase == NULL)) { WPRINTF("%s NOT PERMITTED queue id %u / num_squeues %u", __func__, qid, sc->num_squeues); pci_nvme_status_tc(&compl->status, NVME_SCT_COMMAND_SPECIFIC, NVME_SC_INVALID_QUEUE_IDENTIFIER); return (1); } sc->submit_queues[qid].qbase = NULL; sc->submit_queues[qid].cqid = 0; pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS); return (1); } static int nvme_opc_create_io_sq(struct pci_nvme_softc* sc, struct nvme_command* command, struct nvme_completion* compl) { if (command->cdw11 & NVME_CMD_CDW11_PC) { uint16_t qid = command->cdw10 & 0xffff; struct nvme_submission_queue *nsq; if ((qid == 0) || (qid > sc->num_squeues) || (sc->submit_queues[qid].qbase != NULL)) { WPRINTF("%s queue index %u > num_squeues %u", __func__, qid, sc->num_squeues); pci_nvme_status_tc(&compl->status, NVME_SCT_COMMAND_SPECIFIC, NVME_SC_INVALID_QUEUE_IDENTIFIER); return (1); } nsq = &sc->submit_queues[qid]; nsq->size = ONE_BASED((command->cdw10 >> 16) & 0xffff); DPRINTF("%s size=%u (max=%u)", __func__, nsq->size, sc->max_qentries); if ((nsq->size < 2) || (nsq->size > sc->max_qentries)) { /* * Queues must specify at least two entries * NOTE: "MAXIMUM QUEUE SIZE EXCEEDED" was renamed to * "INVALID QUEUE SIZE" in the NVM Express 1.3 Spec */ pci_nvme_status_tc(&compl->status, NVME_SCT_COMMAND_SPECIFIC, NVME_SC_MAXIMUM_QUEUE_SIZE_EXCEEDED); return (1); } nsq->head = nsq->tail = 0; nsq->cqid = (command->cdw11 >> 16) & 0xffff; if ((nsq->cqid == 0) || (nsq->cqid > sc->num_cqueues)) { pci_nvme_status_tc(&compl->status, NVME_SCT_COMMAND_SPECIFIC, NVME_SC_INVALID_QUEUE_IDENTIFIER); return (1); } if (sc->compl_queues[nsq->cqid].qbase == NULL) { pci_nvme_status_tc(&compl->status, NVME_SCT_COMMAND_SPECIFIC, NVME_SC_COMPLETION_QUEUE_INVALID); return (1); } nsq->qpriority = (command->cdw11 >> 1) & 0x03; nsq->qbase = vm_map_gpa(sc->nsc_pi->pi_vmctx, command->prp1, sizeof(struct nvme_command) * (size_t)nsq->size); DPRINTF("%s sq %u size %u gaddr %p cqid %u", __func__, qid, nsq->size, nsq->qbase, nsq->cqid); pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS); DPRINTF("%s completed creating IOSQ qid %u", __func__, qid); } else { /* * Guest sent non-cont submission queue request. * This setting is unsupported by this emulation. */ WPRINTF("%s unsupported non-contig (list-based) " "create i/o submission queue", __func__); pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD); } return (1); } static int nvme_opc_delete_io_cq(struct pci_nvme_softc* sc, struct nvme_command* command, struct nvme_completion* compl) { uint16_t qid = command->cdw10 & 0xffff; uint16_t sqid; DPRINTF("%s DELETE_IO_CQ %u", __func__, qid); if (qid == 0 || qid > sc->num_cqueues || (sc->compl_queues[qid].qbase == NULL)) { WPRINTF("%s queue index %u / num_cqueues %u", __func__, qid, sc->num_cqueues); pci_nvme_status_tc(&compl->status, NVME_SCT_COMMAND_SPECIFIC, NVME_SC_INVALID_QUEUE_IDENTIFIER); return (1); } /* Deleting an Active CQ is an error */ for (sqid = 1; sqid < sc->num_squeues + 1; sqid++) if (sc->submit_queues[sqid].cqid == qid) { pci_nvme_status_tc(&compl->status, NVME_SCT_COMMAND_SPECIFIC, NVME_SC_INVALID_QUEUE_DELETION); return (1); } sc->compl_queues[qid].qbase = NULL; pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS); return (1); } static int nvme_opc_create_io_cq(struct pci_nvme_softc* sc, struct nvme_command* command, struct nvme_completion* compl) { struct nvme_completion_queue *ncq; uint16_t qid = command->cdw10 & 0xffff; /* Only support Physically Contiguous queues */ if ((command->cdw11 & NVME_CMD_CDW11_PC) == 0) { WPRINTF("%s unsupported non-contig (list-based) " "create i/o completion queue", __func__); pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD); return (1); } if ((qid == 0) || (qid > sc->num_cqueues) || (sc->compl_queues[qid].qbase != NULL)) { WPRINTF("%s queue index %u > num_cqueues %u", __func__, qid, sc->num_cqueues); pci_nvme_status_tc(&compl->status, NVME_SCT_COMMAND_SPECIFIC, NVME_SC_INVALID_QUEUE_IDENTIFIER); return (1); } ncq = &sc->compl_queues[qid]; ncq->intr_en = (command->cdw11 & NVME_CMD_CDW11_IEN) >> 1; ncq->intr_vec = (command->cdw11 >> 16) & 0xffff; if (ncq->intr_vec > (sc->max_queues + 1)) { pci_nvme_status_tc(&compl->status, NVME_SCT_COMMAND_SPECIFIC, NVME_SC_INVALID_INTERRUPT_VECTOR); return (1); } ncq->size = ONE_BASED((command->cdw10 >> 16) & 0xffff); if ((ncq->size < 2) || (ncq->size > sc->max_qentries)) { /* * Queues must specify at least two entries * NOTE: "MAXIMUM QUEUE SIZE EXCEEDED" was renamed to * "INVALID QUEUE SIZE" in the NVM Express 1.3 Spec */ pci_nvme_status_tc(&compl->status, NVME_SCT_COMMAND_SPECIFIC, NVME_SC_MAXIMUM_QUEUE_SIZE_EXCEEDED); return (1); } ncq->head = ncq->tail = 0; ncq->qbase = vm_map_gpa(sc->nsc_pi->pi_vmctx, command->prp1, sizeof(struct nvme_command) * (size_t)ncq->size); pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS); return (1); } static int nvme_opc_get_log_page(struct pci_nvme_softc* sc, struct nvme_command* command, struct nvme_completion* compl) { uint64_t logoff; uint32_t logsize; uint8_t logpage; pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS); /* * Command specifies the number of dwords to return in fields NUMDU * and NUMDL. This is a zero-based value. */ logpage = command->cdw10 & 0xFF; logsize = ((command->cdw11 << 16) | (command->cdw10 >> 16)) + 1; logsize *= sizeof(uint32_t); logoff = ((uint64_t)(command->cdw13) << 32) | command->cdw12; DPRINTF("%s log page %u offset %lu len %u", __func__, logpage, logoff, logsize); switch (logpage) { case NVME_LOG_ERROR: if (logoff >= sizeof(sc->err_log)) { pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD); break; } nvme_prp_memcpy(sc->nsc_pi->pi_vmctx, command->prp1, command->prp2, (uint8_t *)&sc->err_log + logoff, MIN(logsize, sizeof(sc->err_log) - logoff), NVME_COPY_TO_PRP); break; case NVME_LOG_HEALTH_INFORMATION: if (logoff >= sizeof(sc->health_log)) { pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD); break; } pthread_mutex_lock(&sc->mtx); memcpy(&sc->health_log.data_units_read, &sc->read_data_units, sizeof(sc->health_log.data_units_read)); memcpy(&sc->health_log.data_units_written, &sc->write_data_units, sizeof(sc->health_log.data_units_written)); memcpy(&sc->health_log.host_read_commands, &sc->read_commands, sizeof(sc->health_log.host_read_commands)); memcpy(&sc->health_log.host_write_commands, &sc->write_commands, sizeof(sc->health_log.host_write_commands)); pthread_mutex_unlock(&sc->mtx); nvme_prp_memcpy(sc->nsc_pi->pi_vmctx, command->prp1, command->prp2, (uint8_t *)&sc->health_log + logoff, MIN(logsize, sizeof(sc->health_log) - logoff), NVME_COPY_TO_PRP); break; case NVME_LOG_FIRMWARE_SLOT: if (logoff >= sizeof(sc->fw_log)) { pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD); break; } nvme_prp_memcpy(sc->nsc_pi->pi_vmctx, command->prp1, command->prp2, (uint8_t *)&sc->fw_log + logoff, MIN(logsize, sizeof(sc->fw_log) - logoff), NVME_COPY_TO_PRP); break; case NVME_LOG_CHANGED_NAMESPACE: if (logoff >= sizeof(sc->ns_log)) { pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD); break; } nvme_prp_memcpy(sc->nsc_pi->pi_vmctx, command->prp1, command->prp2, (uint8_t *)&sc->ns_log + logoff, MIN(logsize, sizeof(sc->ns_log) - logoff), NVME_COPY_TO_PRP); memset(&sc->ns_log, 0, sizeof(sc->ns_log)); break; default: DPRINTF("%s get log page %x command not supported", __func__, logpage); pci_nvme_status_tc(&compl->status, NVME_SCT_COMMAND_SPECIFIC, NVME_SC_INVALID_LOG_PAGE); } return (1); } static int nvme_opc_identify(struct pci_nvme_softc* sc, struct nvme_command* command, struct nvme_completion* compl) { void *dest; uint16_t status; DPRINTF("%s identify 0x%x nsid 0x%x", __func__, command->cdw10 & 0xFF, command->nsid); status = 0; pci_nvme_status_genc(&status, NVME_SC_SUCCESS); switch (command->cdw10 & 0xFF) { case 0x00: /* return Identify Namespace data structure */ /* Global NS only valid with NS Management */ if (command->nsid == NVME_GLOBAL_NAMESPACE_TAG) { pci_nvme_status_genc(&status, NVME_SC_INVALID_NAMESPACE_OR_FORMAT); break; } nvme_prp_memcpy(sc->nsc_pi->pi_vmctx, command->prp1, command->prp2, (uint8_t *)&sc->nsdata, sizeof(sc->nsdata), NVME_COPY_TO_PRP); break; case 0x01: /* return Identify Controller data structure */ nvme_prp_memcpy(sc->nsc_pi->pi_vmctx, command->prp1, command->prp2, (uint8_t *)&sc->ctrldata, sizeof(sc->ctrldata), NVME_COPY_TO_PRP); break; case 0x02: /* list of 1024 active NSIDs > CDW1.NSID */ dest = vm_map_gpa(sc->nsc_pi->pi_vmctx, command->prp1, sizeof(uint32_t) * 1024); /* All unused entries shall be zero */ memset(dest, 0, sizeof(uint32_t) * 1024); ((uint32_t *)dest)[0] = 1; break; case 0x03: /* list of NSID structures in CDW1.NSID, 4096 bytes */ if (command->nsid != 1) { pci_nvme_status_genc(&status, NVME_SC_INVALID_NAMESPACE_OR_FORMAT); break; } dest = vm_map_gpa(sc->nsc_pi->pi_vmctx, command->prp1, sizeof(uint32_t) * 1024); /* All bytes after the descriptor shall be zero */ memset(dest, 0, sizeof(uint32_t) * 1024); /* Return NIDT=1 (i.e. EUI64) descriptor */ ((uint8_t *)dest)[0] = 1; ((uint8_t *)dest)[1] = sizeof(uint64_t); memcpy(((uint8_t *)dest) + 4, sc->nsdata.eui64, sizeof(uint64_t)); break; case 0x13: /* * Controller list is optional but used by UNH tests. Return * a valid but empty list. */ dest = vm_map_gpa(sc->nsc_pi->pi_vmctx, command->prp1, sizeof(uint16_t) * 2048); memset(dest, 0, sizeof(uint16_t) * 2048); break; default: DPRINTF("%s unsupported identify command requested 0x%x", __func__, command->cdw10 & 0xFF); pci_nvme_status_genc(&status, NVME_SC_INVALID_FIELD); break; } compl->status = status; return (1); } static const char * nvme_fid_to_name(uint8_t fid) { const char *name; switch (fid) { case NVME_FEAT_ARBITRATION: name = "Arbitration"; break; case NVME_FEAT_POWER_MANAGEMENT: name = "Power Management"; break; case NVME_FEAT_LBA_RANGE_TYPE: name = "LBA Range Type"; break; case NVME_FEAT_TEMPERATURE_THRESHOLD: name = "Temperature Threshold"; break; case NVME_FEAT_ERROR_RECOVERY: name = "Error Recovery"; break; case NVME_FEAT_VOLATILE_WRITE_CACHE: name = "Volatile Write Cache"; break; case NVME_FEAT_NUMBER_OF_QUEUES: name = "Number of Queues"; break; case NVME_FEAT_INTERRUPT_COALESCING: name = "Interrupt Coalescing"; break; case NVME_FEAT_INTERRUPT_VECTOR_CONFIGURATION: name = "Interrupt Vector Configuration"; break; case NVME_FEAT_WRITE_ATOMICITY: name = "Write Atomicity Normal"; break; case NVME_FEAT_ASYNC_EVENT_CONFIGURATION: name = "Asynchronous Event Configuration"; break; case NVME_FEAT_AUTONOMOUS_POWER_STATE_TRANSITION: name = "Autonomous Power State Transition"; break; case NVME_FEAT_HOST_MEMORY_BUFFER: name = "Host Memory Buffer"; break; case NVME_FEAT_TIMESTAMP: name = "Timestamp"; break; case NVME_FEAT_KEEP_ALIVE_TIMER: name = "Keep Alive Timer"; break; case NVME_FEAT_HOST_CONTROLLED_THERMAL_MGMT: name = "Host Controlled Thermal Management"; break; case NVME_FEAT_NON_OP_POWER_STATE_CONFIG: name = "Non-Operation Power State Config"; break; case NVME_FEAT_READ_RECOVERY_LEVEL_CONFIG: name = "Read Recovery Level Config"; break; case NVME_FEAT_PREDICTABLE_LATENCY_MODE_CONFIG: name = "Predictable Latency Mode Config"; break; case NVME_FEAT_PREDICTABLE_LATENCY_MODE_WINDOW: name = "Predictable Latency Mode Window"; break; case NVME_FEAT_LBA_STATUS_INFORMATION_ATTRIBUTES: name = "LBA Status Information Report Interval"; break; case NVME_FEAT_HOST_BEHAVIOR_SUPPORT: name = "Host Behavior Support"; break; case NVME_FEAT_SANITIZE_CONFIG: name = "Sanitize Config"; break; case NVME_FEAT_ENDURANCE_GROUP_EVENT_CONFIGURATION: name = "Endurance Group Event Configuration"; break; case NVME_FEAT_SOFTWARE_PROGRESS_MARKER: name = "Software Progress Marker"; break; case NVME_FEAT_HOST_IDENTIFIER: name = "Host Identifier"; break; case NVME_FEAT_RESERVATION_NOTIFICATION_MASK: name = "Reservation Notification Mask"; break; case NVME_FEAT_RESERVATION_PERSISTENCE: name = "Reservation Persistence"; break; case NVME_FEAT_NAMESPACE_WRITE_PROTECTION_CONFIG: name = "Namespace Write Protection Config"; break; default: name = "Unknown"; break; } return (name); } static void nvme_feature_invalid_cb(struct pci_nvme_softc *sc __unused, struct nvme_feature_obj *feat __unused, struct nvme_command *command __unused, struct nvme_completion *compl) { pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD); } static void nvme_feature_iv_config(struct pci_nvme_softc *sc, struct nvme_feature_obj *feat __unused, struct nvme_command *command, struct nvme_completion *compl) { uint32_t i; uint32_t cdw11 = command->cdw11; uint16_t iv; bool cd; pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD); iv = cdw11 & 0xffff; cd = cdw11 & (1 << 16); if (iv > (sc->max_queues + 1)) { return; } /* No Interrupt Coalescing (i.e. not Coalescing Disable) for Admin Q */ if ((iv == 0) && !cd) return; /* Requested Interrupt Vector must be used by a CQ */ for (i = 0; i < sc->num_cqueues + 1; i++) { if (sc->compl_queues[i].intr_vec == iv) { pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS); } } } #define NVME_ASYNC_EVENT_ENDURANCE_GROUP (0x4000) static void nvme_feature_async_event(struct pci_nvme_softc *sc __unused, struct nvme_feature_obj *feat __unused, struct nvme_command *command, struct nvme_completion *compl) { if (command->cdw11 & NVME_ASYNC_EVENT_ENDURANCE_GROUP) pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD); } #define NVME_TEMP_THRESH_OVER 0 #define NVME_TEMP_THRESH_UNDER 1 static void nvme_feature_temperature(struct pci_nvme_softc *sc, struct nvme_feature_obj *feat __unused, struct nvme_command *command, struct nvme_completion *compl) { uint16_t tmpth; /* Temperature Threshold */ uint8_t tmpsel; /* Threshold Temperature Select */ uint8_t thsel; /* Threshold Type Select */ bool set_crit = false; bool report_crit; tmpth = command->cdw11 & 0xffff; tmpsel = (command->cdw11 >> 16) & 0xf; thsel = (command->cdw11 >> 20) & 0x3; DPRINTF("%s: tmpth=%#x tmpsel=%#x thsel=%#x", __func__, tmpth, tmpsel, thsel); /* Check for unsupported values */ if (((tmpsel != 0) && (tmpsel != 0xf)) || (thsel > NVME_TEMP_THRESH_UNDER)) { pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD); return; } if (((thsel == NVME_TEMP_THRESH_OVER) && (NVME_TEMPERATURE >= tmpth)) || ((thsel == NVME_TEMP_THRESH_UNDER) && (NVME_TEMPERATURE <= tmpth))) set_crit = true; pthread_mutex_lock(&sc->mtx); if (set_crit) sc->health_log.critical_warning |= NVME_CRIT_WARN_ST_TEMPERATURE; else sc->health_log.critical_warning &= ~NVME_CRIT_WARN_ST_TEMPERATURE; pthread_mutex_unlock(&sc->mtx); report_crit = sc->feat[NVME_FEAT_ASYNC_EVENT_CONFIGURATION].cdw11 & NVME_CRIT_WARN_ST_TEMPERATURE; if (set_crit && report_crit) pci_nvme_aen_post(sc, PCI_NVME_AE_TYPE_SMART, sc->health_log.critical_warning); DPRINTF("%s: set_crit=%c critical_warning=%#x status=%#x", __func__, set_crit ? 'T':'F', sc->health_log.critical_warning, compl->status); } static void nvme_feature_num_queues(struct pci_nvme_softc *sc, struct nvme_feature_obj *feat __unused, struct nvme_command *command, struct nvme_completion *compl) { uint16_t nqr; /* Number of Queues Requested */ if (sc->num_q_is_set) { WPRINTF("%s: Number of Queues already set", __func__); pci_nvme_status_genc(&compl->status, NVME_SC_COMMAND_SEQUENCE_ERROR); return; } nqr = command->cdw11 & 0xFFFF; if (nqr == 0xffff) { WPRINTF("%s: Illegal NSQR value %#x", __func__, nqr); pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD); return; } sc->num_squeues = ONE_BASED(nqr); if (sc->num_squeues > sc->max_queues) { DPRINTF("NSQR=%u is greater than max %u", sc->num_squeues, sc->max_queues); sc->num_squeues = sc->max_queues; } nqr = (command->cdw11 >> 16) & 0xFFFF; if (nqr == 0xffff) { WPRINTF("%s: Illegal NCQR value %#x", __func__, nqr); pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD); return; } sc->num_cqueues = ONE_BASED(nqr); if (sc->num_cqueues > sc->max_queues) { DPRINTF("NCQR=%u is greater than max %u", sc->num_cqueues, sc->max_queues); sc->num_cqueues = sc->max_queues; } /* Patch the command value which will be saved on callback's return */ command->cdw11 = NVME_FEATURE_NUM_QUEUES(sc); compl->cdw0 = NVME_FEATURE_NUM_QUEUES(sc); sc->num_q_is_set = true; } static int nvme_opc_set_features(struct pci_nvme_softc *sc, struct nvme_command *command, struct nvme_completion *compl) { struct nvme_feature_obj *feat; uint32_t nsid = command->nsid; uint8_t fid = NVMEV(NVME_FEAT_SET_FID, command->cdw10); bool sv = NVMEV(NVME_FEAT_SET_SV, command->cdw10); DPRINTF("%s: Feature ID 0x%x (%s)", __func__, fid, nvme_fid_to_name(fid)); if (fid >= NVME_FID_MAX) { DPRINTF("%s invalid feature 0x%x", __func__, fid); pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD); return (1); } if (sv) { pci_nvme_status_tc(&compl->status, NVME_SCT_COMMAND_SPECIFIC, NVME_SC_FEATURE_NOT_SAVEABLE); return (1); } feat = &sc->feat[fid]; if (feat->namespace_specific && (nsid == NVME_GLOBAL_NAMESPACE_TAG)) { pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD); return (1); } if (!feat->namespace_specific && !((nsid == 0) || (nsid == NVME_GLOBAL_NAMESPACE_TAG))) { pci_nvme_status_tc(&compl->status, NVME_SCT_COMMAND_SPECIFIC, NVME_SC_FEATURE_NOT_NS_SPECIFIC); return (1); } compl->cdw0 = 0; pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS); if (feat->set) feat->set(sc, feat, command, compl); else { pci_nvme_status_tc(&compl->status, NVME_SCT_COMMAND_SPECIFIC, NVME_SC_FEATURE_NOT_CHANGEABLE); return (1); } DPRINTF("%s: status=%#x cdw11=%#x", __func__, compl->status, command->cdw11); if (compl->status == NVME_SC_SUCCESS) { feat->cdw11 = command->cdw11; if ((fid == NVME_FEAT_ASYNC_EVENT_CONFIGURATION) && (command->cdw11 != 0)) pci_nvme_aen_notify(sc); } return (0); } #define NVME_FEATURES_SEL_SUPPORTED 0x3 #define NVME_FEATURES_NS_SPECIFIC (1 << 1) static int nvme_opc_get_features(struct pci_nvme_softc* sc, struct nvme_command* command, struct nvme_completion* compl) { struct nvme_feature_obj *feat; uint8_t fid = command->cdw10 & 0xFF; uint8_t sel = (command->cdw10 >> 8) & 0x7; DPRINTF("%s: Feature ID 0x%x (%s)", __func__, fid, nvme_fid_to_name(fid)); if (fid >= NVME_FID_MAX) { DPRINTF("%s invalid feature 0x%x", __func__, fid); pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD); return (1); } compl->cdw0 = 0; pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS); feat = &sc->feat[fid]; if (feat->get) { feat->get(sc, feat, command, compl); } if (compl->status == NVME_SC_SUCCESS) { if ((sel == NVME_FEATURES_SEL_SUPPORTED) && feat->namespace_specific) compl->cdw0 = NVME_FEATURES_NS_SPECIFIC; else compl->cdw0 = feat->cdw11; } return (0); } static int nvme_opc_format_nvm(struct pci_nvme_softc* sc, struct nvme_command* command, struct nvme_completion* compl) { uint8_t ses, lbaf, pi; /* Only supports Secure Erase Setting - User Data Erase */ ses = (command->cdw10 >> 9) & 0x7; if (ses > 0x1) { pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD); return (1); } /* Only supports a single LBA Format */ lbaf = command->cdw10 & 0xf; if (lbaf != 0) { pci_nvme_status_tc(&compl->status, NVME_SCT_COMMAND_SPECIFIC, NVME_SC_INVALID_FORMAT); return (1); } /* Doesn't support Protection Information */ pi = (command->cdw10 >> 5) & 0x7; if (pi != 0) { pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD); return (1); } if (sc->nvstore.type == NVME_STOR_RAM) { if (sc->nvstore.ctx) free(sc->nvstore.ctx); sc->nvstore.ctx = calloc(1, sc->nvstore.size); pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS); } else { struct pci_nvme_ioreq *req; int err; req = pci_nvme_get_ioreq(sc); if (req == NULL) { pci_nvme_status_genc(&compl->status, NVME_SC_INTERNAL_DEVICE_ERROR); WPRINTF("%s: unable to allocate IO req", __func__); return (1); } req->nvme_sq = &sc->submit_queues[0]; req->sqid = 0; req->opc = command->opc; req->cid = command->cid; req->nsid = command->nsid; req->io_req.br_offset = 0; req->io_req.br_resid = sc->nvstore.size; req->io_req.br_callback = pci_nvme_io_done; err = blockif_delete(sc->nvstore.ctx, &req->io_req); if (err) { pci_nvme_status_genc(&compl->status, NVME_SC_INTERNAL_DEVICE_ERROR); pci_nvme_release_ioreq(sc, req); } else compl->status = NVME_NO_STATUS; } return (1); } static int nvme_opc_abort(struct pci_nvme_softc *sc __unused, struct nvme_command *command, struct nvme_completion *compl) { DPRINTF("%s submission queue %u, command ID 0x%x", __func__, command->cdw10 & 0xFFFF, (command->cdw10 >> 16) & 0xFFFF); /* TODO: search for the command ID and abort it */ compl->cdw0 = 1; pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS); return (1); } static int nvme_opc_async_event_req(struct pci_nvme_softc* sc, struct nvme_command* command, struct nvme_completion* compl) { DPRINTF("%s async event request count=%u aerl=%u cid=%#x", __func__, sc->aer_count, sc->ctrldata.aerl, command->cid); /* Don't exceed the Async Event Request Limit (AERL). */ if (pci_nvme_aer_limit_reached(sc)) { pci_nvme_status_tc(&compl->status, NVME_SCT_COMMAND_SPECIFIC, NVME_SC_ASYNC_EVENT_REQUEST_LIMIT_EXCEEDED); return (1); } if (pci_nvme_aer_add(sc, command->cid)) { pci_nvme_status_tc(&compl->status, NVME_SCT_GENERIC, NVME_SC_INTERNAL_DEVICE_ERROR); return (1); } /* * Raise events when they happen based on the Set Features cmd. * These events happen async, so only set completion successful if * there is an event reflective of the request to get event. */ compl->status = NVME_NO_STATUS; pci_nvme_aen_notify(sc); return (0); } static void pci_nvme_handle_admin_cmd(struct pci_nvme_softc* sc, uint64_t value) { struct nvme_completion compl; struct nvme_command *cmd; struct nvme_submission_queue *sq; struct nvme_completion_queue *cq; uint16_t sqhead; DPRINTF("%s index %u", __func__, (uint32_t)value); sq = &sc->submit_queues[0]; cq = &sc->compl_queues[0]; pthread_mutex_lock(&sq->mtx); sqhead = sq->head; DPRINTF("sqhead %u, tail %u", sqhead, sq->tail); while (sqhead != atomic_load_acq_short(&sq->tail)) { cmd = &(sq->qbase)[sqhead]; compl.cdw0 = 0; compl.status = 0; switch (cmd->opc) { case NVME_OPC_DELETE_IO_SQ: DPRINTF("%s command DELETE_IO_SQ", __func__); nvme_opc_delete_io_sq(sc, cmd, &compl); break; case NVME_OPC_CREATE_IO_SQ: DPRINTF("%s command CREATE_IO_SQ", __func__); nvme_opc_create_io_sq(sc, cmd, &compl); break; case NVME_OPC_DELETE_IO_CQ: DPRINTF("%s command DELETE_IO_CQ", __func__); nvme_opc_delete_io_cq(sc, cmd, &compl); break; case NVME_OPC_CREATE_IO_CQ: DPRINTF("%s command CREATE_IO_CQ", __func__); nvme_opc_create_io_cq(sc, cmd, &compl); break; case NVME_OPC_GET_LOG_PAGE: DPRINTF("%s command GET_LOG_PAGE", __func__); nvme_opc_get_log_page(sc, cmd, &compl); break; case NVME_OPC_IDENTIFY: DPRINTF("%s command IDENTIFY", __func__); nvme_opc_identify(sc, cmd, &compl); break; case NVME_OPC_ABORT: DPRINTF("%s command ABORT", __func__); nvme_opc_abort(sc, cmd, &compl); break; case NVME_OPC_SET_FEATURES: DPRINTF("%s command SET_FEATURES", __func__); nvme_opc_set_features(sc, cmd, &compl); break; case NVME_OPC_GET_FEATURES: DPRINTF("%s command GET_FEATURES", __func__); nvme_opc_get_features(sc, cmd, &compl); break; case NVME_OPC_FIRMWARE_ACTIVATE: DPRINTF("%s command FIRMWARE_ACTIVATE", __func__); pci_nvme_status_tc(&compl.status, NVME_SCT_COMMAND_SPECIFIC, NVME_SC_INVALID_FIRMWARE_SLOT); break; case NVME_OPC_ASYNC_EVENT_REQUEST: DPRINTF("%s command ASYNC_EVENT_REQ", __func__); nvme_opc_async_event_req(sc, cmd, &compl); break; case NVME_OPC_FORMAT_NVM: DPRINTF("%s command FORMAT_NVM", __func__); if (NVMEV(NVME_CTRLR_DATA_OACS_FORMAT, sc->ctrldata.oacs) == 0) { pci_nvme_status_genc(&compl.status, NVME_SC_INVALID_OPCODE); break; } nvme_opc_format_nvm(sc, cmd, &compl); break; case NVME_OPC_SECURITY_SEND: case NVME_OPC_SECURITY_RECEIVE: case NVME_OPC_SANITIZE: case NVME_OPC_GET_LBA_STATUS: DPRINTF("%s command OPC=%#x (unsupported)", __func__, cmd->opc); /* Valid but unsupported opcodes */ pci_nvme_status_genc(&compl.status, NVME_SC_INVALID_FIELD); break; default: DPRINTF("%s command OPC=%#X (not implemented)", __func__, cmd->opc); pci_nvme_status_genc(&compl.status, NVME_SC_INVALID_OPCODE); } sqhead = (sqhead + 1) % sq->size; if (NVME_COMPLETION_VALID(compl)) { pci_nvme_cq_update(sc, &sc->compl_queues[0], compl.cdw0, cmd->cid, 0, /* SQID */ compl.status); } } DPRINTF("setting sqhead %u", sqhead); sq->head = sqhead; if (cq->head != cq->tail) pci_generate_msix(sc->nsc_pi, 0); pthread_mutex_unlock(&sq->mtx); } /* * Update the Write and Read statistics reported in SMART data * * NVMe defines "data unit" as thousand's of 512 byte blocks and is rounded up. * E.g. 1 data unit is 1 - 1,000 512 byte blocks. 3 data units are 2,001 - 3,000 * 512 byte blocks. Rounding up is achieved by initializing the remainder to 999. */ static void pci_nvme_stats_write_read_update(struct pci_nvme_softc *sc, uint8_t opc, size_t bytes, uint16_t status) { pthread_mutex_lock(&sc->mtx); switch (opc) { case NVME_OPC_WRITE: sc->write_commands++; if (status != NVME_SC_SUCCESS) break; sc->write_dunits_remainder += (bytes / 512); while (sc->write_dunits_remainder >= 1000) { sc->write_data_units++; sc->write_dunits_remainder -= 1000; } break; case NVME_OPC_READ: sc->read_commands++; if (status != NVME_SC_SUCCESS) break; sc->read_dunits_remainder += (bytes / 512); while (sc->read_dunits_remainder >= 1000) { sc->read_data_units++; sc->read_dunits_remainder -= 1000; } break; default: DPRINTF("%s: Invalid OPC 0x%02x for stats", __func__, opc); break; } pthread_mutex_unlock(&sc->mtx); } /* * Check if the combination of Starting LBA (slba) and number of blocks * exceeds the range of the underlying storage. * * Because NVMe specifies the SLBA in blocks as a uint64_t and blockif stores * the capacity in bytes as a uint64_t, care must be taken to avoid integer * overflow. */ static bool pci_nvme_out_of_range(struct pci_nvme_blockstore *nvstore, uint64_t slba, uint32_t nblocks) { size_t offset, bytes; /* Overflow check of multiplying Starting LBA by the sector size */ if (slba >> (64 - nvstore->sectsz_bits)) return (true); offset = slba << nvstore->sectsz_bits; bytes = nblocks << nvstore->sectsz_bits; /* Overflow check of Number of Logical Blocks */ if ((nvstore->size <= offset) || ((nvstore->size - offset) < bytes)) return (true); return (false); } static int pci_nvme_append_iov_req(struct pci_nvme_softc *sc __unused, struct pci_nvme_ioreq *req, uint64_t gpaddr, size_t size, uint64_t offset) { int iovidx; bool range_is_contiguous; if (req == NULL) return (-1); if (req->io_req.br_iovcnt == NVME_MAX_IOVEC) { return (-1); } /* * Minimize the number of IOVs by concatenating contiguous address * ranges. If the IOV count is zero, there is no previous range to * concatenate. */ if (req->io_req.br_iovcnt == 0) range_is_contiguous = false; else range_is_contiguous = (req->prev_gpaddr + req->prev_size) == gpaddr; if (range_is_contiguous) { iovidx = req->io_req.br_iovcnt - 1; req->io_req.br_iov[iovidx].iov_base = paddr_guest2host(req->sc->nsc_pi->pi_vmctx, req->prev_gpaddr, size); if (req->io_req.br_iov[iovidx].iov_base == NULL) return (-1); req->prev_size += size; req->io_req.br_resid += size; req->io_req.br_iov[iovidx].iov_len = req->prev_size; } else { iovidx = req->io_req.br_iovcnt; if (iovidx == 0) { req->io_req.br_offset = offset; req->io_req.br_resid = 0; req->io_req.br_param = req; } req->io_req.br_iov[iovidx].iov_base = paddr_guest2host(req->sc->nsc_pi->pi_vmctx, gpaddr, size); if (req->io_req.br_iov[iovidx].iov_base == NULL) return (-1); req->io_req.br_iov[iovidx].iov_len = size; req->prev_gpaddr = gpaddr; req->prev_size = size; req->io_req.br_resid += size; req->io_req.br_iovcnt++; } return (0); } static void pci_nvme_set_completion(struct pci_nvme_softc *sc, struct nvme_submission_queue *sq, int sqid, uint16_t cid, uint16_t status) { struct nvme_completion_queue *cq = &sc->compl_queues[sq->cqid]; DPRINTF("%s sqid %d cqid %u cid %u status: 0x%x 0x%x", __func__, sqid, sq->cqid, cid, NVME_STATUS_GET_SCT(status), NVME_STATUS_GET_SC(status)); pci_nvme_cq_update(sc, cq, 0, cid, sqid, status); if (cq->head != cq->tail) { if (cq->intr_en & NVME_CQ_INTEN) { pci_generate_msix(sc->nsc_pi, cq->intr_vec); } else { DPRINTF("%s: CQ%u interrupt disabled", __func__, sq->cqid); } } } static void pci_nvme_release_ioreq(struct pci_nvme_softc *sc, struct pci_nvme_ioreq *req) { req->sc = NULL; req->nvme_sq = NULL; req->sqid = 0; pthread_mutex_lock(&sc->mtx); STAILQ_INSERT_TAIL(&sc->ioreqs_free, req, link); sc->pending_ios--; /* when no more IO pending, can set to ready if device reset/enabled */ if (sc->pending_ios == 0 && NVME_CC_GET_EN(sc->regs.cc) && !(NVME_CSTS_GET_RDY(sc->regs.csts))) sc->regs.csts |= NVME_CSTS_RDY; pthread_mutex_unlock(&sc->mtx); sem_post(&sc->iosemlock); } static struct pci_nvme_ioreq * pci_nvme_get_ioreq(struct pci_nvme_softc *sc) { struct pci_nvme_ioreq *req = NULL; sem_wait(&sc->iosemlock); pthread_mutex_lock(&sc->mtx); req = STAILQ_FIRST(&sc->ioreqs_free); assert(req != NULL); STAILQ_REMOVE_HEAD(&sc->ioreqs_free, link); req->sc = sc; sc->pending_ios++; pthread_mutex_unlock(&sc->mtx); req->io_req.br_iovcnt = 0; req->io_req.br_offset = 0; req->io_req.br_resid = 0; req->io_req.br_param = req; req->prev_gpaddr = 0; req->prev_size = 0; return req; } static void pci_nvme_io_done(struct blockif_req *br, int err) { struct pci_nvme_ioreq *req = br->br_param; struct nvme_submission_queue *sq = req->nvme_sq; uint16_t code, status; DPRINTF("%s error %d %s", __func__, err, strerror(err)); /* TODO return correct error */ code = err ? NVME_SC_DATA_TRANSFER_ERROR : NVME_SC_SUCCESS; status = 0; pci_nvme_status_genc(&status, code); pci_nvme_set_completion(req->sc, sq, req->sqid, req->cid, status); pci_nvme_stats_write_read_update(req->sc, req->opc, req->bytes, status); pci_nvme_release_ioreq(req->sc, req); } /* * Implements the Flush command. The specification states: * If a volatile write cache is not present, Flush commands complete * successfully and have no effect * in the description of the Volatile Write Cache (VWC) field of the Identify * Controller data. Therefore, set status to Success if the command is * not supported (i.e. RAM or as indicated by the blockif). */ static bool nvme_opc_flush(struct pci_nvme_softc *sc __unused, struct nvme_command *cmd __unused, struct pci_nvme_blockstore *nvstore, struct pci_nvme_ioreq *req, uint16_t *status) { bool pending = false; if (nvstore->type == NVME_STOR_RAM) { pci_nvme_status_genc(status, NVME_SC_SUCCESS); } else { int err; req->io_req.br_callback = pci_nvme_io_done; err = blockif_flush(nvstore->ctx, &req->io_req); switch (err) { case 0: pending = true; break; case EOPNOTSUPP: pci_nvme_status_genc(status, NVME_SC_SUCCESS); break; default: pci_nvme_status_genc(status, NVME_SC_INTERNAL_DEVICE_ERROR); } } return (pending); } static uint16_t nvme_write_read_ram(struct pci_nvme_softc *sc, struct pci_nvme_blockstore *nvstore, uint64_t prp1, uint64_t prp2, size_t offset, uint64_t bytes, bool is_write) { uint8_t *buf = nvstore->ctx; enum nvme_copy_dir dir; uint16_t status; if (is_write) dir = NVME_COPY_TO_PRP; else dir = NVME_COPY_FROM_PRP; status = 0; if (nvme_prp_memcpy(sc->nsc_pi->pi_vmctx, prp1, prp2, buf + offset, bytes, dir)) pci_nvme_status_genc(&status, NVME_SC_DATA_TRANSFER_ERROR); else pci_nvme_status_genc(&status, NVME_SC_SUCCESS); return (status); } static uint16_t nvme_write_read_blockif(struct pci_nvme_softc *sc, struct pci_nvme_blockstore *nvstore, struct pci_nvme_ioreq *req, uint64_t prp1, uint64_t prp2, size_t offset, uint64_t bytes, bool is_write) { uint64_t size; int err; uint16_t status = NVME_NO_STATUS; size = MIN(PAGE_SIZE - (prp1 % PAGE_SIZE), bytes); if (pci_nvme_append_iov_req(sc, req, prp1, size, offset)) { err = -1; goto out; } offset += size; bytes -= size; if (bytes == 0) { ; } else if (bytes <= PAGE_SIZE) { size = bytes; if (pci_nvme_append_iov_req(sc, req, prp2, size, offset)) { err = -1; goto out; } } else { void *vmctx = sc->nsc_pi->pi_vmctx; uint64_t *prp_list = &prp2; uint64_t *last = prp_list; /* PRP2 is pointer to a physical region page list */ while (bytes) { /* Last entry in list points to the next list */ if ((prp_list == last) && (bytes > PAGE_SIZE)) { uint64_t prp = *prp_list; prp_list = paddr_guest2host(vmctx, prp, PAGE_SIZE - (prp % PAGE_SIZE)); if (prp_list == NULL) { err = -1; goto out; } last = prp_list + (NVME_PRP2_ITEMS - 1); } size = MIN(bytes, PAGE_SIZE); if (pci_nvme_append_iov_req(sc, req, *prp_list, size, offset)) { err = -1; goto out; } offset += size; bytes -= size; prp_list++; } } req->io_req.br_callback = pci_nvme_io_done; if (is_write) err = blockif_write(nvstore->ctx, &req->io_req); else err = blockif_read(nvstore->ctx, &req->io_req); out: if (err) pci_nvme_status_genc(&status, NVME_SC_DATA_TRANSFER_ERROR); return (status); } static bool nvme_opc_write_read(struct pci_nvme_softc *sc, struct nvme_command *cmd, struct pci_nvme_blockstore *nvstore, struct pci_nvme_ioreq *req, uint16_t *status) { uint64_t lba, nblocks, bytes; size_t offset; bool is_write = cmd->opc == NVME_OPC_WRITE; bool pending = false; lba = ((uint64_t)cmd->cdw11 << 32) | cmd->cdw10; nblocks = (cmd->cdw12 & 0xFFFF) + 1; bytes = nblocks << nvstore->sectsz_bits; if (bytes > NVME_MAX_DATA_SIZE) { WPRINTF("%s command would exceed MDTS", __func__); pci_nvme_status_genc(status, NVME_SC_INVALID_FIELD); goto out; } if (pci_nvme_out_of_range(nvstore, lba, nblocks)) { WPRINTF("%s command would exceed LBA range(slba=%#lx nblocks=%#lx)", __func__, lba, nblocks); pci_nvme_status_genc(status, NVME_SC_LBA_OUT_OF_RANGE); goto out; } offset = lba << nvstore->sectsz_bits; req->bytes = bytes; req->io_req.br_offset = lba; /* PRP bits 1:0 must be zero */ cmd->prp1 &= ~0x3UL; cmd->prp2 &= ~0x3UL; if (nvstore->type == NVME_STOR_RAM) { *status = nvme_write_read_ram(sc, nvstore, cmd->prp1, cmd->prp2, offset, bytes, is_write); } else { *status = nvme_write_read_blockif(sc, nvstore, req, cmd->prp1, cmd->prp2, offset, bytes, is_write); if (*status == NVME_NO_STATUS) pending = true; } out: if (!pending) pci_nvme_stats_write_read_update(sc, cmd->opc, bytes, *status); return (pending); } static void pci_nvme_dealloc_sm(struct blockif_req *br, int err) { struct pci_nvme_ioreq *req = br->br_param; struct pci_nvme_softc *sc = req->sc; bool done = true; uint16_t status; status = 0; if (err) { pci_nvme_status_genc(&status, NVME_SC_INTERNAL_DEVICE_ERROR); } else if ((req->prev_gpaddr + 1) == (req->prev_size)) { pci_nvme_status_genc(&status, NVME_SC_SUCCESS); } else { struct iovec *iov = req->io_req.br_iov; req->prev_gpaddr++; iov += req->prev_gpaddr; /* The iov_* values already include the sector size */ req->io_req.br_offset = (off_t)iov->iov_base; req->io_req.br_resid = iov->iov_len; if (blockif_delete(sc->nvstore.ctx, &req->io_req)) { pci_nvme_status_genc(&status, NVME_SC_INTERNAL_DEVICE_ERROR); } else done = false; } if (done) { pci_nvme_set_completion(sc, req->nvme_sq, req->sqid, req->cid, status); pci_nvme_release_ioreq(sc, req); } } static bool nvme_opc_dataset_mgmt(struct pci_nvme_softc *sc, struct nvme_command *cmd, struct pci_nvme_blockstore *nvstore, struct pci_nvme_ioreq *req, uint16_t *status) { struct nvme_dsm_range *range = NULL; uint32_t nr, r, non_zero, dr; int err; bool pending = false; if ((sc->ctrldata.oncs & NVME_ONCS_DSM) == 0) { pci_nvme_status_genc(status, NVME_SC_INVALID_OPCODE); goto out; } nr = cmd->cdw10 & 0xff; /* copy locally because a range entry could straddle PRPs */ #ifdef __FreeBSD__ range = calloc(1, NVME_MAX_DSM_TRIM); #else _Static_assert(NVME_MAX_DSM_TRIM % sizeof(struct nvme_dsm_range) == 0, "NVME_MAX_DSM_TRIM is not a multiple of struct size"); range = calloc(NVME_MAX_DSM_TRIM / sizeof (*range), sizeof (*range)); #endif if (range == NULL) { pci_nvme_status_genc(status, NVME_SC_INTERNAL_DEVICE_ERROR); goto out; } nvme_prp_memcpy(sc->nsc_pi->pi_vmctx, cmd->prp1, cmd->prp2, (uint8_t *)range, NVME_MAX_DSM_TRIM, NVME_COPY_FROM_PRP); /* Check for invalid ranges and the number of non-zero lengths */ non_zero = 0; for (r = 0; r <= nr; r++) { if (pci_nvme_out_of_range(nvstore, range[r].starting_lba, range[r].length)) { pci_nvme_status_genc(status, NVME_SC_LBA_OUT_OF_RANGE); goto out; } if (range[r].length != 0) non_zero++; } if (cmd->cdw11 & NVME_DSM_ATTR_DEALLOCATE) { size_t offset, bytes; int sectsz_bits = sc->nvstore.sectsz_bits; /* * DSM calls are advisory only, and compliant controllers * may choose to take no actions (i.e. return Success). */ if (!nvstore->deallocate) { pci_nvme_status_genc(status, NVME_SC_SUCCESS); goto out; } /* If all ranges have a zero length, return Success */ if (non_zero == 0) { pci_nvme_status_genc(status, NVME_SC_SUCCESS); goto out; } if (req == NULL) { pci_nvme_status_genc(status, NVME_SC_INTERNAL_DEVICE_ERROR); goto out; } offset = range[0].starting_lba << sectsz_bits; bytes = range[0].length << sectsz_bits; /* * If the request is for more than a single range, store * the ranges in the br_iov. Optimize for the common case * of a single range. * * Note that NVMe Number of Ranges is a zero based value */ req->io_req.br_iovcnt = 0; req->io_req.br_offset = offset; req->io_req.br_resid = bytes; if (nr == 0) { req->io_req.br_callback = pci_nvme_io_done; } else { struct iovec *iov = req->io_req.br_iov; for (r = 0, dr = 0; r <= nr; r++) { offset = range[r].starting_lba << sectsz_bits; bytes = range[r].length << sectsz_bits; if (bytes == 0) continue; if ((nvstore->size - offset) < bytes) { pci_nvme_status_genc(status, NVME_SC_LBA_OUT_OF_RANGE); goto out; } iov[dr].iov_base = (void *)offset; iov[dr].iov_len = bytes; dr++; } req->io_req.br_callback = pci_nvme_dealloc_sm; /* * Use prev_gpaddr to track the current entry and * prev_size to track the number of entries */ req->prev_gpaddr = 0; req->prev_size = dr; } err = blockif_delete(nvstore->ctx, &req->io_req); if (err) pci_nvme_status_genc(status, NVME_SC_INTERNAL_DEVICE_ERROR); else pending = true; } out: free(range); return (pending); } static void pci_nvme_handle_io_cmd(struct pci_nvme_softc* sc, uint16_t idx) { struct nvme_submission_queue *sq; uint16_t status; uint16_t sqhead; /* handle all submissions up to sq->tail index */ sq = &sc->submit_queues[idx]; pthread_mutex_lock(&sq->mtx); sqhead = sq->head; DPRINTF("nvme_handle_io qid %u head %u tail %u cmdlist %p", idx, sqhead, sq->tail, sq->qbase); while (sqhead != atomic_load_acq_short(&sq->tail)) { struct nvme_command *cmd; struct pci_nvme_ioreq *req; uint32_t nsid; bool pending; pending = false; req = NULL; status = 0; cmd = &sq->qbase[sqhead]; sqhead = (sqhead + 1) % sq->size; nsid = le32toh(cmd->nsid); if ((nsid == 0) || (nsid > sc->ctrldata.nn)) { pci_nvme_status_genc(&status, NVME_SC_INVALID_NAMESPACE_OR_FORMAT); status |= NVMEM(NVME_STATUS_DNR); goto complete; } req = pci_nvme_get_ioreq(sc); if (req == NULL) { pci_nvme_status_genc(&status, NVME_SC_INTERNAL_DEVICE_ERROR); WPRINTF("%s: unable to allocate IO req", __func__); goto complete; } req->nvme_sq = sq; req->sqid = idx; req->opc = cmd->opc; req->cid = cmd->cid; req->nsid = cmd->nsid; switch (cmd->opc) { case NVME_OPC_FLUSH: pending = nvme_opc_flush(sc, cmd, &sc->nvstore, req, &status); break; case NVME_OPC_WRITE: case NVME_OPC_READ: pending = nvme_opc_write_read(sc, cmd, &sc->nvstore, req, &status); break; case NVME_OPC_WRITE_ZEROES: /* TODO: write zeroes WPRINTF("%s write zeroes lba 0x%lx blocks %u", __func__, lba, cmd->cdw12 & 0xFFFF); */ pci_nvme_status_genc(&status, NVME_SC_SUCCESS); break; case NVME_OPC_DATASET_MANAGEMENT: pending = nvme_opc_dataset_mgmt(sc, cmd, &sc->nvstore, req, &status); break; default: WPRINTF("%s unhandled io command 0x%x", __func__, cmd->opc); pci_nvme_status_genc(&status, NVME_SC_INVALID_OPCODE); } complete: if (!pending) { pci_nvme_set_completion(sc, sq, idx, cmd->cid, status); if (req != NULL) pci_nvme_release_ioreq(sc, req); } } sq->head = sqhead; pthread_mutex_unlock(&sq->mtx); } /* * Check for invalid doorbell write values * See NVM Express Base Specification, revision 2.0 * "Asynchronous Event Information - Error Status" for details */ static bool pci_nvme_sq_doorbell_valid(struct nvme_submission_queue *sq, uint64_t value) { uint64_t capacity; /* * Queue empty : head == tail * Queue full : head is one more than tail accounting for wrap * Therefore, can never have more than (size - 1) entries */ if (sq->head == sq->tail) capacity = sq->size - 1; else if (sq->head > sq->tail) capacity = sq->size - (sq->head - sq->tail) - 1; else capacity = sq->tail - sq->head - 1; if ((value == sq->tail) || /* same as previous */ (value > capacity)) { /* exceeds queue capacity */ EPRINTLN("%s: SQ size=%u head=%u tail=%u capacity=%lu value=%lu", __func__, sq->size, sq->head, sq->tail, capacity, value); return false; } return true; } static void pci_nvme_handle_doorbell(struct pci_nvme_softc* sc, uint64_t idx, int is_sq, uint64_t value) { DPRINTF("nvme doorbell %lu, %s, val 0x%lx", idx, is_sq ? "SQ" : "CQ", value & 0xFFFF); if (is_sq) { if (idx > sc->num_squeues) { WPRINTF("%s queue index %lu overflow from " "guest (max %u)", __func__, idx, sc->num_squeues); pci_nvme_aen_post(sc, PCI_NVME_AE_TYPE_ERROR, PCI_NVME_AEI_ERROR_INVALID_DB); return; } if (sc->submit_queues[idx].qbase == NULL) { WPRINTF("%s write to SQ %lu before created", __func__, idx); pci_nvme_aen_post(sc, PCI_NVME_AE_TYPE_ERROR, PCI_NVME_AEI_ERROR_INVALID_DB); return; } if (!pci_nvme_sq_doorbell_valid(&sc->submit_queues[idx], value)) { EPRINTLN("%s write to SQ %lu of %lu invalid", __func__, idx, value); pci_nvme_aen_post(sc, PCI_NVME_AE_TYPE_ERROR, PCI_NVME_AEI_ERROR_INVALID_DB_VALUE); return; } atomic_store_short(&sc->submit_queues[idx].tail, (uint16_t)value); if (idx == 0) pci_nvme_handle_admin_cmd(sc, value); else { /* submission queue; handle new entries in SQ */ pci_nvme_handle_io_cmd(sc, (uint16_t)idx); } } else { if (idx > sc->num_cqueues) { WPRINTF("%s queue index %lu overflow from " "guest (max %u)", __func__, idx, sc->num_cqueues); pci_nvme_aen_post(sc, PCI_NVME_AE_TYPE_ERROR, PCI_NVME_AEI_ERROR_INVALID_DB); return; } if (sc->compl_queues[idx].qbase == NULL) { WPRINTF("%s write to CQ %lu before created", __func__, idx); pci_nvme_aen_post(sc, PCI_NVME_AE_TYPE_ERROR, PCI_NVME_AEI_ERROR_INVALID_DB); return; } atomic_store_short(&sc->compl_queues[idx].head, (uint16_t)value); } } static void pci_nvme_bar0_reg_dumps(const char *func, uint64_t offset, int iswrite) { const char *s = iswrite ? "WRITE" : "READ"; switch (offset) { case NVME_CR_CAP_LOW: DPRINTF("%s %s NVME_CR_CAP_LOW", func, s); break; case NVME_CR_CAP_HI: DPRINTF("%s %s NVME_CR_CAP_HI", func, s); break; case NVME_CR_VS: DPRINTF("%s %s NVME_CR_VS", func, s); break; case NVME_CR_INTMS: DPRINTF("%s %s NVME_CR_INTMS", func, s); break; case NVME_CR_INTMC: DPRINTF("%s %s NVME_CR_INTMC", func, s); break; case NVME_CR_CC: DPRINTF("%s %s NVME_CR_CC", func, s); break; case NVME_CR_CSTS: DPRINTF("%s %s NVME_CR_CSTS", func, s); break; case NVME_CR_NSSR: DPRINTF("%s %s NVME_CR_NSSR", func, s); break; case NVME_CR_AQA: DPRINTF("%s %s NVME_CR_AQA", func, s); break; case NVME_CR_ASQ_LOW: DPRINTF("%s %s NVME_CR_ASQ_LOW", func, s); break; case NVME_CR_ASQ_HI: DPRINTF("%s %s NVME_CR_ASQ_HI", func, s); break; case NVME_CR_ACQ_LOW: DPRINTF("%s %s NVME_CR_ACQ_LOW", func, s); break; case NVME_CR_ACQ_HI: DPRINTF("%s %s NVME_CR_ACQ_HI", func, s); break; default: DPRINTF("unknown nvme bar-0 offset 0x%lx", offset); } } static void pci_nvme_write_bar_0(struct pci_nvme_softc *sc, uint64_t offset, int size, uint64_t value) { uint32_t ccreg; if (offset >= NVME_DOORBELL_OFFSET) { uint64_t belloffset = offset - NVME_DOORBELL_OFFSET; uint64_t idx = belloffset / 8; /* door bell size = 2*int */ int is_sq = (belloffset % 8) < 4; if ((sc->regs.csts & NVME_CSTS_RDY) == 0) { WPRINTF("doorbell write prior to RDY (offset=%#lx)\n", offset); return; } if (belloffset > ((sc->max_queues+1) * 8 - 4)) { WPRINTF("guest attempted an overflow write offset " "0x%lx, val 0x%lx in %s", offset, value, __func__); return; } if (is_sq) { if (sc->submit_queues[idx].qbase == NULL) return; } else if (sc->compl_queues[idx].qbase == NULL) return; pci_nvme_handle_doorbell(sc, idx, is_sq, value); return; } DPRINTF("nvme-write offset 0x%lx, size %d, value 0x%lx", offset, size, value); if (size != 4) { WPRINTF("guest wrote invalid size %d (offset 0x%lx, " "val 0x%lx) to bar0 in %s", size, offset, value, __func__); /* TODO: shutdown device */ return; } pci_nvme_bar0_reg_dumps(__func__, offset, 1); pthread_mutex_lock(&sc->mtx); switch (offset) { case NVME_CR_CAP_LOW: case NVME_CR_CAP_HI: /* readonly */ break; case NVME_CR_VS: /* readonly */ break; case NVME_CR_INTMS: /* MSI-X, so ignore */ break; case NVME_CR_INTMC: /* MSI-X, so ignore */ break; case NVME_CR_CC: ccreg = (uint32_t)value; DPRINTF("%s NVME_CR_CC en %x css %x shn %x iosqes %u " "iocqes %u", __func__, NVME_CC_GET_EN(ccreg), NVME_CC_GET_CSS(ccreg), NVME_CC_GET_SHN(ccreg), NVME_CC_GET_IOSQES(ccreg), NVME_CC_GET_IOCQES(ccreg)); if (NVME_CC_GET_SHN(ccreg)) { /* perform shutdown - flush out data to backend */ sc->regs.csts &= ~NVMEM(NVME_CSTS_REG_SHST); sc->regs.csts |= NVMEF(NVME_CSTS_REG_SHST, NVME_SHST_COMPLETE); } if (NVME_CC_GET_EN(ccreg) != NVME_CC_GET_EN(sc->regs.cc)) { if (NVME_CC_GET_EN(ccreg) == 0) /* transition 1-> causes controller reset */ pci_nvme_reset_locked(sc); else pci_nvme_init_controller(sc); } /* Insert the iocqes, iosqes and en bits from the write */ sc->regs.cc &= ~NVME_CC_WRITE_MASK; sc->regs.cc |= ccreg & NVME_CC_WRITE_MASK; if (NVME_CC_GET_EN(ccreg) == 0) { /* Insert the ams, mps and css bit fields */ sc->regs.cc &= ~NVME_CC_NEN_WRITE_MASK; sc->regs.cc |= ccreg & NVME_CC_NEN_WRITE_MASK; sc->regs.csts &= ~NVME_CSTS_RDY; } else if ((sc->pending_ios == 0) && !(sc->regs.csts & NVME_CSTS_CFS)) { sc->regs.csts |= NVME_CSTS_RDY; } break; case NVME_CR_CSTS: break; case NVME_CR_NSSR: /* ignore writes; don't support subsystem reset */ break; case NVME_CR_AQA: sc->regs.aqa = (uint32_t)value; break; case NVME_CR_ASQ_LOW: sc->regs.asq = (sc->regs.asq & (0xFFFFFFFF00000000)) | (0xFFFFF000 & value); break; case NVME_CR_ASQ_HI: sc->regs.asq = (sc->regs.asq & (0x00000000FFFFFFFF)) | (value << 32); break; case NVME_CR_ACQ_LOW: sc->regs.acq = (sc->regs.acq & (0xFFFFFFFF00000000)) | (0xFFFFF000 & value); break; case NVME_CR_ACQ_HI: sc->regs.acq = (sc->regs.acq & (0x00000000FFFFFFFF)) | (value << 32); break; default: DPRINTF("%s unknown offset 0x%lx, value 0x%lx size %d", __func__, offset, value, size); } pthread_mutex_unlock(&sc->mtx); } static void pci_nvme_write(struct pci_devinst *pi, int baridx, uint64_t offset, int size, uint64_t value) { struct pci_nvme_softc* sc = pi->pi_arg; if (baridx == pci_msix_table_bar(pi) || baridx == pci_msix_pba_bar(pi)) { DPRINTF("nvme-write baridx %d, msix: off 0x%lx, size %d, " " value 0x%lx", baridx, offset, size, value); pci_emul_msix_twrite(pi, offset, size, value); return; } switch (baridx) { case 0: pci_nvme_write_bar_0(sc, offset, size, value); break; default: DPRINTF("%s unknown baridx %d, val 0x%lx", __func__, baridx, value); } } static uint64_t pci_nvme_read_bar_0(struct pci_nvme_softc* sc, uint64_t offset, int size) { uint64_t value; pci_nvme_bar0_reg_dumps(__func__, offset, 0); if (offset < NVME_DOORBELL_OFFSET) { void *p = &(sc->regs); pthread_mutex_lock(&sc->mtx); memcpy(&value, (void *)((uintptr_t)p + offset), size); pthread_mutex_unlock(&sc->mtx); } else { value = 0; WPRINTF("pci_nvme: read invalid offset %ld", offset); } switch (size) { case 1: value &= 0xFF; break; case 2: value &= 0xFFFF; break; case 4: value &= 0xFFFFFFFF; break; } DPRINTF(" nvme-read offset 0x%lx, size %d -> value 0x%x", offset, size, (uint32_t)value); return (value); } static uint64_t pci_nvme_read(struct pci_devinst *pi, int baridx, uint64_t offset, int size) { struct pci_nvme_softc* sc = pi->pi_arg; if (baridx == pci_msix_table_bar(pi) || baridx == pci_msix_pba_bar(pi)) { DPRINTF("nvme-read bar: %d, msix: regoff 0x%lx, size %d", baridx, offset, size); return pci_emul_msix_tread(pi, offset, size); } switch (baridx) { case 0: return pci_nvme_read_bar_0(sc, offset, size); default: DPRINTF("unknown bar %d, 0x%lx", baridx, offset); } return (0); } static int pci_nvme_parse_config(struct pci_nvme_softc *sc, nvlist_t *nvl) { char bident[sizeof("XXX:XXX")]; const char *value; uint32_t sectsz; sc->max_queues = NVME_QUEUES; sc->max_qentries = NVME_MAX_QENTRIES; sc->ioslots = NVME_IOSLOTS; sc->num_squeues = sc->max_queues; sc->num_cqueues = sc->max_queues; sc->dataset_management = NVME_DATASET_MANAGEMENT_AUTO; sectsz = 0; #ifdef __FreeBSD__ snprintf(sc->ctrldata.sn, sizeof(sc->ctrldata.sn), "NVME-%d-%d", sc->nsc_pi->pi_slot, sc->nsc_pi->pi_func); #else snprintf((char *)sc->ctrldata.sn, sizeof(sc->ctrldata.sn), "NVME-%d-%d", sc->nsc_pi->pi_slot, sc->nsc_pi->pi_func); #endif value = get_config_value_node(nvl, "maxq"); if (value != NULL) sc->max_queues = atoi(value); value = get_config_value_node(nvl, "qsz"); if (value != NULL) { sc->max_qentries = atoi(value); if (sc->max_qentries <= 0) { EPRINTLN("nvme: Invalid qsz option %d", sc->max_qentries); return (-1); } } value = get_config_value_node(nvl, "ioslots"); if (value != NULL) { sc->ioslots = atoi(value); if (sc->ioslots <= 0) { EPRINTLN("Invalid ioslots option %d", sc->ioslots); return (-1); } } value = get_config_value_node(nvl, "sectsz"); if (value != NULL) sectsz = atoi(value); value = get_config_value_node(nvl, "ser"); if (value != NULL) { /* * This field indicates the Product Serial Number in * 7-bit ASCII, unused bytes should be space characters. * Ref: NVMe v1.3c. */ cpywithpad((char *)sc->ctrldata.sn, sizeof(sc->ctrldata.sn), value, ' '); } value = get_config_value_node(nvl, "eui64"); if (value != NULL) sc->nvstore.eui64 = htobe64(strtoull(value, NULL, 0)); value = get_config_value_node(nvl, "dsm"); if (value != NULL) { if (strcmp(value, "auto") == 0) sc->dataset_management = NVME_DATASET_MANAGEMENT_AUTO; else if (strcmp(value, "enable") == 0) sc->dataset_management = NVME_DATASET_MANAGEMENT_ENABLE; else if (strcmp(value, "disable") == 0) sc->dataset_management = NVME_DATASET_MANAGEMENT_DISABLE; } value = get_config_value_node(nvl, "bootindex"); if (value != NULL) { if (pci_emul_add_boot_device(sc->nsc_pi, atoi(value))) { EPRINTLN("Invalid bootindex %d", atoi(value)); return (-1); } } value = get_config_value_node(nvl, "ram"); if (value != NULL) { uint64_t sz = strtoull(value, NULL, 10); sc->nvstore.type = NVME_STOR_RAM; sc->nvstore.size = sz * 1024 * 1024; sc->nvstore.ctx = calloc(1, sc->nvstore.size); sc->nvstore.sectsz = 4096; sc->nvstore.sectsz_bits = 12; if (sc->nvstore.ctx == NULL) { EPRINTLN("nvme: Unable to allocate RAM"); return (-1); } } else { snprintf(bident, sizeof(bident), "%u:%u", sc->nsc_pi->pi_slot, sc->nsc_pi->pi_func); sc->nvstore.ctx = blockif_open(nvl, bident); if (sc->nvstore.ctx == NULL) { EPRINTLN("nvme: Could not open backing file: %s", strerror(errno)); return (-1); } sc->nvstore.type = NVME_STOR_BLOCKIF; sc->nvstore.size = blockif_size(sc->nvstore.ctx); } if (sectsz == 512 || sectsz == 4096 || sectsz == 8192) sc->nvstore.sectsz = sectsz; else if (sc->nvstore.type != NVME_STOR_RAM) sc->nvstore.sectsz = blockif_sectsz(sc->nvstore.ctx); for (sc->nvstore.sectsz_bits = 9; (1U << sc->nvstore.sectsz_bits) < sc->nvstore.sectsz; sc->nvstore.sectsz_bits++); if (sc->max_queues <= 0 || sc->max_queues > NVME_QUEUES) sc->max_queues = NVME_QUEUES; return (0); } static void pci_nvme_resized(struct blockif_ctxt *bctxt __unused, void *arg, size_t new_size) { struct pci_nvme_softc *sc; struct pci_nvme_blockstore *nvstore; struct nvme_namespace_data *nd; sc = arg; nvstore = &sc->nvstore; nd = &sc->nsdata; nvstore->size = new_size; pci_nvme_init_nsdata_size(nvstore, nd); /* Add changed NSID to list */ sc->ns_log.ns[0] = 1; sc->ns_log.ns[1] = 0; pci_nvme_aen_post(sc, PCI_NVME_AE_TYPE_NOTICE, PCI_NVME_AEI_NOTICE_NS_ATTR_CHANGED); } static int pci_nvme_init(struct pci_devinst *pi, nvlist_t *nvl) { struct pci_nvme_softc *sc; uint32_t pci_membar_sz; int error; error = 0; sc = calloc(1, sizeof(struct pci_nvme_softc)); pi->pi_arg = sc; sc->nsc_pi = pi; error = pci_nvme_parse_config(sc, nvl); if (error < 0) goto done; else error = 0; STAILQ_INIT(&sc->ioreqs_free); sc->ioreqs = calloc(sc->ioslots, sizeof(struct pci_nvme_ioreq)); for (uint32_t i = 0; i < sc->ioslots; i++) { STAILQ_INSERT_TAIL(&sc->ioreqs_free, &sc->ioreqs[i], link); } pci_set_cfgdata16(pi, PCIR_DEVICE, 0x0A0A); pci_set_cfgdata16(pi, PCIR_VENDOR, 0xFB5D); pci_set_cfgdata8(pi, PCIR_CLASS, PCIC_STORAGE); pci_set_cfgdata8(pi, PCIR_SUBCLASS, PCIS_STORAGE_NVM); pci_set_cfgdata8(pi, PCIR_PROGIF, PCIP_STORAGE_NVM_ENTERPRISE_NVMHCI_1_0); /* * Allocate size of NVMe registers + doorbell space for all queues. * * The specification requires a minimum memory I/O window size of 16K. * The Windows driver will refuse to start a device with a smaller * window. */ pci_membar_sz = sizeof(struct nvme_registers) + 2 * sizeof(uint32_t) * (sc->max_queues + 1); pci_membar_sz = MAX(pci_membar_sz, NVME_MMIO_SPACE_MIN); DPRINTF("nvme membar size: %u", pci_membar_sz); error = pci_emul_alloc_bar(pi, 0, PCIBAR_MEM64, pci_membar_sz); if (error) { WPRINTF("%s pci alloc mem bar failed", __func__); goto done; } error = pci_emul_add_msixcap(pi, sc->max_queues + 1, NVME_MSIX_BAR); if (error) { WPRINTF("%s pci add msixcap failed", __func__); goto done; } error = pci_emul_add_pciecap(pi, PCIEM_TYPE_ROOT_INT_EP); if (error) { WPRINTF("%s pci add Express capability failed", __func__); goto done; } pthread_mutex_init(&sc->mtx, NULL); sem_init(&sc->iosemlock, 0, sc->ioslots); blockif_register_resize_callback(sc->nvstore.ctx, pci_nvme_resized, sc); pci_nvme_init_queues(sc, sc->max_queues, sc->max_queues); /* * Controller data depends on Namespace data so initialize Namespace * data first. */ pci_nvme_init_nsdata(sc, &sc->nsdata, 1, &sc->nvstore); pci_nvme_init_ctrldata(sc); pci_nvme_init_logpages(sc); pci_nvme_init_features(sc); pci_nvme_aer_init(sc); pci_nvme_aen_init(sc); pci_nvme_reset(sc); done: return (error); } static int pci_nvme_legacy_config(nvlist_t *nvl, const char *opts) { char *cp, *ram; if (opts == NULL) return (0); if (strncmp(opts, "ram=", 4) == 0) { cp = strchr(opts, ','); if (cp == NULL) { set_config_value_node(nvl, "ram", opts + 4); return (0); } ram = strndup(opts + 4, cp - opts - 4); set_config_value_node(nvl, "ram", ram); free(ram); return (pci_parse_legacy_config(nvl, cp + 1)); } else return (blockif_legacy_config(nvl, opts)); } static const struct pci_devemu pci_de_nvme = { .pe_emu = "nvme", .pe_init = pci_nvme_init, .pe_legacy_config = pci_nvme_legacy_config, .pe_barwrite = pci_nvme_write, .pe_barread = pci_nvme_read }; PCI_EMUL_SET(pci_de_nvme); /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2011 NetApp, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "config.h" #include "debug.h" #include "pci_passthru.h" #include "mem.h" #define LEGACY_SUPPORT 1 #define MSIX_TABLE_COUNT(ctrl) (((ctrl) & PCIM_MSIXCTRL_TABLE_SIZE) + 1) #define MSIX_CAPLEN 12 struct passthru_softc { struct pci_devinst *psc_pi; /* ROM is handled like a BAR */ struct pcibar psc_bar[PCI_BARMAX_WITH_ROM + 1]; struct { int capoff; int msgctrl; int emulated; } psc_msi; struct { int capoff; } psc_msix; int pptfd; int msi_limit; int msix_limit; cfgread_handler psc_pcir_rhandler[PCI_REGMAX + 1]; cfgwrite_handler psc_pcir_whandler[PCI_REGMAX + 1]; }; static int msi_caplen(int msgctrl) { int len; len = 10; /* minimum length of msi capability */ if (msgctrl & PCIM_MSICTRL_64BIT) len += 4; #if 0 /* * Ignore the 'mask' and 'pending' bits in the MSI capability. * We'll let the guest manipulate them directly. */ if (msgctrl & PCIM_MSICTRL_VECTOR) len += 10; #endif return (len); } static uint32_t passthru_read_config(const struct passthru_softc *sc, long reg, int width) { struct ppt_cfg_io pi; pi.pci_off = reg; pi.pci_width = width; if (ioctl(sc->pptfd, PPT_CFG_READ, &pi) != 0) { return (0); } return (pi.pci_data); } static void passthru_write_config(const struct passthru_softc *sc, long reg, int width, uint32_t data) { struct ppt_cfg_io pi; pi.pci_off = reg; pi.pci_width = width; pi.pci_data = data; (void) ioctl(sc->pptfd, PPT_CFG_WRITE, &pi); } static int passthru_get_bar(struct passthru_softc *sc, int bar, enum pcibar_type *type, uint64_t *base, uint64_t *size) { struct ppt_bar_query pb; pb.pbq_baridx = bar; if (ioctl(sc->pptfd, PPT_BAR_QUERY, &pb) != 0) { return (-1); } switch (pb.pbq_type) { case PCI_ADDR_IO: *type = PCIBAR_IO; break; case PCI_ADDR_MEM32: *type = PCIBAR_MEM32; break; case PCI_ADDR_MEM64: *type = PCIBAR_MEM64; break; default: err(1, "unrecognized BAR type: %u\n", pb.pbq_type); break; } *base = pb.pbq_base; *size = pb.pbq_size; return (0); } static int passthru_dev_open(const char *path, int *pptfdp) { int pptfd; if ((pptfd = open(path, O_RDWR)) < 0) { return (errno); } /* XXX: verify fd with ioctl? */ *pptfdp = pptfd; return (0); } #ifdef LEGACY_SUPPORT static int passthru_add_msicap(struct pci_devinst *pi, int msgnum, int nextptr) { int capoff; struct msicap msicap; u_char *capdata; pci_populate_msicap(&msicap, msgnum, nextptr); /* * XXX * Copy the msi capability structure in the last 16 bytes of the * config space. This is wrong because it could shadow something * useful to the device. */ capoff = 256 - roundup(sizeof(msicap), 4); capdata = (u_char *)&msicap; for (size_t i = 0; i < sizeof(msicap); i++) pci_set_cfgdata8(pi, capoff + i, capdata[i]); return (capoff); } #endif /* LEGACY_SUPPORT */ static void passthru_intr_limit(struct passthru_softc *sc, struct msixcap *msixcap) { struct pci_devinst *pi = sc->psc_pi; int off; /* Reduce the number of MSI vectors if higher than OS limit */ if ((off = sc->psc_msi.capoff) != 0 && sc->msi_limit != -1) { int msi_limit, mmc; msi_limit = sc->msi_limit > 16 ? PCIM_MSICTRL_MMC_32 : sc->msi_limit > 8 ? PCIM_MSICTRL_MMC_16 : sc->msi_limit > 4 ? PCIM_MSICTRL_MMC_8 : sc->msi_limit > 2 ? PCIM_MSICTRL_MMC_4 : sc->msi_limit > 1 ? PCIM_MSICTRL_MMC_2 : PCIM_MSICTRL_MMC_1; mmc = sc->psc_msi.msgctrl & PCIM_MSICTRL_MMC_MASK; if (mmc > msi_limit) { sc->psc_msi.msgctrl &= ~PCIM_MSICTRL_MMC_MASK; sc->psc_msi.msgctrl |= msi_limit; pci_set_cfgdata16(pi, off + 2, sc->psc_msi.msgctrl); } } /* Reduce the number of MSI-X vectors if higher than OS limit */ if ((off = sc->psc_msix.capoff) != 0 && sc->msix_limit != -1) { if (MSIX_TABLE_COUNT(msixcap->msgctrl) > sc->msix_limit) { msixcap->msgctrl &= ~PCIM_MSIXCTRL_TABLE_SIZE; msixcap->msgctrl |= sc->msix_limit - 1; pci_set_cfgdata16(pi, off + 2, msixcap->msgctrl); } } } static int cfginitmsi(struct passthru_softc *sc) { int i, ptr, capptr, cap, sts, caplen, table_size; uint32_t u32; struct pci_devinst *pi = sc->psc_pi; struct msixcap msixcap; char *msixcap_ptr; /* * Parse the capabilities and cache the location of the MSI * and MSI-X capabilities. */ sts = passthru_read_config(sc, PCIR_STATUS, 2); if (sts & PCIM_STATUS_CAPPRESENT) { ptr = passthru_read_config(sc, PCIR_CAP_PTR, 1); while (ptr != 0 && ptr != 0xff) { cap = passthru_read_config(sc, ptr + PCICAP_ID, 1); if (cap == PCIY_MSI) { /* * Copy the MSI capability into the config * space of the emulated pci device */ sc->psc_msi.capoff = ptr; sc->psc_msi.msgctrl = passthru_read_config(sc, ptr + 2, 2); sc->psc_msi.emulated = 0; caplen = msi_caplen(sc->psc_msi.msgctrl); capptr = ptr; while (caplen > 0) { u32 = passthru_read_config(sc, capptr, 4); pci_set_cfgdata32(pi, capptr, u32); caplen -= 4; capptr += 4; } } else if (cap == PCIY_MSIX) { /* * Copy the MSI-X capability */ sc->psc_msix.capoff = ptr; caplen = 12; msixcap_ptr = (char *)&msixcap; capptr = ptr; while (caplen > 0) { u32 = passthru_read_config(sc, capptr, 4); memcpy(msixcap_ptr, &u32, 4); pci_set_cfgdata32(pi, capptr, u32); caplen -= 4; capptr += 4; msixcap_ptr += 4; } } ptr = passthru_read_config(sc, ptr + PCICAP_NEXTPTR, 1); } } passthru_intr_limit(sc, &msixcap); if (sc->psc_msix.capoff != 0) { pi->pi_msix.pba_bar = msixcap.pba_info & PCIM_MSIX_BIR_MASK; pi->pi_msix.pba_offset = msixcap.pba_info & ~PCIM_MSIX_BIR_MASK; pi->pi_msix.table_bar = msixcap.table_info & PCIM_MSIX_BIR_MASK; pi->pi_msix.table_offset = msixcap.table_info & ~PCIM_MSIX_BIR_MASK; pi->pi_msix.table_count = MSIX_TABLE_COUNT(msixcap.msgctrl); pi->pi_msix.pba_size = PBA_SIZE(pi->pi_msix.table_count); /* Allocate the emulated MSI-X table array */ table_size = pi->pi_msix.table_count * MSIX_TABLE_ENTRY_SIZE; pi->pi_msix.table = calloc(1, table_size); /* Mask all table entries */ for (i = 0; i < pi->pi_msix.table_count; i++) { pi->pi_msix.table[i].vector_control |= PCIM_MSIX_VCTRL_MASK; } } #ifdef LEGACY_SUPPORT /* * If the passthrough device does not support MSI then craft a * MSI capability for it. We link the new MSI capability at the * head of the list of capabilities. */ if ((sts & PCIM_STATUS_CAPPRESENT) != 0 && sc->psc_msi.capoff == 0) { int origptr, msiptr; origptr = passthru_read_config(sc, PCIR_CAP_PTR, 1); msiptr = passthru_add_msicap(pi, 1, origptr); sc->psc_msi.capoff = msiptr; sc->psc_msi.msgctrl = pci_get_cfgdata16(pi, msiptr + 2); sc->psc_msi.emulated = 1; pci_set_cfgdata8(pi, PCIR_CAP_PTR, msiptr); } #endif /* Make sure one of the capabilities is present */ if (sc->psc_msi.capoff == 0 && sc->psc_msix.capoff == 0) return (-1); else return (0); } static uint64_t msix_table_read(struct passthru_softc *sc, uint64_t offset, int size) { struct pci_devinst *pi; struct msix_table_entry *entry; uint8_t *src8; uint16_t *src16; uint32_t *src32; uint64_t *src64; uint64_t data; size_t entry_offset; uint32_t table_offset; int index, table_count; pi = sc->psc_pi; table_offset = pi->pi_msix.table_offset; table_count = pi->pi_msix.table_count; if (offset < table_offset || offset >= table_offset + table_count * MSIX_TABLE_ENTRY_SIZE) { switch (size) { case 1: src8 = (uint8_t *)(pi->pi_msix.mapped_addr + offset); data = *src8; break; case 2: src16 = (uint16_t *)(pi->pi_msix.mapped_addr + offset); data = *src16; break; case 4: src32 = (uint32_t *)(pi->pi_msix.mapped_addr + offset); data = *src32; break; case 8: src64 = (uint64_t *)(pi->pi_msix.mapped_addr + offset); data = *src64; break; default: return (-1); } return (data); } offset -= table_offset; index = offset / MSIX_TABLE_ENTRY_SIZE; assert(index < table_count); entry = &pi->pi_msix.table[index]; entry_offset = offset % MSIX_TABLE_ENTRY_SIZE; switch (size) { case 1: src8 = (uint8_t *)((uint8_t *)entry + entry_offset); data = *src8; break; case 2: src16 = (uint16_t *)((uint8_t *)entry + entry_offset); data = *src16; break; case 4: src32 = (uint32_t *)((uint8_t *)entry + entry_offset); data = *src32; break; case 8: src64 = (uint64_t *)((uint8_t *)entry + entry_offset); data = *src64; break; default: return (-1); } return (data); } static void msix_table_write(struct vmctx *ctx, struct passthru_softc *sc, uint64_t offset, int size, uint64_t data) { struct pci_devinst *pi; struct msix_table_entry *entry; uint8_t *dest8; uint16_t *dest16; uint32_t *dest32; uint64_t *dest64; size_t entry_offset; uint32_t table_offset, vector_control; int index, table_count; pi = sc->psc_pi; table_offset = pi->pi_msix.table_offset; table_count = pi->pi_msix.table_count; if (offset < table_offset || offset >= table_offset + table_count * MSIX_TABLE_ENTRY_SIZE) { switch (size) { case 1: dest8 = (uint8_t *)(pi->pi_msix.mapped_addr + offset); *dest8 = data; break; case 2: dest16 = (uint16_t *)(pi->pi_msix.mapped_addr + offset); *dest16 = data; break; case 4: dest32 = (uint32_t *)(pi->pi_msix.mapped_addr + offset); *dest32 = data; break; case 8: dest64 = (uint64_t *)(pi->pi_msix.mapped_addr + offset); *dest64 = data; break; } return; } offset -= table_offset; index = offset / MSIX_TABLE_ENTRY_SIZE; assert(index < table_count); entry = &pi->pi_msix.table[index]; entry_offset = offset % MSIX_TABLE_ENTRY_SIZE; /* Only 4 byte naturally-aligned writes are supported */ assert(size == 4); assert(entry_offset % 4 == 0); vector_control = entry->vector_control; dest32 = (uint32_t *)((uint8_t *)entry + entry_offset); *dest32 = data; /* If MSI-X hasn't been enabled, do nothing */ if (pi->pi_msix.enabled) { /* If the entry is masked, don't set it up */ if ((entry->vector_control & PCIM_MSIX_VCTRL_MASK) == 0 || (vector_control & PCIM_MSIX_VCTRL_MASK) == 0) { (void) vm_setup_pptdev_msix(ctx, sc->pptfd, index, entry->addr, entry->msg_data, entry->vector_control); } } } static int init_msix_table(struct vmctx *ctx __unused, struct passthru_softc *sc) { struct pci_devinst *pi = sc->psc_pi; uint32_t table_size, table_offset; int i; i = pci_msix_table_bar(pi); assert(i >= 0); /* * Map the region of the BAR containing the MSI-X table. This is * necessary for two reasons: * 1. The PBA may reside in the first or last page containing the MSI-X * table. * 2. While PCI devices are not supposed to use the page(s) containing * the MSI-X table for other purposes, some do in practice. */ /* * Mapping pptfd provides access to the BAR containing the MSI-X * table. See ppt_devmap() in usr/src/uts/intel/io/vmm/io/ppt.c * * This maps the whole BAR and then mprotect(PROT_NONE) is used below * to prevent access to pages that don't contain the MSI-X table. * When porting this, it was tempting to just map the MSI-X table pages * but that would mean updating everywhere that assumes that * pi->pi_msix.mapped_addr points to the start of the BAR. For now, * keep closer to upstream. */ pi->pi_msix.mapped_size = sc->psc_bar[i].size; pi->pi_msix.mapped_addr = (uint8_t *)mmap(NULL, pi->pi_msix.mapped_size, PROT_READ | PROT_WRITE, MAP_SHARED, sc->pptfd, 0); if (pi->pi_msix.mapped_addr == MAP_FAILED) { warn("Failed to map MSI-X table BAR on %d", sc->pptfd); return (-1); } table_offset = rounddown2(pi->pi_msix.table_offset, 4096); table_size = pi->pi_msix.table_offset - table_offset; table_size += pi->pi_msix.table_count * MSIX_TABLE_ENTRY_SIZE; table_size = roundup2(table_size, 4096); /* * Unmap any pages not containing the table, we do not need to emulate * accesses to them. Avoid releasing address space to help ensure that * a buggy out-of-bounds access causes a crash. */ if (table_offset != 0) if (mprotect((caddr_t)pi->pi_msix.mapped_addr, table_offset, PROT_NONE) != 0) warn("Failed to unmap MSI-X table BAR region"); if (table_offset + table_size != pi->pi_msix.mapped_size) if (mprotect((caddr_t) pi->pi_msix.mapped_addr + table_offset + table_size, pi->pi_msix.mapped_size - (table_offset + table_size), PROT_NONE) != 0) warn("Failed to unmap MSI-X table BAR region"); return (0); } static int cfginitbar(struct vmctx *ctx __unused, struct passthru_softc *sc) { struct pci_devinst *pi = sc->psc_pi; uint_t i; /* * Initialize BAR registers */ for (i = 0; i <= PCI_BARMAX; i++) { enum pcibar_type bartype; uint64_t base, size; int error; if (passthru_get_bar(sc, i, &bartype, &base, &size) != 0) { continue; } if (bartype != PCIBAR_IO) { if (((base | size) & PAGE_MASK) != 0) { warnx("passthru device %d BAR %d: " "base %#lx or size %#lx not page aligned\n", sc->pptfd, i, base, size); return (-1); } } /* Cache information about the "real" BAR */ sc->psc_bar[i].type = bartype; sc->psc_bar[i].size = size; sc->psc_bar[i].addr = base; sc->psc_bar[i].lobits = 0; /* Allocate the BAR in the guest I/O or MMIO space */ error = pci_emul_alloc_bar(pi, i, bartype, size); if (error) return (-1); /* Use same lobits as physical bar */ uint8_t lobits = passthru_read_config(sc, PCIR_BAR(i), 0x01); if (bartype == PCIBAR_MEM32 || bartype == PCIBAR_MEM64) { lobits &= ~PCIM_BAR_MEM_BASE; } else { lobits &= ~PCIM_BAR_IO_BASE; } sc->psc_bar[i].lobits = lobits; pi->pi_bar[i].lobits = lobits; /* * 64-bit BAR takes up two slots so skip the next one. */ if (bartype == PCIBAR_MEM64) { i++; assert(i <= PCI_BARMAX); sc->psc_bar[i].type = PCIBAR_MEMHI64; } } return (0); } static int cfginit(struct vmctx *ctx, struct passthru_softc *sc) { int error; struct pci_devinst *pi = sc->psc_pi; uint16_t cmd; uint8_t intline, intpin; /* * Copy physical PCI header to virtual config space. COMMAND, * INTLINE and INTPIN shouldn't be aligned with their physical value * and they are already set by pci_emul_init(). */ cmd = pci_get_cfgdata16(pi, PCIR_COMMAND); intline = pci_get_cfgdata8(pi, PCIR_INTLINE); intpin = pci_get_cfgdata8(pi, PCIR_INTPIN); for (int i = 0; i <= PCIR_MAXLAT; i += 4) { #ifdef __FreeBSD__ pci_set_cfgdata32(pi, i, read_config(&sc->psc_sel, i, 4)); #else pci_set_cfgdata32(pi, i, passthru_read_config(sc, i, 4)); #endif } pci_set_cfgdata16(pi, PCIR_COMMAND, cmd); pci_set_cfgdata8(pi, PCIR_INTLINE, intline); pci_set_cfgdata8(pi, PCIR_INTPIN, intpin); if (cfginitmsi(sc) != 0) { warnx("failed to initialize MSI for PCI %d", sc->pptfd); return (-1); } if (cfginitbar(ctx, sc) != 0) { warnx("failed to initialize BARs for PCI %d", sc->pptfd); return (-1); } if (pci_msix_table_bar(pi) >= 0) { error = init_msix_table(ctx, sc); if (error != 0) { warnx("failed to initialize MSI-X table for PCI %d", sc->pptfd); goto done; } } /* Emulate most PCI header register. */ if ((error = set_pcir_handler(sc, 0, PCIR_MAXLAT + 1, passthru_cfgread_emulate, passthru_cfgwrite_emulate)) != 0) goto done; /* Allow access to the physical status register. */ if ((error = set_pcir_handler(sc, PCIR_COMMAND, 0x04, NULL, NULL)) != 0) goto done; error = 0; /* success */ done: return (error); } int set_pcir_handler(struct passthru_softc *sc, int reg, int len, cfgread_handler rhandler, cfgwrite_handler whandler) { if (reg > PCI_REGMAX || reg + len > PCI_REGMAX + 1) return (-1); for (int i = reg; i < reg + len; ++i) { assert(sc->psc_pcir_rhandler[i] == NULL || rhandler == NULL); assert(sc->psc_pcir_whandler[i] == NULL || whandler == NULL); sc->psc_pcir_rhandler[i] = rhandler; sc->psc_pcir_whandler[i] = whandler; } return (0); } static int passthru_legacy_config(nvlist_t *nvl, const char *opt) { char *config, *name, *tofree, *value; if (opt == NULL) return (0); config = tofree = strdup(opt); while ((name = strsep(&config, ",")) != NULL) { value = strchr(name, '='); if (value != NULL) { *value++ = '\0'; set_config_value_node(nvl, name, value); } else { if (strncmp(name, "/dev/ppt", 8) != 0) { EPRINTLN("passthru: invalid path \"%s\"", name); free(tofree); return (-1); } set_config_value_node(nvl, "path", name); } } free(tofree); return (0); } static int passthru_init_rom(struct vmctx *const ctx __unused, struct passthru_softc *const sc, const char *const romfile) { if (romfile == NULL) { return (0); } const int fd = open(romfile, O_RDONLY); if (fd < 0) { warnx("%s: can't open romfile \"%s\"", __func__, romfile); return (-1); } struct stat sbuf; if (fstat(fd, &sbuf) < 0) { warnx("%s: can't fstat romfile \"%s\"", __func__, romfile); close(fd); return (-1); } const uint64_t rom_size = sbuf.st_size; void *const rom_data = mmap(NULL, rom_size, PROT_READ, MAP_SHARED, fd, 0); if (rom_data == MAP_FAILED) { warnx("%s: unable to mmap romfile \"%s\" (%d)", __func__, romfile, errno); close(fd); return (-1); } void *rom_addr; int error = pci_emul_alloc_rom(sc->psc_pi, rom_size, &rom_addr); if (error) { warnx("%s: failed to alloc rom segment", __func__); munmap(rom_data, rom_size); close(fd); return (error); } memcpy(rom_addr, rom_data, rom_size); sc->psc_bar[PCI_ROM_IDX].type = PCIBAR_ROM; sc->psc_bar[PCI_ROM_IDX].addr = (uint64_t)rom_addr; sc->psc_bar[PCI_ROM_IDX].size = rom_size; munmap(rom_data, rom_size); close(fd); return (0); } static int passthru_init(struct pci_devinst *pi, nvlist_t *nvl) { int error, memflags, pptfd; struct passthru_softc *sc; const char *path; struct vmctx *ctx = pi->pi_vmctx; pptfd = -1; sc = NULL; error = 1; memflags = vm_get_memflags(ctx); if (!(memflags & VM_MEM_F_WIRED)) { warnx("passthru requires guest memory to be wired"); goto done; } path = get_config_value_node(nvl, "path"); if (path == NULL || passthru_dev_open(path, &pptfd) != 0) { warnx("invalid passthru options"); goto done; } if (vm_assign_pptdev(ctx, pptfd) != 0) { warnx("PCI device at %d is not using the ppt driver", pptfd); goto done; } sc = calloc(1, sizeof(struct passthru_softc)); pi->pi_arg = sc; sc->psc_pi = pi; sc->pptfd = pptfd; if ((error = vm_get_pptdev_limits(ctx, pptfd, &sc->msi_limit, &sc->msix_limit)) != 0) goto done; #ifndef __FreeBSD__ /* * If this function uses legacy interrupt messages, then request one for * the guest in case drivers expect to see it. Note that nothing in the * hypervisor is currently wired up do deliver such an interrupt should * the guest actually rely upon it. */ uint8_t intpin = passthru_read_config(sc, PCIR_INTPIN, 1); if (intpin > 0 && intpin < 5) pci_lintr_request(sc->psc_pi); #endif /* initialize config space */ if ((error = cfginit(ctx, sc)) != 0) goto done; /* initialize ROM */ if ((error = passthru_init_rom(ctx, sc, get_config_value_node(nvl, "rom"))) != 0) { goto done; } done: if (error) { free(sc); if (pptfd != -1) vm_unassign_pptdev(ctx, pptfd); } return (error); } static int msicap_access(struct passthru_softc *sc, int coff) { int caplen; if (sc->psc_msi.capoff == 0) return (0); caplen = msi_caplen(sc->psc_msi.msgctrl); if (coff >= sc->psc_msi.capoff && coff < sc->psc_msi.capoff + caplen) return (1); else return (0); } static int msixcap_access(struct passthru_softc *sc, int coff) { if (sc->psc_msix.capoff == 0) return (0); return (coff >= sc->psc_msix.capoff && coff < sc->psc_msix.capoff + MSIX_CAPLEN); } static int passthru_cfgread_default(struct passthru_softc *sc, struct pci_devinst *pi __unused, int coff, int bytes, uint32_t *rv) { /* * MSI capability is emulated. */ if (msicap_access(sc, coff) || msixcap_access(sc, coff)) return (PE_CFGRW_DEFAULT); /* * MSI-X is also emulated since a limit on interrupts may be imposed by * the OS, altering the perceived register state. */ if (msixcap_access(sc, coff)) return (PE_CFGRW_DEFAULT); /* * Emulate the command register. If a single read reads both the * command and status registers, read the status register from the * device's config space. */ if (coff == PCIR_COMMAND) { if (bytes <= 2) return (PE_CFGRW_DEFAULT); *rv = passthru_read_config(sc, PCIR_STATUS, 2) << 16 | pci_get_cfgdata16(pi, PCIR_COMMAND); return (PE_CFGRW_DROP); } /* Everything else just read from the device's config space */ *rv = passthru_read_config(sc, coff, bytes); return (PE_CFGRW_DROP); } int passthru_cfgread_emulate(struct passthru_softc *sc __unused, struct pci_devinst *pi __unused, int coff __unused, int bytes __unused, uint32_t *rv __unused) { return (PE_CFGRW_DEFAULT); } static int passthru_cfgread(struct pci_devinst *pi, int coff, int bytes, uint32_t *rv) { struct passthru_softc *sc; sc = pi->pi_arg; if (sc->psc_pcir_rhandler[coff] != NULL) return (sc->psc_pcir_rhandler[coff](sc, pi, coff, bytes, rv)); return (passthru_cfgread_default(sc, pi, coff, bytes, rv)); } static int passthru_cfgwrite_default(struct passthru_softc *sc, struct pci_devinst *pi, int coff, int bytes, uint32_t val) { int error, msix_table_entries, i; uint16_t cmd_old; struct vmctx *ctx = pi->pi_vmctx; /* * MSI capability is emulated */ if (msicap_access(sc, coff)) { pci_emul_capwrite(pi, coff, bytes, val, sc->psc_msi.capoff, PCIY_MSI); error = vm_setup_pptdev_msi(ctx, sc->pptfd, pi->pi_msi.addr, pi->pi_msi.msg_data, pi->pi_msi.maxmsgnum); if (error != 0) err(1, "vm_setup_pptdev_msi"); return (PE_CFGRW_DROP); } if (msixcap_access(sc, coff)) { pci_emul_capwrite(pi, coff, bytes, val, sc->psc_msix.capoff, PCIY_MSIX); if (pi->pi_msix.enabled) { msix_table_entries = pi->pi_msix.table_count; for (i = 0; i < msix_table_entries; i++) { error = vm_setup_pptdev_msix(ctx, sc->pptfd, i, pi->pi_msix.table[i].addr, pi->pi_msix.table[i].msg_data, pi->pi_msix.table[i].vector_control); if (error) err(1, "vm_setup_pptdev_msix"); } } else { error = vm_disable_pptdev_msix(ctx, sc->pptfd); if (error) err(1, "vm_disable_pptdev_msix"); } return (PE_CFGRW_DROP); } /* * The command register is emulated, but the status register * is passed through. */ if (coff == PCIR_COMMAND) { if (bytes <= 2) return (PE_CFGRW_DEFAULT); /* Update the physical status register. */ passthru_write_config(sc, PCIR_STATUS, 2, val >> 16); /* Update the virtual command register. */ cmd_old = pci_get_cfgdata16(pi, PCIR_COMMAND); pci_set_cfgdata16(pi, PCIR_COMMAND, val & 0xffff); pci_emul_cmd_changed(pi, cmd_old); return (PE_CFGRW_DROP); } passthru_write_config(sc, coff, bytes, val); return (PE_CFGRW_DROP); } int passthru_cfgwrite_emulate(struct passthru_softc *sc __unused, struct pci_devinst *pi __unused, int coff __unused, int bytes __unused, uint32_t val __unused) { return (PE_CFGRW_DEFAULT); } static int passthru_cfgwrite(struct pci_devinst *pi, int coff, int bytes, uint32_t val) { struct passthru_softc *sc; sc = pi->pi_arg; if (sc->psc_pcir_whandler[coff] != NULL) return (sc->psc_pcir_whandler[coff](sc, pi, coff, bytes, val)); return (passthru_cfgwrite_default(sc, pi, coff, bytes, val)); } static void passthru_write(struct pci_devinst *pi, int baridx, uint64_t offset, int size, uint64_t value) { struct passthru_softc *sc = pi->pi_arg; struct vmctx *ctx = pi->pi_vmctx; if (baridx == pci_msix_table_bar(pi)) { msix_table_write(ctx, sc, offset, size, value); } else { struct ppt_bar_io pbi; assert(pi->pi_bar[baridx].type == PCIBAR_IO); pbi.pbi_bar = baridx; pbi.pbi_width = size; pbi.pbi_off = offset; pbi.pbi_data = value; (void) ioctl(sc->pptfd, PPT_BAR_WRITE, &pbi); } } static uint64_t passthru_read(struct pci_devinst *pi, int baridx, uint64_t offset, int size) { struct passthru_softc *sc = pi->pi_arg; uint64_t val; if (baridx == pci_msix_table_bar(pi)) { val = msix_table_read(sc, offset, size); } else { struct ppt_bar_io pbi; assert(pi->pi_bar[baridx].type == PCIBAR_IO); pbi.pbi_bar = baridx; pbi.pbi_width = size; pbi.pbi_off = offset; if (ioctl(sc->pptfd, PPT_BAR_READ, &pbi) == 0) { val = pbi.pbi_data; } else { val = 0; } } return (val); } static void passthru_msix_addr(struct vmctx *ctx, struct pci_devinst *pi, int baridx, int enabled, uint64_t address) { struct passthru_softc *sc; size_t remaining; uint32_t table_size, table_offset; sc = pi->pi_arg; table_offset = rounddown2(pi->pi_msix.table_offset, 4096); if (table_offset > 0) { if (!enabled) { if (vm_unmap_pptdev_mmio(ctx, sc->pptfd, address, table_offset) != 0) warnx("pci_passthru: unmap_pptdev_mmio failed"); } else { if (vm_map_pptdev_mmio(ctx, sc->pptfd, address, table_offset, sc->psc_bar[baridx].addr) != 0) warnx("pci_passthru: map_pptdev_mmio failed"); } } table_size = pi->pi_msix.table_offset - table_offset; table_size += pi->pi_msix.table_count * MSIX_TABLE_ENTRY_SIZE; table_size = roundup2(table_size, 4096); remaining = pi->pi_bar[baridx].size - table_offset - table_size; if (remaining > 0) { address += table_offset + table_size; if (!enabled) { if (vm_unmap_pptdev_mmio(ctx, sc->pptfd, address, remaining) != 0) warnx("pci_passthru: unmap_pptdev_mmio failed"); } else { if (vm_map_pptdev_mmio(ctx, sc->pptfd, address, remaining, sc->psc_bar[baridx].addr + table_offset + table_size) != 0) warnx("pci_passthru: map_pptdev_mmio failed"); } } } static void passthru_mmio_addr(struct vmctx *ctx, struct pci_devinst *pi, int baridx, int enabled, uint64_t address) { struct passthru_softc *sc; sc = pi->pi_arg; if (!enabled) { if (vm_unmap_pptdev_mmio(ctx, sc->pptfd, address, sc->psc_bar[baridx].size) != 0) warnx("pci_passthru: unmap_pptdev_mmio failed"); } else { if (vm_map_pptdev_mmio(ctx, sc->pptfd, address, sc->psc_bar[baridx].size, sc->psc_bar[baridx].addr) != 0) warnx("pci_passthru: map_pptdev_mmio failed"); } } static void passthru_addr_rom(struct pci_devinst *const pi, const int idx, const int enabled) { const uint64_t addr = pi->pi_bar[idx].addr; const uint64_t size = pi->pi_bar[idx].size; if (!enabled) { if (vm_munmap_memseg(pi->pi_vmctx, addr, size) != 0) { errx(4, "%s: munmap_memseg @ [%016lx - %016lx] failed", __func__, addr, addr + size); } } else { if (vm_mmap_memseg(pi->pi_vmctx, addr, VM_PCIROM, pi->pi_romoffset, size, PROT_READ | PROT_EXEC) != 0) { errx(4, "%s: mmap_memseg @ [%016lx - %016lx] failed", __func__, addr, addr + size); } } } static void passthru_addr(struct pci_devinst *pi, int baridx, int enabled, uint64_t address) { struct vmctx *ctx = pi->pi_vmctx; switch (pi->pi_bar[baridx].type) { case PCIBAR_IO: /* IO BARs are emulated */ break; case PCIBAR_ROM: passthru_addr_rom(pi, baridx, enabled); break; case PCIBAR_MEM32: case PCIBAR_MEM64: if (baridx == pci_msix_table_bar(pi)) passthru_msix_addr(ctx, pi, baridx, enabled, address); else passthru_mmio_addr(ctx, pi, baridx, enabled, address); break; default: errx(4, "%s: invalid BAR type %d", __func__, pi->pi_bar[baridx].type); } } static const struct pci_devemu passthru = { .pe_emu = "passthru", .pe_init = passthru_init, .pe_legacy_config = passthru_legacy_config, .pe_cfgwrite = passthru_cfgwrite, .pe_cfgread = passthru_cfgread, .pe_barwrite = passthru_write, .pe_barread = passthru_read, .pe_baraddr = passthru_addr, }; PCI_EMUL_SET(passthru); /* * This isn't the right place for these functions which, on FreeBSD, can * read or write from arbitrary devices. They are not supported on illumos; * not least because bhyve is generally run in a non-global zone which doesn't * have access to the devinfo tree. */ uint32_t pci_host_read_config(const struct pcisel *sel __unused, long reg __unused, int width __unused) { return (-1); } void pci_host_write_config(const struct pcisel *sel __unused, long reg __unused, int width __unused, uint32_t data __unused) { errx(4, "pci_host_write_config() unimplemented on illumos"); } /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2020 Beckhoff Automation GmbH & Co. KG * Author: Corvin Khne */ #ifndef _PCI_PASSTHRU_H_ #define _PCI_PASSTHRU_H_ #include #include "pci_emul.h" struct passthru_softc; typedef int (*cfgread_handler)(struct passthru_softc *sc, struct pci_devinst *pi, int coff, int bytes, uint32_t *rv); typedef int (*cfgwrite_handler)(struct passthru_softc *sc, struct pci_devinst *pi, int coff, int bytes, uint32_t val); uint32_t pci_host_read_config(const struct pcisel *sel, long reg, int width); void pci_host_write_config(const struct pcisel *sel, long reg, int width, uint32_t data); int passthru_cfgread_emulate(struct passthru_softc *sc, struct pci_devinst *pi, int coff, int bytes, uint32_t *rv); int passthru_cfgwrite_emulate(struct passthru_softc *sc, struct pci_devinst *pi, int coff, int bytes, uint32_t val); int set_pcir_handler(struct passthru_softc *sc, int reg, int len, cfgread_handler rhandler, cfgwrite_handler whandler); #endif /* _PCI_PASSTHRU_H_ */ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2012 NetApp, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include #include #include "bhyverun.h" #include "config.h" #include "debug.h" #include "pci_emul.h" #include "uart_emul.h" /* * Pick a PCI vid/did of a chip with a single uart at * BAR0, that most versions of FreeBSD can understand: * Siig CyberSerial 1-port. */ #define COM_VENDOR 0x131f #define COM_DEV 0x2000 static void pci_uart_intr_assert(void *arg) { struct pci_devinst *pi = arg; pci_lintr_assert(pi); } static void pci_uart_intr_deassert(void *arg) { struct pci_devinst *pi = arg; pci_lintr_deassert(pi); } static void pci_uart_write(struct pci_devinst *pi, int baridx, uint64_t offset, int size, uint64_t value) { assert(baridx == 0); assert(size == 1); uart_ns16550_write(pi->pi_arg, offset, value); } static uint64_t pci_uart_read(struct pci_devinst *pi, int baridx, uint64_t offset, int size) { uint8_t val; assert(baridx == 0); assert(size == 1); val = uart_ns16550_read(pi->pi_arg, offset); return (val); } static int pci_uart_legacy_config(nvlist_t *nvl, const char *opts) { if (opts != NULL) set_config_value_node(nvl, "path", opts); return (0); } static int pci_uart_init(struct pci_devinst *pi, nvlist_t *nvl) { struct uart_ns16550_softc *sc; const char *device; pci_emul_alloc_bar(pi, 0, PCIBAR_IO, UART_NS16550_IO_BAR_SIZE); pci_lintr_request(pi); /* initialize config space */ pci_set_cfgdata16(pi, PCIR_DEVICE, COM_DEV); pci_set_cfgdata16(pi, PCIR_VENDOR, COM_VENDOR); pci_set_cfgdata8(pi, PCIR_CLASS, PCIC_SIMPLECOMM); pci_set_cfgdata8(pi, PCIR_SUBCLASS, PCIS_SIMPLECOMM_UART); pci_set_cfgdata8(pi, PCIR_PROGIF, PCIP_SIMPLECOMM_UART_16550A); sc = uart_ns16550_init(pci_uart_intr_assert, pci_uart_intr_deassert, pi); pi->pi_arg = sc; device = get_config_value_node(nvl, "path"); if (device != NULL && uart_ns16550_tty_open(sc, device) != 0) { EPRINTLN("Unable to initialize backend '%s' for " "pci uart at %d:%d", device, pi->pi_slot, pi->pi_func); return (-1); } return (0); } static const struct pci_devemu pci_de_com = { .pe_emu = "uart", .pe_init = pci_uart_init, .pe_legacy_config = pci_uart_legacy_config, .pe_barwrite = pci_uart_write, .pe_barread = pci_uart_read }; PCI_EMUL_SET(pci_de_com); /*- * Copyright (c) 2015 iXsystems Inc. * Copyright (c) 2017-2018 Jakub Klama * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer * in this position and unchanged. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * This file and its contents are supplied under the terms of the * Common Development and Distribution License ("CDDL"), version 1.0. * You may only use this file in accordance with the terms of version * 1.0 of the CDDL. * * A full copy of the text of the CDDL should have accompanied this * source. A copy of the CDDL is also available via the Internet at * http://www.illumos.org/license/CDDL. */ /* This file is dual-licensed; see usr/src/contrib/bhyve/LICENSE */ /* * Copyright 2026 Oxide Computer Company */ /* * VirtIO filesystem passthrough using 9p protocol. */ #include #include #include #ifndef __FreeBSD__ #include #endif #ifndef WITHOUT_CAPSICUM #include #endif #include #include #include #include #include #include #include #include #include #include #include "bhyverun.h" #include "config.h" #include "debug.h" #include "pci_emul.h" #include "virtio.h" #ifndef __FreeBSD__ #include "privileges.h" #endif #define VT9P_MAX_IOV 128 #define VT9P_RINGSZ 256 #define VT9P_MAXTAGSZ 256 #define VT9P_CONFIGSPACESZ (VT9P_MAXTAGSZ + sizeof(uint16_t)) static int pci_vt9p_debug; #define DPRINTF(params) if (pci_vt9p_debug) printf params #define WPRINTF(params) printf params /* Capability bits */ #define VT9P_MOUNT_TAG (1 << 0) /* Mount point in config var */ /* * Per-device softc */ struct pci_vt9p_softc { struct virtio_softc vsc_vs; struct vqueue_info vsc_vq; pthread_mutex_t vsc_mtx; uint64_t vsc_cfg; uint64_t vsc_features; char * vsc_rootpath; struct pci_vt9p_config * vsc_config; struct l9p_backend * vsc_fs_backend; struct l9p_server * vsc_server; struct l9p_connection * vsc_conn; }; struct pci_vt9p_request { struct pci_vt9p_softc * vsr_sc; struct iovec * vsr_iov; size_t vsr_niov; size_t vsr_respidx; size_t vsr_iolen; uint16_t vsr_idx; }; struct pci_vt9p_config { uint16_t tag_len; #ifdef __FreeBSD__ char tag[0]; #else char tag[VT9P_MAXTAGSZ]; #endif } __attribute__((packed)); static int pci_vt9p_send(struct l9p_request *, const struct iovec *, const size_t, const size_t, void *); static void pci_vt9p_drop(struct l9p_request *, const struct iovec *, size_t, void *); static void pci_vt9p_reset(void *); static void pci_vt9p_notify(void *, struct vqueue_info *); static int pci_vt9p_cfgread(void *, int, int, uint32_t *); static void pci_vt9p_neg_features(void *, uint64_t *); static virtio_capstr_t vt9p_caps[] = { { VT9P_MOUNT_TAG, "VT9P_MOUNT_TAG" }, }; static struct virtio_consts vt9p_vi_consts = { .vc_name = "vt9p", .vc_nvq = 1, .vc_cfgsize = VT9P_CONFIGSPACESZ, .vc_reset = pci_vt9p_reset, .vc_qnotify = pci_vt9p_notify, .vc_cfgread = pci_vt9p_cfgread, .vc_apply_features = pci_vt9p_neg_features, .vc_hv_caps_legacy = VT9P_MOUNT_TAG, .vc_hv_caps_modern = VT9P_MOUNT_TAG, .vc_capstr = vt9p_caps, .vc_ncapstr = ARRAY_SIZE(vt9p_caps), }; static void pci_vt9p_reset(void *vsc) { struct pci_vt9p_softc *sc; sc = vsc; DPRINTF(("vt9p: device reset requested !\n")); vi_reset_dev(&sc->vsc_vs); } static void pci_vt9p_neg_features(void *vsc, uint64_t *negotiated_features) { struct pci_vt9p_softc *sc = vsc; sc->vsc_features = *negotiated_features; } static int pci_vt9p_cfgread(void *vsc, int offset, int size, uint32_t *retval) { struct pci_vt9p_softc *sc = vsc; void *ptr; ptr = (uint8_t *)sc->vsc_config + offset; memcpy(retval, ptr, size); return (0); } static int pci_vt9p_get_buffer(struct l9p_request *req, struct iovec *iov, size_t *niov, void *arg __unused) { struct pci_vt9p_request *preq = req->lr_aux; size_t n = preq->vsr_niov - preq->vsr_respidx; memcpy(iov, preq->vsr_iov + preq->vsr_respidx, n * sizeof(struct iovec)); *niov = n; return (0); } static int pci_vt9p_send(struct l9p_request *req, const struct iovec *iov __unused, const size_t niov __unused, const size_t iolen, void *arg __unused) { struct pci_vt9p_request *preq = req->lr_aux; struct pci_vt9p_softc *sc = preq->vsr_sc; preq->vsr_iolen = iolen; pthread_mutex_lock(&sc->vsc_mtx); vq_relchain(&sc->vsc_vq, preq->vsr_idx, preq->vsr_iolen); vq_endchains(&sc->vsc_vq, 1); pthread_mutex_unlock(&sc->vsc_mtx); free(preq); return (0); } static void pci_vt9p_drop(struct l9p_request *req, const struct iovec *iov __unused, size_t niov __unused, void *arg __unused) { struct pci_vt9p_request *preq = req->lr_aux; struct pci_vt9p_softc *sc = preq->vsr_sc; pthread_mutex_lock(&sc->vsc_mtx); vq_relchain(&sc->vsc_vq, preq->vsr_idx, 0); vq_endchains(&sc->vsc_vq, 1); pthread_mutex_unlock(&sc->vsc_mtx); free(preq); } static void pci_vt9p_notify(void *vsc, struct vqueue_info *vq) { struct iovec iov[VT9P_MAX_IOV]; struct pci_vt9p_softc *sc; struct pci_vt9p_request *preq; struct vi_req req; int n; sc = vsc; while (vq_has_descs(vq)) { n = vq_getchain(vq, iov, VT9P_MAX_IOV, &req); assert(n >= 1 && n <= VT9P_MAX_IOV); preq = calloc(1, sizeof(struct pci_vt9p_request)); #ifndef __FreeBSD__ if (preq == NULL) { EPRINTLN("virtio-9p: allocation failure: %s", strerror(errno)); break; } #endif preq->vsr_sc = sc; preq->vsr_idx = req.idx; preq->vsr_iov = iov; preq->vsr_niov = n; preq->vsr_respidx = req.readable; for (int i = 0; i < n; i++) { DPRINTF(("vt9p: vt9p_notify(): desc%d base=%p, " "len=%zu\r\n", i, iov[i].iov_base, iov[i].iov_len)); } l9p_connection_recv(sc->vsc_conn, iov, preq->vsr_respidx, preq); } } static int pci_vt9p_legacy_config(nvlist_t *nvl, const char *opts) { char *sharename = NULL, *tofree, *token, *tokens; if (opts == NULL) return (0); tokens = tofree = strdup(opts); while ((token = strsep(&tokens, ",")) != NULL) { if (strchr(token, '=') != NULL) { if (sharename != NULL) { EPRINTLN( "virtio-9p: more than one share name given"); return (-1); } sharename = strsep(&token, "="); set_config_value_node(nvl, "sharename", sharename); set_config_value_node(nvl, "path", token); } else set_config_bool_node(nvl, token, true); } free(tofree); return (0); } static int pci_vt9p_init(struct pci_devinst *pi, nvlist_t *nvl) { struct pci_vt9p_softc *sc; const char *value; const char *sharename; int rootfd; bool ro; #ifndef WITHOUT_CAPSICUM cap_rights_t rootcap; #endif ro = get_config_bool_node_default(nvl, "ro", false); #ifndef __FreeBSD__ illumos_priv_add_min(PRIV_FILE_DAC_READ, "vt9p"); illumos_priv_add_min(PRIV_FILE_DAC_SEARCH, "vt9p"); if (!ro) { illumos_priv_add_min(PRIV_FILE_CHOWN, "vt9p"); illumos_priv_add_min(PRIV_FILE_CHOWN_SELF, "vt9p"); illumos_priv_add_min(PRIV_FILE_WRITE, "vt9p"); illumos_priv_add_min(PRIV_FILE_DAC_WRITE, "vt9p"); illumos_priv_add_min(PRIV_FILE_OWNER, "vt9p"); illumos_priv_add_min(PRIV_FILE_LINK_ANY, "vt9p"); } #endif value = get_config_value_node(nvl, "path"); if (value == NULL) { EPRINTLN("virtio-9p: path required"); return (1); } rootfd = open(value, O_DIRECTORY); if (rootfd < 0) { EPRINTLN("virtio-9p: failed to open '%s': %s", value, strerror(errno)); return (-1); } sharename = get_config_value_node(nvl, "sharename"); if (sharename == NULL) { EPRINTLN("virtio-9p: share name required"); return (1); } if (strlen(sharename) > VT9P_MAXTAGSZ) { EPRINTLN("virtio-9p: share name too long"); return (1); } sc = calloc(1, sizeof(struct pci_vt9p_softc)); #ifdef __FreeBSD__ sc->vsc_config = calloc(1, sizeof(struct pci_vt9p_config) + VT9P_MAXTAGSZ); #else if (sc == NULL) { EPRINTLN("virtio-9p: soft state allocation failure: %s", strerror(errno)); return (1); } sc->vsc_config = calloc(1, sizeof(struct pci_vt9p_config)); if (sc == NULL) { EPRINTLN("virtio-9p: vsc_config allocation failure: %s", strerror(errno)); return (1); } #endif pthread_mutex_init(&sc->vsc_mtx, NULL); #ifndef WITHOUT_CAPSICUM cap_rights_init(&rootcap, CAP_LOOKUP, CAP_ACL_CHECK, CAP_ACL_DELETE, CAP_ACL_GET, CAP_ACL_SET, CAP_READ, CAP_WRITE, CAP_SEEK, CAP_FSTAT, CAP_CREATE, CAP_FCHMODAT, CAP_FCHOWNAT, CAP_FTRUNCATE, CAP_LINKAT_SOURCE, CAP_LINKAT_TARGET, CAP_MKDIRAT, CAP_MKNODAT, CAP_PREAD, CAP_PWRITE, CAP_RENAMEAT_SOURCE, CAP_RENAMEAT_TARGET, CAP_SEEK, CAP_SYMLINKAT, CAP_UNLINKAT, CAP_EXTATTR_DELETE, CAP_EXTATTR_GET, CAP_EXTATTR_LIST, CAP_EXTATTR_SET, CAP_FUTIMES, CAP_FSTATFS, CAP_FSYNC, CAP_FPATHCONF); if (cap_rights_limit(rootfd, &rootcap) != 0) return (1); #endif sc->vsc_config->tag_len = (uint16_t)strlen(sharename); memcpy(sc->vsc_config->tag, sharename, sc->vsc_config->tag_len); if (l9p_backend_fs_init(&sc->vsc_fs_backend, rootfd, ro) != 0) { errno = ENXIO; return (1); } if (l9p_server_init(&sc->vsc_server, sc->vsc_fs_backend) != 0) { errno = ENXIO; return (1); } if (l9p_connection_init(sc->vsc_server, &sc->vsc_conn) != 0) { errno = EIO; return (1); } sc->vsc_conn->lc_msize = L9P_MAX_IOV * PAGE_SIZE; sc->vsc_conn->lc_lt.lt_get_response_buffer = pci_vt9p_get_buffer; sc->vsc_conn->lc_lt.lt_send_response = pci_vt9p_send; sc->vsc_conn->lc_lt.lt_drop_response = pci_vt9p_drop; vi_softc_linkup(&sc->vsc_vs, &vt9p_vi_consts, sc, pi, &sc->vsc_vq); sc->vsc_vs.vs_mtx = &sc->vsc_mtx; sc->vsc_vq.vq_qsize = VT9P_RINGSZ; /* initialize config space */ vi_pci_init(pi, VIRTIO_MODE_TRANSITIONAL, VIRTIO_DEV_9P, VIRTIO_ID_9P, PCIC_STORAGE); if (!vi_intr_init(&sc->vsc_vs, fbsdrun_virtio_msix())) return (1); if (!vi_pcibar_setup(&sc->vsc_vs)) return (1); return (0); } static const struct pci_devemu pci_de_v9p = { .pe_emu = "virtio-9p", .pe_legacy_config = pci_vt9p_legacy_config, .pe_init = pci_vt9p_init, .pe_cfgwrite = vi_pci_cfgwrite, .pe_cfgread = vi_pci_cfgread, .pe_barwrite = vi_pci_write, .pe_barread = vi_pci_read }; PCI_EMUL_SET(pci_de_v9p); /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2011 NetApp, Inc. * All rights reserved. * Copyright 2020-2021 Joyent, Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * This file and its contents are supplied under the terms of the * Common Development and Distribution License ("CDDL"), version 1.0. * You may only use this file in accordance with the terms of version * 1.0 of the CDDL. * * A full copy of the text of the CDDL should have accompanied this * source. A copy of the CDDL is also available via the Internet at * http://www.illumos.org/license/CDDL. */ /* This file is dual-licensed; see usr/src/contrib/bhyve/LICENSE */ /* * Copyright 2014 Pluribus Networks Inc. * Copyright 2026 Oxide Computer Company */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "bhyverun.h" #include "config.h" #include "debug.h" #include "iov.h" #include "pci_emul.h" #include "virtio.h" #include "block_if.h" #define VTBLK_BSIZE 512 #define VTBLK_RINGSZ 128 _Static_assert(VTBLK_RINGSZ <= BLOCKIF_RING_MAX, "Each ring entry must be able to queue a request"); #define VTBLK_S_OK 0 #define VTBLK_S_IOERR 1 #define VTBLK_S_UNSUPP 2 #define VTBLK_BLK_ID_BYTES 20 + 1 /* Capability bits */ #define VTBLK_F_BARRIER (1 << 0) /* Does host support barriers? */ #define VTBLK_F_SIZE_MAX (1 << 1) /* Indicates maximum segment size */ #define VTBLK_F_SEG_MAX (1 << 2) /* Indicates maximum # of segments */ #define VTBLK_F_GEOMETRY (1 << 4) /* Legacy geometry available */ #define VTBLK_F_RO (1 << 5) /* Disk is read-only */ #define VTBLK_F_BLK_SIZE (1 << 6) /* Block size of disk is available */ #define VTBLK_F_SCSI (1 << 7) /* Supports scsi command passthru */ #define VTBLK_F_FLUSH (1 << 9) /* Writeback enabled after reset */ #define VTBLK_F_WCE (1 << 9) /* Legacy alias for FLUSH */ #define VTBLK_F_TOPOLOGY (1 << 10) /* Topology information available */ #define VTBLK_F_CONFIG_WCE (1 << 11) /* Writeback mode avail in config */ #define VTBLK_F_MQ (1 << 12) /* Multi-Queue */ #define VTBLK_F_DISCARD (1 << 13) /* Trim blocks */ #define VTBLK_F_WRITE_ZEROES (1 << 14) /* Write zeros */ /* * Host capabilities */ #define VTBLK_S_HOSTCAPS \ (VTBLK_F_SEG_MAX | \ VTBLK_F_BLK_SIZE | \ VTBLK_F_FLUSH | \ VTBLK_F_TOPOLOGY | \ VIRTIO_RING_F_INDIRECT_DESC) /* indirect descriptors */ /* * The current blockif_delete() interface only allows a single delete * request at a time. */ #define VTBLK_MAX_DISCARD_SEG 1 /* * An arbitrary limit to prevent excessive latency due to large * delete requests. */ #define VTBLK_MAX_DISCARD_SECT ((16 << 20) / VTBLK_BSIZE) /* 16 MiB */ /* * Config space "registers" */ struct vtblk_config { uint64_t vbc_capacity; uint32_t vbc_size_max; uint32_t vbc_seg_max; struct { uint16_t cylinders; uint8_t heads; uint8_t sectors; } vbc_geometry; uint32_t vbc_blk_size; struct { uint8_t physical_block_exp; uint8_t alignment_offset; uint16_t min_io_size; uint32_t opt_io_size; } vbc_topology; uint8_t vbc_writeback; uint8_t unused0[1]; uint16_t num_queues; uint32_t max_discard_sectors; uint32_t max_discard_seg; uint32_t discard_sector_alignment; uint32_t max_write_zeroes_sectors; uint32_t max_write_zeroes_seg; uint8_t write_zeroes_may_unmap; uint8_t unused1[3]; } __packed; /* * Fixed-size block header */ struct virtio_blk_hdr { #define VBH_OP_READ 0 #define VBH_OP_WRITE 1 #define VBH_OP_SCSI_CMD 2 #define VBH_OP_SCSI_CMD_OUT 3 #define VBH_OP_FLUSH 4 #define VBH_OP_FLUSH_OUT 5 #define VBH_OP_IDENT 8 #define VBH_OP_DISCARD 11 #define VBH_OP_WRITE_ZEROES 13 #define VBH_FLAG_BARRIER 0x80000000 /* OR'ed into vbh_type */ uint32_t vbh_type; uint32_t vbh_ioprio; uint64_t vbh_sector; } __packed; /* * Debug printf */ static int pci_vtblk_debug; #define DPRINTF(params) if (pci_vtblk_debug) PRINTLN params #define WPRINTF(params) PRINTLN params struct pci_vtblk_ioreq { struct blockif_req io_req; struct pci_vtblk_softc *io_sc; uint8_t *io_status; uint16_t io_idx; }; struct virtio_blk_discard_write_zeroes { uint64_t sector; uint32_t num_sectors; struct { uint32_t unmap:1; uint32_t reserved:31; } flags; }; /* * Per-device softc */ struct pci_vtblk_softc { struct virtio_softc vbsc_vs; pthread_mutex_t vsc_mtx; struct vqueue_info vbsc_vq; struct vtblk_config vbsc_cfg; struct virtio_consts vbsc_consts; struct blockif_ctxt *bc; #ifndef __FreeBSD__ int vbsc_wce; #endif char vbsc_ident[VTBLK_BLK_ID_BYTES]; struct pci_vtblk_ioreq vbsc_ios[VTBLK_RINGSZ]; }; static void pci_vtblk_reset(void *); static void pci_vtblk_notify(void *, struct vqueue_info *); static int pci_vtblk_cfgread(void *, int, int, uint32_t *); static int pci_vtblk_cfgwrite(void *, int, int, uint32_t); #ifndef __FreeBSD__ static void pci_vtblk_apply_feats(void *, uint64_t *); #endif static virtio_capstr_t vtblk_caps[] = { { VTBLK_F_BARRIER, "VTBLK_F_BARRIER" }, { VTBLK_F_SIZE_MAX, "VTBLK_F_SIZE_MAX" }, { VTBLK_F_SEG_MAX, "VTBLK_F_SEG_MAX" }, { VTBLK_F_GEOMETRY, "VTBLK_F_GEOMETRY" }, { VTBLK_F_RO, "VTBLK_F_RO" }, { VTBLK_F_BLK_SIZE, "VTBLK_F_BLK_SIZE" }, { VTBLK_F_SCSI, "VTBLK_F_SCSI" }, { VTBLK_F_FLUSH, "VTBLK_F_FLUSH" }, { VTBLK_F_WCE, "VTBLK_F_WCE" }, { VTBLK_F_TOPOLOGY, "VTBLK_F_TOPOLOGY" }, { VTBLK_F_CONFIG_WCE, "VTBLK_F_CONFIG_WCE" }, { VTBLK_F_MQ, "VTBLK_F_MQ" }, { VTBLK_F_DISCARD, "VTBLK_F_DISCARD" }, { VTBLK_F_WRITE_ZEROES, "VTBLK_F_WRITE_ZEROES" }, }; static struct virtio_consts vtblk_vi_consts = { .vc_name = "vtblk", .vc_nvq = 1, .vc_cfgsize = sizeof (struct vtblk_config), .vc_reset = pci_vtblk_reset, .vc_qnotify = pci_vtblk_notify, .vc_cfgread = pci_vtblk_cfgread, .vc_cfgwrite = pci_vtblk_cfgwrite, #ifndef __FreeBSD__ .vc_apply_features = pci_vtblk_apply_feats, #else .vc_apply_features = NULL, #endif .vc_hv_caps_legacy = VTBLK_S_HOSTCAPS, .vc_hv_caps_modern = VTBLK_S_HOSTCAPS, .vc_capstr = vtblk_caps, .vc_ncapstr = ARRAY_SIZE(vtblk_caps), }; static void pci_vtblk_reset(void *vsc) { struct pci_vtblk_softc *sc = vsc; DPRINTF(("vtblk: device reset requested !")); vi_reset_dev(&sc->vbsc_vs); #ifndef __FreeBSD__ /* Disable write cache until FLUSH feature is negotiated */ (void) blockif_set_wce(sc->bc, 0); sc->vbsc_wce = 0; #endif } static void pci_vtblk_done_locked(struct pci_vtblk_ioreq *io, int err) { struct pci_vtblk_softc *sc = io->io_sc; /* convert errno into a virtio block error return */ if (err == EOPNOTSUPP || err == ENOSYS) *io->io_status = VTBLK_S_UNSUPP; else if (err != 0) *io->io_status = VTBLK_S_IOERR; else *io->io_status = VTBLK_S_OK; /* * Return the descriptor back to the host. * We wrote 1 byte (our status) to host. */ vq_relchain(&sc->vbsc_vq, io->io_idx, 1); vq_endchains(&sc->vbsc_vq, 0); } static void pci_vtblk_done(struct blockif_req *br, int err) { struct pci_vtblk_ioreq *io = br->br_param; struct pci_vtblk_softc *sc = io->io_sc; pthread_mutex_lock(&sc->vsc_mtx); pci_vtblk_done_locked(io, err); pthread_mutex_unlock(&sc->vsc_mtx); } static void pci_vtblk_proc(struct pci_vtblk_softc *sc, struct vqueue_info *vq) { struct virtio_blk_hdr vbh; struct pci_vtblk_ioreq *io; int niov; int err; bool writeop; int type; struct vi_req req; struct iovec iov[BLOCKIF_IOV_MAX + 2]; struct iovec *siov; iov_bunch_t iob; size_t len; niov = vq_getchain(vq, iov, BLOCKIF_IOV_MAX + 2, &req); /* * As a transitional device we cannot make any assumptions about the * descriptor layout. We know that there will always be at least two * descriptors since every request contains at least one RO and one RW * descriptor but it's perfectly valid (although extremely unlikely) * for a driver to combine things like the last data block in a read * request with the final status byte, or the first data block with * the header in a write. */ if (niov < 2 || niov >= BLOCKIF_IOV_MAX + 2 || req.readable == 0 || req.writable == 0) { EPRINTLN("vioblk: invalid chain niov=0x%x ro=%x rw=%x", niov, req.readable, req.writable); vq_relchain(vq, req.idx, 0); return; } len = iov_bunch_init(&iob, iov, niov); if (!iov_bunch_copy(&iob, &vbh, sizeof (vbh))) { EPRINTLN("vioblk: control header copy failed, chain len 0x%x", len); vq_relchain(vq, req.idx, 0); return; } io = &sc->vbsc_ios[req.idx]; io->io_req.br_offset = vbh.vbh_sector * VTBLK_BSIZE; /* * The IO status byte is the last byte in the last descriptor which we * know is writable having checked above. */ siov = &iov[niov - 1]; io->io_status = (uint8_t *)&siov->iov_base[siov->iov_len - 1]; iob.ib_remain--; /* * The guest should not be setting the BARRIER flag because * we don't advertise the capability. */ type = vbh.vbh_type & ~VBH_FLAG_BARRIER; writeop = (type == VBH_OP_WRITE || type == VBH_OP_DISCARD); switch (type) { case VBH_OP_DISCARD: { struct virtio_blk_discard_write_zeroes discard; /* * We currently only support a single request, as advertised in * the configuration space. If the guest has submitted a * request that doesn't conform to the requirements, we return * a error. */ if (!iov_bunch_copy(&iob, &discard, sizeof (discard)) || iob.ib_remain != 0) { EPRINTLN("vioblk: bad discard message"); pci_vtblk_done_locked(io, EINVAL); return; } /* * virtio v1.1 5.2.6.2: * The device MUST set the status byte to VIRTIO_BLK_S_UNSUPP * for discard and write zeroes commands if any unknown flag is * set. Furthermore, the device MUST set the status byte to * VIRTIO_BLK_S_UNSUPP for discard commands if the unmap flag * is set. * * Currently there are no known flags for a DISCARD request. */ if (discard.flags.unmap != 0 || discard.flags.reserved != 0) { pci_vtblk_done_locked(io, ENOTSUP); return; } /* Make sure the request doesn't exceed our size limit */ if (discard.num_sectors > VTBLK_MAX_DISCARD_SECT) { pci_vtblk_done_locked(io, EINVAL); return; } io->io_req.br_iovcnt = 0; io->io_req.br_offset = discard.sector * VTBLK_BSIZE; io->io_req.br_resid = discard.num_sectors * VTBLK_BSIZE; DPRINTF(("virtio-block: discard op, %zd bytes, offset %ld", io->io_req.br_resid, io->io_req.br_offset)); err = blockif_delete(sc->bc, &io->io_req); return; } case VBH_OP_IDENT: { char *buf; size_t len; int err; len = iob.ib_remain; buf = calloc(len, sizeof (char)); if (buf == NULL) { pci_vtblk_done_locked(io, ENOMEM); return; } len = MIN(len, sizeof (sc->vbsc_ident)); strncpy(buf, sc->vbsc_ident, len); DPRINTF(("virtio-block: ident op, '%.*s'", len, buf)); err = buf_to_iov_bunch(&iob, buf, len) ? 0 : ENOSPC; free(buf); pci_vtblk_done_locked(io, err); return; } default: break; } /* * Accumulate the remainder of the data into the IO request iov. */ io->io_req.br_resid = iob.ib_remain; iov_bunch_to_iov(&iob, (struct iovec *)&io->io_req.br_iov, &io->io_req.br_iovcnt, ARRAY_SIZE(io->io_req.br_iov)); DPRINTF(("virtio-block: %s op, %zd bytes, %d segs, offset %ld", writeop ? "write" : "read", io->io_req.br_resid, io->io_req.br_iovcnt, io->io_req.br_offset)); switch (type) { case VBH_OP_READ: err = blockif_read(sc->bc, &io->io_req); break; case VBH_OP_WRITE: err = blockif_write(sc->bc, &io->io_req); break; case VBH_OP_FLUSH: case VBH_OP_FLUSH_OUT: err = blockif_flush(sc->bc, &io->io_req); break; default: pci_vtblk_done_locked(io, EOPNOTSUPP); return; } assert(err == 0); } static void pci_vtblk_notify(void *vsc, struct vqueue_info *vq) { struct pci_vtblk_softc *sc = vsc; while (vq_has_descs(vq)) pci_vtblk_proc(sc, vq); } static void pci_vtblk_resized(struct blockif_ctxt *bctxt __unused, void *arg, size_t new_size) { struct pci_vtblk_softc *sc; sc = arg; sc->vbsc_cfg.vbc_capacity = new_size / VTBLK_BSIZE; /* 512-byte units */ vq_devcfg_changed(&sc->vbsc_vs); } static int pci_vtblk_init(struct pci_devinst *pi, nvlist_t *nvl) { char bident[sizeof ("XXX:XXX")]; struct blockif_ctxt *bctxt; const char *path, *serial; MD5_CTX mdctx; uchar_t digest[16]; struct pci_vtblk_softc *sc; off_t size; int i, sectsz, sts, sto; /* * The supplied backing file has to exist */ snprintf(bident, sizeof (bident), "%u:%u", pi->pi_slot, pi->pi_func); bctxt = blockif_open(nvl, bident); if (bctxt == NULL) { perror("Could not open backing file"); return (1); } if (blockif_add_boot_device(pi, bctxt)) { perror("Invalid boot device"); return (1); } size = blockif_size(bctxt); sectsz = blockif_sectsz(bctxt); blockif_psectsz(bctxt, &sts, &sto); sc = calloc(1, sizeof (struct pci_vtblk_softc)); sc->bc = bctxt; if (get_config_bool_default("virtio.blk.debug", false)) pci_vtblk_debug = 1; vi_set_debug(&sc->vbsc_vs, pci_vtblk_debug); for (i = 0; i < VTBLK_RINGSZ; i++) { struct pci_vtblk_ioreq *io = &sc->vbsc_ios[i]; io->io_req.br_callback = pci_vtblk_done; io->io_req.br_param = io; io->io_sc = sc; io->io_idx = i; } bcopy(&vtblk_vi_consts, &sc->vbsc_consts, sizeof (vtblk_vi_consts)); if (blockif_candelete(sc->bc)) { sc->vbsc_consts.vc_hv_caps_legacy |= VTBLK_F_DISCARD; sc->vbsc_consts.vc_hv_caps_modern |= VTBLK_F_DISCARD; } #ifndef __FreeBSD__ /* Disable write cache until FLUSH feature is negotiated */ (void) blockif_set_wce(sc->bc, 0); sc->vbsc_wce = 0; #endif pthread_mutex_init(&sc->vsc_mtx, NULL); /* init virtio softc and virtqueues */ vi_softc_linkup(&sc->vbsc_vs, &sc->vbsc_consts, sc, pi, &sc->vbsc_vq); sc->vbsc_vs.vs_mtx = &sc->vsc_mtx; sc->vbsc_vq.vq_qsize = VTBLK_RINGSZ; /* sc->vbsc_vq.vq_notify = we have no per-queue notify */ /* * If an explicit identifier is not given, create an * identifier using parts of the md5 sum of the filename. */ bzero(sc->vbsc_ident, VTBLK_BLK_ID_BYTES); if ((serial = get_config_value_node(nvl, "serial")) != NULL || (serial = get_config_value_node(nvl, "ser")) != NULL) { strlcpy(sc->vbsc_ident, serial, VTBLK_BLK_ID_BYTES); } else { path = get_config_value_node(nvl, "path"); MD5Init(&mdctx); MD5Update(&mdctx, path, strlen(path)); MD5Final(digest, &mdctx); snprintf(sc->vbsc_ident, VTBLK_BLK_ID_BYTES, "BHYVE-%02X%02X-%02X%02X-%02X%02X", digest[0], digest[1], digest[2], digest[3], digest[4], digest[5]); } /* setup virtio block config space */ sc->vbsc_cfg.vbc_capacity = size / VTBLK_BSIZE; /* 512-byte units */ sc->vbsc_cfg.vbc_size_max = 0; /* not negotiated */ /* * If Linux is presented with a seg_max greater than the virtio queue * size, it can stumble into situations where it violates its own * invariants and panics. For safety, we keep seg_max clamped, paying * heed to the two extra descriptors needed for the header and status * of a request. */ sc->vbsc_cfg.vbc_seg_max = MIN(VTBLK_RINGSZ - 2, BLOCKIF_IOV_MAX); sc->vbsc_cfg.vbc_geometry.cylinders = 0; /* no geometry */ sc->vbsc_cfg.vbc_geometry.heads = 0; sc->vbsc_cfg.vbc_geometry.sectors = 0; sc->vbsc_cfg.vbc_blk_size = sectsz; sc->vbsc_cfg.vbc_topology.physical_block_exp = (sts > sectsz) ? (ffsll(sts / sectsz) - 1) : 0; sc->vbsc_cfg.vbc_topology.alignment_offset = (sto != 0) ? ((sts - sto) / sectsz) : 0; sc->vbsc_cfg.vbc_topology.min_io_size = 0; sc->vbsc_cfg.vbc_topology.opt_io_size = 0; sc->vbsc_cfg.vbc_writeback = 0; sc->vbsc_cfg.max_discard_sectors = VTBLK_MAX_DISCARD_SECT; sc->vbsc_cfg.max_discard_seg = VTBLK_MAX_DISCARD_SEG; sc->vbsc_cfg.discard_sector_alignment = MAX(sectsz, sts) / VTBLK_BSIZE; vi_pci_init(pi, VIRTIO_MODE_TRANSITIONAL, VIRTIO_DEV_BLOCK, VIRTIO_ID_BLOCK, PCIC_STORAGE); if (!vi_intr_init(&sc->vbsc_vs, fbsdrun_virtio_msix())) goto fail; if (!vi_pcibar_setup(&sc->vbsc_vs)) goto fail; blockif_register_resize_callback(sc->bc, pci_vtblk_resized, sc); return (0); fail: blockif_close(sc->bc); free(sc); return (1); } static int pci_vtblk_cfgwrite(void *vsc __unused, int offset, int size __unused, uint32_t value __unused) { DPRINTF(("vtblk: write to readonly reg %d", offset)); return (1); } static int pci_vtblk_cfgread(void *vsc, int offset, int size, uint32_t *retval) { struct pci_vtblk_softc *sc = vsc; void *ptr; /* our caller has already verified offset and size */ ptr = (uint8_t *)&sc->vbsc_cfg + offset; memcpy(retval, ptr, size); return (0); } #ifndef __FreeBSD__ void pci_vtblk_apply_feats(void *vsc, uint64_t *caps) { struct pci_vtblk_softc *sc = vsc; const int wce_next = ((*caps & VTBLK_F_FLUSH) != 0) ? 1 : 0; if (sc->vbsc_wce != wce_next) { (void) blockif_set_wce(sc->bc, wce_next); sc->vbsc_wce = wce_next; } } #endif /* __FreeBSD__ */ static const struct pci_devemu pci_de_vblk = { .pe_emu = "virtio-blk", .pe_init = pci_vtblk_init, .pe_legacy_config = blockif_legacy_config, .pe_cfgwrite = vi_pci_cfgwrite, .pe_cfgread = vi_pci_cfgread, .pe_barwrite = vi_pci_write, .pe_barread = vi_pci_read, }; PCI_EMUL_SET(pci_de_vblk); /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2016 iXsystems Inc. * All rights reserved. * * This software was developed by Jakub Klama * under sponsorship from iXsystems Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer * in this position and unchanged. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * This file and its contents are supplied under the terms of the * Common Development and Distribution License ("CDDL"), version 1.0. * You may only use this file in accordance with the terms of version * 1.0 of the CDDL. * * A full copy of the text of the CDDL should have accompanied this * source. A copy of the CDDL is also available via the Internet at * http://www.illumos.org/license/CDDL. */ /* This file is dual-licensed; see usr/src/contrib/bhyve/LICENSE */ /* * Copyright 2018 Joyent, Inc. * Copyright 2025 OmniOS Community Edition (OmniOSce) Association. * Copyright 2026 Oxide Computer Company */ #include #ifndef WITHOUT_CAPSICUM #include #endif #include #include #include #include #include #ifndef __FreeBSD__ #include #include #endif #ifndef WITHOUT_CAPSICUM #include #endif #include #include #include #include #include #include #include #include #include #include #include #include #include "bhyverun.h" #include "config.h" #include "debug.h" #include "pci_emul.h" #include "virtio.h" #include "mevent.h" #include "sockstream.h" #define VTCON_RINGSZ 64 #define VTCON_MAXPORTS 16 #define VTCON_MAXQ (VTCON_MAXPORTS * 2 + 2) #define VTCON_DEVICE_READY 0 #define VTCON_DEVICE_ADD 1 #define VTCON_DEVICE_REMOVE 2 #define VTCON_PORT_READY 3 #define VTCON_CONSOLE_PORT 4 #define VTCON_CONSOLE_RESIZE 5 #define VTCON_PORT_OPEN 6 #define VTCON_PORT_NAME 7 #define VTCON_F_SIZE 0 #define VTCON_F_MULTIPORT 1 #define VTCON_F_EMERG_WRITE 2 #define VTCON_S_HOSTCAPS \ (VTCON_F_SIZE | VTCON_F_MULTIPORT | VTCON_F_EMERG_WRITE) static int pci_vtcon_debug; #define DPRINTF(params) if (pci_vtcon_debug) PRINTLN params #define WPRINTF(params) PRINTLN params struct pci_vtcon_softc; struct pci_vtcon_port; struct pci_vtcon_config; typedef void (pci_vtcon_cb_t)(struct pci_vtcon_port *, void *, struct iovec *, int); struct pci_vtcon_port { struct pci_vtcon_softc * vsp_sc; int vsp_id; const char * vsp_name; bool vsp_enabled; bool vsp_console; bool vsp_rx_ready; bool vsp_open; int vsp_rxq; int vsp_txq; void * vsp_arg; pci_vtcon_cb_t * vsp_cb; }; struct pci_vtcon_sock { struct pci_vtcon_port * vss_port; const char * vss_path; struct mevent * vss_server_evp; struct mevent * vss_conn_evp; int vss_server_fd; int vss_conn_fd; bool vss_open; }; struct pci_vtcon_softc { struct virtio_softc vsc_vs; struct vqueue_info vsc_queues[VTCON_MAXQ]; pthread_mutex_t vsc_mtx; uint64_t vsc_cfg; uint64_t vsc_features; char * vsc_rootdir; int vsc_kq; bool vsc_ready; struct pci_vtcon_port vsc_control_port; struct pci_vtcon_port vsc_ports[VTCON_MAXPORTS]; struct pci_vtcon_config *vsc_config; }; struct pci_vtcon_config { uint16_t cols; uint16_t rows; uint32_t max_nr_ports; uint32_t emerg_wr; } __attribute__((packed)); struct pci_vtcon_control { uint32_t id; uint16_t event; uint16_t value; } __attribute__((packed)); struct pci_vtcon_console_resize { uint16_t cols; uint16_t rows; } __attribute__((packed)); static void pci_vtcon_reset(void *); static void pci_vtcon_notify_rx(void *, struct vqueue_info *); static void pci_vtcon_notify_tx(void *, struct vqueue_info *); static int pci_vtcon_cfgread(void *, int, int, uint32_t *); static int pci_vtcon_cfgwrite(void *, int, int, uint32_t); static void pci_vtcon_neg_features(void *, uint64_t *); static void pci_vtcon_sock_accept(int, enum ev_type, void *); static void pci_vtcon_sock_rx(int, enum ev_type, void *); static void pci_vtcon_sock_tx(struct pci_vtcon_port *, void *, struct iovec *, int); static void pci_vtcon_control_send(struct pci_vtcon_softc *, struct pci_vtcon_control *, const void *, size_t); static void pci_vtcon_announce_port(struct pci_vtcon_port *); static void pci_vtcon_open_port(struct pci_vtcon_port *, bool); static virtio_capstr_t vtcon_caps[] = { { VTCON_F_SIZE, "VTCON_F_SIZE" }, { VTCON_F_MULTIPORT, "VTCON_F_MULTIPORT" }, { VTCON_F_EMERG_WRITE, "VTCON_F_EMERG_WRITE" }, }; static struct virtio_consts vtcon_vi_consts = { .vc_name = "vtcon", .vc_nvq = VTCON_MAXQ, .vc_cfgsize = sizeof(struct pci_vtcon_config), .vc_reset = pci_vtcon_reset, .vc_cfgread = pci_vtcon_cfgread, .vc_cfgwrite = pci_vtcon_cfgwrite, .vc_apply_features = pci_vtcon_neg_features, .vc_hv_caps_legacy = VTCON_S_HOSTCAPS, .vc_hv_caps_modern = VTCON_S_HOSTCAPS, .vc_capstr = vtcon_caps, .vc_ncapstr = ARRAY_SIZE(vtcon_caps), }; static void pci_vtcon_reset(void *vsc) { struct pci_vtcon_softc *sc; sc = vsc; DPRINTF(("vtcon: device reset requested!")); vi_reset_dev(&sc->vsc_vs); } static void pci_vtcon_neg_features(void *vsc, uint64_t *negotiated_features) { struct pci_vtcon_softc *sc = vsc; sc->vsc_features = *negotiated_features; } static int pci_vtcon_cfgread(void *vsc, int offset, int size, uint32_t *retval) { struct pci_vtcon_softc *sc = vsc; void *ptr; ptr = (uint8_t *)sc->vsc_config + offset; memcpy(retval, ptr, size); return (0); } static int pci_vtcon_cfgwrite(void *vsc __unused, int offset __unused, int size __unused, uint32_t val __unused) { return (0); } static inline struct pci_vtcon_port * pci_vtcon_vq_to_port(struct pci_vtcon_softc *sc, struct vqueue_info *vq) { uint16_t num = vq->vq_num; if (num == 0 || num == 1) return (&sc->vsc_ports[0]); if (num == 2 || num == 3) return (&sc->vsc_control_port); return (&sc->vsc_ports[(num / 2) - 1]); } static inline struct vqueue_info * pci_vtcon_port_to_vq(struct pci_vtcon_port *port, bool tx_queue) { int qnum; qnum = tx_queue ? port->vsp_txq : port->vsp_rxq; return (&port->vsp_sc->vsc_queues[qnum]); } static struct pci_vtcon_port * pci_vtcon_port_add(struct pci_vtcon_softc *sc, int port_id, const char *name, pci_vtcon_cb_t *cb, void *arg) { struct pci_vtcon_port *port; port = &sc->vsc_ports[port_id]; if (port->vsp_enabled) { errno = EBUSY; return (NULL); } port->vsp_id = port_id; port->vsp_sc = sc; port->vsp_name = name; port->vsp_cb = cb; port->vsp_arg = arg; if (port->vsp_id == 0) { /* port0 */ port->vsp_txq = 0; port->vsp_rxq = 1; } else { port->vsp_txq = (port_id + 1) * 2; port->vsp_rxq = port->vsp_txq + 1; } port->vsp_enabled = true; return (port); } static int pci_vtcon_sock_add(struct pci_vtcon_softc *sc, const char *port_name, const nvlist_t *nvl) { struct pci_vtcon_sock *sock = NULL; #ifdef __FreeBSD__ struct sockaddr_un sun; #else /* Our compiler #defines 'sun' as '1'. Awesome. */ struct sockaddr_un addr; #endif const char *name, *path; char *cp, *pathcopy; long port; #ifdef __FreeBSD__ int s = -1, fd = -1, error = 0; #else int s = -1, error = 0; #endif #ifndef WITHOUT_CAPSICUM cap_rights_t rights; #endif port = strtol(port_name, &cp, 0); if (*cp != '\0' || port < 0 || port >= VTCON_MAXPORTS) { EPRINTLN("vtcon: Invalid port %s", port_name); error = -1; goto out; } path = get_config_value_node(nvl, "path"); if (path == NULL) { EPRINTLN("vtcon: required path missing for port %ld", port); error = -1; goto out; } sock = calloc(1, sizeof(struct pci_vtcon_sock)); if (sock == NULL) { error = -1; goto out; } s = socket(AF_UNIX, SOCK_STREAM, 0); if (s < 0) { error = -1; goto out; } #ifdef __FreeBSD__ pathcopy = strdup(path); if (pathcopy == NULL) { error = -1; goto out; } fd = open(dirname(pathcopy), O_RDONLY | O_DIRECTORY); if (fd < 0) { free(pathcopy); error = -1; goto out; } sun.sun_family = AF_UNIX; sun.sun_len = sizeof(struct sockaddr_un); strcpy(pathcopy, path); strlcpy(sun.sun_path, basename(pathcopy), sizeof(sun.sun_path)); free(pathcopy); if (bindat(fd, s, (struct sockaddr *)&sun, sun.sun_len) < 0) { error = -1; goto out; } #else /* __FreeBSD__ */ /* Do a simple bind rather than the FreeBSD bindat() */ pathcopy = (char *)path; addr.sun_family = AF_UNIX; (void) strlcpy(addr.sun_path, pathcopy, sizeof (addr.sun_path)); (void) unlink(addr.sun_path); if (bind(s, (struct sockaddr *)&addr, sizeof (addr)) < 0) { error = -1; goto out; } #endif /* __FreeBSD__ */ if (fcntl(s, F_SETFL, O_NONBLOCK) < 0) { error = -1; goto out; } if (listen(s, 1) < 0) { error = -1; goto out; } #ifndef WITHOUT_CAPSICUM cap_rights_init(&rights, CAP_ACCEPT, CAP_EVENT, CAP_READ, CAP_WRITE); if (caph_rights_limit(s, &rights) == -1) errx(EX_OSERR, "Unable to apply rights for sandbox"); #endif name = get_config_value_node(nvl, "name"); if (name == NULL) { EPRINTLN("vtcon: required name missing for port %ld", port); error = -1; goto out; } sock->vss_port = pci_vtcon_port_add(sc, port, name, pci_vtcon_sock_tx, sock); if (sock->vss_port == NULL) { error = -1; goto out; } sock->vss_open = false; sock->vss_conn_fd = -1; sock->vss_server_fd = s; sock->vss_server_evp = mevent_add(s, EVF_READ, pci_vtcon_sock_accept, sock); if (sock->vss_server_evp == NULL) { error = -1; goto out; } out: #ifdef __FreeBSD__ if (fd != -1) close(fd); #endif if (error != 0) { if (s != -1) close(s); free(sock); } return (error); } static void pci_vtcon_sock_accept(int fd __unused, enum ev_type t __unused, void *arg) { struct pci_vtcon_sock *sock = (struct pci_vtcon_sock *)arg; int s; s = accept(sock->vss_server_fd, NULL, NULL); if (s < 0) return; if (sock->vss_open) { close(s); return; } sock->vss_open = true; sock->vss_conn_fd = s; sock->vss_conn_evp = mevent_add(s, EVF_READ, pci_vtcon_sock_rx, sock); pci_vtcon_open_port(sock->vss_port, true); } static void pci_vtcon_sock_rx(int fd __unused, enum ev_type t __unused, void *arg) { struct pci_vtcon_port *port; struct pci_vtcon_sock *sock = (struct pci_vtcon_sock *)arg; struct vqueue_info *vq; struct vi_req req; struct iovec iov; static char dummybuf[2048]; int len, n; port = sock->vss_port; vq = pci_vtcon_port_to_vq(port, true); if (!sock->vss_open || !port->vsp_rx_ready) { len = read(sock->vss_conn_fd, dummybuf, sizeof(dummybuf)); if (len == 0) goto close; return; } if (!vq_has_descs(vq)) { len = read(sock->vss_conn_fd, dummybuf, sizeof(dummybuf)); vq_endchains(vq, 1); if (len == 0) goto close; return; } do { n = vq_getchain(vq, &iov, 1, &req); assert(n == 1); len = readv(sock->vss_conn_fd, &iov, n); if (len == 0 || (len < 0 && errno == EWOULDBLOCK)) { vq_retchains(vq, 1); vq_endchains(vq, 0); if (len == 0) goto close; return; } vq_relchain(vq, req.idx, len); } while (vq_has_descs(vq)); vq_endchains(vq, 1); close: mevent_delete_close(sock->vss_conn_evp); sock->vss_conn_fd = -1; sock->vss_open = false; } static void pci_vtcon_sock_tx(struct pci_vtcon_port *port __unused, void *arg __unused, struct iovec *iov, int niov) { struct pci_vtcon_sock *sock; int i, ret; #ifndef __FreeBSD__ ret = 0; #endif sock = (struct pci_vtcon_sock *)arg; if (sock->vss_conn_fd == -1) return; for (i = 0; i < niov; i++) { ret = stream_write(sock->vss_conn_fd, iov[i].iov_base, iov[i].iov_len); if (ret <= 0) break; } if (ret <= 0) { mevent_delete_close(sock->vss_conn_evp); sock->vss_conn_fd = -1; sock->vss_open = false; } } static void pci_vtcon_control_tx(struct pci_vtcon_port *port, void *arg __unused, struct iovec *iov, int niov) { struct pci_vtcon_softc *sc; struct pci_vtcon_port *tmp; struct pci_vtcon_control resp, *ctrl; int i; assert(niov == 1); sc = port->vsp_sc; ctrl = (struct pci_vtcon_control *)iov->iov_base; switch (ctrl->event) { case VTCON_DEVICE_READY: sc->vsc_ready = true; /* set port ready events for registered ports */ for (i = 0; i < VTCON_MAXPORTS; i++) { tmp = &sc->vsc_ports[i]; if (tmp->vsp_enabled) pci_vtcon_announce_port(tmp); if (tmp->vsp_open) pci_vtcon_open_port(tmp, true); } break; case VTCON_PORT_READY: tmp = &sc->vsc_ports[ctrl->id]; if (ctrl->id >= VTCON_MAXPORTS || !tmp->vsp_enabled) { WPRINTF(("VTCON_PORT_READY event for unknown port %d", ctrl->id)); return; } /* Guest port ready; send additional configuration */ if (ctrl->value == 1 && tmp->vsp_name != NULL) { resp.id = ctrl->id; resp.event = VTCON_PORT_NAME; resp.value = 1; pci_vtcon_control_send(sc, &resp, tmp->vsp_name, strlen(tmp->vsp_name)); } if (tmp->vsp_console) { resp.event = VTCON_CONSOLE_PORT; resp.id = ctrl->id; resp.value = 1; pci_vtcon_control_send(sc, &resp, NULL, 0); } break; } } static void pci_vtcon_announce_port(struct pci_vtcon_port *port) { struct pci_vtcon_control event; event.id = port->vsp_id; event.event = VTCON_DEVICE_ADD; event.value = 1; pci_vtcon_control_send(port->vsp_sc, &event, NULL, 0); } static void pci_vtcon_open_port(struct pci_vtcon_port *port, bool open) { struct pci_vtcon_control event; if (!port->vsp_sc->vsc_ready) { port->vsp_open = true; return; } event.id = port->vsp_id; event.event = VTCON_PORT_OPEN; event.value = (int)open; pci_vtcon_control_send(port->vsp_sc, &event, NULL, 0); } static void pci_vtcon_control_send(struct pci_vtcon_softc *sc, struct pci_vtcon_control *ctrl, const void *payload, size_t len) { struct vqueue_info *vq; struct vi_req req; struct iovec iov; int n; if (len > SIZE_T_MAX - sizeof(struct pci_vtcon_control)) return; vq = pci_vtcon_port_to_vq(&sc->vsc_control_port, true); if (!vq_has_descs(vq)) return; n = vq_getchain(vq, &iov, 1, &req); assert(n == 1); if (iov.iov_len < sizeof(struct pci_vtcon_control) + len) goto out; memcpy(iov.iov_base, ctrl, sizeof(struct pci_vtcon_control)); if (len > 0) memcpy((uint8_t *)iov.iov_base + sizeof(struct pci_vtcon_control), payload, len); out: vq_relchain(vq, req.idx, sizeof(struct pci_vtcon_control) + len); vq_endchains(vq, 1); } static void pci_vtcon_notify_tx(void *vsc, struct vqueue_info *vq) { struct pci_vtcon_softc *sc; struct pci_vtcon_port *port; struct iovec iov[1]; struct vi_req req; int n; sc = vsc; port = pci_vtcon_vq_to_port(sc, vq); while (vq_has_descs(vq)) { n = vq_getchain(vq, iov, 1, &req); assert(n == 1); if (port != NULL) port->vsp_cb(port, port->vsp_arg, iov, 1); /* * Release this chain and handle more */ vq_relchain(vq, req.idx, 0); } vq_endchains(vq, 1); /* Generate interrupt if appropriate. */ } static void pci_vtcon_notify_rx(void *vsc, struct vqueue_info *vq) { struct pci_vtcon_softc *sc; struct pci_vtcon_port *port; sc = vsc; port = pci_vtcon_vq_to_port(sc, vq); if (!port->vsp_rx_ready) { port->vsp_rx_ready = 1; vq_kick_disable(vq); } } /* * Each console device has a "port" node which contains nodes for * each port. Ports are numbered starting at 0. */ static int pci_vtcon_legacy_config_port(nvlist_t *nvl, int port, char *opt) { char *name, *path; char node_name[sizeof("XX")]; nvlist_t *port_nvl; name = strsep(&opt, "="); path = opt; if (path == NULL) { EPRINTLN("vtcon: port %s requires a path", name); return (-1); } if (port >= VTCON_MAXPORTS) { EPRINTLN("vtcon: too many ports"); return (-1); } snprintf(node_name, sizeof(node_name), "%d", port); port_nvl = create_relative_config_node(nvl, node_name); set_config_value_node(port_nvl, "name", name); set_config_value_node(port_nvl, "path", path); return (0); } static int pci_vtcon_legacy_config(nvlist_t *nvl, const char *opts) { char *opt, *str, *tofree; nvlist_t *ports_nvl; int error, port; ports_nvl = create_relative_config_node(nvl, "port"); tofree = str = strdup(opts); error = 0; port = 0; while ((opt = strsep(&str, ",")) != NULL) { error = pci_vtcon_legacy_config_port(ports_nvl, port, opt); if (error) break; port++; } free(tofree); return (error); } static int pci_vtcon_init(struct pci_devinst *pi, nvlist_t *nvl) { struct pci_vtcon_softc *sc; nvlist_t *ports_nvl; int i; sc = calloc(1, sizeof(struct pci_vtcon_softc)); sc->vsc_config = calloc(1, sizeof(struct pci_vtcon_config)); sc->vsc_config->max_nr_ports = VTCON_MAXPORTS; sc->vsc_config->cols = 80; sc->vsc_config->rows = 25; pthread_mutex_init(&sc->vsc_mtx, NULL); vi_softc_linkup(&sc->vsc_vs, &vtcon_vi_consts, sc, pi, sc->vsc_queues); sc->vsc_vs.vs_mtx = &sc->vsc_mtx; for (i = 0; i < VTCON_MAXQ; i++) { sc->vsc_queues[i].vq_qsize = VTCON_RINGSZ; sc->vsc_queues[i].vq_notify = i % 2 == 0 ? pci_vtcon_notify_rx : pci_vtcon_notify_tx; } /* initialize config space */ vi_pci_init(pi, VIRTIO_MODE_TRANSITIONAL, VIRTIO_DEV_CONSOLE, VIRTIO_ID_CONSOLE, PCIC_SIMPLECOMM); if (!vi_intr_init(&sc->vsc_vs, fbsdrun_virtio_msix())) return (1); if (!vi_pcibar_setup(&sc->vsc_vs)) return (1); /* create control port */ sc->vsc_control_port.vsp_sc = sc; sc->vsc_control_port.vsp_txq = 2; sc->vsc_control_port.vsp_rxq = 3; sc->vsc_control_port.vsp_cb = pci_vtcon_control_tx; sc->vsc_control_port.vsp_enabled = true; ports_nvl = find_relative_config_node(nvl, "port"); if (ports_nvl != NULL) { const char *name; void *cookie; int type; cookie = NULL; while ((name = nvlist_next(ports_nvl, &type, &cookie)) != NULL) { if (type != NV_TYPE_NVLIST) continue; if (pci_vtcon_sock_add(sc, name, nvlist_get_nvlist(ports_nvl, name)) < 0) { EPRINTLN("cannot create port %s: %s", name, strerror(errno)); return (1); } } } return (0); } static const struct pci_devemu pci_de_vcon = { .pe_emu = "virtio-console", .pe_init = pci_vtcon_init, .pe_cfgwrite = vi_pci_cfgwrite, .pe_cfgread = vi_pci_cfgread, .pe_barwrite = vi_pci_write, .pe_barread = vi_pci_read, .pe_legacy_config = pci_vtcon_legacy_config, }; PCI_EMUL_SET(pci_de_vcon); /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2021 Beckhoff Automation GmbH & Co. KG * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer * in this position and unchanged. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * This file and its contents are supplied under the terms of the * Common Development and Distribution License ("CDDL"), version 1.0. * You may only use this file in accordance with the terms of version * 1.0 of the CDDL. * * A full copy of the text of the CDDL should have accompanied this * source. A copy of the CDDL is also available via the Internet at * http://www.illumos.org/license/CDDL. */ /* This file is dual-licensed; see usr/src/contrib/bhyve/LICENSE */ /* * Copyright 2026 Oxide Computer Company */ /* * virtio input device emulation. */ #include #ifndef WITHOUT_CAPSICUM #include #include #endif #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "bhyverun.h" #include "config.h" #include "debug.h" #include "mevent.h" #include "pci_emul.h" #include "virtio.h" #define VTINPUT_RINGSZ 64 #define VTINPUT_MAX_PKT_LEN 10 /* * Queue definitions. */ #define VTINPUT_EVENTQ 0 #define VTINPUT_STATUSQ 1 #define VTINPUT_MAXQ 2 static int pci_vtinput_debug; #define DPRINTF(params) \ if (pci_vtinput_debug) \ PRINTLN params #define WPRINTF(params) PRINTLN params enum vtinput_config_select { VTINPUT_CFG_UNSET = 0x00, VTINPUT_CFG_ID_NAME = 0x01, VTINPUT_CFG_ID_SERIAL = 0x02, VTINPUT_CFG_ID_DEVIDS = 0x03, VTINPUT_CFG_PROP_BITS = 0x10, VTINPUT_CFG_EV_BITS = 0x11, VTINPUT_CFG_ABS_INFO = 0x12 }; struct vtinput_absinfo { uint32_t min; uint32_t max; uint32_t fuzz; uint32_t flat; uint32_t res; } __packed; struct vtinput_devids { uint16_t bustype; uint16_t vendor; uint16_t product; uint16_t version; } __packed; struct vtinput_config { uint8_t select; uint8_t subsel; uint8_t size; uint8_t reserved[5]; union { char string[128]; uint8_t bitmap[128]; struct vtinput_absinfo abs; struct vtinput_devids ids; } u; } __packed; struct vtinput_event { uint16_t type; uint16_t code; uint32_t value; } __packed; struct vtinput_event_elem { struct vtinput_event event; struct iovec iov; uint16_t idx; }; struct vtinput_eventqueue { struct vtinput_event_elem *events; uint32_t size; uint32_t idx; }; /* * Per-device softc */ struct pci_vtinput_softc { struct virtio_softc vsc_vs; struct vqueue_info vsc_queues[VTINPUT_MAXQ]; pthread_mutex_t vsc_mtx; const char *vsc_evdev; int vsc_fd; struct vtinput_config vsc_config; int vsc_config_valid; struct mevent *vsc_evp; struct vtinput_eventqueue vsc_eventqueue; }; static void pci_vtinput_reset(void *); static int pci_vtinput_cfgread(void *, int, int, uint32_t *); static int pci_vtinput_cfgwrite(void *, int, int, uint32_t); static struct virtio_consts vtinput_vi_consts = { .vc_name = "vtinput", .vc_nvq = VTINPUT_MAXQ, .vc_cfgsize = sizeof(struct vtinput_config), .vc_reset = pci_vtinput_reset, .vc_cfgread = pci_vtinput_cfgread, .vc_cfgwrite = pci_vtinput_cfgwrite, .vc_hv_caps_legacy = 0, .vc_hv_caps_modern = 0, }; static void pci_vtinput_reset(void *vsc) { struct pci_vtinput_softc *sc = vsc; DPRINTF(("%s: device reset requested", __func__)); vi_reset_dev(&sc->vsc_vs); } static void pci_vtinput_notify_eventq(void *vsc __unused, struct vqueue_info *vq __unused) { DPRINTF(("%s", __func__)); } static void pci_vtinput_notify_statusq(void *vsc, struct vqueue_info *vq) { struct pci_vtinput_softc *sc = vsc; while (vq_has_descs(vq)) { /* get descriptor chain */ struct iovec iov; struct vi_req req; const int n = vq_getchain(vq, &iov, 1, &req); if (n <= 0) { WPRINTF(("%s: invalid descriptor: %d", __func__, n)); return; } /* get event */ struct vtinput_event event; memcpy(&event, iov.iov_base, sizeof(event)); /* * on multi touch devices: * - host send EV_MSC to guest * - guest sends EV_MSC back to host * - host writes EV_MSC to evdev * - evdev saves EV_MSC in it's event buffer * - host receives an extra EV_MSC by reading the evdev event * buffer * - frames become larger and larger * avoid endless loops by ignoring EV_MSC */ if (event.type == EV_MSC) { vq_relchain(vq, req.idx, sizeof(event)); continue; } /* send event to evdev */ struct input_event host_event; host_event.type = event.type; host_event.code = event.code; host_event.value = event.value; if (gettimeofday(&host_event.time, NULL) != 0) { WPRINTF(("%s: failed gettimeofday", __func__)); } if (write(sc->vsc_fd, &host_event, sizeof(host_event)) == -1) { WPRINTF(("%s: failed to write host_event", __func__)); } vq_relchain(vq, req.idx, sizeof(event)); } vq_endchains(vq, 1); } static int pci_vtinput_get_bitmap(struct pci_vtinput_softc *sc, int cmd, int count) { if (count <= 0 || !sc) { return (-1); } /* query bitmap */ memset(sc->vsc_config.u.bitmap, 0, sizeof(sc->vsc_config.u.bitmap)); if (ioctl(sc->vsc_fd, cmd, sc->vsc_config.u.bitmap) < 0) { return (-1); } /* get number of set bytes in bitmap */ for (int i = count - 1; i >= 0; i--) { if (sc->vsc_config.u.bitmap[i]) { return i + 1; } } return (-1); } static int pci_vtinput_read_config_id_name(struct pci_vtinput_softc *sc) { char name[128]; if (ioctl(sc->vsc_fd, EVIOCGNAME(sizeof(name) - 1), name) < 0) { return (1); } memcpy(sc->vsc_config.u.string, name, sizeof(name)); sc->vsc_config.size = strnlen(name, sizeof(name)); return (0); } static int pci_vtinput_read_config_id_serial(struct pci_vtinput_softc *sc) { /* serial isn't supported */ sc->vsc_config.size = 0; return (0); } static int pci_vtinput_read_config_id_devids(struct pci_vtinput_softc *sc) { struct input_id devids; if (ioctl(sc->vsc_fd, EVIOCGID, &devids)) { return (1); } sc->vsc_config.u.ids.bustype = devids.bustype; sc->vsc_config.u.ids.vendor = devids.vendor; sc->vsc_config.u.ids.product = devids.product; sc->vsc_config.u.ids.version = devids.version; sc->vsc_config.size = sizeof(struct vtinput_devids); return (0); } static int pci_vtinput_read_config_prop_bits(struct pci_vtinput_softc *sc) { /* * Evdev bitmap countains 1 bit per count. Additionally evdev bitmaps * are arrays of longs instead of chars. Calculate how many longs are * required for evdev bitmap. Multiply that with sizeof(long) to get the * number of elements. */ const int count = howmany(INPUT_PROP_CNT, sizeof(long) * 8) * sizeof(long); const unsigned int cmd = EVIOCGPROP(count); const int size = pci_vtinput_get_bitmap(sc, cmd, count); if (size <= 0) { return (1); } sc->vsc_config.size = size; return (0); } static int pci_vtinput_read_config_ev_bits(struct pci_vtinput_softc *sc, uint8_t type) { int count; switch (type) { case EV_KEY: count = KEY_CNT; break; case EV_REL: count = REL_CNT; break; case EV_ABS: count = ABS_CNT; break; case EV_MSC: count = MSC_CNT; break; case EV_SW: count = SW_CNT; break; case EV_LED: count = LED_CNT; break; default: return (1); } /* * Evdev bitmap countains 1 bit per count. Additionally evdev bitmaps * are arrays of longs instead of chars. Calculate how many longs are * required for evdev bitmap. Multiply that with sizeof(long) to get the * number of elements. */ count = howmany(count, sizeof(long) * 8) * sizeof(long); const unsigned int cmd = EVIOCGBIT(sc->vsc_config.subsel, count); const int size = pci_vtinput_get_bitmap(sc, cmd, count); if (size <= 0) { return (1); } sc->vsc_config.size = size; return (0); } static int pci_vtinput_read_config_abs_info(struct pci_vtinput_softc *sc) { /* check if evdev has EV_ABS */ if (!pci_vtinput_read_config_ev_bits(sc, EV_ABS)) { return (1); } /* get abs information */ struct input_absinfo abs; if (ioctl(sc->vsc_fd, EVIOCGABS(sc->vsc_config.subsel), &abs) < 0) { return (1); } /* save abs information */ sc->vsc_config.u.abs.min = abs.minimum; sc->vsc_config.u.abs.max = abs.maximum; sc->vsc_config.u.abs.fuzz = abs.fuzz; sc->vsc_config.u.abs.flat = abs.flat; sc->vsc_config.u.abs.res = abs.resolution; sc->vsc_config.size = sizeof(struct vtinput_absinfo); return (0); } static int pci_vtinput_read_config(struct pci_vtinput_softc *sc) { switch (sc->vsc_config.select) { case VTINPUT_CFG_UNSET: return (0); case VTINPUT_CFG_ID_NAME: return pci_vtinput_read_config_id_name(sc); case VTINPUT_CFG_ID_SERIAL: return pci_vtinput_read_config_id_serial(sc); case VTINPUT_CFG_ID_DEVIDS: return pci_vtinput_read_config_id_devids(sc); case VTINPUT_CFG_PROP_BITS: return pci_vtinput_read_config_prop_bits(sc); case VTINPUT_CFG_EV_BITS: return pci_vtinput_read_config_ev_bits( sc, sc->vsc_config.subsel); case VTINPUT_CFG_ABS_INFO: return pci_vtinput_read_config_abs_info(sc); default: return (1); } } static int pci_vtinput_cfgread(void *vsc, int offset, int size, uint32_t *retval) { struct pci_vtinput_softc *sc = vsc; /* check for valid offset and size */ if (offset + size > (int)sizeof(struct vtinput_config)) { WPRINTF(("%s: read to invalid offset/size %d/%d", __func__, offset, size)); memset(retval, 0, size); return (0); } /* read new config values, if select and subsel changed. */ if (!sc->vsc_config_valid) { if (pci_vtinput_read_config(sc) != 0) { DPRINTF(("%s: could not read config %d/%d", __func__, sc->vsc_config.select, sc->vsc_config.subsel)); memset(retval, 0, size); return (0); } sc->vsc_config_valid = 1; } uint8_t *ptr = (uint8_t *)&sc->vsc_config; memcpy(retval, ptr + offset, size); return (0); } static int pci_vtinput_cfgwrite(void *vsc, int offset, int size, uint32_t value) { struct pci_vtinput_softc *sc = vsc; /* guest can only write to select and subsel fields */ if (offset + size > 2) { WPRINTF(("%s: write to readonly reg %d", __func__, offset)); return (1); } /* copy value into config */ uint8_t *ptr = (uint8_t *)&sc->vsc_config; memcpy(ptr + offset, &value, size); /* select/subsel changed, query new config on next cfgread */ sc->vsc_config_valid = 0; /* notify the guest the device configuration has been changed */ vq_devcfg_changed(&sc->vsc_vs); return (0); } static int vtinput_eventqueue_add_event( struct vtinput_eventqueue *queue, struct input_event *e) { /* check if queue is full */ if (queue->idx >= queue->size) { /* alloc new elements for queue */ const uint32_t newSize = queue->idx; void *newPtr = realloc(queue->events, queue->size * sizeof(struct vtinput_event_elem)); if (newPtr == NULL) { WPRINTF(("%s: realloc memory for eventqueue failed!", __func__)); return (1); } queue->events = newPtr; queue->size = newSize; } /* save event */ struct vtinput_event *event = &queue->events[queue->idx].event; event->type = e->type; event->code = e->code; event->value = e->value; queue->idx++; return (0); } static void vtinput_eventqueue_clear(struct vtinput_eventqueue *queue) { /* just reset index to clear queue */ queue->idx = 0; } static void vtinput_eventqueue_send_events( struct vtinput_eventqueue *queue, struct vqueue_info *vq) { /* * First iteration through eventqueue: * Get descriptor chains. */ for (uint32_t i = 0; i < queue->idx; ++i) { /* get descriptor */ if (!vq_has_descs(vq)) { /* * We don't have enough descriptors for all events. * Return chains back to guest. */ vq_retchains(vq, i); WPRINTF(( "%s: not enough available descriptors, dropping %d events", __func__, queue->idx)); goto done; } /* get descriptor chain */ struct iovec iov; struct vi_req req; const int n = vq_getchain(vq, &iov, 1, &req); if (n <= 0) { WPRINTF(("%s: invalid descriptor: %d", __func__, n)); return; } if (n != 1) { WPRINTF( ("%s: invalid number of descriptors in chain: %d", __func__, n)); /* release invalid chain */ vq_relchain(vq, req.idx, 0); return; } if (iov.iov_len < sizeof(struct vtinput_event)) { WPRINTF(("%s: invalid descriptor length: %lu", __func__, iov.iov_len)); /* release invalid chain */ vq_relchain(vq, req.idx, 0); return; } /* save descriptor */ queue->events[i].iov = iov; queue->events[i].idx = req.idx; } /* * Second iteration through eventqueue: * Send events to guest by releasing chains */ for (uint32_t i = 0; i < queue->idx; ++i) { struct vtinput_event_elem event = queue->events[i]; memcpy(event.iov.iov_base, &event.event, sizeof(struct vtinput_event)); vq_relchain(vq, event.idx, sizeof(struct vtinput_event)); } done: /* clear queue and send interrupt to guest */ vtinput_eventqueue_clear(queue); vq_endchains(vq, 1); } static int vtinput_read_event_from_host(int fd, struct input_event *event) { const int len = read(fd, event, sizeof(struct input_event)); if (len != sizeof(struct input_event)) { if (len == -1 && errno != EAGAIN) { WPRINTF(("%s: event read failed! len = %d, errno = %d", __func__, len, errno)); } /* host doesn't have more events for us */ return (1); } return (0); } static void vtinput_read_event(int fd __attribute((unused)), enum ev_type t __attribute__((unused)), void *arg __attribute__((unused))) { struct pci_vtinput_softc *sc = arg; /* skip if driver isn't ready */ if (!(sc->vsc_vs.vs_status & VIRTIO_CONFIG_STATUS_DRIVER_OK)) return; /* read all events from host */ struct input_event event; while (vtinput_read_event_from_host(sc->vsc_fd, &event) == 0) { /* add events to our queue */ vtinput_eventqueue_add_event(&sc->vsc_eventqueue, &event); /* only send events to guest on EV_SYN or SYN_REPORT */ if (event.type != EV_SYN || event.type != SYN_REPORT) { continue; } /* send host events to guest */ vtinput_eventqueue_send_events( &sc->vsc_eventqueue, &sc->vsc_queues[VTINPUT_EVENTQ]); } } static int pci_vtinput_legacy_config(nvlist_t *nvl, const char *opts) { if (opts == NULL) return (-1); /* * parse opts: * virtio-input,/dev/input/eventX */ char *cp = strchr(opts, ','); if (cp == NULL) { set_config_value_node(nvl, "path", opts); return (0); } char *path = strndup(opts, cp - opts); set_config_value_node(nvl, "path", path); free(path); return (pci_parse_legacy_config(nvl, cp + 1)); } static int pci_vtinput_init(struct pci_devinst *pi, nvlist_t *nvl) { struct pci_vtinput_softc *sc; /* * Keep it here. * Else it's possible to access it uninitialized by jumping to failed. */ pthread_mutexattr_t mtx_attr = NULL; sc = calloc(1, sizeof(struct pci_vtinput_softc)); sc->vsc_evdev = get_config_value_node(nvl, "path"); if (sc->vsc_evdev == NULL) { WPRINTF(("%s: missing required path config value", __func__)); goto failed; } /* * open evdev by using non blocking I/O: * read from /dev/input/eventX would block our thread otherwise */ sc->vsc_fd = open(sc->vsc_evdev, O_RDWR | O_NONBLOCK); if (sc->vsc_fd < 0) { WPRINTF(("%s: failed to open %s", __func__, sc->vsc_evdev)); goto failed; } /* check if evdev is really a evdev */ int evversion; int error = ioctl(sc->vsc_fd, EVIOCGVERSION, &evversion); if (error < 0) { WPRINTF(("%s: %s is no evdev", __func__, sc->vsc_evdev)); goto failed; } /* gain exclusive access to evdev */ error = ioctl(sc->vsc_fd, EVIOCGRAB, 1); if (error < 0) { WPRINTF(("%s: failed to grab %s", __func__, sc->vsc_evdev)); goto failed; } if (pthread_mutexattr_init(&mtx_attr)) { WPRINTF(("%s: init mutexattr failed", __func__)); goto failed; } if (pthread_mutexattr_settype(&mtx_attr, PTHREAD_MUTEX_RECURSIVE)) { WPRINTF(("%s: settype mutexattr failed", __func__)); goto failed; } if (pthread_mutex_init(&sc->vsc_mtx, &mtx_attr)) { WPRINTF(("%s: init mutex failed", __func__)); goto failed; } /* init softc */ sc->vsc_eventqueue.idx = 0; sc->vsc_eventqueue.size = VTINPUT_MAX_PKT_LEN; sc->vsc_eventqueue.events = calloc( sc->vsc_eventqueue.size, sizeof(struct vtinput_event_elem)); sc->vsc_config_valid = 0; if (sc->vsc_eventqueue.events == NULL) { WPRINTF(("%s: failed to alloc eventqueue", __func__)); goto failed; } /* register event handler */ sc->vsc_evp = mevent_add(sc->vsc_fd, EVF_READ, vtinput_read_event, sc); if (sc->vsc_evp == NULL) { WPRINTF(("%s: could not register mevent", __func__)); goto failed; } #ifndef WITHOUT_CAPSICUM cap_rights_t rights; cap_rights_init(&rights, CAP_EVENT, CAP_IOCTL, CAP_READ, CAP_WRITE); if (caph_rights_limit(sc->vsc_fd, &rights) == -1) { errx(EX_OSERR, "Unable to apply rights for sandbox"); } #endif /* link virtio to softc */ vi_softc_linkup( &sc->vsc_vs, &vtinput_vi_consts, sc, pi, sc->vsc_queues); sc->vsc_vs.vs_mtx = &sc->vsc_mtx; /* init virtio queues */ sc->vsc_queues[VTINPUT_EVENTQ].vq_qsize = VTINPUT_RINGSZ; sc->vsc_queues[VTINPUT_EVENTQ].vq_notify = pci_vtinput_notify_eventq; sc->vsc_queues[VTINPUT_STATUSQ].vq_qsize = VTINPUT_RINGSZ; sc->vsc_queues[VTINPUT_STATUSQ].vq_notify = pci_vtinput_notify_statusq; /* initialize config space */ vi_pci_init(pi, VIRTIO_MODE_TRANSITIONAL, VIRTIO_DEV_INPUT, VIRTIO_ID_INPUT, PCIC_INPUTDEV); pci_set_cfgdata8(pi, PCIR_SUBCLASS, PCIS_INPUTDEV_OTHER); pci_set_cfgdata8(pi, PCIR_REVID, VIRTIO_REV_INPUT); pci_set_cfgdata16(pi, PCIR_SUBDEV_0, VIRTIO_SUBDEV_INPUT); pci_set_cfgdata16(pi, PCIR_SUBVEND_0, VIRTIO_SUBVEN_INPUT); /* add MSI-X table BAR */ if (!vi_intr_init(&sc->vsc_vs, fbsdrun_virtio_msix())) goto failed; /* add VirtIO BARs */ if (!vi_pcibar_setup(&sc->vsc_vs)) goto failed; return (0); failed: if (sc == NULL) { return (-1); } if (sc->vsc_evp) mevent_delete(sc->vsc_evp); if (sc->vsc_eventqueue.events) free(sc->vsc_eventqueue.events); if (sc->vsc_mtx) pthread_mutex_destroy(&sc->vsc_mtx); if (mtx_attr) pthread_mutexattr_destroy(&mtx_attr); if (sc->vsc_fd) close(sc->vsc_fd); free(sc); return (-1); } static const struct pci_devemu pci_de_vinput = { .pe_emu = "virtio-input", .pe_init = pci_vtinput_init, .pe_legacy_config = pci_vtinput_legacy_config, .pe_barwrite = vi_pci_write, .pe_barread = vi_pci_read, }; PCI_EMUL_SET(pci_de_vinput); /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2011 NetApp, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * This file and its contents are supplied under the terms of the * Common Development and Distribution License ("CDDL"), version 1.0. * You may only use this file in accordance with the terms of version * 1.0 of the CDDL. * * A full copy of the text of the CDDL should have accompanied this * source. A copy of the CDDL is also available via the Internet at * http://www.illumos.org/license/CDDL. */ /* This file is dual-licensed; see usr/src/contrib/bhyve/LICENSE */ /* * Copyright 2026 Oxide Computer Company */ #include #include #include #include #include #include #include /* IFNAMSIZ */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include "bhyverun.h" #include "config.h" #include "debug.h" #include "pci_emul.h" #include "mevent.h" #include "virtio.h" #include "net_utils.h" #include "net_backends.h" #include "iov.h" #define VTNET_RINGSZ 1024 #define VTNET_MAXSEGS 256 #define VTNET_MAX_PKT_LEN (65536 + 64) #define VTNET_MIN_MTU ETHERMIN #define VTNET_MAX_MTU 65535 #define VTNET_S_HOSTCAPS_MODERN \ (VIRTIO_NET_F_MAC | VIRTIO_NET_F_STATUS | \ VIRTIO_RING_F_INDIRECT_DESC | VIRTIO_NET_F_MRG_RXBUF) #define VTNET_S_HOSTCAPS_LEGACY \ (VTNET_S_HOSTCAPS_MODERN | VIRTIO_F_NOTIFY_ON_EMPTY) /* * Queue definitions. */ #define VTNET_RXQ 0 #define VTNET_TXQ 1 #define VTNET_CTLQ 2 /* NB: not yet supported */ #define VTNET_MAXQ 3 /* * Debug printf */ static int pci_vtnet_debug; #define DPRINTF(params) if (pci_vtnet_debug) PRINTLN params #define WPRINTF(params) PRINTLN params /* * Per-device softc */ struct pci_vtnet_softc { struct virtio_softc vsc_vs; struct vqueue_info vsc_queues[VTNET_MAXQ - 1]; pthread_mutex_t vsc_mtx; net_backend_t *vsc_be; bool features_negotiated; /* protected by rx_mtx */ int resetting; /* protected by tx_mtx */ uint64_t vsc_features; /* negotiated features */ pthread_mutex_t rx_mtx; int rx_merge; /* merged rx bufs in use */ pthread_t tx_tid; pthread_mutex_t tx_mtx; pthread_cond_t tx_cond; int tx_in_progress; size_t vhdrlen; size_t be_vhdrlen; struct virtio_net_config vsc_config; struct virtio_consts vsc_consts; }; static void pci_vtnet_reset(void *); /* static void pci_vtnet_notify(void *, struct vqueue_info *); */ static int pci_vtnet_cfgread(void *, int, int, uint32_t *); static int pci_vtnet_cfgwrite(void *, int, int, uint32_t); static void pci_vtnet_neg_features(void *, uint64_t *); static virtio_capstr_t vtnet_caps[] = { { VIRTIO_NET_F_CSUM, "VIRTIO_NET_F_CSUM" }, { VIRTIO_NET_F_GUEST_CSUM, "VIRTIO_NET_F_GUEST_CSUM" }, { VIRTIO_NET_F_MTU, "VIRTIO_NET_F_MTU" }, { VIRTIO_NET_F_MAC, "VIRTIO_NET_F_MAC" }, { VIRTIO_NET_F_GSO_DEPREC, "VIRTIO_NET_F_GSO_DEPREC" }, { VIRTIO_NET_F_GUEST_TSO4, "VIRTIO_NET_F_GUEST_TSO4" }, { VIRTIO_NET_F_GUEST_TSO6, "VIRTIO_NET_F_GUEST_TSO6" }, { VIRTIO_NET_F_GUEST_ECN, "VIRTIO_NET_F_GUEST_ECN" }, { VIRTIO_NET_F_GUEST_UFO, "VIRTIO_NET_F_GUEST_UFO" }, { VIRTIO_NET_F_HOST_TSO4, "VIRTIO_NET_F_HOST_TSO4" }, { VIRTIO_NET_F_HOST_TSO6, "VIRTIO_NET_F_HOST_TSO6" }, { VIRTIO_NET_F_HOST_ECN, "VIRTIO_NET_F_HOST_ECN" }, { VIRTIO_NET_F_HOST_UFO, "VIRTIO_NET_F_HOST_UFO" }, { VIRTIO_NET_F_MRG_RXBUF, "VIRTIO_NET_F_MRG_RXBUF" }, { VIRTIO_NET_F_STATUS, "VIRTIO_NET_F_STATUS" }, { VIRTIO_NET_F_CTRL_VQ, "VIRTIO_NET_F_CTRL_VQ" }, { VIRTIO_NET_F_CTRL_RX, "VIRTIO_NET_F_CTRL_RX" }, { VIRTIO_NET_F_CTRL_VLAN, "VIRTIO_NET_F_CTRL_VLAN" }, { VIRTIO_NET_F_GUEST_ANNOUNCE, "VIRTIO_NET_F_GUEST_ANNOUNCE" }, { VIRTIO_NET_F_MQ, "VIRTIO_NET_F_MQ" }, }; static struct virtio_consts vtnet_vi_consts = { .vc_name = "vtnet", .vc_nvq = VTNET_MAXQ - 1, .vc_cfgsize = sizeof (struct virtio_net_config), .vc_reset = pci_vtnet_reset, .vc_cfgread = pci_vtnet_cfgread, .vc_cfgwrite = pci_vtnet_cfgwrite, .vc_apply_features = pci_vtnet_neg_features, .vc_hv_caps_legacy = VTNET_S_HOSTCAPS_LEGACY, .vc_hv_caps_modern = VTNET_S_HOSTCAPS_MODERN, .vc_capstr = vtnet_caps, .vc_ncapstr = ARRAY_SIZE(vtnet_caps), }; static void pci_vtnet_reset(void *vsc) { struct pci_vtnet_softc *sc = vsc; DPRINTF(("vtnet: device reset requested !")); /* Acquire the RX lock to block RX processing. */ pthread_mutex_lock(&sc->rx_mtx); /* * Make sure receive operation is disabled at least until we * re-negotiate the features, since receive operation depends * on the value of sc->rx_merge and the header length, which * are both set in pci_vtnet_neg_features(). * Receive operation will be enabled again once the guest adds * the first receive buffers and kicks us. */ sc->features_negotiated = false; netbe_rx_disable(sc->vsc_be); /* Set sc->resetting and give a chance to the TX thread to stop. */ pthread_mutex_lock(&sc->tx_mtx); sc->resetting = 1; while (sc->tx_in_progress) { pthread_mutex_unlock(&sc->tx_mtx); usleep(10000); pthread_mutex_lock(&sc->tx_mtx); } /* * Now reset rings, MSI-X vectors, and negotiated capabilities. * Do that with the TX lock held, since we need to reset * sc->resetting. */ vi_reset_dev(&sc->vsc_vs); sc->resetting = 0; pthread_mutex_unlock(&sc->tx_mtx); pthread_mutex_unlock(&sc->rx_mtx); } static __inline struct iovec * iov_trim_hdr(struct iovec *iov, int *iovcnt, unsigned int hlen) { struct iovec *riov; if (iov[0].iov_len < hlen) { /* * Not enough header space in the first fragment. * That's not ok for us. */ return (NULL); } iov[0].iov_len -= hlen; if (iov[0].iov_len == 0) { *iovcnt -= 1; if (*iovcnt == 0) { /* * Only space for the header. That's not * enough for us. */ return (NULL); } riov = &iov[1]; } else { iov[0].iov_base = (void *)((uintptr_t)iov[0].iov_base + hlen); riov = &iov[0]; } return (riov); } struct virtio_mrg_rxbuf_info { uint16_t idx; uint16_t pad; uint32_t len; }; static void pci_vtnet_rx(struct pci_vtnet_softc *sc) { int prepend_hdr_len = sc->vhdrlen - sc->be_vhdrlen; struct virtio_mrg_rxbuf_info info[VTNET_MAXSEGS]; struct iovec iov[VTNET_MAXSEGS + 1]; struct vqueue_info *vq; struct vi_req req; vq = &sc->vsc_queues[VTNET_RXQ]; /* Features must be negotiated */ if (!sc->features_negotiated) { return; } for (;;) { struct virtio_net_rxhdr *hdr; uint32_t riov_bytes; struct iovec *riov; uint32_t ulen; int riov_len; int n_chains; ssize_t rlen; ssize_t plen; plen = netbe_peek_recvlen(sc->vsc_be); if (plen <= 0) { /* * No more packets (plen == 0), or backend errored * (plen < 0). Interrupt if needed and stop. */ vq_endchains(vq, /* used_all_avail= */0); return; } plen += prepend_hdr_len; /* * Get a descriptor chain to store the next ingress * packet. In case of mergeable rx buffers, get as * many chains as necessary in order to make room * for plen bytes. */ riov_bytes = 0; riov_len = 0; riov = iov; n_chains = 0; do { int n = vq_getchain(vq, riov, VTNET_MAXSEGS - riov_len, &req); info[n_chains].idx = req.idx; if (n == 0) { /* * No rx buffers. Enable RX kicks and double * check. */ vq_kick_enable(vq); if (!vq_has_descs(vq)) { /* * Still no buffers. Return the unused * chains (if any), interrupt if needed * (including for NOTIFY_ON_EMPTY), and * disable the backend until the next * kick. */ vq_retchains(vq, n_chains); vq_endchains(vq, /* used_all_avail */1); netbe_rx_disable(sc->vsc_be); return; } /* More rx buffers found, so keep going. */ vq_kick_disable(vq); continue; } #ifndef __FreeBSD__ if (n == -1) { /* * An error from vq_getchain() means that * an invalid descriptor was found. */ vq_retchains(vq, n_chains); vq_endchains(vq, /* used_all_avail= */0); return; } #endif assert(n >= 1 && riov_len + n <= VTNET_MAXSEGS); riov_len += n; if (!sc->rx_merge) { n_chains = 1; break; } #ifndef __FreeBSD__ size_t c = count_iov(riov, n); if (c > UINT32_MAX) { vq_retchains(vq, n_chains); vq_endchains(vq, /* used_all_avail= */0); return; } info[n_chains].len = (uint32_t)c; #else info[n_chains].len = (uint32_t)count_iov(riov, n); #endif riov_bytes += info[n_chains].len; riov += n; n_chains++; } while (riov_bytes < plen && riov_len < VTNET_MAXSEGS); riov = iov; #ifdef __FreeBSD__ hdr = riov[0].iov_base; #else hdr = (struct virtio_net_rxhdr *)riov[0].iov_base; #endif if (prepend_hdr_len > 0) { /* * The frontend uses a virtio-net header, but the * backend does not. We need to prepend a zeroed * header. */ riov = iov_trim_hdr(riov, &riov_len, prepend_hdr_len); if (riov == NULL) { /* * The first collected chain is nonsensical, * as it is not even enough to store the * virtio-net header. Just drop it. */ vq_relchain(vq, info[0].idx, 0); vq_retchains(vq, n_chains - 1); continue; } memset(hdr, 0, prepend_hdr_len); } rlen = netbe_recv(sc->vsc_be, riov, riov_len); if (rlen != plen - prepend_hdr_len) { /* * If this happens it means there is something * wrong with the backend (e.g., some other * process is stealing our packets). */ WPRINTF(( "netbe_recv: expected %zd bytes, got %zd", plen - prepend_hdr_len, rlen)); vq_retchains(vq, n_chains); continue; } ulen = (uint32_t)plen; /* * Publish the used buffers to the guest, reporting the * number of bytes that we wrote. */ if (!sc->rx_merge) { vq_relchain(vq, info[0].idx, ulen); } else { uint32_t iolen; int i = 0; do { iolen = info[i].len; if (iolen > ulen) { iolen = ulen; } vq_relchain_prepare(vq, info[i].idx, iolen); ulen -= iolen; i++; } while (ulen > 0); hdr->vrh_bufs = i; vq_relchain_publish(vq); assert(i == n_chains); } } } /* * Called when there is read activity on the backend file descriptor. * Each buffer posted by the guest is assumed to be able to contain * an entire ethernet frame + rx header. */ static void pci_vtnet_rx_callback(int fd __unused, enum ev_type type __unused, void *param) { struct pci_vtnet_softc *sc = param; pthread_mutex_lock(&sc->rx_mtx); pci_vtnet_rx(sc); pthread_mutex_unlock(&sc->rx_mtx); } /* Called on RX kick. */ static void pci_vtnet_ping_rxq(void *vsc, struct vqueue_info *vq) { struct pci_vtnet_softc *sc = vsc; /* * A qnotify means that the rx process can now begin. * Enable RX only if features are negotiated. */ pthread_mutex_lock(&sc->rx_mtx); if (!sc->features_negotiated) { pthread_mutex_unlock(&sc->rx_mtx); return; } vq_kick_disable(vq); netbe_rx_enable(sc->vsc_be); pthread_mutex_unlock(&sc->rx_mtx); } /* TX virtqueue processing, called by the TX thread. */ static void pci_vtnet_proctx(struct pci_vtnet_softc *sc, struct vqueue_info *vq) { struct iovec iov[VTNET_MAXSEGS + 1]; struct iovec *siov = iov; struct vi_req req; ssize_t len; int n; /* * Obtain chain of descriptors. The first descriptor also * contains the virtio-net header. */ n = vq_getchain(vq, iov, VTNET_MAXSEGS, &req); assert(n >= 1 && n <= VTNET_MAXSEGS); if (sc->vhdrlen != sc->be_vhdrlen) { /* * The frontend uses a virtio-net header, but the backend * does not. We simply strip the header and ignore it, as * it should be zero-filled. */ siov = iov_trim_hdr(siov, &n, sc->vhdrlen); } if (siov == NULL) { /* The chain is nonsensical. Just drop it. */ len = 0; } else { len = netbe_send(sc->vsc_be, siov, n); if (len < 0) { /* * If send failed, report that 0 bytes * were read. */ len = 0; } } /* * Return the processed chain to the guest, reporting * the number of bytes that we read. */ vq_relchain(vq, req.idx, len); } /* Called on TX kick. */ static void pci_vtnet_ping_txq(void *vsc, struct vqueue_info *vq) { struct pci_vtnet_softc *sc = vsc; /* * Any ring entries to process? */ if (!vq_has_descs(vq)) return; /* Signal the tx thread for processing */ pthread_mutex_lock(&sc->tx_mtx); vq_kick_disable(vq); if (sc->tx_in_progress == 0) pthread_cond_signal(&sc->tx_cond); pthread_mutex_unlock(&sc->tx_mtx); } /* * Thread which will handle processing of TX desc */ static void * pci_vtnet_tx_thread(void *param) { struct pci_vtnet_softc *sc = param; struct vqueue_info *vq; int error; vq = &sc->vsc_queues[VTNET_TXQ]; /* * Let us wait till the tx queue pointers get initialised & * first tx signaled */ pthread_mutex_lock(&sc->tx_mtx); error = pthread_cond_wait(&sc->tx_cond, &sc->tx_mtx); assert(error == 0); for (;;) { /* note - tx mutex is locked here */ while (sc->resetting || !vq_has_descs(vq)) { vq_kick_enable(vq); if (!sc->resetting && vq_has_descs(vq)) break; sc->tx_in_progress = 0; error = pthread_cond_wait(&sc->tx_cond, &sc->tx_mtx); assert(error == 0); } vq_kick_disable(vq); sc->tx_in_progress = 1; pthread_mutex_unlock(&sc->tx_mtx); do { /* * Run through entries, placing them into * iovecs and sending when an end-of-packet * is found */ pci_vtnet_proctx(sc, vq); } while (vq_has_descs(vq)); /* * Generate an interrupt if needed. */ vq_endchains(vq, /* used_all_avail= */1); pthread_mutex_lock(&sc->tx_mtx); } #ifndef __FreeBSD__ return (NULL); #endif } #ifdef notyet static void pci_vtnet_ping_ctlq(void *vsc, struct vqueue_info *vq) { DPRINTF(("vtnet: control qnotify!")); } #endif static int pci_vtnet_free_softstate(struct pci_vtnet_softc *sc, int ret) { pthread_mutex_destroy(&sc->vsc_mtx); if (sc->vsc_be != NULL) netbe_cleanup(sc->vsc_be); free(sc); return (ret); } static int pci_vtnet_init(struct pci_devinst *pi, nvlist_t *nvl) { struct pci_vtnet_softc *sc; const char *value; char tname[MAXCOMLEN + 1]; unsigned long mtu = ETHERMTU; int err; /* * Allocate data structures for further virtio initializations. * sc also contains a copy of vtnet_vi_consts, since capabilities * change depending on the backend. */ sc = calloc(1, sizeof (struct pci_vtnet_softc)); if (sc == NULL) return (errno); if (get_config_bool_default("virtio.net.debug", false)) pci_vtnet_debug = 1; vi_set_debug(&sc->vsc_vs, pci_vtnet_debug); sc->vsc_consts = vtnet_vi_consts; pthread_mutex_init(&sc->vsc_mtx, NULL); sc->vsc_queues[VTNET_RXQ].vq_qsize = VTNET_RINGSZ; sc->vsc_queues[VTNET_RXQ].vq_notify = pci_vtnet_ping_rxq; sc->vsc_queues[VTNET_TXQ].vq_qsize = VTNET_RINGSZ; sc->vsc_queues[VTNET_TXQ].vq_notify = pci_vtnet_ping_txq; #ifdef notyet sc->vsc_queues[VTNET_CTLQ].vq_qsize = VTNET_RINGSZ; sc->vsc_queues[VTNET_CTLQ].vq_notify = pci_vtnet_ping_ctlq; #endif value = get_config_value_node(nvl, "mac"); if (value != NULL) { err = net_parsemac(value, sc->vsc_config.vnc_macaddr); if (err != 0) return (pci_vtnet_free_softstate(sc, err)); } else net_genmac(pi, sc->vsc_config.vnc_macaddr); value = get_config_value_node(nvl, "mtu"); if (value != NULL) { err = net_parsemtu(value, &mtu); if (err != 0) return (pci_vtnet_free_softstate(sc, err)); if (mtu < VTNET_MIN_MTU || mtu > VTNET_MAX_MTU) { errno = EINVAL; return (pci_vtnet_free_softstate(sc, errno)); } sc->vsc_consts.vc_hv_caps_legacy |= VIRTIO_NET_F_MTU; sc->vsc_consts.vc_hv_caps_modern |= VIRTIO_NET_F_MTU; } sc->vsc_config.vnc_mtu = mtu; /* Permit interfaces without a configured backend. */ if (get_config_value_node(nvl, "backend") != NULL) { err = netbe_init(&sc->vsc_be, nvl, pci_vtnet_rx_callback, sc); if (err != 0) return (pci_vtnet_free_softstate(sc, err)); #ifndef __FreeBSD__ size_t buflen = sizeof (sc->vsc_config.vnc_macaddr); err = netbe_get_mac(sc->vsc_be, sc->vsc_config.vnc_macaddr, &buflen); if (err != 0) return (pci_vtnet_free_softstate(sc, err)); #endif } sc->vsc_consts.vc_hv_caps_legacy |= netbe_get_cap(sc->vsc_be); sc->vsc_consts.vc_hv_caps_modern |= netbe_get_cap(sc->vsc_be); /* * Since we do not actually support multiqueue, * set the maximum virtqueue pairs to 1. */ sc->vsc_config.vnc_max_qpair = 1; vi_softc_linkup(&sc->vsc_vs, &sc->vsc_consts, sc, pi, sc->vsc_queues); sc->vsc_vs.vs_mtx = &sc->vsc_mtx; /* initialize config space */ vi_pci_init(pi, VIRTIO_MODE_TRANSITIONAL, VIRTIO_DEV_NET, VIRTIO_ID_NETWORK, PCIC_NETWORK); /* Link is always up. */ sc->vsc_config.vnc_status = VIRTIO_NET_S_LINK_UP; /* use BAR 1 to map MSI-X table and PBA, if we're using MSI-X */ if (!vi_intr_init(&sc->vsc_vs, fbsdrun_virtio_msix())) return (pci_vtnet_free_softstate(sc, EIO)); if (!vi_pcibar_setup(&sc->vsc_vs)) return (pci_vtnet_free_softstate(sc, EIO)); sc->resetting = 0; sc->rx_merge = 0; sc->vhdrlen = sizeof (struct virtio_net_rxhdr) - 2; pthread_mutex_init(&sc->rx_mtx, NULL); /* * Initialize tx semaphore & spawn TX processing thread. * As of now, only one thread for TX desc processing is * spawned. */ sc->tx_in_progress = 0; pthread_mutex_init(&sc->tx_mtx, NULL); pthread_cond_init(&sc->tx_cond, NULL); pthread_create(&sc->tx_tid, NULL, pci_vtnet_tx_thread, (void *)sc); snprintf(tname, sizeof (tname), "vtnet-%d:%d tx", pi->pi_slot, pi->pi_func); pthread_set_name_np(sc->tx_tid, tname); return (0); } static int pci_vtnet_cfgwrite(void *vsc, int offset, int size, uint32_t value) { struct pci_vtnet_softc *sc = vsc; void *ptr; if (offset < (int)sizeof (sc->vsc_config.vnc_macaddr)) { assert(offset + size <= (int)sizeof (sc->vsc_config.vnc_macaddr)); /* * The driver is allowed to change the MAC address */ ptr = &sc->vsc_config.vnc_macaddr[offset]; memcpy(ptr, &value, size); vq_devcfg_changed(&sc->vsc_vs); } else { /* silently ignore other writes */ DPRINTF(("vtnet: write to readonly reg %d", offset)); } return (0); } static int pci_vtnet_cfgread(void *vsc, int offset, int size, uint32_t *retval) { struct pci_vtnet_softc *sc = vsc; void *ptr; ptr = (uint8_t *)&sc->vsc_config + offset; memcpy(retval, ptr, size); return (0); } static void pci_vtnet_neg_features(void *vsc, uint64_t *negotiated_features) { struct pci_vtnet_softc *sc = vsc; sc->vsc_features = *negotiated_features; if ((*negotiated_features & VIRTIO_NET_F_MRG_RXBUF) != 0) { sc->vhdrlen = sizeof (struct virtio_net_rxhdr); sc->rx_merge = 1; } else { /* * Without mergeable rx buffers, virtio-net header is 2 * bytes shorter than sizeof (struct virtio_net_rxhdr) unless, * that is, we are operating in modern mode. */ sc->vhdrlen = sizeof (struct virtio_net_rxhdr); if (!vi_is_modern(&sc->vsc_vs)) sc->vhdrlen -= 2; sc->rx_merge = 0; } /* Tell the backend to enable some capabilities it has advertised. */ netbe_set_cap(sc->vsc_be, *negotiated_features, sc->vhdrlen); sc->be_vhdrlen = netbe_get_vnet_hdr_len(sc->vsc_be); assert(sc->be_vhdrlen == 0 || sc->be_vhdrlen == sc->vhdrlen); pthread_mutex_lock(&sc->rx_mtx); sc->features_negotiated = true; pthread_mutex_unlock(&sc->rx_mtx); } static const struct pci_devemu pci_de_vnet = { .pe_emu = "virtio-net", .pe_init = pci_vtnet_init, .pe_legacy_config = netbe_legacy_config, .pe_cfgwrite = vi_pci_cfgwrite, .pe_cfgread = vi_pci_cfgread, .pe_barwrite = vi_pci_write, .pe_barread = vi_pci_read, }; PCI_EMUL_SET(pci_de_vnet); /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2014 Nahanni Systems Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer * in this position and unchanged. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * This file and its contents are supplied under the terms of the * Common Development and Distribution License ("CDDL"), version 1.0. * You may only use this file in accordance with the terms of version * 1.0 of the CDDL. * * A full copy of the text of the CDDL should have accompanied this * source. A copy of the CDDL is also available via the Internet at * http://www.illumos.org/license/CDDL. */ /* This file is dual-licensed; see usr/src/contrib/bhyve/LICENSE */ /* * Copyright 2026 Oxide Computer Company */ /* * virtio entropy device emulation. * Randomness is sourced from /dev/random which does not block * once it has been seeded at bootup. */ #include #ifndef WITHOUT_CAPSICUM #include #endif #include #include #ifndef WITHOUT_CAPSICUM #include #endif #include #include #include #include #include #include #include #include #include #include #include "bhyverun.h" #include "debug.h" #include "pci_emul.h" #include "virtio.h" #define VTRND_RINGSZ 64 static int pci_vtrnd_debug; #define DPRINTF(params) if (pci_vtrnd_debug) PRINTLN params #define WPRINTF(params) PRINTLN params /* * Per-device softc */ struct pci_vtrnd_softc { struct virtio_softc vrsc_vs; struct vqueue_info vrsc_vq; pthread_mutex_t vrsc_mtx; uint64_t vrsc_cfg; int vrsc_fd; }; static void pci_vtrnd_reset(void *); static void pci_vtrnd_notify(void *, struct vqueue_info *); static struct virtio_consts vtrnd_vi_consts = { .vc_name = "vtrnd", .vc_nvq = 1, .vc_cfgsize = 0, .vc_reset = pci_vtrnd_reset, .vc_qnotify = pci_vtrnd_notify, .vc_hv_caps_legacy = 0, .vc_hv_caps_modern = 0, }; static void pci_vtrnd_reset(void *vsc) { struct pci_vtrnd_softc *sc; sc = vsc; DPRINTF(("vtrnd: device reset requested !")); vi_reset_dev(&sc->vrsc_vs); } static void pci_vtrnd_notify(void *vsc, struct vqueue_info *vq) { struct iovec iov; struct pci_vtrnd_softc *sc; struct vi_req req; int len, n; sc = vsc; if (sc->vrsc_fd < 0) { vq_endchains(vq, 0); return; } while (vq_has_descs(vq)) { n = vq_getchain(vq, &iov, 1, &req); assert(n == 1); len = read(sc->vrsc_fd, iov.iov_base, iov.iov_len); DPRINTF(("vtrnd: vtrnd_notify(): %d", len)); /* Catastrophe if unable to read from /dev/random */ assert(len > 0); /* * Release this chain and handle more */ vq_relchain(vq, req.idx, len); } vq_endchains(vq, 1); /* Generate interrupt if appropriate. */ } static int pci_vtrnd_init(struct pci_devinst *pi, nvlist_t *nvl __unused) { struct pci_vtrnd_softc *sc; int fd; int len; uint8_t v; #ifndef WITHOUT_CAPSICUM cap_rights_t rights; #endif /* * Should always be able to open /dev/random. */ fd = open("/dev/random", O_RDONLY | O_NONBLOCK); assert(fd >= 0); #ifndef WITHOUT_CAPSICUM cap_rights_init(&rights, CAP_READ); if (caph_rights_limit(fd, &rights) == -1) errx(EX_OSERR, "Unable to apply rights for sandbox"); #endif /* * Check that device is seeded and non-blocking. */ len = read(fd, &v, sizeof(v)); if (len <= 0) { WPRINTF(("vtrnd: /dev/random not ready, read(): %d", len)); close(fd); return (1); } sc = calloc(1, sizeof(struct pci_vtrnd_softc)); pthread_mutex_init(&sc->vrsc_mtx, NULL); vi_softc_linkup(&sc->vrsc_vs, &vtrnd_vi_consts, sc, pi, &sc->vrsc_vq); sc->vrsc_vs.vs_mtx = &sc->vrsc_mtx; sc->vrsc_vq.vq_qsize = VTRND_RINGSZ; /* keep /dev/random opened while emulating */ sc->vrsc_fd = fd; /* initialize config space */ vi_pci_init(pi, VIRTIO_MODE_TRANSITIONAL, VIRTIO_DEV_RANDOM, VIRTIO_ID_ENTROPY, PCIC_CRYPTO); if (!vi_intr_init(&sc->vrsc_vs, fbsdrun_virtio_msix())) return (1); if (!vi_pcibar_setup(&sc->vrsc_vs)) return (1); return (0); } static const struct pci_devemu pci_de_vrnd = { .pe_emu = "virtio-rnd", .pe_init = pci_vtrnd_init, .pe_cfgwrite = vi_pci_cfgwrite, .pe_cfgread = vi_pci_cfgread, .pe_barwrite = vi_pci_write, .pe_barread = vi_pci_read, }; PCI_EMUL_SET(pci_de_vrnd); /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2016 Jakub Klama . * Copyright (c) 2018 Marcelo Araujo . * Copyright (c) 2026 Hans Rosenfeld * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer * in this position and unchanged. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * This file and its contents are supplied under the terms of the * Common Development and Distribution License ("CDDL"), version 1.0. * You may only use this file in accordance with the terms of version * 1.0 of the CDDL. * * A full copy of the text of the CDDL should have accompanied this * source. A copy of the CDDL is also available via the Internet at * http://www.illumos.org/license/CDDL. */ /* This file is dual-licensed; see usr/src/contrib/bhyve/LICENSE */ /* * Copyright 2026 Oxide Computer Company */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "bhyverun.h" #include "config.h" #include "debug.h" #include "pci_emul.h" #include "virtio.h" #include "iov.h" #include "pci_virtio_scsi.h" enum pci_vtscsi_walk { PCI_VTSCSI_WALK_CONTINUE = 0, PCI_VTSCSI_WALK_STOP, }; typedef enum pci_vtscsi_walk pci_vtscsi_walk_t; typedef pci_vtscsi_walk_t pci_vtscsi_walk_request_queue_cb_t( struct pci_vtscsi_queue *, struct pci_vtscsi_request *, void *); static void pci_vtscsi_print_supported_backends(void); static void *pci_vtscsi_proc(void *); static void pci_vtscsi_reset(void *); static void pci_vtscsi_neg_features(void *, uint64_t *); static int pci_vtscsi_cfgread(void *, int, int, uint32_t *); static int pci_vtscsi_cfgwrite(void *, int, int, uint32_t); static pci_vtscsi_walk_request_queue_cb_t pci_vtscsi_tmf_handle_abort_task; static pci_vtscsi_walk_request_queue_cb_t pci_vtscsi_tmf_handle_abort_task_set; static pci_vtscsi_walk_request_queue_cb_t pci_vtscsi_tmf_handle_clear_aca; static pci_vtscsi_walk_request_queue_cb_t pci_vtscsi_tmf_handle_clear_task_set; static pci_vtscsi_walk_request_queue_cb_t pci_vtscsi_tmf_handle_i_t_nexus_reset; static pci_vtscsi_walk_request_queue_cb_t pci_vtscsi_tmf_handle_lun_reset; static pci_vtscsi_walk_request_queue_cb_t pci_vtscsi_tmf_handle_query_task; static pci_vtscsi_walk_request_queue_cb_t pci_vtscsi_tmf_handle_query_task_set; static pci_vtscsi_walk_t pci_vtscsi_walk_request_queue( struct pci_vtscsi_queue *, pci_vtscsi_walk_request_queue_cb_t *, void *); static void pci_vtscsi_tmf_handle(struct pci_vtscsi_softc *, struct pci_vtscsi_ctrl_tmf *); static void pci_vtscsi_an_handle(struct pci_vtscsi_softc *, struct pci_vtscsi_ctrl_an *); static void pci_vtscsi_control_handle(struct pci_vtscsi_softc *, void *, size_t); static struct pci_vtscsi_request *pci_vtscsi_alloc_request( struct pci_vtscsi_softc *); static void pci_vtscsi_free_request(struct pci_vtscsi_softc *, struct pci_vtscsi_request *); static struct pci_vtscsi_request *pci_vtscsi_get_request( struct pci_vtscsi_req_queue *); static void pci_vtscsi_put_request(struct pci_vtscsi_req_queue *, struct pci_vtscsi_request *); static void pci_vtscsi_queue_request(struct pci_vtscsi_softc *, struct vqueue_info *); static void pci_vtscsi_return_request(struct pci_vtscsi_queue *, struct pci_vtscsi_request *, int); static int pci_vtscsi_request_handle(struct pci_vtscsi_softc *, int, struct pci_vtscsi_request *); static void pci_vtscsi_controlq_notify(void *, struct vqueue_info *); static void pci_vtscsi_eventq_notify(void *, struct vqueue_info *); static void pci_vtscsi_requestq_notify(void *, struct vqueue_info *); static int pci_vtscsi_add_target_config(nvlist_t *, const char *, int); static int pci_vtscsi_init_queue(struct pci_vtscsi_softc *, struct pci_vtscsi_queue *, int); static int pci_vtscsi_init(struct pci_devinst *, nvlist_t *); SET_DECLARE(pci_vtscsi_backend_set, struct pci_vtscsi_backend); static struct virtio_consts vtscsi_vi_consts = { .vc_name = "vtscsi", .vc_nvq = VTSCSI_DEF_REQUESTQ + VIRTIO_SCSI_ADDL_Q, .vc_cfgsize = sizeof(struct pci_vtscsi_config), .vc_reset = pci_vtscsi_reset, .vc_cfgread = pci_vtscsi_cfgread, .vc_cfgwrite = pci_vtscsi_cfgwrite, .vc_apply_features = pci_vtscsi_neg_features, .vc_hv_caps_legacy = VIRTIO_RING_F_INDIRECT_DESC, .vc_hv_caps_modern = VIRTIO_RING_F_INDIRECT_DESC, }; static const struct pci_vtscsi_config vtscsi_config = { .num_queues = VTSCSI_DEF_REQUESTQ, /* Leave room for the request and the response. */ .seg_max = VTSCSI_DEF_MAXSEG - VIRTIO_SCSI_HDR_SEG, .max_sectors = 0, .cmd_per_lun = 1, .event_info_size = sizeof(struct pci_vtscsi_event), .sense_size = 96, .cdb_size = 32, .max_channel = VIRTIO_SCSI_MAX_CHANNEL, .max_target = VIRTIO_SCSI_MAX_TARGET, .max_lun = VIRTIO_SCSI_MAX_LUN }; int pci_vtscsi_debug = 0; static void pci_vtscsi_print_supported_backends(void) { struct pci_vtscsi_backend **vbpp; if (SET_COUNT(pci_vtscsi_backend_set) == 0) { printf("No virtio-scsi backends available"); return; } SET_FOREACH(vbpp, pci_vtscsi_backend_set) { struct pci_vtscsi_backend *vbp = *vbpp; printf("%s\n", vbp->vsb_name); } } static void * pci_vtscsi_proc(void *arg) { struct pci_vtscsi_worker *worker = (struct pci_vtscsi_worker *)arg; struct pci_vtscsi_queue *q = worker->vsw_queue; struct pci_vtscsi_softc *sc = q->vsq_sc; for (;;) { struct pci_vtscsi_request *req; uint8_t target; int iolen; int fd; pthread_mutex_lock(&q->vsq_rmtx); while (STAILQ_EMPTY(&q->vsq_requests) && !worker->vsw_exiting) pthread_cond_wait(&q->vsq_cv, &q->vsq_rmtx); if (worker->vsw_exiting) { pthread_mutex_unlock(&q->vsq_rmtx); return (NULL); } req = pci_vtscsi_get_request(&q->vsq_requests); pthread_mutex_unlock(&q->vsq_rmtx); target = pci_vtscsi_get_target(sc, req->vsr_cmd_rd->lun); fd = sc->vss_targets[target].vst_fd; DPRINTF("I/O request tgt %u, lun %d, data_niov_in %zu, " "data_niov_out %zu", target, pci_vtscsi_get_lun(sc, req->vsr_cmd_rd->lun), req->vsr_data_niov_in, req->vsr_data_niov_out); iolen = pci_vtscsi_request_handle(sc, fd, req); pci_vtscsi_return_request(q, req, iolen); } } static void pci_vtscsi_reset(void *vsc) { struct pci_vtscsi_softc *sc; sc = vsc; DPRINTF("device reset requested"); vi_reset_dev(&sc->vss_vs); /* initialize config structure */ sc->vss_config = sc->vss_default_config; sc->vss_config.max_target = MAX(1, sc->vss_num_target) - 1; sc->vss_backend->vsb_reset(sc); } static void pci_vtscsi_neg_features(void *vsc, uint64_t *negotiated_features) { struct pci_vtscsi_softc *sc = vsc; sc->vss_features = *negotiated_features; } static int pci_vtscsi_cfgread(void *vsc, int offset, int size, uint32_t *retval) { struct pci_vtscsi_softc *sc = vsc; void *ptr; ptr = (uint8_t *)&sc->vss_config + offset; memcpy(retval, ptr, size); return (0); } static int pci_vtscsi_cfgwrite(void *vsc __unused, int offset __unused, int size __unused, uint32_t val __unused) { return (0); } /* * ABORT TASK: Abort the specifed task queued for this LUN. * * We can stop once we have found the specified task queued for this LUN. */ static pci_vtscsi_walk_t pci_vtscsi_tmf_handle_abort_task(struct pci_vtscsi_queue *q, struct pci_vtscsi_request *req, void *arg) { struct pci_vtscsi_ctrl_tmf *tmf = arg; assert(tmf->subtype == VIRTIO_SCSI_T_TMF_ABORT_TASK); if (pci_vtscsi_get_target(q->vsq_sc, tmf->lun) != pci_vtscsi_get_target(q->vsq_sc, req->vsr_cmd_rd->lun)) return (PCI_VTSCSI_WALK_CONTINUE); if (pci_vtscsi_get_lun(q->vsq_sc, tmf->lun) != pci_vtscsi_get_lun(q->vsq_sc, req->vsr_cmd_rd->lun)) return (PCI_VTSCSI_WALK_CONTINUE); if (tmf->id != req->vsr_cmd_rd->id) return (PCI_VTSCSI_WALK_CONTINUE); req->vsr_cmd_wr->response = VIRTIO_SCSI_S_ABORTED; STAILQ_REMOVE(&q->vsq_requests, req, pci_vtscsi_request, vsr_link); pci_vtscsi_return_request(q, req, 0); return (PCI_VTSCSI_WALK_STOP); } /* * ABORT TASK SET: Abort all tasks queued for this LUN. */ static pci_vtscsi_walk_t pci_vtscsi_tmf_handle_abort_task_set(struct pci_vtscsi_queue *q, struct pci_vtscsi_request *req, void *arg) { struct pci_vtscsi_ctrl_tmf *tmf = arg; assert(tmf->subtype == VIRTIO_SCSI_T_TMF_ABORT_TASK_SET); if (pci_vtscsi_get_target(q->vsq_sc, tmf->lun) != pci_vtscsi_get_target(q->vsq_sc, req->vsr_cmd_rd->lun)) return (PCI_VTSCSI_WALK_CONTINUE); if (pci_vtscsi_get_lun(q->vsq_sc, tmf->lun) != pci_vtscsi_get_lun(q->vsq_sc, req->vsr_cmd_rd->lun)) return (PCI_VTSCSI_WALK_CONTINUE); req->vsr_cmd_wr->response = VIRTIO_SCSI_S_ABORTED; STAILQ_REMOVE(&q->vsq_requests, req, pci_vtscsi_request, vsr_link); pci_vtscsi_return_request(q, req, 0); return (PCI_VTSCSI_WALK_CONTINUE); } /* * CLEAR ACA: Clear ACA (auto contingent allegiance) state. */ static pci_vtscsi_walk_t pci_vtscsi_tmf_handle_clear_aca(struct pci_vtscsi_queue *q __unused, struct pci_vtscsi_request *req __unused, void *arg) { struct pci_vtscsi_ctrl_tmf *tmf = arg; assert(tmf->subtype == VIRTIO_SCSI_T_TMF_CLEAR_ACA); /* * We don't implement handling of NACA=1 in the CONTROL byte at all. * * Thus, we probably should start filtering NORMACA in INQUIRY and * reject any command that sets NACA=1. * * In any case, there isn't anything we need to do with our queued * requests, so stop right here. */ return (PCI_VTSCSI_WALK_STOP); } /* * CLEAR TASK SET: Clear all tasks queued for this LUN. * * All tasks in our queue were placed there by us, so there can be no other * I_T nexus involved. Hence, this is handled the same as ABORT TASK SET. */ static pci_vtscsi_walk_t pci_vtscsi_tmf_handle_clear_task_set(struct pci_vtscsi_queue *q, struct pci_vtscsi_request *req, void *arg) { struct pci_vtscsi_ctrl_tmf *tmf = arg; assert(tmf->subtype == VIRTIO_SCSI_T_TMF_CLEAR_TASK_SET); if (pci_vtscsi_get_target(q->vsq_sc, tmf->lun) != pci_vtscsi_get_target(q->vsq_sc, req->vsr_cmd_rd->lun)) return (PCI_VTSCSI_WALK_CONTINUE); if (pci_vtscsi_get_lun(q->vsq_sc, tmf->lun) != pci_vtscsi_get_lun(q->vsq_sc, req->vsr_cmd_rd->lun)) return (PCI_VTSCSI_WALK_CONTINUE); req->vsr_cmd_wr->response = VIRTIO_SCSI_S_ABORTED; STAILQ_REMOVE(&q->vsq_requests, req, pci_vtscsi_request, vsr_link); pci_vtscsi_return_request(q, req, 0); return (PCI_VTSCSI_WALK_CONTINUE); } /* * I_T NEXUS RESET: Abort all tasks queued for any LUN of this target. */ static pci_vtscsi_walk_t pci_vtscsi_tmf_handle_i_t_nexus_reset(struct pci_vtscsi_queue *q, struct pci_vtscsi_request *req, void *arg) { struct pci_vtscsi_ctrl_tmf *tmf = arg; assert(tmf->subtype == VIRTIO_SCSI_T_TMF_I_T_NEXUS_RESET); if (pci_vtscsi_get_target(q->vsq_sc, tmf->lun) != pci_vtscsi_get_target(q->vsq_sc, req->vsr_cmd_rd->lun)) return (PCI_VTSCSI_WALK_CONTINUE); /* * T10 "06-026r4 SAM-4 TASK ABORTED status clarifications" indicates * that we should actually return ABORTED here, but other documents * such as the VirtIO spec suggest RESET. */ req->vsr_cmd_wr->response = VIRTIO_SCSI_S_RESET; STAILQ_REMOVE(&q->vsq_requests, req, pci_vtscsi_request, vsr_link); pci_vtscsi_return_request(q, req, 0); return (PCI_VTSCSI_WALK_CONTINUE); } /* * LOGICAL UNIT RESET: Abort all tasks queued for this LUN. */ static pci_vtscsi_walk_t pci_vtscsi_tmf_handle_lun_reset(struct pci_vtscsi_queue *q, struct pci_vtscsi_request *req, void *arg) { struct pci_vtscsi_ctrl_tmf *tmf = arg; assert(tmf->subtype == VIRTIO_SCSI_T_TMF_LOGICAL_UNIT_RESET); if (pci_vtscsi_get_target(q->vsq_sc, tmf->lun) != pci_vtscsi_get_target(q->vsq_sc, req->vsr_cmd_rd->lun)) return (PCI_VTSCSI_WALK_CONTINUE); if (pci_vtscsi_get_lun(q->vsq_sc, tmf->lun) != pci_vtscsi_get_lun(q->vsq_sc, req->vsr_cmd_rd->lun)) return (PCI_VTSCSI_WALK_CONTINUE); /* * T10 "06-026r4 SAM-4 TASK ABORTED status clarifications" indicates * that we should actually return ABORTED here, but other documents * such as the VirtIO spec suggest RESET. */ req->vsr_cmd_wr->response = VIRTIO_SCSI_S_RESET; STAILQ_REMOVE(&q->vsq_requests, req, pci_vtscsi_request, vsr_link); pci_vtscsi_return_request(q, req, 0); return (PCI_VTSCSI_WALK_CONTINUE); } /* * QUERY TASK: Is the specified task present in this LUN? * * We can stop once we have found the specified task queued for this LUN. */ static pci_vtscsi_walk_t pci_vtscsi_tmf_handle_query_task(struct pci_vtscsi_queue *q, struct pci_vtscsi_request *req, void *arg) { struct pci_vtscsi_ctrl_tmf *tmf = arg; assert(tmf->subtype == VIRTIO_SCSI_T_TMF_QUERY_TASK); if (pci_vtscsi_get_target(q->vsq_sc, tmf->lun) != pci_vtscsi_get_target(q->vsq_sc, req->vsr_cmd_rd->lun)) return (PCI_VTSCSI_WALK_CONTINUE); if (pci_vtscsi_get_lun(q->vsq_sc, tmf->lun) != pci_vtscsi_get_lun(q->vsq_sc, req->vsr_cmd_rd->lun)) return (PCI_VTSCSI_WALK_CONTINUE); if (tmf->id != req->vsr_cmd_rd->id) return (PCI_VTSCSI_WALK_CONTINUE); tmf->response = VIRTIO_SCSI_S_FUNCTION_SUCCEEDED; return (PCI_VTSCSI_WALK_STOP); } /* * QUERY TASK SET: Are there any tasks present in this LUN? * * We can stop as soon as we've found at least one task queued for this LUN. */ static pci_vtscsi_walk_t pci_vtscsi_tmf_handle_query_task_set(struct pci_vtscsi_queue *q, struct pci_vtscsi_request *req, void *arg) { struct pci_vtscsi_ctrl_tmf *tmf = arg; assert(tmf->subtype == VIRTIO_SCSI_T_TMF_QUERY_TASK_SET); if (pci_vtscsi_get_target(q->vsq_sc, tmf->lun) != pci_vtscsi_get_target(q->vsq_sc, req->vsr_cmd_rd->lun)) return (PCI_VTSCSI_WALK_CONTINUE); if (pci_vtscsi_get_lun(q->vsq_sc, tmf->lun) != pci_vtscsi_get_lun(q->vsq_sc, req->vsr_cmd_rd->lun)) return (PCI_VTSCSI_WALK_CONTINUE); tmf->response = VIRTIO_SCSI_S_FUNCTION_SUCCEEDED; return (PCI_VTSCSI_WALK_STOP); } static pci_vtscsi_walk_t pci_vtscsi_walk_request_queue(struct pci_vtscsi_queue *q, pci_vtscsi_walk_request_queue_cb_t cb, void *arg) { struct pci_vtscsi_request *req, *tmp; STAILQ_FOREACH_SAFE(req, &q->vsq_requests, vsr_link, tmp) { if (cb(q, req, arg) == PCI_VTSCSI_WALK_STOP) return (PCI_VTSCSI_WALK_STOP); } return (PCI_VTSCSI_WALK_CONTINUE); } static pci_vtscsi_walk_request_queue_cb_t *pci_vtscsi_tmf_handler_cb[] = { pci_vtscsi_tmf_handle_abort_task, pci_vtscsi_tmf_handle_abort_task_set, pci_vtscsi_tmf_handle_clear_aca, pci_vtscsi_tmf_handle_clear_task_set, pci_vtscsi_tmf_handle_i_t_nexus_reset, pci_vtscsi_tmf_handle_lun_reset, pci_vtscsi_tmf_handle_query_task, pci_vtscsi_tmf_handle_query_task_set }; static void pci_vtscsi_tmf_handle(struct pci_vtscsi_softc *sc, struct pci_vtscsi_ctrl_tmf *tmf) { uint8_t target; int fd; if (tmf->subtype > VIRTIO_SCSI_T_TMF_MAX_FUNC) { WPRINTF("pci_vtscsi_tmf_handle: invalid subtype %u", tmf->subtype); tmf->response = VIRTIO_SCSI_S_FUNCTION_REJECTED; return; } if (pci_vtscsi_check_lun(sc, tmf->lun) == false) { DPRINTF("TMF request to invalid LUN %.2hhx%.2hhx-%.2hhx%.2hhx-" "%.2hhx%.2hhx-%.2hhx%.2hhx", tmf->lun[0], tmf->lun[1], tmf->lun[2], tmf->lun[3], tmf->lun[4], tmf->lun[5], tmf->lun[6], tmf->lun[7]); tmf->response = VIRTIO_SCSI_S_BAD_TARGET; return; } target = pci_vtscsi_get_target(sc, tmf->lun); fd = sc->vss_targets[target].vst_fd; DPRINTF("TMF request tgt %d, lun %d, subtype %d, id %lu", target, pci_vtscsi_get_lun(sc, tmf->lun), tmf->subtype, tmf->id); /* * Lock out all the worker threads from processing any waiting requests * while we're processing the TMF request. This also effectively blocks * pci_vtscsi_requestq_notify() from adding any new requests to the * request queue. This does not prevent any requests currently being * processed by the backend from being completed and returned, which we * must guarantee to adhere to the ordering requirements for any TMF * function which aborts tasks. */ for (uint32_t i = 0; i < sc->vss_config.num_queues; i++) { struct pci_vtscsi_queue *q = &sc->vss_queues[i]; pthread_mutex_lock(&q->vsq_rmtx); } /* * The backend may set response to FAILURE for the TMF request. * * The default response of all TMF functions is FUNCTION COMPLETE if * there was no error, regardless of whether it actually succeeded or * not. The two notable exceptions are QUERY TASK and QUERY TASK SET, * which will explicitly return FUNCTION SUCCEEDED if the specified * task or any task was active in the target/LUN, respectively. * * Thus, we will call the backend first. Only if the response we get * is FUNCTION COMPLETE we'll continue processing the TMF function on * our queues. */ sc->vss_backend->vsb_tmf_hdl(sc, fd, tmf); if (tmf->response != VIRTIO_SCSI_S_FUNCTION_COMPLETE) { /* * If this is either a FAILURE or FUNCTION REJECTED, we must * not continue to process the TMF function on our queued * requests. * * If it is FUNCTION SUCCEEDED, we do not need to process the * TMF function on our queued requests. * * If it is anything else, log a warning, but handle it the * same as above. */ if (tmf->response != VIRTIO_SCSI_S_FAILURE && tmf->response != VIRTIO_SCSI_S_FUNCTION_REJECTED && tmf->response != VIRTIO_SCSI_S_FUNCTION_SUCCEEDED) { WPRINTF("pci_vtscsi_tmf_hdl: unexpected response from " "backend: %d", tmf->response); } } else { pci_vtscsi_walk_t ret = PCI_VTSCSI_WALK_CONTINUE; uint32_t i; for (i = 0; i < sc->vss_config.num_queues; i++) { struct pci_vtscsi_queue *q = &sc->vss_queues[i]; ret = pci_vtscsi_walk_request_queue(q, pci_vtscsi_tmf_handler_cb[tmf->subtype], tmf); if (ret == PCI_VTSCSI_WALK_STOP) break; } } /* Unlock the request queues before we return. */ for (uint32_t i = 0; i < sc->vss_config.num_queues; i++) { struct pci_vtscsi_queue *q = &sc->vss_queues[i]; pthread_mutex_unlock(&q->vsq_rmtx); } } static void pci_vtscsi_an_handle(struct pci_vtscsi_softc *sc, struct pci_vtscsi_ctrl_an *an) { int target; int fd; if (pci_vtscsi_check_lun(sc, an->lun) == false) { DPRINTF("AN request to invalid LUN %.2hhx%.2hhx-%.2hhx%.2hhx-" "%.2hhx%.2hhx-%.2hhx%.2hhx", an->lun[0], an->lun[1], an->lun[2], an->lun[3], an->lun[4], an->lun[5], an->lun[6], an->lun[7]); an->response = VIRTIO_SCSI_S_BAD_TARGET; return; } target = pci_vtscsi_get_target(sc, an->lun); fd = sc->vss_targets[target].vst_fd; DPRINTF("AN request tgt %d, lun %d, event requested %x", target, pci_vtscsi_get_lun(sc, an->lun), an->event_requested); sc->vss_backend->vsb_an_hdl(sc, fd, an); } static void pci_vtscsi_control_handle(struct pci_vtscsi_softc *sc, void *buf, size_t bufsize) { uint32_t type; if (bufsize < sizeof(uint32_t)) { WPRINTF("ignoring truncated control request"); return; } type = *(uint32_t *)buf; if (type == VIRTIO_SCSI_T_TMF) { if (bufsize != sizeof(struct pci_vtscsi_ctrl_tmf)) { WPRINTF("ignoring TMF request with size %zu", bufsize); return; } pci_vtscsi_tmf_handle(sc, buf); } else if (type == VIRTIO_SCSI_T_AN_QUERY) { if (bufsize != sizeof(struct pci_vtscsi_ctrl_an)) { WPRINTF("ignoring AN request with size %zu", bufsize); return; } pci_vtscsi_an_handle(sc, buf); } else { WPRINTF("ignoring unknown control request type = %u", type); } } static struct pci_vtscsi_request * pci_vtscsi_alloc_request(struct pci_vtscsi_softc *sc) { struct pci_vtscsi_request *req; req = calloc(1, sizeof(struct pci_vtscsi_request)); if (req == NULL) goto alloc_fail; req->vsr_iov = calloc(sc->vss_config.seg_max + VIRTIO_SCSI_HDR_SEG + SPLIT_IOV_ADDL_IOV, sizeof(struct iovec)); if (req->vsr_iov == NULL) goto alloc_fail; req->vsr_cmd_rd = calloc(1, VTSCSI_IN_HEADER_LEN(sc)); if (req->vsr_cmd_rd == NULL) goto alloc_fail; req->vsr_cmd_wr = calloc(1, VTSCSI_OUT_HEADER_LEN(sc)); if (req->vsr_cmd_wr == NULL) goto alloc_fail; req->vsr_backend = sc->vss_backend->vsb_req_alloc(sc); if (req->vsr_backend == NULL) goto alloc_fail; return (req); alloc_fail: EPRINTLN("failed to allocate request: %s", strerror(errno)); if (req != NULL) pci_vtscsi_free_request(sc, req); return (NULL); } static void pci_vtscsi_free_request(struct pci_vtscsi_softc *sc, struct pci_vtscsi_request *req) { if (req->vsr_backend != NULL) sc->vss_backend->vsb_req_free(req->vsr_backend); if (req->vsr_cmd_rd != NULL) free(req->vsr_cmd_rd); if (req->vsr_cmd_wr != NULL) free(req->vsr_cmd_wr); if (req->vsr_iov != NULL) free(req->vsr_iov); free(req); } static struct pci_vtscsi_request * pci_vtscsi_get_request(struct pci_vtscsi_req_queue *req_queue) { struct pci_vtscsi_request *req; assert(!STAILQ_EMPTY(req_queue)); req = STAILQ_FIRST(req_queue); STAILQ_REMOVE_HEAD(req_queue, vsr_link); return (req); } static void pci_vtscsi_put_request(struct pci_vtscsi_req_queue *req_queue, struct pci_vtscsi_request *req) { STAILQ_INSERT_TAIL(req_queue, req, vsr_link); } static void pci_vtscsi_queue_request(struct pci_vtscsi_softc *sc, struct vqueue_info *vq) { struct pci_vtscsi_queue *q; struct pci_vtscsi_request *req; struct vi_req vireq; int n, numseg; q = &sc->vss_queues[vq->vq_num - VIRTIO_SCSI_ADDL_Q]; pthread_mutex_lock(&q->vsq_fmtx); req = pci_vtscsi_get_request(&q->vsq_free_requests); assert(req != NULL); pthread_mutex_unlock(&q->vsq_fmtx); numseg = (int)(sc->vss_config.seg_max + VIRTIO_SCSI_HDR_SEG); n = vq_getchain(vq, req->vsr_iov, numseg, &vireq); assert(n >= 1 && n <= numseg); req->vsr_idx = vireq.idx; req->vsr_queue = q; req->vsr_iov_in = &req->vsr_iov[0]; req->vsr_niov_in = vireq.readable; req->vsr_iov_out = &req->vsr_iov[vireq.readable]; req->vsr_niov_out = vireq.writable; /* * Make sure we got at least enough space for the VirtIO-SCSI * command headers. If not, return this request immediately. */ if (check_iov_len(req->vsr_iov_out, req->vsr_niov_out, VTSCSI_OUT_HEADER_LEN(q->vsq_sc)) == false) { WPRINTF("ignoring request with insufficient output"); req->vsr_cmd_wr->response = VIRTIO_SCSI_S_FAILURE; pci_vtscsi_return_request(q, req, 1); return; } if (check_iov_len(req->vsr_iov_in, req->vsr_niov_in, VTSCSI_IN_HEADER_LEN(q->vsq_sc)) == false) { WPRINTF("ignoring request with incomplete header"); req->vsr_cmd_wr->response = VIRTIO_SCSI_S_FAILURE; pci_vtscsi_return_request(q, req, 1); return; } /* * We have to split the iovec array into a header and data portion each * for input and output. * * We need to start with the output section (at the end of iov) in case * the iovec covering the final part of the output header needs to be * split, in which case split_iov() will move all reamaining iovecs up * by one to make room for a new iovec covering the first part of the * output data portion. */ req->vsr_data_iov_out = split_iov(req->vsr_iov_out, &req->vsr_niov_out, VTSCSI_OUT_HEADER_LEN(q->vsq_sc), &req->vsr_data_niov_out); /* * Similarly, to not overwrite the first iovec of the output section, * the 2nd call to split_iov() to split the input section must actually * cover the entire iovec array (both input and the already split output * sections). */ req->vsr_niov_in += req->vsr_niov_out + req->vsr_data_niov_out; req->vsr_data_iov_in = split_iov(req->vsr_iov_in, &req->vsr_niov_in, VTSCSI_IN_HEADER_LEN(q->vsq_sc), &req->vsr_data_niov_in); /* * And of course we now have to adjust data_niov_in accordingly. */ req->vsr_data_niov_in -= req->vsr_niov_out + req->vsr_data_niov_out; /* * iov_to_buf() realloc()s the buffer given as 3rd argument to the * total size of all iovecs it will be copying. Since we've just * truncated it in split_iov(), we know that the size will be * VTSCSI_IN_HEADER_LEN(q->vsq_sc). * * Since we pre-allocated req->vsr_cmd_rd to this size, the realloc() * should never fail. * * This will have to change if we begin allowing config space writes * to change sense size. */ assert(iov_to_buf(req->vsr_iov_in, req->vsr_niov_in, (void **)&req->vsr_cmd_rd) == VTSCSI_IN_HEADER_LEN(q->vsq_sc)); /* Make sure this request addresses a valid LUN. */ if (pci_vtscsi_check_lun(sc, req->vsr_cmd_rd->lun) == false) { DPRINTF("I/O request to invalid LUN " "%.2hhx%.2hhx-%.2hhx%.2hhx-%.2hhx%.2hhx-%.2hhx%.2hhx", req->vsr_cmd_rd->lun[0], req->vsr_cmd_rd->lun[1], req->vsr_cmd_rd->lun[2], req->vsr_cmd_rd->lun[3], req->vsr_cmd_rd->lun[4], req->vsr_cmd_rd->lun[5], req->vsr_cmd_rd->lun[6], req->vsr_cmd_rd->lun[7]); req->vsr_cmd_wr->response = VIRTIO_SCSI_S_BAD_TARGET; pci_vtscsi_return_request(q, req, 1); return; } pthread_mutex_lock(&q->vsq_rmtx); pci_vtscsi_put_request(&q->vsq_requests, req); pthread_cond_signal(&q->vsq_cv); pthread_mutex_unlock(&q->vsq_rmtx); DPRINTF("request enqueued", vireq.idx); } static void pci_vtscsi_return_request(struct pci_vtscsi_queue *q, struct pci_vtscsi_request *req, int iolen) { struct pci_vtscsi_softc *sc = q->vsq_sc; void *iov = req->vsr_iov; void *cmd_rd = req->vsr_cmd_rd; void *cmd_wr = req->vsr_cmd_wr; void *backend = req->vsr_backend; int idx = req->vsr_idx; DPRINTF("request completed, response %d", idx, req->vsr_cmd_wr->response); iolen += buf_to_iov(cmd_wr, VTSCSI_OUT_HEADER_LEN(q->vsq_sc), req->vsr_iov_out, req->vsr_niov_out); sc->vss_backend->vsb_req_clear(backend); memset(iov, 0, sizeof(struct iovec) * (sc->vss_config.seg_max + VIRTIO_SCSI_HDR_SEG + SPLIT_IOV_ADDL_IOV)); memset(cmd_rd, 0, VTSCSI_IN_HEADER_LEN(q->vsq_sc)); memset(cmd_wr, 0, VTSCSI_OUT_HEADER_LEN(q->vsq_sc)); memset(req, 0, sizeof(struct pci_vtscsi_request)); req->vsr_iov = iov; req->vsr_cmd_rd = cmd_rd; req->vsr_cmd_wr = cmd_wr; req->vsr_backend = backend; pthread_mutex_lock(&q->vsq_fmtx); pci_vtscsi_put_request(&q->vsq_free_requests, req); pthread_mutex_unlock(&q->vsq_fmtx); pthread_mutex_lock(&q->vsq_qmtx); vq_relchain(q->vsq_vq, idx, iolen); vq_endchains(q->vsq_vq, 0); pthread_mutex_unlock(&q->vsq_qmtx); } static int pci_vtscsi_request_handle(struct pci_vtscsi_softc *sc, int fd, struct pci_vtscsi_request *req) { return (sc->vss_backend->vsb_req_hdl(sc, fd, req)); } static void pci_vtscsi_controlq_notify(void *vsc, struct vqueue_info *vq) { struct pci_vtscsi_softc *sc = vsc; int numseg = (int)(sc->vss_config.seg_max + VIRTIO_SCSI_HDR_SEG); struct iovec iov[numseg]; struct vi_req req; void *buf = NULL; size_t bufsize; int n; while (vq_has_descs(vq)) { n = vq_getchain(vq, iov, numseg, &req); assert(n >= 1 && n <= numseg); bufsize = iov_to_buf(iov, n, &buf); pci_vtscsi_control_handle(sc, buf, bufsize); buf_to_iov((uint8_t *)buf, bufsize, iov, n); /* * Release this chain and handle more */ vq_relchain(vq, req.idx, bufsize); } vq_endchains(vq, 1); /* Generate interrupt if appropriate. */ free(buf); } static void pci_vtscsi_eventq_notify(void *vsc __unused, struct vqueue_info *vq) { vq_kick_disable(vq); } static void pci_vtscsi_requestq_notify(void *vsc, struct vqueue_info *vq) { while (vq_has_descs(vq)) { pci_vtscsi_queue_request(vsc, vq); } } static int pci_vtscsi_init_queue(struct pci_vtscsi_softc *sc, struct pci_vtscsi_queue *queue, int num) { char tname[MAXCOMLEN + 1]; uint32_t i; queue->vsq_sc = sc; queue->vsq_vq = &sc->vss_vq[num]; pthread_mutex_init(&queue->vsq_rmtx, NULL); pthread_mutex_init(&queue->vsq_fmtx, NULL); pthread_mutex_init(&queue->vsq_qmtx, NULL); pthread_cond_init(&queue->vsq_cv, NULL); STAILQ_INIT(&queue->vsq_requests); STAILQ_INIT(&queue->vsq_free_requests); LIST_INIT(&queue->vsq_workers); for (i = 0; i < sc->vss_req_ringsz; i++) { struct pci_vtscsi_request *req; req = pci_vtscsi_alloc_request(sc); if (req == NULL) return (-1); pci_vtscsi_put_request(&queue->vsq_free_requests, req); } for (i = 0; i < sc->vss_thr_per_q; i++) { struct pci_vtscsi_worker *worker; worker = calloc(1, sizeof(struct pci_vtscsi_worker)); if (worker == NULL) return (-1); worker->vsw_queue = queue; pthread_create(&worker->vsw_thread, NULL, &pci_vtscsi_proc, (void *)worker); snprintf(tname, sizeof(tname), "vtscsi:%d-%d", num, i); pthread_set_name_np(worker->vsw_thread, tname); LIST_INSERT_HEAD(&queue->vsq_workers, worker, vsw_link); } return (0); } /* * Create a target config node, return target id. If the target number isn't * given as part of the path argument, use last_id + 1. */ static int pci_vtscsi_add_target_config(nvlist_t *nvl, const char *path, int last_id) { uint64_t target; char *id; char tmp[4]; if (path == NULL) { EPRINTLN("target path must be specified"); return (-1); } if (path[0] != '/' && (id = strchr(path, ':')) != NULL) { const char *errstr; int len = id - path; id = strndup(path, len); if (id == NULL) { EPRINTLN("failed to get id string: %s", strerror(errno)); return (-1); } target = strtonumx(id, 0, VIRTIO_SCSI_MAX_TARGET, &errstr, 0); if (errstr != NULL) { EPRINTLN("invalid target %s: target ID is %s", id, errstr); free(id); return (-1); } free(id); path += len + 1; } else { target = last_id + 1; if (target > VIRTIO_SCSI_MAX_TARGET) { EPRINTLN("max target (%d) reached, can't add another", VIRTIO_SCSI_MAX_TARGET); return (-1); } } snprintf(tmp, sizeof(tmp), "%lu", target); if (get_config_value_node(nvl, tmp) != NULL) { EPRINTLN("cannot add '%s' as target %s: already exits as '%s'", path, tmp, get_config_value_node(nvl, tmp)); return (-1); } set_config_value_node(nvl, tmp, path); return (target); } /* * The following forms are accepted for legacy config options to configure a * single target: * * (0) -s B:D:F,virtio-scsi * (1) -s B:D:F,virtio-scsi, * (2) -s B:D:F,virtio-scsi,,,... * (3) -s B:D:F,virtio-scsi,,... * (4) -s B:D:F,virtio-scsi, * * To configure multiple targets, the following form is accepted: * (5) -s B:D:F,virtio-scsi,[target=[id:],...] */ static int pci_vtscsi_legacy_config(nvlist_t *nvl, const char *opts) { int last_id = -1; char *config, *tofree, *name, *value; nvlist_t *targets; size_t n; /* Make sure no one accidentally sets "dev" anymore. */ (void) create_relative_config_node(nvl, "dev"); targets = create_relative_config_node(nvl, "target"); /* Legacy form (0) is handled in pci_vtscsi_init(). */ if (opts == NULL) return (0); if (strcmp("help", opts) == 0) { pci_vtscsi_print_supported_backends(); exit(0); } n = strcspn(opts, ",="); /* Handle legacy form (1) and (2). */ if (opts[n] == ',' || opts[n] == '\0') { char *tmp = strndup(opts, n); last_id = pci_vtscsi_add_target_config(targets, tmp, last_id); free(tmp); if (last_id < 0) return (-1); opts += n; if (opts[0] == ',' && opts[1] != '\0') opts++; } /* If this was form (1), we're done. */ if (opts[0] == '\0') return (0); /* * For form (2), (3), (4), and (5), parse the remaining options. * * Contrary to other options, multiple target= options create a new * target for each such option. * * For compatibility reasons we also accept dev= options for * targets. */ config = tofree = strdup(opts); while ((name = strsep(&config, ",")) != NULL) { value = strchr(name, '='); if (value != NULL) *value++ = '\0'; if (strcmp(name, "dev") == 0 || strcmp(name, "target") == 0) { int new_id = pci_vtscsi_add_target_config(targets, value, last_id); if (new_id < 0) { free(tofree); return (-1); } if (new_id > last_id) last_id = new_id; } else if (value != NULL) { set_config_value_node(nvl, name, value); } else { set_config_bool_node(nvl, name, true); } } free(tofree); return (0); } static int pci_vtscsi_count_targets(const char *prefix __unused, const nvlist_t *parent __unused, const char *name, int type, void *arg) { struct pci_vtscsi_softc *sc = arg; const char *errstr; uint64_t target; if (type != NV_TYPE_STRING) { EPRINTLN("invalid target \"%s\" type: not a string", name); errno = EINVAL; return (-1); } target = strtonumx(name, 0, VIRTIO_SCSI_MAX_TARGET, &errstr, 0); if (errstr != NULL) { EPRINTLN("invalid target %s: target ID is %s", name, errstr); return (-1); } if (target >= sc->vss_num_target) sc->vss_num_target = target + 1; return (0); } static int pci_vtscsi_init_target(const char *prefix __unused, const nvlist_t *parent, const char *name, int type, void *arg) { struct pci_vtscsi_softc *sc = arg; const char *value; const char *errstr; uint64_t target; int ret; assert(type == NV_TYPE_STRING); /* * Get the numeric value of the target id from 'name'. */ target = strtonumx(name, 0, sc->vss_num_target, &errstr, 0); assert(errstr == NULL); sc->vss_targets[target].vst_target = target; /* * 'value' contains the backend path. Call the backend to open it. */ value = nvlist_get_string(parent, name); ret = sc->vss_backend->vsb_open(sc, value, target); if (ret != 0) EPRINTLN("cannot open target %lu at %s: %s", target, value, strerror(errno)); return (ret); } static int pci_vtscsi_get_config_num(nvlist_t *nvl, const char *name, uint32_t lim_lo, uint32_t lim_hi, uint32_t *res) { const char *value; const char *errstr; long long val; value = get_config_value_node(nvl, name); if (value == NULL) return (0); val = strtonumx(value, lim_lo, lim_hi, &errstr, 0); if (errstr != NULL) { EPRINTLN("Invalid value for %s: %s", name, value); return (-1); } *res = (uint32_t)val; return (0); } static int pci_vtscsi_init(struct pci_devinst *pi, nvlist_t *nvl) { struct pci_vtscsi_softc *sc; struct pci_vtscsi_backend *backend, **vbpp; const char *value; uint32_t val; int err; sc = calloc(1, sizeof(struct pci_vtscsi_softc)); if (sc == NULL) return (-1); sc->vss_vi_consts = vtscsi_vi_consts; sc->vss_ctl_ringsz = VTSCSI_DEF_RINGSZ; sc->vss_evt_ringsz = VTSCSI_DEF_RINGSZ; sc->vss_req_ringsz = VTSCSI_DEF_RINGSZ; sc->vss_thr_per_q = VTSCSI_DEF_THR_PER_Q; sc->vss_default_config = vtscsi_config; value = get_config_value_node(nvl, "bootindex"); if (value != NULL) { if (pci_emul_add_boot_device(pi, atoi(value))) { EPRINTLN("Invalid bootindex %d", atoi(value)); errno = EINVAL; goto fail; } } val = vtscsi_config.seg_max; if (pci_vtscsi_get_config_num(nvl, "seg_max", VTSCSI_MIN_MAXSEG, VTSCSI_MAX_MAXSEG, &val) != 0) goto fail; sc->vss_default_config.seg_max = val; val = vtscsi_config.num_queues; if (pci_vtscsi_get_config_num(nvl, "num_queues", VTSCSI_MIN_REQUESTQ, VTSCSI_MAX_REQUESTQ, &val) != 0) goto fail; sc->vss_default_config.num_queues = val; /* * num_queues is only the number of request queues, but nvq must * account for the control and event queues. */ sc->vss_vi_consts.vc_nvq = val + VIRTIO_SCSI_ADDL_Q; /* * Allocate queues early, so that they're there for the call to * vi_softc_linkup(). */ sc->vss_vq = calloc(sc->vss_vi_consts.vc_nvq, sizeof(struct vqueue_info)); if (sc->vss_vq == NULL) { EPRINTLN("can't allocate space for %d virtqueues", sc->vss_vi_consts.vc_nvq); goto fail; } sc->vss_queues = calloc(sc->vss_default_config.num_queues, sizeof(struct pci_vtscsi_queue)); if (sc->vss_queues == NULL) { EPRINTLN("can't allocate space for %d request queues", sc->vss_config.num_queues); goto fail; } if (pci_vtscsi_get_config_num(nvl, "ctl_ringsz", VTSCSI_MIN_RINGSZ, VTSCSI_MAX_RINGSZ, &sc->vss_ctl_ringsz) != 0) goto fail; if (pci_vtscsi_get_config_num(nvl, "evt_ringsz", VTSCSI_MIN_RINGSZ, VTSCSI_MAX_RINGSZ, &sc->vss_evt_ringsz) != 0) goto fail; if (pci_vtscsi_get_config_num(nvl, "req_ringsz", VTSCSI_MIN_RINGSZ, VTSCSI_MAX_RINGSZ, &sc->vss_req_ringsz) != 0) goto fail; if (pci_vtscsi_get_config_num(nvl, "thr_per_q", VTSCSI_MIN_THR_PER_Q, VTSCSI_MAX_THR_PER_Q, &sc->vss_thr_per_q) != 0) goto fail; value = get_config_value_node(nvl, "backend"); if (value == NULL) { if (SET_COUNT(pci_vtscsi_backend_set) == 0) { WPRINTF("No virtio-scsi backends available"); errno = EINVAL; goto fail; } backend = SET_ITEM(pci_vtscsi_backend_set, 0); } else { backend = NULL; SET_FOREACH(vbpp, pci_vtscsi_backend_set) { if (strcasecmp(value, (*vbpp)->vsb_name) == 0) { backend = *vbpp; break; } } if (backend == NULL) { WPRINTF("No such virtio-scsi backend: %s", value); errno = EINVAL; goto fail; } } err = backend->vsb_init(sc, backend, nvl); if (err != 0) { errno = EINVAL; goto fail; } nvl = find_relative_config_node(nvl, "target"); if (nvl != NULL) { err = walk_config_nodes("", nvl, sc, pci_vtscsi_count_targets); if (err != 0) goto fail; } if (sc->vss_num_target > 0) { sc->vss_targets = malloc(sc->vss_num_target * sizeof(struct pci_vtscsi_target)); if (sc->vss_targets == NULL) { EPRINTLN("can't allocate space for %lu targets", sc->vss_num_target); goto fail; } memset(sc->vss_targets, -1, sc->vss_num_target * sizeof(struct pci_vtscsi_target)); err = walk_config_nodes("", nvl, sc, pci_vtscsi_init_target); if (err != 0) goto fail; } /* * All targets should be open now and have a valid fd. */ for (size_t i = 0; i < sc->vss_num_target; i++) { if (sc->vss_targets[i].vst_target == i && sc->vss_targets[i].vst_fd < 0) { goto fail; } } pthread_mutex_init(&sc->vss_mtx, NULL); vi_softc_linkup(&sc->vss_vs, &sc->vss_vi_consts, sc, pi, sc->vss_vq); sc->vss_vs.vs_mtx = &sc->vss_mtx; /* * Perform a "reset" before we set up our queues. * * This will write the default config into vss_config, which is used * by the rest of the driver to get the request header size. Note that * if we ever allow the guest to override sense size through config * space writes, pre-allocation of I/O requests will have to change * accordingly. */ pthread_mutex_lock(&sc->vss_mtx); pci_vtscsi_reset(sc); pthread_mutex_unlock(&sc->vss_mtx); /* virtqueue 0: control queue */ sc->vss_vq[0].vq_qsize = sc->vss_ctl_ringsz; sc->vss_vq[0].vq_notify = pci_vtscsi_controlq_notify; /* virtqueue 1: event queue */ sc->vss_vq[1].vq_qsize = sc->vss_evt_ringsz; sc->vss_vq[1].vq_notify = pci_vtscsi_eventq_notify; /* virtqueue 2-n: request queues */ for (int i = VIRTIO_SCSI_ADDL_Q; i < sc->vss_vi_consts.vc_nvq; i++) { int rq = i - VIRTIO_SCSI_ADDL_Q; sc->vss_vq[i].vq_qsize = sc->vss_req_ringsz; sc->vss_vq[i].vq_notify = pci_vtscsi_requestq_notify; err = pci_vtscsi_init_queue(sc, &sc->vss_queues[rq], i); if (err != 0) { free(sc->vss_targets); goto fail; } } /* initialize config space */ vi_pci_init(pi, VIRTIO_MODE_TRANSITIONAL, VIRTIO_DEV_SCSI, VIRTIO_ID_SCSI, PCIC_STORAGE); if (!vi_intr_init(&sc->vss_vs, fbsdrun_virtio_msix())) { free(sc->vss_targets); goto fail; } if (!vi_pcibar_setup(&sc->vss_vs)) { free(sc->vss_targets); goto fail; } return (0); fail: free(sc); return (-1); } static const struct pci_devemu pci_de_vscsi = { .pe_emu = "virtio-scsi", .pe_init = pci_vtscsi_init, .pe_legacy_config = pci_vtscsi_legacy_config, .pe_cfgwrite = vi_pci_cfgwrite, .pe_cfgread = vi_pci_cfgread, .pe_barwrite = vi_pci_write, .pe_barread = vi_pci_read }; PCI_EMUL_SET(pci_de_vscsi); /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2016 Jakub Klama . * Copyright (c) 2018 Marcelo Araujo . * Copyright (c) 2026 Hans Rosenfeld * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer * in this position and unchanged. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef _PCI_VIRTIO_SCSI_H_ #define _PCI_VIRTIO_SCSI_H_ #include "iov.h" extern int pci_vtscsi_debug; #define WPRINTF(msg, params...) PRINTLN("virtio-scsi: " msg, ##params) #define DPRINTF(msg, params...) if (pci_vtscsi_debug) WPRINTF(msg, ##params) /* Absolute limits given by the VirtIO SCSI spec */ #define VIRTIO_SCSI_MAX_CHANNEL 0 #define VIRTIO_SCSI_MAX_TARGET 255 #define VIRTIO_SCSI_MAX_LUN 16383 #define VIRTIO_SCSI_HDR_SEG 2 #define VIRTIO_SCSI_ADDL_Q 2 /* Features specific to VirtIO SCSI, none of which we currently support */ #define VIRTIO_SCSI_F_INOUT (1 << 0) #define VIRTIO_SCSI_F_HOTPLUG (1 << 1) #define VIRTIO_SCSI_F_CHANGE (1 << 2) /* Default limits which we set. All of these are configurable. */ #define VTSCSI_DEF_RINGSZ 64 #define VTSCSI_MIN_RINGSZ 4 #define VTSCSI_MAX_RINGSZ 4096 #define VTSCSI_DEF_THR_PER_Q 16 #define VTSCSI_MIN_THR_PER_Q 1 #define VTSCSI_MAX_THR_PER_Q 256 #define VTSCSI_DEF_MAXSEG 64 #define VTSCSI_MIN_MAXSEG (VIRTIO_SCSI_HDR_SEG + 1) #define VTSCSI_MAX_MAXSEG \ (4096 - VIRTIO_SCSI_HDR_SEG - SPLIT_IOV_ADDL_IOV) #define VTSCSI_DEF_REQUESTQ 1 #define VTSCSI_MIN_REQUESTQ 1 #define VTSCSI_MAX_REQUESTQ (32 - VIRTIO_SCSI_ADDL_Q) /* * Device-specific config space registers * * The guest driver may try to modify cdb_size and sense_size by writing the * respective config space registers. Since we currently ignore all writes to * config space, these macros are essentially constant. */ #define VTSCSI_IN_HEADER_LEN(_sc) \ (sizeof(struct pci_vtscsi_req_cmd_rd) + _sc->vss_config.cdb_size) #define VTSCSI_OUT_HEADER_LEN(_sc) \ (sizeof(struct pci_vtscsi_req_cmd_wr) + _sc->vss_config.sense_size) struct pci_vtscsi_config { uint32_t num_queues; uint32_t seg_max; uint32_t max_sectors; uint32_t cmd_per_lun; uint32_t event_info_size; uint32_t sense_size; uint32_t cdb_size; uint16_t max_channel; uint16_t max_target; uint32_t max_lun; } __attribute__((packed)); /* * I/O request state * * Each pci_vtscsi_queue has configurable number of pci_vtscsi_request * structures pre-allocated on vsq_free_requests. For each I/O request * coming in on the I/O virtqueue, the request queue handler takes * pci_vtscsi_request off vsq_free_requests, fills in the data from the * I/O virtqueue, puts it on vsq_requests, and signals vsq_cv. * * Each pci_vtscsi_queue will have a configurable number of worker threads, * which wait on vsq_cv. When signalled, they repeatedly take a single * pci_vtscsi_request off vsq_requests and hand it to the backend, which * processes it synchronously. After completion, the pci_vtscsi_request * is re-initialized and put back onto vsq_free_requests. * * The worker threads exit when vsq_cv is signalled after vsw_exiting was set. * * The I/O vectors for each request are kept in the preallocated iovec array * vsr_iov, and pointers to the respective header/data in/out portions are set * up to point into the array when the request is queued for processing. * * The number of iovecs preallocated for vsr_iov is derived from the configured * 'seg_max' parameter defined by the virtio spec: * - 'seg_max' parameter specifies the maximum number of I/O data vectors * we support in any request * - we need 2 additional iovecs for the I/O headers (VIRTIO_SCSI_HDR_SEG) * - we need another 2 additional iovecs for split_iov() (SPLIT_IOV_ADDL_IOV) * * The only time we explicitly need the full size of vsr_iov after preallocation * is during re-initialization after completing a request, and implicitly in the * calls to split_iov() the set up the pointers. In all other cases, we use only * 'seg_max' + VIRTIO_SCSI_HDR_SEG, and we advertise only 'seg_max' to the guest * in accordance to the virtio spec. */ STAILQ_HEAD(pci_vtscsi_req_queue, pci_vtscsi_request); struct pci_vtscsi_queue { struct pci_vtscsi_softc *vsq_sc; struct vqueue_info *vsq_vq; pthread_mutex_t vsq_rmtx; pthread_mutex_t vsq_fmtx; pthread_mutex_t vsq_qmtx; pthread_cond_t vsq_cv; struct pci_vtscsi_req_queue vsq_requests; struct pci_vtscsi_req_queue vsq_free_requests; LIST_HEAD(, pci_vtscsi_worker) vsq_workers; }; struct pci_vtscsi_worker { struct pci_vtscsi_queue *vsw_queue; pthread_t vsw_thread; bool vsw_exiting; LIST_ENTRY(pci_vtscsi_worker) vsw_link; }; struct pci_vtscsi_request { struct pci_vtscsi_queue *vsr_queue; struct iovec *vsr_iov; struct iovec *vsr_iov_in; struct iovec *vsr_iov_out; struct iovec *vsr_data_iov_in; struct iovec *vsr_data_iov_out; struct pci_vtscsi_req_cmd_rd *vsr_cmd_rd; struct pci_vtscsi_req_cmd_wr *vsr_cmd_wr; void *vsr_backend; size_t vsr_niov_in; size_t vsr_niov_out; size_t vsr_data_niov_in; size_t vsr_data_niov_out; uint32_t vsr_idx; STAILQ_ENTRY(pci_vtscsi_request) vsr_link; }; /* * Per-target state. */ struct pci_vtscsi_target { uint8_t vst_target; int vst_fd; int vst_max_sectors; }; /* * Per-device softc */ struct pci_vtscsi_softc { struct virtio_softc vss_vs; struct virtio_consts vss_vi_consts; struct vqueue_info *vss_vq; struct pci_vtscsi_queue *vss_queues; pthread_mutex_t vss_mtx; uint32_t vss_features; size_t vss_num_target; uint32_t vss_ctl_ringsz; uint32_t vss_evt_ringsz; uint32_t vss_req_ringsz; uint32_t vss_thr_per_q; struct pci_vtscsi_config vss_default_config; struct pci_vtscsi_config vss_config; struct pci_vtscsi_target *vss_targets; struct pci_vtscsi_backend *vss_backend; }; /* * VirtIO-SCSI Task Management Function control requests */ #define VIRTIO_SCSI_T_TMF 0 #define VIRTIO_SCSI_T_TMF_ABORT_TASK 0 #define VIRTIO_SCSI_T_TMF_ABORT_TASK_SET 1 #define VIRTIO_SCSI_T_TMF_CLEAR_ACA 2 #define VIRTIO_SCSI_T_TMF_CLEAR_TASK_SET 3 #define VIRTIO_SCSI_T_TMF_I_T_NEXUS_RESET 4 #define VIRTIO_SCSI_T_TMF_LOGICAL_UNIT_RESET 5 #define VIRTIO_SCSI_T_TMF_QUERY_TASK 6 #define VIRTIO_SCSI_T_TMF_QUERY_TASK_SET 7 #define VIRTIO_SCSI_T_TMF_MAX_FUNC VIRTIO_SCSI_T_TMF_QUERY_TASK_SET /* command-specific response values */ #define VIRTIO_SCSI_S_FUNCTION_COMPLETE 0 #define VIRTIO_SCSI_S_FUNCTION_SUCCEEDED 10 #define VIRTIO_SCSI_S_FUNCTION_REJECTED 11 struct pci_vtscsi_ctrl_tmf { const uint32_t type; const uint32_t subtype; const uint8_t lun[8]; const uint64_t id; uint8_t response; } __attribute__((packed)); /* * VirtIO-SCSI Asynchronous Notification control requests */ #define VIRTIO_SCSI_T_AN_QUERY 1 #define VIRTIO_SCSI_EVT_ASYNC_OPERATIONAL_CHANGE 2 #define VIRTIO_SCSI_EVT_ASYNC_POWER_MGMT 4 #define VIRTIO_SCSI_EVT_ASYNC_EXTERNAL_REQUEST 8 #define VIRTIO_SCSI_EVT_ASYNC_MEDIA_CHANGE 16 #define VIRTIO_SCSI_EVT_ASYNC_MULTI_HOST 32 #define VIRTIO_SCSI_EVT_ASYNC_DEVICE_BUSY 64 struct pci_vtscsi_ctrl_an { const uint32_t type; const uint8_t lun[8]; const uint32_t event_requested; uint32_t event_actual; uint8_t response; } __attribute__((packed)); /* command-specific response values */ #define VIRTIO_SCSI_S_OK 0 #define VIRTIO_SCSI_S_OVERRUN 1 #define VIRTIO_SCSI_S_ABORTED 2 #define VIRTIO_SCSI_S_BAD_TARGET 3 #define VIRTIO_SCSI_S_RESET 4 #define VIRTIO_SCSI_S_BUSY 5 #define VIRTIO_SCSI_S_TRANSPORT_FAILURE 6 #define VIRTIO_SCSI_S_TARGET_FAILURE 7 #define VIRTIO_SCSI_S_NEXUS_FAILURE 8 #define VIRTIO_SCSI_S_FAILURE 9 #define VIRTIO_SCSI_S_INCORRECT_LUN 12 struct pci_vtscsi_event { uint32_t event; uint8_t lun[8]; uint32_t reason; } __attribute__((packed)); /* * VirtIO-SCSI I/O requests */ struct pci_vtscsi_req_cmd_rd { const uint8_t lun[8]; const uint64_t id; const uint8_t task_attr; const uint8_t prio; const uint8_t crn; const uint8_t cdb[]; } __attribute__((packed)); /* task_attr */ #define VIRTIO_SCSI_S_SIMPLE 0 #define VIRTIO_SCSI_S_ORDERED 1 #define VIRTIO_SCSI_S_HEAD 2 #define VIRTIO_SCSI_S_ACA 3 struct pci_vtscsi_req_cmd_wr { uint32_t sense_len; uint32_t residual; uint16_t status_qualifier; uint8_t status; uint8_t response; uint8_t sense[]; } __attribute__((packed)); /* * Backend interface */ struct pci_vtscsi_backend { const char *vsb_name; int (*vsb_init)(struct pci_vtscsi_softc *, struct pci_vtscsi_backend *, nvlist_t *); int (*vsb_open)(struct pci_vtscsi_softc *, const char *, long); void (*vsb_reset)(struct pci_vtscsi_softc *); void* (*vsb_req_alloc)(struct pci_vtscsi_softc *); void (*vsb_req_clear)(void *); void (*vsb_req_free)(void *); void (*vsb_tmf_hdl)(struct pci_vtscsi_softc *, int, struct pci_vtscsi_ctrl_tmf *); void (*vsb_an_hdl)(struct pci_vtscsi_softc *, int, struct pci_vtscsi_ctrl_an *); int (*vsb_req_hdl)(struct pci_vtscsi_softc *, int, struct pci_vtscsi_request *); }; #define PCI_VTSCSI_BACKEND_SET(x) DATA_SET(pci_vtscsi_backend_set, x) /* * LUN address parsing * * The LUN address consists of 8 bytes. While the spec describes this as 0x01, * followed by the target byte, followed by a "single-level LUN structure", * this is actually the same as a hierarchical LUN address as defined by SAM-5, * consisting of four levels of addressing, where in each level the two MSB of * byte 0 select the address mode used in the remaining bits and bytes. * * * Only the first two levels are acutally used by virtio-scsi: * * Level 1: 0x01, 0xTT: Peripheral Device Addressing: Bus 1, Target 0-255 * Level 2: 0xLL, 0xLL: Peripheral Device Addressing: Bus MBZ, LUN 0-255 * or: Flat Space Addressing: LUN (0-16383) * Level 3 and 4: not used, MBZ * * * Alternatively, the first level may contain an extended LUN address to select * the REPORT_LUNS well-known logical unit: * * Level 1: 0xC1, 0x01: Extended LUN Adressing, Well-Known LUN 1 (REPORT_LUNS) * Level 2, 3, and 4: not used, MBZ * * The virtio spec says that we SHOULD implement the REPORT_LUNS well-known * logical unit but we currently don't. * * According to the virtio spec, these are the only LUNS address formats to be * used with virtio-scsi. */ /* * Check that the given LUN address conforms to the virtio spec, does not * address an unknown target, and especially does not address the REPORT_LUNS * well-known logical unit. */ static inline bool pci_vtscsi_check_lun(struct pci_vtscsi_softc *sc, const uint8_t *lun) { if (lun[0] == 0xC1) return (false); if (lun[0] != 0x01) return (false); if (lun[1] >= sc->vss_num_target) return (false); if (lun[1] != sc->vss_targets[lun[1]].vst_target) return (false); if (sc->vss_targets[lun[1]].vst_fd < 0) return (false); if (lun[2] != 0x00 && (lun[2] & 0xc0) != 0x40) return (false); if (lun[4] != 0 || lun[5] != 0 || lun[6] != 0 || lun[7] != 0) return (false); return (true); } /* * Get the target id from a LUN address. * * Every code path using this function must have called pci_vtscsi_check_lun() * before to make sure the LUN address is valid. */ static inline uint8_t pci_vtscsi_get_target(struct pci_vtscsi_softc *sc, const uint8_t *lun) { assert(lun[0] == 0x01); assert(lun[1] < sc->vss_num_target); assert(lun[1] == sc->vss_targets[lun[1]].vst_target); assert(sc->vss_targets[lun[1]].vst_fd >= 0); assert(lun[2] == 0x00 || (lun[2] & 0xc0) == 0x40); return (lun[1]); } /* * Get the LUN id from a LUN address. * * Every code path using this function must have called pci_vtscsi_check_lun() * before to make sure the LUN address is valid. */ static inline uint16_t pci_vtscsi_get_lun(struct pci_vtscsi_softc *sc, const uint8_t *lun) { assert(lun[0] == 0x01); assert(lun[1] < sc->vss_num_target); assert(lun[1] == sc->vss_targets[lun[1]].vst_target); assert(sc->vss_targets[lun[1]].vst_fd >= 0); assert(lun[2] == 0x00 || (lun[2] & 0xc0) == 0x40); return (((lun[2] << 8) | lun[3]) & 0x3fff); } #endif /* _PCI_VIRTIO_SCSI_H_ */ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2016 Jakub Klama . * Copyright (c) 2018 Marcelo Araujo . * Copyright (c) 2026 Hans Rosenfeld * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer * in this position and unchanged. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "bhyverun.h" #include "config.h" #include "debug.h" #include "pci_emul.h" #include "virtio.h" #include "iov.h" #include "pci_virtio_scsi.h" struct vtscsi_ctl_backend { struct pci_vtscsi_backend vcb_backend; int vcb_iid; }; static int vtscsi_ctl_init(struct pci_vtscsi_softc *, struct pci_vtscsi_backend *, nvlist_t *); static int vtscsi_ctl_open(struct pci_vtscsi_softc *, const char *, long); static void vtscsi_ctl_reset(struct pci_vtscsi_softc *); static void *vtscsi_ctl_req_alloc(struct pci_vtscsi_softc *); static void vtscsi_ctl_req_clear(void *); static void vtscsi_ctl_req_free(void *); static void vtscsi_ctl_tmf_hdl(struct pci_vtscsi_softc *, int, struct pci_vtscsi_ctrl_tmf *); static void vtscsi_ctl_an_hdl(struct pci_vtscsi_softc *, int, struct pci_vtscsi_ctrl_an *); static int vtscsi_ctl_req_hdl(struct pci_vtscsi_softc *, int, struct pci_vtscsi_request *); static int vtscsi_ctl_count_targets(const char *prefix __unused, const nvlist_t *parent __unused, const char *name __unused, int type, void *arg) { int *count = arg; if (type != NV_TYPE_STRING) { EPRINTLN("invalid target \"%s\" type: not a string", name); errno = EINVAL; return (-1); } (*count)++; return (0); } static int vtscsi_ctl_init(struct pci_vtscsi_softc *sc, struct pci_vtscsi_backend *backend, nvlist_t *nvl) { int count = 0; int ret = 0; struct vtscsi_ctl_backend *ctl_backend; const char *value; ctl_backend = calloc(1, sizeof(struct vtscsi_ctl_backend)); if (ctl_backend == NULL) { EPRINTLN("failed to allocate backend data: %s", strerror(errno)); return (-1); } ctl_backend->vcb_backend = *backend; sc->vss_backend = &ctl_backend->vcb_backend; value = get_config_value_node(nvl, "iid"); if (value != NULL) ctl_backend->vcb_iid = strtoul(value, NULL, 10); /* * Count configured targets. If no targets were configured, use * /dev/cam/ctl to remain compatible with previous versions. */ nvl = find_relative_config_node(nvl, "target"); if (nvl != NULL) ret = walk_config_nodes("", nvl, &count, vtscsi_ctl_count_targets); if (ret != 0) return (ret); if (count == 0) set_config_value_node(nvl, "0", "/dev/cam/ctl"); return (0); } static int vtscsi_ctl_open(struct pci_vtscsi_softc *sc __unused, const char *devname, long target) { struct pci_vtscsi_target *tgt = &sc->vss_targets[target]; tgt->vst_fd = open(devname, O_RDWR); if (tgt->vst_fd < 0) return (-1); return (0); } static void vtscsi_ctl_reset(struct pci_vtscsi_softc *sc __unused) { /* * There doesn't seem to be a limit to the maximum number of * sectors CTL can transfer in one request. */ sc->vss_config.max_sectors = INT32_MAX; } static void *vtscsi_ctl_req_alloc(struct pci_vtscsi_softc *sc) { struct vtscsi_ctl_backend *ctl = (struct vtscsi_ctl_backend *)sc; union ctl_io *io = ctl_scsi_alloc_io(ctl->vcb_iid); if (io != NULL) ctl_scsi_zero_io(io); return (io); } static void vtscsi_ctl_req_clear(void *io) { ctl_scsi_zero_io(io); } static void vtscsi_ctl_req_free(void *io) { ctl_scsi_free_io(io); } static void vtscsi_ctl_tmf_hdl(struct pci_vtscsi_softc *sc, int fd, struct pci_vtscsi_ctrl_tmf *tmf) { struct vtscsi_ctl_backend *ctl; union ctl_io *io; int err; ctl = (struct vtscsi_ctl_backend *)sc->vss_backend; io = vtscsi_ctl_req_alloc(sc); if (io == NULL) { tmf->response = VIRTIO_SCSI_S_FAILURE; return; } vtscsi_ctl_req_clear(io); io->io_hdr.io_type = CTL_IO_TASK; io->io_hdr.nexus.initid = ctl->vcb_iid; io->io_hdr.nexus.targ_lun = pci_vtscsi_get_lun(sc, tmf->lun); io->taskio.tag_type = CTL_TAG_SIMPLE; io->taskio.tag_num = tmf->id; io->io_hdr.flags |= CTL_FLAG_USER_TAG; switch (tmf->subtype) { case VIRTIO_SCSI_T_TMF_ABORT_TASK: io->taskio.task_action = CTL_TASK_ABORT_TASK; break; case VIRTIO_SCSI_T_TMF_ABORT_TASK_SET: io->taskio.task_action = CTL_TASK_ABORT_TASK_SET; break; case VIRTIO_SCSI_T_TMF_CLEAR_ACA: io->taskio.task_action = CTL_TASK_CLEAR_ACA; break; case VIRTIO_SCSI_T_TMF_CLEAR_TASK_SET: io->taskio.task_action = CTL_TASK_CLEAR_TASK_SET; break; case VIRTIO_SCSI_T_TMF_I_T_NEXUS_RESET: io->taskio.task_action = CTL_TASK_I_T_NEXUS_RESET; break; case VIRTIO_SCSI_T_TMF_LOGICAL_UNIT_RESET: io->taskio.task_action = CTL_TASK_LUN_RESET; break; case VIRTIO_SCSI_T_TMF_QUERY_TASK: io->taskio.task_action = CTL_TASK_QUERY_TASK; break; case VIRTIO_SCSI_T_TMF_QUERY_TASK_SET: io->taskio.task_action = CTL_TASK_QUERY_TASK_SET; break; } if (pci_vtscsi_debug) { struct sbuf *sb = sbuf_new_auto(); ctl_io_sbuf(io, sb); sbuf_finish(sb); DPRINTF("%s", sbuf_data(sb)); sbuf_delete(sb); } err = ioctl(fd, CTL_IO, io); if (err != 0) { WPRINTF("CTL_IO: err=%d (%s)", errno, strerror(errno)); tmf->response = VIRTIO_SCSI_S_FAILURE; } else { tmf->response = io->taskio.task_status; } vtscsi_ctl_req_free(io); } static void vtscsi_ctl_an_hdl(struct pci_vtscsi_softc *sc __unused, int fd __unused, struct pci_vtscsi_ctrl_an *an) { an->response = VIRTIO_SCSI_S_FAILURE; } static int vtscsi_ctl_req_hdl(struct pci_vtscsi_softc *sc, int fd, struct pci_vtscsi_request *req) { union ctl_io *io = req->vsr_backend; void *ext_data_ptr = NULL; uint32_t ext_data_len = 0, ext_sg_entries = 0; struct vtscsi_ctl_backend *ctl; int err, nxferred; ctl = (struct vtscsi_ctl_backend *)sc->vss_backend; io->io_hdr.nexus.initid = ctl->vcb_iid; io->io_hdr.nexus.targ_lun = pci_vtscsi_get_lun(sc, req->vsr_cmd_rd->lun); io->io_hdr.io_type = CTL_IO_SCSI; if (req->vsr_data_niov_in > 0) { ext_data_ptr = (void *)req->vsr_data_iov_in; ext_sg_entries = req->vsr_data_niov_in; ext_data_len = count_iov(req->vsr_data_iov_in, req->vsr_data_niov_in); io->io_hdr.flags |= CTL_FLAG_DATA_OUT; } else if (req->vsr_data_niov_out > 0) { ext_data_ptr = (void *)req->vsr_data_iov_out; ext_sg_entries = req->vsr_data_niov_out; ext_data_len = count_iov(req->vsr_data_iov_out, req->vsr_data_niov_out); io->io_hdr.flags |= CTL_FLAG_DATA_IN; } io->scsiio.sense_len = sc->vss_config.sense_size; io->scsiio.tag_num = req->vsr_cmd_rd->id; io->io_hdr.flags |= CTL_FLAG_USER_TAG; switch (req->vsr_cmd_rd->task_attr) { case VIRTIO_SCSI_S_ORDERED: io->scsiio.tag_type = CTL_TAG_ORDERED; break; case VIRTIO_SCSI_S_HEAD: io->scsiio.tag_type = CTL_TAG_HEAD_OF_QUEUE; break; case VIRTIO_SCSI_S_ACA: io->scsiio.tag_type = CTL_TAG_ACA; break; case VIRTIO_SCSI_S_SIMPLE: default: io->scsiio.tag_type = CTL_TAG_SIMPLE; break; } io->scsiio.ext_sg_entries = ext_sg_entries; io->scsiio.ext_data_ptr = ext_data_ptr; io->scsiio.ext_data_len = ext_data_len; io->scsiio.ext_data_filled = 0; io->scsiio.cdb_len = sc->vss_config.cdb_size; memcpy(io->scsiio.cdb, req->vsr_cmd_rd->cdb, sc->vss_config.cdb_size); if (pci_vtscsi_debug) { struct sbuf *sb = sbuf_new_auto(); ctl_io_sbuf(io, sb); sbuf_finish(sb); DPRINTF("%s", sbuf_data(sb)); sbuf_delete(sb); } err = ioctl(fd, CTL_IO, io); if (err != 0) { WPRINTF("CTL_IO: err=%d (%s)", errno, strerror(errno)); req->vsr_cmd_wr->response = VIRTIO_SCSI_S_FAILURE; } else { req->vsr_cmd_wr->sense_len = MIN(io->scsiio.sense_len, sc->vss_config.sense_size); req->vsr_cmd_wr->residual = ext_data_len - io->scsiio.ext_data_filled; req->vsr_cmd_wr->status = io->scsiio.scsi_status; req->vsr_cmd_wr->response = VIRTIO_SCSI_S_OK; memcpy(&req->vsr_cmd_wr->sense, &io->scsiio.sense_data, req->vsr_cmd_wr->sense_len); } nxferred = io->scsiio.ext_data_filled; return (nxferred); } static const struct pci_vtscsi_backend vtscsi_ctl_backend = { .vsb_name = "ctl", .vsb_init = vtscsi_ctl_init, .vsb_open = vtscsi_ctl_open, .vsb_reset = vtscsi_ctl_reset, .vsb_req_alloc = vtscsi_ctl_req_alloc, .vsb_req_clear = vtscsi_ctl_req_clear, .vsb_req_free = vtscsi_ctl_req_free, .vsb_tmf_hdl = vtscsi_ctl_tmf_hdl, .vsb_an_hdl = vtscsi_ctl_an_hdl, .vsb_req_hdl = vtscsi_ctl_req_hdl }; PCI_VTSCSI_BACKEND_SET(vtscsi_ctl_backend); /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2016 Jakub Klama . * Copyright (c) 2018 Marcelo Araujo . * Copyright (c) 2026 Hans Rosenfeld * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer * in this position and unchanged. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "bhyverun.h" #include "config.h" #include "debug.h" #include "pci_emul.h" #include "virtio.h" #include "iov.h" #include "privileges.h" #include "pci_virtio_scsi.h" struct vtscsi_uscsi_backend { struct pci_vtscsi_backend vub_backend; libscsi_hdl_t *vub_scsi_hdl; }; static int vtscsi_uscsi_init(struct pci_vtscsi_softc *, struct pci_vtscsi_backend *, nvlist_t *); static int vtscsi_uscsi_open(struct pci_vtscsi_softc *, const char *, long); static void vtscsi_uscsi_reset(struct pci_vtscsi_softc *); static void *vtscsi_uscsi_req_alloc(struct pci_vtscsi_softc *); static void vtscsi_uscsi_req_clear(void *); static void vtscsi_uscsi_req_free(void *); static void vtscsi_uscsi_tmf_hdl(struct pci_vtscsi_softc *, int, struct pci_vtscsi_ctrl_tmf *); static void vtscsi_uscsi_an_hdl(struct pci_vtscsi_softc *, int, struct pci_vtscsi_ctrl_an *); static int vtscsi_uscsi_req_hdl(struct pci_vtscsi_softc *, int, struct pci_vtscsi_request *); static void vtscsi_uscsi_make_check_condition(struct uscsi_cmd *, char, char, char); static void vtscsi_uscsi_filter_post_report_luns(struct uscsi_cmd *); static void vtscsi_uscsi_filter_post(struct uscsi_cmd *); static int vtscsi_uscsi_init(struct pci_vtscsi_softc *sc, struct pci_vtscsi_backend *backend, nvlist_t *nvl __unused) { struct vtscsi_uscsi_backend *uscsi_backend; libscsi_errno_t serr; uscsi_backend = calloc(1, sizeof (struct vtscsi_uscsi_backend)); if (uscsi_backend == NULL) { EPRINTLN("failed to allocate backend data: %s", strerror(errno)); return (-1); } uscsi_backend->vub_backend = *backend; uscsi_backend->vub_scsi_hdl = libscsi_init(LIBSCSI_VERSION, &serr); if (uscsi_backend->vub_scsi_hdl == NULL) { EPRINTLN("failed to initialize libscsi: %s", libscsi_strerror(serr)); free(uscsi_backend); return (-1); } sc->vss_backend = &uscsi_backend->vub_backend; return (0); } static int vtscsi_uscsi_open(struct pci_vtscsi_softc *sc, const char *path, long target) { struct pci_vtscsi_target *tgt = &sc->vss_targets[target]; uscsi_xfer_t maxxfer = 0; /* * Most SCSI target drivers require the SYS_DEVICES privilege to send * USCSI commands. */ illumos_priv_add_min(PRIV_SYS_DEVICES, "scsi"); /* * Open the target. */ tgt->vst_fd = open(path, O_RDWR); if (tgt->vst_fd < 0) return (-1); /* * Get the maximum transfer size of the backend device. */ if (ioctl(tgt->vst_fd, USCSIMAXXFER, &maxxfer) < 0) { int errno_save = errno; if (errno == ENOTTY) { /* * The underlying device doesn't support this ioctl. * Limit max_sectors to 128MB, which is as good as * any other assumption. */ tgt->vst_max_sectors = 128 << (20 - 9); return (0); } WPRINTF("USCSIMAXXFER: unexpected error: errno=%d (%s)", strerrorname_np(errno), strerror(errno)); (void) close(tgt->vst_fd); tgt->vst_fd = -1; errno = errno_save; return (-1); } /* * Even though the virtio spec isn't particularly verbose about what * "max_sectors" actually means and what size a sector is, Linux seems * to treat it as a number of 512b sectors. * * In any case, we need to limit maxxfer such that it fits into a signed * 32bit int. */ if (maxxfer > INT32_MAX) maxxfer = INT32_MAX; tgt->vst_max_sectors = maxxfer >> 9; return (0); } static void vtscsi_uscsi_reset(struct pci_vtscsi_softc *sc) { size_t i; sc->vss_config.max_sectors = INT32_MAX; /* * As we may be configured to use a variety of differing backend devices * with varying maximum transfer sizes but virtio-scsi supports only one * max_sectors limit per instance, we'll use the smallest maximum * transfer size found. */ for (i = 0; i < sc->vss_num_target; i++) { struct pci_vtscsi_target *tgt = &sc->vss_targets[i]; if (tgt->vst_max_sectors < sc->vss_config.max_sectors) sc->vss_config.max_sectors = tgt->vst_max_sectors; } } static void * vtscsi_uscsi_req_alloc(struct pci_vtscsi_softc *sc) { return (calloc(1, sizeof (struct uscsi_cmd))); } static void vtscsi_uscsi_req_clear(void *io) { bzero(io, sizeof (struct uscsi_cmd)); } static void vtscsi_uscsi_req_free(void *io) { free(io); } static void vtscsi_uscsi_tmf_hdl(struct pci_vtscsi_softc *sc __unused, int fd, struct pci_vtscsi_ctrl_tmf *tmf) { struct uscsi_cmd cmd; int err; /* We currently support only LUN 0. */ if (pci_vtscsi_get_lun(sc, tmf->lun) != 0) { tmf->response = VIRTIO_SCSI_S_BAD_TARGET; return; } tmf->response = VIRTIO_SCSI_S_FUNCTION_COMPLETE; memset(&cmd, 0, sizeof (cmd)); cmd.uscsi_status = -1; cmd.uscsi_flags = USCSI_DIAGNOSE | USCSI_SILENT; /* The only TMF requests that we can handle here are RESETs. */ switch (tmf->subtype) { case VIRTIO_SCSI_T_TMF_I_T_NEXUS_RESET: cmd.uscsi_flags |= USCSI_RESET_TARGET; break; case VIRTIO_SCSI_T_TMF_LOGICAL_UNIT_RESET: cmd.uscsi_flags |= USCSI_RESET_LUN; break; default: /* * For all other TMF requests, return FUNCTION COMPLETE as * there is nothing we can or need to do for them. * * See the comments in pci_vtscsi_tmf_handle() for additional * information on how the common code and the backend-specific * code interact for TMF requests. */ tmf->response = VIRTIO_SCSI_S_FUNCTION_COMPLETE; return; } err = ioctl(fd, USCSICMD, &cmd); if (err != 0) { WPRINTF("USCSICMD: unexpected TMF error, errno=%d (%s)", strerrorname_np(errno), strerror(errno)); tmf->response = VIRTIO_SCSI_S_FAILURE; } } static void vtscsi_uscsi_an_hdl(struct pci_vtscsi_softc *sc __unused, int fd __unused, struct pci_vtscsi_ctrl_an *an) { /* We currently support only LUN 0. */ if (pci_vtscsi_get_lun(sc, an->lun) != 0) { an->response = VIRTIO_SCSI_S_BAD_TARGET; return; } an->response = VIRTIO_SCSI_S_FAILURE; } static int vtscsi_uscsi_req_hdl(struct pci_vtscsi_softc *sc, int fd, struct pci_vtscsi_request *req) { struct vtscsi_uscsi_backend *uscsi = (struct vtscsi_uscsi_backend *)sc->vss_backend; struct uscsi_cmd *cmd = req->vsr_backend; void *ext_data = NULL; ssize_t ext_data_len = 0; int nxferred = 0; /* We currently support only LUN 0. */ if (pci_vtscsi_get_lun(sc, req->vsr_cmd_rd->lun) != 0) { req->vsr_cmd_wr->response = VIRTIO_SCSI_S_BAD_TARGET; return (0); } if (req->vsr_data_niov_in > 0) { ext_data_len = iov_to_buf(req->vsr_data_iov_in, req->vsr_data_niov_in, &ext_data); cmd->uscsi_flags |= USCSI_WRITE; } else if (req->vsr_data_niov_out > 0) { ext_data_len = count_iov(req->vsr_data_iov_out, req->vsr_data_niov_out); ext_data = malloc(ext_data_len); cmd->uscsi_flags |= USCSI_READ; } /* Stop here if we failed to allocate ext_data. */ if (ext_data == NULL && ext_data_len != 0) { WPRINTF("failed to allocate buffer for ext_data"); req->vsr_cmd_wr->response = VIRTIO_SCSI_S_FAILURE; return (0); } cmd->uscsi_buflen = ext_data_len; cmd->uscsi_bufaddr = ext_data; cmd->uscsi_cdb = (caddr_t)req->vsr_cmd_rd->cdb; cmd->uscsi_cdblen = libscsi_cmd_cdblen(uscsi->vub_scsi_hdl, req->vsr_cmd_rd->cdb[0]); cmd->uscsi_status = -1; /* * We set an unreasonably large timeout here. The virtio spec doesn't * provide a way for the guest driver to pass a I/O timeout value to * the device, but if our timeout here is larger than any timeout the * guest uses, we can expect them to abort the command before we would. * * INT16_MAX corresponds to a bit over 9 hours, which should be enough. */ cmd->uscsi_timeout = INT16_MAX; cmd->uscsi_flags |= USCSI_DIAGNOSE; cmd->uscsi_rqlen = sc->vss_config.sense_size; cmd->uscsi_rqbuf = (caddr_t)req->vsr_cmd_wr->sense; cmd->uscsi_flags |= USCSI_RQENABLE; switch (req->vsr_cmd_rd->task_attr) { case VIRTIO_SCSI_S_ORDERED: cmd->uscsi_flags |= USCSI_OTAG; break; case VIRTIO_SCSI_S_HEAD: cmd->uscsi_flags |= USCSI_HEAD|USCSI_HTAG; break; case VIRTIO_SCSI_S_SIMPLE: break; case VIRTIO_SCSI_S_ACA: /* * I haven't found any indication in our code that would * suggest that we support ACA in any way in illumos. In * fact, scsi_transport() asserts that NACA isn't set in * a packet, and scsi_uscsi_pktinit() warns about it and * clears the flag if found set. There's a tunable to * override that behaviour (scsi_pkt_allow_naca), but there * really seems to be no code properly handling ACA or * setting the ACA flag. * * I guess this makes sense since we're doing ARQ anyway, * so let's just pretend no target is ever in ACA state * and thus no packet will ever require this. */ default: WPRINTF("USCSICMD: unexpected task attr in request: 0x%x", req->vsr_cmd_rd->task_attr); req->vsr_cmd_wr->response = VIRTIO_SCSI_S_FAILURE; return (0); } errno = 0; (void) ioctl(fd, USCSICMD, cmd); switch (errno) { case EIO: /* * EIO may indicate that a SCSI error occured. If that's the * case, uscsi_status should have been set to a valid value, * and we want to continue to process the request normally. */ if (cmd->uscsi_status == -1) { req->vsr_cmd_wr->response = VIRTIO_SCSI_S_FAILURE; break; } /*FALLTHRU*/ case 0: /* * If the command completed successfully, apply any necessary * post-completion filtering. */ if (cmd->uscsi_status == STATUS_GOOD) vtscsi_uscsi_filter_post(cmd); req->vsr_cmd_wr->sense_len = sc->vss_config.sense_size - cmd->uscsi_rqresid; req->vsr_cmd_wr->residual = cmd->uscsi_resid; req->vsr_cmd_wr->status = cmd->uscsi_status; req->vsr_cmd_wr->response = VIRTIO_SCSI_S_OK; nxferred = ext_data_len - req->vsr_cmd_wr->residual; if (req->vsr_data_niov_out > 0) { (void) buf_to_iov(ext_data, nxferred, req->vsr_data_iov_out, req->vsr_data_niov_out); } break; case EAGAIN: /* * Despite not being documented in uscsi(4I), sd(4D) returns * this when the device is busy formatting. */ req->vsr_cmd_wr->response = VIRTIO_SCSI_S_BUSY; break; case EINVAL: /* * This may happen if packet allocation fails, which in turn * may happen if we didn't honor USCSIMAXXFER. */ req->vsr_cmd_wr->response = VIRTIO_SCSI_S_OVERRUN; break; case EFAULT: /* * EFAULT should never happen as we never send bogus memory * addresses in our USCSI commands. */ case EPERM: /* * EPERM should never happen as we have the SYS_DEVICES * privilege. */ default: WPRINTF("USCSICMD: unexpected I/O error: errno=%d (%s)", strerrorname_np(errno), strerror(errno)); abort(); } free(ext_data); return (nxferred); } /* * Return a CHECK CONDITION and fill in the sense data with the given sense key, * additional sense code, and additional sense qualifier. */ static void vtscsi_uscsi_make_check_condition(struct uscsi_cmd *cmd, char key, char asc, char qual) { cmd->uscsi_status = STATUS_CHECK; cmd->uscsi_resid = cmd->uscsi_buflen; cmd->uscsi_rqstatus = STATUS_GOOD; bzero(cmd->uscsi_rqbuf, cmd->uscsi_rqlen); if (cmd->uscsi_rqlen >= 1) cmd->uscsi_rqbuf[0] = 0x70; if (cmd->uscsi_rqlen >= 3) cmd->uscsi_rqbuf[2] = key; if (cmd->uscsi_rqlen >= 8) cmd->uscsi_rqbuf[7] = cmd->uscsi_rqlen - 8; if (cmd->uscsi_rqlen >= 13) cmd->uscsi_rqbuf[12] = asc; if (cmd->uscsi_rqlen >= 14) cmd->uscsi_rqbuf[13] = qual; } /* * We currently only support LUN 0. Make sure we never report anything else. * * We make no assumption about the buffer size. If it's large enough to hold the * LUN list length, we'll set the LUN list length to 8. The resid is adjusted if * the buffer size is larger than 16 bytes, which is the length needed to hold * one LUN address. */ static void vtscsi_uscsi_filter_post_report_luns(struct uscsi_cmd *cmd) { uint8_t report = (uint8_t)cmd->uscsi_cdb[2]; bzero(cmd->uscsi_bufaddr, cmd->uscsi_buflen); switch (report) { case 0: case 2: /* * We'll overwrite the output from the device to report just one * LUN with an all-zero address: * - LUN list length is 8 * - LUN 1 address is 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 */ if (cmd->uscsi_buflen >= 4) cmd->uscsi_bufaddr[3] = 8; if (cmd->uscsi_buflen >= 16) cmd->uscsi_resid = cmd->uscsi_buflen - 16; break; case 1: /* * We don't report any Well-Known LUNs either, because we have * no way to address them anyway using USCSICMD. */ cmd->uscsi_resid = cmd->uscsi_buflen; break; default: /* * All other values for "select report" are either invalid or * vendor-specific and thus unsupported. Return the command with * CHECK CONDITION, and fill in sense data to report a ILLEGAL * REQUEST with INVALID FIELD IN CDB. */ vtscsi_uscsi_make_check_condition(cmd, KEY_ILLEGAL_REQUEST, 0x24, 0x00); } } static void vtscsi_uscsi_filter_post(struct uscsi_cmd *cmd) { switch ((uint8_t)cmd->uscsi_cdb[0]) { case SCMD_REPORT_LUNS: vtscsi_uscsi_filter_post_report_luns(cmd); break; default: break; } } static const struct pci_vtscsi_backend vtscsi_uscsi_backend = { .vsb_name = "uscsi", .vsb_init = vtscsi_uscsi_init, .vsb_open = vtscsi_uscsi_open, .vsb_reset = vtscsi_uscsi_reset, .vsb_req_alloc = vtscsi_uscsi_req_alloc, .vsb_req_clear = vtscsi_uscsi_req_clear, .vsb_req_free = vtscsi_uscsi_req_free, .vsb_tmf_hdl = vtscsi_uscsi_tmf_hdl, .vsb_an_hdl = vtscsi_uscsi_an_hdl, .vsb_req_hdl = vtscsi_uscsi_req_hdl }; PCI_VTSCSI_BACKEND_SET(vtscsi_uscsi_backend); /* * Copyright (c) 2011 NetApp, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * This file and its contents are supplied under the terms of the * Common Development and Distribution License ("CDDL"), version 1.0. * You may only use this file in accordance with the terms of version * 1.0 of the CDDL. * * A full copy of the text of the CDDL should have accompanied this * source. A copy of the CDDL is also available via the Internet at * http://www.illumos.org/license/CDDL. * * Copyright 2015 Pluribus Networks Inc. * Copyright 2019 Joyent, Inc. * Copyright 2022 OmniOS Community Edition (OmniOSce) Association. * Copyright 2026 Oxide Computer Company */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "bhyverun.h" #include "config.h" #include "debug.h" #include "pci_emul.h" #include "virtio.h" #include "iov.h" #include "virtio_net.h" /* * This is the default number of queues allocated and advertised via the * multi-queue feature. It can be overridden via the `qpair` device option. */ #define VIONA_DEFAULT_MAX_QPAIR 8 /* * These are the default TX and RX ring sizes. They can be overidden with the * `vqsize`, `rxvqsize` and `txvqsize` options. The first sets both values to * the same while the last two allow setting different values for TX and RX. * * It is common to have asymmetry here as the RX queue must absorb host-side * burstiness while some backpressure is desirable on the TX side. */ #define VIONA_DEFAULT_TX_RINGSZ 256 #define VIONA_DEFAULT_RX_RINGSZ 1024 #define VIONA_CTLQ_SIZE 64 #define VIONA_CTLQ_MAXSEGS 32 #define VIONA_DEFAULT_LINK_SPEED 1000 /* In Mb/s, so this is 1Gb/s */ /* * These macros work in terms of TX/RX queues only, which is always what we * need when interfacing with viona since the control queue is implemented * entirely in userspace. */ #define VIONA_P2QS(p) ((p) * 2) #define VIONA_NRINGS(sc) ((sc)->vsc_consts.vc_nvq - 1) #define VIONA_USABLE_RINGS(sc) (VIONA_P2QS((sc)->vsc_vq_usepairs)) #define VIONA_RING_VALID(sc, r) ((r) < VIONA_NRINGS(sc)) #define VIONA_RING(sc, n) (&(sc)->vsc_queues[(n)]) #define VIONA_RXQ(sc, n) (VIONA_RING(sc, (n) * 2)) #define VIONA_TXQ(sc, n) (VIONA_RING(sc, (n) * 2 + 1)) #define IS_RXQ(i) ((i) % 2 == 0) #define IS_TXQ(i) ((i) % 2 != 0) /* * The control queue is always in the last slot of allocated rings, regardless * of how many rings are in use. */ #define VIONA_CTLQ_NUM(sc) (VIONA_NRINGS(sc)) #define VIONA_RING_CTLQ(sc, r) (r == VIONA_CTLQ_NUM(sc)) #define VIONA_CTLQ(sc) (VIONA_RING(sc, VIONA_CTLQ_NUM(sc))) /* * Debug printf */ static volatile int pci_viona_debug; #define DPRINTF(fmt, arg...) \ do { \ if (pci_viona_debug) { \ FPRINTLN(stdout, fmt, ##arg); \ fflush(stdout); \ } \ } while (0) #define WPRINTF(fmt, arg...) FPRINTLN(stderr, fmt, ##arg) /* * Per-device softc */ struct pci_viona_softc { struct virtio_softc vsc_vs; struct virtio_consts vsc_consts; struct vqueue_info *vsc_queues; pthread_mutex_t vsc_mtx; struct virtio_net_config vsc_config; datalink_id_t vsc_linkid; int vsc_vnafd; /* Configurable parameters */ char vsc_linkname[MAXLINKNAMELEN]; uint64_t vsc_feature_mask; uint16_t vsc_vq_usepairs; /* RX/TX pairs in use */ uint16_t vsc_vq_rxsize; /* size of each RX queue */ uint16_t vsc_vq_txsize; /* size of each TX queue */ bool vsc_resetting; bool vsc_msix_active; viona_promisc_t vsc_promisc; /* Current promisc mode */ bool vsc_promisc_promisc; /* PROMISC enabled */ bool vsc_promisc_allmulti; /* ALLMULTI enabled */ bool vsc_promisc_umac; /* unicast MACs sent */ bool vsc_promisc_mmac; /* multicast MACs sent */ }; static int pci_viona_cfgread(void *, int, int, uint32_t *); static int pci_viona_cfgwrite(void *, int, int, uint32_t); static uint64_t pci_viona_get_hv_features(void *, bool); static void pci_viona_set_hv_features(void *, uint64_t *); static void pci_viona_qnotify(void *, struct vqueue_info *); static void pci_viona_ctlqnotify(void *, struct vqueue_info *); static void pci_viona_ring_init(void *, uint64_t, bool); static void pci_viona_reset(void *); static void pci_viona_ring_set_msix(void *, int); static void pci_viona_update_msix(void *, uint64_t); static virtio_capstr_t viona_caps[] = { { VIRTIO_NET_F_CSUM, "VIRTIO_NET_F_CSUM" }, { VIRTIO_NET_F_GUEST_CSUM, "VIRTIO_NET_F_GUEST_CSUM" }, { VIRTIO_NET_F_MTU, "VIRTIO_NET_F_MTU" }, { VIRTIO_NET_F_MAC, "VIRTIO_NET_F_MAC" }, { VIRTIO_NET_F_GSO_DEPREC, "VIRTIO_NET_F_GSO_DEPREC" }, { VIRTIO_NET_F_GUEST_TSO4, "VIRTIO_NET_F_GUEST_TSO4" }, { VIRTIO_NET_F_GUEST_TSO6, "VIRTIO_NET_F_GUEST_TSO6" }, { VIRTIO_NET_F_GUEST_ECN, "VIRTIO_NET_F_GUEST_ECN" }, { VIRTIO_NET_F_GUEST_UFO, "VIRTIO_NET_F_GUEST_UFO" }, { VIRTIO_NET_F_HOST_TSO4, "VIRTIO_NET_F_HOST_TSO4" }, { VIRTIO_NET_F_HOST_TSO6, "VIRTIO_NET_F_HOST_TSO6" }, { VIRTIO_NET_F_HOST_ECN, "VIRTIO_NET_F_HOST_ECN" }, { VIRTIO_NET_F_HOST_UFO, "VIRTIO_NET_F_HOST_UFO" }, { VIRTIO_NET_F_MRG_RXBUF, "VIRTIO_NET_F_MRG_RXBUF" }, { VIRTIO_NET_F_STATUS, "VIRTIO_NET_F_STATUS" }, { VIRTIO_NET_F_CTRL_VQ, "VIRTIO_NET_F_CTRL_VQ" }, { VIRTIO_NET_F_CTRL_RX, "VIRTIO_NET_F_CTRL_RX" }, { VIRTIO_NET_F_CTRL_VLAN, "VIRTIO_NET_F_CTRL_VLAN" }, { VIRTIO_NET_F_GUEST_ANNOUNCE, "VIRTIO_NET_F_GUEST_ANNOUNCE" }, { VIRTIO_NET_F_MQ, "VIRTIO_NET_F_MQ" }, { VIRTIO_F_CTRL_MAC_ADDR, "VIRTIO_F_CTRL_MAC_ADDR" }, { VIRTIO_NET_F_SPEED_DUPLEX, "VIRTIO_NET_F_SPEED_DUPLEX" }, }; static struct virtio_consts viona_vi_consts = { .vc_name = "viona", .vc_nvq = 0, /* set in pci_viona_qalloc() */ .vc_max_nvq = 0, /* set in pci_viona_parse_opts() */ .vc_cfgsize = sizeof (struct virtio_net_config), .vc_cfgread = pci_viona_cfgread, .vc_cfgwrite = pci_viona_cfgwrite, .vc_set_msix = pci_viona_ring_set_msix, .vc_update_msix = pci_viona_update_msix, .vc_reset = pci_viona_reset, .vc_qinit = pci_viona_ring_init, .vc_qnotify = pci_viona_qnotify, .vc_hv_features = pci_viona_get_hv_features, .vc_apply_features = pci_viona_set_hv_features, /* * The following fields are populated using the response from the * viona driver during initialisation, augmented with the additional * capabilities emulated in userspace. */ .vc_hv_caps_legacy = 0, .vc_hv_caps_modern = 0, .vc_capstr = viona_caps, .vc_ncapstr = ARRAY_SIZE(viona_caps), }; static void pci_viona_ring_reset(struct pci_viona_softc *sc, int ring) { DPRINTF("viona: ring reset 0x%x", ring); for (;;) { int res; res = ioctl(sc->vsc_vnafd, VNA_IOC_RING_RESET, ring); if (res == 0) { break; } else if (errno != EINTR) { WPRINTF("ioctl viona ring 0x%x reset failed %d", ring, errno); return; } } } static bool pci_viona_set_usepairs(struct pci_viona_softc *sc, uint16_t pairs) { const uint16_t opairs = sc->vsc_vq_usepairs; DPRINTF("QUSE pairs 0x%x -> 0x%x", opairs, pairs); if (opairs == pairs) return (true); if (ioctl(sc->vsc_vnafd, VNA_IOC_SET_USEPAIRS, pairs) != 0) { WPRINTF("error setting viona use pairs(0x%x): %s", pairs, strerror(errno)); return (false); } sc->vsc_vq_usepairs = pairs; return (true); } static bool pci_viona_qalloc(struct pci_viona_softc *sc, int pairs) { struct virtio_consts *vc = &sc->vsc_consts; struct vqueue_info *queues; int nqueues, oqueues; oqueues = vc->vc_nvq; /* Add one for the control queue */ nqueues = VIONA_P2QS(pairs) + 1; DPRINTF("QALLOC pairs 0x%x (0x%x -> 0x%x)", pairs, oqueues, nqueues); if (oqueues == nqueues) return (true); queues = recallocarray(sc->vsc_queues, oqueues, nqueues, sizeof (struct vqueue_info)); if (queues == NULL) { WPRINTF("Failed to allocate memory changing queues from " "0x%x to 0x%x", oqueues, nqueues); return (false); } sc->vsc_queues = queues; vc->vc_nvq = nqueues; for (uint_t i = 0; i < vc->vc_nvq; i++) { sc->vsc_queues[i].vq_qsize = IS_RXQ(i) ? sc->vsc_vq_rxsize : sc->vsc_vq_txsize; sc->vsc_queues[i].vq_notify = NULL; } VIONA_CTLQ(sc)->vq_qsize = VIONA_CTLQ_SIZE; VIONA_CTLQ(sc)->vq_notify = pci_viona_ctlqnotify; vi_queue_linkup(&sc->vsc_vs, sc->vsc_queues); if (ioctl(sc->vsc_vnafd, VNA_IOC_SET_PAIRS, pairs) != 0) { WPRINTF("error setting viona queue pairs(0x%x): %s", pairs, strerror(errno)); return (false); } return (true); } static void pci_viona_reset(void *vsc) { struct pci_viona_softc *sc = vsc; DPRINTF("viona: device reset requested !"); vi_reset_dev(&sc->vsc_vs); /* Reset all TX/RX rings */ for (uint16_t i = 0; i < VIONA_NRINGS(sc); i++) pci_viona_ring_reset(sc, i); /* Shrink back down to one queue pair */ VERIFY(pci_viona_set_usepairs(sc, 1)); VERIFY(pci_viona_qalloc(sc, 1)); } static const char * pci_viona_promisc_descr(viona_promisc_t mode) { switch (mode) { case VIONA_PROMISC_NONE: return ("none"); case VIONA_PROMISC_MULTI: return ("multicast"); case VIONA_PROMISC_ALL: return ("all"); default: abort(); } } static int pci_viona_eval_promisc(struct pci_viona_softc *sc) { viona_promisc_t mode = VIONA_PROMISC_NONE; int err = 0; /* * If the guest has explicitly requested promiscuous mode or has sent a * non-empty unicast MAC address table, then set viona to promiscuous * mode. Otherwise, if the guest has explicitly requested multicast * promiscuity or has sent a non-empty multicast MAC address table, * then set viona to multicast promiscuous mode. */ if (sc->vsc_promisc_promisc || sc->vsc_promisc_umac) mode = VIONA_PROMISC_ALL; else if (sc->vsc_promisc_allmulti || sc->vsc_promisc_mmac) mode = VIONA_PROMISC_MULTI; if (mode != sc->vsc_promisc) { DPRINTF("viona: setting promiscuous mode to %d (%s)", mode, pci_viona_promisc_descr(mode)); DPRINTF(" promisc=%u, umac=%u, allmulti=%u, mmac=%u", sc->vsc_promisc_promisc, sc->vsc_promisc_umac, sc->vsc_promisc_allmulti, sc->vsc_promisc_mmac); err = ioctl(sc->vsc_vnafd, VNA_IOC_SET_PROMISC, mode); if (err == 0) sc->vsc_promisc = mode; else WPRINTF("ioctl viona set promisc failed %d", errno); } return (err); } static uint8_t pci_viona_control_rx(struct vqueue_info *vq, const virtio_net_ctrl_hdr_t *hdr, iov_bunch_t *iob) { struct pci_viona_softc *sc = (struct pci_viona_softc *)vq->vq_vs; uint8_t v; if (!iov_bunch_copy(iob, &v, sizeof (v))) { EPRINTLN("viona: bad control RX data"); return (VIRTIO_NET_CQ_ERR); } switch (hdr->vnch_command) { case VIRTIO_NET_CTRL_RX_PROMISC: DPRINTF("viona: ctrl RX promisc %d", v); sc->vsc_promisc_promisc = (v != 0); break; case VIRTIO_NET_CTRL_RX_ALLMULTI: DPRINTF("viona: ctrl RX allmulti %d", v); sc->vsc_promisc_allmulti = (v != 0); break; default: /* * VIRTIO_NET_F_CTRL_RX_EXTRA was not offered so no other * commands are expected. */ EPRINTLN("viona: unrecognised RX control cmd %u", hdr->vnch_command); return (VIRTIO_NET_CQ_ERR); } if (pci_viona_eval_promisc(sc) == 0) return (VIRTIO_NET_CQ_OK); return (VIRTIO_NET_CQ_ERR); } static void pci_viona_control_mac_dump(const char *tag, iov_bunch_t *iob, uint32_t cnt) { if (!pci_viona_debug) { (void) iov_bunch_skip(iob, cnt * ETHERADDRL); return; } DPRINTF("-- %s MAC TABLE (entries: %u)", tag, cnt); for (uint32_t i = 0; i < cnt; i++) { ether_addr_t mac; if (!iov_bunch_copy(iob, &mac, sizeof (mac))) return; DPRINTF(" [%2d] %s", i, ether_ntoa((struct ether_addr *)mac)); } } static uint8_t pci_viona_control_mac(struct vqueue_info *vq, const virtio_net_ctrl_hdr_t *hdr, iov_bunch_t *iob) { struct pci_viona_softc *sc = (struct pci_viona_softc *)vq->vq_vs; switch (hdr->vnch_command) { case VIRTIO_NET_CTRL_MAC_TABLE_SET: { virtio_net_ctrl_mac_t table; DPRINTF("viona: ctrl MAC table set"); /* * We advertise VIRTIO_NET_F_CTRL_RX and therefore need to * accept VIRTIO_NET_CTRL_MAC, but we don't support passing * changes in the MAC address lists down to viona. * Instead, we set flags to indicate if the guest has sent * any MAC addresses for each table, and use these to determine * the resulting promiscuous mode, see pci_viona_eval_promisc() * above. */ /* Unicast MAC table */ if (!iov_bunch_copy(iob, &table.vncm_entries, sizeof (table.vncm_entries))) { EPRINTLN("viona: bad control MAC unicast header"); return (VIRTIO_NET_CQ_ERR); } sc->vsc_promisc_umac = (table.vncm_entries != 0); pci_viona_control_mac_dump("UNICAST", iob, table.vncm_entries); /* Multicast MAC table */ if (!iov_bunch_copy(iob, &table.vncm_entries, sizeof (table.vncm_entries))) { EPRINTLN("viona: bad control MAC multicast header"); return (VIRTIO_NET_CQ_ERR); } sc->vsc_promisc_mmac = (table.vncm_entries != 0); pci_viona_control_mac_dump("MULTICAST", iob, table.vncm_entries); break; } case VIRTIO_NET_CTRL_MAC_ADDR_SET: /* disallow setting the primary filter MAC address */ DPRINTF("viona: ctrl MAC addr set with 0x%x bytes", iob->ib_remain); return (VIRTIO_NET_CQ_ERR); default: EPRINTLN("viona: unrecognised MAC control cmd %u", hdr->vnch_command); return (VIRTIO_NET_CQ_ERR); } if (pci_viona_eval_promisc(sc) == 0) return (VIRTIO_NET_CQ_OK); return (VIRTIO_NET_CQ_ERR); } static uint8_t pci_viona_control_mq(struct vqueue_info *vq, const virtio_net_ctrl_hdr_t *hdr, iov_bunch_t *iob) { struct pci_viona_softc *sc = (struct pci_viona_softc *)vq->vq_vs; switch (hdr->vnch_command) { case VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET: { virtio_net_ctrl_mq_t mq; if (!iov_bunch_copy(iob, &mq, sizeof (mq))) { EPRINTLN("viona: bad control MQ data"); return (VIRTIO_NET_CQ_ERR); } DPRINTF("VQ PAIRS SET 0x%x", mq.virtqueue_pairs); if (mq.virtqueue_pairs < 1 || VIONA_P2QS(mq.virtqueue_pairs) > VIONA_NRINGS(sc)) { EPRINTLN("viona: invalid VQ pairs from guest 0x%x", mq.virtqueue_pairs); return (VIRTIO_NET_CQ_ERR); } if (!pci_viona_set_usepairs(sc, mq.virtqueue_pairs)) return (VIRTIO_NET_CQ_ERR); break; } default: EPRINTLN("viona: unrecognised MQ control cmd %u", hdr->vnch_command); return (VIRTIO_NET_CQ_ERR); } return (VIRTIO_NET_CQ_OK); } static void pci_viona_control(struct vqueue_info *vq) { struct iovec iov[VIONA_CTLQ_MAXSEGS]; virtio_net_ctrl_hdr_t hdr; struct vi_req req = { 0 }; iov_bunch_t iob; uint8_t *ackp; uint32_t wlen = 0; size_t len; int niov; niov = vq_getchain(vq, iov, VIONA_CTLQ_MAXSEGS, &req); assert(niov >= 1 && niov <= VIONA_CTLQ_MAXSEGS); /* * Since we support the modern interface we must accept a flexible * layout here. Even with that we can do some basic checks - there have * to be at least two descriptors and only a single writable one sized * for one byte. Check the incoming message to make sure it matches * this layout and drop the entire chain if not. */ if (niov < 2 || req.writable != 1 || req.readable + 1 != niov || iov[req.readable].iov_len != sizeof (uint8_t)) { EPRINTLN("viona: bad control chain, niov=0x%x, w=0x%x, r=0x%x", niov, req.writable, req.readable); goto drop; } len = iov_bunch_init(&iob, iov, niov); if (!iov_bunch_copy(&iob, &hdr, sizeof (hdr))) { EPRINTLN("viona: control header copy failed, len=0x%x", len); goto drop; } /* * Writable iovecs start at iov[req.readable], and we've already * checked that there is only one writable, it's at the end, and the * right size; it's the acknowledgement byte. */ ackp = (uint8_t *)iov[req.readable].iov_base; iob.ib_remain--; switch (hdr.vnch_class) { case VIRTIO_NET_CTRL_RX: *ackp = pci_viona_control_rx(vq, &hdr, &iob); break; case VIRTIO_NET_CTRL_MAC: *ackp = pci_viona_control_mac(vq, &hdr, &iob); break; case VIRTIO_NET_CTRL_MQ: *ackp = pci_viona_control_mq(vq, &hdr, &iob); break; default: EPRINTLN("viona: unrecognised control class %u, cmd %u", hdr.vnch_class, hdr.vnch_command); *ackp = VIRTIO_NET_CQ_ERR; break; } /* We've written the status byte */ wlen++; drop: vq_relchain(vq, req.idx, wlen); } static void pci_viona_process_ctrlq(struct vqueue_info *vq) { for (;;) { vq_kick_disable(vq); while (vq_has_descs(vq)) pci_viona_control(vq); vq_kick_enable(vq); /* * One more check in case a late addition raced with * re-enabling kicks. Note that vq_kick_enable() includes a * memory barrier. */ if (!vq_has_descs(vq)) break; } vq_endchains(vq, /* used_all_avail= */1); } static void * pci_viona_poll_thread(void *param) { struct pci_viona_softc *sc = param; pollfd_t pollset; const int fd = sc->vsc_vnafd; pollset.fd = fd; pollset.events = POLLRDBAND; for (;;) { if (poll(&pollset, 1, -1) < 0) { if (errno == EINTR || errno == EAGAIN) { continue; } else { WPRINTF("pci_viona_poll_thread poll() error %d", errno); break; } } if (pollset.revents & POLLRDBAND) { uint_t i; int entries; bool assert_lintr = false; const bool do_msix = pci_msix_enabled(sc->vsc_vs.vs_pi); vioc_intr_poll_mq_t vipm; vipm.vipm_nrings = VIONA_USABLE_RINGS(sc); entries = ioctl(fd, VNA_IOC_INTR_POLL_MQ, &vipm); for (i = 0; entries > 0 && i < vipm.vipm_nrings && i < VIONA_USABLE_RINGS(sc); i++) { if (!VIONA_INTR_TEST(&vipm, i)) continue; entries--; if (do_msix) { pci_generate_msix(sc->vsc_vs.vs_pi, sc->vsc_queues[i].vq_msix_idx); } else { assert_lintr = true; } if (ioctl(fd, VNA_IOC_RING_INTR_CLR, i) != 0) { WPRINTF("ioctl viona vq %d intr " "clear failed %d", i, errno); } } if (assert_lintr) { pthread_mutex_lock(&sc->vsc_mtx); sc->vsc_vs.vs_isr |= VIRTIO_PCI_ISR_INTR; pci_lintr_assert(sc->vsc_vs.vs_pi); pthread_mutex_unlock(&sc->vsc_mtx); } } } pthread_exit(NULL); } static void pci_viona_ring_init(void *vsc, uint64_t pfn, bool modern) { struct pci_viona_softc *sc = vsc; int qnum = sc->vsc_vs.vs_curq; struct vqueue_info *vq = &sc->vsc_queues[qnum]; vioc_ring_init_modern_t vna_rim = { 0 }; int error; const bool ctlq = VIONA_RING_CTLQ(sc, qnum); DPRINTF("viona: ring init 0x%x", qnum); if (!ctlq && !VIONA_RING_VALID(sc, qnum)) { DPRINTF("viona: bad ring 0x%d", qnum); return; } if (modern) vi_vq_init(&sc->vsc_vs); else vi_legacy_vq_init(&sc->vsc_vs, pfn); if (ctlq) return; vna_rim.rim_index = qnum; vna_rim.rim_qsize = vq->vq_qsize; vna_rim.rim_qaddr_desc = vq->vq_desc_gpa; vna_rim.rim_qaddr_avail = vq->vq_avail_gpa; vna_rim.rim_qaddr_used = vq->vq_used_gpa; error = ioctl(sc->vsc_vnafd, VNA_IOC_RING_INIT_MODERN, &vna_rim); if (error != 0) { WPRINTF("ioctl viona ring 0x%x init failed %d", qnum, errno); } } static int pci_viona_viona_init(struct vmctx *ctx, struct pci_viona_softc *sc) { vioc_create_t vna_create; int error, version; sc->vsc_vnafd = open("/dev/viona", O_RDWR | O_EXCL); if (sc->vsc_vnafd == -1) { WPRINTF("open viona ctl failed: %d", errno); return (-1); } version = ioctl(sc->vsc_vnafd, VNA_IOC_VERSION, 0); if (version != VIONA_CURRENT_INTERFACE_VERSION) { (void) close(sc->vsc_vnafd); WPRINTF( "ioctl interface version %d != expected %d", version, VIONA_CURRENT_INTERFACE_VERSION); return (-1); } vna_create.c_linkid = sc->vsc_linkid; vna_create.c_vmfd = vm_get_device_fd(ctx); error = ioctl(sc->vsc_vnafd, VNA_IOC_CREATE, &vna_create); if (error != 0) { (void) close(sc->vsc_vnafd); WPRINTF("ioctl viona create failed %d", errno); return (-1); } return (0); } static int pci_viona_legacy_config(nvlist_t *nvl, const char *opt) { char *config, *name, *tofree, *value; if (opt == NULL) return (0); config = tofree = strdup(opt); while ((name = strsep(&config, ",")) != NULL) { value = strchr(name, '='); if (value != NULL) { *value++ = '\0'; set_config_value_node(nvl, name, value); } else { set_config_value_node(nvl, "vnic", name); } } free(tofree); return (0); } static int pci_viona_parse_opts(struct pci_viona_softc *sc, nvlist_t *nvl) { const char *value, *errstr; long long num; int err = 0; sc->vsc_config.vnc_max_qpair = VIONA_DEFAULT_MAX_QPAIR; sc->vsc_vq_txsize = VIONA_DEFAULT_TX_RINGSZ; sc->vsc_vq_rxsize = VIONA_DEFAULT_RX_RINGSZ; sc->vsc_config.vnc_speed = VIONA_DEFAULT_LINK_SPEED; sc->vsc_feature_mask = 0; sc->vsc_linkname[0] = '\0'; value = get_config_value_node(nvl, "feature_mask"); if (value != NULL) { num = strtonumx(value, 0, UINT64_MAX, &errstr, 0); if (errstr != NULL) { EPRINTLN("viona: invalid feature_mask '%s': %s", value, errstr); err = -1; } else { sc->vsc_feature_mask = num; } } value = get_config_value_node(nvl, "vqsize"); if (value != NULL) { num = strtonumx(value, 2, 32768, &errstr, 0); if (errstr != NULL) { EPRINTLN("viona: invalid vqsize '%s': %s", value, errstr); err = -1; } else if ((1 << (ffsll(num) - 1)) != num) { EPRINTLN("viona: vqsize '%s' must be power of 2", value); err = -1; } else { sc->vsc_vq_txsize = num; sc->vsc_vq_rxsize = num; } } value = get_config_value_node(nvl, "txvqsize"); if (value != NULL) { num = strtonumx(value, 2, 32768, &errstr, 0); if (errstr != NULL) { EPRINTLN("viona: invalid txvqsize '%s': %s", value, errstr); err = -1; } else if ((1 << (ffsll(num) - 1)) != num) { EPRINTLN("viona: txvqsize '%s' must be power of 2", value); err = -1; } else { sc->vsc_vq_txsize = num; } } value = get_config_value_node(nvl, "rxvqsize"); if (value != NULL) { num = strtonumx(value, 2, 32768, &errstr, 0); if (errstr != NULL) { EPRINTLN("viona: invalid rxvqsize '%s': %s", value, errstr); err = -1; } else if ((1 << (ffsll(num) - 1)) != num) { EPRINTLN("viona: rxvqsize '%s' must be power of 2", value); err = -1; } else { sc->vsc_vq_rxsize = num; } } value = get_config_value_node(nvl, "qpair"); if (value != NULL) { num = strtonumx(value, VIONA_MIN_QPAIR, VIONA_MAX_QPAIR, &errstr, 0); if (errstr != NULL) { EPRINTLN("viona: invalid qpair '%s': %s", value, errstr); err = -1; } else { sc->vsc_config.vnc_max_qpair = num; } } value = get_config_value_node(nvl, "speed"); if (value != NULL) { num = strtonumx(value, 0, INT32_MAX, &errstr, 0); if (errstr != NULL) { EPRINTLN("viona: invalid speed '%s': %s", value, errstr); err = -1; } else { sc->vsc_config.vnc_speed = num; } } value = get_config_value_node(nvl, "vnic"); if (value == NULL) { EPRINTLN("viona: vnic name required"); err = -1; } else { (void) strlcpy(sc->vsc_linkname, value, MAXLINKNAMELEN); } DPRINTF("viona=%p dev=%s vqsize(TX/RX)=0x%x/0x%x qpair=0x%x " "feature_mask=0x%" PRIx64, sc, sc->vsc_linkname, sc->vsc_vq_txsize, sc->vsc_vq_rxsize, sc->vsc_config.vnc_max_qpair, sc->vsc_feature_mask); return (err); } static uint16_t pci_viona_query_mtu(dladm_handle_t handle, datalink_id_t linkid) { char buf[DLADM_PROP_VAL_MAX]; char *propval = buf; uint_t propcnt = 1; if (dladm_get_linkprop(handle, linkid, DLADM_PROP_VAL_CURRENT, "mtu", &propval, &propcnt) == DLADM_STATUS_OK && propcnt == 1) { ulong_t parsed = strtoul(buf, NULL, 10); /* * The virtio spec notes that for devices implementing * VIRTIO_NET_F_MTU, that the noted MTU MUST be between * 68-65535, inclusive. */ if (parsed >= 68 && parsed <= 65535) return (parsed); } /* * Default to 1500 if query is unsuccessful or the result is out of * bounds. */ return (1500); } static int pci_viona_free_softstate(struct pci_viona_softc *sc, int err) { pthread_mutex_destroy(&sc->vsc_mtx); free(sc->vsc_queues); free(sc); return (err); } static int pci_viona_init(struct pci_devinst *pi, nvlist_t *nvl) { dladm_handle_t handle; dladm_status_t status; dladm_vnic_attr_t attr; char errmsg[DLADM_STRSIZE]; char tname[MAXCOMLEN + 1]; int error; struct pci_viona_softc *sc; struct virtio_consts *vc; const char *vnic; pthread_t tid; vnic = get_config_value_node(nvl, "vnic"); if (vnic == NULL) { WPRINTF("virtio-viona: vnic required"); return (1); } sc = calloc(1, sizeof (struct pci_viona_softc)); if (sc == NULL) { WPRINTF("Failed to allocate memory for soft state"); return (1); } vc = &sc->vsc_consts; *vc = viona_vi_consts; pthread_mutex_init(&sc->vsc_mtx, NULL); if (get_config_bool_default("viona.debug", false)) pci_viona_debug = 1; vi_set_debug(&sc->vsc_vs, pci_viona_debug); if (pci_viona_parse_opts(sc, nvl) != 0) return (pci_viona_free_softstate(sc, 1)); /* Add one for the control queue */ vc->vc_max_nvq = VIONA_P2QS(sc->vsc_config.vnc_max_qpair) + 1; if ((status = dladm_open(&handle)) != DLADM_STATUS_OK) { WPRINTF("could not open /dev/dld"); return (pci_viona_free_softstate(sc, 1)); } if ((status = dladm_name2info(handle, sc->vsc_linkname, &sc->vsc_linkid, NULL, NULL, NULL)) != DLADM_STATUS_OK) { WPRINTF("dladm_name2info() for %s failed: %s", vnic, dladm_status2str(status, errmsg)); dladm_close(handle); return (pci_viona_free_softstate(sc, 1)); } if ((status = dladm_vnic_info(handle, sc->vsc_linkid, &attr, DLADM_OPT_ACTIVE)) != DLADM_STATUS_OK) { WPRINTF("dladm_vnic_info() for %s failed: %s", vnic, dladm_status2str(status, errmsg)); dladm_close(handle); return (pci_viona_free_softstate(sc, 1)); } memcpy(sc->vsc_config.vnc_macaddr, attr.va_mac_addr, ETHERADDRL); sc->vsc_config.vnc_status = VIRTIO_NET_S_LINK_UP; /* link always up */ sc->vsc_config.vnc_duplex = VIRTIO_NET_DUPLEX_FULL; sc->vsc_config.vnc_mtu = pci_viona_query_mtu(handle, sc->vsc_linkid); dladm_close(handle); error = pci_viona_viona_init(pi->pi_vmctx, sc); if (error != 0) return (pci_viona_free_softstate(sc, 1)); if (ioctl(sc->vsc_vnafd, VNA_IOC_SET_MTU, sc->vsc_config.vnc_mtu) != 0) { WPRINTF("error setting viona MTU(%u): %s", sc->vsc_config.vnc_mtu, strerror(errno)); } /* link virtio to softc */ vi_softc_linkup(&sc->vsc_vs, vc, sc, pi, sc->vsc_queues); sc->vsc_vs.vs_mtx = &sc->vsc_mtx; /* * We initially need to configure a single queue pair. Some legacy * drivers will set these up before completing feature negotiation, * which is before we know if they support multi-queue. */ if (!pci_viona_qalloc(sc, 1)) return (pci_viona_free_softstate(sc, 1)); /* Until the guest tells us otherwise, we'll only use a single pair */ sc->vsc_vq_usepairs = 1; error = pthread_create(&tid, NULL, pci_viona_poll_thread, sc); assert(error == 0); snprintf(tname, sizeof (tname), "vionapoll:%s", vnic); pthread_set_name_np(tid, tname); /* initialize config space */ vi_pci_init(pi, VIRTIO_MODE_TRANSITIONAL, VIRTIO_DEV_NET, VIRTIO_ID_NETWORK, PCIC_NETWORK); /* * Guests that do not support CTRL_RX_MAC still generally need to * receive multicast packets. Guests that do support this feature will * end up setting this flag indirectly via messages on the control * queue but it does not hurt to default to multicast promiscuity here * and it is what older version of viona did. */ sc->vsc_promisc_mmac = true; pci_viona_eval_promisc(sc); /* Viona always uses MSI-X */ if (!vi_intr_init(&sc->vsc_vs, true)) return (pci_viona_free_softstate(sc, 1)); if (!vi_pcibar_setup(&sc->vsc_vs)) return (pci_viona_free_softstate(sc, 1)); return (0); } static int pci_viona_cfgwrite(void *vsc, int offset, int size, uint32_t value) { struct pci_viona_softc *sc = vsc; void *ptr; /* We will only ever end up here with an 8, 16 or 32-bit size */ ASSERT(size == 1 || size == 2 || size == 4); /* * The driver is allowed to change the MAC address. * vnc_macaddr is the first element of vsc_config */ if (offset < (int)sizeof (sc->vsc_config.vnc_macaddr) && offset + size <= (int)sizeof (sc->vsc_config.vnc_macaddr)) { ptr = &sc->vsc_config.vnc_macaddr[offset]; memcpy(ptr, &value, size); vq_devcfg_changed(&sc->vsc_vs); } else { /* silently ignore other writes */ DPRINTF("viona: write to readonly reg 0x%x", offset); } return (0); } static int pci_viona_cfgread(void *vsc, int offset, int size, uint32_t *retval) { struct pci_viona_softc *sc = vsc; void *ptr; ptr = (uint8_t *)&sc->vsc_config + offset; memcpy(retval, ptr, size); return (0); } static void pci_viona_ring_set_msix(void *vsc, int ring) { struct pci_viona_softc *sc = vsc; struct pci_devinst *pi = sc->vsc_vs.vs_pi; struct msix_table_entry mte; uint16_t tab_index; vioc_ring_msi_t vrm; int res; if (!VIONA_RING_VALID(sc, ring)) return; vrm.rm_index = ring; vrm.rm_addr = 0; vrm.rm_msg = 0; tab_index = sc->vsc_queues[ring].vq_msix_idx; if (tab_index != VIRTIO_MSI_NO_VECTOR && sc->vsc_msix_active) { mte = pi->pi_msix.table[tab_index]; if ((mte.vector_control & PCIM_MSIX_VCTRL_MASK) == 0) { vrm.rm_addr = mte.addr; vrm.rm_msg = mte.msg_data; } } DPRINTF("SET MSI ring=0x%x addr=0x%" PRIx64 " msg=0x%" PRIx64, vrm.rm_index, vrm.rm_addr, vrm.rm_msg); res = ioctl(sc->vsc_vnafd, VNA_IOC_RING_SET_MSI, &vrm); if (res != 0) { WPRINTF("ioctl viona set_msi %d failed %d", ring, errno); } } static void pci_viona_lintrupdate(struct pci_devinst *pi) { struct pci_viona_softc *sc = pi->pi_arg; bool msix_on = false; pthread_mutex_lock(&sc->vsc_mtx); msix_on = pci_msix_enabled(pi) && (pi->pi_msix.function_mask == 0); if ((sc->vsc_msix_active && !msix_on) || (msix_on && !sc->vsc_msix_active)) { DPRINTF("MSIX %u -> %u", sc->vsc_msix_active, msix_on); sc->vsc_msix_active = msix_on; /* Update in-kernel ring configs */ for (uint16_t i = 0; i < VIONA_NRINGS(sc); i++) pci_viona_ring_set_msix(sc, VIONA_RING(sc, i)->vq_num); } pthread_mutex_unlock(&sc->vsc_mtx); } static void pci_viona_update_msix(void *vsc, uint64_t offset) { struct pci_viona_softc *sc = vsc; uint_t tab_index; pthread_mutex_lock(&sc->vsc_mtx); if (!sc->vsc_msix_active) { pthread_mutex_unlock(&sc->vsc_mtx); return; } /* * Rather than update every possible MSI-X vector, cheat and use the * offset to calculate the entry within the table. Since this should * only be called when a write to the table succeeds, the index should * be valid. */ tab_index = offset / MSIX_TABLE_ENTRY_SIZE; for (uint16_t i = 0; i < VIONA_NRINGS(sc); i++) { struct vqueue_info *vq = VIONA_RING(sc, i); if (vq->vq_msix_idx == tab_index) pci_viona_ring_set_msix(vsc, vq->vq_num); } pthread_mutex_unlock(&sc->vsc_mtx); } static void pci_viona_ctlqnotify(void *vsc, struct vqueue_info *vq) { if (vq_has_descs(vq)) pci_viona_process_ctrlq(vq); } static void pci_viona_qnotify(void *vsc, struct vqueue_info *vq) { struct pci_viona_softc *sc = vsc; int ring = vq->vq_num; int error; if (!VIONA_RING_VALID(sc, ring)) return; error = ioctl(sc->vsc_vnafd, VNA_IOC_RING_KICK, ring); if (error != 0) WPRINTF("ioctl viona ring 0x%x kick failed %d", ring, errno); } static void pci_viona_baraddr(struct pci_devinst *pi, int baridx, int enabled, uint64_t address) { struct pci_viona_softc *sc = pi->pi_arg; int err; DPRINTF("BAR%d ADDRESS %" PRIx64 " (%s)", baridx, address, enabled == 1 ? "enable": "disable"); switch (baridx) { case VIRTIO_LEGACY_BAR: { uint64_t ioport; if (enabled == 0) { err = ioctl(sc->vsc_vnafd, VNA_IOC_SET_NOTIFY_IOP, 0); if (err != 0) WPRINTF("Uninstall ioport hook fail %d", errno); break; } /* * Install ioport hook for virtqueue notification. * This is part of the virtio common configuration area so the * address does not change with MSI-X status. */ ioport = address + VIRTIO_PCI_QUEUE_NOTIFY; err = ioctl(sc->vsc_vnafd, VNA_IOC_SET_NOTIFY_IOP, ioport); if (err != 0) { WPRINTF("Install ioport hook at 0x%x failed %d", ioport, errno); } break; } case VIRTIO_MODERN_BAR: { virtio_pci_capcfg_t *cfg; vioc_notify_mmio_t vim; if (enabled == 0) { err = ioctl(sc->vsc_vnafd, VNA_IOC_SET_NOTIFY_MMIO, 0); if (err != 0) WPRINTF("Uninstall MMIO hook fail %d", errno); break; } cfg = vi_pci_cfg_bytype(&sc->vsc_vs, VIRTIO_PCI_CAP_NOTIFY_CFG); if (cfg == NULL) break; vim.vim_address = address + cfg->c_baroff; vim.vim_size = cfg->c_barlen; DPRINTF("MODERN BAR NOTIFY address 0x%" PRIx64 " size 0x%x", vim.vim_address, vim.vim_size); err = ioctl(sc->vsc_vnafd, VNA_IOC_SET_NOTIFY_MMIO, &vim); if (err != 0) { WPRINTF( "Install MMIO hook at 0x%" PRIx64 "+0x%x failed %d", vim.vim_address, vim.vim_size, errno); } break; } default: break; } } static uint64_t pci_viona_get_hv_features(void *vsc, bool modern) { struct pci_viona_softc *sc = vsc; uint64_t value; int err; err = ioctl(sc->vsc_vnafd, VNA_IOC_GET_FEATURES, &value); if (err != 0) WPRINTF("ioctl get host features returned err = %d", errno); /* * Supplementary device capabilities provided in the userspace * component. */ value |= VIRTIO_NET_F_MAC; value |= VIRTIO_NET_F_STATUS; value |= VIRTIO_NET_F_MTU; value |= VIRTIO_NET_F_CTRL_VQ; value |= VIRTIO_NET_F_CTRL_RX; value |= VIRTIO_NET_F_MQ; if (modern) value |= VIRTIO_NET_F_SPEED_DUPLEX; value &= ~sc->vsc_feature_mask; if (modern) { value |= VIRTIO_F_VERSION_1; sc->vsc_consts.vc_hv_caps_modern = value; } else { /* * To be a conforming transitional device we must support * arbitrary descriptor layouts on the legacy interface. The * specification is a little ambiguous as it mandates this but * also provides details on how descriptors should be laid out * by transitional drivers that don't negotiate this. Since we * don't make any assumptions about descriptor layout we may as * well set this. */ value |= VIRTIO_F_ANY_LAYOUT; sc->vsc_consts.vc_hv_caps_legacy = value; } return (value); } static void pci_viona_set_hv_features(void *vsc, uint64_t *value) { struct pci_viona_softc *sc = vsc; int err; *value &= ~sc->vsc_feature_mask; err = ioctl(sc->vsc_vnafd, VNA_IOC_SET_FEATURES, value); if (err != 0) WPRINTF("ioctl feature negotiation returned err = %d", errno); if (*value & VIRTIO_NET_F_MQ) { /* * If multi-queue is negotiated then we need to provision all * of the queues we can support. Even if the guest chooses not * to use all of them it will still set them up. */ DPRINTF("Going MULTIQUEUE with 0x%x pairs!", sc->vsc_config.vnc_max_qpair); if (!pci_viona_qalloc(sc, sc->vsc_config.vnc_max_qpair)) { vi_error(&sc->vsc_vs, "Failed to allocate 0x%x queue pairs", sc->vsc_config.vnc_max_qpair); } } } struct pci_devemu pci_de_viona = { .pe_emu = "virtio-net-viona", .pe_init = pci_viona_init, .pe_legacy_config = pci_viona_legacy_config, .pe_cfgwrite = vi_pci_cfgwrite, .pe_cfgread = vi_pci_cfgread, .pe_barwrite = vi_pci_write, .pe_barread = vi_pci_read, .pe_baraddr = pci_viona_baraddr, .pe_lintrupdate = pci_viona_lintrupdate }; PCI_EMUL_SET(pci_de_viona); /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2014 Leon Dang * Copyright 2018 Joyent, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* XHCI options: -s ,xhci,{devices} devices: tablet USB tablet mouse */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "bhyverun.h" #include "config.h" #include "debug.h" #include "pci_emul.h" #include "pci_xhci.h" #include "usb_emul.h" static int xhci_debug = 0; #define DPRINTF(params) if (xhci_debug) PRINTLN params #define WPRINTF(params) PRINTLN params #define XHCI_NAME "xhci" #define XHCI_MAX_DEVS 8 /* 4 USB3 + 4 USB2 devs */ #define XHCI_MAX_SLOTS 64 /* min allowed by Windows drivers */ /* * XHCI data structures can be up to 64k, but limit paddr_guest2host mapping * to 4k to avoid going over the guest physical memory barrier. */ #define XHCI_PADDR_SZ 4096 /* paddr_guest2host max size */ #define XHCI_ERST_MAX 0 /* max 2^entries event ring seg tbl */ #define XHCI_CAPLEN (4*8) /* offset of op register space */ #define XHCI_HCCPRAMS2 0x1C /* offset of HCCPARAMS2 register */ #define XHCI_PORTREGS_START 0x400 #define XHCI_DOORBELL_MAX 256 #define XHCI_STREAMS_MAX 1 /* 4-15 in XHCI spec */ /* caplength and hci-version registers */ #define XHCI_SET_CAPLEN(x) ((x) & 0xFF) #define XHCI_SET_HCIVERSION(x) (((x) & 0xFFFF) << 16) #define XHCI_GET_HCIVERSION(x) (((x) >> 16) & 0xFFFF) /* hcsparams1 register */ #define XHCI_SET_HCSP1_MAXSLOTS(x) ((x) & 0xFF) #define XHCI_SET_HCSP1_MAXINTR(x) (((x) & 0x7FF) << 8) #define XHCI_SET_HCSP1_MAXPORTS(x) (((x) & 0xFF) << 24) /* hcsparams2 register */ #define XHCI_SET_HCSP2_IST(x) ((x) & 0x0F) #define XHCI_SET_HCSP2_ERSTMAX(x) (((x) & 0x0F) << 4) #define XHCI_SET_HCSP2_MAXSCRATCH_HI(x) (((x) & 0x1F) << 21) #define XHCI_SET_HCSP2_MAXSCRATCH_LO(x) (((x) & 0x1F) << 27) /* hcsparams3 register */ #define XHCI_SET_HCSP3_U1EXITLATENCY(x) ((x) & 0xFF) #define XHCI_SET_HCSP3_U2EXITLATENCY(x) (((x) & 0xFFFF) << 16) /* hccparams1 register */ #define XHCI_SET_HCCP1_AC64(x) ((x) & 0x01) #define XHCI_SET_HCCP1_BNC(x) (((x) & 0x01) << 1) #define XHCI_SET_HCCP1_CSZ(x) (((x) & 0x01) << 2) #define XHCI_SET_HCCP1_PPC(x) (((x) & 0x01) << 3) #define XHCI_SET_HCCP1_PIND(x) (((x) & 0x01) << 4) #define XHCI_SET_HCCP1_LHRC(x) (((x) & 0x01) << 5) #define XHCI_SET_HCCP1_LTC(x) (((x) & 0x01) << 6) #define XHCI_SET_HCCP1_NSS(x) (((x) & 0x01) << 7) #define XHCI_SET_HCCP1_PAE(x) (((x) & 0x01) << 8) #define XHCI_SET_HCCP1_SPC(x) (((x) & 0x01) << 9) #define XHCI_SET_HCCP1_SEC(x) (((x) & 0x01) << 10) #define XHCI_SET_HCCP1_CFC(x) (((x) & 0x01) << 11) #define XHCI_SET_HCCP1_MAXPSA(x) (((x) & 0x0F) << 12) #define XHCI_SET_HCCP1_XECP(x) (((x) & 0xFFFF) << 16) /* hccparams2 register */ #define XHCI_SET_HCCP2_U3C(x) ((x) & 0x01) #define XHCI_SET_HCCP2_CMC(x) (((x) & 0x01) << 1) #define XHCI_SET_HCCP2_FSC(x) (((x) & 0x01) << 2) #define XHCI_SET_HCCP2_CTC(x) (((x) & 0x01) << 3) #define XHCI_SET_HCCP2_LEC(x) (((x) & 0x01) << 4) #define XHCI_SET_HCCP2_CIC(x) (((x) & 0x01) << 5) /* other registers */ #define XHCI_SET_DOORBELL(x) ((x) & ~0x03) #define XHCI_SET_RTSOFFSET(x) ((x) & ~0x0F) /* register masks */ #define XHCI_PS_PLS_MASK (0xF << 5) /* port link state */ #define XHCI_PS_SPEED_MASK (0xF << 10) /* port speed */ #define XHCI_PS_PIC_MASK (0x3 << 14) /* port indicator */ /* port register set */ #define XHCI_PORTREGS_BASE 0x400 /* base offset */ #define XHCI_PORTREGS_PORT0 0x3F0 #define XHCI_PORTREGS_SETSZ 0x10 /* size of a set */ #define MASK_64_HI(x) ((x) & ~0xFFFFFFFFULL) #define MASK_64_LO(x) ((x) & 0xFFFFFFFFULL) #define FIELD_REPLACE(a,b,m,s) (((a) & ~((m) << (s))) | \ (((b) & (m)) << (s))) #define FIELD_COPY(a,b,m,s) (((a) & ~((m) << (s))) | \ (((b) & ((m) << (s))))) struct pci_xhci_trb_ring { uint64_t ringaddr; /* current dequeue guest address */ uint32_t ccs; /* consumer cycle state */ }; /* device endpoint transfer/stream rings */ struct pci_xhci_dev_ep { union { struct xhci_trb *_epu_tr; struct xhci_stream_ctx *_epu_sctx; } _ep_trbsctx; #define ep_tr _ep_trbsctx._epu_tr #define ep_sctx _ep_trbsctx._epu_sctx /* * Caches the value of MaxPStreams from the endpoint context * when an endpoint is initialized and is used to validate the * use of ep_ringaddr vs ep_sctx_trbs[] as well as the length * of ep_sctx_trbs[]. */ uint32_t ep_MaxPStreams; union { struct pci_xhci_trb_ring _epu_trb; struct pci_xhci_trb_ring *_epu_sctx_trbs; } _ep_trb_rings; #define ep_ringaddr _ep_trb_rings._epu_trb.ringaddr #define ep_ccs _ep_trb_rings._epu_trb.ccs #define ep_sctx_trbs _ep_trb_rings._epu_sctx_trbs struct usb_data_xfer *ep_xfer; /* transfer chain */ }; /* device context base address array: maps slot->device context */ struct xhci_dcbaa { uint64_t dcba[USB_MAX_DEVICES+1]; /* xhci_dev_ctx ptrs */ }; /* port status registers */ struct pci_xhci_portregs { uint32_t portsc; /* port status and control */ uint32_t portpmsc; /* port pwr mgmt status & control */ uint32_t portli; /* port link info */ uint32_t porthlpmc; /* port hardware LPM control */ } __packed; #define XHCI_PS_SPEED_SET(x) (((x) & 0xF) << 10) /* xHC operational registers */ struct pci_xhci_opregs { uint32_t usbcmd; /* usb command */ uint32_t usbsts; /* usb status */ uint32_t pgsz; /* page size */ uint32_t dnctrl; /* device notification control */ uint64_t crcr; /* command ring control */ uint64_t dcbaap; /* device ctx base addr array ptr */ uint32_t config; /* configure */ /* guest mapped addresses: */ struct xhci_trb *cr_p; /* crcr dequeue */ struct xhci_dcbaa *dcbaa_p; /* dev ctx array ptr */ }; /* xHC runtime registers */ struct pci_xhci_rtsregs { uint32_t mfindex; /* microframe index */ struct { /* interrupter register set */ uint32_t iman; /* interrupter management */ uint32_t imod; /* interrupter moderation */ uint32_t erstsz; /* event ring segment table size */ uint32_t rsvd; uint64_t erstba; /* event ring seg-tbl base addr */ uint64_t erdp; /* event ring dequeue ptr */ } intrreg __packed; /* guest mapped addresses */ struct xhci_event_ring_seg *erstba_p; struct xhci_trb *erst_p; /* event ring segment tbl */ int er_deq_seg; /* event ring dequeue segment */ int er_enq_idx; /* event ring enqueue index - xHCI */ int er_enq_seg; /* event ring enqueue segment */ uint32_t er_events_cnt; /* number of events in ER */ uint32_t event_pcs; /* producer cycle state flag */ }; struct pci_xhci_softc; /* * USB device emulation container. * This is referenced from usb_hci->hci_sc; 1 pci_xhci_dev_emu for each * emulated device instance. */ struct pci_xhci_dev_emu { struct pci_xhci_softc *xsc; /* XHCI contexts */ struct xhci_dev_ctx *dev_ctx; struct pci_xhci_dev_ep eps[XHCI_MAX_ENDPOINTS]; int dev_slotstate; struct usb_devemu *dev_ue; /* USB emulated dev */ void *dev_sc; /* device's softc */ struct usb_hci hci; }; struct pci_xhci_softc { struct pci_devinst *xsc_pi; pthread_mutex_t mtx; uint32_t caplength; /* caplen & hciversion */ uint32_t hcsparams1; /* structural parameters 1 */ uint32_t hcsparams2; /* structural parameters 2 */ uint32_t hcsparams3; /* structural parameters 3 */ uint32_t hccparams1; /* capability parameters 1 */ uint32_t dboff; /* doorbell offset */ uint32_t rtsoff; /* runtime register space offset */ uint32_t hccparams2; /* capability parameters 2 */ uint32_t regsend; /* end of configuration registers */ struct pci_xhci_opregs opregs; struct pci_xhci_rtsregs rtsregs; struct pci_xhci_portregs *portregs; struct pci_xhci_dev_emu **devices; /* XHCI[port] = device */ struct pci_xhci_dev_emu **slots; /* slots assigned from 1 */ int usb2_port_start; int usb3_port_start; }; /* port and slot numbering start from 1 */ #define XHCI_PORTREG_PTR(x,n) &((x)->portregs[(n) - 1]) #define XHCI_DEVINST_PTR(x,n) ((x)->devices[(n) - 1]) #define XHCI_SLOTDEV_PTR(x,n) ((x)->slots[(n) - 1]) #define XHCI_HALTED(sc) ((sc)->opregs.usbsts & XHCI_STS_HCH) #define XHCI_GADDR(sc,a) paddr_guest2host((sc)->xsc_pi->pi_vmctx, \ (a), \ XHCI_PADDR_SZ - ((a) & (XHCI_PADDR_SZ-1))) static int xhci_in_use; /* map USB errors to XHCI */ static const int xhci_usb_errors[USB_ERR_MAX] = { [USB_ERR_NORMAL_COMPLETION] = XHCI_TRB_ERROR_SUCCESS, [USB_ERR_PENDING_REQUESTS] = XHCI_TRB_ERROR_RESOURCE, [USB_ERR_NOT_STARTED] = XHCI_TRB_ERROR_ENDP_NOT_ON, [USB_ERR_INVAL] = XHCI_TRB_ERROR_INVALID, [USB_ERR_NOMEM] = XHCI_TRB_ERROR_RESOURCE, [USB_ERR_CANCELLED] = XHCI_TRB_ERROR_STOPPED, [USB_ERR_BAD_ADDRESS] = XHCI_TRB_ERROR_PARAMETER, [USB_ERR_BAD_BUFSIZE] = XHCI_TRB_ERROR_PARAMETER, [USB_ERR_BAD_FLAG] = XHCI_TRB_ERROR_PARAMETER, [USB_ERR_NO_CALLBACK] = XHCI_TRB_ERROR_STALL, [USB_ERR_IN_USE] = XHCI_TRB_ERROR_RESOURCE, [USB_ERR_NO_ADDR] = XHCI_TRB_ERROR_RESOURCE, [USB_ERR_NO_PIPE] = XHCI_TRB_ERROR_RESOURCE, [USB_ERR_ZERO_NFRAMES] = XHCI_TRB_ERROR_UNDEFINED, [USB_ERR_ZERO_MAXP] = XHCI_TRB_ERROR_UNDEFINED, [USB_ERR_SET_ADDR_FAILED] = XHCI_TRB_ERROR_RESOURCE, [USB_ERR_NO_POWER] = XHCI_TRB_ERROR_ENDP_NOT_ON, [USB_ERR_TOO_DEEP] = XHCI_TRB_ERROR_RESOURCE, [USB_ERR_IOERROR] = XHCI_TRB_ERROR_TRB, [USB_ERR_NOT_CONFIGURED] = XHCI_TRB_ERROR_ENDP_NOT_ON, [USB_ERR_TIMEOUT] = XHCI_TRB_ERROR_CMD_ABORTED, [USB_ERR_SHORT_XFER] = XHCI_TRB_ERROR_SHORT_PKT, [USB_ERR_STALLED] = XHCI_TRB_ERROR_STALL, [USB_ERR_INTERRUPTED] = XHCI_TRB_ERROR_CMD_ABORTED, [USB_ERR_DMA_LOAD_FAILED] = XHCI_TRB_ERROR_DATA_BUF, [USB_ERR_BAD_CONTEXT] = XHCI_TRB_ERROR_TRB, [USB_ERR_NO_ROOT_HUB] = XHCI_TRB_ERROR_UNDEFINED, [USB_ERR_NO_INTR_THREAD] = XHCI_TRB_ERROR_UNDEFINED, [USB_ERR_NOT_LOCKED] = XHCI_TRB_ERROR_UNDEFINED, }; #define USB_TO_XHCI_ERR(e) ((e) < USB_ERR_MAX ? xhci_usb_errors[(e)] : \ XHCI_TRB_ERROR_INVALID) static int pci_xhci_insert_event(struct pci_xhci_softc *sc, struct xhci_trb *evtrb, int do_intr); static void pci_xhci_dump_trb(struct xhci_trb *trb); static void pci_xhci_assert_interrupt(struct pci_xhci_softc *sc); static void pci_xhci_reset_slot(struct pci_xhci_softc *sc, int slot); static void pci_xhci_reset_port(struct pci_xhci_softc *sc, int portn, int warm); static void pci_xhci_update_ep_ring(struct pci_xhci_softc *sc, struct pci_xhci_dev_emu *dev, struct pci_xhci_dev_ep *devep, struct xhci_endp_ctx *ep_ctx, uint32_t streamid, uint64_t ringaddr, int ccs); static int pci_xhci_validate_slot(uint32_t slot); static void pci_xhci_set_evtrb(struct xhci_trb *evtrb, uint64_t port, uint32_t errcode, uint32_t evtype) { evtrb->qwTrb0 = port << 24; evtrb->dwTrb2 = XHCI_TRB_2_ERROR_SET(errcode); evtrb->dwTrb3 = XHCI_TRB_3_TYPE_SET(evtype); } /* controller reset */ static void pci_xhci_reset(struct pci_xhci_softc *sc) { int i; sc->rtsregs.er_enq_idx = 0; sc->rtsregs.er_events_cnt = 0; sc->rtsregs.event_pcs = 1; for (i = 1; i <= XHCI_MAX_SLOTS; i++) { pci_xhci_reset_slot(sc, i); } } static uint32_t pci_xhci_usbcmd_write(struct pci_xhci_softc *sc, uint32_t cmd) { int do_intr = 0; int i; if (cmd & XHCI_CMD_RS) { do_intr = (sc->opregs.usbcmd & XHCI_CMD_RS) == 0; sc->opregs.usbcmd |= XHCI_CMD_RS; sc->opregs.usbsts &= ~XHCI_STS_HCH; sc->opregs.usbsts |= XHCI_STS_PCD; /* Queue port change event on controller run from stop */ if (do_intr) for (i = 1; i <= XHCI_MAX_DEVS; i++) { struct pci_xhci_dev_emu *dev; struct pci_xhci_portregs *port; struct xhci_trb evtrb; if ((dev = XHCI_DEVINST_PTR(sc, i)) == NULL) continue; port = XHCI_PORTREG_PTR(sc, i); port->portsc |= XHCI_PS_CSC | XHCI_PS_CCS; port->portsc &= ~XHCI_PS_PLS_MASK; /* * XHCI 4.19.3 USB2 RxDetect->Polling, * USB3 Polling->U0 */ if (dev->dev_ue->ue_usbver == 2) port->portsc |= XHCI_PS_PLS_SET(UPS_PORT_LS_POLL); else port->portsc |= XHCI_PS_PLS_SET(UPS_PORT_LS_U0); pci_xhci_set_evtrb(&evtrb, i, XHCI_TRB_ERROR_SUCCESS, XHCI_TRB_EVENT_PORT_STS_CHANGE); if (pci_xhci_insert_event(sc, &evtrb, 0) != XHCI_TRB_ERROR_SUCCESS) break; } } else { sc->opregs.usbcmd &= ~XHCI_CMD_RS; sc->opregs.usbsts |= XHCI_STS_HCH; sc->opregs.usbsts &= ~XHCI_STS_PCD; } /* start execution of schedule; stop when set to 0 */ cmd |= sc->opregs.usbcmd & XHCI_CMD_RS; if (cmd & XHCI_CMD_HCRST) { /* reset controller */ pci_xhci_reset(sc); cmd &= ~XHCI_CMD_HCRST; } cmd &= ~(XHCI_CMD_CSS | XHCI_CMD_CRS); if (do_intr) pci_xhci_assert_interrupt(sc); return (cmd); } static void pci_xhci_portregs_write(struct pci_xhci_softc *sc, uint64_t offset, uint64_t value) { struct xhci_trb evtrb; struct pci_xhci_portregs *p; int port; uint32_t oldpls, newpls; if (sc->portregs == NULL) return; port = (offset - XHCI_PORTREGS_PORT0) / XHCI_PORTREGS_SETSZ; offset = (offset - XHCI_PORTREGS_PORT0) % XHCI_PORTREGS_SETSZ; DPRINTF(("pci_xhci: portregs wr offset 0x%lx, port %u: 0x%lx", offset, port, value)); assert(port >= 0); if (port > XHCI_MAX_DEVS) { DPRINTF(("pci_xhci: portregs_write port %d > ndevices", port)); return; } if (XHCI_DEVINST_PTR(sc, port) == NULL) { DPRINTF(("pci_xhci: portregs_write to unattached port %d", port)); } p = XHCI_PORTREG_PTR(sc, port); switch (offset) { case 0: /* port reset or warm reset */ if (value & (XHCI_PS_PR | XHCI_PS_WPR)) { pci_xhci_reset_port(sc, port, value & XHCI_PS_WPR); break; } if ((p->portsc & XHCI_PS_PP) == 0) { WPRINTF(("pci_xhci: portregs_write to unpowered " "port %d", port)); break; } /* Port status and control register */ oldpls = XHCI_PS_PLS_GET(p->portsc); newpls = XHCI_PS_PLS_GET(value); #ifndef __FreeBSD__ p->portsc &= XHCI_PS_PED | XHCI_PS_PP | XHCI_PS_PLS_MASK | XHCI_PS_SPEED_MASK | XHCI_PS_PIC_MASK; #else p->portsc &= XHCI_PS_PED | XHCI_PS_PLS_MASK | XHCI_PS_SPEED_MASK | XHCI_PS_PIC_MASK; #endif if (XHCI_DEVINST_PTR(sc, port)) p->portsc |= XHCI_PS_CCS; p->portsc |= (value & ~(XHCI_PS_OCA | XHCI_PS_PR | XHCI_PS_PED | XHCI_PS_PLS_MASK | /* link state */ XHCI_PS_SPEED_MASK | XHCI_PS_PIC_MASK | /* port indicator */ XHCI_PS_LWS | XHCI_PS_DR | XHCI_PS_WPR)); /* clear control bits */ p->portsc &= ~(value & (XHCI_PS_CSC | XHCI_PS_PEC | XHCI_PS_WRC | XHCI_PS_OCC | XHCI_PS_PRC | XHCI_PS_PLC | XHCI_PS_CEC | XHCI_PS_CAS)); /* port disable request; for USB3, don't care */ if (value & XHCI_PS_PED) DPRINTF(("Disable port %d request", port)); if (!(value & XHCI_PS_LWS)) break; DPRINTF(("Port new PLS: %d", newpls)); switch (newpls) { case 0: /* U0 */ case 3: /* U3 */ if (oldpls != newpls) { p->portsc &= ~XHCI_PS_PLS_MASK; p->portsc |= XHCI_PS_PLS_SET(newpls) | XHCI_PS_PLC; if (oldpls != 0 && newpls == 0) { pci_xhci_set_evtrb(&evtrb, port, XHCI_TRB_ERROR_SUCCESS, XHCI_TRB_EVENT_PORT_STS_CHANGE); pci_xhci_insert_event(sc, &evtrb, 1); } } break; default: DPRINTF(("Unhandled change port %d PLS %u", port, newpls)); break; } break; case 4: /* Port power management status and control register */ p->portpmsc = value; break; case 8: /* Port link information register */ DPRINTF(("pci_xhci attempted write to PORTLI, port %d", port)); break; case 12: /* * Port hardware LPM control register. * For USB3, this register is reserved. */ p->porthlpmc = value; break; default: DPRINTF(("pci_xhci: unaligned portreg write offset %#lx", offset)); break; } } static struct xhci_dev_ctx * pci_xhci_get_dev_ctx(struct pci_xhci_softc *sc, uint32_t slot) { uint64_t devctx_addr; struct xhci_dev_ctx *devctx; assert(slot > 0 && slot <= XHCI_MAX_SLOTS); assert(XHCI_SLOTDEV_PTR(sc, slot) != NULL); assert(sc->opregs.dcbaa_p != NULL); devctx_addr = sc->opregs.dcbaa_p->dcba[slot]; if (devctx_addr == 0) { DPRINTF(("get_dev_ctx devctx_addr == 0")); return (NULL); } DPRINTF(("pci_xhci: get dev ctx, slot %u devctx addr %016lx", slot, devctx_addr)); devctx = XHCI_GADDR(sc, devctx_addr & ~0x3FUL); return (devctx); } static struct xhci_trb * pci_xhci_trb_next(struct pci_xhci_softc *sc, struct xhci_trb *curtrb, uint64_t *guestaddr) { struct xhci_trb *next; assert(curtrb != NULL); if (XHCI_TRB_3_TYPE_GET(curtrb->dwTrb3) == XHCI_TRB_TYPE_LINK) { if (guestaddr) *guestaddr = curtrb->qwTrb0 & ~0xFUL; next = XHCI_GADDR(sc, curtrb->qwTrb0 & ~0xFUL); } else { if (guestaddr) *guestaddr += sizeof(struct xhci_trb) & ~0xFUL; next = curtrb + 1; } return (next); } static void pci_xhci_assert_interrupt(struct pci_xhci_softc *sc) { sc->rtsregs.intrreg.erdp |= XHCI_ERDP_LO_BUSY; sc->rtsregs.intrreg.iman |= XHCI_IMAN_INTR_PEND; sc->opregs.usbsts |= XHCI_STS_EINT; /* only trigger interrupt if permitted */ if ((sc->opregs.usbcmd & XHCI_CMD_INTE) && (sc->rtsregs.intrreg.iman & XHCI_IMAN_INTR_ENA)) { if (pci_msi_enabled(sc->xsc_pi)) pci_generate_msi(sc->xsc_pi, 0); else pci_lintr_assert(sc->xsc_pi); } } static void pci_xhci_deassert_interrupt(struct pci_xhci_softc *sc) { if (!pci_msi_enabled(sc->xsc_pi)) pci_lintr_assert(sc->xsc_pi); } static void pci_xhci_init_ep(struct pci_xhci_dev_emu *dev, int epid) { struct xhci_dev_ctx *dev_ctx; struct pci_xhci_dev_ep *devep; struct xhci_endp_ctx *ep_ctx; uint32_t i, pstreams; dev_ctx = dev->dev_ctx; ep_ctx = &dev_ctx->ctx_ep[epid]; devep = &dev->eps[epid]; pstreams = XHCI_EPCTX_0_MAXP_STREAMS_GET(ep_ctx->dwEpCtx0); if (pstreams > 0) { DPRINTF(("init_ep %d with pstreams %u", epid, pstreams)); assert(devep->ep_sctx_trbs == NULL); devep->ep_sctx = XHCI_GADDR(dev->xsc, ep_ctx->qwEpCtx2 & XHCI_EPCTX_2_TR_DQ_PTR_MASK); devep->ep_sctx_trbs = calloc(pstreams, sizeof(struct pci_xhci_trb_ring)); for (i = 0; i < pstreams; i++) { devep->ep_sctx_trbs[i].ringaddr = devep->ep_sctx[i].qwSctx0 & XHCI_SCTX_0_TR_DQ_PTR_MASK; devep->ep_sctx_trbs[i].ccs = XHCI_SCTX_0_DCS_GET(devep->ep_sctx[i].qwSctx0); } } else { DPRINTF(("init_ep %d with no pstreams", epid)); devep->ep_ringaddr = ep_ctx->qwEpCtx2 & XHCI_EPCTX_2_TR_DQ_PTR_MASK; devep->ep_ccs = XHCI_EPCTX_2_DCS_GET(ep_ctx->qwEpCtx2); devep->ep_tr = XHCI_GADDR(dev->xsc, devep->ep_ringaddr); DPRINTF(("init_ep tr DCS %x", devep->ep_ccs)); } devep->ep_MaxPStreams = pstreams; if (devep->ep_xfer == NULL) { devep->ep_xfer = malloc(sizeof(struct usb_data_xfer)); USB_DATA_XFER_INIT(devep->ep_xfer); } } static void pci_xhci_disable_ep(struct pci_xhci_dev_emu *dev, int epid) { struct xhci_dev_ctx *dev_ctx; struct pci_xhci_dev_ep *devep; struct xhci_endp_ctx *ep_ctx; DPRINTF(("pci_xhci disable_ep %d", epid)); dev_ctx = dev->dev_ctx; ep_ctx = &dev_ctx->ctx_ep[epid]; ep_ctx->dwEpCtx0 = (ep_ctx->dwEpCtx0 & ~0x7) | XHCI_ST_EPCTX_DISABLED; devep = &dev->eps[epid]; if (devep->ep_MaxPStreams > 0) free(devep->ep_sctx_trbs); if (devep->ep_xfer != NULL) { free(devep->ep_xfer); devep->ep_xfer = NULL; } memset(devep, 0, sizeof(struct pci_xhci_dev_ep)); } /* reset device at slot and data structures related to it */ static void pci_xhci_reset_slot(struct pci_xhci_softc *sc, int slot) { struct pci_xhci_dev_emu *dev; dev = XHCI_SLOTDEV_PTR(sc, slot); if (!dev) { DPRINTF(("xhci reset unassigned slot (%d)?", slot)); } else { dev->dev_slotstate = XHCI_ST_DISABLED; } /* TODO: reset ring buffer pointers */ } static int pci_xhci_insert_event(struct pci_xhci_softc *sc, struct xhci_trb *evtrb, int do_intr) { struct pci_xhci_rtsregs *rts; uint64_t erdp; int erdp_idx; int err; struct xhci_trb *evtrbptr; err = XHCI_TRB_ERROR_SUCCESS; rts = &sc->rtsregs; erdp = rts->intrreg.erdp & ~0xF; erdp_idx = (erdp - rts->erstba_p[rts->er_deq_seg].qwEvrsTablePtr) / sizeof(struct xhci_trb); DPRINTF(("pci_xhci: insert event 0[%lx] 2[%x] 3[%x]", evtrb->qwTrb0, evtrb->dwTrb2, evtrb->dwTrb3)); DPRINTF(("\terdp idx %d/seg %d, enq idx %d/seg %d, pcs %u", erdp_idx, rts->er_deq_seg, rts->er_enq_idx, rts->er_enq_seg, rts->event_pcs)); DPRINTF(("\t(erdp=0x%lx, erst=0x%lx, tblsz=%u, do_intr %d)", erdp, rts->erstba_p->qwEvrsTablePtr, rts->erstba_p->dwEvrsTableSize, do_intr)); evtrbptr = &rts->erst_p[rts->er_enq_idx]; /* TODO: multi-segment table */ if (rts->er_events_cnt >= rts->erstba_p->dwEvrsTableSize) { DPRINTF(("pci_xhci[%d] cannot insert event; ring full", __LINE__)); err = XHCI_TRB_ERROR_EV_RING_FULL; goto done; } if (rts->er_events_cnt == rts->erstba_p->dwEvrsTableSize - 1) { struct xhci_trb errev; if ((evtrbptr->dwTrb3 & 0x1) == (rts->event_pcs & 0x1)) { DPRINTF(("pci_xhci[%d] insert evt err: ring full", __LINE__)); errev.qwTrb0 = 0; errev.dwTrb2 = XHCI_TRB_2_ERROR_SET( XHCI_TRB_ERROR_EV_RING_FULL); errev.dwTrb3 = XHCI_TRB_3_TYPE_SET( XHCI_TRB_EVENT_HOST_CTRL) | rts->event_pcs; rts->er_events_cnt++; memcpy(&rts->erst_p[rts->er_enq_idx], &errev, sizeof(struct xhci_trb)); rts->er_enq_idx = (rts->er_enq_idx + 1) % rts->erstba_p->dwEvrsTableSize; err = XHCI_TRB_ERROR_EV_RING_FULL; do_intr = 1; goto done; } } else { rts->er_events_cnt++; } evtrb->dwTrb3 &= ~XHCI_TRB_3_CYCLE_BIT; evtrb->dwTrb3 |= rts->event_pcs; memcpy(&rts->erst_p[rts->er_enq_idx], evtrb, sizeof(struct xhci_trb)); rts->er_enq_idx = (rts->er_enq_idx + 1) % rts->erstba_p->dwEvrsTableSize; if (rts->er_enq_idx == 0) rts->event_pcs ^= 1; done: if (do_intr) pci_xhci_assert_interrupt(sc); return (err); } static uint32_t pci_xhci_cmd_enable_slot(struct pci_xhci_softc *sc, uint32_t *slot) { struct pci_xhci_dev_emu *dev; uint32_t cmderr; int i; cmderr = XHCI_TRB_ERROR_NO_SLOTS; if (sc->portregs != NULL) for (i = 1; i <= XHCI_MAX_SLOTS; i++) { dev = XHCI_SLOTDEV_PTR(sc, i); if (dev && dev->dev_slotstate == XHCI_ST_DISABLED) { *slot = i; dev->dev_slotstate = XHCI_ST_ENABLED; cmderr = XHCI_TRB_ERROR_SUCCESS; dev->hci.hci_address = i; break; } } DPRINTF(("pci_xhci enable slot (error=%d) slot %u", cmderr != XHCI_TRB_ERROR_SUCCESS, *slot)); return (cmderr); } static uint32_t pci_xhci_cmd_disable_slot(struct pci_xhci_softc *sc, uint32_t slot) { struct pci_xhci_dev_emu *dev; uint32_t cmderr; DPRINTF(("pci_xhci disable slot %u", slot)); if (sc->portregs == NULL) { cmderr = XHCI_TRB_ERROR_NO_SLOTS; goto done; } cmderr = pci_xhci_validate_slot(slot); if (cmderr != XHCI_TRB_ERROR_SUCCESS) goto done; dev = XHCI_SLOTDEV_PTR(sc, slot); if (dev) { if (dev->dev_slotstate == XHCI_ST_DISABLED) { cmderr = XHCI_TRB_ERROR_SLOT_NOT_ON; } else { dev->dev_slotstate = XHCI_ST_DISABLED; /* TODO: reset events and endpoints */ } } else cmderr = XHCI_TRB_ERROR_SLOT_NOT_ON; done: return (cmderr); } static uint32_t pci_xhci_cmd_reset_device(struct pci_xhci_softc *sc, uint32_t slot) { struct pci_xhci_dev_emu *dev; struct xhci_dev_ctx *dev_ctx; struct xhci_endp_ctx *ep_ctx; uint32_t cmderr; int i; if (sc->portregs == NULL) { cmderr = XHCI_TRB_ERROR_NO_SLOTS; goto done; } DPRINTF(("pci_xhci reset device slot %u", slot)); cmderr = pci_xhci_validate_slot(slot); if (cmderr != XHCI_TRB_ERROR_SUCCESS) goto done; dev = XHCI_SLOTDEV_PTR(sc, slot); if (!dev || dev->dev_slotstate == XHCI_ST_DISABLED) cmderr = XHCI_TRB_ERROR_SLOT_NOT_ON; else { dev->dev_slotstate = XHCI_ST_DEFAULT; dev->hci.hci_address = 0; dev_ctx = pci_xhci_get_dev_ctx(sc, slot); if (dev_ctx == NULL) { cmderr = XHCI_TRB_ERROR_PARAMETER; goto done; } /* slot state */ dev_ctx->ctx_slot.dwSctx3 = FIELD_REPLACE( dev_ctx->ctx_slot.dwSctx3, XHCI_ST_SLCTX_DEFAULT, 0x1F, 27); /* number of contexts */ dev_ctx->ctx_slot.dwSctx0 = FIELD_REPLACE( dev_ctx->ctx_slot.dwSctx0, 1, 0x1F, 27); /* reset all eps other than ep-0 */ for (i = 2; i <= 31; i++) { ep_ctx = &dev_ctx->ctx_ep[i]; ep_ctx->dwEpCtx0 = FIELD_REPLACE( ep_ctx->dwEpCtx0, XHCI_ST_EPCTX_DISABLED, 0x7, 0); } } pci_xhci_reset_slot(sc, slot); done: return (cmderr); } static uint32_t pci_xhci_cmd_address_device(struct pci_xhci_softc *sc, uint32_t slot, struct xhci_trb *trb) { struct pci_xhci_dev_emu *dev; struct xhci_input_dev_ctx *input_ctx; struct xhci_slot_ctx *islot_ctx; struct xhci_dev_ctx *dev_ctx; struct xhci_endp_ctx *ep0_ctx; uint32_t cmderr; input_ctx = XHCI_GADDR(sc, trb->qwTrb0 & ~0xFUL); islot_ctx = &input_ctx->ctx_slot; ep0_ctx = &input_ctx->ctx_ep[1]; DPRINTF(("pci_xhci: address device, input ctl: D 0x%08x A 0x%08x,", input_ctx->ctx_input.dwInCtx0, input_ctx->ctx_input.dwInCtx1)); DPRINTF((" slot %08x %08x %08x %08x", islot_ctx->dwSctx0, islot_ctx->dwSctx1, islot_ctx->dwSctx2, islot_ctx->dwSctx3)); DPRINTF((" ep0 %08x %08x %016lx %08x", ep0_ctx->dwEpCtx0, ep0_ctx->dwEpCtx1, ep0_ctx->qwEpCtx2, ep0_ctx->dwEpCtx4)); /* when setting address: drop-ctx=0, add-ctx=slot+ep0 */ if ((input_ctx->ctx_input.dwInCtx0 != 0) || (input_ctx->ctx_input.dwInCtx1 & 0x03) != 0x03) { DPRINTF(("pci_xhci: address device, input ctl invalid")); cmderr = XHCI_TRB_ERROR_TRB; goto done; } cmderr = pci_xhci_validate_slot(slot); if (cmderr != XHCI_TRB_ERROR_SUCCESS) goto done; /* assign address to slot */ dev_ctx = pci_xhci_get_dev_ctx(sc, slot); if (dev_ctx == NULL) { cmderr = XHCI_TRB_ERROR_PARAMETER; goto done; } DPRINTF(("pci_xhci: address device, dev ctx")); DPRINTF((" slot %08x %08x %08x %08x", dev_ctx->ctx_slot.dwSctx0, dev_ctx->ctx_slot.dwSctx1, dev_ctx->ctx_slot.dwSctx2, dev_ctx->ctx_slot.dwSctx3)); dev = XHCI_SLOTDEV_PTR(sc, slot); assert(dev != NULL); dev->hci.hci_address = slot; dev->dev_ctx = dev_ctx; if (dev->dev_ue->ue_reset == NULL || dev->dev_ue->ue_reset(dev->dev_sc) < 0) { cmderr = XHCI_TRB_ERROR_ENDP_NOT_ON; goto done; } memcpy(&dev_ctx->ctx_slot, islot_ctx, sizeof(struct xhci_slot_ctx)); dev_ctx->ctx_slot.dwSctx3 = XHCI_SCTX_3_SLOT_STATE_SET(XHCI_ST_SLCTX_ADDRESSED) | XHCI_SCTX_3_DEV_ADDR_SET(slot); memcpy(&dev_ctx->ctx_ep[1], ep0_ctx, sizeof(struct xhci_endp_ctx)); ep0_ctx = &dev_ctx->ctx_ep[1]; ep0_ctx->dwEpCtx0 = (ep0_ctx->dwEpCtx0 & ~0x7) | XHCI_EPCTX_0_EPSTATE_SET(XHCI_ST_EPCTX_RUNNING); pci_xhci_init_ep(dev, 1); dev->dev_slotstate = XHCI_ST_ADDRESSED; DPRINTF(("pci_xhci: address device, output ctx")); DPRINTF((" slot %08x %08x %08x %08x", dev_ctx->ctx_slot.dwSctx0, dev_ctx->ctx_slot.dwSctx1, dev_ctx->ctx_slot.dwSctx2, dev_ctx->ctx_slot.dwSctx3)); DPRINTF((" ep0 %08x %08x %016lx %08x", ep0_ctx->dwEpCtx0, ep0_ctx->dwEpCtx1, ep0_ctx->qwEpCtx2, ep0_ctx->dwEpCtx4)); done: return (cmderr); } static uint32_t pci_xhci_cmd_config_ep(struct pci_xhci_softc *sc, uint32_t slot, struct xhci_trb *trb) { struct xhci_input_dev_ctx *input_ctx; struct pci_xhci_dev_emu *dev; struct xhci_dev_ctx *dev_ctx; struct xhci_endp_ctx *ep_ctx, *iep_ctx; uint32_t cmderr; int i; DPRINTF(("pci_xhci config_ep slot %u", slot)); cmderr = pci_xhci_validate_slot(slot); if (cmderr != XHCI_TRB_ERROR_SUCCESS) goto done; dev = XHCI_SLOTDEV_PTR(sc, slot); assert(dev != NULL); if ((trb->dwTrb3 & XHCI_TRB_3_DCEP_BIT) != 0) { DPRINTF(("pci_xhci config_ep - deconfigure ep slot %u", slot)); if (dev->dev_ue->ue_stop != NULL) dev->dev_ue->ue_stop(dev->dev_sc); dev->dev_slotstate = XHCI_ST_ADDRESSED; dev->hci.hci_address = 0; dev_ctx = pci_xhci_get_dev_ctx(sc, slot); if (dev_ctx == NULL) { cmderr = XHCI_TRB_ERROR_PARAMETER; goto done; } /* number of contexts */ dev_ctx->ctx_slot.dwSctx0 = FIELD_REPLACE( dev_ctx->ctx_slot.dwSctx0, 1, 0x1F, 27); /* slot state */ dev_ctx->ctx_slot.dwSctx3 = FIELD_REPLACE( dev_ctx->ctx_slot.dwSctx3, XHCI_ST_SLCTX_ADDRESSED, 0x1F, 27); /* disable endpoints */ for (i = 2; i < 32; i++) pci_xhci_disable_ep(dev, i); cmderr = XHCI_TRB_ERROR_SUCCESS; goto done; } if (dev->dev_slotstate < XHCI_ST_ADDRESSED) { DPRINTF(("pci_xhci: config_ep slotstate x%x != addressed", dev->dev_slotstate)); cmderr = XHCI_TRB_ERROR_SLOT_NOT_ON; goto done; } /* In addressed/configured state; * for each drop endpoint ctx flag: * ep->state = DISABLED * for each add endpoint ctx flag: * cp(ep-in, ep-out) * ep->state = RUNNING * for each drop+add endpoint flag: * reset ep resources * cp(ep-in, ep-out) * ep->state = RUNNING * if input->DisabledCtx[2-31] < 30: (at least 1 ep not disabled) * slot->state = configured */ input_ctx = XHCI_GADDR(sc, trb->qwTrb0 & ~0xFUL); dev_ctx = dev->dev_ctx; DPRINTF(("pci_xhci: config_ep inputctx: D:x%08x A:x%08x 7:x%08x", input_ctx->ctx_input.dwInCtx0, input_ctx->ctx_input.dwInCtx1, input_ctx->ctx_input.dwInCtx7)); for (i = 2; i <= 31; i++) { ep_ctx = &dev_ctx->ctx_ep[i]; if (input_ctx->ctx_input.dwInCtx0 & XHCI_INCTX_0_DROP_MASK(i)) { DPRINTF((" config ep - dropping ep %d", i)); pci_xhci_disable_ep(dev, i); } if (input_ctx->ctx_input.dwInCtx1 & XHCI_INCTX_1_ADD_MASK(i)) { iep_ctx = &input_ctx->ctx_ep[i]; DPRINTF((" enable ep[%d] %08x %08x %016lx %08x", i, iep_ctx->dwEpCtx0, iep_ctx->dwEpCtx1, iep_ctx->qwEpCtx2, iep_ctx->dwEpCtx4)); memcpy(ep_ctx, iep_ctx, sizeof(struct xhci_endp_ctx)); pci_xhci_init_ep(dev, i); /* ep state */ ep_ctx->dwEpCtx0 = FIELD_REPLACE( ep_ctx->dwEpCtx0, XHCI_ST_EPCTX_RUNNING, 0x7, 0); } } /* slot state to configured */ dev_ctx->ctx_slot.dwSctx3 = FIELD_REPLACE( dev_ctx->ctx_slot.dwSctx3, XHCI_ST_SLCTX_CONFIGURED, 0x1F, 27); dev_ctx->ctx_slot.dwSctx0 = FIELD_COPY( dev_ctx->ctx_slot.dwSctx0, input_ctx->ctx_slot.dwSctx0, 0x1F, 27); dev->dev_slotstate = XHCI_ST_CONFIGURED; DPRINTF(("EP configured; slot %u [0]=0x%08x [1]=0x%08x [2]=0x%08x " "[3]=0x%08x", slot, dev_ctx->ctx_slot.dwSctx0, dev_ctx->ctx_slot.dwSctx1, dev_ctx->ctx_slot.dwSctx2, dev_ctx->ctx_slot.dwSctx3)); done: return (cmderr); } static uint32_t pci_xhci_cmd_reset_ep(struct pci_xhci_softc *sc, uint32_t slot, struct xhci_trb *trb) { struct pci_xhci_dev_emu *dev; struct pci_xhci_dev_ep *devep; struct xhci_dev_ctx *dev_ctx; struct xhci_endp_ctx *ep_ctx; uint32_t cmderr, epid; uint32_t type; epid = XHCI_TRB_3_EP_GET(trb->dwTrb3); DPRINTF(("pci_xhci: reset ep %u: slot %u", epid, slot)); cmderr = pci_xhci_validate_slot(slot); if (cmderr != XHCI_TRB_ERROR_SUCCESS) goto done; dev = XHCI_SLOTDEV_PTR(sc, slot); assert(dev != NULL); type = XHCI_TRB_3_TYPE_GET(trb->dwTrb3); if (type == XHCI_TRB_TYPE_STOP_EP && (trb->dwTrb3 & XHCI_TRB_3_SUSP_EP_BIT) != 0) { /* XXX suspend endpoint for 10ms */ } if (epid < 1 || epid > 31) { DPRINTF(("pci_xhci: reset ep: invalid epid %u", epid)); cmderr = XHCI_TRB_ERROR_TRB; goto done; } devep = &dev->eps[epid]; if (devep->ep_xfer != NULL) USB_DATA_XFER_RESET(devep->ep_xfer); dev_ctx = dev->dev_ctx; assert(dev_ctx != NULL); ep_ctx = &dev_ctx->ctx_ep[epid]; ep_ctx->dwEpCtx0 = (ep_ctx->dwEpCtx0 & ~0x7) | XHCI_ST_EPCTX_STOPPED; if (devep->ep_MaxPStreams == 0) ep_ctx->qwEpCtx2 = devep->ep_ringaddr | devep->ep_ccs; DPRINTF(("pci_xhci: reset ep[%u] %08x %08x %016lx %08x", epid, ep_ctx->dwEpCtx0, ep_ctx->dwEpCtx1, ep_ctx->qwEpCtx2, ep_ctx->dwEpCtx4)); if (type == XHCI_TRB_TYPE_RESET_EP && (dev->dev_ue->ue_reset == NULL || dev->dev_ue->ue_reset(dev->dev_sc) < 0)) { cmderr = XHCI_TRB_ERROR_ENDP_NOT_ON; goto done; } done: return (cmderr); } static uint32_t pci_xhci_find_stream(struct pci_xhci_softc *sc, struct xhci_endp_ctx *ep, struct pci_xhci_dev_ep *devep, uint32_t streamid) { struct xhci_stream_ctx *sctx; if (devep->ep_MaxPStreams == 0) return (XHCI_TRB_ERROR_TRB); if (devep->ep_MaxPStreams > XHCI_STREAMS_MAX) return (XHCI_TRB_ERROR_INVALID_SID); if (XHCI_EPCTX_0_LSA_GET(ep->dwEpCtx0) == 0) { DPRINTF(("pci_xhci: find_stream; LSA bit not set")); return (XHCI_TRB_ERROR_INVALID_SID); } /* only support primary stream */ if (streamid >= devep->ep_MaxPStreams) return (XHCI_TRB_ERROR_STREAM_TYPE); sctx = (struct xhci_stream_ctx *)XHCI_GADDR(sc, ep->qwEpCtx2 & ~0xFUL) + streamid; if (!XHCI_SCTX_0_SCT_GET(sctx->qwSctx0)) return (XHCI_TRB_ERROR_STREAM_TYPE); return (XHCI_TRB_ERROR_SUCCESS); } static uint32_t pci_xhci_cmd_set_tr(struct pci_xhci_softc *sc, uint32_t slot, struct xhci_trb *trb) { struct pci_xhci_dev_emu *dev; struct pci_xhci_dev_ep *devep; struct xhci_dev_ctx *dev_ctx; struct xhci_endp_ctx *ep_ctx; uint32_t cmderr, epid; uint32_t streamid; cmderr = pci_xhci_validate_slot(slot); if (cmderr != XHCI_TRB_ERROR_SUCCESS) goto done; dev = XHCI_SLOTDEV_PTR(sc, slot); assert(dev != NULL); DPRINTF(("pci_xhci set_tr: new-tr x%016lx, SCT %u DCS %u", (trb->qwTrb0 & ~0xF), (uint32_t)((trb->qwTrb0 >> 1) & 0x7), (uint32_t)(trb->qwTrb0 & 0x1))); DPRINTF((" stream-id %u, slot %u, epid %u, C %u", (trb->dwTrb2 >> 16) & 0xFFFF, XHCI_TRB_3_SLOT_GET(trb->dwTrb3), XHCI_TRB_3_EP_GET(trb->dwTrb3), trb->dwTrb3 & 0x1)); epid = XHCI_TRB_3_EP_GET(trb->dwTrb3); if (epid < 1 || epid > 31) { DPRINTF(("pci_xhci: set_tr_deq: invalid epid %u", epid)); cmderr = XHCI_TRB_ERROR_TRB; goto done; } dev_ctx = dev->dev_ctx; assert(dev_ctx != NULL); ep_ctx = &dev_ctx->ctx_ep[epid]; devep = &dev->eps[epid]; switch (XHCI_EPCTX_0_EPSTATE_GET(ep_ctx->dwEpCtx0)) { case XHCI_ST_EPCTX_STOPPED: case XHCI_ST_EPCTX_ERROR: break; default: DPRINTF(("pci_xhci cmd set_tr invalid state %x", XHCI_EPCTX_0_EPSTATE_GET(ep_ctx->dwEpCtx0))); cmderr = XHCI_TRB_ERROR_CONTEXT_STATE; goto done; } streamid = XHCI_TRB_2_STREAM_GET(trb->dwTrb2); if (devep->ep_MaxPStreams > 0) { cmderr = pci_xhci_find_stream(sc, ep_ctx, devep, streamid); if (cmderr == XHCI_TRB_ERROR_SUCCESS) { assert(devep->ep_sctx != NULL); devep->ep_sctx[streamid].qwSctx0 = trb->qwTrb0; devep->ep_sctx_trbs[streamid].ringaddr = trb->qwTrb0 & ~0xF; devep->ep_sctx_trbs[streamid].ccs = XHCI_EPCTX_2_DCS_GET(trb->qwTrb0); } } else { if (streamid != 0) { DPRINTF(("pci_xhci cmd set_tr streamid %x != 0", streamid)); } ep_ctx->qwEpCtx2 = trb->qwTrb0 & ~0xFUL; devep->ep_ringaddr = ep_ctx->qwEpCtx2 & ~0xFUL; devep->ep_ccs = trb->qwTrb0 & 0x1; devep->ep_tr = XHCI_GADDR(sc, devep->ep_ringaddr); DPRINTF(("pci_xhci set_tr first TRB:")); pci_xhci_dump_trb(devep->ep_tr); } ep_ctx->dwEpCtx0 = (ep_ctx->dwEpCtx0 & ~0x7) | XHCI_ST_EPCTX_STOPPED; done: return (cmderr); } static uint32_t pci_xhci_cmd_eval_ctx(struct pci_xhci_softc *sc, uint32_t slot, struct xhci_trb *trb) { struct xhci_input_dev_ctx *input_ctx; struct xhci_slot_ctx *islot_ctx; struct xhci_dev_ctx *dev_ctx; struct xhci_endp_ctx *ep0_ctx; uint32_t cmderr; input_ctx = XHCI_GADDR(sc, trb->qwTrb0 & ~0xFUL); islot_ctx = &input_ctx->ctx_slot; ep0_ctx = &input_ctx->ctx_ep[1]; DPRINTF(("pci_xhci: eval ctx, input ctl: D 0x%08x A 0x%08x,", input_ctx->ctx_input.dwInCtx0, input_ctx->ctx_input.dwInCtx1)); DPRINTF((" slot %08x %08x %08x %08x", islot_ctx->dwSctx0, islot_ctx->dwSctx1, islot_ctx->dwSctx2, islot_ctx->dwSctx3)); DPRINTF((" ep0 %08x %08x %016lx %08x", ep0_ctx->dwEpCtx0, ep0_ctx->dwEpCtx1, ep0_ctx->qwEpCtx2, ep0_ctx->dwEpCtx4)); /* this command expects drop-ctx=0 & add-ctx=slot+ep0 */ if ((input_ctx->ctx_input.dwInCtx0 != 0) || (input_ctx->ctx_input.dwInCtx1 & 0x03) == 0) { DPRINTF(("pci_xhci: eval ctx, input ctl invalid")); cmderr = XHCI_TRB_ERROR_TRB; goto done; } cmderr = pci_xhci_validate_slot(slot); if (cmderr != XHCI_TRB_ERROR_SUCCESS) goto done; /* assign address to slot; in this emulation, slot_id = address */ dev_ctx = pci_xhci_get_dev_ctx(sc, slot); if (dev_ctx == NULL) { cmderr = XHCI_TRB_ERROR_PARAMETER; goto done; } DPRINTF(("pci_xhci: eval ctx, dev ctx")); DPRINTF((" slot %08x %08x %08x %08x", dev_ctx->ctx_slot.dwSctx0, dev_ctx->ctx_slot.dwSctx1, dev_ctx->ctx_slot.dwSctx2, dev_ctx->ctx_slot.dwSctx3)); if (input_ctx->ctx_input.dwInCtx1 & 0x01) { /* slot ctx */ /* set max exit latency */ dev_ctx->ctx_slot.dwSctx1 = FIELD_COPY( dev_ctx->ctx_slot.dwSctx1, input_ctx->ctx_slot.dwSctx1, 0xFFFF, 0); /* set interrupter target */ dev_ctx->ctx_slot.dwSctx2 = FIELD_COPY( dev_ctx->ctx_slot.dwSctx2, input_ctx->ctx_slot.dwSctx2, 0x3FF, 22); } if (input_ctx->ctx_input.dwInCtx1 & 0x02) { /* control ctx */ /* set max packet size */ dev_ctx->ctx_ep[1].dwEpCtx1 = FIELD_COPY( dev_ctx->ctx_ep[1].dwEpCtx1, ep0_ctx->dwEpCtx1, 0xFFFF, 16); ep0_ctx = &dev_ctx->ctx_ep[1]; } DPRINTF(("pci_xhci: eval ctx, output ctx")); DPRINTF((" slot %08x %08x %08x %08x", dev_ctx->ctx_slot.dwSctx0, dev_ctx->ctx_slot.dwSctx1, dev_ctx->ctx_slot.dwSctx2, dev_ctx->ctx_slot.dwSctx3)); DPRINTF((" ep0 %08x %08x %016lx %08x", ep0_ctx->dwEpCtx0, ep0_ctx->dwEpCtx1, ep0_ctx->qwEpCtx2, ep0_ctx->dwEpCtx4)); done: return (cmderr); } static int pci_xhci_complete_commands(struct pci_xhci_softc *sc) { struct xhci_trb evtrb; struct xhci_trb *trb; uint64_t crcr; uint32_t ccs; /* cycle state (XHCI 4.9.2) */ uint32_t type; uint32_t slot; uint32_t cmderr; int error; error = 0; sc->opregs.crcr |= XHCI_CRCR_LO_CRR; trb = sc->opregs.cr_p; ccs = sc->opregs.crcr & XHCI_CRCR_LO_RCS; crcr = sc->opregs.crcr & ~0xF; while (1) { sc->opregs.cr_p = trb; type = XHCI_TRB_3_TYPE_GET(trb->dwTrb3); if ((trb->dwTrb3 & XHCI_TRB_3_CYCLE_BIT) != (ccs & XHCI_TRB_3_CYCLE_BIT)) break; DPRINTF(("pci_xhci: cmd type 0x%x, Trb0 x%016lx dwTrb2 x%08x" " dwTrb3 x%08x, TRB_CYCLE %u/ccs %u", type, trb->qwTrb0, trb->dwTrb2, trb->dwTrb3, trb->dwTrb3 & XHCI_TRB_3_CYCLE_BIT, ccs)); cmderr = XHCI_TRB_ERROR_SUCCESS; evtrb.dwTrb2 = 0; evtrb.dwTrb3 = (ccs & XHCI_TRB_3_CYCLE_BIT) | XHCI_TRB_3_TYPE_SET(XHCI_TRB_EVENT_CMD_COMPLETE); slot = 0; switch (type) { case XHCI_TRB_TYPE_LINK: /* 0x06 */ if (trb->dwTrb3 & XHCI_TRB_3_TC_BIT) ccs ^= XHCI_CRCR_LO_RCS; break; case XHCI_TRB_TYPE_ENABLE_SLOT: /* 0x09 */ cmderr = pci_xhci_cmd_enable_slot(sc, &slot); break; case XHCI_TRB_TYPE_DISABLE_SLOT: /* 0x0A */ slot = XHCI_TRB_3_SLOT_GET(trb->dwTrb3); cmderr = pci_xhci_cmd_disable_slot(sc, slot); break; case XHCI_TRB_TYPE_ADDRESS_DEVICE: /* 0x0B */ slot = XHCI_TRB_3_SLOT_GET(trb->dwTrb3); cmderr = pci_xhci_cmd_address_device(sc, slot, trb); break; case XHCI_TRB_TYPE_CONFIGURE_EP: /* 0x0C */ slot = XHCI_TRB_3_SLOT_GET(trb->dwTrb3); cmderr = pci_xhci_cmd_config_ep(sc, slot, trb); break; case XHCI_TRB_TYPE_EVALUATE_CTX: /* 0x0D */ slot = XHCI_TRB_3_SLOT_GET(trb->dwTrb3); cmderr = pci_xhci_cmd_eval_ctx(sc, slot, trb); break; case XHCI_TRB_TYPE_RESET_EP: /* 0x0E */ DPRINTF(("Reset Endpoint on slot %d", slot)); slot = XHCI_TRB_3_SLOT_GET(trb->dwTrb3); cmderr = pci_xhci_cmd_reset_ep(sc, slot, trb); break; case XHCI_TRB_TYPE_STOP_EP: /* 0x0F */ DPRINTF(("Stop Endpoint on slot %d", slot)); slot = XHCI_TRB_3_SLOT_GET(trb->dwTrb3); cmderr = pci_xhci_cmd_reset_ep(sc, slot, trb); break; case XHCI_TRB_TYPE_SET_TR_DEQUEUE: /* 0x10 */ slot = XHCI_TRB_3_SLOT_GET(trb->dwTrb3); cmderr = pci_xhci_cmd_set_tr(sc, slot, trb); break; case XHCI_TRB_TYPE_RESET_DEVICE: /* 0x11 */ slot = XHCI_TRB_3_SLOT_GET(trb->dwTrb3); cmderr = pci_xhci_cmd_reset_device(sc, slot); break; case XHCI_TRB_TYPE_FORCE_EVENT: /* 0x12 */ /* TODO: */ break; case XHCI_TRB_TYPE_NEGOTIATE_BW: /* 0x13 */ break; case XHCI_TRB_TYPE_SET_LATENCY_TOL: /* 0x14 */ break; case XHCI_TRB_TYPE_GET_PORT_BW: /* 0x15 */ break; case XHCI_TRB_TYPE_FORCE_HEADER: /* 0x16 */ break; case XHCI_TRB_TYPE_NOOP_CMD: /* 0x17 */ break; default: DPRINTF(("pci_xhci: unsupported cmd %x", type)); break; } if (type != XHCI_TRB_TYPE_LINK) { /* * insert command completion event and assert intr */ evtrb.qwTrb0 = crcr; evtrb.dwTrb2 |= XHCI_TRB_2_ERROR_SET(cmderr); evtrb.dwTrb3 |= XHCI_TRB_3_SLOT_SET(slot); DPRINTF(("pci_xhci: command 0x%x result: 0x%x", type, cmderr)); pci_xhci_insert_event(sc, &evtrb, 1); } trb = pci_xhci_trb_next(sc, trb, &crcr); } sc->opregs.crcr = crcr | (sc->opregs.crcr & XHCI_CRCR_LO_CA) | ccs; sc->opregs.crcr &= ~XHCI_CRCR_LO_CRR; return (error); } static void pci_xhci_dump_trb(struct xhci_trb *trb) { static const char *trbtypes[] = { "RESERVED", "NORMAL", "SETUP_STAGE", "DATA_STAGE", "STATUS_STAGE", "ISOCH", "LINK", "EVENT_DATA", "NOOP", "ENABLE_SLOT", "DISABLE_SLOT", "ADDRESS_DEVICE", "CONFIGURE_EP", "EVALUATE_CTX", "RESET_EP", "STOP_EP", "SET_TR_DEQUEUE", "RESET_DEVICE", "FORCE_EVENT", "NEGOTIATE_BW", "SET_LATENCY_TOL", "GET_PORT_BW", "FORCE_HEADER", "NOOP_CMD" }; uint32_t type; type = XHCI_TRB_3_TYPE_GET(trb->dwTrb3); DPRINTF(("pci_xhci: trb[@%p] type x%02x %s 0:x%016lx 2:x%08x 3:x%08x", trb, type, type <= XHCI_TRB_TYPE_NOOP_CMD ? trbtypes[type] : "INVALID", trb->qwTrb0, trb->dwTrb2, trb->dwTrb3)); } static int pci_xhci_xfer_complete(struct pci_xhci_softc *sc, struct usb_data_xfer *xfer, uint32_t slot, uint32_t epid, int *do_intr) { struct pci_xhci_dev_emu *dev; struct pci_xhci_dev_ep *devep; struct xhci_dev_ctx *dev_ctx; struct xhci_endp_ctx *ep_ctx; struct xhci_trb *trb; struct xhci_trb evtrb; uint32_t trbflags; uint32_t edtla; int i, err; dev = XHCI_SLOTDEV_PTR(sc, slot); devep = &dev->eps[epid]; dev_ctx = pci_xhci_get_dev_ctx(sc, slot); if (dev_ctx == NULL) { return XHCI_TRB_ERROR_PARAMETER; } ep_ctx = &dev_ctx->ctx_ep[epid]; err = XHCI_TRB_ERROR_SUCCESS; *do_intr = 0; edtla = 0; /* go through list of TRBs and insert event(s) */ for (i = xfer->head; xfer->ndata > 0; ) { evtrb.qwTrb0 = (uint64_t)xfer->data[i].hci_data; trb = XHCI_GADDR(sc, evtrb.qwTrb0); trbflags = trb->dwTrb3; DPRINTF(("pci_xhci: xfer[%d] done?%u:%d trb %x %016lx %x " "(err %d) IOC?%d", i, xfer->data[i].processed, xfer->data[i].blen, XHCI_TRB_3_TYPE_GET(trbflags), evtrb.qwTrb0, trbflags, err, trb->dwTrb3 & XHCI_TRB_3_IOC_BIT ? 1 : 0)); if (!xfer->data[i].processed) { xfer->head = i; break; } xfer->ndata--; edtla += xfer->data[i].bdone; trb->dwTrb3 = (trb->dwTrb3 & ~0x1) | (xfer->data[i].ccs); pci_xhci_update_ep_ring(sc, dev, devep, ep_ctx, xfer->data[i].streamid, xfer->data[i].trbnext, xfer->data[i].ccs); /* Only interrupt if IOC or short packet */ if (!(trb->dwTrb3 & XHCI_TRB_3_IOC_BIT) && !((err == XHCI_TRB_ERROR_SHORT_PKT) && (trb->dwTrb3 & XHCI_TRB_3_ISP_BIT))) { i = (i + 1) % USB_MAX_XFER_BLOCKS; continue; } evtrb.dwTrb2 = XHCI_TRB_2_ERROR_SET(err) | XHCI_TRB_2_REM_SET(xfer->data[i].blen); evtrb.dwTrb3 = XHCI_TRB_3_TYPE_SET(XHCI_TRB_EVENT_TRANSFER) | XHCI_TRB_3_SLOT_SET(slot) | XHCI_TRB_3_EP_SET(epid); if (XHCI_TRB_3_TYPE_GET(trbflags) == XHCI_TRB_TYPE_EVENT_DATA) { DPRINTF(("pci_xhci EVENT_DATA edtla %u", edtla)); evtrb.qwTrb0 = trb->qwTrb0; evtrb.dwTrb2 = (edtla & 0xFFFFF) | XHCI_TRB_2_ERROR_SET(err); evtrb.dwTrb3 |= XHCI_TRB_3_ED_BIT; edtla = 0; } *do_intr = 1; err = pci_xhci_insert_event(sc, &evtrb, 0); if (err != XHCI_TRB_ERROR_SUCCESS) { break; } i = (i + 1) % USB_MAX_XFER_BLOCKS; } return (err); } static void pci_xhci_update_ep_ring(struct pci_xhci_softc *sc, struct pci_xhci_dev_emu *dev __unused, struct pci_xhci_dev_ep *devep, struct xhci_endp_ctx *ep_ctx, uint32_t streamid, uint64_t ringaddr, int ccs) { if (devep->ep_MaxPStreams != 0) { devep->ep_sctx[streamid].qwSctx0 = (ringaddr & ~0xFUL) | (ccs & 0x1); devep->ep_sctx_trbs[streamid].ringaddr = ringaddr & ~0xFUL; devep->ep_sctx_trbs[streamid].ccs = ccs & 0x1; ep_ctx->qwEpCtx2 = (ep_ctx->qwEpCtx2 & ~0x1) | (ccs & 0x1); DPRINTF(("xhci update ep-ring stream %d, addr %lx", streamid, devep->ep_sctx[streamid].qwSctx0)); } else { devep->ep_ringaddr = ringaddr & ~0xFUL; devep->ep_ccs = ccs & 0x1; devep->ep_tr = XHCI_GADDR(sc, ringaddr & ~0xFUL); ep_ctx->qwEpCtx2 = (ringaddr & ~0xFUL) | (ccs & 0x1); DPRINTF(("xhci update ep-ring, addr %lx", (devep->ep_ringaddr | devep->ep_ccs))); } } static int pci_xhci_validate_slot(uint32_t slot) { if (slot == 0) return (XHCI_TRB_ERROR_TRB); else if (slot > XHCI_MAX_SLOTS) return (XHCI_TRB_ERROR_SLOT_NOT_ON); else return (XHCI_TRB_ERROR_SUCCESS); } /* * Outstanding transfer still in progress (device NAK'd earlier) so retry * the transfer again to see if it succeeds. */ static int pci_xhci_try_usb_xfer(struct pci_xhci_softc *sc, struct pci_xhci_dev_emu *dev, struct pci_xhci_dev_ep *devep, struct xhci_endp_ctx *ep_ctx, uint32_t slot, uint32_t epid) { struct usb_data_xfer *xfer; int err; int do_intr; ep_ctx->dwEpCtx0 = FIELD_REPLACE( ep_ctx->dwEpCtx0, XHCI_ST_EPCTX_RUNNING, 0x7, 0); err = 0; do_intr = 0; xfer = devep->ep_xfer; #ifdef __FreeBSD__ USB_DATA_XFER_LOCK(xfer); #else /* * At least one caller needs to hold this lock across the call to this * function and other code. To avoid deadlock from a recursive mutex * enter, we ensure that all callers hold this lock. */ assert(USB_DATA_XFER_LOCK_HELD(xfer)); #endif /* outstanding requests queued up */ if (dev->dev_ue->ue_data != NULL) { err = dev->dev_ue->ue_data(dev->dev_sc, xfer, epid & 0x1 ? USB_XFER_IN : USB_XFER_OUT, epid/2); if (err == USB_ERR_CANCELLED) { if (USB_DATA_GET_ERRCODE(&xfer->data[xfer->head]) == USB_NAK) err = XHCI_TRB_ERROR_SUCCESS; } else { err = pci_xhci_xfer_complete(sc, xfer, slot, epid, &do_intr); if (err == XHCI_TRB_ERROR_SUCCESS && do_intr) { pci_xhci_assert_interrupt(sc); } /* XXX should not do it if error? */ USB_DATA_XFER_RESET(xfer); } } #ifdef __FreeBSD__ USB_DATA_XFER_UNLOCK(xfer); #endif return (err); } static int pci_xhci_handle_transfer(struct pci_xhci_softc *sc, struct pci_xhci_dev_emu *dev, struct pci_xhci_dev_ep *devep, struct xhci_endp_ctx *ep_ctx, struct xhci_trb *trb, uint32_t slot, uint32_t epid, uint64_t addr, uint32_t ccs, uint32_t streamid) { struct xhci_trb *setup_trb; struct usb_data_xfer *xfer; struct usb_data_xfer_block *xfer_block; uint64_t val; uint32_t trbflags; int do_intr, err; int do_retry; ep_ctx->dwEpCtx0 = FIELD_REPLACE(ep_ctx->dwEpCtx0, XHCI_ST_EPCTX_RUNNING, 0x7, 0); xfer = devep->ep_xfer; USB_DATA_XFER_LOCK(xfer); DPRINTF(("pci_xhci handle_transfer slot %u", slot)); retry: err = XHCI_TRB_ERROR_INVALID; do_retry = 0; do_intr = 0; setup_trb = NULL; while (1) { pci_xhci_dump_trb(trb); trbflags = trb->dwTrb3; if (XHCI_TRB_3_TYPE_GET(trbflags) != XHCI_TRB_TYPE_LINK && (trbflags & XHCI_TRB_3_CYCLE_BIT) != (ccs & XHCI_TRB_3_CYCLE_BIT)) { DPRINTF(("Cycle-bit changed trbflags %x, ccs %x", trbflags & XHCI_TRB_3_CYCLE_BIT, ccs)); break; } xfer_block = NULL; switch (XHCI_TRB_3_TYPE_GET(trbflags)) { case XHCI_TRB_TYPE_LINK: if (trb->dwTrb3 & XHCI_TRB_3_TC_BIT) ccs ^= 0x1; xfer_block = usb_data_xfer_append(xfer, NULL, 0, (void *)addr, ccs); xfer_block->processed = 1; break; case XHCI_TRB_TYPE_SETUP_STAGE: if ((trbflags & XHCI_TRB_3_IDT_BIT) == 0 || XHCI_TRB_2_BYTES_GET(trb->dwTrb2) != 8) { DPRINTF(("pci_xhci: invalid setup trb")); err = XHCI_TRB_ERROR_TRB; goto errout; } setup_trb = trb; val = trb->qwTrb0; if (!xfer->ureq) xfer->ureq = malloc( sizeof(struct usb_device_request)); memcpy(xfer->ureq, &val, sizeof(struct usb_device_request)); xfer_block = usb_data_xfer_append(xfer, NULL, 0, (void *)addr, ccs); xfer_block->processed = 1; break; case XHCI_TRB_TYPE_NORMAL: case XHCI_TRB_TYPE_ISOCH: if (setup_trb != NULL) { DPRINTF(("pci_xhci: trb not supposed to be in " "ctl scope")); err = XHCI_TRB_ERROR_TRB; goto errout; } /* fall through */ case XHCI_TRB_TYPE_DATA_STAGE: xfer_block = usb_data_xfer_append(xfer, (void *)(trbflags & XHCI_TRB_3_IDT_BIT ? &trb->qwTrb0 : XHCI_GADDR(sc, trb->qwTrb0)), trb->dwTrb2 & 0x1FFFF, (void *)addr, ccs); break; case XHCI_TRB_TYPE_STATUS_STAGE: xfer_block = usb_data_xfer_append(xfer, NULL, 0, (void *)addr, ccs); break; case XHCI_TRB_TYPE_NOOP: xfer_block = usb_data_xfer_append(xfer, NULL, 0, (void *)addr, ccs); xfer_block->processed = 1; break; case XHCI_TRB_TYPE_EVENT_DATA: xfer_block = usb_data_xfer_append(xfer, NULL, 0, (void *)addr, ccs); if ((epid > 1) && (trbflags & XHCI_TRB_3_IOC_BIT)) { xfer_block->processed = 1; } break; default: DPRINTF(("pci_xhci: handle xfer unexpected trb type " "0x%x", XHCI_TRB_3_TYPE_GET(trbflags))); err = XHCI_TRB_ERROR_TRB; goto errout; } trb = pci_xhci_trb_next(sc, trb, &addr); DPRINTF(("pci_xhci: next trb: 0x%lx", (uint64_t)trb)); if (xfer_block) { xfer_block->trbnext = addr; xfer_block->streamid = streamid; } if (!setup_trb && !(trbflags & XHCI_TRB_3_CHAIN_BIT) && XHCI_TRB_3_TYPE_GET(trbflags) != XHCI_TRB_TYPE_LINK) { break; } /* handle current batch that requires interrupt on complete */ if (trbflags & XHCI_TRB_3_IOC_BIT) { DPRINTF(("pci_xhci: trb IOC bit set")); if (epid == 1) do_retry = 1; break; } } DPRINTF(("pci_xhci[%d]: xfer->ndata %u", __LINE__, xfer->ndata)); if (xfer->ndata <= 0) goto errout; if (epid == 1) { int usberr; if (dev->dev_ue->ue_request != NULL) usberr = dev->dev_ue->ue_request(dev->dev_sc, xfer); else usberr = USB_ERR_NOT_STARTED; err = USB_TO_XHCI_ERR(usberr); if (err == XHCI_TRB_ERROR_SUCCESS || err == XHCI_TRB_ERROR_STALL || err == XHCI_TRB_ERROR_SHORT_PKT) { err = pci_xhci_xfer_complete(sc, xfer, slot, epid, &do_intr); if (err != XHCI_TRB_ERROR_SUCCESS) do_retry = 0; } } else { /* handle data transfer */ pci_xhci_try_usb_xfer(sc, dev, devep, ep_ctx, slot, epid); err = XHCI_TRB_ERROR_SUCCESS; } errout: if (err == XHCI_TRB_ERROR_EV_RING_FULL) DPRINTF(("pci_xhci[%d]: event ring full", __LINE__)); if (!do_retry) USB_DATA_XFER_UNLOCK(xfer); if (do_intr) pci_xhci_assert_interrupt(sc); if (do_retry) { USB_DATA_XFER_RESET(xfer); DPRINTF(("pci_xhci[%d]: retry:continuing with next TRBs", __LINE__)); goto retry; } if (epid == 1) USB_DATA_XFER_RESET(xfer); return (err); } static void pci_xhci_device_doorbell(struct pci_xhci_softc *sc, uint32_t slot, uint32_t epid, uint32_t streamid) { struct pci_xhci_dev_emu *dev; struct pci_xhci_dev_ep *devep; struct xhci_dev_ctx *dev_ctx; struct xhci_endp_ctx *ep_ctx; struct pci_xhci_trb_ring *sctx_tr; struct xhci_trb *trb; uint64_t ringaddr; uint32_t ccs; int error; DPRINTF(("pci_xhci doorbell slot %u epid %u stream %u", slot, epid, streamid)); if (slot == 0 || slot > XHCI_MAX_SLOTS) { DPRINTF(("pci_xhci: invalid doorbell slot %u", slot)); return; } if (epid == 0 || epid >= XHCI_MAX_ENDPOINTS) { DPRINTF(("pci_xhci: invalid endpoint %u", epid)); return; } dev = XHCI_SLOTDEV_PTR(sc, slot); devep = &dev->eps[epid]; dev_ctx = pci_xhci_get_dev_ctx(sc, slot); if (!dev_ctx) { return; } ep_ctx = &dev_ctx->ctx_ep[epid]; sctx_tr = NULL; DPRINTF(("pci_xhci: device doorbell ep[%u] %08x %08x %016lx %08x", epid, ep_ctx->dwEpCtx0, ep_ctx->dwEpCtx1, ep_ctx->qwEpCtx2, ep_ctx->dwEpCtx4)); if (ep_ctx->qwEpCtx2 == 0) return; /* handle pending transfers */ if (devep->ep_xfer->ndata > 0) { #ifndef __FreeBSD__ USB_DATA_XFER_LOCK(devep->ep_xfer); #endif pci_xhci_try_usb_xfer(sc, dev, devep, ep_ctx, slot, epid); #ifndef __FreeBSD__ USB_DATA_XFER_UNLOCK(devep->ep_xfer); #endif return; } /* get next trb work item */ if (devep->ep_MaxPStreams != 0) { /* * Stream IDs of 0, 65535 (any stream), and 65534 * (prime) are invalid. */ if (streamid == 0 || streamid == 65534 || streamid == 65535) { DPRINTF(("pci_xhci: invalid stream %u", streamid)); return; } error = pci_xhci_find_stream(sc, ep_ctx, devep, streamid); if (error != XHCI_TRB_ERROR_SUCCESS) { DPRINTF(("pci_xhci: invalid stream %u: %d", streamid, error)); return; } sctx_tr = &devep->ep_sctx_trbs[streamid]; ringaddr = sctx_tr->ringaddr; ccs = sctx_tr->ccs; trb = XHCI_GADDR(sc, sctx_tr->ringaddr & ~0xFUL); DPRINTF(("doorbell, stream %u, ccs %lx, trb ccs %x", streamid, ep_ctx->qwEpCtx2 & XHCI_TRB_3_CYCLE_BIT, trb->dwTrb3 & XHCI_TRB_3_CYCLE_BIT)); } else { if (streamid != 0) { DPRINTF(("pci_xhci: invalid stream %u", streamid)); return; } ringaddr = devep->ep_ringaddr; ccs = devep->ep_ccs; trb = devep->ep_tr; DPRINTF(("doorbell, ccs %lx, trb ccs %x", ep_ctx->qwEpCtx2 & XHCI_TRB_3_CYCLE_BIT, trb->dwTrb3 & XHCI_TRB_3_CYCLE_BIT)); } if (XHCI_TRB_3_TYPE_GET(trb->dwTrb3) == 0) { DPRINTF(("pci_xhci: ring %lx trb[%lx] EP %u is RESERVED?", ep_ctx->qwEpCtx2, devep->ep_ringaddr, epid)); return; } pci_xhci_handle_transfer(sc, dev, devep, ep_ctx, trb, slot, epid, ringaddr, ccs, streamid); } static void pci_xhci_dbregs_write(struct pci_xhci_softc *sc, uint64_t offset, uint64_t value) { offset = (offset - sc->dboff) / sizeof(uint32_t); DPRINTF(("pci_xhci: doorbell write offset 0x%lx: 0x%lx", offset, value)); if (XHCI_HALTED(sc)) { DPRINTF(("pci_xhci: controller halted")); return; } if (offset == 0) pci_xhci_complete_commands(sc); else if (sc->portregs != NULL) pci_xhci_device_doorbell(sc, offset, XHCI_DB_TARGET_GET(value), XHCI_DB_SID_GET(value)); } static void pci_xhci_rtsregs_write(struct pci_xhci_softc *sc, uint64_t offset, uint64_t value) { struct pci_xhci_rtsregs *rts; offset -= sc->rtsoff; if (offset == 0) { DPRINTF(("pci_xhci attempted write to MFINDEX")); return; } DPRINTF(("pci_xhci: runtime regs write offset 0x%lx: 0x%lx", offset, value)); offset -= 0x20; /* start of intrreg */ rts = &sc->rtsregs; switch (offset) { case 0x00: if (value & XHCI_IMAN_INTR_PEND) rts->intrreg.iman &= ~XHCI_IMAN_INTR_PEND; rts->intrreg.iman = (value & XHCI_IMAN_INTR_ENA) | (rts->intrreg.iman & XHCI_IMAN_INTR_PEND); if (!(value & XHCI_IMAN_INTR_ENA)) pci_xhci_deassert_interrupt(sc); break; case 0x04: rts->intrreg.imod = value; break; case 0x08: rts->intrreg.erstsz = value & 0xFFFF; break; case 0x10: /* ERSTBA low bits */ rts->intrreg.erstba = MASK_64_HI(sc->rtsregs.intrreg.erstba) | (value & ~0x3F); break; case 0x14: /* ERSTBA high bits */ rts->intrreg.erstba = (value << 32) | MASK_64_LO(sc->rtsregs.intrreg.erstba); rts->erstba_p = XHCI_GADDR(sc, sc->rtsregs.intrreg.erstba & ~0x3FUL); rts->erst_p = XHCI_GADDR(sc, sc->rtsregs.erstba_p->qwEvrsTablePtr & ~0x3FUL); rts->er_enq_idx = 0; rts->er_events_cnt = 0; DPRINTF(("pci_xhci: wr erstba erst (%p) ptr 0x%lx, sz %u", rts->erstba_p, rts->erstba_p->qwEvrsTablePtr, rts->erstba_p->dwEvrsTableSize)); break; case 0x18: /* ERDP low bits */ rts->intrreg.erdp = MASK_64_HI(sc->rtsregs.intrreg.erdp) | (rts->intrreg.erdp & XHCI_ERDP_LO_BUSY) | (value & ~0xF); if (value & XHCI_ERDP_LO_BUSY) { rts->intrreg.erdp &= ~XHCI_ERDP_LO_BUSY; rts->intrreg.iman &= ~XHCI_IMAN_INTR_PEND; } rts->er_deq_seg = XHCI_ERDP_LO_SINDEX(value); break; case 0x1C: /* ERDP high bits */ rts->intrreg.erdp = (value << 32) | MASK_64_LO(sc->rtsregs.intrreg.erdp); if (rts->er_events_cnt > 0) { uint64_t erdp; int erdp_i; erdp = rts->intrreg.erdp & ~0xF; erdp_i = (erdp - rts->erstba_p->qwEvrsTablePtr) / sizeof(struct xhci_trb); if (erdp_i <= rts->er_enq_idx) rts->er_events_cnt = rts->er_enq_idx - erdp_i; else rts->er_events_cnt = rts->erstba_p->dwEvrsTableSize - (erdp_i - rts->er_enq_idx); DPRINTF(("pci_xhci: erdp 0x%lx, events cnt %u", erdp, rts->er_events_cnt)); } break; default: DPRINTF(("pci_xhci attempted write to RTS offset 0x%lx", offset)); break; } } static uint64_t pci_xhci_portregs_read(struct pci_xhci_softc *sc, uint64_t offset) { struct pci_xhci_portregs *portregs; int port; uint32_t reg; if (sc->portregs == NULL) return (0); port = (offset - XHCI_PORTREGS_PORT0) / XHCI_PORTREGS_SETSZ; offset = (offset - XHCI_PORTREGS_PORT0) % XHCI_PORTREGS_SETSZ; if (port > XHCI_MAX_DEVS) { DPRINTF(("pci_xhci: portregs_read port %d >= XHCI_MAX_DEVS", port)); /* return default value for unused port */ return (XHCI_PS_SPEED_SET(3)); } portregs = XHCI_PORTREG_PTR(sc, port); switch (offset) { case 0: reg = portregs->portsc; break; case 4: reg = portregs->portpmsc; break; case 8: reg = portregs->portli; break; case 12: reg = portregs->porthlpmc; break; default: DPRINTF(("pci_xhci: unaligned portregs read offset %#lx", offset)); reg = 0xffffffff; break; } DPRINTF(("pci_xhci: portregs read offset 0x%lx port %u -> 0x%x", offset, port, reg)); return (reg); } static void pci_xhci_hostop_write(struct pci_xhci_softc *sc, uint64_t offset, uint64_t value) { offset -= XHCI_CAPLEN; if (offset < 0x400) DPRINTF(("pci_xhci: hostop write offset 0x%lx: 0x%lx", offset, value)); switch (offset) { case XHCI_USBCMD: sc->opregs.usbcmd = pci_xhci_usbcmd_write(sc, value & 0x3F0F); break; case XHCI_USBSTS: /* clear bits on write */ sc->opregs.usbsts &= ~(value & (XHCI_STS_HSE|XHCI_STS_EINT|XHCI_STS_PCD|XHCI_STS_SSS| XHCI_STS_RSS|XHCI_STS_SRE|XHCI_STS_CNR)); break; case XHCI_PAGESIZE: /* read only */ break; case XHCI_DNCTRL: sc->opregs.dnctrl = value & 0xFFFF; break; case XHCI_CRCR_LO: if (sc->opregs.crcr & XHCI_CRCR_LO_CRR) { sc->opregs.crcr &= ~(XHCI_CRCR_LO_CS|XHCI_CRCR_LO_CA); sc->opregs.crcr |= value & (XHCI_CRCR_LO_CS|XHCI_CRCR_LO_CA); } else { sc->opregs.crcr = MASK_64_HI(sc->opregs.crcr) | (value & (0xFFFFFFC0 | XHCI_CRCR_LO_RCS)); } break; case XHCI_CRCR_HI: if (!(sc->opregs.crcr & XHCI_CRCR_LO_CRR)) { sc->opregs.crcr = MASK_64_LO(sc->opregs.crcr) | (value << 32); sc->opregs.cr_p = XHCI_GADDR(sc, sc->opregs.crcr & ~0xF); } if (sc->opregs.crcr & XHCI_CRCR_LO_CS) { /* Stop operation of Command Ring */ } if (sc->opregs.crcr & XHCI_CRCR_LO_CA) { /* Abort command */ } break; case XHCI_DCBAAP_LO: sc->opregs.dcbaap = MASK_64_HI(sc->opregs.dcbaap) | (value & 0xFFFFFFC0); break; case XHCI_DCBAAP_HI: sc->opregs.dcbaap = MASK_64_LO(sc->opregs.dcbaap) | (value << 32); sc->opregs.dcbaa_p = XHCI_GADDR(sc, sc->opregs.dcbaap & ~0x3FUL); DPRINTF(("pci_xhci: opregs dcbaap = 0x%lx (vaddr 0x%lx)", sc->opregs.dcbaap, (uint64_t)sc->opregs.dcbaa_p)); break; case XHCI_CONFIG: sc->opregs.config = value & 0x03FF; break; default: if (offset >= 0x400) pci_xhci_portregs_write(sc, offset, value); break; } } static void pci_xhci_write(struct pci_devinst *pi, int baridx, uint64_t offset, int size __unused, uint64_t value) { struct pci_xhci_softc *sc; sc = pi->pi_arg; assert(baridx == 0); pthread_mutex_lock(&sc->mtx); if (offset < XHCI_CAPLEN) /* read only registers */ WPRINTF(("pci_xhci: write RO-CAPs offset %ld", offset)); else if (offset < sc->dboff) pci_xhci_hostop_write(sc, offset, value); else if (offset < sc->rtsoff) pci_xhci_dbregs_write(sc, offset, value); else if (offset < sc->regsend) pci_xhci_rtsregs_write(sc, offset, value); else WPRINTF(("pci_xhci: write invalid offset %ld", offset)); pthread_mutex_unlock(&sc->mtx); } static uint64_t pci_xhci_hostcap_read(struct pci_xhci_softc *sc, uint64_t offset) { uint64_t value; switch (offset) { case XHCI_CAPLENGTH: /* 0x00 */ value = sc->caplength; break; case XHCI_HCSPARAMS1: /* 0x04 */ value = sc->hcsparams1; break; case XHCI_HCSPARAMS2: /* 0x08 */ value = sc->hcsparams2; break; case XHCI_HCSPARAMS3: /* 0x0C */ value = sc->hcsparams3; break; case XHCI_HCSPARAMS0: /* 0x10 */ value = sc->hccparams1; break; case XHCI_DBOFF: /* 0x14 */ value = sc->dboff; break; case XHCI_RTSOFF: /* 0x18 */ value = sc->rtsoff; break; case XHCI_HCCPRAMS2: /* 0x1C */ value = sc->hccparams2; break; default: value = 0; break; } DPRINTF(("pci_xhci: hostcap read offset 0x%lx -> 0x%lx", offset, value)); return (value); } static uint64_t pci_xhci_hostop_read(struct pci_xhci_softc *sc, uint64_t offset) { uint64_t value; offset = (offset - XHCI_CAPLEN); switch (offset) { case XHCI_USBCMD: /* 0x00 */ value = sc->opregs.usbcmd; break; case XHCI_USBSTS: /* 0x04 */ value = sc->opregs.usbsts; break; case XHCI_PAGESIZE: /* 0x08 */ value = sc->opregs.pgsz; break; case XHCI_DNCTRL: /* 0x14 */ value = sc->opregs.dnctrl; break; case XHCI_CRCR_LO: /* 0x18 */ value = sc->opregs.crcr & XHCI_CRCR_LO_CRR; break; case XHCI_CRCR_HI: /* 0x1C */ value = 0; break; case XHCI_DCBAAP_LO: /* 0x30 */ value = sc->opregs.dcbaap & 0xFFFFFFFF; break; case XHCI_DCBAAP_HI: /* 0x34 */ value = (sc->opregs.dcbaap >> 32) & 0xFFFFFFFF; break; case XHCI_CONFIG: /* 0x38 */ value = sc->opregs.config; break; default: if (offset >= 0x400) value = pci_xhci_portregs_read(sc, offset); else value = 0; break; } if (offset < 0x400) DPRINTF(("pci_xhci: hostop read offset 0x%lx -> 0x%lx", offset, value)); return (value); } static uint64_t pci_xhci_dbregs_read(struct pci_xhci_softc *sc __unused, uint64_t offset __unused) { /* read doorbell always returns 0 */ return (0); } static uint64_t pci_xhci_rtsregs_read(struct pci_xhci_softc *sc, uint64_t offset) { uint32_t value; offset -= sc->rtsoff; value = 0; if (offset == XHCI_MFINDEX) { value = sc->rtsregs.mfindex; } else if (offset >= 0x20) { int item; uint32_t *p; offset -= 0x20; item = offset % 32; assert(offset < sizeof(sc->rtsregs.intrreg)); p = &sc->rtsregs.intrreg.iman; p += item / sizeof(uint32_t); value = *p; } DPRINTF(("pci_xhci: rtsregs read offset 0x%lx -> 0x%x", offset, value)); return (value); } static uint64_t pci_xhci_xecp_read(struct pci_xhci_softc *sc, uint64_t offset) { uint32_t value; offset -= sc->regsend; value = 0; switch (offset) { case 0: /* rev major | rev minor | next-cap | cap-id */ value = (0x02 << 24) | (4 << 8) | XHCI_ID_PROTOCOLS; break; case 4: /* name string = "USB" */ value = 0x20425355; break; case 8: /* psic | proto-defined | compat # | compat offset */ value = ((XHCI_MAX_DEVS/2) << 8) | sc->usb2_port_start; break; case 12: break; case 16: /* rev major | rev minor | next-cap | cap-id */ value = (0x03 << 24) | XHCI_ID_PROTOCOLS; break; case 20: /* name string = "USB" */ value = 0x20425355; break; case 24: /* psic | proto-defined | compat # | compat offset */ value = ((XHCI_MAX_DEVS/2) << 8) | sc->usb3_port_start; break; case 28: break; default: DPRINTF(("pci_xhci: xecp invalid offset 0x%lx", offset)); break; } DPRINTF(("pci_xhci: xecp read offset 0x%lx -> 0x%x", offset, value)); return (value); } static uint64_t pci_xhci_read(struct pci_devinst *pi, int baridx, uint64_t offset, int size) { struct pci_xhci_softc *sc; uint32_t value; sc = pi->pi_arg; assert(baridx == 0); pthread_mutex_lock(&sc->mtx); if (offset < XHCI_CAPLEN) value = pci_xhci_hostcap_read(sc, offset); else if (offset < sc->dboff) value = pci_xhci_hostop_read(sc, offset); else if (offset < sc->rtsoff) value = pci_xhci_dbregs_read(sc, offset); else if (offset < sc->regsend) value = pci_xhci_rtsregs_read(sc, offset); else if (offset < (sc->regsend + 4*32)) value = pci_xhci_xecp_read(sc, offset); else { value = 0; WPRINTF(("pci_xhci: read invalid offset %ld", offset)); } pthread_mutex_unlock(&sc->mtx); switch (size) { case 1: value &= 0xFF; break; case 2: value &= 0xFFFF; break; case 4: value &= 0xFFFFFFFF; break; } return (value); } static void pci_xhci_reset_port(struct pci_xhci_softc *sc, int portn, int warm) { struct pci_xhci_portregs *port; struct pci_xhci_dev_emu *dev; struct xhci_trb evtrb; int error; assert(portn <= XHCI_MAX_DEVS); DPRINTF(("xhci reset port %d", portn)); port = XHCI_PORTREG_PTR(sc, portn); dev = XHCI_DEVINST_PTR(sc, portn); if (dev) { port->portsc &= ~(XHCI_PS_PLS_MASK | XHCI_PS_PR | XHCI_PS_PRC); port->portsc |= XHCI_PS_PED | XHCI_PS_SPEED_SET(dev->dev_ue->ue_usbspeed); if (warm && dev->dev_ue->ue_usbver == 3) { port->portsc |= XHCI_PS_WRC; } if ((port->portsc & XHCI_PS_PRC) == 0) { port->portsc |= XHCI_PS_PRC; pci_xhci_set_evtrb(&evtrb, portn, XHCI_TRB_ERROR_SUCCESS, XHCI_TRB_EVENT_PORT_STS_CHANGE); error = pci_xhci_insert_event(sc, &evtrb, 1); if (error != XHCI_TRB_ERROR_SUCCESS) DPRINTF(("xhci reset port insert event " "failed")); } } } static void pci_xhci_init_port(struct pci_xhci_softc *sc, int portn) { struct pci_xhci_portregs *port; struct pci_xhci_dev_emu *dev; port = XHCI_PORTREG_PTR(sc, portn); dev = XHCI_DEVINST_PTR(sc, portn); if (dev) { port->portsc = XHCI_PS_CCS | /* connected */ XHCI_PS_PP; /* port power */ if (dev->dev_ue->ue_usbver == 2) { port->portsc |= XHCI_PS_PLS_SET(UPS_PORT_LS_POLL) | XHCI_PS_SPEED_SET(dev->dev_ue->ue_usbspeed); } else { port->portsc |= XHCI_PS_PLS_SET(UPS_PORT_LS_U0) | XHCI_PS_PED | /* enabled */ XHCI_PS_SPEED_SET(dev->dev_ue->ue_usbspeed); } DPRINTF(("Init port %d 0x%x", portn, port->portsc)); } else { port->portsc = XHCI_PS_PLS_SET(UPS_PORT_LS_RX_DET) | XHCI_PS_PP; DPRINTF(("Init empty port %d 0x%x", portn, port->portsc)); } } static int pci_xhci_dev_intr(struct usb_hci *hci, int epctx) { struct pci_xhci_dev_emu *dev; struct xhci_dev_ctx *dev_ctx; struct xhci_trb evtrb; struct pci_xhci_softc *sc; struct pci_xhci_portregs *p; struct xhci_endp_ctx *ep_ctx; int error = 0; int dir_in; int epid; dir_in = epctx & 0x80; epid = epctx & ~0x80; /* HW endpoint contexts are 0-15; convert to epid based on dir */ epid = (epid * 2) + (dir_in ? 1 : 0); assert(epid >= 1 && epid <= 31); dev = hci->hci_sc; sc = dev->xsc; /* check if device is ready; OS has to initialise it */ if (sc->rtsregs.erstba_p == NULL || (sc->opregs.usbcmd & XHCI_CMD_RS) == 0 || dev->dev_ctx == NULL) return (0); p = XHCI_PORTREG_PTR(sc, hci->hci_port); /* raise event if link U3 (suspended) state */ if (XHCI_PS_PLS_GET(p->portsc) == 3) { p->portsc &= ~XHCI_PS_PLS_MASK; p->portsc |= XHCI_PS_PLS_SET(UPS_PORT_LS_RESUME); if ((p->portsc & XHCI_PS_PLC) != 0) return (0); p->portsc |= XHCI_PS_PLC; pci_xhci_set_evtrb(&evtrb, hci->hci_port, XHCI_TRB_ERROR_SUCCESS, XHCI_TRB_EVENT_PORT_STS_CHANGE); error = pci_xhci_insert_event(sc, &evtrb, 0); if (error != XHCI_TRB_ERROR_SUCCESS) goto done; } dev_ctx = dev->dev_ctx; ep_ctx = &dev_ctx->ctx_ep[epid]; if ((ep_ctx->dwEpCtx0 & 0x7) == XHCI_ST_EPCTX_DISABLED) { DPRINTF(("xhci device interrupt on disabled endpoint %d", epid)); return (0); } DPRINTF(("xhci device interrupt on endpoint %d", epid)); pci_xhci_device_doorbell(sc, hci->hci_port, epid, 0); done: return (error); } static int pci_xhci_dev_event(struct usb_hci *hci, enum hci_usbev evid __unused, void *param __unused) { DPRINTF(("xhci device event port %d", hci->hci_port)); return (0); } /* * Each controller contains a "slot" node which contains a list of * child nodes each of which is a device. Each slot node's name * corresponds to a specific controller slot. These nodes * contain a "device" variable identifying the device model of the * USB device. For example: * * pci.0.1.0 * .device="xhci" * .slot * .1 * .device="tablet" */ static int pci_xhci_legacy_config(nvlist_t *nvl, const char *opts) { char node_name[16]; nvlist_t *slots_nvl, *slot_nvl; char *cp, *opt, *str, *tofree; int slot; if (opts == NULL) return (0); slots_nvl = create_relative_config_node(nvl, "slot"); slot = 1; tofree = str = strdup(opts); while ((opt = strsep(&str, ",")) != NULL) { /* device[=] */ cp = strchr(opt, '='); if (cp != NULL) { *cp = '\0'; cp++; } snprintf(node_name, sizeof(node_name), "%d", slot); slot++; slot_nvl = create_relative_config_node(slots_nvl, node_name); set_config_value_node(slot_nvl, "device", opt); /* * NB: Given that we split on commas above, the legacy * format only supports a single option. */ if (cp != NULL && *cp != '\0') pci_parse_legacy_config(slot_nvl, cp); } free(tofree); return (0); } static int pci_xhci_parse_devices(struct pci_xhci_softc *sc, nvlist_t *nvl) { struct pci_xhci_dev_emu *dev; struct usb_devemu *ue; const nvlist_t *slots_nvl, *slot_nvl; const char *name, *device; char *cp; void *devsc, *cookie; long slot; int type, usb3_port, usb2_port, i, ndevices; usb3_port = sc->usb3_port_start; usb2_port = sc->usb2_port_start; sc->devices = calloc(XHCI_MAX_DEVS, sizeof(struct pci_xhci_dev_emu *)); sc->slots = calloc(XHCI_MAX_SLOTS, sizeof(struct pci_xhci_dev_emu *)); ndevices = 0; slots_nvl = find_relative_config_node(nvl, "slot"); if (slots_nvl == NULL) goto portsfinal; cookie = NULL; while ((name = nvlist_next(slots_nvl, &type, &cookie)) != NULL) { if (usb2_port == ((sc->usb2_port_start) + XHCI_MAX_DEVS/2) || usb3_port == ((sc->usb3_port_start) + XHCI_MAX_DEVS/2)) { WPRINTF(("pci_xhci max number of USB 2 or 3 " "devices reached, max %d", XHCI_MAX_DEVS/2)); goto bad; } if (type != NV_TYPE_NVLIST) { EPRINTLN( "pci_xhci: config variable '%s' under slot node", name); goto bad; } slot = strtol(name, &cp, 0); if (*cp != '\0' || slot <= 0 || slot > XHCI_MAX_SLOTS) { EPRINTLN("pci_xhci: invalid slot '%s'", name); goto bad; } if (XHCI_SLOTDEV_PTR(sc, slot) != NULL) { EPRINTLN("pci_xhci: duplicate slot '%s'", name); goto bad; } slot_nvl = nvlist_get_nvlist(slots_nvl, name); device = get_config_value_node(slot_nvl, "device"); if (device == NULL) { EPRINTLN( "pci_xhci: missing \"device\" value for slot '%s'", name); goto bad; } ue = usb_emu_finddev(device); if (ue == NULL) { EPRINTLN("pci_xhci: unknown device model \"%s\"", device); goto bad; } DPRINTF(("pci_xhci adding device %s", device)); dev = calloc(1, sizeof(struct pci_xhci_dev_emu)); dev->xsc = sc; dev->hci.hci_sc = dev; dev->hci.hci_intr = pci_xhci_dev_intr; dev->hci.hci_event = pci_xhci_dev_event; if (ue->ue_usbver == 2) { if (usb2_port == sc->usb2_port_start + XHCI_MAX_DEVS / 2) { WPRINTF(("pci_xhci max number of USB 2 devices " "reached, max %d", XHCI_MAX_DEVS / 2)); goto bad; } dev->hci.hci_port = usb2_port; usb2_port++; } else { if (usb3_port == sc->usb3_port_start + XHCI_MAX_DEVS / 2) { WPRINTF(("pci_xhci max number of USB 3 devices " "reached, max %d", XHCI_MAX_DEVS / 2)); goto bad; } dev->hci.hci_port = usb3_port; usb3_port++; } XHCI_DEVINST_PTR(sc, dev->hci.hci_port) = dev; dev->hci.hci_address = 0; devsc = ue->ue_init(&dev->hci, nvl); if (devsc == NULL) { goto bad; } dev->dev_ue = ue; dev->dev_sc = devsc; XHCI_SLOTDEV_PTR(sc, slot) = dev; ndevices++; } portsfinal: sc->portregs = calloc(XHCI_MAX_DEVS, sizeof(struct pci_xhci_portregs)); if (ndevices > 0) { for (i = 1; i <= XHCI_MAX_DEVS; i++) { pci_xhci_init_port(sc, i); } } else { WPRINTF(("pci_xhci no USB devices configured")); } return (0); bad: for (i = 1; i <= XHCI_MAX_DEVS; i++) { free(XHCI_DEVINST_PTR(sc, i)); } free(sc->devices); free(sc->slots); return (-1); } static int pci_xhci_init(struct pci_devinst *pi, nvlist_t *nvl) { struct pci_xhci_softc *sc; int error; #ifndef __FreeBSD__ if (get_config_bool_default("xhci.debug", false)) xhci_debug = 1; #endif if (xhci_in_use) { WPRINTF(("pci_xhci controller already defined")); return (-1); } xhci_in_use = 1; sc = calloc(1, sizeof(struct pci_xhci_softc)); pi->pi_arg = sc; sc->xsc_pi = pi; sc->usb2_port_start = (XHCI_MAX_DEVS/2) + 1; sc->usb3_port_start = 1; /* discover devices */ error = pci_xhci_parse_devices(sc, nvl); if (error < 0) goto done; else error = 0; sc->caplength = XHCI_SET_CAPLEN(XHCI_CAPLEN) | XHCI_SET_HCIVERSION(0x0100); sc->hcsparams1 = XHCI_SET_HCSP1_MAXPORTS(XHCI_MAX_DEVS) | XHCI_SET_HCSP1_MAXINTR(1) | /* interrupters */ XHCI_SET_HCSP1_MAXSLOTS(XHCI_MAX_SLOTS); sc->hcsparams2 = XHCI_SET_HCSP2_ERSTMAX(XHCI_ERST_MAX) | XHCI_SET_HCSP2_IST(0x04); sc->hcsparams3 = 0; /* no latency */ sc->hccparams1 = XHCI_SET_HCCP1_AC64(1) | /* 64-bit addrs */ XHCI_SET_HCCP1_NSS(1) | /* no 2nd-streams */ XHCI_SET_HCCP1_SPC(1) | /* short packet */ XHCI_SET_HCCP1_MAXPSA(XHCI_STREAMS_MAX); sc->hccparams2 = XHCI_SET_HCCP2_LEC(1) | XHCI_SET_HCCP2_U3C(1); sc->dboff = XHCI_SET_DOORBELL(XHCI_CAPLEN + XHCI_PORTREGS_START + XHCI_MAX_DEVS * sizeof(struct pci_xhci_portregs)); /* dboff must be 32-bit aligned */ if (sc->dboff & 0x3) sc->dboff = (sc->dboff + 0x3) & ~0x3; /* rtsoff must be 32-bytes aligned */ sc->rtsoff = XHCI_SET_RTSOFFSET(sc->dboff + (XHCI_MAX_SLOTS+1) * 32); if (sc->rtsoff & 0x1F) sc->rtsoff = (sc->rtsoff + 0x1F) & ~0x1F; DPRINTF(("pci_xhci dboff: 0x%x, rtsoff: 0x%x", sc->dboff, sc->rtsoff)); sc->opregs.usbsts = XHCI_STS_HCH; sc->opregs.pgsz = XHCI_PAGESIZE_4K; pci_xhci_reset(sc); sc->regsend = sc->rtsoff + 0x20 + 32; /* only 1 intrpter */ /* * Set extended capabilities pointer to be after regsend; * value of xecp field is 32-bit offset. */ sc->hccparams1 |= XHCI_SET_HCCP1_XECP(sc->regsend/4); pci_set_cfgdata16(pi, PCIR_DEVICE, 0x1E31); pci_set_cfgdata16(pi, PCIR_VENDOR, 0x8086); pci_set_cfgdata8(pi, PCIR_CLASS, PCIC_SERIALBUS); pci_set_cfgdata8(pi, PCIR_SUBCLASS, PCIS_SERIALBUS_USB); pci_set_cfgdata8(pi, PCIR_PROGIF,PCIP_SERIALBUS_USB_XHCI); pci_set_cfgdata8(pi, PCI_USBREV, PCI_USB_REV_3_0); pci_emul_add_msicap(pi, 1); /* regsend + xecp registers */ pci_emul_alloc_bar(pi, 0, PCIBAR_MEM32, sc->regsend + 4*32); DPRINTF(("pci_xhci pci_emu_alloc: %d", sc->regsend + 4*32)); pci_lintr_request(pi); pthread_mutex_init(&sc->mtx, NULL); done: if (error) { free(sc); } return (error); } static const struct pci_devemu pci_de_xhci = { .pe_emu = "xhci", .pe_init = pci_xhci_init, .pe_legacy_config = pci_xhci_legacy_config, .pe_barwrite = pci_xhci_write, .pe_barread = pci_xhci_read }; PCI_EMUL_SET(pci_de_xhci); /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2014 Leon Dang * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef _PCI_XHCI_H_ #define _PCI_XHCI_H_ #define PCI_USBREV 0x60 /* USB protocol revision */ enum { /* dsc_slotstate */ XHCI_ST_DISABLED, XHCI_ST_ENABLED, XHCI_ST_DEFAULT, XHCI_ST_ADDRESSED, XHCI_ST_CONFIGURED, XHCI_ST_MAX }; enum { XHCI_ST_SLCTX_DISABLED, XHCI_ST_SLCTX_DEFAULT, XHCI_ST_SLCTX_ADDRESSED, XHCI_ST_SLCTX_CONFIGURED }; enum { XHCI_ST_EPCTX_DISABLED, XHCI_ST_EPCTX_RUNNING, XHCI_ST_EPCTX_HALTED, XHCI_ST_EPCTX_STOPPED, XHCI_ST_EPCTX_ERROR }; #define XHCI_MAX_DEVICES MIN(USB_MAX_DEVICES, 128) #define XHCI_MAX_ENDPOINTS 32 /* hardcoded - do not change */ #define XHCI_MAX_SCRATCHPADS 32 #define XHCI_MAX_EVENTS (16 * 13) #define XHCI_MAX_COMMANDS (16 * 1) #define XHCI_MAX_RSEG 1 #define XHCI_MAX_TRANSFERS 4 #if USB_MAX_EP_STREAMS == 8 #define XHCI_MAX_STREAMS 8 #define XHCI_MAX_STREAMS_LOG 3 #elif USB_MAX_EP_STREAMS == 1 #define XHCI_MAX_STREAMS 1 #define XHCI_MAX_STREAMS_LOG 0 #else #error "The USB_MAX_EP_STREAMS value is not supported." #endif #define XHCI_DEV_CTX_ADDR_ALIGN 64 /* bytes */ #define XHCI_DEV_CTX_ALIGN 64 /* bytes */ #define XHCI_INPUT_CTX_ALIGN 64 /* bytes */ #define XHCI_SLOT_CTX_ALIGN 32 /* bytes */ #define XHCI_ENDP_CTX_ALIGN 32 /* bytes */ #define XHCI_STREAM_CTX_ALIGN 16 /* bytes */ #define XHCI_TRANS_RING_SEG_ALIGN 16 /* bytes */ #define XHCI_CMD_RING_SEG_ALIGN 64 /* bytes */ #define XHCI_EVENT_RING_SEG_ALIGN 64 /* bytes */ #define XHCI_SCRATCH_BUF_ARRAY_ALIGN 64 /* bytes */ #define XHCI_SCRATCH_BUFFER_ALIGN USB_PAGE_SIZE #define XHCI_TRB_ALIGN 16 /* bytes */ #define XHCI_TD_ALIGN 64 /* bytes */ #define XHCI_PAGE_SIZE 4096 /* bytes */ struct xhci_slot_ctx { uint32_t dwSctx0; #define XHCI_SCTX_0_ROUTE_SET(x) ((x) & 0xFFFFF) #define XHCI_SCTX_0_ROUTE_GET(x) ((x) & 0xFFFFF) #define XHCI_SCTX_0_SPEED_SET(x) (((x) & 0xF) << 20) #define XHCI_SCTX_0_SPEED_GET(x) (((x) >> 20) & 0xF) #define XHCI_SCTX_0_MTT_SET(x) (((x) & 0x1) << 25) #define XHCI_SCTX_0_MTT_GET(x) (((x) >> 25) & 0x1) #define XHCI_SCTX_0_HUB_SET(x) (((x) & 0x1) << 26) #define XHCI_SCTX_0_HUB_GET(x) (((x) >> 26) & 0x1) #define XHCI_SCTX_0_CTX_NUM_SET(x) (((x) & 0x1F) << 27) #define XHCI_SCTX_0_CTX_NUM_GET(x) (((x) >> 27) & 0x1F) uint32_t dwSctx1; #define XHCI_SCTX_1_MAX_EL_SET(x) ((x) & 0xFFFF) #define XHCI_SCTX_1_MAX_EL_GET(x) ((x) & 0xFFFF) #define XHCI_SCTX_1_RH_PORT_SET(x) (((x) & 0xFF) << 16) #define XHCI_SCTX_1_RH_PORT_GET(x) (((x) >> 16) & 0xFF) #define XHCI_SCTX_1_NUM_PORTS_SET(x) (((x) & 0xFF) << 24) #define XHCI_SCTX_1_NUM_PORTS_GET(x) (((x) >> 24) & 0xFF) uint32_t dwSctx2; #define XHCI_SCTX_2_TT_HUB_SID_SET(x) ((x) & 0xFF) #define XHCI_SCTX_2_TT_HUB_SID_GET(x) ((x) & 0xFF) #define XHCI_SCTX_2_TT_PORT_NUM_SET(x) (((x) & 0xFF) << 8) #define XHCI_SCTX_2_TT_PORT_NUM_GET(x) (((x) >> 8) & 0xFF) #define XHCI_SCTX_2_TT_THINK_TIME_SET(x) (((x) & 0x3) << 16) #define XHCI_SCTX_2_TT_THINK_TIME_GET(x) (((x) >> 16) & 0x3) #define XHCI_SCTX_2_IRQ_TARGET_SET(x) (((x) & 0x3FF) << 22) #define XHCI_SCTX_2_IRQ_TARGET_GET(x) (((x) >> 22) & 0x3FF) uint32_t dwSctx3; #define XHCI_SCTX_3_DEV_ADDR_SET(x) ((x) & 0xFF) #define XHCI_SCTX_3_DEV_ADDR_GET(x) ((x) & 0xFF) #define XHCI_SCTX_3_SLOT_STATE_SET(x) (((x) & 0x1F) << 27) #define XHCI_SCTX_3_SLOT_STATE_GET(x) (((x) >> 27) & 0x1F) uint32_t dwSctx4; uint32_t dwSctx5; uint32_t dwSctx6; uint32_t dwSctx7; }; struct xhci_endp_ctx { uint32_t dwEpCtx0; #define XHCI_EPCTX_0_EPSTATE_SET(x) ((x) & 0x7) #define XHCI_EPCTX_0_EPSTATE_GET(x) ((x) & 0x7) #define XHCI_EPCTX_0_MULT_SET(x) (((x) & 0x3) << 8) #define XHCI_EPCTX_0_MULT_GET(x) (((x) >> 8) & 0x3) #define XHCI_EPCTX_0_MAXP_STREAMS_SET(x) (((x) & 0x1F) << 10) #define XHCI_EPCTX_0_MAXP_STREAMS_GET(x) (((x) >> 10) & 0x1F) #define XHCI_EPCTX_0_LSA_SET(x) (((x) & 0x1) << 15) #define XHCI_EPCTX_0_LSA_GET(x) (((x) >> 15) & 0x1) #define XHCI_EPCTX_0_IVAL_SET(x) (((x) & 0xFF) << 16) #define XHCI_EPCTX_0_IVAL_GET(x) (((x) >> 16) & 0xFF) uint32_t dwEpCtx1; #define XHCI_EPCTX_1_CERR_SET(x) (((x) & 0x3) << 1) #define XHCI_EPCTX_1_CERR_GET(x) (((x) >> 1) & 0x3) #define XHCI_EPCTX_1_EPTYPE_SET(x) (((x) & 0x7) << 3) #define XHCI_EPCTX_1_EPTYPE_GET(x) (((x) >> 3) & 0x7) #define XHCI_EPCTX_1_HID_SET(x) (((x) & 0x1) << 7) #define XHCI_EPCTX_1_HID_GET(x) (((x) >> 7) & 0x1) #define XHCI_EPCTX_1_MAXB_SET(x) (((x) & 0xFF) << 8) #define XHCI_EPCTX_1_MAXB_GET(x) (((x) >> 8) & 0xFF) #define XHCI_EPCTX_1_MAXP_SIZE_SET(x) (((x) & 0xFFFF) << 16) #define XHCI_EPCTX_1_MAXP_SIZE_GET(x) (((x) >> 16) & 0xFFFF) uint64_t qwEpCtx2; #define XHCI_EPCTX_2_DCS_SET(x) ((x) & 0x1) #define XHCI_EPCTX_2_DCS_GET(x) ((x) & 0x1) #define XHCI_EPCTX_2_TR_DQ_PTR_MASK 0xFFFFFFFFFFFFFFF0U uint32_t dwEpCtx4; #define XHCI_EPCTX_4_AVG_TRB_LEN_SET(x) ((x) & 0xFFFF) #define XHCI_EPCTX_4_AVG_TRB_LEN_GET(x) ((x) & 0xFFFF) #define XHCI_EPCTX_4_MAX_ESIT_PAYLOAD_SET(x) (((x) & 0xFFFF) << 16) #define XHCI_EPCTX_4_MAX_ESIT_PAYLOAD_GET(x) (((x) >> 16) & 0xFFFF) uint32_t dwEpCtx5; uint32_t dwEpCtx6; uint32_t dwEpCtx7; }; struct xhci_input_ctx { #define XHCI_INCTX_NON_CTRL_MASK 0xFFFFFFFCU uint32_t dwInCtx0; #define XHCI_INCTX_0_DROP_MASK(n) (1U << (n)) uint32_t dwInCtx1; #define XHCI_INCTX_1_ADD_MASK(n) (1U << (n)) uint32_t dwInCtx2; uint32_t dwInCtx3; uint32_t dwInCtx4; uint32_t dwInCtx5; uint32_t dwInCtx6; uint32_t dwInCtx7; }; struct xhci_input_dev_ctx { struct xhci_input_ctx ctx_input; union { struct xhci_slot_ctx u_slot; struct xhci_endp_ctx u_ep[XHCI_MAX_ENDPOINTS]; } ctx_dev_slep; }; struct xhci_dev_ctx { union { struct xhci_slot_ctx u_slot; struct xhci_endp_ctx u_ep[XHCI_MAX_ENDPOINTS]; } ctx_dev_slep; } __aligned(XHCI_DEV_CTX_ALIGN); #define ctx_slot ctx_dev_slep.u_slot #define ctx_ep ctx_dev_slep.u_ep struct xhci_stream_ctx { uint64_t qwSctx0; #define XHCI_SCTX_0_DCS_GET(x) ((x) & 0x1) #define XHCI_SCTX_0_DCS_SET(x) ((x) & 0x1) #define XHCI_SCTX_0_SCT_SET(x) (((x) & 0x7) << 1) #define XHCI_SCTX_0_SCT_GET(x) (((x) >> 1) & 0x7) #define XHCI_SCTX_0_SCT_SEC_TR_RING 0x0 #define XHCI_SCTX_0_SCT_PRIM_TR_RING 0x1 #define XHCI_SCTX_0_SCT_PRIM_SSA_8 0x2 #define XHCI_SCTX_0_SCT_PRIM_SSA_16 0x3 #define XHCI_SCTX_0_SCT_PRIM_SSA_32 0x4 #define XHCI_SCTX_0_SCT_PRIM_SSA_64 0x5 #define XHCI_SCTX_0_SCT_PRIM_SSA_128 0x6 #define XHCI_SCTX_0_SCT_PRIM_SSA_256 0x7 #define XHCI_SCTX_0_TR_DQ_PTR_MASK 0xFFFFFFFFFFFFFFF0U uint32_t dwSctx2; uint32_t dwSctx3; }; struct xhci_trb { uint64_t qwTrb0; #define XHCI_TRB_0_DIR_IN_MASK (0x80ULL << 0) #define XHCI_TRB_0_WLENGTH_MASK (0xFFFFULL << 48) uint32_t dwTrb2; #define XHCI_TRB_2_ERROR_GET(x) (((x) >> 24) & 0xFF) #define XHCI_TRB_2_ERROR_SET(x) (((x) & 0xFF) << 24) #define XHCI_TRB_2_TDSZ_GET(x) (((x) >> 17) & 0x1F) #define XHCI_TRB_2_TDSZ_SET(x) (((x) & 0x1F) << 17) #define XHCI_TRB_2_REM_GET(x) ((x) & 0xFFFFFF) #define XHCI_TRB_2_REM_SET(x) ((x) & 0xFFFFFF) #define XHCI_TRB_2_BYTES_GET(x) ((x) & 0x1FFFF) #define XHCI_TRB_2_BYTES_SET(x) ((x) & 0x1FFFF) #define XHCI_TRB_2_IRQ_GET(x) (((x) >> 22) & 0x3FF) #define XHCI_TRB_2_IRQ_SET(x) (((x) & 0x3FF) << 22) #define XHCI_TRB_2_STREAM_GET(x) (((x) >> 16) & 0xFFFF) #define XHCI_TRB_2_STREAM_SET(x) (((x) & 0xFFFF) << 16) uint32_t dwTrb3; #define XHCI_TRB_3_TYPE_GET(x) (((x) >> 10) & 0x3F) #define XHCI_TRB_3_TYPE_SET(x) (((x) & 0x3F) << 10) #define XHCI_TRB_3_CYCLE_BIT (1U << 0) #define XHCI_TRB_3_TC_BIT (1U << 1) /* command ring only */ #define XHCI_TRB_3_ENT_BIT (1U << 1) /* transfer ring only */ #define XHCI_TRB_3_ISP_BIT (1U << 2) #define XHCI_TRB_3_ED_BIT (1U << 2) #define XHCI_TRB_3_NSNOOP_BIT (1U << 3) #define XHCI_TRB_3_CHAIN_BIT (1U << 4) #define XHCI_TRB_3_IOC_BIT (1U << 5) #define XHCI_TRB_3_IDT_BIT (1U << 6) #define XHCI_TRB_3_TBC_GET(x) (((x) >> 7) & 3) #define XHCI_TRB_3_TBC_SET(x) (((x) & 3) << 7) #define XHCI_TRB_3_BEI_BIT (1U << 9) #define XHCI_TRB_3_DCEP_BIT (1U << 9) #define XHCI_TRB_3_PRSV_BIT (1U << 9) #define XHCI_TRB_3_BSR_BIT (1U << 9) #define XHCI_TRB_3_TRT_MASK (3U << 16) #define XHCI_TRB_3_TRT_NONE (0U << 16) #define XHCI_TRB_3_TRT_OUT (2U << 16) #define XHCI_TRB_3_TRT_IN (3U << 16) #define XHCI_TRB_3_DIR_IN (1U << 16) #define XHCI_TRB_3_TLBPC_GET(x) (((x) >> 16) & 0xF) #define XHCI_TRB_3_TLBPC_SET(x) (((x) & 0xF) << 16) #define XHCI_TRB_3_EP_GET(x) (((x) >> 16) & 0x1F) #define XHCI_TRB_3_EP_SET(x) (((x) & 0x1F) << 16) #define XHCI_TRB_3_FRID_GET(x) (((x) >> 20) & 0x7FF) #define XHCI_TRB_3_FRID_SET(x) (((x) & 0x7FF) << 20) #define XHCI_TRB_3_ISO_SIA_BIT (1U << 31) #define XHCI_TRB_3_SUSP_EP_BIT (1U << 23) #define XHCI_TRB_3_SLOT_GET(x) (((x) >> 24) & 0xFF) #define XHCI_TRB_3_SLOT_SET(x) (((x) & 0xFF) << 24) /* Commands */ #define XHCI_TRB_TYPE_RESERVED 0x00 #define XHCI_TRB_TYPE_NORMAL 0x01 #define XHCI_TRB_TYPE_SETUP_STAGE 0x02 #define XHCI_TRB_TYPE_DATA_STAGE 0x03 #define XHCI_TRB_TYPE_STATUS_STAGE 0x04 #define XHCI_TRB_TYPE_ISOCH 0x05 #define XHCI_TRB_TYPE_LINK 0x06 #define XHCI_TRB_TYPE_EVENT_DATA 0x07 #define XHCI_TRB_TYPE_NOOP 0x08 #define XHCI_TRB_TYPE_ENABLE_SLOT 0x09 #define XHCI_TRB_TYPE_DISABLE_SLOT 0x0A #define XHCI_TRB_TYPE_ADDRESS_DEVICE 0x0B #define XHCI_TRB_TYPE_CONFIGURE_EP 0x0C #define XHCI_TRB_TYPE_EVALUATE_CTX 0x0D #define XHCI_TRB_TYPE_RESET_EP 0x0E #define XHCI_TRB_TYPE_STOP_EP 0x0F #define XHCI_TRB_TYPE_SET_TR_DEQUEUE 0x10 #define XHCI_TRB_TYPE_RESET_DEVICE 0x11 #define XHCI_TRB_TYPE_FORCE_EVENT 0x12 #define XHCI_TRB_TYPE_NEGOTIATE_BW 0x13 #define XHCI_TRB_TYPE_SET_LATENCY_TOL 0x14 #define XHCI_TRB_TYPE_GET_PORT_BW 0x15 #define XHCI_TRB_TYPE_FORCE_HEADER 0x16 #define XHCI_TRB_TYPE_NOOP_CMD 0x17 /* Events */ #define XHCI_TRB_EVENT_TRANSFER 0x20 #define XHCI_TRB_EVENT_CMD_COMPLETE 0x21 #define XHCI_TRB_EVENT_PORT_STS_CHANGE 0x22 #define XHCI_TRB_EVENT_BW_REQUEST 0x23 #define XHCI_TRB_EVENT_DOORBELL 0x24 #define XHCI_TRB_EVENT_HOST_CTRL 0x25 #define XHCI_TRB_EVENT_DEVICE_NOTIFY 0x26 #define XHCI_TRB_EVENT_MFINDEX_WRAP 0x27 /* Error codes */ #define XHCI_TRB_ERROR_INVALID 0x00 #define XHCI_TRB_ERROR_SUCCESS 0x01 #define XHCI_TRB_ERROR_DATA_BUF 0x02 #define XHCI_TRB_ERROR_BABBLE 0x03 #define XHCI_TRB_ERROR_XACT 0x04 #define XHCI_TRB_ERROR_TRB 0x05 #define XHCI_TRB_ERROR_STALL 0x06 #define XHCI_TRB_ERROR_RESOURCE 0x07 #define XHCI_TRB_ERROR_BANDWIDTH 0x08 #define XHCI_TRB_ERROR_NO_SLOTS 0x09 #define XHCI_TRB_ERROR_STREAM_TYPE 0x0A #define XHCI_TRB_ERROR_SLOT_NOT_ON 0x0B #define XHCI_TRB_ERROR_ENDP_NOT_ON 0x0C #define XHCI_TRB_ERROR_SHORT_PKT 0x0D #define XHCI_TRB_ERROR_RING_UNDERRUN 0x0E #define XHCI_TRB_ERROR_RING_OVERRUN 0x0F #define XHCI_TRB_ERROR_VF_RING_FULL 0x10 #define XHCI_TRB_ERROR_PARAMETER 0x11 #define XHCI_TRB_ERROR_BW_OVERRUN 0x12 #define XHCI_TRB_ERROR_CONTEXT_STATE 0x13 #define XHCI_TRB_ERROR_NO_PING_RESP 0x14 #define XHCI_TRB_ERROR_EV_RING_FULL 0x15 #define XHCI_TRB_ERROR_INCOMPAT_DEV 0x16 #define XHCI_TRB_ERROR_MISSED_SERVICE 0x17 #define XHCI_TRB_ERROR_CMD_RING_STOP 0x18 #define XHCI_TRB_ERROR_CMD_ABORTED 0x19 #define XHCI_TRB_ERROR_STOPPED 0x1A #define XHCI_TRB_ERROR_LENGTH 0x1B #define XHCI_TRB_ERROR_BAD_MELAT 0x1D #define XHCI_TRB_ERROR_ISOC_OVERRUN 0x1F #define XHCI_TRB_ERROR_EVENT_LOST 0x20 #define XHCI_TRB_ERROR_UNDEFINED 0x21 #define XHCI_TRB_ERROR_INVALID_SID 0x22 #define XHCI_TRB_ERROR_SEC_BW 0x23 #define XHCI_TRB_ERROR_SPLIT_XACT 0x24 } __aligned(4); struct xhci_dev_endpoint_trbs { struct xhci_trb trb[(XHCI_MAX_STREAMS * XHCI_MAX_TRANSFERS) + XHCI_MAX_STREAMS]; }; struct xhci_event_ring_seg { uint64_t qwEvrsTablePtr; uint32_t dwEvrsTableSize; uint32_t dwEvrsReserved; }; #endif /* _PCI_XHCI_H_ */ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2020 Adam Fenn * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * Emulation of selected legacy test/debug interfaces expected by KVM-unit-tests */ #include #include #include #include #include #include #include #include #include #include "debug.h" #include "inout.h" #include "mem.h" #include "pctestdev.h" #define DEBUGEXIT_BASE 0xf4 #define DEBUGEXIT_LEN 4 #define DEBUGEXIT_NAME "isa-debug-exit" #define IOMEM_BASE 0xff000000 #define IOMEM_LEN 0x10000 #define IOMEM_NAME "pc-testdev-iomem" #define IOPORT_BASE 0xe0 #define IOPORT_LEN 4 #define IOPORT_NAME "pc-testdev-ioport" #define IRQ_BASE 0x2000 #define IRQ_IOAPIC_PINCOUNT_MIN 24 #define IRQ_IOAPIC_PINCOUNT_MAX 32 #define IRQ_NAME "pc-testdev-irq-line" #define PCTESTDEV_NAME "pc-testdev" static bool pctestdev_inited; static uint8_t pctestdev_iomem_buf[IOMEM_LEN]; static uint32_t pctestdev_ioport_data; static int pctestdev_debugexit_io(struct vmctx *ctx, int in, int port, int bytes, uint32_t *eax, void *arg); static int pctestdev_iomem_io(struct vcpu *vcpu, int dir, uint64_t addr, int size, uint64_t *val, void *arg1, long arg2); static int pctestdev_ioport_io(struct vmctx *ctx, int in, int port, int bytes, uint32_t *eax, void *arg); static int pctestdev_irq_io(struct vmctx *ctx, int in, int port, int bytes, uint32_t *eax, void *arg); const char * pctestdev_getname(void) { return (PCTESTDEV_NAME); } int pctestdev_init(struct vmctx *ctx) { struct mem_range iomem; struct inout_port debugexit, ioport, irq; int err, pincount; if (pctestdev_inited) { EPRINTLN("Only one pc-testdev device is allowed."); return (-1); } err = vm_ioapic_pincount(ctx, &pincount); if (err != 0) { EPRINTLN("pc-testdev: Failed to obtain IOAPIC pin count."); return (-1); } if (pincount < IRQ_IOAPIC_PINCOUNT_MIN || pincount > IRQ_IOAPIC_PINCOUNT_MAX) { EPRINTLN("pc-testdev: Unsupported IOAPIC pin count: %d.", pincount); return (-1); } debugexit.name = DEBUGEXIT_NAME; debugexit.port = DEBUGEXIT_BASE; debugexit.size = DEBUGEXIT_LEN; debugexit.flags = IOPORT_F_INOUT; debugexit.handler = pctestdev_debugexit_io; debugexit.arg = NULL; iomem.name = IOMEM_NAME; iomem.flags = MEM_F_RW | MEM_F_IMMUTABLE; iomem.handler = pctestdev_iomem_io; iomem.arg1 = NULL; iomem.arg2 = 0; iomem.base = IOMEM_BASE; iomem.size = IOMEM_LEN; ioport.name = IOPORT_NAME; ioport.port = IOPORT_BASE; ioport.size = IOPORT_LEN; ioport.flags = IOPORT_F_INOUT; ioport.handler = pctestdev_ioport_io; ioport.arg = NULL; irq.name = IRQ_NAME; irq.port = IRQ_BASE; irq.size = pincount; irq.flags = IOPORT_F_INOUT; irq.handler = pctestdev_irq_io; irq.arg = NULL; err = register_inout(&debugexit); if (err != 0) goto fail; err = register_inout(&ioport); if (err != 0) goto fail_after_debugexit_reg; err = register_inout(&irq); if (err != 0) goto fail_after_ioport_reg; err = register_mem(&iomem); if (err != 0) goto fail_after_irq_reg; pctestdev_inited = true; return (0); fail_after_irq_reg: (void)unregister_inout(&irq); fail_after_ioport_reg: (void)unregister_inout(&ioport); fail_after_debugexit_reg: (void)unregister_inout(&debugexit); fail: return (err); } static int pctestdev_debugexit_io(struct vmctx *ctx __unused, int in, int port __unused, int bytes __unused, uint32_t *eax, void *arg __unused) { if (in) *eax = 0; else exit((*eax << 1) | 1); return (0); } static int pctestdev_iomem_io(struct vcpu *vcpu __unused, int dir, uint64_t addr, int size, uint64_t *val, void *arg1 __unused, long arg2 __unused) { uint64_t offset; if (addr + size > IOMEM_BASE + IOMEM_LEN) return (-1); offset = addr - IOMEM_BASE; if (dir == MEM_F_READ) { (void)memcpy(val, pctestdev_iomem_buf + offset, size); } else { assert(dir == MEM_F_WRITE); (void)memcpy(pctestdev_iomem_buf + offset, val, size); } return (0); } static int pctestdev_ioport_io(struct vmctx *ctx __unused, int in, int port, int bytes, uint32_t *eax, void *arg __unused) { uint32_t mask; int lsb; if (port + bytes > IOPORT_BASE + IOPORT_LEN) return (-1); lsb = (port & 0x3) * 8; mask = (-1UL >> (32 - (bytes * 8))) << lsb; if (in) *eax = (pctestdev_ioport_data & mask) >> lsb; else { pctestdev_ioport_data &= ~mask; pctestdev_ioport_data |= *eax << lsb; } return (0); } static int pctestdev_irq_io(struct vmctx *ctx, int in, int port, int bytes, uint32_t *eax, void *arg __unused) { int irq; if (bytes != 1) return (-1); if (in) { *eax = 0; return (0); } else { irq = port - IRQ_BASE; if (irq < 16) { if (*eax) return (vm_isa_assert_irq(ctx, irq, irq)); else return (vm_isa_deassert_irq(ctx, irq, irq)); } else { if (*eax) return (vm_ioapic_assert_irq(ctx, irq)); else return (vm_ioapic_deassert_irq(ctx, irq)); } } } /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2020 Adam Fenn * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * Emulation of selected legacy test/debug interfaces expected by KVM-unit-tests */ #ifndef _PCTESTDEV_H_ #define _PCTESTDEV_H_ struct vmctx; const char *pctestdev_getname(void); int pctestdev_init(struct vmctx *ctx); #endif /* * This file and its contents are supplied under the terms of the * Common Development and Distribution License ("CDDL"), version 1.0. * You may only use this file in accordance with the terms of version * 1.0 of the CDDL. * * A full copy of the text of the CDDL should have accompanied this * source. A copy of the CDDL is also available via the Internet at * http://www.illumos.org/license/CDDL. */ /* * Copyright 2021 OmniOS Community Edition (OmniOSce) Association. */ /* * Two privilege sets are maintained. One which is the minimal privileges * necessary to run bhyve - 'bhyve_priv_min' - and one which is the maximum * privileges required - 'bhyve_priv_max'. This second set starts off identical * to the first, and is added to by modules during initialisation depending on * their requirements and their configuration. * Once all modules are initialised, and before the VM is set running, the * privileges are locked by setting the 'Permitted' privilege set from the * contents of the maximum set, which is now the upper bound, and dropping back * to the minimum set. * * Privileges are process wide, and this framework does not support threaded * access. The ID of the thread that first initialises privileges is recorded, * and all subsequent privilege operations must be done by the same thread. */ #include #include #include #include #include #include #include #include "config.h" #include "debug.h" #include "privileges.h" static pthread_t priv_thread; static priv_set_t *bhyve_priv_init; static priv_set_t *bhyve_priv_min; static priv_set_t *bhyve_priv_max; static bool priv_debug = false; #define DPRINTF(params) \ if (priv_debug) do { PRINTLN params; fflush(stdout); } while (0) static void illumos_priv_printset(const char *tag, priv_set_t *set) { char *s; s = priv_set_to_str(set, ',', PRIV_STR_LIT); if (s == NULL) { warn("priv_set_to_str(%s) failed", tag); return; } DPRINTF((" + %s: %s", tag, s)); free(s); } void illumos_priv_reduce(void) { VERIFY3S(pthread_equal(pthread_self(), priv_thread), !=, 0); DPRINTF((" + Reducing privileges to minimum")); if (setppriv(PRIV_SET, PRIV_EFFECTIVE, bhyve_priv_min) != 0) err(4, "failed to reduce privileges"); } static void illumos_priv_add_set(priv_set_t *set, const char *priv, const char *src) { VERIFY3S(pthread_equal(pthread_self(), priv_thread), !=, 0); DPRINTF((" + Adding privilege %s (%s)", priv, src)); if (!priv_ismember(bhyve_priv_init, priv)) errx(4, "Privilege %s (for %s) not in initial set", priv, src); if (priv_addset(set, priv) != 0) err(4, "Failed to add %s privilege for %s", priv, src); } void illumos_priv_add(const char *priv, const char *src) { illumos_priv_add_set(bhyve_priv_max, priv, src); } void illumos_priv_add_min(const char *priv, const char *src) { illumos_priv_add_set(bhyve_priv_min, priv, src); illumos_priv_add_set(bhyve_priv_max, priv, src); } void illumos_priv_init(void) { priv_debug = get_config_bool_default("privileges.debug", false); DPRINTF((" + Initialising privileges.")); if (priv_debug) (void) setpflags(PRIV_DEBUG, 1); priv_thread = pthread_self(); if ((bhyve_priv_init = priv_allocset()) == NULL) err(4, "failed to allocate memory for initial priv set"); if ((bhyve_priv_min = priv_allocset()) == NULL) err(4, "failed to allocate memory for minimum priv set"); if ((bhyve_priv_max = priv_allocset()) == NULL) err(4, "failed to allocate memory for maximum priv set"); if (getppriv(PRIV_EFFECTIVE, bhyve_priv_init) != 0) err(4, "failed to fetch current privileges"); if (priv_debug) illumos_priv_printset("initial", bhyve_priv_init); /* * file_read is left in the minimum set to allow for lazy library * loading. */ priv_basicset(bhyve_priv_min); VERIFY0(priv_delset(bhyve_priv_min, PRIV_FILE_LINK_ANY)); VERIFY0(priv_delset(bhyve_priv_min, PRIV_FILE_WRITE)); VERIFY0(priv_delset(bhyve_priv_min, PRIV_NET_ACCESS)); VERIFY0(priv_delset(bhyve_priv_min, PRIV_PROC_EXEC)); VERIFY0(priv_delset(bhyve_priv_min, PRIV_PROC_FORK)); VERIFY0(priv_delset(bhyve_priv_min, PRIV_PROC_INFO)); VERIFY0(priv_delset(bhyve_priv_min, PRIV_PROC_SECFLAGS)); VERIFY0(priv_delset(bhyve_priv_min, PRIV_PROC_SESSION)); priv_intersect(bhyve_priv_init, bhyve_priv_min); priv_copyset(bhyve_priv_min, bhyve_priv_max); /* * These are privileges that we know will always be needed. * Other privileges may be added by modules as necessary during * initialisation. */ illumos_priv_add(PRIV_FILE_WRITE, "init"); /* * bhyve can work without proc_clock_highres so don't enforce that * it is present. */ if (priv_ismember(bhyve_priv_init, PRIV_PROC_CLOCK_HIGHRES)) illumos_priv_add_min(PRIV_PROC_CLOCK_HIGHRES, "init"); else warnx("The 'proc_clock_highres' privilege is not available"); } void illumos_priv_lock(void) { VERIFY3S(pthread_equal(pthread_self(), priv_thread), !=, 0); if (bhyve_priv_init == NULL) { const char *msg = "attempted to re-lock privileges"; upanic(msg, strlen(msg)); } priv_intersect(bhyve_priv_init, bhyve_priv_max); if (priv_debug) { DPRINTF((" + Locking privileges")); illumos_priv_printset("min", bhyve_priv_min); illumos_priv_printset("max", bhyve_priv_max); } if (setppriv(PRIV_SET, PRIV_PERMITTED, bhyve_priv_max) != 0) { const char *fail = "failed to reduce permitted privileges"; upanic(fail, strlen(fail)); } if (setppriv(PRIV_SET, PRIV_LIMIT, bhyve_priv_max) != 0) { const char *fail = "failed to reduce limit privileges"; upanic(fail, strlen(fail)); } illumos_priv_reduce(); priv_freeset(bhyve_priv_init); bhyve_priv_init = NULL; } /* * This file and its contents are supplied under the terms of the * Common Development and Distribution License ("CDDL"), version 1.0. * You may only use this file in accordance with the terms of version * 1.0 of the CDDL. * * A full copy of the text of the CDDL should have accompanied this * source. A copy of the CDDL is also available via the Internet at * http://www.illumos.org/license/CDDL. */ /* * Copyright 2021 OmniOS Community Edition (OmniOSce) Association. */ #ifndef _BHYVE_PRIVILEGES_H #define _BHYVE_PRIVILEGES_H #include #include void illumos_priv_init(void); void illumos_priv_lock(void); void illumos_priv_add(const char *, const char *); void illumos_priv_add_min(const char *, const char *); void illumos_priv_reduce(void); #endif /* _BHYVE_PRIVILEGES_H */ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2021 Beckhoff Automation GmbH & Co. KG * Author: Corvin Köhne */ #include #ifdef __FreeBSD__ #include #else #include #endif #include #include #include #include #include #include #include #include #include #include #include "acpi_device.h" #include "bhyverun.h" #include "inout.h" #include "pci_lpc.h" #include "qemu_fwcfg.h" #define QEMU_FWCFG_ACPI_DEVICE_NAME "FWCF" #define QEMU_FWCFG_ACPI_HARDWARE_ID "QEMU0002" #define QEMU_FWCFG_SELECTOR_PORT_NUMBER 0x510 #define QEMU_FWCFG_SELECTOR_PORT_SIZE 1 #define QEMU_FWCFG_SELECTOR_PORT_FLAGS IOPORT_F_INOUT #define QEMU_FWCFG_DATA_PORT_NUMBER 0x511 #define QEMU_FWCFG_DATA_PORT_SIZE 1 #define QEMU_FWCFG_DATA_PORT_FLAGS \ IOPORT_F_INOUT /* QEMU v2.4+ ignores writes */ #define QEMU_FWCFG_ARCHITECTURE_MASK 0x0001 #define QEMU_FWCFG_INDEX_MASK 0x3FFF #define QEMU_FWCFG_SELECT_READ 0 #define QEMU_FWCFG_SELECT_WRITE 1 #define QEMU_FWCFG_ARCHITECTURE_GENERIC 0 #define QEMU_FWCFG_ARCHITECTURE_SPECIFIC 1 #define QEMU_FWCFG_INDEX_SIGNATURE 0x00 #define QEMU_FWCFG_INDEX_ID 0x01 #define QEMU_FWCFG_INDEX_NB_CPUS 0x05 #define QEMU_FWCFG_INDEX_MAX_CPUS 0x0F #define QEMU_FWCFG_INDEX_FILE_DIR 0x19 #define QEMU_FWCFG_FIRST_FILE_INDEX 0x20 #define QEMU_FWCFG_MIN_FILES 10 #pragma pack(1) union qemu_fwcfg_selector { struct { uint16_t index : 14; uint16_t writeable : 1; uint16_t architecture : 1; }; uint16_t bits; }; struct qemu_fwcfg_signature { uint8_t signature[4]; }; struct qemu_fwcfg_id { uint32_t interface : 1; /* always set */ uint32_t DMA : 1; uint32_t reserved : 30; }; struct qemu_fwcfg_file { uint32_t be_size; uint16_t be_selector; uint16_t reserved; uint8_t name[QEMU_FWCFG_MAX_NAME]; }; struct qemu_fwcfg_directory { uint32_t be_count; struct qemu_fwcfg_file files[0]; }; #pragma pack() struct qemu_fwcfg_softc { struct acpi_device *acpi_dev; uint32_t data_offset; union qemu_fwcfg_selector selector; struct qemu_fwcfg_item items[QEMU_FWCFG_MAX_ARCHS] [QEMU_FWCFG_MAX_ENTRIES]; struct qemu_fwcfg_directory *directory; }; static struct qemu_fwcfg_softc fwcfg_sc; struct qemu_fwcfg_user_file { STAILQ_ENTRY(qemu_fwcfg_user_file) chain; uint8_t name[QEMU_FWCFG_MAX_NAME]; uint32_t size; void *data; }; static STAILQ_HEAD(qemu_fwcfg_user_file_list, qemu_fwcfg_user_file) user_files = STAILQ_HEAD_INITIALIZER(user_files); static int qemu_fwcfg_selector_port_handler(struct vmctx *const ctx __unused, const int in, const int port __unused, const int bytes, uint32_t *const eax, void *const arg __unused) { if (bytes != sizeof(uint16_t)) { warnx("%s: invalid size (%d) of IO port access", __func__, bytes); return (-1); } if (in) { *eax = htole16(fwcfg_sc.selector.bits); return (0); } fwcfg_sc.data_offset = 0; fwcfg_sc.selector.bits = le16toh(*eax); return (0); } static int qemu_fwcfg_data_port_handler(struct vmctx *const ctx __unused, const int in, const int port __unused, const int bytes, uint32_t *const eax, void *const arg __unused) { if (bytes != sizeof(uint8_t)) { warnx("%s: invalid size (%d) of IO port access", __func__, bytes); return (-1); } if (!in) { warnx("%s: Writes to qemu fwcfg data port aren't allowed", __func__); return (-1); } /* get fwcfg item */ struct qemu_fwcfg_item *const item = &fwcfg_sc.items[fwcfg_sc.selector.architecture] [fwcfg_sc.selector.index]; if (item->data == NULL) { warnx( "%s: qemu fwcfg item doesn't exist (architecture %s index 0x%x)", __func__, fwcfg_sc.selector.architecture ? "specific" : "generic", fwcfg_sc.selector.index); *eax = 0x00; return (0); } else if (fwcfg_sc.data_offset >= item->size) { warnx( "%s: qemu fwcfg item read exceeds size (architecture %s index 0x%x size 0x%x offset 0x%x)", __func__, fwcfg_sc.selector.architecture ? "specific" : "generic", fwcfg_sc.selector.index, item->size, fwcfg_sc.data_offset); *eax = 0x00; return (0); } /* return item data */ *eax = item->data[fwcfg_sc.data_offset]; fwcfg_sc.data_offset++; return (0); } static int qemu_fwcfg_add_item(const uint16_t architecture, const uint16_t index, const uint32_t size, void *const data) { /* truncate architecture and index to their desired size */ const uint16_t arch = architecture & QEMU_FWCFG_ARCHITECTURE_MASK; const uint16_t idx = index & QEMU_FWCFG_INDEX_MASK; /* get pointer to item specified by selector */ struct qemu_fwcfg_item *const fwcfg_item = &fwcfg_sc.items[arch][idx]; /* check if item is already used */ if (fwcfg_item->data != NULL) { warnx("%s: qemu fwcfg item exists (architecture %s index 0x%x)", __func__, arch ? "specific" : "generic", idx); return (EEXIST); } /* save data of the item */ fwcfg_item->size = size; fwcfg_item->data = data; return (0); } static int qemu_fwcfg_add_item_file_dir(void) { const size_t size = sizeof(struct qemu_fwcfg_directory) + QEMU_FWCFG_MIN_FILES * sizeof(struct qemu_fwcfg_file); #ifdef __FreeBSD__ struct qemu_fwcfg_directory *const fwcfg_directory = calloc(1, size); #else // SMATCH: double check that we're allocating correct size: 4 vs 644 void *ptr = calloc(1, size); struct qemu_fwcfg_directory *const fwcfg_directory = (struct qemu_fwcfg_directory *)ptr; #endif if (fwcfg_directory == NULL) { return (ENOMEM); } fwcfg_sc.directory = fwcfg_directory; return (qemu_fwcfg_add_item(QEMU_FWCFG_ARCHITECTURE_GENERIC, QEMU_FWCFG_INDEX_FILE_DIR, sizeof(struct qemu_fwcfg_directory), (uint8_t *)fwcfg_sc.directory)); } static int qemu_fwcfg_add_item_id(void) { struct qemu_fwcfg_id *const fwcfg_id = calloc(1, sizeof(struct qemu_fwcfg_id)); if (fwcfg_id == NULL) { return (ENOMEM); } fwcfg_id->interface = 1; fwcfg_id->DMA = 0; uint32_t *const le_fwcfg_id_ptr = (uint32_t *)fwcfg_id; *le_fwcfg_id_ptr = htole32(*le_fwcfg_id_ptr); return (qemu_fwcfg_add_item(QEMU_FWCFG_ARCHITECTURE_GENERIC, QEMU_FWCFG_INDEX_ID, sizeof(struct qemu_fwcfg_id), (uint8_t *)fwcfg_id)); } static int qemu_fwcfg_add_item_max_cpus(void) { uint16_t *fwcfg_max_cpus = calloc(1, sizeof(uint16_t)); if (fwcfg_max_cpus == NULL) { return (ENOMEM); } /* * We don't support cpu hotplug yet. For that reason, use guest_ncpus instead * of maxcpus. */ *fwcfg_max_cpus = htole16(guest_ncpus); return (qemu_fwcfg_add_item(QEMU_FWCFG_ARCHITECTURE_GENERIC, QEMU_FWCFG_INDEX_MAX_CPUS, sizeof(uint16_t), fwcfg_max_cpus)); } static int qemu_fwcfg_add_item_nb_cpus(void) { uint16_t *fwcfg_max_cpus = calloc(1, sizeof(uint16_t)); if (fwcfg_max_cpus == NULL) { return (ENOMEM); } *fwcfg_max_cpus = htole16(guest_ncpus); return (qemu_fwcfg_add_item(QEMU_FWCFG_ARCHITECTURE_GENERIC, QEMU_FWCFG_INDEX_NB_CPUS, sizeof(uint16_t), fwcfg_max_cpus)); } static int qemu_fwcfg_add_item_signature(void) { struct qemu_fwcfg_signature *const fwcfg_signature = calloc(1, sizeof(struct qemu_fwcfg_signature)); if (fwcfg_signature == NULL) { return (ENOMEM); } fwcfg_signature->signature[0] = 'Q'; fwcfg_signature->signature[1] = 'E'; fwcfg_signature->signature[2] = 'M'; fwcfg_signature->signature[3] = 'U'; return (qemu_fwcfg_add_item(QEMU_FWCFG_ARCHITECTURE_GENERIC, QEMU_FWCFG_INDEX_SIGNATURE, sizeof(struct qemu_fwcfg_signature), (uint8_t *)fwcfg_signature)); } static int qemu_fwcfg_register_port(const char *const name, const int port, const int size, const int flags, const inout_func_t handler) { struct inout_port iop; bzero(&iop, sizeof(iop)); iop.name = name; iop.port = port; iop.size = size; iop.flags = flags; iop.handler = handler; return (register_inout(&iop)); } int qemu_fwcfg_add_file(const char *name, const uint32_t size, void *const data) { if (strlen(name) >= QEMU_FWCFG_MAX_NAME) return (EINVAL); /* * QEMU specifies count as big endian. * Convert it to host endian to work with it. */ const uint32_t count = be32toh(fwcfg_sc.directory->be_count) + 1; /* add file to items list */ const uint32_t index = QEMU_FWCFG_FIRST_FILE_INDEX + count - 1; const int error = qemu_fwcfg_add_item(QEMU_FWCFG_ARCHITECTURE_GENERIC, index, size, data); if (error != 0) { return (error); } /* * files should be sorted alphabetical, get index for new file */ uint32_t file_index; for (file_index = 0; file_index < count - 1; ++file_index) { #ifdef __FreeBSD__ if (strcmp(name, fwcfg_sc.directory->files[file_index].name) < 0) break; #else if (strcmp((char *)name, (char *)fwcfg_sc.directory->files[file_index].name) < 0) { break; } #endif } if (count > QEMU_FWCFG_MIN_FILES) { /* alloc new file directory */ const uint64_t new_size = sizeof(struct qemu_fwcfg_directory) + count * sizeof(struct qemu_fwcfg_file); struct qemu_fwcfg_directory *const new_directory = calloc(1, new_size); if (new_directory == NULL) { warnx( "%s: Unable to allocate a new qemu fwcfg files directory (count %d)", __func__, count); return (ENOMEM); } /* copy files below file_index to new directory */ memcpy(new_directory->files, fwcfg_sc.directory->files, file_index * sizeof(struct qemu_fwcfg_file)); /* copy files above file_index to directory */ memcpy(&new_directory->files[file_index + 1], &fwcfg_sc.directory->files[file_index], (count - file_index - 1) * sizeof(struct qemu_fwcfg_file)); /* free old directory */ free(fwcfg_sc.directory); /* set directory pointer to new directory */ fwcfg_sc.directory = new_directory; /* adjust directory pointer */ fwcfg_sc.items[0][QEMU_FWCFG_INDEX_FILE_DIR].data = (uint8_t *)fwcfg_sc.directory; } else { /* shift files behind file_index */ for (uint32_t i = QEMU_FWCFG_MIN_FILES - 1; i > file_index; --i) { memcpy(&fwcfg_sc.directory->files[i], &fwcfg_sc.directory->files[i - 1], sizeof(struct qemu_fwcfg_file)); } } /* * QEMU specifies count, size and index as big endian. * Save these values in big endian to simplify guest reads of these * values. */ fwcfg_sc.directory->be_count = htobe32(count); fwcfg_sc.directory->files[file_index].be_size = htobe32(size); fwcfg_sc.directory->files[file_index].be_selector = htobe16(index); #ifdef __FreeBSD__ strcpy(fwcfg_sc.directory->files[file_index].name, name); #else strcpy((char *)fwcfg_sc.directory->files[file_index].name, (char *)name); #endif /* set new size for the fwcfg_file_directory */ fwcfg_sc.items[0][QEMU_FWCFG_INDEX_FILE_DIR].size = sizeof(struct qemu_fwcfg_directory) + count * sizeof(struct qemu_fwcfg_file); return (0); } static int qemu_fwcfg_add_user_files(void) { const struct qemu_fwcfg_user_file *fwcfg_file; int error; STAILQ_FOREACH(fwcfg_file, &user_files, chain) { #ifdef __FreeBSD__ error = qemu_fwcfg_add_file(fwcfg_file->name, fwcfg_file->size, fwcfg_file->data); #else error = qemu_fwcfg_add_file((char *)fwcfg_file->name, fwcfg_file->size, fwcfg_file->data); #endif if (error) return (error); } return (0); } static const struct acpi_device_emul qemu_fwcfg_acpi_device_emul = { .name = QEMU_FWCFG_ACPI_DEVICE_NAME, .hid = QEMU_FWCFG_ACPI_HARDWARE_ID, }; int qemu_fwcfg_init(struct vmctx *const ctx) { int error; /* * Bhyve supports fwctl (bhyve) and fwcfg (qemu) as firmware interfaces. * Both are using the same ports. So, it's not possible to provide both * interfaces at the same time to the guest. Therefore, only create acpi * tables and register io ports for fwcfg, if it's used. */ if (strcmp(lpc_fwcfg(), "qemu") == 0) { error = acpi_device_create(&fwcfg_sc.acpi_dev, &fwcfg_sc, ctx, &qemu_fwcfg_acpi_device_emul); if (error) { warnx("%s: failed to create ACPI device for QEMU FwCfg", __func__); goto done; } error = acpi_device_add_res_fixed_ioport(fwcfg_sc.acpi_dev, QEMU_FWCFG_SELECTOR_PORT_NUMBER, 2); if (error) { warnx("%s: failed to add fixed IO port for QEMU FwCfg", __func__); goto done; } /* add handlers for fwcfg ports */ if ((error = qemu_fwcfg_register_port("qemu_fwcfg_selector", QEMU_FWCFG_SELECTOR_PORT_NUMBER, QEMU_FWCFG_SELECTOR_PORT_SIZE, QEMU_FWCFG_SELECTOR_PORT_FLAGS, qemu_fwcfg_selector_port_handler)) != 0) { warnx( "%s: Unable to register qemu fwcfg selector port 0x%x", __func__, QEMU_FWCFG_SELECTOR_PORT_NUMBER); goto done; } if ((error = qemu_fwcfg_register_port("qemu_fwcfg_data", QEMU_FWCFG_DATA_PORT_NUMBER, QEMU_FWCFG_DATA_PORT_SIZE, QEMU_FWCFG_DATA_PORT_FLAGS, qemu_fwcfg_data_port_handler)) != 0) { warnx( "%s: Unable to register qemu fwcfg data port 0x%x", __func__, QEMU_FWCFG_DATA_PORT_NUMBER); goto done; } } /* add common fwcfg items */ if ((error = qemu_fwcfg_add_item_signature()) != 0) { warnx("%s: Unable to add signature item", __func__); goto done; } if ((error = qemu_fwcfg_add_item_id()) != 0) { warnx("%s: Unable to add id item", __func__); goto done; } if ((error = qemu_fwcfg_add_item_nb_cpus()) != 0) { warnx("%s: Unable to add nb_cpus item", __func__); goto done; } if ((error = qemu_fwcfg_add_item_max_cpus()) != 0) { warnx("%s: Unable to add max_cpus item", __func__); goto done; } if ((error = qemu_fwcfg_add_item_file_dir()) != 0) { warnx("%s: Unable to add file_dir item", __func__); goto done; } /* add user defined fwcfg files */ if ((error = qemu_fwcfg_add_user_files()) != 0) { warnx("%s: Unable to add user files", __func__); goto done; } done: if (error) { acpi_device_destroy(fwcfg_sc.acpi_dev); } return (error); } static void qemu_fwcfg_usage(const char *opt) { warnx("Invalid fw_cfg option \"%s\"", opt); warnx("-f [name=],(string|file)="); } /* * Parses the cmdline argument for user defined fw_cfg items. The cmdline * argument has the format: * "-f [name=],(string|file)=" * * E.g.: "-f opt/com.page/example,string=Hello" */ int qemu_fwcfg_parse_cmdline_arg(const char *opt) { struct qemu_fwcfg_user_file *fwcfg_file; struct stat sb; const char *opt_ptr, *opt_end; ssize_t bytes_read; int fd; fwcfg_file = malloc(sizeof(*fwcfg_file)); if (fwcfg_file == NULL) { warnx("Unable to allocate fw_cfg_user_file"); return (ENOMEM); } /* get pointer to */ opt_ptr = opt; /* If [name=] is specified, skip it */ if (strncmp(opt_ptr, "name=", sizeof("name=") - 1) == 0) { opt_ptr += sizeof("name=") - 1; } /* get the end of */ opt_end = strchr(opt_ptr, ','); if (opt_end == NULL) { qemu_fwcfg_usage(opt); #ifndef __FreeBSD__ free(fwcfg_file); #endif return (EINVAL); } /* check if is too long */ if (opt_end - opt_ptr >= QEMU_FWCFG_MAX_NAME) { warnx("fw_cfg name too long: \"%s\"", opt); #ifndef __FreeBSD__ free(fwcfg_file); #endif return (EINVAL); } /* save */ #ifdef __FreeBSD__ strncpy(fwcfg_file->name, opt_ptr, opt_end - opt_ptr); #else strncpy((char *)fwcfg_file->name, opt_ptr, opt_end - opt_ptr); #endif fwcfg_file->name[opt_end - opt_ptr] = '\0'; /* set opt_ptr and opt_end to */ opt_ptr = opt_end + 1; opt_end = opt_ptr + strlen(opt_ptr); if (strncmp(opt_ptr, "string=", sizeof("string=") - 1) == 0) { opt_ptr += sizeof("string=") - 1; fwcfg_file->data = strdup(opt_ptr); if (fwcfg_file->data == NULL) { warnx("Can't duplicate fw_cfg_user_file string \"%s\"", opt_ptr); return (ENOMEM); } fwcfg_file->size = strlen(opt_ptr) + 1; } else if (strncmp(opt_ptr, "file=", sizeof("file=") - 1) == 0) { opt_ptr += sizeof("file=") - 1; fd = open(opt_ptr, O_RDONLY); if (fd < 0) { warn("Can't open fw_cfg_user_file file \"%s\"", opt_ptr); return (EINVAL); } if (fstat(fd, &sb) < 0) { warn("Unable to get size of file \"%s\"", opt_ptr); close(fd); return (-1); } fwcfg_file->data = malloc(sb.st_size); if (fwcfg_file->data == NULL) { warnx( "Can't allocate fw_cfg_user_file file \"%s\" (size: 0x%16lx)", opt_ptr, sb.st_size); close(fd); return (ENOMEM); } bytes_read = read(fd, fwcfg_file->data, sb.st_size); if (bytes_read < 0 || bytes_read != sb.st_size) { warn("Unable to read file \"%s\"", opt_ptr); free(fwcfg_file->data); close(fd); return (-1); } fwcfg_file->size = bytes_read; close(fd); } else { qemu_fwcfg_usage(opt); return (EINVAL); } STAILQ_INSERT_TAIL(&user_files, fwcfg_file, chain); return (0); } /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2021 Beckhoff Automation GmbH & Co. KG * Author: Corvin Köhne */ #ifndef __QEMU_FWCFG_H__ #define __QEMU_FWCFG_H__ #pragma once #include #define QEMU_FWCFG_MAX_ARCHS 0x2 #define QEMU_FWCFG_MAX_ENTRIES 0x4000 #define QEMU_FWCFG_MAX_NAME 56 #define QEMU_FWCFG_FILE_TABLE_LOADER "etc/table-loader" struct qemu_fwcfg_item { uint32_t size; uint8_t *data; }; int qemu_fwcfg_add_file(const char *name, const uint32_t size, void *const data); int qemu_fwcfg_init(struct vmctx *const ctx); int qemu_fwcfg_parse_cmdline_arg(const char *opt); #endif /* !__QEMU_FWCFG_H__ */ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2022 Beckhoff Automation GmbH & Co. KG * Author: Corvin Köhne */ #include #include #include #include #include #include #include #include #include #include #include #include "qemu_fwcfg.h" #include "qemu_loader.h" #ifndef __FreeBSD__ /* * This is better than the number of ifdef blocks that would be otherwise * required throughout this code. Hopefully upstream will clear up the * char* /uint8_t * confusion. */ #define strlen(x) strlen((char *)(x)) #define strncpy(p, q, n) strncpy((char *)(p), (char *)(q), (n)) #endif struct qemu_loader_entry { uint32_t cmd_le; union { struct { uint8_t name[QEMU_FWCFG_MAX_NAME]; uint32_t alignment_le; uint8_t zone; } alloc; struct { uint8_t dest_name[QEMU_FWCFG_MAX_NAME]; uint8_t src_name[QEMU_FWCFG_MAX_NAME]; uint32_t off_le; uint8_t size; } add_pointer; struct { uint8_t name[QEMU_FWCFG_MAX_NAME]; uint32_t off_le; uint32_t start_le; uint32_t len_le; } add_checksum; struct { uint8_t dest_name[QEMU_FWCFG_MAX_NAME]; uint8_t src_name[QEMU_FWCFG_MAX_NAME]; uint32_t dest_off_le; uint32_t src_off_le; uint8_t size; } write_pointer; /* padding */ uint8_t pad[124]; }; } __packed; enum qemu_loader_command { QEMU_LOADER_CMD_ALLOC = 0x1, QEMU_LOADER_CMD_ADD_POINTER = 0x2, QEMU_LOADER_CMD_ADD_CHECKSUM = 0x3, QEMU_LOADER_CMD_WRITE_POINTER = 0x4, }; struct qemu_loader_element { STAILQ_ENTRY(qemu_loader_element) chain; struct qemu_loader_entry entry; }; struct qemu_loader { uint8_t fwcfg_name[QEMU_FWCFG_MAX_NAME]; STAILQ_HEAD(qemu_loader_list, qemu_loader_element) list; }; int qemu_loader_alloc(struct qemu_loader *const loader, const uint8_t *name, const uint32_t alignment, const enum qemu_loader_zone zone) { struct qemu_loader_element *element; if (strlen(name) >= QEMU_FWCFG_MAX_NAME) return (EINVAL); element = calloc(1, sizeof(struct qemu_loader_element)); if (element == NULL) { warnx("%s: failed to allocate command", __func__); return (ENOMEM); } element->entry.cmd_le = htole32(QEMU_LOADER_CMD_ALLOC); strncpy(element->entry.alloc.name, name, QEMU_FWCFG_MAX_NAME); element->entry.alloc.alignment_le = htole32(alignment); element->entry.alloc.zone = zone; /* * The guest always works on copies of the fwcfg item, which where * loaded into guest memory. Loading a fwcfg item is caused by ALLOC. * For that reason, ALLOC should be scheduled in front of any other * commands. */ STAILQ_INSERT_HEAD(&loader->list, element, chain); return (0); } int qemu_loader_add_checksum(struct qemu_loader *const loader, const uint8_t *name, const uint32_t off, const uint32_t start, const uint32_t len) { struct qemu_loader_element *element; if (strlen(name) >= QEMU_FWCFG_MAX_NAME) return (EINVAL); element = calloc(1, sizeof(struct qemu_loader_element)); if (element == NULL) { warnx("%s: failed to allocate command", __func__); return (ENOMEM); } element->entry.cmd_le = htole32(QEMU_LOADER_CMD_ADD_CHECKSUM); strncpy(element->entry.add_checksum.name, name, QEMU_FWCFG_MAX_NAME); element->entry.add_checksum.off_le = htole32(off); element->entry.add_checksum.start_le = htole32(start); element->entry.add_checksum.len_le = htole32(len); STAILQ_INSERT_TAIL(&loader->list, element, chain); return (0); } int qemu_loader_add_pointer(struct qemu_loader *const loader, const uint8_t *dest_name, const uint8_t *src_name, const uint32_t off, const uint8_t size) { struct qemu_loader_element *element; if (strlen(dest_name) >= QEMU_FWCFG_MAX_NAME || strlen(src_name) >= QEMU_FWCFG_MAX_NAME) return (EINVAL); element = calloc(1, sizeof(struct qemu_loader_element)); if (element == NULL) { warnx("%s: failed to allocate command", __func__); return (ENOMEM); } element->entry.cmd_le = htole32(QEMU_LOADER_CMD_ADD_POINTER); strncpy(element->entry.add_pointer.dest_name, dest_name, QEMU_FWCFG_MAX_NAME); strncpy(element->entry.add_pointer.src_name, src_name, QEMU_FWCFG_MAX_NAME); element->entry.add_pointer.off_le = htole32(off); element->entry.add_pointer.size = size; STAILQ_INSERT_TAIL(&loader->list, element, chain); return (0); } int qemu_loader_create(struct qemu_loader **const new_loader, const uint8_t *fwcfg_name) { struct qemu_loader *loader; if (new_loader == NULL || strlen(fwcfg_name) >= QEMU_FWCFG_MAX_NAME) { return (EINVAL); } loader = calloc(1, sizeof(struct qemu_loader)); if (loader == NULL) { warnx("%s: failed to allocate loader", __func__); return (ENOMEM); } strncpy(loader->fwcfg_name, fwcfg_name, QEMU_FWCFG_MAX_NAME); STAILQ_INIT(&loader->list); *new_loader = loader; return (0); } static const uint8_t * qemu_loader_get_zone_name(const enum qemu_loader_zone zone) { #ifdef __FreeBSD__ switch (zone) { case QEMU_LOADER_ALLOC_HIGH: return ("HIGH"); case QEMU_LOADER_ALLOC_FSEG: return ("FSEG"); default: return ("Unknown"); } #else switch (zone) { case QEMU_LOADER_ALLOC_HIGH: return ((uint8_t *)"HIGH"); case QEMU_LOADER_ALLOC_FSEG: return ((uint8_t *)"FSEG"); default: return ((uint8_t *)"Unknown"); } #endif } static void __unused qemu_loader_dump_entry(const struct qemu_loader_entry *const entry) { switch (le32toh(entry->cmd_le)) { case QEMU_LOADER_CMD_ALLOC: printf("CMD_ALLOC\n\r"); printf(" name : %s\n\r", entry->alloc.name); printf(" alignment: %8x\n\r", le32toh(entry->alloc.alignment_le)); printf(" zone : %s\n\r", qemu_loader_get_zone_name(entry->alloc.zone)); break; case QEMU_LOADER_CMD_ADD_POINTER: printf("CMD_ADD_POINTER\n\r"); printf(" dest_name: %s\n\r", entry->add_pointer.dest_name); printf(" src_name : %s\n\r", entry->add_pointer.src_name); printf(" off : %8x\n\r", le32toh(entry->add_pointer.off_le)); printf(" size : %8x\n\r", entry->add_pointer.size); break; case QEMU_LOADER_CMD_ADD_CHECKSUM: printf("CMD_ADD_CHECKSUM\n\r"); printf(" name : %s\n\r", entry->add_checksum.name); printf(" off : %8x\n\r", le32toh(entry->add_checksum.off_le)); printf(" start : %8x\n\r", le32toh(entry->add_checksum.start_le)); printf(" length : %8x\n\r", le32toh(entry->add_checksum.len_le)); break; case QEMU_LOADER_CMD_WRITE_POINTER: printf("CMD_WRITE_POINTER\n\r"); printf(" dest_name: %s\n\r", entry->write_pointer.dest_name); printf(" src_name : %s\n\r", entry->write_pointer.src_name); printf(" dest_off : %8x\n\r", le32toh(entry->write_pointer.dest_off_le)); printf(" src_off : %8x\n\r", le32toh(entry->write_pointer.src_off_le)); printf(" size : %8x\n\r", entry->write_pointer.size); break; default: printf("UNKNOWN\n\r"); break; } } int qemu_loader_finish(struct qemu_loader *const loader) { struct qemu_loader_element *element; struct qemu_loader_entry *data; size_t len = 0; STAILQ_FOREACH(element, &loader->list, chain) { len += sizeof(struct qemu_loader_entry); } if (len == 0) { warnx("%s: bios loader empty", __func__); return (EFAULT); } data = calloc(1, len); if (data == NULL) { warnx("%s: failed to allocate fwcfg data", __func__); return (ENOMEM); } int i = 0; STAILQ_FOREACH(element, &loader->list, chain) { memcpy(&data[i], &element->entry, sizeof(struct qemu_loader_entry)); ++i; } #ifdef __FreeBSD__ return (qemu_fwcfg_add_file(loader->fwcfg_name, len, data)); #else return (qemu_fwcfg_add_file((const char *)loader->fwcfg_name, len, data)); #endif } /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2022 Beckhoff Automation GmbH & Co. KG * Author: Corvin Köhne */ #pragma once #include "qemu_fwcfg.h" struct qemu_loader; /* * Some guest bios like seabios assume the RSDP to be located in the FSEG. Bhyve * only supports OVMF which has no such requirement. */ enum qemu_loader_zone { QEMU_LOADER_ALLOC_HIGH = 1, QEMU_LOADER_ALLOC_FSEG, /* 0x0F000000 - 0x100000 */ }; /** * Loads a fwcfg item into guest memory. This command has to be issued before * any subsequent command can be used. * * @param loader Qemu loader instance the command should be added to. * @param name Name of the fwcfg item which should be allocated. * @param alignment Alignment required by the data. * @param zone Memory zone in which it should be loaded. */ int qemu_loader_alloc(struct qemu_loader *loader, const uint8_t *name, uint32_t alignment, enum qemu_loader_zone zone); /** * Calculates a checksum for @p name and writes it to @p name + @p off . The * checksum calculation ranges from @p start to @p start + @p len. The checksum * field is always one byte large and all bytes in the specified range, * including the checksum, have to sum up to 0. * * @param loader Qemu loader instance the command should be added to. * @param name Name of the fwcfg item which should be patched. * @param off Offset into @p name . * @param start Start offset of checksum calculation. * @param len Length of the checksum calculation. */ int qemu_loader_add_checksum(struct qemu_loader *loader, const uint8_t *name, uint32_t off, uint32_t start, uint32_t len); /** * Adds the address of @p src_name to the value at @p dest_name + @p off . The * size of the pointer is determined by @p dest_size and should be 1, 2, 4 or 8. * * @param loader Qemu loader instance the command should be added to. * @param dest_name Name of the fwcfg item which should be patched. * @param src_name Name of the fwcfg item which address should be written to * @p dest_name + @p off. * @param off Offset into @p dest_name . * @param size Size of the pointer (1, 2, 4 or 8). */ int qemu_loader_add_pointer(struct qemu_loader *loader, const uint8_t *dest_name, const uint8_t *src_name, uint32_t off, uint8_t size); /** * Creates a qemu loader instance. * * @param new_loader Returns the newly created qemu loader instance. * @param fwcfg_name Name of the FwCfg item which represents the qemu loader */ int qemu_loader_create(struct qemu_loader **new_loader, const uint8_t *fwcfg_name); /** * Signals that all commands are written to the qemu loader. This function * creates a proper FwCfg item and registers it. * * @param loader Qemu loader instance which should be finished. */ int qemu_loader_finish(struct qemu_loader *loader); /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2015 Tycho Nightingale * Copyright (c) 2015 Leon Dang * Copyright 2020 Joyent, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * This file and its contents are supplied under the terms of the * Common Development and Distribution License ("CDDL"), version 1.0. * You may only use this file in accordance with the terms of version * 1.0 of the CDDL. * * A full copy of the text of the CDDL should have accompanied this * source. A copy of the CDDL is also available via the Internet at * http://www.illumos.org/license/CDDL. * * Copyright 2022 OmniOS Community Edition (OmniOSce) Association. */ /* * References to the RFB protocol specification refer to: * - [1] https://github.com/rfbproto/rfbproto/blob/master/rfbproto.rst */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifndef NO_OPENSSL #include #endif #include #include #include #include #include #ifndef WITHOUT_CAPSICUM #include #include #include #endif #include "bhyvegc.h" #include "config.h" #include "debug.h" #include "console.h" #include "rfb.h" #include "rfb_impl.h" #include "sockstream.h" static uint_t rfb_debug = 0; static list_t rfb_list; static id_space_t *rfb_idspace; static bool rfb_sse42; static pthread_once_t rfb_once = PTHREAD_ONCE_INIT; extern int raw_stdio; static void rfb_send_extended_keyevent_update_msg(rfb_client_t *); static void rfb_printf(rfb_client_t *c, rfb_loglevel_t level, const char *fmt, ...) { FILE *fp = stdout; va_list ap; switch (level) { case RFB_LOGDEBUG: if (rfb_debug == 0) return; /* FALLTHROUGH */ case RFB_LOGERR: fp = stderr; /* FALLTHROUGH */ case RFB_LOGWARN: if (c != NULL) (void) fprintf(fp, "rfb%u: ", c->rc_instance); else (void) fprintf(fp, "rfb: "); va_start(ap, fmt); (void) vfprintf(fp, fmt, ap); va_end(ap); if (raw_stdio) (void) fprintf(fp, "\r\n"); else (void) fprintf(fp, "\n"); (void) fflush(fp); } } static void rfb_init_once(void) { uint_t cpu_registers[4], ecx; do_cpuid(1, cpu_registers); ecx = cpu_registers[2]; rfb_sse42 = (ecx & CPUID2_SSE42) != 0; if (rfb_sse42) rfb_printf(NULL, RFB_LOGDEBUG, "enabled fast crc32"); else rfb_printf(NULL, RFB_LOGWARN, "no support for fast crc32"); if (get_config_bool_default("rfb.debug", false)) rfb_debug = 1; list_create(&rfb_list, sizeof (rfb_server_t), offsetof(rfb_server_t, rs_node)); rfb_idspace = id_space_create("rfb", 0, INT32_MAX); } static void rfb_free_client(rfb_client_t *c) { free(c->rc_crc); free(c->rc_crc_tmp); free(c->rc_zbuf); free(c->rc_gci.data); if (c->rc_encodings & RFB_ENCODING_ZLIB) (void) deflateEnd(&c->rc_zstream); if (c->rc_fd != -1) (void) close(c->rc_fd); free(c); } /* * Calculate CRC32 using SSE4.2; Intel or AMD Bulldozer+ CPUs only */ static inline uint32_t fast_crc32(void *buf, int len, uint32_t crcval) { uint32_t q = len / sizeof (uint32_t); uint32_t *p = (uint32_t *)buf; while (q--) { /* BEGIN CSTYLED */ asm volatile ( /* crc32l %ecx,%esi */ ".byte 0xf2, 0xf, 0x38, 0xf1, 0xf1;" :"=S" (crcval) :"0" (crcval), "c" (*p) ); /* END CSTYLED */ p++; } return (crcval); } static void rfb_send_client_status(rfb_client_t *c, uint32_t status, const char *msg) { rfb_printf(c, RFB_LOGDEBUG, "sending client status %u (%s)", status, msg ? msg : "NULL"); status = htonl(status); (void) stream_write(c->rc_fd, &status, sizeof (status)); if (msg != NULL && status != 0 && c->rc_cver == RFB_CVER_3_8) { char buf[4]; rfb_printf(c, RFB_LOGWARN, msg); be32enc(buf, strlen((char *)msg)); (void) stream_write(c->rc_fd, buf, 4); (void) stream_write(c->rc_fd, msg, strlen((char *)msg)); } } static bool rfb_handshake_version(rfb_client_t *c) { unsigned char buf[RFB_VERSION_LEN]; ssize_t l; rfb_printf(c, RFB_LOGDEBUG, "handshake version"); if (stream_write(c->rc_fd, RFB_VERSION, RFB_VERSION_LEN) != RFB_VERSION_LEN) { rfb_printf(c, RFB_LOGWARN, "could not send server version."); return (false); } l = stream_read(c->rc_fd, buf, sizeof (buf)); if (l <= 0) { rfb_printf(c, RFB_LOGWARN, "client version not read"); return (false); } else if (l != RFB_VERSION_LEN) { rfb_printf(c, RFB_LOGWARN, "client sent short version - '%.*s'", l, buf); return (false); } rfb_printf(c, RFB_LOGDEBUG, "version handshake, client ver '%.*s'", l - 1, buf); if (strncmp(RFB_VERSION, (char *)buf, RFB_VERSION_LEN - 2) != 0) { rfb_printf(c, RFB_LOGERR, "bad client version '%.*s'", l, buf); return (false); } switch (buf[RFB_VERSION_LEN - 2]) { case '8': c->rc_cver = RFB_CVER_3_8; break; case '7': c->rc_cver = RFB_CVER_3_7; break; case '5': /* * From the RFB specification[1], section 7.1.1: * "version 3.5 was wrongly reported by some clients, but this * should be interpreted by all servers as 3.3." */ case '3': c->rc_cver = RFB_CVER_3_3; break; default: rfb_printf(c, RFB_LOGERR, "unsupported client version '%.*s'", l - 1, buf); return (false); } return (true); } static bool rfb_handshake_auth(rfb_client_t *c) { unsigned char buf[RFBP_SECURITY_VNC_AUTH_LEN]; int auth_type; rfb_printf(c, RFB_LOGDEBUG, "handshake auth"); auth_type = RFBP_SECURITY_NONE; #ifndef NO_OPENSSL if (c->rc_s->rs_password != NULL) auth_type = RFBP_SECURITY_VNC_AUTH; #endif switch (c->rc_cver) { case RFB_CVER_3_3: /* * RFB specification[1] section 7.1.2: * The server decides the security type and sends a single word. */ be32enc(buf, auth_type); (void) stream_write(c->rc_fd, buf, 4); break; case RFB_CVER_3_7: case RFB_CVER_3_8: /* Send list of supported types. */ buf[0] = 1; /* list length */ buf[1] = auth_type; (void) stream_write(c->rc_fd, buf, 2); /* Read agreed security type. */ if (stream_read(c->rc_fd, buf, 1) != 1) { rfb_printf(c, RFB_LOGWARN, "auth fail, no type from client"); return (false); } if (buf[0] != auth_type) { rfb_send_client_status(c, 1, "Auth failed: authentication type mismatch"); return (false); } break; } if (auth_type == RFBP_SECURITY_NONE) { /* * According to the RFB specification[1], section 7.2.1, for a * security type of 'None', client versions 3.3 and 3.7 expect * to move straight to the ClientInit phase, without the server * sending a response. For version 3.8, a SecurityResult word * needs to be sent indicating success. */ switch (c->rc_cver) { case RFB_CVER_3_3: case RFB_CVER_3_7: break; case RFB_CVER_3_8: rfb_send_client_status(c, 0, NULL); break; } return (true); } /* Perform VNC authentication. */ #ifdef NO_OPENSSL rfb_printf(c, RFB_LOGERR, "Auth not supported, no OpenSSL in your system"); rfb_send_client_status(c, 1, "Auth failed."); return (false); #else unsigned char challenge[RFBP_SECURITY_VNC_AUTH_LEN]; unsigned char keystr[RFBP_SECURITY_VNC_PASSWD_LEN]; unsigned char crypt_expected[RFBP_SECURITY_VNC_AUTH_LEN]; DES_key_schedule ks; /* * The client encrypts the challenge with DES, using a password * supplied by the user as the key. * To form the key, the password is truncated to eight characters, or * padded with null bytes on the right. * The client then sends the resulting 16-bytes response. */ (void) strncpy((char *)keystr, c->rc_s->rs_password, RFBP_SECURITY_VNC_PASSWD_LEN); /* * VNC clients encrypt the challenge with all the bit fields in each * byte of the password mirrored. * Here we flip each byte of the keystr. */ for (uint_t i = 0; i < RFBP_SECURITY_VNC_PASSWD_LEN; i++) { keystr[i] = (keystr[i] & 0xf0) >> 4 | (keystr[i] & 0x0f) << 4; keystr[i] = (keystr[i] & 0xcc) >> 2 | (keystr[i] & 0x33) << 2; keystr[i] = (keystr[i] & 0xaa) >> 1 | (keystr[i] & 0x55) << 1; } /* Initialize a 16-byte random challenge. */ arc4random_buf(challenge, sizeof (challenge)); /* Send the challenge to the client. */ if (stream_write(c->rc_fd, challenge, RFBP_SECURITY_VNC_AUTH_LEN) != RFBP_SECURITY_VNC_AUTH_LEN) { rfb_printf(c, RFB_LOGERR, "failed to send challenge to client"); return (false); } /* Receive the 16-byte challenge response. */ if (stream_read(c->rc_fd, buf, RFBP_SECURITY_VNC_AUTH_LEN) != RFBP_SECURITY_VNC_AUTH_LEN) { rfb_send_client_status(c, 1, "Challenge response read failed"); return (false); } memcpy(crypt_expected, challenge, RFBP_SECURITY_VNC_AUTH_LEN); /* Encrypt the Challenge with DES. */ DES_set_key_unchecked((const_DES_cblock *)keystr, &ks); DES_ecb_encrypt((const_DES_cblock *)challenge, (const_DES_cblock *)crypt_expected, &ks, DES_ENCRYPT); DES_ecb_encrypt( (const_DES_cblock *)(challenge + RFBP_SECURITY_VNC_PASSWD_LEN), (const_DES_cblock *)(crypt_expected + RFBP_SECURITY_VNC_PASSWD_LEN), &ks, DES_ENCRYPT); if (memcmp(crypt_expected, buf, RFBP_SECURITY_VNC_AUTH_LEN) != 0) { rfb_send_client_status(c, 1, "Auth failed: Invalid password."); return (false); } rfb_printf(c, RFB_LOGDEBUG, "authentication succeeded"); rfb_send_client_status(c, 0, NULL); #endif return (true); } static bool rfb_handshake_init_message(rfb_client_t *c) { struct bhyvegc_image *gci; char buf[1]; char *name; rfb_printf(c, RFB_LOGDEBUG, "handshake server init"); /* Read the client init message. */ if (stream_read(c->rc_fd, buf, 1) != 1) { rfb_printf(c, RFB_LOGWARN, "client did not send init"); return (false); } if (buf[0] == 0) { rfb_client_t *oc; rfb_printf(c, RFB_LOGDEBUG, "client requested exclusive access"); pthread_mutex_lock(&c->rc_s->rs_clientlock); c->rc_s->rs_exclusive = true; /* Disconnect all other clients. */ for (oc = list_head(&c->rc_s->rs_clients); oc != NULL; oc = list_next(&c->rc_s->rs_clients, oc)) { if (oc != c) oc->rc_closing = true; } pthread_mutex_unlock(&c->rc_s->rs_clientlock); } else { rfb_printf(c, RFB_LOGDEBUG, "client requested shared access"); pthread_mutex_lock(&c->rc_s->rs_clientlock); if (c->rc_s->rs_exclusive) { rfb_printf(c, RFB_LOGWARN, "deny due to existing exclusive session"); pthread_mutex_unlock(&c->rc_s->rs_clientlock); return (false); } pthread_mutex_unlock(&c->rc_s->rs_clientlock); } gci = console_get_image(); c->rc_sinfo.rsi_width = htons(gci->width); c->rc_sinfo.rsi_height = htons(gci->height); c->rc_width = gci->width; c->rc_height = gci->height; if (c->rc_s->rs_name != NULL) name = (char *)c->rc_s->rs_name; else name = "bhyve"; c->rc_sinfo.rsi_namelen = htonl(strlen(name)); (void) stream_write(c->rc_fd, &c->rc_sinfo, sizeof (c->rc_sinfo)); (void) stream_write(c->rc_fd, name, strlen(name)); return (true); } static bool rfb_handshake(rfb_client_t *c) { if (!rfb_handshake_version(c)) return (false); if (!rfb_handshake_auth(c)) return (false); if (!rfb_handshake_init_message(c)) return (false); return (true); } static void rfb_print_pixfmt(rfb_client_t *c, rfb_pixfmt_t *px, rfb_loglevel_t level) { rfb_printf(c, level, "%20s: %u", "bpp", px->rp_bpp); rfb_printf(c, level, "%20s: %u", "depth", px->rp_depth); rfb_printf(c, level, "%20s: %u", "bigendian", px->rp_bigendian); rfb_printf(c, level, "%20s: %u", "truecolour", px->rp_truecolour); rfb_printf(c, level, "%20s: %u", "r_max", ntohs(px->rp_r_max)); rfb_printf(c, level, "%20s: %u", "g_max", ntohs(px->rp_g_max)); rfb_printf(c, level, "%20s: %u", "b_max", ntohs(px->rp_b_max)); rfb_printf(c, level, "%20s: %u", "r_shift", px->rp_r_shift); rfb_printf(c, level, "%20s: %u", "g_shift", px->rp_g_shift); rfb_printf(c, level, "%20s: %u", "b_shift", px->rp_b_shift); } static bool rfb_recv_set_pixel_format(rfb_client_t *c) { rfb_cs_pixfmt_msg_t msg; rfb_pixfmt_t *newpx = &msg.rp_pixfmt; rfb_pixfmt_t *oldpx = &c->rc_sinfo.rsi_pixfmt; rfb_pixfmt_t *spx = &c->rc_s->rs_pixfmt; rfb_printf(c, RFB_LOGDEBUG, "received pixel format"); if (stream_read(c->rc_fd, &msg, sizeof (msg)) != sizeof (msg)) return (false); /* * The client has sent its desired pixel format. The protocol does not * have a mechanism to reject this, we are supposed to just start using * the requested format from the next update. * * At present, we can only support alternative rgb-shift values and * will accept (and ignore) a new depth value. */ if (oldpx->rp_bpp != newpx->rp_bpp || oldpx->rp_bigendian != newpx->rp_bigendian || oldpx->rp_truecolour != newpx->rp_truecolour || oldpx->rp_r_max != newpx->rp_r_max || oldpx->rp_g_max != newpx->rp_g_max || oldpx->rp_b_max != newpx->rp_b_max) { rfb_printf(c, RFB_LOGWARN, "unsupported pixfmt from client"); rfb_print_pixfmt(c, newpx, RFB_LOGWARN); return (false); } rfb_print_pixfmt(c, newpx, RFB_LOGDEBUG); /* Check if the new shifts match the server's native values. */ if (newpx->rp_r_shift != spx->rp_r_shift || newpx->rp_g_shift != spx->rp_g_shift || newpx->rp_b_shift != spx->rp_b_shift) { c->rc_custom_pixfmt = true; rfb_printf(c, RFB_LOGDEBUG, "Using custom pixfmt"); } else { c->rc_custom_pixfmt = false; rfb_printf(c, RFB_LOGDEBUG, "Using native pixfmt"); } c->rc_sinfo.rsi_pixfmt = msg.rp_pixfmt; c->rc_crc_reset = true; return (true); } static bool rfb_recv_set_encodings(rfb_client_t *c) { rfb_cs_encodings_msg_t msg; rfb_printf(c, RFB_LOGDEBUG, "received encodings"); if (stream_read(c->rc_fd, &msg, sizeof (msg)) != sizeof (msg)) return (false); msg.re_numencs = htons(msg.re_numencs); rfb_printf(c, RFB_LOGDEBUG, "%d values", msg.re_numencs); for (uint_t i = 0; i < msg.re_numencs; i++) { uint32_t enc; if (stream_read(c->rc_fd, &enc, sizeof (enc)) != sizeof (enc)) return (false); enc = htonl(enc); switch (enc) { case RFBP_ENCODING_RAW: rfb_printf(c, RFB_LOGDEBUG, "client supports raw encoding"); c->rc_encodings |= RFB_ENCODING_RAW; break; case RFBP_ENCODING_ZLIB: rfb_printf(c, RFB_LOGDEBUG, "client supports zlib encoding"); if (!(c->rc_encodings & RFB_ENCODING_ZLIB)) { if (deflateInit(&c->rc_zstream, Z_BEST_SPEED) != Z_OK) { return (false); } c->rc_encodings |= RFB_ENCODING_ZLIB; } break; case RFBP_ENCODING_RESIZE: rfb_printf(c, RFB_LOGDEBUG, "client supports resize"); c->rc_encodings |= RFB_ENCODING_RESIZE; break; case RFBP_ENCODING_EXT_KEVENT: rfb_printf(c, RFB_LOGDEBUG, "client supports ext key event"); c->rc_encodings |= RFB_ENCODING_EXT_KEVENT; break; case RFBP_ENCODING_DESKTOP_NAME: rfb_printf(c, RFB_LOGDEBUG, "client supports desktop name"); c->rc_encodings |= RFB_ENCODING_DESKTOP_NAME; break; default: rfb_printf(c, RFB_LOGDEBUG, "client supports encoding %d", (int32_t)enc); } } return (true); } static bool rfb_recv_update(rfb_client_t *c) { rfb_cs_update_msg_t msg; if (stream_read(c->rc_fd, &msg, sizeof (msg)) != sizeof (msg)) return (false); if (!c->rc_keyevent_sent && (c->rc_encodings & RFB_ENCODING_EXT_KEVENT)) { /* * Take this opportunity to tell the client that we * accept QEMU Extended Key Event Pseudo-encoding. */ c->rc_keyevent_sent = true; rfb_send_extended_keyevent_update_msg(c); } c->rc_pending = true; if (msg.rum_incremental == 0) { rfb_printf(c, RFB_LOGDEBUG, "client requested full screen update"); c->rc_send_fullscreen = true; } return (true); } static bool rfb_recv_key_event(rfb_client_t *c) { rfb_cs_key_event_msg_t msg; if (stream_read(c->rc_fd, &msg, sizeof (msg)) != sizeof (msg)) return (false); msg.rke_sym = htonl(msg.rke_sym); rfb_printf(c, RFB_LOGDEBUG, "received key %s %x", msg.rke_down == 0 ? "up" : "down", msg.rke_sym); console_key_event(msg.rke_down, msg.rke_sym, htonl(0)); c->rc_input_detected = true; return (true); } static bool rfb_recv_pointer_event(rfb_client_t *c) { rfb_cs_pointer_event_msg_t msg; if (stream_read(c->rc_fd, &msg, sizeof (msg)) != sizeof (msg)) return (false); msg.rpe_x = htons(msg.rpe_x); msg.rpe_y = htons(msg.rpe_y); if (rfb_debug > 1) { rfb_printf(c, RFB_LOGDEBUG, "received pointer event @ %dx%d", msg.rpe_x, msg.rpe_y); } console_ptr_event(msg.rpe_button, msg.rpe_x, msg.rpe_y); c->rc_input_detected = true; return (true); } static bool rfb_recv_cut_text(rfb_client_t *c) { rfb_cs_cut_text_msg_t msg; unsigned char buf[32]; rfb_printf(c, RFB_LOGDEBUG, "received cut text event"); if (stream_read(c->rc_fd, &msg, sizeof (msg)) != sizeof (msg)) return (false); msg.rct_length = htonl(msg.rct_length); rfb_printf(c, RFB_LOGDEBUG, "%u bytes in buffer", msg.rct_length); /* Consume the buffer */ while (msg.rct_length > 0) { ssize_t l; l = stream_read(c->rc_fd, buf, MIN(sizeof (buf), msg.rct_length)); if (l <= 0) return (false); msg.rct_length -= l; } return (true); } static bool rfb_recv_qemu(rfb_client_t *c) { rfb_cs_qemu_msg_t msg; rfb_printf(c, RFB_LOGDEBUG, "received QEMU event"); if (stream_read(c->rc_fd, &msg, sizeof (msg)) != sizeof (msg)) return (false); switch (msg.rq_subtype) { case RFBP_CS_QEMU_KEVENT: { rfb_cs_qemu_extended_key_msg_t keymsg; if (stream_read(c->rc_fd, &keymsg, sizeof (keymsg)) != sizeof (keymsg)) { return (false); } keymsg.rqek_sym = htonl(keymsg.rqek_sym); keymsg.rqek_code = htonl(keymsg.rqek_code); rfb_printf(c, RFB_LOGDEBUG, "QEMU key %s %x / %x", keymsg.rqek_down == 0 ? "up" : "down", keymsg.rqek_sym, keymsg.rqek_code); console_key_event((int)keymsg.rqek_down, keymsg.rqek_sym, keymsg.rqek_code); c->rc_input_detected = true; break; } default: rfb_printf(c, RFB_LOGWARN, "Unknown QEMU event subtype: %d\n", msg.rq_subtype); return (false); } return (true); } static bool rfb_send_update_header(rfb_client_t *c, int numrects) { rfb_server_update_msg_t msg; msg.rss_type = RFBP_SC_UPDATE; msg.rss_pad = 0; msg.rss_numrects = htons(numrects); return (stream_write(c->rc_fd, &msg, sizeof (msg)) == sizeof (msg)); } static void rfb_send_resize_update_msg(rfb_client_t *c) { rfb_rect_hdr_t rect; rfb_printf(c, RFB_LOGDEBUG, "sending screen resize %dx%d", c->rc_width, c->rc_height); (void) rfb_send_update_header(c, 1); rect.rr_x = htons(0); rect.rr_y = htons(0); rect.rr_width = htons(c->rc_width); rect.rr_height = htons(c->rc_height); rect.rr_encoding = htonl(RFBP_ENCODING_RESIZE); (void) stream_write(c->rc_fd, &rect, sizeof (rect)); } static void rfb_send_extended_keyevent_update_msg(rfb_client_t *c) { rfb_rect_hdr_t rect; rfb_printf(c, RFB_LOGDEBUG, "sending extended keyevent update message"); (void) rfb_send_update_header(c, 1); rect.rr_x = htons(0); rect.rr_y = htons(0); rect.rr_width = htons(c->rc_width); rect.rr_height = htons(c->rc_height); rect.rr_encoding = htonl(RFBP_ENCODING_EXT_KEVENT); (void) stream_write(c->rc_fd, &rect, sizeof (rect)); } static void translate_pixels(rfb_client_t *c, struct bhyvegc_image *gci, int x1, int y1, int x2, int y2) { rfb_pixfmt_t *px = &c->rc_sinfo.rsi_pixfmt; rfb_pixfmt_t *spx = &c->rc_s->rs_pixfmt; int w, h; w = gci->width; h = gci->height; VERIFY3S(gci->width, ==, c->rc_gci.width); VERIFY3S(gci->height, ==, c->rc_gci.height); for (uint_t y = y1; y < h && y < y2; y++) { for (uint_t x = x1; x < w && x < x2; x++) { uint32_t p; p = gci->data[y * w + x]; c->rc_gci.data[y * w + x] = 0xff000000 | ((p >> spx->rp_r_shift) & 0xff) << px->rp_r_shift | ((p >> spx->rp_g_shift) & 0xff) << px->rp_g_shift | ((p >> spx->rp_b_shift) & 0xff) << px->rp_b_shift; } } } static bool rfb_send_rect(rfb_client_t *c, struct bhyvegc_image *gci, int x, int y, int w, int h) { rfb_rect_hdr_t rect; unsigned long zlen; ssize_t nwrite, total; int err; uint32_t *p; uint8_t *zbufp; if (rfb_debug > 1) { rfb_printf(c, RFB_LOGDEBUG, "send rect %dx%d %dx%d", x, y, w, h); } /* Rectangle header. */ rect.rr_x = htons(x); rect.rr_y = htons(y); rect.rr_width = htons(w); rect.rr_height = htons(h); uint32_t *data = gci->data; if (c->rc_custom_pixfmt) { translate_pixels(c, gci, x, y, x + w, y + h); data = c->rc_gci.data; } h = y + h; w *= sizeof (uint32_t); if (c->rc_encodings & RFB_ENCODING_ZLIB) { zbufp = c->rc_zbuf; c->rc_zstream.total_in = 0; c->rc_zstream.total_out = 0; for (p = &data[y * gci->width + x]; y < h; y++) { c->rc_zstream.next_in = (Bytef *)p; c->rc_zstream.avail_in = w; c->rc_zstream.next_out = (Bytef *)zbufp; c->rc_zstream.avail_out = RFB_ZLIB_BUFSZ + 16 - c->rc_zstream.total_out; c->rc_zstream.data_type = Z_BINARY; /* Compress with zlib. */ err = deflate(&c->rc_zstream, Z_SYNC_FLUSH); if (err != Z_OK) { rfb_printf(c, RFB_LOGWARN, "zlib[rect] deflate err: %d", err); goto doraw; } zbufp = c->rc_zbuf + c->rc_zstream.total_out; p += gci->width; } rect.rr_encoding = htonl(RFBP_ENCODING_ZLIB); nwrite = stream_write(c->rc_fd, &rect, sizeof (rect)); if (nwrite <= 0) return (false); zlen = htonl(c->rc_zstream.total_out); nwrite = stream_write(c->rc_fd, &zlen, sizeof (uint32_t)); if (nwrite <= 0) return (false); return (stream_write(c->rc_fd, c->rc_zbuf, c->rc_zstream.total_out) == c->rc_zstream.total_out); } doraw: total = 0; zbufp = c->rc_zbuf; for (p = &data[y * gci->width + x]; y < h; y++) { memcpy(zbufp, p, w); zbufp += w; total += w; p += gci->width; } rect.rr_encoding = htonl(RFBP_ENCODING_RAW); nwrite = stream_write(c->rc_fd, &rect, sizeof (rect)); if (nwrite <= 0) return (false); return (stream_write(c->rc_fd, c->rc_zbuf, total) == total); } static bool rfb_send_all(rfb_client_t *c, struct bhyvegc_image *gci) { rfb_rect_hdr_t rect; ssize_t nwrite; unsigned long zlen; int err; rfb_printf(c, RFB_LOGDEBUG, "send entire screen"); /* Just the one (big) rect. */ if (!rfb_send_update_header(c, 1)) return (false); rect.rr_x = 0; rect.rr_y = 0; rect.rr_width = htons(gci->width); rect.rr_height = htons(gci->height); uint32_t *data = gci->data; if (c->rc_custom_pixfmt) { translate_pixels(c, gci, 0, 0, gci->width, gci->height); data = c->rc_gci.data; } if (c->rc_encodings & RFB_ENCODING_ZLIB) { c->rc_zstream.next_in = (Bytef *)data; c->rc_zstream.avail_in = gci->width * gci->height * sizeof (uint32_t); c->rc_zstream.next_out = (Bytef *)c->rc_zbuf; c->rc_zstream.avail_out = RFB_ZLIB_BUFSZ + 16; c->rc_zstream.data_type = Z_BINARY; c->rc_zstream.total_in = 0; c->rc_zstream.total_out = 0; /* Compress with zlib. */ err = deflate(&c->rc_zstream, Z_SYNC_FLUSH); if (err != Z_OK) { rfb_printf(c, RFB_LOGWARN, "zlib deflate err: %d", err); goto doraw; } rect.rr_encoding = htonl(RFBP_ENCODING_ZLIB); nwrite = stream_write(c->rc_fd, &rect, sizeof (rect)); if (nwrite <= 0) return (false); zlen = htonl(c->rc_zstream.total_out); nwrite = stream_write(c->rc_fd, &zlen, sizeof (uint32_t)); if (nwrite <= 0) return (false); return (stream_write(c->rc_fd, c->rc_zbuf, c->rc_zstream.total_out) == c->rc_zstream.total_out); } doraw: rect.rr_encoding = htonl(RFBP_ENCODING_RAW); nwrite = stream_write(c->rc_fd, &rect, sizeof (rect)); if (nwrite <= 0) return (false); nwrite = gci->width * gci->height * sizeof (uint32_t); return (stream_write(c->rc_fd, data, nwrite) == nwrite); } static bool rfb_send_screen(rfb_client_t *c) { struct bhyvegc_image *gci; bool retval = true; bool sendall = false; int xcells, ycells; int rem_x, rem_y; uint32_t *p, *ncrc, *ocrc; uint_t changes, perc, x, y; /* Updates require a preceding client update request. */ if (atomic_exchange(&c->rc_pending, false) == false) return (true); console_refresh(); gci = console_get_image(); /* * It's helpful if the image size or data address does not change * underneath us. */ pthread_mutex_lock(&gci->mtx); /* Check for screen resolution changes. */ if (c->rc_width != gci->width || c->rc_height != gci->height) { c->rc_width = gci->width; c->rc_height = gci->height; c->rc_crc_reset = true; c->rc_send_fullscreen = true; /* If the client supports it, send a resize event. */ if (c->rc_encodings & RFB_ENCODING_RESIZE) { rfb_send_resize_update_msg(c); /* * A resize message counts as an update in response to * the client's preceding request so rc->pending does * not need to be reset here. */ goto done; } } /* Clear old CRC values. */ if (atomic_exchange(&c->rc_crc_reset, false)) memset(c->rc_crc, '\0', c->rc_cells * sizeof (uint32_t)); if (c->rc_custom_pixfmt && (c->rc_gci.data == NULL || c->rc_gci.width != c->rc_width || c->rc_gci.height != c->rc_height)) { c->rc_gci.data = reallocarray(c->rc_gci.data, c->rc_width * c->rc_height, sizeof (uint32_t)); if (c->rc_gci.data == NULL) { retval = false; goto done; } c->rc_gci.width = c->rc_width; c->rc_gci.height = c->rc_height; } else if (!c->rc_custom_pixfmt && c->rc_gci.data != NULL) { free(c->rc_gci.data); c->rc_gci.data = NULL; } sendall = atomic_exchange(&c->rc_send_fullscreen, false); /* * Calculate a checksum for each 32x32 cell. Send all that have * changed since the last scan. */ xcells = howmany(gci->width, RFB_PIX_PER_CELL); ycells = howmany(gci->height, RFB_PIX_PER_CELL); rem_x = gci->width & RFB_PIXCELL_MASK; rem_y = gci->height & RFB_PIXCELL_MASK; if (rem_y == 0) rem_y = RFB_PIX_PER_CELL; p = gci->data; ncrc = c->rc_crc_tmp - xcells; ocrc = c->rc_crc - xcells; changes = 0; memset(c->rc_crc_tmp, '\0', sizeof (uint32_t) * xcells * ycells); for (y = 0; y < gci->height; y++) { if ((y & RFB_PIXCELL_MASK) == 0) { ncrc += xcells; ocrc += xcells; } for (x = 0; x < xcells; x++) { uint_t cellwidth; if (x == xcells - 1 && rem_x > 0) cellwidth = rem_x; else cellwidth = RFB_PIX_PER_CELL; if (rfb_sse42) { ncrc[x] = fast_crc32(p, cellwidth * sizeof (uint32_t), ncrc[x]); } else { ncrc[x] = (uint32_t)crc32(ncrc[x], (Bytef *)p, cellwidth * sizeof (uint32_t)); } p += cellwidth; /* check for crc delta if last row in cell. */ if ((y & RFB_PIXCELL_MASK) == RFB_PIXCELL_MASK || y == gci->height - 1) { if (ocrc[x] != ncrc[x]) { ocrc[x] = ncrc[x]; ncrc[x] = 1; changes++; } else { ncrc[x] = 0; } } } } perc = (changes * 100) / (xcells * ycells); if (rfb_debug > 1 && changes > 0) { rfb_printf(c, RFB_LOGDEBUG, "scanned and found %u changed cell(s) - %u%%", changes, perc); } /* * If there are no changes, don't send an update. Restore the pending * flag since we still owe the client an update. */ if (!sendall && !changes) { c->rc_pending = true; goto done; } /* If there are a lot of changes, send the whole screen. */ if (perc >= RFB_SENDALL_THRESH) sendall = true; if (sendall) { retval = rfb_send_all(c, gci); goto done; } if (!rfb_send_update_header(c, changes)) { retval = false; goto done; } /* Send the changed cells as separate rects. */ ncrc = c->rc_crc_tmp; for (y = 0; y < gci->height; y += RFB_PIX_PER_CELL) { /* Previous cell's row. */ int celly = (y >> RFB_PIXCELL_SHIFT); /* Delta check crc to previous set. */ for (x = 0; x < xcells; x++) { uint_t cellwidth; if (*ncrc++ == 0) continue; if (x == xcells - 1 && rem_x > 0) cellwidth = rem_x; else cellwidth = RFB_PIX_PER_CELL; if (!rfb_send_rect(c, gci, x * RFB_PIX_PER_CELL, celly * RFB_PIX_PER_CELL, cellwidth, y + RFB_PIX_PER_CELL >= gci->height ? rem_y : RFB_PIX_PER_CELL)) { retval = false; goto done; } } } done: pthread_mutex_unlock(&gci->mtx); return (retval); } static void * rfb_client_rx_thread(void *arg) { rfb_client_t *c = arg; unsigned char cmd; bool ret = true; while (ret && !c->rc_closing && (read(c->rc_fd, &cmd, 1) == 1)) { switch (cmd) { case RFBP_CS_SET_PIXEL_FORMAT: ret = rfb_recv_set_pixel_format(c); break; case RFBP_CS_SET_ENCODINGS: ret = rfb_recv_set_encodings(c); break; case RFBP_CS_UPDATE_REQUEST: ret = rfb_recv_update(c); break; case RFBP_CS_KEY_EVENT: ret = rfb_recv_key_event(c); break; case RFBP_CS_POINTER_EVENT: ret = rfb_recv_pointer_event(c); break; case RFBP_CS_CUT_TEXT: ret = rfb_recv_cut_text(c); break; case RFBP_CS_QEMU: ret = rfb_recv_qemu(c); break; default: rfb_printf(c, RFB_LOGWARN, "unknown cs code %d", cmd & 0xff); ret = false; } } rfb_printf(c, RFB_LOGDEBUG, "client rx thread exiting"); c->rc_closing = true; return (NULL); } static void * rfb_client_tx_thread(void *arg) { rfb_client_t *c = arg; rfb_server_t *s = c->rc_s; char tname[MAXCOMLEN + 1]; uint_t counter = 0; hrtime_t tprev; void *status; int err; (void) snprintf(tname, sizeof (tname), "rfb%u tx", c->rc_instance); (void) pthread_set_name_np(c->rc_tx_tid, tname); c->rc_sinfo.rsi_pixfmt = c->rc_s->rs_pixfmt; c->rc_encodings = RFB_ENCODING_RAW; if (!rfb_handshake(c)) { rfb_printf(c, RFB_LOGWARN, "handshake failure"); goto out; } c->rc_cells = howmany(RFB_MAX_WIDTH * RFB_MAX_HEIGHT, RFB_PIX_PER_CELL); if ((c->rc_crc = calloc(c->rc_cells, sizeof (uint32_t))) == NULL || (c->rc_crc_tmp = calloc(c->rc_cells, sizeof (uint32_t))) == NULL) { perror("calloc crc"); goto out; } err = pthread_create(&c->rc_rx_tid, NULL, rfb_client_rx_thread, c); if (err != 0) { perror("pthread_create client rx thread"); goto out; } (void) snprintf(tname, sizeof (tname), "rfb%u rx", c->rc_instance); (void) pthread_set_name_np(c->rc_rx_tid, tname); tprev = gethrtime(); while (!c->rc_closing) { struct timeval tv; hrtime_t tnow; int64_t tdiff; fd_set rfds; int err; FD_ZERO(&rfds); FD_SET(c->rc_fd, &rfds); tv.tv_sec = 0; tv.tv_usec = RFB_SEL_DELAY_US; err = select(c->rc_fd + 1, &rfds, NULL, NULL, &tv); if (err < 0) break; /* Determine if its time to push the screen; ~24hz. */ tnow = gethrtime(); tdiff = NSEC2USEC(tnow - tprev); if (tdiff >= RFB_SCREEN_POLL_DELAY) { bool input; tprev = tnow; input = atomic_exchange(&c->rc_input_detected, false); /* * Refresh the screen on every second trip through the * loop, or if keyboard/mouse input has been detected. */ if ((++counter & 1) != 0 || input) { if (!rfb_send_screen(c)) break; } } else { (void) usleep(RFB_SCREEN_POLL_DELAY - tdiff); } } out: rfb_printf(c, RFB_LOGWARN, "disconnected"); (void) pthread_join(c->rc_rx_tid, &status); pthread_mutex_lock(&s->rs_clientlock); s->rs_clientcount--; list_remove(&s->rs_clients, c); if (s->rs_exclusive && s->rs_clientcount == 0) s->rs_exclusive = false; id_free(rfb_idspace, c->rc_instance); pthread_mutex_unlock(&s->rs_clientlock); rfb_free_client(c); return (NULL); } static void rfb_accept(int sfd, enum ev_type event, void *arg) { rfb_server_t *s = arg; rfb_client_t *c = NULL; struct sockaddr_storage cliaddr; socklen_t len; char host[NI_MAXHOST], port[NI_MAXSERV]; int cfd, err; uint_t cc; rfb_printf(c, RFB_LOGDEBUG, "incoming connection"); len = sizeof (cliaddr); cfd = accept(sfd, (struct sockaddr *)&cliaddr, &len); if (cfd == -1) { perror("client accept"); return; } *host = *port = '\0'; if (cliaddr.ss_family == AF_UNIX) { rfb_printf(NULL, RFB_LOGDEBUG, "connection on UNIX socket"); (void) strlcpy(host, "", sizeof (host)); } else { err = getnameinfo((struct sockaddr *)&cliaddr, len, host, sizeof (host), port, sizeof (port), NI_NUMERICHOST | NI_NUMERICSERV); if (err != 0) { rfb_printf(NULL, RFB_LOGERR, "getnameinfo: %s", gai_strerror(err)); *host = *port = '\0'; } else { rfb_printf(NULL, RFB_LOGDEBUG, "connection from %s:%s", host, port); } } pthread_mutex_lock(&s->rs_clientlock); cc = s->rs_clientcount; pthread_mutex_unlock(&s->rs_clientlock); if (cc >= RFB_MAX_CLIENTS) { rfb_printf(NULL, RFB_LOGERR, "too many clients, closing connection."); goto fail; } if ((c = calloc(1, sizeof (rfb_client_t))) == NULL) { perror("calloc client"); goto fail; } c->rc_fd = cfd; c->rc_s = s; c->rc_zbuf = malloc(RFB_ZLIB_BUFSZ + 16); if (c->rc_zbuf == NULL) goto fail; pthread_mutex_lock(&s->rs_clientlock); err = pthread_create(&c->rc_tx_tid, NULL, rfb_client_tx_thread, c); if (err != 0) { perror("pthread_create client tx thread"); pthread_mutex_unlock(&s->rs_clientlock); goto fail; } s->rs_clientcount++; list_insert_tail(&s->rs_clients, c); c->rc_instance = id_allocff(rfb_idspace); pthread_mutex_unlock(&s->rs_clientlock); (void) pthread_detach(c->rc_tx_tid); rfb_printf(c, RFB_LOGWARN, "connection from %s", host); return; fail: (void) close(cfd); free(c); } int rfb_init(char *hostname, int port, int wait, const char *password, const char *name) { rfb_server_t *s; #ifndef WITHOUT_CAPSICUM cap_rights_t rights; #endif (void) pthread_once(&rfb_once, rfb_init_once); if (rfb_idspace == NULL) { rfb_printf(NULL, RFB_LOGERR, "rfb_idspace could not be allocated"); return (-1); } if ((s = calloc(1, sizeof (rfb_server_t))) == NULL) { perror("calloc"); return (-1); } s->rs_fd = -1; s->rs_name = name; if (password != NULL && strlen(password) > 0) s->rs_password = password; if (pthread_mutex_init(&s->rs_clientlock, NULL) != 0) { perror("pthread_mutex_init"); free(s); return (-1); } list_create(&s->rs_clients, sizeof (rfb_client_t), offsetof(rfb_client_t, rc_node)); /* Server pixel format. */ s->rs_pixfmt.rp_bpp = RFB_PIX_BPP; s->rs_pixfmt.rp_depth = RFB_PIX_DEPTH; s->rs_pixfmt.rp_bigendian = 0; s->rs_pixfmt.rp_truecolour = 1; s->rs_pixfmt.rp_r_max = htons(RFB_PIX_RMAX); s->rs_pixfmt.rp_g_max = htons(RFB_PIX_GMAX); s->rs_pixfmt.rp_b_max = htons(RFB_PIX_BMAX); s->rs_pixfmt.rp_r_shift = RFB_PIX_RSHIFT; s->rs_pixfmt.rp_g_shift = RFB_PIX_GSHIFT; s->rs_pixfmt.rp_b_shift = RFB_PIX_BSHIFT; /* UNIX socket. */ if (port == -1 && hostname != NULL && *hostname == '/') { struct sockaddr_un sock; s->rs_fd = socket(PF_UNIX, SOCK_STREAM, 0); if (s->rs_fd < 0) { perror("socket"); goto fail; } sock.sun_family = AF_UNIX; if (strlcpy(sock.sun_path, hostname, sizeof (sock.sun_path)) >= sizeof (sock.sun_path)) { rfb_printf(NULL, RFB_LOGERR, "socket path '%s' too long\n", hostname); goto fail; } (void) unlink(hostname); if (bind(s->rs_fd, (struct sockaddr *)&sock, sizeof (sock)) < 0) { perror("bind"); goto fail; } } else { struct addrinfo hints, *ai = NULL; char servname[6]; int e; (void) snprintf(servname, sizeof (servname), "%d", port ? port : RFB_DEFAULT_PORT); if (hostname == NULL || strlen(hostname) == 0) { #if defined(INET) hostname = "127.0.0.1"; #elif defined(INET6) hostname = "[::1]"; #endif } memset(&hints, '\0', sizeof (hints)); hints.ai_family = AF_UNSPEC; hints.ai_socktype = SOCK_STREAM; hints.ai_flags = AI_NUMERICHOST | AI_NUMERICSERV | AI_PASSIVE; if ((e = getaddrinfo(hostname, servname, &hints, &ai)) != 0) { rfb_printf(NULL, RFB_LOGERR, "getaddrinfo: %s", gai_strerror(e)); goto fail; } s->rs_fd = socket(ai->ai_family, ai->ai_socktype, 0); if (s->rs_fd < 0) { perror("socket"); freeaddrinfo(ai); goto fail; } e = 1; (void) setsockopt(s->rs_fd, SOL_SOCKET, SO_REUSEADDR, &e, sizeof (e)); if (bind(s->rs_fd, ai->ai_addr, ai->ai_addrlen) < 0) { perror("bind"); freeaddrinfo(ai); goto fail; } freeaddrinfo(ai); } if (listen(s->rs_fd, 5) < 0) { perror("listen"); goto fail; } #ifndef WITHOUT_CAPSICUM cap_rights_init(&rights, CAP_ACCEPT, CAP_EVENT, CAP_READ, CAP_WRITE); if (caph_rights_limit(s->rs_fd, &rights) == -1) errx(EX_OSERR, "Unable to apply rights for sandbox"); #endif s->rs_connevent = mevent_add(s->rs_fd, EVF_READ, rfb_accept, s); if (s->rs_connevent == NULL) { rfb_printf(NULL, RFB_LOGERR, "Failed to set up rfb connection mevent"); goto fail; } list_insert_tail(&rfb_list, s); /* * Wait for first connection. Since the mevent thread is * not yet running, we can't rely on normal incoming connection * handling. */ if (wait != 0) { fd_set rfds; int e; rfb_printf(NULL, RFB_LOGWARN, "holding boot until first client connection"); for (;;) { FD_ZERO(&rfds); FD_SET(s->rs_fd, &rfds); e = select(s->rs_fd + 1, &rfds, NULL, NULL, NULL); if (e < 0 && errno == EINTR) continue; if (e < 0 || FD_ISSET(s->rs_fd, &rfds)) break; } rfb_printf(NULL, RFB_LOGWARN, "continuing boot"); } return (0); fail: if (s->rs_fd != -1) VERIFY3S(close(s->rs_fd), ==, 0); (void) pthread_mutex_destroy(&s->rs_clientlock); list_destroy(&s->rs_clients); free(s); return (-1); } /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2015 Tycho Nightingale * Copyright 2018 Joyent, Inc. * Copyright 2022 OmniOS Community Edition (OmniOSce) Association. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef _RFB_H_ #define _RFB_H_ #define RFB_DEFAULT_PORT 5900 int rfb_init(char *hostname, int port, int wait, const char *password, const char *name); #endif /* _RFB_H_ */ /* * This file and its contents are supplied under the terms of the * Common Development and Distribution License ("CDDL"), version 1.0. * You may only use this file in accordance with the terms of version * 1.0 of the CDDL. * * A full copy of the text of the CDDL should have accompanied this * source. A copy of the CDDL is also available via the Internet at * http://www.illumos.org/license/CDDL. */ /* * Copyright 2022 OmniOS Community Edition (OmniOSce) Association. */ #ifndef _RFB_IMPL_H #define _RFB_IMPL_H #include #include #include #include #include #include "mevent.h" /* * The ProtocolVersion message consists of 12 bytes interpreted as a string of * ASCII characters in the format "RFB xxx.yyy\n" where xxx and yyy are the * major and minor version numbers, padded with zeros. */ #define RFB_VERSION "RFB 003.008\n" #define RFB_VERSION_LEN (sizeof (RFB_VERSION) - 1) _Static_assert(RFB_VERSION_LEN == 12, "RFB_VERSION length incorrect"); /* Keep synchronised with pci_fbuf.c */ #define RFB_MAX_WIDTH 1920 #define RFB_MAX_HEIGHT 1200 #define RFB_MAX_CLIENTS 10 /* Framebuffer pixel format */ #define RFB_PIX_BPP 32 #define RFB_PIX_DEPTH 24 #define RFB_PIX_RSHIFT 16 #define RFB_PIX_GSHIFT 8 #define RFB_PIX_BSHIFT 0 #define RFB_PIX_RMAX 255 #define RFB_PIX_GMAX 255 #define RFB_PIX_BMAX 255 #define RFB_ZLIB_BUFSZ (RFB_MAX_WIDTH * RFB_MAX_HEIGHT * 4) #define RFB_PIX_PER_CELL 32 #define RFB_PIXCELL_SHIFT 5 #define RFB_PIXCELL_MASK 0x1f #define RFB_SENDALL_THRESH 25 #define RFB_SEL_DELAY_US 10000 #define RFB_SCREEN_REFRESH_DELAY 33300 /* 30 Hz */ #define RFB_SCREEN_POLL_DELAY (RFB_SCREEN_REFRESH_DELAY / 2) /* Client-to-server message types */ #define RFBP_CS_SET_PIXEL_FORMAT 0 #define RFBP_CS_SET_ENCODINGS 2 #define RFBP_CS_UPDATE_REQUEST 3 #define RFBP_CS_KEY_EVENT 4 #define RFBP_CS_POINTER_EVENT 5 #define RFBP_CS_CUT_TEXT 6 #define RFBP_CS_QEMU 255 #define RFBP_CS_QEMU_KEVENT 0 /* Server-to-client message types */ #define RFBP_SC_UPDATE 0 #define RFBP_SC_SET_COLOURMAP_ENTRIES 1 #define RFBP_SC_BELL 2 #define RFBP_SC_CUT_TEXT 3 /* Encodings */ #define RFBP_ENCODING_RAW 0 #define RFBP_ENCODING_ZLIB 6 /* Pseudo-encodings */ #define RFBP_ENCODING_RESIZE -223 #define RFBP_ENCODING_EXT_KEVENT -258 /* QEMU ext. key event */ #define RFBP_ENCODING_DESKTOP_NAME -307 /* Security types */ #define RFBP_SECURITY_INVALID 0 #define RFBP_SECURITY_NONE 1 #define RFBP_SECURITY_VNC_AUTH 2 #define RFBP_SECURITY_VNC_AUTH_LEN 16 #define RFBP_SECURITY_VNC_PASSWD_LEN 8 typedef enum rfb_loglevel { RFB_LOGDEBUG, RFB_LOGWARN, RFB_LOGERR } rfb_loglevel_t; typedef enum rfb_encodings { RFB_ENCODING_RAW = (1ULL << 0), RFB_ENCODING_ZLIB = (1ULL << 1), RFB_ENCODING_RESIZE = (1ULL << 2), RFB_ENCODING_EXT_KEVENT = (1ULL << 3), RFB_ENCODING_DESKTOP_NAME = (1ULL << 4) } rfb_encodings_t; typedef enum rfb_cver { RFB_CVER_3_3, RFB_CVER_3_7, RFB_CVER_3_8 } rfb_cver_t; typedef struct rfb_pixfmt { uint8_t rp_bpp; uint8_t rp_depth; uint8_t rp_bigendian; uint8_t rp_truecolour; uint16_t rp_r_max; uint16_t rp_g_max; uint16_t rp_b_max; uint8_t rp_r_shift; uint8_t rp_g_shift; uint8_t rp_b_shift; uint8_t rp_pad[3]; } __packed rfb_pixfmt_t; /* Server-to-client message formats */ typedef struct rfb_server_info { uint16_t rsi_width; uint16_t rsi_height; rfb_pixfmt_t rsi_pixfmt; uint32_t rsi_namelen; } __packed rfb_server_info_t; typedef struct rfb_server_update_msg { uint8_t rss_type; uint8_t rss_pad; uint16_t rss_numrects; } __packed rfb_server_update_msg_t; typedef struct rfb_rect_hdr { uint16_t rr_x; uint16_t rr_y; uint16_t rr_width; uint16_t rr_height; uint32_t rr_encoding; } __packed rfb_rect_hdr_t; /* Client-to-server message formats */ typedef struct rfb_cs_pixfmt_msg { uint8_t rp_pad[3]; rfb_pixfmt_t rp_pixfmt; } __packed rfb_cs_pixfmt_msg_t; typedef struct rfb_cs_update_msg { uint8_t rum_incremental; uint16_t rum_x; uint16_t rum_y; uint16_t rum_width; uint16_t rum_height; } __packed rfb_cs_update_msg_t; typedef struct rfb_cs_encodings_msg { uint8_t re_pad; uint16_t re_numencs; } __packed rfb_cs_encodings_msg_t; typedef struct rfb_cs_key_event_msg { uint8_t rke_down; uint16_t rke_pad; uint32_t rke_sym; } __packed rfb_cs_key_event_msg_t; typedef struct rfb_cs_pointer_event_msg { uint8_t rpe_button; uint16_t rpe_x; uint16_t rpe_y; } __packed rfb_cs_pointer_event_msg_t; typedef struct rfb_cs_cut_text_msg { uint8_t rct_padding[3]; uint32_t rct_length; } __packed rfb_cs_cut_text_msg_t; typedef struct rfb_cs_qemu_msg { uint8_t rq_subtype; } __packed rfb_cs_qemu_msg_t; typedef struct rfb_cs_qemu_extended_key_msg { uint16_t rqek_down; uint32_t rqek_sym; uint32_t rqek_code; } __packed rfb_cs_qemu_extended_key_msg_t; /* Client/server data structures */ typedef struct rfb_server { list_node_t rs_node; int rs_fd; const char *rs_name; const char *rs_password; struct mevent *rs_connevent; uint_t rs_clientcount; pthread_mutex_t rs_clientlock; list_t rs_clients; bool rs_exclusive; rfb_pixfmt_t rs_pixfmt; } rfb_server_t; typedef struct rfb_client { list_node_t rc_node; uint_t rc_instance; int rc_fd; rfb_server_t *rc_s; rfb_server_info_t rc_sinfo; pthread_t rc_rx_tid; pthread_t rc_tx_tid; int rc_width; int rc_height; size_t rc_cells; uint32_t *rc_crc; uint32_t *rc_crc_tmp; z_stream rc_zstream; uint8_t *rc_zbuf; struct bhyvegc_image rc_gci; rfb_cver_t rc_cver; rfb_encodings_t rc_encodings; atomic_bool rc_closing; atomic_bool rc_pending; atomic_bool rc_input_detected; atomic_bool rc_crc_reset; atomic_bool rc_send_fullscreen; bool rc_custom_pixfmt; bool rc_keyevent_sent; } rfb_client_t; #endif /* _RFB_IMPL_H */ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2014 Tycho Nightingale * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include #include #include #include #include #include #include #include #include #include #include #ifndef __FreeBSD__ #include #endif #include "bhyverun.h" #include "config.h" #include "debug.h" #include "smbiostbl.h" #define MB (1024*1024) #define GB (1024ULL*1024*1024) #define SMBIOS_BASE 0xF1000 #define FIRMWARE_VERSION "14.0" /* The SMBIOS specification defines the date format to be mm/dd/yyyy */ #define FIRMWARE_RELEASE_DATE "10/10/2021" /* BHYVE_ACPI_BASE - SMBIOS_BASE) */ #define SMBIOS_MAX_LENGTH (0xF2400 - 0xF1000) #define SMBIOS_TYPE_BIOS 0 #define SMBIOS_TYPE_SYSTEM 1 #define SMBIOS_TYPE_BOARD 2 #define SMBIOS_TYPE_CHASSIS 3 #define SMBIOS_TYPE_PROCESSOR 4 #define SMBIOS_TYPE_MEMARRAY 16 #define SMBIOS_TYPE_MEMDEVICE 17 #define SMBIOS_TYPE_MEMARRAYMAP 19 #define SMBIOS_TYPE_BOOT 32 #define SMBIOS_TYPE_EOT 127 struct smbios_structure { uint8_t type; uint8_t length; uint16_t handle; } __packed; struct smbios_string { const char *node; const char *value; }; typedef int (*initializer_func_t)(const struct smbios_structure *template_entry, const struct smbios_string *template_strings, char *curaddr, char **endaddr, uint16_t *n); struct smbios_template_entry { const struct smbios_structure *entry; const struct smbios_string *strings; initializer_func_t initializer; }; /* * SMBIOS Structure Table Entry Point */ #define SMBIOS_ENTRY_EANCHOR "_SM_" #define SMBIOS_ENTRY_EANCHORLEN 4 #define SMBIOS_ENTRY_IANCHOR "_DMI_" #define SMBIOS_ENTRY_IANCHORLEN 5 struct smbios_entry_point { char eanchor[4]; /* anchor tag */ uint8_t echecksum; /* checksum of entry point structure */ uint8_t eplen; /* length in bytes of entry point */ uint8_t major; /* major version of the SMBIOS spec */ uint8_t minor; /* minor version of the SMBIOS spec */ uint16_t maxssize; /* maximum size in bytes of a struct */ uint8_t revision; /* entry point structure revision */ uint8_t format[5]; /* entry point rev-specific data */ char ianchor[5]; /* intermediate anchor tag */ uint8_t ichecksum; /* intermediate checksum */ uint16_t stlen; /* len in bytes of structure table */ uint32_t staddr; /* physical addr of structure table */ uint16_t stnum; /* number of structure table entries */ uint8_t bcdrev; /* BCD value representing DMI ver */ } __packed; /* * BIOS Information */ #define SMBIOS_FL_ISA 0x00000010 /* ISA is supported */ #define SMBIOS_FL_PCI 0x00000080 /* PCI is supported */ #define SMBIOS_FL_SHADOW 0x00001000 /* BIOS shadowing is allowed */ #define SMBIOS_FL_CDBOOT 0x00008000 /* Boot from CD is supported */ #define SMBIOS_FL_SELBOOT 0x00010000 /* Selectable Boot supported */ #define SMBIOS_FL_EDD 0x00080000 /* EDD Spec is supported */ #define SMBIOS_XB1_FL_ACPI 0x00000001 /* ACPI is supported */ #define SMBIOS_XB2_FL_BBS 0x00000001 /* BIOS Boot Specification */ #define SMBIOS_XB2_FL_VM 0x00000010 /* Virtual Machine */ struct smbios_table_type0 { struct smbios_structure header; uint8_t vendor; /* vendor string */ uint8_t version; /* version string */ uint16_t segment; /* address segment location */ uint8_t rel_date; /* release date */ uint8_t size; /* rom size */ uint64_t cflags; /* characteristics */ uint8_t xc_bytes[2]; /* characteristics ext bytes */ uint8_t sb_major_rel; /* system bios version */ uint8_t sb_minor_rele; uint8_t ecfw_major_rel; /* embedded ctrl fw version */ uint8_t ecfw_minor_rel; } __packed; /* * System Information */ #define SMBIOS_WAKEUP_SWITCH 0x06 /* power switch */ struct smbios_table_type1 { struct smbios_structure header; uint8_t manufacturer; /* manufacturer string */ uint8_t product; /* product name string */ uint8_t version; /* version string */ uint8_t serial; /* serial number string */ uint8_t uuid[16]; /* uuid byte array */ uint8_t wakeup; /* wake-up event */ uint8_t sku; /* sku number string */ uint8_t family; /* family name string */ } __packed; /* * Baseboard (or Module) Information */ #define SMBIOS_BRF_HOSTING 0x1 #define SMBIOS_BRT_MOTHERBOARD 0xa struct smbios_table_type2 { struct smbios_structure header; uint8_t manufacturer; /* manufacturer string */ uint8_t product; /* product name string */ uint8_t version; /* version string */ uint8_t serial; /* serial number string */ uint8_t asset; /* asset tag string */ uint8_t fflags; /* feature flags */ uint8_t location; /* location in chassis */ uint16_t chandle; /* chassis handle */ uint8_t type; /* board type */ uint8_t n_objs; /* number of contained object handles */ } __packed; /* * System Enclosure or Chassis */ #define SMBIOS_CHT_UNKNOWN 0x02 /* unknown */ #define SMBIOS_CHT_DESKTOP 0x03 /* desktop */ #define SMBIOS_CHST_SAFE 0x03 /* safe */ #define SMBIOS_CHSC_NONE 0x03 /* none */ struct smbios_table_type3 { struct smbios_structure header; uint8_t manufacturer; /* manufacturer string */ uint8_t type; /* type */ uint8_t version; /* version string */ uint8_t serial; /* serial number string */ uint8_t asset; /* asset tag string */ uint8_t bustate; /* boot-up state */ uint8_t psstate; /* power supply state */ uint8_t tstate; /* thermal state */ uint8_t security; /* security status */ uint32_t oemdata; /* OEM-specific data */ uint8_t uheight; /* height in 'u's */ uint8_t cords; /* number of power cords */ uint8_t elems; /* number of element records */ uint8_t elemlen; /* length of records */ uint8_t sku; /* sku number string */ } __packed; /* * Processor Information */ #define SMBIOS_PRT_CENTRAL 0x03 /* central processor */ #define SMBIOS_PRF_OTHER 0x01 /* other */ #define SMBIOS_PRS_PRESENT 0x40 /* socket is populated */ #define SMBIOS_PRS_ENABLED 0x1 /* enabled */ #define SMBIOS_PRU_NONE 0x06 /* none */ #define SMBIOS_PFL_64B 0x04 /* 64-bit capable */ struct smbios_table_type4 { struct smbios_structure header; uint8_t socket; /* socket designation string */ uint8_t type; /* processor type */ uint8_t family; /* processor family */ uint8_t manufacturer; /* manufacturer string */ uint64_t cpuid; /* processor cpuid */ uint8_t version; /* version string */ uint8_t voltage; /* voltage */ uint16_t clkspeed; /* ext clock speed in mhz */ uint16_t maxspeed; /* maximum speed in mhz */ uint16_t curspeed; /* current speed in mhz */ uint8_t status; /* status */ uint8_t upgrade; /* upgrade */ uint16_t l1handle; /* l1 cache handle */ uint16_t l2handle; /* l2 cache handle */ uint16_t l3handle; /* l3 cache handle */ uint8_t serial; /* serial number string */ uint8_t asset; /* asset tag string */ uint8_t part; /* part number string */ uint8_t cores; /* cores per socket */ uint8_t ecores; /* enabled cores */ uint8_t threads; /* threads per socket */ uint16_t cflags; /* processor characteristics */ uint16_t family2; /* processor family 2 */ } __packed; /* * Physical Memory Array */ #define SMBIOS_MAL_SYSMB 0x03 /* system board or motherboard */ #define SMBIOS_MAU_SYSTEM 0x03 /* system memory */ #define SMBIOS_MAE_NONE 0x03 /* none */ struct smbios_table_type16 { struct smbios_structure header; uint8_t location; /* physical device location */ uint8_t use; /* device functional purpose */ uint8_t ecc; /* err detect/correct method */ uint32_t size; /* max mem capacity in kb */ uint16_t errhand; /* handle of error (if any) */ uint16_t ndevs; /* num of slots or sockets */ uint64_t xsize; /* max mem capacity in bytes */ } __packed; /* * Memory Device */ #define SMBIOS_MDFF_UNKNOWN 0x02 /* unknown */ #define SMBIOS_MDT_UNKNOWN 0x02 /* unknown */ #define SMBIOS_MDF_UNKNOWN 0x0004 /* unknown */ struct smbios_table_type17 { struct smbios_structure header; uint16_t arrayhand; /* handle of physl mem array */ uint16_t errhand; /* handle of mem error data */ uint16_t twidth; /* total width in bits */ uint16_t dwidth; /* data width in bits */ uint16_t size; /* size in kb or mb */ uint8_t form; /* form factor */ uint8_t set; /* set */ uint8_t dloc; /* device locator string */ uint8_t bloc; /* phys bank locator string */ uint8_t type; /* memory type */ uint16_t flags; /* memory characteristics */ uint16_t maxspeed; /* maximum speed in mhz */ uint8_t manufacturer; /* manufacturer string */ uint8_t serial; /* serial number string */ uint8_t asset; /* asset tag string */ uint8_t part; /* part number string */ uint8_t attributes; /* attributes */ uint32_t xsize; /* extended size in mb */ uint16_t curspeed; /* current speed in mhz */ uint16_t minvoltage; /* minimum voltage */ uint16_t maxvoltage; /* maximum voltage */ uint16_t curvoltage; /* configured voltage */ } __packed; /* * Memory Array Mapped Address */ struct smbios_table_type19 { struct smbios_structure header; uint32_t saddr; /* start phys addr in kb */ uint32_t eaddr; /* end phys addr in kb */ uint16_t arrayhand; /* physical mem array handle */ uint8_t width; /* num of dev in row */ uint64_t xsaddr; /* start phys addr in bytes */ uint64_t xeaddr; /* end phys addr in bytes */ } __packed; /* * System Boot Information */ #define SMBIOS_BOOT_NORMAL 0 /* no errors detected */ struct smbios_table_type32 { struct smbios_structure header; uint8_t reserved[6]; uint8_t status; /* boot status */ } __packed; /* * End-of-Table */ struct smbios_table_type127 { struct smbios_structure header; } __packed; static const struct smbios_table_type0 smbios_type0_template = { { SMBIOS_TYPE_BIOS, sizeof (struct smbios_table_type0), 0 }, 1, /* bios vendor string */ 2, /* bios version string */ 0xF000, /* bios address segment location */ 3, /* bios release date */ 0x0, /* bios size (64k * (n + 1) is the size in bytes) */ SMBIOS_FL_ISA | SMBIOS_FL_PCI | SMBIOS_FL_SHADOW | SMBIOS_FL_CDBOOT | SMBIOS_FL_EDD, { SMBIOS_XB1_FL_ACPI, SMBIOS_XB2_FL_BBS | SMBIOS_XB2_FL_VM }, 0x0, /* bios major release */ 0x0, /* bios minor release */ 0xff, /* embedded controller firmware major release */ 0xff /* embedded controller firmware minor release */ }; static const struct smbios_string smbios_type0_strings[] = { { "bios.vendor", "BHYVE" }, /* vendor string */ { "bios.version", FIRMWARE_VERSION }, /* bios version string */ { "bios.release_date", FIRMWARE_RELEASE_DATE }, /* bios release date string */ { 0 } }; static const struct smbios_table_type1 smbios_type1_template = { { SMBIOS_TYPE_SYSTEM, sizeof (struct smbios_table_type1), 0 }, 1, /* manufacturer string */ 2, /* product string */ 3, /* version string */ 4, /* serial number string */ { 0 }, SMBIOS_WAKEUP_SWITCH, 5, /* sku string */ 6 /* family string */ }; static int smbios_type1_initializer(const struct smbios_structure *template_entry, const struct smbios_string *template_strings, char *curaddr, char **endaddr, uint16_t *n); static const struct smbios_string smbios_type1_strings[] = { { "system.manufacturer", "illumos" }, /* manufacturer string */ { "system.product_name", "BHYVE" }, /* product string */ { "system.version", "1.0" }, /* version string */ { "system.serial_number", "None" }, /* serial number string */ { "system.sku", "None" }, /* sku string */ { "system.family_name", "Virtual Machine" }, /* family string */ { 0 } }; static const struct smbios_table_type2 smbios_type2_template = { { SMBIOS_TYPE_BOARD, sizeof (struct smbios_table_type2), 0 }, 1, /* manufacturer string */ 2, /* product string */ 3, /* version string */ 4, /* serial number string */ 5, /* asset tag string */ SMBIOS_BRF_HOSTING, /* feature flags */ 6, /* location string */ SMBIOS_CHT_DESKTOP, /* chassis handle */ SMBIOS_BRT_MOTHERBOARD, /* board type */ 0 }; static const struct smbios_string smbios_type2_strings[] = { { "board.manufacturer", "illumos" }, /* manufacturer string */ { "board.product_name", "BHYVE" }, /* product name string */ { "board.version", "1.0" }, /* version string */ { "board.serial_number", "None" }, /* serial number string */ { "board.asset_tag", "None" }, /* asset tag string */ { "board.location", "None" }, /* location string */ { 0 } }; static const struct smbios_table_type3 smbios_type3_template = { { SMBIOS_TYPE_CHASSIS, sizeof (struct smbios_table_type3), 0 }, 1, /* manufacturer string */ SMBIOS_CHT_UNKNOWN, 2, /* version string */ 3, /* serial number string */ 4, /* asset tag string */ SMBIOS_CHST_SAFE, SMBIOS_CHST_SAFE, SMBIOS_CHST_SAFE, SMBIOS_CHSC_NONE, 0, /* OEM specific data, we have none */ 0, /* height in 'u's (0=enclosure height unspecified) */ 0, /* number of power cords (0=number unspecified) */ 0, /* number of contained element records */ 0, /* length of records */ 5 /* sku number string */ }; static const struct smbios_string smbios_type3_strings[] = { { "chassis.manufacturer", "illumos" }, /* manufacturer string */ { "chassis.version", "1.0" }, /* version string */ { "chassis.serial_number", "None" }, /* serial number string */ { "chassis.asset_tag", "None" }, /* asset tag string */ { "chassis.sku", "None" }, /* sku number string */ { 0 } }; static const struct smbios_table_type4 smbios_type4_template = { { SMBIOS_TYPE_PROCESSOR, sizeof (struct smbios_table_type4), 0 }, 1, /* socket designation string */ SMBIOS_PRT_CENTRAL, SMBIOS_PRF_OTHER, 2, /* manufacturer string */ 0, /* cpuid */ 3, /* version string */ 0, /* voltage */ 0, /* external clock frequency in mhz (0=unknown) */ 0, /* maximum frequency in mhz (0=unknown) */ 0, /* current frequency in mhz (0=unknown) */ SMBIOS_PRS_PRESENT | SMBIOS_PRS_ENABLED, SMBIOS_PRU_NONE, -1, /* l1 cache handle */ -1, /* l2 cache handle */ -1, /* l3 cache handle */ 4, /* serial number string */ 5, /* asset tag string */ 6, /* part number string */ 0, /* cores per socket (0=unknown) */ 0, /* enabled cores per socket (0=unknown) */ 0, /* threads per socket (0=unknown) */ SMBIOS_PFL_64B, SMBIOS_PRF_OTHER }; static const struct smbios_string smbios_type4_strings[] = { { NULL, " " }, /* socket designation string */ { NULL, " " }, /* manufacturer string */ { NULL, " " }, /* version string */ { NULL, "None" }, /* serial number string */ { NULL, "None" }, /* asset tag string */ { NULL, "None" }, /* part number string */ { 0 } }; static int smbios_type4_initializer( const struct smbios_structure *template_entry, const struct smbios_string *template_strings, char *curaddr, char **endaddr, uint16_t *n); static const struct smbios_table_type16 smbios_type16_template = { { SMBIOS_TYPE_MEMARRAY, sizeof (struct smbios_table_type16), 0 }, SMBIOS_MAL_SYSMB, SMBIOS_MAU_SYSTEM, SMBIOS_MAE_NONE, 0x80000000, /* max mem capacity in kb (0x80000000=use extended) */ -1, /* handle of error (if any) */ 0, /* number of slots or sockets (TBD) */ 0 /* extended maximum memory capacity in bytes (TBD) */ }; static int smbios_type16_initializer( const struct smbios_structure *template_entry, const struct smbios_string *template_strings, char *curaddr, char **endaddr, uint16_t *n); static const struct smbios_table_type17 smbios_type17_template = { { SMBIOS_TYPE_MEMDEVICE, sizeof (struct smbios_table_type17), 0 }, -1, /* handle of physical memory array */ -1, /* handle of memory error data */ 64, /* total width in bits including ecc */ 64, /* data width in bits */ 0, /* size in kb or mb (0x7fff=use extended)*/ SMBIOS_MDFF_UNKNOWN, 0, /* set (0x00=none, 0xff=unknown) */ 1, /* device locator string */ 2, /* physical bank locator string */ SMBIOS_MDT_UNKNOWN, SMBIOS_MDF_UNKNOWN, 0, /* maximum memory speed in mhz (0=unknown) */ 3, /* manufacturer string */ 4, /* serial number string */ 5, /* asset tag string */ 6, /* part number string */ 0, /* attributes (0=unknown rank information) */ 0, /* extended size in mb (TBD) */ 0, /* current speed in mhz (0=unknown) */ 0, /* minimum voltage in mv (0=unknown) */ 0, /* maximum voltage in mv (0=unknown) */ 0 /* configured voltage in mv (0=unknown) */ }; static const struct smbios_string smbios_type17_strings[] = { { NULL, " " }, /* device locator string */ { NULL, " " }, /* physical bank locator string */ { NULL, " " }, /* manufacturer string */ { NULL, "None" }, /* serial number string */ { NULL, "None" }, /* asset tag string */ { NULL, "None" }, /* part number string */ { 0 } }; static int smbios_type17_initializer( const struct smbios_structure *template_entry, const struct smbios_string *template_strings, char *curaddr, char **endaddr, uint16_t *n); static const struct smbios_table_type19 smbios_type19_template = { { SMBIOS_TYPE_MEMARRAYMAP, sizeof (struct smbios_table_type19), 0 }, 0xffffffff, /* starting phys addr in kb (0xffffffff=use ext) */ 0xffffffff, /* ending phys addr in kb (0xffffffff=use ext) */ -1, /* physical memory array handle */ 1, /* number of devices that form a row */ 0, /* extended starting phys addr in bytes (TDB) */ 0 /* extended ending phys addr in bytes (TDB) */ }; static int smbios_type19_initializer( const struct smbios_structure *template_entry, const struct smbios_string *template_strings, char *curaddr, char **endaddr, uint16_t *n); static struct smbios_table_type32 smbios_type32_template = { { SMBIOS_TYPE_BOOT, sizeof (struct smbios_table_type32), 0 }, { 0, 0, 0, 0, 0, 0 }, SMBIOS_BOOT_NORMAL }; static const struct smbios_table_type127 smbios_type127_template = { { SMBIOS_TYPE_EOT, sizeof (struct smbios_table_type127), 0 } }; static int smbios_generic_initializer( const struct smbios_structure *template_entry, const struct smbios_string *template_strings, char *curaddr, char **endaddr, uint16_t *n); static struct smbios_template_entry smbios_template[] = { { (const struct smbios_structure *)&smbios_type0_template, smbios_type0_strings, smbios_generic_initializer }, { (const struct smbios_structure *)&smbios_type1_template, smbios_type1_strings, smbios_type1_initializer }, { (const struct smbios_structure *)&smbios_type2_template, smbios_type2_strings, smbios_generic_initializer }, { (const struct smbios_structure *)&smbios_type3_template, smbios_type3_strings, smbios_generic_initializer }, { (const struct smbios_structure *)&smbios_type4_template, smbios_type4_strings, smbios_type4_initializer }, { (const struct smbios_structure *)&smbios_type16_template, NULL, smbios_type16_initializer }, { (const struct smbios_structure *)&smbios_type17_template, smbios_type17_strings, smbios_type17_initializer }, { (const struct smbios_structure *)&smbios_type19_template, NULL, smbios_type19_initializer }, { (const struct smbios_structure *)&smbios_type32_template, NULL, smbios_generic_initializer }, { (const struct smbios_structure *)&smbios_type127_template, NULL, smbios_generic_initializer }, { NULL,NULL, NULL } }; static uint64_t guest_lomem, guest_himem, guest_himem_base; static uint16_t type16_handle; static int smbios_generic_initializer(const struct smbios_structure *template_entry, const struct smbios_string *template_strings, char *curaddr, char **endaddr, uint16_t *n) { struct smbios_structure *entry; memcpy(curaddr, template_entry, template_entry->length); entry = (struct smbios_structure *)curaddr; entry->handle = *n + 1; curaddr += entry->length; if (template_strings != NULL) { int i; for (i = 0; template_strings[i].value != NULL; i++) { const char *string; int len; if (template_strings[i].node == NULL) { string = template_strings[i].value; } else { set_config_value_if_unset( template_strings[i].node, template_strings[i].value); string = get_config_value( template_strings[i].node); } len = strlen(string) + 1; memcpy(curaddr, string, len); curaddr += len; } *curaddr = '\0'; curaddr++; } else { /* Minimum string section is double nul */ *curaddr = '\0'; curaddr++; *curaddr = '\0'; curaddr++; } (*n)++; *endaddr = curaddr; return (0); } static int smbios_type1_initializer(const struct smbios_structure *template_entry, const struct smbios_string *template_strings, char *curaddr, char **endaddr, uint16_t *n) { struct smbios_table_type1 *type1; const char *guest_uuid_str; smbios_generic_initializer(template_entry, template_strings, curaddr, endaddr, n); type1 = (struct smbios_table_type1 *)curaddr; guest_uuid_str = get_config_value("uuid"); if (guest_uuid_str != NULL) { uuid_t uuid; uint32_t status; uuid_from_string(guest_uuid_str, &uuid, &status); if (status != uuid_s_ok) { EPRINTLN("Invalid UUID"); return (-1); } uuid_enc_le(&type1->uuid, &uuid); } else { MD5_CTX mdctx; u_char digest[16]; char hostname[MAXHOSTNAMELEN]; const char *vmname; /* * Universally unique and yet reproducible are an * oxymoron, however reproducible is desirable in * this case. */ if (gethostname(hostname, sizeof(hostname))) return (-1); MD5Init(&mdctx); vmname = get_config_value("name"); MD5Update(&mdctx, vmname, strlen(vmname)); MD5Update(&mdctx, hostname, sizeof(hostname)); MD5Final(digest, &mdctx); /* * Set the variant and version number. */ digest[6] &= 0x0F; digest[6] |= 0x30; /* version 3 */ digest[8] &= 0x3F; digest[8] |= 0x80; memcpy(&type1->uuid, digest, sizeof (digest)); } return (0); } static int smbios_type4_initializer(const struct smbios_structure *template_entry, const struct smbios_string *template_strings, char *curaddr, char **endaddr, uint16_t *n) { int i; for (i = 0; i < cpu_sockets; i++) { struct smbios_table_type4 *type4; char *p; int nstrings, len; smbios_generic_initializer(template_entry, template_strings, curaddr, endaddr, n); type4 = (struct smbios_table_type4 *)curaddr; p = curaddr + sizeof (struct smbios_table_type4); nstrings = 0; while (p < *endaddr - 1) { if (*p++ == '\0') nstrings++; } len = sprintf(*endaddr - 1, "CPU #%d", i) + 1; *endaddr += len - 1; *(*endaddr) = '\0'; (*endaddr)++; type4->socket = nstrings + 1; /* Revise cores and threads after update to smbios 3.0 */ if (cpu_cores > 254) type4->cores = 0; else type4->cores = cpu_cores; /* This threads is total threads in a socket */ if (cpu_cores * cpu_threads > 254) type4->threads = 0; else type4->threads = cpu_cores * cpu_threads; curaddr = *endaddr; } return (0); } static int smbios_type16_initializer(const struct smbios_structure *template_entry, const struct smbios_string *template_strings, char *curaddr, char **endaddr, uint16_t *n) { struct smbios_table_type16 *type16; type16_handle = *n; smbios_generic_initializer(template_entry, template_strings, curaddr, endaddr, n); type16 = (struct smbios_table_type16 *)curaddr; type16->xsize = guest_lomem + guest_himem; type16->ndevs = guest_himem > 0 ? 2 : 1; return (0); } static int smbios_type17_initializer(const struct smbios_structure *template_entry, const struct smbios_string *template_strings, char *curaddr, char **endaddr, uint16_t *n) { struct smbios_table_type17 *type17; uint64_t memsize, size_KB, size_MB; smbios_generic_initializer(template_entry, template_strings, curaddr, endaddr, n); type17 = (struct smbios_table_type17 *)curaddr; type17->arrayhand = type16_handle; memsize = guest_lomem + guest_himem; size_KB = memsize / 1024; size_MB = memsize / MB; /* A single Type 17 entry can't represent more than ~2PB RAM */ if (size_MB > 0x7FFFFFFF) { printf("Warning: guest memory too big for SMBIOS Type 17 table: " "%luMB greater than max supported 2147483647MB\n", size_MB); size_MB = 0x7FFFFFFF; } /* See SMBIOS 2.7.0 section 7.18 - Memory Device (Type 17) */ if (size_KB <= 0x7FFF) { /* Can represent up to 32767KB with the top bit set */ type17->size = size_KB | (1 << 15); } else if (size_MB < 0x7FFF) { /* Can represent up to 32766MB with the top bit unset */ type17->size = size_MB & 0x7FFF; } else { type17->size = 0x7FFF; /* * Can represent up to 2147483647MB (~2PB) * The top bit is reserved */ type17->xsize = size_MB & 0x7FFFFFFF; } return (0); } static int smbios_type19_initializer(const struct smbios_structure *template_entry, const struct smbios_string *template_strings, char *curaddr, char **endaddr, uint16_t *n) { struct smbios_table_type19 *type19; smbios_generic_initializer(template_entry, template_strings, curaddr, endaddr, n); type19 = (struct smbios_table_type19 *)curaddr; type19->arrayhand = type16_handle; type19->xsaddr = 0; type19->xeaddr = guest_lomem; if (guest_himem > 0) { curaddr = *endaddr; smbios_generic_initializer(template_entry, template_strings, curaddr, endaddr, n); type19 = (struct smbios_table_type19 *)curaddr; type19->arrayhand = type16_handle; type19->xsaddr = guest_himem_base; type19->xeaddr = guest_himem_base + guest_himem; } return (0); } static void smbios_ep_initializer(struct smbios_entry_point *smbios_ep, uint32_t staddr) { memset(smbios_ep, 0, sizeof(*smbios_ep)); memcpy(smbios_ep->eanchor, SMBIOS_ENTRY_EANCHOR, SMBIOS_ENTRY_EANCHORLEN); smbios_ep->eplen = 0x1F; assert(sizeof (struct smbios_entry_point) == smbios_ep->eplen); smbios_ep->major = 2; smbios_ep->minor = 6; smbios_ep->revision = 0; memcpy(smbios_ep->ianchor, SMBIOS_ENTRY_IANCHOR, SMBIOS_ENTRY_IANCHORLEN); smbios_ep->staddr = staddr; smbios_ep->bcdrev = (smbios_ep->major & 0xf) << 4 | (smbios_ep->minor & 0xf); } static void smbios_ep_finalizer(struct smbios_entry_point *smbios_ep, uint16_t len, uint16_t num, uint16_t maxssize) { uint8_t checksum; int i; smbios_ep->maxssize = maxssize; smbios_ep->stlen = len; smbios_ep->stnum = num; checksum = 0; for (i = 0x10; i < 0x1f; i++) { checksum -= ((uint8_t *)smbios_ep)[i]; } smbios_ep->ichecksum = checksum; checksum = 0; for (i = 0; i < 0x1f; i++) { checksum -= ((uint8_t *)smbios_ep)[i]; } smbios_ep->echecksum = checksum; } #ifndef __FreeBSD__ /* * bhyve on illumos previously used configuration keys starting with 'smbios.' * to control type 1 SMBIOS information. Since these may still be present in * bhyve configuration files, the following table is used to translate them * to their new key names. */ static struct { const char *oldkey; const char *newkey; } smbios_legacy_config_map[] = { { "smbios.manufacturer", "system.manufacturer" }, { "smbios.family", "system.family_name" }, { "smbios.product", "system.product_name" }, { "smbios.serial", "system.serial_number" }, { "smbios.sku", "system.sku" }, { "smbios.version", "system.version" }, }; #endif int smbios_build(struct vmctx *ctx) { struct smbios_entry_point *smbios_ep; uint16_t n; uint16_t maxssize; char *curaddr, *startaddr, *ststartaddr; int i; int err; guest_lomem = vm_get_lowmem_size(ctx); guest_himem = vm_get_highmem_size(ctx); guest_himem_base = vm_get_highmem_base(ctx); startaddr = paddr_guest2host(ctx, SMBIOS_BASE, SMBIOS_MAX_LENGTH); if (startaddr == NULL) { EPRINTLN("smbios table requires mapped mem"); return (ENOMEM); } #ifndef __FreeBSD__ /* Translate legacy illumos configuration keys */ for (uint_t i = 0; i < ARRAY_SIZE(smbios_legacy_config_map); i++) { const char *v; v = get_config_value(smbios_legacy_config_map[i].oldkey); if (v != NULL) { set_config_value_if_unset( smbios_legacy_config_map[i].newkey, v); } } #endif curaddr = startaddr; smbios_ep = (struct smbios_entry_point *)curaddr; smbios_ep_initializer(smbios_ep, SMBIOS_BASE + sizeof(struct smbios_entry_point)); curaddr += sizeof(struct smbios_entry_point); ststartaddr = curaddr; n = 0; maxssize = 0; for (i = 0; smbios_template[i].entry != NULL; i++) { const struct smbios_structure *entry; const struct smbios_string *strings; initializer_func_t initializer; char *endaddr; size_t size; entry = smbios_template[i].entry; strings = smbios_template[i].strings; initializer = smbios_template[i].initializer; err = (*initializer)(entry, strings, curaddr, &endaddr, &n); if (err != 0) return (err); size = endaddr - curaddr; assert(size <= UINT16_MAX); if (size > maxssize) maxssize = (uint16_t)size; curaddr = endaddr; } assert(curaddr - startaddr < SMBIOS_MAX_LENGTH); smbios_ep_finalizer(smbios_ep, curaddr - ststartaddr, n, maxssize); return (0); } #ifndef __FreeBSD__ static struct { uint_t type; const char *key; char *val; } smbios_legacy_map[] = { { 1, "product", "product_name" }, { 1, "serial", "serial_number" }, { 1, "family", "family_name" }, }; static const struct smbios_string *smbios_tbl_map[] = { smbios_type0_strings, smbios_type1_strings, smbios_type2_strings, smbios_type3_strings, }; /* * This function accepts an option of the form * type,[key=value][,key=value]... * and sets smbios data for the given type. Keys for type X are defined in the * smbios_typeX_strings tables above, but for type 1 there are also some * legacy values which were accepted in earlier versions of bhyve on illumos * which need to be mapped. */ int smbios_parse(const char *opts) { char *buf, *lasts, *token, *typekey = NULL; const char *errstr; const struct smbios_string *tbl; nvlist_t *nvl; uint_t i; long type; if ((buf = strdup(opts)) == NULL) { (void) fprintf(stderr, "out of memory\n"); return (-1); } if ((token = strtok_r(buf, ",", &lasts)) == NULL) { (void) fprintf(stderr, "too few fields\n"); goto fail; } type = strtonum(token, 0, 3, &errstr); if (errstr != NULL) { fprintf(stderr, "First token (type) is %s\n", errstr); goto fail; } tbl = smbios_tbl_map[type]; /* Extract the config key for this type */ typekey = strdup(tbl[0].node); if (typekey == NULL) { (void) fprintf(stderr, "out of memory\n"); goto fail; } token = strchr(typekey, '.'); assert(token != NULL); *token = '\0'; nvl = create_config_node(typekey); if (nvl == NULL) { (void) fprintf(stderr, "out of memory\n"); goto fail; } while ((token = strtok_r(NULL, ",", &lasts)) != NULL) { char *val; if ((val = strchr(token, '=')) == NULL || val[1] == '\0') { (void) fprintf(stderr, "invalid key=value: '%s'\n", token); goto fail; } *val++ = '\0'; /* UUID is a top-level config item, but -U takes priority */ if (strcmp(token, "uuid") == 0) { set_config_value_if_unset(token, val); continue; } /* Translate legacy keys */ for (i = 0; i < ARRAY_SIZE(smbios_legacy_map); i++) { if (type == smbios_legacy_map[i].type && strcmp(token, smbios_legacy_map[i].key) == 0) { token = smbios_legacy_map[i].val; break; } } for (i = 0; tbl[i].value != NULL; i++) { if (strcmp(tbl[i].node + strlen(typekey) + 1, token) == 0) { /* Found match */ break; } } if (tbl[i].value == NULL) { (void) fprintf(stderr, "Unknown SMBIOS key %s for type %d\n", token, type); goto fail; } set_config_value_node(nvl, token, val); } free(typekey); return (0); fail: free(buf); free(typekey); return (-1); } #endif /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2014 Tycho Nightingale * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * Copyright 2018 Joyent, Inc. */ #ifndef _SMBIOSTBL_H_ #define _SMBIOSTBL_H_ struct vmctx; int smbios_build(struct vmctx *ctx); #ifndef __FreeBSD__ int smbios_parse(const char *opts); void smbios_apply(void); #endif #endif /* _SMBIOSTBL_H_ */ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2015 Nahanni Systems, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include #include #ifndef __FreeBSD__ #include #endif #include #include "sockstream.h" ssize_t stream_read(int fd, void *buf, ssize_t nbytes) { uint8_t *p; ssize_t len = 0; ssize_t n; p = buf; while (len < nbytes) { n = read(fd, p + len, nbytes - len); if (n == 0) break; if (n < 0) { if (errno == EINTR || errno == EAGAIN) continue; return (n); } len += n; } return (len); } ssize_t stream_write(int fd, const void *buf, ssize_t nbytes) { const uint8_t *p; ssize_t len = 0; ssize_t n; p = buf; while (len < nbytes) { #ifdef __FreeBSD__ n = write(fd, p + len, nbytes - len); #else n = send(fd, p + len, nbytes - len, MSG_NOSIGNAL); #endif if (n == 0) break; if (n < 0) { if (errno == EINTR || errno == EAGAIN) continue; return (n); } len += n; } return (len); } /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2015 Nahanni Systems, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include #include ssize_t stream_read(int fd, void *buf, ssize_t nbytes); ssize_t stream_write(int fd, const void *buf, ssize_t nbytes); /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2023 Beckhoff Automation GmbH & Co. KG * Author: Corvin Köhne */ #include #include #include #include #include #include #ifndef __FreeBSD__ #include #include #endif #include #include "acpi.h" #include "acpi_device.h" #include "config.h" #include "tpm_device.h" #include "tpm_emul.h" #include "tpm_intf.h" #include "tpm_ppi.h" #define TPM_ACPI_DEVICE_NAME "TPM" #define TPM_ACPI_HARDWARE_ID "MSFT0101" SET_DECLARE(tpm_emul_set, struct tpm_emul); SET_DECLARE(tpm_intf_set, struct tpm_intf); SET_DECLARE(tpm_ppi_set, struct tpm_ppi); struct tpm_device { struct vmctx *vm_ctx; struct acpi_device *acpi_dev; struct tpm_emul *emul; void *emul_sc; struct tpm_intf *intf; void *intf_sc; struct tpm_ppi *ppi; void *ppi_sc; }; static int tpm_build_acpi_table(const struct acpi_device *const dev) { const struct tpm_device *const tpm = acpi_device_get_softc(dev); if (tpm->intf->build_acpi_table == NULL) { return (0); } return (tpm->intf->build_acpi_table(tpm->intf_sc, tpm->vm_ctx)); } static int tpm_write_dsdt(const struct acpi_device *const dev) { int error; const struct tpm_device *const tpm = acpi_device_get_softc(dev); const struct tpm_ppi *const ppi = tpm->ppi; /* * packages for returns */ dsdt_line("Name(TPM2, Package(2) {0, 0})"); dsdt_line("Name(TPM3, Package(3) {0, 0, 0})"); if (ppi->write_dsdt_regions) { error = ppi->write_dsdt_regions(tpm->ppi_sc); if (error) { warnx("%s: failed to write ppi dsdt regions\n", __func__); return (error); } } /* * Device Specific Method * Arg0: UUID * Arg1: Revision ID * Arg2: Function Index * Arg3: Arguments */ dsdt_line("Method(_DSM, 4, Serialized)"); dsdt_line("{"); dsdt_indent(1); if (ppi->write_dsdt_dsm) { error = ppi->write_dsdt_dsm(tpm->ppi_sc); if (error) { warnx("%s: failed to write ppi dsdt dsm\n", __func__); return (error); } } dsdt_unindent(1); dsdt_line("}"); return (0); } static const struct acpi_device_emul tpm_acpi_device_emul = { .name = TPM_ACPI_DEVICE_NAME, .hid = TPM_ACPI_HARDWARE_ID, .build_table = tpm_build_acpi_table, .write_dsdt = tpm_write_dsdt, }; void tpm_device_destroy(struct tpm_device *const dev) { if (dev == NULL) return; if (dev->ppi != NULL && dev->ppi->deinit != NULL) dev->ppi->deinit(dev->ppi_sc); if (dev->intf != NULL && dev->intf->deinit != NULL) dev->intf->deinit(dev->intf_sc); if (dev->emul != NULL && dev->emul->deinit != NULL) dev->emul->deinit(dev->emul_sc); acpi_device_destroy(dev->acpi_dev); free(dev); } int tpm_device_create(struct tpm_device **const new_dev, struct vmctx *const vm_ctx, nvlist_t *const nvl) { struct tpm_device *dev = NULL; struct tpm_emul **ppemul; struct tpm_intf **ppintf; struct tpm_ppi **pp_ppi; const char *value; int error; if (new_dev == NULL || vm_ctx == NULL) { error = EINVAL; goto err_out; } set_config_value_node_if_unset(nvl, "intf", "crb"); set_config_value_node_if_unset(nvl, "ppi", "qemu"); value = get_config_value_node(nvl, "version"); assert(value != NULL); if (strcmp(value, "2.0")) { warnx("%s: unsupported tpm version %s", __func__, value); error = EINVAL; goto err_out; } dev = calloc(1, sizeof(*dev)); if (dev == NULL) { error = ENOMEM; goto err_out; } dev->vm_ctx = vm_ctx; error = acpi_device_create(&dev->acpi_dev, dev, dev->vm_ctx, &tpm_acpi_device_emul); if (error) goto err_out; value = get_config_value_node(nvl, "type"); assert(value != NULL); SET_FOREACH(ppemul, tpm_emul_set) { if (strcmp(value, (*ppemul)->name)) continue; dev->emul = *ppemul; break; } if (dev->emul == NULL) { warnx("TPM emulation \"%s\" not found", value); error = EINVAL; goto err_out; } if (dev->emul->init) { error = dev->emul->init(&dev->emul_sc, nvl); if (error) goto err_out; } value = get_config_value_node(nvl, "intf"); SET_FOREACH(ppintf, tpm_intf_set) { if (strcmp(value, (*ppintf)->name)) { continue; } dev->intf = *ppintf; break; } if (dev->intf == NULL) { warnx("TPM interface \"%s\" not found", value); error = EINVAL; goto err_out; } if (dev->intf->init) { error = dev->intf->init(&dev->intf_sc, dev->emul, dev->emul_sc, dev->acpi_dev); if (error) goto err_out; } value = get_config_value_node(nvl, "ppi"); SET_FOREACH(pp_ppi, tpm_ppi_set) { if (strcmp(value, (*pp_ppi)->name)) { continue; } dev->ppi = *pp_ppi; break; } if (dev->ppi == NULL) { warnx("TPM PPI \"%s\" not found\n", value); error = EINVAL; goto err_out; } if (dev->ppi->init) { error = dev->ppi->init(&dev->ppi_sc); if (error) goto err_out; } *new_dev = dev; return (0); err_out: tpm_device_destroy(dev); return (error); } static struct tpm_device *lpc_tpm; int init_tpm(struct vmctx *ctx) { nvlist_t *nvl; int error; nvl = find_config_node("tpm"); if (nvl == NULL) return (0); error = tpm_device_create(&lpc_tpm, ctx, nvl); if (error) { warnx("%s: unable to create a TPM device (%d)", __func__, error); return (error); } return (0); } /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2023 Beckhoff Automation GmbH & Co. KG * Author: Corvin Köhne */ #pragma once #include #include "config.h" struct tpm_device; int tpm_device_create(struct tpm_device **new_dev, struct vmctx *vm_ctx, nvlist_t *nvl); void tpm_device_destroy(struct tpm_device *dev); int init_tpm(struct vmctx *ctx); /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2023 Beckhoff Automation GmbH & Co. KG * Author: Corvin Köhne */ #pragma once #include #include "config.h" struct tpm_device; struct tpm_emul { const char *name; int (*init)(void **sc, nvlist_t *nvl); void (*deinit)(void *sc); int (*execute_cmd)(void *sc, void *cmd, uint32_t cmd_size, void *rsp, uint32_t rsp_size); }; #define TPM_EMUL_SET(x) DATA_SET(tpm_emul_set, x) /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2023 Beckhoff Automation GmbH & Co. KG * Author: Corvin Köhne */ #include #include #include #include #ifdef __FreeBSD__ #include #else #include #endif #include #include #include "config.h" #include "tpm_device.h" #include "tpm_emul.h" struct tpm_passthru { int fd; }; struct tpm_resp_hdr { uint16_t tag; uint32_t len; uint32_t errcode; } __packed; static int tpm_passthru_init(void **sc, nvlist_t *nvl) { #ifndef __FreeBSD__ /* * Until illumos has a TPM 2.0 driver, we can't finish plumbing the TPM * pass-through. */ errx(4, "TPM pass-through devices are not yet supported on illumos"); #else struct tpm_passthru *tpm; const char *path; tpm = calloc(1, sizeof(struct tpm_passthru)); if (tpm == NULL) { warnx("%s: failed to allocate tpm passthru", __func__); return (ENOMEM); } path = get_config_value_node(nvl, "path"); tpm->fd = open(path, O_RDWR); if (tpm->fd < 0) { warnx("%s: unable to open tpm device \"%s\"", __func__, path); return (ENOENT); } *sc = tpm; #endif /* __FreeBSD__ */ return (0); } static int tpm_passthru_execute_cmd(void *sc, void *cmd, uint32_t cmd_size, void *rsp, uint32_t rsp_size) { struct tpm_passthru *tpm; ssize_t len; if (rsp_size < (ssize_t)sizeof(struct tpm_resp_hdr)) { warn("%s: rsp_size of %u is too small", __func__, rsp_size); return (EINVAL); } tpm = sc; len = write(tpm->fd, cmd, cmd_size); if (len != cmd_size) { warn("%s: cmd write failed (bytes written: %zd / %d)", __func__, len, cmd_size); return (EFAULT); } len = read(tpm->fd, rsp, rsp_size); if (len < (ssize_t)sizeof(struct tpm_resp_hdr)) { warn("%s: rsp read failed (bytes read: %zd / %d)", __func__, len, rsp_size); return (EFAULT); } return (0); } static void tpm_passthru_deinit(void *sc) { struct tpm_passthru *tpm; tpm = sc; if (tpm == NULL) return; if (tpm->fd >= 0) close(tpm->fd); free(tpm); } static const struct tpm_emul tpm_emul_passthru = { .name = "passthru", .init = tpm_passthru_init, .deinit = tpm_passthru_deinit, .execute_cmd = tpm_passthru_execute_cmd, }; TPM_EMUL_SET(tpm_emul_passthru); /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2024 Hans Rosenfeld * Author: Hans Rosenfeld */ #include #include #include #include #include #include #ifdef __FreeBSD__ #include #else #include #endif #include #include #include #include #include "config.h" #include "tpm_device.h" #include "tpm_emul.h" struct tpm_swtpm { int fd; }; struct tpm_resp_hdr { uint16_t tag; uint32_t len; uint32_t errcode; } __packed; static int tpm_swtpm_init(void **sc, nvlist_t *nvl) { struct tpm_swtpm *tpm; const char *path; struct sockaddr_un tpm_addr; tpm = calloc(1, sizeof (struct tpm_swtpm)); if (tpm == NULL) { warnx("%s: failed to allocate tpm_swtpm", __func__); return (ENOMEM); } path = get_config_value_node(nvl, "path"); if (path == NULL) { warnx("%s: no socket path specified", __func__); return (ENOENT); } tpm->fd = socket(PF_UNIX, SOCK_STREAM | SOCK_CLOEXEC, 0); if (tpm->fd < 0) { warnx("%s: unable to open tpm socket", __func__); return (ENOENT); } bzero(&tpm_addr, sizeof (tpm_addr)); tpm_addr.sun_family = AF_UNIX; strlcpy(tpm_addr.sun_path, path, sizeof (tpm_addr.sun_path) - 1); if (connect(tpm->fd, (struct sockaddr *)&tpm_addr, sizeof (tpm_addr)) == -1) { warnx("%s: unable to connect to tpm socket \"%s\"", __func__, path); return (ENOENT); } *sc = tpm; return (0); } static int tpm_swtpm_execute_cmd(void *sc, void *cmd, uint32_t cmd_size, void *rsp, uint32_t rsp_size) { struct tpm_swtpm *tpm; ssize_t len; if (rsp_size < (ssize_t)sizeof(struct tpm_resp_hdr)) { warn("%s: rsp_size of %u is too small", __func__, rsp_size); return (EINVAL); } tpm = sc; #ifdef __FreeBSD__ len = send(tpm->fd, cmd, cmd_size, MSG_NOSIGNAL|MSG_DONTWAIT); if (len == -1) err(1, "%s: cmd send failed, is swtpm running?", __func__); if (len != cmd_size) { warn("%s: cmd write failed (bytes written: %zd / %d)", __func__, len, cmd_size); return (EFAULT); } len = recv(tpm->fd, rsp, rsp_size, 0); if (len == -1) err(1, "%s: rsp recv failed, is swtpm running?", __func__); if (len < (ssize_t)sizeof(struct tpm_resp_hdr)) { warn("%s: rsp read failed (bytes read: %zd / %d)", __func__, len, rsp_size); return (EFAULT); } #else while (cmd_size > 0) { len = send(tpm->fd, cmd, cmd_size, MSG_NOSIGNAL|MSG_DONTWAIT); if (len == -1) { if (errno == EINTR) continue; err(1, "%s: cmd send failed, is swtpm running?", __func__); } cmd += len; cmd_size -= len; } size_t buflen = rsp_size; size_t rcvd = 0; while (buflen > 0 && rcvd < sizeof (struct tpm_resp_hdr)) { len = recv(tpm->fd, rsp, buflen, 0); if (len == -1) { if (errno == EINTR) continue; err(1, "%s: rsp recv failed, is swtpm running?", __func__); } if (len == 0) break; rsp += len; buflen -= len; rcvd += len; } #endif return (0); } static void tpm_swtpm_deinit(void *sc) { struct tpm_swtpm *tpm; tpm = sc; if (tpm == NULL) return; if (tpm->fd >= 0) close(tpm->fd); free(tpm); } static const struct tpm_emul tpm_emul_swtpm = { .name = "swtpm", .init = tpm_swtpm_init, .deinit = tpm_swtpm_deinit, .execute_cmd = tpm_swtpm_execute_cmd, }; TPM_EMUL_SET(tpm_emul_swtpm); /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2022 Beckhoff Automation GmbH & Co. KG * Author: Corvin Köhne */ #pragma once #include #include "acpi_device.h" #include "config.h" #include "tpm_device.h" #include "tpm_emul.h" #define TPM_INTF_TYPE_FIFO_PTP 0x0 #define TPM_INTF_TYPE_CRB 0x1 #define TPM_INTF_TYPE_FIFO_TIS 0xF #define TPM_INTF_VERSION_FIFO 0 #define TPM_INTF_VERSION_CRB 1 #define TPM_INTF_CAP_CRB_DATA_XFER_SIZE_4 0 #define TPM_INTF_CAP_CRB_DATA_XFER_SIZE_8 1 #define TPM_INTF_CAP_CRB_DATA_XFER_SIZE_32 2 #define TPM_INTF_CAP_CRB_DATA_XFER_SIZE_64 3 #define TPM_INTF_SELECTOR_FIFO 0 #define TPM_INTF_SELECTOR_CRB 1 struct tpm_intf { const char *name; int (*init)(void **sc, struct tpm_emul *emul, void *emul_sc, struct acpi_device *acpi_dev); void (*deinit)(void *sc); int (*build_acpi_table)(void *sc, struct vmctx *vm_ctx); }; #define TPM_INTF_SET(x) DATA_SET(tpm_intf_set, x) /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2022 Beckhoff Automation GmbH & Co. KG * Author: Corvin Köhne */ #include #include #include #include #include #include #include #include #include #include #include #include #include #ifndef __FreeBSD__ #include #endif #include "basl.h" #include "config.h" #include "mem.h" #include "qemu_fwcfg.h" #include "tpm_device.h" #include "tpm_intf.h" #define TPM_CRB_ADDRESS 0xFED40000 #define TPM_CRB_REGS_SIZE 0x1000 #define TPM_CRB_CONTROL_AREA_ADDRESS \ (TPM_CRB_ADDRESS + offsetof(struct tpm_crb_regs, ctrl_req)) #define TPM_CRB_CONTROL_AREA_SIZE TPM_CRB_REGS_SIZE #define TPM_CRB_DATA_BUFFER_ADDRESS \ (TPM_CRB_ADDRESS + offsetof(struct tpm_crb_regs, data_buffer)) #define TPM_CRB_DATA_BUFFER_SIZE 0xF80 #define TPM_CRB_LOCALITIES_MAX 5 #define TPM_CRB_LOG_AREA_MINIMUM_SIZE (64 * 1024) #define TPM_CRB_LOG_AREA_FWCFG_NAME "etc/tpm/log" #define TPM_CRB_INTF_NAME "crb" struct tpm_crb_regs { union tpm_crb_reg_loc_state { struct { uint32_t tpm_established : 1; uint32_t loc_assigned : 1; uint32_t active_locality : 3; uint32_t _reserved : 2; uint32_t tpm_req_valid_sts : 1; }; uint32_t val; } loc_state; /* 0h */ uint8_t _reserved1[4]; /* 4h */ union tpm_crb_reg_loc_ctrl { struct { uint32_t request_access : 1; uint32_t relinquish : 1; uint32_t seize : 1; uint32_t reset_establishment_bit : 1; }; uint32_t val; } loc_ctrl; /* 8h */ union tpm_crb_reg_loc_sts { struct { uint32_t granted : 1; uint32_t been_seized : 1; }; uint32_t val; } loc_sts; /* Ch */ uint8_t _reserved2[0x20]; /* 10h */ union tpm_crb_reg_intf_id { struct { uint64_t interface_type : 4; uint64_t interface_version : 4; uint64_t cap_locality : 1; uint64_t cap_crb_idle_bypass : 1; uint64_t _reserved1 : 1; uint64_t cap_data_xfer_size_support : 2; uint64_t cap_fifo : 1; uint64_t cap_crb : 1; uint64_t _reserved2 : 2; uint64_t interface_selector : 2; uint64_t intf_sel_lock : 1; uint64_t _reserved3 : 4; uint64_t rid : 8; uint64_t vid : 16; uint64_t did : 16; }; uint64_t val; } intf_id; /* 30h */ union tpm_crb_reg_ctrl_ext { struct { uint32_t clear; uint32_t remaining_bytes; }; uint64_t val; } ctrl_ext; /* 38 */ union tpm_crb_reg_ctrl_req { struct { uint32_t cmd_ready : 1; uint32_t go_idle : 1; }; uint32_t val; } ctrl_req; /* 40h */ union tpm_crb_reg_ctrl_sts { struct { uint32_t tpm_sts : 1; uint32_t tpm_idle : 1; }; uint32_t val; } ctrl_sts; /* 44h */ union tpm_crb_reg_ctrl_cancel { struct { uint32_t cancel : 1; }; uint32_t val; } ctrl_cancel; /* 48h */ union tpm_crb_reg_ctrl_start { struct { uint32_t start : 1; }; uint32_t val; } ctrl_start; /* 4Ch*/ uint32_t int_enable; /* 50h */ uint32_t int_sts; /* 54h */ uint32_t cmd_size; /* 58h */ uint32_t cmd_addr_lo; /* 5Ch */ uint32_t cmd_addr_hi; /* 60h */ uint32_t rsp_size; /* 64h */ uint64_t rsp_addr; /* 68h */ uint8_t _reserved3[0x10]; /* 70h */ uint8_t data_buffer[TPM_CRB_DATA_BUFFER_SIZE]; /* 80h */ } __packed; static_assert(sizeof(struct tpm_crb_regs) == TPM_CRB_REGS_SIZE, "Invalid size of tpm_crb"); #define CRB_CMD_SIZE_READ(regs) (regs.cmd_size) #define CRB_CMD_SIZE_WRITE(regs, val) \ do { \ regs.cmd_size = val; \ } while (0) #define CRB_CMD_ADDR_READ(regs) \ (((uint64_t)regs.cmd_addr_hi << 32) | regs.cmd_addr_lo) #define CRB_CMD_ADDR_WRITE(regs, val) \ do { \ regs.cmd_addr_lo = val & 0xFFFFFFFF; \ regs.cmd_addr_hi = val >> 32; \ } while (0) #define CRB_RSP_SIZE_READ(regs) (regs.rsp_size) #define CRB_RSP_SIZE_WRITE(regs, val) \ do { \ regs.rsp_size = val; \ } while (0) #define CRB_RSP_ADDR_READ(regs) (regs.rsp_addr) #define CRB_RSP_ADDR_WRITE(regs, val) \ do { \ regs.rsp_addr = val; \ } while (0) struct tpm_cmd_hdr { uint16_t tag; uint32_t len; union { uint32_t ordinal; uint32_t errcode; }; } __packed; struct tpm_crb { struct tpm_emul *emul; void *emul_sc; uint8_t tpm_log_area[TPM_CRB_LOG_AREA_MINIMUM_SIZE]; struct tpm_crb_regs regs; pthread_t thread; pthread_mutex_t mutex; pthread_cond_t cond; bool closing; }; static void * tpm_crb_thread(void *const arg) { struct tpm_crb *const crb = arg; pthread_mutex_lock(&crb->mutex); for (;;) { /* * We're releasing the lock after wake up. Therefore, we have to * check the closing condition before and after going to sleep. */ if (crb->closing) break; pthread_cond_wait(&crb->cond, &crb->mutex); if (crb->closing) break; const uint64_t cmd_addr = CRB_CMD_ADDR_READ(crb->regs); const uint64_t rsp_addr = CRB_RSP_ADDR_READ(crb->regs); const uint32_t cmd_size = CRB_CMD_SIZE_READ(crb->regs); const uint32_t rsp_size = CRB_RSP_SIZE_READ(crb->regs); if ((cmd_addr < TPM_CRB_DATA_BUFFER_ADDRESS) || (cmd_size < sizeof (struct tpm_cmd_hdr)) || (cmd_size > TPM_CRB_DATA_BUFFER_SIZE) || (cmd_addr + cmd_size > TPM_CRB_DATA_BUFFER_ADDRESS + TPM_CRB_DATA_BUFFER_SIZE)) { warnx("%s: invalid cmd [%16lx/%8x] outside of TPM " "buffer", __func__, cmd_addr, cmd_size); break; } if ((rsp_addr < TPM_CRB_DATA_BUFFER_ADDRESS) || (rsp_size < sizeof (struct tpm_cmd_hdr)) || (rsp_size > TPM_CRB_DATA_BUFFER_SIZE) || (rsp_addr + rsp_size > TPM_CRB_DATA_BUFFER_ADDRESS + TPM_CRB_DATA_BUFFER_SIZE)) { warnx("%s: invalid rsp [%16lx/%8x] outside of TPM " "buffer", __func__, rsp_addr, rsp_size); break; } const uint64_t cmd_off = cmd_addr - TPM_CRB_DATA_BUFFER_ADDRESS; const uint64_t rsp_off = rsp_addr - TPM_CRB_DATA_BUFFER_ADDRESS; if (cmd_off > TPM_CRB_DATA_BUFFER_SIZE || cmd_off + cmd_size > TPM_CRB_DATA_BUFFER_SIZE || rsp_off > TPM_CRB_DATA_BUFFER_SIZE || rsp_off + rsp_size > TPM_CRB_DATA_BUFFER_SIZE) { warnx( "%s: invalid cmd [%16lx, %16lx] --> [%16lx, %16lx]\n\r", __func__, cmd_addr, cmd_addr + cmd_size, rsp_addr, rsp_addr + rsp_size); break; } uint8_t cmd[TPM_CRB_DATA_BUFFER_SIZE]; memcpy(cmd, crb->regs.data_buffer, TPM_CRB_DATA_BUFFER_SIZE); /* * Do a basic sanity check of the TPM request header. We'll need * the TPM request length for execute_cmd() below. */ struct tpm_cmd_hdr *req = (struct tpm_cmd_hdr *)&cmd[cmd_off]; if (be32toh(req->len) < sizeof (struct tpm_cmd_hdr) || be32toh(req->len) > cmd_size) { warnx("%s: invalid TPM request header", __func__); break; } /* * A TPM command can take multiple seconds to execute. As we've * copied all required values and buffers at this point, we can * release the mutex. */ pthread_mutex_unlock(&crb->mutex); /* * The command response buffer interface uses a single buffer * for sending a command to and receiving a response from the * tpm. To avoid reading old data from the command buffer which * might be a security issue, we zero out the command buffer * before writing the response into it. The rsp_size parameter * is controlled by the guest and it's not guaranteed that the * response has a size of rsp_size (e.g. if the tpm returned an * error, the response would have a different size than * expected). For that reason, use a second buffer for the * response. */ uint8_t rsp[TPM_CRB_DATA_BUFFER_SIZE] = { 0 }; (void) crb->emul->execute_cmd(crb->emul_sc, req, be32toh(req->len), &rsp[rsp_off], rsp_size); pthread_mutex_lock(&crb->mutex); memset(crb->regs.data_buffer, 0, TPM_CRB_DATA_BUFFER_SIZE); memcpy(&crb->regs.data_buffer[rsp_off], &rsp[rsp_off], rsp_size); crb->regs.ctrl_start.start = false; } pthread_mutex_unlock(&crb->mutex); return (NULL); } static int tpm_crb_mmiocpy(void *const dst, void *const src, const int size) { if (!(size == 1 || size == 2 || size == 4 || size == 8)) return (EINVAL); memcpy(dst, src, size); return (0); } static int tpm_crb_mem_handler(struct vcpu *vcpu __unused, const int dir, const uint64_t addr, const int size, uint64_t *const val, void *const arg1, const long arg2 __unused) { struct tpm_crb *crb; uint8_t *ptr; uint64_t off, shift; int error = 0; if ((addr & (size - 1)) != 0) { warnx("%s: unaligned %s access @ %16lx [size = %x]", __func__, (dir == MEM_F_READ) ? "read" : "write", addr, size); return (EINVAL); } crb = arg1; off = addr - TPM_CRB_ADDRESS; if (off > TPM_CRB_REGS_SIZE || off + size >= TPM_CRB_REGS_SIZE) { return (EINVAL); } shift = 8 * (off & 3); ptr = (uint8_t *)&crb->regs + off; if (dir == MEM_F_READ) { error = tpm_crb_mmiocpy(val, ptr, size); if (error) goto err_out; } else { switch (off & ~0x3) { case offsetof(struct tpm_crb_regs, loc_ctrl): { union tpm_crb_reg_loc_ctrl loc_ctrl; if ((size_t)size > sizeof(loc_ctrl)) goto err_out; *val = *val << shift; tpm_crb_mmiocpy(&loc_ctrl, val, size); if (loc_ctrl.relinquish) { crb->regs.loc_sts.granted = false; crb->regs.loc_state.loc_assigned = false; } else if (loc_ctrl.request_access) { crb->regs.loc_sts.granted = true; crb->regs.loc_state.loc_assigned = true; } break; } case offsetof(struct tpm_crb_regs, ctrl_req): { union tpm_crb_reg_ctrl_req req; if ((size_t)size > sizeof(req)) goto err_out; *val = *val << shift; tpm_crb_mmiocpy(&req, val, size); if (req.cmd_ready && !req.go_idle) { crb->regs.ctrl_sts.tpm_idle = false; } else if (!req.cmd_ready && req.go_idle) { crb->regs.ctrl_sts.tpm_idle = true; } break; } case offsetof(struct tpm_crb_regs, ctrl_cancel): { /* TODO: cancel the tpm command */ warnx( "%s: cancelling a TPM command is not implemented yet", __func__); break; } case offsetof(struct tpm_crb_regs, int_enable): /* No interrupt support. Ignore writes to int_enable. */ break; case offsetof(struct tpm_crb_regs, ctrl_start): { union tpm_crb_reg_ctrl_start start; if ((size_t)size > sizeof(start)) goto err_out; *val = *val << shift; pthread_mutex_lock(&crb->mutex); tpm_crb_mmiocpy(&start, val, size); if (!start.start || crb->regs.ctrl_start.start) { pthread_mutex_unlock(&crb->mutex); break; } crb->regs.ctrl_start.start = true; pthread_cond_signal(&crb->cond); pthread_mutex_unlock(&crb->mutex); break; } case offsetof(struct tpm_crb_regs, cmd_size): case offsetof(struct tpm_crb_regs, cmd_addr_lo): case offsetof(struct tpm_crb_regs, cmd_addr_hi): case offsetof(struct tpm_crb_regs, rsp_size): case offsetof(struct tpm_crb_regs, rsp_addr) ... offsetof(struct tpm_crb_regs, rsp_addr) + 4: case offsetof(struct tpm_crb_regs, data_buffer) ... offsetof(struct tpm_crb_regs, data_buffer) + TPM_CRB_DATA_BUFFER_SIZE / 4: /* * Those fields are used to execute a TPM command. The * crb_thread will access them. For that reason, we have * to acquire the crb mutex in order to write them. */ pthread_mutex_lock(&crb->mutex); error = tpm_crb_mmiocpy(ptr, val, size); pthread_mutex_unlock(&crb->mutex); if (error) goto err_out; break; default: /* * The other fields are either readonly or we do not * support writing them. */ error = EINVAL; goto err_out; } } return (0); err_out: warnx("%s: invalid %s @ %16lx [size = %d]", __func__, dir == MEM_F_READ ? "read" : "write", addr, size); return (error); } static int tpm_crb_modify_mmio_registration(const bool registration, void *const arg1) { struct mem_range crb_mmio = { .name = "crb-mmio", .base = TPM_CRB_ADDRESS, .size = TPM_CRB_LOCALITIES_MAX * TPM_CRB_CONTROL_AREA_SIZE, .flags = MEM_F_RW, .arg1 = arg1, .handler = tpm_crb_mem_handler, }; if (registration) return (register_mem(&crb_mmio)); else return (unregister_mem(&crb_mmio)); } static int tpm_crb_init(void **sc, struct tpm_emul *emul, void *emul_sc, struct acpi_device *acpi_dev) { struct tpm_crb *crb = NULL; int error; assert(sc != NULL); assert(emul != NULL); crb = calloc(1, sizeof(struct tpm_crb)); if (crb == NULL) { warnx("%s: failed to allocate tpm crb", __func__); error = ENOMEM; goto err_out; } memset(crb, 0, sizeof(*crb)); crb->emul = emul; crb->emul_sc = emul_sc; crb->regs.loc_state.tpm_req_valid_sts = true; crb->regs.loc_state.tpm_established = true; crb->regs.intf_id.interface_type = TPM_INTF_TYPE_CRB; crb->regs.intf_id.interface_version = TPM_INTF_VERSION_CRB; crb->regs.intf_id.cap_locality = false; crb->regs.intf_id.cap_crb_idle_bypass = false; crb->regs.intf_id.cap_data_xfer_size_support = TPM_INTF_CAP_CRB_DATA_XFER_SIZE_64; crb->regs.intf_id.cap_fifo = false; crb->regs.intf_id.cap_crb = true; crb->regs.intf_id.interface_selector = TPM_INTF_SELECTOR_CRB; crb->regs.intf_id.intf_sel_lock = false; crb->regs.intf_id.rid = 0; crb->regs.intf_id.vid = 0x1014; /* IBM */ crb->regs.intf_id.did = 0x1014; /* IBM */ crb->regs.ctrl_sts.tpm_idle = true; CRB_CMD_SIZE_WRITE(crb->regs, TPM_CRB_DATA_BUFFER_SIZE); CRB_CMD_ADDR_WRITE(crb->regs, TPM_CRB_DATA_BUFFER_ADDRESS); CRB_RSP_SIZE_WRITE(crb->regs, TPM_CRB_DATA_BUFFER_SIZE); CRB_RSP_ADDR_WRITE(crb->regs, TPM_CRB_DATA_BUFFER_ADDRESS); error = qemu_fwcfg_add_file(TPM_CRB_LOG_AREA_FWCFG_NAME, TPM_CRB_LOG_AREA_MINIMUM_SIZE, crb->tpm_log_area); if (error) { warnx("%s: failed to add fwcfg file", __func__); goto err_out; } error = acpi_device_add_res_fixed_memory32(acpi_dev, false, TPM_CRB_ADDRESS, TPM_CRB_CONTROL_AREA_SIZE); if (error) { warnx("%s: failed to add acpi resources\n", __func__); goto err_out; } error = tpm_crb_modify_mmio_registration(true, crb); if (error) { warnx("%s: failed to register crb mmio", __func__); goto err_out; } error = pthread_mutex_init(&crb->mutex, NULL); if (error) { warnc(error, "%s: failed to init mutex", __func__); goto err_out; } error = pthread_cond_init(&crb->cond, NULL); if (error) { warnc(error, "%s: failed to init cond", __func__); goto err_out; } error = pthread_create(&crb->thread, NULL, tpm_crb_thread, crb); if (error) { warnx("%s: failed to create thread\n", __func__); goto err_out; } pthread_set_name_np(crb->thread, "tpm_intf_crb"); *sc = crb; return (0); err_out: free(crb); return (error); } static void tpm_crb_deinit(void *sc) { struct tpm_crb *crb; int error; if (sc == NULL) { return; } crb = sc; crb->closing = true; pthread_cond_signal(&crb->cond); pthread_join(crb->thread, NULL); pthread_cond_destroy(&crb->cond); pthread_mutex_destroy(&crb->mutex); error = tpm_crb_modify_mmio_registration(false, NULL); assert(error == 0); free(crb); } static int tpm_crb_build_acpi_table(void *sc __unused, struct vmctx *vm_ctx) { struct basl_table *table; BASL_EXEC(basl_table_create(&table, vm_ctx, ACPI_SIG_TPM2, BASL_TABLE_ALIGNMENT)); /* Header */ BASL_EXEC(basl_table_append_header(table, ACPI_SIG_TPM2, 4, 1)); /* Platform Class */ BASL_EXEC(basl_table_append_int(table, 0, 2)); /* Reserved */ BASL_EXEC(basl_table_append_int(table, 0, 2)); /* Control Address */ BASL_EXEC( basl_table_append_int(table, TPM_CRB_CONTROL_AREA_ADDRESS, 8)); /* Start Method == (7) Command Response Buffer */ BASL_EXEC(basl_table_append_int(table, 7, 4)); /* Start Method Specific Parameters */ uint8_t parameters[12] = { 0 }; BASL_EXEC(basl_table_append_bytes(table, parameters, 12)); /* Log Area Minimum Length */ BASL_EXEC( basl_table_append_int(table, TPM_CRB_LOG_AREA_MINIMUM_SIZE, 4)); /* Log Area Start Address */ #ifdef __FreeBSD__ BASL_EXEC( basl_table_append_fwcfg(table, TPM_CRB_LOG_AREA_FWCFG_NAME, 1, 8)); #else BASL_EXEC( basl_table_append_fwcfg(table, (const uint8_t *)TPM_CRB_LOG_AREA_FWCFG_NAME, 1, 8)); #endif BASL_EXEC(basl_table_register_to_rsdt(table)); return (0); } static struct tpm_intf tpm_intf_crb = { .name = TPM_CRB_INTF_NAME, .init = tpm_crb_init, .deinit = tpm_crb_deinit, .build_acpi_table = tpm_crb_build_acpi_table, }; TPM_INTF_SET(tpm_intf_crb); /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2022 Beckhoff Automation GmbH & Co. KG * Author: Corvin Köhne */ #pragma once #include "config.h" struct tpm_ppi { const char *name; int (*init)(void **sc); void (*deinit)(void *sc); int (*write_dsdt_regions)(void *sc); int (*write_dsdt_dsm)(void *sc); }; #define TPM_PPI_SET(x) DATA_SET(tpm_ppi_set, x) /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2022 Beckhoff Automation GmbH & Co. KG * Author: Corvin Köhne */ #include #include #include #include #include #include #include #include #include #include "acpi.h" #include "acpi_device.h" #include "config.h" #include "mem.h" #include "qemu_fwcfg.h" #include "tpm_ppi.h" #define TPM_PPI_ADDRESS 0xFED45000 #define TPM_PPI_SIZE 0x400 #define TPM_PPI_FWCFG_FILE "etc/tpm/config" #define TPM_PPI_QEMU_NAME "qemu" struct tpm_ppi_qemu { uint8_t func[256]; // FUNC uint8_t in; // PPIN uint32_t ip; // PPIP uint32_t response; // PPRP uint32_t request; // PPRQ uint32_t request_parameter; // PPRM uint32_t last_request; // LPPR uint32_t func_ret; // FRET uint8_t _reserved1[0x40]; // RES1 uint8_t next_step; // next_step } __packed; static_assert(sizeof(struct tpm_ppi_qemu) <= TPM_PPI_SIZE, "Wrong size of tpm_ppi_qemu"); struct tpm_ppi_fwcfg { uint32_t ppi_address; uint8_t tpm_version; uint8_t ppi_version; } __packed; static int tpm_ppi_mem_handler(struct vcpu *const vcpu __unused, const int dir, const uint64_t addr, const int size, uint64_t *const val, void *const arg1, const long arg2 __unused) { struct tpm_ppi_qemu *ppi; uint8_t *ptr; uint64_t off; if ((addr & (size - 1)) != 0) { warnx("%s: unaligned %s access @ %16lx [size = %x]", __func__, (dir == MEM_F_READ) ? "read" : "write", addr, size); } ppi = arg1; off = addr - TPM_PPI_ADDRESS; ptr = (uint8_t *)ppi + off; if (off > TPM_PPI_SIZE || off + size > TPM_PPI_SIZE) { return (EINVAL); } assert(size == 1 || size == 2 || size == 4 || size == 8); if (dir == MEM_F_READ) { memcpy(val, ptr, size); } else { memcpy(ptr, val, size); } return (0); } static struct mem_range ppi_mmio = { .name = "ppi-mmio", .base = TPM_PPI_ADDRESS, .size = TPM_PPI_SIZE, .flags = MEM_F_RW, .handler = tpm_ppi_mem_handler, }; static int tpm_ppi_init(void **sc) { struct tpm_ppi_qemu *ppi = NULL; struct tpm_ppi_fwcfg *fwcfg = NULL; int error; #ifdef __FreeBSD__ ppi = calloc(1, TPM_PPI_SIZE); #else // SMATCH: double check that we're allocating correct size: 346 vs 1024 void *ptr = calloc(1, TPM_PPI_SIZE); ppi = (struct tpm_ppi_qemu *)ptr; #endif if (ppi == NULL) { warnx("%s: failed to allocate acpi region for ppi", __func__); error = ENOMEM; goto err_out; } fwcfg = calloc(1, sizeof(struct tpm_ppi_fwcfg)); if (fwcfg == NULL) { warnx("%s: failed to allocate fwcfg item", __func__); error = ENOMEM; goto err_out; } fwcfg->ppi_address = htole32(TPM_PPI_ADDRESS); fwcfg->tpm_version = 2; fwcfg->ppi_version = 1; error = qemu_fwcfg_add_file(TPM_PPI_FWCFG_FILE, sizeof(struct tpm_ppi_fwcfg), fwcfg); if (error) { warnx("%s: failed to add fwcfg file", __func__); goto err_out; } /* * We would just need to create some guest memory for the PPI region. * Sadly, bhyve has a strange memory interface. We can't just add more * memory to the VM. So, create a trap instead which reads and writes to * the ppi region. It's very slow but ppi shouldn't be used frequently. */ ppi_mmio.arg1 = ppi; error = register_mem(&ppi_mmio); if (error) { warnx("%s: failed to create trap for ppi accesses", __func__); goto err_out; } *sc = ppi; return (0); err_out: free(fwcfg); free(ppi); return (error); } static void tpm_ppi_deinit(void *sc) { struct tpm_ppi_qemu *ppi; int error; if (sc == NULL) return; ppi = sc; error = unregister_mem(&ppi_mmio); assert(error == 0); free(ppi); } static int tpm_ppi_write_dsdt_regions(void *sc __unused) { /* * struct tpm_ppi_qemu */ /* * According to qemu the Windows ACPI parser has a bug that DerefOf is * broken for SYSTEM_MEMORY. Due to that bug, qemu uses a dynamic * operation region inside a method. */ dsdt_line("Method(TPFN, 1, Serialized)"); dsdt_line("{"); dsdt_line(" If(LGreaterEqual(Arg0, 0x100))"); dsdt_line(" {"); dsdt_line(" Return(Zero)"); dsdt_line(" }"); dsdt_line( " OperationRegion(TPP1, SystemMemory, Add(0x%8x, Arg0), One)", TPM_PPI_ADDRESS); dsdt_line(" Field(TPP1, ByteAcc, NoLock, Preserve)"); dsdt_line(" {"); dsdt_line(" TPPF, 8,"); dsdt_line(" }"); dsdt_line(" Return(TPPF)"); dsdt_line("}"); dsdt_line("OperationRegion(TPP2, SystemMemory, 0x%8x, 0x%x)", TPM_PPI_ADDRESS + 0x100, 0x5A); dsdt_line("Field(TPP2, AnyAcc, NoLock, Preserve)"); dsdt_line("{"); dsdt_line(" PPIN, 8,"); dsdt_line(" PPIP, 32,"); dsdt_line(" PPRP, 32,"); dsdt_line(" PPRQ, 32,"); dsdt_line(" PPRM, 32,"); dsdt_line(" LPPR, 32,"); dsdt_line("}"); /* * Used for TCG Platform Reset Attack Mitigation */ dsdt_line("OperationRegion(TPP3, SystemMemory, 0x%8x, 1)", TPM_PPI_ADDRESS + sizeof(struct tpm_ppi_qemu)); dsdt_line("Field(TPP3, ByteAcc, NoLock, Preserve)"); dsdt_line("{"); dsdt_line(" MOVV, 8,"); dsdt_line("}"); return (0); } static int tpm_ppi_write_dsdt_dsm(void *sc __unused) { /* * Physical Presence Interface */ dsdt_line( "If(LEqual(Arg0, ToUUID(\"3DDDFAA6-361B-4EB4-A424-8D10089D1653\"))) /* UUID */"); dsdt_line("{"); /* * Function 0 - _DSM Query Function * Arguments: * Empty Package * Return: * Buffer - Index field of supported functions */ dsdt_line(" If(LEqual(Arg2, 0)) /* Function */"); dsdt_line(" {"); dsdt_line(" Return(Buffer(0x02)"); dsdt_line(" {"); dsdt_line(" 0xFF, 0x01"); dsdt_line(" })"); dsdt_line(" }"); /* * Function 1 - Get Physical Presence Interface Version * Arguments: * Empty Package * Return: * String - Supported Physical Presence Interface revision */ dsdt_line(" If(LEqual(Arg2, 1)) /* Function */"); dsdt_line(" {"); dsdt_line(" Return(\"1.3\")"); dsdt_line(" }"); /* * Function 2 - Submit TPM Operation Request to Pre-OS Environment * !!!DEPRECATED BUT MANDATORY!!! * Arguments: * Integer - Operation Value of the Request * Return: * Integer - Function Return Code * 0 - Success * 1 - Operation Value of the Request Not Supported * 2 - General Failure */ dsdt_line(" If(LEqual(Arg2, 2)) /* Function */"); dsdt_line(" {"); dsdt_line(" Store(DerefOf(Index(Arg3, 0)), Local0)"); dsdt_line(" Store(TPFN(Local0), Local1)"); dsdt_line(" If (LEqual(And(Local1, 7), 0))"); dsdt_line(" {"); dsdt_line(" Return(1)"); dsdt_line(" }"); dsdt_line(" Store(Local0, PPRQ)"); dsdt_line(" Store(0, PPRM)"); dsdt_line(" Return(0)"); dsdt_line(" }"); /* * Function 3 - Get Pending TPM Operation Request By the OS * Arguments: * Empty Package * Return: * Package * Integer 1 - Function Return Code * 0 - Success * 1 - General Failure * Integer 2 - Pending operation requested by the OS * 0 - None * >0 - Operation Value of the Pending Request * Integer 3 - Optional argument to pending operation requested by * the OS * 0 - None * >0 - Argument of the Pending Request */ dsdt_line(" If(LEqual(Arg2, 3)) /* Function */"); dsdt_line(" {"); dsdt_line(" If(LEqual(Arg1, 1)) /* Revision */"); dsdt_line(" {"); dsdt_line(" Store(PPRQ, Index(TPM2, 1))"); dsdt_line(" Return(TPM2)"); dsdt_line(" }"); dsdt_line(" If(LEqual(Arg1, 2)) /* Revision */"); dsdt_line(" {"); dsdt_line(" Store(PPRQ, Index(TPM3, 1))"); dsdt_line(" Store(PPRM, Index(TPM3, 2))"); dsdt_line(" Return(TPM3)"); dsdt_line(" }"); dsdt_line(" }"); /* * Function 4 - Get Platform-Specific Action to Transition to Pre-OS * Environment * Arguments: * Empty Package * Return: * Integer - Action that the OS should take to transition to the * pre-OS environment for execution of a requested operation * 0 - None * 1 - Shutdown * 2 - Reboot * 3 - OS Vendor-specific */ dsdt_line(" If(LEqual(Arg2, 4)) /* Function */"); dsdt_line(" {"); dsdt_line(" Return(2)"); dsdt_line(" }"); /* * Function 5 - Return TPM Operation Response to OS Environment * Arguments: * Empty Package * Return: * Package * Integer 1 - Function Return Code * 0 - Success * 1 - General Failure * Integer 2 - Most recent operation request * 0 - None * >0 - Operation value of the most recent request * Integer 3 - Response to the most recent operation request * 0 - Success * 0x00000001..0x000000FF - Corresponding TPM error code * 0xFFFFFFF0 - User Abort or timeout of dialog * 0xFFFFFFF1 - firmware failure */ dsdt_line(" If(LEqual(Arg2, 5)) /* Function */"); dsdt_line(" {"); dsdt_line(" Store(LPPR, Index(TPM3, 1))"); dsdt_line(" Store(PPRP, Index(TPM3, 2))"); dsdt_line(" Return(TPM3)"); dsdt_line(" }"); /* * Function 6 - Submit preferred user language * !!!DEPRECATED BUT MANDATORY!!! * Arguments: * Package * String - Preferred language code * Return: * Integer * 3 - Not implemented */ dsdt_line(" If(LEqual(Arg2, 6)) /* Function */"); dsdt_line(" {"); dsdt_line(" Return(3)"); dsdt_line(" }"); /* * Function 7 - Submit TPM Operation Request to Pre-OS Environment 2 * Arguments: * Package * Integer 1 - Operation Value of the Request * Integer 2 - Argument for Operation * Return: * Integer - Function Return Code * 0 - Success * 1 - Not Implemented * 2 - General Failure * 3 - Operation blocked by current firmware settings */ dsdt_line(" If(LEqual(Arg2, 7)) /* Function */"); dsdt_line(" {"); dsdt_line(" Store(DerefOf(Index(Arg3, 0)), Local0)"); dsdt_line(" Store(TPFN(Local0), Local1)"); dsdt_line(" If (LEqual(And(Local1, 7), 0)) /* Not Implemented */"); dsdt_line(" {"); dsdt_line(" Return(1)"); dsdt_line(" }"); dsdt_line(" If (LEqual(And(Local1, 7), 2)) /* Blocked */ "); dsdt_line(" {"); dsdt_line(" Return(3)"); dsdt_line(" }"); dsdt_line(" If(LEqual(Arg1, 1)) /* Revision */"); dsdt_line(" {"); dsdt_line(" Store(Local0, PPRQ)"); dsdt_line(" Store(0, PPRM)"); dsdt_line(" }"); dsdt_line(" If(LEqual(Arg1, 2)) /* Revision */"); dsdt_line(" {"); dsdt_line(" Store(Local0, PPRQ)"); dsdt_line(" Store(DerefOf(Index(Arg3, 1)), PPRM)"); dsdt_line(" }"); dsdt_line(" Return(0)"); dsdt_line(" }"); /* * Function 8 - Get User Confirmation Status for Operation * Arguments: * Package * Integer - Operation Value that may need user confirmation * Return: * Integer - Function Return Code * 0 - Not implemented * 1 - Firmware only * 2 - Blocked for OS by firmware configuration * 3 - Allowed and physically present user required * 4 - Allowed and physically present user not required */ dsdt_line(" If(LEqual(Arg2, 8)) /* Function */"); dsdt_line(" {"); dsdt_line(" Store(DerefOf(Index(Arg3, 0)), Local0)"); dsdt_line(" Store(TPFN(Local0), Local1)"); dsdt_line(" Return(And(Local1, 7))"); dsdt_line(" }"); /* * Unknown function */ dsdt_line(" Return(Buffer(1)"); dsdt_line(" {"); dsdt_line(" 0x00"); dsdt_line(" })"); dsdt_line("}"); /* * TCG Platform Reset Attack Mitigation */ dsdt_line( "If(LEqual(Arg0, ToUUID(\"376054ED-CC13-4675-901C-4756D7F2D45D\"))) /* UUID */"); dsdt_line("{"); /* * Function 0 - _DSM Query Function * Arguments: * Empty Package * Return: * Buffer - Index field of supported functions */ dsdt_line(" If(LEqual(Arg2, 0)) /* Function */"); dsdt_line(" {"); dsdt_line(" Return(Buffer(1)"); dsdt_line(" {"); dsdt_line(" 0x03"); dsdt_line(" })"); dsdt_line(" }"); /* * Function 1 - Memory Clear * Arguments: * Package * Integer - Operation Value of the Request * Return: * Integer - Function Return Code * 0 - Success * 1 - General Failure */ dsdt_line(" If(LEqual(Arg2, 1)) /* Function */"); dsdt_line(" {"); dsdt_line(" Store(DerefOf(Index(Arg3, 0)), Local0)"); dsdt_line(" Store(Local0, MOVV)"); dsdt_line(" Return(0)"); dsdt_line(" }"); dsdt_line("}"); return (0); } static struct tpm_ppi tpm_ppi_qemu = { .name = TPM_PPI_QEMU_NAME, .init = tpm_ppi_init, .deinit = tpm_ppi_deinit, .write_dsdt_regions = tpm_ppi_write_dsdt_regions, .write_dsdt_dsm = tpm_ppi_write_dsdt_dsm, }; TPM_PPI_SET(tpm_ppi_qemu); /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2012 NetApp, Inc. * Copyright (c) 2013 Neel Natu * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include #include #include #ifndef WITHOUT_CAPSICUM #include #endif #include #include #include #include #include #include #include #include #include #ifndef __FreeBSD__ #include #include #endif #include "debug.h" #include "mevent.h" #include "uart_backend.h" #include "uart_emul.h" struct ttyfd { bool opened; int rfd; /* fd for reading */ int wfd; /* fd for writing, may be == rfd */ }; #ifndef __FreeBSD__ struct sockfd { bool sock; int clifd; /* console client unix domain socket */ int servfd; /* console server unix domain socket */ struct mevent *servmev; /* mevent for server socket */ void (*drain)(int, enum ev_type, void *); void *drainarg; }; #endif #define FIFOSZ 16 struct fifo { uint8_t buf[FIFOSZ]; int rindex; /* index to read from */ int windex; /* index to write to */ int num; /* number of characters in the fifo */ int size; /* size of the fifo */ }; struct uart_softc { struct ttyfd tty; #ifndef __FreeBSD__ struct sockfd usc_sock; #endif struct fifo rxfifo; struct mevent *mev; pthread_mutex_t mtx; }; static bool uart_stdio; /* stdio in use for i/o */ static struct termios tio_stdio_orig; static void ttyclose(void) { tcsetattr(STDIN_FILENO, TCSANOW, &tio_stdio_orig); } static void ttyopen(struct ttyfd *tf) { struct termios orig, new; tcgetattr(tf->rfd, &orig); new = orig; cfmakeraw(&new); new.c_cflag |= CLOCAL; tcsetattr(tf->rfd, TCSANOW, &new); if (uart_stdio) { tio_stdio_orig = orig; atexit(ttyclose); } raw_stdio = 1; } static int ttyread(struct ttyfd *tf) { unsigned char rb; if (read(tf->rfd, &rb, 1) == 1) return (rb); else return (-1); } static void ttywrite(struct ttyfd *tf, unsigned char wb) { (void)write(tf->wfd, &wb, 1); } #ifndef __FreeBSD__ static void sockwrite(struct uart_softc *sc, unsigned char wb) { (void) write(sc->usc_sock.clifd, &wb, 1); } #endif static bool rxfifo_available(struct uart_softc *sc) { return (sc->rxfifo.num < sc->rxfifo.size); } int uart_rxfifo_getchar(struct uart_softc *sc) { struct fifo *fifo; int c, error, wasfull; wasfull = 0; fifo = &sc->rxfifo; if (fifo->num > 0) { if (!rxfifo_available(sc)) wasfull = 1; c = fifo->buf[fifo->rindex]; fifo->rindex = (fifo->rindex + 1) % fifo->size; fifo->num--; if (wasfull) { if (sc->tty.opened) { error = mevent_enable(sc->mev); assert(error == 0); } #ifndef __FreeBSD__ if (sc->usc_sock.sock && sc->usc_sock.clifd != -1) { error = mevent_enable(sc->mev); assert(error == 0); } #endif /* __FreeBSD__ */ } return (c); } else return (-1); } int uart_rxfifo_numchars(struct uart_softc *sc) { return (sc->rxfifo.num); } static int rxfifo_putchar(struct uart_softc *sc, uint8_t ch) { struct fifo *fifo; int error; fifo = &sc->rxfifo; if (fifo->num < fifo->size) { fifo->buf[fifo->windex] = ch; fifo->windex = (fifo->windex + 1) % fifo->size; fifo->num++; if (!rxfifo_available(sc)) { if (sc->tty.opened) { /* * Disable mevent callback if the FIFO is full. */ error = mevent_disable(sc->mev); assert(error == 0); } #ifndef __FreeBSD__ if (sc->usc_sock.sock && sc->usc_sock.clifd != -1) { /* * Disable mevent callback if the FIFO is full. */ error = mevent_disable(sc->mev); assert(error == 0); } #endif /* __FreeBSD__ */ } return (0); } else return (-1); } void uart_rxfifo_drain(struct uart_softc *sc, bool loopback) { int ch; if (loopback) { (void)ttyread(&sc->tty); } else { while (rxfifo_available(sc) && ((ch = ttyread(&sc->tty)) != -1)) rxfifo_putchar(sc, ch); } } #ifndef __FreeBSD__ void uart_rxfifo_sock_drain(struct uart_softc *sc, bool loopback) { int ch; if (loopback) { (void) read(sc->usc_sock.clifd, &ch, 1); } else { bool err_close = false; while (rxfifo_available(sc)) { int res; res = read(sc->usc_sock.clifd, &ch, 1); if (res == 0) { err_close = true; break; } else if (res == -1) { if (errno != EAGAIN && errno != EINTR) { err_close = true; } break; } rxfifo_putchar(sc, ch); } if (err_close) { (void) fprintf(stderr, "uart: closing client conn\n"); (void) shutdown(sc->usc_sock.clifd, SHUT_RDWR); mevent_delete_close(sc->mev); sc->mev = NULL; sc->usc_sock.clifd = -1; } } } #endif int uart_rxfifo_putchar(struct uart_softc *sc, uint8_t ch, bool loopback) { if (loopback) { return (rxfifo_putchar(sc, ch)); } else if (sc->tty.opened) { ttywrite(&sc->tty, ch); return (0); #ifndef __FreeBSD__ } else if (sc->usc_sock.sock) { sockwrite(sc, ch); return (0); #endif } else { /* Drop on the floor. */ return (0); } } void uart_rxfifo_reset(struct uart_softc *sc, int size) { char flushbuf[32]; struct fifo *fifo; ssize_t nread; int error; fifo = &sc->rxfifo; bzero(fifo, sizeof(struct fifo)); fifo->size = size; if (sc->tty.opened) { /* * Flush any unread input from the tty buffer. */ while (1) { nread = read(sc->tty.rfd, flushbuf, sizeof(flushbuf)); if (nread != sizeof(flushbuf)) break; } /* * Enable mevent to trigger when new characters are available * on the tty fd. */ error = mevent_enable(sc->mev); assert(error == 0); } #ifndef __FreeBSD__ if (sc->usc_sock.sock && sc->usc_sock.clifd != -1) { /* Flush any unread input from the socket buffer. */ do { nread = read(sc->usc_sock.clifd, flushbuf, sizeof (flushbuf)); } while (nread == sizeof (flushbuf)); /* Enable mevent to trigger when new data available on sock */ error = mevent_enable(sc->mev); assert(error == 0); } #endif /* __FreeBSD__ */ } int uart_rxfifo_size(struct uart_softc *sc __unused) { return (FIFOSZ); } #ifdef BHYVE_SNAPSHOT int uart_rxfifo_snapshot(struct uart_softc *sc, struct vm_snapshot_meta *meta) { int ret; SNAPSHOT_VAR_OR_LEAVE(sc->rxfifo.rindex, meta, ret, done); SNAPSHOT_VAR_OR_LEAVE(sc->rxfifo.windex, meta, ret, done); SNAPSHOT_VAR_OR_LEAVE(sc->rxfifo.num, meta, ret, done); SNAPSHOT_VAR_OR_LEAVE(sc->rxfifo.size, meta, ret, done); SNAPSHOT_BUF_OR_LEAVE(sc->rxfifo.buf, sizeof(sc->rxfifo.buf), meta, ret, done); done: return (ret); } #endif static int uart_stdio_backend(struct uart_softc *sc) { #ifndef WITHOUT_CAPSICUM cap_rights_t rights; cap_ioctl_t cmds[] = { TIOCGETA, TIOCSETA, TIOCGWINSZ }; #endif if (uart_stdio) return (-1); sc->tty.rfd = STDIN_FILENO; sc->tty.wfd = STDOUT_FILENO; sc->tty.opened = true; if (fcntl(sc->tty.rfd, F_SETFL, O_NONBLOCK) != 0) return (-1); if (fcntl(sc->tty.wfd, F_SETFL, O_NONBLOCK) != 0) return (-1); #ifndef WITHOUT_CAPSICUM cap_rights_init(&rights, CAP_EVENT, CAP_IOCTL, CAP_READ); if (caph_rights_limit(sc->tty.rfd, &rights) == -1) errx(EX_OSERR, "Unable to apply rights for sandbox"); if (caph_ioctls_limit(sc->tty.rfd, cmds, nitems(cmds)) == -1) errx(EX_OSERR, "Unable to apply rights for sandbox"); #endif uart_stdio = true; return (0); } static int uart_tty_backend(struct uart_softc *sc, const char *path) { #ifndef WITHOUT_CAPSICUM cap_rights_t rights; cap_ioctl_t cmds[] = { TIOCGETA, TIOCSETA, TIOCGWINSZ }; #endif int fd; fd = open(path, O_RDWR | O_NONBLOCK); if (fd < 0) return (-1); if (!isatty(fd)) { close(fd); return (-1); } sc->tty.rfd = sc->tty.wfd = fd; sc->tty.opened = true; #ifndef WITHOUT_CAPSICUM cap_rights_init(&rights, CAP_EVENT, CAP_IOCTL, CAP_READ, CAP_WRITE); if (caph_rights_limit(fd, &rights) == -1) errx(EX_OSERR, "Unable to apply rights for sandbox"); if (caph_ioctls_limit(fd, cmds, nitems(cmds)) == -1) errx(EX_OSERR, "Unable to apply rights for sandbox"); #endif return (0); } #ifndef __FreeBSD__ static void uart_sock_accept(int fd, enum ev_type ev, void *arg) { struct uart_softc *sc = arg; int connfd; connfd = accept(sc->usc_sock.servfd, NULL, NULL); if (connfd == -1) { return; } /* * Do client connection management under protection of the softc lock * to avoid racing with concurrent UART events. */ pthread_mutex_lock(&sc->mtx); if (sc->usc_sock.clifd != -1) { /* we're already handling a client */ (void) fprintf(stderr, "uart: unexpected client conn\n"); (void) shutdown(connfd, SHUT_RDWR); (void) close(connfd); } else { if (fcntl(connfd, F_SETFL, O_NONBLOCK) < 0) { perror("uart: fcntl(O_NONBLOCK)"); (void) shutdown(connfd, SHUT_RDWR); (void) close(connfd); } else { sc->usc_sock.clifd = connfd; sc->mev = mevent_add(sc->usc_sock.clifd, EVF_READ, sc->usc_sock.drain, sc->usc_sock.drainarg); } } pthread_mutex_unlock(&sc->mtx); } static int init_sock(const char *path) { int servfd; struct sockaddr_un servaddr; bzero(&servaddr, sizeof (servaddr)); servaddr.sun_family = AF_UNIX; if (strlcpy(servaddr.sun_path, path, sizeof (servaddr.sun_path)) >= sizeof (servaddr.sun_path)) { (void) fprintf(stderr, "uart: path '%s' too long\n", path); return (-1); } if ((servfd = socket(AF_UNIX, SOCK_STREAM, 0)) == -1) { (void) fprintf(stderr, "uart: socket() error - %s\n", strerror(errno)); return (-1); } (void) unlink(servaddr.sun_path); if (bind(servfd, (struct sockaddr *)&servaddr, sizeof (servaddr)) == -1) { (void) fprintf(stderr, "uart: bind() error - %s\n", strerror(errno)); goto out; } if (listen(servfd, 1) == -1) { (void) fprintf(stderr, "uart: listen() error - %s\n", strerror(errno)); goto out; } return (servfd); out: (void) unlink(servaddr.sun_path); (void) close(servfd); return (-1); } static int uart_sock_backend(struct uart_softc *sc, const char *inopts, void (*drain)(int, enum ev_type, void *), void *drainarg) { char *opts, *tofree; char *opt; char *nextopt; char *path = NULL; if (strncmp(inopts, "socket,", 7) != 0) { return (-1); } if ((opts = strdup(inopts + 7)) == NULL) { return (-1); } tofree = nextopt = opts; for (opt = strsep(&nextopt, ","); opt != NULL; opt = strsep(&nextopt, ",")) { if (path == NULL && *opt == '/') { path = opt; continue; } /* * XXX check for server and client options here. For now, * everything is a server */ free(tofree); return (-1); } sc->usc_sock.clifd = -1; if ((sc->usc_sock.servfd = init_sock(path)) == -1) { free(tofree); return (-1); } sc->usc_sock.sock = true; sc->tty.rfd = sc->tty.wfd = -1; sc->usc_sock.servmev = mevent_add(sc->usc_sock.servfd, EVF_READ, uart_sock_accept, sc); assert(sc->usc_sock.servmev != NULL); sc->usc_sock.drain = drain; sc->usc_sock.drainarg = drainarg; free(tofree); return (0); } #endif /* not __FreeBSD__ */ struct uart_softc * uart_init(void) { struct uart_softc *sc = calloc(1, sizeof(struct uart_softc)); if (sc == NULL) return (NULL); pthread_mutex_init(&sc->mtx, NULL); return (sc); } int uart_tty_open(struct uart_softc *sc, const char *path, void (*drain)(int, enum ev_type, void *), void *arg) { int retval; #ifndef __FreeBSD__ if (strncmp("socket,", path, 7) == 0) return (uart_sock_backend(sc, path, drain, arg)); #endif if (strcmp("stdio", path) == 0) retval = uart_stdio_backend(sc); else retval = uart_tty_backend(sc, path); if (retval == 0) { ttyopen(&sc->tty); sc->mev = mevent_add(sc->tty.rfd, EVF_READ, drain, arg); assert(sc->mev != NULL); } return (retval); } void uart_softc_lock(struct uart_softc *sc) { pthread_mutex_lock(&sc->mtx); } void uart_softc_unlock(struct uart_softc *sc) { pthread_mutex_unlock(&sc->mtx); } /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2012 NetApp, Inc. * Copyright (c) 2013 Neel Natu * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef _UART_BACKEND_H_ #define _UART_BACKEND_H_ #include #include "mevent.h" struct uart_softc; struct vm_snapshot_meta; void uart_rxfifo_drain(struct uart_softc *sc, bool loopback); #ifndef __FreeBSD__ void uart_rxfifo_sock_drain(struct uart_softc *sc, bool loopback); #endif int uart_rxfifo_getchar(struct uart_softc *sc); int uart_rxfifo_numchars(struct uart_softc *sc); int uart_rxfifo_putchar(struct uart_softc *sc, uint8_t ch, bool loopback); void uart_rxfifo_reset(struct uart_softc *sc, int size); int uart_rxfifo_size(struct uart_softc *sc); struct uart_softc *uart_init(void); int uart_tty_open(struct uart_softc *sc, const char *path, void (*drain)(int, enum ev_type, void *), void *arg); void uart_softc_lock(struct uart_softc *sc); void uart_softc_unlock(struct uart_softc *sc); #endif /* _UART_BACKEND_H_ */ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2012 NetApp, Inc. * Copyright (c) 2013 Neel Natu * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include #include #include #include #include #include #include #include #include #include #include "uart_backend.h" #include "uart_emul.h" #define COM1_BASE 0x3F8 #define COM1_IRQ 4 #define COM2_BASE 0x2F8 #define COM2_IRQ 3 #define COM3_BASE 0x3E8 #define COM3_IRQ 4 #define COM4_BASE 0x2E8 #define COM4_IRQ 3 #define DEFAULT_RCLK 1843200 #define DEFAULT_BAUD 115200 #define FCR_RX_MASK 0xC0 #define MCR_OUT1 0x04 #define MCR_OUT2 0x08 #define MSR_DELTA_MASK 0x0f #ifndef REG_SCR #define REG_SCR com_scr #endif static struct { int baseaddr; int irq; bool inuse; } uart_lres[] = { { COM1_BASE, COM1_IRQ, false}, { COM2_BASE, COM2_IRQ, false}, { COM3_BASE, COM3_IRQ, false}, { COM4_BASE, COM4_IRQ, false}, }; #define UART_NLDEVS (sizeof(uart_lres) / sizeof(uart_lres[0])) struct uart_ns16550_softc { struct uart_softc *backend; uint8_t data; /* Data register (R/W) */ uint8_t ier; /* Interrupt enable register (R/W) */ uint8_t lcr; /* Line control register (R/W) */ uint8_t mcr; /* Modem control register (R/W) */ uint8_t lsr; /* Line status register (R/W) */ uint8_t msr; /* Modem status register (R/W) */ uint8_t fcr; /* FIFO control register (W) */ uint8_t scr; /* Scratch register (R/W) */ uint8_t dll; /* Baudrate divisor latch LSB */ uint8_t dlh; /* Baudrate divisor latch MSB */ bool thre_int_pending; /* THRE interrupt pending */ void *arg; uart_intr_func_t intr_assert; uart_intr_func_t intr_deassert; }; static uint8_t modem_status(uint8_t mcr) { uint8_t msr; if (mcr & MCR_LOOPBACK) { /* * In the loopback mode certain bits from the MCR are * reflected back into MSR. */ msr = 0; if (mcr & MCR_RTS) msr |= MSR_CTS; if (mcr & MCR_DTR) msr |= MSR_DSR; if (mcr & MCR_OUT1) msr |= MSR_RI; if (mcr & MCR_OUT2) msr |= MSR_DCD; } else { /* * Always assert DCD and DSR so tty open doesn't block * even if CLOCAL is turned off. */ msr = MSR_DCD | MSR_DSR; } assert((msr & MSR_DELTA_MASK) == 0); return (msr); } /* * The IIR returns a prioritized interrupt reason: * - receive data available * - transmit holding register empty * - modem status change * * Return an interrupt reason if one is available. */ static int uart_intr_reason(struct uart_ns16550_softc *sc) { if ((sc->lsr & LSR_OE) != 0 && (sc->ier & IER_ERLS) != 0) return (IIR_RLS); else if (uart_rxfifo_numchars(sc->backend) > 0 && (sc->ier & IER_ERXRDY) != 0) return (IIR_RXTOUT); else if (sc->thre_int_pending && (sc->ier & IER_ETXRDY) != 0) return (IIR_TXRDY); else if ((sc->msr & MSR_DELTA_MASK) != 0 && (sc->ier & IER_EMSC) != 0) return (IIR_MLSC); else return (IIR_NOPEND); } static void uart_reset(struct uart_ns16550_softc *sc) { uint16_t divisor; divisor = DEFAULT_RCLK / DEFAULT_BAUD / 16; sc->dll = divisor; #ifdef __FreeBSD__ sc->dlh = divisor >> 16; #else sc->dlh = 0; #endif sc->msr = modem_status(sc->mcr); uart_rxfifo_reset(sc->backend, 1); } /* * Toggle the COM port's intr pin depending on whether or not we have an * interrupt condition to report to the processor. */ static void uart_toggle_intr(struct uart_ns16550_softc *sc) { uint8_t intr_reason; intr_reason = uart_intr_reason(sc); if (intr_reason == IIR_NOPEND) (*sc->intr_deassert)(sc->arg); else (*sc->intr_assert)(sc->arg); } static void uart_drain(int fd __unused, enum ev_type ev, void *arg) { struct uart_ns16550_softc *sc; bool loopback; sc = arg; assert(ev == EVF_READ); /* * This routine is called in the context of the mevent thread * to take out the softc lock to protect against concurrent * access from a vCPU i/o exit */ uart_softc_lock(sc->backend); loopback = (sc->mcr & MCR_LOOPBACK) != 0; uart_rxfifo_drain(sc->backend, loopback); if (!loopback) uart_toggle_intr(sc); uart_softc_unlock(sc->backend); } #ifndef __FreeBSD__ void uart_sock_drain(int fd, enum ev_type ev, void *arg) { struct uart_ns16550_softc *sc; bool loopback; sc = arg; /* * Take the softc lock to protect against concurrent * access from a vCPU i/o exit */ uart_softc_lock(sc->backend); loopback = (sc->mcr & MCR_LOOPBACK) != 0; uart_rxfifo_sock_drain(sc->backend, loopback); if (!loopback) uart_toggle_intr(sc); uart_softc_unlock(sc->backend); } #endif /* !__FreeBSD__ */ void uart_ns16550_write(struct uart_ns16550_softc *sc, int offset, uint8_t value) { int fifosz; uint8_t msr; uart_softc_lock(sc->backend); /* * Take care of the special case DLAB accesses first */ if ((sc->lcr & LCR_DLAB) != 0) { if (offset == REG_DLL) { sc->dll = value; goto done; } if (offset == REG_DLH) { sc->dlh = value; goto done; } } switch (offset) { case REG_DATA: if (uart_rxfifo_putchar(sc->backend, value, (sc->mcr & MCR_LOOPBACK) != 0)) sc->lsr |= LSR_OE; sc->thre_int_pending = true; break; case REG_IER: /* Set pending when IER_ETXRDY is raised (edge-triggered). */ if ((sc->ier & IER_ETXRDY) == 0 && (value & IER_ETXRDY) != 0) sc->thre_int_pending = true; /* * Apply mask so that bits 4-7 are 0 * Also enables bits 0-3 only if they're 1 */ sc->ier = value & 0x0F; break; case REG_FCR: /* * When moving from FIFO and 16450 mode and vice versa, * the FIFO contents are reset. */ if ((sc->fcr & FCR_ENABLE) ^ (value & FCR_ENABLE)) { fifosz = (value & FCR_ENABLE) ? uart_rxfifo_size(sc->backend) : 1; uart_rxfifo_reset(sc->backend, fifosz); } /* * The FCR_ENABLE bit must be '1' for the programming * of other FCR bits to be effective. */ if ((value & FCR_ENABLE) == 0) { sc->fcr = 0; } else { if ((value & FCR_RCV_RST) != 0) uart_rxfifo_reset(sc->backend, uart_rxfifo_size(sc->backend)); sc->fcr = value & (FCR_ENABLE | FCR_DMA | FCR_RX_MASK); } break; case REG_LCR: sc->lcr = value; break; case REG_MCR: /* Apply mask so that bits 5-7 are 0 */ sc->mcr = value & 0x1F; msr = modem_status(sc->mcr); /* * Detect if there has been any change between the * previous and the new value of MSR. If there is * then assert the appropriate MSR delta bit. */ if ((msr & MSR_CTS) ^ (sc->msr & MSR_CTS)) sc->msr |= MSR_DCTS; if ((msr & MSR_DSR) ^ (sc->msr & MSR_DSR)) sc->msr |= MSR_DDSR; if ((msr & MSR_DCD) ^ (sc->msr & MSR_DCD)) sc->msr |= MSR_DDCD; if ((sc->msr & MSR_RI) != 0 && (msr & MSR_RI) == 0) sc->msr |= MSR_TERI; /* * Update the value of MSR while retaining the delta * bits. */ sc->msr &= MSR_DELTA_MASK; sc->msr |= msr; break; case REG_LSR: /* * Line status register is not meant to be written to * during normal operation. */ break; case REG_MSR: /* * As far as I can tell MSR is a read-only register. */ break; case REG_SCR: sc->scr = value; break; default: break; } done: uart_toggle_intr(sc); uart_softc_unlock(sc->backend); } uint8_t uart_ns16550_read(struct uart_ns16550_softc *sc, int offset) { uint8_t iir, intr_reason, reg; uart_softc_lock(sc->backend); /* * Take care of the special case DLAB accesses first */ if ((sc->lcr & LCR_DLAB) != 0) { if (offset == REG_DLL) { reg = sc->dll; goto done; } if (offset == REG_DLH) { reg = sc->dlh; goto done; } } switch (offset) { case REG_DATA: reg = uart_rxfifo_getchar(sc->backend); break; case REG_IER: reg = sc->ier; break; case REG_IIR: iir = (sc->fcr & FCR_ENABLE) ? IIR_FIFO_MASK : 0; intr_reason = uart_intr_reason(sc); /* * Deal with side effects of reading the IIR register */ if (intr_reason == IIR_TXRDY) sc->thre_int_pending = false; iir |= intr_reason; reg = iir; break; case REG_LCR: reg = sc->lcr; break; case REG_MCR: reg = sc->mcr; break; case REG_LSR: /* Transmitter is always ready for more data */ sc->lsr |= LSR_TEMT | LSR_THRE; /* Check for new receive data */ if (uart_rxfifo_numchars(sc->backend) > 0) sc->lsr |= LSR_RXRDY; else sc->lsr &= ~LSR_RXRDY; reg = sc->lsr; /* The LSR_OE bit is cleared on LSR read */ sc->lsr &= ~LSR_OE; break; case REG_MSR: /* * MSR delta bits are cleared on read */ reg = sc->msr; sc->msr &= ~MSR_DELTA_MASK; break; case REG_SCR: reg = sc->scr; break; default: reg = 0xFF; break; } done: uart_toggle_intr(sc); uart_softc_unlock(sc->backend); return (reg); } int uart_legacy_alloc(int which, int *baseaddr, int *irq) { if (which < 0 || which >= (int)UART_NLDEVS || uart_lres[which].inuse) return (-1); uart_lres[which].inuse = true; *baseaddr = uart_lres[which].baseaddr; *irq = uart_lres[which].irq; return (0); } struct uart_ns16550_softc * uart_ns16550_init(uart_intr_func_t intr_assert, uart_intr_func_t intr_deassert, void *arg) { struct uart_ns16550_softc *sc; sc = calloc(1, sizeof(struct uart_ns16550_softc)); sc->arg = arg; sc->intr_assert = intr_assert; sc->intr_deassert = intr_deassert; sc->backend = uart_init(); uart_reset(sc); return (sc); } int uart_ns16550_tty_open(struct uart_ns16550_softc *sc, const char *device) { #ifndef __FreeBSD__ if (strncmp("socket,", device, 7) == 0) { return (uart_tty_open(sc->backend, device, uart_sock_drain, sc)); } #endif return (uart_tty_open(sc->backend, device, uart_drain, sc)); } /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2013 Neel Natu * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef _UART_EMUL_H_ #define _UART_EMUL_H_ #define UART_NS16550_IO_BAR_SIZE 8 struct uart_ns16550_softc; typedef void (*uart_intr_func_t)(void *arg); int uart_legacy_alloc(int unit, int *ioaddr, int *irq); struct uart_ns16550_softc *uart_ns16550_init(uart_intr_func_t intr_assert, uart_intr_func_t intr_deassert, void *arg); uint8_t uart_ns16550_read(struct uart_ns16550_softc *sc, int offset); void uart_ns16550_write(struct uart_ns16550_softc *sc, int offset, uint8_t value); int uart_ns16550_tty_open(struct uart_ns16550_softc *sc, const char *device); #endif /* _UART_EMUL_H_ */ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2014 Nahanni Systems Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include #include #include #include #include #include #include #include "usb_emul.h" SET_DECLARE(usb_emu_set, struct usb_devemu); struct usb_devemu * usb_emu_finddev(const char *name) { struct usb_devemu **udpp, *udp; SET_FOREACH(udpp, usb_emu_set) { udp = *udpp; if (!strcmp(udp->ue_emu, name)) return (udp); } return (NULL); } struct usb_data_xfer_block * usb_data_xfer_append(struct usb_data_xfer *xfer, void *buf, int blen, void *hci_data, int ccs) { struct usb_data_xfer_block *xb; if (xfer->ndata >= USB_MAX_XFER_BLOCKS) return (NULL); xb = &xfer->data[xfer->tail]; xb->buf = buf; xb->blen = blen; xb->hci_data = hci_data; xb->ccs = ccs; xb->processed = 0; xb->bdone = 0; xfer->ndata++; xfer->tail = (xfer->tail + 1) % USB_MAX_XFER_BLOCKS; return (xb); } /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2014 Leon Dang * Copyright 2018 Joyent, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef _USB_EMUL_H_ #define _USB_EMUL_H_ #include #include #include #include #ifndef __FreeBSD__ #include #endif #define USB_MAX_XFER_BLOCKS 8 #define USB_XFER_OUT 0 #define USB_XFER_IN 1 struct usb_hci; struct usb_device_request; struct usb_data_xfer; /* Device emulation handlers */ struct usb_devemu { const char *ue_emu; /* name of device emulation */ int ue_usbver; /* usb version: 2 or 3 */ int ue_usbspeed; /* usb device speed */ /* instance creation */ void *(*ue_init)(struct usb_hci *hci, nvlist_t *nvl); /* handlers */ int (*ue_request)(void *sc, struct usb_data_xfer *xfer); int (*ue_data)(void *sc, struct usb_data_xfer *xfer, int dir, int epctx); int (*ue_reset)(void *sc); int (*ue_remove)(void *sc); int (*ue_stop)(void *sc); }; #define USB_EMUL_SET(x) DATA_SET(usb_emu_set, x) /* * USB device events to notify HCI when state changes */ enum hci_usbev { USBDEV_ATTACH, USBDEV_RESET, USBDEV_STOP, USBDEV_REMOVE, }; /* usb controller, ie xhci, ehci */ struct usb_hci { int (*hci_intr)(struct usb_hci *hci, int epctx); int (*hci_event)(struct usb_hci *hci, enum hci_usbev evid, void *param); void *hci_sc; /* private softc for hci */ /* controller managed fields */ int hci_address; int hci_port; }; /* * Each xfer block is mapped to the hci transfer block. * On input into the device handler, blen is set to the length of buf. * The device handler is to update blen to reflect on the residual size * of the buffer, i.e. len(buf) - len(consumed). */ struct usb_data_xfer_block { void *buf; /* IN or OUT pointer */ int blen; /* in:len(buf), out:len(remaining) */ int bdone; /* bytes transferred */ uint32_t processed; /* device processed this + errcode */ void *hci_data; /* HCI private reference */ int ccs; uint32_t streamid; uint64_t trbnext; /* next TRB guest address */ }; struct usb_data_xfer { struct usb_data_xfer_block data[USB_MAX_XFER_BLOCKS]; struct usb_device_request *ureq; /* setup ctl request */ int ndata; /* # of data items */ int head; int tail; pthread_mutex_t mtx; }; enum USB_ERRCODE { USB_ACK, USB_NAK, USB_STALL, USB_NYET, USB_ERR, USB_SHORT }; #define USB_DATA_GET_ERRCODE(x) (x)->processed >> 8 #define USB_DATA_SET_ERRCODE(x,e) do { \ (x)->processed = ((x)->processed & 0xFF) | (e << 8); \ } while (0) #define USB_DATA_OK(x,i) ((x)->data[(i)].buf != NULL) #define USB_DATA_XFER_INIT(x) do { \ memset((x), 0, sizeof(*(x))); \ pthread_mutex_init(&((x)->mtx), NULL); \ } while (0) #define USB_DATA_XFER_RESET(x) do { \ memset((x)->data, 0, sizeof((x)->data)); \ (x)->ndata = 0; \ (x)->head = (x)->tail = 0; \ } while (0) #define USB_DATA_XFER_LOCK(x) do { \ pthread_mutex_lock(&((x)->mtx)); \ } while (0) #define USB_DATA_XFER_UNLOCK(x) do { \ pthread_mutex_unlock(&((x)->mtx)); \ } while (0) #ifndef __FreeBSD__ #define USB_DATA_XFER_LOCK_HELD(x) MUTEX_HELD(&((x)->mtx)) #endif struct usb_devemu *usb_emu_finddev(const char *name); struct usb_data_xfer_block *usb_data_xfer_append(struct usb_data_xfer *xfer, void *buf, int blen, void *hci_data, int ccs); #endif /* _USB_EMUL_H_ */ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2014 Leon Dang * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include #include #include #include #include #include #include #include "usb_emul.h" #include "console.h" #include "bhyvegc.h" #include "debug.h" static int umouse_debug = 0; #define DPRINTF(params) if (umouse_debug) PRINTLN params #define WPRINTF(params) PRINTLN params /* USB endpoint context (1-15) for reporting mouse data events*/ #define UMOUSE_INTR_ENDPT 1 #define UMOUSE_REPORT_DESC_TYPE 0x22 #define UMOUSE_GET_REPORT 0x01 #define UMOUSE_GET_IDLE 0x02 #define UMOUSE_GET_PROTOCOL 0x03 #define UMOUSE_SET_REPORT 0x09 #define UMOUSE_SET_IDLE 0x0A #define UMOUSE_SET_PROTOCOL 0x0B #define HSETW(ptr, val) ptr = { (uint8_t)(val), (uint8_t)((val) >> 8) } enum { UMSTR_LANG, UMSTR_MANUFACTURER, UMSTR_PRODUCT, UMSTR_SERIAL, UMSTR_CONFIG, UMSTR_MAX }; static const char *umouse_desc_strings[] = { "\x09\x04", "BHYVE", "HID Tablet", "01", "HID Tablet Device", }; struct umouse_hid_descriptor { uint8_t bLength; uint8_t bDescriptorType; uint8_t bcdHID[2]; uint8_t bCountryCode; uint8_t bNumDescriptors; uint8_t bReportDescriptorType; uint8_t wItemLength[2]; } __packed; struct umouse_config_desc { struct usb_config_descriptor confd; struct usb_interface_descriptor ifcd; struct umouse_hid_descriptor hidd; struct usb_endpoint_descriptor endpd; struct usb_endpoint_ss_comp_descriptor sscompd; } __packed; #define MOUSE_MAX_X 0x8000 #define MOUSE_MAX_Y 0x8000 static const uint8_t umouse_report_desc[] = { 0x05, 0x01, /* USAGE_PAGE (Generic Desktop) */ 0x09, 0x02, /* USAGE (Mouse) */ 0xa1, 0x01, /* COLLECTION (Application) */ 0x09, 0x01, /* USAGE (Pointer) */ 0xa1, 0x00, /* COLLECTION (Physical) */ 0x05, 0x09, /* USAGE_PAGE (Button) */ 0x19, 0x01, /* USAGE_MINIMUM (Button 1) */ 0x29, 0x03, /* USAGE_MAXIMUM (Button 3) */ 0x15, 0x00, /* LOGICAL_MINIMUM (0) */ 0x25, 0x01, /* LOGICAL_MAXIMUM (1) */ 0x75, 0x01, /* REPORT_SIZE (1) */ 0x95, 0x03, /* REPORT_COUNT (3) */ 0x81, 0x02, /* INPUT (Data,Var,Abs); 3 buttons */ 0x75, 0x05, /* REPORT_SIZE (5) */ 0x95, 0x01, /* REPORT_COUNT (1) */ 0x81, 0x03, /* INPUT (Cnst,Var,Abs); padding */ 0x05, 0x01, /* USAGE_PAGE (Generic Desktop) */ 0x09, 0x30, /* USAGE (X) */ 0x09, 0x31, /* USAGE (Y) */ 0x35, 0x00, /* PHYSICAL_MINIMUM (0) */ 0x46, 0xff, 0x7f, /* PHYSICAL_MAXIMUM (0x7fff) */ 0x15, 0x00, /* LOGICAL_MINIMUM (0) */ 0x26, 0xff, 0x7f, /* LOGICAL_MAXIMUM (0x7fff) */ 0x75, 0x10, /* REPORT_SIZE (16) */ 0x95, 0x02, /* REPORT_COUNT (2) */ 0x81, 0x02, /* INPUT (Data,Var,Abs) */ 0x05, 0x01, /* USAGE Page (Generic Desktop) */ 0x09, 0x38, /* USAGE (Wheel) */ 0x35, 0x00, /* PHYSICAL_MINIMUM (0) */ 0x45, 0x00, /* PHYSICAL_MAXIMUM (0) */ 0x15, 0x81, /* LOGICAL_MINIMUM (-127) */ 0x25, 0x7f, /* LOGICAL_MAXIMUM (127) */ 0x75, 0x08, /* REPORT_SIZE (8) */ 0x95, 0x01, /* REPORT_COUNT (1) */ 0x81, 0x06, /* INPUT (Data,Var,Rel) */ 0xc0, /* END_COLLECTION */ 0xc0 /* END_COLLECTION */ }; struct umouse_report { uint8_t buttons; /* bits: 0 left, 1 right, 2 middle */ int16_t x; /* x position */ int16_t y; /* y position */ int8_t z; /* z wheel position */ } __packed; #define MSETW(ptr, val) ptr = { (uint8_t)(val), (uint8_t)((val) >> 8) } static struct usb_device_descriptor umouse_dev_desc = { .bLength = sizeof(umouse_dev_desc), .bDescriptorType = UDESC_DEVICE, MSETW(.bcdUSB, UD_USB_3_0), .bMaxPacketSize = 9, /* max pkt size, 2^9 = 512 */ MSETW(.idVendor, 0xFB5D), /* vendor */ MSETW(.idProduct, 0x0001), /* product */ MSETW(.bcdDevice, 0), /* device version */ .iManufacturer = UMSTR_MANUFACTURER, .iProduct = UMSTR_PRODUCT, .iSerialNumber = UMSTR_SERIAL, .bNumConfigurations = 1, }; static struct umouse_config_desc umouse_confd = { .confd = { .bLength = sizeof(umouse_confd.confd), .bDescriptorType = UDESC_CONFIG, .wTotalLength[0] = sizeof(umouse_confd), .bNumInterface = 1, .bConfigurationValue = 1, .iConfiguration = UMSTR_CONFIG, .bmAttributes = UC_BUS_POWERED | UC_REMOTE_WAKEUP, .bMaxPower = 0, }, .ifcd = { .bLength = sizeof(umouse_confd.ifcd), .bDescriptorType = UDESC_INTERFACE, .bNumEndpoints = 1, .bInterfaceClass = UICLASS_HID, .bInterfaceSubClass = UISUBCLASS_BOOT, .bInterfaceProtocol = UIPROTO_MOUSE, }, .hidd = { .bLength = sizeof(umouse_confd.hidd), .bDescriptorType = 0x21, .bcdHID = { 0x01, 0x10 }, .bCountryCode = 0, .bNumDescriptors = 1, .bReportDescriptorType = UMOUSE_REPORT_DESC_TYPE, .wItemLength = { sizeof(umouse_report_desc), 0 }, }, .endpd = { .bLength = sizeof(umouse_confd.endpd), .bDescriptorType = UDESC_ENDPOINT, .bEndpointAddress = UE_DIR_IN | UMOUSE_INTR_ENDPT, .bmAttributes = UE_INTERRUPT, .wMaxPacketSize[0] = 8, .bInterval = 0xA, }, .sscompd = { .bLength = sizeof(umouse_confd.sscompd), .bDescriptorType = UDESC_ENDPOINT_SS_COMP, .bMaxBurst = 0, .bmAttributes = 0, MSETW(.wBytesPerInterval, 0), }, }; struct umouse_bos_desc { struct usb_bos_descriptor bosd; struct usb_devcap_ss_descriptor usbssd; } __packed; static struct umouse_bos_desc umouse_bosd = { .bosd = { .bLength = sizeof(umouse_bosd.bosd), .bDescriptorType = UDESC_BOS, HSETW(.wTotalLength, sizeof(umouse_bosd)), .bNumDeviceCaps = 1, }, .usbssd = { .bLength = sizeof(umouse_bosd.usbssd), .bDescriptorType = UDESC_DEVICE_CAPABILITY, .bDevCapabilityType = 3, .bmAttributes = 0, HSETW(.wSpeedsSupported, 0x08), .bFunctionalitySupport = 3, .bU1DevExitLat = 0xa, /* dummy - not used */ .wU2DevExitLat = { 0x20, 0x00 }, } }; struct umouse_softc { struct usb_hci *hci; struct umouse_report um_report; int newdata; struct { uint8_t idle; uint8_t protocol; uint8_t feature; } hid; pthread_mutex_t mtx; pthread_mutex_t ev_mtx; int polling; struct timeval prev_evt; }; static void umouse_event(uint8_t button, int x, int y, void *arg) { struct umouse_softc *sc; struct bhyvegc_image *gc; gc = console_get_image(); if (gc == NULL) { /* not ready */ return; } sc = arg; pthread_mutex_lock(&sc->mtx); sc->um_report.buttons = 0; sc->um_report.z = 0; if (button & 0x01) sc->um_report.buttons |= 0x01; /* left */ if (button & 0x02) sc->um_report.buttons |= 0x04; /* middle */ if (button & 0x04) sc->um_report.buttons |= 0x02; /* right */ if (button & 0x8) sc->um_report.z = 1; if (button & 0x10) sc->um_report.z = -1; /* scale coords to mouse resolution */ sc->um_report.x = MOUSE_MAX_X * x / gc->width; sc->um_report.y = MOUSE_MAX_Y * y / gc->height; sc->newdata = 1; pthread_mutex_unlock(&sc->mtx); pthread_mutex_lock(&sc->ev_mtx); sc->hci->hci_intr(sc->hci, UE_DIR_IN | UMOUSE_INTR_ENDPT); pthread_mutex_unlock(&sc->ev_mtx); } static void * umouse_init(struct usb_hci *hci, nvlist_t *nvl) { struct umouse_softc *sc; sc = calloc(1, sizeof(struct umouse_softc)); sc->hci = hci; sc->hid.protocol = 1; /* REPORT protocol */ pthread_mutex_init(&sc->mtx, NULL); pthread_mutex_init(&sc->ev_mtx, NULL); console_ptr_register(umouse_event, sc, 10); return (sc); } #define UREQ(x,y) ((x) | ((y) << 8)) static int umouse_request(void *scarg, struct usb_data_xfer *xfer) { struct umouse_softc *sc; struct usb_data_xfer_block *data; const char *str; uint16_t value; uint16_t index; uint16_t len; uint16_t slen; uint8_t *udata; int err; int i, idx; int eshort; sc = scarg; data = NULL; udata = NULL; idx = xfer->head; for (i = 0; i < xfer->ndata; i++) { xfer->data[idx].bdone = 0; if (data == NULL && USB_DATA_OK(xfer,idx)) { data = &xfer->data[idx]; udata = data->buf; } xfer->data[idx].processed = 1; idx = (idx + 1) % USB_MAX_XFER_BLOCKS; } err = USB_ERR_NORMAL_COMPLETION; eshort = 0; if (!xfer->ureq) { DPRINTF(("umouse_request: port %d", sc->hci->hci_port)); goto done; } value = UGETW(xfer->ureq->wValue); index = UGETW(xfer->ureq->wIndex); len = UGETW(xfer->ureq->wLength); DPRINTF(("umouse_request: port %d, type 0x%x, req 0x%x, val 0x%x, " "idx 0x%x, len %u", sc->hci->hci_port, xfer->ureq->bmRequestType, xfer->ureq->bRequest, value, index, len)); switch (UREQ(xfer->ureq->bRequest, xfer->ureq->bmRequestType)) { case UREQ(UR_GET_CONFIG, UT_READ_DEVICE): DPRINTF(("umouse: (UR_GET_CONFIG, UT_READ_DEVICE)")); if (!data) break; *udata = umouse_confd.confd.bConfigurationValue; data->blen = len > 0 ? len - 1 : 0; eshort = data->blen > 0; data->bdone += 1; break; case UREQ(UR_GET_DESCRIPTOR, UT_READ_DEVICE): DPRINTF(("umouse: (UR_GET_DESCRIPTOR, UT_READ_DEVICE) val %x", value >> 8)); if (!data) break; switch (value >> 8) { case UDESC_DEVICE: DPRINTF(("umouse: (->UDESC_DEVICE) len %u ?= " "sizeof(umouse_dev_desc) %lu", len, sizeof(umouse_dev_desc))); if ((value & 0xFF) != 0) { err = USB_ERR_STALLED; goto done; } if (len > sizeof(umouse_dev_desc)) { data->blen = len - sizeof(umouse_dev_desc); len = sizeof(umouse_dev_desc); } else data->blen = 0; memcpy(data->buf, &umouse_dev_desc, len); data->bdone += len; break; case UDESC_CONFIG: DPRINTF(("umouse: (->UDESC_CONFIG)")); if ((value & 0xFF) != 0) { err = USB_ERR_STALLED; goto done; } if (len > sizeof(umouse_confd)) { data->blen = len - sizeof(umouse_confd); len = sizeof(umouse_confd); } else data->blen = 0; memcpy(data->buf, &umouse_confd, len); data->bdone += len; break; case UDESC_STRING: DPRINTF(("umouse: (->UDESC_STRING)")); str = NULL; if ((value & 0xFF) < UMSTR_MAX) str = umouse_desc_strings[value & 0xFF]; else goto done; if ((value & 0xFF) == UMSTR_LANG) { udata[0] = 4; udata[1] = UDESC_STRING; data->blen = len - 2; len -= 2; data->bdone += 2; if (len >= 2) { udata[2] = str[0]; udata[3] = str[1]; data->blen -= 2; data->bdone += 2; } else data->blen = 0; goto done; } slen = 2 + strlen(str) * 2; udata[0] = slen; udata[1] = UDESC_STRING; if (len > slen) { data->blen = len - slen; len = slen; } else data->blen = 0; for (i = 2; i < len; i += 2) { udata[i] = *str++; udata[i+1] = '\0'; } data->bdone += slen; break; case UDESC_BOS: DPRINTF(("umouse: USB3 BOS")); if (len > sizeof(umouse_bosd)) { data->blen = len - sizeof(umouse_bosd); len = sizeof(umouse_bosd); } else data->blen = 0; memcpy(udata, &umouse_bosd, len); data->bdone += len; break; default: DPRINTF(("umouse: unknown(%d)->ERROR", value >> 8)); err = USB_ERR_STALLED; goto done; } eshort = data->blen > 0; break; case UREQ(UR_GET_DESCRIPTOR, UT_READ_INTERFACE): DPRINTF(("umouse: (UR_GET_DESCRIPTOR, UT_READ_INTERFACE) " "0x%x", (value >> 8))); if (!data) break; switch (value >> 8) { case UMOUSE_REPORT_DESC_TYPE: if (len > sizeof(umouse_report_desc)) { data->blen = len - sizeof(umouse_report_desc); len = sizeof(umouse_report_desc); } else data->blen = 0; memcpy(data->buf, umouse_report_desc, len); data->bdone += len; break; default: DPRINTF(("umouse: IO ERROR")); err = USB_ERR_STALLED; goto done; } eshort = data->blen > 0; break; case UREQ(UR_GET_INTERFACE, UT_READ_INTERFACE): DPRINTF(("umouse: (UR_GET_INTERFACE, UT_READ_INTERFACE)")); if (index != 0) { DPRINTF(("umouse get_interface, invalid index %d", index)); err = USB_ERR_STALLED; goto done; } if (!data) break; if (len > 0) { *udata = 0; data->blen = len - 1; } eshort = data->blen > 0; data->bdone += 1; break; case UREQ(UR_GET_STATUS, UT_READ_DEVICE): DPRINTF(("umouse: (UR_GET_STATUS, UT_READ_DEVICE)")); if (data == NULL) break; if (len > 1) { if (sc->hid.feature == UF_DEVICE_REMOTE_WAKEUP) USETW(udata, UDS_REMOTE_WAKEUP); else USETW(udata, 0); data->blen = len - 2; data->bdone += 2; } eshort = data->blen > 0; break; case UREQ(UR_GET_STATUS, UT_READ_INTERFACE): case UREQ(UR_GET_STATUS, UT_READ_ENDPOINT): DPRINTF(("umouse: (UR_GET_STATUS, UT_READ_INTERFACE)")); if (data == NULL) break; if (len > 1) { USETW(udata, 0); data->blen = len - 2; data->bdone += 2; } eshort = data->blen > 0; break; case UREQ(UR_SET_ADDRESS, UT_WRITE_DEVICE): /* XXX Controller should've handled this */ DPRINTF(("umouse set address %u", value)); break; case UREQ(UR_SET_CONFIG, UT_WRITE_DEVICE): DPRINTF(("umouse set config %u", value)); break; case UREQ(UR_SET_DESCRIPTOR, UT_WRITE_DEVICE): DPRINTF(("umouse set descriptor %u", value)); break; case UREQ(UR_CLEAR_FEATURE, UT_WRITE_DEVICE): DPRINTF(("umouse: (UR_SET_FEATURE, UT_WRITE_DEVICE) %x", value)); if (value == UF_DEVICE_REMOTE_WAKEUP) sc->hid.feature = 0; break; case UREQ(UR_SET_FEATURE, UT_WRITE_DEVICE): DPRINTF(("umouse: (UR_SET_FEATURE, UT_WRITE_DEVICE) %x", value)); if (value == UF_DEVICE_REMOTE_WAKEUP) sc->hid.feature = UF_DEVICE_REMOTE_WAKEUP; break; case UREQ(UR_CLEAR_FEATURE, UT_WRITE_INTERFACE): case UREQ(UR_CLEAR_FEATURE, UT_WRITE_ENDPOINT): case UREQ(UR_SET_FEATURE, UT_WRITE_INTERFACE): case UREQ(UR_SET_FEATURE, UT_WRITE_ENDPOINT): DPRINTF(("umouse: (UR_CLEAR_FEATURE, UT_WRITE_INTERFACE)")); err = USB_ERR_STALLED; goto done; case UREQ(UR_SET_INTERFACE, UT_WRITE_INTERFACE): DPRINTF(("umouse set interface %u", value)); break; case UREQ(UR_ISOCH_DELAY, UT_WRITE_DEVICE): DPRINTF(("umouse set isoch delay %u", value)); break; case UREQ(UR_SET_SEL, 0): DPRINTF(("umouse set sel")); break; case UREQ(UR_SYNCH_FRAME, UT_WRITE_ENDPOINT): DPRINTF(("umouse synch frame")); break; /* HID device requests */ case UREQ(UMOUSE_GET_REPORT, UT_READ_CLASS_INTERFACE): DPRINTF(("umouse: (UMOUSE_GET_REPORT, UT_READ_CLASS_INTERFACE) " "0x%x", (value >> 8))); if (!data) break; if ((value >> 8) == 0x01 && len >= sizeof(sc->um_report)) { /* TODO read from backend */ if (len > sizeof(sc->um_report)) { data->blen = len - sizeof(sc->um_report); len = sizeof(sc->um_report); } else data->blen = 0; memcpy(data->buf, &sc->um_report, len); data->bdone += len; } else { err = USB_ERR_STALLED; goto done; } eshort = data->blen > 0; break; case UREQ(UMOUSE_GET_IDLE, UT_READ_CLASS_INTERFACE): if (data == NULL) break; if (len > 0) { *udata = sc->hid.idle; data->blen = len - 1; data->bdone += 1; } eshort = data->blen > 0; break; case UREQ(UMOUSE_GET_PROTOCOL, UT_READ_CLASS_INTERFACE): if (data == NULL) break; if (len > 0) { *udata = sc->hid.protocol; data->blen = len - 1; data->bdone += 1; } eshort = data->blen > 0; break; case UREQ(UMOUSE_SET_REPORT, UT_WRITE_CLASS_INTERFACE): DPRINTF(("umouse: (UMOUSE_SET_REPORT, UT_WRITE_CLASS_INTERFACE) ignored")); break; case UREQ(UMOUSE_SET_IDLE, UT_WRITE_CLASS_INTERFACE): sc->hid.idle = UGETW(xfer->ureq->wValue) >> 8; DPRINTF(("umouse: (UMOUSE_SET_IDLE, UT_WRITE_CLASS_INTERFACE) %x", sc->hid.idle)); break; case UREQ(UMOUSE_SET_PROTOCOL, UT_WRITE_CLASS_INTERFACE): sc->hid.protocol = UGETW(xfer->ureq->wValue) >> 8; DPRINTF(("umouse: (UR_CLEAR_FEATURE, UT_WRITE_CLASS_INTERFACE) %x", sc->hid.protocol)); break; default: DPRINTF(("**** umouse request unhandled")); err = USB_ERR_STALLED; break; } done: /* UT_WRITE is 0, so this is condition is never true. */ #ifdef __FreeBSD__ if (xfer->ureq && (xfer->ureq->bmRequestType & UT_WRITE) && (err == USB_ERR_NORMAL_COMPLETION) && (data != NULL)) data->blen = 0; else if (eshort) err = USB_ERR_SHORT_XFER; #else if (eshort) err = USB_ERR_SHORT_XFER; #endif DPRINTF(("umouse request error code %d (0=ok), blen %u txlen %u", err, (data ? data->blen : 0), (data ? data->bdone : 0))); return (err); } static int umouse_data_handler(void *scarg, struct usb_data_xfer *xfer, int dir, int epctx) { struct umouse_softc *sc; struct usb_data_xfer_block *data; uint8_t *udata; int len, i, idx; int err; DPRINTF(("umouse handle data - DIR=%s|EP=%d, blen %d", dir ? "IN" : "OUT", epctx, xfer->data[0].blen)); /* find buffer to add data */ udata = NULL; err = USB_ERR_NORMAL_COMPLETION; /* handle xfer at first unprocessed item with buffer */ data = NULL; idx = xfer->head; for (i = 0; i < xfer->ndata; i++) { data = &xfer->data[idx]; if (data->buf != NULL && data->blen != 0) { break; } else { data->processed = 1; data = NULL; } idx = (idx + 1) % USB_MAX_XFER_BLOCKS; } if (!data) goto done; udata = data->buf; len = data->blen; if (udata == NULL) { DPRINTF(("umouse no buffer provided for input")); err = USB_ERR_NOMEM; goto done; } sc = scarg; if (dir) { pthread_mutex_lock(&sc->mtx); if (!sc->newdata) { err = USB_ERR_CANCELLED; USB_DATA_SET_ERRCODE(&xfer->data[xfer->head], USB_NAK); pthread_mutex_unlock(&sc->mtx); goto done; } if (sc->polling) { err = USB_ERR_STALLED; USB_DATA_SET_ERRCODE(data, USB_STALL); pthread_mutex_unlock(&sc->mtx); goto done; } sc->polling = 1; if (len > 0) { sc->newdata = 0; data->processed = 1; data->bdone += 6; memcpy(udata, &sc->um_report, 6); data->blen = len - 6; if (data->blen > 0) err = USB_ERR_SHORT_XFER; } sc->polling = 0; pthread_mutex_unlock(&sc->mtx); } else { USB_DATA_SET_ERRCODE(data, USB_STALL); err = USB_ERR_STALLED; } done: return (err); } static int umouse_reset(void *scarg) { struct umouse_softc *sc; sc = scarg; sc->newdata = 0; return (0); } static int umouse_remove(void *scarg) { return (0); } static int umouse_stop(void *scarg) { return (0); } static struct usb_devemu ue_mouse = { .ue_emu = "tablet", .ue_usbver = 3, .ue_usbspeed = USB_SPEED_HIGH, .ue_init = umouse_init, .ue_request = umouse_request, .ue_data = umouse_data_handler, .ue_reset = umouse_reset, .ue_remove = umouse_remove, .ue_stop = umouse_stop }; USB_EMUL_SET(ue_mouse); /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2013 Chris Torek * All rights reserved. * Copyright (c) 2019 Joyent, Inc. * Copyright (c) 2021 The FreeBSD Foundation * * Portions of this software were developed by Ka Ho Ng * under sponsorship of the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * This file and its contents are supplied under the terms of the * Common Development and Distribution License ("CDDL"), version 1.0. * You may only use this file in accordance with the terms of version * 1.0 of the CDDL. * * A full copy of the text of the CDDL should have accompanied this * source. A copy of the CDDL is also available via the Internet at * http://www.illumos.org/license/CDDL. */ /* This file is dual-licensed; see usr/src/contrib/bhyve/LICENSE */ /* * Copyright 2026 Oxide Computer Company */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "bhyverun.h" #include "config.h" #include "debug.h" #include "pci_emul.h" #include "virtio.h" /* * Functions for dealing with generalized "virtual devices" as * defined by * * The reference for the implementation of virtio modern is on * */ #define DPRINTF(vs, fmt, arg...) \ do { \ if ((((vs)->vs_flags) & VIRTIO_DEBUG) != 0) { \ FPRINTLN(stdout, fmt, ##arg); \ fflush(stdout); \ } \ } while (0) #define VQ_NOTIFY_OFF_MULTIPLIER sizeof (uint32_t) /* * In case we decide to relax the "virtio softc comes at the * front of virtio-based device softc" constraint, let's use * this to convert. */ #define DEV_SOFTC(vs) ((void *)(vs)) #define VI_MASK(nbytes) \ (((nbytes) >= 4) ? 0xFFFFFFFFu : (~0u >> (32 - 8 * (nbytes)))) static uint64_t vi_modern_pci_read(struct virtio_softc *, int, uint64_t, int); static void vi_modern_pci_write(struct virtio_softc *, int, uint64_t, int, uint64_t); void vi_queue_linkup(struct virtio_softc *vs, struct vqueue_info *queues) { struct virtio_consts *vc = vs->vs_vc; vs->vs_queues = queues; for (int i = 0; i < vc->vc_nvq; i++) { vs->vs_queues[i].vq_vs = vs; vs->vs_queues[i].vq_num = i; } } /* * Link a virtio_softc to its constants, the device softc, and * the PCI emulation. */ void vi_softc_linkup(struct virtio_softc *vs, struct virtio_consts *vc, void *dev_softc, struct pci_devinst *pi, struct vqueue_info *queues) { /* vs and dev_softc addresses must match */ assert((void *)vs == dev_softc); vs->vs_vc = vc; vs->vs_pi = pi; pi->pi_arg = vs; vi_queue_linkup(vs, queues); } /* * Reset device (device-wide). This erases all queues, i.e., * all the queues become invalid (though we don't wipe out the * internal pointers, we just clear the VQ_ALLOC flag). * * It resets negotiated features to "none". * * If MSI-X is enabled, this also resets all the vectors to NO_VECTOR. */ void vi_reset_dev(struct virtio_softc *vs) { struct vqueue_info *vq; int i, nvq; if (vs->vs_mtx) assert(pthread_mutex_isowned_np(vs->vs_mtx)); nvq = vs->vs_vc->vc_nvq; for (vq = vs->vs_queues, i = 0; i < nvq; vq++, i++) { vq->vq_flags = 0; vq->vq_last_avail = 0; vq->vq_next_used = 0; vq->vq_save_used = 0; vq->vq_pfn = 0; vq->vq_desc_gpa = vq->vq_avail_gpa = vq->vq_used_gpa = 0; vq->vq_msix_idx = VIRTIO_MSI_NO_VECTOR; } vs->vs_negotiated_caps = 0; vs->vs_curq = 0; if (vs->vs_isr != 0) pci_lintr_deassert(vs->vs_pi); vs->vs_isr = 0; vs->vs_msix_cfg_idx = VIRTIO_MSI_NO_VECTOR; } /* * These are the capability bits common to all virtio devices. */ static const virtio_capstr_t virtio_caps[] = { { VIRTIO_F_NOTIFY_ON_EMPTY, "VIRTIO_F_NOTIFY_ON_EMPTY" }, { VIRTIO_F_ANY_LAYOUT, "VIRTIO_F_ANY_LAYOUT" }, { VIRTIO_RING_F_INDIRECT_DESC, "VIRTIO_RING_F_INDIRECT_DESC" }, { VIRTIO_RING_F_EVENT_IDX, "VIRTIO_RING_F_EVENT_IDX" }, { VIRTIO_F_BAD_FEATURE, "VIRTIO_F_BAD_FEATURE" }, { VIRTIO_F_VERSION_1, "VIRTIO_F_VERSION_1" }, }; static void vi_print_caps(struct virtio_softc *vs, uint64_t caps) { struct virtio_consts *vc = vs->vs_vc; if ((vs->vs_flags & VIRTIO_DEBUG) == 0) return; for (size_t i = 0; i < vc->vc_ncapstr; i++) { if ((caps & vc->vc_capstr[i].vp_flag) != 0) FPRINTLN(stdout, " -> %s", vc->vc_capstr[i].vp_name); } for (size_t i = 0; i < ARRAY_SIZE(virtio_caps); i++) { if ((caps & virtio_caps[i].vp_flag) != 0) FPRINTLN(stdout, " -> %s", virtio_caps[i].vp_name); } fflush(stdout); } void vi_set_debug(struct virtio_softc *vs, bool debug) { if (debug) vs->vs_flags |= VIRTIO_DEBUG; else vs->vs_flags &= ~VIRTIO_DEBUG; } bool vi_is_modern(struct virtio_softc *vs) { return (vs->vs_negotiated_caps & VIRTIO_F_VERSION_1) != 0; } void __PRINTFLIKE(2) vi_error(struct virtio_softc *vs, const char *fmt, ...) { va_list ap; va_start(ap, fmt); vfprintf(stderr, fmt, ap); fprintf(stderr, "%s", raw_stdio ? "\r\n" : "\n"); va_end(ap); if (vi_is_modern(vs)) { vs->vs_status |= VTCFG_STATUS_NEEDS_RST; vq_devcfg_changed(vs); } vs->vs_flags |= VIRTIO_BROKEN; } /* * Set I/O BAR (usually 0) to map legacy PCI config registers. */ static bool vi_legacy_iobar_setup(struct virtio_softc *vs, int barnum) { size_t size; /* * We set the size to that which will accommodate the configuration * space with MSI-X enabled, plus the configuration size. */ size = VIRTIO_PCI_CONFIG_OFF(1) + vs->vs_vc->vc_cfgsize; if (pci_emul_alloc_bar(vs->vs_pi, barnum, PCIBAR_IO, size) != 0) return (false); return (true); } virtio_pci_capcfg_t * vi_pci_cfg_bytype(struct virtio_softc *vs, uint8_t cfgtype) { for (uint_t i = 0; i < vs->vs_ncaps; i++) { if (vs->vs_caps[i].c_captype == cfgtype) return (&vs->vs_caps[i]); } return (NULL); } virtio_pci_capcfg_t * vi_pci_cfg_bycapaddr(struct virtio_softc *vs, uint32_t start, uint32_t size) { if (size == 0 || start > UINT32_MAX - size) return (NULL); const uint32_t end = start + size; for (uint_t i = 0; i < vs->vs_ncaps; i++) { virtio_pci_capcfg_t *cfg = &vs->vs_caps[i]; const uint32_t cap_start = cfg->c_capoff; const uint32_t cap_end = cap_start + cfg->c_caplen; if (cap_start <= start && end <= cap_end) return (cfg); } return (NULL); } virtio_pci_capcfg_t * vi_pci_cfg_bybaraddr(struct virtio_softc *vs, uint8_t bar, uint64_t offset, uint32_t size) { /* * We currently don't use the larger capabilities introduced in VirtIO * 1.2 that allow for 64-bit offsets and sizes. */ if (size == 0 || offset > UINT32_MAX - size) return (NULL); const uint32_t end = offset + size; for (uint_t i = 0; i < vs->vs_ncaps; i++) { virtio_pci_capcfg_t *cfg = &vs->vs_caps[i]; if (cfg->c_baridx != bar) continue; const uint32_t bar_start = cfg->c_baroff; const uint32_t bar_end = bar_start + cfg->c_barlen; if (bar_start <= offset && end <= bar_end) return (cfg); } return (NULL); } /* * Add a modern configuration structure capability. */ static bool vi_modern_add_cfg(struct virtio_softc *vs, struct virtio_pci_cap *cap, int barnum, uint32_t baroff, uint32_t barlen, uint8_t caplen, uint8_t cfgtype) { int capoff; cap->cap_vndr = PCIY_VENDOR; cap->cap_len = caplen; cap->cfg_type = cfgtype; cap->bar = barnum; cap->id = 0; cap->offset = baroff; cap->length = barlen; if (pci_emul_add_capability(vs->vs_pi, (u_char *)cap, caplen, &capoff) != 0) { return (false); } vs->vs_caps[vs->vs_ncaps].c_captype = cfgtype; vs->vs_caps[vs->vs_ncaps].c_baridx = cap->bar; vs->vs_caps[vs->vs_ncaps].c_baroff = cap->offset; vs->vs_caps[vs->vs_ncaps].c_barlen = cap->length; vs->vs_caps[vs->vs_ncaps].c_capoff = capoff; vs->vs_caps[vs->vs_ncaps].c_caplen = caplen; vs->vs_ncaps++; VERIFY3U(vs->vs_ncaps, <=, sizeof (vs->vs_caps)); return (true); } /* * Add COMMON_CFG configuration structure capability. */ static bool vi_modern_add_common_cfg(struct virtio_softc *vs, int barnum, uint32_t *offp) { struct virtio_pci_cap cap; uint32_t bardatalen; *offp = roundup2(*offp, VIRTIO_PCI_CAP_COMMON_CFG_ALIGN); /* * We choose to round this BAR area up to a page size in common with * other hypervisors. */ bardatalen = roundup2(sizeof (struct virtio_pci_common_cfg), PAGE_SIZE); memset(&cap, 0, sizeof (cap)); if (vi_modern_add_cfg(vs, &cap, barnum, *offp, bardatalen, sizeof (cap), VIRTIO_PCI_CAP_COMMON_CFG)) { *offp += bardatalen; return (true); } return (false); } /* * Add NOTIFY_CFG configuration structure capability. */ static bool vi_modern_add_notify_cfg(struct virtio_softc *vs, int barnum, uint32_t *offp) { struct virtio_pci_notify_cap cap; struct virtio_consts *vc = vs->vs_vc; int numq = MAX(vc->vc_max_nvq, vc->vc_nvq); uint32_t bardatalen; VERIFY3S(numq, >, 0); VERIFY3S(numq, <=, UINT16_MAX); *offp = roundup2(*offp, VIRTIO_PCI_CAP_NOTIFY_CFG_ALIGN); /* * We choose to round this BAR area up to a page size in common with * other hypervisors. */ uint64_t datalen = (uint64_t)numq * VQ_NOTIFY_OFF_MULTIPLIER; VERIFY3U(datalen, <=, UINT32_MAX - (PAGE_SIZE - 1)); bardatalen = roundup2((uint32_t)datalen, PAGE_SIZE); memset(&cap, 0, sizeof (cap)); cap.notify_off_multiplier = VQ_NOTIFY_OFF_MULTIPLIER; if (vi_modern_add_cfg(vs, &cap.cap, barnum, *offp, bardatalen, sizeof (cap), VIRTIO_PCI_CAP_NOTIFY_CFG)) { *offp += bardatalen; return (true); } return (false); } /* * Add ISR_CFG configuration structure capability. */ static bool vi_modern_add_isr_cfg(struct virtio_softc *vs, int barnum, uint32_t *offp) { struct virtio_pci_cap cap; uint32_t bardatalen; *offp = roundup2(*offp, VIRTIO_PCI_CAP_ISR_CFG_ALIGN); /* * While this capability could point to a single byte in the BAR, we * choose to round up to a page in common with other hypervisors. */ bardatalen = PAGE_SIZE; memset(&cap, 0, sizeof (cap)); if (vi_modern_add_cfg(vs, &cap, barnum, *offp, bardatalen, sizeof (cap), VIRTIO_PCI_CAP_ISR_CFG)) { *offp += bardatalen; return (true); } return (false); } /* * Add DEV_CFG configuration structure capability. */ static bool vi_modern_add_dev_cfg(struct virtio_softc *vs, int barnum, uint32_t *offp) { struct virtio_pci_cap cap; uint32_t bardatalen; *offp = roundup2(*offp, VIRTIO_PCI_CAP_DEVICE_CFG_ALIGN); /* * We choose to round this BAR area up to a page size in common with * other hypervisors. */ bardatalen = PAGE_SIZE; memset(&cap, 0, sizeof (cap)); if (vi_modern_add_cfg(vs, &cap, barnum, *offp, bardatalen, sizeof (cap), VIRTIO_PCI_CAP_DEVICE_CFG)) { *offp += bardatalen; return (true); } return (false); } /* * Add PCI_CFG configuration structure capability. */ static bool vi_modern_add_pci_cfg(struct virtio_softc *vs) { struct virtio_pci_cfg_cap cap; memset(&cap, 0, sizeof (cap)); memset(cap.pci_cfg_data, 0xff, sizeof (cap.pci_cfg_data)); if (vi_modern_add_cfg(vs, &cap.cap, 0, 0, 0, sizeof (cap), VIRTIO_PCI_CAP_PCI_CFG)) { vs->vs_pcicap = &vs->vs_caps[vs->vs_ncaps - 1]; return (true); } return (false); } /* * Set up Virtio modern device pci configuration space */ static bool vi_modern_membar_setup(struct virtio_softc *vs, int barnum) { uint32_t baroff = 0; bool ret = false; ret |= vi_modern_add_common_cfg(vs, barnum, &baroff); ret |= vi_modern_add_notify_cfg(vs, barnum, &baroff); ret |= vi_modern_add_dev_cfg(vs, barnum, &baroff); ret |= vi_modern_add_isr_cfg(vs, barnum, &baroff); ret |= vi_modern_add_pci_cfg(vs); if (!ret) return (false); if (pci_emul_alloc_bar(vs->vs_pi, barnum, PCIBAR_MEM64, baroff) != 0) return (false); return (true); } void vi_pci_init(struct pci_devinst *pi, virtio_mode_t mode, uint16_t legacy, uint16_t device_id, uint8_t class) { struct virtio_softc *vs = pi->pi_arg; DPRINTF(vs, "VIRTIO %s PCI init mode=%x, legacy=0x%x devid=0x%x", vs->vs_vc->vc_name, mode, legacy, device_id); /* * We provide global options to force transitional devices to present * as pure legacy or modern. This is mostly to support testing guest * drivers or bhyve itself. * * TRANSITIONAL mode usually exposes both interfaces * - virtio.legacy=false forces a modern-only device * - virtio.modern=false forces a legacy-only device */ if (mode == VIRTIO_MODE_TRANSITIONAL) { if (!get_config_bool_default("virtio.legacy", true)) mode = VIRTIO_MODE_MODERN; else if (!get_config_bool_default("virtio.modern", true)) mode = VIRTIO_MODE_LEGACY; } vs->vs_mode = mode; pci_set_cfgdata16(pi, PCIR_VENDOR, VIRTIO_VENDOR); pci_set_cfgdata16(pi, PCIR_SUBVEND_0, VIRTIO_VENDOR); pci_set_cfgdata8(pi, PCIR_CLASS, class); if (mode == VIRTIO_MODE_MODERN) { /* * Pure modern / non-transitional device. * * Virtio 1.2, 4.1.2.1: * - PCI Device ID = 0x1040 + virtio device ID * - PCI Revision ID> >= 1 * - PCI Subsystem Device ID >= 0x40 * * `device_id` here is the virtio Device ID from section 5 * [0x0-0x3f]. */ VERIFY3U(device_id, <=, 0x3f); pci_set_cfgdata16(pi, PCIR_DEVICE, VIRTIO_PCI_DEVICEID_MODERN_MIN + device_id); /* * For modern devices the spec only recommends that the * Subsystem Device ID be >= 0x40 to avoid legacy binding. * We choose to mirror the main device ID here so that the * (vendor,device) and (subvendor,subdevice) pairs line up. */ pci_set_cfgdata16(pi, PCIR_SUBDEV_0, VIRTIO_PCI_DEVICEID_MODERN_MIN + device_id); pci_set_cfgdata16(pi, PCIR_REVID, 1); } else { /* * Legacy-only or transitional device. * * For *transitional* devices, virtio 1.2, 4.1.2.3 requires: * - PCI Device ID in [0x1000, 0x103f] * - PCI Revision ID == 0 * - PCI Subsystem Device ID == virtio Device ID * * We rely on the caller to pass: * - `legacy` the 0x1000-0x103f PCI Device ID * - `device_id` the virtio Device ID from section 5 * * For a true legacy-only device this layout is also compatible * with old drivers. */ VERIFY(legacy >= 0x1000 && legacy <= 0x103f); pci_set_cfgdata16(pi, PCIR_DEVICE, legacy); pci_set_cfgdata16(pi, PCIR_SUBDEV_0, device_id); pci_set_cfgdata16(pi, PCIR_REVID, 0); } } /* * Set up Virtio device pci configuration space. */ bool vi_pcibar_setup(struct virtio_softc *vs) { DPRINTF(vs, "VIRTIO %s set up PCI BARs", vs->vs_vc->vc_name); assert(vs->vs_mode != VIRTIO_MODE_UNSET); if (vs->vs_mode == VIRTIO_MODE_LEGACY || vs->vs_mode == VIRTIO_MODE_TRANSITIONAL) { if (!vi_legacy_iobar_setup(vs, VIRTIO_LEGACY_BAR)) return (false); } if (vs->vs_mode == VIRTIO_MODE_MODERN || vs->vs_mode == VIRTIO_MODE_TRANSITIONAL) { if (!vi_modern_membar_setup(vs, VIRTIO_MODERN_BAR)) return (false); } return (true); } /* * Configure interrupt delivery for this VirtIO device. * * If requested, enable MSI-X and allocate one vector per queue plus * a configuration vector. Regardless, always establish the mandatory * legacy (INTx) interrupt as VirtIO devices do not support MSI and * require a fixed interrupt line for compatibility. */ bool vi_intr_init(struct virtio_softc *vs, bool use_msix) { if (use_msix) { struct virtio_consts *vc = vs->vs_vc; int nvec = MIN(MAX(vc->vc_max_nvq, vc->vc_nvq) + 1, MAX_MSIX_TABLE_ENTRIES); vs->vs_flags |= VIRTIO_USE_MSIX; VS_LOCK(vs); vi_reset_dev(vs); /* set all vectors to NO_VECTOR */ VS_UNLOCK(vs); if (pci_emul_add_msixcap(vs->vs_pi, nvec, VIRTIO_MSIX_BAR) != 0) return (false); } else { vs->vs_flags &= ~VIRTIO_USE_MSIX; } /* Legacy interrupts are mandatory for virtio devices */ pci_lintr_request(vs->vs_pi); return (true); } /* * Initialize the currently-selected virtio queue (vs->vs_curq) */ void vi_vq_init(struct virtio_softc *vs) { struct vqueue_info *vq; uint64_t phys; size_t size; char *base; vq = &vs->vs_queues[vs->vs_curq]; phys = vq->vq_desc_gpa; size = vq->vq_qsize * sizeof (struct vring_desc); base = paddr_guest2host(vs->vs_pi->pi_vmctx, phys, size); if (base == NULL) { vi_error(vs, "Could not map queue 0x%x phys 0x%" PRIx64, vq->vq_num, phys); return; } vq->vq_desc = (struct vring_desc *)base; phys = vq->vq_avail_gpa; size = sizeof (struct vring_avail) + sizeof (uint16_t) + vq->vq_qsize * sizeof (uint16_t); base = paddr_guest2host(vs->vs_pi->pi_vmctx, phys, size); if (base == NULL) { vi_error(vs, "Could not map queue 0x%x phys 0x%" PRIx64, vq->vq_num, phys); return; } vq->vq_avail = (struct vring_avail *)base; phys = vq->vq_used_gpa; size = sizeof (struct vring_used) + sizeof (uint16_t) + vq->vq_qsize * sizeof (struct vring_used_elem); base = paddr_guest2host(vs->vs_pi->pi_vmctx, phys, size); if (base == NULL) { vi_error(vs, "Could not map queue 0x%x phys 0x%" PRIx64, vq->vq_num, phys); return; } vq->vq_used = (struct vring_used *)base; /* Mark queue as allocated, and start at 0 when we use it. */ vq->vq_flags = VQ_ALLOC; vq->vq_last_avail = 0; vq->vq_next_used = 0; vq->vq_save_used = 0; } /* * Initialize the currently-selected virtio queue (vs->vs_curq). * The guest just gave us a page frame number, from which we can * calculate the addresses of the queue components. */ void vi_legacy_vq_init(struct virtio_softc *vs, uint32_t pfn) { struct vqueue_info *vq; uint64_t phys; vq = &vs->vs_queues[vs->vs_curq]; vq->vq_pfn = pfn; phys = (uint64_t)pfn << LEGACY_VRING_PFN; /* First page(s) are descriptors... */ vq->vq_desc_gpa = phys; phys += vq->vq_qsize * sizeof (struct vring_desc); /* ... immediately followed by "avail" ring (entirely uint16_t's) */ vq->vq_avail_gpa = phys; phys += sizeof (struct vring_avail) + sizeof (uint16_t) + vq->vq_qsize * sizeof (uint16_t); /* Then it's rounded up to the next page... */ phys = roundup2(phys, LEGACY_VRING_ALIGN); /* ... and the last page(s) are the used ring. */ vq->vq_used_gpa = phys; vi_vq_init(vs); } /* * Helper inline for vq_getchain(): record the i'th "real" * descriptor. */ static inline void _vq_record(struct virtio_softc *vs, int i, struct vring_desc *vd, struct iovec *iov, int n_iov, struct vi_req *reqp) { struct vmctx *ctx; uint32_t len; uint64_t addr; ctx = vs->vs_pi->pi_vmctx; if (i >= n_iov) return; len = atomic_load_32(&vd->len); addr = atomic_load_64(&vd->addr); iov[i].iov_len = len; iov[i].iov_base = paddr_guest2host(ctx, addr, len); if ((vd->flags & VRING_DESC_F_WRITE) == 0) reqp->readable++; else reqp->writable++; } #define VQ_MAX_DESCRIPTORS 512 /* see below */ /* * Examine the chain of descriptors starting at the "next one" to * make sure that they describe a sensible request. If so, return * the number of "real" descriptors that would be needed/used in * acting on this request. This may be smaller than the number of * available descriptors, e.g., if there are two available but * they are two separate requests, this just returns 1. Or, it * may be larger: if there are indirect descriptors involved, * there may only be one descriptor available but it may be an * indirect pointing to eight more. We return 8 in this case, * i.e., we do not count the indirect descriptors, only the "real" * ones. * * Basically, this vets the "flags" and "next" field of each * descriptor and tells you how many are involved. Since some may * be indirect, this also needs the vmctx (in the pci_devinst * at vs->vs_pi) so that it can find indirect descriptors. * * As we process each descriptor, we copy and adjust it (guest to * host address wise, also using the vmtctx) into the given iov[] * array (of the given size). If the array overflows, we stop * placing values into the array but keep processing descriptors, * up to VQ_MAX_DESCRIPTORS, before giving up and returning -1. * So you, the caller, must not assume that iov[] is as big as the * return value (you can process the same thing twice to allocate * a larger iov array if needed, or supply a zero length to find * out how much space is needed). * * If some descriptor(s) are invalid, this prints a diagnostic message * and returns -1. If no descriptors are ready now it simply returns 0. * * You are assumed to have done a vq_ring_ready() if needed (note * that vq_has_descs() does one). */ int vq_getchain(struct vqueue_info *vq, struct iovec *iov, int niov, struct vi_req *reqp) { int i; u_int ndesc, n_indir; u_int idx, next; struct vi_req req; struct vring_desc *vdir, *vindir, *vp; struct vmctx *ctx; struct virtio_softc *vs; const char *name; vs = vq->vq_vs; name = vs->vs_vc->vc_name; memset(&req, 0, sizeof (req)); /* * Note: it's the responsibility of the guest not to * update vq->vq_avail->idx until all of the descriptors * the guest has written are valid (including all their * "next" fields and "flags"). * * Compute (vq_avail->idx - last_avail) in integers mod 2**16. This is * the number of descriptors the device has made available * since the last time we updated vq->vq_last_avail. * * We just need to do the subtraction as an unsigned int, * then trim off excess bits. */ idx = vq->vq_last_avail; ndesc = (uint16_t)((u_int)vq->vq_avail->idx - idx); if (ndesc == 0) return (0); if (ndesc > vq->vq_qsize) { vi_error(vs, "%s: ndesc (%u) out of range, driver confused?", name, (u_int)ndesc); return (-1); } /* * Now count/parse "involved" descriptors starting from * the head of the chain. * * To prevent loops, we could be more complicated and * check whether we're re-visiting a previously visited * index, but we just abort if the count gets excessive. */ ctx = vs->vs_pi->pi_vmctx; req.idx = next = vq->vq_avail->ring[idx & (vq->vq_qsize - 1)]; vq->vq_last_avail++; for (i = 0; i < VQ_MAX_DESCRIPTORS; next = vdir->next) { if (next >= vq->vq_qsize) { vi_error(vs, "%s: descriptor index %u out of range, " "driver confused?", name, next); return (-1); } vdir = &vq->vq_desc[next]; if ((vdir->flags & VRING_DESC_F_INDIRECT) == 0) { _vq_record(vs, i, vdir, iov, niov, &req); i++; } else if ((vs->vs_negotiated_caps & VIRTIO_RING_F_INDIRECT_DESC) == 0) { vi_error(vs, "%s: descriptor has forbidden INDIRECT flag, " "driver confused?", name); return (-1); } else { n_indir = vdir->len / 16; if ((vdir->len & 0xf) || n_indir == 0) { vi_error(vs, "%s: invalid indir len 0x%x, " "driver confused?", name, (u_int)vdir->len); return (-1); } vindir = paddr_guest2host(ctx, vdir->addr, vdir->len); /* * Indirects start at the 0th, then follow * their own embedded "next"s until those run * out. Each one's indirect flag must be off * (we don't really have to check, could just * ignore errors...). */ next = 0; for (;;) { vp = &vindir[next]; if (vp->flags & VRING_DESC_F_INDIRECT) { vi_error(vs, "%s: indirect desc has INDIR flag," " driver confused?", name); return (-1); } _vq_record(vs, i, vp, iov, niov, &req); if (++i > VQ_MAX_DESCRIPTORS) goto loopy; if ((vp->flags & VRING_DESC_F_NEXT) == 0) break; next = vp->next; if (next >= n_indir) { vi_error(vs, "%s: invalid next %u > %u, " "driver confused?", name, (u_int)next, n_indir); return (-1); } } } if ((vdir->flags & VRING_DESC_F_NEXT) == 0) goto done; } loopy: vi_error(vs, "%s: descriptor loop? count > %d - driver confused?", name, i); return (-1); done: *reqp = req; return (i); } /* * Return the first n_chain request chains back to the available queue. * * (These chains are the ones you handled when you called vq_getchain() * and used its positive return value.) */ void vq_retchains(struct vqueue_info *vq, uint16_t n_chains) { vq->vq_last_avail -= n_chains; } void vq_relchain_prepare(struct vqueue_info *vq, uint16_t idx, uint32_t iolen) { struct vring_used *vuh; struct vring_used_elem *vue; uint16_t mask; /* * Notes: * - mask is N-1 where N is a power of 2 so computes x % N * - vuh points to the "used" data shared with guest * - vue points to the "used" ring entry we want to update */ mask = vq->vq_qsize - 1; vuh = vq->vq_used; vue = &vuh->ring[vq->vq_next_used++ & mask]; vue->id = idx; vue->len = iolen; } void vq_relchain_publish(struct vqueue_info *vq) { /* * Ensure the used descriptor is visible before updating the index. * This is necessary on ISAs with memory ordering less strict than x86 * (and even on x86 to act as a compiler barrier). */ atomic_thread_fence_rel(); vq->vq_used->idx = vq->vq_next_used; } /* * Return specified request chain to the guest, setting its I/O length * to the provided value. * * (This chain is the one you handled when you called vq_getchain() * and used its positive return value.) */ void vq_relchain(struct vqueue_info *vq, uint16_t idx, uint32_t iolen) { vq_relchain_prepare(vq, idx, iolen); vq_relchain_publish(vq); } /* * Driver has finished processing "available" chains and calling * vq_relchain on each one. If driver used all the available * chains, used_all should be set. * * If the "used" index moved we may need to inform the guest, i.e., * deliver an interrupt. Even if the used index did NOT move we * may need to deliver an interrupt, if the avail ring is empty and * we are supposed to interrupt on empty. * * Note that used_all_avail is provided by the caller because it's * a snapshot of the ring state when he decided to finish interrupt * processing -- it's possible that descriptors became available after * that point. (It's also typically a constant 1/True as well.) */ void vq_endchains(struct vqueue_info *vq, int used_all_avail) { struct virtio_softc *vs; uint16_t event_idx, new_idx, old_idx; int intr; /* * Interrupt generation: if we're using EVENT_IDX, * interrupt if we've crossed the event threshold. * Otherwise interrupt is generated if we added "used" entries, * but suppressed by VRING_AVAIL_F_NO_INTERRUPT. * * In any case, though, if NOTIFY_ON_EMPTY is set and the * entire avail was processed, we need to interrupt always. */ vs = vq->vq_vs; old_idx = vq->vq_save_used; vq->vq_save_used = new_idx = vq->vq_used->idx; /* * Use full memory barrier between "idx" store from preceding * vq_relchain() call and the loads from VQ_USED_EVENT_IDX() or * "flags" field below. */ atomic_thread_fence_seq_cst(); if (used_all_avail && (vs->vs_negotiated_caps & VIRTIO_F_NOTIFY_ON_EMPTY)) { intr = 1; } else if (vs->vs_negotiated_caps & VIRTIO_RING_F_EVENT_IDX) { event_idx = VQ_USED_EVENT_IDX(vq); /* * This calculation is per docs and the kernel * (see src/sys/dev/virtio/virtio_ring.h). */ intr = (uint16_t)(new_idx - event_idx - 1) < (uint16_t)(new_idx - old_idx); } else { intr = new_idx != old_idx && !(vq->vq_avail->flags & VRING_AVAIL_F_NO_INTERRUPT); } if (intr) vq_interrupt(vs, vq); } /* Note: these are in sorted order to make for a fast search */ static struct config_reg { uint16_t cr_offset; /* register offset */ uint8_t cr_size; /* size (bytes) */ uint8_t cr_ro; /* true => reg is read only */ const char *cr_name; /* name of reg */ } legacy_cfg_regs[] = { { VIRTIO_PCI_HOST_FEATURES, 4, 1, "HOST_FEATURES" }, { VIRTIO_PCI_GUEST_FEATURES, 4, 0, "GUEST_FEATURES" }, { VIRTIO_PCI_QUEUE_PFN, 4, 0, "QUEUE_PFN" }, { VIRTIO_PCI_QUEUE_NUM, 2, 1, "QUEUE_NUM" }, { VIRTIO_PCI_QUEUE_SEL, 2, 0, "QUEUE_SEL" }, { VIRTIO_PCI_QUEUE_NOTIFY, 2, 0, "QUEUE_NOTIFY" }, { VIRTIO_PCI_STATUS, 1, 0, "STATUS" }, { VIRTIO_PCI_ISR, 1, 0, "ISR" }, { VIRTIO_MSI_CONFIG_VECTOR, 2, 0, "CONFIG_VECTOR" }, { VIRTIO_MSI_QUEUE_VECTOR, 2, 0, "QUEUE_VECTOR" }, }, common_cfg_regs[] = { { VIRTIO_PCI_COMMON_DFSELECT, 4, 0, "DFSELECT" }, { VIRTIO_PCI_COMMON_DF, 4, 1, "DF" }, { VIRTIO_PCI_COMMON_GFSELECT, 4, 0, "GFSELECT" }, { VIRTIO_PCI_COMMON_GF, 4, 0, "GF" }, { VIRTIO_PCI_COMMON_MSIX, 2, 0, "MSIX" }, { VIRTIO_PCI_COMMON_NUMQ, 2, 1, "NUMQ" }, { VIRTIO_PCI_COMMON_STATUS, 1, 0, "STATUS" }, { VIRTIO_PCI_COMMON_CFGGENERATION, 1, 1, "CFGGENERATION" }, { VIRTIO_PCI_COMMON_Q_SELECT, 2, 0, "Q_SELECT" }, { VIRTIO_PCI_COMMON_Q_SIZE, 2, 0, "Q_SIZE" }, { VIRTIO_PCI_COMMON_Q_MSIX, 2, 0, "Q_MSIX" }, { VIRTIO_PCI_COMMON_Q_ENABLE, 2, 0, "Q_ENABLE" }, { VIRTIO_PCI_COMMON_Q_NOFF, 2, 1, "Q_NOFF" }, { VIRTIO_PCI_COMMON_Q_DESCLO, 4, 0, "Q_DESCLO" }, { VIRTIO_PCI_COMMON_Q_DESCHI, 4, 0, "Q_DESCHI" }, { VIRTIO_PCI_COMMON_Q_AVAILLO, 4, 0, "Q_AVAILLO" }, { VIRTIO_PCI_COMMON_Q_AVAILHI, 4, 0, "Q_AVAILHI" }, { VIRTIO_PCI_COMMON_Q_USEDLO, 4, 0, "Q_USEDLO" }, { VIRTIO_PCI_COMMON_Q_USEDHI, 4, 0, "Q_USEDHI" }, }; static inline struct config_reg * vi_find_cr(struct config_reg *regstbl, size_t n, int offset) { u_int hi, lo, mid; struct config_reg *cr; lo = 0; hi = n - 1; while (hi >= lo) { mid = (hi + lo) >> 1; cr = ®stbl[mid]; if (cr->cr_offset == offset) return (cr); if (cr->cr_offset < offset) lo = mid + 1; else hi = mid - 1; } return (NULL); } static uint64_t vi_hv_features(struct virtio_softc *vs, bool modern) { return (modern ? vs->vs_vc->vc_hv_caps_modern | VIRTIO_F_VERSION_1 : vs->vs_vc->vc_hv_caps_legacy); } /* * Handle legacy pci config space reads. * * If it's part of the legacy virtio config structure, do that. * Otherwise dispatch to the actual device backend's config read * callback. */ static uint64_t vi_legacy_pci_read(struct virtio_softc *vs, uint64_t offset, int size) { struct virtio_consts *vc; struct config_reg *cr; uint64_t virtio_config_size; const char *name; uint32_t newoff; uint32_t value; int error; /* Checked by caller */ assert(size == 1 || size == 2 || size == 4); vc = vs->vs_vc; name = vc->vc_name; value = VI_MASK(size); virtio_config_size = VIRTIO_PCI_CONFIG_OFF(pci_msix_enabled(vs->vs_pi)); if (offset >= virtio_config_size) { /* * Subtract off the standard size (including MSI-X * registers if enabled) and dispatch to underlying driver. * If that fails, fall into general code. */ newoff = offset - virtio_config_size; if (newoff + size > vc->vc_cfgsize) goto bad; if (vc->vc_cfgread != NULL) { error = (*vc->vc_cfgread)(DEV_SOFTC(vs), newoff, size, &value); } else { error = 0; } if (error == 0) { DPRINTF(vs, "VIRTIO %s LEGACY PCI devcfg read[0x%" PRIx64 "] = 0x%x", name, newoff, value); goto done; } } bad: cr = vi_find_cr(legacy_cfg_regs, nitems(legacy_cfg_regs), offset); if (cr == NULL || cr->cr_size != size) { if (cr != NULL) { /* offset must be OK, so size must be bad */ EPRINTLN( "%s: read from %s: bad size %d", name, cr->cr_name, size); } else { EPRINTLN( "%s: read from bad offset/size %jd/%d", name, (uintmax_t)offset, size); } goto done; } switch (offset) { case VIRTIO_PCI_HOST_FEATURES: /* Caps for legacy PCI configuration layout is only 32bit */ if (vc->vc_hv_features != NULL) value = vc->vc_hv_features(DEV_SOFTC(vs), false); else value = vi_hv_features(vs, false); break; case VIRTIO_PCI_GUEST_FEATURES: value = vs->vs_negotiated_caps; break; case VIRTIO_PCI_QUEUE_PFN: if (!vi_is_modern(vs) && vs->vs_curq < vc->vc_nvq) value = vs->vs_queues[vs->vs_curq].vq_pfn; break; case VIRTIO_PCI_QUEUE_NUM: value = vs->vs_curq < vc->vc_nvq ? vs->vs_queues[vs->vs_curq].vq_qsize : 0; break; case VIRTIO_PCI_QUEUE_SEL: value = vs->vs_curq; break; case VIRTIO_PCI_QUEUE_NOTIFY: value = 0; /* XXX */ break; case VIRTIO_PCI_STATUS: value = vs->vs_status; break; case VIRTIO_PCI_ISR: value = vs->vs_isr; vs->vs_isr = 0; /* a read clears this flag */ if (value != 0) pci_lintr_deassert(vs->vs_pi); break; case VIRTIO_MSI_CONFIG_VECTOR: value = vs->vs_msix_cfg_idx; break; case VIRTIO_MSI_QUEUE_VECTOR: value = vs->vs_curq < vc->vc_nvq ? vs->vs_queues[vs->vs_curq].vq_msix_idx : VIRTIO_MSI_NO_VECTOR; break; } DPRINTF(vs, "VIRTIO %s LEGACY READ %s = 0x%x", name, cr->cr_name, value); switch (offset) { case VIRTIO_PCI_GUEST_FEATURES: case VIRTIO_PCI_HOST_FEATURES: vi_print_caps(vs, value); break; } done: return (value); } /* * Handle legacy pci config space writes. * * If it's part of the legacy virtio config structure, do that. * Otherwise dispatch to the actual device backend's config write * callback. */ static void vi_legacy_pci_write(struct virtio_softc *vs, uint64_t offset, int size, uint64_t value) { struct vqueue_info *vq; struct virtio_consts *vc; struct config_reg *cr; uint64_t virtio_config_size; const char *name; uint32_t newoff; int error; /* Checked by caller */ assert(size == 1 || size == 2 || size == 4); vc = vs->vs_vc; name = vc->vc_name; virtio_config_size = VIRTIO_PCI_CONFIG_OFF(pci_msix_enabled(vs->vs_pi)); if (offset >= virtio_config_size) { /* * Subtract off the standard size (including MSI-X * registers if enabled) and dispatch to underlying driver. */ newoff = offset - virtio_config_size; if (newoff + size > vc->vc_cfgsize) goto bad; if (vc->vc_cfgwrite != NULL) { error = (*vc->vc_cfgwrite)(DEV_SOFTC(vs), newoff, size, value); } else { error = 0; } if (error == 0) { DPRINTF(vs, "VIRTIO %s LEGACY PCI devcfg write[0x%" PRIx64 "] = 0x%x", name, newoff, value); return; } } bad: cr = vi_find_cr(legacy_cfg_regs, nitems(legacy_cfg_regs), offset); if (cr == NULL || cr->cr_size != size || cr->cr_ro) { if (cr != NULL) { /* offset must be OK, wrong size and/or reg is R/O */ if (cr->cr_size != size) EPRINTLN( "%s: write to %s: bad size %d", name, cr->cr_name, size); if (cr->cr_ro) EPRINTLN( "%s: write to read-only reg %s", name, cr->cr_name); } else { EPRINTLN( "%s: write to bad offset/size %jd/%d", name, (uintmax_t)offset, size); } return; } DPRINTF(vs, "VIRTIO %s LEGACY WRITE %s = 0x%x", name, cr->cr_name, value); switch (offset) { case VIRTIO_PCI_GUEST_FEATURES: if (vc->vc_hv_features != NULL) value &= vc->vc_hv_features(DEV_SOFTC(vs), false); else value &= vi_hv_features(vs, false); vs->vs_negotiated_caps = value; if (vc->vc_apply_features != NULL) { (*vc->vc_apply_features)(DEV_SOFTC(vs), &vs->vs_negotiated_caps); } DPRINTF(vs, "NEGOTIATED FEATURES 0x%" PRIx64 " (%s)", vs->vs_negotiated_caps, vi_is_modern(vs) ? "modern" : "legacy"); vi_print_caps(vs, vs->vs_negotiated_caps); break; case VIRTIO_PCI_QUEUE_PFN: if (vs->vs_curq >= vc->vc_nvq) goto bad_qindex; if (vc->vc_qinit != NULL) vc->vc_qinit(DEV_SOFTC(vs), value, false); else vi_legacy_vq_init(vs, value); break; case VIRTIO_PCI_QUEUE_SEL: /* * Note that the guest is allowed to select an * invalid queue; we just need to return a QNUM * of 0 while the bad queue is selected. */ vs->vs_curq = value; break; case VIRTIO_PCI_QUEUE_NOTIFY: if (value >= (unsigned int)vc->vc_nvq) { EPRINTLN("%s: queue %d notify out of range", name, (int)value); break; } if ((vs->vs_flags & VIRTIO_BROKEN) != 0) break; vq = &vs->vs_queues[value]; if (vq->vq_notify != NULL) { (*vq->vq_notify)(DEV_SOFTC(vs), vq); } else if (vc->vc_qnotify != NULL) { (*vc->vc_qnotify)(DEV_SOFTC(vs), vq); } else { EPRINTLN("%s: qnotify queue %d: missing vq/vc notify", name, (int)value); } break; case VIRTIO_PCI_STATUS: vs->vs_status = value; if (value == 0) { DPRINTF(vs, "VIRTIO %s RESET", name); DPRINTF(vs, "**************************************"); vc->vc_reset(DEV_SOFTC(vs)); } break; case VIRTIO_MSI_CONFIG_VECTOR: vs->vs_msix_cfg_idx = value; break; case VIRTIO_MSI_QUEUE_VECTOR: if (vs->vs_curq >= vc->vc_nvq) goto bad_qindex; vq = &vs->vs_queues[vs->vs_curq]; vq->vq_msix_idx = value; if (vc->vc_set_msix != NULL) vc->vc_set_msix(DEV_SOFTC(vs), vs->vs_curq); break; } return; bad_qindex: EPRINTLN( "%s: write config reg %s: curq %d >= max %d", name, cr->cr_name, vs->vs_curq, vc->vc_nvq); } #define VI_HIGH(x) (((x) >> 32) & 0xffffffff) #define VI_LOW(x) ((x) & 0xffffffff) /* * Virtio modern: * Handle pci config space reads to common config structure. */ static uint64_t vi_pci_common_cfg_read(struct virtio_softc *vs, uint64_t offset, int size) { uint64_t value = -1; struct virtio_consts *vc; struct vqueue_info *vq; struct config_reg *cr; const char *name; uint64_t capval = 0; /* Checked by caller */ assert(size == 1 || size == 2 || size == 4); vc = vs->vs_vc; name = vc->vc_name; cr = vi_find_cr(common_cfg_regs, nitems(common_cfg_regs), offset); if (cr == NULL) { EPRINTLN("%s: read from bad offset/size 0x%jx/0x%x", name, (uintmax_t)offset, size); goto done; } /* * We check that the requested size matches the register at this * offset, and refuse to process it if there is a mismatch. */ if (cr->cr_size != size) { EPRINTLN("%s: read from %s: bad size 0x%x", name, cr->cr_name, size); goto done; } vq = (vs->vs_curq < vc->vc_nvq ? &vs->vs_queues[vs->vs_curq] : NULL); switch (offset) { case VIRTIO_PCI_COMMON_DFSELECT: value = vs->vs_dfselect; break; case VIRTIO_PCI_COMMON_DF: if (vc->vc_hv_features != NULL) value = vc->vc_hv_features(DEV_SOFTC(vs), true); else value = vi_hv_features(vs, true); switch (vs->vs_dfselect) { case 0: capval = value = VI_LOW(value); break; case 1: value = VI_HIGH(value); capval = value << 32; break; default: value = capval = 0; break; } /* capval is debug printed below */ break; case VIRTIO_PCI_COMMON_GFSELECT: value = vs->vs_gfselect; break; case VIRTIO_PCI_COMMON_GF: value = vs->vs_negotiated_caps; switch (vs->vs_gfselect) { case 0: capval = value = VI_LOW(value); break; case 1: value = VI_HIGH(value); capval = value << 32; break; default: value = capval = 0; break; } /* capval is debug printed below */ break; case VIRTIO_PCI_COMMON_MSIX: value = vs->vs_msix_cfg_idx; break; case VIRTIO_PCI_COMMON_NUMQ: value = vc->vc_nvq; break; case VIRTIO_PCI_COMMON_STATUS: value = vs->vs_status; break; case VIRTIO_PCI_COMMON_CFGGENERATION: if ((vs->vs_flags & VIRTIO_DEVCFG_CHG) != 0) { vs->vs_devcfg_gen++; vs->vs_flags &= ~VIRTIO_DEVCFG_CHG; } value = vs->vs_devcfg_gen; break; case VIRTIO_PCI_COMMON_Q_SELECT: value = vs->vs_curq; break; case VIRTIO_PCI_COMMON_Q_SIZE: value = vq != NULL ? vq->vq_qsize : 0; break; case VIRTIO_PCI_COMMON_Q_MSIX: if (vq != NULL) value = vq->vq_msix_idx; break; case VIRTIO_PCI_COMMON_Q_ENABLE: value = vq != NULL ? !!(vq->vq_flags & VQ_ENABLED) : 0; break; case VIRTIO_PCI_COMMON_Q_NOFF: /* queue_notify_off is equal to qid for now */ value = vs->vs_curq; break; case VIRTIO_PCI_COMMON_Q_DESCLO: if (vq != NULL) value = VI_LOW(vq->vq_desc_gpa); break; case VIRTIO_PCI_COMMON_Q_DESCHI: if (vq != NULL) value = VI_HIGH(vq->vq_desc_gpa); break; case VIRTIO_PCI_COMMON_Q_AVAILLO: if (vq != NULL) value = VI_LOW(vq->vq_avail_gpa); break; case VIRTIO_PCI_COMMON_Q_AVAILHI: if (vq != NULL) value = VI_HIGH(vq->vq_avail_gpa); break; case VIRTIO_PCI_COMMON_Q_USEDLO: if (vq != NULL) value = VI_LOW(vq->vq_used_gpa); break; case VIRTIO_PCI_COMMON_Q_USEDHI: if (vq != NULL) value = VI_HIGH(vq->vq_used_gpa); break; } done: value &= VI_MASK(size); DPRINTF(vs, "VIRTIO %s COMMON %s read = 0x%x", name, cr->cr_name, value); switch (offset) { case VIRTIO_PCI_COMMON_DF: case VIRTIO_PCI_COMMON_GF: vi_print_caps(vs, capval); break; } return (value); } /* * Virtio modern: * Handle pci config space writes to common config structure. */ static void vi_pci_common_cfg_write(struct virtio_softc *vs, uint64_t offset, int size, uint64_t value) { uint64_t capval = 0; struct virtio_consts *vc; struct vqueue_info *vq; struct config_reg *cr; const char *name; /* Checked by caller */ assert(size == 1 || size == 2 || size == 4); vc = vs->vs_vc; name = vc->vc_name; value &= VI_MASK(size); cr = vi_find_cr(common_cfg_regs, nitems(common_cfg_regs), offset); if (cr == NULL) { EPRINTLN( "%s: write to %s: bad size 0x%x", name, cr->cr_name, size); return; } /* * We check that the requested size matches the register at this * offset, and refuse to process it if there is a mismatch. */ if (cr->cr_size != size) { EPRINTLN("%s: write to bad offset/size 0x%jx/0x%x", name, (uintmax_t)offset, size); return; } DPRINTF(vs, "VIRTIO %s COMMON %s write 0x%x", name, cr->cr_name, value); vq = NULL; switch (offset) { case VIRTIO_PCI_COMMON_Q_SIZE: case VIRTIO_PCI_COMMON_Q_MSIX: case VIRTIO_PCI_COMMON_Q_ENABLE: case VIRTIO_PCI_COMMON_Q_DESCLO: case VIRTIO_PCI_COMMON_Q_DESCHI: case VIRTIO_PCI_COMMON_Q_AVAILLO: case VIRTIO_PCI_COMMON_Q_AVAILHI: case VIRTIO_PCI_COMMON_Q_USEDLO: case VIRTIO_PCI_COMMON_Q_USEDHI: if (vs->vs_curq >= vc->vc_nvq) { EPRINTLN("%s: write queue %d out of range", name, vs->vs_curq); goto bad_write; } vq = &vs->vs_queues[vs->vs_curq]; break; default: break; } switch (offset) { case VIRTIO_PCI_COMMON_DFSELECT: vs->vs_dfselect = value; break; case VIRTIO_PCI_COMMON_GFSELECT: vs->vs_gfselect = value; break; case VIRTIO_PCI_COMMON_GF: switch (vs->vs_gfselect) { case 0: capval = value; vs->vs_negotiated_caps = (VI_HIGH(vs->vs_negotiated_caps) << 32) | value; break; case 1: capval = value << 32; vs->vs_negotiated_caps = capval | VI_LOW(vs->vs_negotiated_caps); break; default: capval = 0; break; } vi_print_caps(vs, capval); uint64_t hvfeat; if (vc->vc_hv_features != NULL) hvfeat = vc->vc_hv_features(DEV_SOFTC(vs), true); else hvfeat = vi_hv_features(vs, true); vs->vs_negotiated_caps &= hvfeat; break; case VIRTIO_PCI_COMMON_MSIX: vs->vs_msix_cfg_idx = value; break; case VIRTIO_PCI_COMMON_STATUS: if (value == 0) { DPRINTF(vs, "VIRTIO %s RESET", name); (*vc->vc_reset)(DEV_SOFTC(vs)); vs->vs_status = value; break; } if ((vs->vs_status & VIRTIO_CONFIG_S_FEATURES_OK) == 0 && (value & VIRTIO_CONFIG_S_FEATURES_OK) != 0) { if (vc->vc_apply_features != NULL) { (*vc->vc_apply_features)(DEV_SOFTC(vs), &vs->vs_negotiated_caps); } DPRINTF(vs, "NEGOTIATED FEATURES 0x%" PRIx64 " (%s)", vs->vs_negotiated_caps, vi_is_modern(vs) ? "modern" : "legacy"); vi_print_caps(vs, vs->vs_negotiated_caps); } vs->vs_status = value; break; case VIRTIO_PCI_COMMON_Q_SELECT: if (value >= vc->vc_nvq) { EPRINTLN("%s: queue select %d out of range", name, (int)value); goto bad_write; } vs->vs_curq = value; break; case VIRTIO_PCI_COMMON_Q_SIZE: /* * If the guest has passed us a queue size that is not a power * of two, something is very wrong. */ if (!ISP2(value)) { vi_error(vs, "Bad queue size 0x%" PRIx64 " for qid 0x%x, not power of 2", value, vq->vq_num); } else { vq->vq_qsize = value; } break; case VIRTIO_PCI_COMMON_Q_MSIX: vq->vq_msix_idx = value; if (vc->vc_set_msix != NULL) vc->vc_set_msix(DEV_SOFTC(vs), vs->vs_curq); break; case VIRTIO_PCI_COMMON_Q_ENABLE: if ((vq->vq_flags & VQ_ENABLED) == 0 && value == 1) { if (vc->vc_qinit != NULL) vc->vc_qinit(DEV_SOFTC(vs), 0, true); else vi_vq_init(vs); vq->vq_flags |= VQ_ENABLED; } else if (value == 0) { /* * The driver is not permitted to write a 0 to this * register. We choose to ignore it rather than fault * the device. */ } break; case VIRTIO_PCI_COMMON_Q_DESCLO: vq->vq_desc_gpa = (VI_HIGH(vq->vq_desc_gpa) << 32) | value; break; case VIRTIO_PCI_COMMON_Q_DESCHI: vq->vq_desc_gpa = (value << 32) | VI_LOW(vq->vq_desc_gpa); break; case VIRTIO_PCI_COMMON_Q_AVAILLO: vq->vq_avail_gpa = (VI_HIGH(vq->vq_avail_gpa) << 32) | value; break; case VIRTIO_PCI_COMMON_Q_AVAILHI: vq->vq_avail_gpa = (value << 32) | VI_LOW(vq->vq_avail_gpa); break; case VIRTIO_PCI_COMMON_Q_USEDLO: vq->vq_used_gpa = (VI_HIGH(vq->vq_used_gpa) << 32) | value; break; case VIRTIO_PCI_COMMON_Q_USEDHI: vq->vq_used_gpa = (value << 32) | VI_LOW(vq->vq_used_gpa); break; default: EPRINTLN("%s: write to bad offset/size %jd/%d", name, (uintmax_t)offset, size); goto bad_write; } return; bad_write: return; } /* * Virtio modern: * Handle pci MMIO reads to the notification structure. * * Reading the structure always returns zero. */ static uint64_t vi_pci_notify_cfg_read(struct virtio_softc *vs, uint64_t offset, int size) { return (0); } /* * Virtio modern: * Handle pci MMIO writes to the notification structure. * * VIRTIO_F_NOTIFICATION_DATA is not a feature that this device advertises * so we only need to consider the simple case where the vq index is written * into the registers. */ static void vi_pci_notify_cfg_write(struct virtio_softc *vs, uint64_t offset, int size, uint64_t value) { struct virtio_consts *vc = vs->vs_vc; const char *name = vc->vc_name; unsigned int qid = value; struct vqueue_info *vq; DPRINTF(vs, "VIRTIO %s notify VQ 0x%x offset 0x%x", name, qid, offset); if (size != 2) { EPRINTLN("%s: bad size 0x%x access at offset 0x%" PRIx64, name, size, offset); return; } if ((vs->vs_status & VIRTIO_CONFIG_STATUS_DRIVER_OK) == 0) { EPRINTLN("%s: attempt to use VQ 0x%x before DRIVER_OK, " "driver confused?", name, qid); return; } if ((vs->vs_flags & VIRTIO_BROKEN) != 0) { EPRINTLN("%s: attempt to use VQ 0x%x while VIRTIO device is " "flagged as broken", name, qid); return; } if (offset != qid * VQ_NOTIFY_OFF_MULTIPLIER) { EPRINTLN( "%s: VQ 0x%x notify does not have matching offset at 0x%" PRIx64, name, qid, offset); return; } if (qid >= vc->vc_nvq) { EPRINTLN("%s: VQ 0x%x notify out of range", name, qid); return; } vq = &vs->vs_queues[qid]; if ((vq->vq_flags & VQ_ENABLED) == 0) return; if (vq->vq_notify != NULL) (*vq->vq_notify)(DEV_SOFTC(vs), vq); else if (vc->vc_qnotify != NULL) (*vc->vc_qnotify)(DEV_SOFTC(vs), vq); else EPRINTLN("%s: qnotify VQ 0x%x: no vq/vc notify", name, qid); } /* * Virtio modern: * Handle pci MMIO reads to ISR structure. * * The ISR structure has a relaxed requirement on alignment. */ static uint64_t vi_pci_isr_cfg_read(struct virtio_softc *vs, uint64_t offset, int size) { uint64_t value; if (offset != 0) return (0); value = vs->vs_isr; vs->vs_isr = 0; if (value != 0) { DPRINTF(vs, "VIRTIO ISR read[0x%" PRIx64 "] = 0x%x", offset, value); pci_lintr_deassert(vs->vs_pi); } return (value); } /* * Virtio modern: * pci MMIO writes to ISR structure are disallowed. */ static void vi_pci_isr_cfg_write(struct virtio_softc *vs, uint64_t offset, int size, uint64_t value) { const char *name = vs->vs_vc->vc_name; EPRINTLN("%s: invalid write into isr cfg", name); } /* * Virtio modern: * Handle pci MMIO reads to device-specific config structure. */ static uint64_t vi_pci_dev_cfg_read(struct virtio_softc *vs, uint64_t offset, int size) { struct virtio_consts *vc = vs->vs_vc; uint32_t value = VI_MASK(size); if (offset + size > vc->vc_cfgsize) return (value); vc->vc_cfgread(DEV_SOFTC(vs), offset, size, &value); DPRINTF(vs, "VIRTIO %s PCI devcfg read[0x%" PRIx64 "] = 0x%x", vs->vs_vc->vc_name, offset, value); return (value); } /* * Virtio modern: * Handle pci MMIO writes to device-specific config structure. */ static void vi_pci_dev_cfg_write(struct virtio_softc *vs, uint64_t offset, int size, uint64_t value) { struct virtio_consts *vc = vs->vs_vc; value &= VI_MASK(size); if (offset + size > vc->vc_cfgsize) return; if (vc->vc_cfgwrite != NULL) vc->vc_cfgwrite(DEV_SOFTC(vs), offset, size, value); DPRINTF(vs, "VIRTIO %s PCI devcfg write[0x%" PRIx64 "] = 0x%x", vs->vs_vc->vc_name, offset, value); } /* * Handle configuration space reads. */ int vi_pci_cfgread(struct pci_devinst *pi, int offset, int bytes, uint32_t *retval) { struct virtio_softc *vs = pi->pi_arg; virtio_pci_capcfg_t *cfg; uint32_t baroff, barlen; int baridx; cfg = vi_pci_cfg_bycapaddr(vs, offset, bytes); /* If this is not a VirtIO cap, use the default cfgspace handler */ if (cfg == NULL) return (PE_CFGRW_DEFAULT); /* Only the PCI cap has special handling */ if (cfg->c_captype != VIRTIO_PCI_CAP_PCI_CFG) return (PE_CFGRW_DEFAULT); /* and then only the data field */ if (offset != vs->vs_pcicap->c_capoff + offsetof(struct virtio_pci_cfg_cap, pci_cfg_data)) { return (PE_CFGRW_DEFAULT); } if (bytes != 1 && bytes != 2 && bytes != 4) return (PE_CFGRW_DROP); if (vs->vs_mtx) pthread_mutex_lock(vs->vs_mtx); baridx = pci_get_cfgdata8(pi, offset + offsetof(struct virtio_pci_cap, bar)); baroff = pci_get_cfgdata32(pi, offset + offsetof(struct virtio_pci_cap, offset)); barlen = pci_get_cfgdata32(pi, offset + offsetof(struct virtio_pci_cap, length)); if (baridx > PCIR_MAX_BAR_0) { *retval = VI_MASK(bytes); goto done; } *retval = vi_modern_pci_read(vs, baridx, baroff, barlen); done: if (vs->vs_mtx) pthread_mutex_unlock(vs->vs_mtx); DPRINTF(vs, "VIRTIO %s PCI READ BAR%u[0x%x+%x] = 0x%x", vs->vs_vc->vc_name, baridx, baroff, barlen, *retval); return (PE_CFGRW_DROP); } /* * Handle configuration space writes. */ int vi_pci_cfgwrite(struct pci_devinst *pi, int offset, int bytes, uint32_t val) { struct virtio_softc *vs = pi->pi_arg; virtio_pci_capcfg_t *cfg; uint32_t baroff, barlen; int baridx; cfg = vi_pci_cfg_bycapaddr(vs, offset, bytes); /* If this is not a VirtIO cap, use the default cfgspace handler */ if (cfg == NULL) return (PE_CFGRW_DEFAULT); /* Only the PCI VirtIO cap can be written to */ if (cfg->c_captype != VIRTIO_PCI_CAP_PCI_CFG) return (PE_CFGRW_DROP); /* and then only the data field needs special handling */ if (offset != vs->vs_pcicap->c_capoff + offsetof(struct virtio_pci_cfg_cap, pci_cfg_data)) { return (PE_CFGRW_DEFAULT); } if (bytes != 1 && bytes != 2 && bytes != 4) return (PE_CFGRW_DROP); if (vs->vs_mtx) pthread_mutex_lock(vs->vs_mtx); baridx = pci_get_cfgdata8(pi, offset + offsetof(struct virtio_pci_cap, bar)); baroff = pci_get_cfgdata32(pi, offset + offsetof(struct virtio_pci_cap, offset)); barlen = pci_get_cfgdata32(pi, offset + offsetof(struct virtio_pci_cap, length)); if (baridx > PCIR_MAX_BAR_0) goto done; vi_modern_pci_write(vs, baridx, baroff, barlen, val); done: if (vs->vs_mtx) pthread_mutex_unlock(vs->vs_mtx); DPRINTF(vs, "VIRTIO %s PCI WRITE BAR%x[0x%x+%x] = 0x%x", vs->vs_vc->vc_name, baridx, baroff, barlen, val); return (PE_CFGRW_DROP); } /* * Handle pci config space reads to virtio-related structures */ static uint64_t vi_modern_pci_read(struct virtio_softc *vs, int baridx, uint64_t offset, int size) { virtio_pci_capcfg_t *cfg; uint64_t value = VI_MASK(size); cfg = vi_pci_cfg_bybaraddr(vs, baridx, offset, size); if (cfg == NULL) return (value); offset -= cfg->c_baroff; switch (cfg->c_captype) { case VIRTIO_PCI_CAP_COMMON_CFG: value = vi_pci_common_cfg_read(vs, offset, size); break; case VIRTIO_PCI_CAP_NOTIFY_CFG: value = vi_pci_notify_cfg_read(vs, offset, size); break; case VIRTIO_PCI_CAP_ISR_CFG: value = vi_pci_isr_cfg_read(vs, offset, size); break; case VIRTIO_PCI_CAP_DEVICE_CFG: value = vi_pci_dev_cfg_read(vs, offset, size); break; default: break; } return (value); } /* * Handle pci config space reads to virtio-related structures */ static void vi_modern_pci_write(struct virtio_softc *vs, int baridx, uint64_t offset, int size, uint64_t value) { virtio_pci_capcfg_t *cfg; cfg = vi_pci_cfg_bybaraddr(vs, baridx, offset, size); if (cfg == NULL) return; offset -= cfg->c_baroff; switch (cfg->c_captype) { case VIRTIO_PCI_CAP_COMMON_CFG: vi_pci_common_cfg_write(vs, offset, size, value); break; case VIRTIO_PCI_CAP_NOTIFY_CFG: vi_pci_notify_cfg_write(vs, offset, size, value); break; case VIRTIO_PCI_CAP_ISR_CFG: vi_pci_isr_cfg_write(vs, offset, size, value); break; case VIRTIO_PCI_CAP_DEVICE_CFG: vi_pci_dev_cfg_write(vs, offset, size, value); break; } } /* * Handle virtio bar reads. * * If it's to the MSI-X info, dispatch the reads to the msix handling code. * Otherwise, dispatch the reads to virtio device code. */ uint64_t vi_pci_read(struct pci_devinst *pi, int baridx, uint64_t offset, int size) { struct virtio_softc *vs = pi->pi_arg; uint64_t value; if ((vs->vs_flags & VIRTIO_USE_MSIX) != 0 && (baridx == pci_msix_table_bar(pi) || baridx == pci_msix_pba_bar(pi))) { return (pci_emul_msix_tread(pi, offset, size)); } if (vs->vs_mtx) pthread_mutex_lock(vs->vs_mtx); value = VI_MASK(size); if (size != 1 && size != 2 && size != 4) goto done; switch (baridx) { case VIRTIO_LEGACY_BAR: value = vi_legacy_pci_read(vs, offset, size); break; case VIRTIO_MODERN_BAR: value = vi_modern_pci_read(vs, baridx, offset, size); break; default: break; } done: if (vs->vs_mtx) pthread_mutex_unlock(vs->vs_mtx); return (value); } /* * Handle virtio bar writes. * * If it's to the MSI-X info, dispatch the writes to the msix handling code. * Otherwise, dispatch the writes to virtio device code. */ void vi_pci_write(struct pci_devinst *pi, int baridx, uint64_t offset, int size, uint64_t value) { struct virtio_softc *vs = pi->pi_arg; struct virtio_consts *vc = vs->vs_vc; if ((vs->vs_flags & VIRTIO_USE_MSIX) != 0 && (baridx == pci_msix_table_bar(pi) || baridx == pci_msix_pba_bar(pi))) { if (pci_emul_msix_twrite(pi, offset, size, value) == 0 && vc->vc_update_msix != NULL) { vc->vc_update_msix(DEV_SOFTC(vs), offset); } return; } if (vs->vs_mtx) pthread_mutex_lock(vs->vs_mtx); if (size != 1 && size != 2 && size != 4) goto done; switch (baridx) { case VIRTIO_LEGACY_BAR: vi_legacy_pci_write(vs, offset, size, value); break; case VIRTIO_MODERN_BAR: vi_modern_pci_write(vs, baridx, offset, size, value); break; default: break; } done: if (vs->vs_mtx) pthread_mutex_unlock(vs->vs_mtx); } /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2013 Chris Torek * All rights reserved. * Copyright (c) 2021 The FreeBSD Foundation * * Portions of this software were developed by Ka Ho Ng * under sponsorship of the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * This file and its contents are supplied under the terms of the * Common Development and Distribution License ("CDDL"), version 1.0. * You may only use this file in accordance with the terms of version * 1.0 of the CDDL. * * A full copy of the text of the CDDL should have accompanied this * source. A copy of the CDDL is also available via the Internet at * http://www.illumos.org/license/CDDL. */ /* This file is dual-licensed; see usr/src/contrib/bhyve/LICENSE */ /* * Copyright 2026 Oxide Computer Company */ #ifndef _BHYVE_VIRTIO_H_ #define _BHYVE_VIRTIO_H_ #include #include #include #include /* * Virtio legacy support is derived from several specifications below: * https://github.com/rustyrussell/virtio-spec * http://people.redhat.com/pbonzini/virtio-spec.pdf * * Virtio modern support is authored with the reference below: * https://docs.oasis-open.org/virtio/virtio/v1.2/virtio-v1.2.pdf */ /* * A virtual device has zero or more "virtual queues" (virtqueue). * For a legacy device, each virtqueue uses at least two 4096-byte pages, laid * out thus: * * +-----------------------------------------------+ * | "desc": descriptors, 16 bytes each | * | ----------------------------------------- | * | "avail": 2 uint16; uint16; 1 uint16 | * | ----------------------------------------- | * | pad to 4k boundary | * +-----------------------------------------------+ * | "used": 2 x uint16; elems; 1 uint16 | * | ----------------------------------------- | * | pad to 4k boundary | * +-----------------------------------------------+ * * The number that appears here is always a power of two and is * limited to no more than 32768 (as it must fit in a 16-bit field). * If is sufficiently large, the above will occupy more than * two pages. In any case, all pages must be physically contiguous * within the guest's physical address space. * * The 16-byte "desc" descriptors consist of a 64-bit guest * physical address , a 32-bit length , a 16-bit * , and a 16-bit field (all in guest byte order). * * There are three flags that may be set : * NEXT descriptor is chained, so use its "next" field * WRITE descriptor is for host to write into guest RAM * (else host is to read from guest RAM) * INDIRECT descriptor address field is (guest physical) * address of a linear array of descriptors * * Unless INDIRECT is set, is the number of bytes that may * be read/written from guest physical address . If * INDIRECT is set, WRITE is ignored and provides the length * of the indirect descriptors (and must be a multiple of * 16). Note that NEXT may still be set in the main descriptor * pointing to the indirect, and should be set in each indirect * descriptor that uses the next descriptor (these should generally * be numbered sequentially). However, INDIRECT must not be set * in the indirect descriptors. Upon reaching an indirect descriptor * without a NEXT bit, control returns to the direct descriptors. * * Except inside an indirect, each value must be in the * range [0 .. N) (i.e., the half-open interval). (Inside an * indirect, each must be in the range [0 .. /16).) * * The "avail" data structures reside in the same pages as the * "desc" structures since both together are used by the device to * pass information to the hypervisor's virtual driver. These * begin with a 16-bit field and 16-bit index , then * have 16-bit values, followed by one final 16-bit * field . The entries are simply indices * into the descriptor ring (and thus must meet the same * constraints as each value). However, is counted * up from 0 (initially) and simply wraps around after 65535; it * is taken mod to find the next available entry. * * The "used" ring occupies a separate page or pages, and contains * values written from the virtual driver back to the guest OS. * This begins with a 16-bit and 16-bit , then there * are "vring_used" elements, followed by a 16-bit . * The "vring_used" elements consist of a 32-bit and a * 32-bit (vu_tlen below). The is simply the index of * the head of a descriptor chain the guest made available * earlier, and the is the number of bytes actually written, * e.g., in the case of a network driver that provided a large * receive buffer but received only a small amount of data. * * The two event fields, and , in the * avail and used rings (respectively -- note the reversal!), are * always provided, but are used only if the virtual device * negotiates the VIRTIO_RING_F_EVENT_IDX feature during feature * negotiation. Similarly, both rings provide a flag -- * VRING_AVAIL_F_NO_INTERRUPT and VRING_USED_F_NO_NOTIFY -- in * their field, indicating that the guest does not need an * interrupt, or that the hypervisor driver does not need a * notify, when descriptors are added to the corresponding ring. * (These are provided only for interrupt optimization and need * not be implemented.) */ #define LEGACY_VRING_ALIGN 4096 /* * Virtio legacy: * * The address of any given virtual queue is determined by a single * Page Frame Number register. The guest writes the PFN into the * PCI config space. However, a device that has two or more * virtqueues can have a different PFN, and size, for each queue. * The number of queues is determinable via the PCI config space * VTCFG_R_QSEL register. Writes to QSEL select the queue: 0 means * queue #0, 1 means queue#1, etc. Once a queue is selected, the * remaining PFN and QNUM registers refer to that queue. * * QNUM is a read-only register containing a nonzero power of two * that indicates the (hypervisor's) queue size. Or, if reading it * produces zero, the hypervisor does not have a corresponding * queue. (The number of possible queues depends on the virtual * device. The block device has just one; the network device * provides either two -- 0 = receive, 1 = transmit -- or three, * with 2 = control.) * * PFN is a read/write register giving the physical page address of * the virtqueue in guest memory (the guest must allocate enough space * based on the hypervisor's provided QNUM). * * QNOTIFY is effectively write-only: when the guest writes a queue * number to the register, the hypervisor should scan the specified * virtqueue. (Reading QNOTIFY currently always gets 0). */ /* * PFN register shift amount */ #define LEGACY_VRING_PFN 12 /* * PCI vendor/device IDs */ #define VIRTIO_VENDOR 0x1AF4 #define VIRTIO_DEV_NET 0x1000 #define VIRTIO_DEV_BLOCK 0x1001 #define VIRTIO_DEV_CONSOLE 0x1003 #define VIRTIO_DEV_SCSI 0x1004 #define VIRTIO_DEV_RANDOM 0x1005 #define VIRTIO_DEV_9P 0x1009 #define VIRTIO_DEV_INPUT 0x1052 /* * PCI revision IDs */ #define VIRTIO_REV_INPUT 1 /* * PCI subvendor IDs */ #define VIRTIO_SUBVEN_INPUT 0x108E /* * PCI subdevice IDs */ #define VIRTIO_SUBDEV_INPUT 0x1100 struct pci_devinst; struct vqueue_info; /* * A virtual device, with some number (possibly 0) of virtual * queues and some size (possibly 0) of configuration-space * registers private to the device. The virtio_softc should come * at the front of each "derived class", so that a pointer to the * virtio_softc is also a pointer to the more specific, derived- * from-virtio driver's softc. * * Note: inside each hypervisor virtio driver, changes to these * data structures must be locked against other threads, if any. * Except for PCI config space register read/write, we assume each * driver does the required locking, but we need a pointer to the * lock (if there is one) for PCI config space read/write ops. * * When the guest reads or writes the device's config space, the * generic layer checks for operations on the special registers * described above. If the offset of the register(s) being read * or written is past the CFG area (CFG0 or CFG1), the request is * passed on to the virtual device, after subtracting off the * generic-layer size. (So, drivers can just use the offset as * an offset into "struct config", for instance.) * * (The virtio layer also makes sure that the read or write is to/ * from a "good" config offset, hence vc_cfgsize, and on BAR #0. * However, the driver must verify the read or write size and offset * and that no one is writing a readonly register.) */ typedef enum virtio_flags { VIRTIO_DEBUG = 1 << 0, VIRTIO_USE_MSIX = 1 << 1, VIRTIO_EVENT_IDX = 1 << 2, /* use the event-index values */ VIRTIO_DEVCFG_CHG = 1 << 3, /* Device configuration changed */ VIRTIO_BROKEN = 1 << 4, } virtio_flags_t; typedef enum virtio_mode { VIRTIO_MODE_UNSET = 0, VIRTIO_MODE_LEGACY, VIRTIO_MODE_TRANSITIONAL, VIRTIO_MODE_MODERN } virtio_mode_t; /* * This describes a Virtio PCI capability in config space. */ typedef struct virtio_pci_capcfg { uint8_t c_captype; /* * The offset and length of the capability in config space. */ uint32_t c_capoff; uint32_t c_caplen; /* * The containing BAR, offset and length of the data to which the * capability points. */ uint32_t c_barlen; uint32_t c_baroff; uint8_t c_baridx; } virtio_pci_capcfg_t; struct virtio_softc { struct virtio_consts *vs_vc; /* constants (see below) */ virtio_flags_t vs_flags; /* VIRTIO_* flags from above */ virtio_mode_t vs_mode; /* VIRTIO_MODE_* values from above */ pthread_mutex_t *vs_mtx; /* POSIX mutex, if any */ struct pci_devinst *vs_pi; /* PCI device instance */ uint64_t vs_negotiated_caps; /* negotiated capabilities */ struct vqueue_info *vs_queues; /* one per vc_nvq */ int vs_curq; /* current queue */ uint8_t vs_status; /* value from last status write */ uint8_t vs_isr; /* ISR flags, if not MSI-X */ uint16_t vs_msix_cfg_idx; /* MSI-X vector for config event */ uint32_t vs_dfselect; /* Current DFSELECT value */ uint32_t vs_gfselect; /* Current GFSELECT value */ uint8_t vs_devcfg_gen; /* Generation of device config space */ virtio_pci_capcfg_t vs_caps[VIRTIO_PCI_CAP_MAX]; /* PCI capabilities */ virtio_pci_capcfg_t *vs_pcicap; /* PCI configuration access cap */ uint_t vs_ncaps; /* Number of PCI capabilities */ }; #define VS_LOCK(vs) \ do { \ if (vs->vs_mtx) \ pthread_mutex_lock(vs->vs_mtx); \ } while (0) #define VS_UNLOCK(vs) \ do { \ if (vs->vs_mtx) \ pthread_mutex_unlock(vs->vs_mtx); \ } while (0) /* * To aid debugging we allow drivers to provide a table to map feature bits to * text. */ typedef struct virtio_capstr { uint64_t vp_flag; const char *vp_name; } virtio_capstr_t; struct virtio_consts { const char *vc_name; /* name of driver (for diagnostics) */ int vc_nvq; /* current number of virtual queues */ int vc_max_nvq; /* max no. queues, for multi-queue */ size_t vc_cfgsize; /* size of dev-specific config regs */ void (*vc_reset)(void *); /* called on virtual device reset */ void (*vc_qinit)(void *, uint64_t, bool); void (*vc_qnotify)(void *, struct vqueue_info *); /* called on QNOTIFY if no VQ notify */ int (*vc_cfgread)(void *, int, int, uint32_t *); /* called to read config regs */ int (*vc_cfgwrite)(void *, int, int, uint32_t); /* called to write config regs */ void (*vc_apply_features)(void *, uint64_t *); /* called to apply negotiated features */ uint64_t (*vc_hv_features)(void *, bool); /* called to read device features */ void (*vc_set_msix)(void *, int); void (*vc_update_msix)(void *, uint64_t); uint64_t vc_hv_caps_legacy; /* hypervisor-provided capabilities (legacy) */ uint64_t vc_hv_caps_modern; /* hypervisor-provided capabilities (modern) */ /* * Optional feature bit map. */ size_t vc_ncapstr; virtio_capstr_t *vc_capstr; }; /* * Data structure allocated (statically) per virtual queue. * * Drivers may change vq_qsize after a reset. When the guest OS * requests a device reset, the hypervisor first calls * vs->vs_vc->vc_reset(); then the data structure below is * reinitialized (for each virtqueue: vs->vs_vc->vc_nvq). * * The remaining fields should only be fussed-with by the generic * code. * * Note: the addresses of vq_desc, vq_avail, and vq_used are all * computable from each other in the legacy interface, but even * there it's a lot simpler if we just keep a pointer to * each one. The event indices are similarly (but more * easily) computable, and this time we'll compute them: * they're just XX_ring[N]. */ #define VQ_ALLOC 0x01 /* set once we have a pfn */ #define VQ_ENABLED 0x02 /* set if the queue was enabled */ struct vqueue_info { uint16_t vq_qsize; /* size of this queue (a power of 2) */ void (*vq_notify)(void *, struct vqueue_info *); /* called instead of vc_notify, if not NULL */ struct virtio_softc *vq_vs; /* backpointer to softc */ uint16_t vq_num; /* we're the num'th queue in the softc */ uint16_t vq_flags; /* flags (see above) */ uint16_t vq_last_avail; /* a recent value of vq_avail->idx */ uint16_t vq_next_used; /* index of the next used slot to be filled */ uint16_t vq_save_used; /* saved vq_used->idx; see vq_endchains */ uint16_t vq_msix_idx; /* MSI-X index, or VIRTIO_MSI_NO_VECTOR */ uint32_t vq_pfn; /* PFN of virt queue (not shifted!) */ uint64_t vq_desc_gpa; /* PA of virtqueue descriptors ring */ uint64_t vq_avail_gpa; /* PA of virtqueue avail ring */ uint64_t vq_used_gpa; /* PA of virtqueue used ring */ struct vring_desc *vq_desc; /* descriptor array */ struct vring_avail *vq_avail; /* the "avail" ring */ struct vring_used *vq_used; /* the "used" ring */ }; /* * As noted above, these are sort of backwards, name-wise. * * Endian helpers must be used when using the following macros. */ #define VQ_AVAIL_EVENT_IDX(vq) \ (*(uint16_t *)&(vq)->vq_used->ring[(vq)->vq_qsize]) #define VQ_USED_EVENT_IDX(vq) \ ((vq)->vq_avail->ring[(vq)->vq_qsize]) /* * Is this ring ready for I/O? */ static inline int vq_ring_ready(struct vqueue_info *vq) { return (vq->vq_flags & VQ_ALLOC); } /* * Are there "available" descriptors? (This does not count * how many, just returns True if there are some.) */ static inline int vq_has_descs(struct vqueue_info *vq) { return (vq_ring_ready(vq) && vq->vq_last_avail != vq->vq_avail->idx); } /* * Deliver an interrupt to the guest for a specific MSI-X queue or * event. */ static inline void vi_interrupt(struct virtio_softc *vs, uint8_t isr, uint16_t msix_idx) { if (!(vs->vs_status & VIRTIO_CONFIG_STATUS_DRIVER_OK)) return; if (pci_msix_enabled(vs->vs_pi)) { pci_generate_msix(vs->vs_pi, msix_idx); } else { vs->vs_isr |= isr; pci_generate_msi(vs->vs_pi, 0); pci_lintr_assert(vs->vs_pi); } } /* * Deliver an interrupt to the guest on the given virtual queue (if * possible, or a generic MSI interrupt if not using MSI-X). */ static inline void vq_interrupt(struct virtio_softc *vs, struct vqueue_info *vq) { vi_interrupt(vs, VIRTIO_PCI_ISR_INTR, vq->vq_msix_idx); } /* * Deliver an interrupt to guest on device-specific configuration changes * (if possible, or a generic MSI interrupt if not using MSI-X). */ static inline void vq_devcfg_changed(struct virtio_softc *vs) { vs->vs_flags |= VIRTIO_DEVCFG_CHG; vi_interrupt(vs, VIRTIO_PCI_ISR_CONFIG, vs->vs_msix_cfg_idx); } static inline void vq_kick_enable(struct vqueue_info *vq) { vq->vq_used->flags &= ~VRING_USED_F_NO_NOTIFY; /* * Full memory barrier to make sure the store to vq_used->flags * happens before the load from vq_avail->idx, which results from a * subsequent call to vq_has_descs(). */ atomic_thread_fence_seq_cst(); } static inline void vq_kick_disable(struct vqueue_info *vq) { vq->vq_used->flags |= VRING_USED_F_NO_NOTIFY; } #define VIRTIO_LEGACY_BAR 0 /* BAR for virtio legacy cfg regs */ #define VIRTIO_MSIX_BAR 1 /* BAR for host MSI-X tables */ #define VIRTIO_MODERN_BAR 2 /* BAR for virtio modern cfg regs */ struct iovec; /* * Request description returned by vq_getchain. * * Writable iovecs start at iov[req.readable]. */ struct vi_req { int readable; /* num of readable iovecs */ int writable; /* num of writable iovecs */ unsigned int idx; /* ring index */ }; void vi_softc_linkup(struct virtio_softc *vs, struct virtio_consts *vc, void *dev_softc, struct pci_devinst *pi, struct vqueue_info *queues); void vi_queue_linkup(struct virtio_softc *vc, struct vqueue_info *queues); bool vi_intr_init(struct virtio_softc *vs, bool use_msix); void vi_pci_init(struct pci_devinst *, virtio_mode_t, uint16_t, uint16_t, uint8_t); bool vi_pcibar_setup(struct virtio_softc *); virtio_pci_capcfg_t *vi_pci_cfg_bytype(struct virtio_softc *, uint8_t); virtio_pci_capcfg_t *vi_pci_cfg_bycapaddr(struct virtio_softc *, uint32_t, uint32_t); virtio_pci_capcfg_t *vi_pci_cfg_bybaraddr(struct virtio_softc *, uint8_t, uint64_t, uint32_t); void vi_reset_dev(struct virtio_softc *); void vi_set_debug(struct virtio_softc *, bool); bool vi_is_modern(struct virtio_softc *); void vi_error(struct virtio_softc *, const char *, ...) __PRINTFLIKE(2); int vq_getchain(struct vqueue_info *vq, struct iovec *iov, int niov, struct vi_req *reqp); void vq_retchains(struct vqueue_info *vq, uint16_t n_chains); void vq_relchain_prepare(struct vqueue_info *vq, uint16_t idx, uint32_t iolen); void vq_relchain_publish(struct vqueue_info *vq); void vq_relchain(struct vqueue_info *vq, uint16_t idx, uint32_t iolen); void vq_endchains(struct vqueue_info *vq, int used_all_avail); int vi_pci_cfgread(struct pci_devinst *pi, int offset, int bytes, uint32_t *retval); int vi_pci_cfgwrite(struct pci_devinst *pi, int offset, int bytes, uint32_t val); uint64_t vi_pci_read(struct pci_devinst *pi, int baridx, uint64_t offset, int size); void vi_pci_write(struct pci_devinst *pi, int baridx, uint64_t offset, int size, uint64_t value); void vi_vq_init(struct virtio_softc *); void vi_legacy_vq_init(struct virtio_softc *, uint32_t); #endif /* _BHYVE_VIRTIO_H_ */ /* * This file and its contents are supplied under the terms of the * Common Development and Distribution License ("CDDL"), version 1.0. * You may only use this file in accordance with the terms of version * 1.0 of the CDDL. * * A full copy of the text of the CDDL should have accompanied this * source. A copy of the CDDL is also available via the Internet at * http://www.illumos.org/license/CDDL. * * Copyright 2022 OmniOS Community Edition (OmniOSce) Association. */ #ifndef _VIRTIO_NET_H_ #define _VIRTIO_NET_H_ #include "mevent.h" #include "net_backends.h" /* * This structure appears at the start of each control virtqueue request. */ typedef struct virtio_net_ctrl_hdr { uint8_t vnch_class; uint8_t vnch_command; } __packed virtio_net_ctrl_hdr_t; /* * This structure is used for the mac address tables associated with the * VIRTIO_NET_CTRL_MAC class. */ typedef struct virtio_net_ctrl_mac { uint32_t vncm_entries; ether_addr_t vncm_mac; } __packed virtio_net_ctrl_mac_t; /* * This structure is used to pass the number of queues. */ typedef struct virtio_net_ctrl_mq { uint16_t virtqueue_pairs; } __packed virtio_net_ctrl_mq_t; /* * Control Queue Classes */ #define VIRTIO_NET_CTRL_RX 0 #define VIRTIO_NET_CTRL_MAC 1 #define VIRTIO_NET_CTRL_VLAN 2 #define VIRTIO_NET_CTRL_ANNOUNCE 3 #define VIRTIO_NET_CTRL_MQ 4 /* * CTRL_RX commands */ #define VIRTIO_NET_CTRL_RX_PROMISC 0 #define VIRTIO_NET_CTRL_RX_ALLMULTI 1 #define VIRTIO_NET_CTRL_RX_ALLUNI 2 #define VIRTIO_NET_CTRL_RX_NOMULTI 3 #define VIRTIO_NET_CTRL_RX_NOUNI 4 #define VIRTIO_NET_CTRL_RX_NOBCAST 5 /* CTRL_MAC commands */ #define VIRTIO_NET_CTRL_MAC_TABLE_SET 0 #define VIRTIO_NET_CTRL_MAC_ADDR_SET 1 /* CTRL_MQ commands */ #define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET 0 #define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN 1 #define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX 0x8000 /* * Control queue ack values */ #define VIRTIO_NET_CQ_OK 0 #define VIRTIO_NET_CQ_ERR 1 #endif /* _VIRTIO_NET_H_ */ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2011 NetApp, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef _VMEXIT_H_ #define _VMEXIT_H_ extern const vmexit_handler_t vmexit_handlers[VM_EXITCODE_MAX]; #ifndef __FreeBSD__ extern int vmentry_init(int); extern struct vm_entry *vmentry_vcpu(int); #endif #endif /* !_VMEXIT_H_ */ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright 2020 Conrad Meyer . All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "acpi.h" #include "bootrom.h" #include "vmgenc.h" static uint64_t vmgen_gpa; void vmgenc_init(struct vmctx *ctx) { char *region; int error; error = bootrom_alloc(ctx, PAGE_SIZE, PROT_READ, 0, ®ion, &vmgen_gpa); if (error != 0) errx(4, "%s: bootrom_alloc", __func__); /* * It is basically harmless to always generate a random ID when * starting a VM. */ error = getentropy(region, sizeof(struct uuid)); if (error == -1) err(4, "%s: getentropy", __func__); /* XXX When we have suspend/resume/rollback. */ #if 0 acpi_raise_gpe(ctx, GPE_VMGENC); #endif } void vmgenc_write_dsdt(void) { dsdt_line(""); dsdt_indent(1); dsdt_line("Scope (_SB)"); dsdt_line("{"); dsdt_line(" Device (GENC)"); dsdt_line(" {"); dsdt_indent(2); dsdt_line("Name (_CID, \"VM_Gen_Counter\")"); dsdt_line("Method (_HID, 0, NotSerialized)"); dsdt_line("{"); dsdt_line(" Return (\"Bhyve_V_Gen_Counter_V1\")"); dsdt_line("}"); dsdt_line("Name (_UID, 0)"); dsdt_line("Name (_DDN, \"VM_Gen_Counter\")"); dsdt_line("Name (ADDR, Package (0x02)"); dsdt_line("{"); dsdt_line(" 0x%08x,", (uint32_t)vmgen_gpa); dsdt_line(" 0x%08x", (uint32_t)(vmgen_gpa >> 32)); dsdt_line("})"); dsdt_unindent(2); dsdt_line(" }"); /* Device (GENC) */ dsdt_line("}"); /* Scope (_SB) */ dsdt_line(""); dsdt_line("Scope (_GPE)"); dsdt_line("{"); dsdt_line(" Method (_E%02x, 0, NotSerialized)", GPE_VMGENC); dsdt_line(" {"); dsdt_line(" Notify (\\_SB.GENC, 0x80)"); dsdt_line(" }"); dsdt_line("}"); dsdt_unindent(1); } /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright 2020 Conrad Meyer . All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #pragma once void vmgenc_init(struct vmctx *); void vmgenc_write_dsdt(void); # # This file and its contents are supplied under the terms of the # Common Development and Distribution License ("CDDL"), version 1.0. # You may only use this file in accordance with the terms of version # 1.0 of the CDDL. # # A full copy of the text of the CDDL should have accompanied this # source. A copy of the CDDL is also available via the Internet at # http://www.illumos.org/license/CDDL. # # # Copyright 2018 Joyent, Inc. # SUBDIRS = tests include Makefile.subdirs # # This file and its contents are supplied under the terms of the # Common Development and Distribution License ("CDDL"), version 1.0. # You may only use this file in accordance with the terms of version # 1.0 of the CDDL. # # A full copy of the text of the CDDL should have accompanied this # source. A copy of the CDDL is also available via the Internet at # http://www.illumos.org/license/CDDL. # # # Copyright 2019 Joyent, Inc. # Copyright 2022 Oxide Computer Company # include $(SRC)/Makefile.master include $(SRC)/cmd/Makefile.cmd include $(SRC)/cmd/Makefile.cmd.64 # # Force c99 for everything # CSTD= $(CSTD_GNU99) CPPFLAGS = -I$(COMPAT)/bhyve -I$(CONTRIB)/bhyve \ -I$(COMPAT)/bhyve/amd64 -I$(CONTRIB)/bhyve/amd64 \ $(CPPFLAGS.master) \ -I$(SRC)/cmd/bhyve/common \ -DWITHOUT_CAPSICUM SMOFF += all_func_returns CLOBBERFILES += $(PROG) # # Install related definitions # ROOTOPTPKG = $(ROOT)/opt/bhyve-tests ROOTTESTS = $(ROOTOPTPKG)/tests TESTDIR = $(ROOTTESTS)/$(TESTSUBDIR) FILEMODE = 0555 LDLIBS = $(LDLIBS.cmd) # # This file and its contents are supplied under the terms of the # Common Development and Distribution License ("CDDL"), version 1.0. # You may only use this file in accordance with the terms of version # 1.0 of the CDDL. # # A full copy of the text of the CDDL should have accompanied this # source. A copy of the CDDL is also available via the Internet at # http://www.illumos.org/license/CDDL. # # # Copyright 2018 Joyent, Inc. # .KEEP_STATE: all : TARGET += all clean : TARGET += clean clobber : TARGET += clobber install : TARGET += install all clean clobber install: $(SUBDIRS) $(filter-out .WAIT,$(SUBDIRS)): FRC @cd $@; pwd; $(MAKE) $(TARGET) FRC: # # This file and its contents are supplied under the terms of the # Common Development and Distribution License ("CDDL"), version 1.0. # You may only use this file in accordance with the terms of version # 1.0 of the CDDL. # # A full copy of the text of the CDDL should have accompanied this # source. A copy of the CDDL is also available via the Internet at # http://www.illumos.org/license/CDDL. # # # Copyright 2018 Joyent, Inc. # Copyright 2022 Oxide Computer Company # $(ROOTOPTPKG): $(INS.dir) $(ROOTTESTS): $(ROOTOPTPKG) $(INS.dir) $(TESTDIR): $(ROOTTESTS) $(INS.dir) $(TESTDIR)/%: % $(TESTDIR) $(INS.file) clean: -$(RM) *.o $(CLEANFILES) clobber: clean -$(RM) $(CLOBBERFILES) # # This file and its contents are supplied under the terms of the # Common Development and Distribution License ("CDDL"), version 1.0. # You may only use this file in accordance with the terms of version # 1.0 of the CDDL. # # A full copy of the text of the CDDL should have accompanied this # source. A copy of the CDDL is also available via the Internet at # http://www.illumos.org/license/CDDL. # # # Copyright 2018 Joyent, Inc. # SUBDIRS = mevent include ../Makefile.subdirs # # This file and its contents are supplied under the terms of the # Common Development and Distribution License ("CDDL"), version 1.0. # You may only use this file in accordance with the terms of version # 1.0 of the CDDL. # # A full copy of the text of the CDDL should have accompanied this # source. A copy of the CDDL is also available via the Internet at # http://www.illumos.org/license/CDDL. # # # Copyright 2018 Joyent, Inc. # Copyright 2022 Oxide Computer Company # Copyright 2022 OmniOS Community Edition (OmniOSce) Association. # TESTSUBDIR = mevent PROG = \ lists_delete \ mevent_test \ read_disable \ read_pause \ read_requeue \ vnode_file \ vnode_zvol SUPOBJS = mevent.o testlib.o include ../../Makefile.com CMDS = $(PROG:%=$(TESTDIR)/%) $(CMDS) : FILEMODE = 0555 all: $(PROG) install: $(TESTDIR) $(CMDS) $(CMDS): $(PROG) vnode_zvol : LDLIBS += -lzfs -lnvpair mevent_test : LDLIBS += -lsocket include ../../Makefile.targ # Hammerhead: Cancel GNU Make built-in compile-and-link rule (%: %.c) %: %.c %: %.o $(SUPOBJS) $(LINK.c) -o $@ $< $(SUPOBJS) $(LDLIBS) $(POST_PROCESS) mevent.o: ../../../common/mevent.c /* * This file and its contents are supplied under the terms of the * Common Development and Distribution License ("CDDL"), version 1.0. * You may only use this file in accordance with the terms of version * 1.0 of the CDDL. * * A full copy of the text of the CDDL should have accompanied this * source. A copy of the CDDL is also available via the Internet at * http://www.illumos.org/license/CDDL. */ /* * Copyright 2018 Joyent, Inc. */ /* * Test: lists.delete * Assertion: mevent_delete() causes the total number of events to decrease * * Strategy: 1. Create a pipe. * 2. Call mevent_add() to be notified of writes to the pipe. The * callback will do nothing other than generate an error if it * is called. * 3. Create another pipe and add a read event watcher to it. The * callback will signal a cv when called. A write to the pipe * followed by a wait on the cv will ensure that async * operations in mevent.c are complete. See flush_and_wait(). * 4. Call flush_and_wait(), then get event count. * 5. Delete the event created in step 2. * 6. Call flush_and_wait(), then get event count. * 7. Verify result in step 6 is one less than result in step 4. */ #include #include #include #include #include #include #include #include #include #include #include "testlib.h" #include "mevent.h" static pthread_mutex_t mtx = PTHREAD_MUTEX_INITIALIZER; static pthread_cond_t cv = PTHREAD_COND_INITIALIZER; static int get_count(void) { int global = -1, change = -1, del_pending = -1; int total; test_mevent_count_lists(&global, &change, &del_pending); ASSERT_INT_NEQ(("count not set"), global, -1); ASSERT_INT_NEQ(("count not set"), change, -1); ASSERT_INT_NEQ(("count not set"), change, -1); ASSERT_INT_EQ(("pending delete not processed"), del_pending, 0); total = global + change + del_pending; VERBOSE(("count = %d (%d + %d + %d)", total, global, change, del_pending)); return (total); } static void not_called_cb(int fd, enum ev_type ev, void *arg) { FAIL(("this callback should never be called")); } static void flush_cb(int fd, enum ev_type ev, void *arg) { char buf[32]; /* Drain the pipe */ while (read(fd, buf, sizeof (buf)) > 0) ; pthread_mutex_lock(&mtx); pthread_cond_signal(&cv); pthread_mutex_unlock(&mtx); } void flush_and_wait(int fd) { uint8_t msg = 42; /* * Lock taken ahead of waking flush_cb so this thread doesn't race * with the event thread. */ pthread_mutex_lock(&mtx); if (write(fd, &msg, sizeof (msg)) != sizeof (msg)) { FAIL(("bad write")); } /* Wait for it to be read */ pthread_cond_wait(&cv, &mtx); pthread_mutex_unlock(&mtx); } int main(int argc, const char *argv[]) { int unused_pipe[2]; int flush_pipe[2]; struct mevent *unused_evp, *flush_evp; int count1, count2; start_test(argv[0], 5); start_event_thread(); /* * Create first pipe and related event */ if (pipe(unused_pipe) != 0) { FAIL_ERRNO("pipe"); } VERBOSE(("unused_pipe[] = { %d, %d }", unused_pipe[0], unused_pipe[1])); if (fcntl(unused_pipe[0], F_SETFL, O_NONBLOCK) != 0) { FAIL_ERRNO("set pipe nonblocking"); } unused_evp = mevent_add(unused_pipe[0], EVF_READ, not_called_cb, NULL); ASSERT_PTR_NEQ(("mevent_add"), unused_evp, NULL); /* * Create flush pipe and related event */ if (pipe(flush_pipe) != 0) { FAIL_ERRNO("pipe"); } VERBOSE(("flush_pipe[] = { %d, %d }", flush_pipe[0], flush_pipe[1])); if (fcntl(flush_pipe[0], F_SETFL, O_NONBLOCK) != 0) { FAIL_ERRNO("set pipe nonblocking"); } flush_evp = mevent_add(flush_pipe[0], EVF_READ, flush_cb, NULL); ASSERT_PTR_NEQ(("mevent_add"), flush_evp, NULL); /* Get count before delete. */ flush_and_wait(flush_pipe[1]); count1 = get_count(); /* * Delete the first event and flush a read after the delete is * complete. */ if (mevent_delete(unused_evp) != 0) { FAIL_ERRNO("mevent_delete"); } /* * Verify count decreased. */ flush_and_wait(flush_pipe[1]); count2 = get_count(); if (count1 - 1 != count2) { FAIL(("mevent_delete() did not decrease count by 1: " "was %d, now %d", count1, count2)); } PASS(); } /* * This file and its contents are supplied under the terms of the * Common Development and Distribution License ("CDDL"), version 1.0. * You may only use this file in accordance with the terms of version * 1.0 of the CDDL. * * A full copy of the text of the CDDL should have accompanied this * source. A copy of the CDDL is also available via the Internet at * http://www.illumos.org/license/CDDL. */ /* * Copyright 2018 Joyent, Inc. */ #include "../../../common/mevent.c" #include "testlib.h" /* * Returns by reference the number of events on the global and change lists. * * Used by tests that wish to ensure that the event count changes as suggested * by mevent_add() and mevent_delete(). Note that a delete does not immediately * delete an event. Events that are pending delete are included in the change * list until the next pass through the change list to process pending changes. */ void test_mevent_count_lists(int *ret_global, int *ret_change, int *ret_del_pending) { struct mevent *mevp; int global = 0; int change = 0; int del_pending = 0; mevent_qlock(); LIST_FOREACH(mevp, &global_head, me_list) { global++; VERBOSE(("on global: type %d fd %d state %d", mevp->me_type, mevp->me_fd, mevp->me_state)); } LIST_FOREACH(mevp, &change_head, me_list) { change++; if (mevp->me_state == EV_DELETE) { del_pending++; } VERBOSE(("on change: type %d fd %d state %d", mevp->me_type, mevp->me_fd, mevp->me_state)); } mevent_qunlock(); *ret_global = global; *ret_change = change; *ret_del_pending = del_pending; } void set_mevent_file_poll_interval_ms(int ms) { mevent_file_poll_interval_ms = ms; } /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2011 NetApp, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * Copyright 2018 Joyent, Inc. */ /* * Test program for the micro event library. Set up a simple TCP echo * service. * * cc mevent_test.c mevent.c -lpthread */ #include #include #ifdef __FreeBSD__ #include #endif #include #include #ifdef __FreeBSD__ #include #endif #include #include #include #include #include "mevent.h" #define TEST_PORT 4321 static pthread_mutex_t accept_mutex = PTHREAD_MUTEX_INITIALIZER; static pthread_cond_t accept_condvar = PTHREAD_COND_INITIALIZER; static struct mevent *tevp; #define MEVENT_ECHO /* Number of timer events to capture */ #define TEVSZ 4096 uint64_t tevbuf[TEVSZ]; static void timer_print(void) { uint64_t min, max, diff, sum; #ifdef __FreeBSD__ uint64_t tsc_freq; size_t len; #endif int j; min = UINT64_MAX; max = 0; sum = 0; #ifdef __FreeBSD__ len = sizeof(tsc_freq); sysctlbyname("machdep.tsc_freq", &tsc_freq, &len, NULL, 0); #endif for (j = 1; j < TEVSZ; j++) { #ifdef __FreeBSD__ /* Convert a tsc diff into microseconds */ diff = (tevbuf[j] - tevbuf[j-1]) * 1000000 / tsc_freq; #else diff = (tevbuf[j] - tevbuf[j-1]) / 1000; #endif sum += diff; if (min > diff) min = diff; if (max < diff) max = diff; } printf("timers done: usecs, min %ld, max %ld, mean %ld\n", min, max, sum/(TEVSZ - 1)); } static void timer_callback(int fd, enum ev_type type, void *param) { static int i; if (i >= TEVSZ) abort(); #ifdef __FreeBSD__ tevbuf[i++] = rdtsc(); #else tevbuf[i++] = gethrtime(); #endif if (i == TEVSZ) { mevent_delete(tevp); timer_print(); } } #ifdef MEVENT_ECHO struct esync { pthread_mutex_t e_mt; pthread_cond_t e_cond; }; static void echoer_callback(int fd, enum ev_type type, void *param) { struct esync *sync = param; pthread_mutex_lock(&sync->e_mt); pthread_cond_signal(&sync->e_cond); pthread_mutex_unlock(&sync->e_mt); } static void * echoer(void *param) { struct esync sync; struct mevent *mev; char buf[128]; int fd = (int)(uintptr_t) param; int len; pthread_mutex_init(&sync.e_mt, NULL); pthread_cond_init(&sync.e_cond, NULL); pthread_mutex_lock(&sync.e_mt); mev = mevent_add(fd, EVF_READ, echoer_callback, &sync); if (mev == NULL) { printf("Could not allocate echoer event\n"); exit(4); } while (!pthread_cond_wait(&sync.e_cond, &sync.e_mt)) { len = read(fd, buf, sizeof(buf)); if (len > 0) { write(fd, buf, len); write(0, buf, len); } else { break; } } mevent_delete_close(mev); pthread_mutex_unlock(&sync.e_mt); pthread_mutex_destroy(&sync.e_mt); pthread_cond_destroy(&sync.e_cond); return (NULL); } #else static void * echoer(void *param) { char buf[128]; int fd = (int)(uintptr_t) param; int len; while ((len = read(fd, buf, sizeof(buf))) > 0) { write(1, buf, len); } return (NULL); } #endif /* MEVENT_ECHO */ static void acceptor_callback(int fd, enum ev_type type, void *param) { pthread_mutex_lock(&accept_mutex); pthread_cond_signal(&accept_condvar); pthread_mutex_unlock(&accept_mutex); } static void * acceptor(void *param) { struct sockaddr_in sin; pthread_t tid; int news; int s; if ((s = socket(AF_INET, SOCK_STREAM, 0)) < 0) { perror("cannot create socket"); exit(4); } #ifdef __FreeBSD__ sin.sin_len = sizeof(sin); #endif sin.sin_family = AF_INET; sin.sin_addr.s_addr = htonl(INADDR_ANY); sin.sin_port = htons(TEST_PORT); if (bind(s, (struct sockaddr *)&sin, sizeof(sin)) < 0) { perror("cannot bind socket"); exit(4); } if (listen(s, 1) < 0) { perror("cannot listen socket"); exit(4); } (void) mevent_add(s, EVF_READ, acceptor_callback, NULL); pthread_mutex_lock(&accept_mutex); while (!pthread_cond_wait(&accept_condvar, &accept_mutex)) { news = accept(s, NULL, NULL); if (news < 0) { perror("accept error"); } else { static int first = 1; if (first) { /* * Start a timer */ first = 0; tevp = mevent_add(1, EVF_TIMER, timer_callback, NULL); } printf("incoming connection, spawning thread\n"); pthread_create(&tid, NULL, echoer, (void *)(uintptr_t)news); } } return (NULL); } int main() { pthread_t tid; pthread_create(&tid, NULL, acceptor, NULL); mevent_dispatch(); return (0); } /* * This file and its contents are supplied under the terms of the * Common Development and Distribution License ("CDDL"), version 1.0. * You may only use this file in accordance with the terms of version * 1.0 of the CDDL. * * A full copy of the text of the CDDL should have accompanied this * source. A copy of the CDDL is also available via the Internet at * http://www.illumos.org/license/CDDL. */ /* * Copyright 2018 Joyent, Inc. */ /* * Test: read.cancel * Assertion: A read is not requeued if mevent_disable() is called while it is * being handled. * * Strategy: 1. Create a pipe * 2. Call mevent_add() to be notified of writes to the pipe. The * callback will signal a cv. * 3. Write to the pipe then wait for a wakeup. * 4. From the read event callback, disable the event then awaken * the main thread. * 5. In the main thread, add a timer event that will awaken the * main thread after a short delay. * 5. Write to the pipe and wait to be awoken. The wakeup should * come from the timer event, not the read event. */ #include #include #include #include #include #include #include #include #include #include #include "testlib.h" #include "mevent.h" typedef enum { CB_NONE, CB_READ, CB_TIMER, } lastwake_t; static lastwake_t lastwake = CB_NONE; static pthread_mutex_t mtx = PTHREAD_MUTEX_INITIALIZER; static pthread_cond_t cv = PTHREAD_COND_INITIALIZER; static struct mevent *read_event; static void munch(int fd, enum ev_type ev, void *arg) { ssize_t nbytes; char buf[32] = { 0 }; int err; if ((nbytes = read(fd, buf, sizeof (buf))) < 0) { FAIL_ERRNO("bad read"); } VERBOSE(("read %ld bytes '%s'", nbytes, buf)); err = mevent_disable(read_event); ASSERT_INT_EQ(("mevent_disable: ", strerror(err)), err, 0); pthread_mutex_lock(&mtx); ASSERT_INT_EQ(("wrong lastwake"), lastwake, CB_NONE); lastwake = CB_READ; pthread_cond_signal(&cv); VERBOSE(("wakeup")); pthread_mutex_unlock(&mtx); } static void tick(int ms, enum ev_type ev, void *arg) { pthread_mutex_lock(&mtx); ASSERT_INT_EQ(("wrong lastwake"), lastwake, CB_READ); lastwake = CB_TIMER; pthread_cond_signal(&cv); VERBOSE(("wakeup")); pthread_mutex_unlock(&mtx); } int main(int argc, const char *argv[]) { int pipefds[2]; struct mevent *timer; ssize_t written; char *msgs[] = { "first", "second" }; char *msg; start_test(argv[0], 5); start_event_thread(); if (pipe(pipefds) != 0) { FAIL_ERRNO("pipe"); } if (fcntl(pipefds[0], F_SETFL, O_NONBLOCK) != 0) { FAIL_ERRNO("set pipe nonblocking"); } /* * First write */ msg = msgs[0]; read_event = mevent_add(pipefds[0], EVF_READ, munch, msg); ASSERT_PTR_NEQ(("mevent_add pipefd"), read_event, NULL); pthread_mutex_lock(&mtx); written = write(pipefds[1], msg, strlen(msg)); if (written < 0) { FAIL_ERRNO("bad write"); } ASSERT_INT64_EQ(("write '%s' failed", msg), written, strlen(msg)); /* * Wait for it to be read */ pthread_cond_wait(&cv, &mtx); ASSERT_INT_EQ(("wrong lastwake"), lastwake, CB_READ); pthread_mutex_unlock(&mtx); /* * Add timer, second write. */ msg = msgs[1]; timer = mevent_add(50, EVF_TIMER, tick, msg); ASSERT_PTR_NEQ(("mevent_add timer"), timer, NULL); pthread_mutex_lock(&mtx); written = write(pipefds[1], msg, strlen(msg)); if (written < 0) { FAIL_ERRNO("bad write"); } ASSERT_INT64_EQ(("write '%s' failed", msg), written, strlen(msg)); /* * Wait for timer to expire */ pthread_cond_wait(&cv, &mtx); ASSERT_INT_EQ(("wrong lastwake"), lastwake, CB_TIMER); pthread_mutex_unlock(&mtx); PASS(); } /* * This file and its contents are supplied under the terms of the * Common Development and Distribution License ("CDDL"), version 1.0. * You may only use this file in accordance with the terms of version * 1.0 of the CDDL. * * A full copy of the text of the CDDL should have accompanied this * source. A copy of the CDDL is also available via the Internet at * http://www.illumos.org/license/CDDL. */ /* * Copyright 2018 Joyent, Inc. */ /* * Test: read.pause * Assertion: mevent_disable() can be used to pause reads. * * Strategy: 1. Create a pipe * 2. Call mevent_add() to be notified of writes to the pipe. The * callback will signal a cv. * 3. In a loop, write to the pipe then wait on the cv. */ #include #include #include #include #include #include #include #include #include #include #include "testlib.h" #include "mevent.h" static pthread_mutex_t mtx = PTHREAD_MUTEX_INITIALIZER; static pthread_cond_t cv = PTHREAD_COND_INITIALIZER; static char cookie[] = "Chocolate chip with fudge stripes"; /* * After this many bytes are sent, writes will get batched up, progress will be * made on the write side via an interval timer */ const int pauseat = 8; static void munch(int fd, enum ev_type ev, void *arg) { static int i = 0; char buf[sizeof (cookie)] = { 0 }; ssize_t nbytes; ssize_t expected; ASSERT_INT_EQ(("bad event"), ev, EVF_READ); ASSERT_PTR_EQ(("bad cookie"), arg, cookie); /* * For the first while, expect data to come a byte at a time. After the * pause, we should get a burst with the rest of the data. */ if (i > pauseat) { expected = strlen(cookie) - pauseat - 1; } else { expected = 1; } if ((nbytes = read(fd, buf, sizeof (buf))) < 0) { FAIL_ERRNO("bad read"); } VERBOSE(("read %ld bytes '%s'", nbytes, buf)); ASSERT_INT64_EQ(("wanted a byte of cookie"), nbytes, expected); if (expected == 1) { ASSERT_CHAR_EQ(("bad byte %d of cookie", i), buf[0], cookie[i]); } else { ASSERT_STR_EQ(("bad last half of cookie"), buf, &cookie[i]); } pthread_mutex_lock(&mtx); pthread_cond_signal(&cv); VERBOSE(("wakeup")); pthread_mutex_unlock(&mtx); i++; } static void tick(int ms, enum ev_type ev, void *arg) { pthread_mutex_lock(&mtx); pthread_cond_signal(&cv); VERBOSE(("wakeup")); pthread_mutex_unlock(&mtx); } int main(int argc, const char *argv[]) { int pipefds[2]; struct mevent *evp, *timer; ssize_t written; start_test(argv[0], 5); start_event_thread(); if (pipe(pipefds) != 0) { FAIL_ERRNO("pipe"); } if (fcntl(pipefds[0], F_SETFL, O_NONBLOCK) != 0) { FAIL_ERRNO("set pipe nonblocking"); } evp = mevent_add(pipefds[0], EVF_READ, munch, cookie); ASSERT_PTR_NEQ(("mevent_add pipefd"), evp, NULL); for (int i = 0; cookie[i] != 0; i++) { pthread_mutex_lock(&mtx); written = write(pipefds[1], cookie + i, 1); if (written < 0) { FAIL_ERRNO("bad write"); } ASSERT_INT64_EQ(("write byte %d of cookie", i), written, 1); /* Wait for it to be read */ pthread_cond_wait(&cv, &mtx); pthread_mutex_unlock(&mtx); if (i == pauseat) { timer = mevent_add(10, EVF_TIMER, tick, &cookie[pauseat]); ASSERT_PTR_NEQ(("mevent_add timer"), timer, NULL); VERBOSE(("disable munch")); mevent_disable(evp); } } pthread_mutex_lock(&mtx); mevent_enable(evp); pthread_cond_wait(&cv, &mtx); pthread_mutex_unlock(&mtx); PASS(); } /* * This file and its contents are supplied under the terms of the * Common Development and Distribution License ("CDDL"), version 1.0. * You may only use this file in accordance with the terms of version * 1.0 of the CDDL. * * A full copy of the text of the CDDL should have accompanied this * source. A copy of the CDDL is also available via the Internet at * http://www.illumos.org/license/CDDL. */ /* * Copyright 2018 Joyent, Inc. */ /* * Test: read.requeue * Assertion: A sequence of writes turns into a sequence of events. * * Strategy: 1. Create a pipe * 2. Call mevent_add() to be notified of writes to the pipe. The * callback will signal a cv. * 3. In a loop, write to the pipe then wait on the cv. */ #include #include #include #include #include #include #include #include #include #include #include "testlib.h" #include "mevent.h" static pthread_mutex_t mtx = PTHREAD_MUTEX_INITIALIZER; static pthread_cond_t cv = PTHREAD_COND_INITIALIZER; static char *cookie = "Chocolate chip with fudge stripes"; static void munch(int fd, enum ev_type ev, void *arg) { static int i = 0; char buf[8] = { 0 }; ssize_t nbytes; ASSERT_INT_EQ(("bad event"), ev, EVF_READ); ASSERT_PTR_EQ(("bad cookie"), arg, cookie); if ((nbytes = read(fd, buf, sizeof (buf))) < 0) { ASSERT_INT64_EQ(("bad read: %s", strerror(errno)), nbytes, 1); } VERBOSE(("read %ld bytes '%s'", nbytes, buf)); ASSERT_INT64_EQ(("wanted a byte of cookie"), nbytes, 1); ASSERT_CHAR_EQ(("bad byte %d of cookie", i), buf[0], cookie[i]); pthread_mutex_lock(&mtx); pthread_cond_signal(&cv); VERBOSE(("wakeup")); pthread_mutex_unlock(&mtx); i++; } int main(int argc, const char *argv[]) { int pipefds[2]; struct mevent *evp; start_test(argv[0], 5); start_event_thread(); if (pipe(pipefds) != 0) { FAIL_ERRNO("pipe"); } if (fcntl(pipefds[0], F_SETFL, O_NONBLOCK) != 0) { FAIL_ERRNO("set pipe nonblocking"); } evp = mevent_add(pipefds[0], EVF_READ, munch, cookie); ASSERT_PTR_NEQ(("mevent_add"), evp, NULL); for (int i = 0; cookie[i] != '\0'; i++) { ssize_t written; pthread_mutex_lock(&mtx); written = write(pipefds[1], cookie + i, 1); if (written < 0) { FAIL_ERRNO("bad write"); } ASSERT_INT64_EQ(("write byte %d of cookie", i), written, 1); /* Wait for it to be read */ pthread_cond_wait(&cv, &mtx); pthread_mutex_unlock(&mtx); } PASS(); } /* * This file and its contents are supplied under the terms of the * Common Development and Distribution License ("CDDL"), version 1.0. * You may only use this file in accordance with the terms of version * 1.0 of the CDDL. * * A full copy of the text of the CDDL should have accompanied this * source. A copy of the CDDL is also available via the Internet at * http://www.illumos.org/license/CDDL. */ /* * Copyright 2018 Joyent, Inc. */ #include #include #include #include #include "testlib.h" #include "mevent.h" const char *testlib_prog; boolean_t testlib_verbose; static void timed_out(int signo) { ASSERT_INT_EQ(("timeout signal"), signo, SIGALRM); FAIL(("Timed out")); } void start_test(const char *argv0, uint32_t timeout) { char *val; testlib_prog = strrchr(argv0, '/'); if (testlib_prog == NULL) { testlib_prog = argv0; } else { testlib_prog++; } testlib_verbose = ((val = getenv("TEST_VERBOSE")) != NULL) && val[0] != '\0'; signal(SIGALRM, timed_out); alarm(timeout); } /* ARGSUSED */ static void * event_thread(void *arg) { mevent_dispatch(); return (NULL); } void start_event_thread(void) { pthread_t tid; if (pthread_create(&tid, NULL, event_thread, NULL) != 0) { FAIL_ERRNO("pthread_create"); } } /* * This file and its contents are supplied under the terms of the * Common Development and Distribution License ("CDDL"), version 1.0. * You may only use this file in accordance with the terms of version * 1.0 of the CDDL. * * A full copy of the text of the CDDL should have accompanied this * source. A copy of the CDDL is also available via the Internet at * http://www.illumos.org/license/CDDL. */ /* * Copyright 2018 Joyent, Inc. */ #ifndef _TESTLIB_H_ #define _TESTLIB_H_ #include #include #include #include #include #include #include #include #include "mevent.h" #define EXIT_PASS 0 #define EXIT_FAIL 1 #define VERBOSE(msg) \ if (testlib_verbose) { \ (void) printf("VERBOSE %s: %s:%d %s: ", testlib_prog, \ __FILE__, __LINE__, __func__); \ (void) printf msg; \ (void) printf("\n"); \ } #define FAIL_PROLOGUE() \ (void) printf("FAIL %s: %s:%d: ", testlib_prog, __FILE__, __LINE__) #define FAIL(msg) \ { \ FAIL_PROLOGUE(); \ (void) printf msg; \ (void) printf("\n"); \ exit(EXIT_FAIL); \ } #define FAIL_ERRNO(msg) FAIL((msg ": %s", strerror(errno))) #define PASS() \ { \ (void) printf("PASS %s\n", testlib_prog); \ exit(EXIT_PASS); \ } #define ASSERT_CMP(msg, got, cmp, exp, nfmt) \ if (!(got cmp exp)) { \ FAIL_PROLOGUE(); \ (void) printf msg; \ (void) printf(": %s=" nfmt " %s %s=" nfmt "\n", \ #got, got, #cmp, #exp, exp); \ exit(EXIT_FAIL); \ } #define ASSERT_CHAR_EQ(msg, got, exp) ASSERT_CMP(msg, got, ==, exp, "%c") #define ASSERT_INT_EQ(msg, got, exp) ASSERT_CMP(msg, got, ==, exp, "%d") #define ASSERT_INT_NEQ(msg, got, exp) ASSERT_CMP(msg, got, !=, exp, "%d") #define ASSERT_INT64_EQ(msg, got, exp) ASSERT_CMP(msg, got, ==, exp, "%ld") #define ASSERT_INT64_NEQ(msg, got, exp) ASSERT_CMP(msg, got, !=, exp, "%ld") #define ASSERT_PTR_EQ(msg, got, exp) ASSERT_CMP(msg, got, ==, exp, "%p") #define ASSERT_PTR_NEQ(msg, got, exp) ASSERT_CMP(msg, got, !=, exp, "%p") #define ASSERT_STR_EQ(msg, got, exp) \ if (strcmp(got, exp) != 0) { \ FAIL_PROLOGUE(); \ (void) printf msg; \ (void) printf(": %s='%s' != %s='%s'\n", \ #got, got, #exp, exp); \ exit(EXIT_FAIL); \ } extern const char *testlib_prog; extern boolean_t testlib_verbose; extern void start_test(const char *, uint32_t); extern void start_event_thread(void); extern void test_mevent_count_lists(int *, int *, int *); extern void set_mevent_file_poll_interval_ms(int); #endif /* _TESTLIB_H_ */ /* * This file and its contents are supplied under the terms of the * Common Development and Distribution License ("CDDL"), version 1.0. * You may only use this file in accordance with the terms of version * 1.0 of the CDDL. * * A full copy of the text of the CDDL should have accompanied this * source. A copy of the CDDL is also available via the Internet at * http://www.illumos.org/license/CDDL. */ /* * Copyright 2018 Joyent, Inc. * Copyright 2022 OmniOS Community Edition (OmniOSce) Association. */ #include #include #include #include #include #include #include #include #include #include #include "testlib.h" #include "mevent.h" static char *cookie = "Shortcake"; static pthread_mutex_t mtx = PTHREAD_MUTEX_INITIALIZER; static pthread_cond_t cv = PTHREAD_COND_INITIALIZER; static void callback(int fd, enum ev_type ev, void *arg) { static off_t size = 0; struct stat st; ASSERT_INT_EQ(("bad event"), ev, EVF_VNODE); ASSERT_PTR_EQ(("bad cookie"), arg, cookie); if (fstat(fd, &st) != 0) FAIL_ERRNO("fstat failed"); ASSERT_INT64_NEQ(("File size has not changed"), size, st.st_size); size = st.st_size; pthread_mutex_lock(&mtx); pthread_cond_signal(&cv); VERBOSE(("wakeup")); pthread_mutex_unlock(&mtx); } static void test_fd(int fd, char *tag) { struct mevent *evp; int err; evp = mevent_add_flags(fd, EVF_VNODE, EVFF_ATTRIB, callback, cookie); ASSERT_PTR_NEQ(("%s: mevent_add", tag), evp, NULL); for (uint_t i = 0; cookie[i] != '\0'; i++) { ssize_t written; if (i > 0) { /* * Check that no events are emitted for writes which do * not alter the size. */ if (lseek(fd, -1, SEEK_CUR) == -1) FAIL_ERRNO("lseek"); if (write(fd, "X", 1) == -1) FAIL_ERRNO("write"); /* * Allow time for the callback to fire if it is going * to. */ VERBOSE(("Write within")); usleep(100); } pthread_mutex_lock(&mtx); written = write(fd, cookie + i, 1); if (written < 0) FAIL_ERRNO("bad write"); ASSERT_INT64_EQ(("write byte %d of cookie", i), written, 1); VERBOSE(("Write extend")); /* Wait for the size change to be processed */ pthread_cond_wait(&cv, &mtx); pthread_mutex_unlock(&mtx); /* * This is a bit unsatisfactory but we need to allow time * for mevent to re-associate the port or the next write could * be missed. */ usleep(100); } err = mevent_disable(evp); ASSERT_INT_EQ(("%s: mevent_disable: %s", tag, strerror(err)), err, 0); (void) printf("PASS %s - %s\n", testlib_prog, tag); } int main(int argc, const char **argv) { int fd; start_test(argv[0], 20); set_mevent_file_poll_interval_ms(500); start_event_thread(); /* Test with a temporary file in /tmp */ char *template = strdup("/tmp/mevent.vnode.XXXXXX"); ASSERT_PTR_NEQ(("strdup"), template, NULL); fd = mkstemp(template); if (fd == -1) FAIL_ERRNO("Couldn't create temporary file with mkstemp"); VERBOSE(("Opened temporary file at '%s'", template)); test_fd(fd, "temporary file"); /* Test with a file which is unlinked from the filesystem */ FILE *fp = tmpfile(); ASSERT_PTR_NEQ(("tmpfile"), fp, NULL); fd = fileno(fp); if (fd == -1) FAIL_ERRNO("Couldn't get file descriptor for temporary file"); test_fd(fd, "anon file"); /* * Defer to here to avoid generating a new event before the disable has * been processed and the port deassociated. */ unlink(template); free(template); PASS(); } /* * This file and its contents are supplied under the terms of the * Common Development and Distribution License ("CDDL"), version 1.0. * You may only use this file in accordance with the terms of version * 1.0 of the CDDL. * * A full copy of the text of the CDDL should have accompanied this * source. A copy of the CDDL is also available via the Internet at * http://www.illumos.org/license/CDDL. */ /* * Copyright 2018 Joyent, Inc. * Copyright 2022 OmniOS Community Edition (OmniOSce) Association. */ #include #include #include #include #include #include #include #include #include #include #include #include #include "testlib.h" #include "mevent.h" #define MB (1024 * 1024) static char *cookie = "Chocolate chip with fudge stripes"; static pthread_mutex_t mtx = PTHREAD_MUTEX_INITIALIZER; static pthread_cond_t cv = PTHREAD_COND_INITIALIZER; static void callback(int fd, enum ev_type ev, void *arg) { static off_t size = 0; struct stat st; ASSERT_INT_EQ(("bad event"), ev, EVF_VNODE); ASSERT_PTR_EQ(("bad cookie"), arg, cookie); if (fstat(fd, &st) != 0) FAIL_ERRNO("fstat failed"); ASSERT_INT64_NEQ(("Size has not changed"), size, st.st_size); size = st.st_size; pthread_mutex_lock(&mtx); pthread_cond_signal(&cv); VERBOSE(("wakeup")); pthread_mutex_unlock(&mtx); } static void destroy_zpool(libzfs_handle_t *zfshdl, zpool_handle_t *poolhdl, zfs_handle_t *volhdl) { if (volhdl != NULL) { if (zfs_destroy(volhdl, B_FALSE) != 0) { FAIL(("Failed to destroy ZVOL - %s", libzfs_error_description(zfshdl))); } } if (poolhdl != NULL) { if (zpool_destroy(poolhdl, testlib_prog) != 0) { FAIL(("Failed to destroy ZPOOL - %s", libzfs_error_description(zfshdl))); } } } static void create_zpool(libzfs_handle_t *zfshdl, const char *pool, const char *file) { nvlist_t *nvroot, *props; nvlist_t *vdevs[1]; nvroot = fnvlist_alloc(); props = fnvlist_alloc(); vdevs[0] = fnvlist_alloc(); fnvlist_add_string(vdevs[0], ZPOOL_CONFIG_PATH, file); fnvlist_add_string(vdevs[0], ZPOOL_CONFIG_TYPE, VDEV_TYPE_FILE); fnvlist_add_uint64(vdevs[0], ZPOOL_CONFIG_IS_LOG, 0); fnvlist_add_string(nvroot, ZPOOL_CONFIG_TYPE, VDEV_TYPE_ROOT); fnvlist_add_nvlist_array(nvroot, ZPOOL_CONFIG_CHILDREN, vdevs, 1); fnvlist_add_string(props, zfs_prop_to_name(ZFS_PROP_MOUNTPOINT), ZFS_MOUNTPOINT_NONE); if (zpool_create(zfshdl, pool, nvroot, NULL, props) != 0) { FAIL(("Failed to create ZPOOL %s using %s - %s", pool, file, libzfs_error_description(zfshdl))); } VERBOSE(("Created ZFS pool %s", pool)); } static bool create_zvol(libzfs_handle_t *zfshdl, const char *vol) { nvlist_t *volprops; int err; volprops = fnvlist_alloc(); fnvlist_add_uint64(volprops, zfs_prop_to_name(ZFS_PROP_VOLSIZE), 1 * MB); err = zfs_create(zfshdl, vol, ZFS_TYPE_VOLUME, volprops); if (err != 0) { (void) printf("Failed to create ZVOL %s - %s", vol, libzfs_error_description(zfshdl)); return (false); } VERBOSE(("Created ZVOL %s", vol)); return (true); } int main(int argc, const char **argv) { libzfs_handle_t *zfshdl; char *template, *pool, *vol, *backend; struct mevent *evp; zpool_handle_t *poolhdl = NULL; zfs_handle_t *volhdl = NULL; int err, fd; start_test(argv[0], 10); set_mevent_file_poll_interval_ms(1000); if (getzoneid() != GLOBAL_ZONEID) FAIL(("Can only be run in the global zone")); if ((zfshdl = libzfs_init()) == NULL) FAIL_ERRNO("Could not open ZFS library"); template = strdup("/tmp/mevent.vnode.zvol.XXXXXX"); ASSERT_PTR_NEQ(("strdup"), template, NULL); fd = mkstemp(template); if (fd == -1) FAIL_ERRNO("Couldn't create temporary file with mkstemp"); VERBOSE(("Opened temporary file at '%s'", template)); err = asprintf(&pool, "mevent_test_%d", getpid()); ASSERT_INT_NEQ(("asprintf pool"), err, -1); err = asprintf(&vol, "%s/test_zvol_%d", pool, getpid()); ASSERT_INT_NEQ(("asprintf vol"), err, -1); err = asprintf(&backend, "/dev/zvol/rdsk/%s", vol); ASSERT_INT_NEQ(("asprintf backend"), err, -1); err = ftruncate(fd, 64 * MB); if (err != 0) FAIL_ERRNO("ftruncate"); (void) close(fd); fd = -1; /* * Create the pool as late as possible to reduce the risk of leaving * a test pool hanging around. */ create_zpool(zfshdl, pool, template); if ((poolhdl = zpool_open(zfshdl, pool)) == NULL) { (void) printf("Could not open ZPOOL - %s\n", libzfs_error_description(zfshdl)); err = EXIT_FAIL; goto out; } if (!create_zvol(zfshdl, vol)) { err = EXIT_FAIL; goto out; } if ((volhdl = zfs_open(zfshdl, vol, ZFS_TYPE_VOLUME)) == NULL) { (void) printf("Could not open ZFS volume - %s\n", libzfs_error_description(zfshdl)); err = EXIT_FAIL; goto out; } if ((fd = open(backend, O_RDWR)) == -1) { (void) printf("Failed to open '%s': %s\n", backend, strerror(errno)); err = EXIT_FAIL; goto out; } VERBOSE(("Opened backend %s", backend)); start_event_thread(); evp = mevent_add_flags(fd, EVF_VNODE, EVFF_ATTRIB, callback, cookie); if (evp == NULL) { (void) printf("mevent_add returned NULL\n"); err = EXIT_FAIL; goto out; } for (uint_t i = 2; i < 4; i++) { ssize_t written; char buf[64]; /* * Check that a write to the volume does not trigger an event. */ if (lseek(fd, 0, SEEK_SET) == -1) FAIL_ERRNO("lseek"); written = write(fd, cookie, strlen(cookie)); if (written < 0) FAIL_ERRNO("bad write"); ASSERT_INT64_EQ(("write cookie", i), written, strlen(cookie)); (void) snprintf(buf, sizeof (buf), "%llu", i * MB); VERBOSE(("Setting volsize to %s", buf)); if (zfs_prop_set(volhdl, zfs_prop_to_name(ZFS_PROP_VOLSIZE), buf) != 0) { (void) printf("Failed to increase ZFS volume size\n"); pthread_mutex_unlock(&mtx); err = EXIT_FAIL; goto out; } /* Wait for the size change to be processed */ pthread_mutex_lock(&mtx); pthread_cond_wait(&cv, &mtx); pthread_mutex_unlock(&mtx); } (void) mevent_disable(evp); err = EXIT_PASS; out: (void) close(fd); destroy_zpool(zfshdl, poolhdl, volhdl); (void) libzfs_fini(zfshdl); (void) unlink(template); if (err == EXIT_PASS) PASS(); exit(err); }