/*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2012 NetApp, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * bhyve ACPI table generator. * * Create the minimal set of ACPI tables required to boot FreeBSD (and * hopefully other o/s's). * * The tables are placed in the guest's ROM area just below 1MB physical, * above the MPTable. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "bhyverun.h" #include "acpi.h" #include "basl.h" #include "pci_emul.h" #include "vmgenc.h" #define BHYVE_ASL_TEMPLATE "bhyve.XXXXXXX" #define BHYVE_ASL_SUFFIX ".aml" #define BHYVE_ASL_COMPILER "/usr/sbin/iasl" #define BHYVE_ADDRESS_IOAPIC 0xFEC00000 #define BHYVE_ADDRESS_HPET 0xFED00000 #define BHYVE_ADDRESS_LAPIC 0xFEE00000 static int basl_keep_temps; static int basl_verbose_iasl; static int basl_ncpu; static uint32_t hpet_capabilities; /* * Contains the full pathname of the template to be passed * to mkstemp/mkstemps(3C) */ static char basl_template[MAXPATHLEN]; static char basl_stemplate[MAXPATHLEN]; /* * State for dsdt_line(), dsdt_indent(), and dsdt_unindent(). */ static FILE *dsdt_fp; static int dsdt_indent_level; static int dsdt_error; struct basl_fio { int fd; FILE *fp; char f_name[MAXPATHLEN]; }; #define EFPRINTF(...) \ if (fprintf(__VA_ARGS__) < 0) goto err_exit #define EFFLUSH(x) \ if (fflush(x) != 0) goto err_exit /* * A list for additional ACPI devices like a TPM. */ struct acpi_device_list_entry { SLIST_ENTRY(acpi_device_list_entry) chain; const struct acpi_device *dev; }; static SLIST_HEAD(acpi_device_list, acpi_device_list_entry) acpi_devices = SLIST_HEAD_INITIALIZER(acpi_devices); int acpi_tables_add_device(const struct acpi_device *const dev) { struct acpi_device_list_entry *const entry = calloc(1, sizeof(*entry)); if (entry == NULL) { return (ENOMEM); } entry->dev = dev; SLIST_INSERT_HEAD(&acpi_devices, entry, chain); return (0); } /* * Helper routines for writing to the DSDT from other modules. */ void dsdt_line(const char *fmt, ...) { va_list ap; if (dsdt_error != 0) return; if (strcmp(fmt, "") != 0) { if (dsdt_indent_level != 0) EFPRINTF(dsdt_fp, "%*c", dsdt_indent_level * 2, ' '); va_start(ap, fmt); if (vfprintf(dsdt_fp, fmt, ap) < 0) { va_end(ap); goto err_exit; } va_end(ap); } EFPRINTF(dsdt_fp, "\n"); return; err_exit: dsdt_error = errno; } void dsdt_indent(int levels) { dsdt_indent_level += levels; assert(dsdt_indent_level >= 0); } void dsdt_unindent(int levels) { assert(dsdt_indent_level >= levels); dsdt_indent_level -= levels; } void dsdt_fixed_ioport(uint16_t iobase, uint16_t length) { dsdt_line("IO (Decode16,"); dsdt_line(" 0x%04X, // Range Minimum", iobase); dsdt_line(" 0x%04X, // Range Maximum", iobase); dsdt_line(" 0x01, // Alignment"); dsdt_line(" 0x%02X, // Length", length); dsdt_line(" )"); } void dsdt_fixed_irq(uint8_t irq) { dsdt_line("IRQNoFlags ()"); dsdt_line(" {%d}", irq); } void dsdt_fixed_mem32(uint32_t base, uint32_t length) { dsdt_line("Memory32Fixed (ReadWrite,"); dsdt_line(" 0x%08X, // Address Base", base); dsdt_line(" 0x%08X, // Address Length", length); dsdt_line(" )"); } static int basl_fwrite_dsdt(FILE *fp) { dsdt_fp = fp; dsdt_error = 0; dsdt_indent_level = 0; dsdt_line("/*"); dsdt_line(" * bhyve DSDT template"); dsdt_line(" */"); dsdt_line("DefinitionBlock (\"bhyve_dsdt.aml\", \"DSDT\", 2," "\"BHYVE \", \"BVDSDT \", 0x00000001)"); dsdt_line("{"); dsdt_line(" Name (_S5, Package ()"); dsdt_line(" {"); dsdt_line(" 0x05,"); dsdt_line(" Zero,"); dsdt_line(" })"); pci_write_dsdt(); dsdt_line(""); dsdt_line(" Scope (_SB.PC00)"); dsdt_line(" {"); dsdt_line(" Device (HPET)"); dsdt_line(" {"); dsdt_line(" Name (_HID, EISAID(\"PNP0103\"))"); dsdt_line(" Name (_UID, 0)"); dsdt_line(" Name (_CRS, ResourceTemplate ()"); dsdt_line(" {"); dsdt_indent(4); dsdt_fixed_mem32(0xFED00000, 0x400); dsdt_unindent(4); dsdt_line(" })"); dsdt_line(" }"); dsdt_line(" }"); vmgenc_write_dsdt(); const struct acpi_device_list_entry *entry; SLIST_FOREACH(entry, &acpi_devices, chain) { BASL_EXEC(acpi_device_write_dsdt(entry->dev)); } dsdt_line("}"); if (dsdt_error != 0) return (dsdt_error); EFFLUSH(fp); return (0); err_exit: return (errno); } static int basl_open(struct basl_fio *bf, int suffix) { int err; err = 0; if (suffix) { strlcpy(bf->f_name, basl_stemplate, MAXPATHLEN); bf->fd = mkstemps(bf->f_name, strlen(BHYVE_ASL_SUFFIX)); } else { strlcpy(bf->f_name, basl_template, MAXPATHLEN); bf->fd = mkstemp(bf->f_name); } if (bf->fd > 0) { bf->fp = fdopen(bf->fd, "w+"); if (bf->fp == NULL) { unlink(bf->f_name); close(bf->fd); } } else { err = 1; } return (err); } static void basl_close(struct basl_fio *bf) { if (!basl_keep_temps) unlink(bf->f_name); fclose(bf->fp); } static int basl_start(struct basl_fio *in, struct basl_fio *out) { int err; err = basl_open(in, 0); if (!err) { err = basl_open(out, 1); if (err) { basl_close(in); } } return (err); } static void basl_end(struct basl_fio *in, struct basl_fio *out) { basl_close(in); basl_close(out); } static int basl_load(struct vmctx *ctx, int fd) { struct stat sb; void *addr; if (fstat(fd, &sb) < 0) return (errno); addr = calloc(1, sb.st_size); if (addr == NULL) return (ENOMEM); if (read(fd, addr, sb.st_size) < 0) return (errno); struct basl_table *table; uint8_t name[ACPI_NAMESEG_SIZE + 1] = { 0 }; memcpy(name, addr, sizeof(name) - 1 /* last char is '\0' */); BASL_EXEC(basl_table_create(&table, ctx, name, BASL_TABLE_ALIGNMENT)); BASL_EXEC(basl_table_append_bytes(table, addr, sb.st_size)); free(addr); return (0); } static int basl_compile(struct vmctx *ctx, int (*fwrite_section)(FILE *)) { struct basl_fio io[2]; static char iaslbuf[3*MAXPATHLEN + 10]; const char *fmt; int err; err = basl_start(&io[0], &io[1]); if (!err) { err = (*fwrite_section)(io[0].fp); if (!err) { /* * iasl sends the results of the compilation to * stdout. Shut this down by using the shell to * redirect stdout to /dev/null, unless the user * has requested verbose output for debugging * purposes */ fmt = basl_verbose_iasl ? "%s -p %s %s" : "/bin/sh -c \"%s -p %s %s\" 1> /dev/null"; snprintf(iaslbuf, sizeof(iaslbuf), fmt, BHYVE_ASL_COMPILER, io[1].f_name, io[0].f_name); err = system(iaslbuf); if (!err) { /* * Copy the aml output file into guest * memory at the specified location */ err = basl_load(ctx, io[1].fd); } } basl_end(&io[0], &io[1]); } return (err); } static int basl_make_templates(void) { const char *tmpdir; int err; int len; err = 0; /* * */ if ((tmpdir = getenv("BHYVE_TMPDIR")) == NULL || *tmpdir == '\0' || (tmpdir = getenv("TMPDIR")) == NULL || *tmpdir == '\0') { tmpdir = _PATH_TMP; } len = strlen(tmpdir); if ((len + sizeof(BHYVE_ASL_TEMPLATE) + 1) < MAXPATHLEN) { strcpy(basl_template, tmpdir); while (len > 0 && basl_template[len - 1] == '/') len--; basl_template[len] = '/'; strcpy(&basl_template[len + 1], BHYVE_ASL_TEMPLATE); } else err = E2BIG; if (!err) { /* * len has been initialized (and maybe adjusted) above */ if ((len + sizeof(BHYVE_ASL_TEMPLATE) + 1 + sizeof(BHYVE_ASL_SUFFIX)) < MAXPATHLEN) { strcpy(basl_stemplate, tmpdir); basl_stemplate[len] = '/'; strcpy(&basl_stemplate[len + 1], BHYVE_ASL_TEMPLATE); len = strlen(basl_stemplate); strcpy(&basl_stemplate[len], BHYVE_ASL_SUFFIX); } else err = E2BIG; } return (err); } static int build_dsdt(struct vmctx *const ctx) { BASL_EXEC(basl_compile(ctx, basl_fwrite_dsdt)); return (0); } static int build_facs(struct vmctx *const ctx) { ACPI_TABLE_FACS facs; struct basl_table *table; BASL_EXEC(basl_table_create(&table, ctx, ACPI_SIG_FACS, BASL_TABLE_ALIGNMENT_FACS)); memset(&facs, 0, sizeof(facs)); memcpy(facs.Signature, ACPI_SIG_FACS, ACPI_NAMESEG_SIZE); facs.Length = sizeof(facs); facs.Version = htole32(2); BASL_EXEC(basl_table_append_bytes(table, &facs, sizeof(facs))); return (0); } static int build_fadt(struct vmctx *const ctx) { ACPI_TABLE_FADT fadt; struct basl_table *table; BASL_EXEC(basl_table_create(&table, ctx, ACPI_SIG_FADT, BASL_TABLE_ALIGNMENT)); memset(&fadt, 0, sizeof(fadt)); BASL_EXEC(basl_table_append_header(table, ACPI_SIG_FADT, 5, 1)); fadt.Facs = htole32(0); /* patched by basl */ fadt.Dsdt = htole32(0); /* patched by basl */ fadt.SciInterrupt = htole16(SCI_INT); fadt.SmiCommand = htole32(SMI_CMD); fadt.AcpiEnable = BHYVE_ACPI_ENABLE; fadt.AcpiDisable = BHYVE_ACPI_DISABLE; fadt.Pm1aEventBlock = htole32(PM1A_EVT_ADDR); fadt.Pm1aControlBlock = htole32(PM1A_CNT_ADDR); fadt.PmTimerBlock = htole32(IO_PMTMR); fadt.Gpe0Block = htole32(IO_GPE0_BLK); fadt.Pm1EventLength = 4; fadt.Pm1ControlLength = 2; fadt.PmTimerLength = 4; fadt.Gpe0BlockLength = IO_GPE0_LEN; fadt.Century = 0x32; fadt.BootFlags = htole16(ACPI_FADT_NO_VGA | ACPI_FADT_NO_ASPM); fadt.Flags = htole32(ACPI_FADT_WBINVD | ACPI_FADT_C1_SUPPORTED | ACPI_FADT_SLEEP_BUTTON | ACPI_FADT_32BIT_TIMER | ACPI_FADT_RESET_REGISTER | ACPI_FADT_HEADLESS | ACPI_FADT_APIC_PHYSICAL); basl_fill_gas(&fadt.ResetRegister, ACPI_ADR_SPACE_SYSTEM_IO, 8, 0, ACPI_GAS_ACCESS_WIDTH_BYTE, 0xCF9); fadt.ResetValue = 6; fadt.MinorRevision = 1; fadt.XFacs = htole64(0); /* patched by basl */ fadt.XDsdt = htole64(0); /* patched by basl */ basl_fill_gas(&fadt.XPm1aEventBlock, ACPI_ADR_SPACE_SYSTEM_IO, 0x20, 0, ACPI_GAS_ACCESS_WIDTH_WORD, PM1A_EVT_ADDR); basl_fill_gas(&fadt.XPm1bEventBlock, ACPI_ADR_SPACE_SYSTEM_IO, 0, 0, ACPI_GAS_ACCESS_WIDTH_UNDEFINED, 0); basl_fill_gas(&fadt.XPm1aControlBlock, ACPI_ADR_SPACE_SYSTEM_IO, 0x10, 0, ACPI_GAS_ACCESS_WIDTH_WORD, PM1A_CNT_ADDR); basl_fill_gas(&fadt.XPm1bControlBlock, ACPI_ADR_SPACE_SYSTEM_IO, 0, 0, ACPI_GAS_ACCESS_WIDTH_UNDEFINED, 0); basl_fill_gas(&fadt.XPm2ControlBlock, ACPI_ADR_SPACE_SYSTEM_IO, 8, 0, ACPI_GAS_ACCESS_WIDTH_UNDEFINED, 0); basl_fill_gas(&fadt.XPmTimerBlock, ACPI_ADR_SPACE_SYSTEM_IO, 0x20, 0, ACPI_GAS_ACCESS_WIDTH_DWORD, IO_PMTMR); basl_fill_gas(&fadt.XGpe0Block, ACPI_ADR_SPACE_SYSTEM_IO, IO_GPE0_LEN * 8, 0, ACPI_GAS_ACCESS_WIDTH_BYTE, IO_GPE0_BLK); basl_fill_gas(&fadt.XGpe1Block, ACPI_ADR_SPACE_SYSTEM_IO, 0, 0, ACPI_GAS_ACCESS_WIDTH_UNDEFINED, 0); basl_fill_gas(&fadt.SleepControl, ACPI_ADR_SPACE_SYSTEM_IO, 8, 0, ACPI_GAS_ACCESS_WIDTH_BYTE, 0); basl_fill_gas(&fadt.SleepStatus, ACPI_ADR_SPACE_SYSTEM_IO, 8, 0, ACPI_GAS_ACCESS_WIDTH_BYTE, 0); BASL_EXEC(basl_table_append_content(table, &fadt, sizeof(fadt))); BASL_EXEC(basl_table_add_pointer(table, ACPI_SIG_FACS, offsetof(ACPI_TABLE_FADT, Facs), sizeof(fadt.Facs))); BASL_EXEC(basl_table_add_pointer(table, ACPI_SIG_DSDT, offsetof(ACPI_TABLE_FADT, Dsdt), sizeof(fadt.Dsdt))); BASL_EXEC(basl_table_add_pointer(table, ACPI_SIG_FACS, offsetof(ACPI_TABLE_FADT, XFacs), sizeof(fadt.XFacs))); BASL_EXEC(basl_table_add_pointer(table, ACPI_SIG_DSDT, offsetof(ACPI_TABLE_FADT, XDsdt), sizeof(fadt.XDsdt))); BASL_EXEC(basl_table_register_to_rsdt(table)); return (0); } static int build_hpet(struct vmctx *const ctx) { ACPI_TABLE_HPET hpet; struct basl_table *table; BASL_EXEC(basl_table_create(&table, ctx, ACPI_SIG_HPET, BASL_TABLE_ALIGNMENT)); memset(&hpet, 0, sizeof(hpet)); BASL_EXEC(basl_table_append_header(table, ACPI_SIG_HPET, 1, 1)); hpet.Id = htole32(hpet_capabilities); basl_fill_gas(&hpet.Address, ACPI_ADR_SPACE_SYSTEM_MEMORY, 0, 0, ACPI_GAS_ACCESS_WIDTH_LEGACY, BHYVE_ADDRESS_HPET); hpet.Flags = ACPI_HPET_PAGE_PROTECT4; BASL_EXEC(basl_table_append_content(table, &hpet, sizeof(hpet))); BASL_EXEC(basl_table_register_to_rsdt(table)); return (0); } static int build_madt(struct vmctx *const ctx) { ACPI_TABLE_MADT madt; ACPI_MADT_LOCAL_APIC madt_lapic; ACPI_MADT_IO_APIC madt_ioapic; ACPI_MADT_INTERRUPT_OVERRIDE madt_irq_override; ACPI_MADT_LOCAL_APIC_NMI madt_lapic_nmi; struct basl_table *table; BASL_EXEC(basl_table_create(&table, ctx, ACPI_SIG_MADT, BASL_TABLE_ALIGNMENT)); memset(&madt, 0, sizeof(madt)); BASL_EXEC(basl_table_append_header(table, ACPI_SIG_MADT, 1, 1)); madt.Address = htole32(BHYVE_ADDRESS_LAPIC); madt.Flags = htole32(ACPI_MADT_PCAT_COMPAT); BASL_EXEC(basl_table_append_content(table, &madt, sizeof(madt))); /* Local APIC for each CPU */ for (int i = 0; i < basl_ncpu; ++i) { memset(&madt_lapic, 0, sizeof(madt_lapic)); madt_lapic.Header.Type = ACPI_MADT_TYPE_LOCAL_APIC; madt_lapic.Header.Length = sizeof(madt_lapic); madt_lapic.ProcessorId = i; madt_lapic.Id = i; madt_lapic.LapicFlags = htole32(ACPI_MADT_ENABLED); BASL_EXEC(basl_table_append_bytes(table, &madt_lapic, sizeof(madt_lapic))); } /* I/O APIC */ memset(&madt_ioapic, 0, sizeof(madt_ioapic)); madt_ioapic.Header.Type = ACPI_MADT_TYPE_IO_APIC; madt_ioapic.Header.Length = sizeof(madt_ioapic); madt_ioapic.Address = htole32(BHYVE_ADDRESS_IOAPIC); BASL_EXEC( basl_table_append_bytes(table, &madt_ioapic, sizeof(madt_ioapic))); /* Legacy IRQ0 is connected to pin 2 of the I/O APIC */ memset(&madt_irq_override, 0, sizeof(madt_irq_override)); madt_irq_override.Header.Type = ACPI_MADT_TYPE_INTERRUPT_OVERRIDE; madt_irq_override.Header.Length = sizeof(madt_irq_override); madt_irq_override.GlobalIrq = htole32(2); madt_irq_override.IntiFlags = htole16( ACPI_MADT_POLARITY_ACTIVE_HIGH | ACPI_MADT_TRIGGER_EDGE); BASL_EXEC(basl_table_append_bytes(table, &madt_irq_override, sizeof(madt_irq_override))); memset(&madt_irq_override, 0, sizeof(madt_irq_override)); madt_irq_override.Header.Type = ACPI_MADT_TYPE_INTERRUPT_OVERRIDE; madt_irq_override.Header.Length = sizeof(madt_irq_override); madt_irq_override.SourceIrq = SCI_INT; madt_irq_override.GlobalIrq = htole32(SCI_INT); madt_irq_override.IntiFlags = htole16( ACPI_MADT_POLARITY_ACTIVE_LOW | ACPI_MADT_TRIGGER_LEVEL); BASL_EXEC(basl_table_append_bytes(table, &madt_irq_override, sizeof(madt_irq_override))); /* Local APIC NMI is conntected to LINT 1 on all CPUs */ memset(&madt_lapic_nmi, 0, sizeof(madt_lapic_nmi)); madt_lapic_nmi.Header.Type = ACPI_MADT_TYPE_LOCAL_APIC_NMI; madt_lapic_nmi.Header.Length = sizeof(madt_lapic_nmi); madt_lapic_nmi.ProcessorId = 0xFF; madt_lapic_nmi.IntiFlags = htole16( ACPI_MADT_POLARITY_ACTIVE_HIGH | ACPI_MADT_TRIGGER_EDGE); madt_lapic_nmi.Lint = 1; BASL_EXEC(basl_table_append_bytes(table, &madt_lapic_nmi, sizeof(madt_lapic_nmi))); BASL_EXEC(basl_table_register_to_rsdt(table)); return (0); } static int build_mcfg(struct vmctx *const ctx) { ACPI_TABLE_MCFG mcfg; ACPI_MCFG_ALLOCATION mcfg_allocation; struct basl_table *table; BASL_EXEC(basl_table_create(&table, ctx, ACPI_SIG_MCFG, BASL_TABLE_ALIGNMENT)); memset(&mcfg, 0, sizeof(mcfg)); BASL_EXEC(basl_table_append_header(table, ACPI_SIG_MCFG, 1, 1)); BASL_EXEC(basl_table_append_content(table, &mcfg, sizeof(mcfg))); memset(&mcfg_allocation, 0, sizeof(mcfg_allocation)); mcfg_allocation.Address = htole64(pci_ecfg_base()); mcfg_allocation.EndBusNumber = 0xFF; BASL_EXEC(basl_table_append_bytes(table, &mcfg_allocation, sizeof(mcfg_allocation))); BASL_EXEC(basl_table_register_to_rsdt(table)); return (0); } static int build_rsdp(struct vmctx *const ctx) { ACPI_TABLE_RSDP rsdp; struct basl_table *table; BASL_EXEC(basl_table_create(&table, ctx, ACPI_RSDP_NAME, BASL_TABLE_ALIGNMENT)); memset(&rsdp, 0, sizeof(rsdp)); memcpy(rsdp.Signature, ACPI_SIG_RSDP, 8); rsdp.Checksum = 0; /* patched by basl */ memcpy(rsdp.OemId, "BHYVE ", ACPI_OEM_ID_SIZE); rsdp.Revision = 2; rsdp.RsdtPhysicalAddress = htole32(0); /* patched by basl */ rsdp.Length = htole32(0); /* patched by basl */ rsdp.XsdtPhysicalAddress = htole64(0); /* patched by basl */ rsdp.ExtendedChecksum = 0; /* patched by basl */ BASL_EXEC(basl_table_append_bytes(table, &rsdp, sizeof(rsdp))); BASL_EXEC(basl_table_add_checksum(table, offsetof(ACPI_TABLE_RSDP, Checksum), 0, 20)); BASL_EXEC(basl_table_add_pointer(table, ACPI_SIG_RSDT, offsetof(ACPI_TABLE_RSDP, RsdtPhysicalAddress), sizeof(rsdp.RsdtPhysicalAddress))); BASL_EXEC(basl_table_add_length(table, offsetof(ACPI_TABLE_RSDP, Length), sizeof(rsdp.Length))); BASL_EXEC(basl_table_add_pointer(table, ACPI_SIG_XSDT, offsetof(ACPI_TABLE_RSDP, XsdtPhysicalAddress), sizeof(rsdp.XsdtPhysicalAddress))); BASL_EXEC(basl_table_add_checksum(table, offsetof(ACPI_TABLE_RSDP, ExtendedChecksum), 0, BASL_TABLE_CHECKSUM_LEN_FULL_TABLE)); return (0); } static int build_spcr(struct vmctx *const ctx) { ACPI_TABLE_SPCR spcr; struct basl_table *table; BASL_EXEC(basl_table_create(&table, ctx, ACPI_SIG_SPCR, BASL_TABLE_ALIGNMENT)); memset(&spcr, 0, sizeof(spcr)); BASL_EXEC(basl_table_append_header(table, ACPI_SIG_SPCR, 1, 1)); spcr.InterfaceType = ACPI_DBG2_16550_COMPATIBLE; basl_fill_gas(&spcr.SerialPort, ACPI_ADR_SPACE_SYSTEM_IO, 8, 0, ACPI_GAS_ACCESS_WIDTH_LEGACY, 0x3F8); spcr.InterruptType = ACPI_SPCR_INTERRUPT_TYPE_8259; spcr.PcInterrupt = 4; spcr.BaudRate = ACPI_SPCR_BAUD_RATE_115200; spcr.Parity = ACPI_SPCR_PARITY_NO_PARITY; spcr.StopBits = ACPI_SPCR_STOP_BITS_1; spcr.FlowControl = 3; /* RTS/CTS | DCD */ spcr.TerminalType = ACPI_SPCR_TERMINAL_TYPE_VT_UTF8; BASL_EXEC(basl_table_append_content(table, &spcr, sizeof(spcr))); BASL_EXEC(basl_table_register_to_rsdt(table)); return (0); } int acpi_build(struct vmctx *ctx, int ncpu) { int err; basl_ncpu = ncpu; err = vm_get_hpet_capabilities(ctx, &hpet_capabilities); if (err != 0) return (err); /* * For debug, allow the user to have iasl compiler output sent * to stdout rather than /dev/null */ if (getenv("BHYVE_ACPI_VERBOSE_IASL")) basl_verbose_iasl = 1; /* * Allow the user to keep the generated ASL files for debugging * instead of deleting them following use */ if (getenv("BHYVE_ACPI_KEEPTMPS")) basl_keep_temps = 1; BASL_EXEC(basl_init(ctx)); BASL_EXEC(basl_make_templates()); /* * Generate ACPI tables and copy them into guest memory. * * According to UEFI Specification v6.3 chapter 5.1 the FADT should be * the first table pointed to by XSDT. For that reason, build it as the * first table after XSDT. */ BASL_EXEC(build_rsdp(ctx)); BASL_EXEC(build_fadt(ctx)); BASL_EXEC(build_madt(ctx)); BASL_EXEC(build_hpet(ctx)); BASL_EXEC(build_mcfg(ctx)); BASL_EXEC(build_facs(ctx)); BASL_EXEC(build_spcr(ctx)); /* Build ACPI device-specific tables such as a TPM2 table. */ const struct acpi_device_list_entry *entry; SLIST_FOREACH(entry, &acpi_devices, chain) { BASL_EXEC(acpi_device_build_table(entry->dev)); } BASL_EXEC(build_dsdt(ctx)); BASL_EXEC(basl_finish()); return (0); } /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2012 NetApp, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef _ACPI_H_ #define _ACPI_H_ #include "acpi_device.h" #define SCI_INT 9 #define SMI_CMD 0xb2 #define BHYVE_ACPI_ENABLE 0xa0 #define BHYVE_ACPI_DISABLE 0xa1 #define PM1A_EVT_ADDR 0x400 #define PM1A_CNT_ADDR 0x404 #define IO_PMTMR 0x408 /* 4-byte i/o port for the timer */ #define IO_GPE0_BLK 0x40c /* 2x 1-byte IO port for GPE0_STS/EN */ #define IO_GPE0_LEN 0x2 #define IO_GPE0_STS IO_GPE0_BLK #define IO_GPE0_EN (IO_GPE0_BLK + (IO_GPE0_LEN / 2)) /* Allocated GPE bits. */ #define GPE_VMGENC 0 struct vmctx; int acpi_build(struct vmctx *ctx, int ncpu); void acpi_raise_gpe(struct vmctx *ctx, unsigned bit); int acpi_tables_add_device(const struct acpi_device *const dev); void dsdt_line(const char *fmt, ...); void dsdt_fixed_ioport(uint16_t iobase, uint16_t length); void dsdt_fixed_irq(uint8_t irq); void dsdt_fixed_mem32(uint32_t base, uint32_t length); void dsdt_indent(int levels); void dsdt_unindent(int levels); void sci_init(struct vmctx *ctx); #ifndef __FreeBSD__ void pmtmr_init(struct vmctx *ctx); #endif #endif /* _ACPI_H_ */ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2021 Beckhoff Automation GmbH & Co. KG * Author: Corvin Köhne */ #include #include #include #include #include #include #include #include "acpi.h" #include "acpi_device.h" #include "basl.h" /** * List entry to enumerate all resources used by an ACPI device. * * @param chain Used to chain multiple elements together. * @param type Type of the ACPI resource. * @param data Data of the ACPI resource. */ struct acpi_resource_list_entry { SLIST_ENTRY(acpi_resource_list_entry) chain; UINT32 type; ACPI_RESOURCE_DATA data; }; /** * Holds information about an ACPI device. * * @param vm_ctx VM context the ACPI device was created in. * @param softc A pointer to the software context of the ACPI device. * @param emul Device emulation struct. It contains some information like the name of the ACPI device and some device specific functions. * @param crs Current resources used by the ACPI device. */ struct acpi_device { struct vmctx *vm_ctx; void *softc; const struct acpi_device_emul *emul; SLIST_HEAD(acpi_resource_list, acpi_resource_list_entry) crs; }; int acpi_device_create(struct acpi_device **const new_dev, void *const softc, struct vmctx *const vm_ctx, const struct acpi_device_emul *const emul) { assert(new_dev != NULL); assert(vm_ctx != NULL); assert(emul != NULL); struct acpi_device *const dev = calloc(1, sizeof(*dev)); if (dev == NULL) { return (ENOMEM); } dev->vm_ctx = vm_ctx; dev->softc = softc; dev->emul = emul; SLIST_INIT(&dev->crs); const int error = acpi_tables_add_device(dev); if (error) { acpi_device_destroy(dev); return (error); } *new_dev = dev; return (0); } void acpi_device_destroy(struct acpi_device *const dev) { if (dev == NULL) { return; } struct acpi_resource_list_entry *res; while (!SLIST_EMPTY(&dev->crs)) { res = SLIST_FIRST(&dev->crs); SLIST_REMOVE_HEAD(&dev->crs, chain); free(res); } free(dev); } int acpi_device_add_res_fixed_ioport(struct acpi_device *const dev, const UINT16 port, const UINT8 length) { if (dev == NULL) { return (EINVAL); } struct acpi_resource_list_entry *const res = calloc(1, sizeof(*res)); if (res == NULL) { return (ENOMEM); } res->type = ACPI_RESOURCE_TYPE_FIXED_IO; res->data.FixedIo.Address = port; res->data.FixedIo.AddressLength = length; SLIST_INSERT_HEAD(&dev->crs, res, chain); return (0); } int acpi_device_add_res_fixed_memory32(struct acpi_device *const dev, const UINT8 write_protected, const UINT32 address, const UINT32 length) { if (dev == NULL) { return (EINVAL); } struct acpi_resource_list_entry *const res = calloc(1, sizeof(*res)); if (res == NULL) { return (ENOMEM); } res->type = ACPI_RESOURCE_TYPE_FIXED_MEMORY32; res->data.FixedMemory32.WriteProtect = write_protected; res->data.FixedMemory32.Address = address; res->data.FixedMemory32.AddressLength = length; SLIST_INSERT_HEAD(&dev->crs, res, chain); return (0); } void * acpi_device_get_softc(const struct acpi_device *const dev) { assert(dev != NULL); return (dev->softc); } int acpi_device_build_table(const struct acpi_device *const dev) { assert(dev != NULL); assert(dev->emul != NULL); if (dev->emul->build_table != NULL) { return (dev->emul->build_table(dev)); } return (0); } static int acpi_device_write_dsdt_crs(const struct acpi_device *const dev) { const struct acpi_resource_list_entry *res; SLIST_FOREACH(res, &dev->crs, chain) { switch (res->type) { case ACPI_RESOURCE_TYPE_FIXED_IO: dsdt_fixed_ioport(res->data.FixedIo.Address, res->data.FixedIo.AddressLength); break; case ACPI_RESOURCE_TYPE_FIXED_MEMORY32: dsdt_fixed_mem32(res->data.FixedMemory32.Address, res->data.FixedMemory32.AddressLength); break; default: assert(0); break; } } return (0); } int acpi_device_write_dsdt(const struct acpi_device *const dev) { assert(dev != NULL); dsdt_line(""); dsdt_line(" Scope (\\_SB)"); dsdt_line(" {"); dsdt_line(" Device (%s)", dev->emul->name); dsdt_line(" {"); dsdt_line(" Name (_HID, \"%s\")", dev->emul->hid); dsdt_line(" Name (_STA, 0x0F)"); dsdt_line(" Name (_CRS, ResourceTemplate ()"); dsdt_line(" {"); dsdt_indent(4); BASL_EXEC(acpi_device_write_dsdt_crs(dev)); dsdt_unindent(4); dsdt_line(" })"); if (dev->emul->write_dsdt != NULL) { dsdt_indent(3); BASL_EXEC(dev->emul->write_dsdt(dev)); dsdt_unindent(3); } dsdt_line(" }"); dsdt_line(" }"); return (0); } /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2021 Beckhoff Automation GmbH & Co. KG * Author: Corvin Köhne */ #pragma once #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wunused-parameter" #include #pragma GCC diagnostic pop struct vmctx; struct acpi_device; /** * Device specific information and emulation. * * @param name Used as device name in the DSDT. * @param hid Used as _HID in the DSDT. * @param build_table Called to build a device specific ACPI table like the TPM2 * table. * @param write_dsdt Called to append the DSDT with device specific * information. */ struct acpi_device_emul { const char *name; const char *hid; int (*build_table)(const struct acpi_device *dev); int (*write_dsdt)(const struct acpi_device *dev); }; /** * Creates an ACPI device. * * @param[out] new_dev Returns the newly create ACPI device. * @param[in] softc Pointer to the software context of the ACPI device. * @param[in] vm_ctx VM context the ACPI device is created in. * @param[in] emul Device emulation struct. It contains some information * like the name of the ACPI device and some device specific * functions. */ int acpi_device_create(struct acpi_device **new_dev, void *softc, struct vmctx *vm_ctx, const struct acpi_device_emul *emul); void acpi_device_destroy(struct acpi_device *dev); int acpi_device_add_res_fixed_ioport(struct acpi_device *dev, UINT16 port, UINT8 length); int acpi_device_add_res_fixed_memory32(struct acpi_device *dev, UINT8 write_protected, UINT32 address, UINT32 length); void *acpi_device_get_softc(const struct acpi_device *dev); int acpi_device_build_table(const struct acpi_device *dev); int acpi_device_write_dsdt(const struct acpi_device *dev); /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 1998 - 2008 Søren Schmidt * Copyright (c) 2009-2012 Alexander Motin * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer, * without modification, immediately at the beginning of the file. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef _AHCI_H_ #define _AHCI_H_ /* ATA register defines */ #define ATA_DATA 0 /* (RW) data */ #define ATA_FEATURE 1 /* (W) feature */ #define ATA_F_DMA 0x01 /* enable DMA */ #define ATA_F_OVL 0x02 /* enable overlap */ #define ATA_COUNT 2 /* (W) sector count */ #define ATA_SECTOR 3 /* (RW) sector # */ #define ATA_CYL_LSB 4 /* (RW) cylinder# LSB */ #define ATA_CYL_MSB 5 /* (RW) cylinder# MSB */ #define ATA_DRIVE 6 /* (W) Sector/Drive/Head */ #define ATA_D_LBA 0x40 /* use LBA addressing */ #define ATA_D_IBM 0xa0 /* 512 byte sectors, ECC */ #define ATA_COMMAND 7 /* (W) command */ #define ATA_ERROR 8 /* (R) error */ #define ATA_E_ILI 0x01 /* illegal length */ #define ATA_E_NM 0x02 /* no media */ #define ATA_E_ABORT 0x04 /* command aborted */ #define ATA_E_MCR 0x08 /* media change request */ #define ATA_E_IDNF 0x10 /* ID not found */ #define ATA_E_MC 0x20 /* media changed */ #define ATA_E_UNC 0x40 /* uncorrectable data */ #define ATA_E_ICRC 0x80 /* UDMA crc error */ #define ATA_E_ATAPI_SENSE_MASK 0xf0 /* ATAPI sense key mask */ #define ATA_IREASON 9 /* (R) interrupt reason */ #define ATA_I_CMD 0x01 /* cmd (1) | data (0) */ #define ATA_I_IN 0x02 /* read (1) | write (0) */ #define ATA_I_RELEASE 0x04 /* released bus (1) */ #define ATA_I_TAGMASK 0xf8 /* tag mask */ #define ATA_STATUS 10 /* (R) status */ #define ATA_ALTSTAT 11 /* (R) alternate status */ #define ATA_S_ERROR 0x01 /* error */ #define ATA_S_INDEX 0x02 /* index */ #define ATA_S_CORR 0x04 /* data corrected */ #define ATA_S_DRQ 0x08 /* data request */ #define ATA_S_DSC 0x10 /* drive seek completed */ #define ATA_S_SERVICE 0x10 /* drive needs service */ #define ATA_S_DWF 0x20 /* drive write fault */ #define ATA_S_DMA 0x20 /* DMA ready */ #define ATA_S_READY 0x40 /* drive ready */ #define ATA_S_BUSY 0x80 /* busy */ #define ATA_CONTROL 12 /* (W) control */ #define ATA_A_IDS 0x02 /* disable interrupts */ #define ATA_A_RESET 0x04 /* RESET controller */ #define ATA_A_4BIT 0x08 /* 4 head bits */ #define ATA_A_HOB 0x80 /* High Order Byte enable */ /* SATA register defines */ #define ATA_SSTATUS 13 #define ATA_SS_DET_MASK 0x0000000f #define ATA_SS_DET_NO_DEVICE 0x00000000 #define ATA_SS_DET_DEV_PRESENT 0x00000001 #define ATA_SS_DET_PHY_ONLINE 0x00000003 #define ATA_SS_DET_PHY_OFFLINE 0x00000004 #define ATA_SS_SPD_MASK 0x000000f0 #define ATA_SS_SPD_NO_SPEED 0x00000000 #define ATA_SS_SPD_GEN1 0x00000010 #define ATA_SS_SPD_GEN2 0x00000020 #define ATA_SS_SPD_GEN3 0x00000030 #define ATA_SS_IPM_MASK 0x00000f00 #define ATA_SS_IPM_NO_DEVICE 0x00000000 #define ATA_SS_IPM_ACTIVE 0x00000100 #define ATA_SS_IPM_PARTIAL 0x00000200 #define ATA_SS_IPM_SLUMBER 0x00000600 #define ATA_SS_IPM_DEVSLEEP 0x00000800 #define ATA_SERROR 14 #define ATA_SE_DATA_CORRECTED 0x00000001 #define ATA_SE_COMM_CORRECTED 0x00000002 #define ATA_SE_DATA_ERR 0x00000100 #define ATA_SE_COMM_ERR 0x00000200 #define ATA_SE_PROT_ERR 0x00000400 #define ATA_SE_HOST_ERR 0x00000800 #define ATA_SE_PHY_CHANGED 0x00010000 #define ATA_SE_PHY_IERROR 0x00020000 #define ATA_SE_COMM_WAKE 0x00040000 #define ATA_SE_DECODE_ERR 0x00080000 #define ATA_SE_PARITY_ERR 0x00100000 #define ATA_SE_CRC_ERR 0x00200000 #define ATA_SE_HANDSHAKE_ERR 0x00400000 #define ATA_SE_LINKSEQ_ERR 0x00800000 #define ATA_SE_TRANSPORT_ERR 0x01000000 #define ATA_SE_UNKNOWN_FIS 0x02000000 #define ATA_SE_EXCHANGED 0x04000000 #define ATA_SCONTROL 15 #define ATA_SC_DET_MASK 0x0000000f #define ATA_SC_DET_IDLE 0x00000000 #define ATA_SC_DET_RESET 0x00000001 #define ATA_SC_DET_DISABLE 0x00000004 #define ATA_SC_SPD_MASK 0x000000f0 #define ATA_SC_SPD_NO_SPEED 0x00000000 #define ATA_SC_SPD_SPEED_GEN1 0x00000010 #define ATA_SC_SPD_SPEED_GEN2 0x00000020 #define ATA_SC_SPD_SPEED_GEN3 0x00000030 #define ATA_SC_IPM_MASK 0x00000f00 #define ATA_SC_IPM_NONE 0x00000000 #define ATA_SC_IPM_DIS_PARTIAL 0x00000100 #define ATA_SC_IPM_DIS_SLUMBER 0x00000200 #define ATA_SC_IPM_DIS_DEVSLEEP 0x00000400 #define ATA_SACTIVE 16 #define AHCI_MAX_PORTS 32 #define AHCI_MAX_SLOTS 32 #define AHCI_MAX_IRQS 16 /* SATA AHCI v1.0 register defines */ #define AHCI_CAP 0x00 #define AHCI_CAP_NPMASK 0x0000001f #define AHCI_CAP_SXS 0x00000020 #define AHCI_CAP_EMS 0x00000040 #define AHCI_CAP_CCCS 0x00000080 #define AHCI_CAP_NCS 0x00001F00 #define AHCI_CAP_NCS_SHIFT 8 #define AHCI_CAP_PSC 0x00002000 #define AHCI_CAP_SSC 0x00004000 #define AHCI_CAP_PMD 0x00008000 #define AHCI_CAP_FBSS 0x00010000 #define AHCI_CAP_SPM 0x00020000 #define AHCI_CAP_SAM 0x00080000 #define AHCI_CAP_ISS 0x00F00000 #define AHCI_CAP_ISS_SHIFT 20 #define AHCI_CAP_SCLO 0x01000000 #define AHCI_CAP_SAL 0x02000000 #define AHCI_CAP_SALP 0x04000000 #define AHCI_CAP_SSS 0x08000000 #define AHCI_CAP_SMPS 0x10000000 #define AHCI_CAP_SSNTF 0x20000000 #define AHCI_CAP_SNCQ 0x40000000 #define AHCI_CAP_64BIT 0x80000000 #define AHCI_GHC 0x04 #define AHCI_GHC_AE 0x80000000 #define AHCI_GHC_MRSM 0x00000004 #define AHCI_GHC_IE 0x00000002 #define AHCI_GHC_HR 0x00000001 #define AHCI_IS 0x08 #define AHCI_PI 0x0c #define AHCI_VS 0x10 #define AHCI_CCCC 0x14 #define AHCI_CCCC_TV_MASK 0xffff0000 #define AHCI_CCCC_TV_SHIFT 16 #define AHCI_CCCC_CC_MASK 0x0000ff00 #define AHCI_CCCC_CC_SHIFT 8 #define AHCI_CCCC_INT_MASK 0x000000f8 #define AHCI_CCCC_INT_SHIFT 3 #define AHCI_CCCC_EN 0x00000001 #define AHCI_CCCP 0x18 #define AHCI_EM_LOC 0x1C #define AHCI_EM_CTL 0x20 #define AHCI_EM_MR 0x00000001 #define AHCI_EM_TM 0x00000100 #define AHCI_EM_RST 0x00000200 #define AHCI_EM_LED 0x00010000 #define AHCI_EM_SAFTE 0x00020000 #define AHCI_EM_SES2 0x00040000 #define AHCI_EM_SGPIO 0x00080000 #define AHCI_EM_SMB 0x01000000 #define AHCI_EM_XMT 0x02000000 #define AHCI_EM_ALHD 0x04000000 #define AHCI_EM_PM 0x08000000 #define AHCI_CAP2 0x24 #define AHCI_CAP2_BOH 0x00000001 #define AHCI_CAP2_NVMP 0x00000002 #define AHCI_CAP2_APST 0x00000004 #define AHCI_CAP2_SDS 0x00000008 #define AHCI_CAP2_SADM 0x00000010 #define AHCI_CAP2_DESO 0x00000020 #define AHCI_OFFSET 0x100 #define AHCI_STEP 0x80 #define AHCI_P_CLB 0x00 #define AHCI_P_CLBU 0x04 #define AHCI_P_FB 0x08 #define AHCI_P_FBU 0x0c #define AHCI_P_IS 0x10 #define AHCI_P_IE 0x14 #define AHCI_P_IX_DHR 0x00000001 #define AHCI_P_IX_PS 0x00000002 #define AHCI_P_IX_DS 0x00000004 #define AHCI_P_IX_SDB 0x00000008 #define AHCI_P_IX_UF 0x00000010 #define AHCI_P_IX_DP 0x00000020 #define AHCI_P_IX_PC 0x00000040 #define AHCI_P_IX_MP 0x00000080 #define AHCI_P_IX_PRC 0x00400000 #define AHCI_P_IX_IPM 0x00800000 #define AHCI_P_IX_OF 0x01000000 #define AHCI_P_IX_INF 0x04000000 #define AHCI_P_IX_IF 0x08000000 #define AHCI_P_IX_HBD 0x10000000 #define AHCI_P_IX_HBF 0x20000000 #define AHCI_P_IX_TFE 0x40000000 #define AHCI_P_IX_CPD 0x80000000 #define AHCI_P_CMD 0x18 #define AHCI_P_CMD_ST 0x00000001 #define AHCI_P_CMD_SUD 0x00000002 #define AHCI_P_CMD_POD 0x00000004 #define AHCI_P_CMD_CLO 0x00000008 #define AHCI_P_CMD_FRE 0x00000010 #define AHCI_P_CMD_CCS_MASK 0x00001f00 #define AHCI_P_CMD_CCS_SHIFT 8 #define AHCI_P_CMD_ISS 0x00002000 #define AHCI_P_CMD_FR 0x00004000 #define AHCI_P_CMD_CR 0x00008000 #define AHCI_P_CMD_CPS 0x00010000 #define AHCI_P_CMD_PMA 0x00020000 #define AHCI_P_CMD_HPCP 0x00040000 #define AHCI_P_CMD_MPSP 0x00080000 #define AHCI_P_CMD_CPD 0x00100000 #define AHCI_P_CMD_ESP 0x00200000 #define AHCI_P_CMD_FBSCP 0x00400000 #define AHCI_P_CMD_APSTE 0x00800000 #define AHCI_P_CMD_ATAPI 0x01000000 #define AHCI_P_CMD_DLAE 0x02000000 #define AHCI_P_CMD_ALPE 0x04000000 #define AHCI_P_CMD_ASP 0x08000000 #define AHCI_P_CMD_ICC_MASK 0xf0000000 #define AHCI_P_CMD_NOOP 0x00000000 #define AHCI_P_CMD_ACTIVE 0x10000000 #define AHCI_P_CMD_PARTIAL 0x20000000 #define AHCI_P_CMD_SLUMBER 0x60000000 #define AHCI_P_CMD_DEVSLEEP 0x80000000 #define AHCI_P_TFD 0x20 #define AHCI_P_SIG 0x24 #define AHCI_P_SSTS 0x28 #define AHCI_P_SCTL 0x2c #define AHCI_P_SERR 0x30 #define AHCI_P_SACT 0x34 #define AHCI_P_CI 0x38 #define AHCI_P_SNTF 0x3C #define AHCI_P_FBS 0x40 #define AHCI_P_FBS_EN 0x00000001 #define AHCI_P_FBS_DEC 0x00000002 #define AHCI_P_FBS_SDE 0x00000004 #define AHCI_P_FBS_DEV 0x00000f00 #define AHCI_P_FBS_DEV_SHIFT 8 #define AHCI_P_FBS_ADO 0x0000f000 #define AHCI_P_FBS_ADO_SHIFT 12 #define AHCI_P_FBS_DWE 0x000f0000 #define AHCI_P_FBS_DWE_SHIFT 16 #define AHCI_P_DEVSLP 0x44 #define AHCI_P_DEVSLP_ADSE 0x00000001 #define AHCI_P_DEVSLP_DSP 0x00000002 #define AHCI_P_DEVSLP_DETO 0x000003fc #define AHCI_P_DEVSLP_DETO_SHIFT 2 #define AHCI_P_DEVSLP_MDAT 0x00007c00 #define AHCI_P_DEVSLP_MDAT_SHIFT 10 #define AHCI_P_DEVSLP_DITO 0x01ff8000 #define AHCI_P_DEVSLP_DITO_SHIFT 15 #define AHCI_P_DEVSLP_DM 0x0e000000 #define AHCI_P_DEVSLP_DM_SHIFT 25 /* Just to be sure, if building as module. */ #if MAXPHYS < 512 * 1024 #undef MAXPHYS #define MAXPHYS 512 * 1024 #endif /* Pessimistic prognosis on number of required S/G entries */ #define AHCI_SG_ENTRIES (roundup(btoc(MAXPHYS) + 1, 8)) /* Command list. 32 commands. First, 1Kbyte aligned. */ #define AHCI_CL_OFFSET 0 #define AHCI_CL_SIZE 32 /* Command tables. Up to 32 commands, Each, 128byte aligned. */ #define AHCI_CT_OFFSET (AHCI_CL_OFFSET + AHCI_CL_SIZE * AHCI_MAX_SLOTS) #define AHCI_CT_SIZE (128 + AHCI_SG_ENTRIES * 16) /* Total main work area. */ #define AHCI_WORK_SIZE (AHCI_CT_OFFSET + AHCI_CT_SIZE * ch->numslots) #endif /* _AHCI_H_ */ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2016 Alex Teaca * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * */ #ifndef WITHOUT_CAPSICUM #include #include #endif #include #include #include #include #include #include #include #include #include #include "audio.h" #include "pci_hda.h" /* * Audio Player internal data structures */ struct audio { int fd; uint8_t dir; uint8_t inited; char dev_name[64]; }; /* * Audio Player module function definitions */ /* * audio_init - initialize an instance of audio player * @dev_name - the backend sound device used to play / capture * @dir - dir = 1 for write mode, dir = 0 for read mode */ struct audio * audio_init(const char *dev_name, uint8_t dir) { struct audio *aud = NULL; #ifndef WITHOUT_CAPSICUM cap_rights_t rights; cap_ioctl_t cmds[] = { SNDCTL_DSP_RESET, SNDCTL_DSP_SETFMT, SNDCTL_DSP_CHANNELS, SNDCTL_DSP_SPEED, #ifdef DEBUG_HDA SNDCTL_DSP_GETOSPACE, SNDCTL_DSP_GETISPACE, #endif }; #endif assert(dev_name); aud = calloc(1, sizeof(*aud)); if (!aud) return NULL; if (strlen(dev_name) < sizeof(aud->dev_name)) memcpy(aud->dev_name, dev_name, strlen(dev_name) + 1); else { DPRINTF("dev_name too big"); free(aud); return NULL; } aud->dir = dir; aud->fd = open(aud->dev_name, aud->dir ? O_WRONLY : O_RDONLY, 0); if (aud->fd == -1) { DPRINTF("Failed to open dev: %s, errno: %d", aud->dev_name, errno); free(aud); return (NULL); } #ifndef WITHOUT_CAPSICUM cap_rights_init(&rights, CAP_IOCTL, CAP_READ, CAP_WRITE); if (caph_rights_limit(aud->fd, &rights) == -1) errx(EX_OSERR, "Unable to apply rights for sandbox"); if (caph_ioctls_limit(aud->fd, cmds, nitems(cmds)) == -1) errx(EX_OSERR, "Unable to limit ioctl rights for sandbox"); #endif return aud; } /* * audio_set_params - reset the sound device and set the audio params * @aud - the audio player to be configured * @params - the audio parameters to be set */ int audio_set_params(struct audio *aud, struct audio_params *params) { int audio_fd; int format, channels, rate; int err; #if DEBUG_HDA == 1 audio_buf_info info; #endif assert(aud); assert(params); if ((audio_fd = aud->fd) < 0) { DPRINTF("Incorrect audio device descriptor for %s", aud->dev_name); return (-1); } /* Reset the device if it was previously opened */ if (aud->inited) { err = ioctl(audio_fd, SNDCTL_DSP_RESET, NULL); if (err == -1) { DPRINTF("Failed to reset fd: %d, errno: %d", aud->fd, errno); return (-1); } } else aud->inited = 1; /* Set the Format (Bits per Sample) */ format = params->format; err = ioctl(audio_fd, SNDCTL_DSP_SETFMT, &format); if (err == -1) { DPRINTF("Fail to set fmt: 0x%x errno: %d", params->format, errno); return -1; } /* The device does not support the requested audio format */ if (format != params->format) { DPRINTF("Mismatch format: 0x%x params->format: 0x%x", format, params->format); return -1; } /* Set the Number of Channels */ channels = params->channels; err = ioctl(audio_fd, SNDCTL_DSP_CHANNELS, &channels); if (err == -1) { DPRINTF("Fail to set channels: %d errno: %d", params->channels, errno); return -1; } /* The device does not support the requested no. of channels */ if (channels != params->channels) { DPRINTF("Mismatch channels: %d params->channels: %d", channels, params->channels); return -1; } /* Set the Sample Rate / Speed */ rate = params->rate; err = ioctl(audio_fd, SNDCTL_DSP_SPEED, &rate); if (err == -1) { DPRINTF("Fail to set speed: %d errno: %d", params->rate, errno); return -1; } /* The device does not support the requested rate / speed */ if (rate != params->rate) { DPRINTF("Mismatch rate: %d params->rate: %d", rate, params->rate); return -1; } #if DEBUG_HDA == 1 err = ioctl(audio_fd, aud->dir ? SNDCTL_DSP_GETOSPACE : SNDCTL_DSP_GETISPACE, &info); if (err == -1) { DPRINTF("Fail to get audio buf info errno: %d", errno); return -1; } DPRINTF("fragstotal: 0x%x fragsize: 0x%x", info.fragstotal, info.fragsize); #endif return 0; } /* * audio_playback - plays samples to the sound device using blocking operations * @aud - the audio player used to play the samples * @buf - the buffer containing the samples * @count - the number of bytes in buffer */ int audio_playback(struct audio *aud, const uint8_t *buf, size_t count) { ssize_t len; size_t total; int audio_fd; assert(aud); assert(aud->dir); assert(buf); audio_fd = aud->fd; assert(audio_fd != -1); for (total = 0; total < count; total += len) { len = write(audio_fd, buf + total, count - total); if (len < 0) { DPRINTF("Fail to write to fd: %d, errno: %d", audio_fd, errno); return -1; } } return 0; } /* * audio_record - records samples from the sound device using * blocking operations. * @aud - the audio player used to capture the samples * @buf - the buffer to receive the samples * @count - the number of bytes to capture in buffer * Returns -1 on error and 0 on success */ int audio_record(struct audio *aud, uint8_t *buf, size_t count) { ssize_t len; size_t total; int audio_fd; assert(aud); assert(!aud->dir); assert(buf); audio_fd = aud->fd; assert(audio_fd != -1); for (total = 0; total < count; total += len) { len = read(audio_fd, buf + total, count - total); if (len < 0) { DPRINTF("Fail to write to fd: %d, errno: %d", audio_fd, errno); return -1; } } return 0; } /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2016 Alex Teaca * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef _AUDIO_EMUL_H_ #define _AUDIO_EMUL_H_ #include #include /* * Audio Player data structures */ struct audio; struct audio_params { int channels; int format; int rate; }; /* * Audio Player API */ /* * audio_init - initialize an instance of audio player * @dev_name - the backend sound device used to play / capture * @dir - dir = 1 for write mode, dir = 0 for read mode * Returns NULL on error and the address of the audio player instance */ struct audio *audio_init(const char *dev_name, uint8_t dir); /* * audio_set_params - reset the sound device and set the audio params * @aud - the audio player to be configured * @params - the audio parameters to be set * Returns -1 on error and 0 on success */ int audio_set_params(struct audio *aud, struct audio_params *params); /* * audio_playback - plays samples to the sound device using blocking operations * @aud - the audio player used to play the samples * @buf - the buffer containing the samples * @count - the number of bytes in buffer * Returns -1 on error and 0 on success */ int audio_playback(struct audio *aud, const uint8_t *buf, size_t count); /* * audio_record - records samples from the sound device using blocking * operations. * @aud - the audio player used to capture the samples * @buf - the buffer to receive the samples * @count - the number of bytes to capture in buffer * Returns -1 on error and 0 on success */ int audio_record(struct audio *aud, uint8_t *buf, size_t count); #endif /* _AUDIO_EMUL_H_ */ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2022 Beckhoff Automation GmbH & Co. KG */ #include #include #include #include #include #include #include #include #include #include #include #include #ifndef __FreeBSD__ #include #endif #include "basl.h" #include "config.h" #include "qemu_loader.h" struct basl_table_checksum { STAILQ_ENTRY(basl_table_checksum) chain; uint32_t off; uint32_t start; uint32_t len; }; struct basl_table_length { STAILQ_ENTRY(basl_table_length) chain; uint32_t off; uint8_t size; }; struct basl_table_pointer { STAILQ_ENTRY(basl_table_pointer) chain; uint8_t src_signature[ACPI_NAMESEG_SIZE]; uint32_t off; uint8_t size; }; struct basl_table { STAILQ_ENTRY(basl_table) chain; struct vmctx *ctx; uint8_t fwcfg_name[QEMU_FWCFG_MAX_NAME]; void *data; uint32_t len; uint32_t off; uint32_t alignment; STAILQ_HEAD(basl_table_checksum_list, basl_table_checksum) checksums; STAILQ_HEAD(basl_table_length_list, basl_table_length) lengths; STAILQ_HEAD(basl_table_pointer_list, basl_table_pointer) pointers; }; static STAILQ_HEAD(basl_table_list, basl_table) basl_tables = STAILQ_HEAD_INITIALIZER( basl_tables); static struct qemu_loader *basl_loader; static struct basl_table *rsdt; static struct basl_table *xsdt; static bool load_into_memory; static __inline uint64_t basl_le_dec(void *pp, size_t len) { assert(len <= 8); switch (len) { case 1: return ((uint8_t *)pp)[0]; case 2: return le16dec(pp); case 4: return le32dec(pp); case 8: return le64dec(pp); } return 0; } static __inline void basl_le_enc(void *pp, uint64_t val, size_t len) { char buf[8]; assert(len <= 8); le64enc(buf, val); memcpy(pp, buf, len); } static int basl_dump_table(const struct basl_table *const table, const bool mem) { const ACPI_TABLE_HEADER *const header = table->data; const uint8_t *data; if (!mem) { data = table->data; } else { data = vm_map_gpa(table->ctx, BHYVE_ACPI_BASE + table->off, table->len); if (data == NULL) { return (ENOMEM); } } #ifdef __FreeBSD__ printf("%.4s @ %8x (%s)\n", header->Signature, BHYVE_ACPI_BASE + table->off, mem ? "Memory" : "FwCfg"); hexdump(data, table->len, NULL, 0); #else (void) printf("%.4s @ %8x (%s)\n", header->Signature, BHYVE_ACPI_BASE + table->off, mem ? "Memory" : "FwCfg"); hexdump_t h; hexdump_init(&h); hexdump_set_addr(&h, BHYVE_ACPI_BASE + table->off); (void) hexdump_fileh(&h, data, table->len, HDF_DEFAULT, stdout); hexdump_fini(&h); (void) printf("\n"); #endif return (0); } static int basl_dump(const bool mem) { struct basl_table *table; STAILQ_FOREACH(table, &basl_tables, chain) { BASL_EXEC(basl_dump_table(table, mem)); } return (0); } void basl_fill_gas(ACPI_GENERIC_ADDRESS *const gas, const uint8_t space_id, const uint8_t bit_width, const uint8_t bit_offset, const uint8_t access_width, const uint64_t address) { assert(gas != NULL); gas->SpaceId = space_id; gas->BitWidth = bit_width; gas->BitOffset = bit_offset; gas->AccessWidth = access_width; gas->Address = htole64(address); } static int basl_finish_install_guest_tables(struct basl_table *const table, uint32_t *const off) { void *gva; table->off = roundup2(*off, table->alignment); *off = table->off + table->len; if (*off <= table->off) { warnx("%s: invalid table length 0x%8x @ offset 0x%8x", __func__, table->len, table->off); return (EFAULT); } /* Cause guest BIOS to copy the ACPI table into guest memory. */ #ifdef __FreeBSD__ BASL_EXEC( qemu_fwcfg_add_file(table->fwcfg_name, table->len, table->data)); #else BASL_EXEC( qemu_fwcfg_add_file((const char *)table->fwcfg_name, table->len, table->data)); #endif BASL_EXEC(qemu_loader_alloc(basl_loader, table->fwcfg_name, table->alignment, QEMU_LOADER_ALLOC_HIGH)); if (!load_into_memory) { return (0); } /* * Install ACPI tables directly in guest memory for use by guests which * do not boot via EFI. EFI ROMs provide a pointer to the firmware * generated ACPI tables instead, but it doesn't hurt to install the * tables always. */ gva = vm_map_gpa(table->ctx, BHYVE_ACPI_BASE + table->off, table->len); if (gva == NULL) { warnx("%s: could not map gpa [ 0x%16lx, 0x%16lx ]", __func__, (uint64_t)BHYVE_ACPI_BASE + table->off, (uint64_t)BHYVE_ACPI_BASE + table->off + table->len); return (ENOMEM); } memcpy(gva, table->data, table->len); return (0); } static int basl_finish_patch_checksums(struct basl_table *const table) { struct basl_table_checksum *checksum; STAILQ_FOREACH(checksum, &table->checksums, chain) { uint8_t *gva, *checksum_gva; uint64_t gpa; uint32_t len; uint8_t sum; len = checksum->len; if (len == BASL_TABLE_CHECKSUM_LEN_FULL_TABLE) { len = table->len; } assert(checksum->off < table->len); assert(checksum->start < table->len); assert(checksum->start + len <= table->len); /* Cause guest BIOS to patch the checksum. */ BASL_EXEC(qemu_loader_add_checksum(basl_loader, table->fwcfg_name, checksum->off, checksum->start, len)); if (!load_into_memory) { continue; } /* * Install ACPI tables directly in guest memory for use by * guests which do not boot via EFI. EFI ROMs provide a pointer * to the firmware generated ACPI tables instead, but it doesn't * hurt to install the tables always. */ gpa = BHYVE_ACPI_BASE + table->off + checksum->start; if ((gpa < BHYVE_ACPI_BASE) || (gpa < BHYVE_ACPI_BASE + table->off)) { warnx("%s: invalid gpa (off 0x%8x start 0x%8x)", __func__, table->off, checksum->start); return (EFAULT); } gva = vm_map_gpa(table->ctx, gpa, len); if (gva == NULL) { warnx("%s: could not map gpa [ 0x%16lx, 0x%16lx ]", __func__, gpa, gpa + len); return (ENOMEM); } checksum_gva = gva + checksum->off; if (checksum_gva < gva) { warnx("%s: invalid checksum offset 0x%8x", __func__, checksum->off); return (EFAULT); } sum = 0; for (uint32_t i = 0; i < len; ++i) { sum += *(gva + i); } *checksum_gva = -sum; } return (0); } static struct basl_table * basl_get_table_by_signature(const uint8_t signature[ACPI_NAMESEG_SIZE]) { struct basl_table *table; STAILQ_FOREACH(table, &basl_tables, chain) { const ACPI_TABLE_HEADER *const header = (const ACPI_TABLE_HEADER *)table->data; #ifdef __FreeBSD__ if (strncmp(header->Signature, signature, sizeof(header->Signature)) == 0) { return (table); #else if (strncmp(header->Signature, (char *)signature, sizeof(header->Signature)) == 0) { return (table); #endif } } warnx("%s: %.4s not found", __func__, signature); return (NULL); } static int basl_finish_patch_pointers(struct basl_table *const table) { struct basl_table_pointer *pointer; STAILQ_FOREACH(pointer, &table->pointers, chain) { const struct basl_table *src_table; uint8_t *gva; uint64_t gpa, val; assert(pointer->off < table->len); assert(pointer->off + pointer->size <= table->len); src_table = basl_get_table_by_signature(pointer->src_signature); if (src_table == NULL) { warnx("%s: could not find ACPI table %.4s", __func__, pointer->src_signature); return (EFAULT); } /* Cause guest BIOS to patch the pointer. */ BASL_EXEC( qemu_loader_add_pointer(basl_loader, table->fwcfg_name, src_table->fwcfg_name, pointer->off, pointer->size)); if (!load_into_memory) { continue; } /* * Install ACPI tables directly in guest memory for use by * guests which do not boot via EFI. EFI ROMs provide a pointer * to the firmware generated ACPI tables instead, but it doesn't * hurt to install the tables always. */ gpa = BHYVE_ACPI_BASE + table->off; if (gpa < BHYVE_ACPI_BASE) { warnx("%s: table offset of 0x%8x is too large", __func__, table->off); return (EFAULT); } gva = vm_map_gpa(table->ctx, gpa, table->len); if (gva == NULL) { warnx("%s: could not map gpa [ 0x%16lx, 0x%16lx ]", __func__, gpa, gpa + table->len); return (ENOMEM); } val = basl_le_dec(gva + pointer->off, pointer->size); val += BHYVE_ACPI_BASE + src_table->off; basl_le_enc(gva + pointer->off, val, pointer->size); } return (0); } static int basl_finish_set_length(struct basl_table *const table) { struct basl_table_length *length; STAILQ_FOREACH(length, &table->lengths, chain) { assert(length->off < table->len); assert(length->off + length->size <= table->len); basl_le_enc((uint8_t *)table->data + length->off, table->len, length->size); } return (0); } int basl_finish(void) { struct basl_table *table; uint32_t off = 0; if (STAILQ_EMPTY(&basl_tables)) { warnx("%s: no ACPI tables found", __func__); return (EINVAL); } /* * If we install ACPI tables by FwCfg and by memory, Windows will use * the tables from memory. This can cause issues when using advanced * features like a TPM log because we aren't able to patch the memory * tables accordingly. */ load_into_memory = get_config_bool_default("acpi_tables_in_memory", true); #ifndef __FreeBSD__ if (get_config_bool_default("basl.debug", false)) basl_dump(false); #endif /* * We have to install all tables before we can patch them. Therefore, * use two loops. The first one installs all tables and the second one * patches them. */ STAILQ_FOREACH(table, &basl_tables, chain) { BASL_EXEC(basl_finish_set_length(table)); BASL_EXEC(basl_finish_install_guest_tables(table, &off)); } STAILQ_FOREACH(table, &basl_tables, chain) { BASL_EXEC(basl_finish_patch_pointers(table)); /* * Calculate the checksum as last step! */ BASL_EXEC(basl_finish_patch_checksums(table)); } BASL_EXEC(qemu_loader_finish(basl_loader)); #ifndef __FreeBSD__ if (get_config_bool_default("basl.debug", false)) basl_dump(true); #endif return (0); } static int basl_init_rsdt(struct vmctx *const ctx) { BASL_EXEC( basl_table_create(&rsdt, ctx, ACPI_SIG_RSDT, BASL_TABLE_ALIGNMENT)); /* Header */ BASL_EXEC(basl_table_append_header(rsdt, ACPI_SIG_RSDT, 1, 1)); /* Pointers (added by basl_table_register_to_rsdt) */ return (0); } static int basl_init_xsdt(struct vmctx *const ctx) { BASL_EXEC( basl_table_create(&xsdt, ctx, ACPI_SIG_XSDT, BASL_TABLE_ALIGNMENT)); /* Header */ BASL_EXEC(basl_table_append_header(xsdt, ACPI_SIG_XSDT, 1, 1)); /* Pointers (added by basl_table_register_to_rsdt) */ return (0); } int basl_init(struct vmctx *const ctx) { BASL_EXEC(basl_init_rsdt(ctx)); BASL_EXEC(basl_init_xsdt(ctx)); #ifdef __FreeBSD__ BASL_EXEC( qemu_loader_create(&basl_loader, QEMU_FWCFG_FILE_TABLE_LOADER)); #else BASL_EXEC( qemu_loader_create(&basl_loader, (uint8_t *)QEMU_FWCFG_FILE_TABLE_LOADER)); #endif return (0); } int basl_table_add_checksum(struct basl_table *const table, const uint32_t off, const uint32_t start, const uint32_t len) { struct basl_table_checksum *checksum; assert(table != NULL); checksum = calloc(1, sizeof(struct basl_table_checksum)); if (checksum == NULL) { warnx("%s: failed to allocate checksum", __func__); return (ENOMEM); } checksum->off = off; checksum->start = start; checksum->len = len; STAILQ_INSERT_TAIL(&table->checksums, checksum, chain); return (0); } int basl_table_add_length(struct basl_table *const table, const uint32_t off, const uint8_t size) { struct basl_table_length *length; assert(table != NULL); assert(size == 4 || size == 8); length = calloc(1, sizeof(struct basl_table_length)); if (length == NULL) { warnx("%s: failed to allocate length", __func__); return (ENOMEM); } length->off = off; length->size = size; STAILQ_INSERT_TAIL(&table->lengths, length, chain); return (0); } int basl_table_add_pointer(struct basl_table *const table, const uint8_t src_signature[ACPI_NAMESEG_SIZE], const uint32_t off, const uint8_t size) { struct basl_table_pointer *pointer; assert(table != NULL); assert(size == 4 || size == 8); pointer = calloc(1, sizeof(struct basl_table_pointer)); if (pointer == NULL) { warnx("%s: failed to allocate pointer", __func__); return (ENOMEM); } memcpy(pointer->src_signature, src_signature, sizeof(pointer->src_signature)); pointer->off = off; pointer->size = size; STAILQ_INSERT_TAIL(&table->pointers, pointer, chain); return (0); } int basl_table_append_bytes(struct basl_table *const table, const void *const bytes, const uint32_t len) { void *end; assert(table != NULL); assert(bytes != NULL); if (table->len + len <= table->len) { warnx("%s: table too large (table->len 0x%8x len 0x%8x)", __func__, table->len, len); return (EFAULT); } table->data = reallocf(table->data, table->len + len); if (table->data == NULL) { warnx("%s: failed to realloc table to length 0x%8x", __func__, table->len + len); table->len = 0; return (ENOMEM); } end = (uint8_t *)table->data + table->len; table->len += len; memcpy(end, bytes, len); return (0); } int basl_table_append_checksum(struct basl_table *const table, const uint32_t start, const uint32_t len) { assert(table != NULL); BASL_EXEC(basl_table_add_checksum(table, table->len, start, len)); BASL_EXEC(basl_table_append_int(table, 0, 1)); return (0); } int basl_table_append_content(struct basl_table *table, void *data, uint32_t len) { assert(data != NULL); assert(len >= sizeof(ACPI_TABLE_HEADER)); return (basl_table_append_bytes(table, (void *)((uintptr_t)(data) + sizeof(ACPI_TABLE_HEADER)), len - sizeof(ACPI_TABLE_HEADER))); } int basl_table_append_fwcfg(struct basl_table *const table, const uint8_t *fwcfg_name, const uint32_t alignment, const uint8_t size) { assert(table != NULL); assert(fwcfg_name != NULL); assert(size <= sizeof(uint64_t)); BASL_EXEC(qemu_loader_alloc(basl_loader, fwcfg_name, alignment, QEMU_LOADER_ALLOC_HIGH)); BASL_EXEC(qemu_loader_add_pointer(basl_loader, table->fwcfg_name, fwcfg_name, table->len, size)); BASL_EXEC(basl_table_append_int(table, 0, size)); return (0); } int basl_table_append_gas(struct basl_table *const table, const uint8_t space_id, const uint8_t bit_width, const uint8_t bit_offset, const uint8_t access_width, const uint64_t address) { ACPI_GENERIC_ADDRESS gas_le = { .SpaceId = space_id, .BitWidth = bit_width, .BitOffset = bit_offset, .AccessWidth = access_width, .Address = htole64(address), }; return (basl_table_append_bytes(table, &gas_le, sizeof(gas_le))); } int basl_table_append_header(struct basl_table *const table, const uint8_t signature[ACPI_NAMESEG_SIZE], const uint8_t revision, const uint32_t oem_revision) { ACPI_TABLE_HEADER header_le; /* + 1 is required for the null terminator */ char oem_table_id[ACPI_OEM_TABLE_ID_SIZE + 1]; assert(table != NULL); assert(table->len == 0); memcpy(header_le.Signature, signature, ACPI_NAMESEG_SIZE); header_le.Length = 0; /* patched by basl_finish */ header_le.Revision = revision; header_le.Checksum = 0; /* patched by basl_finish */ memcpy(header_le.OemId, "BHYVE ", ACPI_OEM_ID_SIZE); snprintf(oem_table_id, ACPI_OEM_TABLE_ID_SIZE, "BV%.4s ", signature); memcpy(header_le.OemTableId, oem_table_id, sizeof(header_le.OemTableId)); header_le.OemRevision = htole32(oem_revision); memcpy(header_le.AslCompilerId, "BASL", ACPI_NAMESEG_SIZE); header_le.AslCompilerRevision = htole32(0x20220504); BASL_EXEC( basl_table_append_bytes(table, &header_le, sizeof(header_le))); BASL_EXEC(basl_table_add_length(table, offsetof(ACPI_TABLE_HEADER, Length), sizeof(header_le.Length))); BASL_EXEC(basl_table_add_checksum(table, offsetof(ACPI_TABLE_HEADER, Checksum), 0, BASL_TABLE_CHECKSUM_LEN_FULL_TABLE)); return (0); } int basl_table_append_int(struct basl_table *const table, const uint64_t val, const uint8_t size) { char buf[8]; assert(size <= sizeof(val)); basl_le_enc(buf, val, size); return (basl_table_append_bytes(table, buf, size)); } int basl_table_append_length(struct basl_table *const table, const uint8_t size) { assert(table != NULL); assert(size <= sizeof(table->len)); BASL_EXEC(basl_table_add_length(table, table->len, size)); BASL_EXEC(basl_table_append_int(table, 0, size)); return (0); } int basl_table_append_pointer(struct basl_table *const table, const uint8_t src_signature[ACPI_NAMESEG_SIZE], const uint8_t size) { assert(table != NULL); assert(size == 4 || size == 8); BASL_EXEC(basl_table_add_pointer(table, src_signature, table->len, size)); BASL_EXEC(basl_table_append_int(table, 0, size)); return (0); } int basl_table_create(struct basl_table **const table, struct vmctx *ctx, const uint8_t *const name, const uint32_t alignment) { struct basl_table *new_table; assert(table != NULL); new_table = calloc(1, sizeof(struct basl_table)); if (new_table == NULL) { warnx("%s: failed to allocate table", __func__); return (ENOMEM); } new_table->ctx = ctx; #ifdef __FreeBSD__ snprintf(new_table->fwcfg_name, sizeof(new_table->fwcfg_name), "etc/acpi/%s", name); #else snprintf((char *)new_table->fwcfg_name, sizeof (new_table->fwcfg_name), "etc/acpi/%s", name); #endif new_table->alignment = alignment; STAILQ_INIT(&new_table->checksums); STAILQ_INIT(&new_table->lengths); STAILQ_INIT(&new_table->pointers); STAILQ_INSERT_TAIL(&basl_tables, new_table, chain); *table = new_table; return (0); } int basl_table_register_to_rsdt(struct basl_table *table) { const ACPI_TABLE_HEADER *header; assert(table != NULL); header = (const ACPI_TABLE_HEADER *)table->data; #ifdef __FreeBSD__ BASL_EXEC(basl_table_append_pointer(rsdt, header->Signature, ACPI_RSDT_ENTRY_SIZE)); BASL_EXEC(basl_table_append_pointer(xsdt, header->Signature, ACPI_XSDT_ENTRY_SIZE)); #else BASL_EXEC(basl_table_append_pointer(rsdt, (uint8_t *)header->Signature, ACPI_RSDT_ENTRY_SIZE)); BASL_EXEC(basl_table_append_pointer(xsdt, (uint8_t *)header->Signature, ACPI_XSDT_ENTRY_SIZE)); #endif return (0); } /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2022 Beckhoff Automation GmbH & Co. KG */ #pragma once #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wunused-parameter" #include #pragma GCC diagnostic pop #ifndef __FreeBSD__ /* Until the in-gate ACPI is updated, map the new name to the old. */ #define ACPI_NAMESEG_SIZE ACPI_NAME_SIZE /* * Also to avoid type conflicts without having to pepper the code with ifdefs, * redefine these constants to be uint8_t * */ #undef ACPI_SIG_FACS #undef ACPI_SIG_MCFG #undef ACPI_SIG_HPET #undef ACPI_SIG_MADT #undef ACPI_SIG_DSDT #undef ACPI_SIG_FADT #undef ACPI_SIG_XSDT #undef ACPI_SIG_RSDT #undef ACPI_RSDP_NAME #undef ACPI_SIG_SPCR #undef ACPI_SIG_TPM2 #define ACPI_SIG_FACS (const uint8_t *)"FACS" #define ACPI_SIG_MCFG (const uint8_t *)"MCFG" #define ACPI_SIG_HPET (const uint8_t *)"HPET" #define ACPI_SIG_MADT (const uint8_t *)"APIC" #define ACPI_SIG_DSDT (const uint8_t *)"DSDT" #define ACPI_SIG_FADT (const uint8_t *)"FACP" #define ACPI_SIG_XSDT (const uint8_t *)"XSDT" #define ACPI_SIG_RSDT (const uint8_t *)"RSDT" #define ACPI_RSDP_NAME (const uint8_t *)"RSDP" #define ACPI_SIG_SPCR (const uint8_t *)"SPCR" #define ACPI_SIG_TPM2 (const uint8_t *)"TPM2" #endif /* !__FreeBSD__ */ #include "qemu_fwcfg.h" #define ACPI_GAS_ACCESS_WIDTH_LEGACY 0 #define ACPI_GAS_ACCESS_WIDTH_UNDEFINED 0 #define ACPI_GAS_ACCESS_WIDTH_BYTE 1 #define ACPI_GAS_ACCESS_WIDTH_WORD 2 #define ACPI_GAS_ACCESS_WIDTH_DWORD 3 #define ACPI_GAS_ACCESS_WIDTH_QWORD 4 #define ACPI_SPCR_INTERRUPT_TYPE_8259 0x1 #define ACPI_SPCR_INTERRUPT_TYPE_APIC 0x2 #define ACPI_SPCR_INTERRUPT_TYPE_SAPIC 0x4 #define ACPI_SPCR_INTERRUPT_TYPE_GIC 0x8 #define ACPI_SPCR_BAUD_RATE_9600 3 #define ACPI_SPCR_BAUD_RATE_19200 4 #define ACPI_SPCR_BAUD_RATE_57600 6 #define ACPI_SPCR_BAUD_RATE_115200 7 #define ACPI_SPCR_PARITY_NO_PARITY 0 #define ACPI_SPCR_STOP_BITS_1 1 #define ACPI_SPCR_FLOW_CONTROL_DCD 0x1 #define ACPI_SPCR_FLOW_CONTROL_RTS_CTS 0x2 #define ACPI_SPCR_FLOW_CONTROL_XON_XOFF 0x4 #define ACPI_SPCR_TERMINAL_TYPE_VT100 0 #define ACPI_SPCR_TERMINAL_TYPE_VT100_PLUS 1 #define ACPI_SPCR_TERMINAL_TYPE_VT_UTF8 2 #define ACPI_SPCR_TERMINAL_TYPE_ANSI 3 #define BHYVE_ACPI_BASE 0xf2400 #define BASL_TABLE_ALIGNMENT 0x10 #define BASL_TABLE_ALIGNMENT_FACS 0x40 #define BASL_TABLE_CHECKSUM_LEN_FULL_TABLE (-1U) #define BASL_EXEC(x) \ do { \ const int error = (x); \ if (error) { \ warnc(error, \ "BASL failed @ %s:%d\n Failed to execute %s", \ __func__, __LINE__, #x); \ return (error); \ } \ } while (0) struct basl_table; void basl_fill_gas(ACPI_GENERIC_ADDRESS *gas, uint8_t space_id, uint8_t bit_width, uint8_t bit_offset, uint8_t access_width, uint64_t address); int basl_finish(void); int basl_init(struct vmctx *ctx); int basl_table_add_checksum(struct basl_table *const table, const uint32_t off, const uint32_t start, const uint32_t len); int basl_table_add_length(struct basl_table *const table, const uint32_t off, const uint8_t size); int basl_table_add_pointer(struct basl_table *const table, const uint8_t src_signature[ACPI_NAMESEG_SIZE], const uint32_t off, const uint8_t size); int basl_table_append_bytes(struct basl_table *table, const void *bytes, uint32_t len); int basl_table_append_checksum(struct basl_table *table, uint32_t start, uint32_t len); /* Add an ACPI_TABLE_* to basl without its header. */ int basl_table_append_content(struct basl_table *table, void *data, uint32_t len); int basl_table_append_fwcfg(struct basl_table *table, const uint8_t *fwcfg_name, uint32_t alignment, uint8_t size); int basl_table_append_gas(struct basl_table *table, uint8_t space_id, uint8_t bit_width, uint8_t bit_offset, uint8_t access_width, uint64_t address); int basl_table_append_header(struct basl_table *table, const uint8_t signature[ACPI_NAMESEG_SIZE], uint8_t revision, uint32_t oem_revision); int basl_table_append_int(struct basl_table *table, uint64_t val, uint8_t size); int basl_table_append_length(struct basl_table *table, uint8_t size); int basl_table_append_pointer(struct basl_table *table, const uint8_t src_signature[ACPI_NAMESEG_SIZE], uint8_t size); int basl_table_create(struct basl_table **table, struct vmctx *ctx, const uint8_t *name, uint32_t alignment); /* Adds the table to RSDT and XSDT */ int basl_table_register_to_rsdt(struct basl_table *table); /* * This file and its contents are supplied under the terms of the * Common Development and Distribution License ("CDDL"), version 1.0. * You may only use this file in accordance with the terms of version * 1.0 of the CDDL. * * A full copy of the text of the CDDL should have accompanied this * source. A copy of the CDDL is also available via the Internet at * http://www.illumos.org/license/CDDL. */ /* This file is dual-licensed; see usr/src/contrib/bhyve/LICENSE */ /* * Copyright 2013 Pluribus Networks Inc. * Copyright 2017 Joyent, Inc. */ #include #include #include /* * Make a pre-existing termios structure into "raw" mode: character-at-a-time * mode with no characters interpreted, 8-bit data path. */ void cfmakeraw(struct termios *t) { t->c_iflag &= ~(IMAXBEL|IXOFF|INPCK|BRKINT|PARMRK|ISTRIP|INLCR|IGNCR| ICRNL|IXON|IGNPAR); t->c_iflag |= IGNBRK; t->c_oflag &= ~OPOST; t->c_lflag &= ~(ECHO|ECHOE|ECHOK|ECHONL|ICANON|ISIG|IEXTEN|NOFLSH| TOSTOP|PENDIN); t->c_cflag &= ~(CSIZE|PARENB); t->c_cflag |= CS8|CREAD; t->c_cc[VMIN] = 1; t->c_cc[VTIME] = 0; } /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2015 Tycho Nightingale * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include #include #include #include #ifndef __FreeBSD__ #include #endif #include "bhyvegc.h" struct bhyvegc { struct bhyvegc_image *gc_image; int raw; }; struct bhyvegc * bhyvegc_init(int width, int height, void *fbaddr) { struct bhyvegc *gc; struct bhyvegc_image *gc_image; gc = calloc(1, sizeof (struct bhyvegc)); gc_image = calloc(1, sizeof(struct bhyvegc_image)); gc_image->width = width; gc_image->height = height; if (fbaddr) { gc_image->data = fbaddr; gc->raw = 1; } else { gc_image->data = calloc(width * height, sizeof (uint32_t)); gc->raw = 0; } gc->gc_image = gc_image; #ifndef __FreeBSD__ pthread_mutex_init(&gc_image->mtx, NULL); #endif return (gc); } void bhyvegc_set_fbaddr(struct bhyvegc *gc, void *fbaddr) { #ifndef __FreeBSD__ pthread_mutex_lock(&gc->gc_image->mtx); #endif gc->raw = 1; if (gc->gc_image->data && gc->gc_image->data != fbaddr) free(gc->gc_image->data); gc->gc_image->data = fbaddr; #ifndef __FreeBSD__ pthread_mutex_unlock(&gc->gc_image->mtx); #endif } void bhyvegc_resize(struct bhyvegc *gc, int width, int height) { struct bhyvegc_image *gc_image; #ifndef __FreeBSD__ pthread_mutex_lock(&gc->gc_image->mtx); #endif gc_image = gc->gc_image; gc_image->width = width; gc_image->height = height; if (!gc->raw) { gc_image->data = reallocarray(gc_image->data, width * height, sizeof (uint32_t)); if (gc_image->data != NULL) memset(gc_image->data, 0, width * height * sizeof (uint32_t)); } #ifndef __FreeBSD__ pthread_mutex_unlock(&gc->gc_image->mtx); #endif } struct bhyvegc_image * bhyvegc_get_image(struct bhyvegc *gc) { if (gc == NULL) return (NULL); return (gc->gc_image); } /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2015 Tycho Nightingale * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef _BHYVEGC_H_ #define _BHYVEGC_H_ #ifndef __FreeBSD__ #include #endif struct bhyvegc; struct bhyvegc_image { int vgamode; int width; int height; uint32_t *data; #ifndef __FreeBSD__ pthread_mutex_t mtx; #endif }; struct bhyvegc *bhyvegc_init(int width, int height, void *fbaddr); void bhyvegc_set_fbaddr(struct bhyvegc *gc, void *fbaddr); void bhyvegc_resize(struct bhyvegc *gc, int width, int height); struct bhyvegc_image *bhyvegc_get_image(struct bhyvegc *gc); #endif /* _BHYVEGC_H_ */ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2011 NetApp, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * This file and its contents are supplied under the terms of the * Common Development and Distribution License ("CDDL"), version 1.0. * You may only use this file in accordance with the terms of version * 1.0 of the CDDL. * * A full copy of the text of the CDDL should have accompanied this * source. A copy of the CDDL is also available via the Internet at * http://www.illumos.org/license/CDDL. * * Copyright 2015 Pluribus Networks Inc. * Copyright 2018 Joyent, Inc. * Copyright 2025 Oxide Computer Company * Copyright 2026 OmniOS Community Edition (OmniOSce) Association. */ #include #ifndef WITHOUT_CAPSICUM #include #endif #include #include #ifdef __FreeBSD__ #include #else #include #include #endif #include #ifndef WITHOUT_CAPSICUM #include #endif #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifndef WITHOUT_CAPSICUM #include #endif #ifdef __FreeBSD__ #include #endif #include #include "acpi.h" #include "bhyverun.h" #include "bootrom.h" #include "config.h" #include "debug.h" #include "gdb.h" #include "ioapic.h" #include "mem.h" #include "mevent.h" #include "pci_emul.h" #include "pci_lpc.h" #include "qemu_fwcfg.h" #include "tpm_device.h" #include "spinup_ap.h" #include "vmgenc.h" #include "vmexit.h" #ifndef __FreeBSD__ #include "smbiostbl.h" #include "privileges.h" #endif #define MB (1024UL * 1024) #define GB (1024UL * MB) int guest_ncpus; uint16_t cpu_cores, cpu_sockets, cpu_threads; int raw_stdio = 0; static const int BSP = 0; static cpuset_t cpumask; static void vm_loop(struct vmctx *ctx, struct vcpu *vcpu); static struct vcpu_info { struct vmctx *ctx; struct vcpu *vcpu; int vcpuid; } *vcpu_info; #ifdef __FreeBSD__ static cpuset_t **vcpumap; #endif /* * XXX This parser is known to have the following issues: * 1. It accepts null key=value tokens ",," as setting "cpus" to an * empty string. * * The acceptance of a null specification ('-c ""') is by design to match the * manual page syntax specification, this results in a topology of 1 vCPU. */ int bhyve_topology_parse(const char *opt) { char *cp, *str, *tofree; if (*opt == '\0') { set_config_value("sockets", "1"); set_config_value("cores", "1"); set_config_value("threads", "1"); set_config_value("cpus", "1"); return (0); } tofree = str = strdup(opt); if (str == NULL) errx(4, "Failed to allocate memory"); while ((cp = strsep(&str, ",")) != NULL) { if (strncmp(cp, "cpus=", strlen("cpus=")) == 0) set_config_value("cpus", cp + strlen("cpus=")); else if (strncmp(cp, "sockets=", strlen("sockets=")) == 0) set_config_value("sockets", cp + strlen("sockets=")); else if (strncmp(cp, "cores=", strlen("cores=")) == 0) set_config_value("cores", cp + strlen("cores=")); else if (strncmp(cp, "threads=", strlen("threads=")) == 0) set_config_value("threads", cp + strlen("threads=")); else if (strchr(cp, '=') != NULL) goto out; else set_config_value("cpus", cp); } free(tofree); return (0); out: free(tofree); return (-1); } static int parse_int_value(const char *key, const char *value, int minval, int maxval) { char *cp; long lval; errno = 0; lval = strtol(value, &cp, 0); if (errno != 0 || *cp != '\0' || cp == value || lval < minval || lval > maxval) errx(4, "Invalid value for %s: '%s'", key, value); return (lval); } /* * Set the sockets, cores, threads, and guest_cpus variables based on * the configured topology. * * The limits of UINT16_MAX are due to the types passed to * vm_set_topology(). vmm.ko may enforce tighter limits. */ static void calc_topology(void) { const char *value; bool explicit_cpus; uint64_t ncpus; value = get_config_value("cpus"); if (value != NULL) { guest_ncpus = parse_int_value("cpus", value, 1, UINT16_MAX); explicit_cpus = true; } else { guest_ncpus = 1; explicit_cpus = false; } value = get_config_value("cores"); if (value != NULL) cpu_cores = parse_int_value("cores", value, 1, UINT16_MAX); else cpu_cores = 1; value = get_config_value("threads"); if (value != NULL) cpu_threads = parse_int_value("threads", value, 1, UINT16_MAX); else cpu_threads = 1; value = get_config_value("sockets"); if (value != NULL) cpu_sockets = parse_int_value("sockets", value, 1, UINT16_MAX); else cpu_sockets = guest_ncpus; /* * Compute sockets * cores * threads avoiding overflow. The * range check above insures these are 16 bit values. */ ncpus = (uint64_t)cpu_sockets * cpu_cores * cpu_threads; if (ncpus > UINT16_MAX) errx(4, "Computed number of vCPUs too high: %ju", (uintmax_t)ncpus); if (explicit_cpus) { if (guest_ncpus != (int)ncpus) errx(4, "Topology (%d sockets, %d cores, %d threads) " "does not match %d vCPUs", cpu_sockets, cpu_cores, cpu_threads, guest_ncpus); } else guest_ncpus = ncpus; } #ifdef __FreeBSD__ int bhyve_pincpu_parse(const char *opt) { int vcpu, pcpu; const char *value; char *newval; char key[16]; if (sscanf(opt, "%d:%d", &vcpu, &pcpu) != 2) { fprintf(stderr, "invalid format: %s\n", opt); return (-1); } if (vcpu < 0) { fprintf(stderr, "invalid vcpu '%d'\n", vcpu); return (-1); } if (pcpu < 0 || pcpu >= CPU_SETSIZE) { fprintf(stderr, "hostcpu '%d' outside valid range from " "0 to %d\n", pcpu, CPU_SETSIZE - 1); return (-1); } snprintf(key, sizeof(key), "vcpu.%d.cpuset", vcpu); value = get_config_value(key); if (asprintf(&newval, "%s%s%d", value != NULL ? value : "", value != NULL ? "," : "", pcpu) == -1) { perror("failed to build new cpuset string"); return (-1); } set_config_value(key, newval); free(newval); return (0); } static void parse_cpuset(int vcpu, const char *list, cpuset_t *set) { char *cp, *token; int pcpu, start; CPU_ZERO(set); start = -1; token = __DECONST(char *, list); for (;;) { pcpu = strtoul(token, &cp, 0); if (cp == token) errx(4, "invalid cpuset for vcpu %d: '%s'", vcpu, list); if (pcpu < 0 || pcpu >= CPU_SETSIZE) errx(4, "hostcpu '%d' outside valid range from 0 to %d", pcpu, CPU_SETSIZE - 1); switch (*cp) { case ',': case '\0': if (start >= 0) { if (start > pcpu) errx(4, "Invalid hostcpu range %d-%d", start, pcpu); while (start < pcpu) { CPU_SET(start, set); start++; } start = -1; } CPU_SET(pcpu, set); break; case '-': if (start >= 0) errx(4, "invalid cpuset for vcpu %d: '%s'", vcpu, list); start = pcpu; break; default: errx(4, "invalid cpuset for vcpu %d: '%s'", vcpu, list); } if (*cp == '\0') break; token = cp + 1; } } static void build_vcpumaps(void) { char key[16]; const char *value; int vcpu; vcpumap = calloc(guest_ncpus, sizeof(*vcpumap)); for (vcpu = 0; vcpu < guest_ncpus; vcpu++) { snprintf(key, sizeof(key), "vcpu.%d.cpuset", vcpu); value = get_config_value(key); if (value == NULL) continue; vcpumap[vcpu] = malloc(sizeof(cpuset_t)); if (vcpumap[vcpu] == NULL) err(4, "Failed to allocate cpuset for vcpu %d", vcpu); parse_cpuset(vcpu, value, vcpumap[vcpu]); } } #endif /* __FreeBSD__ */ void * paddr_guest2host(struct vmctx *ctx, uintptr_t gaddr, size_t len) { return (vm_map_gpa(ctx, gaddr, len)); } bool fbsdrun_virtio_msix(void) { /* * The configuration key used to be the non-namespaced `virtio_msix`. * Check for that too in case anyone is using the old name in a * configuration file. If either of these exists and is set to false * we'll disable MSI-X for Virtio. */ return (get_config_bool_default("virtio.msix", true) && get_config_bool_default("virtio_msix", true)); } struct vcpu * fbsdrun_vcpu(int vcpuid) { return (vcpu_info[vcpuid].vcpu); } static void * fbsdrun_start_thread(void *param) { char tname[MAXCOMLEN + 1]; struct vcpu_info *vi = param; #ifdef __FreeBSD__ int error; #endif snprintf(tname, sizeof(tname), "vcpu %d", vi->vcpuid); pthread_set_name_np(pthread_self(), tname); #ifdef __FreeBSD__ if (vcpumap[vi->vcpuid] != NULL) { error = pthread_setaffinity_np(pthread_self(), sizeof(cpuset_t), vcpumap[vi->vcpuid]); assert(error == 0); } #endif gdb_cpu_add(vi->vcpu); vm_loop(vi->ctx, vi->vcpu); /* not reached */ exit(1); return (NULL); } void fbsdrun_addcpu(int vcpuid, bool suspend) { struct vcpu_info *vi; pthread_t thr; int error; vi = &vcpu_info[vcpuid]; error = vm_activate_cpu(vi->vcpu); if (error != 0) err(EX_OSERR, "could not activate CPU %d", vi->vcpuid); CPU_SET_ATOMIC(vcpuid, &cpumask); if (suspend) { error = vm_suspend_cpu(vi->vcpu); assert(error == 0); } error = pthread_create(&thr, NULL, fbsdrun_start_thread, vi); assert(error == 0); } void fbsdrun_deletecpu(int vcpu) { static pthread_mutex_t resetcpu_mtx = PTHREAD_MUTEX_INITIALIZER; static pthread_cond_t resetcpu_cond = PTHREAD_COND_INITIALIZER; pthread_mutex_lock(&resetcpu_mtx); if (!CPU_ISSET(vcpu, &cpumask)) { EPRINTLN("Attempting to delete unknown cpu %d", vcpu); exit(4); } CPU_CLR(vcpu, &cpumask); if (vcpu != BSP) { pthread_cond_signal(&resetcpu_cond); pthread_mutex_unlock(&resetcpu_mtx); pthread_exit(NULL); /* NOTREACHED */ } while (!CPU_EMPTY(&cpumask)) { pthread_cond_wait(&resetcpu_cond, &resetcpu_mtx); } pthread_mutex_unlock(&resetcpu_mtx); } static void vm_loop(struct vmctx *ctx, struct vcpu *vcpu) { struct vm_exit vme; int error, rc; enum vm_exitcode exitcode; cpuset_t active_cpus; struct vm_entry *ventry; error = vm_active_cpus(ctx, &active_cpus); assert(CPU_ISSET(vcpu_id(vcpu), &active_cpus)); ventry = vmentry_vcpu(vcpu_id(vcpu)); while (1) { error = vm_run(vcpu, ventry, &vme); if (error != 0) break; if (ventry->cmd != VEC_DEFAULT) { /* * Discard any lingering entry state after it has been * submitted via vm_run(). */ bzero(ventry, sizeof (*ventry)); } exitcode = vme.exitcode; if (exitcode >= VM_EXITCODE_MAX || vmexit_handlers[exitcode] == NULL) { fprintf(stderr, "vm_loop: unexpected exitcode 0x%x\n", exitcode); exit(4); } rc = (*vmexit_handlers[exitcode])(ctx, vcpu, &vme); switch (rc) { case VMEXIT_CONTINUE: break; case VMEXIT_ABORT: abort(); default: exit(4); } } EPRINTLN("vm_run error %d, errno %d", error, errno); } static int num_vcpus_allowed(struct vmctx *ctx, struct vcpu *vcpu) { uint16_t sockets, cores, threads, maxcpus; #ifdef __FreeBSD__ int tmp, error; /* * The guest is allowed to spinup more than one processor only if the * UNRESTRICTED_GUEST capability is available. */ error = vm_get_capability(vcpu, VM_CAP_UNRESTRICTED_GUEST, &tmp); if (error != 0) return (1); #else int error; /* Unrestricted Guest is always enabled on illumos */ #endif /* __FreeBSD__ */ error = vm_get_topology(ctx, &sockets, &cores, &threads, &maxcpus); if (error == 0) return (maxcpus); else return (1); } static struct vmctx * do_open(const char *vmname) { struct vmctx *ctx; int error; bool reinit, romboot; reinit = romboot = false; romboot = bootrom_boot(); #ifndef __FreeBSD__ uint64_t create_flags = 0; if (get_config_bool_default("memory.use_reservoir", false)) { create_flags |= VCF_RESERVOIR_MEM; } error = vm_create(vmname, create_flags); #else error = vm_create(vmname); #endif /* __FreeBSD__ */ if (error) { if (errno == EEXIST) { if (romboot) { reinit = true; } else { /* * The virtual machine has been setup by the * userspace bootloader. */ } } else { perror("vm_create"); exit(4); } } else { if (!romboot) { /* * If the virtual machine was just created then a * bootrom must be configured to boot it. */ fprintf(stderr, "virtual machine cannot be booted\n"); exit(4); } } ctx = vm_open(vmname); if (ctx == NULL) { perror("vm_open"); #ifndef __FreeBSD__ if (errno == ENOENT) { fprintf(stderr, "Does the VM already exist in the global or " "another zone?\n"); } #endif exit(4); } #ifndef WITHOUT_CAPSICUM if (vm_limit_rights(ctx) != 0) err(EX_OSERR, "vm_limit_rights"); #endif if (reinit) { #ifndef __FreeBSD__ error = vm_reinit(ctx, 0); #else error = vm_reinit(ctx); #endif if (error) { perror("vm_reinit"); exit(4); } } error = vm_set_topology(ctx, cpu_sockets, cpu_cores, cpu_threads, 0); if (error) errx(EX_OSERR, "vm_set_topology"); return (ctx); } bool bhyve_parse_config_option(const char *option) { const char *value; char *path; value = strchr(option, '='); if (value == NULL || value[1] == '\0') return (false); path = strndup(option, value - option); if (path == NULL) err(4, "Failed to allocate memory"); set_config_value(path, value + 1); free(path); return (true); } void bhyve_parse_simple_config_file(const char *path) { FILE *fp; char *line, *cp; size_t linecap; unsigned int lineno; fp = fopen(path, "r"); if (fp == NULL) err(4, "Failed to open configuration file %s", path); line = NULL; linecap = 0; lineno = 1; for (lineno = 1; getline(&line, &linecap, fp) > 0; lineno++) { if (*line == '#' || *line == '\n') continue; cp = strchr(line, '\n'); if (cp != NULL) *cp = '\0'; if (!bhyve_parse_config_option(line)) errx(4, "%s line %u: invalid config option '%s'", path, lineno, line); } free(line); fclose(fp); } void bhyve_parse_gdb_options(const char *opt) { const char *sport; char *colon; if (opt[0] == 'w') { set_config_bool("gdb.wait", true); opt++; } colon = strrchr(opt, ':'); if (colon == NULL) { sport = opt; } else { *colon = '\0'; colon++; sport = colon; set_config_value("gdb.address", opt); } set_config_value("gdb.port", sport); } int main(int argc, char *argv[]) { int error; int max_vcpus, memflags; struct vcpu *bsp; struct vmctx *ctx; size_t memsize; const char *value, *vmname; bhyve_init_config(); bhyve_optparse(argc, argv); argc -= optind; argv += optind; if (argc > 1) bhyve_usage(1); if (argc == 1) set_config_value("name", argv[0]); vmname = get_config_value("name"); if (vmname == NULL) bhyve_usage(1); if (get_config_bool_default("config.dump", false)) { dump_config(); exit(1); } #ifndef __FreeBSD__ illumos_priv_init(); #endif calc_topology(); #ifdef __FreeBSD__ build_vcpumaps(); #endif value = get_config_value("memory.size"); error = vm_parse_memsize(value, &memsize); if (error) errx(EX_USAGE, "invalid memsize '%s'", value); ctx = do_open(vmname); bsp = vm_vcpu_open(ctx, BSP); max_vcpus = num_vcpus_allowed(ctx, bsp); if (guest_ncpus > max_vcpus) { fprintf(stderr, "%d vCPUs requested but only %d available\n", guest_ncpus, max_vcpus); exit(4); } bhyve_init_vcpu(bsp); /* Allocate per-VCPU resources. */ vcpu_info = calloc(guest_ncpus, sizeof(*vcpu_info)); for (int vcpuid = 0; vcpuid < guest_ncpus; vcpuid++) { vcpu_info[vcpuid].ctx = ctx; vcpu_info[vcpuid].vcpuid = vcpuid; if (vcpuid == BSP) vcpu_info[vcpuid].vcpu = bsp; else vcpu_info[vcpuid].vcpu = vm_vcpu_open(ctx, vcpuid); } memflags = 0; if (get_config_bool_default("memory.wired", false)) memflags |= VM_MEM_F_WIRED; if (get_config_bool_default("memory.guest_in_core", false)) memflags |= VM_MEM_F_INCORE; vm_set_memflags(ctx, memflags); #ifdef __FreeBSD__ error = vm_setup_memory(ctx, memsize, VM_MMAP_ALL); #else int _errno; do { errno = 0; error = vm_setup_memory(ctx, memsize, VM_MMAP_ALL); _errno = errno; if (error != 0 && _errno == ENOMEM) { (void) fprintf(stderr, "Unable to allocate memory " "(%llu), retrying in 1 second\n", memsize); sleep(1); } } while (_errno == ENOMEM); #endif if (error) { fprintf(stderr, "Unable to set up memory (%d)\n", errno); exit(4); } init_mem(guest_ncpus); init_bootrom(ctx); if (bhyve_init_platform(ctx, bsp) != 0) exit(4); if (qemu_fwcfg_init(ctx) != 0) { fprintf(stderr, "qemu fwcfg initialization error\n"); exit(4); } if (qemu_fwcfg_add_file("opt/bhyve/hw.ncpu", sizeof(guest_ncpus), &guest_ncpus) != 0) { fprintf(stderr, "Could not add qemu fwcfg opt/bhyve/hw.ncpu\n"); exit(4); } /* * Exit if a device emulation finds an error in its initialization */ if (init_pci(ctx) != 0) { EPRINTLN("Device emulation initialization error: %s", strerror(errno)); exit(4); } if (init_tpm(ctx) != 0) { EPRINTLN("Failed to init TPM device"); exit(4); } /* * Initialize after PCI, to allow a bootrom file to reserve the high * region. */ if (get_config_bool("acpi_tables")) vmgenc_init(ctx); #ifdef __FreeBSD__ init_gdb(ctx); #else if (value != NULL) { int port = atoi(value); if (port < 0) init_mdb(ctx); else init_gdb(ctx); } #endif if (bootrom_boot()) { #ifdef __FreeBSD__ if (vm_set_capability(bsp, VM_CAP_UNRESTRICTED_GUEST, 1)) { fprintf(stderr, "ROM boot failed: unrestricted guest " "capability not available\n"); exit(4); } #else /* Unrestricted Guest is always enabled on illumos */ #endif error = vcpu_reset(bsp); assert(error == 0); } if (bhyve_init_platform_late(ctx, bsp) != 0) exit(4); /* * Change the proc title to include the VM name. */ setproctitle("%s", vmname); #ifndef WITHOUT_CAPSICUM caph_cache_catpages(); if (caph_limit_stdout() == -1 || caph_limit_stderr() == -1) errx(EX_OSERR, "Unable to apply rights for sandbox"); if (caph_enter() == -1) errx(EX_OSERR, "cap_enter() failed"); #endif #ifndef __FreeBSD__ illumos_priv_lock(); #endif #ifndef __FreeBSD__ if (vmentry_init(guest_ncpus) != 0) err(EX_OSERR, "vmentry_init() failed"); #endif /* * Add all vCPUs. */ for (int vcpuid = 0; vcpuid < guest_ncpus; vcpuid++) { #ifdef __FreeBSD__ bool suspend = (vcpuid != BSP); #else bool suspend = vcpuid == BSP && get_config_bool_default("suspend_at_boot", false); #endif bhyve_start_vcpu(vcpu_info[vcpuid].vcpu, vcpuid == BSP, suspend); } /* * Head off to the main event dispatch loop */ mevent_dispatch(); exit(4); } /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2011 NetApp, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * This file and its contents are supplied under the terms of the * Common Development and Distribution License ("CDDL"), version 1.0. * You may only use this file in accordance with the terms of version * 1.0 of the CDDL. * * A full copy of the text of the CDDL should have accompanied this * source. A copy of the CDDL is also available via the Internet at * http://www.illumos.org/license/CDDL. * * Copyright 2015 Pluribus Networks Inc. * Copyright 2025 Oxide Computer Company */ #ifndef _BHYVERUN_H_ #define _BHYVERUN_H_ #define VMEXIT_CONTINUE (0) #define VMEXIT_ABORT (-1) #include extern int guest_ncpus; extern uint16_t cpu_cores, cpu_sockets, cpu_threads; struct vcpu; struct vmctx; struct vm_exit; extern void *paddr_guest2host(struct vmctx *ctx, uintptr_t addr, size_t len); struct vcpu *fbsdrun_vcpu(int vcpuid); void fbsdrun_addcpu(int vcpuid, bool); void fbsdrun_deletecpu(int vcpuid); bool fbsdrun_virtio_msix(void); typedef int (*vmexit_handler_t)(struct vmctx *, struct vcpu *, struct vm_exit *); extern int vmexit_task_switch(struct vmctx *, struct vcpu *, struct vm_exit *); /* Interfaces implemented by machine-dependent code. */ void bhyve_init_config(void); void bhyve_init_vcpu(struct vcpu *vcpu); void bhyve_start_vcpu(struct vcpu *vcpu, bool bsp, bool suspend); int bhyve_init_platform(struct vmctx *ctx, struct vcpu *bsp); int bhyve_init_platform_late(struct vmctx *ctx, struct vcpu *bsp); void bhyve_optparse(int argc, char **argv); void bhyve_usage(int code); /* Interfaces used by command-line option-parsing code. */ bool bhyve_parse_config_option(const char *option); void bhyve_parse_simple_config_file(const char *path); void bhyve_parse_gdb_options(const char *opt); #ifdef __FreeBSD__ int bhyve_pincpu_parse(const char *opt); #endif int bhyve_topology_parse(const char *opt); #endif /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2013 Peter Grehan * All rights reserved. * Copyright 2020 Joyent, Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * Copyright 2020 Joyent, Inc. */ #include #ifndef WITHOUT_CAPSICUM #include #endif #include #include #include #include #include #ifndef __FreeBSD__ #include #include #include #endif #include #ifndef WITHOUT_CAPSICUM #include #endif #include #include #include #include #include #include #include #include #include #include #include #include "bhyverun.h" #include "config.h" #include "debug.h" #include "mevent.h" #include "pci_emul.h" #include "block_if.h" #define BLOCKIF_SIG 0xb109b109 #ifdef __FreeBSD__ #define BLOCKIF_NUMTHR 8 #else /* Enlarge to keep pace with the virtio-block ring size */ #define BLOCKIF_NUMTHR 16 #endif #define BLOCKIF_MAXREQ (BLOCKIF_RING_MAX + BLOCKIF_NUMTHR) enum blockop { BOP_READ, BOP_WRITE, #ifndef __FreeBSD__ BOP_WRITE_SYNC, #endif BOP_FLUSH, BOP_DELETE }; enum blockstat { BST_FREE, BST_BLOCK, BST_PEND, BST_BUSY, BST_DONE }; struct blockif_elem { TAILQ_ENTRY(blockif_elem) be_link; struct blockif_req *be_req; enum blockop be_op; enum blockstat be_status; pthread_t be_tid; off_t be_block; }; #ifndef __FreeBSD__ enum blockif_wce { WCE_NONE = 0, WCE_IOCTL, WCE_FCNTL }; #endif struct blockif_ctxt { unsigned int bc_magic; int bc_fd; int bc_ischr; int bc_isgeom; int bc_candelete; #ifndef __FreeBSD__ enum blockif_wce bc_wce; #endif int bc_rdonly; off_t bc_size; int bc_sectsz; int bc_psectsz; int bc_psectoff; int bc_closing; pthread_t bc_btid[BLOCKIF_NUMTHR]; pthread_mutex_t bc_mtx; pthread_cond_t bc_cond; blockif_resize_cb *bc_resize_cb; void *bc_resize_cb_arg; struct mevent *bc_resize_event; /* Request elements and free/pending/busy queues */ TAILQ_HEAD(, blockif_elem) bc_freeq; TAILQ_HEAD(, blockif_elem) bc_pendq; TAILQ_HEAD(, blockif_elem) bc_busyq; struct blockif_elem bc_reqs[BLOCKIF_MAXREQ]; int bc_bootindex; }; static pthread_once_t blockif_once = PTHREAD_ONCE_INIT; struct blockif_sig_elem { pthread_mutex_t bse_mtx; pthread_cond_t bse_cond; int bse_pending; struct blockif_sig_elem *bse_next; }; static struct blockif_sig_elem *blockif_bse_head; static int blockif_enqueue(struct blockif_ctxt *bc, struct blockif_req *breq, enum blockop op) { struct blockif_elem *be, *tbe; off_t off; int i; be = TAILQ_FIRST(&bc->bc_freeq); assert(be != NULL); assert(be->be_status == BST_FREE); TAILQ_REMOVE(&bc->bc_freeq, be, be_link); be->be_req = breq; be->be_op = op; switch (op) { case BOP_READ: case BOP_WRITE: #ifndef __FreeBSD__ case BOP_WRITE_SYNC: #endif case BOP_DELETE: off = breq->br_offset; for (i = 0; i < breq->br_iovcnt; i++) off += breq->br_iov[i].iov_len; break; default: off = OFF_MAX; } be->be_block = off; TAILQ_FOREACH(tbe, &bc->bc_pendq, be_link) { if (tbe->be_block == breq->br_offset) break; } if (tbe == NULL) { TAILQ_FOREACH(tbe, &bc->bc_busyq, be_link) { if (tbe->be_block == breq->br_offset) break; } } if (tbe == NULL) be->be_status = BST_PEND; else be->be_status = BST_BLOCK; TAILQ_INSERT_TAIL(&bc->bc_pendq, be, be_link); return (be->be_status == BST_PEND); } static int blockif_dequeue(struct blockif_ctxt *bc, pthread_t t, struct blockif_elem **bep) { struct blockif_elem *be; TAILQ_FOREACH(be, &bc->bc_pendq, be_link) { if (be->be_status == BST_PEND) break; assert(be->be_status == BST_BLOCK); } if (be == NULL) return (0); TAILQ_REMOVE(&bc->bc_pendq, be, be_link); be->be_status = BST_BUSY; be->be_tid = t; TAILQ_INSERT_TAIL(&bc->bc_busyq, be, be_link); *bep = be; return (1); } static void blockif_complete(struct blockif_ctxt *bc, struct blockif_elem *be) { struct blockif_elem *tbe; if (be->be_status == BST_DONE || be->be_status == BST_BUSY) TAILQ_REMOVE(&bc->bc_busyq, be, be_link); else TAILQ_REMOVE(&bc->bc_pendq, be, be_link); TAILQ_FOREACH(tbe, &bc->bc_pendq, be_link) { if (tbe->be_req->br_offset == be->be_block) tbe->be_status = BST_PEND; } be->be_tid = 0; be->be_status = BST_FREE; be->be_req = NULL; TAILQ_INSERT_TAIL(&bc->bc_freeq, be, be_link); } static int blockif_flush_bc(struct blockif_ctxt *bc) { #ifdef __FreeBSD__ if (bc->bc_ischr) { if (ioctl(bc->bc_fd, DIOCGFLUSH)) return (errno); } else if (fsync(bc->bc_fd)) return (errno); #else /* * This fsync() should be adequate to flush the cache of a file * or device. In VFS, the VOP_SYNC operation is converted to * the appropriate ioctl in both sdev (for real devices) and * zfs (for zvols). */ if (fsync(bc->bc_fd)) return (errno); #endif return (0); } static void blockif_proc(struct blockif_ctxt *bc, struct blockif_elem *be, uint8_t *buf) { #ifdef __FreeBSD__ struct spacectl_range range; #endif struct blockif_req *br; #ifdef __FreeBSD__ off_t arg[2]; #endif ssize_t n; size_t clen, len, off, boff, voff; int i, err; br = be->be_req; assert(br->br_resid >= 0); if (br->br_iovcnt <= 1) buf = NULL; err = 0; switch (be->be_op) { case BOP_READ: if (buf == NULL) { if ((n = preadv(bc->bc_fd, br->br_iov, br->br_iovcnt, br->br_offset)) < 0) err = errno; else br->br_resid -= n; break; } i = 0; off = voff = 0; while (br->br_resid > 0) { len = MIN(br->br_resid, MAXPHYS); n = pread(bc->bc_fd, buf, len, br->br_offset + off); if (n < 0) { err = errno; break; } len = (size_t)n; boff = 0; do { clen = MIN(len - boff, br->br_iov[i].iov_len - voff); memcpy((uint8_t *)br->br_iov[i].iov_base + voff, buf + boff, clen); if (clen < br->br_iov[i].iov_len - voff) voff += clen; else { i++; voff = 0; } boff += clen; } while (boff < len); off += len; br->br_resid -= len; } break; case BOP_WRITE: if (bc->bc_rdonly) { err = EROFS; break; } if (buf == NULL) { if ((n = pwritev(bc->bc_fd, br->br_iov, br->br_iovcnt, br->br_offset)) < 0) err = errno; else br->br_resid -= n; break; } i = 0; off = voff = 0; while (br->br_resid > 0) { len = MIN(br->br_resid, MAXPHYS); boff = 0; do { clen = MIN(len - boff, br->br_iov[i].iov_len - voff); memcpy(buf + boff, (uint8_t *)br->br_iov[i].iov_base + voff, clen); if (clen < br->br_iov[i].iov_len - voff) voff += clen; else { i++; voff = 0; } boff += clen; } while (boff < len); n = pwrite(bc->bc_fd, buf, len, br->br_offset + off); if (n < 0) { err = errno; break; } off += n; br->br_resid -= n; } break; case BOP_FLUSH: err = blockif_flush_bc(bc); break; case BOP_DELETE: if (!bc->bc_candelete) err = EOPNOTSUPP; else if (bc->bc_rdonly) err = EROFS; #ifdef __FreeBSD__ else if (bc->bc_ischr) { arg[0] = br->br_offset; arg[1] = br->br_resid; if (ioctl(bc->bc_fd, DIOCGDELETE, arg)) err = errno; else br->br_resid = 0; } else { range.r_offset = br->br_offset; range.r_len = br->br_resid; while (range.r_len > 0) { if (fspacectl(bc->bc_fd, SPACECTL_DEALLOC, &range, 0, &range) != 0) { err = errno; break; } } if (err == 0) br->br_resid = 0; } #else else if (bc->bc_ischr) { dkioc_free_list_t dfl = { .dfl_num_exts = 1, .dfl_offset = 0, .dfl_flags = 0, .dfl_exts = { { .dfle_start = br->br_offset, .dfle_length = br->br_resid } } }; if (ioctl(bc->bc_fd, DKIOCFREE, &dfl)) err = errno; else br->br_resid = 0; } else { struct flock fl = { .l_whence = 0, .l_type = F_WRLCK, .l_start = br->br_offset, .l_len = br->br_resid }; if (fcntl(bc->bc_fd, F_FREESP, &fl)) err = errno; else br->br_resid = 0; } #endif break; default: err = EINVAL; break; } be->be_status = BST_DONE; (*br->br_callback)(br, err); } static inline bool blockif_empty(const struct blockif_ctxt *bc) { return (TAILQ_EMPTY(&bc->bc_pendq) && TAILQ_EMPTY(&bc->bc_busyq)); } static void * blockif_thr(void *arg) { struct blockif_ctxt *bc; struct blockif_elem *be; pthread_t t; uint8_t *buf; bc = arg; if (bc->bc_isgeom) buf = malloc(MAXPHYS); else buf = NULL; t = pthread_self(); pthread_mutex_lock(&bc->bc_mtx); for (;;) { while (blockif_dequeue(bc, t, &be)) { pthread_mutex_unlock(&bc->bc_mtx); blockif_proc(bc, be, buf); pthread_mutex_lock(&bc->bc_mtx); blockif_complete(bc, be); } /* Check ctxt status here to see if exit requested */ if (bc->bc_closing) break; pthread_cond_wait(&bc->bc_cond, &bc->bc_mtx); } pthread_mutex_unlock(&bc->bc_mtx); if (buf) free(buf); pthread_exit(NULL); return (NULL); } #ifdef __FreeBSD__ static void blockif_sigcont_handler(int signal __unused, enum ev_type type __unused, void *arg __unused) #else static void blockif_sigcont_handler(int signal __unused) #endif { struct blockif_sig_elem *bse; for (;;) { /* * Process the entire list even if not intended for * this thread. */ do { bse = blockif_bse_head; if (bse == NULL) return; } while (!atomic_cmpset_ptr((uintptr_t *)&blockif_bse_head, (uintptr_t)bse, (uintptr_t)bse->bse_next)); pthread_mutex_lock(&bse->bse_mtx); bse->bse_pending = 0; pthread_cond_signal(&bse->bse_cond); pthread_mutex_unlock(&bse->bse_mtx); } } static void blockif_init(void) { #ifdef __FreeBSD__ mevent_add(SIGCONT, EVF_SIGNAL, blockif_sigcont_handler, NULL); (void) signal(SIGCONT, SIG_IGN); #else (void) sigset(SIGCONT, blockif_sigcont_handler); #endif } int blockif_legacy_config(nvlist_t *nvl, const char *opts) { char *cp, *path; if (opts == NULL) return (0); cp = strchr(opts, ','); if (cp == NULL) { set_config_value_node(nvl, "path", opts); return (0); } path = strndup(opts, cp - opts); set_config_value_node(nvl, "path", path); free(path); return (pci_parse_legacy_config(nvl, cp + 1)); } int blockif_add_boot_device(struct pci_devinst *const pi, struct blockif_ctxt *const bc) { if (bc->bc_bootindex < 0) return (0); return (pci_emul_add_boot_device(pi, bc->bc_bootindex)); } struct blockif_ctxt * blockif_open(nvlist_t *nvl, const char *ident) { char tname[MAXCOMLEN + 1]; #ifdef __FreeBSD__ char name[MAXPATHLEN]; #endif const char *path, *pssval, *ssval, *bootindex_val; char *cp; struct blockif_ctxt *bc; struct stat sbuf; #ifdef __FreeBSD__ struct diocgattr_arg arg; #else enum blockif_wce wce = WCE_NONE; #endif off_t size, psectsz, psectoff; int extra, fd, i, sectsz; int ro, candelete, geom, ssopt, pssopt; int nodelete; int bootindex; #ifndef WITHOUT_CAPSICUM cap_rights_t rights; cap_ioctl_t cmds[] = { DIOCGFLUSH, DIOCGDELETE, DIOCGMEDIASIZE }; #endif pthread_once(&blockif_once, blockif_init); fd = -1; extra = 0; ssopt = 0; #ifndef __FreeBSD__ pssopt = 0; #endif ro = 0; nodelete = 0; bootindex = -1; if (get_config_bool_node_default(nvl, "nocache", false)) extra |= O_DIRECT; if (get_config_bool_node_default(nvl, "nodelete", false)) nodelete = 1; if (get_config_bool_node_default(nvl, "sync", false) || get_config_bool_node_default(nvl, "direct", false)) extra |= O_SYNC; if (get_config_bool_node_default(nvl, "ro", false)) ro = 1; ssval = get_config_value_node(nvl, "sectorsize"); if (ssval != NULL) { ssopt = strtol(ssval, &cp, 10); if (cp == ssval) { EPRINTLN("Invalid sector size \"%s\"", ssval); goto err; } if (*cp == '\0') { pssopt = ssopt; } else if (*cp == '/') { pssval = cp + 1; pssopt = strtol(pssval, &cp, 10); if (cp == pssval || *cp != '\0') { EPRINTLN("Invalid sector size \"%s\"", ssval); goto err; } } else { EPRINTLN("Invalid sector size \"%s\"", ssval); goto err; } } bootindex_val = get_config_value_node(nvl, "bootindex"); if (bootindex_val != NULL) { bootindex = atoi(bootindex_val); } path = get_config_value_node(nvl, "path"); if (path == NULL) { EPRINTLN("Missing \"path\" for block device."); goto err; } fd = open(path, (ro ? O_RDONLY : O_RDWR) | extra); if (fd < 0 && !ro) { /* Attempt a r/w fail with a r/o open */ fd = open(path, O_RDONLY | extra); ro = 1; } if (fd < 0) { warn("Could not open backing file: %s", path); goto err; } if (fstat(fd, &sbuf) < 0) { warn("Could not stat backing file %s", path); goto err; } #ifndef WITHOUT_CAPSICUM cap_rights_init(&rights, CAP_FSYNC, CAP_IOCTL, CAP_READ, CAP_SEEK, CAP_WRITE, CAP_FSTAT, CAP_EVENT, CAP_FPATHCONF); if (ro) cap_rights_clear(&rights, CAP_FSYNC, CAP_WRITE); if (caph_rights_limit(fd, &rights) == -1) errx(EX_OSERR, "Unable to apply rights for sandbox"); #endif /* * Deal with raw devices */ size = sbuf.st_size; sectsz = DEV_BSIZE; psectsz = psectoff = 0; candelete = geom = 0; #ifdef __FreeBSD__ if (S_ISCHR(sbuf.st_mode)) { if (ioctl(fd, DIOCGMEDIASIZE, &size) < 0 || ioctl(fd, DIOCGSECTORSIZE, §sz)) { perror("Could not fetch dev blk/sector size"); goto err; } assert(size != 0); assert(sectsz != 0); if (ioctl(fd, DIOCGSTRIPESIZE, &psectsz) == 0 && psectsz > 0) ioctl(fd, DIOCGSTRIPEOFFSET, &psectoff); strlcpy(arg.name, "GEOM::candelete", sizeof(arg.name)); arg.len = sizeof(arg.value.i); if (nodelete == 0 && ioctl(fd, DIOCGATTR, &arg) == 0) candelete = arg.value.i; if (ioctl(fd, DIOCGPROVIDERNAME, name) == 0) geom = 1; } else { psectsz = sbuf.st_blksize; /* Avoid fallback implementation */ candelete = fpathconf(fd, _PC_DEALLOC_PRESENT) == 1; } #else psectsz = sbuf.st_blksize; if (S_ISCHR(sbuf.st_mode)) { struct dk_minfo_ext dkmext; int wce_val; /* Look for a more accurate physical block/media size */ if (ioctl(fd, DKIOCGMEDIAINFOEXT, &dkmext) == 0) { psectsz = dkmext.dki_pbsize; size = dkmext.dki_lbsize * dkmext.dki_capacity; } /* See if a configurable write cache is present and working */ if (ioctl(fd, DKIOCGETWCE, &wce_val) == 0) { /* * If WCE is already active, disable it until the * specific device driver calls for its return. If it * is not active, toggle it on and off to verify that * such actions are possible. */ if (wce_val != 0) { wce_val = 0; /* * Inability to disable the cache is a threat * to data durability. */ assert(ioctl(fd, DKIOCSETWCE, &wce_val) == 0); wce = WCE_IOCTL; } else { int r1, r2; wce_val = 1; r1 = ioctl(fd, DKIOCSETWCE, &wce_val); wce_val = 0; r2 = ioctl(fd, DKIOCSETWCE, &wce_val); if (r1 == 0 && r2 == 0) { wce = WCE_IOCTL; } else { /* * If the cache cache toggle was not * successful, ensure that the cache * was not left enabled. */ assert(r1 != 0); } } } if (nodelete == 0 && ioctl(fd, DKIOC_CANFREE, &candelete)) candelete = 0; } else { int flags; if ((flags = fcntl(fd, F_GETFL)) >= 0) { flags |= O_DSYNC; if (fcntl(fd, F_SETFL, flags) != -1) { wce = WCE_FCNTL; } } /* * We don't have a way to discover if a file supports the * FREESP fcntl cmd (other than trying it). However, * zfs, ufs, tmpfs, and udfs all support the FREESP fcntl cmd. * Nfsv4 and nfsv4 also forward the FREESP request * to the server, so we always enable it for file based * volumes. Anyone trying to run volumes on an unsupported * configuration is on their own, and should be prepared * for the requests to fail. */ if (nodelete == 0) candelete = 1; } #endif #ifndef WITHOUT_CAPSICUM if (caph_ioctls_limit(fd, cmds, nitems(cmds)) == -1) errx(EX_OSERR, "Unable to apply rights for sandbox"); #endif if (ssopt != 0) { if (!powerof2(ssopt) || !powerof2(pssopt) || ssopt < 512 || ssopt > pssopt) { EPRINTLN("Invalid sector size %d/%d", ssopt, pssopt); goto err; } /* * Some backend drivers (e.g. cd0, ada0) require that the I/O * size be a multiple of the device's sector size. * * Validate that the emulated sector size complies with this * requirement. */ if (S_ISCHR(sbuf.st_mode)) { if (ssopt < sectsz || (ssopt % sectsz) != 0) { EPRINTLN("Sector size %d incompatible " "with underlying device sector size %d", ssopt, sectsz); goto err; } } sectsz = ssopt; psectsz = pssopt; psectoff = 0; } bc = calloc(1, sizeof(struct blockif_ctxt)); if (bc == NULL) { perror("calloc"); goto err; } bc->bc_magic = BLOCKIF_SIG; bc->bc_fd = fd; bc->bc_ischr = S_ISCHR(sbuf.st_mode); bc->bc_isgeom = geom; bc->bc_candelete = candelete; #ifndef __FreeBSD__ bc->bc_wce = wce; #endif bc->bc_rdonly = ro; bc->bc_size = size; bc->bc_sectsz = sectsz; bc->bc_psectsz = psectsz; bc->bc_psectoff = psectoff; pthread_mutex_init(&bc->bc_mtx, NULL); pthread_cond_init(&bc->bc_cond, NULL); TAILQ_INIT(&bc->bc_freeq); TAILQ_INIT(&bc->bc_pendq); TAILQ_INIT(&bc->bc_busyq); bc->bc_bootindex = bootindex; for (i = 0; i < BLOCKIF_MAXREQ; i++) { bc->bc_reqs[i].be_status = BST_FREE; TAILQ_INSERT_HEAD(&bc->bc_freeq, &bc->bc_reqs[i], be_link); } for (i = 0; i < BLOCKIF_NUMTHR; i++) { pthread_create(&bc->bc_btid[i], NULL, blockif_thr, bc); snprintf(tname, sizeof(tname), "blk-%s-%d", ident, i); pthread_set_name_np(bc->bc_btid[i], tname); } return (bc); err: if (fd >= 0) close(fd); return (NULL); } static void blockif_resized(int fd, enum ev_type type __unused, void *arg) { struct blockif_ctxt *bc; struct stat sb; off_t mediasize; if (fstat(fd, &sb) != 0) return; #ifdef __FreeBSD__ if (S_ISCHR(sb.st_mode)) { if (ioctl(fd, DIOCGMEDIASIZE, &mediasize) < 0) { EPRINTLN("blockif_resized: get mediasize failed: %s", strerror(errno)); return; } } else mediasize = sb.st_size; #else mediasize = sb.st_size; if (S_ISCHR(sb.st_mode)) { struct dk_minfo dkm; if (ioctl(fd, DKIOCGMEDIAINFO, &dkm) == 0) mediasize = dkm.dki_lbsize * dkm.dki_capacity; } #endif bc = arg; pthread_mutex_lock(&bc->bc_mtx); if (mediasize != bc->bc_size) { bc->bc_size = mediasize; bc->bc_resize_cb(bc, bc->bc_resize_cb_arg, bc->bc_size); } pthread_mutex_unlock(&bc->bc_mtx); } int blockif_register_resize_callback(struct blockif_ctxt *bc, blockif_resize_cb *cb, void *cb_arg) { struct stat sb; int err; if (cb == NULL) return (EINVAL); err = 0; pthread_mutex_lock(&bc->bc_mtx); if (bc->bc_resize_cb != NULL) { err = EBUSY; goto out; } assert(bc->bc_closing == 0); if (fstat(bc->bc_fd, &sb) != 0) { err = errno; goto out; } bc->bc_resize_event = mevent_add_flags(bc->bc_fd, EVF_VNODE, EVFF_ATTRIB, blockif_resized, bc); if (bc->bc_resize_event == NULL) { err = ENXIO; goto out; } bc->bc_resize_cb = cb; bc->bc_resize_cb_arg = cb_arg; out: pthread_mutex_unlock(&bc->bc_mtx); return (err); } static int blockif_request(struct blockif_ctxt *bc, struct blockif_req *breq, enum blockop op) { int err; err = 0; pthread_mutex_lock(&bc->bc_mtx); if (!TAILQ_EMPTY(&bc->bc_freeq)) { /* * Enqueue and inform the block i/o thread * that there is work available */ if (blockif_enqueue(bc, breq, op)) pthread_cond_signal(&bc->bc_cond); } else { /* * Callers are not allowed to enqueue more than * the specified blockif queue limit. Return an * error to indicate that the queue length has been * exceeded. */ err = E2BIG; } pthread_mutex_unlock(&bc->bc_mtx); return (err); } int blockif_read(struct blockif_ctxt *bc, struct blockif_req *breq) { assert(bc->bc_magic == BLOCKIF_SIG); return (blockif_request(bc, breq, BOP_READ)); } int blockif_write(struct blockif_ctxt *bc, struct blockif_req *breq) { assert(bc->bc_magic == BLOCKIF_SIG); return (blockif_request(bc, breq, BOP_WRITE)); } int blockif_flush(struct blockif_ctxt *bc, struct blockif_req *breq) { assert(bc->bc_magic == BLOCKIF_SIG); return (blockif_request(bc, breq, BOP_FLUSH)); } int blockif_delete(struct blockif_ctxt *bc, struct blockif_req *breq) { assert(bc->bc_magic == BLOCKIF_SIG); return (blockif_request(bc, breq, BOP_DELETE)); } int blockif_cancel(struct blockif_ctxt *bc, struct blockif_req *breq) { struct blockif_elem *be; assert(bc->bc_magic == BLOCKIF_SIG); pthread_mutex_lock(&bc->bc_mtx); /* * Check pending requests. */ TAILQ_FOREACH(be, &bc->bc_pendq, be_link) { if (be->be_req == breq) break; } if (be != NULL) { /* * Found it. */ blockif_complete(bc, be); pthread_mutex_unlock(&bc->bc_mtx); return (0); } /* * Check in-flight requests. */ TAILQ_FOREACH(be, &bc->bc_busyq, be_link) { if (be->be_req == breq) break; } if (be == NULL) { /* * Didn't find it. */ pthread_mutex_unlock(&bc->bc_mtx); return (EINVAL); } /* * Interrupt the processing thread to force it return * prematurely via it's normal callback path. */ while (be->be_status == BST_BUSY) { struct blockif_sig_elem bse, *old_head; pthread_mutex_init(&bse.bse_mtx, NULL); pthread_cond_init(&bse.bse_cond, NULL); bse.bse_pending = 1; do { old_head = blockif_bse_head; bse.bse_next = old_head; } while (!atomic_cmpset_ptr((uintptr_t *)&blockif_bse_head, (uintptr_t)old_head, (uintptr_t)&bse)); pthread_kill(be->be_tid, SIGCONT); pthread_mutex_lock(&bse.bse_mtx); while (bse.bse_pending) pthread_cond_wait(&bse.bse_cond, &bse.bse_mtx); pthread_mutex_unlock(&bse.bse_mtx); } pthread_mutex_unlock(&bc->bc_mtx); /* * The processing thread has been interrupted. Since it's not * clear if the callback has been invoked yet, return EBUSY. */ return (EBUSY); } int blockif_close(struct blockif_ctxt *bc) { void *jval; int i; assert(bc->bc_magic == BLOCKIF_SIG); /* * Stop the block i/o thread */ pthread_mutex_lock(&bc->bc_mtx); bc->bc_closing = 1; if (bc->bc_resize_event != NULL) mevent_disable(bc->bc_resize_event); pthread_mutex_unlock(&bc->bc_mtx); pthread_cond_broadcast(&bc->bc_cond); for (i = 0; i < BLOCKIF_NUMTHR; i++) pthread_join(bc->bc_btid[i], &jval); /* XXX Cancel queued i/o's ??? */ /* * Release resources */ bc->bc_magic = 0; close(bc->bc_fd); free(bc); return (0); } /* * Return virtual C/H/S values for a given block. Use the algorithm * outlined in the VHD specification to calculate values. */ void blockif_chs(struct blockif_ctxt *bc, uint16_t *c, uint8_t *h, uint8_t *s) { off_t sectors; /* total sectors of the block dev */ off_t hcyl; /* cylinders times heads */ uint16_t secpt; /* sectors per track */ uint8_t heads; assert(bc->bc_magic == BLOCKIF_SIG); sectors = bc->bc_size / bc->bc_sectsz; /* Clamp the size to the largest possible with CHS */ if (sectors > 65535L * 16 * 255) sectors = 65535L * 16 * 255; if (sectors >= 65536L * 16 * 63) { secpt = 255; heads = 16; hcyl = sectors / secpt; } else { secpt = 17; hcyl = sectors / secpt; heads = (hcyl + 1023) / 1024; if (heads < 4) heads = 4; if (hcyl >= (heads * 1024) || heads > 16) { secpt = 31; heads = 16; hcyl = sectors / secpt; } if (hcyl >= (heads * 1024)) { secpt = 63; heads = 16; hcyl = sectors / secpt; } } *c = hcyl / heads; *h = heads; *s = secpt; } /* * Accessors */ off_t blockif_size(struct blockif_ctxt *bc) { assert(bc->bc_magic == BLOCKIF_SIG); return (bc->bc_size); } int blockif_sectsz(struct blockif_ctxt *bc) { assert(bc->bc_magic == BLOCKIF_SIG); return (bc->bc_sectsz); } void blockif_psectsz(struct blockif_ctxt *bc, int *size, int *off) { assert(bc->bc_magic == BLOCKIF_SIG); *size = bc->bc_psectsz; *off = bc->bc_psectoff; } int blockif_queuesz(struct blockif_ctxt *bc) { assert(bc->bc_magic == BLOCKIF_SIG); return (BLOCKIF_MAXREQ - 1); } int blockif_is_ro(struct blockif_ctxt *bc) { assert(bc->bc_magic == BLOCKIF_SIG); return (bc->bc_rdonly); } int blockif_candelete(struct blockif_ctxt *bc) { assert(bc->bc_magic == BLOCKIF_SIG); return (bc->bc_candelete); } #ifndef __FreeBSD__ int blockif_set_wce(struct blockif_ctxt *bc, int wc_enable) { int res = 0, flags; int clean_val = (wc_enable != 0) ? 1 : 0; (void) pthread_mutex_lock(&bc->bc_mtx); switch (bc->bc_wce) { case WCE_IOCTL: res = ioctl(bc->bc_fd, DKIOCSETWCE, &clean_val); break; case WCE_FCNTL: if ((flags = fcntl(bc->bc_fd, F_GETFL)) >= 0) { if (wc_enable == 0) { flags |= O_DSYNC; } else { flags &= ~O_DSYNC; } if (fcntl(bc->bc_fd, F_SETFL, flags) == -1) { res = -1; } } else { res = -1; } break; default: break; } /* * After a successful disable of the write cache, ensure that any * lingering data in the cache is synced out. */ if (res == 0 && wc_enable == 0) { res = fsync(bc->bc_fd); } (void) pthread_mutex_unlock(&bc->bc_mtx); return (res); } #endif /* __FreeBSD__ */ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2013 Peter Grehan * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * The block API to be used by bhyve block-device emulations. The routines * are thread safe, with no assumptions about the context of the completion * callback - it may occur in the caller's context, or asynchronously in * another thread. */ #ifndef _BLOCK_IF_H_ #define _BLOCK_IF_H_ #include #include #include /* * BLOCKIF_IOV_MAX is the maximum number of scatter/gather entries in * a single request. BLOCKIF_RING_MAX is the maxmimum number of * pending requests that can be queued. */ #define BLOCKIF_IOV_MAX 128 /* not practical to be IOV_MAX */ #define BLOCKIF_RING_MAX 128 struct blockif_req { int br_iovcnt; off_t br_offset; ssize_t br_resid; void (*br_callback)(struct blockif_req *req, int err); void *br_param; struct iovec br_iov[BLOCKIF_IOV_MAX]; }; struct pci_devinst; struct blockif_ctxt; typedef void blockif_resize_cb(struct blockif_ctxt *, void *, size_t); int blockif_legacy_config(nvlist_t *nvl, const char *opts); int blockif_add_boot_device(struct pci_devinst *const pi, struct blockif_ctxt *const bc); struct blockif_ctxt *blockif_open(nvlist_t *nvl, const char *ident); int blockif_register_resize_callback(struct blockif_ctxt *bc, blockif_resize_cb *cb, void *cb_arg); off_t blockif_size(struct blockif_ctxt *bc); void blockif_chs(struct blockif_ctxt *bc, uint16_t *c, uint8_t *h, uint8_t *s); int blockif_sectsz(struct blockif_ctxt *bc); void blockif_psectsz(struct blockif_ctxt *bc, int *size, int *off); int blockif_queuesz(struct blockif_ctxt *bc); int blockif_is_ro(struct blockif_ctxt *bc); int blockif_candelete(struct blockif_ctxt *bc); #ifndef __FreeBSD__ int blockif_set_wce(struct blockif_ctxt *bc, int enable); #endif int blockif_read(struct blockif_ctxt *bc, struct blockif_req *breq); int blockif_write(struct blockif_ctxt *bc, struct blockif_req *breq); int blockif_flush(struct blockif_ctxt *bc, struct blockif_req *breq); int blockif_delete(struct blockif_ctxt *bc, struct blockif_req *breq); int blockif_cancel(struct blockif_ctxt *bc, struct blockif_req *breq); int blockif_close(struct blockif_ctxt *bc); #endif /* _BLOCK_IF_H_ */ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2015 Neel Natu * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "bhyverun.h" #include "bootrom.h" #include "debug.h" #include "mem.h" #define BOOTROM_SIZE (16 * 1024 * 1024) /* 16 MB */ /* * ROM region is 16 MB at the top of 4GB ("low") memory. * * The size is limited so it doesn't encroach into reserved MMIO space (e.g., * APIC, HPET, MSI). * * It is allocated in page-multiple blocks on a first-come first-serve basis, * from high to low, during initialization, and does not change at runtime. */ static char *romptr; /* Pointer to userspace-mapped bootrom region. */ static vm_paddr_t gpa_base; /* GPA of low end of region. */ static vm_paddr_t gpa_allocbot; /* Low GPA of free region. */ static vm_paddr_t gpa_alloctop; /* High GPA, minus 1, of free region. */ #define CFI_BCS_WRITE_BYTE 0x10 #define CFI_BCS_CLEAR_STATUS 0x50 #define CFI_BCS_READ_STATUS 0x70 #define CFI_BCS_READ_ARRAY 0xff static struct bootrom_var_state { uint8_t *mmap; uint64_t gpa; off_t size; uint8_t cmd; } var = { NULL, 0, 0, CFI_BCS_READ_ARRAY }; /* * Emulate just those CFI basic commands that will convince EDK II * that the Firmware Volume area is writable and persistent. */ static int bootrom_var_mem_handler(struct vcpu *vcpu __unused, int dir, uint64_t addr, int size, uint64_t *val, void *arg1 __unused, long arg2 __unused) { off_t offset; offset = addr - var.gpa; if (offset + size > var.size || offset < 0 || offset + size <= offset) return (EINVAL); if (dir == MEM_F_WRITE) { switch (var.cmd) { case CFI_BCS_WRITE_BYTE: memcpy(var.mmap + offset, val, size); var.cmd = CFI_BCS_READ_ARRAY; break; default: var.cmd = *(uint8_t *)val; } } else { switch (var.cmd) { case CFI_BCS_CLEAR_STATUS: case CFI_BCS_READ_STATUS: memset(val, 0, size); var.cmd = CFI_BCS_READ_ARRAY; break; default: memcpy(val, var.mmap + offset, size); break; } } return (0); } void init_bootrom(struct vmctx *ctx) { vm_paddr_t highmem; romptr = vm_create_devmem(ctx, VM_BOOTROM, "bootrom", BOOTROM_SIZE); if (romptr == MAP_FAILED) err(4, "%s: vm_create_devmem", __func__); highmem = vm_get_highmem_base(ctx); gpa_base = highmem - BOOTROM_SIZE; gpa_allocbot = gpa_base; gpa_alloctop = highmem - 1; } int bootrom_alloc(struct vmctx *ctx, size_t len, int prot, int flags, char **region_out, uint64_t *gpa_out) { static const int bootrom_valid_flags = BOOTROM_ALLOC_TOP; vm_paddr_t gpa; vm_ooffset_t segoff; if (flags & ~bootrom_valid_flags) { warnx("%s: Invalid flags: %x", __func__, flags & ~bootrom_valid_flags); return (EINVAL); } if (prot & ~_PROT_ALL) { warnx("%s: Invalid protection: %x", __func__, prot & ~_PROT_ALL); return (EINVAL); } if (len == 0 || len > BOOTROM_SIZE) { warnx("ROM size %zu is invalid", len); return (EINVAL); } if (len & PAGE_MASK) { warnx("ROM size %zu is not a multiple of the page size", len); return (EINVAL); } if (flags & BOOTROM_ALLOC_TOP) { gpa = (gpa_alloctop - len) + 1; if (gpa < gpa_allocbot) { warnx("No room for %zu ROM in bootrom region", len); return (ENOMEM); } } else { gpa = gpa_allocbot; if (gpa > (gpa_alloctop - len) + 1) { warnx("No room for %zu ROM in bootrom region", len); return (ENOMEM); } } segoff = gpa - gpa_base; if (vm_mmap_memseg(ctx, gpa, VM_BOOTROM, segoff, len, prot) != 0) { int serrno = errno; warn("%s: vm_mmap_mapseg", __func__); return (serrno); } if (flags & BOOTROM_ALLOC_TOP) gpa_alloctop = gpa - 1; else gpa_allocbot = gpa + len; *region_out = romptr + segoff; if (gpa_out != NULL) *gpa_out = gpa; return (0); } int bootrom_loadrom(struct vmctx *ctx) { struct stat sbuf; ssize_t rlen; off_t rom_size, var_size, total_size; char *ptr, *romfile; int fd, varfd, i, rv; const char *bootrom, *varfile; rv = -1; varfd = -1; bootrom = get_config_value("bootrom"); if (bootrom == NULL) { return (0); } /* * get_config_value_node may use a thread local buffer to return * variables. So, when we query the second variable, the first variable * might get overwritten. For that reason, the bootrom should be * duplicated. */ romfile = strdup(bootrom); if (romfile == NULL) { return (-1); } fd = open(romfile, O_RDONLY); if (fd < 0) { EPRINTLN("Error opening bootrom \"%s\": %s", romfile, strerror(errno)); goto done; } if (fstat(fd, &sbuf) < 0) { EPRINTLN("Could not fstat bootrom file \"%s\": %s", romfile, strerror(errno)); goto done; } rom_size = sbuf.st_size; varfile = get_config_value("bootvars"); var_size = 0; if (varfile != NULL) { varfd = open(varfile, O_RDWR); if (varfd < 0) { EPRINTLN("Error opening bootrom variable file " "\"%s\": %s", varfile, strerror(errno)); goto done; } if (fstat(varfd, &sbuf) < 0) { EPRINTLN( "Could not fstat bootrom variable file \"%s\": %s", varfile, strerror(errno)); goto done; } var_size = sbuf.st_size; } if (var_size > BOOTROM_SIZE || (var_size != 0 && var_size < PAGE_SIZE)) { EPRINTLN("Invalid bootrom variable size %ld", var_size); goto done; } total_size = rom_size + var_size; if (total_size > BOOTROM_SIZE) { EPRINTLN("Invalid bootrom and variable aggregate size %ld", total_size); goto done; } /* Map the bootrom into the guest address space */ if (bootrom_alloc(ctx, rom_size, PROT_READ | PROT_EXEC, BOOTROM_ALLOC_TOP, &ptr, NULL) != 0) { goto done; } /* Read 'romfile' into the guest address space */ for (i = 0; i < rom_size / PAGE_SIZE; i++) { rlen = read(fd, ptr + i * PAGE_SIZE, PAGE_SIZE); if (rlen != PAGE_SIZE) { EPRINTLN("Incomplete read of page %d of bootrom " "file %s: %ld bytes", i, romfile, rlen); goto done; } } if (varfd >= 0) { #ifdef __FreeBSD__ var.mmap = mmap(NULL, var_size, PROT_READ | PROT_WRITE, MAP_SHARED, varfd, 0); #else var.mmap = (uint8_t *)mmap(NULL, var_size, PROT_READ | PROT_WRITE, MAP_SHARED, varfd, 0); #endif if (var.mmap == MAP_FAILED) goto done; var.size = var_size; var.gpa = (gpa_alloctop - var_size) + 1; gpa_alloctop = var.gpa - 1; rv = register_mem(&(struct mem_range){ .name = "bootrom variable", .flags = MEM_F_RW, .handler = bootrom_var_mem_handler, .base = var.gpa, .size = var.size, }); if (rv != 0) goto done; } rv = 0; done: if (varfd >= 0) close(varfd); if (fd >= 0) close(fd); free(romfile); return (rv); } /* * Are we relying on a bootrom to initialize the guest's CPU context? */ bool bootrom_boot(void) { return (get_config_value("bootrom") != NULL); } /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2015 Neel Natu * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef _BOOTROM_H_ #define _BOOTROM_H_ #include #include #include #include #include "config.h" struct vmctx; void init_bootrom(struct vmctx *ctx); enum { BOOTROM_ALLOC_TOP = 0x80, _FORCE_INT = INT_MIN, }; int bootrom_alloc(struct vmctx *ctx, size_t len, int prot, int flags, char **region_out, uint64_t *gpa_out); bool bootrom_boot(void); int bootrom_loadrom(struct vmctx *ctx); #endif /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2021 John H. Baldwin * Copyright 2026 Hans Rosenfeld * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include #include #include #include #include #ifndef __FreeBSD__ #include #endif #include "config.h" static nvlist_t *config_root; void init_config(void) { config_root = nvlist_create(0); if (config_root == NULL) err(4, "Failed to create configuration root nvlist"); } static nvlist_t * _lookup_config_node(nvlist_t *parent, const char *path, bool create) { char *copy, *name, *tofree; nvlist_t *nvl, *new_nvl; copy = strdup(path); if (copy == NULL) errx(4, "Failed to allocate memory"); tofree = copy; nvl = parent; while ((name = strsep(©, ".")) != NULL) { if (*name == '\0') { warnx("Invalid configuration node: %s", path); nvl = NULL; break; } if (nvlist_exists_nvlist(nvl, name)) /* * XXX-MJ it is incorrect to cast away the const * qualifier like this since the contract with nvlist * says that values are immutable, and some consumers * will indeed add nodes to the returned nvlist. In * practice, however, it appears to be harmless with the * current nvlist implementation, so we just live with * it until the implementation is reworked. */ nvl = __DECONST(nvlist_t *, nvlist_get_nvlist(nvl, name)); else if (nvlist_exists(nvl, name)) { for (copy = tofree; copy < name; copy++) if (*copy == '\0') *copy = '.'; warnx( "Configuration node %s is a child of existing variable %s", path, tofree); nvl = NULL; break; } else if (create) { /* * XXX-MJ as with the case above, "new_nvl" shouldn't be * mutated after its ownership is given to "nvl". */ new_nvl = nvlist_create(0); if (new_nvl == NULL) errx(4, "Failed to allocate memory"); #ifdef __FreeBSD__ nvlist_move_nvlist(nvl, name, new_nvl); #else if (nvlist_add_nvlist(nvl, name, new_nvl) != 0) errx(4, "Failed to allocate memory"); (void) nvlist_free(new_nvl); if (nvlist_lookup_nvlist(nvl, name, &new_nvl) != 0) errx(4, "Failed to retrieve new nvlist"); #endif nvl = new_nvl; } else { nvl = NULL; break; } } free(tofree); return (nvl); } nvlist_t * create_config_node(const char *path) { return (_lookup_config_node(config_root, path, true)); } nvlist_t * find_config_node(const char *path) { return (_lookup_config_node(config_root, path, false)); } nvlist_t * create_relative_config_node(nvlist_t *parent, const char *path) { return (_lookup_config_node(parent, path, true)); } nvlist_t * find_relative_config_node(nvlist_t *parent, const char *path) { return (_lookup_config_node(parent, path, false)); } void set_config_value_node(nvlist_t *parent, const char *name, const char *value) { if (strchr(name, '.') != NULL) errx(4, "Invalid config node name %s", name); if (parent == NULL) parent = config_root; if (nvlist_exists_string(parent, name)) nvlist_free_string(parent, name); else if (nvlist_exists(parent, name)) errx(4, "Attempting to add value %s to existing node %s of list %p", value, name, parent); nvlist_add_string(parent, name, value); } void set_config_value_node_if_unset(nvlist_t *const parent, const char *const name, const char *const value) { if (get_config_value_node(parent, name) != NULL) { return; } set_config_value_node(parent, name, value); } void set_config_value(const char *path, const char *value) { const char *name; char *node_name; nvlist_t *nvl; /* Look for last separator. */ name = strrchr(path, '.'); if (name == NULL) { nvl = config_root; name = path; } else { node_name = strndup(path, name - path); if (node_name == NULL) errx(4, "Failed to allocate memory"); nvl = create_config_node(node_name); if (nvl == NULL) errx(4, "Failed to create configuration node %s", node_name); free(node_name); /* Skip over '.'. */ name++; } if (nvlist_exists_nvlist(nvl, name)) errx(4, "Attempting to add value %s to existing node %s", value, path); set_config_value_node(nvl, name, value); } void set_config_value_if_unset(const char *const path, const char *const value) { if (get_config_value(path) != NULL) { return; } set_config_value(path, value); } static const char * get_raw_config_value(const char *path) { const char *name; char *node_name; nvlist_t *nvl; /* Look for last separator. */ name = strrchr(path, '.'); if (name == NULL) { nvl = config_root; name = path; } else { node_name = strndup(path, name - path); if (node_name == NULL) errx(4, "Failed to allocate memory"); nvl = find_config_node(node_name); free(node_name); if (nvl == NULL) return (NULL); /* Skip over '.'. */ name++; } if (nvlist_exists_string(nvl, name)) return (nvlist_get_string(nvl, name)); if (nvlist_exists_nvlist(nvl, name)) warnx("Attempting to fetch value of node %s", path); return (NULL); } static char * _expand_config_value(const char *value, int depth) { FILE *valfp; const char *cp, *vp; char *nestedval, *path, *valbuf; size_t valsize; valfp = open_memstream(&valbuf, &valsize); if (valfp == NULL) errx(4, "Failed to allocate memory"); vp = value; while (*vp != '\0') { switch (*vp) { case '%': if (depth > 15) { warnx( "Too many recursive references in configuration value"); fputc('%', valfp); vp++; break; } if (vp[1] != '(' || vp[2] == '\0') cp = NULL; else cp = strchr(vp + 2, ')'); if (cp == NULL) { warnx( "Invalid reference in configuration value \"%s\"", value); fputc('%', valfp); vp++; break; } vp += 2; if (cp == vp) { warnx( "Empty reference in configuration value \"%s\"", value); vp++; break; } /* Allocate a C string holding the path. */ path = strndup(vp, cp - vp); if (path == NULL) errx(4, "Failed to allocate memory"); /* Advance 'vp' past the reference. */ vp = cp + 1; /* Fetch the referenced value. */ cp = get_raw_config_value(path); if (cp == NULL) warnx( "Failed to fetch referenced configuration variable %s", path); else { nestedval = _expand_config_value(cp, depth + 1); fputs(nestedval, valfp); free(nestedval); } free(path); break; case '\\': vp++; if (*vp == '\0') { warnx( "Trailing \\ in configuration value \"%s\"", value); break; } /* FALLTHROUGH */ default: fputc(*vp, valfp); vp++; break; } } fclose(valfp); return (valbuf); } static const char * expand_config_value(const char *value) { static char *valbuf; if (strchr(value, '%') == NULL) return (value); free(valbuf); valbuf = _expand_config_value(value, 0); return (valbuf); } const char * get_config_value(const char *path) { const char *value; value = get_raw_config_value(path); if (value == NULL) return (NULL); return (expand_config_value(value)); } const char * get_config_value_node(const nvlist_t *parent, const char *name) { if (strchr(name, '.') != NULL) errx(4, "Invalid config node name %s", name); if (parent == NULL) parent = config_root; if (nvlist_exists_nvlist(parent, name)) warnx("Attempt to fetch value of node %s of list %p", name, parent); if (!nvlist_exists_string(parent, name)) return (NULL); return (expand_config_value(nvlist_get_string(parent, name))); } static bool _bool_value(const char *name, const char *value) { if (strcasecmp(value, "true") == 0 || strcasecmp(value, "on") == 0 || strcasecmp(value, "yes") == 0 || strcmp(value, "1") == 0) return (true); if (strcasecmp(value, "false") == 0 || strcasecmp(value, "off") == 0 || strcasecmp(value, "no") == 0 || strcmp(value, "0") == 0) return (false); err(4, "Invalid value %s for boolean variable %s", value, name); } bool get_config_bool(const char *path) { const char *value; value = get_config_value(path); if (value == NULL) err(4, "Failed to fetch boolean variable %s", path); return (_bool_value(path, value)); } bool get_config_bool_default(const char *path, bool def) { const char *value; value = get_config_value(path); if (value == NULL) return (def); return (_bool_value(path, value)); } bool get_config_bool_node(const nvlist_t *parent, const char *name) { const char *value; value = get_config_value_node(parent, name); if (value == NULL) err(4, "Failed to fetch boolean variable %s", name); return (_bool_value(name, value)); } bool get_config_bool_node_default(const nvlist_t *parent, const char *name, bool def) { const char *value; value = get_config_value_node(parent, name); if (value == NULL) return (def); return (_bool_value(name, value)); } void set_config_bool(const char *path, bool value) { set_config_value(path, value ? "true" : "false"); } void set_config_bool_node(nvlist_t *parent, const char *name, bool value) { set_config_value_node(parent, name, value ? "true" : "false"); } int walk_config_nodes(const char *prefix, const nvlist_t *parent, void *arg, int (*cb)(const char *, const nvlist_t *, const char *, int, void *)) { void *cookie = NULL; const char *name; int type; while ((name = nvlist_next(parent, &type, &cookie)) != NULL) { int ret; ret = cb(prefix, parent, name, type, arg); if (ret != 0) return (ret); } return (0); } static int dump_node_cb(const char *prefix, const nvlist_t *parent, const char *name, int type, void *arg) { if (type == NV_TYPE_NVLIST) { char *new_prefix; int ret; asprintf(&new_prefix, "%s%s.", prefix, name); ret = walk_config_nodes(new_prefix, nvlist_get_nvlist(parent, name), arg, dump_node_cb); free(new_prefix); return (ret); } assert(type == NV_TYPE_STRING); printf("%s%s=%s\n", prefix, name, nvlist_get_string(parent, name)); return (0); } void dump_config(void) { (void) walk_config_nodes("", config_root, NULL, dump_node_cb); } /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2021 John H. Baldwin * Copyright 2026 Hans Rosenfeld * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef __CONFIG_H__ #define __CONFIG_H__ #include /*- * Manages a configuration database backed by an nv(9) list. * * The database only stores string values. Callers should parse * values into other types if needed. String values can reference * other configuration variables using a '%(name)' syntax. In this * case, the name must be the full path of the configuration * variable. The % character can be escaped with a preceding \ to * avoid expansion. Any \ characters must be escaped. * * Configuration variables are stored in a tree. The full path of a * variable is specified as a dot-separated name similar to sysctl(8) * OIDs. */ /* * Walk the nodes under a parent nvlist. For each node found, call the given * callback function passing the current prefix, nvlist, node name and type, * and the given argument. */ int walk_config_nodes(const char *, const nvlist_t *, void *, int (*cb)(const char *, const nvlist_t *, const char *, int, void *)); /* * Fetches the value of a configuration variable. If the "raw" value * contains references to other configuration variables, this function * expands those references and returns a pointer to the parsed * string. The string's storage is only stable until the next call to * this function. * * If no node is found, returns NULL. * * If 'parent' is NULL, 'name' is assumed to be a top-level variable. */ const char *get_config_value_node(const nvlist_t *parent, const char *name); /* * Similar to get_config_value_node but expects a full path to the * leaf node. */ const char *get_config_value(const char *path); /* Initializes the tree to an empty state. */ void init_config(void); /* * Creates an existing configuration node via a dot-separated OID * path. Will fail if the path names an existing leaf configuration * variable. If the node already exists, this returns a pointer to * the existing node. */ nvlist_t *create_config_node(const char *path); /* * Looks for an existing configuration node via a dot-separated OID * path. Will fail if the path names an existing leaf configuration * variable. */ nvlist_t *find_config_node(const char *path); /* * Similar to the above, but treats the path relative to an existing * 'parent' node rather than as an absolute path. */ nvlist_t *create_relative_config_node(nvlist_t *parent, const char *path); nvlist_t *find_relative_config_node(nvlist_t *parent, const char *path); /* * Adds or replaces the value of the specified variable. * * If 'parent' is NULL, 'name' is assumed to be a top-level variable. */ void set_config_value_node(nvlist_t *parent, const char *name, const char *value); /* * Similar to set_config_value_node but only sets value if it's unset yet. */ void set_config_value_node_if_unset(nvlist_t *const parent, const char *const name, const char *const value); /* * Similar to set_config_value_node but expects a full path to the * leaf node. */ void set_config_value(const char *path, const char *value); /* * Similar to set_config_value but only sets the value if it's unset yet. */ void set_config_value_if_unset(const char *const path, const char *const value); /* Convenience wrappers for boolean variables. */ bool get_config_bool(const char *path); bool get_config_bool_node(const nvlist_t *parent, const char *name); bool get_config_bool_default(const char *path, bool def); bool get_config_bool_node_default(const nvlist_t *parent, const char *name, bool def); void set_config_bool(const char *path, bool value); void set_config_bool_node(nvlist_t *parent, const char *name, bool value); void dump_config(void); #endif /* !__CONFIG_H__ */ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2015 Tycho Nightingale * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include #include "bhyvegc.h" #include "console.h" static struct { struct bhyvegc *gc; fb_render_func_t fb_render_cb; void *fb_arg; kbd_event_func_t kbd_event_cb; void *kbd_arg; int kbd_priority; ptr_event_func_t ptr_event_cb; void *ptr_arg; int ptr_priority; } console; void console_init(int w, int h, void *fbaddr) { console.gc = bhyvegc_init(w, h, fbaddr); } void console_set_fbaddr(void *fbaddr) { bhyvegc_set_fbaddr(console.gc, fbaddr); } struct bhyvegc_image * console_get_image(void) { struct bhyvegc_image *bhyvegc_image; bhyvegc_image = bhyvegc_get_image(console.gc); return (bhyvegc_image); } void console_fb_register(fb_render_func_t render_cb, void *arg) { console.fb_render_cb = render_cb; console.fb_arg = arg; } void console_refresh(void) { if (console.fb_render_cb) (*console.fb_render_cb)(console.gc, console.fb_arg); } void console_kbd_register(kbd_event_func_t event_cb, void *arg, int pri) { if (pri > console.kbd_priority) { console.kbd_event_cb = event_cb; console.kbd_arg = arg; console.kbd_priority = pri; } } void console_ptr_register(ptr_event_func_t event_cb, void *arg, int pri) { if (pri > console.ptr_priority) { console.ptr_event_cb = event_cb; console.ptr_arg = arg; console.ptr_priority = pri; } } void console_key_event(int down, uint32_t keysym, uint32_t keycode) { if (console.kbd_event_cb) (*console.kbd_event_cb)(down, keysym, keycode, console.kbd_arg); } void console_ptr_event(uint8_t button, int x, int y) { if (console.ptr_event_cb) (*console.ptr_event_cb)(button, x, y, console.ptr_arg); } /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2015 Tycho Nightingale * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef _CONSOLE_H_ #define _CONSOLE_H_ struct bhyvegc; typedef void (*fb_render_func_t)(struct bhyvegc *gc, void *arg); typedef void (*kbd_event_func_t)(int down, uint32_t keysym, uint32_t keycode, void *arg); typedef void (*ptr_event_func_t)(uint8_t mask, int x, int y, void *arg); void console_init(int w, int h, void *fbaddr); void console_set_fbaddr(void *fbaddr); struct bhyvegc_image *console_get_image(void); void console_fb_register(fb_render_func_t render_cb, void *arg); void console_refresh(void); void console_kbd_register(kbd_event_func_t event_cb, void *arg, int pri); void console_key_event(int down, uint32_t keysym, uint32_t keycode); void console_ptr_register(ptr_event_func_t event_cb, void *arg, int pri); void console_ptr_event(uint8_t button, int x, int y); #endif /* _CONSOLE_H_ */ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2017, Fedor Uporov * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include #include "crc16.h" /* CRC table for the CRC-16. The poly is 0x8005 (x16 + x15 + x2 + 1). */ uint16_t const crc16_table[256] = { 0x0000, 0xC0C1, 0xC181, 0x0140, 0xC301, 0x03C0, 0x0280, 0xC241, 0xC601, 0x06C0, 0x0780, 0xC741, 0x0500, 0xC5C1, 0xC481, 0x0440, 0xCC01, 0x0CC0, 0x0D80, 0xCD41, 0x0F00, 0xCFC1, 0xCE81, 0x0E40, 0x0A00, 0xCAC1, 0xCB81, 0x0B40, 0xC901, 0x09C0, 0x0880, 0xC841, 0xD801, 0x18C0, 0x1980, 0xD941, 0x1B00, 0xDBC1, 0xDA81, 0x1A40, 0x1E00, 0xDEC1, 0xDF81, 0x1F40, 0xDD01, 0x1DC0, 0x1C80, 0xDC41, 0x1400, 0xD4C1, 0xD581, 0x1540, 0xD701, 0x17C0, 0x1680, 0xD641, 0xD201, 0x12C0, 0x1380, 0xD341, 0x1100, 0xD1C1, 0xD081, 0x1040, 0xF001, 0x30C0, 0x3180, 0xF141, 0x3300, 0xF3C1, 0xF281, 0x3240, 0x3600, 0xF6C1, 0xF781, 0x3740, 0xF501, 0x35C0, 0x3480, 0xF441, 0x3C00, 0xFCC1, 0xFD81, 0x3D40, 0xFF01, 0x3FC0, 0x3E80, 0xFE41, 0xFA01, 0x3AC0, 0x3B80, 0xFB41, 0x3900, 0xF9C1, 0xF881, 0x3840, 0x2800, 0xE8C1, 0xE981, 0x2940, 0xEB01, 0x2BC0, 0x2A80, 0xEA41, 0xEE01, 0x2EC0, 0x2F80, 0xEF41, 0x2D00, 0xEDC1, 0xEC81, 0x2C40, 0xE401, 0x24C0, 0x2580, 0xE541, 0x2700, 0xE7C1, 0xE681, 0x2640, 0x2200, 0xE2C1, 0xE381, 0x2340, 0xE101, 0x21C0, 0x2080, 0xE041, 0xA001, 0x60C0, 0x6180, 0xA141, 0x6300, 0xA3C1, 0xA281, 0x6240, 0x6600, 0xA6C1, 0xA781, 0x6740, 0xA501, 0x65C0, 0x6480, 0xA441, 0x6C00, 0xACC1, 0xAD81, 0x6D40, 0xAF01, 0x6FC0, 0x6E80, 0xAE41, 0xAA01, 0x6AC0, 0x6B80, 0xAB41, 0x6900, 0xA9C1, 0xA881, 0x6840, 0x7800, 0xB8C1, 0xB981, 0x7940, 0xBB01, 0x7BC0, 0x7A80, 0xBA41, 0xBE01, 0x7EC0, 0x7F80, 0xBF41, 0x7D00, 0xBDC1, 0xBC81, 0x7C40, 0xB401, 0x74C0, 0x7580, 0xB541, 0x7700, 0xB7C1, 0xB681, 0x7640, 0x7200, 0xB2C1, 0xB381, 0x7340, 0xB101, 0x71C0, 0x7080, 0xB041, 0x5000, 0x90C1, 0x9181, 0x5140, 0x9301, 0x53C0, 0x5280, 0x9241, 0x9601, 0x56C0, 0x5780, 0x9741, 0x5500, 0x95C1, 0x9481, 0x5440, 0x9C01, 0x5CC0, 0x5D80, 0x9D41, 0x5F00, 0x9FC1, 0x9E81, 0x5E40, 0x5A00, 0x9AC1, 0x9B81, 0x5B40, 0x9901, 0x59C0, 0x5880, 0x9841, 0x8801, 0x48C0, 0x4980, 0x8941, 0x4B00, 0x8BC1, 0x8A81, 0x4A40, 0x4E00, 0x8EC1, 0x8F81, 0x4F40, 0x8D01, 0x4DC0, 0x4C80, 0x8C41, 0x4400, 0x84C1, 0x8581, 0x4540, 0x8701, 0x47C0, 0x4680, 0x8641, 0x8201, 0x42C0, 0x4380, 0x8341, 0x4100, 0x81C1, 0x8081, 0x4040 }; /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2017, Fedor Uporov * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include #ifndef _CRC16_H_ #define _CRC16_H_ extern uint16_t const crc16_table[256]; static inline uint16_t crc16(uint16_t crc, const void *buffer, unsigned int len) { const unsigned char *cp = buffer; while (len--) crc = (((crc >> 8) & 0xffU) ^ crc16_table[(crc ^ *cp++) & 0xffU]) & 0x0000ffffU; return crc; } #endif /* !_CRC16_H_ */ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2019 Vincenzo Maffione * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef _DEBUG_H_ #define _DEBUG_H_ extern int raw_stdio; #define FPRINTLN(filep, fmt, arg...) \ do { \ if (raw_stdio) \ fprintf(filep, fmt "\r\n", ##arg); \ else \ fprintf(filep, fmt "\n", ##arg); \ } while (0) #define PRINTLN(fmt, arg...) FPRINTLN(stdout, fmt, ##arg) #define EPRINTLN(fmt, arg...) FPRINTLN(stderr, fmt, ##arg) #endif /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2017-2018 John H. Baldwin * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include #ifndef WITHOUT_CAPSICUM #include #endif #ifdef __FreeBSD__ #include #else #include #endif #include #include #include #include #include #include #include #include #include #ifndef WITHOUT_CAPSICUM #include #endif #include #include #include #include #include #include #include #include #include #include #include #include #include #include "bhyverun.h" #include "config.h" #include "debug.h" #include "gdb.h" #include "mem.h" #include "mevent.h" /* * GDB_SIGNAL_* numbers are part of the GDB remote protocol. Most stops * use SIGTRAP. */ #define GDB_SIGNAL_TRAP 5 #define GDB_BP_SIZE 1 #define GDB_BP_INSTR (uint8_t []){0xcc} #define GDB_PC_REGNAME VM_REG_GUEST_RIP _Static_assert(sizeof(GDB_BP_INSTR) == GDB_BP_SIZE, "GDB_BP_INSTR has wrong size"); static void gdb_resume_vcpus(void); static void check_command(int fd); static struct mevent *read_event, *write_event; static cpuset_t vcpus_active, vcpus_suspended, vcpus_waiting; static pthread_mutex_t gdb_lock; static pthread_cond_t idle_vcpus; static bool first_stop, report_next_stop, swbreak_enabled; /* * An I/O buffer contains 'capacity' bytes of room at 'data'. For a * read buffer, 'start' is unused and 'len' contains the number of * valid bytes in the buffer. For a write buffer, 'start' is set to * the index of the next byte in 'data' to send, and 'len' contains * the remaining number of valid bytes to send. */ struct io_buffer { uint8_t *data; size_t capacity; size_t start; size_t len; }; struct breakpoint { uint64_t gpa; uint8_t shadow_inst[GDB_BP_SIZE]; TAILQ_ENTRY(breakpoint) link; }; /* * When a vCPU stops to due to an event that should be reported to the * debugger, information about the event is stored in this structure. * The vCPU thread then sets 'stopped_vcpu' if it is not already set * and stops other vCPUs so the event can be reported. The * report_stop() function reports the event for the 'stopped_vcpu' * vCPU. When the debugger resumes execution via continue or step, * the event for 'stopped_vcpu' is cleared. vCPUs will loop in their * event handlers until the associated event is reported or disabled. * * An idle vCPU will have all of the boolean fields set to false. * * When a vCPU is stepped, 'stepping' is set to true when the vCPU is * released to execute the stepped instruction. When the vCPU reports * the stepping trap, 'stepped' is set. * * When a vCPU hits a breakpoint set by the debug server, * 'hit_swbreak' is set to true. */ struct vcpu_state { bool stepping; bool stepped; bool hit_swbreak; }; static struct io_buffer cur_comm, cur_resp; static uint8_t cur_csum; static struct vmctx *ctx; static int cur_fd = -1; static TAILQ_HEAD(, breakpoint) breakpoints; static struct vcpu_state *vcpu_state; static struct vcpu **vcpus; static int cur_vcpu, stopped_vcpu; static bool gdb_active = false; struct gdb_reg { enum vm_reg_name id; int size; }; static const struct gdb_reg gdb_regset[] = { { .id = VM_REG_GUEST_RAX, .size = 8 }, { .id = VM_REG_GUEST_RBX, .size = 8 }, { .id = VM_REG_GUEST_RCX, .size = 8 }, { .id = VM_REG_GUEST_RDX, .size = 8 }, { .id = VM_REG_GUEST_RSI, .size = 8 }, { .id = VM_REG_GUEST_RDI, .size = 8 }, { .id = VM_REG_GUEST_RBP, .size = 8 }, { .id = VM_REG_GUEST_RSP, .size = 8 }, { .id = VM_REG_GUEST_R8, .size = 8 }, { .id = VM_REG_GUEST_R9, .size = 8 }, { .id = VM_REG_GUEST_R10, .size = 8 }, { .id = VM_REG_GUEST_R11, .size = 8 }, { .id = VM_REG_GUEST_R12, .size = 8 }, { .id = VM_REG_GUEST_R13, .size = 8 }, { .id = VM_REG_GUEST_R14, .size = 8 }, { .id = VM_REG_GUEST_R15, .size = 8 }, { .id = VM_REG_GUEST_RIP, .size = 8 }, { .id = VM_REG_GUEST_RFLAGS, .size = 4 }, { .id = VM_REG_GUEST_CS, .size = 4 }, { .id = VM_REG_GUEST_SS, .size = 4 }, { .id = VM_REG_GUEST_DS, .size = 4 }, { .id = VM_REG_GUEST_ES, .size = 4 }, { .id = VM_REG_GUEST_FS, .size = 4 }, { .id = VM_REG_GUEST_GS, .size = 4 }, }; #ifdef GDB_LOG #include #include static void __printflike(1, 2) debug(const char *fmt, ...) { static FILE *logfile; va_list ap; if (logfile == NULL) { logfile = fopen("/tmp/bhyve_gdb.log", "w"); if (logfile == NULL) return; #ifndef WITHOUT_CAPSICUM if (caph_limit_stream(fileno(logfile), CAPH_WRITE) == -1) { fclose(logfile); logfile = NULL; return; } #endif setlinebuf(logfile); } va_start(ap, fmt); vfprintf(logfile, fmt, ap); va_end(ap); } #else #ifndef __FreeBSD__ /* * A totally empty debug() makes the compiler grumpy due to how its used with * some control flow here. */ #define debug(...) do { } while (0) #else #define debug(...) #endif #endif static void remove_all_sw_breakpoints(void); static int guest_paging_info(struct vcpu *vcpu, struct vm_guest_paging *paging) { uint64_t regs[4]; const int regset[4] = { VM_REG_GUEST_CR0, VM_REG_GUEST_CR3, VM_REG_GUEST_CR4, VM_REG_GUEST_EFER }; if (vm_get_register_set(vcpu, nitems(regset), regset, regs) == -1) return (-1); /* * For the debugger, always pretend to be the kernel (CPL 0), * and if long-mode is enabled, always parse addresses as if * in 64-bit mode. */ paging->cr3 = regs[1]; paging->cpl = 0; if (regs[3] & EFER_LMA) paging->cpu_mode = CPU_MODE_64BIT; else if (regs[0] & CR0_PE) paging->cpu_mode = CPU_MODE_PROTECTED; else paging->cpu_mode = CPU_MODE_REAL; if (!(regs[0] & CR0_PG)) paging->paging_mode = PAGING_MODE_FLAT; else if (!(regs[2] & CR4_PAE)) paging->paging_mode = PAGING_MODE_32; else if (regs[3] & EFER_LME) paging->paging_mode = PAGING_MODE_64; else paging->paging_mode = PAGING_MODE_PAE; return (0); } /* * Map a guest virtual address to a physical address (for a given vcpu). * If a guest virtual address is valid, return 1. If the address is * not valid, return 0. If an error occurs obtaining the mapping, * return -1. */ static int guest_vaddr2paddr(struct vcpu *vcpu, uint64_t vaddr, uint64_t *paddr) { struct vm_guest_paging paging; int fault; if (guest_paging_info(vcpu, &paging) == -1) return (-1); /* * Always use PROT_READ. We really care if the VA is * accessible, not if the current vCPU can write. */ if (vm_gla2gpa_nofault(vcpu, &paging, vaddr, PROT_READ, paddr, &fault) == -1) return (-1); if (fault) return (0); return (1); } static uint64_t guest_pc(struct vm_exit *vme) { return (vme->rip); } static void io_buffer_reset(struct io_buffer *io) { io->start = 0; io->len = 0; } /* Available room for adding data. */ static size_t io_buffer_avail(struct io_buffer *io) { return (io->capacity - (io->start + io->len)); } static uint8_t * io_buffer_head(struct io_buffer *io) { return (io->data + io->start); } static uint8_t * io_buffer_tail(struct io_buffer *io) { return (io->data + io->start + io->len); } static void io_buffer_advance(struct io_buffer *io, size_t amount) { assert(amount <= io->len); io->start += amount; io->len -= amount; } static void io_buffer_consume(struct io_buffer *io, size_t amount) { io_buffer_advance(io, amount); if (io->len == 0) { io->start = 0; return; } /* * XXX: Consider making this move optional and compacting on a * future read() before realloc(). */ memmove(io->data, io_buffer_head(io), io->len); io->start = 0; } static void io_buffer_grow(struct io_buffer *io, size_t newsize) { uint8_t *new_data; size_t avail, new_cap; avail = io_buffer_avail(io); if (newsize <= avail) return; new_cap = io->capacity + (newsize - avail); new_data = realloc(io->data, new_cap); if (new_data == NULL) err(1, "Failed to grow GDB I/O buffer"); io->data = new_data; io->capacity = new_cap; } static bool response_pending(void) { if (cur_resp.start == 0 && cur_resp.len == 0) return (false); if (cur_resp.start + cur_resp.len == 1 && cur_resp.data[0] == '+') return (false); return (true); } static void close_connection(void) { /* * XXX: This triggers a warning because mevent does the close * before the EV_DELETE. */ pthread_mutex_lock(&gdb_lock); mevent_delete(write_event); mevent_delete_close(read_event); write_event = NULL; read_event = NULL; io_buffer_reset(&cur_comm); io_buffer_reset(&cur_resp); cur_fd = -1; remove_all_sw_breakpoints(); /* Clear any pending events. */ memset(vcpu_state, 0, guest_ncpus * sizeof(*vcpu_state)); /* Resume any stopped vCPUs. */ gdb_resume_vcpus(); pthread_mutex_unlock(&gdb_lock); } static uint8_t hex_digit(uint8_t nibble) { if (nibble <= 9) return (nibble + '0'); else return (nibble + 'a' - 10); } static uint8_t parse_digit(uint8_t v) { if (v >= '0' && v <= '9') return (v - '0'); if (v >= 'a' && v <= 'f') return (v - 'a' + 10); if (v >= 'A' && v <= 'F') return (v - 'A' + 10); return (0xF); } /* Parses big-endian hexadecimal. */ static uintmax_t parse_integer(const uint8_t *p, size_t len) { uintmax_t v; v = 0; while (len > 0) { v <<= 4; v |= parse_digit(*p); p++; len--; } return (v); } static uint8_t parse_byte(const uint8_t *p) { return (parse_digit(p[0]) << 4 | parse_digit(p[1])); } static void send_pending_data(int fd) { ssize_t nwritten; if (cur_resp.len == 0) { mevent_disable(write_event); return; } nwritten = write(fd, io_buffer_head(&cur_resp), cur_resp.len); if (nwritten == -1) { warn("Write to GDB socket failed"); close_connection(); } else { io_buffer_advance(&cur_resp, nwritten); if (cur_resp.len == 0) mevent_disable(write_event); else mevent_enable(write_event); } } /* Append a single character to the output buffer. */ static void send_char(uint8_t data) { io_buffer_grow(&cur_resp, 1); *io_buffer_tail(&cur_resp) = data; cur_resp.len++; } /* Append an array of bytes to the output buffer. */ static void send_data(const uint8_t *data, size_t len) { io_buffer_grow(&cur_resp, len); memcpy(io_buffer_tail(&cur_resp), data, len); cur_resp.len += len; } static void format_byte(uint8_t v, uint8_t *buf) { buf[0] = hex_digit(v >> 4); buf[1] = hex_digit(v & 0xf); } /* * Append a single byte (formatted as two hex characters) to the * output buffer. */ static void send_byte(uint8_t v) { uint8_t buf[2]; format_byte(v, buf); send_data(buf, sizeof(buf)); } static void start_packet(void) { send_char('$'); cur_csum = 0; } static void finish_packet(void) { send_char('#'); send_byte(cur_csum); debug("-> %.*s\n", (int)cur_resp.len, io_buffer_head(&cur_resp)); } /* * Append a single character (for the packet payload) and update the * checksum. */ static void append_char(uint8_t v) { send_char(v); cur_csum += v; } /* * Append an array of bytes (for the packet payload) and update the * checksum. */ static void append_packet_data(const uint8_t *data, size_t len) { send_data(data, len); while (len > 0) { cur_csum += *data; data++; len--; } } static void append_string(const char *str) { #ifdef __FreeBSD__ append_packet_data(str, strlen(str)); #else append_packet_data((const uint8_t *)str, strlen(str)); #endif } static void append_byte(uint8_t v) { uint8_t buf[2]; format_byte(v, buf); append_packet_data(buf, sizeof(buf)); } static void append_unsigned_native(uintmax_t value, size_t len) { size_t i; for (i = 0; i < len; i++) { append_byte(value); value >>= 8; } } static void append_unsigned_be(uintmax_t value, size_t len) { char buf[len * 2]; size_t i; for (i = 0; i < len; i++) { #ifdef __FreeBSD__ format_byte(value, buf + (len - i - 1) * 2); #else format_byte(value, (uint8_t *)(buf + (len - i - 1) * 2)); #endif value >>= 8; } #ifdef __FreeBSD__ append_packet_data(buf, sizeof(buf)); #else append_packet_data((const uint8_t *)buf, sizeof(buf)); #endif } static void append_integer(unsigned int value) { if (value == 0) append_char('0'); else append_unsigned_be(value, (fls(value) + 7) / 8); } static void append_asciihex(const char *str) { while (*str != '\0') { append_byte(*str); str++; } } static void send_empty_response(void) { start_packet(); finish_packet(); } static void send_error(int error) { start_packet(); append_char('E'); append_byte(error); finish_packet(); } static void send_ok(void) { start_packet(); append_string("OK"); finish_packet(); } static int parse_threadid(const uint8_t *data, size_t len) { if (len == 1 && *data == '0') return (0); if (len == 2 && memcmp(data, "-1", 2) == 0) return (-1); if (len == 0) return (-2); return (parse_integer(data, len)); } /* * Report the current stop event to the debugger. If the stop is due * to an event triggered on a specific vCPU such as a breakpoint or * stepping trap, stopped_vcpu will be set to the vCPU triggering the * stop. If 'set_cur_vcpu' is true, then cur_vcpu will be updated to * the reporting vCPU for vCPU events. */ static void report_stop(bool set_cur_vcpu) { struct vcpu_state *vs; start_packet(); if (stopped_vcpu == -1) { append_char('S'); append_byte(GDB_SIGNAL_TRAP); } else { vs = &vcpu_state[stopped_vcpu]; if (set_cur_vcpu) cur_vcpu = stopped_vcpu; append_char('T'); append_byte(GDB_SIGNAL_TRAP); append_string("thread:"); append_integer(stopped_vcpu + 1); append_char(';'); if (vs->hit_swbreak) { debug("$vCPU %d reporting swbreak\n", stopped_vcpu); if (swbreak_enabled) append_string("swbreak:;"); } else if (vs->stepped) debug("$vCPU %d reporting step\n", stopped_vcpu); else debug("$vCPU %d reporting ???\n", stopped_vcpu); } finish_packet(); report_next_stop = false; } /* * If this stop is due to a vCPU event, clear that event to mark it as * acknowledged. */ static void discard_stop(void) { struct vcpu_state *vs; if (stopped_vcpu != -1) { vs = &vcpu_state[stopped_vcpu]; vs->hit_swbreak = false; vs->stepped = false; stopped_vcpu = -1; } report_next_stop = true; } static void gdb_finish_suspend_vcpus(void) { if (first_stop) { first_stop = false; stopped_vcpu = -1; } else if (report_next_stop) { assert(!response_pending()); report_stop(true); send_pending_data(cur_fd); } } /* * vCPU threads invoke this function whenever the vCPU enters the * debug server to pause or report an event. vCPU threads wait here * as long as the debug server keeps them suspended. */ static void _gdb_cpu_suspend(struct vcpu *vcpu, bool report_stop) { int vcpuid = vcpu_id(vcpu); debug("$vCPU %d suspending\n", vcpuid); CPU_SET(vcpuid, &vcpus_waiting); if (report_stop && CPU_CMP(&vcpus_waiting, &vcpus_suspended) == 0) gdb_finish_suspend_vcpus(); while (CPU_ISSET(vcpuid, &vcpus_suspended)) pthread_cond_wait(&idle_vcpus, &gdb_lock); CPU_CLR(vcpuid, &vcpus_waiting); debug("$vCPU %d resuming\n", vcpuid); } /* * Invoked at the start of a vCPU thread's execution to inform the * debug server about the new thread. */ void gdb_cpu_add(struct vcpu *vcpu) { int vcpuid; if (!gdb_active) return; vcpuid = vcpu_id(vcpu); debug("$vCPU %d starting\n", vcpuid); pthread_mutex_lock(&gdb_lock); assert(vcpuid < guest_ncpus); assert(vcpus[vcpuid] == NULL); vcpus[vcpuid] = vcpu; CPU_SET(vcpuid, &vcpus_active); if (!TAILQ_EMPTY(&breakpoints)) { vm_set_capability(vcpu, VM_CAP_BPT_EXIT, 1); debug("$vCPU %d enabled breakpoint exits\n", vcpuid); } /* * If a vcpu is added while vcpus are stopped, suspend the new * vcpu so that it will pop back out with a debug exit before * executing the first instruction. */ if (!CPU_EMPTY(&vcpus_suspended)) { CPU_SET(vcpuid, &vcpus_suspended); _gdb_cpu_suspend(vcpu, false); } pthread_mutex_unlock(&gdb_lock); } /* * Invoked by vCPU before resuming execution. This enables stepping * if the vCPU is marked as stepping. */ static void gdb_cpu_resume(struct vcpu *vcpu) { struct vcpu_state *vs; int error; vs = &vcpu_state[vcpu_id(vcpu)]; /* * Any pending event should already be reported before * resuming. */ assert(vs->hit_swbreak == false); assert(vs->stepped == false); if (vs->stepping) { error = vm_set_capability(vcpu, VM_CAP_MTRAP_EXIT, 1); assert(error == 0); } } /* * Handler for VM_EXITCODE_DEBUG used to suspend a vCPU when the guest * has been suspended due to an event on different vCPU or in response * to a guest-wide suspend such as Ctrl-C or the stop on attach. */ void gdb_cpu_suspend(struct vcpu *vcpu) { if (!gdb_active) return; pthread_mutex_lock(&gdb_lock); _gdb_cpu_suspend(vcpu, true); gdb_cpu_resume(vcpu); pthread_mutex_unlock(&gdb_lock); } static void gdb_suspend_vcpus(void) { assert(pthread_mutex_isowned_np(&gdb_lock)); debug("suspending all CPUs\n"); vcpus_suspended = vcpus_active; vm_suspend_all_cpus(ctx); if (CPU_CMP(&vcpus_waiting, &vcpus_suspended) == 0) gdb_finish_suspend_vcpus(); } /* * Handler for VM_EXITCODE_MTRAP reported when a vCPU single-steps via * the VT-x-specific MTRAP exit. */ void gdb_cpu_mtrap(struct vcpu *vcpu) { struct vcpu_state *vs; int vcpuid; if (!gdb_active) return; vcpuid = vcpu_id(vcpu); debug("$vCPU %d MTRAP\n", vcpuid); pthread_mutex_lock(&gdb_lock); vs = &vcpu_state[vcpuid]; if (vs->stepping) { vs->stepping = false; vs->stepped = true; vm_set_capability(vcpu, VM_CAP_MTRAP_EXIT, 0); while (vs->stepped) { if (stopped_vcpu == -1) { debug("$vCPU %d reporting step\n", vcpuid); stopped_vcpu = vcpuid; gdb_suspend_vcpus(); } _gdb_cpu_suspend(vcpu, true); } gdb_cpu_resume(vcpu); } pthread_mutex_unlock(&gdb_lock); } static struct breakpoint * find_breakpoint(uint64_t gpa) { struct breakpoint *bp; TAILQ_FOREACH(bp, &breakpoints, link) { if (bp->gpa == gpa) return (bp); } return (NULL); } void gdb_cpu_breakpoint(struct vcpu *vcpu, struct vm_exit *vmexit) { struct breakpoint *bp; struct vcpu_state *vs; uint64_t gpa; int error, vcpuid; if (!gdb_active) { EPRINTLN("vm_loop: unexpected VMEXIT_DEBUG"); exit(4); } vcpuid = vcpu_id(vcpu); pthread_mutex_lock(&gdb_lock); error = guest_vaddr2paddr(vcpu, guest_pc(vmexit), &gpa); assert(error == 1); bp = find_breakpoint(gpa); if (bp != NULL) { vs = &vcpu_state[vcpuid]; assert(vs->stepping == false); assert(vs->stepped == false); assert(vs->hit_swbreak == false); vs->hit_swbreak = true; vm_set_register(vcpu, GDB_PC_REGNAME, guest_pc(vmexit)); for (;;) { if (stopped_vcpu == -1) { debug("$vCPU %d reporting breakpoint at rip %#lx\n", vcpuid, guest_pc(vmexit)); stopped_vcpu = vcpuid; gdb_suspend_vcpus(); } _gdb_cpu_suspend(vcpu, true); if (!vs->hit_swbreak) { /* Breakpoint reported. */ break; } bp = find_breakpoint(gpa); if (bp == NULL) { /* Breakpoint was removed. */ vs->hit_swbreak = false; break; } } gdb_cpu_resume(vcpu); } else { debug("$vCPU %d injecting breakpoint at rip %#lx\n", vcpuid, guest_pc(vmexit)); error = vm_set_register(vcpu, VM_REG_GUEST_ENTRY_INST_LENGTH, vmexit->u.bpt.inst_length); assert(error == 0); error = vm_inject_exception(vcpu, IDT_BP, 0, 0, 0); assert(error == 0); } pthread_mutex_unlock(&gdb_lock); } static bool gdb_step_vcpu(struct vcpu *vcpu) { int error, val, vcpuid; vcpuid = vcpu_id(vcpu); debug("$vCPU %d step\n", vcpuid); error = vm_get_capability(vcpu, VM_CAP_MTRAP_EXIT, &val); if (error < 0) return (false); discard_stop(); vcpu_state[vcpuid].stepping = true; vm_resume_cpu(vcpu); CPU_CLR(vcpuid, &vcpus_suspended); pthread_cond_broadcast(&idle_vcpus); return (true); } static void gdb_resume_vcpus(void) { assert(pthread_mutex_isowned_np(&gdb_lock)); vm_resume_all_cpus(ctx); debug("resuming all CPUs\n"); CPU_ZERO(&vcpus_suspended); pthread_cond_broadcast(&idle_vcpus); } static void gdb_read_regs(void) { uint64_t regvals[nitems(gdb_regset)]; int regnums[nitems(gdb_regset)]; for (size_t i = 0; i < nitems(gdb_regset); i++) regnums[i] = gdb_regset[i].id; if (vm_get_register_set(vcpus[cur_vcpu], nitems(gdb_regset), regnums, regvals) == -1) { send_error(errno); return; } start_packet(); for (size_t i = 0; i < nitems(gdb_regset); i++) append_unsigned_native(regvals[i], gdb_regset[i].size); finish_packet(); } static void gdb_read_one_reg(const uint8_t *data, size_t len) { uint64_t regval; uintmax_t reg; reg = parse_integer(data, len); if (reg >= nitems(gdb_regset)) { send_error(EINVAL); return; } if (vm_get_register(vcpus[cur_vcpu], gdb_regset[reg].id, ®val) == -1) { send_error(errno); return; } start_packet(); append_unsigned_native(regval, gdb_regset[reg].size); finish_packet(); } static void gdb_read_mem(const uint8_t *data, size_t len) { uint64_t gpa, gva, val; uint8_t *cp; size_t resid, todo, bytes; bool started; int error; assert(len >= 1); /* Skip 'm' */ data += 1; len -= 1; /* Parse and consume address. */ cp = memchr(data, ',', len); if (cp == NULL || cp == data) { send_error(EINVAL); return; } gva = parse_integer(data, cp - data); len -= (cp - data) + 1; data += (cp - data) + 1; /* Parse length. */ resid = parse_integer(data, len); started = false; while (resid > 0) { error = guest_vaddr2paddr(vcpus[cur_vcpu], gva, &gpa); if (error == -1) { if (started) finish_packet(); else send_error(errno); return; } if (error == 0) { if (started) finish_packet(); else send_error(EFAULT); return; } /* Read bytes from current page. */ todo = getpagesize() - gpa % getpagesize(); if (todo > resid) todo = resid; cp = paddr_guest2host(ctx, gpa, todo); if (cp != NULL) { /* * If this page is guest RAM, read it a byte * at a time. */ if (!started) { start_packet(); started = true; } while (todo > 0) { append_byte(*cp); cp++; gpa++; gva++; resid--; todo--; } } else { /* * If this page isn't guest RAM, try to handle * it via MMIO. For MMIO requests, use * aligned reads of words when possible. */ while (todo > 0) { if (gpa & 1 || todo == 1) bytes = 1; else if (gpa & 2 || todo == 2) bytes = 2; else bytes = 4; error = read_mem(vcpus[cur_vcpu], gpa, &val, bytes); if (error == 0) { if (!started) { start_packet(); started = true; } gpa += bytes; gva += bytes; resid -= bytes; todo -= bytes; while (bytes > 0) { append_byte(val); val >>= 8; bytes--; } } else { if (started) finish_packet(); else send_error(EFAULT); return; } } } assert(resid == 0 || gpa % getpagesize() == 0); } if (!started) start_packet(); finish_packet(); } static void gdb_write_mem(const uint8_t *data, size_t len) { uint64_t gpa, gva, val; uint8_t *cp; size_t resid, todo, bytes; int error; assert(len >= 1); /* Skip 'M' */ data += 1; len -= 1; /* Parse and consume address. */ cp = memchr(data, ',', len); if (cp == NULL || cp == data) { send_error(EINVAL); return; } gva = parse_integer(data, cp - data); len -= (cp - data) + 1; data += (cp - data) + 1; /* Parse and consume length. */ cp = memchr(data, ':', len); if (cp == NULL || cp == data) { send_error(EINVAL); return; } resid = parse_integer(data, cp - data); len -= (cp - data) + 1; data += (cp - data) + 1; /* Verify the available bytes match the length. */ if (len != resid * 2) { send_error(EINVAL); return; } while (resid > 0) { error = guest_vaddr2paddr(vcpus[cur_vcpu], gva, &gpa); if (error == -1) { send_error(errno); return; } if (error == 0) { send_error(EFAULT); return; } /* Write bytes to current page. */ todo = getpagesize() - gpa % getpagesize(); if (todo > resid) todo = resid; cp = paddr_guest2host(ctx, gpa, todo); if (cp != NULL) { /* * If this page is guest RAM, write it a byte * at a time. */ while (todo > 0) { assert(len >= 2); *cp = parse_byte(data); data += 2; len -= 2; cp++; gpa++; gva++; resid--; todo--; } } else { /* * If this page isn't guest RAM, try to handle * it via MMIO. For MMIO requests, use * aligned writes of words when possible. */ while (todo > 0) { if (gpa & 1 || todo == 1) { bytes = 1; val = parse_byte(data); } else if (gpa & 2 || todo == 2) { bytes = 2; val = be16toh(parse_integer(data, 4)); } else { bytes = 4; val = be32toh(parse_integer(data, 8)); } error = write_mem(vcpus[cur_vcpu], gpa, val, bytes); if (error == 0) { gpa += bytes; gva += bytes; resid -= bytes; todo -= bytes; data += 2 * bytes; len -= 2 * bytes; } else { send_error(EFAULT); return; } } } assert(resid == 0 || gpa % getpagesize() == 0); } assert(len == 0); send_ok(); } static bool set_breakpoint_caps(bool enable) { cpuset_t mask; int vcpu; mask = vcpus_active; while (!CPU_EMPTY(&mask)) { vcpu = CPU_FFS(&mask) - 1; CPU_CLR(vcpu, &mask); if (vm_set_capability(vcpus[vcpu], VM_CAP_BPT_EXIT, enable ? 1 : 0) < 0) return (false); debug("$vCPU %d %sabled breakpoint exits\n", vcpu, enable ? "en" : "dis"); } return (true); } static void remove_all_sw_breakpoints(void) { struct breakpoint *bp, *nbp; uint8_t *cp; if (TAILQ_EMPTY(&breakpoints)) return; TAILQ_FOREACH_SAFE(bp, &breakpoints, link, nbp) { debug("remove breakpoint at %#lx\n", bp->gpa); cp = paddr_guest2host(ctx, bp->gpa, sizeof(bp->shadow_inst)); memcpy(cp, bp->shadow_inst, sizeof(bp->shadow_inst)); TAILQ_REMOVE(&breakpoints, bp, link); free(bp); } TAILQ_INIT(&breakpoints); set_breakpoint_caps(false); } static void update_sw_breakpoint(uint64_t gva, int kind, bool insert) { struct breakpoint *bp; uint64_t gpa; uint8_t *cp; int error; if (kind != GDB_BP_SIZE) { send_error(EINVAL); return; } error = guest_vaddr2paddr(vcpus[cur_vcpu], gva, &gpa); if (error == -1) { send_error(errno); return; } if (error == 0) { send_error(EFAULT); return; } cp = paddr_guest2host(ctx, gpa, sizeof(bp->shadow_inst)); /* Only permit breakpoints in guest RAM. */ if (cp == NULL) { send_error(EFAULT); return; } /* Find any existing breakpoint. */ bp = find_breakpoint(gpa); /* * Silently ignore duplicate commands since the protocol * requires these packets to be idempotent. */ if (insert) { if (bp == NULL) { if (TAILQ_EMPTY(&breakpoints) && !set_breakpoint_caps(true)) { send_empty_response(); return; } bp = malloc(sizeof(*bp)); bp->gpa = gpa; memcpy(bp->shadow_inst, cp, sizeof(bp->shadow_inst)); memcpy(cp, GDB_BP_INSTR, sizeof(bp->shadow_inst)); TAILQ_INSERT_TAIL(&breakpoints, bp, link); debug("new breakpoint at %#lx\n", gpa); } } else { if (bp != NULL) { debug("remove breakpoint at %#lx\n", gpa); memcpy(cp, bp->shadow_inst, sizeof(bp->shadow_inst)); TAILQ_REMOVE(&breakpoints, bp, link); free(bp); if (TAILQ_EMPTY(&breakpoints)) set_breakpoint_caps(false); } } send_ok(); } static void parse_breakpoint(const uint8_t *data, size_t len) { uint64_t gva; uint8_t *cp; bool insert; int kind, type; insert = data[0] == 'Z'; /* Skip 'Z/z' */ data += 1; len -= 1; /* Parse and consume type. */ cp = memchr(data, ',', len); if (cp == NULL || cp == data) { send_error(EINVAL); return; } type = parse_integer(data, cp - data); len -= (cp - data) + 1; data += (cp - data) + 1; /* Parse and consume address. */ cp = memchr(data, ',', len); if (cp == NULL || cp == data) { send_error(EINVAL); return; } gva = parse_integer(data, cp - data); len -= (cp - data) + 1; data += (cp - data) + 1; /* Parse and consume kind. */ cp = memchr(data, ';', len); if (cp == data) { send_error(EINVAL); return; } if (cp != NULL) { /* * We do not advertise support for either the * ConditionalBreakpoints or BreakpointCommands * features, so we should not be getting conditions or * commands from the remote end. */ send_empty_response(); return; } kind = parse_integer(data, len); data += len; len = 0; switch (type) { case 0: update_sw_breakpoint(gva, kind, insert); break; default: send_empty_response(); break; } } static bool command_equals(const uint8_t *data, size_t len, const char *cmd) { if (strlen(cmd) > len) return (false); return (memcmp(data, cmd, strlen(cmd)) == 0); } static void check_features(const uint8_t *data, size_t len) { char *feature, *next_feature, *str, *value; bool supported; str = malloc(len + 1); memcpy(str, data, len); str[len] = '\0'; next_feature = str; while ((feature = strsep(&next_feature, ";")) != NULL) { /* * Null features shouldn't exist, but skip if they * do. */ if (strcmp(feature, "") == 0) continue; /* * Look for the value or supported / not supported * flag. */ value = strchr(feature, '='); if (value != NULL) { *value = '\0'; value++; supported = true; } else { value = feature + strlen(feature) - 1; switch (*value) { case '+': supported = true; break; case '-': supported = false; break; default: /* * This is really a protocol error, * but we just ignore malformed * features for ease of * implementation. */ continue; } value = NULL; } if (strcmp(feature, "swbreak") == 0) swbreak_enabled = supported; #ifndef __FreeBSD__ /* * The compiler dislikes 'supported' being set but never used. * Make it happy here. */ if (supported) { debug("feature '%s' supported\n", feature); } #endif /* __FreeBSD__ */ } free(str); start_packet(); /* This is an arbitrary limit. */ append_string("PacketSize=4096"); append_string(";swbreak+"); finish_packet(); } static void gdb_query(const uint8_t *data, size_t len) { /* * TODO: * - qSearch */ if (command_equals(data, len, "qAttached")) { start_packet(); append_char('1'); finish_packet(); } else if (command_equals(data, len, "qC")) { start_packet(); append_string("QC"); append_integer(cur_vcpu + 1); finish_packet(); } else if (command_equals(data, len, "qfThreadInfo")) { cpuset_t mask; bool first; int vcpu; if (CPU_EMPTY(&vcpus_active)) { send_error(EINVAL); return; } mask = vcpus_active; start_packet(); append_char('m'); first = true; while (!CPU_EMPTY(&mask)) { vcpu = CPU_FFS(&mask) - 1; CPU_CLR(vcpu, &mask); if (first) first = false; else append_char(','); append_integer(vcpu + 1); } finish_packet(); } else if (command_equals(data, len, "qsThreadInfo")) { start_packet(); append_char('l'); finish_packet(); } else if (command_equals(data, len, "qSupported")) { data += strlen("qSupported"); len -= strlen("qSupported"); check_features(data, len); } else if (command_equals(data, len, "qThreadExtraInfo")) { char buf[16]; int tid; data += strlen("qThreadExtraInfo"); len -= strlen("qThreadExtraInfo"); if (len == 0 || *data != ',') { send_error(EINVAL); return; } tid = parse_threadid(data + 1, len - 1); if (tid <= 0 || !CPU_ISSET(tid - 1, &vcpus_active)) { send_error(EINVAL); return; } snprintf(buf, sizeof(buf), "vCPU %d", tid - 1); start_packet(); append_asciihex(buf); finish_packet(); } else send_empty_response(); } static void handle_command(const uint8_t *data, size_t len) { /* Reject packets with a sequence-id. */ if (len >= 3 && data[0] >= '0' && data[0] <= '9' && data[0] >= '0' && data[0] <= '9' && data[2] == ':') { send_empty_response(); return; } switch (*data) { case 'c': if (len != 1) { send_error(EINVAL); break; } discard_stop(); gdb_resume_vcpus(); break; case 'D': send_ok(); /* TODO: Resume any stopped CPUs. */ break; case 'g': gdb_read_regs(); break; case 'p': gdb_read_one_reg(data + 1, len - 1); break; case 'H': { int tid; if (len < 2 || (data[1] != 'g' && data[1] != 'c')) { send_error(EINVAL); break; } tid = parse_threadid(data + 2, len - 2); if (tid == -2) { send_error(EINVAL); break; } if (CPU_EMPTY(&vcpus_active)) { send_error(EINVAL); break; } if (tid == -1 || tid == 0) cur_vcpu = CPU_FFS(&vcpus_active) - 1; else if (CPU_ISSET(tid - 1, &vcpus_active)) cur_vcpu = tid - 1; else { send_error(EINVAL); break; } send_ok(); break; } case 'm': gdb_read_mem(data, len); break; case 'M': gdb_write_mem(data, len); break; case 'T': { int tid; tid = parse_threadid(data + 1, len - 1); if (tid <= 0 || !CPU_ISSET(tid - 1, &vcpus_active)) { send_error(EINVAL); return; } send_ok(); break; } case 'q': gdb_query(data, len); break; case 's': if (len != 1) { send_error(EINVAL); break; } /* Don't send a reply until a stop occurs. */ if (!gdb_step_vcpu(vcpus[cur_vcpu])) { send_error(EOPNOTSUPP); break; } break; case 'z': case 'Z': parse_breakpoint(data, len); break; case '?': report_stop(false); break; case 'G': /* TODO */ case 'v': /* Handle 'vCont' */ /* 'vCtrlC' */ case 'P': /* TODO */ case 'Q': /* TODO */ case 't': /* TODO */ case 'X': /* TODO */ default: send_empty_response(); } } /* Check for a valid packet in the command buffer. */ static void check_command(int fd) { uint8_t *head, *hash, *p, sum; size_t avail, plen; for (;;) { avail = cur_comm.len; if (avail == 0) return; head = io_buffer_head(&cur_comm); switch (*head) { case 0x03: debug("<- Ctrl-C\n"); io_buffer_consume(&cur_comm, 1); gdb_suspend_vcpus(); break; case '+': /* ACK of previous response. */ debug("<- +\n"); if (response_pending()) io_buffer_reset(&cur_resp); io_buffer_consume(&cur_comm, 1); if (stopped_vcpu != -1 && report_next_stop) { report_stop(true); send_pending_data(fd); } break; case '-': /* NACK of previous response. */ debug("<- -\n"); if (response_pending()) { cur_resp.len += cur_resp.start; cur_resp.start = 0; if (cur_resp.data[0] == '+') io_buffer_advance(&cur_resp, 1); debug("-> %.*s\n", (int)cur_resp.len, io_buffer_head(&cur_resp)); } io_buffer_consume(&cur_comm, 1); send_pending_data(fd); break; case '$': /* Packet. */ if (response_pending()) { warnx("New GDB command while response in " "progress"); io_buffer_reset(&cur_resp); } /* Is packet complete? */ hash = memchr(head, '#', avail); if (hash == NULL) return; plen = (hash - head + 1) + 2; if (avail < plen) return; debug("<- %.*s\n", (int)plen, head); /* Verify checksum. */ for (sum = 0, p = head + 1; p < hash; p++) sum += *p; if (sum != parse_byte(hash + 1)) { io_buffer_consume(&cur_comm, plen); debug("-> -\n"); send_char('-'); send_pending_data(fd); break; } send_char('+'); handle_command(head + 1, hash - (head + 1)); io_buffer_consume(&cur_comm, plen); if (!response_pending()) { debug("-> +\n"); } send_pending_data(fd); break; default: /* XXX: Possibly drop connection instead. */ debug("-> %02x\n", *head); io_buffer_consume(&cur_comm, 1); break; } } } static void gdb_readable(int fd, enum ev_type event __unused, void *arg __unused) { size_t pending; ssize_t nread; int n; if (ioctl(fd, FIONREAD, &n) == -1) { warn("FIONREAD on GDB socket"); return; } assert(n >= 0); pending = n; /* * 'pending' might be zero due to EOF. We need to call read * with a non-zero length to detect EOF. */ if (pending == 0) pending = 1; /* Ensure there is room in the command buffer. */ io_buffer_grow(&cur_comm, pending); assert(io_buffer_avail(&cur_comm) >= pending); nread = read(fd, io_buffer_tail(&cur_comm), io_buffer_avail(&cur_comm)); if (nread == 0) { close_connection(); } else if (nread == -1) { if (errno == EAGAIN) return; warn("Read from GDB socket"); close_connection(); } else { cur_comm.len += nread; pthread_mutex_lock(&gdb_lock); check_command(fd); pthread_mutex_unlock(&gdb_lock); } } static void gdb_writable(int fd, enum ev_type event __unused, void *arg __unused) { send_pending_data(fd); } static void new_connection(int fd, enum ev_type event __unused, void *arg) { int optval, s; s = accept4(fd, NULL, NULL, SOCK_NONBLOCK); if (s == -1) { if (arg != NULL) err(1, "Failed accepting initial GDB connection"); /* Silently ignore errors post-startup. */ return; } optval = 1; if (setsockopt(s, SOL_SOCKET, SO_NOSIGPIPE, &optval, sizeof(optval)) == -1) { warn("Failed to disable SIGPIPE for GDB connection"); close(s); return; } pthread_mutex_lock(&gdb_lock); if (cur_fd != -1) { close(s); warnx("Ignoring additional GDB connection."); } read_event = mevent_add(s, EVF_READ, gdb_readable, NULL); if (read_event == NULL) { if (arg != NULL) err(1, "Failed to setup initial GDB connection"); pthread_mutex_unlock(&gdb_lock); return; } write_event = mevent_add(s, EVF_WRITE, gdb_writable, NULL); if (write_event == NULL) { if (arg != NULL) err(1, "Failed to setup initial GDB connection"); mevent_delete_close(read_event); read_event = NULL; } cur_fd = s; cur_vcpu = 0; stopped_vcpu = -1; /* Break on attach. */ first_stop = true; report_next_stop = false; gdb_suspend_vcpus(); pthread_mutex_unlock(&gdb_lock); } #ifndef WITHOUT_CAPSICUM static void limit_gdb_socket(int s) { cap_rights_t rights; unsigned long ioctls[] = { FIONREAD }; cap_rights_init(&rights, CAP_ACCEPT, CAP_EVENT, CAP_READ, CAP_WRITE, CAP_SETSOCKOPT, CAP_IOCTL); if (caph_rights_limit(s, &rights) == -1) errx(EX_OSERR, "Unable to apply rights for sandbox"); if (caph_ioctls_limit(s, ioctls, nitems(ioctls)) == -1) errx(EX_OSERR, "Unable to apply rights for sandbox"); } #endif #ifndef __FreeBSD__ /* * Equivalent to init_gdb() below, but without configuring the listening socket. * This will allow the bhyve process to tolerate mdb attaching/detaching from * the instance while it is running. */ void init_mdb(struct vmctx *_ctx) { int error; bool wait; wait = get_config_bool_default("gdb.wait", false); error = pthread_mutex_init(&gdb_lock, NULL); if (error != 0) errc(1, error, "gdb mutex init"); error = pthread_cond_init(&idle_vcpus, NULL); if (error != 0) errc(1, error, "gdb cv init"); ctx = _ctx; stopped_vcpu = -1; TAILQ_INIT(&breakpoints); vcpu_state = calloc(guest_ncpus, sizeof(*vcpu_state)); if (wait) { /* * Set vcpu 0 in vcpus_suspended. This will trigger the * logic in gdb_cpu_add() to suspend the first vcpu before * it starts execution. The vcpu will remain suspended * until a debugger connects. */ CPU_SET(0, &vcpus_suspended); stopped_vcpu = 0; } } #endif void init_gdb(struct vmctx *_ctx) { int error, flags, optval, s; struct addrinfo hints; struct addrinfo *gdbaddr; const char *saddr, *value; char *sport; bool wait; value = get_config_value("gdb.port"); if (value == NULL) return; sport = strdup(value); if (sport == NULL) errx(4, "Failed to allocate memory"); wait = get_config_bool_default("gdb.wait", false); saddr = get_config_value("gdb.address"); if (saddr == NULL) { saddr = "localhost"; } debug("==> starting on %s:%s, %swaiting\n", saddr, sport, wait ? "" : "not "); error = pthread_mutex_init(&gdb_lock, NULL); if (error != 0) errc(1, error, "gdb mutex init"); error = pthread_cond_init(&idle_vcpus, NULL); if (error != 0) errc(1, error, "gdb cv init"); memset(&hints, 0, sizeof(hints)); hints.ai_family = AF_UNSPEC; hints.ai_socktype = SOCK_STREAM; hints.ai_flags = AI_NUMERICSERV | AI_PASSIVE; error = getaddrinfo(saddr, sport, &hints, &gdbaddr); if (error != 0) errx(1, "gdb address resolution: %s", gai_strerror(error)); ctx = _ctx; s = socket(gdbaddr->ai_family, gdbaddr->ai_socktype, 0); if (s < 0) err(1, "gdb socket create"); optval = 1; (void)setsockopt(s, SOL_SOCKET, SO_REUSEADDR, &optval, sizeof(optval)); if (bind(s, gdbaddr->ai_addr, gdbaddr->ai_addrlen) < 0) err(1, "gdb socket bind"); if (listen(s, 1) < 0) err(1, "gdb socket listen"); stopped_vcpu = -1; TAILQ_INIT(&breakpoints); vcpus = calloc(guest_ncpus, sizeof(*vcpus)); vcpu_state = calloc(guest_ncpus, sizeof(*vcpu_state)); if (wait) { /* * Set vcpu 0 in vcpus_suspended. This will trigger the * logic in gdb_cpu_add() to suspend the first vcpu before * it starts execution. The vcpu will remain suspended * until a debugger connects. */ CPU_SET(0, &vcpus_suspended); stopped_vcpu = 0; } flags = fcntl(s, F_GETFL); if (fcntl(s, F_SETFL, flags | O_NONBLOCK) == -1) err(1, "Failed to mark gdb socket non-blocking"); #ifndef WITHOUT_CAPSICUM limit_gdb_socket(s); #endif mevent_add(s, EVF_READ, new_connection, NULL); gdb_active = true; freeaddrinfo(gdbaddr); free(sport); } /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2017 John H. Baldwin * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef __GDB_H__ #define __GDB_H__ void gdb_cpu_add(struct vcpu *vcpu); void gdb_cpu_breakpoint(struct vcpu *vcpu, struct vm_exit *vmexit); void gdb_cpu_mtrap(struct vcpu *vcpu); void gdb_cpu_suspend(struct vcpu *vcpu); void init_gdb(struct vmctx *ctx); #ifndef __FreeBSD__ void init_mdb(struct vmctx *ctx); #endif #endif /* !__GDB_H__ */ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2016 Alex Teaca * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * */ #include #include #include #include "pci_hda.h" #include "audio.h" /* * HDA Codec defines */ #define INTEL_VENDORID 0x8086 #define HDA_CODEC_SUBSYSTEM_ID ((INTEL_VENDORID << 16) | 0x01) #define HDA_CODEC_ROOT_NID 0x00 #define HDA_CODEC_FG_NID 0x01 #define HDA_CODEC_AUDIO_OUTPUT_NID 0x02 #define HDA_CODEC_PIN_OUTPUT_NID 0x03 #define HDA_CODEC_AUDIO_INPUT_NID 0x04 #define HDA_CODEC_PIN_INPUT_NID 0x05 #define HDA_CODEC_STREAMS_COUNT 0x02 #define HDA_CODEC_STREAM_OUTPUT 0x00 #define HDA_CODEC_STREAM_INPUT 0x01 #define HDA_CODEC_PARAMS_COUNT 0x14 #define HDA_CODEC_CONN_LIST_COUNT 0x01 #define HDA_CODEC_RESPONSE_EX_UNSOL 0x10 #define HDA_CODEC_RESPONSE_EX_SOL 0x00 #define HDA_CODEC_AMP_NUMSTEPS 0x4a #define HDA_CODEC_SUPP_STREAM_FORMATS_PCM \ (1 << HDA_PARAM_SUPP_STREAM_FORMATS_PCM_SHIFT) #define HDA_CODEC_FMT_BASE_MASK (0x01 << 14) #define HDA_CODEC_FMT_MULT_MASK (0x07 << 11) #define HDA_CODEC_FMT_MULT_2 (0x01 << 11) #define HDA_CODEC_FMT_MULT_3 (0x02 << 11) #define HDA_CODEC_FMT_MULT_4 (0x03 << 11) #define HDA_CODEC_FMT_DIV_MASK 0x07 #define HDA_CODEC_FMT_DIV_SHIFT 8 #define HDA_CODEC_FMT_BITS_MASK (0x07 << 4) #define HDA_CODEC_FMT_BITS_8 (0x00 << 4) #define HDA_CODEC_FMT_BITS_16 (0x01 << 4) #define HDA_CODEC_FMT_BITS_24 (0x03 << 4) #define HDA_CODEC_FMT_BITS_32 (0x04 << 4) #define HDA_CODEC_FMT_CHAN_MASK (0x0f << 0) #define HDA_CODEC_AUDIO_WCAP_OUTPUT \ (0x00 << HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_SHIFT) #define HDA_CODEC_AUDIO_WCAP_INPUT \ (0x01 << HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_SHIFT) #define HDA_CODEC_AUDIO_WCAP_PIN \ (0x04 << HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_SHIFT) #define HDA_CODEC_AUDIO_WCAP_CONN_LIST \ (1 << HDA_PARAM_AUDIO_WIDGET_CAP_CONN_LIST_SHIFT) #define HDA_CODEC_AUDIO_WCAP_FORMAT_OVR \ (1 << HDA_PARAM_AUDIO_WIDGET_CAP_FORMAT_OVR_SHIFT) #define HDA_CODEC_AUDIO_WCAP_AMP_OVR \ (1 << HDA_PARAM_AUDIO_WIDGET_CAP_AMP_OVR_SHIFT) #define HDA_CODEC_AUDIO_WCAP_OUT_AMP \ (1 << HDA_PARAM_AUDIO_WIDGET_CAP_OUT_AMP_SHIFT) #define HDA_CODEC_AUDIO_WCAP_IN_AMP \ (1 << HDA_PARAM_AUDIO_WIDGET_CAP_IN_AMP_SHIFT) #define HDA_CODEC_AUDIO_WCAP_STEREO \ (1 << HDA_PARAM_AUDIO_WIDGET_CAP_STEREO_SHIFT) #define HDA_CODEC_PIN_CAP_OUTPUT \ (1 << HDA_PARAM_PIN_CAP_OUTPUT_CAP_SHIFT) #define HDA_CODEC_PIN_CAP_INPUT \ (1 << HDA_PARAM_PIN_CAP_INPUT_CAP_SHIFT) #define HDA_CODEC_PIN_CAP_PRESENCE_DETECT \ (1 << HDA_PARAM_PIN_CAP_PRESENCE_DETECT_CAP_SHIFT) #define HDA_CODEC_OUTPUT_AMP_CAP_MUTE_CAP \ (1 << HDA_PARAM_OUTPUT_AMP_CAP_MUTE_CAP_SHIFT) #define HDA_CODEC_OUTPUT_AMP_CAP_STEPSIZE \ (0x03 << HDA_PARAM_OUTPUT_AMP_CAP_STEPSIZE_SHIFT) #define HDA_CODEC_OUTPUT_AMP_CAP_NUMSTEPS \ (HDA_CODEC_AMP_NUMSTEPS << HDA_PARAM_OUTPUT_AMP_CAP_NUMSTEPS_SHIFT) #define HDA_CODEC_OUTPUT_AMP_CAP_OFFSET \ (HDA_CODEC_AMP_NUMSTEPS << HDA_PARAM_OUTPUT_AMP_CAP_OFFSET_SHIFT) #define HDA_CODEC_SET_AMP_GAIN_MUTE_MUTE 0x80 #define HDA_CODEC_SET_AMP_GAIN_MUTE_GAIN_MASK 0x7f #define HDA_CODEC_PIN_SENSE_PRESENCE_PLUGGED (1 << 31) #define HDA_CODEC_PIN_WIDGET_CTRL_OUT_ENABLE \ (1 << HDA_CMD_GET_PIN_WIDGET_CTRL_OUT_ENABLE_SHIFT) #define HDA_CODEC_PIN_WIDGET_CTRL_IN_ENABLE \ (1 << HDA_CMD_GET_PIN_WIDGET_CTRL_IN_ENABLE_SHIFT) #define HDA_CONFIG_DEFAULTCONF_COLOR_BLACK \ (0x01 << HDA_CONFIG_DEFAULTCONF_COLOR_SHIFT) #define HDA_CONFIG_DEFAULTCONF_COLOR_RED \ (0x05 << HDA_CONFIG_DEFAULTCONF_COLOR_SHIFT) #define HDA_CODEC_BUF_SIZE HDA_FIFO_SIZE #define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0])) /* * HDA Audio Context data structures */ typedef void (*transfer_func_t)(void *arg); typedef int (*setup_func_t)(void *arg); struct hda_audio_ctxt { char name[64]; uint8_t run; uint8_t started; void *priv; pthread_t tid; pthread_mutex_t mtx; pthread_cond_t cond; setup_func_t do_setup; transfer_func_t do_transfer; }; /* * HDA Audio Context module function declarations */ static void *hda_audio_ctxt_thr(void *arg); static int hda_audio_ctxt_init(struct hda_audio_ctxt *actx, const char *tname, transfer_func_t do_transfer, setup_func_t do_setup, void *priv); static int hda_audio_ctxt_start(struct hda_audio_ctxt *actx); static int hda_audio_ctxt_stop(struct hda_audio_ctxt *actx); /* * HDA Codec data structures */ struct hda_codec_softc; typedef uint32_t (*verb_func_t)(struct hda_codec_softc *sc, uint16_t verb, uint16_t payload); struct hda_codec_stream { uint8_t buf[HDA_CODEC_BUF_SIZE]; uint8_t channel; uint16_t fmt; uint8_t stream; uint8_t left_gain; uint8_t right_gain; uint8_t left_mute; uint8_t right_mute; struct audio *aud; struct hda_audio_ctxt actx; }; struct hda_codec_softc { uint32_t no_nodes; uint32_t subsystem_id; const uint32_t (*get_parameters)[HDA_CODEC_PARAMS_COUNT]; const uint8_t (*conn_list)[HDA_CODEC_CONN_LIST_COUNT]; const uint32_t *conf_default; const uint8_t *pin_ctrl_default; const verb_func_t *verb_handlers; struct hda_codec_inst *hci; struct hda_codec_stream streams[HDA_CODEC_STREAMS_COUNT]; }; /* * HDA Codec module function declarations */ static int hda_codec_init(struct hda_codec_inst *hci, const char *play, const char *rec); static int hda_codec_reset(struct hda_codec_inst *hci); static int hda_codec_command(struct hda_codec_inst *hci, uint32_t cmd_data); static int hda_codec_notify(struct hda_codec_inst *hci, uint8_t run, uint8_t stream, uint8_t dir); static int hda_codec_parse_format(uint16_t fmt, struct audio_params *params); static uint32_t hda_codec_audio_output_nid(struct hda_codec_softc *sc, uint16_t verb, uint16_t payload); static void hda_codec_audio_output_do_transfer(void *arg); static int hda_codec_audio_output_do_setup(void *arg); static uint32_t hda_codec_audio_input_nid(struct hda_codec_softc *sc, uint16_t verb, uint16_t payload); static void hda_codec_audio_input_do_transfer(void *arg); static int hda_codec_audio_input_do_setup(void *arg); static uint32_t hda_codec_audio_inout_nid(struct hda_codec_stream *st, uint16_t verb, uint16_t payload); /* * HDA Codec global data */ #define HDA_CODEC_ROOT_DESC \ [HDA_CODEC_ROOT_NID] = { \ [HDA_PARAM_VENDOR_ID] = INTEL_VENDORID, \ [HDA_PARAM_REVISION_ID] = 0xffff, \ /* 1 Subnode, StartNid = 1 */ \ [HDA_PARAM_SUB_NODE_COUNT] = 0x00010001, \ }, \ #define HDA_CODEC_FG_COMMON_DESC \ [HDA_PARAM_FCT_GRP_TYPE] = HDA_PARAM_FCT_GRP_TYPE_NODE_TYPE_AUDIO,\ /* B8 - B32, 8.0 - 192.0kHz */ \ [HDA_PARAM_SUPP_PCM_SIZE_RATE] = (0x1f << 16) | 0x7ff, \ [HDA_PARAM_SUPP_STREAM_FORMATS] = HDA_CODEC_SUPP_STREAM_FORMATS_PCM,\ [HDA_PARAM_INPUT_AMP_CAP] = 0x00, /* None */ \ [HDA_PARAM_OUTPUT_AMP_CAP] = 0x00, /* None */ \ [HDA_PARAM_GPIO_COUNT] = 0x00, \ #define HDA_CODEC_FG_OUTPUT_DESC \ [HDA_CODEC_FG_NID] = { \ /* 2 Subnodes, StartNid = 2 */ \ [HDA_PARAM_SUB_NODE_COUNT] = 0x00020002, \ HDA_CODEC_FG_COMMON_DESC \ }, \ #define HDA_CODEC_FG_INPUT_DESC \ [HDA_CODEC_FG_NID] = { \ /* 2 Subnodes, StartNid = 4 */ \ [HDA_PARAM_SUB_NODE_COUNT] = 0x00040002, \ HDA_CODEC_FG_COMMON_DESC \ }, \ #define HDA_CODEC_FG_DUPLEX_DESC \ [HDA_CODEC_FG_NID] = { \ /* 4 Subnodes, StartNid = 2 */ \ [HDA_PARAM_SUB_NODE_COUNT] = 0x00020004, \ HDA_CODEC_FG_COMMON_DESC \ }, \ #define HDA_CODEC_OUTPUT_DESC \ [HDA_CODEC_AUDIO_OUTPUT_NID] = { \ [HDA_PARAM_AUDIO_WIDGET_CAP] = \ HDA_CODEC_AUDIO_WCAP_OUTPUT | \ HDA_CODEC_AUDIO_WCAP_FORMAT_OVR | \ HDA_CODEC_AUDIO_WCAP_AMP_OVR | \ HDA_CODEC_AUDIO_WCAP_OUT_AMP | \ HDA_CODEC_AUDIO_WCAP_STEREO, \ /* B16, 16.0 - 192.0kHz */ \ [HDA_PARAM_SUPP_PCM_SIZE_RATE] = (0x02 << 16) | 0x7fc, \ [HDA_PARAM_SUPP_STREAM_FORMATS] = \ HDA_CODEC_SUPP_STREAM_FORMATS_PCM, \ [HDA_PARAM_INPUT_AMP_CAP] = 0x00, /* None */ \ [HDA_PARAM_CONN_LIST_LENGTH] = 0x00, \ [HDA_PARAM_OUTPUT_AMP_CAP] = \ HDA_CODEC_OUTPUT_AMP_CAP_MUTE_CAP | \ HDA_CODEC_OUTPUT_AMP_CAP_STEPSIZE | \ HDA_CODEC_OUTPUT_AMP_CAP_NUMSTEPS | \ HDA_CODEC_OUTPUT_AMP_CAP_OFFSET, \ }, \ [HDA_CODEC_PIN_OUTPUT_NID] = { \ [HDA_PARAM_AUDIO_WIDGET_CAP] = \ HDA_CODEC_AUDIO_WCAP_PIN | \ HDA_CODEC_AUDIO_WCAP_CONN_LIST | \ HDA_CODEC_AUDIO_WCAP_STEREO, \ [HDA_PARAM_PIN_CAP] = HDA_CODEC_PIN_CAP_OUTPUT | \ HDA_CODEC_PIN_CAP_PRESENCE_DETECT,\ [HDA_PARAM_INPUT_AMP_CAP] = 0x00, /* None */ \ [HDA_PARAM_CONN_LIST_LENGTH] = 0x01, \ [HDA_PARAM_OUTPUT_AMP_CAP] = 0x00, /* None */ \ }, \ #define HDA_CODEC_INPUT_DESC \ [HDA_CODEC_AUDIO_INPUT_NID] = { \ [HDA_PARAM_AUDIO_WIDGET_CAP] = \ HDA_CODEC_AUDIO_WCAP_INPUT | \ HDA_CODEC_AUDIO_WCAP_CONN_LIST | \ HDA_CODEC_AUDIO_WCAP_FORMAT_OVR | \ HDA_CODEC_AUDIO_WCAP_AMP_OVR | \ HDA_CODEC_AUDIO_WCAP_IN_AMP | \ HDA_CODEC_AUDIO_WCAP_STEREO, \ /* B16, 16.0 - 192.0kHz */ \ [HDA_PARAM_SUPP_PCM_SIZE_RATE] = (0x02 << 16) | 0x7fc, \ [HDA_PARAM_SUPP_STREAM_FORMATS] = \ HDA_CODEC_SUPP_STREAM_FORMATS_PCM, \ [HDA_PARAM_OUTPUT_AMP_CAP] = 0x00, /* None */ \ [HDA_PARAM_CONN_LIST_LENGTH] = 0x01, \ [HDA_PARAM_INPUT_AMP_CAP] = \ HDA_CODEC_OUTPUT_AMP_CAP_MUTE_CAP | \ HDA_CODEC_OUTPUT_AMP_CAP_STEPSIZE | \ HDA_CODEC_OUTPUT_AMP_CAP_NUMSTEPS | \ HDA_CODEC_OUTPUT_AMP_CAP_OFFSET, \ }, \ [HDA_CODEC_PIN_INPUT_NID] = { \ [HDA_PARAM_AUDIO_WIDGET_CAP] = \ HDA_CODEC_AUDIO_WCAP_PIN | \ HDA_CODEC_AUDIO_WCAP_STEREO, \ [HDA_PARAM_PIN_CAP] = HDA_CODEC_PIN_CAP_INPUT | \ HDA_CODEC_PIN_CAP_PRESENCE_DETECT, \ [HDA_PARAM_INPUT_AMP_CAP] = 0x00, /* None */ \ [HDA_PARAM_OUTPUT_AMP_CAP] = 0x00, /* None */ \ }, \ static const uint32_t hda_codec_output_parameters[][HDA_CODEC_PARAMS_COUNT] = { HDA_CODEC_ROOT_DESC HDA_CODEC_FG_OUTPUT_DESC HDA_CODEC_OUTPUT_DESC }; static const uint32_t hda_codec_input_parameters[][HDA_CODEC_PARAMS_COUNT] = { HDA_CODEC_ROOT_DESC HDA_CODEC_FG_INPUT_DESC HDA_CODEC_INPUT_DESC }; static const uint32_t hda_codec_duplex_parameters[][HDA_CODEC_PARAMS_COUNT] = { HDA_CODEC_ROOT_DESC HDA_CODEC_FG_DUPLEX_DESC HDA_CODEC_OUTPUT_DESC HDA_CODEC_INPUT_DESC }; #define HDA_CODEC_NODES_COUNT (ARRAY_SIZE(hda_codec_duplex_parameters)) static const uint8_t hda_codec_conn_list[HDA_CODEC_NODES_COUNT][HDA_CODEC_CONN_LIST_COUNT] = { [HDA_CODEC_PIN_OUTPUT_NID] = {HDA_CODEC_AUDIO_OUTPUT_NID}, [HDA_CODEC_AUDIO_INPUT_NID] = {HDA_CODEC_PIN_INPUT_NID}, }; static const uint32_t hda_codec_conf_default[HDA_CODEC_NODES_COUNT] = { [HDA_CODEC_PIN_OUTPUT_NID] = \ HDA_CONFIG_DEFAULTCONF_CONNECTIVITY_JACK | HDA_CONFIG_DEFAULTCONF_DEVICE_LINE_OUT | HDA_CONFIG_DEFAULTCONF_COLOR_BLACK | (0x01 << HDA_CONFIG_DEFAULTCONF_ASSOCIATION_SHIFT), [HDA_CODEC_PIN_INPUT_NID] = HDA_CONFIG_DEFAULTCONF_CONNECTIVITY_JACK | HDA_CONFIG_DEFAULTCONF_DEVICE_LINE_IN | HDA_CONFIG_DEFAULTCONF_COLOR_RED | (0x02 << HDA_CONFIG_DEFAULTCONF_ASSOCIATION_SHIFT), }; static const uint8_t hda_codec_pin_ctrl_default[HDA_CODEC_NODES_COUNT] = { [HDA_CODEC_PIN_OUTPUT_NID] = HDA_CODEC_PIN_WIDGET_CTRL_OUT_ENABLE, [HDA_CODEC_PIN_INPUT_NID] = HDA_CODEC_PIN_WIDGET_CTRL_IN_ENABLE, }; static const verb_func_t hda_codec_verb_handlers[HDA_CODEC_NODES_COUNT] = { [HDA_CODEC_AUDIO_OUTPUT_NID] = hda_codec_audio_output_nid, [HDA_CODEC_AUDIO_INPUT_NID] = hda_codec_audio_input_nid, }; /* * HDA Codec module function definitions */ static int hda_codec_init(struct hda_codec_inst *hci, const char *play, const char *rec) { struct hda_codec_softc *sc = NULL; struct hda_codec_stream *st = NULL; int err; if (!(play || rec)) return (-1); sc = calloc(1, sizeof(*sc)); if (!sc) return (-1); if (play && rec) sc->get_parameters = hda_codec_duplex_parameters; else { if (play) sc->get_parameters = hda_codec_output_parameters; else sc->get_parameters = hda_codec_input_parameters; } sc->subsystem_id = HDA_CODEC_SUBSYSTEM_ID; sc->no_nodes = HDA_CODEC_NODES_COUNT; sc->conn_list = hda_codec_conn_list; sc->conf_default = hda_codec_conf_default; sc->pin_ctrl_default = hda_codec_pin_ctrl_default; sc->verb_handlers = hda_codec_verb_handlers; DPRINTF("HDA Codec nodes: %d", sc->no_nodes); /* * Initialize the Audio Output stream */ if (play) { st = &sc->streams[HDA_CODEC_STREAM_OUTPUT]; err = hda_audio_ctxt_init(&st->actx, "hda-audio-output", hda_codec_audio_output_do_transfer, hda_codec_audio_output_do_setup, sc); assert(!err); st->aud = audio_init(play, 1); if (!st->aud) { DPRINTF("Fail to init the output audio player"); return (-1); } } /* * Initialize the Audio Input stream */ if (rec) { st = &sc->streams[HDA_CODEC_STREAM_INPUT]; err = hda_audio_ctxt_init(&st->actx, "hda-audio-input", hda_codec_audio_input_do_transfer, hda_codec_audio_input_do_setup, sc); assert(!err); st->aud = audio_init(rec, 0); if (!st->aud) { DPRINTF("Fail to init the input audio player"); return (-1); } } sc->hci = hci; hci->priv = sc; return (0); } static int hda_codec_reset(struct hda_codec_inst *hci) { const struct hda_ops *hops = NULL; struct hda_codec_softc *sc = NULL; struct hda_codec_stream *st = NULL; int i; assert(hci); hops = hci->hops; assert(hops); sc = (struct hda_codec_softc *)hci->priv; assert(sc); for (i = 0; i < HDA_CODEC_STREAMS_COUNT; i++) { st = &sc->streams[i]; st->left_gain = HDA_CODEC_AMP_NUMSTEPS; st->right_gain = HDA_CODEC_AMP_NUMSTEPS; st->left_mute = HDA_CODEC_SET_AMP_GAIN_MUTE_MUTE; st->right_mute = HDA_CODEC_SET_AMP_GAIN_MUTE_MUTE; } DPRINTF("cad: 0x%x", hci->cad); if (!hops->signal) { DPRINTF("The controller ops does not implement \ the signal function"); return (-1); } return (hops->signal(hci)); } static int hda_codec_command(struct hda_codec_inst *hci, uint32_t cmd_data) { const struct hda_ops *hops = NULL; struct hda_codec_softc *sc = NULL; uint8_t cad = 0, nid = 0; uint16_t verb = 0, payload = 0; uint32_t res = 0; /* 4 bits */ cad = (cmd_data >> HDA_CMD_CAD_SHIFT) & 0x0f; /* 8 bits */ nid = (cmd_data >> HDA_CMD_NID_SHIFT) & 0xff; if ((cmd_data & 0x70000) == 0x70000) { /* 12 bits */ verb = (cmd_data >> HDA_CMD_VERB_12BIT_SHIFT) & 0x0fff; /* 8 bits */ payload = cmd_data & 0xff; } else { /* 4 bits */ verb = (cmd_data >> HDA_CMD_VERB_4BIT_SHIFT) & 0x0f; /* 16 bits */ payload = cmd_data & 0xffff; } assert(hci); hops = hci->hops; assert(hops); sc = (struct hda_codec_softc *)hci->priv; assert(sc); if (cad != hci->cad || nid >= sc->no_nodes) { DPRINTF("Invalid command data"); return (-1); } if (!hops->response) { DPRINTF("The controller ops does not implement \ the response function"); return (-1); } switch (verb) { case HDA_CMD_VERB_GET_PARAMETER: if (payload < HDA_CODEC_PARAMS_COUNT) res = sc->get_parameters[nid][payload]; break; case HDA_CMD_VERB_GET_CONN_LIST_ENTRY: res = sc->conn_list[nid][0]; break; case HDA_CMD_VERB_GET_PIN_WIDGET_CTRL: res = sc->pin_ctrl_default[nid]; break; case HDA_CMD_VERB_GET_PIN_SENSE: res = HDA_CODEC_PIN_SENSE_PRESENCE_PLUGGED; break; case HDA_CMD_VERB_GET_CONFIGURATION_DEFAULT: res = sc->conf_default[nid]; break; case HDA_CMD_VERB_GET_SUBSYSTEM_ID: res = sc->subsystem_id; break; default: assert(sc->verb_handlers); if (sc->verb_handlers[nid]) res = sc->verb_handlers[nid](sc, verb, payload); else DPRINTF("Unknown VERB: 0x%x", verb); break; } DPRINTF("cad: 0x%x nid: 0x%x verb: 0x%x payload: 0x%x response: 0x%x", cad, nid, verb, payload, res); return (hops->response(hci, res, HDA_CODEC_RESPONSE_EX_SOL)); } static int hda_codec_notify(struct hda_codec_inst *hci, uint8_t run, uint8_t stream, uint8_t dir) { struct hda_codec_softc *sc = NULL; struct hda_codec_stream *st = NULL; struct hda_audio_ctxt *actx = NULL; int i; int err; assert(hci); assert(stream); sc = (struct hda_codec_softc *)hci->priv; assert(sc); i = dir ? HDA_CODEC_STREAM_OUTPUT : HDA_CODEC_STREAM_INPUT; st = &sc->streams[i]; DPRINTF("run: %d, stream: 0x%x, st->stream: 0x%x dir: %d", run, stream, st->stream, dir); if (stream != st->stream) { DPRINTF("Stream not found"); return (0); } actx = &st->actx; if (run) err = hda_audio_ctxt_start(actx); else err = hda_audio_ctxt_stop(actx); return (err); } static int hda_codec_parse_format(uint16_t fmt, struct audio_params *params) { uint8_t div = 0; assert(params); /* Compute the Sample Rate */ params->rate = (fmt & HDA_CODEC_FMT_BASE_MASK) ? 44100 : 48000; switch (fmt & HDA_CODEC_FMT_MULT_MASK) { case HDA_CODEC_FMT_MULT_2: params->rate *= 2; break; case HDA_CODEC_FMT_MULT_3: params->rate *= 3; break; case HDA_CODEC_FMT_MULT_4: params->rate *= 4; break; } div = (fmt >> HDA_CODEC_FMT_DIV_SHIFT) & HDA_CODEC_FMT_DIV_MASK; params->rate /= (div + 1); /* Compute the Bits per Sample */ switch (fmt & HDA_CODEC_FMT_BITS_MASK) { case HDA_CODEC_FMT_BITS_8: params->format = AFMT_U8; break; case HDA_CODEC_FMT_BITS_16: params->format = AFMT_S16_LE; break; case HDA_CODEC_FMT_BITS_24: params->format = AFMT_S24_LE; break; case HDA_CODEC_FMT_BITS_32: params->format = AFMT_S32_LE; break; default: DPRINTF("Unknown format bits: 0x%x", fmt & HDA_CODEC_FMT_BITS_MASK); return (-1); } /* Compute the Number of Channels */ params->channels = (fmt & HDA_CODEC_FMT_CHAN_MASK) + 1; return (0); } static uint32_t hda_codec_audio_output_nid(struct hda_codec_softc *sc, uint16_t verb, uint16_t payload) { struct hda_codec_stream *st = &sc->streams[HDA_CODEC_STREAM_OUTPUT]; int res; res = hda_codec_audio_inout_nid(st, verb, payload); return (res); } static void hda_codec_audio_output_do_transfer(void *arg) { const struct hda_ops *hops = NULL; struct hda_codec_softc *sc = (struct hda_codec_softc *)arg; struct hda_codec_inst *hci = NULL; struct hda_codec_stream *st = NULL; struct audio *aud = NULL; int err; hci = sc->hci; assert(hci); hops = hci->hops; assert(hops); st = &sc->streams[HDA_CODEC_STREAM_OUTPUT]; aud = st->aud; err = hops->transfer(hci, st->stream, 1, st->buf, sizeof(st->buf)); if (err) return; err = audio_playback(aud, st->buf, sizeof(st->buf)); assert(!err); } static int hda_codec_audio_output_do_setup(void *arg) { struct hda_codec_softc *sc = (struct hda_codec_softc *)arg; struct hda_codec_stream *st = NULL; struct audio *aud = NULL; struct audio_params params; int err; st = &sc->streams[HDA_CODEC_STREAM_OUTPUT]; aud = st->aud; err = hda_codec_parse_format(st->fmt, ¶ms); if (err) return (-1); DPRINTF("rate: %d, channels: %d, format: 0x%x", params.rate, params.channels, params.format); return (audio_set_params(aud, ¶ms)); } static uint32_t hda_codec_audio_input_nid(struct hda_codec_softc *sc, uint16_t verb, uint16_t payload) { struct hda_codec_stream *st = &sc->streams[HDA_CODEC_STREAM_INPUT]; int res; res = hda_codec_audio_inout_nid(st, verb, payload); return (res); } static void hda_codec_audio_input_do_transfer(void *arg) { const struct hda_ops *hops = NULL; struct hda_codec_softc *sc = (struct hda_codec_softc *)arg; struct hda_codec_inst *hci = NULL; struct hda_codec_stream *st = NULL; struct audio *aud = NULL; int err; hci = sc->hci; assert(hci); hops = hci->hops; assert(hops); st = &sc->streams[HDA_CODEC_STREAM_INPUT]; aud = st->aud; err = audio_record(aud, st->buf, sizeof(st->buf)); assert(!err); hops->transfer(hci, st->stream, 0, st->buf, sizeof(st->buf)); } static int hda_codec_audio_input_do_setup(void *arg) { struct hda_codec_softc *sc = (struct hda_codec_softc *)arg; struct hda_codec_stream *st = NULL; struct audio *aud = NULL; struct audio_params params; int err; st = &sc->streams[HDA_CODEC_STREAM_INPUT]; aud = st->aud; err = hda_codec_parse_format(st->fmt, ¶ms); if (err) return (-1); DPRINTF("rate: %d, channels: %d, format: 0x%x", params.rate, params.channels, params.format); return (audio_set_params(aud, ¶ms)); } static uint32_t hda_codec_audio_inout_nid(struct hda_codec_stream *st, uint16_t verb, uint16_t payload) { uint32_t res = 0; uint8_t mute = 0; uint8_t gain = 0; DPRINTF("%s verb: 0x%x, payload, 0x%x", st->actx.name, verb, payload); switch (verb) { case HDA_CMD_VERB_GET_CONV_FMT: res = st->fmt; break; case HDA_CMD_VERB_SET_CONV_FMT: st->fmt = payload; break; case HDA_CMD_VERB_GET_AMP_GAIN_MUTE: if (payload & HDA_CMD_GET_AMP_GAIN_MUTE_LEFT) { res = st->left_gain | st->left_mute; DPRINTF("GET_AMP_GAIN_MUTE_LEFT: 0x%x", res); } else { res = st->right_gain | st->right_mute; DPRINTF("GET_AMP_GAIN_MUTE_RIGHT: 0x%x", res); } break; case HDA_CMD_VERB_SET_AMP_GAIN_MUTE: mute = payload & HDA_CODEC_SET_AMP_GAIN_MUTE_MUTE; gain = payload & HDA_CODEC_SET_AMP_GAIN_MUTE_GAIN_MASK; if (payload & HDA_CMD_SET_AMP_GAIN_MUTE_LEFT) { st->left_mute = mute; st->left_gain = gain; DPRINTF("SET_AMP_GAIN_MUTE_LEFT: \ mute: 0x%x gain: 0x%x", mute, gain); } if (payload & HDA_CMD_SET_AMP_GAIN_MUTE_RIGHT) { st->right_mute = mute; st->right_gain = gain; DPRINTF("SET_AMP_GAIN_MUTE_RIGHT: \ mute: 0x%x gain: 0x%x", mute, gain); } break; case HDA_CMD_VERB_GET_CONV_STREAM_CHAN: res = (st->stream << 4) | st->channel; break; case HDA_CMD_VERB_SET_CONV_STREAM_CHAN: st->channel = payload & 0x0f; st->stream = (payload >> 4) & 0x0f; DPRINTF("st->channel: 0x%x st->stream: 0x%x", st->channel, st->stream); if (!st->stream) hda_audio_ctxt_stop(&st->actx); break; default: DPRINTF("Unknown VERB: 0x%x", verb); break; } return (res); } static const struct hda_codec_class hda_codec = { .name = "hda_codec", .init = hda_codec_init, .reset = hda_codec_reset, .command = hda_codec_command, .notify = hda_codec_notify, }; HDA_EMUL_SET(hda_codec); /* * HDA Audio Context module function definitions */ static void * hda_audio_ctxt_thr(void *arg) { struct hda_audio_ctxt *actx = arg; DPRINTF("Start Thread: %s", actx->name); pthread_mutex_lock(&actx->mtx); while (1) { while (!actx->run) pthread_cond_wait(&actx->cond, &actx->mtx); actx->do_transfer(actx->priv); } pthread_mutex_unlock(&actx->mtx); pthread_exit(NULL); return (NULL); } static int hda_audio_ctxt_init(struct hda_audio_ctxt *actx, const char *tname, transfer_func_t do_transfer, setup_func_t do_setup, void *priv) { int err; assert(actx); assert(tname); assert(do_transfer); assert(do_setup); assert(priv); memset(actx, 0, sizeof(*actx)); actx->run = 0; actx->do_transfer = do_transfer; actx->do_setup = do_setup; actx->priv = priv; if (strlen(tname) < sizeof(actx->name)) memcpy(actx->name, tname, strlen(tname) + 1); else strcpy(actx->name, "unknown"); err = pthread_mutex_init(&actx->mtx, NULL); assert(!err); err = pthread_cond_init(&actx->cond, NULL); assert(!err); err = pthread_create(&actx->tid, NULL, hda_audio_ctxt_thr, actx); assert(!err); pthread_set_name_np(actx->tid, tname); actx->started = 1; return (0); } static int hda_audio_ctxt_start(struct hda_audio_ctxt *actx) { int err = 0; assert(actx); assert(actx->started); /* The stream is supposed to be stopped */ if (actx->run) return (-1); pthread_mutex_lock(&actx->mtx); err = (* actx->do_setup)(actx->priv); if (!err) { actx->run = 1; pthread_cond_signal(&actx->cond); } pthread_mutex_unlock(&actx->mtx); return (err); } static int hda_audio_ctxt_stop(struct hda_audio_ctxt *actx) { actx->run = 0; return (0); } /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2006 Stephane E. Potvin * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef _HDA_REG_H_ #define _HDA_REG_H_ /**************************************************************************** * HDA Device Verbs ****************************************************************************/ /* HDA Command */ #define HDA_CMD_VERB_MASK 0x000fffff #define HDA_CMD_VERB_SHIFT 0 #define HDA_CMD_NID_MASK 0x0ff00000 #define HDA_CMD_NID_SHIFT 20 #define HDA_CMD_CAD_MASK 0xf0000000 #define HDA_CMD_CAD_SHIFT 28 #define HDA_CMD_VERB_4BIT_SHIFT 16 #define HDA_CMD_VERB_12BIT_SHIFT 8 #define HDA_CMD_VERB_4BIT(verb, payload) \ (((verb) << HDA_CMD_VERB_4BIT_SHIFT) | (payload)) #define HDA_CMD_4BIT(cad, nid, verb, payload) \ (((cad) << HDA_CMD_CAD_SHIFT) | \ ((nid) << HDA_CMD_NID_SHIFT) | \ (HDA_CMD_VERB_4BIT((verb), (payload)))) #define HDA_CMD_VERB_12BIT(verb, payload) \ (((verb) << HDA_CMD_VERB_12BIT_SHIFT) | (payload)) #define HDA_CMD_12BIT(cad, nid, verb, payload) \ (((cad) << HDA_CMD_CAD_SHIFT) | \ ((nid) << HDA_CMD_NID_SHIFT) | \ (HDA_CMD_VERB_12BIT((verb), (payload)))) /* Get Parameter */ #define HDA_CMD_VERB_GET_PARAMETER 0xf00 #define HDA_CMD_GET_PARAMETER(cad, nid, payload) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_GET_PARAMETER, (payload))) /* Connection Select Control */ #define HDA_CMD_VERB_GET_CONN_SELECT_CONTROL 0xf01 #define HDA_CMD_VERB_SET_CONN_SELECT_CONTROL 0x701 #define HDA_CMD_GET_CONN_SELECT_CONTROL(cad, nid) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_GET_CONN_SELECT_CONTROL, 0x0)) #define HDA_CMD_SET_CONNECTION_SELECT_CONTROL(cad, nid, payload) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_SET_CONN_SELECT_CONTROL, (payload))) /* Connection List Entry */ #define HDA_CMD_VERB_GET_CONN_LIST_ENTRY 0xf02 #define HDA_CMD_GET_CONN_LIST_ENTRY(cad, nid, payload) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_GET_CONN_LIST_ENTRY, (payload))) #define HDA_CMD_GET_CONN_LIST_ENTRY_SIZE_SHORT 1 #define HDA_CMD_GET_CONN_LIST_ENTRY_SIZE_LONG 2 /* Processing State */ #define HDA_CMD_VERB_GET_PROCESSING_STATE 0xf03 #define HDA_CMD_VERB_SET_PROCESSING_STATE 0x703 #define HDA_CMD_GET_PROCESSING_STATE(cad, nid) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_GET_PROCESSING_STATE, 0x0)) #define HDA_CMD_SET_PROCESSING_STATE(cad, nid, payload) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_SET_PROCESSING_STATE, (payload))) #define HDA_CMD_GET_PROCESSING_STATE_STATE_OFF 0x00 #define HDA_CMD_GET_PROCESSING_STATE_STATE_ON 0x01 #define HDA_CMD_GET_PROCESSING_STATE_STATE_BENIGN 0x02 /* Coefficient Index */ #define HDA_CMD_VERB_GET_COEFF_INDEX 0xd #define HDA_CMD_VERB_SET_COEFF_INDEX 0x5 #define HDA_CMD_GET_COEFF_INDEX(cad, nid) \ (HDA_CMD_4BIT((cad), (nid), \ HDA_CMD_VERB_GET_COEFF_INDEX, 0x0)) #define HDA_CMD_SET_COEFF_INDEX(cad, nid, payload) \ (HDA_CMD_4BIT((cad), (nid), \ HDA_CMD_VERB_SET_COEFF_INDEX, (payload))) /* Processing Coefficient */ #define HDA_CMD_VERB_GET_PROCESSING_COEFF 0xc #define HDA_CMD_VERB_SET_PROCESSING_COEFF 0x4 #define HDA_CMD_GET_PROCESSING_COEFF(cad, nid) \ (HDA_CMD_4BIT((cad), (nid), \ HDA_CMD_VERB_GET_PROCESSING_COEFF, 0x0)) #define HDA_CMD_SET_PROCESSING_COEFF(cad, nid, payload) \ (HDA_CMD_4BIT((cad), (nid), \ HDA_CMD_VERB_SET_PROCESSING_COEFF, (payload))) /* Amplifier Gain/Mute */ #define HDA_CMD_VERB_GET_AMP_GAIN_MUTE 0xb #define HDA_CMD_VERB_SET_AMP_GAIN_MUTE 0x3 #define HDA_CMD_GET_AMP_GAIN_MUTE(cad, nid, payload) \ (HDA_CMD_4BIT((cad), (nid), \ HDA_CMD_VERB_GET_AMP_GAIN_MUTE, (payload))) #define HDA_CMD_SET_AMP_GAIN_MUTE(cad, nid, payload) \ (HDA_CMD_4BIT((cad), (nid), \ HDA_CMD_VERB_SET_AMP_GAIN_MUTE, (payload))) #define HDA_CMD_GET_AMP_GAIN_MUTE_INPUT 0x0000 #define HDA_CMD_GET_AMP_GAIN_MUTE_OUTPUT 0x8000 #define HDA_CMD_GET_AMP_GAIN_MUTE_RIGHT 0x0000 #define HDA_CMD_GET_AMP_GAIN_MUTE_LEFT 0x2000 #define HDA_CMD_GET_AMP_GAIN_MUTE_MUTE_MASK 0x00000008 #define HDA_CMD_GET_AMP_GAIN_MUTE_MUTE_SHIFT 7 #define HDA_CMD_GET_AMP_GAIN_MUTE_GAIN_MASK 0x00000007 #define HDA_CMD_GET_AMP_GAIN_MUTE_GAIN_SHIFT 0 #define HDA_CMD_GET_AMP_GAIN_MUTE_MUTE(rsp) \ (((rsp) & HDA_CMD_GET_AMP_GAIN_MUTE_MUTE_MASK) >> \ HDA_CMD_GET_AMP_GAIN_MUTE_MUTE_SHIFT) #define HDA_CMD_GET_AMP_GAIN_MUTE_GAIN(rsp) \ (((rsp) & HDA_CMD_GET_AMP_GAIN_MUTE_GAIN_MASK) >> \ HDA_CMD_GET_AMP_GAIN_MUTE_GAIN_SHIFT) #define HDA_CMD_SET_AMP_GAIN_MUTE_OUTPUT 0x8000 #define HDA_CMD_SET_AMP_GAIN_MUTE_INPUT 0x4000 #define HDA_CMD_SET_AMP_GAIN_MUTE_LEFT 0x2000 #define HDA_CMD_SET_AMP_GAIN_MUTE_RIGHT 0x1000 #define HDA_CMD_SET_AMP_GAIN_MUTE_INDEX_MASK 0x0f00 #define HDA_CMD_SET_AMP_GAIN_MUTE_INDEX_SHIFT 8 #define HDA_CMD_SET_AMP_GAIN_MUTE_MUTE 0x0080 #define HDA_CMD_SET_AMP_GAIN_MUTE_GAIN_MASK 0x0007 #define HDA_CMD_SET_AMP_GAIN_MUTE_GAIN_SHIFT 0 #define HDA_CMD_SET_AMP_GAIN_MUTE_INDEX(index) \ (((index) << HDA_CMD_SET_AMP_GAIN_MUTE_INDEX_SHIFT) & \ HDA_CMD_SET_AMP_GAIN_MUTE_INDEX_MASK) #define HDA_CMD_SET_AMP_GAIN_MUTE_GAIN(index) \ (((index) << HDA_CMD_SET_AMP_GAIN_MUTE_GAIN_SHIFT) & \ HDA_CMD_SET_AMP_GAIN_MUTE_GAIN_MASK) /* Converter format */ #define HDA_CMD_VERB_GET_CONV_FMT 0xa #define HDA_CMD_VERB_SET_CONV_FMT 0x2 #define HDA_CMD_GET_CONV_FMT(cad, nid) \ (HDA_CMD_4BIT((cad), (nid), \ HDA_CMD_VERB_GET_CONV_FMT, 0x0)) #define HDA_CMD_SET_CONV_FMT(cad, nid, payload) \ (HDA_CMD_4BIT((cad), (nid), \ HDA_CMD_VERB_SET_CONV_FMT, (payload))) /* Digital Converter Control */ #define HDA_CMD_VERB_GET_DIGITAL_CONV_FMT1 0xf0d #define HDA_CMD_VERB_GET_DIGITAL_CONV_FMT2 0xf0e #define HDA_CMD_VERB_SET_DIGITAL_CONV_FMT1 0x70d #define HDA_CMD_VERB_SET_DIGITAL_CONV_FMT2 0x70e #define HDA_CMD_GET_DIGITAL_CONV_FMT(cad, nid) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_GET_DIGITAL_CONV_FMT1, 0x0)) #define HDA_CMD_SET_DIGITAL_CONV_FMT1(cad, nid, payload) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_SET_DIGITAL_CONV_FMT1, (payload))) #define HDA_CMD_SET_DIGITAL_CONV_FMT2(cad, nid, payload) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_SET_DIGITAL_CONV_FMT2, (payload))) #define HDA_CMD_GET_DIGITAL_CONV_FMT_CC_MASK 0x7f00 #define HDA_CMD_GET_DIGITAL_CONV_FMT_CC_SHIFT 8 #define HDA_CMD_GET_DIGITAL_CONV_FMT_L_MASK 0x0080 #define HDA_CMD_GET_DIGITAL_CONV_FMT_L_SHIFT 7 #define HDA_CMD_GET_DIGITAL_CONV_FMT_PRO_MASK 0x0040 #define HDA_CMD_GET_DIGITAL_CONV_FMT_PRO_SHIFT 6 #define HDA_CMD_GET_DIGITAL_CONV_FMT_NAUDIO_MASK 0x0020 #define HDA_CMD_GET_DIGITAL_CONV_FMT_NAUDIO_SHIFT 5 #define HDA_CMD_GET_DIGITAL_CONV_FMT_COPY_MASK 0x0010 #define HDA_CMD_GET_DIGITAL_CONV_FMT_COPY_SHIFT 4 #define HDA_CMD_GET_DIGITAL_CONV_FMT_PRE_MASK 0x0008 #define HDA_CMD_GET_DIGITAL_CONV_FMT_PRE_SHIFT 3 #define HDA_CMD_GET_DIGITAL_CONV_FMT_VCFG_MASK 0x0004 #define HDA_CMD_GET_DIGITAL_CONV_FMT_VCFG_SHIFT 2 #define HDA_CMD_GET_DIGITAL_CONV_FMT_V_MASK 0x0002 #define HDA_CMD_GET_DIGITAL_CONV_FMT_V_SHIFT 1 #define HDA_CMD_GET_DIGITAL_CONV_FMT_DIGEN_MASK 0x0001 #define HDA_CMD_GET_DIGITAL_CONV_FMT_DIGEN_SHIFT 0 #define HDA_CMD_GET_DIGITAL_CONV_FMT_CC(rsp) \ (((rsp) & HDA_CMD_GET_DIGITAL_CONV_FMT_CC_MASK) >> \ HDA_CMD_GET_DIGITAL_CONV_FMT_CC_SHIFT) #define HDA_CMD_GET_DIGITAL_CONV_FMT_L(rsp) \ (((rsp) & HDA_CMD_GET_DIGITAL_CONV_FMT_L_MASK) >> \ HDA_CMD_GET_DIGITAL_CONV_FMT_L_SHIFT) #define HDA_CMD_GET_DIGITAL_CONV_FMT_PRO(rsp) \ (((rsp) & HDA_CMD_GET_DIGITAL_CONV_FMT_PRO_MASK) >> \ HDA_CMD_GET_DIGITAL_CONV_FMT_PRO_SHIFT) #define HDA_CMD_GET_DIGITAL_CONV_FMT_NAUDIO(rsp) \ (((rsp) & HDA_CMD_GET_DIGITAL_CONV_FMT_NAUDIO_MASK) >> \ HDA_CMD_GET_DIGITAL_CONV_FMT_NAUDIO_SHIFT) #define HDA_CMD_GET_DIGITAL_CONV_FMT_COPY(rsp) \ (((rsp) & HDA_CMD_GET_DIGITAL_CONV_FMT_COPY_MASK) >> \ HDA_CMD_GET_DIGITAL_CONV_FMT_COPY_SHIFT) #define HDA_CMD_GET_DIGITAL_CONV_FMT_PRE(rsp) \ (((rsp) & HDA_CMD_GET_DIGITAL_CONV_FMT_PRE_MASK) >> \ HDA_CMD_GET_DIGITAL_CONV_FMT_PRE_SHIFT) #define HDA_CMD_GET_DIGITAL_CONV_FMT_VCFG(rsp) \ (((rsp) & HDA_CMD_GET_DIGITAL_CONV_FMT_VCFG_MASK) >> \ HDA_CMD_GET_DIGITAL_CONV_FMT_VCFG_SHIFT) #define HDA_CMD_GET_DIGITAL_CONV_FMT_V(rsp) \ (((rsp) & HDA_CMD_GET_DIGITAL_CONV_FMT_V_MASK) >> \ HDA_CMD_GET_DIGITAL_CONV_FMT_V_SHIFT) #define HDA_CMD_GET_DIGITAL_CONV_FMT_DIGEN(rsp) \ (((rsp) & HDA_CMD_GET_DIGITAL_CONV_FMT_DIGEN_MASK) >> \ HDA_CMD_GET_DIGITAL_CONV_FMT_DIGEN_SHIFT) #define HDA_CMD_SET_DIGITAL_CONV_FMT1_L 0x80 #define HDA_CMD_SET_DIGITAL_CONV_FMT1_PRO 0x40 #define HDA_CMD_SET_DIGITAL_CONV_FMT1_NAUDIO 0x20 #define HDA_CMD_SET_DIGITAL_CONV_FMT1_COPY 0x10 #define HDA_CMD_SET_DIGITAL_CONV_FMT1_PRE 0x08 #define HDA_CMD_SET_DIGITAL_CONV_FMT1_VCFG 0x04 #define HDA_CMD_SET_DIGITAL_CONV_FMT1_V 0x02 #define HDA_CMD_SET_DIGITAL_CONV_FMT1_DIGEN 0x01 /* Power State */ #define HDA_CMD_VERB_GET_POWER_STATE 0xf05 #define HDA_CMD_VERB_SET_POWER_STATE 0x705 #define HDA_CMD_GET_POWER_STATE(cad, nid) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_GET_POWER_STATE, 0x0)) #define HDA_CMD_SET_POWER_STATE(cad, nid, payload) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_SET_POWER_STATE, (payload))) #define HDA_CMD_POWER_STATE_D0 0x00 #define HDA_CMD_POWER_STATE_D1 0x01 #define HDA_CMD_POWER_STATE_D2 0x02 #define HDA_CMD_POWER_STATE_D3 0x03 #define HDA_CMD_POWER_STATE_ACT_MASK 0x000000f0 #define HDA_CMD_POWER_STATE_ACT_SHIFT 4 #define HDA_CMD_POWER_STATE_SET_MASK 0x0000000f #define HDA_CMD_POWER_STATE_SET_SHIFT 0 #define HDA_CMD_GET_POWER_STATE_ACT(rsp) \ (((rsp) & HDA_CMD_POWER_STATE_ACT_MASK) >> \ HDA_CMD_POWER_STATE_ACT_SHIFT) #define HDA_CMD_GET_POWER_STATE_SET(rsp) \ (((rsp) & HDA_CMD_POWER_STATE_SET_MASK) >> \ HDA_CMD_POWER_STATE_SET_SHIFT) #define HDA_CMD_SET_POWER_STATE_ACT(ps) \ (((ps) << HDA_CMD_POWER_STATE_ACT_SHIFT) & \ HDA_CMD_POWER_STATE_ACT_MASK) #define HDA_CMD_SET_POWER_STATE_SET(ps) \ (((ps) << HDA_CMD_POWER_STATE_SET_SHIFT) & \ HDA_CMD_POWER_STATE_ACT_MASK) /* Converter Stream, Channel */ #define HDA_CMD_VERB_GET_CONV_STREAM_CHAN 0xf06 #define HDA_CMD_VERB_SET_CONV_STREAM_CHAN 0x706 #define HDA_CMD_GET_CONV_STREAM_CHAN(cad, nid) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_GET_CONV_STREAM_CHAN, 0x0)) #define HDA_CMD_SET_CONV_STREAM_CHAN(cad, nid, payload) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_SET_CONV_STREAM_CHAN, (payload))) #define HDA_CMD_CONV_STREAM_CHAN_STREAM_MASK 0x000000f0 #define HDA_CMD_CONV_STREAM_CHAN_STREAM_SHIFT 4 #define HDA_CMD_CONV_STREAM_CHAN_CHAN_MASK 0x0000000f #define HDA_CMD_CONV_STREAM_CHAN_CHAN_SHIFT 0 #define HDA_CMD_GET_CONV_STREAM_CHAN_STREAM(rsp) \ (((rsp) & HDA_CMD_CONV_STREAM_CHAN_STREAM_MASK) >> \ HDA_CMD_CONV_STREAM_CHAN_STREAM_SHIFT) #define HDA_CMD_GET_CONV_STREAM_CHAN_CHAN(rsp) \ (((rsp) & HDA_CMD_CONV_STREAM_CHAN_CHAN_MASK) >> \ HDA_CMD_CONV_STREAM_CHAN_CHAN_SHIFT) #define HDA_CMD_SET_CONV_STREAM_CHAN_STREAM(param) \ (((param) << HDA_CMD_CONV_STREAM_CHAN_STREAM_SHIFT) & \ HDA_CMD_CONV_STREAM_CHAN_STREAM_MASK) #define HDA_CMD_SET_CONV_STREAM_CHAN_CHAN(param) \ (((param) << HDA_CMD_CONV_STREAM_CHAN_CHAN_SHIFT) & \ HDA_CMD_CONV_STREAM_CHAN_CHAN_MASK) /* Input Converter SDI Select */ #define HDA_CMD_VERB_GET_INPUT_CONVERTER_SDI_SELECT 0xf04 #define HDA_CMD_VERB_SET_INPUT_CONVERTER_SDI_SELECT 0x704 #define HDA_CMD_GET_INPUT_CONVERTER_SDI_SELECT(cad, nid) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_GET_INPUT_CONVERTER_SDI_SELECT, 0x0)) #define HDA_CMD_SET_INPUT_CONVERTER_SDI_SELECT(cad, nid, payload) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_SET_INPUT_CONVERTER_SDI_SELECT, (payload))) /* Pin Widget Control */ #define HDA_CMD_VERB_GET_PIN_WIDGET_CTRL 0xf07 #define HDA_CMD_VERB_SET_PIN_WIDGET_CTRL 0x707 #define HDA_CMD_GET_PIN_WIDGET_CTRL(cad, nid) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_GET_PIN_WIDGET_CTRL, 0x0)) #define HDA_CMD_SET_PIN_WIDGET_CTRL(cad, nid, payload) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_SET_PIN_WIDGET_CTRL, (payload))) #define HDA_CMD_GET_PIN_WIDGET_CTRL_HPHN_ENABLE_MASK 0x00000080 #define HDA_CMD_GET_PIN_WIDGET_CTRL_HPHN_ENABLE_SHIFT 7 #define HDA_CMD_GET_PIN_WIDGET_CTRL_OUT_ENABLE_MASK 0x00000040 #define HDA_CMD_GET_PIN_WIDGET_CTRL_OUT_ENABLE_SHIFT 6 #define HDA_CMD_GET_PIN_WIDGET_CTRL_IN_ENABLE_MASK 0x00000020 #define HDA_CMD_GET_PIN_WIDGET_CTRL_IN_ENABLE_SHIFT 5 #define HDA_CMD_GET_PIN_WIDGET_CTRL_VREF_ENABLE_MASK 0x00000007 #define HDA_CMD_GET_PIN_WIDGET_CTRL_VREF_ENABLE_SHIFT 0 #define HDA_CMD_GET_PIN_WIDGET_CTRL_HPHN_ENABLE(rsp) \ (((rsp) & HDA_CMD_GET_PIN_WIDGET_CTRL_HPHN_ENABLE_MASK) >> \ HDA_CMD_GET_PIN_WIDGET_CTRL_HPHN_ENABLE_SHIFT) #define HDA_CMD_GET_PIN_WIDGET_CTRL_OUT_ENABLE(rsp) \ (((rsp) & HDA_CMD_GET_PIN_WIDGET_CTRL_OUT_ENABLE_MASK) >> \ HDA_GET_CMD_PIN_WIDGET_CTRL_OUT_ENABLE_SHIFT) #define HDA_CMD_GET_PIN_WIDGET_CTRL_IN_ENABLE(rsp) \ (((rsp) & HDA_CMD_GET_PIN_WIDGET_CTRL_IN_ENABLE_MASK) >> \ HDA_CMD_GET_PIN_WIDGET_CTRL_IN_ENABLE_SHIFT) #define HDA_CMD_GET_PIN_WIDGET_CTRL_VREF_ENABLE(rsp) \ (((rsp) & HDA_CMD_GET_PIN_WIDGET_CTRL_VREF_ENABLE_MASK) >> \ HDA_CMD_GET_PIN_WIDGET_CTRL_VREF_ENABLE_SHIFT) #define HDA_CMD_SET_PIN_WIDGET_CTRL_HPHN_ENABLE 0x80 #define HDA_CMD_SET_PIN_WIDGET_CTRL_OUT_ENABLE 0x40 #define HDA_CMD_SET_PIN_WIDGET_CTRL_IN_ENABLE 0x20 #define HDA_CMD_SET_PIN_WIDGET_CTRL_VREF_ENABLE_MASK 0x07 #define HDA_CMD_SET_PIN_WIDGET_CTRL_VREF_ENABLE_SHIFT 0 #define HDA_CMD_SET_PIN_WIDGET_CTRL_VREF_ENABLE(param) \ (((param) << HDA_CMD_SET_PIN_WIDGET_CTRL_VREF_ENABLE_SHIFT) & \ HDA_CMD_SET_PIN_WIDGET_CTRL_VREF_ENABLE_MASK) #define HDA_CMD_PIN_WIDGET_CTRL_VREF_ENABLE_HIZ 0 #define HDA_CMD_PIN_WIDGET_CTRL_VREF_ENABLE_50 1 #define HDA_CMD_PIN_WIDGET_CTRL_VREF_ENABLE_GROUND 2 #define HDA_CMD_PIN_WIDGET_CTRL_VREF_ENABLE_80 4 #define HDA_CMD_PIN_WIDGET_CTRL_VREF_ENABLE_100 5 /* Unsolicited Response */ #define HDA_CMD_VERB_GET_UNSOLICITED_RESPONSE 0xf08 #define HDA_CMD_VERB_SET_UNSOLICITED_RESPONSE 0x708 #define HDA_CMD_GET_UNSOLICITED_RESPONSE(cad, nid) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_GET_UNSOLICITED_RESPONSE, 0x0)) #define HDA_CMD_SET_UNSOLICITED_RESPONSE(cad, nid, payload) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_SET_UNSOLICITED_RESPONSE, (payload))) #define HDA_CMD_GET_UNSOLICITED_RESPONSE_ENABLE_MASK 0x00000080 #define HDA_CMD_GET_UNSOLICITED_RESPONSE_ENABLE_SHIFT 7 #define HDA_CMD_GET_UNSOLICITED_RESPONSE_TAG_MASK 0x0000001f #define HDA_CMD_GET_UNSOLICITED_RESPONSE_TAG_SHIFT 0 #define HDA_CMD_GET_UNSOLICITED_RESPONSE_ENABLE(rsp) \ (((rsp) & HDA_CMD_GET_UNSOLICITED_RESPONSE_ENABLE_MASK) >> \ HDA_CMD_GET_UNSOLICITED_RESPONSE_ENABLE_SHIFT) #define HDA_CMD_GET_UNSOLICITED_RESPONSE_TAG(rsp) \ (((rsp) & HDA_CMD_GET_UNSOLICITED_RESPONSE_TAG_MASK) >> \ HDA_CMD_GET_UNSOLICITED_RESPONSE_TAG_SHIFT) #define HDA_CMD_SET_UNSOLICITED_RESPONSE_ENABLE 0x80 #define HDA_CMD_SET_UNSOLICITED_RESPONSE_TAG_MASK 0x3f #define HDA_CMD_SET_UNSOLICITED_RESPONSE_TAG_SHIFT 0 #define HDA_CMD_SET_UNSOLICITED_RESPONSE_TAG(param) \ (((param) << HDA_CMD_SET_UNSOLICITED_RESPONSE_TAG_SHIFT) & \ HDA_CMD_SET_UNSOLICITED_RESPONSE_TAG_MASK) /* Pin Sense */ #define HDA_CMD_VERB_GET_PIN_SENSE 0xf09 #define HDA_CMD_VERB_SET_PIN_SENSE 0x709 #define HDA_CMD_GET_PIN_SENSE(cad, nid) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_GET_PIN_SENSE, 0x0)) #define HDA_CMD_SET_PIN_SENSE(cad, nid, payload) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_SET_PIN_SENSE, (payload))) #define HDA_CMD_GET_PIN_SENSE_PRESENCE_DETECT 0x80000000 #define HDA_CMD_GET_PIN_SENSE_ELD_VALID 0x40000000 #define HDA_CMD_GET_PIN_SENSE_IMP_SENSE_MASK 0x7fffffff #define HDA_CMD_GET_PIN_SENSE_IMP_SENSE_SHIFT 0 #define HDA_CMD_GET_PIN_SENSE_IMP_SENSE(rsp) \ (((rsp) & HDA_CMD_GET_PIN_SENSE_IMP_SENSE_MASK) >> \ HDA_CMD_GET_PIN_SENSE_IMP_SENSE_SHIFT) #define HDA_CMD_GET_PIN_SENSE_IMP_SENSE_INVALID 0x7fffffff #define HDA_CMD_SET_PIN_SENSE_LEFT_CHANNEL 0x00 #define HDA_CMD_SET_PIN_SENSE_RIGHT_CHANNEL 0x01 /* EAPD/BTL Enable */ #define HDA_CMD_VERB_GET_EAPD_BTL_ENABLE 0xf0c #define HDA_CMD_VERB_SET_EAPD_BTL_ENABLE 0x70c #define HDA_CMD_GET_EAPD_BTL_ENABLE(cad, nid) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_GET_EAPD_BTL_ENABLE, 0x0)) #define HDA_CMD_SET_EAPD_BTL_ENABLE(cad, nid, payload) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_SET_EAPD_BTL_ENABLE, (payload))) #define HDA_CMD_GET_EAPD_BTL_ENABLE_LR_SWAP_MASK 0x00000004 #define HDA_CMD_GET_EAPD_BTL_ENABLE_LR_SWAP_SHIFT 2 #define HDA_CMD_GET_EAPD_BTL_ENABLE_EAPD_MASK 0x00000002 #define HDA_CMD_GET_EAPD_BTL_ENABLE_EAPD_SHIFT 1 #define HDA_CMD_GET_EAPD_BTL_ENABLE_BTL_MASK 0x00000001 #define HDA_CMD_GET_EAPD_BTL_ENABLE_BTL_SHIFT 0 #define HDA_CMD_GET_EAPD_BTL_ENABLE_LR_SWAP(rsp) \ (((rsp) & HDA_CMD_GET_EAPD_BTL_ENABLE_LR_SWAP_MASK) >> \ HDA_CMD_GET_EAPD_BTL_ENABLE_LR_SWAP_SHIFT) #define HDA_CMD_GET_EAPD_BTL_ENABLE_EAPD(rsp) \ (((rsp) & HDA_CMD_GET_EAPD_BTL_ENABLE_EAPD_MASK) >> \ HDA_CMD_GET_EAPD_BTL_ENABLE_EAPD_SHIFT) #define HDA_CMD_GET_EAPD_BTL_ENABLE_BTL(rsp) \ (((rsp) & HDA_CMD_GET_EAPD_BTL_ENABLE_BTL_MASK) >> \ HDA_CMD_GET_EAPD_BTL_ENABLE_BTL_SHIFT) #define HDA_CMD_SET_EAPD_BTL_ENABLE_LR_SWAP 0x04 #define HDA_CMD_SET_EAPD_BTL_ENABLE_EAPD 0x02 #define HDA_CMD_SET_EAPD_BTL_ENABLE_BTL 0x01 /* GPI Data */ #define HDA_CMD_VERB_GET_GPI_DATA 0xf10 #define HDA_CMD_VERB_SET_GPI_DATA 0x710 #define HDA_CMD_GET_GPI_DATA(cad, nid) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_GET_GPI_DATA, 0x0)) #define HDA_CMD_SET_GPI_DATA(cad, nid) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_SET_GPI_DATA, (payload))) /* GPI Wake Enable Mask */ #define HDA_CMD_VERB_GET_GPI_WAKE_ENABLE_MASK 0xf11 #define HDA_CMD_VERB_SET_GPI_WAKE_ENABLE_MASK 0x711 #define HDA_CMD_GET_GPI_WAKE_ENABLE_MASK(cad, nid) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_GET_GPI_WAKE_ENABLE_MASK, 0x0)) #define HDA_CMD_SET_GPI_WAKE_ENABLE_MASK(cad, nid, payload) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_SET_GPI_WAKE_ENABLE_MASK, (payload))) /* GPI Unsolicited Enable Mask */ #define HDA_CMD_VERB_GET_GPI_UNSOLICITED_ENABLE_MASK 0xf12 #define HDA_CMD_VERB_SET_GPI_UNSOLICITED_ENABLE_MASK 0x712 #define HDA_CMD_GET_GPI_UNSOLICITED_ENABLE_MASK(cad, nid) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_GET_GPI_UNSOLICITED_ENABLE_MASK, 0x0)) #define HDA_CMD_SET_GPI_UNSOLICITED_ENABLE_MASK(cad, nid, payload) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_SET_GPI_UNSOLICITED_ENABLE_MASK, (payload))) /* GPI Sticky Mask */ #define HDA_CMD_VERB_GET_GPI_STICKY_MASK 0xf13 #define HDA_CMD_VERB_SET_GPI_STICKY_MASK 0x713 #define HDA_CMD_GET_GPI_STICKY_MASK(cad, nid) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_GET_GPI_STICKY_MASK, 0x0)) #define HDA_CMD_SET_GPI_STICKY_MASK(cad, nid, payload) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_SET_GPI_STICKY_MASK, (payload))) /* GPO Data */ #define HDA_CMD_VERB_GET_GPO_DATA 0xf14 #define HDA_CMD_VERB_SET_GPO_DATA 0x714 #define HDA_CMD_GET_GPO_DATA(cad, nid) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_GET_GPO_DATA, 0x0)) #define HDA_CMD_SET_GPO_DATA(cad, nid, payload) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_SET_GPO_DATA, (payload))) /* GPIO Data */ #define HDA_CMD_VERB_GET_GPIO_DATA 0xf15 #define HDA_CMD_VERB_SET_GPIO_DATA 0x715 #define HDA_CMD_GET_GPIO_DATA(cad, nid) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_GET_GPIO_DATA, 0x0)) #define HDA_CMD_SET_GPIO_DATA(cad, nid, payload) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_SET_GPIO_DATA, (payload))) /* GPIO Enable Mask */ #define HDA_CMD_VERB_GET_GPIO_ENABLE_MASK 0xf16 #define HDA_CMD_VERB_SET_GPIO_ENABLE_MASK 0x716 #define HDA_CMD_GET_GPIO_ENABLE_MASK(cad, nid) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_GET_GPIO_ENABLE_MASK, 0x0)) #define HDA_CMD_SET_GPIO_ENABLE_MASK(cad, nid, payload) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_SET_GPIO_ENABLE_MASK, (payload))) /* GPIO Direction */ #define HDA_CMD_VERB_GET_GPIO_DIRECTION 0xf17 #define HDA_CMD_VERB_SET_GPIO_DIRECTION 0x717 #define HDA_CMD_GET_GPIO_DIRECTION(cad, nid) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_GET_GPIO_DIRECTION, 0x0)) #define HDA_CMD_SET_GPIO_DIRECTION(cad, nid, payload) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_SET_GPIO_DIRECTION, (payload))) /* GPIO Wake Enable Mask */ #define HDA_CMD_VERB_GET_GPIO_WAKE_ENABLE_MASK 0xf18 #define HDA_CMD_VERB_SET_GPIO_WAKE_ENABLE_MASK 0x718 #define HDA_CMD_GET_GPIO_WAKE_ENABLE_MASK(cad, nid) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_GET_GPIO_WAKE_ENABLE_MASK, 0x0)) #define HDA_CMD_SET_GPIO_WAKE_ENABLE_MASK(cad, nid, payload) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_SET_GPIO_WAKE_ENABLE_MASK, (payload))) /* GPIO Unsolicited Enable Mask */ #define HDA_CMD_VERB_GET_GPIO_UNSOLICITED_ENABLE_MASK 0xf19 #define HDA_CMD_VERB_SET_GPIO_UNSOLICITED_ENABLE_MASK 0x719 #define HDA_CMD_GET_GPIO_UNSOLICITED_ENABLE_MASK(cad, nid) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_GET_GPIO_UNSOLICITED_ENABLE_MASK, 0x0)) #define HDA_CMD_SET_GPIO_UNSOLICITED_ENABLE_MASK(cad, nid, payload) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_SET_GPIO_UNSOLICITED_ENABLE_MASK, (payload))) /* GPIO_STICKY_MASK */ #define HDA_CMD_VERB_GET_GPIO_STICKY_MASK 0xf1a #define HDA_CMD_VERB_SET_GPIO_STICKY_MASK 0x71a #define HDA_CMD_GET_GPIO_STICKY_MASK(cad, nid) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_GET_GPIO_STICKY_MASK, 0x0)) #define HDA_CMD_SET_GPIO_STICKY_MASK(cad, nid, payload) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_SET_GPIO_STICKY_MASK, (payload))) /* Beep Generation */ #define HDA_CMD_VERB_GET_BEEP_GENERATION 0xf0a #define HDA_CMD_VERB_SET_BEEP_GENERATION 0x70a #define HDA_CMD_GET_BEEP_GENERATION(cad, nid) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_GET_BEEP_GENERATION, 0x0)) #define HDA_CMD_SET_BEEP_GENERATION(cad, nid, payload) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_SET_BEEP_GENERATION, (payload))) /* Volume Knob */ #define HDA_CMD_VERB_GET_VOLUME_KNOB 0xf0f #define HDA_CMD_VERB_SET_VOLUME_KNOB 0x70f #define HDA_CMD_GET_VOLUME_KNOB(cad, nid) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_GET_VOLUME_KNOB, 0x0)) #define HDA_CMD_SET_VOLUME_KNOB(cad, nid, payload) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_SET_VOLUME_KNOB, (payload))) /* Subsystem ID */ #define HDA_CMD_VERB_GET_SUBSYSTEM_ID 0xf20 #define HDA_CMD_VERB_SET_SUSBYSTEM_ID1 0x720 #define HDA_CMD_VERB_SET_SUBSYSTEM_ID2 0x721 #define HDA_CMD_VERB_SET_SUBSYSTEM_ID3 0x722 #define HDA_CMD_VERB_SET_SUBSYSTEM_ID4 0x723 #define HDA_CMD_GET_SUBSYSTEM_ID(cad, nid) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_GET_SUBSYSTEM_ID, 0x0)) #define HDA_CMD_SET_SUBSYSTEM_ID1(cad, nid, payload) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_SET_SUSBYSTEM_ID1, (payload))) #define HDA_CMD_SET_SUBSYSTEM_ID2(cad, nid, payload) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_SET_SUSBYSTEM_ID2, (payload))) #define HDA_CMD_SET_SUBSYSTEM_ID3(cad, nid, payload) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_SET_SUSBYSTEM_ID3, (payload))) #define HDA_CMD_SET_SUBSYSTEM_ID4(cad, nid, payload) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_SET_SUSBYSTEM_ID4, (payload))) /* Configuration Default */ #define HDA_CMD_VERB_GET_CONFIGURATION_DEFAULT 0xf1c #define HDA_CMD_VERB_SET_CONFIGURATION_DEFAULT1 0x71c #define HDA_CMD_VERB_SET_CONFIGURATION_DEFAULT2 0x71d #define HDA_CMD_VERB_SET_CONFIGURATION_DEFAULT3 0x71e #define HDA_CMD_VERB_SET_CONFIGURATION_DEFAULT4 0x71f #define HDA_CMD_GET_CONFIGURATION_DEFAULT(cad, nid) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_GET_CONFIGURATION_DEFAULT, 0x0)) #define HDA_CMD_SET_CONFIGURATION_DEFAULT1(cad, nid, payload) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_SET_CONFIGURATION_DEFAULT1, (payload))) #define HDA_CMD_SET_CONFIGURATION_DEFAULT2(cad, nid, payload) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_SET_CONFIGURATION_DEFAULT2, (payload))) #define HDA_CMD_SET_CONFIGURATION_DEFAULT3(cad, nid, payload) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_SET_CONFIGURATION_DEFAULT3, (payload))) #define HDA_CMD_SET_CONFIGURATION_DEFAULT4(cad, nid, payload) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_SET_CONFIGURATION_DEFAULT4, (payload))) /* Stripe Control */ #define HDA_CMD_VERB_GET_STRIPE_CONTROL 0xf24 #define HDA_CMD_VERB_SET_STRIPE_CONTROL 0x724 #define HDA_CMD_GET_STRIPE_CONTROL(cad, nid) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_GET_STRIPE_CONTROL, 0x0)) #define HDA_CMD_SET_STRIPE_CONTROL(cad, nid, payload) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_SET_STRIPE_CONTROL, (payload))) /* Channel Count Control */ #define HDA_CMD_VERB_GET_CONV_CHAN_COUNT 0xf2d #define HDA_CMD_VERB_SET_CONV_CHAN_COUNT 0x72d #define HDA_CMD_GET_CONV_CHAN_COUNT(cad, nid) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_GET_CONV_CHAN_COUNT, 0x0)) #define HDA_CMD_SET_CONV_CHAN_COUNT(cad, nid, payload) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_SET_CONV_CHAN_COUNT, (payload))) #define HDA_CMD_VERB_GET_HDMI_DIP_SIZE 0xf2e #define HDA_CMD_GET_HDMI_DIP_SIZE(cad, nid, arg) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_GET_HDMI_DIP_SIZE, (arg))) #define HDA_CMD_VERB_GET_HDMI_ELDD 0xf2f #define HDA_CMD_GET_HDMI_ELDD(cad, nid, off) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_GET_HDMI_ELDD, (off))) #define HDA_CMD_VERB_GET_HDMI_DIP_INDEX 0xf30 #define HDA_CMD_VERB_SET_HDMI_DIP_INDEX 0x730 #define HDA_CMD_GET_HDMI_DIP_INDEX(cad, nid) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_GET_HDMI_DIP_INDEX, 0x0)) #define HDA_CMD_SET_HDMI_DIP_INDEX(cad, nid, payload) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_SET_HDMI_DIP_INDEX, (payload))) #define HDA_CMD_VERB_GET_HDMI_DIP_DATA 0xf31 #define HDA_CMD_VERB_SET_HDMI_DIP_DATA 0x731 #define HDA_CMD_GET_HDMI_DIP_DATA(cad, nid) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_GET_HDMI_DIP_DATA, 0x0)) #define HDA_CMD_SET_HDMI_DIP_DATA(cad, nid, payload) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_SET_HDMI_DIP_DATA, (payload))) #define HDA_CMD_VERB_GET_HDMI_DIP_XMIT 0xf32 #define HDA_CMD_VERB_SET_HDMI_DIP_XMIT 0x732 #define HDA_CMD_GET_HDMI_DIP_XMIT(cad, nid) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_GET_HDMI_DIP_XMIT, 0x0)) #define HDA_CMD_SET_HDMI_DIP_XMIT(cad, nid, payload) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_SET_HDMI_DIP_XMIT, (payload))) #define HDA_CMD_VERB_GET_HDMI_CP_CTRL 0xf33 #define HDA_CMD_VERB_SET_HDMI_CP_CTRL 0x733 #define HDA_CMD_VERB_GET_HDMI_CHAN_SLOT 0xf34 #define HDA_CMD_VERB_SET_HDMI_CHAN_SLOT 0x734 #define HDA_CMD_GET_HDMI_CHAN_SLOT(cad, nid) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_GET_HDMI_CHAN_SLOT, 0x0)) #define HDA_CMD_SET_HDMI_CHAN_SLOT(cad, nid, payload) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_SET_HDMI_CHAN_SLOT, (payload))) #define HDA_HDMI_CODING_TYPE_REF_STREAM_HEADER 0 #define HDA_HDMI_CODING_TYPE_LPCM 1 #define HDA_HDMI_CODING_TYPE_AC3 2 #define HDA_HDMI_CODING_TYPE_MPEG1 3 #define HDA_HDMI_CODING_TYPE_MP3 4 #define HDA_HDMI_CODING_TYPE_MPEG2 5 #define HDA_HDMI_CODING_TYPE_AACLC 6 #define HDA_HDMI_CODING_TYPE_DTS 7 #define HDA_HDMI_CODING_TYPE_ATRAC 8 #define HDA_HDMI_CODING_TYPE_SACD 9 #define HDA_HDMI_CODING_TYPE_EAC3 10 #define HDA_HDMI_CODING_TYPE_DTS_HD 11 #define HDA_HDMI_CODING_TYPE_MLP 12 #define HDA_HDMI_CODING_TYPE_DST 13 #define HDA_HDMI_CODING_TYPE_WMAPRO 14 #define HDA_HDMI_CODING_TYPE_REF_CTX 15 /* Function Reset */ #define HDA_CMD_VERB_FUNCTION_RESET 0x7ff #define HDA_CMD_FUNCTION_RESET(cad, nid) \ (HDA_CMD_12BIT((cad), (nid), \ HDA_CMD_VERB_FUNCTION_RESET, 0x0)) /**************************************************************************** * HDA Device Parameters ****************************************************************************/ /* Vendor ID */ #define HDA_PARAM_VENDOR_ID 0x00 #define HDA_PARAM_VENDOR_ID_VENDOR_ID_MASK 0xffff0000 #define HDA_PARAM_VENDOR_ID_VENDOR_ID_SHIFT 16 #define HDA_PARAM_VENDOR_ID_DEVICE_ID_MASK 0x0000ffff #define HDA_PARAM_VENDOR_ID_DEVICE_ID_SHIFT 0 #define HDA_PARAM_VENDOR_ID_VENDOR_ID(param) \ (((param) & HDA_PARAM_VENDOR_ID_VENDOR_ID_MASK) >> \ HDA_PARAM_VENDOR_ID_VENDOR_ID_SHIFT) #define HDA_PARAM_VENDOR_ID_DEVICE_ID(param) \ (((param) & HDA_PARAM_VENDOR_ID_DEVICE_ID_MASK) >> \ HDA_PARAM_VENDOR_ID_DEVICE_ID_SHIFT) /* Revision ID */ #define HDA_PARAM_REVISION_ID 0x02 #define HDA_PARAM_REVISION_ID_MAJREV_MASK 0x00f00000 #define HDA_PARAM_REVISION_ID_MAJREV_SHIFT 20 #define HDA_PARAM_REVISION_ID_MINREV_MASK 0x000f0000 #define HDA_PARAM_REVISION_ID_MINREV_SHIFT 16 #define HDA_PARAM_REVISION_ID_REVISION_ID_MASK 0x0000ff00 #define HDA_PARAM_REVISION_ID_REVISION_ID_SHIFT 8 #define HDA_PARAM_REVISION_ID_STEPPING_ID_MASK 0x000000ff #define HDA_PARAM_REVISION_ID_STEPPING_ID_SHIFT 0 #define HDA_PARAM_REVISION_ID_MAJREV(param) \ (((param) & HDA_PARAM_REVISION_ID_MAJREV_MASK) >> \ HDA_PARAM_REVISION_ID_MAJREV_SHIFT) #define HDA_PARAM_REVISION_ID_MINREV(param) \ (((param) & HDA_PARAM_REVISION_ID_MINREV_MASK) >> \ HDA_PARAM_REVISION_ID_MINREV_SHIFT) #define HDA_PARAM_REVISION_ID_REVISION_ID(param) \ (((param) & HDA_PARAM_REVISION_ID_REVISION_ID_MASK) >> \ HDA_PARAM_REVISION_ID_REVISION_ID_SHIFT) #define HDA_PARAM_REVISION_ID_STEPPING_ID(param) \ (((param) & HDA_PARAM_REVISION_ID_STEPPING_ID_MASK) >> \ HDA_PARAM_REVISION_ID_STEPPING_ID_SHIFT) /* Subordinate Node Cound */ #define HDA_PARAM_SUB_NODE_COUNT 0x04 #define HDA_PARAM_SUB_NODE_COUNT_START_MASK 0x00ff0000 #define HDA_PARAM_SUB_NODE_COUNT_START_SHIFT 16 #define HDA_PARAM_SUB_NODE_COUNT_TOTAL_MASK 0x000000ff #define HDA_PARAM_SUB_NODE_COUNT_TOTAL_SHIFT 0 #define HDA_PARAM_SUB_NODE_COUNT_START(param) \ (((param) & HDA_PARAM_SUB_NODE_COUNT_START_MASK) >> \ HDA_PARAM_SUB_NODE_COUNT_START_SHIFT) #define HDA_PARAM_SUB_NODE_COUNT_TOTAL(param) \ (((param) & HDA_PARAM_SUB_NODE_COUNT_TOTAL_MASK) >> \ HDA_PARAM_SUB_NODE_COUNT_TOTAL_SHIFT) /* Function Group Type */ #define HDA_PARAM_FCT_GRP_TYPE 0x05 #define HDA_PARAM_FCT_GRP_TYPE_UNSOL_MASK 0x00000100 #define HDA_PARAM_FCT_GRP_TYPE_UNSOL_SHIFT 8 #define HDA_PARAM_FCT_GRP_TYPE_NODE_TYPE_MASK 0x000000ff #define HDA_PARAM_FCT_GRP_TYPE_NODE_TYPE_SHIFT 0 #define HDA_PARAM_FCT_GRP_TYPE_UNSOL(param) \ (((param) & HDA_PARAM_FCT_GRP_TYPE_UNSOL_MASK) >> \ HDA_PARAM_FCT_GROUP_TYPE_UNSOL_SHIFT) #define HDA_PARAM_FCT_GRP_TYPE_NODE_TYPE(param) \ (((param) & HDA_PARAM_FCT_GRP_TYPE_NODE_TYPE_MASK) >> \ HDA_PARAM_FCT_GRP_TYPE_NODE_TYPE_SHIFT) #define HDA_PARAM_FCT_GRP_TYPE_NODE_TYPE_AUDIO 0x01 #define HDA_PARAM_FCT_GRP_TYPE_NODE_TYPE_MODEM 0x02 /* Audio Function Group Capabilities */ #define HDA_PARAM_AUDIO_FCT_GRP_CAP 0x08 #define HDA_PARAM_AUDIO_FCT_GRP_CAP_BEEP_GEN_MASK 0x00010000 #define HDA_PARAM_AUDIO_FCT_GRP_CAP_BEEP_GEN_SHIFT 16 #define HDA_PARAM_AUDIO_FCT_GRP_CAP_INPUT_DELAY_MASK 0x00000f00 #define HDA_PARAM_AUDIO_FCT_GRP_CAP_INPUT_DELAY_SHIFT 8 #define HDA_PARAM_AUDIO_FCT_GRP_CAP_OUTPUT_DELAY_MASK 0x0000000f #define HDA_PARAM_AUDIO_FCT_GRP_CAP_OUTPUT_DELAY_SHIFT 0 #define HDA_PARAM_AUDIO_FCT_GRP_CAP_BEEP_GEN(param) \ (((param) & HDA_PARAM_AUDIO_FCT_GRP_CAP_BEEP_GEN_MASK) >> \ HDA_PARAM_AUDIO_FCT_GRP_CAP_BEEP_GEN_SHIFT) #define HDA_PARAM_AUDIO_FCT_GRP_CAP_INPUT_DELAY(param) \ (((param) & HDA_PARAM_AUDIO_FCT_GRP_CAP_INPUT_DELAY_MASK) >> \ HDA_PARAM_AUDIO_FCT_GRP_CAP_INPUT_DELAY_SHIFT) #define HDA_PARAM_AUDIO_FCT_GRP_CAP_OUTPUT_DELAY(param) \ (((param) & HDA_PARAM_AUDIO_FCT_GRP_CAP_OUTPUT_DELAY_MASK) >> \ HDA_PARAM_AUDIO_FCT_GRP_CAP_OUTPUT_DELAY_SHIFT) /* Audio Widget Capabilities */ #define HDA_PARAM_AUDIO_WIDGET_CAP 0x09 #define HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_MASK 0x00f00000 #define HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_SHIFT 20 #define HDA_PARAM_AUDIO_WIDGET_CAP_DELAY_MASK 0x000f0000 #define HDA_PARAM_AUDIO_WIDGET_CAP_DELAY_SHIFT 16 #define HDA_PARAM_AUDIO_WIDGET_CAP_CC_EXT_MASK 0x0000e000 #define HDA_PARAM_AUDIO_WIDGET_CAP_CC_EXT_SHIFT 13 #define HDA_PARAM_AUDIO_WIDGET_CAP_CP_MASK 0x00001000 #define HDA_PARAM_AUDIO_WIDGET_CAP_CP_SHIFT 12 #define HDA_PARAM_AUDIO_WIDGET_CAP_LR_SWAP_MASK 0x00000800 #define HDA_PARAM_AUDIO_WIDGET_CAP_LR_SWAP_SHIFT 11 #define HDA_PARAM_AUDIO_WIDGET_CAP_POWER_CTRL_MASK 0x00000400 #define HDA_PARAM_AUDIO_WIDGET_CAP_POWER_CTRL_SHIFT 10 #define HDA_PARAM_AUDIO_WIDGET_CAP_DIGITAL_MASK 0x00000200 #define HDA_PARAM_AUDIO_WIDGET_CAP_DIGITAL_SHIFT 9 #define HDA_PARAM_AUDIO_WIDGET_CAP_CONN_LIST_MASK 0x00000100 #define HDA_PARAM_AUDIO_WIDGET_CAP_CONN_LIST_SHIFT 8 #define HDA_PARAM_AUDIO_WIDGET_CAP_UNSOL_CAP_MASK 0x00000080 #define HDA_PARAM_AUDIO_WIDGET_CAP_UNSOL_CAP_SHIFT 7 #define HDA_PARAM_AUDIO_WIDGET_CAP_PROC_WIDGET_MASK 0x00000040 #define HDA_PARAM_AUDIO_WIDGET_CAP_PROC_WIDGET_SHIFT 6 #define HDA_PARAM_AUDIO_WIDGET_CAP_STRIPE_MASK 0x00000020 #define HDA_PARAM_AUDIO_WIDGET_CAP_STRIPE_SHIFT 5 #define HDA_PARAM_AUDIO_WIDGET_CAP_FORMAT_OVR_MASK 0x00000010 #define HDA_PARAM_AUDIO_WIDGET_CAP_FORMAT_OVR_SHIFT 4 #define HDA_PARAM_AUDIO_WIDGET_CAP_AMP_OVR_MASK 0x00000008 #define HDA_PARAM_AUDIO_WIDGET_CAP_AMP_OVR_SHIFT 3 #define HDA_PARAM_AUDIO_WIDGET_CAP_OUT_AMP_MASK 0x00000004 #define HDA_PARAM_AUDIO_WIDGET_CAP_OUT_AMP_SHIFT 2 #define HDA_PARAM_AUDIO_WIDGET_CAP_IN_AMP_MASK 0x00000002 #define HDA_PARAM_AUDIO_WIDGET_CAP_IN_AMP_SHIFT 1 #define HDA_PARAM_AUDIO_WIDGET_CAP_STEREO_MASK 0x00000001 #define HDA_PARAM_AUDIO_WIDGET_CAP_STEREO_SHIFT 0 #define HDA_PARAM_AUDIO_WIDGET_CAP_TYPE(param) \ (((param) & HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_MASK) >> \ HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_SHIFT) #define HDA_PARAM_AUDIO_WIDGET_CAP_DELAY(param) \ (((param) & HDA_PARAM_AUDIO_WIDGET_CAP_DELAY_MASK) >> \ HDA_PARAM_AUDIO_WIDGET_CAP_DELAY_SHIFT) #define HDA_PARAM_AUDIO_WIDGET_CAP_CC(param) \ ((((param) & HDA_PARAM_AUDIO_WIDGET_CAP_CC_EXT_MASK) >> \ (HDA_PARAM_AUDIO_WIDGET_CAP_CC_EXT_SHIFT - 1)) | \ (((param) & HDA_PARAM_AUDIO_WIDGET_CAP_STEREO_MASK) >> \ HDA_PARAM_AUDIO_WIDGET_CAP_STEREO_SHIFT)) #define HDA_PARAM_AUDIO_WIDGET_CAP_CP(param) \ (((param) & HDA_PARAM_AUDIO_WIDGET_CAP_CP_MASK) >> \ HDA_PARAM_AUDIO_WIDGET_CAP_CP_SHIFT) #define HDA_PARAM_AUDIO_WIDGET_CAP_LR_SWAP(param) \ (((param) & HDA_PARAM_AUDIO_WIDGET_CAP_LR_SWAP_MASK) >> \ HDA_PARAM_AUDIO_WIDGET_CAP_LR_SWAP_SHIFT) #define HDA_PARAM_AUDIO_WIDGET_CAP_POWER_CTRL(param) \ (((param) & HDA_PARAM_AUDIO_WIDGET_CAP_POWER_CTRL_MASK) >> \ HDA_PARAM_AUDIO_WIDGET_CAP_POWER_CTRL_SHIFT) #define HDA_PARAM_AUDIO_WIDGET_CAP_DIGITAL(param) \ (((param) & HDA_PARAM_AUDIO_WIDGET_CAP_DIGITAL_MASK) >> \ HDA_PARAM_AUDIO_WIDGET_CAP_DIGITAL_SHIFT) #define HDA_PARAM_AUDIO_WIDGET_CAP_CONN_LIST(param) \ (((param) & HDA_PARAM_AUDIO_WIDGET_CAP_CONN_LIST_MASK) >> \ HDA_PARAM_AUDIO_WIDGET_CAP_CONN_LIST_SHIFT) #define HDA_PARAM_AUDIO_WIDGET_CAP_UNSOL_CAP(param) \ (((param) & HDA_PARAM_AUDIO_WIDGET_CAP_UNSOL_CAP_MASK) >> \ HDA_PARAM_AUDIO_WIDGET_CAP_UNSOL_CAP_SHIFT) #define HDA_PARAM_AUDIO_WIDGET_CAP_PROC_WIDGET(param) \ (((param) & HDA_PARAM_AUDIO_WIDGET_CAP_PROC_WIDGET_MASK) >> \ HDA_PARAM_AUDIO_WIDGET_CAP_PROC_WIDGET_SHIFT) #define HDA_PARAM_AUDIO_WIDGET_CAP_STRIPE(param) \ (((param) & HDA_PARAM_AUDIO_WIDGET_CAP_STRIPE_MASK) >> \ HDA_PARAM_AUDIO_WIDGET_CAP_STRIPE_SHIFT) #define HDA_PARAM_AUDIO_WIDGET_CAP_FORMAT_OVR(param) \ (((param) & HDA_PARAM_AUDIO_WIDGET_CAP_FORMAT_OVR_MASK) >> \ HDA_PARAM_AUDIO_WIDGET_CAP_FORMAT_OVR_SHIFT) #define HDA_PARAM_AUDIO_WIDGET_CAP_AMP_OVR(param) \ (((param) & HDA_PARAM_AUDIO_WIDGET_CAP_AMP_OVR_MASK) >> \ HDA_PARAM_AUDIO_WIDGET_CAP_AMP_OVR_SHIFT) #define HDA_PARAM_AUDIO_WIDGET_CAP_OUT_AMP(param) \ (((param) & HDA_PARAM_AUDIO_WIDGET_CAP_OUT_AMP_MASK) >> \ HDA_PARAM_AUDIO_WIDGET_CAP_OUT_AMP_SHIFT) #define HDA_PARAM_AUDIO_WIDGET_CAP_IN_AMP(param) \ (((param) & HDA_PARAM_AUDIO_WIDGET_CAP_IN_AMP_MASK) >> \ HDA_PARAM_AUDIO_WIDGET_CAP_IN_AMP_SHIFT) #define HDA_PARAM_AUDIO_WIDGET_CAP_STEREO(param) \ (((param) & HDA_PARAM_AUDIO_WIDGET_CAP_STEREO_MASK) >> \ HDA_PARAM_AUDIO_WIDGET_CAP_STEREO_SHIFT) #define HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_AUDIO_OUTPUT 0x0 #define HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_AUDIO_INPUT 0x1 #define HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_AUDIO_MIXER 0x2 #define HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_AUDIO_SELECTOR 0x3 #define HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_PIN_COMPLEX 0x4 #define HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_POWER_WIDGET 0x5 #define HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_VOLUME_WIDGET 0x6 #define HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_BEEP_WIDGET 0x7 #define HDA_PARAM_AUDIO_WIDGET_CAP_TYPE_VENDOR_WIDGET 0xf /* Supported PCM Size, Rates */ #define HDA_PARAM_SUPP_PCM_SIZE_RATE 0x0a #define HDA_PARAM_SUPP_PCM_SIZE_RATE_32BIT_MASK 0x00100000 #define HDA_PARAM_SUPP_PCM_SIZE_RATE_32BIT_SHIFT 20 #define HDA_PARAM_SUPP_PCM_SIZE_RATE_24BIT_MASK 0x00080000 #define HDA_PARAM_SUPP_PCM_SIZE_RATE_24BIT_SHIFT 19 #define HDA_PARAM_SUPP_PCM_SIZE_RATE_20BIT_MASK 0x00040000 #define HDA_PARAM_SUPP_PCM_SIZE_RATE_20BIT_SHIFT 18 #define HDA_PARAM_SUPP_PCM_SIZE_RATE_16BIT_MASK 0x00020000 #define HDA_PARAM_SUPP_PCM_SIZE_RATE_16BIT_SHIFT 17 #define HDA_PARAM_SUPP_PCM_SIZE_RATE_8BIT_MASK 0x00010000 #define HDA_PARAM_SUPP_PCM_SIZE_RATE_8BIT_SHIFT 16 #define HDA_PARAM_SUPP_PCM_SIZE_RATE_8KHZ_MASK 0x00000001 #define HDA_PARAM_SUPP_PCM_SIZE_RATE_8KHZ_SHIFT 0 #define HDA_PARAM_SUPP_PCM_SIZE_RATE_11KHZ_MASK 0x00000002 #define HDA_PARAM_SUPP_PCM_SIZE_RATE_11KHZ_SHIFT 1 #define HDA_PARAM_SUPP_PCM_SIZE_RATE_16KHZ_MASK 0x00000004 #define HDA_PARAM_SUPP_PCM_SIZE_RATE_16KHZ_SHIFT 2 #define HDA_PARAM_SUPP_PCM_SIZE_RATE_22KHZ_MASK 0x00000008 #define HDA_PARAM_SUPP_PCM_SIZE_RATE_22KHZ_SHIFT 3 #define HDA_PARAM_SUPP_PCM_SIZE_RATE_32KHZ_MASK 0x00000010 #define HDA_PARAM_SUPP_PCM_SIZE_RATE_32KHZ_SHIFT 4 #define HDA_PARAM_SUPP_PCM_SIZE_RATE_44KHZ_MASK 0x00000020 #define HDA_PARAM_SUPP_PCM_SIZE_RATE_44KHZ_SHIFT 5 #define HDA_PARAM_SUPP_PCM_SIZE_RATE_48KHZ_MASK 0x00000040 #define HDA_PARAM_SUPP_PCM_SIZE_RATE_48KHZ_SHIFT 6 #define HDA_PARAM_SUPP_PCM_SIZE_RATE_88KHZ_MASK 0x00000080 #define HDA_PARAM_SUPP_PCM_SIZE_RATE_88KHZ_SHIFT 7 #define HDA_PARAM_SUPP_PCM_SIZE_RATE_96KHZ_MASK 0x00000100 #define HDA_PARAM_SUPP_PCM_SIZE_RATE_96KHZ_SHIFT 8 #define HDA_PARAM_SUPP_PCM_SIZE_RATE_176KHZ_MASK 0x00000200 #define HDA_PARAM_SUPP_PCM_SIZE_RATE_176KHZ_SHIFT 9 #define HDA_PARAM_SUPP_PCM_SIZE_RATE_192KHZ_MASK 0x00000400 #define HDA_PARAM_SUPP_PCM_SIZE_RATE_192KHZ_SHIFT 10 #define HDA_PARAM_SUPP_PCM_SIZE_RATE_384KHZ_MASK 0x00000800 #define HDA_PARAM_SUPP_PCM_SIZE_RATE_384KHZ_SHIFT 11 #define HDA_PARAM_SUPP_PCM_SIZE_RATE_32BIT(param) \ (((param) & HDA_PARAM_SUPP_PCM_SIZE_RATE_32BIT_MASK) >> \ HDA_PARAM_SUPP_PCM_SIZE_RATE_32BIT_SHIFT) #define HDA_PARAM_SUPP_PCM_SIZE_RATE_24BIT(param) \ (((param) & HDA_PARAM_SUPP_PCM_SIZE_RATE_24BIT_MASK) >> \ HDA_PARAM_SUPP_PCM_SIZE_RATE_24BIT_SHIFT) #define HDA_PARAM_SUPP_PCM_SIZE_RATE_20BIT(param) \ (((param) & HDA_PARAM_SUPP_PCM_SIZE_RATE_20BIT_MASK) >> \ HDA_PARAM_SUPP_PCM_SIZE_RATE_20BIT_SHIFT) #define HDA_PARAM_SUPP_PCM_SIZE_RATE_16BIT(param) \ (((param) & HDA_PARAM_SUPP_PCM_SIZE_RATE_16BIT_MASK) >> \ HDA_PARAM_SUPP_PCM_SIZE_RATE_16BIT_SHIFT) #define HDA_PARAM_SUPP_PCM_SIZE_RATE_8BIT(param) \ (((param) & HDA_PARAM_SUPP_PCM_SIZE_RATE_8BIT_MASK) >> \ HDA_PARAM_SUPP_PCM_SIZE_RATE_8BIT_SHIFT) #define HDA_PARAM_SUPP_PCM_SIZE_RATE_8KHZ(param) \ (((param) & HDA_PARAM_SUPP_PCM_SIZE_RATE_8KHZ_MASK) >> \ HDA_PARAM_SUPP_PCM_SIZE_RATE_8KHZ_SHIFT) #define HDA_PARAM_SUPP_PCM_SIZE_RATE_11KHZ(param) \ (((param) & HDA_PARAM_SUPP_PCM_SIZE_RATE_11KHZ_MASK) >> \ HDA_PARAM_SUPP_PCM_SIZE_RATE_11KHZ_SHIFT) #define HDA_PARAM_SUPP_PCM_SIZE_RATE_16KHZ(param) \ (((param) & HDA_PARAM_SUPP_PCM_SIZE_RATE_16KHZ_MASK) >> \ HDA_PARAM_SUPP_PCM_SIZE_RATE_16KHZ_SHIFT) #define HDA_PARAM_SUPP_PCM_SIZE_RATE_22KHZ(param) \ (((param) & HDA_PARAM_SUPP_PCM_SIZE_RATE_22KHZ_MASK) >> \ HDA_PARAM_SUPP_PCM_SIZE_RATE_22KHZ_SHIFT) #define HDA_PARAM_SUPP_PCM_SIZE_RATE_32KHZ(param) \ (((param) & HDA_PARAM_SUPP_PCM_SIZE_RATE_32KHZ_MASK) >> \ HDA_PARAM_SUPP_PCM_SIZE_RATE_32KHZ_SHIFT) #define HDA_PARAM_SUPP_PCM_SIZE_RATE_44KHZ(param) \ (((param) & HDA_PARAM_SUPP_PCM_SIZE_RATE_44KHZ_MASK) >> \ HDA_PARAM_SUPP_PCM_SIZE_RATE_44KHZ_SHIFT) #define HDA_PARAM_SUPP_PCM_SIZE_RATE_48KHZ(param) \ (((param) & HDA_PARAM_SUPP_PCM_SIZE_RATE_48KHZ_MASK) >> \ HDA_PARAM_SUPP_PCM_SIZE_RATE_48KHZ_SHIFT) #define HDA_PARAM_SUPP_PCM_SIZE_RATE_88KHZ(param) \ (((param) & HDA_PARAM_SUPP_PCM_SIZE_RATE_88KHZ_MASK) >> \ HDA_PARAM_SUPP_PCM_SIZE_RATE_88KHZ_SHIFT) #define HDA_PARAM_SUPP_PCM_SIZE_RATE_96KHZ(param) \ (((param) & HDA_PARAM_SUPP_PCM_SIZE_RATE_96KHZ_MASK) >> \ HDA_PARAM_SUPP_PCM_SIZE_RATE_96KHZ_SHIFT) #define HDA_PARAM_SUPP_PCM_SIZE_RATE_176KHZ(param) \ (((param) & HDA_PARAM_SUPP_PCM_SIZE_RATE_176KHZ_MASK) >> \ HDA_PARAM_SUPP_PCM_SIZE_RATE_176KHZ_SHIFT) #define HDA_PARAM_SUPP_PCM_SIZE_RATE_192KHZ(param) \ (((param) & HDA_PARAM_SUPP_PCM_SIZE_RATE_192KHZ_MASK) >> \ HDA_PARAM_SUPP_PCM_SIZE_RATE_192KHZ_SHIFT) #define HDA_PARAM_SUPP_PCM_SIZE_RATE_384KHZ(param) \ (((param) & HDA_PARAM_SUPP_PCM_SIZE_RATE_384KHZ_MASK) >> \ HDA_PARAM_SUPP_PCM_SIZE_RATE_384KHZ_SHIFT) /* Supported Stream Formats */ #define HDA_PARAM_SUPP_STREAM_FORMATS 0x0b #define HDA_PARAM_SUPP_STREAM_FORMATS_AC3_MASK 0x00000004 #define HDA_PARAM_SUPP_STREAM_FORMATS_AC3_SHIFT 2 #define HDA_PARAM_SUPP_STREAM_FORMATS_FLOAT32_MASK 0x00000002 #define HDA_PARAM_SUPP_STREAM_FORMATS_FLOAT32_SHIFT 1 #define HDA_PARAM_SUPP_STREAM_FORMATS_PCM_MASK 0x00000001 #define HDA_PARAM_SUPP_STREAM_FORMATS_PCM_SHIFT 0 #define HDA_PARAM_SUPP_STREAM_FORMATS_AC3(param) \ (((param) & HDA_PARAM_SUPP_STREAM_FORMATS_AC3_MASK) >> \ HDA_PARAM_SUPP_STREAM_FORMATS_AC3_SHIFT) #define HDA_PARAM_SUPP_STREAM_FORMATS_FLOAT32(param) \ (((param) & HDA_PARAM_SUPP_STREAM_FORMATS_FLOAT32_MASK) >> \ HDA_PARAM_SUPP_STREAM_FORMATS_FLOAT32_SHIFT) #define HDA_PARAM_SUPP_STREAM_FORMATS_PCM(param) \ (((param) & HDA_PARAM_SUPP_STREAM_FORMATS_PCM_MASK) >> \ HDA_PARAM_SUPP_STREAM_FORMATS_PCM_SHIFT) /* Pin Capabilities */ #define HDA_PARAM_PIN_CAP 0x0c #define HDA_PARAM_PIN_CAP_HBR_MASK 0x08000000 #define HDA_PARAM_PIN_CAP_HBR_SHIFT 27 #define HDA_PARAM_PIN_CAP_DP_MASK 0x01000000 #define HDA_PARAM_PIN_CAP_DP_SHIFT 24 #define HDA_PARAM_PIN_CAP_EAPD_CAP_MASK 0x00010000 #define HDA_PARAM_PIN_CAP_EAPD_CAP_SHIFT 16 #define HDA_PARAM_PIN_CAP_VREF_CTRL_MASK 0x0000ff00 #define HDA_PARAM_PIN_CAP_VREF_CTRL_SHIFT 8 #define HDA_PARAM_PIN_CAP_VREF_CTRL_100_MASK 0x00002000 #define HDA_PARAM_PIN_CAP_VREF_CTRL_100_SHIFT 13 #define HDA_PARAM_PIN_CAP_VREF_CTRL_80_MASK 0x00001000 #define HDA_PARAM_PIN_CAP_VREF_CTRL_80_SHIFT 12 #define HDA_PARAM_PIN_CAP_VREF_CTRL_GROUND_MASK 0x00000400 #define HDA_PARAM_PIN_CAP_VREF_CTRL_GROUND_SHIFT 10 #define HDA_PARAM_PIN_CAP_VREF_CTRL_50_MASK 0x00000200 #define HDA_PARAM_PIN_CAP_VREF_CTRL_50_SHIFT 9 #define HDA_PARAM_PIN_CAP_VREF_CTRL_HIZ_MASK 0x00000100 #define HDA_PARAM_PIN_CAP_VREF_CTRL_HIZ_SHIFT 8 #define HDA_PARAM_PIN_CAP_HDMI_MASK 0x00000080 #define HDA_PARAM_PIN_CAP_HDMI_SHIFT 7 #define HDA_PARAM_PIN_CAP_BALANCED_IO_PINS_MASK 0x00000040 #define HDA_PARAM_PIN_CAP_BALANCED_IO_PINS_SHIFT 6 #define HDA_PARAM_PIN_CAP_INPUT_CAP_MASK 0x00000020 #define HDA_PARAM_PIN_CAP_INPUT_CAP_SHIFT 5 #define HDA_PARAM_PIN_CAP_OUTPUT_CAP_MASK 0x00000010 #define HDA_PARAM_PIN_CAP_OUTPUT_CAP_SHIFT 4 #define HDA_PARAM_PIN_CAP_HEADPHONE_CAP_MASK 0x00000008 #define HDA_PARAM_PIN_CAP_HEADPHONE_CAP_SHIFT 3 #define HDA_PARAM_PIN_CAP_PRESENCE_DETECT_CAP_MASK 0x00000004 #define HDA_PARAM_PIN_CAP_PRESENCE_DETECT_CAP_SHIFT 2 #define HDA_PARAM_PIN_CAP_TRIGGER_REQD_MASK 0x00000002 #define HDA_PARAM_PIN_CAP_TRIGGER_REQD_SHIFT 1 #define HDA_PARAM_PIN_CAP_IMP_SENSE_CAP_MASK 0x00000001 #define HDA_PARAM_PIN_CAP_IMP_SENSE_CAP_SHIFT 0 #define HDA_PARAM_PIN_CAP_HBR(param) \ (((param) & HDA_PARAM_PIN_CAP_HBR_MASK) >> \ HDA_PARAM_PIN_CAP_HBR_SHIFT) #define HDA_PARAM_PIN_CAP_DP(param) \ (((param) & HDA_PARAM_PIN_CAP_DP_MASK) >> \ HDA_PARAM_PIN_CAP_DP_SHIFT) #define HDA_PARAM_PIN_CAP_EAPD_CAP(param) \ (((param) & HDA_PARAM_PIN_CAP_EAPD_CAP_MASK) >> \ HDA_PARAM_PIN_CAP_EAPD_CAP_SHIFT) #define HDA_PARAM_PIN_CAP_VREF_CTRL(param) \ (((param) & HDA_PARAM_PIN_CAP_VREF_CTRL_MASK) >> \ HDA_PARAM_PIN_CAP_VREF_CTRL_SHIFT) #define HDA_PARAM_PIN_CAP_VREF_CTRL_100(param) \ (((param) & HDA_PARAM_PIN_CAP_VREF_CTRL_100_MASK) >> \ HDA_PARAM_PIN_CAP_VREF_CTRL_100_SHIFT) #define HDA_PARAM_PIN_CAP_VREF_CTRL_80(param) \ (((param) & HDA_PARAM_PIN_CAP_VREF_CTRL_80_MASK) >> \ HDA_PARAM_PIN_CAP_VREF_CTRL_80_SHIFT) #define HDA_PARAM_PIN_CAP_VREF_CTRL_GROUND(param) \ (((param) & HDA_PARAM_PIN_CAP_VREF_CTRL_GROUND_MASK) >> \ HDA_PARAM_PIN_CAP_VREF_CTRL_GROUND_SHIFT) #define HDA_PARAM_PIN_CAP_VREF_CTRL_50(param) \ (((param) & HDA_PARAM_PIN_CAP_VREF_CTRL_50_MASK) >> \ HDA_PARAM_PIN_CAP_VREF_CTRL_50_SHIFT) #define HDA_PARAM_PIN_CAP_VREF_CTRL_HIZ(param) \ (((param) & HDA_PARAM_PIN_CAP_VREF_CTRL_HIZ_MASK) >> \ HDA_PARAM_PIN_CAP_VREF_CTRL_HIZ_SHIFT) #define HDA_PARAM_PIN_CAP_HDMI(param) \ (((param) & HDA_PARAM_PIN_CAP_HDMI_MASK) >> \ HDA_PARAM_PIN_CAP_HDMI_SHIFT) #define HDA_PARAM_PIN_CAP_BALANCED_IO_PINS(param) \ (((param) & HDA_PARAM_PIN_CAP_BALANCED_IO_PINS_MASK) >> \ HDA_PARAM_PIN_CAP_BALANCED_IO_PINS_SHIFT) #define HDA_PARAM_PIN_CAP_INPUT_CAP(param) \ (((param) & HDA_PARAM_PIN_CAP_INPUT_CAP_MASK) >> \ HDA_PARAM_PIN_CAP_INPUT_CAP_SHIFT) #define HDA_PARAM_PIN_CAP_OUTPUT_CAP(param) \ (((param) & HDA_PARAM_PIN_CAP_OUTPUT_CAP_MASK) >> \ HDA_PARAM_PIN_CAP_OUTPUT_CAP_SHIFT) #define HDA_PARAM_PIN_CAP_HEADPHONE_CAP(param) \ (((param) & HDA_PARAM_PIN_CAP_HEADPHONE_CAP_MASK) >> \ HDA_PARAM_PIN_CAP_HEADPHONE_CAP_SHIFT) #define HDA_PARAM_PIN_CAP_PRESENCE_DETECT_CAP(param) \ (((param) & HDA_PARAM_PIN_CAP_PRESENCE_DETECT_CAP_MASK) >> \ HDA_PARAM_PIN_CAP_PRESENCE_DETECT_CAP_SHIFT) #define HDA_PARAM_PIN_CAP_TRIGGER_REQD(param) \ (((param) & HDA_PARAM_PIN_CAP_TRIGGER_REQD_MASK) >> \ HDA_PARAM_PIN_CAP_TRIGGER_REQD_SHIFT) #define HDA_PARAM_PIN_CAP_IMP_SENSE_CAP(param) \ (((param) & HDA_PARAM_PIN_CAP_IMP_SENSE_CAP_MASK) >> \ HDA_PARAM_PIN_CAP_IMP_SENSE_CAP_SHIFT) /* Input Amplifier Capabilities */ #define HDA_PARAM_INPUT_AMP_CAP 0x0d #define HDA_PARAM_INPUT_AMP_CAP_MUTE_CAP_MASK 0x80000000 #define HDA_PARAM_INPUT_AMP_CAP_MUTE_CAP_SHIFT 31 #define HDA_PARAM_INPUT_AMP_CAP_STEPSIZE_MASK 0x007f0000 #define HDA_PARAM_INPUT_AMP_CAP_STEPSIZE_SHIFT 16 #define HDA_PARAM_INPUT_AMP_CAP_NUMSTEPS_MASK 0x00007f00 #define HDA_PARAM_INPUT_AMP_CAP_NUMSTEPS_SHIFT 8 #define HDA_PARAM_INPUT_AMP_CAP_OFFSET_MASK 0x0000007f #define HDA_PARAM_INPUT_AMP_CAP_OFFSET_SHIFT 0 #define HDA_PARAM_INPUT_AMP_CAP_MUTE_CAP(param) \ (((param) & HDA_PARAM_INPUT_AMP_CAP_MUTE_CAP_MASK) >> \ HDA_PARAM_INPUT_AMP_CAP_MUTE_CAP_SHIFT) #define HDA_PARAM_INPUT_AMP_CAP_STEPSIZE(param) \ (((param) & HDA_PARAM_INPUT_AMP_CAP_STEPSIZE_MASK) >> \ HDA_PARAM_INPUT_AMP_CAP_STEPSIZE_SHIFT) #define HDA_PARAM_INPUT_AMP_CAP_NUMSTEPS(param) \ (((param) & HDA_PARAM_INPUT_AMP_CAP_NUMSTEPS_MASK) >> \ HDA_PARAM_INPUT_AMP_CAP_NUMSTEPS_SHIFT) #define HDA_PARAM_INPUT_AMP_CAP_OFFSET(param) \ (((param) & HDA_PARAM_INPUT_AMP_CAP_OFFSET_MASK) >> \ HDA_PARAM_INPUT_AMP_CAP_OFFSET_SHIFT) /* Output Amplifier Capabilities */ #define HDA_PARAM_OUTPUT_AMP_CAP 0x12 #define HDA_PARAM_OUTPUT_AMP_CAP_MUTE_CAP_MASK 0x80000000 #define HDA_PARAM_OUTPUT_AMP_CAP_MUTE_CAP_SHIFT 31 #define HDA_PARAM_OUTPUT_AMP_CAP_STEPSIZE_MASK 0x007f0000 #define HDA_PARAM_OUTPUT_AMP_CAP_STEPSIZE_SHIFT 16 #define HDA_PARAM_OUTPUT_AMP_CAP_NUMSTEPS_MASK 0x00007f00 #define HDA_PARAM_OUTPUT_AMP_CAP_NUMSTEPS_SHIFT 8 #define HDA_PARAM_OUTPUT_AMP_CAP_OFFSET_MASK 0x0000007f #define HDA_PARAM_OUTPUT_AMP_CAP_OFFSET_SHIFT 0 #define HDA_PARAM_OUTPUT_AMP_CAP_MUTE_CAP(param) \ (((param) & HDA_PARAM_OUTPUT_AMP_CAP_MUTE_CAP_MASK) >> \ HDA_PARAM_OUTPUT_AMP_CAP_MUTE_CAP_SHIFT) #define HDA_PARAM_OUTPUT_AMP_CAP_STEPSIZE(param) \ (((param) & HDA_PARAM_OUTPUT_AMP_CAP_STEPSIZE_MASK) >> \ HDA_PARAM_OUTPUT_AMP_CAP_STEPSIZE_SHIFT) #define HDA_PARAM_OUTPUT_AMP_CAP_NUMSTEPS(param) \ (((param) & HDA_PARAM_OUTPUT_AMP_CAP_NUMSTEPS_MASK) >> \ HDA_PARAM_OUTPUT_AMP_CAP_NUMSTEPS_SHIFT) #define HDA_PARAM_OUTPUT_AMP_CAP_OFFSET(param) \ (((param) & HDA_PARAM_OUTPUT_AMP_CAP_OFFSET_MASK) >> \ HDA_PARAM_OUTPUT_AMP_CAP_OFFSET_SHIFT) /* Connection List Length */ #define HDA_PARAM_CONN_LIST_LENGTH 0x0e #define HDA_PARAM_CONN_LIST_LENGTH_LONG_FORM_MASK 0x00000080 #define HDA_PARAM_CONN_LIST_LENGTH_LONG_FORM_SHIFT 7 #define HDA_PARAM_CONN_LIST_LENGTH_LIST_LENGTH_MASK 0x0000007f #define HDA_PARAM_CONN_LIST_LENGTH_LIST_LENGTH_SHIFT 0 #define HDA_PARAM_CONN_LIST_LENGTH_LONG_FORM(param) \ (((param) & HDA_PARAM_CONN_LIST_LENGTH_LONG_FORM_MASK) >> \ HDA_PARAM_CONN_LIST_LENGTH_LONG_FORM_SHIFT) #define HDA_PARAM_CONN_LIST_LENGTH_LIST_LENGTH(param) \ (((param) & HDA_PARAM_CONN_LIST_LENGTH_LIST_LENGTH_MASK) >> \ HDA_PARAM_CONN_LIST_LENGTH_LIST_LENGTH_SHIFT) /* Supported Power States */ #define HDA_PARAM_SUPP_POWER_STATES 0x0f #define HDA_PARAM_SUPP_POWER_STATES_D3_MASK 0x00000008 #define HDA_PARAM_SUPP_POWER_STATES_D3_SHIFT 3 #define HDA_PARAM_SUPP_POWER_STATES_D2_MASK 0x00000004 #define HDA_PARAM_SUPP_POWER_STATES_D2_SHIFT 2 #define HDA_PARAM_SUPP_POWER_STATES_D1_MASK 0x00000002 #define HDA_PARAM_SUPP_POWER_STATES_D1_SHIFT 1 #define HDA_PARAM_SUPP_POWER_STATES_D0_MASK 0x00000001 #define HDA_PARAM_SUPP_POWER_STATES_D0_SHIFT 0 #define HDA_PARAM_SUPP_POWER_STATES_D3(param) \ (((param) & HDA_PARAM_SUPP_POWER_STATES_D3_MASK) >> \ HDA_PARAM_SUPP_POWER_STATES_D3_SHIFT) #define HDA_PARAM_SUPP_POWER_STATES_D2(param) \ (((param) & HDA_PARAM_SUPP_POWER_STATES_D2_MASK) >> \ HDA_PARAM_SUPP_POWER_STATES_D2_SHIFT) #define HDA_PARAM_SUPP_POWER_STATES_D1(param) \ (((param) & HDA_PARAM_SUPP_POWER_STATES_D1_MASK) >> \ HDA_PARAM_SUPP_POWER_STATES_D1_SHIFT) #define HDA_PARAM_SUPP_POWER_STATES_D0(param) \ (((param) & HDA_PARAM_SUPP_POWER_STATES_D0_MASK) >> \ HDA_PARAM_SUPP_POWER_STATES_D0_SHIFT) /* Processing Capabilities */ #define HDA_PARAM_PROCESSING_CAP 0x10 #define HDA_PARAM_PROCESSING_CAP_NUMCOEFF_MASK 0x0000ff00 #define HDA_PARAM_PROCESSING_CAP_NUMCOEFF_SHIFT 8 #define HDA_PARAM_PROCESSING_CAP_BENIGN_MASK 0x00000001 #define HDA_PARAM_PROCESSING_CAP_BENIGN_SHIFT 0 #define HDA_PARAM_PROCESSING_CAP_NUMCOEFF(param) \ (((param) & HDA_PARAM_PROCESSING_CAP_NUMCOEFF_MASK) >> \ HDA_PARAM_PROCESSING_CAP_NUMCOEFF_SHIFT) #define HDA_PARAM_PROCESSING_CAP_BENIGN(param) \ (((param) & HDA_PARAM_PROCESSING_CAP_BENIGN_MASK) >> \ HDA_PARAM_PROCESSING_CAP_BENIGN_SHIFT) /* GPIO Count */ #define HDA_PARAM_GPIO_COUNT 0x11 #define HDA_PARAM_GPIO_COUNT_GPI_WAKE_MASK 0x80000000 #define HDA_PARAM_GPIO_COUNT_GPI_WAKE_SHIFT 31 #define HDA_PARAM_GPIO_COUNT_GPI_UNSOL_MASK 0x40000000 #define HDA_PARAM_GPIO_COUNT_GPI_UNSOL_SHIFT 30 #define HDA_PARAM_GPIO_COUNT_NUM_GPI_MASK 0x00ff0000 #define HDA_PARAM_GPIO_COUNT_NUM_GPI_SHIFT 16 #define HDA_PARAM_GPIO_COUNT_NUM_GPO_MASK 0x0000ff00 #define HDA_PARAM_GPIO_COUNT_NUM_GPO_SHIFT 8 #define HDA_PARAM_GPIO_COUNT_NUM_GPIO_MASK 0x000000ff #define HDA_PARAM_GPIO_COUNT_NUM_GPIO_SHIFT 0 #define HDA_PARAM_GPIO_COUNT_GPI_WAKE(param) \ (((param) & HDA_PARAM_GPIO_COUNT_GPI_WAKE_MASK) >> \ HDA_PARAM_GPIO_COUNT_GPI_WAKE_SHIFT) #define HDA_PARAM_GPIO_COUNT_GPI_UNSOL(param) \ (((param) & HDA_PARAM_GPIO_COUNT_GPI_UNSOL_MASK) >> \ HDA_PARAM_GPIO_COUNT_GPI_UNSOL_SHIFT) #define HDA_PARAM_GPIO_COUNT_NUM_GPI(param) \ (((param) & HDA_PARAM_GPIO_COUNT_NUM_GPI_MASK) >> \ HDA_PARAM_GPIO_COUNT_NUM_GPI_SHIFT) #define HDA_PARAM_GPIO_COUNT_NUM_GPO(param) \ (((param) & HDA_PARAM_GPIO_COUNT_NUM_GPO_MASK) >> \ HDA_PARAM_GPIO_COUNT_NUM_GPO_SHIFT) #define HDA_PARAM_GPIO_COUNT_NUM_GPIO(param) \ (((param) & HDA_PARAM_GPIO_COUNT_NUM_GPIO_MASK) >> \ HDA_PARAM_GPIO_COUNT_NUM_GPIO_SHIFT) /* Volume Knob Capabilities */ #define HDA_PARAM_VOLUME_KNOB_CAP 0x13 #define HDA_PARAM_VOLUME_KNOB_CAP_DELTA_MASK 0x00000080 #define HDA_PARAM_VOLUME_KNOB_CAP_DELTA_SHIFT 7 #define HDA_PARAM_VOLUME_KNOB_CAP_NUM_STEPS_MASK 0x0000007f #define HDA_PARAM_VOLUME_KNOB_CAP_NUM_STEPS_SHIFT 0 #define HDA_PARAM_VOLUME_KNOB_CAP_DELTA(param) \ (((param) & HDA_PARAM_VOLUME_KNOB_CAP_DELTA_MASK) >> \ HDA_PARAM_VOLUME_KNOB_CAP_DELTA_SHIFT) #define HDA_PARAM_VOLUME_KNOB_CAP_NUM_STEPS(param) \ (((param) & HDA_PARAM_VOLUME_KNOB_CAP_NUM_STEPS_MASK) >> \ HDA_PARAM_VOLUME_KNOB_CAP_NUM_STEPS_SHIFT) #define HDA_CONFIG_DEFAULTCONF_SEQUENCE_MASK 0x0000000f #define HDA_CONFIG_DEFAULTCONF_SEQUENCE_SHIFT 0 #define HDA_CONFIG_DEFAULTCONF_ASSOCIATION_MASK 0x000000f0 #define HDA_CONFIG_DEFAULTCONF_ASSOCIATION_SHIFT 4 #define HDA_CONFIG_DEFAULTCONF_MISC_MASK 0x00000f00 #define HDA_CONFIG_DEFAULTCONF_MISC_SHIFT 8 #define HDA_CONFIG_DEFAULTCONF_COLOR_MASK 0x0000f000 #define HDA_CONFIG_DEFAULTCONF_COLOR_SHIFT 12 #define HDA_CONFIG_DEFAULTCONF_CONNECTION_TYPE_MASK 0x000f0000 #define HDA_CONFIG_DEFAULTCONF_CONNECTION_TYPE_SHIFT 16 #define HDA_CONFIG_DEFAULTCONF_DEVICE_MASK 0x00f00000 #define HDA_CONFIG_DEFAULTCONF_DEVICE_SHIFT 20 #define HDA_CONFIG_DEFAULTCONF_LOCATION_MASK 0x3f000000 #define HDA_CONFIG_DEFAULTCONF_LOCATION_SHIFT 24 #define HDA_CONFIG_DEFAULTCONF_CONNECTIVITY_MASK 0xc0000000 #define HDA_CONFIG_DEFAULTCONF_CONNECTIVITY_SHIFT 30 #define HDA_CONFIG_DEFAULTCONF_SEQUENCE(conf) \ (((conf) & HDA_CONFIG_DEFAULTCONF_SEQUENCE_MASK) >> \ HDA_CONFIG_DEFAULTCONF_SEQUENCE_SHIFT) #define HDA_CONFIG_DEFAULTCONF_ASSOCIATION(conf) \ (((conf) & HDA_CONFIG_DEFAULTCONF_ASSOCIATION_MASK) >> \ HDA_CONFIG_DEFAULTCONF_ASSOCIATION_SHIFT) #define HDA_CONFIG_DEFAULTCONF_MISC(conf) \ (((conf) & HDA_CONFIG_DEFAULTCONF_MISC_MASK) >> \ HDA_CONFIG_DEFAULTCONF_MISC_SHIFT) #define HDA_CONFIG_DEFAULTCONF_COLOR(conf) \ (((conf) & HDA_CONFIG_DEFAULTCONF_COLOR_MASK) >> \ HDA_CONFIG_DEFAULTCONF_COLOR_SHIFT) #define HDA_CONFIG_DEFAULTCONF_CONNECTION_TYPE(conf) \ (((conf) & HDA_CONFIG_DEFAULTCONF_CONNECTION_TYPE_MASK) >> \ HDA_CONFIG_DEFAULTCONF_CONNECTION_TYPE_SHIFT) #define HDA_CONFIG_DEFAULTCONF_DEVICE(conf) \ (((conf) & HDA_CONFIG_DEFAULTCONF_DEVICE_MASK) >> \ HDA_CONFIG_DEFAULTCONF_DEVICE_SHIFT) #define HDA_CONFIG_DEFAULTCONF_LOCATION(conf) \ (((conf) & HDA_CONFIG_DEFAULTCONF_LOCATION_MASK) >> \ HDA_CONFIG_DEFAULTCONF_LOCATION_SHIFT) #define HDA_CONFIG_DEFAULTCONF_CONNECTIVITY(conf) \ (((conf) & HDA_CONFIG_DEFAULTCONF_CONNECTIVITY_MASK) >> \ HDA_CONFIG_DEFAULTCONF_CONNECTIVITY_SHIFT) #define HDA_CONFIG_DEFAULTCONF_CONNECTIVITY_JACK (0<<30) #define HDA_CONFIG_DEFAULTCONF_CONNECTIVITY_NONE (1<<30) #define HDA_CONFIG_DEFAULTCONF_CONNECTIVITY_FIXED (2<<30) #define HDA_CONFIG_DEFAULTCONF_CONNECTIVITY_BOTH (3<<30) #define HDA_CONFIG_DEFAULTCONF_DEVICE_LINE_OUT (0<<20) #define HDA_CONFIG_DEFAULTCONF_DEVICE_SPEAKER (1<<20) #define HDA_CONFIG_DEFAULTCONF_DEVICE_HP_OUT (2<<20) #define HDA_CONFIG_DEFAULTCONF_DEVICE_CD (3<<20) #define HDA_CONFIG_DEFAULTCONF_DEVICE_SPDIF_OUT (4<<20) #define HDA_CONFIG_DEFAULTCONF_DEVICE_DIGITAL_OTHER_OUT (5<<20) #define HDA_CONFIG_DEFAULTCONF_DEVICE_MODEM_LINE (6<<20) #define HDA_CONFIG_DEFAULTCONF_DEVICE_MODEM_HANDSET (7<<20) #define HDA_CONFIG_DEFAULTCONF_DEVICE_LINE_IN (8<<20) #define HDA_CONFIG_DEFAULTCONF_DEVICE_AUX (9<<20) #define HDA_CONFIG_DEFAULTCONF_DEVICE_MIC_IN (10<<20) #define HDA_CONFIG_DEFAULTCONF_DEVICE_TELEPHONY (11<<20) #define HDA_CONFIG_DEFAULTCONF_DEVICE_SPDIF_IN (12<<20) #define HDA_CONFIG_DEFAULTCONF_DEVICE_DIGITAL_OTHER_IN (13<<20) #define HDA_CONFIG_DEFAULTCONF_DEVICE_OTHER (15<<20) #endif /* _HDA_REG_H_ */ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2006 Stephane E. Potvin * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef _HDAC_REG_H_ #define _HDAC_REG_H_ /**************************************************************************** * HDA Controller Register Set ****************************************************************************/ #define HDAC_GCAP 0x00 /* 2 - Global Capabilities*/ #define HDAC_VMIN 0x02 /* 1 - Minor Version */ #define HDAC_VMAJ 0x03 /* 1 - Major Version */ #define HDAC_OUTPAY 0x04 /* 2 - Output Payload Capability */ #define HDAC_INPAY 0x06 /* 2 - Input Payload Capability */ #define HDAC_GCTL 0x08 /* 4 - Global Control */ #define HDAC_WAKEEN 0x0c /* 2 - Wake Enable */ #define HDAC_STATESTS 0x0e /* 2 - State Change Status */ #define HDAC_GSTS 0x10 /* 2 - Global Status */ #define HDAC_OUTSTRMPAY 0x18 /* 2 - Output Stream Payload Capability */ #define HDAC_INSTRMPAY 0x1a /* 2 - Input Stream Payload Capability */ #define HDAC_INTCTL 0x20 /* 4 - Interrupt Control */ #define HDAC_INTSTS 0x24 /* 4 - Interrupt Status */ #define HDAC_WALCLK 0x30 /* 4 - Wall Clock Counter */ #define HDAC_SSYNC 0x38 /* 4 - Stream Synchronization */ #define HDAC_CORBLBASE 0x40 /* 4 - CORB Lower Base Address */ #define HDAC_CORBUBASE 0x44 /* 4 - CORB Upper Base Address */ #define HDAC_CORBWP 0x48 /* 2 - CORB Write Pointer */ #define HDAC_CORBRP 0x4a /* 2 - CORB Read Pointer */ #define HDAC_CORBCTL 0x4c /* 1 - CORB Control */ #define HDAC_CORBSTS 0x4d /* 1 - CORB Status */ #define HDAC_CORBSIZE 0x4e /* 1 - CORB Size */ #define HDAC_RIRBLBASE 0x50 /* 4 - RIRB Lower Base Address */ #define HDAC_RIRBUBASE 0x54 /* 4 - RIRB Upper Base Address */ #define HDAC_RIRBWP 0x58 /* 2 - RIRB Write Pointer */ #define HDAC_RINTCNT 0x5a /* 2 - Response Interrupt Count */ #define HDAC_RIRBCTL 0x5c /* 1 - RIRB Control */ #define HDAC_RIRBSTS 0x5d /* 1 - RIRB Status */ #define HDAC_RIRBSIZE 0x5e /* 1 - RIRB Size */ #define HDAC_ICOI 0x60 /* 4 - Immediate Command Output Interface */ #define HDAC_ICII 0x64 /* 4 - Immediate Command Input Interface */ #define HDAC_ICIS 0x68 /* 2 - Immediate Command Status */ #define HDAC_DPIBLBASE 0x70 /* 4 - DMA Position Buffer Lower Base */ #define HDAC_DPIBUBASE 0x74 /* 4 - DMA Position Buffer Upper Base */ #define HDAC_SDCTL0 0x80 /* 3 - Stream Descriptor Control */ #define HDAC_SDCTL1 0x81 /* 3 - Stream Descriptor Control */ #define HDAC_SDCTL2 0x82 /* 3 - Stream Descriptor Control */ #define HDAC_SDSTS 0x83 /* 1 - Stream Descriptor Status */ #define HDAC_SDLPIB 0x84 /* 4 - Link Position in Buffer */ #define HDAC_SDCBL 0x88 /* 4 - Cyclic Buffer Length */ #define HDAC_SDLVI 0x8C /* 2 - Last Valid Index */ #define HDAC_SDFIFOS 0x90 /* 2 - FIFOS */ #define HDAC_SDFMT 0x92 /* 2 - fmt */ #define HDAC_SDBDPL 0x98 /* 4 - Buffer Descriptor Pointer Lower Base */ #define HDAC_SDBDPU 0x9C /* 4 - Buffer Descriptor Pointer Upper Base */ #define _HDAC_ISDOFFSET(n, iss, oss) (0x80 + ((n) * 0x20)) #define _HDAC_ISDCTL(n, iss, oss) (0x00 + _HDAC_ISDOFFSET(n, iss, oss)) #define _HDAC_ISDSTS(n, iss, oss) (0x03 + _HDAC_ISDOFFSET(n, iss, oss)) #define _HDAC_ISDPICB(n, iss, oss) (0x04 + _HDAC_ISDOFFSET(n, iss, oss)) #define _HDAC_ISDCBL(n, iss, oss) (0x08 + _HDAC_ISDOFFSET(n, iss, oss)) #define _HDAC_ISDLVI(n, iss, oss) (0x0c + _HDAC_ISDOFFSET(n, iss, oss)) #define _HDAC_ISDFIFOD(n, iss, oss) (0x10 + _HDAC_ISDOFFSET(n, iss, oss)) #define _HDAC_ISDFMT(n, iss, oss) (0x12 + _HDAC_ISDOFFSET(n, iss, oss)) #define _HDAC_ISDBDPL(n, iss, oss) (0x18 + _HDAC_ISDOFFSET(n, iss, oss)) #define _HDAC_ISDBDPU(n, iss, oss) (0x1c + _HDAC_ISDOFFSET(n, iss, oss)) #define _HDAC_OSDOFFSET(n, iss, oss) (0x80 + ((iss) * 0x20) + ((n) * 0x20)) #define _HDAC_OSDCTL(n, iss, oss) (0x00 + _HDAC_OSDOFFSET(n, iss, oss)) #define _HDAC_OSDSTS(n, iss, oss) (0x03 + _HDAC_OSDOFFSET(n, iss, oss)) #define _HDAC_OSDPICB(n, iss, oss) (0x04 + _HDAC_OSDOFFSET(n, iss, oss)) #define _HDAC_OSDCBL(n, iss, oss) (0x08 + _HDAC_OSDOFFSET(n, iss, oss)) #define _HDAC_OSDLVI(n, iss, oss) (0x0c + _HDAC_OSDOFFSET(n, iss, oss)) #define _HDAC_OSDFIFOD(n, iss, oss) (0x10 + _HDAC_OSDOFFSET(n, iss, oss)) #define _HDAC_OSDFMT(n, iss, oss) (0x12 + _HDAC_OSDOFFSET(n, iss, oss)) #define _HDAC_OSDBDPL(n, iss, oss) (0x18 + _HDAC_OSDOFFSET(n, iss, oss)) #define _HDAC_OSDBDPU(n, iss, oss) (0x1c + _HDAC_OSDOFFSET(n, iss, oss)) #define _HDAC_BSDOFFSET(n, iss, oss) \ (0x80 + ((iss) * 0x20) + ((oss) * 0x20) + ((n) * 0x20)) #define _HDAC_BSDCTL(n, iss, oss) (0x00 + _HDAC_BSDOFFSET(n, iss, oss)) #define _HDAC_BSDSTS(n, iss, oss) (0x03 + _HDAC_BSDOFFSET(n, iss, oss)) #define _HDAC_BSDPICB(n, iss, oss) (0x04 + _HDAC_BSDOFFSET(n, iss, oss)) #define _HDAC_BSDCBL(n, iss, oss) (0x08 + _HDAC_BSDOFFSET(n, iss, oss)) #define _HDAC_BSDLVI(n, iss, oss) (0x0c + _HDAC_BSDOFFSET(n, iss, oss)) #define _HDAC_BSDFIFOD(n, iss, oss) (0x10 + _HDAC_BSDOFFSET(n, iss, oss)) #define _HDAC_BSDFMT(n, iss, oss) (0x12 + _HDAC_BSDOFFSET(n, iss, oss)) #define _HDAC_BSDBDPL(n, iss, oss) (0x18 + _HDAC_BSDOFFSET(n, iss, oss)) #define _HDAC_BSDBDBU(n, iss, oss) (0x1c + _HDAC_BSDOFFSET(n, iss, oss)) /**************************************************************************** * HDA Controller Register Fields ****************************************************************************/ /* GCAP - Global Capabilities */ #define HDAC_GCAP_64OK 0x0001 #define HDAC_GCAP_NSDO_MASK 0x0006 #define HDAC_GCAP_NSDO_SHIFT 1 #define HDAC_GCAP_BSS_MASK 0x00f8 #define HDAC_GCAP_BSS_SHIFT 3 #define HDAC_GCAP_ISS_MASK 0x0f00 #define HDAC_GCAP_ISS_SHIFT 8 #define HDAC_GCAP_OSS_MASK 0xf000 #define HDAC_GCAP_OSS_SHIFT 12 #define HDAC_GCAP_NSDO_1SDO 0x00 #define HDAC_GCAP_NSDO_2SDO 0x02 #define HDAC_GCAP_NSDO_4SDO 0x04 #define HDAC_GCAP_BSS(gcap) \ (((gcap) & HDAC_GCAP_BSS_MASK) >> HDAC_GCAP_BSS_SHIFT) #define HDAC_GCAP_ISS(gcap) \ (((gcap) & HDAC_GCAP_ISS_MASK) >> HDAC_GCAP_ISS_SHIFT) #define HDAC_GCAP_OSS(gcap) \ (((gcap) & HDAC_GCAP_OSS_MASK) >> HDAC_GCAP_OSS_SHIFT) #define HDAC_GCAP_NSDO(gcap) \ (((gcap) & HDAC_GCAP_NSDO_MASK) >> HDAC_GCAP_NSDO_SHIFT) /* GCTL - Global Control */ #define HDAC_GCTL_CRST 0x00000001 #define HDAC_GCTL_FCNTRL 0x00000002 #define HDAC_GCTL_UNSOL 0x00000100 /* WAKEEN - Wake Enable */ #define HDAC_WAKEEN_SDIWEN_MASK 0x7fff #define HDAC_WAKEEN_SDIWEN_SHIFT 0 /* STATESTS - State Change Status */ #define HDAC_STATESTS_SDIWAKE_MASK 0x7fff #define HDAC_STATESTS_SDIWAKE_SHIFT 0 #define HDAC_STATESTS_SDIWAKE(statests, n) \ (((((statests) & HDAC_STATESTS_SDIWAKE_MASK) >> \ HDAC_STATESTS_SDIWAKE_SHIFT) >> (n)) & 0x0001) /* GSTS - Global Status */ #define HDAC_GSTS_FSTS 0x0002 /* INTCTL - Interrut Control */ #define HDAC_INTCTL_SIE_MASK 0x3fffffff #define HDAC_INTCTL_SIE_SHIFT 0 #define HDAC_INTCTL_CIE 0x40000000 #define HDAC_INTCTL_GIE 0x80000000 /* INTSTS - Interrupt Status */ #define HDAC_INTSTS_SIS_MASK 0x3fffffff #define HDAC_INTSTS_SIS_SHIFT 0 #define HDAC_INTSTS_CIS 0x40000000 #define HDAC_INTSTS_GIS 0x80000000 /* SSYNC - Stream Synchronization */ #define HDAC_SSYNC_SSYNC_MASK 0x3fffffff #define HDAC_SSYNC_SSYNC_SHIFT 0 /* CORBWP - CORB Write Pointer */ #define HDAC_CORBWP_CORBWP_MASK 0x00ff #define HDAC_CORBWP_CORBWP_SHIFT 0 /* CORBRP - CORB Read Pointer */ #define HDAC_CORBRP_CORBRP_MASK 0x00ff #define HDAC_CORBRP_CORBRP_SHIFT 0 #define HDAC_CORBRP_CORBRPRST 0x8000 /* CORBCTL - CORB Control */ #define HDAC_CORBCTL_CMEIE 0x01 #define HDAC_CORBCTL_CORBRUN 0x02 /* CORBSTS - CORB Status */ #define HDAC_CORBSTS_CMEI 0x01 /* CORBSIZE - CORB Size */ #define HDAC_CORBSIZE_CORBSIZE_MASK 0x03 #define HDAC_CORBSIZE_CORBSIZE_SHIFT 0 #define HDAC_CORBSIZE_CORBSZCAP_MASK 0xf0 #define HDAC_CORBSIZE_CORBSZCAP_SHIFT 4 #define HDAC_CORBSIZE_CORBSIZE_2 0x00 #define HDAC_CORBSIZE_CORBSIZE_16 0x01 #define HDAC_CORBSIZE_CORBSIZE_256 0x02 #define HDAC_CORBSIZE_CORBSZCAP_2 0x10 #define HDAC_CORBSIZE_CORBSZCAP_16 0x20 #define HDAC_CORBSIZE_CORBSZCAP_256 0x40 #define HDAC_CORBSIZE_CORBSIZE(corbsize) \ (((corbsize) & HDAC_CORBSIZE_CORBSIZE_MASK) >> HDAC_CORBSIZE_CORBSIZE_SHIFT) /* RIRBWP - RIRB Write Pointer */ #define HDAC_RIRBWP_RIRBWP_MASK 0x00ff #define HDAC_RIRBWP_RIRBWP_SHIFT 0 #define HDAC_RIRBWP_RIRBWPRST 0x8000 /* RINTCTN - Response Interrupt Count */ #define HDAC_RINTCNT_MASK 0x00ff #define HDAC_RINTCNT_SHIFT 0 /* RIRBCTL - RIRB Control */ #define HDAC_RIRBCTL_RINTCTL 0x01 #define HDAC_RIRBCTL_RIRBDMAEN 0x02 #define HDAC_RIRBCTL_RIRBOIC 0x04 /* RIRBSTS - RIRB Status */ #define HDAC_RIRBSTS_RINTFL 0x01 #define HDAC_RIRBSTS_RIRBOIS 0x04 /* RIRBSIZE - RIRB Size */ #define HDAC_RIRBSIZE_RIRBSIZE_MASK 0x03 #define HDAC_RIRBSIZE_RIRBSIZE_SHIFT 0 #define HDAC_RIRBSIZE_RIRBSZCAP_MASK 0xf0 #define HDAC_RIRBSIZE_RIRBSZCAP_SHIFT 4 #define HDAC_RIRBSIZE_RIRBSIZE_2 0x00 #define HDAC_RIRBSIZE_RIRBSIZE_16 0x01 #define HDAC_RIRBSIZE_RIRBSIZE_256 0x02 #define HDAC_RIRBSIZE_RIRBSZCAP_2 0x10 #define HDAC_RIRBSIZE_RIRBSZCAP_16 0x20 #define HDAC_RIRBSIZE_RIRBSZCAP_256 0x40 #define HDAC_RIRBSIZE_RIRBSIZE(rirbsize) \ (((rirbsize) & HDAC_RIRBSIZE_RIRBSIZE_MASK) >> HDAC_RIRBSIZE_RIRBSIZE_SHIFT) /* DPLBASE - DMA Position Lower Base Address */ #define HDAC_DPLBASE_DPLBASE_MASK 0xffffff80 #define HDAC_DPLBASE_DPLBASE_SHIFT 7 #define HDAC_DPLBASE_DPLBASE_DMAPBE 0x00000001 /* SDCTL - Stream Descriptor Control */ #define HDAC_SDCTL_SRST 0x000001 #define HDAC_SDCTL_RUN 0x000002 #define HDAC_SDCTL_IOCE 0x000004 #define HDAC_SDCTL_FEIE 0x000008 #define HDAC_SDCTL_DEIE 0x000010 #define HDAC_SDCTL2_STRIPE_MASK 0x03 #define HDAC_SDCTL2_STRIPE_SHIFT 0 #define HDAC_SDCTL2_TP 0x04 #define HDAC_SDCTL2_DIR 0x08 #define HDAC_SDCTL2_STRM_MASK 0xf0 #define HDAC_SDCTL2_STRM_SHIFT 4 #define HDAC_SDSTS_DESE (1 << 4) #define HDAC_SDSTS_FIFOE (1 << 3) #define HDAC_SDSTS_BCIS (1 << 2) #endif /* _HDAC_REG_H_ */ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2016 Jakub Klama . * Copyright (c) 2018 Alexander Motin * Copyright (c) 2026 Hans Rosenfeld * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer * in this position and unchanged. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * This file and its contents are supplied under the terms of the * Common Development and Distribution License ("CDDL"), version 1.0. * You may only use this file in accordance with the terms of version * 1.0 of the CDDL. * * A full copy of the text of the CDDL should have accompanied this * source. A copy of the CDDL is also available via the Internet at * http://www.illumos.org/license/CDDL. */ /* This file is dual-licensed; see usr/src/contrib/bhyve/LICENSE */ /* * Copyright 2025 Oxide Computer Company */ #include #include #include #include #include #include #include #include #include #include #include #include "iov.h" /* * Given an array of iovecs iov, the number of valid iovecs niov, and an * offset, truncate iov at offset. If necessary, split the final iovec, * moving the remaining iovecs up by one in iov. Return a pointer to the * iovec beginning at offset, and the total number of remaining iovecs. * * The caller must take care that iov contains enough space for at least * niov+1 iovecs so the remainder of the iovec array may be moved up by one. */ struct iovec * split_iov(struct iovec *iov, size_t *niov, size_t offset, size_t *niov_rem) { size_t remainder = 0; struct iovec *iov_rem; size_t i; /* * Handle the special case of offset == 0: Return the whole iovec array * as the remainder. */ if (offset == 0) { *niov_rem = *niov; *niov = 0; return (iov); } /* Seek to the requested offset and truncate the final iovec. */ for (i = 0; i < *niov && offset > iov[i].iov_len; i++) { /* * We're seeking past this iovec. Adjust the offset and move on. */ offset -= iov[i].iov_len; } /* We've reached the end of the array without reaching the offset. */ if (i == *niov) { *niov_rem = 0; return (NULL); } /* * We found the iovec covering offset. Calculate the remainder and * truncate at offset. */ remainder = iov[i].iov_len - offset; iov[i].iov_len = offset; *niov_rem = *niov - i - 1; *niov = i + 1; iov_rem = &iov[*niov]; /* * If there's no remainder in this iovec, we're done. Return the * pointer to the next iovec after the offset, or NULL if there * are no more iovecs beyond offset. */ if (remainder == 0) { if (*niov_rem == 0) iov_rem = NULL; return (iov_rem); } /* * In the (unlikely, ideally) case where there is a remainder from the * final iovec before the split, make room for a new iovec covering the * remainder by moving all following iovecs up. It is the caller's * responsibility that there is enough spare space for this extra iovec. */ for (struct iovec *tmp = &iov_rem[*niov_rem]; tmp != iov_rem; tmp[0] = tmp[-1], tmp--) { ; } /* * Fill in the new first iovec, covering the remainder from the split. */ iov_rem[0].iov_len = remainder; iov_rem[0].iov_base = (char *)iov[i].iov_base + offset; (*niov_rem)++; return (iov_rem); } size_t count_iov(const struct iovec *iov, size_t niov) { size_t total = 0; size_t i; for (i = 0; i < niov; i++) total += iov[i].iov_len; return (total); } bool check_iov_len(const struct iovec *iov, size_t niov, size_t len) { size_t total = 0; size_t i; for (i = 0; i < niov; i++) { total += iov[i].iov_len; if (total >= len) return (true); } return (false); } ssize_t iov_to_buf(const struct iovec *iov, size_t niov, void **buf) { size_t ptr, total; size_t i; total = count_iov(iov, niov); *buf = reallocf(*buf, total); if (*buf == NULL) return (-1); for (i = 0, ptr = 0; i < niov; i++) { memcpy((uint8_t *)*buf + ptr, iov[i].iov_base, iov[i].iov_len); ptr += iov[i].iov_len; } return (total); } ssize_t buf_to_iov(const void *buf, size_t buflen, const struct iovec *iov, size_t niov) { size_t off = 0, len; size_t i; for (i = 0; i < niov && off < buflen; i++) { len = MIN(iov[i].iov_len, buflen - off); memcpy(iov[i].iov_base, (const uint8_t *)buf + off, len); off += len; } return ((ssize_t)off); } size_t iov_bunch_init(iov_bunch_t *iob, struct iovec *iov, int niov) { bzero(iob, sizeof (*iob)); iob->ib_iov = iov; iob->ib_remain = count_iov(iov, niov); return (iob->ib_remain); } /* * Copy `sz` bytes from iovecs contained in `iob` to `dst`. * * Returns `true` if copy was successful (implying adequate data was remaining * in the iov_bunch_t). */ bool iov_bunch_copy(iov_bunch_t *iob, void *dst, size_t sz) { if (sz > iob->ib_remain) return (false); if (sz == 0) return (true); caddr_t dest = dst; do { struct iovec *iov = iob->ib_iov; ASSERT3U(iov->iov_len, !=, 0); /* ib_offset is the offset within the current head of ib_iov */ const size_t iov_avail = iov->iov_len - iob->ib_offset; const size_t to_copy = MIN(sz, iov_avail); if (to_copy != 0 && dest != NULL) { bcopy((caddr_t)iov->iov_base + iob->ib_offset, dest, to_copy); dest += to_copy; } sz -= to_copy; iob->ib_remain -= to_copy; iob->ib_offset += to_copy; ASSERT3U(iob->ib_offset, <=, iov->iov_len); if (iob->ib_offset == iov->iov_len) { iob->ib_iov++; iob->ib_offset = 0; } } while (sz > 0); return (true); } /* * Skip `sz` bytes from iovecs contained in `iob`. * * Returns `true` if the skip was successful (implying adequate data was * remaining in the iov_bunch_t). */ bool iov_bunch_skip(iov_bunch_t *iob, size_t sz) { return (iov_bunch_copy(iob, NULL, sz)); } /* * Get the data pointer and length of the current head iovec, less any * offsetting from prior copy operations. This will advance the iov_bunch_t as * if the caller had performed a copy of that chunk length. * * Returns `true` if the iov_bunch_t had at least one iovec (unconsumed bytes) * remaining, setting `chunk` and `chunk_sz` to the chunk pointer and size, * respectively. */ bool iov_bunch_next_chunk(iov_bunch_t *iob, caddr_t *chunk, size_t *chunk_sz) { if (iob->ib_remain == 0) { *chunk = NULL; *chunk_sz = 0; return (false); } *chunk_sz = iob->ib_iov->iov_len - iob->ib_offset; *chunk = (caddr_t)iob->ib_iov->iov_base + iob->ib_offset; iob->ib_remain -= *chunk_sz; iob->ib_iov++; iob->ib_offset = 0; return (true); } /* * Extract the remaining data in an iov_bunch_t into a new iovec. */ void iov_bunch_to_iov(iov_bunch_t *iob, struct iovec *iov, int *niov, uint_t size) { *niov = 0; while (size-- > 0) { caddr_t chunk; size_t sz; if (!iov_bunch_next_chunk(iob, &chunk, &sz)) break; iov->iov_base = chunk; iov->iov_len = sz; iov++; (*niov)++; } } /* * Extract the remaining data in an iov_bunch_t into a buffer, reallocating it * to the required size. If there are no bytes remaining in the iov_bunch_t any * supplied buffer will be freed and this function will return 0. */ ssize_t iov_bunch_to_buf(iov_bunch_t *iob, void **buf) { size_t total = iob->ib_remain; if (total == 0) { free(*buf); *buf = NULL; return (0); } *buf = reallocf(*buf, total); if (*buf == NULL) return (-1); if (!iov_bunch_copy(iob, buf, total)) return (-1); if (total > SSIZE_MAX) { errno = EOVERFLOW; return (-1); } return (total); } /* * Copy data from a buffer into an iob_bunch_t. Returns true if there was * sufficient space for the data, false otherwise. */ bool buf_to_iov_bunch(iov_bunch_t *iob, const void *buf, size_t len) { const char *src = buf; if (iob->ib_remain < len) return (false); do { struct iovec *iov = iob->ib_iov; /* ib_offset is the offset within the current head of ib_iov */ const size_t iov_avail = iov->iov_len - iob->ib_offset; const size_t to_copy = MIN(len, iov_avail); if (to_copy != 0) { bcopy(src, (caddr_t)iov->iov_base + iob->ib_offset, to_copy); } src += to_copy; len -= to_copy; iob->ib_remain -= to_copy; iob->ib_offset += to_copy; ASSERT3U(iob->ib_offset, <=, iov->iov_len); if (iob->ib_offset == iov->iov_len) { iob->ib_iov++; iob->ib_offset = 0; } } while (len > 0); return (true); } /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2016 Jakub Klama . * Copyright (c) 2018 Alexander Motin * Copyright (c) 2026 Hans Rosenfeld * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer * in this position and unchanged. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * This file and its contents are supplied under the terms of the * Common Development and Distribution License ("CDDL"), version 1.0. * You may only use this file in accordance with the terms of version * 1.0 of the CDDL. * * A full copy of the text of the CDDL should have accompanied this * source. A copy of the CDDL is also available via the Internet at * http://www.illumos.org/license/CDDL. */ /* This file is dual-licensed; see usr/src/contrib/bhyve/LICENSE */ /* * Copyright 2025 Oxide Computer Company */ #ifndef _IOV_H_ #define _IOV_H_ #include /* Number of additional iovecs required for split_iov() */ #define SPLIT_IOV_ADDL_IOV 2 struct iovec *split_iov(struct iovec *, size_t *, size_t, size_t *); size_t count_iov(const struct iovec *, size_t); bool check_iov_len(const struct iovec *, size_t, size_t); ssize_t iov_to_buf(const struct iovec *, size_t, void **); ssize_t buf_to_iov(const void *, size_t, const struct iovec *, size_t); /* * Helpers for performing operations on an array of iovec entries. */ typedef struct iov_bunch { /* * The current head of an array of iovec entries, which have an iov_len * sum covering ib_remain bytes. This moves as the bunch is traversed. */ struct iovec *ib_iov; /* * Byte offset in current ib_iov entry. */ size_t ib_offset; /* * Bytes remaining in entries covered by ib_iov entries, not including * the offset specified by ib_offset */ size_t ib_remain; } iov_bunch_t; extern size_t iov_bunch_init(iov_bunch_t *, struct iovec *, int); extern bool iov_bunch_copy(iov_bunch_t *, void *, size_t); extern bool iov_bunch_skip(iov_bunch_t *, size_t); extern bool iov_bunch_next_chunk(iov_bunch_t *, caddr_t *, size_t *); extern void iov_bunch_to_iov(iov_bunch_t *, struct iovec *, int *, uint_t); extern ssize_t iov_bunch_to_buf(iov_bunch_t *iob, void **); extern bool buf_to_iov_bunch(iov_bunch_t *, const void *, size_t); #endif /* _IOV_H_ */ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2012 NetApp, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * This file and its contents are supplied under the terms of the * Common Development and Distribution License ("CDDL"), version 1.0. * You may only use this file in accordance with the terms of version * 1.0 of the CDDL. * * A full copy of the text of the CDDL should have accompanied this * source. A copy of the CDDL is also available via the Internet at * http://www.illumos.org/license/CDDL. * * Copyright 2020 Oxide Computer Company * Copyright 2026 OmniOS Community Edition (OmniOSce) Association. */ /* * Memory ranges are represented with an RB tree. On insertion, the range * is checked for overlaps. On lookup, the key has the same base and limit * so it can be searched within the range. */ #include #include #include #include #include #include #include #include #include #include #include "mem.h" #include "debug.h" struct mmio_rb_range { RB_ENTRY(mmio_rb_range) mr_link; /* RB tree links */ struct mem_range mr_param; uint64_t mr_base; uint64_t mr_end; }; struct mmio_rb_tree; RB_PROTOTYPE(mmio_rb_tree, mmio_rb_range, mr_link, mmio_rb_range_compare); static RB_HEAD(mmio_rb_tree, mmio_rb_range) mmio_rb_root, mmio_rb_fallback; /* * Per-vCPU cache. Since most accesses from a vCPU will be to * consecutive addresses in a range, it makes sense to cache the * result of a lookup. */ static struct mmio_rb_range **mmio_hint; static int mmio_ncpu; static pthread_rwlock_t mmio_rwlock; static int mmio_rb_range_compare(struct mmio_rb_range *a, struct mmio_rb_range *b) { if (a->mr_end < b->mr_base) return (-1); else if (a->mr_base > b->mr_end) return (1); return (0); } static int mmio_rb_lookup(struct mmio_rb_tree *rbt, uint64_t addr, struct mmio_rb_range **entry) { struct mmio_rb_range find, *res; find.mr_base = find.mr_end = addr; res = RB_FIND(mmio_rb_tree, rbt, &find); if (res != NULL) { *entry = res; return (0); } return (ENOENT); } static void mmio_rb_dump(struct mmio_rb_tree *rbt) { int perror; struct mmio_rb_range *np; pthread_rwlock_rdlock(&mmio_rwlock); RB_FOREACH(np, mmio_rb_tree, rbt) { PRINTLN(" %lx:%lx, %s", np->mr_base, np->mr_end, np->mr_param.name); } perror = pthread_rwlock_unlock(&mmio_rwlock); assert(perror == 0); } static int mmio_rb_add(struct mmio_rb_tree *rbt, struct mmio_rb_range *new) { struct mmio_rb_range *overlap; overlap = RB_INSERT(mmio_rb_tree, rbt, new); if (overlap != NULL) { EPRINTLN("overlap detected: new %lx:%lx, tree %lx:%lx, '%s' " "claims region already claimed for '%s'", new->mr_base, new->mr_end, overlap->mr_base, overlap->mr_end, new->mr_param.name, overlap->mr_param.name); return (EEXIST); } return (0); } RB_GENERATE(mmio_rb_tree, mmio_rb_range, mr_link, mmio_rb_range_compare); typedef int (mem_cb_t)(struct vcpu *vcpu, uint64_t gpa, struct mem_range *mr, void *arg); static int mem_read(struct vcpu *vcpu, uint64_t gpa, uint64_t *rval, int size, void *arg) { int error; struct mem_range *mr = arg; error = (*mr->handler)(vcpu, MEM_F_READ, gpa, size, rval, mr->arg1, mr->arg2); return (error); } static int mem_write(struct vcpu *vcpu, uint64_t gpa, uint64_t wval, int size, void *arg) { int error; struct mem_range *mr = arg; error = (*mr->handler)(vcpu, MEM_F_WRITE, gpa, size, &wval, mr->arg1, mr->arg2); return (error); } static int access_memory(struct vcpu *vcpu, uint64_t paddr, mem_cb_t *cb, void *arg) { struct mmio_rb_range *entry; int err, perror, immutable, vcpuid; vcpuid = vcpu_id(vcpu); pthread_rwlock_rdlock(&mmio_rwlock); /* * First check the per-vCPU cache */ if (mmio_hint[vcpuid] && paddr >= mmio_hint[vcpuid]->mr_base && paddr <= mmio_hint[vcpuid]->mr_end) { entry = mmio_hint[vcpuid]; } else entry = NULL; if (entry == NULL) { if (mmio_rb_lookup(&mmio_rb_root, paddr, &entry) == 0) { /* Update the per-vCPU cache */ mmio_hint[vcpuid] = entry; } else if (mmio_rb_lookup(&mmio_rb_fallback, paddr, &entry)) { perror = pthread_rwlock_unlock(&mmio_rwlock); assert(perror == 0); return (ESRCH); } } assert(entry != NULL); /* * An 'immutable' memory range is guaranteed to be never removed * so there is no need to hold 'mmio_rwlock' while calling the * handler. * * XXX writes to the PCIR_COMMAND register can cause register_mem() * to be called. If the guest is using PCI extended config space * to modify the PCIR_COMMAND register then register_mem() can * deadlock on 'mmio_rwlock'. However by registering the extended * config space window as 'immutable' the deadlock can be avoided. */ immutable = (entry->mr_param.flags & MEM_F_IMMUTABLE); if (immutable) { perror = pthread_rwlock_unlock(&mmio_rwlock); assert(perror == 0); } err = cb(vcpu, paddr, &entry->mr_param, arg); if (!immutable) { perror = pthread_rwlock_unlock(&mmio_rwlock); assert(perror == 0); } return (err); } static int emulate_mem_cb(struct vcpu *vcpu, uint64_t paddr, struct mem_range *mr, void *arg) { struct vm_mmio *mmio; int err = 0; mmio = arg; if (mmio->read != 0) { err = mem_read(vcpu, paddr, &mmio->data, mmio->bytes, mr); } else { err = mem_write(vcpu, paddr, mmio->data, mmio->bytes, mr); } return (err); } int emulate_mem(struct vcpu *vcpu, struct vm_mmio *mmio) { return (access_memory(vcpu, mmio->gpa, emulate_mem_cb, mmio)); } struct rw_mem_args { uint64_t *val; int size; int operation; }; static int rw_mem_cb(struct vcpu *vcpu, uint64_t paddr, struct mem_range *mr, void *arg) { struct rw_mem_args *rma; rma = arg; return (mr->handler(vcpu, rma->operation, paddr, rma->size, rma->val, mr->arg1, mr->arg2)); } int read_mem(struct vcpu *vcpu, uint64_t gpa, uint64_t *rval, int size) { struct rw_mem_args rma; rma.val = rval; rma.size = size; rma.operation = MEM_F_READ; return (access_memory(vcpu, gpa, rw_mem_cb, &rma)); } int write_mem(struct vcpu *vcpu, uint64_t gpa, uint64_t wval, int size) { struct rw_mem_args rma; rma.val = &wval; rma.size = size; rma.operation = MEM_F_WRITE; return (access_memory(vcpu, gpa, rw_mem_cb, &rma)); } static int register_mem_int(struct mmio_rb_tree *rbt, struct mem_range *memp) { struct mmio_rb_range *entry, *mrp; int err, perror; err = 0; mrp = malloc(sizeof(struct mmio_rb_range)); if (mrp == NULL) { warn("%s: couldn't allocate memory for mrp\n", __func__); err = ENOMEM; } else { mrp->mr_param = *memp; mrp->mr_base = memp->base; mrp->mr_end = memp->base + memp->size - 1; pthread_rwlock_wrlock(&mmio_rwlock); if (mmio_rb_lookup(rbt, memp->base, &entry) != 0) err = mmio_rb_add(rbt, mrp); #ifndef __FreeBSD__ else /* smatch warn: possible memory leak of 'mrp' */ free(mrp); #endif perror = pthread_rwlock_unlock(&mmio_rwlock); #ifdef __FreeBSD__ assert(perror == 0); #else if (perror != 0) { mmio_rb_dump(rbt); exit(4); } #endif if (err) free(mrp); } return (err); } int register_mem(struct mem_range *memp) { return (register_mem_int(&mmio_rb_root, memp)); } int register_mem_fallback(struct mem_range *memp) { return (register_mem_int(&mmio_rb_fallback, memp)); } int unregister_mem(struct mem_range *memp) { struct mem_range *mr; struct mmio_rb_range *entry = NULL; int err, perror, i; pthread_rwlock_wrlock(&mmio_rwlock); err = mmio_rb_lookup(&mmio_rb_root, memp->base, &entry); if (err == 0) { mr = &entry->mr_param; assert(mr->name == memp->name); assert(mr->base == memp->base && mr->size == memp->size); assert((mr->flags & MEM_F_IMMUTABLE) == 0); RB_REMOVE(mmio_rb_tree, &mmio_rb_root, entry); /* flush Per-vCPU cache */ for (i = 0; i < mmio_ncpu; i++) { if (mmio_hint[i] == entry) mmio_hint[i] = NULL; } } perror = pthread_rwlock_unlock(&mmio_rwlock); assert(perror == 0); if (entry) free(entry); return (err); } void init_mem(int ncpu) { mmio_ncpu = ncpu; mmio_hint = calloc(ncpu, sizeof(*mmio_hint)); RB_INIT(&mmio_rb_root); RB_INIT(&mmio_rb_fallback); pthread_rwlock_init(&mmio_rwlock, NULL); } /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2012 NetApp, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef _MEM_H_ #define _MEM_H_ #include struct vcpu; typedef int (*mem_func_t)(struct vcpu *vcpu, int dir, uint64_t addr, int size, uint64_t *val, void *arg1, long arg2); struct mem_range { const char *name; int flags; mem_func_t handler; void *arg1; long arg2; uint64_t base; uint64_t size; }; #define MEM_F_READ 0x1 #define MEM_F_WRITE 0x2 #define MEM_F_RW 0x3 #define MEM_F_IMMUTABLE 0x4 /* mem_range cannot be unregistered */ void init_mem(int ncpu); int emulate_mem(struct vcpu *vcpu, struct vm_mmio *mmio); int read_mem(struct vcpu *vpu, uint64_t gpa, uint64_t *rval, int size); int register_mem(struct mem_range *memp); int register_mem_fallback(struct mem_range *memp); int unregister_mem(struct mem_range *memp); int write_mem(struct vcpu *vcpu, uint64_t gpa, uint64_t wval, int size); #endif /* _MEM_H_ */ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2011 NetApp, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * Copyright 2018 Joyent, Inc. * Copyright 2022 OmniOS Community Edition (OmniOSce) Association. */ /* * Micro event library for FreeBSD, designed for a single i/o thread * using kqueue, and having events be persistent by default. */ #include #ifndef WITHOUT_CAPSICUM #include #endif #include #include #include #include #include #include #include #include #include #ifndef WITHOUT_CAPSICUM #include #endif #ifdef __FreeBSD__ #include #else #include #include #include #include #include #include #endif #include #include #include #include "mevent.h" #define MEVENT_MAX 64 #ifndef __FreeBSD__ #define EV_ENABLE 0x01 #define EV_ADD EV_ENABLE #define EV_DISABLE 0x02 #define EV_DELETE 0x04 static int mevent_file_poll_interval_ms = 5000; #endif static pthread_t mevent_tid; static pthread_once_t mevent_once = PTHREAD_ONCE_INIT; #ifdef __FreeBSD__ static int mevent_timid = 43; #endif static int mevent_pipefd[2]; static int mfd; static pthread_mutex_t mevent_lmutex = PTHREAD_MUTEX_INITIALIZER; struct mevent { void (*me_func)(int, enum ev_type, void *); #define me_msecs me_fd int me_fd; #ifdef __FreeBSD__ int me_timid; #else timer_t me_timid; #endif enum ev_type me_type; void *me_param; int me_cq; int me_state; /* Desired kevent flags. */ int me_closefd; int me_fflags; #ifndef __FreeBSD__ port_notify_t me_notify; struct sigevent me_sigev; boolean_t me_auto_requeue; struct { int mp_fd; off_t mp_size; void (*mp_func)(int, enum ev_type, void *); void *mp_param; } me_poll; #endif LIST_ENTRY(mevent) me_list; }; static LIST_HEAD(listhead, mevent) global_head, change_head; static void mevent_qlock(void) { pthread_mutex_lock(&mevent_lmutex); } static void mevent_qunlock(void) { pthread_mutex_unlock(&mevent_lmutex); } static void mevent_pipe_read(int fd, enum ev_type type __unused, void *param __unused) { char buf[MEVENT_MAX]; int status; /* * Drain the pipe read side. The fd is non-blocking so this is * safe to do. */ do { status = read(fd, buf, sizeof(buf)); } while (status == MEVENT_MAX); } static void mevent_notify(void) { char c = '\0'; /* * If calling from outside the i/o thread, write a byte on the * pipe to force the i/o thread to exit the blocking kevent call. */ if (mevent_pipefd[1] != 0 && pthread_self() != mevent_tid) { write(mevent_pipefd[1], &c, 1); } } static void mevent_init(void) { #ifndef WITHOUT_CAPSICUM cap_rights_t rights; #endif #ifdef __FreeBSD__ mfd = kqueue(); #else mfd = port_create(); #endif assert(mfd > 0); #ifndef WITHOUT_CAPSICUM cap_rights_init(&rights, CAP_KQUEUE); if (caph_rights_limit(mfd, &rights) == -1) errx(EX_OSERR, "Unable to apply rights for sandbox"); #endif LIST_INIT(&change_head); LIST_INIT(&global_head); } #ifdef __FreeBSD__ static int mevent_kq_filter(struct mevent *mevp) { int retval; retval = 0; if (mevp->me_type == EVF_READ) retval = EVFILT_READ; if (mevp->me_type == EVF_WRITE) retval = EVFILT_WRITE; if (mevp->me_type == EVF_TIMER) retval = EVFILT_TIMER; if (mevp->me_type == EVF_SIGNAL) retval = EVFILT_SIGNAL; if (mevp->me_type == EVF_VNODE) retval = EVFILT_VNODE; return (retval); } static int mevent_kq_flags(struct mevent *mevp) { int retval; retval = mevp->me_state; if (mevp->me_type == EVF_VNODE) retval |= EV_CLEAR; return (retval); } static int mevent_kq_fflags(struct mevent *mevp) { int retval; retval = 0; switch (mevp->me_type) { case EVF_VNODE: if ((mevp->me_fflags & EVFF_ATTRIB) != 0) retval |= NOTE_ATTRIB; break; case EVF_READ: case EVF_WRITE: case EVF_TIMER: case EVF_SIGNAL: break; } return (retval); } static void mevent_populate(struct mevent *mevp, struct kevent *kev) { if (mevp->me_type == EVF_TIMER) { kev->ident = mevp->me_timid; kev->data = mevp->me_msecs; } else { kev->ident = mevp->me_fd; kev->data = 0; } kev->filter = mevent_kq_filter(mevp); kev->flags = mevent_kq_flags(mevp); kev->fflags = mevent_kq_fflags(mevp); kev->udata = mevp; } static int mevent_build(struct kevent *kev) { struct mevent *mevp, *tmpp; int i; i = 0; mevent_qlock(); LIST_FOREACH_SAFE(mevp, &change_head, me_list, tmpp) { if (mevp->me_closefd) { /* * A close of the file descriptor will remove the * event */ close(mevp->me_fd); } else { assert((mevp->me_state & EV_ADD) == 0); mevent_populate(mevp, &kev[i]); i++; } mevp->me_cq = 0; LIST_REMOVE(mevp, me_list); if (mevp->me_state & EV_DELETE) { free(mevp); } else { LIST_INSERT_HEAD(&global_head, mevp, me_list); } assert(i < MEVENT_MAX); } mevent_qunlock(); return (i); } static void mevent_handle(struct kevent *kev, int numev) { struct mevent *mevp; int i; for (i = 0; i < numev; i++) { mevp = kev[i].udata; /* XXX check for EV_ERROR ? */ (*mevp->me_func)(mevp->me_fd, mevp->me_type, mevp->me_param); } } #else /* __FreeBSD__ */ static boolean_t mevent_clarify_state(struct mevent *mevp) { const int state = mevp->me_state; if ((state & EV_DELETE) != 0) { /* All other intents are overriden by delete. */ mevp->me_state = EV_DELETE; return (B_TRUE); } /* * Without a distinction between EV_ADD and EV_ENABLE in our emulation, * handling the add-disabled case means eliding the portfs operation * when both flags are present. * * This is not a concern for subsequent enable/disable operations, as * mevent_update() toggles the flags properly so they are not left in * conflict. */ if (state == (EV_ENABLE|EV_DISABLE)) { mevp->me_state = EV_DISABLE; return (B_FALSE); } return (B_TRUE); } static void mevent_poll_file_attrib(int fd, enum ev_type type, void *param) { struct mevent *mevp = param; struct stat st; if (fstat(mevp->me_poll.mp_fd, &st) != 0) { (void) fprintf(stderr, "%s: fstat(%d) failed: %s\n", __func__, fd, strerror(errno)); return; } /* * The only current consumer of file attribute monitoring is * blockif, which wants to know about size changes. */ if (mevp->me_poll.mp_size != st.st_size) { mevp->me_poll.mp_size = st.st_size; (*mevp->me_poll.mp_func)(mevp->me_poll.mp_fd, EVF_VNODE, mevp->me_poll.mp_param); } } static void mevent_update_one_readwrite(struct mevent *mevp) { int portfd = mevp->me_notify.portnfy_port; mevp->me_auto_requeue = B_FALSE; switch (mevp->me_state) { case EV_ENABLE: { const int events = (mevp->me_type == EVF_READ) ? POLLIN : POLLOUT; if (port_associate(portfd, PORT_SOURCE_FD, mevp->me_fd, events, mevp) != 0) { (void) fprintf(stderr, "port_associate fd %d %p failed: %s\n", mevp->me_fd, mevp, strerror(errno)); } return; } case EV_DISABLE: case EV_DELETE: /* * A disable that comes in while an event is being * handled will result in an ENOENT. */ if (port_dissociate(portfd, PORT_SOURCE_FD, mevp->me_fd) != 0 && errno != ENOENT) { (void) fprintf(stderr, "port_dissociate " "portfd %d fd %d mevp %p failed: %s\n", portfd, mevp->me_fd, mevp, strerror(errno)); } return; default: (void) fprintf(stderr, "%s: unhandled state %d\n", __func__, mevp->me_state); abort(); } } static void mevent_update_one_timer(struct mevent *mevp) { mevp->me_auto_requeue = B_TRUE; switch (mevp->me_state) { case EV_ENABLE: { struct itimerspec it = { 0 }; mevp->me_sigev.sigev_notify = SIGEV_PORT; mevp->me_sigev.sigev_value.sival_ptr = &mevp->me_notify; if (timer_create(CLOCK_REALTIME, &mevp->me_sigev, &mevp->me_timid) != 0) { (void) fprintf(stderr, "timer_create failed: %s", strerror(errno)); return; } /* The first timeout */ it.it_value.tv_sec = mevp->me_msecs / MILLISEC; it.it_value.tv_nsec = MSEC2NSEC(mevp->me_msecs % MILLISEC); /* Repeat at the same interval */ it.it_interval = it.it_value; if (timer_settime(mevp->me_timid, 0, &it, NULL) != 0) { (void) fprintf(stderr, "timer_settime failed: %s", strerror(errno)); } return; } case EV_DISABLE: case EV_DELETE: if (timer_delete(mevp->me_timid) != 0) { (void) fprintf(stderr, "timer_delete failed: %s", strerror(errno)); } mevp->me_timid = -1; return; default: (void) fprintf(stderr, "%s: unhandled state %d\n", __func__, mevp->me_state); abort(); } } static void mevent_update_one_vnode(struct mevent *mevp) { switch (mevp->me_state) { case EV_ENABLE: { struct stat st; int events = 0; if ((mevp->me_fflags & EVFF_ATTRIB) != 0) events |= FILE_ATTRIB; assert(events != 0); /* * It is tempting to use the PORT_SOURCE_FILE type for this in * conjunction with the FILE_ATTRIB event type. Unfortunately * this event type triggers on any change to the file's * ctime, and therefore for every write as well as attribute * changes. It also does not work for ZVOLs. * * Convert this to a timer event and poll for the file * attribute changes that we care about. */ if (fstat(mevp->me_fd, &st) != 0) { (void) fprintf(stderr, "fstat(%d) failed: %s\n", mevp->me_fd, strerror(errno)); return; } mevp->me_poll.mp_fd = mevp->me_fd; mevp->me_poll.mp_size = st.st_size; mevp->me_poll.mp_func = mevp->me_func; mevp->me_poll.mp_param = mevp->me_param; mevp->me_func = mevent_poll_file_attrib; mevp->me_param = mevp; mevp->me_type = EVF_TIMER; mevp->me_timid = -1; mevp->me_msecs = mevent_file_poll_interval_ms; mevent_update_one_timer(mevp); return; } case EV_DISABLE: case EV_DELETE: /* * These events do not really exist as they are converted to * timers; fall through to abort. */ default: (void) fprintf(stderr, "%s: unhandled state %d\n", __func__, mevp->me_state); abort(); } } static void mevent_update_one(struct mevent *mevp) { switch (mevp->me_type) { case EVF_READ: case EVF_WRITE: mevent_update_one_readwrite(mevp); break; case EVF_TIMER: mevent_update_one_timer(mevp); break; case EVF_VNODE: mevent_update_one_vnode(mevp); break; case EVF_SIGNAL: /* EVF_SIGNAL not yet implemented. */ default: (void) fprintf(stderr, "%s: unhandled event type %d\n", __func__, mevp->me_type); abort(); } } static void mevent_populate(struct mevent *mevp) { mevp->me_notify.portnfy_port = mfd; mevp->me_notify.portnfy_user = mevp; } static void mevent_update_pending() { struct mevent *mevp, *tmpp; mevent_qlock(); LIST_FOREACH_SAFE(mevp, &change_head, me_list, tmpp) { mevent_populate(mevp); if (mevp->me_closefd) { /* * A close of the file descriptor will remove the * event */ (void) close(mevp->me_fd); mevp->me_fd = -1; } else { if (mevent_clarify_state(mevp)) { mevent_update_one(mevp); } } mevp->me_cq = 0; LIST_REMOVE(mevp, me_list); if (mevp->me_state & EV_DELETE) { free(mevp); } else { LIST_INSERT_HEAD(&global_head, mevp, me_list); } } mevent_qunlock(); } static void mevent_handle_pe(port_event_t *pe) { struct mevent *mevp = pe->portev_user; (*mevp->me_func)(mevp->me_fd, mevp->me_type, mevp->me_param); mevent_qlock(); if (!mevp->me_cq && !mevp->me_auto_requeue) { mevent_update_one(mevp); } mevent_qunlock(); } #endif static struct mevent * mevent_add_state(int tfd, enum ev_type type, void (*func)(int, enum ev_type, void *), void *param, int state, int fflags) { #ifdef __FreeBSD__ struct kevent kev; #endif struct mevent *lp, *mevp; #ifdef __FreeBSD__ int ret; #endif if (tfd < 0 || func == NULL) { return (NULL); } mevp = NULL; pthread_once(&mevent_once, mevent_init); mevent_qlock(); /* * Verify that the fd/type tuple is not present in any list */ LIST_FOREACH(lp, &global_head, me_list) { if (type != EVF_TIMER && lp->me_fd == tfd && lp->me_type == type) { goto exit; } } LIST_FOREACH(lp, &change_head, me_list) { if (type != EVF_TIMER && lp->me_fd == tfd && lp->me_type == type) { goto exit; } } /* * Allocate an entry and populate it. */ mevp = calloc(1, sizeof(struct mevent)); if (mevp == NULL) { goto exit; } if (type == EVF_TIMER) { mevp->me_msecs = tfd; #ifdef __FreeBSD__ mevp->me_timid = mevent_timid++; #else mevp->me_timid = -1; #endif } else mevp->me_fd = tfd; mevp->me_type = type; mevp->me_func = func; mevp->me_param = param; mevp->me_state = state; mevp->me_fflags = fflags; /* * Try to add the event. If this fails, report the failure to * the caller. */ #ifdef __FreeBSD__ mevent_populate(mevp, &kev); ret = kevent(mfd, &kev, 1, NULL, 0, NULL); if (ret == -1) { free(mevp); mevp = NULL; goto exit; } mevp->me_state &= ~EV_ADD; #else mevent_populate(mevp); if (mevent_clarify_state(mevp)) mevent_update_one(mevp); #endif LIST_INSERT_HEAD(&global_head, mevp, me_list); exit: mevent_qunlock(); return (mevp); } struct mevent * mevent_add(int tfd, enum ev_type type, void (*func)(int, enum ev_type, void *), void *param) { return (mevent_add_state(tfd, type, func, param, EV_ADD, 0)); } struct mevent * mevent_add_flags(int tfd, enum ev_type type, int fflags, void (*func)(int, enum ev_type, void *), void *param) { return (mevent_add_state(tfd, type, func, param, EV_ADD, fflags)); } struct mevent * mevent_add_disabled(int tfd, enum ev_type type, void (*func)(int, enum ev_type, void *), void *param) { return (mevent_add_state(tfd, type, func, param, EV_ADD | EV_DISABLE, 0)); } static int mevent_update(struct mevent *evp, bool enable) { int newstate; mevent_qlock(); /* * It's not possible to enable/disable a deleted event */ assert((evp->me_state & EV_DELETE) == 0); newstate = evp->me_state; if (enable) { newstate |= EV_ENABLE; newstate &= ~EV_DISABLE; } else { newstate |= EV_DISABLE; newstate &= ~EV_ENABLE; } /* * No update needed if state isn't changing */ if (evp->me_state != newstate) { evp->me_state = newstate; /* * Place the entry onto the changed list if not * already there. */ if (evp->me_cq == 0) { evp->me_cq = 1; LIST_REMOVE(evp, me_list); LIST_INSERT_HEAD(&change_head, evp, me_list); mevent_notify(); } } mevent_qunlock(); return (0); } int mevent_enable(struct mevent *evp) { return (mevent_update(evp, true)); } int mevent_disable(struct mevent *evp) { return (mevent_update(evp, false)); } static int mevent_delete_event(struct mevent *evp, int closefd) { mevent_qlock(); /* * Place the entry onto the changed list if not already there, and * mark as to be deleted. */ if (evp->me_cq == 0) { evp->me_cq = 1; LIST_REMOVE(evp, me_list); LIST_INSERT_HEAD(&change_head, evp, me_list); mevent_notify(); } evp->me_state = EV_DELETE; if (closefd) evp->me_closefd = 1; mevent_qunlock(); return (0); } int mevent_delete(struct mevent *evp) { return (mevent_delete_event(evp, 0)); } int mevent_delete_close(struct mevent *evp) { return (mevent_delete_event(evp, 1)); } static void mevent_set_name(void) { pthread_set_name_np(mevent_tid, "mevent"); } void mevent_dispatch(void) { #ifdef __FreeBSD__ struct kevent changelist[MEVENT_MAX]; struct kevent eventlist[MEVENT_MAX]; struct mevent *pipev; int numev; #else struct mevent *pipev; #endif int ret; #ifndef WITHOUT_CAPSICUM cap_rights_t rights; #endif mevent_tid = pthread_self(); mevent_set_name(); pthread_once(&mevent_once, mevent_init); /* * Open the pipe that will be used for other threads to force * the blocking kqueue call to exit by writing to it. Set the * descriptor to non-blocking. */ ret = pipe(mevent_pipefd); if (ret < 0) { perror("pipe"); exit(0); } #ifndef WITHOUT_CAPSICUM cap_rights_init(&rights, CAP_EVENT, CAP_READ, CAP_WRITE); if (caph_rights_limit(mevent_pipefd[0], &rights) == -1) errx(EX_OSERR, "Unable to apply rights for sandbox"); if (caph_rights_limit(mevent_pipefd[1], &rights) == -1) errx(EX_OSERR, "Unable to apply rights for sandbox"); #endif /* * Add internal event handler for the pipe write fd */ pipev = mevent_add(mevent_pipefd[0], EVF_READ, mevent_pipe_read, NULL); assert(pipev != NULL); for (;;) { #ifdef __FreeBSD__ /* * Build changelist if required. * XXX the changelist can be put into the blocking call * to eliminate the extra syscall. Currently better for * debug. */ numev = mevent_build(changelist); if (numev) { ret = kevent(mfd, changelist, numev, NULL, 0, NULL); if (ret == -1) { perror("Error return from kevent change"); } } /* * Block awaiting events */ ret = kevent(mfd, NULL, 0, eventlist, MEVENT_MAX, NULL); if (ret == -1 && errno != EINTR) { perror("Error return from kevent monitor"); } /* * Handle reported events */ mevent_handle(eventlist, ret); #else /* __FreeBSD__ */ port_event_t pev; /* Handle any pending updates */ mevent_update_pending(); /* Block awaiting events */ ret = port_get(mfd, &pev, NULL); if (ret != 0) { if (errno != EINTR) perror("Error return from port_get"); continue; } /* Handle reported event */ mevent_handle_pe(&pev); #endif /* __FreeBSD__ */ } } /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2011 NetApp, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef _MEVENT_H_ #define _MEVENT_H_ enum ev_type { EVF_READ, EVF_WRITE, EVF_TIMER, EVF_SIGNAL, EVF_VNODE, }; /* Filter flags for EVF_VNODE */ #define EVFF_ATTRIB 0x0001 struct mevent; struct mevent *mevent_add(int fd, enum ev_type type, void (*func)(int, enum ev_type, void *), void *param); struct mevent *mevent_add_flags(int fd, enum ev_type type, int fflags, void (*func)(int, enum ev_type, void *), void *param); struct mevent *mevent_add_disabled(int fd, enum ev_type type, void (*func)(int, enum ev_type, void *), void *param); int mevent_enable(struct mevent *evp); int mevent_disable(struct mevent *evp); int mevent_delete(struct mevent *evp); int mevent_delete_close(struct mevent *evp); void mevent_dispatch(void); #endif /* _MEVENT_H_ */ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright 2024 OmniOS Community Edition (OmniOSce) Association. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, * OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ /* * The illumos dlpi backend */ #include #include #include #include #include #include #include #include "config.h" #include "debug.h" #include "iov.h" #include "mevent.h" #include "net_backends.h" #include "net_backends_priv.h" /* * The size of the bounce buffer used to implement the peek callback. * This value should be big enough to accommodate the largest of all possible * frontend packet lengths. The value here matches the definition of * VTNET_MAX_PKT_LEN in pci_virtio_net.c */ #define DLPI_BBUF_SIZE (65536 + 64) typedef struct be_dlpi_priv { dlpi_handle_t bdp_dhp; struct mevent *bdp_mevp; /* * A bounce buffer that allows us to implement the peek_recvlen * callback. Each structure is only used by a single thread so * one is enough. */ uint8_t bdp_bbuf[DLPI_BBUF_SIZE]; ssize_t bdp_bbuflen; } be_dlpi_priv_t; static void be_dlpi_cleanup(net_backend_t *be) { be_dlpi_priv_t *priv = NET_BE_PRIV(be); if (priv->bdp_dhp != NULL) dlpi_close(priv->bdp_dhp); priv->bdp_dhp = NULL; if (priv->bdp_mevp != NULL) mevent_delete(priv->bdp_mevp); priv->bdp_mevp = NULL; priv->bdp_bbuflen = 0; be->fd = -1; } static void be_dlpi_err(int ret, const char *dev, char *msg) { EPRINTLN("%s: %s (%s)", dev, msg, dlpi_strerror(ret)); } static int be_dlpi_init(net_backend_t *be, const char *devname __unused, nvlist_t *nvl, net_be_rxeof_t cb, void *param) { be_dlpi_priv_t *priv = NET_BE_PRIV(be); const char *vnic; int ret; if (cb == NULL) { EPRINTLN("dlpi backend requires non-NULL callback"); return (-1); } vnic = get_config_value_node(nvl, "vnic"); if (vnic == NULL) { EPRINTLN("dlpi backend requires a VNIC"); return (-1); } priv->bdp_bbuflen = 0; ret = dlpi_open(vnic, &priv->bdp_dhp, DLPI_RAW); if (ret != DLPI_SUCCESS) { be_dlpi_err(ret, vnic, "open failed"); goto error; } if ((ret = dlpi_bind(priv->bdp_dhp, DLPI_ANY_SAP, NULL)) != DLPI_SUCCESS) { be_dlpi_err(ret, vnic, "bind failed"); goto error; } if (get_config_bool_node_default(nvl, "promiscrxonly", true)) { if ((ret = dlpi_promiscon(priv->bdp_dhp, DL_PROMISC_RX_ONLY)) != DLPI_SUCCESS) { be_dlpi_err(ret, vnic, "enable promiscuous mode(rxonly) failed"); goto error; } } if (get_config_bool_node_default(nvl, "promiscphys", false)) { if ((ret = dlpi_promiscon(priv->bdp_dhp, DL_PROMISC_PHYS)) != DLPI_SUCCESS) { be_dlpi_err(ret, vnic, "enable promiscuous mode(physical) failed"); goto error; } } if (get_config_bool_node_default(nvl, "promiscsap", true)) { if ((ret = dlpi_promiscon(priv->bdp_dhp, DL_PROMISC_SAP)) != DLPI_SUCCESS) { be_dlpi_err(ret, vnic, "enable promiscuous mode(SAP) failed"); goto error; } } if (get_config_bool_node_default(nvl, "promiscmulti", true)) { if ((ret = dlpi_promiscon(priv->bdp_dhp, DL_PROMISC_MULTI)) != DLPI_SUCCESS) { be_dlpi_err(ret, vnic, "enable promiscuous mode(muticast) failed"); goto error; } } be->fd = dlpi_fd(priv->bdp_dhp); if (fcntl(be->fd, F_SETFL, O_NONBLOCK) < 0) { EPRINTLN("%s: enable O_NONBLOCK failed", vnic); goto error; } priv->bdp_mevp = mevent_add_disabled(be->fd, EVF_READ, cb, param); if (priv->bdp_mevp == NULL) { EPRINTLN("Could not register event"); goto error; } return (0); error: be_dlpi_cleanup(be); return (-1); } /* * Called to send a buffer chain out to the dlpi device */ static ssize_t be_dlpi_send(net_backend_t *be, const struct iovec *iov, int iovcnt) { be_dlpi_priv_t *priv = NET_BE_PRIV(be); ssize_t len = 0; int ret; if (iovcnt == 1) { len = iov[0].iov_len; ret = dlpi_send(priv->bdp_dhp, NULL, 0, iov[0].iov_base, len, NULL); } else { void *buf = NULL; len = iov_to_buf(iov, iovcnt, &buf); if (len <= 0 || buf == NULL) return (-1); ret = dlpi_send(priv->bdp_dhp, NULL, 0, buf, len, NULL); free(buf); } if (ret != DLPI_SUCCESS) return (-1); return (len); } static ssize_t be_dlpi_peek_recvlen(net_backend_t *be) { be_dlpi_priv_t *priv = NET_BE_PRIV(be); dlpi_recvinfo_t recv; size_t len; int ret; /* * We already have a packet in the bounce buffer. * Just return its length. */ if (priv->bdp_bbuflen > 0) return (priv->bdp_bbuflen); /* * Read the next packet (if any) into the bounce buffer, so * that we get to know its length and we can return that * to the caller. */ len = sizeof (priv->bdp_bbuf); ret = dlpi_recv(priv->bdp_dhp, NULL, NULL, priv->bdp_bbuf, &len, 0, &recv); if (ret == DL_SYSERR) { if (errno == EWOULDBLOCK) return (0); return (-1); } else if (ret == DLPI_ETIMEDOUT) { return (0); } else if (ret != DLPI_SUCCESS) { return (-1); } if (recv.dri_totmsglen > sizeof (priv->bdp_bbuf)) { EPRINTLN("DLPI bounce buffer was too small! - needed %x bytes", recv.dri_totmsglen); } priv->bdp_bbuflen = len; return (len); } static ssize_t be_dlpi_recv(net_backend_t *be, const struct iovec *iov, int iovcnt) { be_dlpi_priv_t *priv = NET_BE_PRIV(be); size_t len; int ret; if (priv->bdp_bbuflen > 0) { /* * A packet is available in the bounce buffer, so * we read it from there. */ len = buf_to_iov(priv->bdp_bbuf, priv->bdp_bbuflen, iov, iovcnt); /* Mark the bounce buffer as empty. */ priv->bdp_bbuflen = 0; if (len == 0) return (-1); return (len); } len = iov[0].iov_len; ret = dlpi_recv(priv->bdp_dhp, NULL, NULL, (uint8_t *)iov[0].iov_base, &len, 0, NULL); if (ret == DL_SYSERR) { if (errno == EWOULDBLOCK) return (0); return (-1); } else if (ret == DLPI_ETIMEDOUT) { return (0); } else if (ret != DLPI_SUCCESS) { return (-1); } return (len); } static void be_dlpi_recv_enable(net_backend_t *be) { be_dlpi_priv_t *priv = NET_BE_PRIV(be); mevent_enable(priv->bdp_mevp); } static void be_dlpi_recv_disable(net_backend_t *be) { be_dlpi_priv_t *priv = NET_BE_PRIV(be); mevent_disable(priv->bdp_mevp); } static uint64_t be_dlpi_get_cap(net_backend_t *be) { return (0); /* no capabilities for now */ } static int be_dlpi_set_cap(net_backend_t *be, uint64_t features, unsigned vnet_hdr_len) { return ((features || vnet_hdr_len) ? -1 : 0); } static int be_dlpi_get_mac(net_backend_t *be, void *buf, size_t *buflen) { be_dlpi_priv_t *priv = NET_BE_PRIV(be); uchar_t physaddr[DLPI_PHYSADDR_MAX]; size_t physaddrlen = DLPI_PHYSADDR_MAX; int ret; if ((ret = dlpi_get_physaddr(priv->bdp_dhp, DL_CURR_PHYS_ADDR, physaddr, &physaddrlen)) != DLPI_SUCCESS) { be_dlpi_err(ret, dlpi_linkname(priv->bdp_dhp), "read MAC address failed"); return (EINVAL); } if (physaddrlen != ETHERADDRL) { EPRINTLN("%s: bad MAC address len %d", dlpi_linkname(priv->bdp_dhp), physaddrlen); return (EINVAL); } if (physaddrlen > *buflen) { EPRINTLN("%s: MAC address too long (%d bytes required)", dlpi_linkname(priv->bdp_dhp), physaddrlen); return (ENOMEM); } *buflen = physaddrlen; memcpy(buf, physaddr, *buflen); return (0); } static struct net_backend dlpi_backend = { .prefix = "dlpi", .priv_size = sizeof(struct be_dlpi_priv), .init = be_dlpi_init, .cleanup = be_dlpi_cleanup, .send = be_dlpi_send, .peek_recvlen = be_dlpi_peek_recvlen, .recv = be_dlpi_recv, .recv_enable = be_dlpi_recv_enable, .recv_disable = be_dlpi_recv_disable, .get_cap = be_dlpi_get_cap, .set_cap = be_dlpi_set_cap, .get_mac = be_dlpi_get_mac, }; DATA_SET(net_backend_set, dlpi_backend); /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2019 Vincenzo Maffione * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, * OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ /* * This file implements multiple network backends (tap, netmap, ...), * to be used by network frontends such as virtio-net and e1000. * The API to access the backend (e.g. send/receive packets, negotiate * features) is exported by net_backends.h. */ #include #ifndef WITHOUT_CAPSICUM #include #endif #include #include #include #include #ifdef __FreeBSD__ #include #endif #include #ifndef WITHOUT_CAPSICUM #include #endif #include #include #include #include #include #include #include #include #include #include #include #include #include #include "config.h" #include "debug.h" #include "iov.h" #include "mevent.h" #include "net_backends.h" #include "net_backends_priv.h" #include "pci_emul.h" #define NET_BE_SIZE(be) (sizeof(*be) + (be)->priv_size) #ifdef __FreeBSD__ void tap_cleanup(struct net_backend *be) { struct tap_priv *priv = NET_BE_PRIV(be); if (priv->mevp) { mevent_delete(priv->mevp); } if (be->fd != -1) { close(be->fd); be->fd = -1; } } static int tap_init(struct net_backend *be, const char *devname, nvlist_t *nvl __unused, net_be_rxeof_t cb, void *param) { struct tap_priv *priv = NET_BE_PRIV(be); char tbuf[80]; int opt = 1, up = IFF_UP; #ifndef WITHOUT_CAPSICUM cap_rights_t rights; #endif if (cb == NULL) { EPRINTLN("TAP backend requires non-NULL callback"); return (-1); } strcpy(tbuf, "/dev/"); strlcat(tbuf, devname, sizeof(tbuf)); be->fd = open(tbuf, O_RDWR); if (be->fd == -1) { EPRINTLN("open of tap device %s failed", tbuf); goto error; } /* * Set non-blocking and register for read * notifications with the event loop */ if (ioctl(be->fd, FIONBIO, &opt) < 0) { EPRINTLN("tap device O_NONBLOCK failed"); goto error; } if (ioctl(be->fd, VMIO_SIOCSIFFLAGS, up)) { EPRINTLN("tap device link up failed"); goto error; } #ifndef WITHOUT_CAPSICUM cap_rights_init(&rights, CAP_EVENT, CAP_READ, CAP_WRITE); if (caph_rights_limit(be->fd, &rights) == -1) errx(EX_OSERR, "Unable to apply rights for sandbox"); #endif memset(priv->bbuf, 0, sizeof(priv->bbuf)); priv->bbuflen = 0; priv->mevp = mevent_add_disabled(be->fd, EVF_READ, cb, param); if (priv->mevp == NULL) { EPRINTLN("Could not register event"); goto error; } return (0); error: tap_cleanup(be); return (-1); } /* * Called to send a buffer chain out to the tap device */ ssize_t tap_send(struct net_backend *be, const struct iovec *iov, int iovcnt) { return (writev(be->fd, iov, iovcnt)); } ssize_t tap_peek_recvlen(struct net_backend *be) { struct tap_priv *priv = NET_BE_PRIV(be); ssize_t ret; if (priv->bbuflen > 0) { /* * We already have a packet in the bounce buffer. * Just return its length. */ return priv->bbuflen; } /* * Read the next packet (if any) into the bounce buffer, so * that we get to know its length and we can return that * to the caller. */ ret = read(be->fd, priv->bbuf, sizeof(priv->bbuf)); if (ret < 0 && errno == EWOULDBLOCK) { return (0); } if (ret > 0) priv->bbuflen = ret; return (ret); } ssize_t tap_recv(struct net_backend *be, const struct iovec *iov, int iovcnt) { struct tap_priv *priv = NET_BE_PRIV(be); ssize_t ret; if (priv->bbuflen > 0) { /* * A packet is available in the bounce buffer, so * we read it from there. */ ret = buf_to_iov(priv->bbuf, priv->bbuflen, iov, iovcnt); /* Mark the bounce buffer as empty. */ priv->bbuflen = 0; return (ret); } ret = readv(be->fd, iov, iovcnt); if (ret < 0 && errno == EWOULDBLOCK) { return (0); } return (ret); } void tap_recv_enable(struct net_backend *be) { struct tap_priv *priv = NET_BE_PRIV(be); mevent_enable(priv->mevp); } void tap_recv_disable(struct net_backend *be) { struct tap_priv *priv = NET_BE_PRIV(be); mevent_disable(priv->mevp); } uint64_t tap_get_cap(struct net_backend *be __unused) { return (0); /* no capabilities for now */ } int tap_set_cap(struct net_backend *be __unused, uint64_t features, unsigned vnet_hdr_len) { return ((features || vnet_hdr_len) ? -1 : 0); } static struct net_backend tap_backend = { .prefix = "tap", .priv_size = sizeof(struct tap_priv), .init = tap_init, .cleanup = tap_cleanup, .send = tap_send, .peek_recvlen = tap_peek_recvlen, .recv = tap_recv, .recv_enable = tap_recv_enable, .recv_disable = tap_recv_disable, .get_cap = tap_get_cap, .set_cap = tap_set_cap, }; /* A clone of the tap backend, with a different prefix. */ static struct net_backend vmnet_backend = { .prefix = "vmnet", .priv_size = sizeof(struct tap_priv), .init = tap_init, .cleanup = tap_cleanup, .send = tap_send, .peek_recvlen = tap_peek_recvlen, .recv = tap_recv, .recv_enable = tap_recv_enable, .recv_disable = tap_recv_disable, .get_cap = tap_get_cap, .set_cap = tap_set_cap, }; DATA_SET(net_backend_set, tap_backend); DATA_SET(net_backend_set, vmnet_backend); #endif /* __FreeBSD__ */ #ifdef __FreeBSD__ int netbe_legacy_config(nvlist_t *nvl, const char *opts) { char *backend, *cp; if (opts == NULL) return (0); cp = strchr(opts, ','); if (cp == NULL) { set_config_value_node(nvl, "backend", opts); return (0); } backend = strndup(opts, cp - opts); set_config_value_node(nvl, "backend", backend); free(backend); return (pci_parse_legacy_config(nvl, cp + 1)); } #else int netbe_legacy_config(nvlist_t *nvl, const char *opts) { char *config, *name, *tofree, *value; if (opts == NULL) return (0); /* Default to the 'dlpi' backend - can still be overridden by opts */ set_config_value_node(nvl, "backend", "dlpi"); set_config_value_node(nvl, "type", "dlpi"); config = tofree = strdup(opts); if (config == NULL) err(4, "netbe_legacy_config strdup()"); while ((name = strsep(&config, ",")) != NULL) { value = strchr(name, '='); if (value != NULL) { *value++ = '\0'; set_config_value_node(nvl, name, value); } else { set_config_value_node(nvl, "vnic", name); } } free(tofree); return (0); } #endif /* * Initialize a backend and attach to the frontend. * This is called during frontend initialization. * @ret is a pointer to the backend to be initialized * @devname is the backend-name as supplied on the command line, * e.g. -s 2:0,frontend-name,backend-name[,other-args] * @cb is the receive callback supplied by the frontend, * and it is invoked in the event loop when a receive * event is generated in the hypervisor, * @param is a pointer to the frontend, and normally used as * the argument for the callback. */ int netbe_init(struct net_backend **ret, nvlist_t *nvl, net_be_rxeof_t cb, void *param) { struct net_backend **pbe, *nbe, *tbe = NULL; const char *value, *type; char *devname; int err; value = get_config_value_node(nvl, "backend"); if (value == NULL) { return (-1); } devname = strdup(value); /* * Use the type given by configuration if exists; otherwise * use the prefix of the backend as the type. */ type = get_config_value_node(nvl, "type"); if (type == NULL) type = devname; /* * Find the network backend that matches the user-provided * device name. net_backend_set is built using a linker set. */ SET_FOREACH(pbe, net_backend_set) { if (strncmp(type, (*pbe)->prefix, strlen((*pbe)->prefix)) == 0) { tbe = *pbe; assert(tbe->init != NULL); assert(tbe->cleanup != NULL); assert(tbe->send != NULL); assert(tbe->recv != NULL); assert(tbe->get_cap != NULL); assert(tbe->set_cap != NULL); break; } } *ret = NULL; if (tbe == NULL) { free(devname); return (EINVAL); } nbe = calloc(1, NET_BE_SIZE(tbe)); *nbe = *tbe; /* copy the template */ nbe->fd = -1; nbe->sc = param; nbe->be_vnet_hdr_len = 0; nbe->fe_vnet_hdr_len = 0; /* Initialize the backend. */ err = nbe->init(nbe, devname, nvl, cb, param); if (err) { free(devname); free(nbe); return (err); } *ret = nbe; free(devname); return (0); } void netbe_cleanup(struct net_backend *be) { if (be != NULL) { be->cleanup(be); free(be); } } uint64_t netbe_get_cap(struct net_backend *be) { assert(be != NULL); return (be->get_cap(be)); } int netbe_set_cap(struct net_backend *be, uint64_t features, unsigned vnet_hdr_len) { int ret; assert(be != NULL); /* There are only three valid lengths, i.e., 0, 10 and 12. */ if (vnet_hdr_len && vnet_hdr_len != VNET_HDR_LEN && vnet_hdr_len != (VNET_HDR_LEN - sizeof(uint16_t))) return (-1); be->fe_vnet_hdr_len = vnet_hdr_len; ret = be->set_cap(be, features, vnet_hdr_len); assert(be->be_vnet_hdr_len == 0 || be->be_vnet_hdr_len == be->fe_vnet_hdr_len); return (ret); } ssize_t netbe_send(struct net_backend *be, const struct iovec *iov, int iovcnt) { return (be->send(be, iov, iovcnt)); } ssize_t netbe_peek_recvlen(struct net_backend *be) { return (be->peek_recvlen(be)); } /* * Try to read a packet from the backend, without blocking. * If no packets are available, return 0. In case of success, return * the length of the packet just read. Return -1 in case of errors. */ ssize_t netbe_recv(struct net_backend *be, const struct iovec *iov, int iovcnt) { return (be->recv(be, iov, iovcnt)); } /* * Read a packet from the backend and discard it. * Returns the size of the discarded packet or zero if no packet was available. * A negative error code is returned in case of read error. */ ssize_t netbe_rx_discard(struct net_backend *be) { /* * MP note: the dummybuf is only used to discard frames, * so there is no need for it to be per-vtnet or locked. * We only make it large enough for TSO-sized segment. */ static uint8_t dummybuf[65536 + 64]; struct iovec iov; #ifdef __FreeBSD__ iov.iov_base = dummybuf; #else iov.iov_base = (caddr_t)dummybuf; #endif iov.iov_len = sizeof(dummybuf); return netbe_recv(be, &iov, 1); } void netbe_rx_disable(struct net_backend *be) { return be->recv_disable(be); } void netbe_rx_enable(struct net_backend *be) { return be->recv_enable(be); } size_t netbe_get_vnet_hdr_len(struct net_backend *be) { return (be->be_vnet_hdr_len); } #ifndef __FreeBSD__ int netbe_get_mac(net_backend_t *be, void *buf, size_t *buflen) { if (be->get_mac == NULL) return (ENOTSUP); return (be->get_mac(be, buf, buflen)); } #endif /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2019 Vincenzo Maffione * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, * OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ /* * This file and its contents are supplied under the terms of the * Common Development and Distribution License ("CDDL"), version 1.0. * You may only use this file in accordance with the terms of version * 1.0 of the CDDL. * * A full copy of the text of the CDDL should have accompanied this * source. A copy of the CDDL is also available via the Internet at * http://www.illumos.org/license/CDDL. */ /* This file is dual-licensed; see usr/src/contrib/bhyve/LICENSE */ /* * Copyright 2025 Oxide Computer Company */ #ifndef __NET_BACKENDS_H__ #define __NET_BACKENDS_H__ #include /* Opaque type representing a network backend. */ typedef struct net_backend net_backend_t; /* Interface between network frontends and the network backends. */ typedef void (*net_be_rxeof_t)(int, enum ev_type, void *param); int netbe_init(net_backend_t **be, nvlist_t *nvl, net_be_rxeof_t cb, void *param); int netbe_legacy_config(nvlist_t *nvl, const char *opts); void netbe_cleanup(net_backend_t *be); uint64_t netbe_get_cap(net_backend_t *be); int netbe_set_cap(net_backend_t *be, uint64_t cap, unsigned vnet_hdr_len); size_t netbe_get_vnet_hdr_len(net_backend_t *be); ssize_t netbe_send(net_backend_t *be, const struct iovec *iov, int iovcnt); ssize_t netbe_peek_recvlen(net_backend_t *be); ssize_t netbe_recv(net_backend_t *be, const struct iovec *iov, int iovcnt); ssize_t netbe_rx_discard(net_backend_t *be); void netbe_rx_disable(net_backend_t *be); void netbe_rx_enable(net_backend_t *be); #ifndef __FreeBSD__ int netbe_get_mac(net_backend_t *, void *, size_t *); #endif /* * Network device capabilities taken from the VirtIO standard. * Despite the name, these capabilities can be used by different frontends * (virtio-net, ptnet) and supported by different backends (netmap, tap, ...). */ #define VIRTIO_NET_F_CSUM (1ULL << 0) /* host handles partial cksum */ #define VIRTIO_NET_F_GUEST_CSUM (1ULL << 1) /* guest handles partial cksum */ #define VIRTIO_NET_F_MTU (1ULL << 3) /* initial MTU advice */ #define VIRTIO_NET_F_MAC (1ULL << 5) /* host supplies MAC */ #define VIRTIO_NET_F_GSO_DEPREC (1ULL << 6) /* deprecated: host handles GSO */ #define VIRTIO_NET_F_GUEST_TSO4 (1ULL << 7) /* guest can rcv TSOv4 */ #define VIRTIO_NET_F_GUEST_TSO6 (1ULL << 8) /* guest can rcv TSOv6 */ #define VIRTIO_NET_F_GUEST_ECN (1ULL << 9) /* guest can rcv TSO with ECN */ #define VIRTIO_NET_F_GUEST_UFO (1ULL << 10) /* guest can rcv UFO */ #define VIRTIO_NET_F_HOST_TSO4 (1ULL << 11) /* host can rcv TSOv4 */ #define VIRTIO_NET_F_HOST_TSO6 (1ULL << 12) /* host can rcv TSOv6 */ #define VIRTIO_NET_F_HOST_ECN (1ULL << 13) /* host can rcv TSO with ECN */ #define VIRTIO_NET_F_HOST_UFO (1ULL << 14) /* host can rcv UFO */ #define VIRTIO_NET_F_MRG_RXBUF (1ULL << 15) /* host can merge RX buffers */ #define VIRTIO_NET_F_STATUS (1ULL << 16) /* config status field available */ #define VIRTIO_NET_F_CTRL_VQ (1ULL << 17) /* control chan available */ #define VIRTIO_NET_F_CTRL_RX (1ULL << 18) /* control chan RX mode support */ #define VIRTIO_NET_F_CTRL_VLAN (1ULL << 19) /* control chan VLAN filtering */ #define VIRTIO_NET_F_GUEST_ANNOUNCE \ (1ULL << 21) /* guest can send gratuit. pkts */ #define VIRTIO_NET_F_MQ (1ULL << 22) /* host supp multiple VQ pairs */ #define VIRTIO_F_CTRL_MAC_ADDR (1ULL << 23) /* set MAC address through ctrlq */ #define VIRTIO_NET_F_SPEED_DUPLEX \ (1ULL << 63) /* speed/duplex fields avail */ /* * PCI config-space "registers"; device configuration fields. */ struct virtio_net_config { uint8_t vnc_macaddr[6]; uint16_t vnc_status; uint16_t vnc_max_qpair; uint16_t vnc_mtu; uint32_t vnc_speed; /* Specified in Mb/s */ uint8_t vnc_duplex; } __packed; #define VIRTIO_NET_S_LINK_UP 1 #define VIRTIO_NET_S_ANNOUNCE 2 #define VIRTIO_NET_SPEED_UNKNOWN UINT32_MAX #define VIRTIO_NET_DUPLEX_HALF 0 #define VIRTIO_NET_DUPLEX_FULL 1 #define VIRTIO_NET_DUPLEX_UNK 0xff /* * Fixed network header size */ struct virtio_net_rxhdr { uint8_t vrh_flags; uint8_t vrh_gso_type; uint16_t vrh_hdr_len; uint16_t vrh_gso_size; uint16_t vrh_csum_start; uint16_t vrh_csum_offset; uint16_t vrh_bufs; } __packed; #endif /* __NET_BACKENDS_H__ */ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2019 Vincenzo Maffione * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, * OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef __NET_BACKENDS_PRIV_H__ #define __NET_BACKENDS_PRIV_H__ #include /* * Each network backend registers a set of function pointers that are * used to implement the net backends API. * This might need to be exposed if we implement backends in separate files. */ struct net_backend { const char *prefix; /* prefix matching this backend */ /* * Routines used to initialize and cleanup the resources needed * by a backend. The cleanup function is used internally, * and should not be called by the frontend. */ int (*init)(struct net_backend *be, const char *devname, nvlist_t *nvl, net_be_rxeof_t cb, void *param); void (*cleanup)(struct net_backend *be); /* * Called to serve a guest transmit request. The scatter-gather * vector provided by the caller has 'iovcnt' elements and contains * the packet to send. */ ssize_t (*send)(struct net_backend *be, const struct iovec *iov, int iovcnt); /* * Get the length of the next packet that can be received from * the backend. If no packets are currently available, this * function returns 0. */ ssize_t (*peek_recvlen)(struct net_backend *be); /* * Called to receive a packet from the backend. When the function * returns a positive value 'len', the scatter-gather vector * provided by the caller contains a packet with such length. * The function returns 0 if the backend doesn't have a new packet to * receive. */ ssize_t (*recv)(struct net_backend *be, const struct iovec *iov, int iovcnt); /* * Ask the backend to enable or disable receive operation in the * backend. On return from a disable operation, it is guaranteed * that the receive callback won't be called until receive is * enabled again. Note however that it is up to the caller to make * sure that netbe_recv() is not currently being executed by another * thread. */ void (*recv_enable)(struct net_backend *be); void (*recv_disable)(struct net_backend *be); /* * Ask the backend for the virtio-net features it is able to * support. Possible features are TSO, UFO and checksum offloading * in both rx and tx direction and for both IPv4 and IPv6. */ uint64_t (*get_cap)(struct net_backend *be); /* * Tell the backend to enable/disable the specified virtio-net * features (capabilities). */ int (*set_cap)(struct net_backend *be, uint64_t features, unsigned int vnet_hdr_len); #ifndef __FreeBSD__ int (*get_mac)(struct net_backend *be, void *, size_t *); #endif struct pci_vtnet_softc *sc; int fd; /* * Length of the virtio-net header used by the backend and the * frontend, respectively. A zero value means that the header * is not used. */ unsigned int be_vnet_hdr_len; unsigned int fe_vnet_hdr_len; /* Size of backend-specific private data. */ size_t priv_size; /* Backend-specific private data follows. */ }; #define NET_BE_PRIV(be) ((void *)((be) + 1)) SET_DECLARE(net_backend_set, struct net_backend); #define VNET_HDR_LEN sizeof(struct virtio_net_rxhdr) #ifdef __FreeBSD__ /* * Export the tap backend routines for the benefit of other backends which have * a similar interface to the kernel, i.e., they send and receive data using * standard I/O system calls with a single file descriptor. */ struct tap_priv { struct mevent *mevp; /* * A bounce buffer that allows us to implement the peek_recvlen * callback. In the future we may get the same information from * the kevent data. */ char bbuf[1 << 16]; ssize_t bbuflen; }; void tap_cleanup(struct net_backend *be); ssize_t tap_send(struct net_backend *be, const struct iovec *iov, int io vcnt); ssize_t tap_recv(struct net_backend *be, const struct iovec *iov, int io vcnt); ssize_t tap_peek_recvlen(struct net_backend *be); void tap_recv_enable(struct net_backend *be); ssize_t tap_recv(struct net_backend *be, const struct iovec *iov, int io vcnt); void tap_recv_disable(struct net_backend *be); uint64_t tap_get_cap(struct net_backend *be); int tap_set_cap(struct net_backend *be, uint64_t features, unsigned vnet_hdr_len); #endif /* __FreeBSD__ */ #endif /* !__NET_BACKENDS_PRIV_H__ */ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2011 NetApp, Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, * OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include #include #include #include #include #include #include #include #include #include "bhyverun.h" #include "config.h" #include "debug.h" #include "net_utils.h" int net_parsemac(const char *mac_str, uint8_t *mac_addr) { struct ether_addr *ea; char zero_addr[ETHER_ADDR_LEN] = { 0, 0, 0, 0, 0, 0 }; if (mac_str == NULL) return (EINVAL); ea = ether_aton(mac_str); if (ea == NULL || ETHER_IS_MULTICAST(ea->octet) || memcmp(ea->octet, zero_addr, ETHER_ADDR_LEN) == 0) { EPRINTLN("Invalid MAC %s", mac_str); return (EINVAL); } else memcpy(mac_addr, ea->octet, ETHER_ADDR_LEN); return (0); } int net_parsemtu(const char *mtu_str, unsigned long *mtu) { char *end; unsigned long val; assert(mtu_str != NULL); if (*mtu_str == '-') goto err; val = strtoul(mtu_str, &end, 0); if (*end != '\0') goto err; if (val == ULONG_MAX) return (ERANGE); if (val == 0 && errno == EINVAL) return (EINVAL); *mtu = val; return (0); err: errno = EINVAL; return (EINVAL); } void net_genmac(struct pci_devinst *pi, uint8_t *macaddr) { /* * The default MAC address is the standard NetApp OUI of 00-a0-98, * followed by an MD5 of the PCI slot/func number and dev name */ MD5_CTX mdctx; unsigned char digest[16]; char nstr[80]; snprintf(nstr, sizeof(nstr), "%d-%d-%s", pi->pi_slot, pi->pi_func, get_config_value("name")); MD5Init(&mdctx); MD5Update(&mdctx, nstr, (unsigned int)strlen(nstr)); MD5Final(digest, &mdctx); macaddr[0] = 0x00; macaddr[1] = 0xa0; macaddr[2] = 0x98; macaddr[3] = digest[0]; macaddr[4] = digest[1]; macaddr[5] = digest[2]; } /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2019 Vincenzo Maffione * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, * OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef _NET_UTILS_H_ #define _NET_UTILS_H_ #include #include "pci_emul.h" void net_genmac(struct pci_devinst *pi, uint8_t *macaddr); int net_parsemac(const char *mac_str, uint8_t *mac_addr); int net_parsemtu(const char *mtu_str, unsigned long *mtu); #endif /* _NET_UTILS_H_ */ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2013 Zhixiang Yu * Copyright (c) 2015-2016 Alexander Motin * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include #include #include #include #include #include #include #include #ifndef __FreeBSD__ #include #endif #include #include #include #include #include #include #include #include #include #include #include #include #include #include "bhyverun.h" #include "config.h" #include "debug.h" #include "pci_emul.h" #include "ahci.h" #include "block_if.h" #define DEF_PORTS 6 /* Intel ICH8 AHCI supports 6 ports */ #define MAX_PORTS 32 /* AHCI supports 32 ports */ #define PxSIG_ATA 0x00000101 /* ATA drive */ #define PxSIG_ATAPI 0xeb140101 /* ATAPI drive */ enum sata_fis_type { FIS_TYPE_REGH2D = 0x27, /* Register FIS - host to device */ FIS_TYPE_REGD2H = 0x34, /* Register FIS - device to host */ FIS_TYPE_DMAACT = 0x39, /* DMA activate FIS - device to host */ FIS_TYPE_DMASETUP = 0x41, /* DMA setup FIS - bidirectional */ FIS_TYPE_DATA = 0x46, /* Data FIS - bidirectional */ FIS_TYPE_BIST = 0x58, /* BIST activate FIS - bidirectional */ FIS_TYPE_PIOSETUP = 0x5F, /* PIO setup FIS - device to host */ FIS_TYPE_SETDEVBITS = 0xA1, /* Set dev bits FIS - device to host */ }; /* * SCSI opcodes */ #define TEST_UNIT_READY 0x00 #define REQUEST_SENSE 0x03 #define INQUIRY 0x12 #define START_STOP_UNIT 0x1B #define PREVENT_ALLOW 0x1E #define READ_CAPACITY 0x25 #define READ_10 0x28 #define POSITION_TO_ELEMENT 0x2B #define READ_TOC 0x43 #define GET_EVENT_STATUS_NOTIFICATION 0x4A #define MODE_SENSE_10 0x5A #define REPORT_LUNS 0xA0 #define READ_12 0xA8 #define READ_CD 0xBE /* * SCSI mode page codes */ #define MODEPAGE_RW_ERROR_RECOVERY 0x01 #define MODEPAGE_CD_CAPABILITIES 0x2A /* * ATA commands */ #define ATA_SF_ENAB_SATA_SF 0x10 #define ATA_SATA_SF_AN 0x05 #define ATA_SF_DIS_SATA_SF 0x90 /* * Debug printf */ #ifdef AHCI_DEBUG static FILE *dbg; #define DPRINTF(format, arg...) do{fprintf(dbg, format, ##arg);fflush(dbg);}while(0) #else #define DPRINTF(format, arg...) #endif #define AHCI_PORT_IDENT 20 + 1 struct ahci_ioreq { struct blockif_req io_req; struct ahci_port *io_pr; STAILQ_ENTRY(ahci_ioreq) io_flist; TAILQ_ENTRY(ahci_ioreq) io_blist; uint8_t *cfis; uint8_t *dsm; uint32_t len; uint32_t done; int slot; int more; }; struct ahci_port { struct blockif_ctxt *bctx; struct pci_ahci_softc *pr_sc; struct ata_params ata_ident; uint8_t *cmd_lst; uint8_t *rfis; int port; int atapi; int reset; int waitforclear; int mult_sectors; uint8_t xfermode; uint8_t err_cfis[20]; uint8_t sense_key; uint8_t asc; u_int ccs; uint32_t pending; uint32_t clb; uint32_t clbu; uint32_t fb; uint32_t fbu; uint32_t is; uint32_t ie; uint32_t cmd; uint32_t unused0; uint32_t tfd; uint32_t sig; uint32_t ssts; uint32_t sctl; uint32_t serr; uint32_t sact; uint32_t ci; uint32_t sntf; uint32_t fbs; /* * i/o request info */ struct ahci_ioreq *ioreq; int ioqsz; STAILQ_HEAD(ahci_fhead, ahci_ioreq) iofhd; TAILQ_HEAD(ahci_bhead, ahci_ioreq) iobhd; }; struct ahci_cmd_hdr { uint16_t flags; uint16_t prdtl; uint32_t prdbc; uint64_t ctba; uint32_t reserved[4]; }; struct ahci_prdt_entry { uint64_t dba; uint32_t reserved; #define DBCMASK 0x3fffff uint32_t dbc; }; struct pci_ahci_softc { struct pci_devinst *asc_pi; pthread_mutex_t mtx; int ports; uint32_t cap; uint32_t ghc; uint32_t is; uint32_t pi; uint32_t vs; uint32_t ccc_ctl; uint32_t ccc_pts; uint32_t em_loc; uint32_t em_ctl; uint32_t cap2; uint32_t bohc; uint32_t lintr; struct ahci_port port[MAX_PORTS]; }; #define ahci_ctx(sc) ((sc)->asc_pi->pi_vmctx) static void ahci_handle_next_trim(struct ahci_port *p, int slot, uint8_t *cfis, uint8_t *buf, uint32_t len, uint32_t done); static void ahci_handle_port(struct ahci_port *p); static inline void lba_to_msf(uint8_t *buf, int lba) { lba += 150; buf[0] = (lba / 75) / 60; buf[1] = (lba / 75) % 60; buf[2] = lba % 75; } /* * Generate HBA interrupts on global IS register write. */ static void ahci_generate_intr(struct pci_ahci_softc *sc, uint32_t mask) { struct pci_devinst *pi = sc->asc_pi; struct ahci_port *p; int i, nmsg; uint32_t mmask; /* Update global IS from PxIS/PxIE. */ for (i = 0; i < sc->ports; i++) { p = &sc->port[i]; if (p->is & p->ie) sc->is |= (1 << i); } DPRINTF("%s(%08x) %08x", __func__, mask, sc->is); /* If there is nothing enabled -- clear legacy interrupt and exit. */ if (sc->is == 0 || (sc->ghc & AHCI_GHC_IE) == 0) { if (sc->lintr) { pci_lintr_deassert(pi); sc->lintr = 0; } return; } /* If there is anything and no MSI -- assert legacy interrupt. */ nmsg = pci_msi_maxmsgnum(pi); if (nmsg == 0) { if (!sc->lintr) { sc->lintr = 1; pci_lintr_assert(pi); } return; } /* Assert respective MSIs for ports that were touched. */ for (i = 0; i < nmsg; i++) { if (sc->ports <= nmsg || i < nmsg - 1) mmask = 1 << i; else mmask = 0xffffffff << i; if (sc->is & mask && mmask & mask) pci_generate_msi(pi, i); } } /* * Generate HBA interrupt on specific port event. */ static void ahci_port_intr(struct ahci_port *p) { struct pci_ahci_softc *sc = p->pr_sc; struct pci_devinst *pi = sc->asc_pi; int nmsg; DPRINTF("%s(%d) %08x/%08x %08x", __func__, p->port, p->is, p->ie, sc->is); /* If there is nothing enabled -- we are done. */ if ((p->is & p->ie) == 0) return; /* In case of non-shared MSI always generate interrupt. */ nmsg = pci_msi_maxmsgnum(pi); if (sc->ports <= nmsg || p->port < nmsg - 1) { sc->is |= (1 << p->port); if ((sc->ghc & AHCI_GHC_IE) == 0) return; pci_generate_msi(pi, p->port); return; } /* If IS for this port is already set -- do nothing. */ if (sc->is & (1 << p->port)) return; sc->is |= (1 << p->port); /* If interrupts are enabled -- generate one. */ if ((sc->ghc & AHCI_GHC_IE) == 0) return; if (nmsg > 0) { pci_generate_msi(pi, nmsg - 1); } else if (!sc->lintr) { sc->lintr = 1; pci_lintr_assert(pi); } } static void ahci_write_fis(struct ahci_port *p, enum sata_fis_type ft, uint8_t *fis) { int offset, len, irq; if (p->rfis == NULL || !(p->cmd & AHCI_P_CMD_FRE)) return; switch (ft) { case FIS_TYPE_REGD2H: offset = 0x40; len = 20; irq = (fis[1] & (1 << 6)) ? AHCI_P_IX_DHR : 0; break; case FIS_TYPE_SETDEVBITS: offset = 0x58; len = 8; irq = (fis[1] & (1 << 6)) ? AHCI_P_IX_SDB : 0; break; case FIS_TYPE_PIOSETUP: offset = 0x20; len = 20; irq = (fis[1] & (1 << 6)) ? AHCI_P_IX_PS : 0; break; default: EPRINTLN("unsupported fis type %d", ft); return; } if (fis[2] & ATA_S_ERROR) { p->waitforclear = 1; irq |= AHCI_P_IX_TFE; } memcpy(p->rfis + offset, fis, len); if (irq) { if (~p->is & irq) { p->is |= irq; ahci_port_intr(p); } } } static void ahci_write_fis_piosetup(struct ahci_port *p) { uint8_t fis[20]; memset(fis, 0, sizeof(fis)); fis[0] = FIS_TYPE_PIOSETUP; ahci_write_fis(p, FIS_TYPE_PIOSETUP, fis); } static void ahci_write_fis_sdb(struct ahci_port *p, int slot, uint8_t *cfis, uint32_t tfd) { uint8_t fis[8]; uint8_t error; error = (tfd >> 8) & 0xff; tfd &= 0x77; memset(fis, 0, sizeof(fis)); fis[0] = FIS_TYPE_SETDEVBITS; fis[1] = (1 << 6); fis[2] = tfd; fis[3] = error; if (fis[2] & ATA_S_ERROR) { p->err_cfis[0] = slot; p->err_cfis[2] = tfd; p->err_cfis[3] = error; memcpy(&p->err_cfis[4], cfis + 4, 16); } else { *(uint32_t *)(fis + 4) = (1 << slot); p->sact &= ~(1 << slot); } p->tfd &= ~0x77; p->tfd |= tfd; ahci_write_fis(p, FIS_TYPE_SETDEVBITS, fis); } static void ahci_write_fis_d2h(struct ahci_port *p, int slot, uint8_t *cfis, uint32_t tfd) { uint8_t fis[20]; uint8_t error; error = (tfd >> 8) & 0xff; memset(fis, 0, sizeof(fis)); fis[0] = FIS_TYPE_REGD2H; fis[1] = (1 << 6); fis[2] = tfd & 0xff; fis[3] = error; fis[4] = cfis[4]; fis[5] = cfis[5]; fis[6] = cfis[6]; fis[7] = cfis[7]; fis[8] = cfis[8]; fis[9] = cfis[9]; fis[10] = cfis[10]; fis[11] = cfis[11]; fis[12] = cfis[12]; fis[13] = cfis[13]; if (fis[2] & ATA_S_ERROR) { p->err_cfis[0] = 0x80; p->err_cfis[2] = tfd & 0xff; p->err_cfis[3] = error; memcpy(&p->err_cfis[4], cfis + 4, 16); } else p->ci &= ~(1 << slot); p->tfd = tfd; ahci_write_fis(p, FIS_TYPE_REGD2H, fis); } static void ahci_write_fis_d2h_ncq(struct ahci_port *p, int slot) { uint8_t fis[20]; p->tfd = ATA_S_READY | ATA_S_DSC; memset(fis, 0, sizeof(fis)); fis[0] = FIS_TYPE_REGD2H; fis[1] = 0; /* No interrupt */ fis[2] = p->tfd; /* Status */ fis[3] = 0; /* No error */ p->ci &= ~(1 << slot); ahci_write_fis(p, FIS_TYPE_REGD2H, fis); } static void ahci_write_reset_fis_d2h(struct ahci_port *p) { uint8_t fis[20]; memset(fis, 0, sizeof(fis)); fis[0] = FIS_TYPE_REGD2H; fis[3] = 1; fis[4] = 1; if (p->atapi) { fis[5] = 0x14; fis[6] = 0xeb; } fis[12] = 1; ahci_write_fis(p, FIS_TYPE_REGD2H, fis); } static void ahci_check_stopped(struct ahci_port *p) { /* * If we are no longer processing the command list and nothing * is in-flight, clear the running bit, the current command * slot, the command issue and active bits. */ if (!(p->cmd & AHCI_P_CMD_ST)) { if (p->pending == 0) { p->ccs = 0; p->cmd &= ~(AHCI_P_CMD_CR | AHCI_P_CMD_CCS_MASK); p->ci = 0; p->sact = 0; p->waitforclear = 0; } } } static void ahci_port_stop(struct ahci_port *p) { struct ahci_ioreq *aior; uint8_t *cfis; int slot; int error; assert(pthread_mutex_isowned_np(&p->pr_sc->mtx)); TAILQ_FOREACH(aior, &p->iobhd, io_blist) { /* * Try to cancel the outstanding blockif request. */ error = blockif_cancel(p->bctx, &aior->io_req); if (error != 0) continue; slot = aior->slot; cfis = aior->cfis; if (cfis[2] == ATA_WRITE_FPDMA_QUEUED || cfis[2] == ATA_READ_FPDMA_QUEUED || cfis[2] == ATA_SEND_FPDMA_QUEUED) p->sact &= ~(1 << slot); /* NCQ */ else p->ci &= ~(1 << slot); /* * This command is now done. */ p->pending &= ~(1 << slot); /* * Delete the blockif request from the busy list */ TAILQ_REMOVE(&p->iobhd, aior, io_blist); /* * Move the blockif request back to the free list */ STAILQ_INSERT_TAIL(&p->iofhd, aior, io_flist); } ahci_check_stopped(p); } static void ahci_port_reset(struct ahci_port *pr) { pr->serr = 0; pr->sact = 0; pr->xfermode = ATA_UDMA6; pr->mult_sectors = 128; if (!pr->bctx) { pr->ssts = ATA_SS_DET_NO_DEVICE; pr->sig = 0xFFFFFFFF; pr->tfd = 0x7F; return; } pr->ssts = ATA_SS_DET_PHY_ONLINE | ATA_SS_IPM_ACTIVE; if (pr->sctl & ATA_SC_SPD_MASK) pr->ssts |= (pr->sctl & ATA_SC_SPD_MASK); else pr->ssts |= ATA_SS_SPD_GEN3; pr->tfd = (1 << 8) | ATA_S_DSC | ATA_S_DMA; if (!pr->atapi) { pr->sig = PxSIG_ATA; pr->tfd |= ATA_S_READY; } else pr->sig = PxSIG_ATAPI; ahci_write_reset_fis_d2h(pr); } static void ahci_reset(struct pci_ahci_softc *sc) { int i; sc->ghc = AHCI_GHC_AE; sc->is = 0; if (sc->lintr) { pci_lintr_deassert(sc->asc_pi); sc->lintr = 0; } for (i = 0; i < sc->ports; i++) { sc->port[i].ie = 0; sc->port[i].is = 0; sc->port[i].cmd = (AHCI_P_CMD_SUD | AHCI_P_CMD_POD); if (sc->port[i].bctx) sc->port[i].cmd |= AHCI_P_CMD_CPS; sc->port[i].sctl = 0; ahci_port_reset(&sc->port[i]); } } static void ata_string(uint8_t *dest, const char *src, int len) { int i; for (i = 0; i < len; i++) { if (*src) dest[i ^ 1] = *src++; else dest[i ^ 1] = ' '; } } static void atapi_string(uint8_t *dest, const char *src, int len) { int i; for (i = 0; i < len; i++) { if (*src) dest[i] = *src++; else dest[i] = ' '; } } /* * Build up the iovec based on the PRDT, 'done' and 'len'. */ static void ahci_build_iov(struct ahci_port *p, struct ahci_ioreq *aior, struct ahci_prdt_entry *prdt, uint16_t prdtl) { struct blockif_req *breq = &aior->io_req; uint32_t dbcsz, extra, left, skip, todo; int i, j; assert(aior->len >= aior->done); /* Copy part of PRDT between 'done' and 'len' bytes into the iov. */ skip = aior->done; left = aior->len - aior->done; todo = 0; for (i = 0, j = 0; i < prdtl && j < BLOCKIF_IOV_MAX && left > 0; i++, prdt++) { dbcsz = (prdt->dbc & DBCMASK) + 1; /* Skip already done part of the PRDT */ if (dbcsz <= skip) { skip -= dbcsz; continue; } dbcsz -= skip; if (dbcsz > left) dbcsz = left; breq->br_iov[j].iov_base = paddr_guest2host(ahci_ctx(p->pr_sc), prdt->dba + skip, dbcsz); breq->br_iov[j].iov_len = dbcsz; todo += dbcsz; left -= dbcsz; skip = 0; j++; } /* If we got limited by IOV length, round I/O down to sector size. */ if (j == BLOCKIF_IOV_MAX) { extra = todo % blockif_sectsz(p->bctx); todo -= extra; assert(todo > 0); while (extra > 0) { if (breq->br_iov[j - 1].iov_len > extra) { breq->br_iov[j - 1].iov_len -= extra; break; } extra -= breq->br_iov[j - 1].iov_len; j--; } } breq->br_iovcnt = j; breq->br_resid = todo; aior->done += todo; aior->more = (aior->done < aior->len && i < prdtl); } static void ahci_handle_rw(struct ahci_port *p, int slot, uint8_t *cfis, uint32_t done) { struct ahci_ioreq *aior; struct blockif_req *breq; struct ahci_prdt_entry *prdt; struct ahci_cmd_hdr *hdr; uint64_t lba; uint32_t len; int err, first, ncq, readop; prdt = (struct ahci_prdt_entry *)(cfis + 0x80); hdr = (struct ahci_cmd_hdr *)(p->cmd_lst + slot * AHCI_CL_SIZE); ncq = 0; readop = 1; first = (done == 0); if (cfis[2] == ATA_WRITE || cfis[2] == ATA_WRITE48 || cfis[2] == ATA_WRITE_MUL || cfis[2] == ATA_WRITE_MUL48 || cfis[2] == ATA_WRITE_DMA || cfis[2] == ATA_WRITE_DMA48 || cfis[2] == ATA_WRITE_FPDMA_QUEUED) readop = 0; if (cfis[2] == ATA_WRITE_FPDMA_QUEUED || cfis[2] == ATA_READ_FPDMA_QUEUED) { lba = ((uint64_t)cfis[10] << 40) | ((uint64_t)cfis[9] << 32) | ((uint64_t)cfis[8] << 24) | ((uint64_t)cfis[6] << 16) | ((uint64_t)cfis[5] << 8) | cfis[4]; len = cfis[11] << 8 | cfis[3]; if (!len) len = 65536; ncq = 1; } else if (cfis[2] == ATA_READ48 || cfis[2] == ATA_WRITE48 || cfis[2] == ATA_READ_MUL48 || cfis[2] == ATA_WRITE_MUL48 || cfis[2] == ATA_READ_DMA48 || cfis[2] == ATA_WRITE_DMA48) { lba = ((uint64_t)cfis[10] << 40) | ((uint64_t)cfis[9] << 32) | ((uint64_t)cfis[8] << 24) | ((uint64_t)cfis[6] << 16) | ((uint64_t)cfis[5] << 8) | cfis[4]; len = cfis[13] << 8 | cfis[12]; if (!len) len = 65536; } else { lba = ((cfis[7] & 0xf) << 24) | (cfis[6] << 16) | (cfis[5] << 8) | cfis[4]; len = cfis[12]; if (!len) len = 256; } lba *= blockif_sectsz(p->bctx); len *= blockif_sectsz(p->bctx); /* Pull request off free list */ aior = STAILQ_FIRST(&p->iofhd); assert(aior != NULL); STAILQ_REMOVE_HEAD(&p->iofhd, io_flist); aior->cfis = cfis; aior->slot = slot; aior->len = len; aior->done = done; breq = &aior->io_req; breq->br_offset = lba + done; ahci_build_iov(p, aior, prdt, hdr->prdtl); /* Mark this command in-flight. */ p->pending |= 1 << slot; /* Stuff request onto busy list. */ TAILQ_INSERT_HEAD(&p->iobhd, aior, io_blist); if (ncq && first) ahci_write_fis_d2h_ncq(p, slot); if (readop) err = blockif_read(p->bctx, breq); else err = blockif_write(p->bctx, breq); assert(err == 0); } static void ahci_handle_flush(struct ahci_port *p, int slot, uint8_t *cfis) { struct ahci_ioreq *aior; struct blockif_req *breq; int err; /* * Pull request off free list */ aior = STAILQ_FIRST(&p->iofhd); assert(aior != NULL); STAILQ_REMOVE_HEAD(&p->iofhd, io_flist); aior->cfis = cfis; aior->slot = slot; aior->len = 0; aior->done = 0; aior->more = 0; breq = &aior->io_req; /* * Mark this command in-flight. */ p->pending |= 1 << slot; /* * Stuff request onto busy list */ TAILQ_INSERT_HEAD(&p->iobhd, aior, io_blist); err = blockif_flush(p->bctx, breq); assert(err == 0); } static inline unsigned int read_prdt(struct ahci_port *p, int slot, uint8_t *cfis, void *buf, unsigned int size) { struct ahci_cmd_hdr *hdr; struct ahci_prdt_entry *prdt; uint8_t *to; unsigned int len; int i; hdr = (struct ahci_cmd_hdr *)(p->cmd_lst + slot * AHCI_CL_SIZE); len = size; to = buf; prdt = (struct ahci_prdt_entry *)(cfis + 0x80); for (i = 0; i < hdr->prdtl && len; i++) { uint8_t *ptr; uint32_t dbcsz; unsigned int sublen; dbcsz = (prdt->dbc & DBCMASK) + 1; ptr = paddr_guest2host(ahci_ctx(p->pr_sc), prdt->dba, dbcsz); sublen = MIN(len, dbcsz); memcpy(to, ptr, sublen); len -= sublen; to += sublen; prdt++; } #ifndef __FreeBSD__ VERIFY3U(size, >=, len); #endif return (size - len); } static void ahci_handle_dsm_trim(struct ahci_port *p, int slot, uint8_t *cfis) { uint32_t len; int ncq; uint8_t *buf; unsigned int nread; buf = NULL; if (cfis[2] == ATA_DATA_SET_MANAGEMENT) { len = (uint16_t)cfis[13] << 8 | cfis[12]; len *= 512; ncq = 0; } else { /* ATA_SEND_FPDMA_QUEUED */ len = (uint16_t)cfis[11] << 8 | cfis[3]; len *= 512; ncq = 1; } /* Support for only a single block is advertised via IDENTIFY. */ if (len > 512) { goto invalid_command; } buf = malloc(len); nread = read_prdt(p, slot, cfis, buf, len); if (nread != len) { goto invalid_command; } ahci_handle_next_trim(p, slot, cfis, buf, len, 0); return; invalid_command: free(buf); if (ncq) { ahci_write_fis_d2h_ncq(p, slot); ahci_write_fis_sdb(p, slot, cfis, (ATA_E_ABORT << 8) | ATA_S_READY | ATA_S_ERROR); } else { ahci_write_fis_d2h(p, slot, cfis, (ATA_E_ABORT << 8) | ATA_S_READY | ATA_S_ERROR); } } static void ahci_handle_next_trim(struct ahci_port *p, int slot, uint8_t *cfis, uint8_t *buf, uint32_t len, uint32_t done) { struct ahci_ioreq *aior; struct blockif_req *breq; uint8_t *entry; uint64_t elba; uint32_t elen; int err; bool first, ncq; #ifndef __FreeBSD__ elba = 0; #endif first = (done == 0); if (cfis[2] == ATA_DATA_SET_MANAGEMENT) { ncq = false; } else { /* ATA_SEND_FPDMA_QUEUED */ ncq = true; } /* Find the next range to TRIM. */ while (done < len) { entry = &buf[done]; elba = ((uint64_t)entry[5] << 40) | ((uint64_t)entry[4] << 32) | ((uint64_t)entry[3] << 24) | ((uint64_t)entry[2] << 16) | ((uint64_t)entry[1] << 8) | entry[0]; elen = (uint16_t)entry[7] << 8 | entry[6]; done += 8; if (elen != 0) break; } /* All remaining ranges were empty. */ if (done == len) { free(buf); if (ncq) { if (first) ahci_write_fis_d2h_ncq(p, slot); ahci_write_fis_sdb(p, slot, cfis, ATA_S_READY | ATA_S_DSC); } else { ahci_write_fis_d2h(p, slot, cfis, ATA_S_READY | ATA_S_DSC); } if (!first) { p->pending &= ~(1 << slot); ahci_check_stopped(p); ahci_handle_port(p); } return; } /* * Pull request off free list */ aior = STAILQ_FIRST(&p->iofhd); assert(aior != NULL); STAILQ_REMOVE_HEAD(&p->iofhd, io_flist); aior->cfis = cfis; aior->slot = slot; aior->len = len; aior->done = done; aior->dsm = buf; aior->more = (len != done); breq = &aior->io_req; breq->br_offset = elba * blockif_sectsz(p->bctx); breq->br_resid = elen * blockif_sectsz(p->bctx); /* * Mark this command in-flight. */ p->pending |= 1 << slot; /* * Stuff request onto busy list */ TAILQ_INSERT_HEAD(&p->iobhd, aior, io_blist); if (ncq && first) ahci_write_fis_d2h_ncq(p, slot); err = blockif_delete(p->bctx, breq); assert(err == 0); } static inline void write_prdt(struct ahci_port *p, int slot, uint8_t *cfis, void *buf, unsigned int size) { struct ahci_cmd_hdr *hdr; struct ahci_prdt_entry *prdt; uint8_t *from; unsigned int len; int i; hdr = (struct ahci_cmd_hdr *)(p->cmd_lst + slot * AHCI_CL_SIZE); len = size; from = buf; prdt = (struct ahci_prdt_entry *)(cfis + 0x80); for (i = 0; i < hdr->prdtl && len; i++) { uint8_t *ptr; uint32_t dbcsz; int sublen; dbcsz = (prdt->dbc & DBCMASK) + 1; ptr = paddr_guest2host(ahci_ctx(p->pr_sc), prdt->dba, dbcsz); sublen = MIN(len, dbcsz); memcpy(ptr, from, sublen); len -= sublen; from += sublen; prdt++; } hdr->prdbc = size - len; } static void ahci_checksum(uint8_t *buf, int size) { int i; uint8_t sum = 0; for (i = 0; i < size - 1; i++) sum += buf[i]; buf[size - 1] = 0x100 - sum; } static void ahci_handle_read_log(struct ahci_port *p, int slot, uint8_t *cfis) { struct ahci_cmd_hdr *hdr; uint32_t buf[128]; uint8_t *buf8 = (uint8_t *)buf; uint16_t *buf16 = (uint16_t *)buf; hdr = (struct ahci_cmd_hdr *)(p->cmd_lst + slot * AHCI_CL_SIZE); if (p->atapi || hdr->prdtl == 0 || cfis[5] != 0 || cfis[9] != 0 || cfis[12] != 1 || cfis[13] != 0) { ahci_write_fis_d2h(p, slot, cfis, (ATA_E_ABORT << 8) | ATA_S_READY | ATA_S_ERROR); return; } memset(buf, 0, sizeof(buf)); if (cfis[4] == 0x00) { /* Log directory */ buf16[0x00] = 1; /* Version -- 1 */ buf16[0x10] = 1; /* NCQ Command Error Log -- 1 page */ buf16[0x13] = 1; /* SATA NCQ Send and Receive Log -- 1 page */ } else if (cfis[4] == 0x10) { /* NCQ Command Error Log */ memcpy(buf8, p->err_cfis, sizeof(p->err_cfis)); ahci_checksum(buf8, sizeof(buf)); } else if (cfis[4] == 0x13) { /* SATA NCQ Send and Receive Log */ if (blockif_candelete(p->bctx) && !blockif_is_ro(p->bctx)) { buf[0x00] = 1; /* SFQ DSM supported */ buf[0x01] = 1; /* SFQ DSM TRIM supported */ } } else { ahci_write_fis_d2h(p, slot, cfis, (ATA_E_ABORT << 8) | ATA_S_READY | ATA_S_ERROR); return; } if (cfis[2] == ATA_READ_LOG_EXT) ahci_write_fis_piosetup(p); write_prdt(p, slot, cfis, (void *)buf, sizeof(buf)); ahci_write_fis_d2h(p, slot, cfis, ATA_S_DSC | ATA_S_READY); } static void handle_identify(struct ahci_port *p, int slot, uint8_t *cfis) { struct ahci_cmd_hdr *hdr; hdr = (struct ahci_cmd_hdr *)(p->cmd_lst + slot * AHCI_CL_SIZE); if (p->atapi || hdr->prdtl == 0) { ahci_write_fis_d2h(p, slot, cfis, (ATA_E_ABORT << 8) | ATA_S_READY | ATA_S_ERROR); } else { ahci_write_fis_piosetup(p); write_prdt(p, slot, cfis, (void*)&p->ata_ident, sizeof(struct ata_params)); ahci_write_fis_d2h(p, slot, cfis, ATA_S_DSC | ATA_S_READY); } } static void ata_identify_init(struct ahci_port* p, int atapi) { struct ata_params* ata_ident = &p->ata_ident; if (atapi) { ata_ident->config = ATA_PROTO_ATAPI | ATA_ATAPI_TYPE_CDROM | ATA_ATAPI_REMOVABLE | ATA_DRQ_FAST; ata_ident->capabilities1 = ATA_SUPPORT_LBA | ATA_SUPPORT_DMA; ata_ident->capabilities2 = (1 << 14 | 1); ata_ident->atavalid = ATA_FLAG_64_70 | ATA_FLAG_88; ata_ident->obsolete62 = 0x3f; ata_ident->mwdmamodes = 7; if (p->xfermode & ATA_WDMA0) ata_ident->mwdmamodes |= (1 << ((p->xfermode & 7) + 8)); ata_ident->apiomodes = 3; ata_ident->mwdmamin = 0x0078; ata_ident->mwdmarec = 0x0078; ata_ident->pioblind = 0x0078; ata_ident->pioiordy = 0x0078; ata_ident->satacapabilities = (ATA_SATA_GEN1 | ATA_SATA_GEN2 | ATA_SATA_GEN3); ata_ident->satacapabilities2 = ((p->ssts & ATA_SS_SPD_MASK) >> 3); ata_ident->satasupport = ATA_SUPPORT_NCQ_STREAM; ata_ident->version_major = 0x3f0; ata_ident->support.command1 = (ATA_SUPPORT_POWERMGT | ATA_SUPPORT_PACKET | ATA_SUPPORT_RESET | ATA_SUPPORT_NOP); ata_ident->support.command2 = (1 << 14); ata_ident->support.extension = (1 << 14); ata_ident->enabled.command1 = (ATA_SUPPORT_POWERMGT | ATA_SUPPORT_PACKET | ATA_SUPPORT_RESET | ATA_SUPPORT_NOP); ata_ident->enabled.extension = (1 << 14); ata_ident->udmamodes = 0x7f; if (p->xfermode & ATA_UDMA0) ata_ident->udmamodes |= (1 << ((p->xfermode & 7) + 8)); ata_ident->transport_major = 0x1020; ata_ident->integrity = 0x00a5; } else { uint64_t sectors; int sectsz, psectsz, psectoff, candelete, ro; uint16_t cyl; uint8_t sech, heads; ro = blockif_is_ro(p->bctx); candelete = blockif_candelete(p->bctx); sectsz = blockif_sectsz(p->bctx); sectors = blockif_size(p->bctx) / sectsz; blockif_chs(p->bctx, &cyl, &heads, &sech); blockif_psectsz(p->bctx, &psectsz, &psectoff); ata_ident->config = ATA_DRQ_FAST; ata_ident->cylinders = cyl; ata_ident->heads = heads; ata_ident->sectors = sech; ata_ident->sectors_intr = (0x8000 | 128); ata_ident->tcg = 0; ata_ident->capabilities1 = ATA_SUPPORT_DMA | ATA_SUPPORT_LBA | ATA_SUPPORT_IORDY; ata_ident->capabilities2 = (1 << 14); ata_ident->atavalid = ATA_FLAG_64_70 | ATA_FLAG_88; if (p->mult_sectors) ata_ident->multi = (ATA_MULTI_VALID | p->mult_sectors); if (sectors <= 0x0fffffff) { ata_ident->lba_size_1 = sectors; ata_ident->lba_size_2 = (sectors >> 16); } else { ata_ident->lba_size_1 = 0xffff; ata_ident->lba_size_2 = 0x0fff; } ata_ident->mwdmamodes = 0x7; if (p->xfermode & ATA_WDMA0) ata_ident->mwdmamodes |= (1 << ((p->xfermode & 7) + 8)); ata_ident->apiomodes = 0x3; ata_ident->mwdmamin = 0x0078; ata_ident->mwdmarec = 0x0078; ata_ident->pioblind = 0x0078; ata_ident->pioiordy = 0x0078; ata_ident->support3 = 0; ata_ident->queue = 31; ata_ident->satacapabilities = (ATA_SATA_GEN1 | ATA_SATA_GEN2 | ATA_SATA_GEN3 | ATA_SUPPORT_NCQ); ata_ident->satacapabilities2 = (ATA_SUPPORT_RCVSND_FPDMA_QUEUED | (p->ssts & ATA_SS_SPD_MASK) >> 3); ata_ident->version_major = 0x3f0; ata_ident->version_minor = 0x28; ata_ident->support.command1 = (ATA_SUPPORT_POWERMGT | ATA_SUPPORT_WRITECACHE | ATA_SUPPORT_LOOKAHEAD | ATA_SUPPORT_NOP); ata_ident->support.command2 = (ATA_SUPPORT_ADDRESS48 | ATA_SUPPORT_FLUSHCACHE | ATA_SUPPORT_FLUSHCACHE48 | 1 << 14); ata_ident->support.extension = (1 << 14); ata_ident->enabled.command1 = (ATA_SUPPORT_POWERMGT | ATA_SUPPORT_WRITECACHE | ATA_SUPPORT_LOOKAHEAD | ATA_SUPPORT_NOP); ata_ident->enabled.command2 = (ATA_SUPPORT_ADDRESS48 | ATA_SUPPORT_FLUSHCACHE | ATA_SUPPORT_FLUSHCACHE48 | 1 << 15); ata_ident->enabled.extension = (1 << 14); ata_ident->udmamodes = 0x7f; if (p->xfermode & ATA_UDMA0) ata_ident->udmamodes |= (1 << ((p->xfermode & 7) + 8)); ata_ident->lba_size48_1 = sectors; ata_ident->lba_size48_2 = (sectors >> 16); ata_ident->lba_size48_3 = (sectors >> 32); ata_ident->lba_size48_4 = (sectors >> 48); if (candelete && !ro) { ata_ident->support3 |= ATA_SUPPORT_RZAT | ATA_SUPPORT_DRAT; ata_ident->max_dsm_blocks = 1; ata_ident->support_dsm = ATA_SUPPORT_DSM_TRIM; } ata_ident->pss = ATA_PSS_VALID_VALUE; ata_ident->lsalign = 0x4000; if (psectsz > sectsz) { ata_ident->pss |= ATA_PSS_MULTLS; ata_ident->pss |= ffsl(psectsz / sectsz) - 1; ata_ident->lsalign |= (psectoff / sectsz); } if (sectsz > 512) { ata_ident->pss |= ATA_PSS_LSSABOVE512; ata_ident->lss_1 = sectsz / 2; ata_ident->lss_2 = ((sectsz / 2) >> 16); } ata_ident->support2 = (ATA_SUPPORT_RWLOGDMAEXT | 1 << 14); ata_ident->enabled2 = (ATA_SUPPORT_RWLOGDMAEXT | 1 << 14); ata_ident->transport_major = 0x1020; ata_ident->integrity = 0x00a5; } ahci_checksum((uint8_t*)ata_ident, sizeof(struct ata_params)); } static void handle_atapi_identify(struct ahci_port *p, int slot, uint8_t *cfis) { if (!p->atapi) { ahci_write_fis_d2h(p, slot, cfis, (ATA_E_ABORT << 8) | ATA_S_READY | ATA_S_ERROR); } else { ahci_write_fis_piosetup(p); write_prdt(p, slot, cfis, (void *)&p->ata_ident, sizeof(struct ata_params)); ahci_write_fis_d2h(p, slot, cfis, ATA_S_DSC | ATA_S_READY); } } static void atapi_inquiry(struct ahci_port *p, int slot, uint8_t *cfis) { uint8_t buf[36]; uint8_t *acmd; unsigned int len; uint32_t tfd; acmd = cfis + 0x40; if (acmd[1] & 1) { /* VPD */ if (acmd[2] == 0) { /* Supported VPD pages */ buf[0] = 0x05; buf[1] = 0; buf[2] = 0; buf[3] = 1; buf[4] = 0; len = 4 + buf[3]; } else { p->sense_key = ATA_SENSE_ILLEGAL_REQUEST; p->asc = 0x24; tfd = (p->sense_key << 12) | ATA_S_READY | ATA_S_ERROR; cfis[4] = (cfis[4] & ~7) | ATA_I_CMD | ATA_I_IN; ahci_write_fis_d2h(p, slot, cfis, tfd); return; } } else { buf[0] = 0x05; buf[1] = 0x80; buf[2] = 0x00; buf[3] = 0x21; buf[4] = 31; buf[5] = 0; buf[6] = 0; buf[7] = 0; atapi_string(buf + 8, "BHYVE", 8); atapi_string(buf + 16, "BHYVE DVD-ROM", 16); atapi_string(buf + 32, "001", 4); len = sizeof(buf); } if (len > acmd[4]) len = acmd[4]; cfis[4] = (cfis[4] & ~7) | ATA_I_CMD | ATA_I_IN; write_prdt(p, slot, cfis, buf, len); ahci_write_fis_d2h(p, slot, cfis, ATA_S_READY | ATA_S_DSC); } static void atapi_read_capacity(struct ahci_port *p, int slot, uint8_t *cfis) { uint8_t buf[8]; uint64_t sectors; sectors = blockif_size(p->bctx) / 2048; be32enc(buf, sectors - 1); be32enc(buf + 4, 2048); cfis[4] = (cfis[4] & ~7) | ATA_I_CMD | ATA_I_IN; write_prdt(p, slot, cfis, buf, sizeof(buf)); ahci_write_fis_d2h(p, slot, cfis, ATA_S_READY | ATA_S_DSC); } static void atapi_read_toc(struct ahci_port *p, int slot, uint8_t *cfis) { uint8_t *acmd; uint8_t format; unsigned int len; acmd = cfis + 0x40; len = be16dec(acmd + 7); format = acmd[9] >> 6; switch (format) { case 0: { size_t size; int msf; uint64_t sectors; uint8_t start_track, buf[20], *bp; msf = (acmd[1] >> 1) & 1; start_track = acmd[6]; if (start_track > 1 && start_track != 0xaa) { uint32_t tfd; p->sense_key = ATA_SENSE_ILLEGAL_REQUEST; p->asc = 0x24; tfd = (p->sense_key << 12) | ATA_S_READY | ATA_S_ERROR; cfis[4] = (cfis[4] & ~7) | ATA_I_CMD | ATA_I_IN; ahci_write_fis_d2h(p, slot, cfis, tfd); return; } bp = buf + 2; *bp++ = 1; *bp++ = 1; if (start_track <= 1) { *bp++ = 0; *bp++ = 0x14; *bp++ = 1; *bp++ = 0; if (msf) { *bp++ = 0; lba_to_msf(bp, 0); bp += 3; } else { *bp++ = 0; *bp++ = 0; *bp++ = 0; *bp++ = 0; } } *bp++ = 0; *bp++ = 0x14; *bp++ = 0xaa; *bp++ = 0; sectors = blockif_size(p->bctx) / blockif_sectsz(p->bctx); sectors >>= 2; if (msf) { *bp++ = 0; lba_to_msf(bp, sectors); bp += 3; } else { be32enc(bp, sectors); bp += 4; } size = bp - buf; be16enc(buf, size - 2); if (len > size) len = size; write_prdt(p, slot, cfis, buf, len); cfis[4] = (cfis[4] & ~7) | ATA_I_CMD | ATA_I_IN; ahci_write_fis_d2h(p, slot, cfis, ATA_S_READY | ATA_S_DSC); break; } case 1: { uint8_t buf[12]; memset(buf, 0, sizeof(buf)); buf[1] = 0xa; buf[2] = 0x1; buf[3] = 0x1; if (len > sizeof(buf)) len = sizeof(buf); write_prdt(p, slot, cfis, buf, len); cfis[4] = (cfis[4] & ~7) | ATA_I_CMD | ATA_I_IN; ahci_write_fis_d2h(p, slot, cfis, ATA_S_READY | ATA_S_DSC); break; } case 2: { size_t size; int msf; uint64_t sectors; uint8_t *bp, buf[50]; msf = (acmd[1] >> 1) & 1; bp = buf + 2; *bp++ = 1; *bp++ = 1; *bp++ = 1; *bp++ = 0x14; *bp++ = 0; *bp++ = 0xa0; *bp++ = 0; *bp++ = 0; *bp++ = 0; *bp++ = 0; *bp++ = 1; *bp++ = 0; *bp++ = 0; *bp++ = 1; *bp++ = 0x14; *bp++ = 0; *bp++ = 0xa1; *bp++ = 0; *bp++ = 0; *bp++ = 0; *bp++ = 0; *bp++ = 1; *bp++ = 0; *bp++ = 0; *bp++ = 1; *bp++ = 0x14; *bp++ = 0; *bp++ = 0xa2; *bp++ = 0; *bp++ = 0; *bp++ = 0; sectors = blockif_size(p->bctx) / blockif_sectsz(p->bctx); sectors >>= 2; if (msf) { *bp++ = 0; lba_to_msf(bp, sectors); bp += 3; } else { be32enc(bp, sectors); bp += 4; } *bp++ = 1; *bp++ = 0x14; *bp++ = 0; *bp++ = 1; *bp++ = 0; *bp++ = 0; *bp++ = 0; if (msf) { *bp++ = 0; lba_to_msf(bp, 0); bp += 3; } else { *bp++ = 0; *bp++ = 0; *bp++ = 0; *bp++ = 0; } size = bp - buf; be16enc(buf, size - 2); if (len > size) len = size; write_prdt(p, slot, cfis, buf, len); cfis[4] = (cfis[4] & ~7) | ATA_I_CMD | ATA_I_IN; ahci_write_fis_d2h(p, slot, cfis, ATA_S_READY | ATA_S_DSC); break; } default: { uint32_t tfd; p->sense_key = ATA_SENSE_ILLEGAL_REQUEST; p->asc = 0x24; tfd = (p->sense_key << 12) | ATA_S_READY | ATA_S_ERROR; cfis[4] = (cfis[4] & ~7) | ATA_I_CMD | ATA_I_IN; ahci_write_fis_d2h(p, slot, cfis, tfd); break; } } } static void atapi_report_luns(struct ahci_port *p, int slot, uint8_t *cfis) { uint8_t buf[16]; memset(buf, 0, sizeof(buf)); buf[3] = 8; cfis[4] = (cfis[4] & ~7) | ATA_I_CMD | ATA_I_IN; write_prdt(p, slot, cfis, buf, sizeof(buf)); ahci_write_fis_d2h(p, slot, cfis, ATA_S_READY | ATA_S_DSC); } static void atapi_read(struct ahci_port *p, int slot, uint8_t *cfis, uint32_t done) { struct ahci_ioreq *aior; struct ahci_cmd_hdr *hdr; struct ahci_prdt_entry *prdt; struct blockif_req *breq; uint8_t *acmd; uint64_t lba; uint32_t len; int err; acmd = cfis + 0x40; hdr = (struct ahci_cmd_hdr *)(p->cmd_lst + slot * AHCI_CL_SIZE); prdt = (struct ahci_prdt_entry *)(cfis + 0x80); lba = be32dec(acmd + 2); if (acmd[0] == READ_10) len = be16dec(acmd + 7); else len = be32dec(acmd + 6); if (len == 0) { cfis[4] = (cfis[4] & ~7) | ATA_I_CMD | ATA_I_IN; ahci_write_fis_d2h(p, slot, cfis, ATA_S_READY | ATA_S_DSC); } lba *= 2048; len *= 2048; /* * Pull request off free list */ aior = STAILQ_FIRST(&p->iofhd); assert(aior != NULL); STAILQ_REMOVE_HEAD(&p->iofhd, io_flist); aior->cfis = cfis; aior->slot = slot; aior->len = len; aior->done = done; breq = &aior->io_req; breq->br_offset = lba + done; ahci_build_iov(p, aior, prdt, hdr->prdtl); /* Mark this command in-flight. */ p->pending |= 1 << slot; /* Stuff request onto busy list. */ TAILQ_INSERT_HEAD(&p->iobhd, aior, io_blist); err = blockif_read(p->bctx, breq); assert(err == 0); } static void atapi_request_sense(struct ahci_port *p, int slot, uint8_t *cfis) { uint8_t buf[64]; uint8_t *acmd; unsigned int len; acmd = cfis + 0x40; len = acmd[4]; if (len > sizeof(buf)) len = sizeof(buf); memset(buf, 0, len); buf[0] = 0x70 | (1 << 7); buf[2] = p->sense_key; buf[7] = 10; buf[12] = p->asc; write_prdt(p, slot, cfis, buf, len); cfis[4] = (cfis[4] & ~7) | ATA_I_CMD | ATA_I_IN; ahci_write_fis_d2h(p, slot, cfis, ATA_S_READY | ATA_S_DSC); } static void atapi_start_stop_unit(struct ahci_port *p, int slot, uint8_t *cfis) { uint8_t *acmd = cfis + 0x40; uint32_t tfd; switch (acmd[4] & 3) { case 0: case 1: case 3: cfis[4] = (cfis[4] & ~7) | ATA_I_CMD | ATA_I_IN; tfd = ATA_S_READY | ATA_S_DSC; break; case 2: /* TODO eject media */ cfis[4] = (cfis[4] & ~7) | ATA_I_CMD | ATA_I_IN; p->sense_key = ATA_SENSE_ILLEGAL_REQUEST; p->asc = 0x53; tfd = (p->sense_key << 12) | ATA_S_READY | ATA_S_ERROR; break; } ahci_write_fis_d2h(p, slot, cfis, tfd); } static void atapi_mode_sense(struct ahci_port *p, int slot, uint8_t *cfis) { uint8_t *acmd; uint32_t tfd = 0; uint8_t pc, code; unsigned int len; acmd = cfis + 0x40; len = be16dec(acmd + 7); pc = acmd[2] >> 6; code = acmd[2] & 0x3f; switch (pc) { case 0: switch (code) { case MODEPAGE_RW_ERROR_RECOVERY: { uint8_t buf[16]; if (len > sizeof(buf)) len = sizeof(buf); memset(buf, 0, sizeof(buf)); be16enc(buf, 16 - 2); buf[2] = 0x70; buf[8] = 0x01; buf[9] = 16 - 10; buf[11] = 0x05; write_prdt(p, slot, cfis, buf, len); tfd = ATA_S_READY | ATA_S_DSC; break; } case MODEPAGE_CD_CAPABILITIES: { uint8_t buf[30]; if (len > sizeof(buf)) len = sizeof(buf); memset(buf, 0, sizeof(buf)); be16enc(buf, 30 - 2); buf[2] = 0x70; buf[8] = 0x2A; buf[9] = 30 - 10; buf[10] = 0x08; buf[12] = 0x71; be16enc(&buf[18], 2); be16enc(&buf[20], 512); write_prdt(p, slot, cfis, buf, len); tfd = ATA_S_READY | ATA_S_DSC; break; } default: goto error; break; } break; case 3: p->sense_key = ATA_SENSE_ILLEGAL_REQUEST; p->asc = 0x39; tfd = (p->sense_key << 12) | ATA_S_READY | ATA_S_ERROR; break; error: case 1: case 2: p->sense_key = ATA_SENSE_ILLEGAL_REQUEST; p->asc = 0x24; tfd = (p->sense_key << 12) | ATA_S_READY | ATA_S_ERROR; break; } cfis[4] = (cfis[4] & ~7) | ATA_I_CMD | ATA_I_IN; ahci_write_fis_d2h(p, slot, cfis, tfd); } static void atapi_get_event_status_notification(struct ahci_port *p, int slot, uint8_t *cfis) { uint8_t *acmd; uint32_t tfd; acmd = cfis + 0x40; /* we don't support asynchronous operation */ if (!(acmd[1] & 1)) { p->sense_key = ATA_SENSE_ILLEGAL_REQUEST; p->asc = 0x24; tfd = (p->sense_key << 12) | ATA_S_READY | ATA_S_ERROR; } else { uint8_t buf[8]; unsigned int len; len = be16dec(acmd + 7); if (len > sizeof(buf)) len = sizeof(buf); memset(buf, 0, sizeof(buf)); be16enc(buf, 8 - 2); buf[2] = 0x04; buf[3] = 0x10; buf[5] = 0x02; write_prdt(p, slot, cfis, buf, len); tfd = ATA_S_READY | ATA_S_DSC; } cfis[4] = (cfis[4] & ~7) | ATA_I_CMD | ATA_I_IN; ahci_write_fis_d2h(p, slot, cfis, tfd); } static void handle_packet_cmd(struct ahci_port *p, int slot, uint8_t *cfis) { uint8_t *acmd; acmd = cfis + 0x40; #ifdef AHCI_DEBUG { int i; DPRINTF("ACMD:"); for (i = 0; i < 16; i++) DPRINTF("%02x ", acmd[i]); DPRINTF(""); } #endif switch (acmd[0]) { case TEST_UNIT_READY: cfis[4] = (cfis[4] & ~7) | ATA_I_CMD | ATA_I_IN; ahci_write_fis_d2h(p, slot, cfis, ATA_S_READY | ATA_S_DSC); break; case INQUIRY: atapi_inquiry(p, slot, cfis); break; case READ_CAPACITY: atapi_read_capacity(p, slot, cfis); break; case PREVENT_ALLOW: /* TODO */ cfis[4] = (cfis[4] & ~7) | ATA_I_CMD | ATA_I_IN; ahci_write_fis_d2h(p, slot, cfis, ATA_S_READY | ATA_S_DSC); break; case READ_TOC: atapi_read_toc(p, slot, cfis); break; case REPORT_LUNS: atapi_report_luns(p, slot, cfis); break; case READ_10: case READ_12: atapi_read(p, slot, cfis, 0); break; case REQUEST_SENSE: atapi_request_sense(p, slot, cfis); break; case START_STOP_UNIT: atapi_start_stop_unit(p, slot, cfis); break; case MODE_SENSE_10: atapi_mode_sense(p, slot, cfis); break; case GET_EVENT_STATUS_NOTIFICATION: atapi_get_event_status_notification(p, slot, cfis); break; default: cfis[4] = (cfis[4] & ~7) | ATA_I_CMD | ATA_I_IN; p->sense_key = ATA_SENSE_ILLEGAL_REQUEST; p->asc = 0x20; ahci_write_fis_d2h(p, slot, cfis, (p->sense_key << 12) | ATA_S_READY | ATA_S_ERROR); break; } } static void ahci_handle_cmd(struct ahci_port *p, int slot, uint8_t *cfis) { p->tfd |= ATA_S_BUSY; switch (cfis[2]) { case ATA_ATA_IDENTIFY: handle_identify(p, slot, cfis); break; case ATA_SETFEATURES: { switch (cfis[3]) { case ATA_SF_ENAB_SATA_SF: switch (cfis[12]) { case ATA_SATA_SF_AN: p->tfd = ATA_S_DSC | ATA_S_READY; break; default: p->tfd = ATA_S_ERROR | ATA_S_READY; p->tfd |= (ATA_ERROR_ABORT << 8); break; } break; case ATA_SF_ENAB_WCACHE: case ATA_SF_DIS_WCACHE: case ATA_SF_ENAB_RCACHE: case ATA_SF_DIS_RCACHE: p->tfd = ATA_S_DSC | ATA_S_READY; break; case ATA_SF_SETXFER: { switch (cfis[12] & 0xf8) { case ATA_PIO: case ATA_PIO0: break; case ATA_WDMA0: case ATA_UDMA0: p->xfermode = (cfis[12] & 0x7); break; } p->tfd = ATA_S_DSC | ATA_S_READY; break; } default: p->tfd = ATA_S_ERROR | ATA_S_READY; p->tfd |= (ATA_ERROR_ABORT << 8); break; } ahci_write_fis_d2h(p, slot, cfis, p->tfd); break; } case ATA_SET_MULTI: if (cfis[12] != 0 && (cfis[12] > 128 || (cfis[12] & (cfis[12] - 1)))) { p->tfd = ATA_S_ERROR | ATA_S_READY; p->tfd |= (ATA_ERROR_ABORT << 8); } else { p->mult_sectors = cfis[12]; p->tfd = ATA_S_DSC | ATA_S_READY; } ahci_write_fis_d2h(p, slot, cfis, p->tfd); break; case ATA_READ: case ATA_WRITE: case ATA_READ48: case ATA_WRITE48: case ATA_READ_MUL: case ATA_WRITE_MUL: case ATA_READ_MUL48: case ATA_WRITE_MUL48: case ATA_READ_DMA: case ATA_WRITE_DMA: case ATA_READ_DMA48: case ATA_WRITE_DMA48: case ATA_READ_FPDMA_QUEUED: case ATA_WRITE_FPDMA_QUEUED: ahci_handle_rw(p, slot, cfis, 0); break; case ATA_FLUSHCACHE: case ATA_FLUSHCACHE48: ahci_handle_flush(p, slot, cfis); break; case ATA_DATA_SET_MANAGEMENT: if (cfis[11] == 0 && cfis[3] == ATA_DSM_TRIM && cfis[13] == 0 && cfis[12] == 1) { ahci_handle_dsm_trim(p, slot, cfis); break; } ahci_write_fis_d2h(p, slot, cfis, (ATA_E_ABORT << 8) | ATA_S_READY | ATA_S_ERROR); break; case ATA_SEND_FPDMA_QUEUED: if ((cfis[13] & 0x1f) == ATA_SFPDMA_DSM && cfis[17] == 0 && cfis[16] == ATA_DSM_TRIM && cfis[11] == 0 && cfis[3] == 1) { ahci_handle_dsm_trim(p, slot, cfis); break; } ahci_write_fis_d2h(p, slot, cfis, (ATA_E_ABORT << 8) | ATA_S_READY | ATA_S_ERROR); break; case ATA_READ_LOG_EXT: case ATA_READ_LOG_DMA_EXT: ahci_handle_read_log(p, slot, cfis); break; case ATA_SECURITY_FREEZE_LOCK: case ATA_SMART_CMD: case ATA_NOP: ahci_write_fis_d2h(p, slot, cfis, (ATA_E_ABORT << 8) | ATA_S_READY | ATA_S_ERROR); break; case ATA_CHECK_POWER_MODE: cfis[12] = 0xff; /* always on */ ahci_write_fis_d2h(p, slot, cfis, ATA_S_READY | ATA_S_DSC); break; case ATA_STANDBY_CMD: case ATA_STANDBY_IMMEDIATE: case ATA_IDLE_CMD: case ATA_IDLE_IMMEDIATE: case ATA_SLEEP: case ATA_READ_VERIFY: case ATA_READ_VERIFY48: ahci_write_fis_d2h(p, slot, cfis, ATA_S_READY | ATA_S_DSC); break; case ATA_ATAPI_IDENTIFY: handle_atapi_identify(p, slot, cfis); break; case ATA_PACKET_CMD: if (!p->atapi) { ahci_write_fis_d2h(p, slot, cfis, (ATA_E_ABORT << 8) | ATA_S_READY | ATA_S_ERROR); } else handle_packet_cmd(p, slot, cfis); break; default: EPRINTLN("Unsupported cmd:%02x", cfis[2]); ahci_write_fis_d2h(p, slot, cfis, (ATA_E_ABORT << 8) | ATA_S_READY | ATA_S_ERROR); break; } } static void ahci_handle_slot(struct ahci_port *p, int slot) { struct ahci_cmd_hdr *hdr; #ifdef AHCI_DEBUG struct ahci_prdt_entry *prdt; #endif struct pci_ahci_softc *sc; uint8_t *cfis; #ifdef AHCI_DEBUG int cfl, i; #endif sc = p->pr_sc; hdr = (struct ahci_cmd_hdr *)(p->cmd_lst + slot * AHCI_CL_SIZE); #ifdef AHCI_DEBUG cfl = (hdr->flags & 0x1f) * 4; #endif cfis = paddr_guest2host(ahci_ctx(sc), hdr->ctba, 0x80 + hdr->prdtl * sizeof(struct ahci_prdt_entry)); #ifdef AHCI_DEBUG prdt = (struct ahci_prdt_entry *)(cfis + 0x80); DPRINTF("cfis:"); for (i = 0; i < cfl; i++) { if (i % 10 == 0) DPRINTF(""); DPRINTF("%02x ", cfis[i]); } DPRINTF(""); for (i = 0; i < hdr->prdtl; i++) { DPRINTF("%d@%08"PRIx64"", prdt->dbc & 0x3fffff, prdt->dba); prdt++; } #endif if (cfis[0] != FIS_TYPE_REGH2D) { EPRINTLN("Not a H2D FIS:%02x", cfis[0]); return; } if (cfis[1] & 0x80) { ahci_handle_cmd(p, slot, cfis); } else { if (cfis[15] & (1 << 2)) p->reset = 1; else if (p->reset) { p->reset = 0; ahci_port_reset(p); } p->ci &= ~(1 << slot); } } static void ahci_handle_port(struct ahci_port *p) { if (!(p->cmd & AHCI_P_CMD_ST)) return; /* * Search for any new commands to issue ignoring those that * are already in-flight. Stop if device is busy or in error. */ for (; (p->ci & ~p->pending) != 0; p->ccs = ((p->ccs + 1) & 31)) { if ((p->tfd & (ATA_S_BUSY | ATA_S_DRQ)) != 0) break; if (p->waitforclear) break; if ((p->ci & ~p->pending & (1 << p->ccs)) != 0) { p->cmd &= ~AHCI_P_CMD_CCS_MASK; p->cmd |= p->ccs << AHCI_P_CMD_CCS_SHIFT; ahci_handle_slot(p, p->ccs); } } } /* * blockif callback routine - this runs in the context of the blockif * i/o thread, so the mutex needs to be acquired. */ static void ata_ioreq_cb(struct blockif_req *br, int err) { struct ahci_cmd_hdr *hdr; struct ahci_ioreq *aior; struct ahci_port *p; struct pci_ahci_softc *sc; uint32_t tfd; uint8_t *cfis, *dsm; int slot, ncq; DPRINTF("%s %d", __func__, err); ncq = 0; aior = br->br_param; p = aior->io_pr; cfis = aior->cfis; slot = aior->slot; sc = p->pr_sc; hdr = (struct ahci_cmd_hdr *)(p->cmd_lst + slot * AHCI_CL_SIZE); if (cfis[2] == ATA_WRITE_FPDMA_QUEUED || cfis[2] == ATA_READ_FPDMA_QUEUED || cfis[2] == ATA_SEND_FPDMA_QUEUED) ncq = 1; dsm = aior->dsm; aior->dsm = NULL; pthread_mutex_lock(&sc->mtx); /* * Delete the blockif request from the busy list */ TAILQ_REMOVE(&p->iobhd, aior, io_blist); /* * Move the blockif request back to the free list */ STAILQ_INSERT_TAIL(&p->iofhd, aior, io_flist); if (!err) hdr->prdbc = aior->done; if (!err && aior->more) { if (dsm != NULL) ahci_handle_next_trim(p, slot, cfis, dsm, aior->len, aior->done); else ahci_handle_rw(p, slot, cfis, aior->done); goto out; } if (!err) tfd = ATA_S_READY | ATA_S_DSC; else tfd = (ATA_E_ABORT << 8) | ATA_S_READY | ATA_S_ERROR; if (ncq) ahci_write_fis_sdb(p, slot, cfis, tfd); else ahci_write_fis_d2h(p, slot, cfis, tfd); /* * This command is now complete. */ p->pending &= ~(1 << slot); ahci_check_stopped(p); ahci_handle_port(p); free(dsm); out: pthread_mutex_unlock(&sc->mtx); DPRINTF("%s exit", __func__); } static void atapi_ioreq_cb(struct blockif_req *br, int err) { struct ahci_cmd_hdr *hdr; struct ahci_ioreq *aior; struct ahci_port *p; struct pci_ahci_softc *sc; uint8_t *cfis; uint32_t tfd; int slot; DPRINTF("%s %d", __func__, err); aior = br->br_param; p = aior->io_pr; cfis = aior->cfis; slot = aior->slot; sc = p->pr_sc; hdr = (struct ahci_cmd_hdr *)(p->cmd_lst + aior->slot * AHCI_CL_SIZE); pthread_mutex_lock(&sc->mtx); /* * Delete the blockif request from the busy list */ TAILQ_REMOVE(&p->iobhd, aior, io_blist); /* * Move the blockif request back to the free list */ STAILQ_INSERT_TAIL(&p->iofhd, aior, io_flist); if (!err) hdr->prdbc = aior->done; if (!err && aior->more) { atapi_read(p, slot, cfis, aior->done); goto out; } if (!err) { tfd = ATA_S_READY | ATA_S_DSC; } else { p->sense_key = ATA_SENSE_ILLEGAL_REQUEST; p->asc = 0x21; tfd = (p->sense_key << 12) | ATA_S_READY | ATA_S_ERROR; } cfis[4] = (cfis[4] & ~7) | ATA_I_CMD | ATA_I_IN; ahci_write_fis_d2h(p, slot, cfis, tfd); /* * This command is now complete. */ p->pending &= ~(1 << slot); ahci_check_stopped(p); ahci_handle_port(p); out: pthread_mutex_unlock(&sc->mtx); DPRINTF("%s exit", __func__); } static void pci_ahci_ioreq_init(struct ahci_port *pr) { struct ahci_ioreq *vr; int i; pr->ioqsz = blockif_queuesz(pr->bctx); pr->ioreq = calloc(pr->ioqsz, sizeof(struct ahci_ioreq)); STAILQ_INIT(&pr->iofhd); /* * Add all i/o request entries to the free queue */ for (i = 0; i < pr->ioqsz; i++) { vr = &pr->ioreq[i]; vr->io_pr = pr; if (!pr->atapi) vr->io_req.br_callback = ata_ioreq_cb; else vr->io_req.br_callback = atapi_ioreq_cb; vr->io_req.br_param = vr; STAILQ_INSERT_TAIL(&pr->iofhd, vr, io_flist); } TAILQ_INIT(&pr->iobhd); } static void pci_ahci_port_write(struct pci_ahci_softc *sc, uint64_t offset, uint64_t value) { int port = (offset - AHCI_OFFSET) / AHCI_STEP; offset = (offset - AHCI_OFFSET) % AHCI_STEP; struct ahci_port *p = &sc->port[port]; DPRINTF("pci_ahci_port %d: write offset 0x%"PRIx64" value 0x%"PRIx64"", port, offset, value); switch (offset) { case AHCI_P_CLB: p->clb = value; break; case AHCI_P_CLBU: p->clbu = value; break; case AHCI_P_FB: p->fb = value; break; case AHCI_P_FBU: p->fbu = value; break; case AHCI_P_IS: p->is &= ~value; ahci_port_intr(p); break; case AHCI_P_IE: p->ie = value & 0xFDC000FF; ahci_port_intr(p); break; case AHCI_P_CMD: { p->cmd &= ~(AHCI_P_CMD_ST | AHCI_P_CMD_SUD | AHCI_P_CMD_POD | AHCI_P_CMD_CLO | AHCI_P_CMD_FRE | AHCI_P_CMD_APSTE | AHCI_P_CMD_ATAPI | AHCI_P_CMD_DLAE | AHCI_P_CMD_ALPE | AHCI_P_CMD_ASP | AHCI_P_CMD_ICC_MASK); p->cmd |= (AHCI_P_CMD_ST | AHCI_P_CMD_SUD | AHCI_P_CMD_POD | AHCI_P_CMD_CLO | AHCI_P_CMD_FRE | AHCI_P_CMD_APSTE | AHCI_P_CMD_ATAPI | AHCI_P_CMD_DLAE | AHCI_P_CMD_ALPE | AHCI_P_CMD_ASP | AHCI_P_CMD_ICC_MASK) & value; if (!(value & AHCI_P_CMD_ST)) { ahci_port_stop(p); } else { uint64_t clb; p->cmd |= AHCI_P_CMD_CR; clb = (uint64_t)p->clbu << 32 | p->clb; p->cmd_lst = paddr_guest2host(ahci_ctx(sc), clb, AHCI_CL_SIZE * AHCI_MAX_SLOTS); } if (value & AHCI_P_CMD_FRE) { uint64_t fb; p->cmd |= AHCI_P_CMD_FR; fb = (uint64_t)p->fbu << 32 | p->fb; /* we don't support FBSCP, so rfis size is 256Bytes */ p->rfis = paddr_guest2host(ahci_ctx(sc), fb, 256); } else { p->cmd &= ~AHCI_P_CMD_FR; } if (value & AHCI_P_CMD_CLO) { p->tfd &= ~(ATA_S_BUSY | ATA_S_DRQ); p->cmd &= ~AHCI_P_CMD_CLO; } if (value & AHCI_P_CMD_ICC_MASK) { p->cmd &= ~AHCI_P_CMD_ICC_MASK; } ahci_handle_port(p); break; } case AHCI_P_TFD: case AHCI_P_SIG: case AHCI_P_SSTS: EPRINTLN("pci_ahci_port: read only registers 0x%"PRIx64"", offset); break; case AHCI_P_SCTL: p->sctl = value; if (!(p->cmd & AHCI_P_CMD_ST)) { if (value & ATA_SC_DET_RESET) ahci_port_reset(p); } break; case AHCI_P_SERR: p->serr &= ~value; break; case AHCI_P_SACT: p->sact |= value; break; case AHCI_P_CI: p->ci |= value; ahci_handle_port(p); break; case AHCI_P_SNTF: case AHCI_P_FBS: default: break; } } static void pci_ahci_host_write(struct pci_ahci_softc *sc, uint64_t offset, uint64_t value) { DPRINTF("pci_ahci_host: write offset 0x%"PRIx64" value 0x%"PRIx64"", offset, value); switch (offset) { case AHCI_CAP: case AHCI_PI: case AHCI_VS: case AHCI_CAP2: DPRINTF("pci_ahci_host: read only registers 0x%"PRIx64"", offset); break; case AHCI_GHC: if (value & AHCI_GHC_HR) { ahci_reset(sc); break; } if (value & AHCI_GHC_IE) sc->ghc |= AHCI_GHC_IE; else sc->ghc &= ~AHCI_GHC_IE; ahci_generate_intr(sc, 0xffffffff); break; case AHCI_IS: sc->is &= ~value; ahci_generate_intr(sc, value); break; default: break; } } static void pci_ahci_write(struct pci_devinst *pi, int baridx, uint64_t offset, int size, uint64_t value) { struct pci_ahci_softc *sc = pi->pi_arg; assert(baridx == 5); assert((offset % 4) == 0 && size == 4); pthread_mutex_lock(&sc->mtx); if (offset < AHCI_OFFSET) pci_ahci_host_write(sc, offset, value); else if (offset < (uint64_t)AHCI_OFFSET + sc->ports * AHCI_STEP) pci_ahci_port_write(sc, offset, value); else EPRINTLN("pci_ahci: unknown i/o write offset 0x%"PRIx64"", offset); pthread_mutex_unlock(&sc->mtx); } static uint64_t pci_ahci_host_read(struct pci_ahci_softc *sc, uint64_t offset) { uint32_t value; switch (offset) { case AHCI_CAP: case AHCI_GHC: case AHCI_IS: case AHCI_PI: case AHCI_VS: case AHCI_CCCC: case AHCI_CCCP: case AHCI_EM_LOC: case AHCI_EM_CTL: case AHCI_CAP2: { uint32_t *p = &sc->cap; p += (offset - AHCI_CAP) / sizeof(uint32_t); value = *p; break; } default: value = 0; break; } DPRINTF("pci_ahci_host: read offset 0x%"PRIx64" value 0x%x", offset, value); return (value); } static uint64_t pci_ahci_port_read(struct pci_ahci_softc *sc, uint64_t offset) { uint32_t value; int port = (offset - AHCI_OFFSET) / AHCI_STEP; offset = (offset - AHCI_OFFSET) % AHCI_STEP; switch (offset) { case AHCI_P_CLB: case AHCI_P_CLBU: case AHCI_P_FB: case AHCI_P_FBU: case AHCI_P_IS: case AHCI_P_IE: case AHCI_P_CMD: case AHCI_P_TFD: case AHCI_P_SIG: case AHCI_P_SSTS: case AHCI_P_SCTL: case AHCI_P_SERR: case AHCI_P_SACT: case AHCI_P_CI: case AHCI_P_SNTF: case AHCI_P_FBS: { uint32_t *p= &sc->port[port].clb; p += (offset - AHCI_P_CLB) / sizeof(uint32_t); value = *p; break; } default: value = 0; break; } DPRINTF("pci_ahci_port %d: read offset 0x%"PRIx64" value 0x%x", port, offset, value); return value; } static uint64_t pci_ahci_read(struct pci_devinst *pi, int baridx, uint64_t regoff, int size) { struct pci_ahci_softc *sc = pi->pi_arg; uint64_t offset; uint32_t value; assert(baridx == 5); assert(size == 1 || size == 2 || size == 4); assert((regoff & (size - 1)) == 0); pthread_mutex_lock(&sc->mtx); offset = regoff & ~0x3; /* round down to a multiple of 4 bytes */ if (offset < AHCI_OFFSET) value = pci_ahci_host_read(sc, offset); else if (offset < (uint64_t)AHCI_OFFSET + sc->ports * AHCI_STEP) value = pci_ahci_port_read(sc, offset); else { value = 0; EPRINTLN("pci_ahci: unknown i/o read offset 0x%"PRIx64"", regoff); } value >>= 8 * (regoff & 0x3); pthread_mutex_unlock(&sc->mtx); return (value); } /* * Each AHCI controller has a "port" node which contains nodes for * each port named after the decimal number of the port (no leading * zeroes). Port nodes contain a "type" ("hd" or "cd"), as well as * options for blockif. For example: * * pci.0.1.0 * .device="ahci" * .port * .0 * .type="hd" * .path="/path/to/image" */ static int pci_ahci_legacy_config_port(nvlist_t *nvl, int port, const char *type, const char *opts) { char node_name[sizeof("XX")]; nvlist_t *port_nvl; snprintf(node_name, sizeof(node_name), "%d", port); port_nvl = create_relative_config_node(nvl, node_name); set_config_value_node(port_nvl, "type", type); return (blockif_legacy_config(port_nvl, opts)); } static int pci_ahci_legacy_config(nvlist_t *nvl, const char *opts) { nvlist_t *ports_nvl; const char *type; char *next, *next2, *str, *tofree; int p, ret; if (opts == NULL) return (0); ports_nvl = create_relative_config_node(nvl, "port"); ret = 1; tofree = str = strdup(opts); for (p = 0; p < MAX_PORTS && str != NULL; p++, str = next) { /* Identify and cut off type of present port. */ if (strncmp(str, "hd:", 3) == 0) { type = "hd"; str += 3; } else if (strncmp(str, "cd:", 3) == 0) { type = "cd"; str += 3; } else type = NULL; /* Find and cut off the next port options. */ next = strstr(str, ",hd:"); next2 = strstr(str, ",cd:"); if (next == NULL || (next2 != NULL && next2 < next)) next = next2; if (next != NULL) { next[0] = 0; next++; } if (str[0] == 0) continue; if (type == NULL) { EPRINTLN("Missing or invalid type for port %d: \"%s\"", p, str); goto out; } if (pci_ahci_legacy_config_port(ports_nvl, p, type, str) != 0) goto out; } ret = 0; out: free(tofree); return (ret); } static int pci_ahci_cd_legacy_config(nvlist_t *nvl, const char *opts) { nvlist_t *ports_nvl; ports_nvl = create_relative_config_node(nvl, "port"); return (pci_ahci_legacy_config_port(ports_nvl, 0, "cd", opts)); } static int pci_ahci_hd_legacy_config(nvlist_t *nvl, const char *opts) { nvlist_t *ports_nvl; ports_nvl = create_relative_config_node(nvl, "port"); return (pci_ahci_legacy_config_port(ports_nvl, 0, "hd", opts)); } static int pci_ahci_init(struct pci_devinst *pi, nvlist_t *nvl) { char bident[sizeof("XXX:XXX:XXX")]; char node_name[sizeof("XX")]; struct blockif_ctxt *bctxt; struct pci_ahci_softc *sc; int atapi, ret, slots, p; MD5_CTX mdctx; u_char digest[16]; const char *path, *type, *value; nvlist_t *ports_nvl, *port_nvl; ret = 0; #ifdef AHCI_DEBUG dbg = fopen("/tmp/log", "w+"); #endif sc = calloc(1, sizeof(struct pci_ahci_softc)); pi->pi_arg = sc; sc->asc_pi = pi; pthread_mutex_init(&sc->mtx, NULL); sc->ports = 0; sc->pi = 0; slots = 32; ports_nvl = find_relative_config_node(nvl, "port"); for (p = 0; ports_nvl != NULL && p < MAX_PORTS; p++) { struct ata_params *ata_ident = &sc->port[p].ata_ident; char ident[AHCI_PORT_IDENT]; snprintf(node_name, sizeof(node_name), "%d", p); port_nvl = find_relative_config_node(ports_nvl, node_name); if (port_nvl == NULL) continue; type = get_config_value_node(port_nvl, "type"); if (type == NULL) continue; if (strcmp(type, "hd") == 0) atapi = 0; else atapi = 1; /* * Attempt to open the backing image. Use the PCI slot/func * and the port number for the identifier string. */ snprintf(bident, sizeof(bident), "%u:%u:%u", pi->pi_slot, pi->pi_func, p); bctxt = blockif_open(port_nvl, bident); if (bctxt == NULL) { sc->ports = p; ret = 1; goto open_fail; } ret = blockif_add_boot_device(pi, bctxt); if (ret) { sc->ports = p; goto open_fail; } sc->port[p].bctx = bctxt; sc->port[p].pr_sc = sc; sc->port[p].port = p; sc->port[p].atapi = atapi; /* * Create an identifier for the backing file. * Use parts of the md5 sum of the filename */ path = get_config_value_node(port_nvl, "path"); MD5Init(&mdctx); MD5Update(&mdctx, path, strlen(path)); MD5Final(digest, &mdctx); snprintf(ident, AHCI_PORT_IDENT, "BHYVE-%02X%02X-%02X%02X-%02X%02X", digest[0], digest[1], digest[2], digest[3], digest[4], digest[5]); memset(ata_ident, 0, sizeof(struct ata_params)); ata_string((uint8_t*)&ata_ident->serial, ident, 20); ata_string((uint8_t*)&ata_ident->revision, "001", 8); if (atapi) ata_string((uint8_t*)&ata_ident->model, "BHYVE SATA DVD ROM", 40); else ata_string((uint8_t*)&ata_ident->model, "BHYVE SATA DISK", 40); value = get_config_value_node(port_nvl, "nmrr"); if (value != NULL) ata_ident->media_rotation_rate = atoi(value); value = get_config_value_node(port_nvl, "ser"); if (value != NULL) ata_string((uint8_t*)(&ata_ident->serial), value, 20); value = get_config_value_node(port_nvl, "rev"); if (value != NULL) ata_string((uint8_t*)(&ata_ident->revision), value, 8); value = get_config_value_node(port_nvl, "model"); if (value != NULL) ata_string((uint8_t*)(&ata_ident->model), value, 40); ata_identify_init(&sc->port[p], atapi); #ifndef __FreeBSD__ /* * Attempt to enable the write cache for this device, as the * guest will issue FLUSH commands when it requires durability. * * Failure here is fine, since an always-sync device will not * have an impact on correctness. */ (void) blockif_set_wce(bctxt, 1); #endif /* * Allocate blockif request structures and add them * to the free list */ pci_ahci_ioreq_init(&sc->port[p]); sc->pi |= (1 << p); if (sc->port[p].ioqsz < slots) slots = sc->port[p].ioqsz; } sc->ports = p; /* Intel ICH8 AHCI */ --slots; if (sc->ports < DEF_PORTS) sc->ports = DEF_PORTS; sc->cap = AHCI_CAP_64BIT | AHCI_CAP_SNCQ | AHCI_CAP_SSNTF | AHCI_CAP_SMPS | AHCI_CAP_SSS | AHCI_CAP_SALP | AHCI_CAP_SAL | AHCI_CAP_SCLO | (0x3 << AHCI_CAP_ISS_SHIFT)| AHCI_CAP_PMD | AHCI_CAP_SSC | AHCI_CAP_PSC | (slots << AHCI_CAP_NCS_SHIFT) | AHCI_CAP_SXS | (sc->ports - 1); sc->vs = 0x10300; sc->cap2 = AHCI_CAP2_APST; ahci_reset(sc); pci_set_cfgdata16(pi, PCIR_DEVICE, 0x2821); pci_set_cfgdata16(pi, PCIR_VENDOR, 0x8086); pci_set_cfgdata8(pi, PCIR_CLASS, PCIC_STORAGE); pci_set_cfgdata8(pi, PCIR_SUBCLASS, PCIS_STORAGE_SATA); pci_set_cfgdata8(pi, PCIR_PROGIF, PCIP_STORAGE_SATA_AHCI_1_0); p = MIN(sc->ports, 16); p = flsl(p) - ((p & (p - 1)) ? 0 : 1); pci_emul_add_msicap(pi, 1 << p); pci_emul_alloc_bar(pi, 5, PCIBAR_MEM32, AHCI_OFFSET + sc->ports * AHCI_STEP); pci_lintr_request(pi); open_fail: if (ret) { for (p = 0; p < sc->ports; p++) { if (sc->port[p].bctx != NULL) blockif_close(sc->port[p].bctx); } free(sc); } return (ret); } /* * Use separate emulation names to distinguish drive and atapi devices */ static const struct pci_devemu pci_de_ahci = { .pe_emu = "ahci", .pe_init = pci_ahci_init, .pe_legacy_config = pci_ahci_legacy_config, .pe_barwrite = pci_ahci_write, .pe_barread = pci_ahci_read, }; PCI_EMUL_SET(pci_de_ahci); static const struct pci_devemu pci_de_ahci_hd = { .pe_emu = "ahci-hd", .pe_legacy_config = pci_ahci_hd_legacy_config, .pe_alias = "ahci", }; PCI_EMUL_SET(pci_de_ahci_hd); static const struct pci_devemu pci_de_ahci_cd = { .pe_emu = "ahci-cd", .pe_legacy_config = pci_ahci_cd_legacy_config, .pe_alias = "ahci", }; PCI_EMUL_SET(pci_de_ahci_cd); /* * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2016 Alexander Motin * Copyright (c) 2015 Peter Grehan * Copyright (c) 2013 Jeremiah Lott, Avere Systems * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer * in this position and unchanged. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include #ifndef WITHOUT_CAPSICUM #include #endif #include #include #include #include #include #include #ifndef WITHOUT_CAPSICUM #include #endif #include #include #include #include #include #include #include #include #include #include #include #include "e1000_regs.h" #include "e1000_defines.h" #include "mii.h" #include "bhyverun.h" #include "config.h" #include "debug.h" #include "pci_emul.h" #include "mevent.h" #include "net_utils.h" #include "net_backends.h" /* Hardware/register definitions XXX: move some to common code. */ #define E82545_VENDOR_ID_INTEL 0x8086 #define E82545_DEV_ID_82545EM_COPPER 0x100F #define E82545_SUBDEV_ID 0x1008 #define E82545_REVISION_4 4 #define E82545_MDIC_DATA_MASK 0x0000FFFF #define E82545_MDIC_OP_MASK 0x0c000000 #define E82545_MDIC_IE 0x20000000 #define E82545_EECD_FWE_DIS 0x00000010 /* Flash writes disabled */ #define E82545_EECD_FWE_EN 0x00000020 /* Flash writes enabled */ #define E82545_EECD_FWE_MASK 0x00000030 /* Flash writes mask */ #define E82545_BAR_REGISTER 0 #define E82545_BAR_REGISTER_LEN (128*1024) #define E82545_BAR_FLASH 1 #define E82545_BAR_FLASH_LEN (64*1024) #define E82545_BAR_IO 2 #define E82545_BAR_IO_LEN 8 #define E82545_IOADDR 0x00000000 #define E82545_IODATA 0x00000004 #define E82545_IO_REGISTER_MAX 0x0001FFFF #define E82545_IO_FLASH_BASE 0x00080000 #define E82545_IO_FLASH_MAX 0x000FFFFF #define E82545_ARRAY_ENTRY(reg, offset) (reg + (offset<<2)) #define E82545_RAR_MAX 15 #define E82545_MTA_MAX 127 #define E82545_VFTA_MAX 127 /* Slightly modified from the driver versions, hardcoded for 3 opcode bits, * followed by 6 address bits. * TODO: make opcode bits and addr bits configurable? * NVM Commands - Microwire */ #define E82545_NVM_OPCODE_BITS 3 #define E82545_NVM_ADDR_BITS 6 #define E82545_NVM_DATA_BITS 16 #define E82545_NVM_OPADDR_BITS (E82545_NVM_OPCODE_BITS + E82545_NVM_ADDR_BITS) #define E82545_NVM_ADDR_MASK ((1 << E82545_NVM_ADDR_BITS)-1) #define E82545_NVM_OPCODE_MASK \ (((1 << E82545_NVM_OPCODE_BITS) - 1) << E82545_NVM_ADDR_BITS) #define E82545_NVM_OPCODE_READ (0x6 << E82545_NVM_ADDR_BITS) /* read */ #define E82545_NVM_OPCODE_WRITE (0x5 << E82545_NVM_ADDR_BITS) /* write */ #define E82545_NVM_OPCODE_ERASE (0x7 << E82545_NVM_ADDR_BITS) /* erase */ #define E82545_NVM_OPCODE_EWEN (0x4 << E82545_NVM_ADDR_BITS) /* wr-enable */ #define E82545_NVM_EEPROM_SIZE 64 /* 64 * 16-bit values == 128K */ #define E1000_ICR_SRPD 0x00010000 /* This is an arbitrary number. There is no hard limit on the chip. */ #define I82545_MAX_TXSEGS 64 /* Legacy receive descriptor */ struct e1000_rx_desc { uint64_t buffer_addr; /* Address of the descriptor's data buffer */ uint16_t length; /* Length of data DMAed into data buffer */ uint16_t csum; /* Packet checksum */ uint8_t status; /* Descriptor status */ uint8_t errors; /* Descriptor Errors */ uint16_t special; }; /* Transmit descriptor types */ #define E1000_TXD_MASK (E1000_TXD_CMD_DEXT | 0x00F00000) #define E1000_TXD_TYP_L (0) #define E1000_TXD_TYP_C (E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_C) #define E1000_TXD_TYP_D (E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D) /* Legacy transmit descriptor */ struct e1000_tx_desc { uint64_t buffer_addr; /* Address of the descriptor's data buffer */ union { uint32_t data; struct { uint16_t length; /* Data buffer length */ uint8_t cso; /* Checksum offset */ uint8_t cmd; /* Descriptor control */ } flags; } lower; union { uint32_t data; struct { uint8_t status; /* Descriptor status */ uint8_t css; /* Checksum start */ uint16_t special; } fields; } upper; }; /* Context descriptor */ struct e1000_context_desc { union { uint32_t ip_config; struct { uint8_t ipcss; /* IP checksum start */ uint8_t ipcso; /* IP checksum offset */ uint16_t ipcse; /* IP checksum end */ } ip_fields; } lower_setup; union { uint32_t tcp_config; struct { uint8_t tucss; /* TCP checksum start */ uint8_t tucso; /* TCP checksum offset */ uint16_t tucse; /* TCP checksum end */ } tcp_fields; } upper_setup; uint32_t cmd_and_length; union { uint32_t data; struct { uint8_t status; /* Descriptor status */ uint8_t hdr_len; /* Header length */ uint16_t mss; /* Maximum segment size */ } fields; } tcp_seg_setup; }; /* Data descriptor */ struct e1000_data_desc { uint64_t buffer_addr; /* Address of the descriptor's buffer address */ union { uint32_t data; struct { uint16_t length; /* Data buffer length */ uint8_t typ_len_ext; uint8_t cmd; } flags; } lower; union { uint32_t data; struct { uint8_t status; /* Descriptor status */ uint8_t popts; /* Packet Options */ uint16_t special; } fields; } upper; }; union e1000_tx_udesc { struct e1000_tx_desc td; struct e1000_context_desc cd; struct e1000_data_desc dd; }; /* Tx checksum info for a packet. */ struct ck_info { int ck_valid; /* ck_info is valid */ uint8_t ck_start; /* start byte of cksum calcuation */ uint8_t ck_off; /* offset of cksum insertion */ uint16_t ck_len; /* length of cksum calc: 0 is to packet-end */ }; /* * Debug printf */ static int e82545_debug = 0; #define WPRINTF(msg,params...) PRINTLN("e82545: " msg, ##params) #define DPRINTF(msg,params...) if (e82545_debug) WPRINTF(msg, params) #define MIN(a,b) (((a)<(b))?(a):(b)) #define MAX(a,b) (((a)>(b))?(a):(b)) /* s/w representation of the RAL/RAH regs */ struct eth_uni { int eu_valid; int eu_addrsel; struct ether_addr eu_eth; }; struct e82545_softc { struct pci_devinst *esc_pi; struct vmctx *esc_ctx; struct mevent *esc_mevpitr; pthread_mutex_t esc_mtx; struct ether_addr esc_mac; net_backend_t *esc_be; /* General */ uint32_t esc_CTRL; /* x0000 device ctl */ uint32_t esc_FCAL; /* x0028 flow ctl addr lo */ uint32_t esc_FCAH; /* x002C flow ctl addr hi */ uint32_t esc_FCT; /* x0030 flow ctl type */ uint32_t esc_VET; /* x0038 VLAN eth type */ uint32_t esc_FCTTV; /* x0170 flow ctl tx timer */ uint32_t esc_LEDCTL; /* x0E00 LED control */ uint32_t esc_PBA; /* x1000 pkt buffer allocation */ /* Interrupt control */ int esc_irq_asserted; uint32_t esc_ICR; /* x00C0 cause read/clear */ uint32_t esc_ITR; /* x00C4 intr throttling */ uint32_t esc_ICS; /* x00C8 cause set */ uint32_t esc_IMS; /* x00D0 mask set/read */ uint32_t esc_IMC; /* x00D8 mask clear */ /* Transmit */ union e1000_tx_udesc *esc_txdesc; struct e1000_context_desc esc_txctx; pthread_t esc_tx_tid; pthread_cond_t esc_tx_cond; int esc_tx_enabled; int esc_tx_active; uint32_t esc_TXCW; /* x0178 transmit config */ uint32_t esc_TCTL; /* x0400 transmit ctl */ uint32_t esc_TIPG; /* x0410 inter-packet gap */ uint16_t esc_AIT; /* x0458 Adaptive Interframe Throttle */ uint64_t esc_tdba; /* verified 64-bit desc table addr */ uint32_t esc_TDBAL; /* x3800 desc table addr, low bits */ uint32_t esc_TDBAH; /* x3804 desc table addr, hi 32-bits */ uint32_t esc_TDLEN; /* x3808 # descriptors in bytes */ uint16_t esc_TDH; /* x3810 desc table head idx */ uint16_t esc_TDHr; /* internal read version of TDH */ uint16_t esc_TDT; /* x3818 desc table tail idx */ uint32_t esc_TIDV; /* x3820 intr delay */ uint32_t esc_TXDCTL; /* x3828 desc control */ uint32_t esc_TADV; /* x382C intr absolute delay */ /* L2 frame acceptance */ struct eth_uni esc_uni[16]; /* 16 x unicast MAC addresses */ uint32_t esc_fmcast[128]; /* Multicast filter bit-match */ uint32_t esc_fvlan[128]; /* VLAN 4096-bit filter */ /* Receive */ struct e1000_rx_desc *esc_rxdesc; pthread_cond_t esc_rx_cond; int esc_rx_enabled; int esc_rx_active; int esc_rx_loopback; uint32_t esc_RCTL; /* x0100 receive ctl */ uint32_t esc_FCRTL; /* x2160 flow cntl thresh, low */ uint32_t esc_FCRTH; /* x2168 flow cntl thresh, hi */ uint64_t esc_rdba; /* verified 64-bit desc table addr */ uint32_t esc_RDBAL; /* x2800 desc table addr, low bits */ uint32_t esc_RDBAH; /* x2804 desc table addr, hi 32-bits*/ uint32_t esc_RDLEN; /* x2808 #descriptors */ uint16_t esc_RDH; /* x2810 desc table head idx */ uint16_t esc_RDT; /* x2818 desc table tail idx */ uint32_t esc_RDTR; /* x2820 intr delay */ uint32_t esc_RXDCTL; /* x2828 desc control */ uint32_t esc_RADV; /* x282C intr absolute delay */ uint32_t esc_RSRPD; /* x2C00 recv small packet detect */ uint32_t esc_RXCSUM; /* x5000 receive cksum ctl */ /* IO Port register access */ uint32_t io_addr; /* Shadow copy of MDIC */ uint32_t mdi_control; /* Shadow copy of EECD */ uint32_t eeprom_control; /* Latest NVM in/out */ uint16_t nvm_data; uint16_t nvm_opaddr; /* stats */ uint32_t missed_pkt_count; /* dropped for no room in rx queue */ uint32_t pkt_rx_by_size[6]; uint32_t pkt_tx_by_size[6]; uint32_t good_pkt_rx_count; uint32_t bcast_pkt_rx_count; uint32_t mcast_pkt_rx_count; uint32_t good_pkt_tx_count; uint32_t bcast_pkt_tx_count; uint32_t mcast_pkt_tx_count; uint32_t oversize_rx_count; uint32_t tso_tx_count; uint64_t good_octets_rx; uint64_t good_octets_tx; uint64_t missed_octets; /* counts missed and oversized */ uint8_t nvm_bits:6; /* number of bits remaining in/out */ uint8_t nvm_mode:2; #define E82545_NVM_MODE_OPADDR 0x0 #define E82545_NVM_MODE_DATAIN 0x1 #define E82545_NVM_MODE_DATAOUT 0x2 /* EEPROM data */ uint16_t eeprom_data[E82545_NVM_EEPROM_SIZE]; }; static void e82545_reset(struct e82545_softc *sc, int dev); static void e82545_rx_enable(struct e82545_softc *sc); static void e82545_rx_disable(struct e82545_softc *sc); static void e82545_rx_callback(int fd, enum ev_type type, void *param); static void e82545_tx_start(struct e82545_softc *sc); static void e82545_tx_enable(struct e82545_softc *sc); static void e82545_tx_disable(struct e82545_softc *sc); static inline int __unused e82545_size_stat_index(uint32_t size) { if (size <= 64) { return 0; } else if (size >= 1024) { return 5; } else { /* should be 1-4 */ return (ffs(size) - 6); } } static void e82545_init_eeprom(struct e82545_softc *sc) { uint16_t checksum, i; /* mac addr */ sc->eeprom_data[NVM_MAC_ADDR] = ((uint16_t)sc->esc_mac.octet[0]) | (((uint16_t)sc->esc_mac.octet[1]) << 8); sc->eeprom_data[NVM_MAC_ADDR+1] = ((uint16_t)sc->esc_mac.octet[2]) | (((uint16_t)sc->esc_mac.octet[3]) << 8); sc->eeprom_data[NVM_MAC_ADDR+2] = ((uint16_t)sc->esc_mac.octet[4]) | (((uint16_t)sc->esc_mac.octet[5]) << 8); /* pci ids */ sc->eeprom_data[NVM_SUB_DEV_ID] = E82545_SUBDEV_ID; sc->eeprom_data[NVM_SUB_VEN_ID] = E82545_VENDOR_ID_INTEL; sc->eeprom_data[NVM_DEV_ID] = E82545_DEV_ID_82545EM_COPPER; sc->eeprom_data[NVM_VEN_ID] = E82545_VENDOR_ID_INTEL; /* fill in the checksum */ checksum = 0; for (i = 0; i < NVM_CHECKSUM_REG; i++) { checksum += sc->eeprom_data[i]; } checksum = NVM_SUM - checksum; sc->eeprom_data[NVM_CHECKSUM_REG] = checksum; DPRINTF("eeprom checksum: 0x%x", checksum); } static void e82545_write_mdi(struct e82545_softc *sc __unused, uint8_t reg_addr, uint8_t phy_addr, uint32_t data) { DPRINTF("Write mdi reg:0x%x phy:0x%x data: 0x%x", reg_addr, phy_addr, data); } static uint32_t e82545_read_mdi(struct e82545_softc *sc __unused, uint8_t reg_addr, uint8_t phy_addr) { //DPRINTF("Read mdi reg:0x%x phy:0x%x", reg_addr, phy_addr); switch (reg_addr) { case PHY_STATUS: return (MII_SR_LINK_STATUS | MII_SR_AUTONEG_CAPS | MII_SR_AUTONEG_COMPLETE); case PHY_AUTONEG_ADV: return NWAY_AR_SELECTOR_FIELD; case PHY_LP_ABILITY: return 0; case PHY_1000T_STATUS: return (SR_1000T_LP_FD_CAPS | SR_1000T_REMOTE_RX_STATUS | SR_1000T_LOCAL_RX_STATUS); case PHY_ID1: return (M88E1011_I_PHY_ID >> 16) & 0xFFFF; case PHY_ID2: return (M88E1011_I_PHY_ID | E82545_REVISION_4) & 0xFFFF; default: DPRINTF("Unknown mdi read reg:0x%x phy:0x%x", reg_addr, phy_addr); return 0; } /* not reached */ } static void e82545_eecd_strobe(struct e82545_softc *sc) { /* Microwire state machine */ /* DPRINTF("eeprom state machine srtobe " "0x%x 0x%x 0x%x 0x%x", sc->nvm_mode, sc->nvm_bits, sc->nvm_opaddr, sc->nvm_data);*/ if (sc->nvm_bits == 0) { DPRINTF("eeprom state machine not expecting data! " "0x%x 0x%x 0x%x 0x%x", sc->nvm_mode, sc->nvm_bits, sc->nvm_opaddr, sc->nvm_data); return; } sc->nvm_bits--; if (sc->nvm_mode == E82545_NVM_MODE_DATAOUT) { /* shifting out */ if (sc->nvm_data & 0x8000) { sc->eeprom_control |= E1000_EECD_DO; } else { sc->eeprom_control &= ~E1000_EECD_DO; } sc->nvm_data <<= 1; if (sc->nvm_bits == 0) { /* read done, back to opcode mode. */ sc->nvm_opaddr = 0; sc->nvm_mode = E82545_NVM_MODE_OPADDR; sc->nvm_bits = E82545_NVM_OPADDR_BITS; } } else if (sc->nvm_mode == E82545_NVM_MODE_DATAIN) { /* shifting in */ sc->nvm_data <<= 1; if (sc->eeprom_control & E1000_EECD_DI) { sc->nvm_data |= 1; } if (sc->nvm_bits == 0) { /* eeprom write */ uint16_t op = sc->nvm_opaddr & E82545_NVM_OPCODE_MASK; uint16_t addr = sc->nvm_opaddr & E82545_NVM_ADDR_MASK; if (op != E82545_NVM_OPCODE_WRITE) { DPRINTF("Illegal eeprom write op 0x%x", sc->nvm_opaddr); } else if (addr >= E82545_NVM_EEPROM_SIZE) { DPRINTF("Illegal eeprom write addr 0x%x", sc->nvm_opaddr); } else { DPRINTF("eeprom write eeprom[0x%x] = 0x%x", addr, sc->nvm_data); sc->eeprom_data[addr] = sc->nvm_data; } /* back to opcode mode */ sc->nvm_opaddr = 0; sc->nvm_mode = E82545_NVM_MODE_OPADDR; sc->nvm_bits = E82545_NVM_OPADDR_BITS; } } else if (sc->nvm_mode == E82545_NVM_MODE_OPADDR) { sc->nvm_opaddr <<= 1; if (sc->eeprom_control & E1000_EECD_DI) { sc->nvm_opaddr |= 1; } if (sc->nvm_bits == 0) { uint16_t op = sc->nvm_opaddr & E82545_NVM_OPCODE_MASK; switch (op) { case E82545_NVM_OPCODE_EWEN: DPRINTF("eeprom write enable: 0x%x", sc->nvm_opaddr); /* back to opcode mode */ sc->nvm_opaddr = 0; sc->nvm_mode = E82545_NVM_MODE_OPADDR; sc->nvm_bits = E82545_NVM_OPADDR_BITS; break; case E82545_NVM_OPCODE_READ: { uint16_t addr = sc->nvm_opaddr & E82545_NVM_ADDR_MASK; sc->nvm_mode = E82545_NVM_MODE_DATAOUT; sc->nvm_bits = E82545_NVM_DATA_BITS; if (addr < E82545_NVM_EEPROM_SIZE) { sc->nvm_data = sc->eeprom_data[addr]; DPRINTF("eeprom read: eeprom[0x%x] = 0x%x", addr, sc->nvm_data); } else { DPRINTF("eeprom illegal read: 0x%x", sc->nvm_opaddr); sc->nvm_data = 0; } break; } case E82545_NVM_OPCODE_WRITE: sc->nvm_mode = E82545_NVM_MODE_DATAIN; sc->nvm_bits = E82545_NVM_DATA_BITS; sc->nvm_data = 0; break; default: DPRINTF("eeprom unknown op: 0x%x", sc->nvm_opaddr); /* back to opcode mode */ sc->nvm_opaddr = 0; sc->nvm_mode = E82545_NVM_MODE_OPADDR; sc->nvm_bits = E82545_NVM_OPADDR_BITS; } } } else { DPRINTF("eeprom state machine wrong state! " "0x%x 0x%x 0x%x 0x%x", sc->nvm_mode, sc->nvm_bits, sc->nvm_opaddr, sc->nvm_data); } } static void e82545_itr_callback(int fd __unused, enum ev_type type __unused, void *param) { uint32_t new; struct e82545_softc *sc = param; pthread_mutex_lock(&sc->esc_mtx); new = sc->esc_ICR & sc->esc_IMS; if (new && !sc->esc_irq_asserted) { DPRINTF("itr callback: lintr assert %x", new); sc->esc_irq_asserted = 1; pci_lintr_assert(sc->esc_pi); } else { mevent_delete(sc->esc_mevpitr); sc->esc_mevpitr = NULL; } pthread_mutex_unlock(&sc->esc_mtx); } static void e82545_icr_assert(struct e82545_softc *sc, uint32_t bits) { uint32_t new; DPRINTF("icr assert: 0x%x", bits); /* * An interrupt is only generated if bits are set that * aren't already in the ICR, these bits are unmasked, * and there isn't an interrupt already pending. */ new = bits & ~sc->esc_ICR & sc->esc_IMS; sc->esc_ICR |= bits; if (new == 0) { DPRINTF("icr assert: masked %x, ims %x", new, sc->esc_IMS); } else if (sc->esc_mevpitr != NULL) { DPRINTF("icr assert: throttled %x, ims %x", new, sc->esc_IMS); } else if (!sc->esc_irq_asserted) { DPRINTF("icr assert: lintr assert %x", new); sc->esc_irq_asserted = 1; pci_lintr_assert(sc->esc_pi); if (sc->esc_ITR != 0) { sc->esc_mevpitr = mevent_add( (sc->esc_ITR + 3905) / 3906, /* 256ns -> 1ms */ EVF_TIMER, e82545_itr_callback, sc); } } } static void e82545_ims_change(struct e82545_softc *sc, uint32_t bits) { uint32_t new; /* * Changing the mask may allow previously asserted * but masked interrupt requests to generate an interrupt. */ new = bits & sc->esc_ICR & ~sc->esc_IMS; sc->esc_IMS |= bits; if (new == 0) { DPRINTF("ims change: masked %x, ims %x", new, sc->esc_IMS); } else if (sc->esc_mevpitr != NULL) { DPRINTF("ims change: throttled %x, ims %x", new, sc->esc_IMS); } else if (!sc->esc_irq_asserted) { DPRINTF("ims change: lintr assert %x", new); sc->esc_irq_asserted = 1; pci_lintr_assert(sc->esc_pi); if (sc->esc_ITR != 0) { sc->esc_mevpitr = mevent_add( (sc->esc_ITR + 3905) / 3906, /* 256ns -> 1ms */ EVF_TIMER, e82545_itr_callback, sc); } } } static void e82545_icr_deassert(struct e82545_softc *sc, uint32_t bits) { DPRINTF("icr deassert: 0x%x", bits); sc->esc_ICR &= ~bits; /* * If there are no longer any interrupt sources and there * was an asserted interrupt, clear it */ if (sc->esc_irq_asserted && !(sc->esc_ICR & sc->esc_IMS)) { DPRINTF("icr deassert: lintr deassert %x", bits); pci_lintr_deassert(sc->esc_pi); sc->esc_irq_asserted = 0; } } static void e82545_intr_write(struct e82545_softc *sc, uint32_t offset, uint32_t value) { DPRINTF("intr_write: off %x, val %x", offset, value); switch (offset) { case E1000_ICR: e82545_icr_deassert(sc, value); break; case E1000_ITR: sc->esc_ITR = value; break; case E1000_ICS: sc->esc_ICS = value; /* not used: store for debug */ e82545_icr_assert(sc, value); break; case E1000_IMS: e82545_ims_change(sc, value); break; case E1000_IMC: sc->esc_IMC = value; /* for debug */ sc->esc_IMS &= ~value; // XXX clear interrupts if all ICR bits now masked // and interrupt was pending ? break; default: break; } } static uint32_t e82545_intr_read(struct e82545_softc *sc, uint32_t offset) { uint32_t retval; retval = 0; DPRINTF("intr_read: off %x", offset); switch (offset) { case E1000_ICR: retval = sc->esc_ICR; sc->esc_ICR = 0; e82545_icr_deassert(sc, ~0); break; case E1000_ITR: retval = sc->esc_ITR; break; case E1000_ICS: /* write-only register */ break; case E1000_IMS: retval = sc->esc_IMS; break; case E1000_IMC: /* write-only register */ break; default: break; } return (retval); } static void e82545_devctl(struct e82545_softc *sc, uint32_t val) { sc->esc_CTRL = val & ~E1000_CTRL_RST; if (val & E1000_CTRL_RST) { DPRINTF("e1k: s/w reset, ctl %x", val); e82545_reset(sc, 1); } /* XXX check for phy reset ? */ } static void e82545_rx_update_rdba(struct e82545_softc *sc) { /* XXX verify desc base/len within phys mem range */ sc->esc_rdba = (uint64_t)sc->esc_RDBAH << 32 | sc->esc_RDBAL; /* Cache host mapping of guest descriptor array */ sc->esc_rxdesc = paddr_guest2host(sc->esc_ctx, sc->esc_rdba, sc->esc_RDLEN); } static void e82545_rx_ctl(struct e82545_softc *sc, uint32_t val) { int on; on = ((val & E1000_RCTL_EN) == E1000_RCTL_EN); /* Save RCTL after stripping reserved bits 31:27,24,21,14,11:10,0 */ sc->esc_RCTL = val & ~0xF9204c01; DPRINTF("rx_ctl - %s RCTL %x, val %x", on ? "on" : "off", sc->esc_RCTL, val); /* state change requested */ if (on != sc->esc_rx_enabled) { if (on) { /* Catch disallowed/unimplemented settings */ //assert(!(val & E1000_RCTL_LBM_TCVR)); if (sc->esc_RCTL & E1000_RCTL_LBM_TCVR) { sc->esc_rx_loopback = 1; } else { sc->esc_rx_loopback = 0; } e82545_rx_update_rdba(sc); e82545_rx_enable(sc); } else { e82545_rx_disable(sc); sc->esc_rx_loopback = 0; sc->esc_rdba = 0; sc->esc_rxdesc = NULL; } } } static void e82545_tx_update_tdba(struct e82545_softc *sc) { /* XXX verify desc base/len within phys mem range */ sc->esc_tdba = (uint64_t)sc->esc_TDBAH << 32 | sc->esc_TDBAL; /* Cache host mapping of guest descriptor array */ sc->esc_txdesc = paddr_guest2host(sc->esc_ctx, sc->esc_tdba, sc->esc_TDLEN); } static void e82545_tx_ctl(struct e82545_softc *sc, uint32_t val) { int on; on = ((val & E1000_TCTL_EN) == E1000_TCTL_EN); /* ignore TCTL_EN settings that don't change state */ if (on == sc->esc_tx_enabled) return; if (on) { e82545_tx_update_tdba(sc); e82545_tx_enable(sc); } else { e82545_tx_disable(sc); sc->esc_tdba = 0; sc->esc_txdesc = NULL; } /* Save TCTL value after stripping reserved bits 31:25,23,2,0 */ sc->esc_TCTL = val & ~0xFE800005; } static int e82545_bufsz(uint32_t rctl) { switch (rctl & (E1000_RCTL_BSEX | E1000_RCTL_SZ_256)) { case (E1000_RCTL_SZ_2048): return (2048); case (E1000_RCTL_SZ_1024): return (1024); case (E1000_RCTL_SZ_512): return (512); case (E1000_RCTL_SZ_256): return (256); case (E1000_RCTL_BSEX|E1000_RCTL_SZ_16384): return (16384); case (E1000_RCTL_BSEX|E1000_RCTL_SZ_8192): return (8192); case (E1000_RCTL_BSEX|E1000_RCTL_SZ_4096): return (4096); } return (256); /* Forbidden value. */ } /* XXX one packet at a time until this is debugged */ static void e82545_rx_callback(int fd __unused, enum ev_type type __unused, void *param) { struct e82545_softc *sc = param; struct e1000_rx_desc *rxd; struct iovec vec[64]; ssize_t len; int left, lim, maxpktsz, maxpktdesc, bufsz, i, n, size; uint32_t cause = 0; uint16_t *tp, tag, head; pthread_mutex_lock(&sc->esc_mtx); DPRINTF("rx_run: head %x, tail %x", sc->esc_RDH, sc->esc_RDT); if (!sc->esc_rx_enabled || sc->esc_rx_loopback) { DPRINTF("rx disabled (!%d || %d) -- packet(s) dropped", sc->esc_rx_enabled, sc->esc_rx_loopback); while (netbe_rx_discard(sc->esc_be) > 0) { } goto done1; } bufsz = e82545_bufsz(sc->esc_RCTL); maxpktsz = (sc->esc_RCTL & E1000_RCTL_LPE) ? 16384 : 1522; maxpktdesc = (maxpktsz + bufsz - 1) / bufsz; size = sc->esc_RDLEN / 16; head = sc->esc_RDH; left = (size + sc->esc_RDT - head) % size; if (left < maxpktdesc) { DPRINTF("rx overflow (%d < %d) -- packet(s) dropped", left, maxpktdesc); while (netbe_rx_discard(sc->esc_be) > 0) { } goto done1; } sc->esc_rx_active = 1; pthread_mutex_unlock(&sc->esc_mtx); for (lim = size / 4; lim > 0 && left >= maxpktdesc; lim -= n) { /* Grab rx descriptor pointed to by the head pointer */ for (i = 0; i < maxpktdesc; i++) { rxd = &sc->esc_rxdesc[(head + i) % size]; vec[i].iov_base = paddr_guest2host(sc->esc_ctx, rxd->buffer_addr, bufsz); vec[i].iov_len = bufsz; } len = netbe_recv(sc->esc_be, vec, maxpktdesc); if (len <= 0) { DPRINTF("netbe_recv() returned %zd", len); goto done; } /* * Adjust the packet length based on whether the CRC needs * to be stripped or if the packet is less than the minimum * eth packet size. */ if (len < ETHER_MIN_LEN - ETHER_CRC_LEN) len = ETHER_MIN_LEN - ETHER_CRC_LEN; if (!(sc->esc_RCTL & E1000_RCTL_SECRC)) len += ETHER_CRC_LEN; n = (len + bufsz - 1) / bufsz; DPRINTF("packet read %zd bytes, %d segs, head %d", len, n, head); /* Apply VLAN filter. */ tp = (uint16_t *)vec[0].iov_base + 6; if ((sc->esc_RCTL & E1000_RCTL_VFE) && (ntohs(tp[0]) == sc->esc_VET)) { tag = ntohs(tp[1]) & 0x0fff; if ((sc->esc_fvlan[tag >> 5] & (1 << (tag & 0x1f))) != 0) { DPRINTF("known VLAN %d", tag); } else { DPRINTF("unknown VLAN %d", tag); n = 0; continue; } } /* Update all consumed descriptors. */ for (i = 0; i < n - 1; i++) { rxd = &sc->esc_rxdesc[(head + i) % size]; rxd->length = bufsz; rxd->csum = 0; rxd->errors = 0; rxd->special = 0; rxd->status = E1000_RXD_STAT_DD; } rxd = &sc->esc_rxdesc[(head + i) % size]; rxd->length = len % bufsz; rxd->csum = 0; rxd->errors = 0; rxd->special = 0; /* XXX signal no checksum for now */ rxd->status = E1000_RXD_STAT_PIF | E1000_RXD_STAT_IXSM | E1000_RXD_STAT_EOP | E1000_RXD_STAT_DD; /* Schedule receive interrupts. */ if ((uint32_t)len <= sc->esc_RSRPD) { cause |= E1000_ICR_SRPD | E1000_ICR_RXT0; } else { /* XXX: RDRT and RADV timers should be here. */ cause |= E1000_ICR_RXT0; } head = (head + n) % size; left -= n; } done: pthread_mutex_lock(&sc->esc_mtx); sc->esc_rx_active = 0; if (sc->esc_rx_enabled == 0) pthread_cond_signal(&sc->esc_rx_cond); sc->esc_RDH = head; /* Respect E1000_RCTL_RDMTS */ left = (size + sc->esc_RDT - head) % size; if (left < (size >> (((sc->esc_RCTL >> 8) & 3) + 1))) cause |= E1000_ICR_RXDMT0; /* Assert all accumulated interrupts. */ if (cause != 0) e82545_icr_assert(sc, cause); done1: DPRINTF("rx_run done: head %x, tail %x", sc->esc_RDH, sc->esc_RDT); pthread_mutex_unlock(&sc->esc_mtx); } static uint16_t e82545_carry(uint32_t sum) { sum = (sum & 0xFFFF) + (sum >> 16); if (sum > 0xFFFF) sum -= 0xFFFF; return (sum); } static uint16_t e82545_buf_checksum(uint8_t *buf, int len) { int i; uint32_t sum = 0; /* Checksum all the pairs of bytes first... */ for (i = 0; i < (len & ~1); i += 2) sum += *((u_int16_t *)(buf + i)); /* * If there's a single byte left over, checksum it, too. * Network byte order is big-endian, so the remaining byte is * the high byte. */ if (i < len) sum += htons(buf[i] << 8); return (e82545_carry(sum)); } static uint16_t e82545_iov_checksum(struct iovec *iov, int iovcnt, unsigned int off, unsigned int len) { unsigned int now, odd; uint32_t sum = 0, s; /* Skip completely unneeded vectors. */ while (iovcnt > 0 && iov->iov_len <= off && off > 0) { off -= iov->iov_len; iov++; iovcnt--; } /* Calculate checksum of requested range. */ odd = 0; while (len > 0 && iovcnt > 0) { now = MIN(len, iov->iov_len - off); s = e82545_buf_checksum((uint8_t *)iov->iov_base + off, now); sum += odd ? (s << 8) : s; odd ^= (now & 1); len -= now; off = 0; iov++; iovcnt--; } return (e82545_carry(sum)); } /* * Return the transmit descriptor type. */ static int e82545_txdesc_type(uint32_t lower) { int type; type = 0; if (lower & E1000_TXD_CMD_DEXT) type = lower & E1000_TXD_MASK; return (type); } static void e82545_transmit_checksum(struct iovec *iov, int iovcnt, struct ck_info *ck) { uint16_t cksum; unsigned int cklen; DPRINTF("tx cksum: iovcnt/s/off/len %d/%d/%d/%d", iovcnt, ck->ck_start, ck->ck_off, ck->ck_len); cklen = ck->ck_len ? ck->ck_len - ck->ck_start + 1U : UINT_MAX; cksum = e82545_iov_checksum(iov, iovcnt, ck->ck_start, cklen); *(uint16_t *)((uint8_t *)iov[0].iov_base + ck->ck_off) = ~cksum; } static void e82545_transmit_backend(struct e82545_softc *sc, struct iovec *iov, int iovcnt) { if (sc->esc_be == NULL) return; (void) netbe_send(sc->esc_be, iov, iovcnt); } static void e82545_transmit_done(struct e82545_softc *sc, uint16_t head, uint16_t tail, uint16_t dsize, int *tdwb) { union e1000_tx_udesc *dsc; for ( ; head != tail; head = (head + 1) % dsize) { dsc = &sc->esc_txdesc[head]; if (dsc->td.lower.data & E1000_TXD_CMD_RS) { dsc->td.upper.data |= E1000_TXD_STAT_DD; *tdwb = 1; } } } static int e82545_transmit(struct e82545_softc *sc, uint16_t head, uint16_t tail, uint16_t dsize, uint16_t *rhead, int *tdwb) { uint8_t *hdr, *hdrp; struct iovec iovb[I82545_MAX_TXSEGS + 2]; struct iovec tiov[I82545_MAX_TXSEGS + 2]; struct e1000_context_desc *cd; struct ck_info ckinfo[2]; struct iovec *iov; union e1000_tx_udesc *dsc; int desc, dtype, ntype, iovcnt, tcp, tso, paylen, seg, tiovcnt, pv; unsigned hdrlen, vlen, pktlen, len, left, mss, now, nnow, nleft, pvoff; uint32_t tcpsum, tcpseq; uint16_t ipcs, tcpcs, ipid, ohead; bool invalid; ckinfo[0].ck_valid = ckinfo[1].ck_valid = 0; iovcnt = 0; ntype = 0; tso = 0; pktlen = 0; ohead = head; invalid = false; /* iovb[0/1] may be used for writable copy of headers. */ iov = &iovb[2]; for (desc = 0; ; desc++, head = (head + 1) % dsize) { if (head == tail) { *rhead = head; return (0); } dsc = &sc->esc_txdesc[head]; dtype = e82545_txdesc_type(dsc->td.lower.data); if (desc == 0) { switch (dtype) { case E1000_TXD_TYP_C: DPRINTF("tx ctxt desc idx %d: %016jx " "%08x%08x", head, dsc->td.buffer_addr, dsc->td.upper.data, dsc->td.lower.data); /* Save context and return */ sc->esc_txctx = dsc->cd; goto done; case E1000_TXD_TYP_L: DPRINTF("tx legacy desc idx %d: %08x%08x", head, dsc->td.upper.data, dsc->td.lower.data); /* * legacy cksum start valid in first descriptor */ ntype = dtype; ckinfo[0].ck_start = dsc->td.upper.fields.css; break; case E1000_TXD_TYP_D: DPRINTF("tx data desc idx %d: %08x%08x", head, dsc->td.upper.data, dsc->td.lower.data); ntype = dtype; break; default: break; } } else { /* Descriptor type must be consistent */ assert(dtype == ntype); DPRINTF("tx next desc idx %d: %08x%08x", head, dsc->td.upper.data, dsc->td.lower.data); } len = (dtype == E1000_TXD_TYP_L) ? dsc->td.lower.flags.length : dsc->dd.lower.data & 0xFFFFF; /* Strip checksum supplied by guest. */ if ((dsc->td.lower.data & E1000_TXD_CMD_EOP) != 0 && (dsc->td.lower.data & E1000_TXD_CMD_IFCS) == 0) { if (len <= 2) { WPRINTF("final descriptor too short (%d) -- dropped", len); invalid = true; } else len -= 2; } if (len > 0 && iovcnt < I82545_MAX_TXSEGS) { iov[iovcnt].iov_base = paddr_guest2host(sc->esc_ctx, dsc->td.buffer_addr, len); iov[iovcnt].iov_len = len; iovcnt++; pktlen += len; } /* * Pull out info that is valid in the final descriptor * and exit descriptor loop. */ if (dsc->td.lower.data & E1000_TXD_CMD_EOP) { if (dtype == E1000_TXD_TYP_L) { if (dsc->td.lower.data & E1000_TXD_CMD_IC) { ckinfo[0].ck_valid = 1; ckinfo[0].ck_off = dsc->td.lower.flags.cso; ckinfo[0].ck_len = 0; } } else { cd = &sc->esc_txctx; if (dsc->dd.lower.data & E1000_TXD_CMD_TSE) tso = 1; if (dsc->dd.upper.fields.popts & E1000_TXD_POPTS_IXSM) ckinfo[0].ck_valid = 1; if (dsc->dd.upper.fields.popts & E1000_TXD_POPTS_IXSM || tso) { ckinfo[0].ck_start = cd->lower_setup.ip_fields.ipcss; ckinfo[0].ck_off = cd->lower_setup.ip_fields.ipcso; ckinfo[0].ck_len = cd->lower_setup.ip_fields.ipcse; } if (dsc->dd.upper.fields.popts & E1000_TXD_POPTS_TXSM) ckinfo[1].ck_valid = 1; if (dsc->dd.upper.fields.popts & E1000_TXD_POPTS_TXSM || tso) { ckinfo[1].ck_start = cd->upper_setup.tcp_fields.tucss; ckinfo[1].ck_off = cd->upper_setup.tcp_fields.tucso; ckinfo[1].ck_len = cd->upper_setup.tcp_fields.tucse; } } break; } } if (invalid) goto done; if (iovcnt > I82545_MAX_TXSEGS) { WPRINTF("tx too many descriptors (%d > %d) -- dropped", iovcnt, I82545_MAX_TXSEGS); goto done; } hdrlen = vlen = 0; /* Estimate writable space for VLAN header insertion. */ if ((sc->esc_CTRL & E1000_CTRL_VME) && (dsc->td.lower.data & E1000_TXD_CMD_VLE)) { hdrlen = ETHER_ADDR_LEN*2; vlen = ETHER_VLAN_ENCAP_LEN; } if (!tso) { /* Estimate required writable space for checksums. */ if (ckinfo[0].ck_valid) hdrlen = MAX(hdrlen, ckinfo[0].ck_off + 2U); if (ckinfo[1].ck_valid) hdrlen = MAX(hdrlen, ckinfo[1].ck_off + 2U); /* Round up writable space to the first vector. */ if (hdrlen != 0 && iov[0].iov_len > hdrlen && iov[0].iov_len < hdrlen + 100) hdrlen = iov[0].iov_len; } else { /* In case of TSO header length provided by software. */ hdrlen = sc->esc_txctx.tcp_seg_setup.fields.hdr_len; /* * Cap the header length at 240 based on 7.2.4.5 of * the Intel 82576EB (Rev 2.63) datasheet. */ if (hdrlen > 240) { WPRINTF("TSO hdrlen too large: %d", hdrlen); goto done; } /* * If VLAN insertion is requested, ensure the header * at least holds the amount of data copied during * VLAN insertion below. * * XXX: Realistic packets will include a full Ethernet * header before the IP header at ckinfo[0].ck_start, * but this check is sufficient to prevent * out-of-bounds access below. */ if (vlen != 0 && hdrlen < ETHER_ADDR_LEN*2) { WPRINTF("TSO hdrlen too small for vlan insertion " "(%d vs %d) -- dropped", hdrlen, ETHER_ADDR_LEN*2); goto done; } /* * Ensure that the header length covers the used fields * in the IP and TCP headers as well as the IP and TCP * checksums. The following fields are accessed below: * * Header | Field | Offset | Length * -------+-------+--------+------- * IPv4 | len | 2 | 2 * IPv4 | ID | 4 | 2 * IPv6 | len | 4 | 2 * TCP | seq # | 4 | 4 * TCP | flags | 13 | 1 * UDP | len | 4 | 4 */ if (hdrlen < ckinfo[0].ck_start + 6U || hdrlen < ckinfo[0].ck_off + 2U) { WPRINTF("TSO hdrlen too small for IP fields (%d) " "-- dropped", hdrlen); goto done; } if (sc->esc_txctx.cmd_and_length & E1000_TXD_CMD_TCP) { if (hdrlen < ckinfo[1].ck_start + 14U) { WPRINTF("TSO hdrlen too small for TCP fields " "(%d) -- dropped", hdrlen); goto done; } } else { if (hdrlen < ckinfo[1].ck_start + 8U) { WPRINTF("TSO hdrlen too small for UDP fields " "(%d) -- dropped", hdrlen); goto done; } } if (ckinfo[1].ck_valid && hdrlen < ckinfo[1].ck_off + 2U) { WPRINTF("TSO hdrlen too small for TCP/UDP fields " "(%d) -- dropped", hdrlen); goto done; } if (ckinfo[1].ck_valid && hdrlen < ckinfo[1].ck_off + 2) { WPRINTF("TSO hdrlen too small for TCP/UDP fields " "(%d) -- dropped", hdrlen); goto done; } } if (pktlen < hdrlen + vlen) { WPRINTF("packet too small for writable header"); goto done; } /* Allocate, fill and prepend writable header vector. */ if (hdrlen + vlen != 0) { hdr = __builtin_alloca(hdrlen + vlen); hdr += vlen; for (left = hdrlen, hdrp = hdr; left > 0; left -= now, hdrp += now) { now = MIN(left, iov->iov_len); memcpy(hdrp, iov->iov_base, now); #ifdef __FreeBSD__ iov->iov_base = (uint8_t *)iov->iov_base + now; #else /* * The type of iov_base changed in SUS (XPG4v2) from * caddr_t (char * - note signed) to 'void *'. On * illumos, bhyve is not currently compiled with XPG4v2 * or higher, and so we can't cast the RHS to unsigned. * error: pointer targets in assignment differ in * signedness * This also means that we need to apply some casts to * (caddr_t) below. */ iov->iov_base += now; #endif iov->iov_len -= now; if (iov->iov_len == 0) { iov++; iovcnt--; } } iov--; iovcnt++; #ifdef __FreeBSD__ iov->iov_base = hdr; #else iov->iov_base = (caddr_t)hdr; #endif iov->iov_len = hdrlen; } else hdr = NULL; /* Insert VLAN tag. */ if (vlen != 0) { hdr -= ETHER_VLAN_ENCAP_LEN; memmove(hdr, hdr + ETHER_VLAN_ENCAP_LEN, ETHER_ADDR_LEN*2); hdrlen += ETHER_VLAN_ENCAP_LEN; hdr[ETHER_ADDR_LEN*2 + 0] = sc->esc_VET >> 8; hdr[ETHER_ADDR_LEN*2 + 1] = sc->esc_VET & 0xff; hdr[ETHER_ADDR_LEN*2 + 2] = dsc->td.upper.fields.special >> 8; hdr[ETHER_ADDR_LEN*2 + 3] = dsc->td.upper.fields.special & 0xff; #ifdef __FreeBSD__ iov->iov_base = hdr; #else iov->iov_base = (caddr_t)hdr; #endif iov->iov_len += ETHER_VLAN_ENCAP_LEN; /* Correct checksum offsets after VLAN tag insertion. */ ckinfo[0].ck_start += ETHER_VLAN_ENCAP_LEN; ckinfo[0].ck_off += ETHER_VLAN_ENCAP_LEN; if (ckinfo[0].ck_len != 0) ckinfo[0].ck_len += ETHER_VLAN_ENCAP_LEN; ckinfo[1].ck_start += ETHER_VLAN_ENCAP_LEN; ckinfo[1].ck_off += ETHER_VLAN_ENCAP_LEN; if (ckinfo[1].ck_len != 0) ckinfo[1].ck_len += ETHER_VLAN_ENCAP_LEN; } /* Simple non-TSO case. */ if (!tso) { /* Calculate checksums and transmit. */ if (ckinfo[0].ck_valid) e82545_transmit_checksum(iov, iovcnt, &ckinfo[0]); if (ckinfo[1].ck_valid) e82545_transmit_checksum(iov, iovcnt, &ckinfo[1]); e82545_transmit_backend(sc, iov, iovcnt); goto done; } /* Doing TSO. */ tcp = (sc->esc_txctx.cmd_and_length & E1000_TXD_CMD_TCP) != 0; mss = sc->esc_txctx.tcp_seg_setup.fields.mss; paylen = (sc->esc_txctx.cmd_and_length & 0x000fffff); DPRINTF("tx %s segmentation offload %d+%d/%u bytes %d iovs", tcp ? "TCP" : "UDP", hdrlen, paylen, mss, iovcnt); ipid = ntohs(*(uint16_t *)&hdr[ckinfo[0].ck_start + 4]); tcpseq = 0; if (tcp) tcpseq = ntohl(*(uint32_t *)&hdr[ckinfo[1].ck_start + 4]); ipcs = *(uint16_t *)&hdr[ckinfo[0].ck_off]; tcpcs = 0; if (ckinfo[1].ck_valid) /* Save partial pseudo-header checksum. */ tcpcs = *(uint16_t *)&hdr[ckinfo[1].ck_off]; pv = 1; pvoff = 0; for (seg = 0, left = paylen; left > 0; seg++, left -= now) { now = MIN(left, mss); /* Construct IOVs for the segment. */ /* Include whole original header. */ #ifdef __FreeBSD__ tiov[0].iov_base = hdr; #else tiov[0].iov_base = (caddr_t)hdr; #endif tiov[0].iov_len = hdrlen; tiovcnt = 1; /* Include respective part of payload IOV. */ for (nleft = now; pv < iovcnt && nleft > 0; nleft -= nnow) { nnow = MIN(nleft, iov[pv].iov_len - pvoff); #ifdef __FreeBSD__ tiov[tiovcnt].iov_base = (uint8_t *)iov[pv].iov_base + pvoff; #else tiov[tiovcnt].iov_base = iov[pv].iov_base + pvoff; #endif tiov[tiovcnt++].iov_len = nnow; if (pvoff + nnow == iov[pv].iov_len) { pv++; pvoff = 0; } else pvoff += nnow; } DPRINTF("tx segment %d %d+%d bytes %d iovs", seg, hdrlen, now, tiovcnt); /* Update IP header. */ if (sc->esc_txctx.cmd_and_length & E1000_TXD_CMD_IP) { /* IPv4 -- set length and ID */ *(uint16_t *)&hdr[ckinfo[0].ck_start + 2] = htons(hdrlen - ckinfo[0].ck_start + now); *(uint16_t *)&hdr[ckinfo[0].ck_start + 4] = htons(ipid + seg); } else { /* IPv6 -- set length */ *(uint16_t *)&hdr[ckinfo[0].ck_start + 4] = htons(hdrlen - ckinfo[0].ck_start - 40 + now); } /* Update pseudo-header checksum. */ tcpsum = tcpcs; tcpsum += htons(hdrlen - ckinfo[1].ck_start + now); /* Update TCP/UDP headers. */ if (tcp) { /* Update sequence number and FIN/PUSH flags. */ *(uint32_t *)&hdr[ckinfo[1].ck_start + 4] = htonl(tcpseq + paylen - left); if (now < left) { hdr[ckinfo[1].ck_start + 13] &= ~(TH_FIN | TH_PUSH); } } else { /* Update payload length. */ *(uint32_t *)&hdr[ckinfo[1].ck_start + 4] = hdrlen - ckinfo[1].ck_start + now; } /* Calculate checksums and transmit. */ if (ckinfo[0].ck_valid) { *(uint16_t *)&hdr[ckinfo[0].ck_off] = ipcs; e82545_transmit_checksum(tiov, tiovcnt, &ckinfo[0]); } if (ckinfo[1].ck_valid) { *(uint16_t *)&hdr[ckinfo[1].ck_off] = e82545_carry(tcpsum); e82545_transmit_checksum(tiov, tiovcnt, &ckinfo[1]); } e82545_transmit_backend(sc, tiov, tiovcnt); } done: head = (head + 1) % dsize; e82545_transmit_done(sc, ohead, head, dsize, tdwb); *rhead = head; return (desc + 1); } static void e82545_tx_run(struct e82545_softc *sc) { uint32_t cause; uint16_t head, rhead, tail, size; int lim, tdwb, sent; size = sc->esc_TDLEN / 16; if (size == 0) return; head = sc->esc_TDH % size; tail = sc->esc_TDT % size; DPRINTF("tx_run: head %x, rhead %x, tail %x", sc->esc_TDH, sc->esc_TDHr, sc->esc_TDT); pthread_mutex_unlock(&sc->esc_mtx); rhead = head; tdwb = 0; for (lim = size / 4; sc->esc_tx_enabled && lim > 0; lim -= sent) { sent = e82545_transmit(sc, head, tail, size, &rhead, &tdwb); if (sent == 0) break; head = rhead; } pthread_mutex_lock(&sc->esc_mtx); sc->esc_TDH = head; sc->esc_TDHr = rhead; cause = 0; if (tdwb) cause |= E1000_ICR_TXDW; if (lim != size / 4 && sc->esc_TDH == sc->esc_TDT) cause |= E1000_ICR_TXQE; if (cause) e82545_icr_assert(sc, cause); DPRINTF("tx_run done: head %x, rhead %x, tail %x", sc->esc_TDH, sc->esc_TDHr, sc->esc_TDT); } static _Noreturn void * e82545_tx_thread(void *param) { struct e82545_softc *sc = param; pthread_mutex_lock(&sc->esc_mtx); for (;;) { while (!sc->esc_tx_enabled || sc->esc_TDHr == sc->esc_TDT) { if (sc->esc_tx_enabled && sc->esc_TDHr != sc->esc_TDT) break; sc->esc_tx_active = 0; if (sc->esc_tx_enabled == 0) pthread_cond_signal(&sc->esc_tx_cond); pthread_cond_wait(&sc->esc_tx_cond, &sc->esc_mtx); } sc->esc_tx_active = 1; /* Process some tx descriptors. Lock dropped inside. */ e82545_tx_run(sc); } } static void e82545_tx_start(struct e82545_softc *sc) { if (sc->esc_tx_active == 0) pthread_cond_signal(&sc->esc_tx_cond); } static void e82545_tx_enable(struct e82545_softc *sc) { sc->esc_tx_enabled = 1; } static void e82545_tx_disable(struct e82545_softc *sc) { sc->esc_tx_enabled = 0; while (sc->esc_tx_active) pthread_cond_wait(&sc->esc_tx_cond, &sc->esc_mtx); } static void e82545_rx_enable(struct e82545_softc *sc) { sc->esc_rx_enabled = 1; } static void e82545_rx_disable(struct e82545_softc *sc) { sc->esc_rx_enabled = 0; while (sc->esc_rx_active) pthread_cond_wait(&sc->esc_rx_cond, &sc->esc_mtx); } static void e82545_write_ra(struct e82545_softc *sc, int reg, uint32_t wval) { struct eth_uni *eu; int idx; idx = reg >> 1; assert(idx < 15); eu = &sc->esc_uni[idx]; if (reg & 0x1) { /* RAH */ eu->eu_valid = ((wval & E1000_RAH_AV) == E1000_RAH_AV); eu->eu_addrsel = (wval >> 16) & 0x3; eu->eu_eth.octet[5] = wval >> 8; eu->eu_eth.octet[4] = wval; } else { /* RAL */ eu->eu_eth.octet[3] = wval >> 24; eu->eu_eth.octet[2] = wval >> 16; eu->eu_eth.octet[1] = wval >> 8; eu->eu_eth.octet[0] = wval; } } static uint32_t e82545_read_ra(struct e82545_softc *sc, int reg) { struct eth_uni *eu; uint32_t retval; int idx; idx = reg >> 1; assert(idx < 15); eu = &sc->esc_uni[idx]; if (reg & 0x1) { /* RAH */ retval = (eu->eu_valid << 31) | (eu->eu_addrsel << 16) | (eu->eu_eth.octet[5] << 8) | eu->eu_eth.octet[4]; } else { /* RAL */ retval = (eu->eu_eth.octet[3] << 24) | (eu->eu_eth.octet[2] << 16) | (eu->eu_eth.octet[1] << 8) | eu->eu_eth.octet[0]; } return (retval); } static void e82545_write_register(struct e82545_softc *sc, uint32_t offset, uint32_t value) { int ridx; if (offset & 0x3) { DPRINTF("Unaligned register write offset:0x%x value:0x%x", offset, value); return; } DPRINTF("Register write: 0x%x value: 0x%x", offset, value); switch (offset) { case E1000_CTRL: case E1000_CTRL_DUP: e82545_devctl(sc, value); break; case E1000_FCAL: sc->esc_FCAL = value; break; case E1000_FCAH: sc->esc_FCAH = value & ~0xFFFF0000; break; case E1000_FCT: sc->esc_FCT = value & ~0xFFFF0000; break; case E1000_VET: sc->esc_VET = value & ~0xFFFF0000; break; case E1000_FCTTV: sc->esc_FCTTV = value & ~0xFFFF0000; break; case E1000_LEDCTL: sc->esc_LEDCTL = value & ~0x30303000; break; case E1000_PBA: sc->esc_PBA = value & 0x0000FF80; break; case E1000_ICR: case E1000_ITR: case E1000_ICS: case E1000_IMS: case E1000_IMC: e82545_intr_write(sc, offset, value); break; case E1000_RCTL: e82545_rx_ctl(sc, value); break; case E1000_FCRTL: sc->esc_FCRTL = value & ~0xFFFF0007; break; case E1000_FCRTH: sc->esc_FCRTH = value & ~0xFFFF0007; break; case E1000_RDBAL(0): sc->esc_RDBAL = value & ~0xF; if (sc->esc_rx_enabled) { /* Apparently legal: update cached address */ e82545_rx_update_rdba(sc); } break; case E1000_RDBAH(0): assert(!sc->esc_rx_enabled); sc->esc_RDBAH = value; break; case E1000_RDLEN(0): assert(!sc->esc_rx_enabled); sc->esc_RDLEN = value & ~0xFFF0007F; break; case E1000_RDH(0): /* XXX should only ever be zero ? Range check ? */ sc->esc_RDH = value; break; case E1000_RDT(0): /* XXX if this opens up the rx ring, do something ? */ sc->esc_RDT = value; break; case E1000_RDTR: /* ignore FPD bit 31 */ sc->esc_RDTR = value & ~0xFFFF0000; break; case E1000_RXDCTL(0): sc->esc_RXDCTL = value & ~0xFEC0C0C0; break; case E1000_RADV: sc->esc_RADV = value & ~0xFFFF0000; break; case E1000_RSRPD: sc->esc_RSRPD = value & ~0xFFFFF000; break; case E1000_RXCSUM: sc->esc_RXCSUM = value & ~0xFFFFF800; break; case E1000_TXCW: sc->esc_TXCW = value & ~0x3FFF0000; break; case E1000_TCTL: e82545_tx_ctl(sc, value); break; case E1000_TIPG: sc->esc_TIPG = value; break; case E1000_AIT: sc->esc_AIT = value; break; case E1000_TDBAL(0): sc->esc_TDBAL = value & ~0xF; if (sc->esc_tx_enabled) e82545_tx_update_tdba(sc); break; case E1000_TDBAH(0): sc->esc_TDBAH = value; if (sc->esc_tx_enabled) e82545_tx_update_tdba(sc); break; case E1000_TDLEN(0): sc->esc_TDLEN = value & ~0xFFF0007F; if (sc->esc_tx_enabled) e82545_tx_update_tdba(sc); break; case E1000_TDH(0): if (sc->esc_tx_enabled) { WPRINTF("ignoring write to TDH while transmit enabled"); break; } if (value != 0) { WPRINTF("ignoring non-zero value written to TDH"); break; } sc->esc_TDHr = sc->esc_TDH = value; break; case E1000_TDT(0): sc->esc_TDT = value; if (sc->esc_tx_enabled) e82545_tx_start(sc); break; case E1000_TIDV: sc->esc_TIDV = value & ~0xFFFF0000; break; case E1000_TXDCTL(0): //assert(!sc->esc_tx_enabled); sc->esc_TXDCTL = value & ~0xC0C0C0; break; case E1000_TADV: sc->esc_TADV = value & ~0xFFFF0000; break; case E1000_RAL(0) ... E1000_RAH(15): /* convert to u32 offset */ ridx = (offset - E1000_RAL(0)) >> 2; e82545_write_ra(sc, ridx, value); break; case E1000_MTA ... (E1000_MTA + (127*4)): sc->esc_fmcast[(offset - E1000_MTA) >> 2] = value; break; case E1000_VFTA ... (E1000_VFTA + (127*4)): sc->esc_fvlan[(offset - E1000_VFTA) >> 2] = value; break; case E1000_EECD: { //DPRINTF("EECD write 0x%x -> 0x%x", sc->eeprom_control, value); /* edge triggered low->high */ uint32_t eecd_strobe = ((sc->eeprom_control & E1000_EECD_SK) ? 0 : (value & E1000_EECD_SK)); uint32_t eecd_mask = (E1000_EECD_SK|E1000_EECD_CS| E1000_EECD_DI|E1000_EECD_REQ); sc->eeprom_control &= ~eecd_mask; sc->eeprom_control |= (value & eecd_mask); /* grant/revoke immediately */ if (value & E1000_EECD_REQ) { sc->eeprom_control |= E1000_EECD_GNT; } else { sc->eeprom_control &= ~E1000_EECD_GNT; } if (eecd_strobe && (sc->eeprom_control & E1000_EECD_CS)) { e82545_eecd_strobe(sc); } return; } case E1000_MDIC: { uint8_t reg_addr = (uint8_t)((value & E1000_MDIC_REG_MASK) >> E1000_MDIC_REG_SHIFT); uint8_t phy_addr = (uint8_t)((value & E1000_MDIC_PHY_MASK) >> E1000_MDIC_PHY_SHIFT); sc->mdi_control = (value & ~(E1000_MDIC_ERROR|E1000_MDIC_DEST)); if ((value & E1000_MDIC_READY) != 0) { DPRINTF("Incorrect MDIC ready bit: 0x%x", value); return; } switch (value & E82545_MDIC_OP_MASK) { case E1000_MDIC_OP_READ: sc->mdi_control &= ~E82545_MDIC_DATA_MASK; sc->mdi_control |= e82545_read_mdi(sc, reg_addr, phy_addr); break; case E1000_MDIC_OP_WRITE: e82545_write_mdi(sc, reg_addr, phy_addr, value & E82545_MDIC_DATA_MASK); break; default: DPRINTF("Unknown MDIC op: 0x%x", value); return; } /* TODO: barrier? */ sc->mdi_control |= E1000_MDIC_READY; if (value & E82545_MDIC_IE) { // TODO: generate interrupt } return; } case E1000_MANC: case E1000_STATUS: return; default: DPRINTF("Unknown write register: 0x%x value:%x", offset, value); return; } } static uint32_t e82545_read_register(struct e82545_softc *sc, uint32_t offset) { uint32_t retval; int ridx; if (offset & 0x3) { DPRINTF("Unaligned register read offset:0x%x", offset); return 0; } DPRINTF("Register read: 0x%x", offset); switch (offset) { case E1000_CTRL: retval = sc->esc_CTRL; break; case E1000_STATUS: retval = E1000_STATUS_FD | E1000_STATUS_LU | E1000_STATUS_SPEED_1000; break; case E1000_FCAL: retval = sc->esc_FCAL; break; case E1000_FCAH: retval = sc->esc_FCAH; break; case E1000_FCT: retval = sc->esc_FCT; break; case E1000_VET: retval = sc->esc_VET; break; case E1000_FCTTV: retval = sc->esc_FCTTV; break; case E1000_LEDCTL: retval = sc->esc_LEDCTL; break; case E1000_PBA: retval = sc->esc_PBA; break; case E1000_ICR: case E1000_ITR: case E1000_ICS: case E1000_IMS: case E1000_IMC: retval = e82545_intr_read(sc, offset); break; case E1000_RCTL: retval = sc->esc_RCTL; break; case E1000_FCRTL: retval = sc->esc_FCRTL; break; case E1000_FCRTH: retval = sc->esc_FCRTH; break; case E1000_RDBAL(0): retval = sc->esc_RDBAL; break; case E1000_RDBAH(0): retval = sc->esc_RDBAH; break; case E1000_RDLEN(0): retval = sc->esc_RDLEN; break; case E1000_RDH(0): retval = sc->esc_RDH; break; case E1000_RDT(0): retval = sc->esc_RDT; break; case E1000_RDTR: retval = sc->esc_RDTR; break; case E1000_RXDCTL(0): retval = sc->esc_RXDCTL; break; case E1000_RADV: retval = sc->esc_RADV; break; case E1000_RSRPD: retval = sc->esc_RSRPD; break; case E1000_RXCSUM: retval = sc->esc_RXCSUM; break; case E1000_TXCW: retval = sc->esc_TXCW; break; case E1000_TCTL: retval = sc->esc_TCTL; break; case E1000_TIPG: retval = sc->esc_TIPG; break; case E1000_AIT: retval = sc->esc_AIT; break; case E1000_TDBAL(0): retval = sc->esc_TDBAL; break; case E1000_TDBAH(0): retval = sc->esc_TDBAH; break; case E1000_TDLEN(0): retval = sc->esc_TDLEN; break; case E1000_TDH(0): retval = sc->esc_TDH; break; case E1000_TDT(0): retval = sc->esc_TDT; break; case E1000_TIDV: retval = sc->esc_TIDV; break; case E1000_TXDCTL(0): retval = sc->esc_TXDCTL; break; case E1000_TADV: retval = sc->esc_TADV; break; case E1000_RAL(0) ... E1000_RAH(15): /* convert to u32 offset */ ridx = (offset - E1000_RAL(0)) >> 2; retval = e82545_read_ra(sc, ridx); break; case E1000_MTA ... (E1000_MTA + (127*4)): retval = sc->esc_fmcast[(offset - E1000_MTA) >> 2]; break; case E1000_VFTA ... (E1000_VFTA + (127*4)): retval = sc->esc_fvlan[(offset - E1000_VFTA) >> 2]; break; case E1000_EECD: //DPRINTF("EECD read %x", sc->eeprom_control); retval = sc->eeprom_control; break; case E1000_MDIC: retval = sc->mdi_control; break; case E1000_MANC: retval = 0; break; /* stats that we emulate. */ case E1000_MPC: retval = sc->missed_pkt_count; break; case E1000_PRC64: retval = sc->pkt_rx_by_size[0]; break; case E1000_PRC127: retval = sc->pkt_rx_by_size[1]; break; case E1000_PRC255: retval = sc->pkt_rx_by_size[2]; break; case E1000_PRC511: retval = sc->pkt_rx_by_size[3]; break; case E1000_PRC1023: retval = sc->pkt_rx_by_size[4]; break; case E1000_PRC1522: retval = sc->pkt_rx_by_size[5]; break; case E1000_GPRC: retval = sc->good_pkt_rx_count; break; case E1000_BPRC: retval = sc->bcast_pkt_rx_count; break; case E1000_MPRC: retval = sc->mcast_pkt_rx_count; break; case E1000_GPTC: case E1000_TPT: retval = sc->good_pkt_tx_count; break; case E1000_GORCL: retval = (uint32_t)sc->good_octets_rx; break; case E1000_GORCH: retval = (uint32_t)(sc->good_octets_rx >> 32); break; case E1000_TOTL: case E1000_GOTCL: retval = (uint32_t)sc->good_octets_tx; break; case E1000_TOTH: case E1000_GOTCH: retval = (uint32_t)(sc->good_octets_tx >> 32); break; case E1000_ROC: retval = sc->oversize_rx_count; break; case E1000_TORL: retval = (uint32_t)(sc->good_octets_rx + sc->missed_octets); break; case E1000_TORH: retval = (uint32_t)((sc->good_octets_rx + sc->missed_octets) >> 32); break; case E1000_TPR: retval = sc->good_pkt_rx_count + sc->missed_pkt_count + sc->oversize_rx_count; break; case E1000_PTC64: retval = sc->pkt_tx_by_size[0]; break; case E1000_PTC127: retval = sc->pkt_tx_by_size[1]; break; case E1000_PTC255: retval = sc->pkt_tx_by_size[2]; break; case E1000_PTC511: retval = sc->pkt_tx_by_size[3]; break; case E1000_PTC1023: retval = sc->pkt_tx_by_size[4]; break; case E1000_PTC1522: retval = sc->pkt_tx_by_size[5]; break; case E1000_MPTC: retval = sc->mcast_pkt_tx_count; break; case E1000_BPTC: retval = sc->bcast_pkt_tx_count; break; case E1000_TSCTC: retval = sc->tso_tx_count; break; /* stats that are always 0. */ case E1000_CRCERRS: case E1000_ALGNERRC: case E1000_SYMERRS: case E1000_RXERRC: case E1000_SCC: case E1000_ECOL: case E1000_MCC: case E1000_LATECOL: case E1000_COLC: case E1000_DC: case E1000_TNCRS: case E1000_SEC: case E1000_CEXTERR: case E1000_RLEC: case E1000_XONRXC: case E1000_XONTXC: case E1000_XOFFRXC: case E1000_XOFFTXC: case E1000_FCRUC: case E1000_RNBC: case E1000_RUC: case E1000_RFC: case E1000_RJC: case E1000_MGTPRC: case E1000_MGTPDC: case E1000_MGTPTC: case E1000_TSCTFC: retval = 0; break; default: DPRINTF("Unknown read register: 0x%x", offset); retval = 0; break; } return (retval); } static void e82545_write(struct pci_devinst *pi, int baridx, uint64_t offset, int size, uint64_t value) { struct e82545_softc *sc; //DPRINTF("Write bar:%d offset:0x%lx value:0x%lx size:%d", baridx, offset, value, size); sc = pi->pi_arg; pthread_mutex_lock(&sc->esc_mtx); switch (baridx) { case E82545_BAR_IO: switch (offset) { case E82545_IOADDR: if (size != 4) { DPRINTF("Wrong io addr write sz:%d value:0x%lx", size, value); } else sc->io_addr = (uint32_t)value; break; case E82545_IODATA: if (size != 4) { DPRINTF("Wrong io data write size:%d value:0x%lx", size, value); } else if (sc->io_addr > E82545_IO_REGISTER_MAX) { DPRINTF("Non-register io write addr:0x%x value:0x%lx", sc->io_addr, value); } else e82545_write_register(sc, sc->io_addr, (uint32_t)value); break; default: DPRINTF("Unknown io bar write offset:0x%lx value:0x%lx size:%d", offset, value, size); break; } break; case E82545_BAR_REGISTER: if (size != 4) { DPRINTF("Wrong register write size:%d offset:0x%lx value:0x%lx", size, offset, value); } else e82545_write_register(sc, (uint32_t)offset, (uint32_t)value); break; default: DPRINTF("Unknown write bar:%d off:0x%lx val:0x%lx size:%d", baridx, offset, value, size); } pthread_mutex_unlock(&sc->esc_mtx); } static uint64_t e82545_read(struct pci_devinst *pi, int baridx, uint64_t offset, int size) { struct e82545_softc *sc; uint64_t retval; //DPRINTF("Read bar:%d offset:0x%lx size:%d", baridx, offset, size); sc = pi->pi_arg; retval = 0; pthread_mutex_lock(&sc->esc_mtx); switch (baridx) { case E82545_BAR_IO: switch (offset) { case E82545_IOADDR: if (size != 4) { DPRINTF("Wrong io addr read sz:%d", size); } else retval = sc->io_addr; break; case E82545_IODATA: if (size != 4) { DPRINTF("Wrong io data read sz:%d", size); } if (sc->io_addr > E82545_IO_REGISTER_MAX) { DPRINTF("Non-register io read addr:0x%x", sc->io_addr); } else retval = e82545_read_register(sc, sc->io_addr); break; default: DPRINTF("Unknown io bar read offset:0x%lx size:%d", offset, size); break; } break; case E82545_BAR_REGISTER: if (size != 4) { DPRINTF("Wrong register read size:%d offset:0x%lx", size, offset); } else retval = e82545_read_register(sc, (uint32_t)offset); break; default: DPRINTF("Unknown read bar:%d offset:0x%lx size:%d", baridx, offset, size); break; } pthread_mutex_unlock(&sc->esc_mtx); return (retval); } static void e82545_reset(struct e82545_softc *sc, int drvr) { int i; e82545_rx_disable(sc); e82545_tx_disable(sc); /* clear outstanding interrupts */ if (sc->esc_irq_asserted) pci_lintr_deassert(sc->esc_pi); /* misc */ if (!drvr) { sc->esc_FCAL = 0; sc->esc_FCAH = 0; sc->esc_FCT = 0; sc->esc_VET = 0; sc->esc_FCTTV = 0; } sc->esc_LEDCTL = 0x07061302; sc->esc_PBA = 0x00100030; /* start nvm in opcode mode. */ sc->nvm_opaddr = 0; sc->nvm_mode = E82545_NVM_MODE_OPADDR; sc->nvm_bits = E82545_NVM_OPADDR_BITS; sc->eeprom_control = E1000_EECD_PRES | E82545_EECD_FWE_EN; e82545_init_eeprom(sc); /* interrupt */ sc->esc_ICR = 0; sc->esc_ITR = 250; sc->esc_ICS = 0; sc->esc_IMS = 0; sc->esc_IMC = 0; /* L2 filters */ if (!drvr) { memset(sc->esc_fvlan, 0, sizeof(sc->esc_fvlan)); memset(sc->esc_fmcast, 0, sizeof(sc->esc_fmcast)); memset(sc->esc_uni, 0, sizeof(sc->esc_uni)); /* XXX not necessary on 82545 ?? */ sc->esc_uni[0].eu_valid = 1; memcpy(sc->esc_uni[0].eu_eth.octet, sc->esc_mac.octet, ETHER_ADDR_LEN); } else { /* Clear RAH valid bits */ for (i = 0; i < 16; i++) sc->esc_uni[i].eu_valid = 0; } /* receive */ if (!drvr) { sc->esc_RDBAL = 0; sc->esc_RDBAH = 0; } sc->esc_RCTL = 0; sc->esc_FCRTL = 0; sc->esc_FCRTH = 0; sc->esc_RDLEN = 0; sc->esc_RDH = 0; sc->esc_RDT = 0; sc->esc_RDTR = 0; sc->esc_RXDCTL = (1 << 24) | (1 << 16); /* default GRAN/WTHRESH */ sc->esc_RADV = 0; sc->esc_RXCSUM = 0; /* transmit */ if (!drvr) { sc->esc_TDBAL = 0; sc->esc_TDBAH = 0; sc->esc_TIPG = 0; sc->esc_AIT = 0; sc->esc_TIDV = 0; sc->esc_TADV = 0; } sc->esc_tdba = 0; sc->esc_txdesc = NULL; sc->esc_TXCW = 0; sc->esc_TCTL = 0; sc->esc_TDLEN = 0; sc->esc_TDT = 0; sc->esc_TDHr = sc->esc_TDH = 0; sc->esc_TXDCTL = 0; } static int e82545_init(struct pci_devinst *pi, nvlist_t *nvl) { char nstr[80]; struct e82545_softc *sc; const char *mac; int err; /* Setup our softc */ sc = calloc(1, sizeof(*sc)); pi->pi_arg = sc; sc->esc_pi = pi; sc->esc_ctx = pi->pi_vmctx; pthread_mutex_init(&sc->esc_mtx, NULL); pthread_cond_init(&sc->esc_rx_cond, NULL); pthread_cond_init(&sc->esc_tx_cond, NULL); pthread_create(&sc->esc_tx_tid, NULL, e82545_tx_thread, sc); snprintf(nstr, sizeof(nstr), "e82545-%d:%d tx", pi->pi_slot, pi->pi_func); pthread_set_name_np(sc->esc_tx_tid, nstr); pci_set_cfgdata16(pi, PCIR_DEVICE, E82545_DEV_ID_82545EM_COPPER); pci_set_cfgdata16(pi, PCIR_VENDOR, E82545_VENDOR_ID_INTEL); pci_set_cfgdata8(pi, PCIR_CLASS, PCIC_NETWORK); pci_set_cfgdata8(pi, PCIR_SUBCLASS, PCIS_NETWORK_ETHERNET); pci_set_cfgdata16(pi, PCIR_SUBDEV_0, E82545_SUBDEV_ID); pci_set_cfgdata16(pi, PCIR_SUBVEND_0, E82545_VENDOR_ID_INTEL); pci_set_cfgdata8(pi, PCIR_HDRTYPE, PCIM_HDRTYPE_NORMAL); pci_set_cfgdata8(pi, PCIR_INTPIN, 0x1); /* TODO: this card also supports msi, but the freebsd driver for it * does not, so I have not implemented it. */ pci_lintr_request(pi); pci_emul_alloc_bar(pi, E82545_BAR_REGISTER, PCIBAR_MEM32, E82545_BAR_REGISTER_LEN); pci_emul_alloc_bar(pi, E82545_BAR_FLASH, PCIBAR_MEM32, E82545_BAR_FLASH_LEN); pci_emul_alloc_bar(pi, E82545_BAR_IO, PCIBAR_IO, E82545_BAR_IO_LEN); mac = get_config_value_node(nvl, "mac"); if (mac != NULL) { err = net_parsemac(mac, sc->esc_mac.octet); if (err) { free(sc); return (err); } } else net_genmac(pi, sc->esc_mac.octet); err = netbe_init(&sc->esc_be, nvl, e82545_rx_callback, sc); if (err) { free(sc); return (err); } #ifndef __FreeBSD__ size_t buflen = sizeof (sc->esc_mac.octet); err = netbe_get_mac(sc->esc_be, sc->esc_mac.octet, &buflen); if (err != 0) { free(sc); return (err); } #endif netbe_rx_enable(sc->esc_be); /* H/w initiated reset */ e82545_reset(sc, 0); return (0); } static const struct pci_devemu pci_de_e82545 = { .pe_emu = "e1000", .pe_init = e82545_init, .pe_legacy_config = netbe_legacy_config, .pe_barwrite = e82545_write, .pe_barread = e82545_read, }; PCI_EMUL_SET(pci_de_e82545); /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2011 NetApp, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * This file and its contents are supplied under the terms of the * Common Development and Distribution License ("CDDL"), version 1.0. * You may only use this file in accordance with the terms of version * 1.0 of the CDDL. * * A full copy of the text of the CDDL should have accompanied this * source. A copy of the CDDL is also available via the Internet at * http://www.illumos.org/license/CDDL. * * Copyright 2014 Pluribus Networks Inc. * Copyright 2018 Joyent, Inc. * Copyright 2025 Oxide Computer Company */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "acpi.h" #include "bhyverun.h" #include "bootrom.h" #include "config.h" #include "debug.h" #include "inout.h" #include "ioapic.h" #include "mem.h" #include "pci_emul.h" #include "pci_irq.h" #include "pci_lpc.h" #include "pci_passthru.h" #include "qemu_fwcfg.h" #define CONF1_ADDR_PORT 0x0cf8 #define CONF1_DATA_PORT 0x0cfc #define CONF1_ENABLE 0x80000000ul #define MAXBUSES (PCI_BUSMAX + 1) #define MAXSLOTS (PCI_SLOTMAX + 1) #define MAXFUNCS (PCI_FUNCMAX + 1) #define GB (1024 * 1024 * 1024UL) struct funcinfo { nvlist_t *fi_config; struct pci_devemu *fi_pde; struct pci_devinst *fi_devi; }; struct intxinfo { int ii_count; int ii_pirq_pin; int ii_ioapic_irq; }; struct slotinfo { struct intxinfo si_intpins[4]; struct funcinfo si_funcs[MAXFUNCS]; }; struct businfo { uint16_t iobase, iolimit; /* I/O window */ uint32_t membase32, memlimit32; /* mmio window below 4GB */ uint64_t membase64, memlimit64; /* mmio window above 4GB */ struct slotinfo slotinfo[MAXSLOTS]; }; static struct businfo *pci_businfo[MAXBUSES]; SET_DECLARE(pci_devemu_set, struct pci_devemu); static uint64_t pci_emul_iobase; static uint8_t *pci_emul_rombase; static uint64_t pci_emul_romoffset; static uint8_t *pci_emul_romlim; static uint64_t pci_emul_membase32; static uint64_t pci_emul_membase64; static uint64_t pci_emul_memlim64; struct pci_bar_allocation { TAILQ_ENTRY(pci_bar_allocation) chain; struct pci_devinst *pdi; int idx; enum pcibar_type type; uint64_t size; }; static TAILQ_HEAD(pci_bar_list, pci_bar_allocation) pci_bars = TAILQ_HEAD_INITIALIZER(pci_bars); struct boot_device { TAILQ_ENTRY(boot_device) boot_device_chain; struct pci_devinst *pdi; int bootindex; }; static TAILQ_HEAD(boot_list, boot_device) boot_devices = TAILQ_HEAD_INITIALIZER( boot_devices); #define PCI_EMUL_IOBASE 0x2000 #define PCI_EMUL_IOLIMIT 0x10000 #define PCI_EMUL_ROMSIZE 0x10000000 #define PCI_EMUL_ECFG_BASE 0xE0000000 /* 3.5GB */ #define PCI_EMUL_ECFG_SIZE (MAXBUSES * 1024 * 1024) /* 1MB per bus */ SYSRES_MEM(PCI_EMUL_ECFG_BASE, PCI_EMUL_ECFG_SIZE); /* * OVMF always uses 0xC0000000 as base address for 32 bit PCI MMIO. Don't * change this address without changing it in OVMF. */ #define PCI_EMUL_MEMBASE32 0xC0000000 #define PCI_EMUL_MEMLIMIT32 PCI_EMUL_ECFG_BASE #define PCI_EMUL_MEMSIZE64 (32*GB) static struct pci_devemu *pci_emul_finddev(const char *name); static void pci_lintr_route(struct pci_devinst *pi); static void pci_lintr_update(struct pci_devinst *pi); static void pci_cfgrw(int in, int bus, int slot, int func, int coff, int bytes, uint32_t *val); static __inline void CFGWRITE(struct pci_devinst *pi, int coff, uint32_t val, int bytes) { if (bytes == 1) pci_set_cfgdata8(pi, coff, val); else if (bytes == 2) pci_set_cfgdata16(pi, coff, val); else pci_set_cfgdata32(pi, coff, val); } static __inline uint32_t CFGREAD(struct pci_devinst *pi, int coff, int bytes) { if (bytes == 1) return (pci_get_cfgdata8(pi, coff)); else if (bytes == 2) return (pci_get_cfgdata16(pi, coff)); else return (pci_get_cfgdata32(pi, coff)); } static int is_pcir_bar(int coff) { return (coff >= PCIR_BAR(0) && coff < PCIR_BAR(PCI_BARMAX + 1)); } static int is_pcir_bios(int coff) { return (coff >= PCIR_BIOS && coff < PCIR_BIOS + 4); } /* * I/O access */ /* * Slot options are in the form: * * ::,[,] * [:],[,] * * slot is 0..31 * func is 0..7 * emul is a string describing the type of PCI device e.g. virtio-net * config is an optional string, depending on the device, that can be * used for configuration. * Examples are: * 1,virtio-net,tap0 * 3:0,dummy */ static void pci_parse_slot_usage(char *aopt) { EPRINTLN("Invalid PCI slot info field \"%s\"", aopt); } /* * Helper function to parse a list of comma-separated options where * each option is formatted as "name[=value]". If no value is * provided, the option is treated as a boolean and is given a value * of true. */ int pci_parse_legacy_config(nvlist_t *nvl, const char *opt) { char *config, *name, *tofree, *value; if (opt == NULL) return (0); config = tofree = strdup(opt); while ((name = strsep(&config, ",")) != NULL) { value = strchr(name, '='); if (value != NULL) { *value = '\0'; value++; set_config_value_node(nvl, name, value); } else set_config_bool_node(nvl, name, true); } free(tofree); return (0); } /* * PCI device configuration is stored in MIBs that encode the device's * location: * * pci... * * Where "bus", "slot", and "func" are all decimal values without * leading zeroes. Each valid device must have a "device" node which * identifies the driver model of the device. * * Device backends can provide a parser for the "config" string. If * a custom parser is not provided, pci_parse_legacy_config() is used * to parse the string. */ int pci_parse_slot(char *opt) { char node_name[sizeof("pci.XXX.XX.X")]; struct pci_devemu *pde; char *emul, *config, *str, *cp; int error, bnum, snum, fnum; nvlist_t *nvl; error = -1; str = strdup(opt); emul = config = NULL; if ((cp = strchr(str, ',')) != NULL) { *cp = '\0'; emul = cp + 1; if ((cp = strchr(emul, ',')) != NULL) { *cp = '\0'; config = cp + 1; } } else { pci_parse_slot_usage(opt); goto done; } /* :: */ if (sscanf(str, "%d:%d:%d", &bnum, &snum, &fnum) != 3) { bnum = 0; /* : */ if (sscanf(str, "%d:%d", &snum, &fnum) != 2) { fnum = 0; /* */ if (sscanf(str, "%d", &snum) != 1) { snum = -1; } } } if (bnum < 0 || bnum >= MAXBUSES || snum < 0 || snum >= MAXSLOTS || fnum < 0 || fnum >= MAXFUNCS) { pci_parse_slot_usage(opt); goto done; } pde = pci_emul_finddev(emul); if (pde == NULL) { EPRINTLN("pci slot %d:%d:%d: unknown device \"%s\"", bnum, snum, fnum, emul); goto done; } snprintf(node_name, sizeof(node_name), "pci.%d.%d.%d", bnum, snum, fnum); nvl = find_config_node(node_name); if (nvl != NULL) { EPRINTLN("pci slot %d:%d:%d already occupied!", bnum, snum, fnum); goto done; } nvl = create_config_node(node_name); if (pde->pe_alias != NULL) set_config_value_node(nvl, "device", pde->pe_alias); else set_config_value_node(nvl, "device", pde->pe_emu); if (pde->pe_legacy_config != NULL) error = pde->pe_legacy_config(nvl, config); else error = pci_parse_legacy_config(nvl, config); done: free(str); return (error); } void pci_print_supported_devices(void) { struct pci_devemu **pdpp, *pdp; SET_FOREACH(pdpp, pci_devemu_set) { pdp = *pdpp; printf("%s\n", pdp->pe_emu); } } uint32_t pci_config_read_reg(const struct pcisel *const host_sel, nvlist_t *nvl, const uint32_t reg, const uint8_t size, const uint32_t def) { const char *config; const nvlist_t *pci_regs; assert(size == 1 || size == 2 || size == 4); pci_regs = find_relative_config_node(nvl, "pcireg"); if (pci_regs == NULL) { return def; } switch (reg) { case PCIR_DEVICE: config = get_config_value_node(pci_regs, "device"); break; case PCIR_VENDOR: config = get_config_value_node(pci_regs, "vendor"); break; case PCIR_REVID: config = get_config_value_node(pci_regs, "revid"); break; case PCIR_SUBVEND_0: config = get_config_value_node(pci_regs, "subvendor"); break; case PCIR_SUBDEV_0: config = get_config_value_node(pci_regs, "subdevice"); break; default: return (-1); } if (config == NULL) { return def; } else if (host_sel != NULL && strcmp(config, "host") == 0) { return pci_host_read_config(host_sel, reg, size); } else { return strtol(config, NULL, 16); } } static int pci_valid_pba_offset(struct pci_devinst *pi, uint64_t offset) { if (offset < pi->pi_msix.pba_offset) return (0); if (offset >= pi->pi_msix.pba_offset + pi->pi_msix.pba_size) { return (0); } return (1); } int pci_emul_msix_twrite(struct pci_devinst *pi, uint64_t offset, int size, uint64_t value) { int msix_entry_offset; int tab_index; char *dest; /* support only 4 or 8 byte writes */ if (size != 4 && size != 8) return (-1); /* * Return if table index is beyond what device supports */ tab_index = offset / MSIX_TABLE_ENTRY_SIZE; if (tab_index >= pi->pi_msix.table_count) return (-1); msix_entry_offset = offset % MSIX_TABLE_ENTRY_SIZE; /* support only aligned writes */ if ((msix_entry_offset % size) != 0) return (-1); dest = (char *)(pi->pi_msix.table + tab_index); dest += msix_entry_offset; if (size == 4) *((uint32_t *)dest) = value; else *((uint64_t *)dest) = value; return (0); } uint64_t pci_emul_msix_tread(struct pci_devinst *pi, uint64_t offset, int size) { char *dest; int msix_entry_offset; int tab_index; uint64_t retval = ~0; /* * The PCI standard only allows 4 and 8 byte accesses to the MSI-X * table but we also allow 1 byte access to accommodate reads from * ddb. */ if (size != 1 && size != 4 && size != 8) return (retval); msix_entry_offset = offset % MSIX_TABLE_ENTRY_SIZE; /* support only aligned reads */ if ((msix_entry_offset % size) != 0) { return (retval); } tab_index = offset / MSIX_TABLE_ENTRY_SIZE; if (tab_index < pi->pi_msix.table_count) { /* valid MSI-X Table access */ dest = (char *)(pi->pi_msix.table + tab_index); dest += msix_entry_offset; if (size == 1) retval = *((uint8_t *)dest); else if (size == 4) retval = *((uint32_t *)dest); else retval = *((uint64_t *)dest); } else if (pci_valid_pba_offset(pi, offset)) { /* return 0 for PBA access */ retval = 0; } return (retval); } int pci_msix_table_bar(struct pci_devinst *pi) { if (pi->pi_msix.table != NULL) return (pi->pi_msix.table_bar); else return (-1); } int pci_msix_pba_bar(struct pci_devinst *pi) { if (pi->pi_msix.table != NULL) return (pi->pi_msix.pba_bar); else return (-1); } static int pci_emul_io_handler(struct vmctx *ctx __unused, int in, int port, int bytes, uint32_t *eax, void *arg) { struct pci_devinst *pdi = arg; struct pci_devemu *pe = pdi->pi_d; uint64_t offset; int i; assert(port >= 0); for (i = 0; i <= PCI_BARMAX; i++) { if (pdi->pi_bar[i].type == PCIBAR_IO && (uint64_t)port >= pdi->pi_bar[i].addr && (uint64_t)port + bytes <= pdi->pi_bar[i].addr + pdi->pi_bar[i].size) { offset = port - pdi->pi_bar[i].addr; if (in) *eax = (*pe->pe_barread)(pdi, i, offset, bytes); else (*pe->pe_barwrite)(pdi, i, offset, bytes, *eax); return (0); } } return (-1); } static int pci_emul_mem_handler(struct vcpu *vcpu __unused, int dir, uint64_t addr, int size, uint64_t *val, void *arg1, long arg2) { struct pci_devinst *pdi = arg1; struct pci_devemu *pe = pdi->pi_d; uint64_t offset; int bidx = (int)arg2; assert(bidx <= PCI_BARMAX); assert(pdi->pi_bar[bidx].type == PCIBAR_MEM32 || pdi->pi_bar[bidx].type == PCIBAR_MEM64); assert(addr >= pdi->pi_bar[bidx].addr && addr + size <= pdi->pi_bar[bidx].addr + pdi->pi_bar[bidx].size); offset = addr - pdi->pi_bar[bidx].addr; if (dir == MEM_F_WRITE) { if (size == 8) { (*pe->pe_barwrite)(pdi, bidx, offset, 4, *val & 0xffffffff); (*pe->pe_barwrite)(pdi, bidx, offset + 4, 4, *val >> 32); } else { (*pe->pe_barwrite)(pdi, bidx, offset, size, *val); } } else { if (size == 8) { *val = (*pe->pe_barread)(pdi, bidx, offset, 4); *val |= (*pe->pe_barread)(pdi, bidx, offset + 4, 4) << 32; } else { *val = (*pe->pe_barread)(pdi, bidx, offset, size); } } return (0); } static int pci_emul_alloc_resource(uint64_t *baseptr, uint64_t limit, uint64_t size, uint64_t *addr) { uint64_t base; assert((size & (size - 1)) == 0); /* must be a power of 2 */ base = roundup2(*baseptr, size); if (base + size <= limit) { *addr = base; *baseptr = base + size; return (0); } else return (-1); } /* * Register (or unregister) the MMIO or I/O region associated with the BAR * register 'idx' of an emulated pci device. */ static void modify_bar_registration(struct pci_devinst *pi, int idx, int registration) { struct pci_devemu *pe; int error; struct inout_port iop; struct mem_range mr; pe = pi->pi_d; switch (pi->pi_bar[idx].type) { case PCIBAR_IO: bzero(&iop, sizeof(struct inout_port)); iop.name = pi->pi_name; iop.port = pi->pi_bar[idx].addr; iop.size = pi->pi_bar[idx].size; if (registration) { iop.flags = IOPORT_F_INOUT; iop.handler = pci_emul_io_handler; iop.arg = pi; error = register_inout(&iop); } else error = unregister_inout(&iop); break; case PCIBAR_MEM32: case PCIBAR_MEM64: bzero(&mr, sizeof(struct mem_range)); mr.name = pi->pi_name; mr.base = pi->pi_bar[idx].addr; mr.size = pi->pi_bar[idx].size; if (registration) { mr.flags = MEM_F_RW; mr.handler = pci_emul_mem_handler; mr.arg1 = pi; mr.arg2 = idx; error = register_mem(&mr); } else error = unregister_mem(&mr); break; case PCIBAR_ROM: error = 0; break; default: error = EINVAL; break; } assert(error == 0); if (pe->pe_baraddr != NULL) (*pe->pe_baraddr)(pi, idx, registration, pi->pi_bar[idx].addr); } static void unregister_bar(struct pci_devinst *pi, int idx) { modify_bar_registration(pi, idx, 0); } static void register_bar(struct pci_devinst *pi, int idx) { modify_bar_registration(pi, idx, 1); } /* Is the ROM enabled for the emulated pci device? */ static int romen(struct pci_devinst *pi) { return (pi->pi_bar[PCI_ROM_IDX].lobits & PCIM_BIOS_ENABLE) == PCIM_BIOS_ENABLE; } /* Are we decoding i/o port accesses for the emulated pci device? */ static int porten(struct pci_devinst *pi) { uint16_t cmd; cmd = pci_get_cfgdata16(pi, PCIR_COMMAND); return (cmd & PCIM_CMD_PORTEN); } /* Are we decoding memory accesses for the emulated pci device? */ static int memen(struct pci_devinst *pi) { uint16_t cmd; cmd = pci_get_cfgdata16(pi, PCIR_COMMAND); return (cmd & PCIM_CMD_MEMEN); } /* * Update the MMIO or I/O address that is decoded by the BAR register. * * If the pci device has enabled the address space decoding then intercept * the address range decoded by the BAR register. */ static void update_bar_address(struct pci_devinst *pi, uint64_t addr, int idx, int type) { int decode; if (pi->pi_bar[idx].type == PCIBAR_IO) decode = porten(pi); else decode = memen(pi); if (decode) unregister_bar(pi, idx); switch (type) { case PCIBAR_IO: case PCIBAR_MEM32: pi->pi_bar[idx].addr = addr; break; case PCIBAR_MEM64: pi->pi_bar[idx].addr &= ~0xffffffffUL; pi->pi_bar[idx].addr |= addr; break; case PCIBAR_MEMHI64: pi->pi_bar[idx].addr &= 0xffffffff; pi->pi_bar[idx].addr |= addr; break; default: assert(0); } if (decode) register_bar(pi, idx); } int pci_emul_alloc_bar(struct pci_devinst *pdi, int idx, enum pcibar_type type, uint64_t size) { assert((type == PCIBAR_ROM) || (idx >= 0 && idx <= PCI_BARMAX)); assert((type != PCIBAR_ROM) || (idx == PCI_ROM_IDX)); if ((size & (size - 1)) != 0) size = 1UL << flsl(size); /* round up to a power of 2 */ /* Enforce minimum BAR sizes required by the PCI standard */ if (type == PCIBAR_IO) { if (size < 4) size = 4; } else if (type == PCIBAR_ROM) { if (size < ~PCIM_BIOS_ADDR_MASK + 1) size = ~PCIM_BIOS_ADDR_MASK + 1; } else { if (size < 16) size = 16; } /* * To reduce fragmentation of the MMIO space, we allocate the BARs by * size. Therefore, don't allocate the BAR yet. We create a list of all * BAR allocation which is sorted by BAR size. When all PCI devices are * initialized, we will assign an address to the BARs. */ /* create a new list entry */ struct pci_bar_allocation *const new_bar = malloc(sizeof(*new_bar)); memset(new_bar, 0, sizeof(*new_bar)); new_bar->pdi = pdi; new_bar->idx = idx; new_bar->type = type; new_bar->size = size; /* * Search for a BAR which size is lower than the size of our newly * allocated BAR. */ struct pci_bar_allocation *bar = NULL; TAILQ_FOREACH(bar, &pci_bars, chain) { if (bar->size < size) { break; } } if (bar == NULL) { /* * Either the list is empty or new BAR is the smallest BAR of * the list. Append it to the end of our list. */ TAILQ_INSERT_TAIL(&pci_bars, new_bar, chain); } else { /* * The found BAR is smaller than our new BAR. For that reason, * insert our new BAR before the found BAR. */ TAILQ_INSERT_BEFORE(bar, new_bar, chain); } #ifdef __FreeBSD__ /* * Enable PCI BARs only if we don't have a boot ROM, i.e., bhyveload was * used to load the initial guest image. Otherwise, we rely on the boot * ROM to handle this. */ if (!get_config_bool_default("pci.enable_bars", !bootrom_boot())) return (0); #else /* * Enable PCI BARs unless specifically requested not to. Bootroms * generally used in illumos do not perform PCI BAR enumeration * themselves and so need the BARs enabling here. */ if (!get_config_bool_default("pci.enable_bars", true)) return (0); #endif /* * pci_passthru devices synchronize their physical and virtual command * register on init. For that reason, the virtual cmd reg should be * updated as early as possible. */ uint16_t enbit = 0; switch (type) { case PCIBAR_IO: enbit = PCIM_CMD_PORTEN; break; case PCIBAR_MEM64: case PCIBAR_MEM32: enbit = PCIM_CMD_MEMEN; break; default: enbit = 0; break; } const uint16_t cmd = pci_get_cfgdata16(pdi, PCIR_COMMAND); pci_set_cfgdata16(pdi, PCIR_COMMAND, cmd | enbit); return (0); } static int pci_emul_assign_bar(struct pci_devinst *const pdi, const int idx, const enum pcibar_type type, const uint64_t size) { int error; uint64_t *baseptr, limit, addr, mask, lobits, bar; switch (type) { case PCIBAR_NONE: baseptr = NULL; addr = mask = lobits = 0; break; case PCIBAR_IO: baseptr = &pci_emul_iobase; limit = PCI_EMUL_IOLIMIT; mask = PCIM_BAR_IO_BASE; lobits = PCIM_BAR_IO_SPACE; break; case PCIBAR_MEM64: /* * XXX * Some drivers do not work well if the 64-bit BAR is allocated * above 4GB. Allow for this by allocating small requests under * 4GB unless then allocation size is larger than some arbitrary * number (128MB currently). */ if (size > 128 * 1024 * 1024) { baseptr = &pci_emul_membase64; limit = pci_emul_memlim64; mask = PCIM_BAR_MEM_BASE; lobits = PCIM_BAR_MEM_SPACE | PCIM_BAR_MEM_64 | PCIM_BAR_MEM_PREFETCH; } else { baseptr = &pci_emul_membase32; limit = PCI_EMUL_MEMLIMIT32; mask = PCIM_BAR_MEM_BASE; lobits = PCIM_BAR_MEM_SPACE | PCIM_BAR_MEM_64; } break; case PCIBAR_MEM32: baseptr = &pci_emul_membase32; limit = PCI_EMUL_MEMLIMIT32; mask = PCIM_BAR_MEM_BASE; lobits = PCIM_BAR_MEM_SPACE | PCIM_BAR_MEM_32; break; case PCIBAR_ROM: /* do not claim memory for ROM. OVMF will do it for us. */ baseptr = NULL; limit = 0; mask = PCIM_BIOS_ADDR_MASK; lobits = 0; break; default: printf("pci_emul_alloc_base: invalid bar type %d\n", type); #ifdef FreeBSD assert(0); #else abort(); #endif } if (baseptr != NULL) { error = pci_emul_alloc_resource(baseptr, limit, size, &addr); if (error != 0) return (error); } else { addr = 0; } pdi->pi_bar[idx].type = type; pdi->pi_bar[idx].addr = addr; pdi->pi_bar[idx].size = size; /* * passthru devices are using same lobits as physical device they set * this property */ if (pdi->pi_bar[idx].lobits != 0) { lobits = pdi->pi_bar[idx].lobits; } else { pdi->pi_bar[idx].lobits = lobits; } /* Initialize the BAR register in config space */ bar = (addr & mask) | lobits; pci_set_cfgdata32(pdi, PCIR_BAR(idx), bar); if (type == PCIBAR_MEM64) { assert(idx + 1 <= PCI_BARMAX); pdi->pi_bar[idx + 1].type = PCIBAR_MEMHI64; pci_set_cfgdata32(pdi, PCIR_BAR(idx + 1), bar >> 32); } switch (type) { case PCIBAR_IO: if (porten(pdi)) register_bar(pdi, idx); break; case PCIBAR_MEM32: case PCIBAR_MEM64: case PCIBAR_MEMHI64: if (memen(pdi)) register_bar(pdi, idx); break; default: break; } return (0); } int pci_emul_alloc_rom(struct pci_devinst *const pdi, const uint64_t size, void **const addr) { /* allocate ROM space once on first call */ if (pci_emul_rombase == 0) { pci_emul_rombase = vm_create_devmem(pdi->pi_vmctx, VM_PCIROM, "pcirom", PCI_EMUL_ROMSIZE); if (pci_emul_rombase == MAP_FAILED) { warnx("%s: failed to create rom segment", __func__); return (-1); } pci_emul_romlim = pci_emul_rombase + PCI_EMUL_ROMSIZE; pci_emul_romoffset = 0; } /* ROM size should be a power of 2 and greater than 2 KB */ const uint64_t rom_size = MAX(1UL << flsl(size), ~PCIM_BIOS_ADDR_MASK + 1); /* check if ROM fits into ROM space */ if (pci_emul_romoffset + rom_size > PCI_EMUL_ROMSIZE) { warnx("%s: no space left in rom segment:", __func__); warnx("%16lu bytes left", PCI_EMUL_ROMSIZE - pci_emul_romoffset); warnx("%16lu bytes required by %d/%d/%d", rom_size, pdi->pi_bus, pdi->pi_slot, pdi->pi_func); return (-1); } /* allocate ROM BAR */ const int error = pci_emul_alloc_bar(pdi, PCI_ROM_IDX, PCIBAR_ROM, rom_size); if (error) return error; /* return address */ *addr = pci_emul_rombase + pci_emul_romoffset; /* save offset into ROM Space */ pdi->pi_romoffset = pci_emul_romoffset; /* increase offset for next ROM */ pci_emul_romoffset += rom_size; return (0); } int pci_emul_add_boot_device(struct pci_devinst *pi, int bootindex) { struct boot_device *new_device, *device; /* don't permit a negative bootindex */ if (bootindex < 0) { errx(4, "Invalid bootindex %d for %s", bootindex, pi->pi_name); } /* alloc new boot device */ new_device = calloc(1, sizeof(struct boot_device)); if (new_device == NULL) { return (ENOMEM); } new_device->pdi = pi; new_device->bootindex = bootindex; /* search for boot device with higher boot index */ TAILQ_FOREACH(device, &boot_devices, boot_device_chain) { if (device->bootindex == bootindex) { errx(4, "Could not set bootindex %d for %s. Bootindex already occupied by %s", bootindex, pi->pi_name, device->pdi->pi_name); } else if (device->bootindex > bootindex) { break; } } /* add boot device to queue */ if (device == NULL) { TAILQ_INSERT_TAIL(&boot_devices, new_device, boot_device_chain); } else { TAILQ_INSERT_BEFORE(device, new_device, boot_device_chain); } return (0); } #define CAP_START_OFFSET 0x40 int pci_emul_add_capability(struct pci_devinst *pi, u_char *capdata, int caplen, int *capoffp) { int i, capoff, reallen; uint16_t sts; assert(caplen > 0); reallen = roundup2(caplen, 4); /* dword aligned */ sts = pci_get_cfgdata16(pi, PCIR_STATUS); if ((sts & PCIM_STATUS_CAPPRESENT) == 0) capoff = CAP_START_OFFSET; else capoff = pi->pi_capend + 1; /* Check if we have enough space */ if (capoff + reallen > PCI_REGMAX + 1) return (-1); /* Set the previous capability pointer */ if ((sts & PCIM_STATUS_CAPPRESENT) == 0) { pci_set_cfgdata8(pi, PCIR_CAP_PTR, capoff); pci_set_cfgdata16(pi, PCIR_STATUS, sts|PCIM_STATUS_CAPPRESENT); } else pci_set_cfgdata8(pi, pi->pi_prevcap + 1, capoff); /* Copy the capability */ for (i = 0; i < caplen; i++) pci_set_cfgdata8(pi, capoff + i, capdata[i]); /* Set the next capability pointer */ pci_set_cfgdata8(pi, capoff + 1, 0); pi->pi_prevcap = capoff; pi->pi_capend = capoff + reallen - 1; if (capoffp != NULL) *capoffp = capoff; return (0); } static struct pci_devemu * pci_emul_finddev(const char *name) { struct pci_devemu **pdpp, *pdp; SET_FOREACH(pdpp, pci_devemu_set) { pdp = *pdpp; if (!strcmp(pdp->pe_emu, name)) { return (pdp); } } return (NULL); } static int pci_emul_init(struct vmctx *ctx, struct pci_devemu *pde, int bus, int slot, int func, struct funcinfo *fi) { struct pci_devinst *pdi; int err; pdi = calloc(1, sizeof(struct pci_devinst)); pdi->pi_vmctx = ctx; pdi->pi_bus = bus; pdi->pi_slot = slot; pdi->pi_func = func; pthread_mutex_init(&pdi->pi_lintr.lock, NULL); pdi->pi_lintr.pin = 0; pdi->pi_lintr.state = IDLE; pdi->pi_lintr.pirq_pin = 0; pdi->pi_lintr.ioapic_irq = 0; pdi->pi_d = pde; snprintf(pdi->pi_name, PI_NAMESZ, "%s@pci.%d.%d.%d", pde->pe_emu, bus, slot, func); /* Disable legacy interrupts */ pci_set_cfgdata8(pdi, PCIR_INTLINE, 255); pci_set_cfgdata8(pdi, PCIR_INTPIN, 0); #ifdef __FreeBSD__ if (get_config_bool_default("pci.enable_bars", !bootrom_boot())) pci_set_cfgdata8(pdi, PCIR_COMMAND, PCIM_CMD_BUSMASTEREN); #else if (get_config_bool_default("pci.enable_bars", true)) pci_set_cfgdata8(pdi, PCIR_COMMAND, PCIM_CMD_BUSMASTEREN); #endif err = (*pde->pe_init)(pdi, fi->fi_config); if (err == 0) fi->fi_devi = pdi; else free(pdi); return (err); } void pci_populate_msicap(struct msicap *msicap, int msgnum, int nextptr) { int mmc; /* Number of msi messages must be a power of 2 between 1 and 32 */ assert((msgnum & (msgnum - 1)) == 0 && msgnum >= 1 && msgnum <= 32); mmc = ffs(msgnum) - 1; bzero(msicap, sizeof(struct msicap)); msicap->capid = PCIY_MSI; msicap->nextptr = nextptr; msicap->msgctrl = PCIM_MSICTRL_64BIT | (mmc << 1); } int pci_emul_add_msicap(struct pci_devinst *pi, int msgnum) { struct msicap msicap; pci_populate_msicap(&msicap, msgnum, 0); return (pci_emul_add_capability(pi, (u_char *)&msicap, sizeof(msicap), NULL)); } static void pci_populate_msixcap(struct msixcap *msixcap, int msgnum, int barnum, uint32_t msix_tab_size) { assert(msix_tab_size % 4096 == 0); bzero(msixcap, sizeof(struct msixcap)); msixcap->capid = PCIY_MSIX; /* * Message Control Register, all fields set to * zero except for the Table Size. * Note: Table size N is encoded as N-1 */ msixcap->msgctrl = msgnum - 1; /* * MSI-X BAR setup: * - MSI-X table start at offset 0 * - PBA table starts at a 4K aligned offset after the MSI-X table */ msixcap->table_info = barnum & PCIM_MSIX_BIR_MASK; msixcap->pba_info = msix_tab_size | (barnum & PCIM_MSIX_BIR_MASK); } static void pci_msix_table_init(struct pci_devinst *pi, int table_entries) { int i, table_size; assert(table_entries > 0); assert(table_entries <= MAX_MSIX_TABLE_ENTRIES); table_size = table_entries * MSIX_TABLE_ENTRY_SIZE; pi->pi_msix.table = calloc(1, table_size); /* set mask bit of vector control register */ for (i = 0; i < table_entries; i++) pi->pi_msix.table[i].vector_control |= PCIM_MSIX_VCTRL_MASK; } int pci_emul_add_msixcap(struct pci_devinst *pi, int msgnum, int barnum) { uint32_t tab_size; struct msixcap msixcap; assert(msgnum >= 1 && msgnum <= MAX_MSIX_TABLE_ENTRIES); assert(barnum >= 0 && barnum <= PCIR_MAX_BAR_0); tab_size = msgnum * MSIX_TABLE_ENTRY_SIZE; /* Align table size to nearest 4K */ tab_size = roundup2(tab_size, 4096); pi->pi_msix.table_bar = barnum; pi->pi_msix.pba_bar = barnum; pi->pi_msix.table_offset = 0; pi->pi_msix.table_count = msgnum; pi->pi_msix.pba_offset = tab_size; pi->pi_msix.pba_size = PBA_SIZE(msgnum); pci_msix_table_init(pi, msgnum); pci_populate_msixcap(&msixcap, msgnum, barnum, tab_size); /* allocate memory for MSI-X Table and PBA */ pci_emul_alloc_bar(pi, barnum, PCIBAR_MEM32, tab_size + pi->pi_msix.pba_size); return (pci_emul_add_capability(pi, (u_char *)&msixcap, sizeof(msixcap), NULL)); } static void msixcap_cfgwrite(struct pci_devinst *pi, int capoff, int offset, int bytes, uint32_t val) { uint16_t msgctrl, rwmask; int off; off = offset - capoff; /* Message Control Register */ if (off == 2 && bytes == 2) { rwmask = PCIM_MSIXCTRL_MSIX_ENABLE | PCIM_MSIXCTRL_FUNCTION_MASK; msgctrl = pci_get_cfgdata16(pi, offset); msgctrl &= ~rwmask; msgctrl |= val & rwmask; val = msgctrl; pi->pi_msix.enabled = val & PCIM_MSIXCTRL_MSIX_ENABLE; pi->pi_msix.function_mask = val & PCIM_MSIXCTRL_FUNCTION_MASK; pci_lintr_update(pi); } CFGWRITE(pi, offset, val, bytes); } static void msicap_cfgwrite(struct pci_devinst *pi, int capoff, int offset, int bytes, uint32_t val) { uint16_t msgctrl, rwmask, msgdata, mme; uint32_t addrlo; /* * If guest is writing to the message control register make sure * we do not overwrite read-only fields. */ if ((offset - capoff) == 2 && bytes == 2) { rwmask = PCIM_MSICTRL_MME_MASK | PCIM_MSICTRL_MSI_ENABLE; msgctrl = pci_get_cfgdata16(pi, offset); msgctrl &= ~rwmask; msgctrl |= val & rwmask; val = msgctrl; } CFGWRITE(pi, offset, val, bytes); msgctrl = pci_get_cfgdata16(pi, capoff + 2); addrlo = pci_get_cfgdata32(pi, capoff + 4); if (msgctrl & PCIM_MSICTRL_64BIT) msgdata = pci_get_cfgdata16(pi, capoff + 12); else msgdata = pci_get_cfgdata16(pi, capoff + 8); mme = msgctrl & PCIM_MSICTRL_MME_MASK; pi->pi_msi.enabled = msgctrl & PCIM_MSICTRL_MSI_ENABLE ? 1 : 0; if (pi->pi_msi.enabled) { pi->pi_msi.addr = addrlo; pi->pi_msi.msg_data = msgdata; pi->pi_msi.maxmsgnum = 1 << (mme >> 4); } else { pi->pi_msi.maxmsgnum = 0; } pci_lintr_update(pi); } static void pciecap_cfgwrite(struct pci_devinst *pi, int capoff __unused, int offset, int bytes, uint32_t val) { /* XXX don't write to the readonly parts */ CFGWRITE(pi, offset, val, bytes); } #define PCIECAP_VERSION 0x2 int pci_emul_add_pciecap(struct pci_devinst *pi, int type) { int err; struct pciecap pciecap; bzero(&pciecap, sizeof(pciecap)); /* * Use the integrated endpoint type for endpoints on a root complex bus. * * NB: bhyve currently only supports a single PCI bus that is the root * complex bus, so all endpoints are integrated. */ if ((type == PCIEM_TYPE_ENDPOINT) && (pi->pi_bus == 0)) type = PCIEM_TYPE_ROOT_INT_EP; pciecap.capid = PCIY_EXPRESS; pciecap.pcie_capabilities = PCIECAP_VERSION | type; if (type != PCIEM_TYPE_ROOT_INT_EP) { pciecap.link_capabilities = 0x411; /* gen1, x1 */ pciecap.link_status = 0x11; /* gen1, x1 */ } err = pci_emul_add_capability(pi, (u_char *)&pciecap, sizeof(pciecap), NULL); return (err); } /* * This function assumes that 'coff' is in the capabilities region of the * config space. A capoff parameter of zero will force a search for the * offset and type. */ void pci_emul_capwrite(struct pci_devinst *pi, int offset, int bytes, uint32_t val, uint8_t capoff, int capid) { uint8_t nextoff; /* Do not allow un-aligned writes */ if ((offset & (bytes - 1)) != 0) return; if (capoff == 0) { /* Find the capability that we want to update */ capoff = CAP_START_OFFSET; while (1) { nextoff = pci_get_cfgdata8(pi, capoff + 1); if (nextoff == 0) break; if (offset >= capoff && offset < nextoff) break; capoff = nextoff; } assert(offset >= capoff); capid = pci_get_cfgdata8(pi, capoff); } /* * Capability ID and Next Capability Pointer are readonly. * However, some o/s's do 4-byte writes that include these. * For this case, trim the write back to 2 bytes and adjust * the data. */ if (offset == capoff || offset == capoff + 1) { if (offset == capoff && bytes == 4) { bytes = 2; offset += 2; val >>= 16; } else return; } switch (capid) { case PCIY_MSI: msicap_cfgwrite(pi, capoff, offset, bytes, val); break; case PCIY_MSIX: msixcap_cfgwrite(pi, capoff, offset, bytes, val); break; case PCIY_EXPRESS: pciecap_cfgwrite(pi, capoff, offset, bytes, val); break; default: break; } } static int pci_emul_iscap(struct pci_devinst *pi, int offset) { uint16_t sts; sts = pci_get_cfgdata16(pi, PCIR_STATUS); if ((sts & PCIM_STATUS_CAPPRESENT) != 0) { if (offset >= CAP_START_OFFSET && offset <= pi->pi_capend) return (1); } return (0); } static int pci_emul_fallback_handler(struct vcpu *vcpu __unused, int dir, uint64_t addr __unused, int size __unused, uint64_t *val, void *arg1 __unused, long arg2 __unused) { /* * Ignore writes; return 0xff's for reads. The mem read code * will take care of truncating to the correct size. */ if (dir == MEM_F_READ) { *val = 0xffffffffffffffff; } return (0); } static int pci_emul_ecfg_handler(struct vcpu *vcpu __unused, int dir, uint64_t addr, int bytes, uint64_t *val, void *arg1 __unused, long arg2 __unused) { int bus, slot, func, coff, in; coff = addr & 0xfff; func = (addr >> 12) & 0x7; slot = (addr >> 15) & 0x1f; bus = (addr >> 20) & 0xff; in = (dir == MEM_F_READ); if (in) *val = ~0UL; pci_cfgrw(in, bus, slot, func, coff, bytes, (uint32_t *)val); return (0); } uint64_t pci_ecfg_base(void) { return (PCI_EMUL_ECFG_BASE); } static int init_bootorder(void) { struct boot_device *device; FILE *fp; char *bootorder; size_t bootorder_len; if (TAILQ_EMPTY(&boot_devices)) return (0); fp = open_memstream(&bootorder, &bootorder_len); TAILQ_FOREACH(device, &boot_devices, boot_device_chain) { fprintf(fp, "/pci@i0cf8/pci@%d,%d\n", device->pdi->pi_slot, device->pdi->pi_func); } fclose(fp); return (qemu_fwcfg_add_file("bootorder", bootorder_len, bootorder)); } #define BUSIO_ROUNDUP 32 #define BUSMEM32_ROUNDUP (1024 * 1024) #define BUSMEM64_ROUNDUP (512 * 1024 * 1024) int init_pci(struct vmctx *ctx) { char node_name[sizeof("pci.XXX.XX.X")]; struct mem_range mr; struct pci_devemu *pde; struct businfo *bi; struct slotinfo *si; struct funcinfo *fi; nvlist_t *nvl; const char *emul; size_t lowmem; int bus, slot, func; int error; if (vm_get_lowmem_limit(ctx) > PCI_EMUL_MEMBASE32) errx(EX_OSERR, "Invalid lowmem limit"); pci_emul_iobase = PCI_EMUL_IOBASE; pci_emul_membase32 = PCI_EMUL_MEMBASE32; pci_emul_membase64 = vm_get_highmem_base(ctx) + vm_get_highmem_size(ctx); pci_emul_membase64 = roundup2(pci_emul_membase64, PCI_EMUL_MEMSIZE64); pci_emul_memlim64 = pci_emul_membase64 + PCI_EMUL_MEMSIZE64; TAILQ_INIT(&boot_devices); for (bus = 0; bus < MAXBUSES; bus++) { snprintf(node_name, sizeof(node_name), "pci.%d", bus); nvl = find_config_node(node_name); if (nvl == NULL) continue; pci_businfo[bus] = calloc(1, sizeof(struct businfo)); bi = pci_businfo[bus]; /* * Keep track of the i/o and memory resources allocated to * this bus. */ bi->iobase = pci_emul_iobase; bi->membase32 = pci_emul_membase32; bi->membase64 = pci_emul_membase64; /* first run: init devices */ for (slot = 0; slot < MAXSLOTS; slot++) { si = &bi->slotinfo[slot]; for (func = 0; func < MAXFUNCS; func++) { fi = &si->si_funcs[func]; snprintf(node_name, sizeof(node_name), "pci.%d.%d.%d", bus, slot, func); nvl = find_config_node(node_name); if (nvl == NULL) continue; fi->fi_config = nvl; emul = get_config_value_node(nvl, "device"); if (emul == NULL) { EPRINTLN("pci slot %d:%d:%d: missing " "\"device\" value", bus, slot, func); return (EINVAL); } pde = pci_emul_finddev(emul); if (pde == NULL) { EPRINTLN("pci slot %d:%d:%d: unknown " "device \"%s\"", bus, slot, func, emul); return (EINVAL); } if (pde->pe_alias != NULL) { EPRINTLN("pci slot %d:%d:%d: legacy " "device \"%s\", use \"%s\" instead", bus, slot, func, emul, pde->pe_alias); return (EINVAL); } fi->fi_pde = pde; error = pci_emul_init(ctx, pde, bus, slot, func, fi); if (error) return (error); } } /* second run: assign BARs and free list */ struct pci_bar_allocation *bar; struct pci_bar_allocation *bar_tmp; TAILQ_FOREACH_SAFE(bar, &pci_bars, chain, bar_tmp) { pci_emul_assign_bar(bar->pdi, bar->idx, bar->type, bar->size); free(bar); } TAILQ_INIT(&pci_bars); /* * Add some slop to the I/O and memory resources decoded by * this bus to give a guest some flexibility if it wants to * reprogram the BARs. */ pci_emul_iobase += BUSIO_ROUNDUP; pci_emul_iobase = roundup2(pci_emul_iobase, BUSIO_ROUNDUP); bi->iolimit = pci_emul_iobase; pci_emul_membase32 += BUSMEM32_ROUNDUP; pci_emul_membase32 = roundup2(pci_emul_membase32, BUSMEM32_ROUNDUP); bi->memlimit32 = pci_emul_membase32; pci_emul_membase64 += BUSMEM64_ROUNDUP; pci_emul_membase64 = roundup2(pci_emul_membase64, BUSMEM64_ROUNDUP); bi->memlimit64 = pci_emul_membase64; } /* * PCI backends are initialized before routing INTx interrupts * so that LPC devices are able to reserve ISA IRQs before * routing PIRQ pins. */ for (bus = 0; bus < MAXBUSES; bus++) { if ((bi = pci_businfo[bus]) == NULL) continue; for (slot = 0; slot < MAXSLOTS; slot++) { si = &bi->slotinfo[slot]; for (func = 0; func < MAXFUNCS; func++) { fi = &si->si_funcs[func]; if (fi->fi_devi == NULL) continue; pci_lintr_route(fi->fi_devi); } } } lpc_pirq_routed(); if ((error = init_bootorder()) != 0) { warnx("%s: Unable to init bootorder", __func__); return (error); } /* * The guest physical memory map looks like the following: * [0, lowmem) guest system memory * [lowmem, 0xC0000000) memory hole (may be absent) * [0xC0000000, 0xE0000000) PCI hole (32-bit BAR allocation) * [0xE0000000, 0xF0000000) PCI extended config window * [0xF0000000, 4GB) LAPIC, IOAPIC, HPET, firmware * [4GB, 4GB + highmem) */ /* * Accesses to memory addresses that are not allocated to system * memory or PCI devices return 0xff's. */ lowmem = vm_get_lowmem_size(ctx); bzero(&mr, sizeof(struct mem_range)); mr.name = "PCI hole"; mr.flags = MEM_F_RW | MEM_F_IMMUTABLE; mr.base = lowmem; mr.size = (4ULL * 1024 * 1024 * 1024) - lowmem; mr.handler = pci_emul_fallback_handler; error = register_mem_fallback(&mr); assert(error == 0); /* PCI extended config space */ bzero(&mr, sizeof(struct mem_range)); mr.name = "PCI ECFG"; mr.flags = MEM_F_RW | MEM_F_IMMUTABLE; mr.base = PCI_EMUL_ECFG_BASE; mr.size = PCI_EMUL_ECFG_SIZE; mr.handler = pci_emul_ecfg_handler; error = register_mem(&mr); assert(error == 0); return (0); } static void pci_apic_prt_entry(int bus __unused, int slot, int pin, int pirq_pin __unused, int ioapic_irq, void *arg __unused) { dsdt_line(" Package ()"); dsdt_line(" {"); dsdt_line(" 0x%X,", slot << 16 | 0xffff); dsdt_line(" 0x%02X,", pin - 1); dsdt_line(" Zero,"); dsdt_line(" 0x%X", ioapic_irq); dsdt_line(" },"); } static void pci_pirq_prt_entry(int bus __unused, int slot, int pin, int pirq_pin, int ioapic_irq __unused, void *arg __unused) { char *name; name = lpc_pirq_name(pirq_pin); if (name == NULL) return; dsdt_line(" Package ()"); dsdt_line(" {"); dsdt_line(" 0x%X,", slot << 16 | 0xffff); dsdt_line(" 0x%02X,", pin - 1); dsdt_line(" %s,", name); dsdt_line(" 0x00"); dsdt_line(" },"); free(name); } /* * A bhyve virtual machine has a flat PCI hierarchy with a root port * corresponding to each PCI bus. */ static void pci_bus_write_dsdt(int bus) { struct businfo *bi; struct slotinfo *si; struct pci_devinst *pi; int count, func, slot; /* * If there are no devices on this 'bus' then just return. */ if ((bi = pci_businfo[bus]) == NULL) { /* * Bus 0 is special because it decodes the I/O ports used * for PCI config space access even if there are no devices * on it. */ if (bus != 0) return; } dsdt_line(" Device (PC%02X)", bus); dsdt_line(" {"); dsdt_line(" Name (_HID, EisaId (\"PNP0A03\"))"); dsdt_line(" Method (_BBN, 0, NotSerialized)"); dsdt_line(" {"); dsdt_line(" Return (0x%08X)", bus); dsdt_line(" }"); dsdt_line(" Name (_CRS, ResourceTemplate ()"); dsdt_line(" {"); dsdt_line(" WordBusNumber (ResourceProducer, MinFixed, " "MaxFixed, PosDecode,"); dsdt_line(" 0x0000, // Granularity"); dsdt_line(" 0x%04X, // Range Minimum", bus); dsdt_line(" 0x%04X, // Range Maximum", bus); dsdt_line(" 0x0000, // Translation Offset"); dsdt_line(" 0x0001, // Length"); dsdt_line(" ,, )"); if (bus == 0) { dsdt_indent(3); dsdt_fixed_ioport(0xCF8, 8); dsdt_unindent(3); dsdt_line(" WordIO (ResourceProducer, MinFixed, MaxFixed, " "PosDecode, EntireRange,"); dsdt_line(" 0x0000, // Granularity"); dsdt_line(" 0x0000, // Range Minimum"); dsdt_line(" 0x0CF7, // Range Maximum"); dsdt_line(" 0x0000, // Translation Offset"); dsdt_line(" 0x0CF8, // Length"); dsdt_line(" ,, , TypeStatic)"); dsdt_line(" WordIO (ResourceProducer, MinFixed, MaxFixed, " "PosDecode, EntireRange,"); dsdt_line(" 0x0000, // Granularity"); dsdt_line(" 0x0D00, // Range Minimum"); dsdt_line(" 0x%04X, // Range Maximum", PCI_EMUL_IOBASE - 1); dsdt_line(" 0x0000, // Translation Offset"); dsdt_line(" 0x%04X, // Length", PCI_EMUL_IOBASE - 0x0D00); dsdt_line(" ,, , TypeStatic)"); if (bi == NULL) { dsdt_line(" })"); goto done; } } assert(bi != NULL); /* i/o window */ dsdt_line(" WordIO (ResourceProducer, MinFixed, MaxFixed, " "PosDecode, EntireRange,"); dsdt_line(" 0x0000, // Granularity"); dsdt_line(" 0x%04X, // Range Minimum", bi->iobase); dsdt_line(" 0x%04X, // Range Maximum", bi->iolimit - 1); dsdt_line(" 0x0000, // Translation Offset"); dsdt_line(" 0x%04X, // Length", bi->iolimit - bi->iobase); dsdt_line(" ,, , TypeStatic)"); /* mmio window (32-bit) */ dsdt_line(" DWordMemory (ResourceProducer, PosDecode, " "MinFixed, MaxFixed, NonCacheable, ReadWrite,"); dsdt_line(" 0x00000000, // Granularity"); dsdt_line(" 0x%08X, // Range Minimum\n", bi->membase32); dsdt_line(" 0x%08X, // Range Maximum\n", bi->memlimit32 - 1); dsdt_line(" 0x00000000, // Translation Offset"); dsdt_line(" 0x%08X, // Length\n", bi->memlimit32 - bi->membase32); dsdt_line(" ,, , AddressRangeMemory, TypeStatic)"); /* mmio window (64-bit) */ dsdt_line(" QWordMemory (ResourceProducer, PosDecode, " "MinFixed, MaxFixed, NonCacheable, ReadWrite,"); dsdt_line(" 0x0000000000000000, // Granularity"); dsdt_line(" 0x%016lX, // Range Minimum\n", bi->membase64); dsdt_line(" 0x%016lX, // Range Maximum\n", bi->memlimit64 - 1); dsdt_line(" 0x0000000000000000, // Translation Offset"); dsdt_line(" 0x%016lX, // Length\n", bi->memlimit64 - bi->membase64); dsdt_line(" ,, , AddressRangeMemory, TypeStatic)"); dsdt_line(" })"); count = pci_count_lintr(bus); if (count != 0) { dsdt_indent(2); dsdt_line("Name (PPRT, Package ()"); dsdt_line("{"); pci_walk_lintr(bus, pci_pirq_prt_entry, NULL); dsdt_line("})"); dsdt_line("Name (APRT, Package ()"); dsdt_line("{"); pci_walk_lintr(bus, pci_apic_prt_entry, NULL); dsdt_line("})"); dsdt_line("Method (_PRT, 0, NotSerialized)"); dsdt_line("{"); dsdt_line(" If (PICM)"); dsdt_line(" {"); dsdt_line(" Return (APRT)"); dsdt_line(" }"); dsdt_line(" Else"); dsdt_line(" {"); dsdt_line(" Return (PPRT)"); dsdt_line(" }"); dsdt_line("}"); dsdt_unindent(2); } dsdt_indent(2); for (slot = 0; slot < MAXSLOTS; slot++) { si = &bi->slotinfo[slot]; for (func = 0; func < MAXFUNCS; func++) { pi = si->si_funcs[func].fi_devi; if (pi != NULL && pi->pi_d->pe_write_dsdt != NULL) pi->pi_d->pe_write_dsdt(pi); } } dsdt_unindent(2); done: dsdt_line(" }"); } void pci_write_dsdt(void) { int bus; dsdt_indent(1); dsdt_line("Name (PICM, 0x00)"); dsdt_line("Method (_PIC, 1, NotSerialized)"); dsdt_line("{"); dsdt_line(" Store (Arg0, PICM)"); dsdt_line("}"); dsdt_line(""); dsdt_line("Scope (_SB)"); dsdt_line("{"); for (bus = 0; bus < MAXBUSES; bus++) pci_bus_write_dsdt(bus); dsdt_line("}"); dsdt_unindent(1); } int pci_bus_configured(int bus) { assert(bus >= 0 && bus < MAXBUSES); return (pci_businfo[bus] != NULL); } int pci_msi_enabled(struct pci_devinst *pi) { return (pi->pi_msi.enabled); } int pci_msi_maxmsgnum(struct pci_devinst *pi) { if (pi->pi_msi.enabled) return (pi->pi_msi.maxmsgnum); else return (0); } int pci_msix_enabled(struct pci_devinst *pi) { return (pi->pi_msix.enabled && !pi->pi_msi.enabled); } void pci_generate_msix(struct pci_devinst *pi, int index) { struct msix_table_entry *mte; if (!pci_msix_enabled(pi)) return; if (pi->pi_msix.function_mask) return; if (index >= pi->pi_msix.table_count) return; mte = &pi->pi_msix.table[index]; if ((mte->vector_control & PCIM_MSIX_VCTRL_MASK) == 0) { /* XXX Set PBA bit if interrupt is disabled */ vm_lapic_msi(pi->pi_vmctx, mte->addr, mte->msg_data); } } void pci_generate_msi(struct pci_devinst *pi, int index) { if (pci_msi_enabled(pi) && index < pci_msi_maxmsgnum(pi)) { vm_lapic_msi(pi->pi_vmctx, pi->pi_msi.addr, pi->pi_msi.msg_data + index); } } static bool pci_lintr_permitted(struct pci_devinst *pi) { uint16_t cmd; cmd = pci_get_cfgdata16(pi, PCIR_COMMAND); return (!(pi->pi_msi.enabled || pi->pi_msix.enabled || (cmd & PCIM_CMD_INTxDIS))); } void pci_lintr_request(struct pci_devinst *pi) { struct businfo *bi; struct slotinfo *si; int bestpin, bestcount, pin; bi = pci_businfo[pi->pi_bus]; assert(bi != NULL); /* * Just allocate a pin from our slot. The pin will be * assigned IRQs later when interrupts are routed. */ si = &bi->slotinfo[pi->pi_slot]; bestpin = 0; bestcount = si->si_intpins[0].ii_count; for (pin = 1; pin < 4; pin++) { if (si->si_intpins[pin].ii_count < bestcount) { bestpin = pin; bestcount = si->si_intpins[pin].ii_count; } } si->si_intpins[bestpin].ii_count++; pi->pi_lintr.pin = bestpin + 1; pci_set_cfgdata8(pi, PCIR_INTPIN, bestpin + 1); } static void pci_lintr_route(struct pci_devinst *pi) { struct businfo *bi; struct intxinfo *ii; if (pi->pi_lintr.pin == 0) return; bi = pci_businfo[pi->pi_bus]; assert(bi != NULL); ii = &bi->slotinfo[pi->pi_slot].si_intpins[pi->pi_lintr.pin - 1]; /* * Attempt to allocate an I/O APIC pin for this intpin if one * is not yet assigned. */ if (ii->ii_ioapic_irq == 0) ii->ii_ioapic_irq = ioapic_pci_alloc_irq(pi); assert(ii->ii_ioapic_irq > 0); /* * Attempt to allocate a PIRQ pin for this intpin if one is * not yet assigned. */ if (ii->ii_pirq_pin == 0) ii->ii_pirq_pin = pirq_alloc_pin(pi); assert(ii->ii_pirq_pin > 0); pi->pi_lintr.ioapic_irq = ii->ii_ioapic_irq; pi->pi_lintr.pirq_pin = ii->ii_pirq_pin; pci_set_cfgdata8(pi, PCIR_INTLINE, pirq_irq(ii->ii_pirq_pin)); } void pci_lintr_assert(struct pci_devinst *pi) { assert(pi->pi_lintr.pin > 0); pthread_mutex_lock(&pi->pi_lintr.lock); if (pi->pi_lintr.state == IDLE) { if (pci_lintr_permitted(pi)) { pi->pi_lintr.state = ASSERTED; pci_irq_assert(pi); } else pi->pi_lintr.state = PENDING; } pthread_mutex_unlock(&pi->pi_lintr.lock); } void pci_lintr_deassert(struct pci_devinst *pi) { assert(pi->pi_lintr.pin > 0); pthread_mutex_lock(&pi->pi_lintr.lock); if (pi->pi_lintr.state == ASSERTED) { pi->pi_lintr.state = IDLE; pci_irq_deassert(pi); } else if (pi->pi_lintr.state == PENDING) pi->pi_lintr.state = IDLE; pthread_mutex_unlock(&pi->pi_lintr.lock); } static void pci_lintr_update(struct pci_devinst *pi) { pthread_mutex_lock(&pi->pi_lintr.lock); if (pi->pi_lintr.state == ASSERTED && !pci_lintr_permitted(pi)) { pci_irq_deassert(pi); pi->pi_lintr.state = PENDING; } else if (pi->pi_lintr.state == PENDING && pci_lintr_permitted(pi)) { pi->pi_lintr.state = ASSERTED; pci_irq_assert(pi); } pthread_mutex_unlock(&pi->pi_lintr.lock); #ifndef __FreeBSD__ if (pi->pi_d->pe_lintrupdate != NULL) { pi->pi_d->pe_lintrupdate(pi); } #endif /* __FreeBSD__ */ } int pci_count_lintr(int bus) { int count, slot, pin; struct slotinfo *slotinfo; count = 0; if (pci_businfo[bus] != NULL) { for (slot = 0; slot < MAXSLOTS; slot++) { slotinfo = &pci_businfo[bus]->slotinfo[slot]; for (pin = 0; pin < 4; pin++) { if (slotinfo->si_intpins[pin].ii_count != 0) count++; } } } return (count); } void pci_walk_lintr(int bus, pci_lintr_cb cb, void *arg) { struct businfo *bi; struct slotinfo *si; struct intxinfo *ii; int slot, pin; if ((bi = pci_businfo[bus]) == NULL) return; for (slot = 0; slot < MAXSLOTS; slot++) { si = &bi->slotinfo[slot]; for (pin = 0; pin < 4; pin++) { ii = &si->si_intpins[pin]; if (ii->ii_count != 0) cb(bus, slot, pin + 1, ii->ii_pirq_pin, ii->ii_ioapic_irq, arg); } } } /* * Return 1 if the emulated device in 'slot' is a multi-function device. * Return 0 otherwise. */ static int pci_emul_is_mfdev(int bus, int slot) { struct businfo *bi; struct slotinfo *si; int f, numfuncs; numfuncs = 0; if ((bi = pci_businfo[bus]) != NULL) { si = &bi->slotinfo[slot]; for (f = 0; f < MAXFUNCS; f++) { if (si->si_funcs[f].fi_devi != NULL) { numfuncs++; } } } return (numfuncs > 1); } /* * Ensure that the PCIM_MFDEV bit is properly set (or unset) depending on * whether or not is a multi-function being emulated in the pci 'slot'. */ static void pci_emul_hdrtype_fixup(int bus, int slot, int off, int bytes, uint32_t *rv) { int mfdev; if (off <= PCIR_HDRTYPE && off + bytes > PCIR_HDRTYPE) { mfdev = pci_emul_is_mfdev(bus, slot); switch (bytes) { case 1: case 2: *rv &= ~PCIM_MFDEV; if (mfdev) { *rv |= PCIM_MFDEV; } break; case 4: *rv &= ~(PCIM_MFDEV << 16); if (mfdev) { *rv |= (PCIM_MFDEV << 16); } break; } } } /* * Update device state in response to changes to the PCI command * register. */ void pci_emul_cmd_changed(struct pci_devinst *pi, uint16_t old) { int i; uint16_t changed, new; new = pci_get_cfgdata16(pi, PCIR_COMMAND); changed = old ^ new; /* * If the MMIO or I/O address space decoding has changed then * register/unregister all BARs that decode that address space. */ for (i = 0; i <= PCI_BARMAX_WITH_ROM; i++) { switch (pi->pi_bar[i].type) { case PCIBAR_NONE: case PCIBAR_MEMHI64: break; case PCIBAR_IO: /* I/O address space decoding changed? */ if (changed & PCIM_CMD_PORTEN) { if (new & PCIM_CMD_PORTEN) register_bar(pi, i); else unregister_bar(pi, i); } break; case PCIBAR_ROM: /* skip (un-)register of ROM if it disabled */ if (!romen(pi)) break; /* fallthrough */ case PCIBAR_MEM32: case PCIBAR_MEM64: /* MMIO address space decoding changed? */ if (changed & PCIM_CMD_MEMEN) { if (new & PCIM_CMD_MEMEN) register_bar(pi, i); else unregister_bar(pi, i); } break; default: assert(0); } } /* * If INTx has been unmasked and is pending, assert the * interrupt. */ pci_lintr_update(pi); } static void pci_emul_cmdsts_write(struct pci_devinst *pi, int coff, uint32_t new, int bytes) { int rshift; uint32_t cmd, old, readonly; cmd = pci_get_cfgdata16(pi, PCIR_COMMAND); /* stash old value */ /* * From PCI Local Bus Specification 3.0 sections 6.2.2 and 6.2.3. * * XXX Bits 8, 11, 12, 13, 14 and 15 in the status register are * 'write 1 to clear'. However these bits are not set to '1' by * any device emulation so it is simpler to treat them as readonly. */ rshift = (coff & 0x3) * 8; readonly = 0xFFFFF880 >> rshift; old = CFGREAD(pi, coff, bytes); new &= ~readonly; new |= (old & readonly); CFGWRITE(pi, coff, new, bytes); /* update config */ pci_emul_cmd_changed(pi, cmd); } static void pci_cfgrw(int in, int bus, int slot, int func, int coff, int bytes, uint32_t *valp) { struct businfo *bi; struct slotinfo *si; struct pci_devinst *pi; struct pci_devemu *pe; int idx, needcfg; uint64_t addr, mask; uint64_t bar = 0; if ((bi = pci_businfo[bus]) != NULL) { si = &bi->slotinfo[slot]; pi = si->si_funcs[func].fi_devi; } else pi = NULL; /* * Just return if there is no device at this slot:func or if the * guest is doing an un-aligned access. */ if (pi == NULL || (bytes != 1 && bytes != 2 && bytes != 4) || (coff & (bytes - 1)) != 0) { if (in) *valp = 0xffffffff; return; } /* * Ignore all writes beyond the standard config space and return all * ones on reads. */ if (coff >= PCI_REGMAX + 1) { if (in) { *valp = 0xffffffff; /* * Extended capabilities begin at offset 256 in config * space. Absence of extended capabilities is signaled * with all 0s in the extended capability header at * offset 256. */ if (coff <= PCI_REGMAX + 4) *valp = 0x00000000; } return; } pe = pi->pi_d; /* * Config read */ if (in) { /* Let the device emulation override the default handler */ needcfg = PE_CFGRW_DEFAULT; if (pe->pe_cfgread != NULL) needcfg = pe->pe_cfgread(pi, coff, bytes, valp); if (needcfg != PE_CFGRW_DROP) *valp = CFGREAD(pi, coff, bytes); pci_emul_hdrtype_fixup(bus, slot, coff, bytes, valp); } else { /* Let the device emulation override the default handler */ if (pe->pe_cfgwrite != NULL && pe->pe_cfgwrite(pi, coff, bytes, *valp) == PE_CFGRW_DROP) { return; } /* * Special handling for write to BAR and ROM registers */ if (is_pcir_bar(coff) || is_pcir_bios(coff)) { /* * Ignore writes to BAR registers that are not * 4-byte aligned. */ if (bytes != 4 || (coff & 0x3) != 0) return; if (is_pcir_bar(coff)) { idx = (coff - PCIR_BAR(0)) / 4; } else if (is_pcir_bios(coff)) { idx = PCI_ROM_IDX; } else { errx(4, "%s: invalid BAR offset %d", __func__, coff); } mask = ~(pi->pi_bar[idx].size - 1); switch (pi->pi_bar[idx].type) { case PCIBAR_NONE: pi->pi_bar[idx].addr = bar = 0; break; case PCIBAR_IO: addr = *valp & mask; addr &= 0xffff; bar = addr | pi->pi_bar[idx].lobits; /* * Register the new BAR value for interception */ if (addr != pi->pi_bar[idx].addr) { update_bar_address(pi, addr, idx, PCIBAR_IO); } break; case PCIBAR_MEM32: addr = bar = *valp & mask; bar |= pi->pi_bar[idx].lobits; if (addr != pi->pi_bar[idx].addr) { update_bar_address(pi, addr, idx, PCIBAR_MEM32); } break; case PCIBAR_MEM64: addr = bar = *valp & mask; bar |= pi->pi_bar[idx].lobits; if (addr != (uint32_t)pi->pi_bar[idx].addr) { update_bar_address(pi, addr, idx, PCIBAR_MEM64); } break; case PCIBAR_MEMHI64: mask = ~(pi->pi_bar[idx - 1].size - 1); addr = ((uint64_t)*valp << 32) & mask; bar = addr >> 32; if (bar != pi->pi_bar[idx - 1].addr >> 32) { update_bar_address(pi, addr, idx - 1, PCIBAR_MEMHI64); } break; case PCIBAR_ROM: addr = bar = *valp & mask; if (memen(pi) && romen(pi)) { unregister_bar(pi, idx); } pi->pi_bar[idx].addr = addr; pi->pi_bar[idx].lobits = *valp & PCIM_BIOS_ENABLE; /* romen could have changed it value */ if (memen(pi) && romen(pi)) { register_bar(pi, idx); } bar |= pi->pi_bar[idx].lobits; break; default: assert(0); } pci_set_cfgdata32(pi, coff, bar); } else if (pci_emul_iscap(pi, coff)) { pci_emul_capwrite(pi, coff, bytes, *valp, 0, 0); } else if (coff >= PCIR_COMMAND && coff < PCIR_REVID) { pci_emul_cmdsts_write(pi, coff, *valp, bytes); } else { CFGWRITE(pi, coff, *valp, bytes); } } } static int cfgenable, cfgbus, cfgslot, cfgfunc, cfgoff; static int pci_emul_cfgaddr(struct vmctx *ctx __unused, int in, int port __unused, int bytes, uint32_t *eax, void *arg __unused) { uint32_t x; if (bytes != 4) { if (in) *eax = (bytes == 2) ? 0xffff : 0xff; return (0); } if (in) { x = (cfgbus << 16) | (cfgslot << 11) | (cfgfunc << 8) | cfgoff; if (cfgenable) x |= CONF1_ENABLE; *eax = x; } else { x = *eax; cfgenable = (x & CONF1_ENABLE) == CONF1_ENABLE; cfgoff = (x & PCI_REGMAX) & ~0x03; cfgfunc = (x >> 8) & PCI_FUNCMAX; cfgslot = (x >> 11) & PCI_SLOTMAX; cfgbus = (x >> 16) & PCI_BUSMAX; } return (0); } INOUT_PORT(pci_cfgaddr, CONF1_ADDR_PORT, IOPORT_F_INOUT, pci_emul_cfgaddr); static int pci_emul_cfgdata(struct vmctx *ctx __unused, int in, int port, int bytes, uint32_t *eax, void *arg __unused) { int coff; assert(bytes == 1 || bytes == 2 || bytes == 4); coff = cfgoff + (port - CONF1_DATA_PORT); if (cfgenable) { pci_cfgrw(in, cfgbus, cfgslot, cfgfunc, coff, bytes, eax); } else { /* Ignore accesses to cfgdata if not enabled by cfgaddr */ if (in) *eax = 0xffffffff; } return (0); } INOUT_PORT(pci_cfgdata, CONF1_DATA_PORT+0, IOPORT_F_INOUT, pci_emul_cfgdata); INOUT_PORT(pci_cfgdata, CONF1_DATA_PORT+1, IOPORT_F_INOUT, pci_emul_cfgdata); INOUT_PORT(pci_cfgdata, CONF1_DATA_PORT+2, IOPORT_F_INOUT, pci_emul_cfgdata); INOUT_PORT(pci_cfgdata, CONF1_DATA_PORT+3, IOPORT_F_INOUT, pci_emul_cfgdata); #define PCI_EMUL_TEST #ifdef PCI_EMUL_TEST /* * Define a dummy test device */ #define DIOSZ 8 #define DMEMSZ 4096 struct pci_emul_dsoftc { uint8_t ioregs[DIOSZ]; uint8_t memregs[2][DMEMSZ]; }; #define PCI_EMUL_MSI_MSGS 4 #define PCI_EMUL_MSIX_MSGS 16 static int pci_emul_dinit(struct pci_devinst *pi, nvlist_t *nvl __unused) { int error; struct pci_emul_dsoftc *sc; sc = calloc(1, sizeof(struct pci_emul_dsoftc)); pi->pi_arg = sc; pci_set_cfgdata16(pi, PCIR_DEVICE, 0x0001); pci_set_cfgdata16(pi, PCIR_VENDOR, 0x10DD); pci_set_cfgdata8(pi, PCIR_CLASS, 0x02); error = pci_emul_add_msicap(pi, PCI_EMUL_MSI_MSGS); assert(error == 0); error = pci_emul_alloc_bar(pi, 0, PCIBAR_IO, DIOSZ); assert(error == 0); error = pci_emul_alloc_bar(pi, 1, PCIBAR_MEM32, DMEMSZ); assert(error == 0); error = pci_emul_alloc_bar(pi, 2, PCIBAR_MEM32, DMEMSZ); assert(error == 0); return (0); } static void pci_emul_diow(struct pci_devinst *pi, int baridx, uint64_t offset, int size, uint64_t value) { int i; struct pci_emul_dsoftc *sc = pi->pi_arg; if (baridx == 0) { if (offset + size > DIOSZ) { printf("diow: iow too large, offset %ld size %d\n", offset, size); return; } if (size == 1) { sc->ioregs[offset] = value & 0xff; } else if (size == 2) { *(uint16_t *)&sc->ioregs[offset] = value & 0xffff; } else if (size == 4) { *(uint32_t *)&sc->ioregs[offset] = value; } else { printf("diow: iow unknown size %d\n", size); } /* * Special magic value to generate an interrupt */ if (offset == 4 && size == 4 && pci_msi_enabled(pi)) pci_generate_msi(pi, value % pci_msi_maxmsgnum(pi)); if (value == 0xabcdef) { for (i = 0; i < pci_msi_maxmsgnum(pi); i++) pci_generate_msi(pi, i); } } if (baridx == 1 || baridx == 2) { if (offset + size > DMEMSZ) { printf("diow: memw too large, offset %ld size %d\n", offset, size); return; } i = baridx - 1; /* 'memregs' index */ if (size == 1) { sc->memregs[i][offset] = value; } else if (size == 2) { *(uint16_t *)&sc->memregs[i][offset] = value; } else if (size == 4) { *(uint32_t *)&sc->memregs[i][offset] = value; } else if (size == 8) { *(uint64_t *)&sc->memregs[i][offset] = value; } else { printf("diow: memw unknown size %d\n", size); } /* * magic interrupt ?? */ } if (baridx > 2 || baridx < 0) { printf("diow: unknown bar idx %d\n", baridx); } } static uint64_t pci_emul_dior(struct pci_devinst *pi, int baridx, uint64_t offset, int size) { struct pci_emul_dsoftc *sc = pi->pi_arg; uint32_t value; int i; value = 0; if (baridx == 0) { if (offset + size > DIOSZ) { printf("dior: ior too large, offset %ld size %d\n", offset, size); return (0); } value = 0; if (size == 1) { value = sc->ioregs[offset]; } else if (size == 2) { value = *(uint16_t *) &sc->ioregs[offset]; } else if (size == 4) { value = *(uint32_t *) &sc->ioregs[offset]; } else { printf("dior: ior unknown size %d\n", size); } } if (baridx == 1 || baridx == 2) { if (offset + size > DMEMSZ) { printf("dior: memr too large, offset %ld size %d\n", offset, size); return (0); } i = baridx - 1; /* 'memregs' index */ if (size == 1) { value = sc->memregs[i][offset]; } else if (size == 2) { value = *(uint16_t *) &sc->memregs[i][offset]; } else if (size == 4) { value = *(uint32_t *) &sc->memregs[i][offset]; } else if (size == 8) { value = *(uint64_t *) &sc->memregs[i][offset]; } else { printf("dior: ior unknown size %d\n", size); } } if (baridx > 2 || baridx < 0) { printf("dior: unknown bar idx %d\n", baridx); return (0); } return (value); } static const struct pci_devemu pci_dummy = { .pe_emu = "dummy", .pe_init = pci_emul_dinit, .pe_barwrite = pci_emul_diow, .pe_barread = pci_emul_dior, }; PCI_EMUL_SET(pci_dummy); #endif /* PCI_EMUL_TEST */ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2011 NetApp, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * Copyright 2018 Joyent, Inc. * Copyright 2025 Oxide Computer Company */ #ifndef _PCI_EMUL_H_ #define _PCI_EMUL_H_ #include #include #include #include #include #include #include #include #define PCI_BARMAX PCIR_MAX_BAR_0 /* BAR registers in a Type 0 header */ #define PCI_BARMAX_WITH_ROM (PCI_BARMAX + 1) #define PCI_ROM_IDX (PCI_BARMAX + 1) struct vmctx; struct pci_devinst; struct memory_region; struct pci_devemu { const char *pe_emu; /* Name of device emulation */ /* instance creation */ int (*pe_init)(struct pci_devinst *, nvlist_t *); int (*pe_legacy_config)(nvlist_t *, const char *); const char *pe_alias; /* ACPI DSDT enumeration */ void (*pe_write_dsdt)(struct pci_devinst *); /* config space read/write callbacks */ int (*pe_cfgwrite)(struct pci_devinst *pi, int offset, int bytes, uint32_t val); int (*pe_cfgread)(struct pci_devinst *pi, int offset, int bytes, uint32_t *retval); /* BAR read/write callbacks */ void (*pe_barwrite)(struct pci_devinst *pi, int baridx, uint64_t offset, int size, uint64_t value); uint64_t (*pe_barread)(struct pci_devinst *pi, int baridx, uint64_t offset, int size); void (*pe_baraddr)(struct pci_devinst *pi, int baridx, int enabled, uint64_t address); #ifndef __FreeBSD__ void (*pe_lintrupdate)(struct pci_devinst *pi); #endif /* __FreeBSD__ */ }; #define PCI_EMUL_SET(x) DATA_SET(pci_devemu_set, x) /* * These values are returned by the config space read/write callbacks * (pe_cfgwrite and pe_cfgread in the above structure). * * A return value of PE_CFGRW_DEFAULT will cause the PCI emulation framework to * continue on and access the configuration space as if the callback did not * exist, whereas PE_CFGRW_DROP will not. */ #define PE_CFGRW_DROP (0) #define PE_CFGRW_DEFAULT (1) enum pcibar_type { PCIBAR_NONE, PCIBAR_IO, PCIBAR_MEM32, PCIBAR_MEM64, PCIBAR_MEMHI64, PCIBAR_ROM, }; struct pcibar { enum pcibar_type type; /* io or memory */ uint64_t size; uint64_t addr; uint8_t lobits; }; #define PI_NAMESZ 40 struct msix_table_entry { uint64_t addr; uint32_t msg_data; uint32_t vector_control; } __packed; /* * In case the structure is modified to hold extra information, use a define * for the size that should be emulated. */ #define MSIX_TABLE_ENTRY_SIZE 16 #define MAX_MSIX_TABLE_ENTRIES 2048 #define PBA_SIZE(msgnum) (roundup2((msgnum), 64) / 8) enum lintr_stat { IDLE, ASSERTED, PENDING }; struct pci_devinst { struct pci_devemu *pi_d; struct vmctx *pi_vmctx; uint8_t pi_bus, pi_slot, pi_func; char pi_name[PI_NAMESZ]; int pi_bar_getsize; int pi_prevcap; int pi_capend; struct { int8_t pin; enum lintr_stat state; int pirq_pin; int ioapic_irq; pthread_mutex_t lock; } pi_lintr; struct { int enabled; uint64_t addr; uint64_t msg_data; int maxmsgnum; } pi_msi; struct { int enabled; int table_bar; int pba_bar; uint32_t table_offset; int table_count; uint32_t pba_offset; int pba_size; int function_mask; struct msix_table_entry *table; /* allocated at runtime */ void *pba_page; int pba_page_offset; uint8_t *mapped_addr; size_t mapped_size; } pi_msix; void *pi_arg; /* devemu-private data */ u_char pi_cfgdata[PCI_REGMAX + 1]; /* ROM is handled like a BAR */ struct pcibar pi_bar[PCI_BARMAX_WITH_ROM + 1]; uint64_t pi_romoffset; }; struct msicap { uint8_t capid; uint8_t nextptr; uint16_t msgctrl; uint32_t addrlo; uint32_t addrhi; uint16_t msgdata; } __packed; static_assert(sizeof(struct msicap) == 14, "compile-time assertion failed"); struct msixcap { uint8_t capid; uint8_t nextptr; uint16_t msgctrl; uint32_t table_info; /* bar index and offset within it */ uint32_t pba_info; /* bar index and offset within it */ } __packed; static_assert(sizeof(struct msixcap) == 12, "compile-time assertion failed"); struct pciecap { uint8_t capid; uint8_t nextptr; uint16_t pcie_capabilities; uint32_t dev_capabilities; /* all devices */ uint16_t dev_control; uint16_t dev_status; uint32_t link_capabilities; /* devices with links */ uint16_t link_control; uint16_t link_status; uint32_t slot_capabilities; /* ports with slots */ uint16_t slot_control; uint16_t slot_status; uint16_t root_control; /* root ports */ uint16_t root_capabilities; uint32_t root_status; uint32_t dev_capabilities2; /* all devices */ uint16_t dev_control2; uint16_t dev_status2; uint32_t link_capabilities2; /* devices with links */ uint16_t link_control2; uint16_t link_status2; uint32_t slot_capabilities2; /* ports with slots */ uint16_t slot_control2; uint16_t slot_status2; } __packed; static_assert(sizeof(struct pciecap) == 60, "compile-time assertion failed"); typedef void (*pci_lintr_cb)(int b, int s, int pin, int pirq_pin, int ioapic_irq, void *arg); int init_pci(struct vmctx *ctx); void pci_callback(void); uint32_t pci_config_read_reg(const struct pcisel *host_sel, nvlist_t *nvl, uint32_t reg, uint8_t size, uint32_t def); int pci_emul_alloc_bar(struct pci_devinst *pdi, int idx, enum pcibar_type type, uint64_t size); int pci_emul_alloc_rom(struct pci_devinst *const pdi, const uint64_t size, void **const addr); int pci_emul_add_boot_device(struct pci_devinst *const pi, const int bootindex); int pci_emul_add_capability(struct pci_devinst *pi, u_char *capdata, int caplen, int *capoffp); int pci_emul_add_msicap(struct pci_devinst *pi, int msgnum); int pci_emul_add_pciecap(struct pci_devinst *pi, int pcie_device_type); void pci_emul_capwrite(struct pci_devinst *pi, int offset, int bytes, uint32_t val, uint8_t capoff, int capid); void pci_emul_cmd_changed(struct pci_devinst *pi, uint16_t old); void pci_generate_msi(struct pci_devinst *pi, int msgnum); void pci_generate_msix(struct pci_devinst *pi, int msgnum); void pci_lintr_assert(struct pci_devinst *pi); void pci_lintr_deassert(struct pci_devinst *pi); void pci_lintr_request(struct pci_devinst *pi); int pci_msi_enabled(struct pci_devinst *pi); int pci_msix_enabled(struct pci_devinst *pi); int pci_msix_table_bar(struct pci_devinst *pi); int pci_msix_pba_bar(struct pci_devinst *pi); int pci_msi_maxmsgnum(struct pci_devinst *pi); int pci_parse_legacy_config(nvlist_t *nvl, const char *opt); int pci_parse_slot(char *opt); void pci_print_supported_devices(void); void pci_populate_msicap(struct msicap *cap, int msgs, int nextptr); int pci_emul_add_msixcap(struct pci_devinst *pi, int msgnum, int barnum); int pci_emul_msix_twrite(struct pci_devinst *pi, uint64_t offset, int size, uint64_t value); uint64_t pci_emul_msix_tread(struct pci_devinst *pi, uint64_t offset, int size); int pci_count_lintr(int bus); void pci_walk_lintr(int bus, pci_lintr_cb cb, void *arg); void pci_write_dsdt(void); uint64_t pci_ecfg_base(void); int pci_bus_configured(int bus); static __inline void pci_set_cfgdata8(struct pci_devinst *pi, int offset, uint8_t val) { assert(offset <= PCI_REGMAX); *(uint8_t *)(pi->pi_cfgdata + offset) = val; } static __inline void pci_set_cfgdata16(struct pci_devinst *pi, int offset, uint16_t val) { assert(offset <= (PCI_REGMAX - 1) && (offset & 1) == 0); *(uint16_t *)(pi->pi_cfgdata + offset) = val; } static __inline void pci_set_cfgdata32(struct pci_devinst *pi, int offset, uint32_t val) { assert(offset <= (PCI_REGMAX - 3) && (offset & 3) == 0); *(uint32_t *)(pi->pi_cfgdata + offset) = val; } static __inline uint8_t pci_get_cfgdata8(struct pci_devinst *pi, int offset) { assert(offset <= PCI_REGMAX); return (*(uint8_t *)(pi->pi_cfgdata + offset)); } static __inline uint16_t pci_get_cfgdata16(struct pci_devinst *pi, int offset) { assert(offset <= (PCI_REGMAX - 1) && (offset & 1) == 0); return (*(uint16_t *)(pi->pi_cfgdata + offset)); } static __inline uint32_t pci_get_cfgdata32(struct pci_devinst *pi, int offset) { assert(offset <= (PCI_REGMAX - 3) && (offset & 3) == 0); return (*(uint32_t *)(pi->pi_cfgdata + offset)); } #endif /* _PCI_EMUL_H_ */ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2015 Nahanni Systems, Inc. * Copyright 2018 Joyent, Inc. * Copyright 2021 OmniOS Community Edition (OmniOSce) Association. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include #include #include #include #include #include #include #include #include #include "bhyvegc.h" #include "bhyverun.h" #include "config.h" #include "debug.h" #include "console.h" #include "inout.h" #include "pci_emul.h" #include "rfb.h" #include "vga.h" /* * bhyve Framebuffer device emulation. * BAR0 points to the current mode information. * BAR1 is the 32-bit framebuffer address. * * -s ,fbuf,wait,vga=on|io|off,rfb=:port,w=width,h=height */ static int fbuf_debug = 1; #define DEBUG_INFO 1 #define DEBUG_VERBOSE 4 #define DPRINTF(level, params) if (level <= fbuf_debug) PRINTLN params #define KB (1024UL) #define MB (1024 * 1024UL) #define DMEMSZ 128 #define FB_SIZE (32*MB) #define COLS_MAX 3840 #define ROWS_MAX 2160 #define COLS_DEFAULT 1024 #define ROWS_DEFAULT 768 #define COLS_MIN 640 #define ROWS_MIN 480 struct pci_fbuf_softc { struct pci_devinst *fsc_pi; struct { uint32_t fbsize; uint16_t width; uint16_t height; uint16_t depth; uint16_t refreshrate; uint8_t reserved[116]; } __packed memregs; /* rfb server */ char *rfb_host; char *rfb_password; int rfb_port; #ifndef __FreeBSD__ const char *rfb_unix; #endif int rfb_wait; int vga_enabled; int vga_full; uint32_t fbaddr; char *fb_base; uint16_t gc_width; uint16_t gc_height; void *vgasc; struct bhyvegc_image *gc_image; }; static struct pci_fbuf_softc *fbuf_sc; #define PCI_FBUF_MSI_MSGS 4 static void pci_fbuf_write(struct pci_devinst *pi, int baridx, uint64_t offset, int size, uint64_t value) { struct pci_fbuf_softc *sc; uint8_t *p; assert(baridx == 0); sc = pi->pi_arg; DPRINTF(DEBUG_VERBOSE, ("fbuf wr: offset 0x%lx, size: %d, value: 0x%lx", offset, size, value)); if (offset + size > DMEMSZ) { printf("fbuf: write too large, offset %ld size %d\n", offset, size); return; } p = (uint8_t *)&sc->memregs + offset; switch (size) { case 1: *p = value; break; case 2: *(uint16_t *)p = value; break; case 4: *(uint32_t *)p = value; break; case 8: *(uint64_t *)p = value; break; default: printf("fbuf: write unknown size %d\n", size); break; } if (!sc->gc_image->vgamode && sc->memregs.width == 0 && sc->memregs.height == 0) { DPRINTF(DEBUG_INFO, ("switching to VGA mode")); sc->gc_image->vgamode = 1; sc->gc_width = 0; sc->gc_height = 0; } else if (sc->gc_image->vgamode && sc->memregs.width != 0 && sc->memregs.height != 0) { DPRINTF(DEBUG_INFO, ("switching to VESA mode")); sc->gc_image->vgamode = 0; } } static uint64_t pci_fbuf_read(struct pci_devinst *pi, int baridx, uint64_t offset, int size) { struct pci_fbuf_softc *sc; uint8_t *p; uint64_t value; assert(baridx == 0); sc = pi->pi_arg; if (offset + size > DMEMSZ) { printf("fbuf: read too large, offset %ld size %d\n", offset, size); return (0); } p = (uint8_t *)&sc->memregs + offset; value = 0; switch (size) { case 1: value = *p; break; case 2: value = *(uint16_t *)p; break; case 4: value = *(uint32_t *)p; break; case 8: value = *(uint64_t *)p; break; default: printf("fbuf: read unknown size %d\n", size); break; } DPRINTF(DEBUG_VERBOSE, ("fbuf rd: offset 0x%lx, size: %d, value: 0x%lx", offset, size, value)); return (value); } static void pci_fbuf_baraddr(struct pci_devinst *pi, int baridx, int enabled, uint64_t address) { struct pci_fbuf_softc *sc; int prot; if (baridx != 1) return; sc = pi->pi_arg; if (!enabled) { if (vm_munmap_memseg(pi->pi_vmctx, sc->fbaddr, FB_SIZE) != 0) EPRINTLN("pci_fbuf: munmap_memseg failed"); sc->fbaddr = 0; } else { prot = PROT_READ | PROT_WRITE; if (vm_mmap_memseg(pi->pi_vmctx, address, VM_FRAMEBUFFER, 0, FB_SIZE, prot) != 0) EPRINTLN("pci_fbuf: mmap_memseg failed"); else sc->fbaddr = address; } } static int pci_fbuf_parse_config(struct pci_fbuf_softc *sc, nvlist_t *nvl) { const char *value; char *cp; sc->rfb_wait = get_config_bool_node_default(nvl, "wait", false); /* Prefer "rfb" to "tcp". */ value = get_config_value_node(nvl, "rfb"); if (value == NULL) value = get_config_value_node(nvl, "tcp"); if (value != NULL) { /* * IPv4 -- host-ip:port * IPv6 -- [host-ip%zone]:port * XXX for now port is mandatory for IPv4. */ if (value[0] == '[') { cp = strchr(value + 1, ']'); if (cp == NULL || cp == value + 1) { EPRINTLN("fbuf: Invalid IPv6 address: \"%s\"", value); return (-1); } sc->rfb_host = strndup(value + 1, cp - (value + 1)); cp++; if (*cp == ':') { cp++; if (*cp == '\0') { EPRINTLN( "fbuf: Missing port number: \"%s\"", value); return (-1); } sc->rfb_port = atoi(cp); } else if (*cp != '\0') { EPRINTLN("fbuf: Invalid IPv6 address: \"%s\"", value); return (-1); } } else { cp = strchr(value, ':'); if (cp == NULL) { sc->rfb_port = atoi(value); } else { sc->rfb_host = strndup(value, cp - value); cp++; if (*cp == '\0') { EPRINTLN( "fbuf: Missing port number: \"%s\"", value); return (-1); } sc->rfb_port = atoi(cp); } } } #ifndef __FreeBSD__ sc->rfb_unix = get_config_value_node(nvl, "unix"); #endif value = get_config_value_node(nvl, "vga"); if (value != NULL) { if (strcmp(value, "off") == 0) { sc->vga_enabled = 0; } else if (strcmp(value, "io") == 0) { sc->vga_enabled = 1; sc->vga_full = 0; } else if (strcmp(value, "on") == 0) { sc->vga_enabled = 1; sc->vga_full = 1; } else { EPRINTLN("fbuf: Invalid vga setting: \"%s\"", value); return (-1); } } value = get_config_value_node(nvl, "w"); if (value != NULL) sc->memregs.width = strtol(value, NULL, 10); value = get_config_value_node(nvl, "h"); if (value != NULL) sc->memregs.height = strtol(value, NULL, 10); if (sc->memregs.width > COLS_MAX || sc->memregs.height > ROWS_MAX) { EPRINTLN("fbuf: max resolution is %ux%u", COLS_MAX, ROWS_MAX); return (-1); } if (sc->memregs.width < COLS_MIN || sc->memregs.height < ROWS_MIN) { EPRINTLN("fbuf: minimum resolution is %ux%u", COLS_MIN, ROWS_MIN); return (-1); } value = get_config_value_node(nvl, "password"); if (value != NULL) sc->rfb_password = strdup(value); return (0); } extern void vga_render(struct bhyvegc *gc, void *arg); static void pci_fbuf_render(struct bhyvegc *gc, void *arg) { struct pci_fbuf_softc *sc; sc = arg; if (sc->vga_full && sc->gc_image->vgamode) { /* TODO: mode switching to vga and vesa should use the special * EFI-bhyve protocol port. */ vga_render(gc, sc->vgasc); return; } if (sc->gc_width != sc->memregs.width || sc->gc_height != sc->memregs.height) { bhyvegc_resize(gc, sc->memregs.width, sc->memregs.height); sc->gc_width = sc->memregs.width; sc->gc_height = sc->memregs.height; } } static int pci_fbuf_init(struct pci_devinst *pi, nvlist_t *nvl) { int error; struct pci_fbuf_softc *sc; if (fbuf_sc != NULL) { EPRINTLN("Only one frame buffer device is allowed."); return (-1); } sc = calloc(1, sizeof(struct pci_fbuf_softc)); pi->pi_arg = sc; /* initialize config space */ pci_set_cfgdata16(pi, PCIR_DEVICE, 0x40FB); pci_set_cfgdata16(pi, PCIR_VENDOR, 0xFB5D); pci_set_cfgdata8(pi, PCIR_CLASS, PCIC_DISPLAY); pci_set_cfgdata8(pi, PCIR_SUBCLASS, PCIS_DISPLAY_VGA); sc->fb_base = vm_create_devmem(pi->pi_vmctx, VM_FRAMEBUFFER, "framebuffer", FB_SIZE); if (sc->fb_base == MAP_FAILED) { error = -1; goto done; } error = pci_emul_alloc_bar(pi, 0, PCIBAR_MEM32, DMEMSZ); assert(error == 0); error = pci_emul_alloc_bar(pi, 1, PCIBAR_MEM32, FB_SIZE); assert(error == 0); error = pci_emul_add_msicap(pi, PCI_FBUF_MSI_MSGS); assert(error == 0); sc->memregs.fbsize = FB_SIZE; sc->memregs.width = COLS_DEFAULT; sc->memregs.height = ROWS_DEFAULT; sc->memregs.depth = 32; sc->vga_enabled = 1; sc->vga_full = 0; sc->fsc_pi = pi; error = pci_fbuf_parse_config(sc, nvl); if (error != 0) goto done; /* XXX until VGA rendering is enabled */ if (sc->vga_full != 0) { EPRINTLN("pci_fbuf: VGA rendering not enabled"); #ifndef __FreeBSD__ errno = ENOTSUP; error = -1; #endif goto done; } DPRINTF(DEBUG_INFO, ("fbuf frame buffer base: %p [sz %lu]", sc->fb_base, FB_SIZE)); console_init(sc->memregs.width, sc->memregs.height, sc->fb_base); console_fb_register(pci_fbuf_render, sc); if (sc->vga_enabled) sc->vgasc = vga_init(!sc->vga_full); sc->gc_image = console_get_image(); fbuf_sc = sc; memset((void *)sc->fb_base, 0, FB_SIZE); #ifdef __FreeBSD__ error = rfb_init(sc->rfb_host, sc->rfb_port, sc->rfb_wait, sc->rfb_password); #else char *name; (void) asprintf(&name, "%s (bhyve)", get_config_value("name")); if (sc->rfb_unix != NULL) { error = rfb_init((char *)sc->rfb_unix, -1, sc->rfb_wait, sc->rfb_password, name != NULL ? name : "bhyve"); } else { error = rfb_init(sc->rfb_host, sc->rfb_port, sc->rfb_wait, sc->rfb_password, name != NULL ? name : "bhyve"); } if (error != 0) free(name); #endif done: if (error) free(sc); return (error); } static const struct pci_devemu pci_fbuf = { .pe_emu = "fbuf", .pe_init = pci_fbuf_init, .pe_barwrite = pci_fbuf_write, .pe_barread = pci_fbuf_read, .pe_baraddr = pci_fbuf_baraddr, }; PCI_EMUL_SET(pci_fbuf); /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2016 Alex Teaca * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * */ #include #include #include "pci_hda.h" #include "bhyverun.h" #include "config.h" #include "pci_emul.h" #include "hdac_reg.h" /* * HDA defines */ #define PCIR_HDCTL 0x40 #define INTEL_VENDORID 0x8086 #define HDA_INTEL_82801G 0x27d8 #define HDA_IOSS_NO 0x08 #define HDA_OSS_NO 0x04 #define HDA_ISS_NO 0x04 #define HDA_CODEC_MAX 0x0f #define HDA_LAST_OFFSET \ (0x2084 + ((HDA_ISS_NO) * 0x20) + ((HDA_OSS_NO) * 0x20)) #define HDA_CORB_ENTRY_LEN 0x04 #define HDA_RIRB_ENTRY_LEN 0x08 #define HDA_BDL_ENTRY_LEN 0x10 #define HDA_DMA_PIB_ENTRY_LEN 0x08 #define HDA_STREAM_TAGS_CNT 0x10 #define HDA_STREAM_REGS_BASE 0x80 #define HDA_STREAM_REGS_LEN 0x20 #define HDA_DMA_ACCESS_LEN (sizeof(uint32_t)) #define HDA_BDL_MAX_LEN 0x0100 #define HDAC_SDSTS_FIFORDY (1 << 5) #define HDA_RIRBSTS_IRQ_MASK (HDAC_RIRBSTS_RINTFL | HDAC_RIRBSTS_RIRBOIS) #define HDA_STATESTS_IRQ_MASK ((1 << HDA_CODEC_MAX) - 1) #define HDA_SDSTS_IRQ_MASK \ (HDAC_SDSTS_DESE | HDAC_SDSTS_FIFOE | HDAC_SDSTS_BCIS) /* * HDA data structures */ struct hda_softc; typedef void (*hda_set_reg_handler)(struct hda_softc *sc, uint32_t offset, uint32_t old); struct hda_bdle { uint32_t addrl; uint32_t addrh; uint32_t len; uint32_t ioc; } __packed; struct hda_bdle_desc { void *addr; uint8_t ioc; uint32_t len; }; struct hda_codec_cmd_ctl { const char *name; void *dma_vaddr; uint8_t run; uint16_t rp; uint16_t size; uint16_t wp; }; struct hda_stream_desc { uint8_t dir; uint8_t run; uint8_t stream; /* bp is the no. of bytes transferred in the current bdle */ uint32_t bp; /* be is the no. of bdles transferred in the bdl */ uint32_t be; uint32_t bdl_cnt; struct hda_bdle_desc bdl[HDA_BDL_MAX_LEN]; }; struct hda_softc { struct pci_devinst *pci_dev; uint32_t regs[HDA_LAST_OFFSET]; uint8_t lintr; uint8_t rirb_cnt; uint64_t wall_clock_start; struct hda_codec_cmd_ctl corb; struct hda_codec_cmd_ctl rirb; uint8_t codecs_no; struct hda_codec_inst *codecs[HDA_CODEC_MAX]; /* Base Address of the DMA Position Buffer */ void *dma_pib_vaddr; struct hda_stream_desc streams[HDA_IOSS_NO]; /* 2 tables for output and input */ uint8_t stream_map[2][HDA_STREAM_TAGS_CNT]; }; /* * HDA module function declarations */ static inline void hda_set_reg_by_offset(struct hda_softc *sc, uint32_t offset, uint32_t value); static inline uint32_t hda_get_reg_by_offset(struct hda_softc *sc, uint32_t offset); static inline void hda_set_field_by_offset(struct hda_softc *sc, uint32_t offset, uint32_t mask, uint32_t value); static struct hda_softc *hda_init(nvlist_t *nvl); static void hda_update_intr(struct hda_softc *sc); static void hda_response_interrupt(struct hda_softc *sc); static int hda_codec_constructor(struct hda_softc *sc, struct hda_codec_class *codec, const char *play, const char *rec); static struct hda_codec_class *hda_find_codec_class(const char *name); static int hda_send_command(struct hda_softc *sc, uint32_t verb); static int hda_notify_codecs(struct hda_softc *sc, uint8_t run, uint8_t stream, uint8_t dir); static void hda_reset(struct hda_softc *sc); static void hda_reset_regs(struct hda_softc *sc); static void hda_stream_reset(struct hda_softc *sc, uint8_t stream_ind); static int hda_stream_start(struct hda_softc *sc, uint8_t stream_ind); static int hda_stream_stop(struct hda_softc *sc, uint8_t stream_ind); static uint32_t hda_read(struct hda_softc *sc, uint32_t offset); static int hda_write(struct hda_softc *sc, uint32_t offset, uint8_t size, uint32_t value); static inline void hda_print_cmd_ctl_data(struct hda_codec_cmd_ctl *p); static int hda_corb_start(struct hda_softc *sc); static int hda_corb_run(struct hda_softc *sc); static int hda_rirb_start(struct hda_softc *sc); static void *hda_dma_get_vaddr(struct hda_softc *sc, uint64_t dma_paddr, size_t len); static void hda_dma_st_dword(void *dma_vaddr, uint32_t data); static uint32_t hda_dma_ld_dword(void *dma_vaddr); static inline uint8_t hda_get_stream_by_offsets(uint32_t offset, uint8_t reg_offset); static inline uint32_t hda_get_offset_stream(uint8_t stream_ind); static void hda_set_gctl(struct hda_softc *sc, uint32_t offset, uint32_t old); static void hda_set_statests(struct hda_softc *sc, uint32_t offset, uint32_t old); static void hda_set_corbwp(struct hda_softc *sc, uint32_t offset, uint32_t old); static void hda_set_corbctl(struct hda_softc *sc, uint32_t offset, uint32_t old); static void hda_set_rirbctl(struct hda_softc *sc, uint32_t offset, uint32_t old); static void hda_set_rirbsts(struct hda_softc *sc, uint32_t offset, uint32_t old); static void hda_set_dpiblbase(struct hda_softc *sc, uint32_t offset, uint32_t old); static void hda_set_sdctl(struct hda_softc *sc, uint32_t offset, uint32_t old); static void hda_set_sdctl2(struct hda_softc *sc, uint32_t offset, uint32_t old); static void hda_set_sdsts(struct hda_softc *sc, uint32_t offset, uint32_t old); static int hda_signal_state_change(struct hda_codec_inst *hci); static int hda_response(struct hda_codec_inst *hci, uint32_t response, uint8_t unsol); static int hda_transfer(struct hda_codec_inst *hci, uint8_t stream, uint8_t dir, uint8_t *buf, size_t count); static void hda_set_pib(struct hda_softc *sc, uint8_t stream_ind, uint32_t pib); static uint64_t hda_get_clock_ns(void); /* * PCI HDA function declarations */ static int pci_hda_init(struct pci_devinst *pi, nvlist_t *nvl); static void pci_hda_write(struct pci_devinst *pi, int baridx, uint64_t offset, int size, uint64_t value); static uint64_t pci_hda_read(struct pci_devinst *pi, int baridx, uint64_t offset, int size); /* * HDA global data */ static const hda_set_reg_handler hda_set_reg_table[] = { [HDAC_GCTL] = hda_set_gctl, [HDAC_STATESTS] = hda_set_statests, [HDAC_CORBWP] = hda_set_corbwp, [HDAC_CORBCTL] = hda_set_corbctl, [HDAC_RIRBCTL] = hda_set_rirbctl, [HDAC_RIRBSTS] = hda_set_rirbsts, [HDAC_DPIBLBASE] = hda_set_dpiblbase, #define HDAC_ISTREAM(n, iss, oss) \ [_HDAC_ISDCTL(n, iss, oss)] = hda_set_sdctl, \ [_HDAC_ISDCTL(n, iss, oss) + 2] = hda_set_sdctl2, \ [_HDAC_ISDSTS(n, iss, oss)] = hda_set_sdsts, \ #define HDAC_OSTREAM(n, iss, oss) \ [_HDAC_OSDCTL(n, iss, oss)] = hda_set_sdctl, \ [_HDAC_OSDCTL(n, iss, oss) + 2] = hda_set_sdctl2, \ [_HDAC_OSDSTS(n, iss, oss)] = hda_set_sdsts, \ HDAC_ISTREAM(0, HDA_ISS_NO, HDA_OSS_NO) HDAC_ISTREAM(1, HDA_ISS_NO, HDA_OSS_NO) HDAC_ISTREAM(2, HDA_ISS_NO, HDA_OSS_NO) HDAC_ISTREAM(3, HDA_ISS_NO, HDA_OSS_NO) HDAC_OSTREAM(0, HDA_ISS_NO, HDA_OSS_NO) HDAC_OSTREAM(1, HDA_ISS_NO, HDA_OSS_NO) HDAC_OSTREAM(2, HDA_ISS_NO, HDA_OSS_NO) HDAC_OSTREAM(3, HDA_ISS_NO, HDA_OSS_NO) }; static const uint16_t hda_corb_sizes[] = { [HDAC_CORBSIZE_CORBSIZE_2] = 2, [HDAC_CORBSIZE_CORBSIZE_16] = 16, [HDAC_CORBSIZE_CORBSIZE_256] = 256, [HDAC_CORBSIZE_CORBSIZE_MASK] = 0, }; static const uint16_t hda_rirb_sizes[] = { [HDAC_RIRBSIZE_RIRBSIZE_2] = 2, [HDAC_RIRBSIZE_RIRBSIZE_16] = 16, [HDAC_RIRBSIZE_RIRBSIZE_256] = 256, [HDAC_RIRBSIZE_RIRBSIZE_MASK] = 0, }; static const struct hda_ops hops = { .signal = hda_signal_state_change, .response = hda_response, .transfer = hda_transfer, }; static const struct pci_devemu pci_de_hda = { .pe_emu = "hda", .pe_init = pci_hda_init, .pe_barwrite = pci_hda_write, .pe_barread = pci_hda_read }; PCI_EMUL_SET(pci_de_hda); SET_DECLARE(hda_codec_class_set, struct hda_codec_class); #if DEBUG_HDA == 1 FILE *dbg; #endif /* * HDA module function definitions */ static inline void hda_set_reg_by_offset(struct hda_softc *sc, uint32_t offset, uint32_t value) { assert(offset < HDA_LAST_OFFSET); sc->regs[offset] = value; } static inline uint32_t hda_get_reg_by_offset(struct hda_softc *sc, uint32_t offset) { assert(offset < HDA_LAST_OFFSET); return sc->regs[offset]; } static inline void hda_set_field_by_offset(struct hda_softc *sc, uint32_t offset, uint32_t mask, uint32_t value) { uint32_t reg_value = 0; reg_value = hda_get_reg_by_offset(sc, offset); reg_value &= ~mask; reg_value |= (value & mask); hda_set_reg_by_offset(sc, offset, reg_value); } static struct hda_softc * hda_init(nvlist_t *nvl) { struct hda_softc *sc = NULL; struct hda_codec_class *codec = NULL; const char *value; char *play; char *rec; int err; #if DEBUG_HDA == 1 dbg = fopen(DEBUG_HDA_FILE, "w+"); #endif sc = calloc(1, sizeof(*sc)); if (!sc) return (NULL); hda_reset_regs(sc); /* * TODO search all configured codecs * For now we play with one single codec */ codec = hda_find_codec_class("hda_codec"); if (codec) { value = get_config_value_node(nvl, "play"); if (value == NULL) play = NULL; else play = strdup(value); value = get_config_value_node(nvl, "rec"); if (value == NULL) rec = NULL; else rec = strdup(value); DPRINTF("play: %s rec: %s", play, rec); if (play != NULL || rec != NULL) { err = hda_codec_constructor(sc, codec, play, rec); assert(!err); } free(play); free(rec); } return (sc); } static void hda_update_intr(struct hda_softc *sc) { struct pci_devinst *pi = sc->pci_dev; uint32_t intctl = hda_get_reg_by_offset(sc, HDAC_INTCTL); uint32_t intsts = 0; uint32_t sdsts = 0; uint32_t rirbsts = 0; uint32_t wakeen = 0; uint32_t statests = 0; uint32_t off = 0; int i; /* update the CIS bits */ rirbsts = hda_get_reg_by_offset(sc, HDAC_RIRBSTS); if (rirbsts & (HDAC_RIRBSTS_RINTFL | HDAC_RIRBSTS_RIRBOIS)) intsts |= HDAC_INTSTS_CIS; wakeen = hda_get_reg_by_offset(sc, HDAC_WAKEEN); statests = hda_get_reg_by_offset(sc, HDAC_STATESTS); if (statests & wakeen) intsts |= HDAC_INTSTS_CIS; /* update the SIS bits */ for (i = 0; i < HDA_IOSS_NO; i++) { off = hda_get_offset_stream(i); sdsts = hda_get_reg_by_offset(sc, off + HDAC_SDSTS); if (sdsts & HDAC_SDSTS_BCIS) intsts |= (1 << i); } /* update the GIS bit */ if (intsts) intsts |= HDAC_INTSTS_GIS; hda_set_reg_by_offset(sc, HDAC_INTSTS, intsts); if ((intctl & HDAC_INTCTL_GIE) && ((intsts & \ ~HDAC_INTSTS_GIS) & intctl)) { if (!sc->lintr) { pci_lintr_assert(pi); sc->lintr = 1; } } else { if (sc->lintr) { pci_lintr_deassert(pi); sc->lintr = 0; } } } static void hda_response_interrupt(struct hda_softc *sc) { uint8_t rirbctl = hda_get_reg_by_offset(sc, HDAC_RIRBCTL); if ((rirbctl & HDAC_RIRBCTL_RINTCTL) && sc->rirb_cnt) { sc->rirb_cnt = 0; hda_set_field_by_offset(sc, HDAC_RIRBSTS, HDAC_RIRBSTS_RINTFL, HDAC_RIRBSTS_RINTFL); hda_update_intr(sc); } } static int hda_codec_constructor(struct hda_softc *sc, struct hda_codec_class *codec, const char *play, const char *rec) { struct hda_codec_inst *hci = NULL; if (sc->codecs_no >= HDA_CODEC_MAX) return (-1); hci = calloc(1, sizeof(struct hda_codec_inst)); if (!hci) return (-1); hci->hda = sc; hci->hops = &hops; hci->cad = sc->codecs_no; hci->codec = codec; sc->codecs[sc->codecs_no++] = hci; if (!codec->init) { DPRINTF("This codec does not implement the init function"); return (-1); } return (codec->init(hci, play, rec)); } static struct hda_codec_class * hda_find_codec_class(const char *name) { struct hda_codec_class **pdpp = NULL, *pdp = NULL; SET_FOREACH(pdpp, hda_codec_class_set) { pdp = *pdpp; if (!strcmp(pdp->name, name)) { return (pdp); } } return (NULL); } static int hda_send_command(struct hda_softc *sc, uint32_t verb) { struct hda_codec_inst *hci = NULL; struct hda_codec_class *codec = NULL; uint8_t cad = (verb >> HDA_CMD_CAD_SHIFT) & 0x0f; if (cad >= sc->codecs_no) return (-1); DPRINTF("cad: 0x%x verb: 0x%x", cad, verb); hci = sc->codecs[cad]; assert(hci); codec = hci->codec; assert(codec); if (!codec->command) { DPRINTF("This codec does not implement the command function"); return (-1); } return (codec->command(hci, verb)); } static int hda_notify_codecs(struct hda_softc *sc, uint8_t run, uint8_t stream, uint8_t dir) { struct hda_codec_inst *hci = NULL; struct hda_codec_class *codec = NULL; int err; int i; /* Notify each codec */ for (i = 0; i < sc->codecs_no; i++) { hci = sc->codecs[i]; assert(hci); codec = hci->codec; assert(codec); if (codec->notify) { err = codec->notify(hci, run, stream, dir); if (!err) break; } } return (i == sc->codecs_no ? (-1) : 0); } static void hda_reset(struct hda_softc *sc) { int i; struct hda_codec_inst *hci = NULL; struct hda_codec_class *codec = NULL; hda_reset_regs(sc); /* Reset each codec */ for (i = 0; i < sc->codecs_no; i++) { hci = sc->codecs[i]; assert(hci); codec = hci->codec; assert(codec); if (codec->reset) codec->reset(hci); } sc->wall_clock_start = hda_get_clock_ns(); } static void hda_reset_regs(struct hda_softc *sc) { uint32_t off = 0; uint8_t i; DPRINTF("Reset the HDA controller registers ..."); memset(sc->regs, 0, sizeof(sc->regs)); hda_set_reg_by_offset(sc, HDAC_GCAP, HDAC_GCAP_64OK | (HDA_ISS_NO << HDAC_GCAP_ISS_SHIFT) | (HDA_OSS_NO << HDAC_GCAP_OSS_SHIFT)); hda_set_reg_by_offset(sc, HDAC_VMAJ, 0x01); hda_set_reg_by_offset(sc, HDAC_OUTPAY, 0x3c); hda_set_reg_by_offset(sc, HDAC_INPAY, 0x1d); hda_set_reg_by_offset(sc, HDAC_CORBSIZE, HDAC_CORBSIZE_CORBSZCAP_256 | HDAC_CORBSIZE_CORBSIZE_256); hda_set_reg_by_offset(sc, HDAC_RIRBSIZE, HDAC_RIRBSIZE_RIRBSZCAP_256 | HDAC_RIRBSIZE_RIRBSIZE_256); for (i = 0; i < HDA_IOSS_NO; i++) { off = hda_get_offset_stream(i); hda_set_reg_by_offset(sc, off + HDAC_SDFIFOS, HDA_FIFO_SIZE); } } static void hda_stream_reset(struct hda_softc *sc, uint8_t stream_ind) { struct hda_stream_desc *st = &sc->streams[stream_ind]; uint32_t off = hda_get_offset_stream(stream_ind); DPRINTF("Reset the HDA stream: 0x%x", stream_ind); /* Reset the Stream Descriptor registers */ memset(sc->regs + HDA_STREAM_REGS_BASE + off, 0, HDA_STREAM_REGS_LEN); /* Reset the Stream Descriptor */ memset(st, 0, sizeof(*st)); hda_set_field_by_offset(sc, off + HDAC_SDSTS, HDAC_SDSTS_FIFORDY, HDAC_SDSTS_FIFORDY); hda_set_field_by_offset(sc, off + HDAC_SDCTL0, HDAC_SDCTL_SRST, HDAC_SDCTL_SRST); } static int hda_stream_start(struct hda_softc *sc, uint8_t stream_ind) { struct hda_stream_desc *st = &sc->streams[stream_ind]; struct hda_bdle_desc *bdle_desc = NULL; struct hda_bdle *bdle = NULL; uint32_t lvi = 0; uint32_t bdl_cnt = 0; uint64_t bdpl = 0; uint64_t bdpu = 0; uint64_t bdl_paddr = 0; void *bdl_vaddr = NULL; uint32_t bdle_sz = 0; uint64_t bdle_addrl = 0; uint64_t bdle_addrh = 0; uint64_t bdle_paddr = 0; void *bdle_vaddr = NULL; uint32_t off = hda_get_offset_stream(stream_ind); uint32_t sdctl = 0; uint8_t strm = 0; uint8_t dir = 0; assert(!st->run); lvi = hda_get_reg_by_offset(sc, off + HDAC_SDLVI); bdpl = hda_get_reg_by_offset(sc, off + HDAC_SDBDPL); bdpu = hda_get_reg_by_offset(sc, off + HDAC_SDBDPU); bdl_cnt = lvi + 1; assert(bdl_cnt <= HDA_BDL_MAX_LEN); bdl_paddr = bdpl | (bdpu << 32); bdl_vaddr = hda_dma_get_vaddr(sc, bdl_paddr, HDA_BDL_ENTRY_LEN * bdl_cnt); if (!bdl_vaddr) { DPRINTF("Fail to get the guest virtual address"); return (-1); } DPRINTF("stream: 0x%x bdl_cnt: 0x%x bdl_paddr: 0x%lx", stream_ind, bdl_cnt, bdl_paddr); st->bdl_cnt = bdl_cnt; bdle = (struct hda_bdle *)bdl_vaddr; for (size_t i = 0; i < bdl_cnt; i++, bdle++) { bdle_sz = bdle->len; assert(!(bdle_sz % HDA_DMA_ACCESS_LEN)); bdle_addrl = bdle->addrl; bdle_addrh = bdle->addrh; bdle_paddr = bdle_addrl | (bdle_addrh << 32); bdle_vaddr = hda_dma_get_vaddr(sc, bdle_paddr, bdle_sz); if (!bdle_vaddr) { DPRINTF("Fail to get the guest virtual address"); return (-1); } bdle_desc = &st->bdl[i]; bdle_desc->addr = bdle_vaddr; bdle_desc->len = bdle_sz; bdle_desc->ioc = bdle->ioc; DPRINTF("bdle: 0x%zx bdle_sz: 0x%x", i, bdle_sz); } sdctl = hda_get_reg_by_offset(sc, off + HDAC_SDCTL0); strm = (sdctl >> 20) & 0x0f; dir = stream_ind >= HDA_ISS_NO; DPRINTF("strm: 0x%x, dir: 0x%x", strm, dir); sc->stream_map[dir][strm] = stream_ind; st->stream = strm; st->dir = dir; st->bp = 0; st->be = 0; hda_set_pib(sc, stream_ind, 0); st->run = 1; hda_notify_codecs(sc, 1, strm, dir); return (0); } static int hda_stream_stop(struct hda_softc *sc, uint8_t stream_ind) { struct hda_stream_desc *st = &sc->streams[stream_ind]; uint8_t strm = st->stream; uint8_t dir = st->dir; DPRINTF("stream: 0x%x, strm: 0x%x, dir: 0x%x", stream_ind, strm, dir); st->run = 0; hda_notify_codecs(sc, 0, strm, dir); return (0); } static uint32_t hda_read(struct hda_softc *sc, uint32_t offset) { if (offset == HDAC_WALCLK) return (24 * (hda_get_clock_ns() - \ sc->wall_clock_start) / 1000); return (hda_get_reg_by_offset(sc, offset)); } static int hda_write(struct hda_softc *sc, uint32_t offset, uint8_t size, uint32_t value) { uint32_t old = hda_get_reg_by_offset(sc, offset); uint32_t masks[] = {0x00000000, 0x000000ff, 0x0000ffff, 0x00ffffff, 0xffffffff}; hda_set_reg_handler set_reg_handler = NULL; if (offset < nitems(hda_set_reg_table)) set_reg_handler = hda_set_reg_table[offset]; hda_set_field_by_offset(sc, offset, masks[size], value); if (set_reg_handler) set_reg_handler(sc, offset, old); return (0); } #if DEBUG_HDA == 1 static inline void hda_print_cmd_ctl_data(struct hda_codec_cmd_ctl *p) { DPRINTF("%s size: %d", p->name, p->size); DPRINTF("%s dma_vaddr: %p", p->name, p->dma_vaddr); DPRINTF("%s wp: 0x%x", p->name, p->wp); DPRINTF("%s rp: 0x%x", p->name, p->rp); } #else static inline void hda_print_cmd_ctl_data(struct hda_codec_cmd_ctl *p __unused) {} #endif static int hda_corb_start(struct hda_softc *sc) { struct hda_codec_cmd_ctl *corb = &sc->corb; uint8_t corbsize = 0; uint64_t corblbase = 0; uint64_t corbubase = 0; uint64_t corbpaddr = 0; corb->name = "CORB"; corbsize = hda_get_reg_by_offset(sc, HDAC_CORBSIZE) & \ HDAC_CORBSIZE_CORBSIZE_MASK; corb->size = hda_corb_sizes[corbsize]; if (!corb->size) { DPRINTF("Invalid corb size"); return (-1); } corblbase = hda_get_reg_by_offset(sc, HDAC_CORBLBASE); corbubase = hda_get_reg_by_offset(sc, HDAC_CORBUBASE); corbpaddr = corblbase | (corbubase << 32); DPRINTF("CORB dma_paddr: %p", (void *)corbpaddr); corb->dma_vaddr = hda_dma_get_vaddr(sc, corbpaddr, HDA_CORB_ENTRY_LEN * corb->size); if (!corb->dma_vaddr) { DPRINTF("Fail to get the guest virtual address"); return (-1); } corb->wp = hda_get_reg_by_offset(sc, HDAC_CORBWP); corb->rp = hda_get_reg_by_offset(sc, HDAC_CORBRP); corb->run = 1; hda_print_cmd_ctl_data(corb); return (0); } static int hda_corb_run(struct hda_softc *sc) { struct hda_codec_cmd_ctl *corb = &sc->corb; uint32_t verb = 0; int err; corb->wp = hda_get_reg_by_offset(sc, HDAC_CORBWP); if (corb->wp >= corb->size) { DPRINTF("Invalid HDAC_CORBWP %u >= size %u", corb->wp, corb->size); return (-1); } while (corb->rp != corb->wp && corb->run) { corb->rp++; corb->rp %= corb->size; verb = hda_dma_ld_dword((uint8_t *)corb->dma_vaddr + HDA_CORB_ENTRY_LEN * corb->rp); err = hda_send_command(sc, verb); assert(!err); } hda_set_reg_by_offset(sc, HDAC_CORBRP, corb->rp); if (corb->run) hda_response_interrupt(sc); return (0); } static int hda_rirb_start(struct hda_softc *sc) { struct hda_codec_cmd_ctl *rirb = &sc->rirb; uint8_t rirbsize = 0; uint64_t rirblbase = 0; uint64_t rirbubase = 0; uint64_t rirbpaddr = 0; rirb->name = "RIRB"; rirbsize = hda_get_reg_by_offset(sc, HDAC_RIRBSIZE) & \ HDAC_RIRBSIZE_RIRBSIZE_MASK; rirb->size = hda_rirb_sizes[rirbsize]; if (!rirb->size) { DPRINTF("Invalid rirb size"); return (-1); } rirblbase = hda_get_reg_by_offset(sc, HDAC_RIRBLBASE); rirbubase = hda_get_reg_by_offset(sc, HDAC_RIRBUBASE); rirbpaddr = rirblbase | (rirbubase << 32); DPRINTF("RIRB dma_paddr: %p", (void *)rirbpaddr); rirb->dma_vaddr = hda_dma_get_vaddr(sc, rirbpaddr, HDA_RIRB_ENTRY_LEN * rirb->size); if (!rirb->dma_vaddr) { DPRINTF("Fail to get the guest virtual address"); return (-1); } rirb->wp = hda_get_reg_by_offset(sc, HDAC_RIRBWP); rirb->rp = 0x0000; rirb->run = 1; hda_print_cmd_ctl_data(rirb); return (0); } static void * hda_dma_get_vaddr(struct hda_softc *sc, uint64_t dma_paddr, size_t len) { struct pci_devinst *pi = sc->pci_dev; assert(pi); return (paddr_guest2host(pi->pi_vmctx, (uintptr_t)dma_paddr, len)); } static void hda_dma_st_dword(void *dma_vaddr, uint32_t data) { *(uint32_t*)dma_vaddr = data; } static uint32_t hda_dma_ld_dword(void *dma_vaddr) { return (*(uint32_t*)dma_vaddr); } static inline uint8_t hda_get_stream_by_offsets(uint32_t offset, uint8_t reg_offset) { uint8_t stream_ind = (offset - reg_offset) >> 5; assert(stream_ind < HDA_IOSS_NO); return (stream_ind); } static inline uint32_t hda_get_offset_stream(uint8_t stream_ind) { return (stream_ind << 5); } static void hda_set_gctl(struct hda_softc *sc, uint32_t offset, uint32_t old __unused) { uint32_t value = hda_get_reg_by_offset(sc, offset); if (!(value & HDAC_GCTL_CRST)) { hda_reset(sc); } } static void hda_set_statests(struct hda_softc *sc, uint32_t offset, uint32_t old) { uint32_t value = hda_get_reg_by_offset(sc, offset); hda_set_reg_by_offset(sc, offset, old); /* clear the corresponding bits written by the software (guest) */ hda_set_field_by_offset(sc, offset, value & HDA_STATESTS_IRQ_MASK, 0); hda_update_intr(sc); } static void hda_set_corbwp(struct hda_softc *sc, uint32_t offset __unused, uint32_t old __unused) { hda_corb_run(sc); } static void hda_set_corbctl(struct hda_softc *sc, uint32_t offset, uint32_t old) { uint32_t value = hda_get_reg_by_offset(sc, offset); int err; struct hda_codec_cmd_ctl *corb = NULL; if (value & HDAC_CORBCTL_CORBRUN) { if (!(old & HDAC_CORBCTL_CORBRUN)) { err = hda_corb_start(sc); assert(!err); } } else { corb = &sc->corb; memset(corb, 0, sizeof(*corb)); } hda_corb_run(sc); } static void hda_set_rirbctl(struct hda_softc *sc, uint32_t offset, uint32_t old __unused) { uint32_t value = hda_get_reg_by_offset(sc, offset); int err; struct hda_codec_cmd_ctl *rirb = NULL; if (value & HDAC_RIRBCTL_RIRBDMAEN) { err = hda_rirb_start(sc); assert(!err); } else { rirb = &sc->rirb; memset(rirb, 0, sizeof(*rirb)); } } static void hda_set_rirbsts(struct hda_softc *sc, uint32_t offset, uint32_t old) { uint32_t value = hda_get_reg_by_offset(sc, offset); hda_set_reg_by_offset(sc, offset, old); /* clear the corresponding bits written by the software (guest) */ hda_set_field_by_offset(sc, offset, value & HDA_RIRBSTS_IRQ_MASK, 0); hda_update_intr(sc); } static void hda_set_dpiblbase(struct hda_softc *sc, uint32_t offset, uint32_t old) { uint32_t value = hda_get_reg_by_offset(sc, offset); uint64_t dpiblbase = 0; uint64_t dpibubase = 0; uint64_t dpibpaddr = 0; if ((value & HDAC_DPLBASE_DPLBASE_DMAPBE) != (old & \ HDAC_DPLBASE_DPLBASE_DMAPBE)) { if (value & HDAC_DPLBASE_DPLBASE_DMAPBE) { dpiblbase = value & HDAC_DPLBASE_DPLBASE_MASK; dpibubase = hda_get_reg_by_offset(sc, HDAC_DPIBUBASE); dpibpaddr = dpiblbase | (dpibubase << 32); DPRINTF("DMA Position In Buffer dma_paddr: %p", (void *)dpibpaddr); sc->dma_pib_vaddr = hda_dma_get_vaddr(sc, dpibpaddr, HDA_DMA_PIB_ENTRY_LEN * HDA_IOSS_NO); if (!sc->dma_pib_vaddr) { DPRINTF("Fail to get the guest \ virtual address"); assert(0); } } else { DPRINTF("DMA Position In Buffer Reset"); sc->dma_pib_vaddr = NULL; } } } static void hda_set_sdctl(struct hda_softc *sc, uint32_t offset, uint32_t old) { uint8_t stream_ind = hda_get_stream_by_offsets(offset, HDAC_SDCTL0); uint32_t value = hda_get_reg_by_offset(sc, offset); int err; DPRINTF("stream_ind: 0x%x old: 0x%x value: 0x%x", stream_ind, old, value); if (value & HDAC_SDCTL_SRST) { hda_stream_reset(sc, stream_ind); } if ((value & HDAC_SDCTL_RUN) != (old & HDAC_SDCTL_RUN)) { if (value & HDAC_SDCTL_RUN) { err = hda_stream_start(sc, stream_ind); assert(!err); } else { err = hda_stream_stop(sc, stream_ind); assert(!err); } } } static void hda_set_sdctl2(struct hda_softc *sc, uint32_t offset, uint32_t old __unused) { uint32_t value = hda_get_reg_by_offset(sc, offset); hda_set_field_by_offset(sc, offset - 2, 0x00ff0000, value << 16); } static void hda_set_sdsts(struct hda_softc *sc, uint32_t offset, uint32_t old) { uint32_t value = hda_get_reg_by_offset(sc, offset); hda_set_reg_by_offset(sc, offset, old); /* clear the corresponding bits written by the software (guest) */ hda_set_field_by_offset(sc, offset, value & HDA_SDSTS_IRQ_MASK, 0); hda_update_intr(sc); } static int hda_signal_state_change(struct hda_codec_inst *hci) { struct hda_softc *sc = NULL; uint32_t sdiwake = 0; assert(hci); assert(hci->hda); DPRINTF("cad: 0x%x", hci->cad); sc = hci->hda; sdiwake = 1 << hci->cad; hda_set_field_by_offset(sc, HDAC_STATESTS, sdiwake, sdiwake); hda_update_intr(sc); return (0); } static int hda_response(struct hda_codec_inst *hci, uint32_t response, uint8_t unsol) { struct hda_softc *sc = NULL; struct hda_codec_cmd_ctl *rirb = NULL; uint32_t response_ex = 0; uint8_t rintcnt = 0; assert(hci); assert(hci->cad <= HDA_CODEC_MAX); response_ex = hci->cad | unsol; sc = hci->hda; assert(sc); rirb = &sc->rirb; if (rirb->run) { rirb->wp++; rirb->wp %= rirb->size; hda_dma_st_dword((uint8_t *)rirb->dma_vaddr + HDA_RIRB_ENTRY_LEN * rirb->wp, response); hda_dma_st_dword((uint8_t *)rirb->dma_vaddr + HDA_RIRB_ENTRY_LEN * rirb->wp + 0x04, response_ex); hda_set_reg_by_offset(sc, HDAC_RIRBWP, rirb->wp); sc->rirb_cnt++; } rintcnt = hda_get_reg_by_offset(sc, HDAC_RINTCNT); if (sc->rirb_cnt == rintcnt) hda_response_interrupt(sc); return (0); } static int hda_transfer(struct hda_codec_inst *hci, uint8_t stream, uint8_t dir, uint8_t *buf, size_t count) { struct hda_softc *sc = NULL; struct hda_stream_desc *st = NULL; struct hda_bdle_desc *bdl = NULL; struct hda_bdle_desc *bdle_desc = NULL; uint8_t stream_ind = 0; uint32_t lpib = 0; uint32_t off = 0; size_t left = 0; uint8_t irq = 0; assert(hci); assert(hci->hda); assert(buf); assert(!(count % HDA_DMA_ACCESS_LEN)); if (!stream) { DPRINTF("Invalid stream"); return (-1); } sc = hci->hda; assert(stream < HDA_STREAM_TAGS_CNT); stream_ind = sc->stream_map[dir][stream]; if (!dir) assert(stream_ind < HDA_ISS_NO); else assert(stream_ind >= HDA_ISS_NO && stream_ind < HDA_IOSS_NO); st = &sc->streams[stream_ind]; if (!st->run) { DPRINTF("Stream 0x%x stopped", stream); return (-1); } assert(st->stream == stream); off = hda_get_offset_stream(stream_ind); lpib = hda_get_reg_by_offset(sc, off + HDAC_SDLPIB); bdl = st->bdl; assert(st->be < st->bdl_cnt); assert(st->bp < bdl[st->be].len); left = count; while (left) { bdle_desc = &bdl[st->be]; if (dir) *(uint32_t *)buf = hda_dma_ld_dword( (uint8_t *)bdle_desc->addr + st->bp); else hda_dma_st_dword((uint8_t *)bdle_desc->addr + st->bp, *(uint32_t *)buf); buf += HDA_DMA_ACCESS_LEN; st->bp += HDA_DMA_ACCESS_LEN; lpib += HDA_DMA_ACCESS_LEN; left -= HDA_DMA_ACCESS_LEN; if (st->bp == bdle_desc->len) { st->bp = 0; if (bdle_desc->ioc) irq = 1; st->be++; if (st->be == st->bdl_cnt) { st->be = 0; lpib = 0; } bdle_desc = &bdl[st->be]; } } hda_set_pib(sc, stream_ind, lpib); if (irq) { hda_set_field_by_offset(sc, off + HDAC_SDSTS, HDAC_SDSTS_BCIS, HDAC_SDSTS_BCIS); hda_update_intr(sc); } return (0); } static void hda_set_pib(struct hda_softc *sc, uint8_t stream_ind, uint32_t pib) { uint32_t off = hda_get_offset_stream(stream_ind); hda_set_reg_by_offset(sc, off + HDAC_SDLPIB, pib); /* LPIB Alias */ hda_set_reg_by_offset(sc, 0x2000 + off + HDAC_SDLPIB, pib); if (sc->dma_pib_vaddr) *(uint32_t *)((uint8_t *)sc->dma_pib_vaddr + stream_ind * HDA_DMA_PIB_ENTRY_LEN) = pib; } static uint64_t hda_get_clock_ns(void) { struct timespec ts; int err; err = clock_gettime(CLOCK_MONOTONIC, &ts); assert(!err); return (ts.tv_sec * 1000000000LL + ts.tv_nsec); } /* * PCI HDA function definitions */ static int pci_hda_init(struct pci_devinst *pi, nvlist_t *nvl) { struct hda_softc *sc = NULL; assert(pi != NULL); pci_set_cfgdata16(pi, PCIR_VENDOR, INTEL_VENDORID); pci_set_cfgdata16(pi, PCIR_DEVICE, HDA_INTEL_82801G); pci_set_cfgdata8(pi, PCIR_SUBCLASS, PCIS_MULTIMEDIA_HDA); pci_set_cfgdata8(pi, PCIR_CLASS, PCIC_MULTIMEDIA); /* select the Intel HDA mode */ pci_set_cfgdata8(pi, PCIR_HDCTL, 0x01); /* allocate one BAR register for the Memory address offsets */ pci_emul_alloc_bar(pi, 0, PCIBAR_MEM32, HDA_LAST_OFFSET); /* allocate an IRQ pin for our slot */ pci_lintr_request(pi); sc = hda_init(nvl); if (!sc) return (-1); sc->pci_dev = pi; pi->pi_arg = sc; return (0); } static void pci_hda_write(struct pci_devinst *pi, int baridx, uint64_t offset, int size, uint64_t value) { struct hda_softc *sc = pi->pi_arg; int err; assert(sc); assert(baridx == 0); assert(size <= 4); DPRINTF("offset: 0x%lx value: 0x%lx", offset, value); err = hda_write(sc, offset, size, value); assert(!err); } static uint64_t pci_hda_read(struct pci_devinst *pi, int baridx, uint64_t offset, int size) { struct hda_softc *sc = pi->pi_arg; uint64_t value = 0; assert(sc); assert(baridx == 0); assert(size <= 4); value = hda_read(sc, offset); DPRINTF("offset: 0x%lx value: 0x%lx", offset, value); return (value); } /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2016 Alex Teaca * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef _HDA_EMUL_H_ #define _HDA_EMUL_H_ #include #include #include #include #include #include #include #include #include "hda_reg.h" /* * HDA Debug Log */ #if DEBUG_HDA == 1 extern FILE *dbg; #define DPRINTF(fmt, arg...) \ do {fprintf(dbg, "%s-%d: " fmt "\n", __func__, __LINE__, ##arg); \ fflush(dbg); } while (0) #ifndef DEBUG_HDA_FILE #define DEBUG_HDA_FILE "/tmp/bhyve_hda.log" #endif #else #define DPRINTF(fmt, arg...) #endif #define HDA_FIFO_SIZE 0x100 struct hda_softc; struct hda_codec_class; struct hda_codec_inst { uint8_t cad; struct hda_codec_class *codec; struct hda_softc *hda; const struct hda_ops *hops; void *priv; }; struct hda_codec_class { const char *name; int (*init)(struct hda_codec_inst *hci, const char *play, const char *rec); int (*reset)(struct hda_codec_inst *hci); int (*command)(struct hda_codec_inst *hci, uint32_t cmd_data); int (*notify)(struct hda_codec_inst *hci, uint8_t run, uint8_t stream, uint8_t dir); }; struct hda_ops { int (*signal)(struct hda_codec_inst *hci); int (*response)(struct hda_codec_inst *hci, uint32_t response, uint8_t unsol); int (*transfer)(struct hda_codec_inst *hci, uint8_t stream, uint8_t dir, uint8_t *buf, size_t count); }; #define HDA_EMUL_SET(x) DATA_SET(hda_codec_class_set, x) #endif /* _HDA_EMUL_H_ */ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2011 NetApp, Inc. * Copyright (c) 2018 Joyent, Inc. * Copyright 2021 Oxide Computer Company * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef __FreeBSD__ #include #include #include #include #include #endif #include #include "config.h" #include "pci_emul.h" #ifndef __FreeBSD__ #include "bhyverun.h" #endif #ifndef __FreeBSD__ static struct pci_hostbridge_model { const char *phm_model; uint16_t phm_vendor; uint16_t phm_device; } pci_hb_models[] = { { "amd", 0x1022, 0x7432 }, /* AMD/made-up */ { "netapp", 0x1275, 0x1275 }, /* NetApp/NetApp */ { "i440fx", 0x8086, 0x1237 }, /* Intel/82441 */ { "q35", 0x8086, 0x29b0 }, /* Intel/Q35 HB */ }; #define NUM_HB_MODELS (sizeof (pci_hb_models) / sizeof (pci_hb_models[0])) #endif static int pci_hostbridge_init(struct pci_devinst *pi, nvlist_t *nvl) { const char *value; u_int vendor, device; #ifdef __FreeBSD__ vendor = 0x1275; /* NetApp */ device = 0x1275; /* NetApp */ #else vendor = device = 0; #endif value = get_config_value_node(nvl, "vendor"); if (value != NULL) vendor = strtol(value, NULL, 0); else vendor = pci_config_read_reg(NULL, nvl, PCIR_VENDOR, 2, vendor); value = get_config_value_node(nvl, "devid"); if (value != NULL) device = strtol(value, NULL, 0); else device = pci_config_read_reg(NULL, nvl, PCIR_DEVICE, 2, device); #ifndef __FreeBSD__ const char *model = get_config_value_node(nvl, "model"); if (model != NULL && (vendor != 0 || device != 0)) { warnx("pci_hostbridge: cannot specify model and vendor/device"); return (-1); } else if ((vendor != 0 && device == 0) || (vendor == 0 && device != 0)) { warnx("pci_hostbridge: must specify both vendor and " "device for custom hostbridge"); return (-1); } if (model == NULL && vendor == 0 && device == 0) model = "netapp"; if (model != NULL) { for (uint_t i = 0; i < NUM_HB_MODELS; i++) { if (strcmp(model, pci_hb_models[i].phm_model) != 0) continue; /* found a model match */ vendor = pci_hb_models[i].phm_vendor; device = pci_hb_models[i].phm_device; break; } if (vendor == 0) { warnx("pci_hostbridge: invalid model '%s'", model); return (-1); } } /* Both i440fx and Q35 chipsets feature the concept of Programmable * Address Memory (PAM), where certain physical address ranges can be * configured to direct reads/writes to either DRAM, or to the PCI MMIO * space instead. At boot, they default to bypassing DRAM, so in order * to cheaply paper over our lack of emulation, the memory in PAM0 * (0xf0000-0xfffff, the System BIOS segment) should be zeroed. * * If this emulation is expanded in the future to truly support PAM * behavior, this hack can be removed. */ if (vendor == 0x8086 && (device == 0x1237 || device == 0x29b0)) { const uintptr_t start = 0xf0000; const size_t len = 0x10000; void *system_bios_region = paddr_guest2host(pi->pi_vmctx, start, len); assert(system_bios_region != NULL); bzero(system_bios_region, len); } #endif /* !__FreeBSD__ */ /* config space */ pci_set_cfgdata16(pi, PCIR_VENDOR, vendor); pci_set_cfgdata16(pi, PCIR_DEVICE, device); pci_set_cfgdata8(pi, PCIR_HDRTYPE, PCIM_HDRTYPE_NORMAL); pci_set_cfgdata8(pi, PCIR_CLASS, PCIC_BRIDGE); pci_set_cfgdata8(pi, PCIR_SUBCLASS, PCIS_BRIDGE_HOST); pci_emul_add_pciecap(pi, PCIEM_TYPE_ROOT_PORT); return (0); } static int pci_amd_hostbridge_legacy_config(nvlist_t *nvl, const char *opts __unused) { nvlist_t *pci_regs; pci_regs = create_relative_config_node(nvl, "pcireg"); if (pci_regs == NULL) { warnx("amd_hostbridge: failed to create pciregs node"); return (-1); } set_config_value_node(pci_regs, "vendor", "0x1022"); /* AMD */ set_config_value_node(pci_regs, "device", "0x7432"); /* made up */ return (0); } static const struct pci_devemu pci_de_amd_hostbridge = { .pe_emu = "amd_hostbridge", .pe_legacy_config = pci_amd_hostbridge_legacy_config, .pe_alias = "hostbridge", }; PCI_EMUL_SET(pci_de_amd_hostbridge); static const struct pci_devemu pci_de_hostbridge = { .pe_emu = "hostbridge", .pe_init = pci_hostbridge_init, }; PCI_EMUL_SET(pci_de_hostbridge); /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2014 Hudson River Trading LLC * Written by: John H. Baldwin * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include #include #include #include #include #include #include #include #include "acpi.h" #include "bootrom.h" #include "inout.h" #include "pci_emul.h" #include "pci_irq.h" #include "pci_lpc.h" /* * Implement an 8 pin PCI interrupt router compatible with the router * present on Intel's ICH10 chip. */ /* Fields in each PIRQ register. */ #define PIRQ_DIS 0x80 #define PIRQ_IRQ 0x0f /* Only IRQs 3-7, 9-12, and 14-15 are permitted. */ #define PERMITTED_IRQS 0xdef8 #define IRQ_PERMITTED(irq) (((1U << (irq)) & PERMITTED_IRQS) != 0) /* IRQ count to disable an IRQ. */ #define IRQ_DISABLED 0xff #define NPIRQS 8 static struct pirq { uint8_t reg; int use_count; int active_count; pthread_mutex_t lock; } pirqs[NPIRQS]; #define NIRQ_COUNTS 16 static u_char irq_counts[NIRQ_COUNTS]; static int pirq_cold = 1; /* * Returns true if this pin is enabled with a valid IRQ. Setting the * register to a reserved IRQ causes interrupts to not be asserted as * if the pin was disabled. */ static bool pirq_valid_irq(int reg) { if (reg & PIRQ_DIS) return (false); return (IRQ_PERMITTED(reg & PIRQ_IRQ)); } uint8_t pirq_read(int pin) { assert(pin > 0 && pin <= NPIRQS); return (pirqs[pin - 1].reg); } void pirq_write(struct vmctx *ctx, int pin, uint8_t val) { struct pirq *pirq; assert(pin > 0 && pin <= NPIRQS); pirq = &pirqs[pin - 1]; pthread_mutex_lock(&pirq->lock); if (pirq->reg != (val & (PIRQ_DIS | PIRQ_IRQ))) { if (pirq->active_count != 0 && pirq_valid_irq(pirq->reg)) vm_isa_deassert_irq(ctx, pirq->reg & PIRQ_IRQ, -1); pirq->reg = val & (PIRQ_DIS | PIRQ_IRQ); if (pirq->active_count != 0 && pirq_valid_irq(pirq->reg)) vm_isa_assert_irq(ctx, pirq->reg & PIRQ_IRQ, -1); } pthread_mutex_unlock(&pirq->lock); } void pci_irq_reserve(int irq) { assert(irq >= 0 && irq < NIRQ_COUNTS); assert(pirq_cold); assert(irq_counts[irq] == 0 || irq_counts[irq] == IRQ_DISABLED); irq_counts[irq] = IRQ_DISABLED; } void pci_irq_use(int irq) { assert(irq >= 0 && irq < NIRQ_COUNTS); assert(pirq_cold); assert(irq_counts[irq] != IRQ_DISABLED); irq_counts[irq]++; } void pci_irq_init(struct vmctx *ctx __unused) { int i; for (i = 0; i < NPIRQS; i++) { pirqs[i].reg = PIRQ_DIS; pirqs[i].use_count = 0; pirqs[i].active_count = 0; pthread_mutex_init(&pirqs[i].lock, NULL); } for (i = 0; i < NIRQ_COUNTS; i++) { if (IRQ_PERMITTED(i)) irq_counts[i] = 0; else irq_counts[i] = IRQ_DISABLED; } } void pci_irq_assert(struct pci_devinst *pi) { struct pirq *pirq; int pin; pin = pi->pi_lintr.pirq_pin; if (pin > 0) { assert(pin <= NPIRQS); pirq = &pirqs[pin - 1]; pthread_mutex_lock(&pirq->lock); pirq->active_count++; if (pirq->active_count == 1 && pirq_valid_irq(pirq->reg)) { vm_isa_assert_irq(pi->pi_vmctx, pirq->reg & PIRQ_IRQ, pi->pi_lintr.ioapic_irq); pthread_mutex_unlock(&pirq->lock); return; } pthread_mutex_unlock(&pirq->lock); } vm_ioapic_assert_irq(pi->pi_vmctx, pi->pi_lintr.ioapic_irq); } void pci_irq_deassert(struct pci_devinst *pi) { struct pirq *pirq; int pin; pin = pi->pi_lintr.pirq_pin; if (pin > 0) { assert(pin <= NPIRQS); pirq = &pirqs[pin - 1]; pthread_mutex_lock(&pirq->lock); pirq->active_count--; if (pirq->active_count == 0 && pirq_valid_irq(pirq->reg)) { vm_isa_deassert_irq(pi->pi_vmctx, pirq->reg & PIRQ_IRQ, pi->pi_lintr.ioapic_irq); pthread_mutex_unlock(&pirq->lock); return; } pthread_mutex_unlock(&pirq->lock); } vm_ioapic_deassert_irq(pi->pi_vmctx, pi->pi_lintr.ioapic_irq); } int pirq_alloc_pin(struct pci_devinst *pi) { struct vmctx *ctx = pi->pi_vmctx; int best_count, best_irq, best_pin, irq, pin; pirq_cold = 0; if (bootrom_boot()) { /* For external bootrom use fixed mapping. */ best_pin = (4 + pi->pi_slot + pi->pi_lintr.pin) % 8; } else { /* Find the least-used PIRQ pin. */ best_pin = 0; best_count = pirqs[0].use_count; for (pin = 1; pin < NPIRQS; pin++) { if (pirqs[pin].use_count < best_count) { best_pin = pin; best_count = pirqs[pin].use_count; } } } pirqs[best_pin].use_count++; /* Second, route this pin to an IRQ. */ if (pirqs[best_pin].reg == PIRQ_DIS) { best_irq = -1; best_count = 0; for (irq = 0; irq < NIRQ_COUNTS; irq++) { if (irq_counts[irq] == IRQ_DISABLED) continue; if (best_irq == -1 || irq_counts[irq] < best_count) { best_irq = irq; best_count = irq_counts[irq]; } } assert(best_irq >= 0); irq_counts[best_irq]++; pirqs[best_pin].reg = best_irq; vm_isa_set_irq_trigger(ctx, best_irq, LEVEL_TRIGGER); } return (best_pin + 1); } int pirq_irq(int pin) { assert(pin > 0 && pin <= NPIRQS); return (pirqs[pin - 1].reg & PIRQ_IRQ); } /* XXX: Generate $PIR table. */ static void pirq_dsdt(void) { char *irq_prs, *old; int irq, pin; irq_prs = NULL; for (irq = 0; irq < NIRQ_COUNTS; irq++) { if (!IRQ_PERMITTED(irq)) continue; if (irq_prs == NULL) asprintf(&irq_prs, "%d", irq); else { old = irq_prs; asprintf(&irq_prs, "%s,%d", old, irq); free(old); } } /* * A helper method to validate a link register's value. This * duplicates pirq_valid_irq(). */ dsdt_line(""); dsdt_line("Method (PIRV, 1, NotSerialized)"); dsdt_line("{"); dsdt_line(" If (And (Arg0, 0x%02X))", PIRQ_DIS); dsdt_line(" {"); dsdt_line(" Return (0x00)"); dsdt_line(" }"); dsdt_line(" And (Arg0, 0x%02X, Local0)", PIRQ_IRQ); dsdt_line(" If (LLess (Local0, 0x03))"); dsdt_line(" {"); dsdt_line(" Return (0x00)"); dsdt_line(" }"); dsdt_line(" If (LEqual (Local0, 0x08))"); dsdt_line(" {"); dsdt_line(" Return (0x00)"); dsdt_line(" }"); dsdt_line(" If (LEqual (Local0, 0x0D))"); dsdt_line(" {"); dsdt_line(" Return (0x00)"); dsdt_line(" }"); dsdt_line(" Return (0x01)"); dsdt_line("}"); for (pin = 0; pin < NPIRQS; pin++) { dsdt_line(""); dsdt_line("Device (LNK%c)", 'A' + pin); dsdt_line("{"); dsdt_line(" Name (_HID, EisaId (\"PNP0C0F\"))"); dsdt_line(" Name (_UID, 0x%02X)", pin + 1); dsdt_line(" Method (_STA, 0, NotSerialized)"); dsdt_line(" {"); dsdt_line(" If (PIRV (PIR%c))", 'A' + pin); dsdt_line(" {"); dsdt_line(" Return (0x0B)"); dsdt_line(" }"); dsdt_line(" Else"); dsdt_line(" {"); dsdt_line(" Return (0x09)"); dsdt_line(" }"); dsdt_line(" }"); dsdt_line(" Name (_PRS, ResourceTemplate ()"); dsdt_line(" {"); dsdt_line(" IRQ (Level, ActiveLow, Shared, )"); dsdt_line(" {%s}", irq_prs); dsdt_line(" })"); dsdt_line(" Name (CB%02X, ResourceTemplate ()", pin + 1); dsdt_line(" {"); dsdt_line(" IRQ (Level, ActiveLow, Shared, )"); dsdt_line(" {}"); dsdt_line(" })"); dsdt_line(" CreateWordField (CB%02X, 0x01, CIR%c)", pin + 1, 'A' + pin); dsdt_line(" Method (_CRS, 0, NotSerialized)"); dsdt_line(" {"); dsdt_line(" And (PIR%c, 0x%02X, Local0)", 'A' + pin, PIRQ_DIS | PIRQ_IRQ); dsdt_line(" If (PIRV (Local0))"); dsdt_line(" {"); dsdt_line(" ShiftLeft (0x01, Local0, CIR%c)", 'A' + pin); dsdt_line(" }"); dsdt_line(" Else"); dsdt_line(" {"); dsdt_line(" Store (0x00, CIR%c)", 'A' + pin); dsdt_line(" }"); dsdt_line(" Return (CB%02X)", pin + 1); dsdt_line(" }"); dsdt_line(" Method (_DIS, 0, NotSerialized)"); dsdt_line(" {"); dsdt_line(" Store (0x80, PIR%c)", 'A' + pin); dsdt_line(" }"); dsdt_line(" Method (_SRS, 1, NotSerialized)"); dsdt_line(" {"); dsdt_line(" CreateWordField (Arg0, 0x01, SIR%c)", 'A' + pin); dsdt_line(" FindSetRightBit (SIR%c, Local0)", 'A' + pin); dsdt_line(" Store (Decrement (Local0), PIR%c)", 'A' + pin); dsdt_line(" }"); dsdt_line("}"); } free(irq_prs); } LPC_DSDT(pirq_dsdt); /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2014 Hudson River Trading LLC * Written by: John H. Baldwin * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef __PCI_IRQ_H__ #define __PCI_IRQ_H__ struct pci_devinst; void pci_irq_assert(struct pci_devinst *pi); void pci_irq_deassert(struct pci_devinst *pi); void pci_irq_init(struct vmctx *ctx); void pci_irq_reserve(int irq); void pci_irq_use(int irq); int pirq_alloc_pin(struct pci_devinst *pi); int pirq_irq(int pin); uint8_t pirq_read(int pin); void pirq_write(struct vmctx *ctx, int pin, uint8_t val); #endif /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2017 Shunsuke Mie * Copyright (c) 2018 Leon Dang * Copyright (c) 2020 Chuck Tuffli * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * bhyve PCIe-NVMe device emulation. * * options: * -s ,nvme,devpath,maxq=#,qsz=#,ioslots=#,sectsz=#,ser=A-Z,eui64=#,dsm= * * accepted devpath: * /dev/blockdev * /path/to/image * ram=size_in_MiB * * maxq = max number of queues * qsz = max elements in each queue * ioslots = max number of concurrent io requests * sectsz = sector size (defaults to blockif sector size) * ser = serial number (20-chars max) * eui64 = IEEE Extended Unique Identifier (8 byte value) * dsm = DataSet Management support. Option is one of auto, enable,disable * */ /* TODO: - create async event for smart and log - intr coalesce */ #include #include #ifdef __FreeBSD__ #include #else #include "crc16.h" #endif #include #ifndef __FreeBSD__ #include #endif #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "bhyverun.h" #include "block_if.h" #include "config.h" #include "debug.h" #include "pci_emul.h" static int nvme_debug = 0; #define DPRINTF(fmt, args...) if (nvme_debug) PRINTLN(fmt, ##args) #define WPRINTF(fmt, args...) PRINTLN(fmt, ##args) /* defaults; can be overridden */ #define NVME_MSIX_BAR 4 #define NVME_IOSLOTS 8 /* The NVMe spec defines bits 13:4 in BAR0 as reserved */ #define NVME_MMIO_SPACE_MIN (1 << 14) #define NVME_QUEUES 16 #define NVME_MAX_QENTRIES 2048 /* Memory Page size Minimum reported in CAP register */ #define NVME_MPSMIN 0 /* MPSMIN converted to bytes */ #define NVME_MPSMIN_BYTES (1 << (12 + NVME_MPSMIN)) #define NVME_PRP2_ITEMS (PAGE_SIZE/sizeof(uint64_t)) #define NVME_MDTS 9 /* Note the + 1 allows for the initial descriptor to not be page aligned */ #define NVME_MAX_IOVEC ((1 << NVME_MDTS) + 1) #define NVME_MAX_DATA_SIZE ((1 << NVME_MDTS) * NVME_MPSMIN_BYTES) /* This is a synthetic status code to indicate there is no status */ #define NVME_NO_STATUS 0xffff #define NVME_COMPLETION_VALID(c) ((c).status != NVME_NO_STATUS) /* Reported temperature in Kelvin (i.e. room temperature) */ #define NVME_TEMPERATURE 296 /* helpers */ /* Convert a zero-based value into a one-based value */ #define ONE_BASED(zero) ((zero) + 1) /* Convert a one-based value into a zero-based value */ #define ZERO_BASED(one) ((one) - 1) /* Encode number of SQ's and CQ's for Set/Get Features */ #define NVME_FEATURE_NUM_QUEUES(sc) \ (ZERO_BASED((sc)->num_squeues) & 0xffff) | \ (ZERO_BASED((sc)->num_cqueues) & 0xffff) << 16 #define NVME_DOORBELL_OFFSET offsetof(struct nvme_registers, doorbell) enum nvme_controller_register_offsets { NVME_CR_CAP_LOW = 0x00, NVME_CR_CAP_HI = 0x04, NVME_CR_VS = 0x08, NVME_CR_INTMS = 0x0c, NVME_CR_INTMC = 0x10, NVME_CR_CC = 0x14, NVME_CR_CSTS = 0x1c, NVME_CR_NSSR = 0x20, NVME_CR_AQA = 0x24, NVME_CR_ASQ_LOW = 0x28, NVME_CR_ASQ_HI = 0x2c, NVME_CR_ACQ_LOW = 0x30, NVME_CR_ACQ_HI = 0x34, }; enum nvme_cmd_cdw11 { NVME_CMD_CDW11_PC = 0x0001, NVME_CMD_CDW11_IEN = 0x0002, NVME_CMD_CDW11_IV = 0xFFFF0000, }; enum nvme_copy_dir { NVME_COPY_TO_PRP, NVME_COPY_FROM_PRP, }; #define NVME_CQ_INTEN 0x01 #define NVME_CQ_INTCOAL 0x02 struct nvme_completion_queue { struct nvme_completion *qbase; pthread_mutex_t mtx; uint32_t size; uint16_t tail; /* nvme progress */ uint16_t head; /* guest progress */ uint16_t intr_vec; uint32_t intr_en; }; struct nvme_submission_queue { struct nvme_command *qbase; pthread_mutex_t mtx; uint32_t size; uint16_t head; /* nvme progress */ uint16_t tail; /* guest progress */ uint16_t cqid; /* completion queue id */ int qpriority; }; enum nvme_storage_type { NVME_STOR_BLOCKIF = 0, NVME_STOR_RAM = 1, }; struct pci_nvme_blockstore { enum nvme_storage_type type; void *ctx; uint64_t size; uint32_t sectsz; uint32_t sectsz_bits; uint64_t eui64; uint32_t deallocate:1; }; /* * Calculate the number of additional page descriptors for guest IO requests * based on the advertised Max Data Transfer (MDTS) and given the number of * default iovec's in a struct blockif_req. */ #define MDTS_PAD_SIZE \ ( NVME_MAX_IOVEC > BLOCKIF_IOV_MAX ? \ NVME_MAX_IOVEC - BLOCKIF_IOV_MAX : \ 0 ) struct pci_nvme_ioreq { struct pci_nvme_softc *sc; STAILQ_ENTRY(pci_nvme_ioreq) link; struct nvme_submission_queue *nvme_sq; uint16_t sqid; /* command information */ uint16_t opc; uint16_t cid; uint32_t nsid; uint64_t prev_gpaddr; size_t prev_size; size_t bytes; struct blockif_req io_req; struct iovec iovpadding[MDTS_PAD_SIZE]; }; enum nvme_dsm_type { /* Dataset Management bit in ONCS reflects backing storage capability */ NVME_DATASET_MANAGEMENT_AUTO, /* Unconditionally set Dataset Management bit in ONCS */ NVME_DATASET_MANAGEMENT_ENABLE, /* Unconditionally clear Dataset Management bit in ONCS */ NVME_DATASET_MANAGEMENT_DISABLE, }; struct pci_nvme_softc; struct nvme_feature_obj; typedef void (*nvme_feature_cb)(struct pci_nvme_softc *, struct nvme_feature_obj *, struct nvme_command *, struct nvme_completion *); struct nvme_feature_obj { uint32_t cdw11; nvme_feature_cb set; nvme_feature_cb get; bool namespace_specific; }; #define NVME_FID_MAX (NVME_FEAT_ENDURANCE_GROUP_EVENT_CONFIGURATION + 1) typedef enum { PCI_NVME_AE_TYPE_ERROR = 0, PCI_NVME_AE_TYPE_SMART, PCI_NVME_AE_TYPE_NOTICE, PCI_NVME_AE_TYPE_IO_CMD = 6, PCI_NVME_AE_TYPE_VENDOR = 7, PCI_NVME_AE_TYPE_MAX /* Must be last */ } pci_nvme_async_type; /* Asynchronous Event Requests */ struct pci_nvme_aer { STAILQ_ENTRY(pci_nvme_aer) link; uint16_t cid; /* Command ID of the submitted AER */ }; /** Asynchronous Event Information - Error */ typedef enum { PCI_NVME_AEI_ERROR_INVALID_DB, PCI_NVME_AEI_ERROR_INVALID_DB_VALUE, PCI_NVME_AEI_ERROR_DIAG_FAILURE, PCI_NVME_AEI_ERROR_PERSISTANT_ERR, PCI_NVME_AEI_ERROR_TRANSIENT_ERR, PCI_NVME_AEI_ERROR_FIRMWARE_LOAD_ERR, PCI_NVME_AEI_ERROR_MAX, } pci_nvme_async_event_info_error; /** Asynchronous Event Information - Notice */ typedef enum { PCI_NVME_AEI_NOTICE_NS_ATTR_CHANGED = 0, PCI_NVME_AEI_NOTICE_FW_ACTIVATION, PCI_NVME_AEI_NOTICE_TELEMETRY_CHANGE, PCI_NVME_AEI_NOTICE_ANA_CHANGE, PCI_NVME_AEI_NOTICE_PREDICT_LATENCY_CHANGE, PCI_NVME_AEI_NOTICE_LBA_STATUS_ALERT, PCI_NVME_AEI_NOTICE_ENDURANCE_GROUP_CHANGE, PCI_NVME_AEI_NOTICE_MAX, } pci_nvme_async_event_info_notice; #define PCI_NVME_AEI_NOTICE_SHIFT 8 #define PCI_NVME_AEI_NOTICE_MASK(event) (1 << (event + PCI_NVME_AEI_NOTICE_SHIFT)) /* Asynchronous Event Notifications */ struct pci_nvme_aen { pci_nvme_async_type atype; uint32_t event_data; bool posted; }; /* * By default, enable all Asynchrnous Event Notifications: * SMART / Health Critical Warnings * Namespace Attribute Notices */ #define PCI_NVME_AEN_DEFAULT_MASK 0x11f typedef enum { NVME_CNTRLTYPE_IO = 1, NVME_CNTRLTYPE_DISCOVERY = 2, NVME_CNTRLTYPE_ADMIN = 3, } pci_nvme_cntrl_type; struct pci_nvme_softc { struct pci_devinst *nsc_pi; pthread_mutex_t mtx; struct nvme_registers regs; struct nvme_namespace_data nsdata; struct nvme_controller_data ctrldata; struct nvme_error_information_entry err_log; struct nvme_health_information_page health_log; struct nvme_firmware_page fw_log; struct nvme_ns_list ns_log; struct pci_nvme_blockstore nvstore; uint16_t max_qentries; /* max entries per queue */ uint32_t max_queues; /* max number of IO SQ's or CQ's */ uint32_t num_cqueues; uint32_t num_squeues; bool num_q_is_set; /* Has host set Number of Queues */ struct pci_nvme_ioreq *ioreqs; STAILQ_HEAD(, pci_nvme_ioreq) ioreqs_free; /* free list of ioreqs */ uint32_t pending_ios; uint32_t ioslots; sem_t iosemlock; /* * Memory mapped Submission and Completion queues * Each array includes both Admin and IO queues */ struct nvme_completion_queue *compl_queues; struct nvme_submission_queue *submit_queues; struct nvme_feature_obj feat[NVME_FID_MAX]; enum nvme_dsm_type dataset_management; /* Accounting for SMART data */ __uint128_t read_data_units; __uint128_t write_data_units; __uint128_t read_commands; __uint128_t write_commands; uint32_t read_dunits_remainder; uint32_t write_dunits_remainder; STAILQ_HEAD(, pci_nvme_aer) aer_list; pthread_mutex_t aer_mtx; uint32_t aer_count; struct pci_nvme_aen aen[PCI_NVME_AE_TYPE_MAX]; pthread_t aen_tid; pthread_mutex_t aen_mtx; pthread_cond_t aen_cond; }; static void pci_nvme_cq_update(struct pci_nvme_softc *sc, struct nvme_completion_queue *cq, uint32_t cdw0, uint16_t cid, uint16_t sqid, uint16_t status); static struct pci_nvme_ioreq *pci_nvme_get_ioreq(struct pci_nvme_softc *); static void pci_nvme_release_ioreq(struct pci_nvme_softc *, struct pci_nvme_ioreq *); static void pci_nvme_io_done(struct blockif_req *, int); /* Controller Configuration utils */ #define NVME_CC_GET_EN(cc) \ NVMEV(NVME_CC_REG_EN, cc) #define NVME_CC_GET_CSS(cc) \ NVMEV(NVME_CC_REG_CSS, cc) #define NVME_CC_GET_SHN(cc) \ NVMEV(NVME_CC_REG_SHN, cc) #define NVME_CC_GET_IOSQES(cc) \ NVMEV(NVME_CC_REG_IOSQES, cc) #define NVME_CC_GET_IOCQES(cc) \ NVMEV(NVME_CC_REG_IOCQES, cc) #define NVME_CC_WRITE_MASK \ (NVMEM(NVME_CC_REG_EN) | \ NVMEM(NVME_CC_REG_IOSQES) | \ NVMEM(NVME_CC_REG_IOCQES)) #define NVME_CC_NEN_WRITE_MASK \ (NVMEM(NVME_CC_REG_CSS) | \ NVMEM(NVME_CC_REG_MPS) | \ NVMEM(NVME_CC_REG_AMS)) /* Controller Status utils */ #define NVME_CSTS_GET_RDY(sts) \ NVMEV(NVME_CSTS_REG_RDY, sts) #define NVME_CSTS_RDY (NVMEF(NVME_CSTS_REG_RDY, 1)) #define NVME_CSTS_CFS (NVMEF(NVME_CSTS_REG_CFS, 1)) /* Completion Queue status word utils */ #define NVME_STATUS_P (NVMEF(NVME_STATUS_P, 1)) #define NVME_STATUS_MASK \ (NVMEM(NVME_STATUS_SCT) | \ NVMEM(NVME_STATUS_SC)) #define NVME_ONCS_DSM NVMEM(NVME_CTRLR_DATA_ONCS_DSM) static void nvme_feature_invalid_cb(struct pci_nvme_softc *, struct nvme_feature_obj *, struct nvme_command *, struct nvme_completion *); static void nvme_feature_temperature(struct pci_nvme_softc *, struct nvme_feature_obj *, struct nvme_command *, struct nvme_completion *); static void nvme_feature_num_queues(struct pci_nvme_softc *, struct nvme_feature_obj *, struct nvme_command *, struct nvme_completion *); static void nvme_feature_iv_config(struct pci_nvme_softc *, struct nvme_feature_obj *, struct nvme_command *, struct nvme_completion *); static void nvme_feature_async_event(struct pci_nvme_softc *, struct nvme_feature_obj *, struct nvme_command *, struct nvme_completion *); static void *aen_thr(void *arg); static __inline void cpywithpad(char *dst, size_t dst_size, const char *src, char pad) { size_t len; len = strnlen(src, dst_size); memset(dst, pad, dst_size); memcpy(dst, src, len); } static __inline void pci_nvme_status_tc(uint16_t *status, uint16_t type, uint16_t code) { *status &= ~NVME_STATUS_MASK; *status |= NVMEF(NVME_STATUS_SCT, type) | NVMEF(NVME_STATUS_SC, code); } static __inline void pci_nvme_status_genc(uint16_t *status, uint16_t code) { pci_nvme_status_tc(status, NVME_SCT_GENERIC, code); } /* * Initialize the requested number or IO Submission and Completion Queues. * Admin queues are allocated implicitly. */ static void pci_nvme_init_queues(struct pci_nvme_softc *sc, uint32_t nsq, uint32_t ncq) { uint32_t i; /* * Allocate and initialize the Submission Queues */ if (nsq > NVME_QUEUES) { WPRINTF("%s: clamping number of SQ from %u to %u", __func__, nsq, NVME_QUEUES); nsq = NVME_QUEUES; } sc->num_squeues = nsq; sc->submit_queues = calloc(sc->num_squeues + 1, sizeof(struct nvme_submission_queue)); if (sc->submit_queues == NULL) { WPRINTF("%s: SQ allocation failed", __func__); sc->num_squeues = 0; } else { struct nvme_submission_queue *sq = sc->submit_queues; for (i = 0; i < sc->num_squeues + 1; i++) pthread_mutex_init(&sq[i].mtx, NULL); } /* * Allocate and initialize the Completion Queues */ if (ncq > NVME_QUEUES) { WPRINTF("%s: clamping number of CQ from %u to %u", __func__, ncq, NVME_QUEUES); ncq = NVME_QUEUES; } sc->num_cqueues = ncq; sc->compl_queues = calloc(sc->num_cqueues + 1, sizeof(struct nvme_completion_queue)); if (sc->compl_queues == NULL) { WPRINTF("%s: CQ allocation failed", __func__); sc->num_cqueues = 0; } else { struct nvme_completion_queue *cq = sc->compl_queues; for (i = 0; i < sc->num_cqueues + 1; i++) pthread_mutex_init(&cq[i].mtx, NULL); } } static void pci_nvme_init_ctrldata(struct pci_nvme_softc *sc) { struct nvme_controller_data *cd = &sc->ctrldata; int ret; cd->vid = 0xFB5D; cd->ssvid = 0x0000; cpywithpad((char *)cd->mn, sizeof(cd->mn), "bhyve-NVMe", ' '); cpywithpad((char *)cd->fr, sizeof(cd->fr), "1.0", ' '); /* Num of submission commands that we can handle at a time (2^rab) */ cd->rab = 4; /* FreeBSD OUI */ cd->ieee[0] = 0xfc; cd->ieee[1] = 0x9c; cd->ieee[2] = 0x58; cd->mic = 0; cd->mdts = NVME_MDTS; /* max data transfer size (2^mdts * CAP.MPSMIN) */ cd->ver = NVME_REV(1,4); cd->cntrltype = NVME_CNTRLTYPE_IO; cd->oacs = NVMEF(NVME_CTRLR_DATA_OACS_FORMAT, 1); cd->oaes = NVMEM(NVME_CTRLR_DATA_OAES_NS_ATTR); cd->acl = 2; cd->aerl = 4; /* Advertise 1, Read-only firmware slot */ cd->frmw = NVMEM(NVME_CTRLR_DATA_FRMW_SLOT1_RO) | NVMEF(NVME_CTRLR_DATA_FRMW_NUM_SLOTS, 1); cd->lpa = 0; /* TODO: support some simple things like SMART */ cd->elpe = 0; /* max error log page entries */ /* * Report a single power state (zero-based value) * power_state[] values are left as zero to indicate "Not reported" */ cd->npss = 0; /* Warning Composite Temperature Threshold */ cd->wctemp = 0x0157; cd->cctemp = 0x0157; /* SANICAP must not be 0 for Revision 1.4 and later NVMe Controllers */ cd->sanicap = NVMEF(NVME_CTRLR_DATA_SANICAP_NODMMAS, NVME_CTRLR_DATA_SANICAP_NODMMAS_NO); cd->sqes = NVMEF(NVME_CTRLR_DATA_SQES_MAX, 6) | NVMEF(NVME_CTRLR_DATA_SQES_MIN, 6); cd->cqes = NVMEF(NVME_CTRLR_DATA_CQES_MAX, 4) | NVMEF(NVME_CTRLR_DATA_CQES_MIN, 4); cd->nn = 1; /* number of namespaces */ cd->oncs = 0; switch (sc->dataset_management) { case NVME_DATASET_MANAGEMENT_AUTO: if (sc->nvstore.deallocate) cd->oncs |= NVME_ONCS_DSM; break; case NVME_DATASET_MANAGEMENT_ENABLE: cd->oncs |= NVME_ONCS_DSM; break; default: break; } cd->fna = NVMEM(NVME_CTRLR_DATA_FNA_FORMAT_ALL); cd->vwc = NVMEF(NVME_CTRLR_DATA_VWC_ALL, NVME_CTRLR_DATA_VWC_ALL_NO); #ifdef __FreeBSD__ ret = snprintf(cd->subnqn, sizeof(cd->subnqn), "nqn.2013-12.org.freebsd:bhyve-%s-%u-%u-%u", get_config_value("name"), sc->nsc_pi->pi_bus, sc->nsc_pi->pi_slot, sc->nsc_pi->pi_func); #else ret = snprintf((char *)cd->subnqn, sizeof (cd->subnqn), "nqn.2013-12.org.illumos:bhyve-%s-%u-%u-%u", get_config_value("name"), sc->nsc_pi->pi_bus, sc->nsc_pi->pi_slot, sc->nsc_pi->pi_func); #endif if ((ret < 0) || ((unsigned)ret > sizeof(cd->subnqn))) EPRINTLN("%s: error setting subnqn (%d)", __func__, ret); } static void pci_nvme_init_nsdata_size(struct pci_nvme_blockstore *nvstore, struct nvme_namespace_data *nd) { /* Get capacity and block size information from backing store */ nd->nsze = nvstore->size / nvstore->sectsz; nd->ncap = nd->nsze; nd->nuse = nd->nsze; } static void pci_nvme_init_nsdata(struct pci_nvme_softc *sc, struct nvme_namespace_data *nd, uint32_t nsid, struct pci_nvme_blockstore *nvstore) { pci_nvme_init_nsdata_size(nvstore, nd); if (nvstore->type == NVME_STOR_BLOCKIF) nvstore->deallocate = blockif_candelete(nvstore->ctx); nd->nlbaf = 0; /* NLBAF is a 0's based value (i.e. 1 LBA Format) */ nd->flbas = 0; /* Create an EUI-64 if user did not provide one */ if (nvstore->eui64 == 0) { char *data = NULL; uint64_t eui64 = nvstore->eui64; asprintf(&data, "%s%u%u%u", get_config_value("name"), sc->nsc_pi->pi_bus, sc->nsc_pi->pi_slot, sc->nsc_pi->pi_func); if (data != NULL) { eui64 = OUI_FREEBSD_NVME_LOW | crc16(0, data, strlen(data)); free(data); } nvstore->eui64 = (eui64 << 16) | (nsid & 0xffff); } be64enc(nd->eui64, nvstore->eui64); /* LBA data-sz = 2^lbads */ nd->lbaf[0] = NVMEF(NVME_NS_DATA_LBAF_LBADS, nvstore->sectsz_bits); } static void pci_nvme_init_logpages(struct pci_nvme_softc *sc) { __uint128_t power_cycles = 1; memset(&sc->err_log, 0, sizeof(sc->err_log)); memset(&sc->health_log, 0, sizeof(sc->health_log)); memset(&sc->fw_log, 0, sizeof(sc->fw_log)); memset(&sc->ns_log, 0, sizeof(sc->ns_log)); /* Set read/write remainder to round up according to spec */ sc->read_dunits_remainder = 999; sc->write_dunits_remainder = 999; /* Set nominal Health values checked by implementations */ sc->health_log.temperature = NVME_TEMPERATURE; sc->health_log.available_spare = 100; sc->health_log.available_spare_threshold = 10; /* Set Active Firmware Info to slot 1 */ sc->fw_log.afi = NVMEF(NVME_FIRMWARE_PAGE_AFI_SLOT, 1); memcpy(&sc->fw_log.revision[0], sc->ctrldata.fr, sizeof(sc->fw_log.revision[0])); memcpy(&sc->health_log.power_cycles, &power_cycles, sizeof(sc->health_log.power_cycles)); } static void pci_nvme_init_features(struct pci_nvme_softc *sc) { enum nvme_feature fid; for (fid = 0; fid < NVME_FID_MAX; fid++) { switch (fid) { case NVME_FEAT_ARBITRATION: case NVME_FEAT_POWER_MANAGEMENT: case NVME_FEAT_INTERRUPT_COALESCING: //XXX case NVME_FEAT_WRITE_ATOMICITY: /* Mandatory but no special handling required */ //XXX hang - case NVME_FEAT_PREDICTABLE_LATENCY_MODE_CONFIG: //XXX hang - case NVME_FEAT_HOST_BEHAVIOR_SUPPORT: // this returns a data buffer break; case NVME_FEAT_TEMPERATURE_THRESHOLD: sc->feat[fid].set = nvme_feature_temperature; break; case NVME_FEAT_ERROR_RECOVERY: sc->feat[fid].namespace_specific = true; break; case NVME_FEAT_NUMBER_OF_QUEUES: sc->feat[fid].set = nvme_feature_num_queues; break; case NVME_FEAT_INTERRUPT_VECTOR_CONFIGURATION: sc->feat[fid].set = nvme_feature_iv_config; break; case NVME_FEAT_ASYNC_EVENT_CONFIGURATION: sc->feat[fid].set = nvme_feature_async_event; /* Enable all AENs by default */ sc->feat[fid].cdw11 = PCI_NVME_AEN_DEFAULT_MASK; break; default: sc->feat[fid].set = nvme_feature_invalid_cb; sc->feat[fid].get = nvme_feature_invalid_cb; } } } static void pci_nvme_aer_reset(struct pci_nvme_softc *sc) { STAILQ_INIT(&sc->aer_list); sc->aer_count = 0; } static void pci_nvme_aer_init(struct pci_nvme_softc *sc) { pthread_mutex_init(&sc->aer_mtx, NULL); pci_nvme_aer_reset(sc); } static void pci_nvme_aer_destroy(struct pci_nvme_softc *sc) { struct pci_nvme_aer *aer = NULL; pthread_mutex_lock(&sc->aer_mtx); while (!STAILQ_EMPTY(&sc->aer_list)) { aer = STAILQ_FIRST(&sc->aer_list); STAILQ_REMOVE_HEAD(&sc->aer_list, link); free(aer); } pthread_mutex_unlock(&sc->aer_mtx); pci_nvme_aer_reset(sc); } static bool pci_nvme_aer_available(struct pci_nvme_softc *sc) { return (sc->aer_count != 0); } static bool pci_nvme_aer_limit_reached(struct pci_nvme_softc *sc) { struct nvme_controller_data *cd = &sc->ctrldata; /* AERL is a zero based value while aer_count is one's based */ return (sc->aer_count == (cd->aerl + 1U)); } /* * Add an Async Event Request * * Stores an AER to be returned later if the Controller needs to notify the * host of an event. * Note that while the NVMe spec doesn't require Controllers to return AER's * in order, this implementation does preserve the order. */ static int pci_nvme_aer_add(struct pci_nvme_softc *sc, uint16_t cid) { struct pci_nvme_aer *aer = NULL; aer = calloc(1, sizeof(struct pci_nvme_aer)); if (aer == NULL) return (-1); /* Save the Command ID for use in the completion message */ aer->cid = cid; pthread_mutex_lock(&sc->aer_mtx); sc->aer_count++; STAILQ_INSERT_TAIL(&sc->aer_list, aer, link); pthread_mutex_unlock(&sc->aer_mtx); return (0); } /* * Get an Async Event Request structure * * Returns a pointer to an AER previously submitted by the host or NULL if * no AER's exist. Caller is responsible for freeing the returned struct. */ static struct pci_nvme_aer * pci_nvme_aer_get(struct pci_nvme_softc *sc) { struct pci_nvme_aer *aer = NULL; pthread_mutex_lock(&sc->aer_mtx); aer = STAILQ_FIRST(&sc->aer_list); if (aer != NULL) { STAILQ_REMOVE_HEAD(&sc->aer_list, link); sc->aer_count--; } pthread_mutex_unlock(&sc->aer_mtx); return (aer); } static void pci_nvme_aen_reset(struct pci_nvme_softc *sc) { uint32_t atype; memset(sc->aen, 0, PCI_NVME_AE_TYPE_MAX * sizeof(struct pci_nvme_aen)); for (atype = 0; atype < PCI_NVME_AE_TYPE_MAX; atype++) { sc->aen[atype].atype = atype; } } static void pci_nvme_aen_init(struct pci_nvme_softc *sc) { char nstr[80]; pci_nvme_aen_reset(sc); pthread_mutex_init(&sc->aen_mtx, NULL); pthread_create(&sc->aen_tid, NULL, aen_thr, sc); snprintf(nstr, sizeof(nstr), "nvme-aen-%d:%d", sc->nsc_pi->pi_slot, sc->nsc_pi->pi_func); pthread_set_name_np(sc->aen_tid, nstr); } static void pci_nvme_aen_destroy(struct pci_nvme_softc *sc) { pci_nvme_aen_reset(sc); } /* Notify the AEN thread of pending work */ static void pci_nvme_aen_notify(struct pci_nvme_softc *sc) { pthread_cond_signal(&sc->aen_cond); } /* * Post an Asynchronous Event Notification */ static int32_t pci_nvme_aen_post(struct pci_nvme_softc *sc, pci_nvme_async_type atype, uint32_t event_data) { struct pci_nvme_aen *aen; if (atype >= PCI_NVME_AE_TYPE_MAX) { return(EINVAL); } pthread_mutex_lock(&sc->aen_mtx); aen = &sc->aen[atype]; /* Has the controller already posted an event of this type? */ if (aen->posted) { pthread_mutex_unlock(&sc->aen_mtx); return(EALREADY); } aen->event_data = event_data; aen->posted = true; pthread_mutex_unlock(&sc->aen_mtx); pci_nvme_aen_notify(sc); return(0); } static void pci_nvme_aen_process(struct pci_nvme_softc *sc) { struct pci_nvme_aer *aer; struct pci_nvme_aen *aen; pci_nvme_async_type atype; uint32_t mask; uint16_t status; uint8_t lid; #ifndef __FreeBSD__ lid = 0; #endif assert(pthread_mutex_isowned_np(&sc->aen_mtx)); for (atype = 0; atype < PCI_NVME_AE_TYPE_MAX; atype++) { aen = &sc->aen[atype]; /* Previous iterations may have depleted the available AER's */ if (!pci_nvme_aer_available(sc)) { DPRINTF("%s: no AER", __func__); break; } if (!aen->posted) { DPRINTF("%s: no AEN posted for atype=%#x", __func__, atype); continue; } status = NVME_SC_SUCCESS; /* Is the event masked? */ mask = sc->feat[NVME_FEAT_ASYNC_EVENT_CONFIGURATION].cdw11; DPRINTF("%s: atype=%#x mask=%#x event_data=%#x", __func__, atype, mask, aen->event_data); switch (atype) { case PCI_NVME_AE_TYPE_ERROR: lid = NVME_LOG_ERROR; break; case PCI_NVME_AE_TYPE_SMART: mask &= 0xff; if ((mask & aen->event_data) == 0) continue; lid = NVME_LOG_HEALTH_INFORMATION; break; case PCI_NVME_AE_TYPE_NOTICE: if (aen->event_data >= PCI_NVME_AEI_NOTICE_MAX) { EPRINTLN("%s unknown AEN notice type %u", __func__, aen->event_data); status = NVME_SC_INTERNAL_DEVICE_ERROR; lid = 0; break; } if ((PCI_NVME_AEI_NOTICE_MASK(aen->event_data) & mask) == 0) continue; switch (aen->event_data) { case PCI_NVME_AEI_NOTICE_NS_ATTR_CHANGED: lid = NVME_LOG_CHANGED_NAMESPACE; break; case PCI_NVME_AEI_NOTICE_FW_ACTIVATION: lid = NVME_LOG_FIRMWARE_SLOT; break; case PCI_NVME_AEI_NOTICE_TELEMETRY_CHANGE: lid = NVME_LOG_TELEMETRY_CONTROLLER_INITIATED; break; case PCI_NVME_AEI_NOTICE_ANA_CHANGE: lid = NVME_LOG_ASYMMETRIC_NAMESPACE_ACCESS; break; case PCI_NVME_AEI_NOTICE_PREDICT_LATENCY_CHANGE: lid = NVME_LOG_PREDICTABLE_LATENCY_EVENT_AGGREGATE; break; case PCI_NVME_AEI_NOTICE_LBA_STATUS_ALERT: lid = NVME_LOG_LBA_STATUS_INFORMATION; break; case PCI_NVME_AEI_NOTICE_ENDURANCE_GROUP_CHANGE: lid = NVME_LOG_ENDURANCE_GROUP_EVENT_AGGREGATE; break; default: lid = 0; } break; default: /* bad type?!? */ EPRINTLN("%s unknown AEN type %u", __func__, atype); status = NVME_SC_INTERNAL_DEVICE_ERROR; lid = 0; break; } aer = pci_nvme_aer_get(sc); assert(aer != NULL); DPRINTF("%s: CID=%#x CDW0=%#x", __func__, aer->cid, (lid << 16) | (aen->event_data << 8) | atype); pci_nvme_cq_update(sc, &sc->compl_queues[0], (lid << 16) | (aen->event_data << 8) | atype, /* cdw0 */ aer->cid, 0, /* SQID */ status); aen->event_data = 0; aen->posted = false; pci_generate_msix(sc->nsc_pi, 0); } } static void * aen_thr(void *arg) { struct pci_nvme_softc *sc; sc = arg; pthread_mutex_lock(&sc->aen_mtx); for (;;) { pci_nvme_aen_process(sc); pthread_cond_wait(&sc->aen_cond, &sc->aen_mtx); } #ifdef __FreeBSD__ /* Smatch spots unreachable code */ pthread_mutex_unlock(&sc->aen_mtx); pthread_exit(NULL); #endif return (NULL); } static void pci_nvme_reset_locked(struct pci_nvme_softc *sc) { uint32_t i; DPRINTF("%s", __func__); sc->regs.cap_lo = (ZERO_BASED(sc->max_qentries) & NVME_CAP_LO_REG_MQES_MASK) | NVMEF(NVME_CAP_LO_REG_CQR, 1) | NVMEF(NVME_CAP_LO_REG_TO, 60); sc->regs.cap_hi = NVMEF(NVME_CAP_HI_REG_CSS_NVM, 1); sc->regs.vs = NVME_REV(1,4); /* NVMe v1.4 */ sc->regs.cc = 0; assert(sc->submit_queues != NULL); for (i = 0; i < sc->num_squeues + 1; i++) { sc->submit_queues[i].qbase = NULL; sc->submit_queues[i].size = 0; sc->submit_queues[i].cqid = 0; sc->submit_queues[i].tail = 0; sc->submit_queues[i].head = 0; } assert(sc->compl_queues != NULL); for (i = 0; i < sc->num_cqueues + 1; i++) { sc->compl_queues[i].qbase = NULL; sc->compl_queues[i].size = 0; sc->compl_queues[i].tail = 0; sc->compl_queues[i].head = 0; } sc->num_q_is_set = false; pci_nvme_aer_destroy(sc); pci_nvme_aen_destroy(sc); /* * Clear CSTS.RDY last to prevent the host from enabling Controller * before cleanup completes */ sc->regs.csts = 0; } static void pci_nvme_reset(struct pci_nvme_softc *sc) { pthread_mutex_lock(&sc->mtx); pci_nvme_reset_locked(sc); pthread_mutex_unlock(&sc->mtx); } static int pci_nvme_init_controller(struct pci_nvme_softc *sc) { uint16_t acqs, asqs; DPRINTF("%s", __func__); /* * NVMe 2.0 states that "enabling a controller while this field is * cleared to 0h produces undefined results" for both ACQS and * ASQS. If zero, set CFS and do not become ready. */ asqs = ONE_BASED(NVMEV(NVME_AQA_REG_ASQS, sc->regs.aqa)); if (asqs < 2) { EPRINTLN("%s: illegal ASQS value %#x (aqa=%#x)", __func__, asqs - 1, sc->regs.aqa); sc->regs.csts |= NVME_CSTS_CFS; return (-1); } sc->submit_queues[0].size = asqs; sc->submit_queues[0].qbase = vm_map_gpa(sc->nsc_pi->pi_vmctx, sc->regs.asq, sizeof(struct nvme_command) * asqs); if (sc->submit_queues[0].qbase == NULL) { EPRINTLN("%s: ASQ vm_map_gpa(%lx) failed", __func__, sc->regs.asq); sc->regs.csts |= NVME_CSTS_CFS; return (-1); } DPRINTF("%s mapping Admin-SQ guest 0x%lx, host: %p", __func__, sc->regs.asq, sc->submit_queues[0].qbase); acqs = ONE_BASED(NVMEV(NVME_AQA_REG_ACQS, sc->regs.aqa)); if (acqs < 2) { EPRINTLN("%s: illegal ACQS value %#x (aqa=%#x)", __func__, acqs - 1, sc->regs.aqa); sc->regs.csts |= NVME_CSTS_CFS; return (-1); } sc->compl_queues[0].size = acqs; sc->compl_queues[0].qbase = vm_map_gpa(sc->nsc_pi->pi_vmctx, sc->regs.acq, sizeof(struct nvme_completion) * acqs); if (sc->compl_queues[0].qbase == NULL) { EPRINTLN("%s: ACQ vm_map_gpa(%lx) failed", __func__, sc->regs.acq); sc->regs.csts |= NVME_CSTS_CFS; return (-1); } sc->compl_queues[0].intr_en = NVME_CQ_INTEN; DPRINTF("%s mapping Admin-CQ guest 0x%lx, host: %p", __func__, sc->regs.acq, sc->compl_queues[0].qbase); return (0); } static int nvme_prp_memcpy(struct vmctx *ctx, uint64_t prp1, uint64_t prp2, uint8_t *b, size_t len, enum nvme_copy_dir dir) { uint8_t *p; size_t bytes; if (len > (8 * 1024)) { return (-1); } /* Copy from the start of prp1 to the end of the physical page */ bytes = PAGE_SIZE - (prp1 & PAGE_MASK); bytes = MIN(bytes, len); p = vm_map_gpa(ctx, prp1, bytes); if (p == NULL) { return (-1); } if (dir == NVME_COPY_TO_PRP) memcpy(p, b, bytes); else memcpy(b, p, bytes); b += bytes; len -= bytes; if (len == 0) { return (0); } len = MIN(len, PAGE_SIZE); p = vm_map_gpa(ctx, prp2, len); if (p == NULL) { return (-1); } if (dir == NVME_COPY_TO_PRP) memcpy(p, b, len); else memcpy(b, p, len); return (0); } /* * Write a Completion Queue Entry update * * Write the completion and update the doorbell value */ static void pci_nvme_cq_update(struct pci_nvme_softc *sc, struct nvme_completion_queue *cq, uint32_t cdw0, uint16_t cid, uint16_t sqid, uint16_t status) { struct nvme_submission_queue *sq = &sc->submit_queues[sqid]; struct nvme_completion *cqe; assert(cq->qbase != NULL); pthread_mutex_lock(&cq->mtx); cqe = &cq->qbase[cq->tail]; /* Flip the phase bit */ status |= (cqe->status ^ NVME_STATUS_P) & NVME_STATUS_P_MASK; cqe->cdw0 = cdw0; cqe->sqhd = sq->head; cqe->sqid = sqid; cqe->cid = cid; cqe->status = status; cq->tail++; if (cq->tail >= cq->size) { cq->tail = 0; } pthread_mutex_unlock(&cq->mtx); } static int nvme_opc_delete_io_sq(struct pci_nvme_softc* sc, struct nvme_command* command, struct nvme_completion* compl) { uint16_t qid = command->cdw10 & 0xffff; DPRINTF("%s DELETE_IO_SQ %u", __func__, qid); if (qid == 0 || qid > sc->num_squeues || (sc->submit_queues[qid].qbase == NULL)) { WPRINTF("%s NOT PERMITTED queue id %u / num_squeues %u", __func__, qid, sc->num_squeues); pci_nvme_status_tc(&compl->status, NVME_SCT_COMMAND_SPECIFIC, NVME_SC_INVALID_QUEUE_IDENTIFIER); return (1); } sc->submit_queues[qid].qbase = NULL; sc->submit_queues[qid].cqid = 0; pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS); return (1); } static int nvme_opc_create_io_sq(struct pci_nvme_softc* sc, struct nvme_command* command, struct nvme_completion* compl) { if (command->cdw11 & NVME_CMD_CDW11_PC) { uint16_t qid = command->cdw10 & 0xffff; struct nvme_submission_queue *nsq; if ((qid == 0) || (qid > sc->num_squeues) || (sc->submit_queues[qid].qbase != NULL)) { WPRINTF("%s queue index %u > num_squeues %u", __func__, qid, sc->num_squeues); pci_nvme_status_tc(&compl->status, NVME_SCT_COMMAND_SPECIFIC, NVME_SC_INVALID_QUEUE_IDENTIFIER); return (1); } nsq = &sc->submit_queues[qid]; nsq->size = ONE_BASED((command->cdw10 >> 16) & 0xffff); DPRINTF("%s size=%u (max=%u)", __func__, nsq->size, sc->max_qentries); if ((nsq->size < 2) || (nsq->size > sc->max_qentries)) { /* * Queues must specify at least two entries * NOTE: "MAXIMUM QUEUE SIZE EXCEEDED" was renamed to * "INVALID QUEUE SIZE" in the NVM Express 1.3 Spec */ pci_nvme_status_tc(&compl->status, NVME_SCT_COMMAND_SPECIFIC, NVME_SC_MAXIMUM_QUEUE_SIZE_EXCEEDED); return (1); } nsq->head = nsq->tail = 0; nsq->cqid = (command->cdw11 >> 16) & 0xffff; if ((nsq->cqid == 0) || (nsq->cqid > sc->num_cqueues)) { pci_nvme_status_tc(&compl->status, NVME_SCT_COMMAND_SPECIFIC, NVME_SC_INVALID_QUEUE_IDENTIFIER); return (1); } if (sc->compl_queues[nsq->cqid].qbase == NULL) { pci_nvme_status_tc(&compl->status, NVME_SCT_COMMAND_SPECIFIC, NVME_SC_COMPLETION_QUEUE_INVALID); return (1); } nsq->qpriority = (command->cdw11 >> 1) & 0x03; nsq->qbase = vm_map_gpa(sc->nsc_pi->pi_vmctx, command->prp1, sizeof(struct nvme_command) * (size_t)nsq->size); DPRINTF("%s sq %u size %u gaddr %p cqid %u", __func__, qid, nsq->size, nsq->qbase, nsq->cqid); pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS); DPRINTF("%s completed creating IOSQ qid %u", __func__, qid); } else { /* * Guest sent non-cont submission queue request. * This setting is unsupported by this emulation. */ WPRINTF("%s unsupported non-contig (list-based) " "create i/o submission queue", __func__); pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD); } return (1); } static int nvme_opc_delete_io_cq(struct pci_nvme_softc* sc, struct nvme_command* command, struct nvme_completion* compl) { uint16_t qid = command->cdw10 & 0xffff; uint16_t sqid; DPRINTF("%s DELETE_IO_CQ %u", __func__, qid); if (qid == 0 || qid > sc->num_cqueues || (sc->compl_queues[qid].qbase == NULL)) { WPRINTF("%s queue index %u / num_cqueues %u", __func__, qid, sc->num_cqueues); pci_nvme_status_tc(&compl->status, NVME_SCT_COMMAND_SPECIFIC, NVME_SC_INVALID_QUEUE_IDENTIFIER); return (1); } /* Deleting an Active CQ is an error */ for (sqid = 1; sqid < sc->num_squeues + 1; sqid++) if (sc->submit_queues[sqid].cqid == qid) { pci_nvme_status_tc(&compl->status, NVME_SCT_COMMAND_SPECIFIC, NVME_SC_INVALID_QUEUE_DELETION); return (1); } sc->compl_queues[qid].qbase = NULL; pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS); return (1); } static int nvme_opc_create_io_cq(struct pci_nvme_softc* sc, struct nvme_command* command, struct nvme_completion* compl) { struct nvme_completion_queue *ncq; uint16_t qid = command->cdw10 & 0xffff; /* Only support Physically Contiguous queues */ if ((command->cdw11 & NVME_CMD_CDW11_PC) == 0) { WPRINTF("%s unsupported non-contig (list-based) " "create i/o completion queue", __func__); pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD); return (1); } if ((qid == 0) || (qid > sc->num_cqueues) || (sc->compl_queues[qid].qbase != NULL)) { WPRINTF("%s queue index %u > num_cqueues %u", __func__, qid, sc->num_cqueues); pci_nvme_status_tc(&compl->status, NVME_SCT_COMMAND_SPECIFIC, NVME_SC_INVALID_QUEUE_IDENTIFIER); return (1); } ncq = &sc->compl_queues[qid]; ncq->intr_en = (command->cdw11 & NVME_CMD_CDW11_IEN) >> 1; ncq->intr_vec = (command->cdw11 >> 16) & 0xffff; if (ncq->intr_vec > (sc->max_queues + 1)) { pci_nvme_status_tc(&compl->status, NVME_SCT_COMMAND_SPECIFIC, NVME_SC_INVALID_INTERRUPT_VECTOR); return (1); } ncq->size = ONE_BASED((command->cdw10 >> 16) & 0xffff); if ((ncq->size < 2) || (ncq->size > sc->max_qentries)) { /* * Queues must specify at least two entries * NOTE: "MAXIMUM QUEUE SIZE EXCEEDED" was renamed to * "INVALID QUEUE SIZE" in the NVM Express 1.3 Spec */ pci_nvme_status_tc(&compl->status, NVME_SCT_COMMAND_SPECIFIC, NVME_SC_MAXIMUM_QUEUE_SIZE_EXCEEDED); return (1); } ncq->head = ncq->tail = 0; ncq->qbase = vm_map_gpa(sc->nsc_pi->pi_vmctx, command->prp1, sizeof(struct nvme_command) * (size_t)ncq->size); pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS); return (1); } static int nvme_opc_get_log_page(struct pci_nvme_softc* sc, struct nvme_command* command, struct nvme_completion* compl) { uint64_t logoff; uint32_t logsize; uint8_t logpage; pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS); /* * Command specifies the number of dwords to return in fields NUMDU * and NUMDL. This is a zero-based value. */ logpage = command->cdw10 & 0xFF; logsize = ((command->cdw11 << 16) | (command->cdw10 >> 16)) + 1; logsize *= sizeof(uint32_t); logoff = ((uint64_t)(command->cdw13) << 32) | command->cdw12; DPRINTF("%s log page %u offset %lu len %u", __func__, logpage, logoff, logsize); switch (logpage) { case NVME_LOG_ERROR: if (logoff >= sizeof(sc->err_log)) { pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD); break; } nvme_prp_memcpy(sc->nsc_pi->pi_vmctx, command->prp1, command->prp2, (uint8_t *)&sc->err_log + logoff, MIN(logsize, sizeof(sc->err_log) - logoff), NVME_COPY_TO_PRP); break; case NVME_LOG_HEALTH_INFORMATION: if (logoff >= sizeof(sc->health_log)) { pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD); break; } pthread_mutex_lock(&sc->mtx); memcpy(&sc->health_log.data_units_read, &sc->read_data_units, sizeof(sc->health_log.data_units_read)); memcpy(&sc->health_log.data_units_written, &sc->write_data_units, sizeof(sc->health_log.data_units_written)); memcpy(&sc->health_log.host_read_commands, &sc->read_commands, sizeof(sc->health_log.host_read_commands)); memcpy(&sc->health_log.host_write_commands, &sc->write_commands, sizeof(sc->health_log.host_write_commands)); pthread_mutex_unlock(&sc->mtx); nvme_prp_memcpy(sc->nsc_pi->pi_vmctx, command->prp1, command->prp2, (uint8_t *)&sc->health_log + logoff, MIN(logsize, sizeof(sc->health_log) - logoff), NVME_COPY_TO_PRP); break; case NVME_LOG_FIRMWARE_SLOT: if (logoff >= sizeof(sc->fw_log)) { pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD); break; } nvme_prp_memcpy(sc->nsc_pi->pi_vmctx, command->prp1, command->prp2, (uint8_t *)&sc->fw_log + logoff, MIN(logsize, sizeof(sc->fw_log) - logoff), NVME_COPY_TO_PRP); break; case NVME_LOG_CHANGED_NAMESPACE: if (logoff >= sizeof(sc->ns_log)) { pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD); break; } nvme_prp_memcpy(sc->nsc_pi->pi_vmctx, command->prp1, command->prp2, (uint8_t *)&sc->ns_log + logoff, MIN(logsize, sizeof(sc->ns_log) - logoff), NVME_COPY_TO_PRP); memset(&sc->ns_log, 0, sizeof(sc->ns_log)); break; default: DPRINTF("%s get log page %x command not supported", __func__, logpage); pci_nvme_status_tc(&compl->status, NVME_SCT_COMMAND_SPECIFIC, NVME_SC_INVALID_LOG_PAGE); } return (1); } static int nvme_opc_identify(struct pci_nvme_softc* sc, struct nvme_command* command, struct nvme_completion* compl) { void *dest; uint16_t status; DPRINTF("%s identify 0x%x nsid 0x%x", __func__, command->cdw10 & 0xFF, command->nsid); status = 0; pci_nvme_status_genc(&status, NVME_SC_SUCCESS); switch (command->cdw10 & 0xFF) { case 0x00: /* return Identify Namespace data structure */ /* Global NS only valid with NS Management */ if (command->nsid == NVME_GLOBAL_NAMESPACE_TAG) { pci_nvme_status_genc(&status, NVME_SC_INVALID_NAMESPACE_OR_FORMAT); break; } nvme_prp_memcpy(sc->nsc_pi->pi_vmctx, command->prp1, command->prp2, (uint8_t *)&sc->nsdata, sizeof(sc->nsdata), NVME_COPY_TO_PRP); break; case 0x01: /* return Identify Controller data structure */ nvme_prp_memcpy(sc->nsc_pi->pi_vmctx, command->prp1, command->prp2, (uint8_t *)&sc->ctrldata, sizeof(sc->ctrldata), NVME_COPY_TO_PRP); break; case 0x02: /* list of 1024 active NSIDs > CDW1.NSID */ dest = vm_map_gpa(sc->nsc_pi->pi_vmctx, command->prp1, sizeof(uint32_t) * 1024); /* All unused entries shall be zero */ memset(dest, 0, sizeof(uint32_t) * 1024); ((uint32_t *)dest)[0] = 1; break; case 0x03: /* list of NSID structures in CDW1.NSID, 4096 bytes */ if (command->nsid != 1) { pci_nvme_status_genc(&status, NVME_SC_INVALID_NAMESPACE_OR_FORMAT); break; } dest = vm_map_gpa(sc->nsc_pi->pi_vmctx, command->prp1, sizeof(uint32_t) * 1024); /* All bytes after the descriptor shall be zero */ memset(dest, 0, sizeof(uint32_t) * 1024); /* Return NIDT=1 (i.e. EUI64) descriptor */ ((uint8_t *)dest)[0] = 1; ((uint8_t *)dest)[1] = sizeof(uint64_t); memcpy(((uint8_t *)dest) + 4, sc->nsdata.eui64, sizeof(uint64_t)); break; case 0x13: /* * Controller list is optional but used by UNH tests. Return * a valid but empty list. */ dest = vm_map_gpa(sc->nsc_pi->pi_vmctx, command->prp1, sizeof(uint16_t) * 2048); memset(dest, 0, sizeof(uint16_t) * 2048); break; default: DPRINTF("%s unsupported identify command requested 0x%x", __func__, command->cdw10 & 0xFF); pci_nvme_status_genc(&status, NVME_SC_INVALID_FIELD); break; } compl->status = status; return (1); } static const char * nvme_fid_to_name(uint8_t fid) { const char *name; switch (fid) { case NVME_FEAT_ARBITRATION: name = "Arbitration"; break; case NVME_FEAT_POWER_MANAGEMENT: name = "Power Management"; break; case NVME_FEAT_LBA_RANGE_TYPE: name = "LBA Range Type"; break; case NVME_FEAT_TEMPERATURE_THRESHOLD: name = "Temperature Threshold"; break; case NVME_FEAT_ERROR_RECOVERY: name = "Error Recovery"; break; case NVME_FEAT_VOLATILE_WRITE_CACHE: name = "Volatile Write Cache"; break; case NVME_FEAT_NUMBER_OF_QUEUES: name = "Number of Queues"; break; case NVME_FEAT_INTERRUPT_COALESCING: name = "Interrupt Coalescing"; break; case NVME_FEAT_INTERRUPT_VECTOR_CONFIGURATION: name = "Interrupt Vector Configuration"; break; case NVME_FEAT_WRITE_ATOMICITY: name = "Write Atomicity Normal"; break; case NVME_FEAT_ASYNC_EVENT_CONFIGURATION: name = "Asynchronous Event Configuration"; break; case NVME_FEAT_AUTONOMOUS_POWER_STATE_TRANSITION: name = "Autonomous Power State Transition"; break; case NVME_FEAT_HOST_MEMORY_BUFFER: name = "Host Memory Buffer"; break; case NVME_FEAT_TIMESTAMP: name = "Timestamp"; break; case NVME_FEAT_KEEP_ALIVE_TIMER: name = "Keep Alive Timer"; break; case NVME_FEAT_HOST_CONTROLLED_THERMAL_MGMT: name = "Host Controlled Thermal Management"; break; case NVME_FEAT_NON_OP_POWER_STATE_CONFIG: name = "Non-Operation Power State Config"; break; case NVME_FEAT_READ_RECOVERY_LEVEL_CONFIG: name = "Read Recovery Level Config"; break; case NVME_FEAT_PREDICTABLE_LATENCY_MODE_CONFIG: name = "Predictable Latency Mode Config"; break; case NVME_FEAT_PREDICTABLE_LATENCY_MODE_WINDOW: name = "Predictable Latency Mode Window"; break; case NVME_FEAT_LBA_STATUS_INFORMATION_ATTRIBUTES: name = "LBA Status Information Report Interval"; break; case NVME_FEAT_HOST_BEHAVIOR_SUPPORT: name = "Host Behavior Support"; break; case NVME_FEAT_SANITIZE_CONFIG: name = "Sanitize Config"; break; case NVME_FEAT_ENDURANCE_GROUP_EVENT_CONFIGURATION: name = "Endurance Group Event Configuration"; break; case NVME_FEAT_SOFTWARE_PROGRESS_MARKER: name = "Software Progress Marker"; break; case NVME_FEAT_HOST_IDENTIFIER: name = "Host Identifier"; break; case NVME_FEAT_RESERVATION_NOTIFICATION_MASK: name = "Reservation Notification Mask"; break; case NVME_FEAT_RESERVATION_PERSISTENCE: name = "Reservation Persistence"; break; case NVME_FEAT_NAMESPACE_WRITE_PROTECTION_CONFIG: name = "Namespace Write Protection Config"; break; default: name = "Unknown"; break; } return (name); } static void nvme_feature_invalid_cb(struct pci_nvme_softc *sc __unused, struct nvme_feature_obj *feat __unused, struct nvme_command *command __unused, struct nvme_completion *compl) { pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD); } static void nvme_feature_iv_config(struct pci_nvme_softc *sc, struct nvme_feature_obj *feat __unused, struct nvme_command *command, struct nvme_completion *compl) { uint32_t i; uint32_t cdw11 = command->cdw11; uint16_t iv; bool cd; pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD); iv = cdw11 & 0xffff; cd = cdw11 & (1 << 16); if (iv > (sc->max_queues + 1)) { return; } /* No Interrupt Coalescing (i.e. not Coalescing Disable) for Admin Q */ if ((iv == 0) && !cd) return; /* Requested Interrupt Vector must be used by a CQ */ for (i = 0; i < sc->num_cqueues + 1; i++) { if (sc->compl_queues[i].intr_vec == iv) { pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS); } } } #define NVME_ASYNC_EVENT_ENDURANCE_GROUP (0x4000) static void nvme_feature_async_event(struct pci_nvme_softc *sc __unused, struct nvme_feature_obj *feat __unused, struct nvme_command *command, struct nvme_completion *compl) { if (command->cdw11 & NVME_ASYNC_EVENT_ENDURANCE_GROUP) pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD); } #define NVME_TEMP_THRESH_OVER 0 #define NVME_TEMP_THRESH_UNDER 1 static void nvme_feature_temperature(struct pci_nvme_softc *sc, struct nvme_feature_obj *feat __unused, struct nvme_command *command, struct nvme_completion *compl) { uint16_t tmpth; /* Temperature Threshold */ uint8_t tmpsel; /* Threshold Temperature Select */ uint8_t thsel; /* Threshold Type Select */ bool set_crit = false; bool report_crit; tmpth = command->cdw11 & 0xffff; tmpsel = (command->cdw11 >> 16) & 0xf; thsel = (command->cdw11 >> 20) & 0x3; DPRINTF("%s: tmpth=%#x tmpsel=%#x thsel=%#x", __func__, tmpth, tmpsel, thsel); /* Check for unsupported values */ if (((tmpsel != 0) && (tmpsel != 0xf)) || (thsel > NVME_TEMP_THRESH_UNDER)) { pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD); return; } if (((thsel == NVME_TEMP_THRESH_OVER) && (NVME_TEMPERATURE >= tmpth)) || ((thsel == NVME_TEMP_THRESH_UNDER) && (NVME_TEMPERATURE <= tmpth))) set_crit = true; pthread_mutex_lock(&sc->mtx); if (set_crit) sc->health_log.critical_warning |= NVME_CRIT_WARN_ST_TEMPERATURE; else sc->health_log.critical_warning &= ~NVME_CRIT_WARN_ST_TEMPERATURE; pthread_mutex_unlock(&sc->mtx); report_crit = sc->feat[NVME_FEAT_ASYNC_EVENT_CONFIGURATION].cdw11 & NVME_CRIT_WARN_ST_TEMPERATURE; if (set_crit && report_crit) pci_nvme_aen_post(sc, PCI_NVME_AE_TYPE_SMART, sc->health_log.critical_warning); DPRINTF("%s: set_crit=%c critical_warning=%#x status=%#x", __func__, set_crit ? 'T':'F', sc->health_log.critical_warning, compl->status); } static void nvme_feature_num_queues(struct pci_nvme_softc *sc, struct nvme_feature_obj *feat __unused, struct nvme_command *command, struct nvme_completion *compl) { uint16_t nqr; /* Number of Queues Requested */ if (sc->num_q_is_set) { WPRINTF("%s: Number of Queues already set", __func__); pci_nvme_status_genc(&compl->status, NVME_SC_COMMAND_SEQUENCE_ERROR); return; } nqr = command->cdw11 & 0xFFFF; if (nqr == 0xffff) { WPRINTF("%s: Illegal NSQR value %#x", __func__, nqr); pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD); return; } sc->num_squeues = ONE_BASED(nqr); if (sc->num_squeues > sc->max_queues) { DPRINTF("NSQR=%u is greater than max %u", sc->num_squeues, sc->max_queues); sc->num_squeues = sc->max_queues; } nqr = (command->cdw11 >> 16) & 0xFFFF; if (nqr == 0xffff) { WPRINTF("%s: Illegal NCQR value %#x", __func__, nqr); pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD); return; } sc->num_cqueues = ONE_BASED(nqr); if (sc->num_cqueues > sc->max_queues) { DPRINTF("NCQR=%u is greater than max %u", sc->num_cqueues, sc->max_queues); sc->num_cqueues = sc->max_queues; } /* Patch the command value which will be saved on callback's return */ command->cdw11 = NVME_FEATURE_NUM_QUEUES(sc); compl->cdw0 = NVME_FEATURE_NUM_QUEUES(sc); sc->num_q_is_set = true; } static int nvme_opc_set_features(struct pci_nvme_softc *sc, struct nvme_command *command, struct nvme_completion *compl) { struct nvme_feature_obj *feat; uint32_t nsid = command->nsid; uint8_t fid = NVMEV(NVME_FEAT_SET_FID, command->cdw10); bool sv = NVMEV(NVME_FEAT_SET_SV, command->cdw10); DPRINTF("%s: Feature ID 0x%x (%s)", __func__, fid, nvme_fid_to_name(fid)); if (fid >= NVME_FID_MAX) { DPRINTF("%s invalid feature 0x%x", __func__, fid); pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD); return (1); } if (sv) { pci_nvme_status_tc(&compl->status, NVME_SCT_COMMAND_SPECIFIC, NVME_SC_FEATURE_NOT_SAVEABLE); return (1); } feat = &sc->feat[fid]; if (feat->namespace_specific && (nsid == NVME_GLOBAL_NAMESPACE_TAG)) { pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD); return (1); } if (!feat->namespace_specific && !((nsid == 0) || (nsid == NVME_GLOBAL_NAMESPACE_TAG))) { pci_nvme_status_tc(&compl->status, NVME_SCT_COMMAND_SPECIFIC, NVME_SC_FEATURE_NOT_NS_SPECIFIC); return (1); } compl->cdw0 = 0; pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS); if (feat->set) feat->set(sc, feat, command, compl); else { pci_nvme_status_tc(&compl->status, NVME_SCT_COMMAND_SPECIFIC, NVME_SC_FEATURE_NOT_CHANGEABLE); return (1); } DPRINTF("%s: status=%#x cdw11=%#x", __func__, compl->status, command->cdw11); if (compl->status == NVME_SC_SUCCESS) { feat->cdw11 = command->cdw11; if ((fid == NVME_FEAT_ASYNC_EVENT_CONFIGURATION) && (command->cdw11 != 0)) pci_nvme_aen_notify(sc); } return (0); } #define NVME_FEATURES_SEL_SUPPORTED 0x3 #define NVME_FEATURES_NS_SPECIFIC (1 << 1) static int nvme_opc_get_features(struct pci_nvme_softc* sc, struct nvme_command* command, struct nvme_completion* compl) { struct nvme_feature_obj *feat; uint8_t fid = command->cdw10 & 0xFF; uint8_t sel = (command->cdw10 >> 8) & 0x7; DPRINTF("%s: Feature ID 0x%x (%s)", __func__, fid, nvme_fid_to_name(fid)); if (fid >= NVME_FID_MAX) { DPRINTF("%s invalid feature 0x%x", __func__, fid); pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD); return (1); } compl->cdw0 = 0; pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS); feat = &sc->feat[fid]; if (feat->get) { feat->get(sc, feat, command, compl); } if (compl->status == NVME_SC_SUCCESS) { if ((sel == NVME_FEATURES_SEL_SUPPORTED) && feat->namespace_specific) compl->cdw0 = NVME_FEATURES_NS_SPECIFIC; else compl->cdw0 = feat->cdw11; } return (0); } static int nvme_opc_format_nvm(struct pci_nvme_softc* sc, struct nvme_command* command, struct nvme_completion* compl) { uint8_t ses, lbaf, pi; /* Only supports Secure Erase Setting - User Data Erase */ ses = (command->cdw10 >> 9) & 0x7; if (ses > 0x1) { pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD); return (1); } /* Only supports a single LBA Format */ lbaf = command->cdw10 & 0xf; if (lbaf != 0) { pci_nvme_status_tc(&compl->status, NVME_SCT_COMMAND_SPECIFIC, NVME_SC_INVALID_FORMAT); return (1); } /* Doesn't support Protection Information */ pi = (command->cdw10 >> 5) & 0x7; if (pi != 0) { pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD); return (1); } if (sc->nvstore.type == NVME_STOR_RAM) { if (sc->nvstore.ctx) free(sc->nvstore.ctx); sc->nvstore.ctx = calloc(1, sc->nvstore.size); pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS); } else { struct pci_nvme_ioreq *req; int err; req = pci_nvme_get_ioreq(sc); if (req == NULL) { pci_nvme_status_genc(&compl->status, NVME_SC_INTERNAL_DEVICE_ERROR); WPRINTF("%s: unable to allocate IO req", __func__); return (1); } req->nvme_sq = &sc->submit_queues[0]; req->sqid = 0; req->opc = command->opc; req->cid = command->cid; req->nsid = command->nsid; req->io_req.br_offset = 0; req->io_req.br_resid = sc->nvstore.size; req->io_req.br_callback = pci_nvme_io_done; err = blockif_delete(sc->nvstore.ctx, &req->io_req); if (err) { pci_nvme_status_genc(&compl->status, NVME_SC_INTERNAL_DEVICE_ERROR); pci_nvme_release_ioreq(sc, req); } else compl->status = NVME_NO_STATUS; } return (1); } static int nvme_opc_abort(struct pci_nvme_softc *sc __unused, struct nvme_command *command, struct nvme_completion *compl) { DPRINTF("%s submission queue %u, command ID 0x%x", __func__, command->cdw10 & 0xFFFF, (command->cdw10 >> 16) & 0xFFFF); /* TODO: search for the command ID and abort it */ compl->cdw0 = 1; pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS); return (1); } static int nvme_opc_async_event_req(struct pci_nvme_softc* sc, struct nvme_command* command, struct nvme_completion* compl) { DPRINTF("%s async event request count=%u aerl=%u cid=%#x", __func__, sc->aer_count, sc->ctrldata.aerl, command->cid); /* Don't exceed the Async Event Request Limit (AERL). */ if (pci_nvme_aer_limit_reached(sc)) { pci_nvme_status_tc(&compl->status, NVME_SCT_COMMAND_SPECIFIC, NVME_SC_ASYNC_EVENT_REQUEST_LIMIT_EXCEEDED); return (1); } if (pci_nvme_aer_add(sc, command->cid)) { pci_nvme_status_tc(&compl->status, NVME_SCT_GENERIC, NVME_SC_INTERNAL_DEVICE_ERROR); return (1); } /* * Raise events when they happen based on the Set Features cmd. * These events happen async, so only set completion successful if * there is an event reflective of the request to get event. */ compl->status = NVME_NO_STATUS; pci_nvme_aen_notify(sc); return (0); } static void pci_nvme_handle_admin_cmd(struct pci_nvme_softc* sc, uint64_t value) { struct nvme_completion compl; struct nvme_command *cmd; struct nvme_submission_queue *sq; struct nvme_completion_queue *cq; uint16_t sqhead; DPRINTF("%s index %u", __func__, (uint32_t)value); sq = &sc->submit_queues[0]; cq = &sc->compl_queues[0]; pthread_mutex_lock(&sq->mtx); sqhead = sq->head; DPRINTF("sqhead %u, tail %u", sqhead, sq->tail); while (sqhead != atomic_load_acq_short(&sq->tail)) { cmd = &(sq->qbase)[sqhead]; compl.cdw0 = 0; compl.status = 0; switch (cmd->opc) { case NVME_OPC_DELETE_IO_SQ: DPRINTF("%s command DELETE_IO_SQ", __func__); nvme_opc_delete_io_sq(sc, cmd, &compl); break; case NVME_OPC_CREATE_IO_SQ: DPRINTF("%s command CREATE_IO_SQ", __func__); nvme_opc_create_io_sq(sc, cmd, &compl); break; case NVME_OPC_DELETE_IO_CQ: DPRINTF("%s command DELETE_IO_CQ", __func__); nvme_opc_delete_io_cq(sc, cmd, &compl); break; case NVME_OPC_CREATE_IO_CQ: DPRINTF("%s command CREATE_IO_CQ", __func__); nvme_opc_create_io_cq(sc, cmd, &compl); break; case NVME_OPC_GET_LOG_PAGE: DPRINTF("%s command GET_LOG_PAGE", __func__); nvme_opc_get_log_page(sc, cmd, &compl); break; case NVME_OPC_IDENTIFY: DPRINTF("%s command IDENTIFY", __func__); nvme_opc_identify(sc, cmd, &compl); break; case NVME_OPC_ABORT: DPRINTF("%s command ABORT", __func__); nvme_opc_abort(sc, cmd, &compl); break; case NVME_OPC_SET_FEATURES: DPRINTF("%s command SET_FEATURES", __func__); nvme_opc_set_features(sc, cmd, &compl); break; case NVME_OPC_GET_FEATURES: DPRINTF("%s command GET_FEATURES", __func__); nvme_opc_get_features(sc, cmd, &compl); break; case NVME_OPC_FIRMWARE_ACTIVATE: DPRINTF("%s command FIRMWARE_ACTIVATE", __func__); pci_nvme_status_tc(&compl.status, NVME_SCT_COMMAND_SPECIFIC, NVME_SC_INVALID_FIRMWARE_SLOT); break; case NVME_OPC_ASYNC_EVENT_REQUEST: DPRINTF("%s command ASYNC_EVENT_REQ", __func__); nvme_opc_async_event_req(sc, cmd, &compl); break; case NVME_OPC_FORMAT_NVM: DPRINTF("%s command FORMAT_NVM", __func__); if (NVMEV(NVME_CTRLR_DATA_OACS_FORMAT, sc->ctrldata.oacs) == 0) { pci_nvme_status_genc(&compl.status, NVME_SC_INVALID_OPCODE); break; } nvme_opc_format_nvm(sc, cmd, &compl); break; case NVME_OPC_SECURITY_SEND: case NVME_OPC_SECURITY_RECEIVE: case NVME_OPC_SANITIZE: case NVME_OPC_GET_LBA_STATUS: DPRINTF("%s command OPC=%#x (unsupported)", __func__, cmd->opc); /* Valid but unsupported opcodes */ pci_nvme_status_genc(&compl.status, NVME_SC_INVALID_FIELD); break; default: DPRINTF("%s command OPC=%#X (not implemented)", __func__, cmd->opc); pci_nvme_status_genc(&compl.status, NVME_SC_INVALID_OPCODE); } sqhead = (sqhead + 1) % sq->size; if (NVME_COMPLETION_VALID(compl)) { pci_nvme_cq_update(sc, &sc->compl_queues[0], compl.cdw0, cmd->cid, 0, /* SQID */ compl.status); } } DPRINTF("setting sqhead %u", sqhead); sq->head = sqhead; if (cq->head != cq->tail) pci_generate_msix(sc->nsc_pi, 0); pthread_mutex_unlock(&sq->mtx); } /* * Update the Write and Read statistics reported in SMART data * * NVMe defines "data unit" as thousand's of 512 byte blocks and is rounded up. * E.g. 1 data unit is 1 - 1,000 512 byte blocks. 3 data units are 2,001 - 3,000 * 512 byte blocks. Rounding up is achieved by initializing the remainder to 999. */ static void pci_nvme_stats_write_read_update(struct pci_nvme_softc *sc, uint8_t opc, size_t bytes, uint16_t status) { pthread_mutex_lock(&sc->mtx); switch (opc) { case NVME_OPC_WRITE: sc->write_commands++; if (status != NVME_SC_SUCCESS) break; sc->write_dunits_remainder += (bytes / 512); while (sc->write_dunits_remainder >= 1000) { sc->write_data_units++; sc->write_dunits_remainder -= 1000; } break; case NVME_OPC_READ: sc->read_commands++; if (status != NVME_SC_SUCCESS) break; sc->read_dunits_remainder += (bytes / 512); while (sc->read_dunits_remainder >= 1000) { sc->read_data_units++; sc->read_dunits_remainder -= 1000; } break; default: DPRINTF("%s: Invalid OPC 0x%02x for stats", __func__, opc); break; } pthread_mutex_unlock(&sc->mtx); } /* * Check if the combination of Starting LBA (slba) and number of blocks * exceeds the range of the underlying storage. * * Because NVMe specifies the SLBA in blocks as a uint64_t and blockif stores * the capacity in bytes as a uint64_t, care must be taken to avoid integer * overflow. */ static bool pci_nvme_out_of_range(struct pci_nvme_blockstore *nvstore, uint64_t slba, uint32_t nblocks) { size_t offset, bytes; /* Overflow check of multiplying Starting LBA by the sector size */ if (slba >> (64 - nvstore->sectsz_bits)) return (true); offset = slba << nvstore->sectsz_bits; bytes = nblocks << nvstore->sectsz_bits; /* Overflow check of Number of Logical Blocks */ if ((nvstore->size <= offset) || ((nvstore->size - offset) < bytes)) return (true); return (false); } static int pci_nvme_append_iov_req(struct pci_nvme_softc *sc __unused, struct pci_nvme_ioreq *req, uint64_t gpaddr, size_t size, uint64_t offset) { int iovidx; bool range_is_contiguous; if (req == NULL) return (-1); if (req->io_req.br_iovcnt == NVME_MAX_IOVEC) { return (-1); } /* * Minimize the number of IOVs by concatenating contiguous address * ranges. If the IOV count is zero, there is no previous range to * concatenate. */ if (req->io_req.br_iovcnt == 0) range_is_contiguous = false; else range_is_contiguous = (req->prev_gpaddr + req->prev_size) == gpaddr; if (range_is_contiguous) { iovidx = req->io_req.br_iovcnt - 1; req->io_req.br_iov[iovidx].iov_base = paddr_guest2host(req->sc->nsc_pi->pi_vmctx, req->prev_gpaddr, size); if (req->io_req.br_iov[iovidx].iov_base == NULL) return (-1); req->prev_size += size; req->io_req.br_resid += size; req->io_req.br_iov[iovidx].iov_len = req->prev_size; } else { iovidx = req->io_req.br_iovcnt; if (iovidx == 0) { req->io_req.br_offset = offset; req->io_req.br_resid = 0; req->io_req.br_param = req; } req->io_req.br_iov[iovidx].iov_base = paddr_guest2host(req->sc->nsc_pi->pi_vmctx, gpaddr, size); if (req->io_req.br_iov[iovidx].iov_base == NULL) return (-1); req->io_req.br_iov[iovidx].iov_len = size; req->prev_gpaddr = gpaddr; req->prev_size = size; req->io_req.br_resid += size; req->io_req.br_iovcnt++; } return (0); } static void pci_nvme_set_completion(struct pci_nvme_softc *sc, struct nvme_submission_queue *sq, int sqid, uint16_t cid, uint16_t status) { struct nvme_completion_queue *cq = &sc->compl_queues[sq->cqid]; DPRINTF("%s sqid %d cqid %u cid %u status: 0x%x 0x%x", __func__, sqid, sq->cqid, cid, NVME_STATUS_GET_SCT(status), NVME_STATUS_GET_SC(status)); pci_nvme_cq_update(sc, cq, 0, cid, sqid, status); if (cq->head != cq->tail) { if (cq->intr_en & NVME_CQ_INTEN) { pci_generate_msix(sc->nsc_pi, cq->intr_vec); } else { DPRINTF("%s: CQ%u interrupt disabled", __func__, sq->cqid); } } } static void pci_nvme_release_ioreq(struct pci_nvme_softc *sc, struct pci_nvme_ioreq *req) { req->sc = NULL; req->nvme_sq = NULL; req->sqid = 0; pthread_mutex_lock(&sc->mtx); STAILQ_INSERT_TAIL(&sc->ioreqs_free, req, link); sc->pending_ios--; /* when no more IO pending, can set to ready if device reset/enabled */ if (sc->pending_ios == 0 && NVME_CC_GET_EN(sc->regs.cc) && !(NVME_CSTS_GET_RDY(sc->regs.csts))) sc->regs.csts |= NVME_CSTS_RDY; pthread_mutex_unlock(&sc->mtx); sem_post(&sc->iosemlock); } static struct pci_nvme_ioreq * pci_nvme_get_ioreq(struct pci_nvme_softc *sc) { struct pci_nvme_ioreq *req = NULL; sem_wait(&sc->iosemlock); pthread_mutex_lock(&sc->mtx); req = STAILQ_FIRST(&sc->ioreqs_free); assert(req != NULL); STAILQ_REMOVE_HEAD(&sc->ioreqs_free, link); req->sc = sc; sc->pending_ios++; pthread_mutex_unlock(&sc->mtx); req->io_req.br_iovcnt = 0; req->io_req.br_offset = 0; req->io_req.br_resid = 0; req->io_req.br_param = req; req->prev_gpaddr = 0; req->prev_size = 0; return req; } static void pci_nvme_io_done(struct blockif_req *br, int err) { struct pci_nvme_ioreq *req = br->br_param; struct nvme_submission_queue *sq = req->nvme_sq; uint16_t code, status; DPRINTF("%s error %d %s", __func__, err, strerror(err)); /* TODO return correct error */ code = err ? NVME_SC_DATA_TRANSFER_ERROR : NVME_SC_SUCCESS; status = 0; pci_nvme_status_genc(&status, code); pci_nvme_set_completion(req->sc, sq, req->sqid, req->cid, status); pci_nvme_stats_write_read_update(req->sc, req->opc, req->bytes, status); pci_nvme_release_ioreq(req->sc, req); } /* * Implements the Flush command. The specification states: * If a volatile write cache is not present, Flush commands complete * successfully and have no effect * in the description of the Volatile Write Cache (VWC) field of the Identify * Controller data. Therefore, set status to Success if the command is * not supported (i.e. RAM or as indicated by the blockif). */ static bool nvme_opc_flush(struct pci_nvme_softc *sc __unused, struct nvme_command *cmd __unused, struct pci_nvme_blockstore *nvstore, struct pci_nvme_ioreq *req, uint16_t *status) { bool pending = false; if (nvstore->type == NVME_STOR_RAM) { pci_nvme_status_genc(status, NVME_SC_SUCCESS); } else { int err; req->io_req.br_callback = pci_nvme_io_done; err = blockif_flush(nvstore->ctx, &req->io_req); switch (err) { case 0: pending = true; break; case EOPNOTSUPP: pci_nvme_status_genc(status, NVME_SC_SUCCESS); break; default: pci_nvme_status_genc(status, NVME_SC_INTERNAL_DEVICE_ERROR); } } return (pending); } static uint16_t nvme_write_read_ram(struct pci_nvme_softc *sc, struct pci_nvme_blockstore *nvstore, uint64_t prp1, uint64_t prp2, size_t offset, uint64_t bytes, bool is_write) { uint8_t *buf = nvstore->ctx; enum nvme_copy_dir dir; uint16_t status; if (is_write) dir = NVME_COPY_TO_PRP; else dir = NVME_COPY_FROM_PRP; status = 0; if (nvme_prp_memcpy(sc->nsc_pi->pi_vmctx, prp1, prp2, buf + offset, bytes, dir)) pci_nvme_status_genc(&status, NVME_SC_DATA_TRANSFER_ERROR); else pci_nvme_status_genc(&status, NVME_SC_SUCCESS); return (status); } static uint16_t nvme_write_read_blockif(struct pci_nvme_softc *sc, struct pci_nvme_blockstore *nvstore, struct pci_nvme_ioreq *req, uint64_t prp1, uint64_t prp2, size_t offset, uint64_t bytes, bool is_write) { uint64_t size; int err; uint16_t status = NVME_NO_STATUS; size = MIN(PAGE_SIZE - (prp1 % PAGE_SIZE), bytes); if (pci_nvme_append_iov_req(sc, req, prp1, size, offset)) { err = -1; goto out; } offset += size; bytes -= size; if (bytes == 0) { ; } else if (bytes <= PAGE_SIZE) { size = bytes; if (pci_nvme_append_iov_req(sc, req, prp2, size, offset)) { err = -1; goto out; } } else { void *vmctx = sc->nsc_pi->pi_vmctx; uint64_t *prp_list = &prp2; uint64_t *last = prp_list; /* PRP2 is pointer to a physical region page list */ while (bytes) { /* Last entry in list points to the next list */ if ((prp_list == last) && (bytes > PAGE_SIZE)) { uint64_t prp = *prp_list; prp_list = paddr_guest2host(vmctx, prp, PAGE_SIZE - (prp % PAGE_SIZE)); if (prp_list == NULL) { err = -1; goto out; } last = prp_list + (NVME_PRP2_ITEMS - 1); } size = MIN(bytes, PAGE_SIZE); if (pci_nvme_append_iov_req(sc, req, *prp_list, size, offset)) { err = -1; goto out; } offset += size; bytes -= size; prp_list++; } } req->io_req.br_callback = pci_nvme_io_done; if (is_write) err = blockif_write(nvstore->ctx, &req->io_req); else err = blockif_read(nvstore->ctx, &req->io_req); out: if (err) pci_nvme_status_genc(&status, NVME_SC_DATA_TRANSFER_ERROR); return (status); } static bool nvme_opc_write_read(struct pci_nvme_softc *sc, struct nvme_command *cmd, struct pci_nvme_blockstore *nvstore, struct pci_nvme_ioreq *req, uint16_t *status) { uint64_t lba, nblocks, bytes; size_t offset; bool is_write = cmd->opc == NVME_OPC_WRITE; bool pending = false; lba = ((uint64_t)cmd->cdw11 << 32) | cmd->cdw10; nblocks = (cmd->cdw12 & 0xFFFF) + 1; bytes = nblocks << nvstore->sectsz_bits; if (bytes > NVME_MAX_DATA_SIZE) { WPRINTF("%s command would exceed MDTS", __func__); pci_nvme_status_genc(status, NVME_SC_INVALID_FIELD); goto out; } if (pci_nvme_out_of_range(nvstore, lba, nblocks)) { WPRINTF("%s command would exceed LBA range(slba=%#lx nblocks=%#lx)", __func__, lba, nblocks); pci_nvme_status_genc(status, NVME_SC_LBA_OUT_OF_RANGE); goto out; } offset = lba << nvstore->sectsz_bits; req->bytes = bytes; req->io_req.br_offset = lba; /* PRP bits 1:0 must be zero */ cmd->prp1 &= ~0x3UL; cmd->prp2 &= ~0x3UL; if (nvstore->type == NVME_STOR_RAM) { *status = nvme_write_read_ram(sc, nvstore, cmd->prp1, cmd->prp2, offset, bytes, is_write); } else { *status = nvme_write_read_blockif(sc, nvstore, req, cmd->prp1, cmd->prp2, offset, bytes, is_write); if (*status == NVME_NO_STATUS) pending = true; } out: if (!pending) pci_nvme_stats_write_read_update(sc, cmd->opc, bytes, *status); return (pending); } static void pci_nvme_dealloc_sm(struct blockif_req *br, int err) { struct pci_nvme_ioreq *req = br->br_param; struct pci_nvme_softc *sc = req->sc; bool done = true; uint16_t status; status = 0; if (err) { pci_nvme_status_genc(&status, NVME_SC_INTERNAL_DEVICE_ERROR); } else if ((req->prev_gpaddr + 1) == (req->prev_size)) { pci_nvme_status_genc(&status, NVME_SC_SUCCESS); } else { struct iovec *iov = req->io_req.br_iov; req->prev_gpaddr++; iov += req->prev_gpaddr; /* The iov_* values already include the sector size */ req->io_req.br_offset = (off_t)iov->iov_base; req->io_req.br_resid = iov->iov_len; if (blockif_delete(sc->nvstore.ctx, &req->io_req)) { pci_nvme_status_genc(&status, NVME_SC_INTERNAL_DEVICE_ERROR); } else done = false; } if (done) { pci_nvme_set_completion(sc, req->nvme_sq, req->sqid, req->cid, status); pci_nvme_release_ioreq(sc, req); } } static bool nvme_opc_dataset_mgmt(struct pci_nvme_softc *sc, struct nvme_command *cmd, struct pci_nvme_blockstore *nvstore, struct pci_nvme_ioreq *req, uint16_t *status) { struct nvme_dsm_range *range = NULL; uint32_t nr, r, non_zero, dr; int err; bool pending = false; if ((sc->ctrldata.oncs & NVME_ONCS_DSM) == 0) { pci_nvme_status_genc(status, NVME_SC_INVALID_OPCODE); goto out; } nr = cmd->cdw10 & 0xff; /* copy locally because a range entry could straddle PRPs */ #ifdef __FreeBSD__ range = calloc(1, NVME_MAX_DSM_TRIM); #else _Static_assert(NVME_MAX_DSM_TRIM % sizeof(struct nvme_dsm_range) == 0, "NVME_MAX_DSM_TRIM is not a multiple of struct size"); range = calloc(NVME_MAX_DSM_TRIM / sizeof (*range), sizeof (*range)); #endif if (range == NULL) { pci_nvme_status_genc(status, NVME_SC_INTERNAL_DEVICE_ERROR); goto out; } nvme_prp_memcpy(sc->nsc_pi->pi_vmctx, cmd->prp1, cmd->prp2, (uint8_t *)range, NVME_MAX_DSM_TRIM, NVME_COPY_FROM_PRP); /* Check for invalid ranges and the number of non-zero lengths */ non_zero = 0; for (r = 0; r <= nr; r++) { if (pci_nvme_out_of_range(nvstore, range[r].starting_lba, range[r].length)) { pci_nvme_status_genc(status, NVME_SC_LBA_OUT_OF_RANGE); goto out; } if (range[r].length != 0) non_zero++; } if (cmd->cdw11 & NVME_DSM_ATTR_DEALLOCATE) { size_t offset, bytes; int sectsz_bits = sc->nvstore.sectsz_bits; /* * DSM calls are advisory only, and compliant controllers * may choose to take no actions (i.e. return Success). */ if (!nvstore->deallocate) { pci_nvme_status_genc(status, NVME_SC_SUCCESS); goto out; } /* If all ranges have a zero length, return Success */ if (non_zero == 0) { pci_nvme_status_genc(status, NVME_SC_SUCCESS); goto out; } if (req == NULL) { pci_nvme_status_genc(status, NVME_SC_INTERNAL_DEVICE_ERROR); goto out; } offset = range[0].starting_lba << sectsz_bits; bytes = range[0].length << sectsz_bits; /* * If the request is for more than a single range, store * the ranges in the br_iov. Optimize for the common case * of a single range. * * Note that NVMe Number of Ranges is a zero based value */ req->io_req.br_iovcnt = 0; req->io_req.br_offset = offset; req->io_req.br_resid = bytes; if (nr == 0) { req->io_req.br_callback = pci_nvme_io_done; } else { struct iovec *iov = req->io_req.br_iov; for (r = 0, dr = 0; r <= nr; r++) { offset = range[r].starting_lba << sectsz_bits; bytes = range[r].length << sectsz_bits; if (bytes == 0) continue; if ((nvstore->size - offset) < bytes) { pci_nvme_status_genc(status, NVME_SC_LBA_OUT_OF_RANGE); goto out; } iov[dr].iov_base = (void *)offset; iov[dr].iov_len = bytes; dr++; } req->io_req.br_callback = pci_nvme_dealloc_sm; /* * Use prev_gpaddr to track the current entry and * prev_size to track the number of entries */ req->prev_gpaddr = 0; req->prev_size = dr; } err = blockif_delete(nvstore->ctx, &req->io_req); if (err) pci_nvme_status_genc(status, NVME_SC_INTERNAL_DEVICE_ERROR); else pending = true; } out: free(range); return (pending); } static void pci_nvme_handle_io_cmd(struct pci_nvme_softc* sc, uint16_t idx) { struct nvme_submission_queue *sq; uint16_t status; uint16_t sqhead; /* handle all submissions up to sq->tail index */ sq = &sc->submit_queues[idx]; pthread_mutex_lock(&sq->mtx); sqhead = sq->head; DPRINTF("nvme_handle_io qid %u head %u tail %u cmdlist %p", idx, sqhead, sq->tail, sq->qbase); while (sqhead != atomic_load_acq_short(&sq->tail)) { struct nvme_command *cmd; struct pci_nvme_ioreq *req; uint32_t nsid; bool pending; pending = false; req = NULL; status = 0; cmd = &sq->qbase[sqhead]; sqhead = (sqhead + 1) % sq->size; nsid = le32toh(cmd->nsid); if ((nsid == 0) || (nsid > sc->ctrldata.nn)) { pci_nvme_status_genc(&status, NVME_SC_INVALID_NAMESPACE_OR_FORMAT); status |= NVMEM(NVME_STATUS_DNR); goto complete; } req = pci_nvme_get_ioreq(sc); if (req == NULL) { pci_nvme_status_genc(&status, NVME_SC_INTERNAL_DEVICE_ERROR); WPRINTF("%s: unable to allocate IO req", __func__); goto complete; } req->nvme_sq = sq; req->sqid = idx; req->opc = cmd->opc; req->cid = cmd->cid; req->nsid = cmd->nsid; switch (cmd->opc) { case NVME_OPC_FLUSH: pending = nvme_opc_flush(sc, cmd, &sc->nvstore, req, &status); break; case NVME_OPC_WRITE: case NVME_OPC_READ: pending = nvme_opc_write_read(sc, cmd, &sc->nvstore, req, &status); break; case NVME_OPC_WRITE_ZEROES: /* TODO: write zeroes WPRINTF("%s write zeroes lba 0x%lx blocks %u", __func__, lba, cmd->cdw12 & 0xFFFF); */ pci_nvme_status_genc(&status, NVME_SC_SUCCESS); break; case NVME_OPC_DATASET_MANAGEMENT: pending = nvme_opc_dataset_mgmt(sc, cmd, &sc->nvstore, req, &status); break; default: WPRINTF("%s unhandled io command 0x%x", __func__, cmd->opc); pci_nvme_status_genc(&status, NVME_SC_INVALID_OPCODE); } complete: if (!pending) { pci_nvme_set_completion(sc, sq, idx, cmd->cid, status); if (req != NULL) pci_nvme_release_ioreq(sc, req); } } sq->head = sqhead; pthread_mutex_unlock(&sq->mtx); } /* * Check for invalid doorbell write values * See NVM Express Base Specification, revision 2.0 * "Asynchronous Event Information - Error Status" for details */ static bool pci_nvme_sq_doorbell_valid(struct nvme_submission_queue *sq, uint64_t value) { uint64_t capacity; /* * Queue empty : head == tail * Queue full : head is one more than tail accounting for wrap * Therefore, can never have more than (size - 1) entries */ if (sq->head == sq->tail) capacity = sq->size - 1; else if (sq->head > sq->tail) capacity = sq->size - (sq->head - sq->tail) - 1; else capacity = sq->tail - sq->head - 1; if ((value == sq->tail) || /* same as previous */ (value > capacity)) { /* exceeds queue capacity */ EPRINTLN("%s: SQ size=%u head=%u tail=%u capacity=%lu value=%lu", __func__, sq->size, sq->head, sq->tail, capacity, value); return false; } return true; } static void pci_nvme_handle_doorbell(struct pci_nvme_softc* sc, uint64_t idx, int is_sq, uint64_t value) { DPRINTF("nvme doorbell %lu, %s, val 0x%lx", idx, is_sq ? "SQ" : "CQ", value & 0xFFFF); if (is_sq) { if (idx > sc->num_squeues) { WPRINTF("%s queue index %lu overflow from " "guest (max %u)", __func__, idx, sc->num_squeues); pci_nvme_aen_post(sc, PCI_NVME_AE_TYPE_ERROR, PCI_NVME_AEI_ERROR_INVALID_DB); return; } if (sc->submit_queues[idx].qbase == NULL) { WPRINTF("%s write to SQ %lu before created", __func__, idx); pci_nvme_aen_post(sc, PCI_NVME_AE_TYPE_ERROR, PCI_NVME_AEI_ERROR_INVALID_DB); return; } if (!pci_nvme_sq_doorbell_valid(&sc->submit_queues[idx], value)) { EPRINTLN("%s write to SQ %lu of %lu invalid", __func__, idx, value); pci_nvme_aen_post(sc, PCI_NVME_AE_TYPE_ERROR, PCI_NVME_AEI_ERROR_INVALID_DB_VALUE); return; } atomic_store_short(&sc->submit_queues[idx].tail, (uint16_t)value); if (idx == 0) pci_nvme_handle_admin_cmd(sc, value); else { /* submission queue; handle new entries in SQ */ pci_nvme_handle_io_cmd(sc, (uint16_t)idx); } } else { if (idx > sc->num_cqueues) { WPRINTF("%s queue index %lu overflow from " "guest (max %u)", __func__, idx, sc->num_cqueues); pci_nvme_aen_post(sc, PCI_NVME_AE_TYPE_ERROR, PCI_NVME_AEI_ERROR_INVALID_DB); return; } if (sc->compl_queues[idx].qbase == NULL) { WPRINTF("%s write to CQ %lu before created", __func__, idx); pci_nvme_aen_post(sc, PCI_NVME_AE_TYPE_ERROR, PCI_NVME_AEI_ERROR_INVALID_DB); return; } atomic_store_short(&sc->compl_queues[idx].head, (uint16_t)value); } } static void pci_nvme_bar0_reg_dumps(const char *func, uint64_t offset, int iswrite) { const char *s = iswrite ? "WRITE" : "READ"; switch (offset) { case NVME_CR_CAP_LOW: DPRINTF("%s %s NVME_CR_CAP_LOW", func, s); break; case NVME_CR_CAP_HI: DPRINTF("%s %s NVME_CR_CAP_HI", func, s); break; case NVME_CR_VS: DPRINTF("%s %s NVME_CR_VS", func, s); break; case NVME_CR_INTMS: DPRINTF("%s %s NVME_CR_INTMS", func, s); break; case NVME_CR_INTMC: DPRINTF("%s %s NVME_CR_INTMC", func, s); break; case NVME_CR_CC: DPRINTF("%s %s NVME_CR_CC", func, s); break; case NVME_CR_CSTS: DPRINTF("%s %s NVME_CR_CSTS", func, s); break; case NVME_CR_NSSR: DPRINTF("%s %s NVME_CR_NSSR", func, s); break; case NVME_CR_AQA: DPRINTF("%s %s NVME_CR_AQA", func, s); break; case NVME_CR_ASQ_LOW: DPRINTF("%s %s NVME_CR_ASQ_LOW", func, s); break; case NVME_CR_ASQ_HI: DPRINTF("%s %s NVME_CR_ASQ_HI", func, s); break; case NVME_CR_ACQ_LOW: DPRINTF("%s %s NVME_CR_ACQ_LOW", func, s); break; case NVME_CR_ACQ_HI: DPRINTF("%s %s NVME_CR_ACQ_HI", func, s); break; default: DPRINTF("unknown nvme bar-0 offset 0x%lx", offset); } } static void pci_nvme_write_bar_0(struct pci_nvme_softc *sc, uint64_t offset, int size, uint64_t value) { uint32_t ccreg; if (offset >= NVME_DOORBELL_OFFSET) { uint64_t belloffset = offset - NVME_DOORBELL_OFFSET; uint64_t idx = belloffset / 8; /* door bell size = 2*int */ int is_sq = (belloffset % 8) < 4; if ((sc->regs.csts & NVME_CSTS_RDY) == 0) { WPRINTF("doorbell write prior to RDY (offset=%#lx)\n", offset); return; } if (belloffset > ((sc->max_queues+1) * 8 - 4)) { WPRINTF("guest attempted an overflow write offset " "0x%lx, val 0x%lx in %s", offset, value, __func__); return; } if (is_sq) { if (sc->submit_queues[idx].qbase == NULL) return; } else if (sc->compl_queues[idx].qbase == NULL) return; pci_nvme_handle_doorbell(sc, idx, is_sq, value); return; } DPRINTF("nvme-write offset 0x%lx, size %d, value 0x%lx", offset, size, value); if (size != 4) { WPRINTF("guest wrote invalid size %d (offset 0x%lx, " "val 0x%lx) to bar0 in %s", size, offset, value, __func__); /* TODO: shutdown device */ return; } pci_nvme_bar0_reg_dumps(__func__, offset, 1); pthread_mutex_lock(&sc->mtx); switch (offset) { case NVME_CR_CAP_LOW: case NVME_CR_CAP_HI: /* readonly */ break; case NVME_CR_VS: /* readonly */ break; case NVME_CR_INTMS: /* MSI-X, so ignore */ break; case NVME_CR_INTMC: /* MSI-X, so ignore */ break; case NVME_CR_CC: ccreg = (uint32_t)value; DPRINTF("%s NVME_CR_CC en %x css %x shn %x iosqes %u " "iocqes %u", __func__, NVME_CC_GET_EN(ccreg), NVME_CC_GET_CSS(ccreg), NVME_CC_GET_SHN(ccreg), NVME_CC_GET_IOSQES(ccreg), NVME_CC_GET_IOCQES(ccreg)); if (NVME_CC_GET_SHN(ccreg)) { /* perform shutdown - flush out data to backend */ sc->regs.csts &= ~NVMEM(NVME_CSTS_REG_SHST); sc->regs.csts |= NVMEF(NVME_CSTS_REG_SHST, NVME_SHST_COMPLETE); } if (NVME_CC_GET_EN(ccreg) != NVME_CC_GET_EN(sc->regs.cc)) { if (NVME_CC_GET_EN(ccreg) == 0) /* transition 1-> causes controller reset */ pci_nvme_reset_locked(sc); else pci_nvme_init_controller(sc); } /* Insert the iocqes, iosqes and en bits from the write */ sc->regs.cc &= ~NVME_CC_WRITE_MASK; sc->regs.cc |= ccreg & NVME_CC_WRITE_MASK; if (NVME_CC_GET_EN(ccreg) == 0) { /* Insert the ams, mps and css bit fields */ sc->regs.cc &= ~NVME_CC_NEN_WRITE_MASK; sc->regs.cc |= ccreg & NVME_CC_NEN_WRITE_MASK; sc->regs.csts &= ~NVME_CSTS_RDY; } else if ((sc->pending_ios == 0) && !(sc->regs.csts & NVME_CSTS_CFS)) { sc->regs.csts |= NVME_CSTS_RDY; } break; case NVME_CR_CSTS: break; case NVME_CR_NSSR: /* ignore writes; don't support subsystem reset */ break; case NVME_CR_AQA: sc->regs.aqa = (uint32_t)value; break; case NVME_CR_ASQ_LOW: sc->regs.asq = (sc->regs.asq & (0xFFFFFFFF00000000)) | (0xFFFFF000 & value); break; case NVME_CR_ASQ_HI: sc->regs.asq = (sc->regs.asq & (0x00000000FFFFFFFF)) | (value << 32); break; case NVME_CR_ACQ_LOW: sc->regs.acq = (sc->regs.acq & (0xFFFFFFFF00000000)) | (0xFFFFF000 & value); break; case NVME_CR_ACQ_HI: sc->regs.acq = (sc->regs.acq & (0x00000000FFFFFFFF)) | (value << 32); break; default: DPRINTF("%s unknown offset 0x%lx, value 0x%lx size %d", __func__, offset, value, size); } pthread_mutex_unlock(&sc->mtx); } static void pci_nvme_write(struct pci_devinst *pi, int baridx, uint64_t offset, int size, uint64_t value) { struct pci_nvme_softc* sc = pi->pi_arg; if (baridx == pci_msix_table_bar(pi) || baridx == pci_msix_pba_bar(pi)) { DPRINTF("nvme-write baridx %d, msix: off 0x%lx, size %d, " " value 0x%lx", baridx, offset, size, value); pci_emul_msix_twrite(pi, offset, size, value); return; } switch (baridx) { case 0: pci_nvme_write_bar_0(sc, offset, size, value); break; default: DPRINTF("%s unknown baridx %d, val 0x%lx", __func__, baridx, value); } } static uint64_t pci_nvme_read_bar_0(struct pci_nvme_softc* sc, uint64_t offset, int size) { uint64_t value; pci_nvme_bar0_reg_dumps(__func__, offset, 0); if (offset < NVME_DOORBELL_OFFSET) { void *p = &(sc->regs); pthread_mutex_lock(&sc->mtx); memcpy(&value, (void *)((uintptr_t)p + offset), size); pthread_mutex_unlock(&sc->mtx); } else { value = 0; WPRINTF("pci_nvme: read invalid offset %ld", offset); } switch (size) { case 1: value &= 0xFF; break; case 2: value &= 0xFFFF; break; case 4: value &= 0xFFFFFFFF; break; } DPRINTF(" nvme-read offset 0x%lx, size %d -> value 0x%x", offset, size, (uint32_t)value); return (value); } static uint64_t pci_nvme_read(struct pci_devinst *pi, int baridx, uint64_t offset, int size) { struct pci_nvme_softc* sc = pi->pi_arg; if (baridx == pci_msix_table_bar(pi) || baridx == pci_msix_pba_bar(pi)) { DPRINTF("nvme-read bar: %d, msix: regoff 0x%lx, size %d", baridx, offset, size); return pci_emul_msix_tread(pi, offset, size); } switch (baridx) { case 0: return pci_nvme_read_bar_0(sc, offset, size); default: DPRINTF("unknown bar %d, 0x%lx", baridx, offset); } return (0); } static int pci_nvme_parse_config(struct pci_nvme_softc *sc, nvlist_t *nvl) { char bident[sizeof("XXX:XXX")]; const char *value; uint32_t sectsz; sc->max_queues = NVME_QUEUES; sc->max_qentries = NVME_MAX_QENTRIES; sc->ioslots = NVME_IOSLOTS; sc->num_squeues = sc->max_queues; sc->num_cqueues = sc->max_queues; sc->dataset_management = NVME_DATASET_MANAGEMENT_AUTO; sectsz = 0; #ifdef __FreeBSD__ snprintf(sc->ctrldata.sn, sizeof(sc->ctrldata.sn), "NVME-%d-%d", sc->nsc_pi->pi_slot, sc->nsc_pi->pi_func); #else snprintf((char *)sc->ctrldata.sn, sizeof(sc->ctrldata.sn), "NVME-%d-%d", sc->nsc_pi->pi_slot, sc->nsc_pi->pi_func); #endif value = get_config_value_node(nvl, "maxq"); if (value != NULL) sc->max_queues = atoi(value); value = get_config_value_node(nvl, "qsz"); if (value != NULL) { sc->max_qentries = atoi(value); if (sc->max_qentries <= 0) { EPRINTLN("nvme: Invalid qsz option %d", sc->max_qentries); return (-1); } } value = get_config_value_node(nvl, "ioslots"); if (value != NULL) { sc->ioslots = atoi(value); if (sc->ioslots <= 0) { EPRINTLN("Invalid ioslots option %d", sc->ioslots); return (-1); } } value = get_config_value_node(nvl, "sectsz"); if (value != NULL) sectsz = atoi(value); value = get_config_value_node(nvl, "ser"); if (value != NULL) { /* * This field indicates the Product Serial Number in * 7-bit ASCII, unused bytes should be space characters. * Ref: NVMe v1.3c. */ cpywithpad((char *)sc->ctrldata.sn, sizeof(sc->ctrldata.sn), value, ' '); } value = get_config_value_node(nvl, "eui64"); if (value != NULL) sc->nvstore.eui64 = htobe64(strtoull(value, NULL, 0)); value = get_config_value_node(nvl, "dsm"); if (value != NULL) { if (strcmp(value, "auto") == 0) sc->dataset_management = NVME_DATASET_MANAGEMENT_AUTO; else if (strcmp(value, "enable") == 0) sc->dataset_management = NVME_DATASET_MANAGEMENT_ENABLE; else if (strcmp(value, "disable") == 0) sc->dataset_management = NVME_DATASET_MANAGEMENT_DISABLE; } value = get_config_value_node(nvl, "bootindex"); if (value != NULL) { if (pci_emul_add_boot_device(sc->nsc_pi, atoi(value))) { EPRINTLN("Invalid bootindex %d", atoi(value)); return (-1); } } value = get_config_value_node(nvl, "ram"); if (value != NULL) { uint64_t sz = strtoull(value, NULL, 10); sc->nvstore.type = NVME_STOR_RAM; sc->nvstore.size = sz * 1024 * 1024; sc->nvstore.ctx = calloc(1, sc->nvstore.size); sc->nvstore.sectsz = 4096; sc->nvstore.sectsz_bits = 12; if (sc->nvstore.ctx == NULL) { EPRINTLN("nvme: Unable to allocate RAM"); return (-1); } } else { snprintf(bident, sizeof(bident), "%u:%u", sc->nsc_pi->pi_slot, sc->nsc_pi->pi_func); sc->nvstore.ctx = blockif_open(nvl, bident); if (sc->nvstore.ctx == NULL) { EPRINTLN("nvme: Could not open backing file: %s", strerror(errno)); return (-1); } sc->nvstore.type = NVME_STOR_BLOCKIF; sc->nvstore.size = blockif_size(sc->nvstore.ctx); } if (sectsz == 512 || sectsz == 4096 || sectsz == 8192) sc->nvstore.sectsz = sectsz; else if (sc->nvstore.type != NVME_STOR_RAM) sc->nvstore.sectsz = blockif_sectsz(sc->nvstore.ctx); for (sc->nvstore.sectsz_bits = 9; (1U << sc->nvstore.sectsz_bits) < sc->nvstore.sectsz; sc->nvstore.sectsz_bits++); if (sc->max_queues <= 0 || sc->max_queues > NVME_QUEUES) sc->max_queues = NVME_QUEUES; return (0); } static void pci_nvme_resized(struct blockif_ctxt *bctxt __unused, void *arg, size_t new_size) { struct pci_nvme_softc *sc; struct pci_nvme_blockstore *nvstore; struct nvme_namespace_data *nd; sc = arg; nvstore = &sc->nvstore; nd = &sc->nsdata; nvstore->size = new_size; pci_nvme_init_nsdata_size(nvstore, nd); /* Add changed NSID to list */ sc->ns_log.ns[0] = 1; sc->ns_log.ns[1] = 0; pci_nvme_aen_post(sc, PCI_NVME_AE_TYPE_NOTICE, PCI_NVME_AEI_NOTICE_NS_ATTR_CHANGED); } static int pci_nvme_init(struct pci_devinst *pi, nvlist_t *nvl) { struct pci_nvme_softc *sc; uint32_t pci_membar_sz; int error; error = 0; sc = calloc(1, sizeof(struct pci_nvme_softc)); pi->pi_arg = sc; sc->nsc_pi = pi; error = pci_nvme_parse_config(sc, nvl); if (error < 0) goto done; else error = 0; STAILQ_INIT(&sc->ioreqs_free); sc->ioreqs = calloc(sc->ioslots, sizeof(struct pci_nvme_ioreq)); for (uint32_t i = 0; i < sc->ioslots; i++) { STAILQ_INSERT_TAIL(&sc->ioreqs_free, &sc->ioreqs[i], link); } pci_set_cfgdata16(pi, PCIR_DEVICE, 0x0A0A); pci_set_cfgdata16(pi, PCIR_VENDOR, 0xFB5D); pci_set_cfgdata8(pi, PCIR_CLASS, PCIC_STORAGE); pci_set_cfgdata8(pi, PCIR_SUBCLASS, PCIS_STORAGE_NVM); pci_set_cfgdata8(pi, PCIR_PROGIF, PCIP_STORAGE_NVM_ENTERPRISE_NVMHCI_1_0); /* * Allocate size of NVMe registers + doorbell space for all queues. * * The specification requires a minimum memory I/O window size of 16K. * The Windows driver will refuse to start a device with a smaller * window. */ pci_membar_sz = sizeof(struct nvme_registers) + 2 * sizeof(uint32_t) * (sc->max_queues + 1); pci_membar_sz = MAX(pci_membar_sz, NVME_MMIO_SPACE_MIN); DPRINTF("nvme membar size: %u", pci_membar_sz); error = pci_emul_alloc_bar(pi, 0, PCIBAR_MEM64, pci_membar_sz); if (error) { WPRINTF("%s pci alloc mem bar failed", __func__); goto done; } error = pci_emul_add_msixcap(pi, sc->max_queues + 1, NVME_MSIX_BAR); if (error) { WPRINTF("%s pci add msixcap failed", __func__); goto done; } error = pci_emul_add_pciecap(pi, PCIEM_TYPE_ROOT_INT_EP); if (error) { WPRINTF("%s pci add Express capability failed", __func__); goto done; } pthread_mutex_init(&sc->mtx, NULL); sem_init(&sc->iosemlock, 0, sc->ioslots); blockif_register_resize_callback(sc->nvstore.ctx, pci_nvme_resized, sc); pci_nvme_init_queues(sc, sc->max_queues, sc->max_queues); /* * Controller data depends on Namespace data so initialize Namespace * data first. */ pci_nvme_init_nsdata(sc, &sc->nsdata, 1, &sc->nvstore); pci_nvme_init_ctrldata(sc); pci_nvme_init_logpages(sc); pci_nvme_init_features(sc); pci_nvme_aer_init(sc); pci_nvme_aen_init(sc); pci_nvme_reset(sc); done: return (error); } static int pci_nvme_legacy_config(nvlist_t *nvl, const char *opts) { char *cp, *ram; if (opts == NULL) return (0); if (strncmp(opts, "ram=", 4) == 0) { cp = strchr(opts, ','); if (cp == NULL) { set_config_value_node(nvl, "ram", opts + 4); return (0); } ram = strndup(opts + 4, cp - opts - 4); set_config_value_node(nvl, "ram", ram); free(ram); return (pci_parse_legacy_config(nvl, cp + 1)); } else return (blockif_legacy_config(nvl, opts)); } static const struct pci_devemu pci_de_nvme = { .pe_emu = "nvme", .pe_init = pci_nvme_init, .pe_legacy_config = pci_nvme_legacy_config, .pe_barwrite = pci_nvme_write, .pe_barread = pci_nvme_read }; PCI_EMUL_SET(pci_de_nvme); /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2011 NetApp, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "config.h" #include "debug.h" #include "pci_passthru.h" #include "mem.h" #define LEGACY_SUPPORT 1 #define MSIX_TABLE_COUNT(ctrl) (((ctrl) & PCIM_MSIXCTRL_TABLE_SIZE) + 1) #define MSIX_CAPLEN 12 struct passthru_softc { struct pci_devinst *psc_pi; /* ROM is handled like a BAR */ struct pcibar psc_bar[PCI_BARMAX_WITH_ROM + 1]; struct { int capoff; int msgctrl; int emulated; } psc_msi; struct { int capoff; } psc_msix; int pptfd; int msi_limit; int msix_limit; cfgread_handler psc_pcir_rhandler[PCI_REGMAX + 1]; cfgwrite_handler psc_pcir_whandler[PCI_REGMAX + 1]; }; static int msi_caplen(int msgctrl) { int len; len = 10; /* minimum length of msi capability */ if (msgctrl & PCIM_MSICTRL_64BIT) len += 4; #if 0 /* * Ignore the 'mask' and 'pending' bits in the MSI capability. * We'll let the guest manipulate them directly. */ if (msgctrl & PCIM_MSICTRL_VECTOR) len += 10; #endif return (len); } static uint32_t passthru_read_config(const struct passthru_softc *sc, long reg, int width) { struct ppt_cfg_io pi; pi.pci_off = reg; pi.pci_width = width; if (ioctl(sc->pptfd, PPT_CFG_READ, &pi) != 0) { return (0); } return (pi.pci_data); } static void passthru_write_config(const struct passthru_softc *sc, long reg, int width, uint32_t data) { struct ppt_cfg_io pi; pi.pci_off = reg; pi.pci_width = width; pi.pci_data = data; (void) ioctl(sc->pptfd, PPT_CFG_WRITE, &pi); } static int passthru_get_bar(struct passthru_softc *sc, int bar, enum pcibar_type *type, uint64_t *base, uint64_t *size) { struct ppt_bar_query pb; pb.pbq_baridx = bar; if (ioctl(sc->pptfd, PPT_BAR_QUERY, &pb) != 0) { return (-1); } switch (pb.pbq_type) { case PCI_ADDR_IO: *type = PCIBAR_IO; break; case PCI_ADDR_MEM32: *type = PCIBAR_MEM32; break; case PCI_ADDR_MEM64: *type = PCIBAR_MEM64; break; default: err(1, "unrecognized BAR type: %u\n", pb.pbq_type); break; } *base = pb.pbq_base; *size = pb.pbq_size; return (0); } static int passthru_dev_open(const char *path, int *pptfdp) { int pptfd; if ((pptfd = open(path, O_RDWR)) < 0) { return (errno); } /* XXX: verify fd with ioctl? */ *pptfdp = pptfd; return (0); } #ifdef LEGACY_SUPPORT static int passthru_add_msicap(struct pci_devinst *pi, int msgnum, int nextptr) { int capoff; struct msicap msicap; u_char *capdata; pci_populate_msicap(&msicap, msgnum, nextptr); /* * XXX * Copy the msi capability structure in the last 16 bytes of the * config space. This is wrong because it could shadow something * useful to the device. */ capoff = 256 - roundup(sizeof(msicap), 4); capdata = (u_char *)&msicap; for (size_t i = 0; i < sizeof(msicap); i++) pci_set_cfgdata8(pi, capoff + i, capdata[i]); return (capoff); } #endif /* LEGACY_SUPPORT */ static void passthru_intr_limit(struct passthru_softc *sc, struct msixcap *msixcap) { struct pci_devinst *pi = sc->psc_pi; int off; /* Reduce the number of MSI vectors if higher than OS limit */ if ((off = sc->psc_msi.capoff) != 0 && sc->msi_limit != -1) { int msi_limit, mmc; msi_limit = sc->msi_limit > 16 ? PCIM_MSICTRL_MMC_32 : sc->msi_limit > 8 ? PCIM_MSICTRL_MMC_16 : sc->msi_limit > 4 ? PCIM_MSICTRL_MMC_8 : sc->msi_limit > 2 ? PCIM_MSICTRL_MMC_4 : sc->msi_limit > 1 ? PCIM_MSICTRL_MMC_2 : PCIM_MSICTRL_MMC_1; mmc = sc->psc_msi.msgctrl & PCIM_MSICTRL_MMC_MASK; if (mmc > msi_limit) { sc->psc_msi.msgctrl &= ~PCIM_MSICTRL_MMC_MASK; sc->psc_msi.msgctrl |= msi_limit; pci_set_cfgdata16(pi, off + 2, sc->psc_msi.msgctrl); } } /* Reduce the number of MSI-X vectors if higher than OS limit */ if ((off = sc->psc_msix.capoff) != 0 && sc->msix_limit != -1) { if (MSIX_TABLE_COUNT(msixcap->msgctrl) > sc->msix_limit) { msixcap->msgctrl &= ~PCIM_MSIXCTRL_TABLE_SIZE; msixcap->msgctrl |= sc->msix_limit - 1; pci_set_cfgdata16(pi, off + 2, msixcap->msgctrl); } } } static int cfginitmsi(struct passthru_softc *sc) { int i, ptr, capptr, cap, sts, caplen, table_size; uint32_t u32; struct pci_devinst *pi = sc->psc_pi; struct msixcap msixcap; char *msixcap_ptr; /* * Parse the capabilities and cache the location of the MSI * and MSI-X capabilities. */ sts = passthru_read_config(sc, PCIR_STATUS, 2); if (sts & PCIM_STATUS_CAPPRESENT) { ptr = passthru_read_config(sc, PCIR_CAP_PTR, 1); while (ptr != 0 && ptr != 0xff) { cap = passthru_read_config(sc, ptr + PCICAP_ID, 1); if (cap == PCIY_MSI) { /* * Copy the MSI capability into the config * space of the emulated pci device */ sc->psc_msi.capoff = ptr; sc->psc_msi.msgctrl = passthru_read_config(sc, ptr + 2, 2); sc->psc_msi.emulated = 0; caplen = msi_caplen(sc->psc_msi.msgctrl); capptr = ptr; while (caplen > 0) { u32 = passthru_read_config(sc, capptr, 4); pci_set_cfgdata32(pi, capptr, u32); caplen -= 4; capptr += 4; } } else if (cap == PCIY_MSIX) { /* * Copy the MSI-X capability */ sc->psc_msix.capoff = ptr; caplen = 12; msixcap_ptr = (char *)&msixcap; capptr = ptr; while (caplen > 0) { u32 = passthru_read_config(sc, capptr, 4); memcpy(msixcap_ptr, &u32, 4); pci_set_cfgdata32(pi, capptr, u32); caplen -= 4; capptr += 4; msixcap_ptr += 4; } } ptr = passthru_read_config(sc, ptr + PCICAP_NEXTPTR, 1); } } passthru_intr_limit(sc, &msixcap); if (sc->psc_msix.capoff != 0) { pi->pi_msix.pba_bar = msixcap.pba_info & PCIM_MSIX_BIR_MASK; pi->pi_msix.pba_offset = msixcap.pba_info & ~PCIM_MSIX_BIR_MASK; pi->pi_msix.table_bar = msixcap.table_info & PCIM_MSIX_BIR_MASK; pi->pi_msix.table_offset = msixcap.table_info & ~PCIM_MSIX_BIR_MASK; pi->pi_msix.table_count = MSIX_TABLE_COUNT(msixcap.msgctrl); pi->pi_msix.pba_size = PBA_SIZE(pi->pi_msix.table_count); /* Allocate the emulated MSI-X table array */ table_size = pi->pi_msix.table_count * MSIX_TABLE_ENTRY_SIZE; pi->pi_msix.table = calloc(1, table_size); /* Mask all table entries */ for (i = 0; i < pi->pi_msix.table_count; i++) { pi->pi_msix.table[i].vector_control |= PCIM_MSIX_VCTRL_MASK; } } #ifdef LEGACY_SUPPORT /* * If the passthrough device does not support MSI then craft a * MSI capability for it. We link the new MSI capability at the * head of the list of capabilities. */ if ((sts & PCIM_STATUS_CAPPRESENT) != 0 && sc->psc_msi.capoff == 0) { int origptr, msiptr; origptr = passthru_read_config(sc, PCIR_CAP_PTR, 1); msiptr = passthru_add_msicap(pi, 1, origptr); sc->psc_msi.capoff = msiptr; sc->psc_msi.msgctrl = pci_get_cfgdata16(pi, msiptr + 2); sc->psc_msi.emulated = 1; pci_set_cfgdata8(pi, PCIR_CAP_PTR, msiptr); } #endif /* Make sure one of the capabilities is present */ if (sc->psc_msi.capoff == 0 && sc->psc_msix.capoff == 0) return (-1); else return (0); } static uint64_t msix_table_read(struct passthru_softc *sc, uint64_t offset, int size) { struct pci_devinst *pi; struct msix_table_entry *entry; uint8_t *src8; uint16_t *src16; uint32_t *src32; uint64_t *src64; uint64_t data; size_t entry_offset; uint32_t table_offset; int index, table_count; pi = sc->psc_pi; table_offset = pi->pi_msix.table_offset; table_count = pi->pi_msix.table_count; if (offset < table_offset || offset >= table_offset + table_count * MSIX_TABLE_ENTRY_SIZE) { switch (size) { case 1: src8 = (uint8_t *)(pi->pi_msix.mapped_addr + offset); data = *src8; break; case 2: src16 = (uint16_t *)(pi->pi_msix.mapped_addr + offset); data = *src16; break; case 4: src32 = (uint32_t *)(pi->pi_msix.mapped_addr + offset); data = *src32; break; case 8: src64 = (uint64_t *)(pi->pi_msix.mapped_addr + offset); data = *src64; break; default: return (-1); } return (data); } offset -= table_offset; index = offset / MSIX_TABLE_ENTRY_SIZE; assert(index < table_count); entry = &pi->pi_msix.table[index]; entry_offset = offset % MSIX_TABLE_ENTRY_SIZE; switch (size) { case 1: src8 = (uint8_t *)((uint8_t *)entry + entry_offset); data = *src8; break; case 2: src16 = (uint16_t *)((uint8_t *)entry + entry_offset); data = *src16; break; case 4: src32 = (uint32_t *)((uint8_t *)entry + entry_offset); data = *src32; break; case 8: src64 = (uint64_t *)((uint8_t *)entry + entry_offset); data = *src64; break; default: return (-1); } return (data); } static void msix_table_write(struct vmctx *ctx, struct passthru_softc *sc, uint64_t offset, int size, uint64_t data) { struct pci_devinst *pi; struct msix_table_entry *entry; uint8_t *dest8; uint16_t *dest16; uint32_t *dest32; uint64_t *dest64; size_t entry_offset; uint32_t table_offset, vector_control; int index, table_count; pi = sc->psc_pi; table_offset = pi->pi_msix.table_offset; table_count = pi->pi_msix.table_count; if (offset < table_offset || offset >= table_offset + table_count * MSIX_TABLE_ENTRY_SIZE) { switch (size) { case 1: dest8 = (uint8_t *)(pi->pi_msix.mapped_addr + offset); *dest8 = data; break; case 2: dest16 = (uint16_t *)(pi->pi_msix.mapped_addr + offset); *dest16 = data; break; case 4: dest32 = (uint32_t *)(pi->pi_msix.mapped_addr + offset); *dest32 = data; break; case 8: dest64 = (uint64_t *)(pi->pi_msix.mapped_addr + offset); *dest64 = data; break; } return; } offset -= table_offset; index = offset / MSIX_TABLE_ENTRY_SIZE; assert(index < table_count); entry = &pi->pi_msix.table[index]; entry_offset = offset % MSIX_TABLE_ENTRY_SIZE; /* Only 4 byte naturally-aligned writes are supported */ assert(size == 4); assert(entry_offset % 4 == 0); vector_control = entry->vector_control; dest32 = (uint32_t *)((uint8_t *)entry + entry_offset); *dest32 = data; /* If MSI-X hasn't been enabled, do nothing */ if (pi->pi_msix.enabled) { /* If the entry is masked, don't set it up */ if ((entry->vector_control & PCIM_MSIX_VCTRL_MASK) == 0 || (vector_control & PCIM_MSIX_VCTRL_MASK) == 0) { (void) vm_setup_pptdev_msix(ctx, sc->pptfd, index, entry->addr, entry->msg_data, entry->vector_control); } } } static int init_msix_table(struct vmctx *ctx __unused, struct passthru_softc *sc) { struct pci_devinst *pi = sc->psc_pi; uint32_t table_size, table_offset; int i; i = pci_msix_table_bar(pi); assert(i >= 0); /* * Map the region of the BAR containing the MSI-X table. This is * necessary for two reasons: * 1. The PBA may reside in the first or last page containing the MSI-X * table. * 2. While PCI devices are not supposed to use the page(s) containing * the MSI-X table for other purposes, some do in practice. */ /* * Mapping pptfd provides access to the BAR containing the MSI-X * table. See ppt_devmap() in usr/src/uts/intel/io/vmm/io/ppt.c * * This maps the whole BAR and then mprotect(PROT_NONE) is used below * to prevent access to pages that don't contain the MSI-X table. * When porting this, it was tempting to just map the MSI-X table pages * but that would mean updating everywhere that assumes that * pi->pi_msix.mapped_addr points to the start of the BAR. For now, * keep closer to upstream. */ pi->pi_msix.mapped_size = sc->psc_bar[i].size; pi->pi_msix.mapped_addr = (uint8_t *)mmap(NULL, pi->pi_msix.mapped_size, PROT_READ | PROT_WRITE, MAP_SHARED, sc->pptfd, 0); if (pi->pi_msix.mapped_addr == MAP_FAILED) { warn("Failed to map MSI-X table BAR on %d", sc->pptfd); return (-1); } table_offset = rounddown2(pi->pi_msix.table_offset, 4096); table_size = pi->pi_msix.table_offset - table_offset; table_size += pi->pi_msix.table_count * MSIX_TABLE_ENTRY_SIZE; table_size = roundup2(table_size, 4096); /* * Unmap any pages not containing the table, we do not need to emulate * accesses to them. Avoid releasing address space to help ensure that * a buggy out-of-bounds access causes a crash. */ if (table_offset != 0) if (mprotect((caddr_t)pi->pi_msix.mapped_addr, table_offset, PROT_NONE) != 0) warn("Failed to unmap MSI-X table BAR region"); if (table_offset + table_size != pi->pi_msix.mapped_size) if (mprotect((caddr_t) pi->pi_msix.mapped_addr + table_offset + table_size, pi->pi_msix.mapped_size - (table_offset + table_size), PROT_NONE) != 0) warn("Failed to unmap MSI-X table BAR region"); return (0); } static int cfginitbar(struct vmctx *ctx __unused, struct passthru_softc *sc) { struct pci_devinst *pi = sc->psc_pi; uint_t i; /* * Initialize BAR registers */ for (i = 0; i <= PCI_BARMAX; i++) { enum pcibar_type bartype; uint64_t base, size; int error; if (passthru_get_bar(sc, i, &bartype, &base, &size) != 0) { continue; } if (bartype != PCIBAR_IO) { if (((base | size) & PAGE_MASK) != 0) { warnx("passthru device %d BAR %d: " "base %#lx or size %#lx not page aligned\n", sc->pptfd, i, base, size); return (-1); } } /* Cache information about the "real" BAR */ sc->psc_bar[i].type = bartype; sc->psc_bar[i].size = size; sc->psc_bar[i].addr = base; sc->psc_bar[i].lobits = 0; /* Allocate the BAR in the guest I/O or MMIO space */ error = pci_emul_alloc_bar(pi, i, bartype, size); if (error) return (-1); /* Use same lobits as physical bar */ uint8_t lobits = passthru_read_config(sc, PCIR_BAR(i), 0x01); if (bartype == PCIBAR_MEM32 || bartype == PCIBAR_MEM64) { lobits &= ~PCIM_BAR_MEM_BASE; } else { lobits &= ~PCIM_BAR_IO_BASE; } sc->psc_bar[i].lobits = lobits; pi->pi_bar[i].lobits = lobits; /* * 64-bit BAR takes up two slots so skip the next one. */ if (bartype == PCIBAR_MEM64) { i++; assert(i <= PCI_BARMAX); sc->psc_bar[i].type = PCIBAR_MEMHI64; } } return (0); } static int cfginit(struct vmctx *ctx, struct passthru_softc *sc) { int error; struct pci_devinst *pi = sc->psc_pi; uint16_t cmd; uint8_t intline, intpin; /* * Copy physical PCI header to virtual config space. COMMAND, * INTLINE and INTPIN shouldn't be aligned with their physical value * and they are already set by pci_emul_init(). */ cmd = pci_get_cfgdata16(pi, PCIR_COMMAND); intline = pci_get_cfgdata8(pi, PCIR_INTLINE); intpin = pci_get_cfgdata8(pi, PCIR_INTPIN); for (int i = 0; i <= PCIR_MAXLAT; i += 4) { #ifdef __FreeBSD__ pci_set_cfgdata32(pi, i, read_config(&sc->psc_sel, i, 4)); #else pci_set_cfgdata32(pi, i, passthru_read_config(sc, i, 4)); #endif } pci_set_cfgdata16(pi, PCIR_COMMAND, cmd); pci_set_cfgdata8(pi, PCIR_INTLINE, intline); pci_set_cfgdata8(pi, PCIR_INTPIN, intpin); if (cfginitmsi(sc) != 0) { warnx("failed to initialize MSI for PCI %d", sc->pptfd); return (-1); } if (cfginitbar(ctx, sc) != 0) { warnx("failed to initialize BARs for PCI %d", sc->pptfd); return (-1); } if (pci_msix_table_bar(pi) >= 0) { error = init_msix_table(ctx, sc); if (error != 0) { warnx("failed to initialize MSI-X table for PCI %d", sc->pptfd); goto done; } } /* Emulate most PCI header register. */ if ((error = set_pcir_handler(sc, 0, PCIR_MAXLAT + 1, passthru_cfgread_emulate, passthru_cfgwrite_emulate)) != 0) goto done; /* Allow access to the physical status register. */ if ((error = set_pcir_handler(sc, PCIR_COMMAND, 0x04, NULL, NULL)) != 0) goto done; error = 0; /* success */ done: return (error); } int set_pcir_handler(struct passthru_softc *sc, int reg, int len, cfgread_handler rhandler, cfgwrite_handler whandler) { if (reg > PCI_REGMAX || reg + len > PCI_REGMAX + 1) return (-1); for (int i = reg; i < reg + len; ++i) { assert(sc->psc_pcir_rhandler[i] == NULL || rhandler == NULL); assert(sc->psc_pcir_whandler[i] == NULL || whandler == NULL); sc->psc_pcir_rhandler[i] = rhandler; sc->psc_pcir_whandler[i] = whandler; } return (0); } static int passthru_legacy_config(nvlist_t *nvl, const char *opt) { char *config, *name, *tofree, *value; if (opt == NULL) return (0); config = tofree = strdup(opt); while ((name = strsep(&config, ",")) != NULL) { value = strchr(name, '='); if (value != NULL) { *value++ = '\0'; set_config_value_node(nvl, name, value); } else { if (strncmp(name, "/dev/ppt", 8) != 0) { EPRINTLN("passthru: invalid path \"%s\"", name); free(tofree); return (-1); } set_config_value_node(nvl, "path", name); } } free(tofree); return (0); } static int passthru_init_rom(struct vmctx *const ctx __unused, struct passthru_softc *const sc, const char *const romfile) { if (romfile == NULL) { return (0); } const int fd = open(romfile, O_RDONLY); if (fd < 0) { warnx("%s: can't open romfile \"%s\"", __func__, romfile); return (-1); } struct stat sbuf; if (fstat(fd, &sbuf) < 0) { warnx("%s: can't fstat romfile \"%s\"", __func__, romfile); close(fd); return (-1); } const uint64_t rom_size = sbuf.st_size; void *const rom_data = mmap(NULL, rom_size, PROT_READ, MAP_SHARED, fd, 0); if (rom_data == MAP_FAILED) { warnx("%s: unable to mmap romfile \"%s\" (%d)", __func__, romfile, errno); close(fd); return (-1); } void *rom_addr; int error = pci_emul_alloc_rom(sc->psc_pi, rom_size, &rom_addr); if (error) { warnx("%s: failed to alloc rom segment", __func__); munmap(rom_data, rom_size); close(fd); return (error); } memcpy(rom_addr, rom_data, rom_size); sc->psc_bar[PCI_ROM_IDX].type = PCIBAR_ROM; sc->psc_bar[PCI_ROM_IDX].addr = (uint64_t)rom_addr; sc->psc_bar[PCI_ROM_IDX].size = rom_size; munmap(rom_data, rom_size); close(fd); return (0); } static int passthru_init(struct pci_devinst *pi, nvlist_t *nvl) { int error, memflags, pptfd; struct passthru_softc *sc; const char *path; struct vmctx *ctx = pi->pi_vmctx; pptfd = -1; sc = NULL; error = 1; memflags = vm_get_memflags(ctx); if (!(memflags & VM_MEM_F_WIRED)) { warnx("passthru requires guest memory to be wired"); goto done; } path = get_config_value_node(nvl, "path"); if (path == NULL || passthru_dev_open(path, &pptfd) != 0) { warnx("invalid passthru options"); goto done; } if (vm_assign_pptdev(ctx, pptfd) != 0) { warnx("PCI device at %d is not using the ppt driver", pptfd); goto done; } sc = calloc(1, sizeof(struct passthru_softc)); pi->pi_arg = sc; sc->psc_pi = pi; sc->pptfd = pptfd; if ((error = vm_get_pptdev_limits(ctx, pptfd, &sc->msi_limit, &sc->msix_limit)) != 0) goto done; #ifndef __FreeBSD__ /* * If this function uses legacy interrupt messages, then request one for * the guest in case drivers expect to see it. Note that nothing in the * hypervisor is currently wired up do deliver such an interrupt should * the guest actually rely upon it. */ uint8_t intpin = passthru_read_config(sc, PCIR_INTPIN, 1); if (intpin > 0 && intpin < 5) pci_lintr_request(sc->psc_pi); #endif /* initialize config space */ if ((error = cfginit(ctx, sc)) != 0) goto done; /* initialize ROM */ if ((error = passthru_init_rom(ctx, sc, get_config_value_node(nvl, "rom"))) != 0) { goto done; } done: if (error) { free(sc); if (pptfd != -1) vm_unassign_pptdev(ctx, pptfd); } return (error); } static int msicap_access(struct passthru_softc *sc, int coff) { int caplen; if (sc->psc_msi.capoff == 0) return (0); caplen = msi_caplen(sc->psc_msi.msgctrl); if (coff >= sc->psc_msi.capoff && coff < sc->psc_msi.capoff + caplen) return (1); else return (0); } static int msixcap_access(struct passthru_softc *sc, int coff) { if (sc->psc_msix.capoff == 0) return (0); return (coff >= sc->psc_msix.capoff && coff < sc->psc_msix.capoff + MSIX_CAPLEN); } static int passthru_cfgread_default(struct passthru_softc *sc, struct pci_devinst *pi __unused, int coff, int bytes, uint32_t *rv) { /* * MSI capability is emulated. */ if (msicap_access(sc, coff) || msixcap_access(sc, coff)) return (PE_CFGRW_DEFAULT); /* * MSI-X is also emulated since a limit on interrupts may be imposed by * the OS, altering the perceived register state. */ if (msixcap_access(sc, coff)) return (PE_CFGRW_DEFAULT); /* * Emulate the command register. If a single read reads both the * command and status registers, read the status register from the * device's config space. */ if (coff == PCIR_COMMAND) { if (bytes <= 2) return (PE_CFGRW_DEFAULT); *rv = passthru_read_config(sc, PCIR_STATUS, 2) << 16 | pci_get_cfgdata16(pi, PCIR_COMMAND); return (PE_CFGRW_DROP); } /* Everything else just read from the device's config space */ *rv = passthru_read_config(sc, coff, bytes); return (PE_CFGRW_DROP); } int passthru_cfgread_emulate(struct passthru_softc *sc __unused, struct pci_devinst *pi __unused, int coff __unused, int bytes __unused, uint32_t *rv __unused) { return (PE_CFGRW_DEFAULT); } static int passthru_cfgread(struct pci_devinst *pi, int coff, int bytes, uint32_t *rv) { struct passthru_softc *sc; sc = pi->pi_arg; if (sc->psc_pcir_rhandler[coff] != NULL) return (sc->psc_pcir_rhandler[coff](sc, pi, coff, bytes, rv)); return (passthru_cfgread_default(sc, pi, coff, bytes, rv)); } static int passthru_cfgwrite_default(struct passthru_softc *sc, struct pci_devinst *pi, int coff, int bytes, uint32_t val) { int error, msix_table_entries, i; uint16_t cmd_old; struct vmctx *ctx = pi->pi_vmctx; /* * MSI capability is emulated */ if (msicap_access(sc, coff)) { pci_emul_capwrite(pi, coff, bytes, val, sc->psc_msi.capoff, PCIY_MSI); error = vm_setup_pptdev_msi(ctx, sc->pptfd, pi->pi_msi.addr, pi->pi_msi.msg_data, pi->pi_msi.maxmsgnum); if (error != 0) err(1, "vm_setup_pptdev_msi"); return (PE_CFGRW_DROP); } if (msixcap_access(sc, coff)) { pci_emul_capwrite(pi, coff, bytes, val, sc->psc_msix.capoff, PCIY_MSIX); if (pi->pi_msix.enabled) { msix_table_entries = pi->pi_msix.table_count; for (i = 0; i < msix_table_entries; i++) { error = vm_setup_pptdev_msix(ctx, sc->pptfd, i, pi->pi_msix.table[i].addr, pi->pi_msix.table[i].msg_data, pi->pi_msix.table[i].vector_control); if (error) err(1, "vm_setup_pptdev_msix"); } } else { error = vm_disable_pptdev_msix(ctx, sc->pptfd); if (error) err(1, "vm_disable_pptdev_msix"); } return (PE_CFGRW_DROP); } /* * The command register is emulated, but the status register * is passed through. */ if (coff == PCIR_COMMAND) { if (bytes <= 2) return (PE_CFGRW_DEFAULT); /* Update the physical status register. */ passthru_write_config(sc, PCIR_STATUS, 2, val >> 16); /* Update the virtual command register. */ cmd_old = pci_get_cfgdata16(pi, PCIR_COMMAND); pci_set_cfgdata16(pi, PCIR_COMMAND, val & 0xffff); pci_emul_cmd_changed(pi, cmd_old); return (PE_CFGRW_DROP); } passthru_write_config(sc, coff, bytes, val); return (PE_CFGRW_DROP); } int passthru_cfgwrite_emulate(struct passthru_softc *sc __unused, struct pci_devinst *pi __unused, int coff __unused, int bytes __unused, uint32_t val __unused) { return (PE_CFGRW_DEFAULT); } static int passthru_cfgwrite(struct pci_devinst *pi, int coff, int bytes, uint32_t val) { struct passthru_softc *sc; sc = pi->pi_arg; if (sc->psc_pcir_whandler[coff] != NULL) return (sc->psc_pcir_whandler[coff](sc, pi, coff, bytes, val)); return (passthru_cfgwrite_default(sc, pi, coff, bytes, val)); } static void passthru_write(struct pci_devinst *pi, int baridx, uint64_t offset, int size, uint64_t value) { struct passthru_softc *sc = pi->pi_arg; struct vmctx *ctx = pi->pi_vmctx; if (baridx == pci_msix_table_bar(pi)) { msix_table_write(ctx, sc, offset, size, value); } else { struct ppt_bar_io pbi; assert(pi->pi_bar[baridx].type == PCIBAR_IO); pbi.pbi_bar = baridx; pbi.pbi_width = size; pbi.pbi_off = offset; pbi.pbi_data = value; (void) ioctl(sc->pptfd, PPT_BAR_WRITE, &pbi); } } static uint64_t passthru_read(struct pci_devinst *pi, int baridx, uint64_t offset, int size) { struct passthru_softc *sc = pi->pi_arg; uint64_t val; if (baridx == pci_msix_table_bar(pi)) { val = msix_table_read(sc, offset, size); } else { struct ppt_bar_io pbi; assert(pi->pi_bar[baridx].type == PCIBAR_IO); pbi.pbi_bar = baridx; pbi.pbi_width = size; pbi.pbi_off = offset; if (ioctl(sc->pptfd, PPT_BAR_READ, &pbi) == 0) { val = pbi.pbi_data; } else { val = 0; } } return (val); } static void passthru_msix_addr(struct vmctx *ctx, struct pci_devinst *pi, int baridx, int enabled, uint64_t address) { struct passthru_softc *sc; size_t remaining; uint32_t table_size, table_offset; sc = pi->pi_arg; table_offset = rounddown2(pi->pi_msix.table_offset, 4096); if (table_offset > 0) { if (!enabled) { if (vm_unmap_pptdev_mmio(ctx, sc->pptfd, address, table_offset) != 0) warnx("pci_passthru: unmap_pptdev_mmio failed"); } else { if (vm_map_pptdev_mmio(ctx, sc->pptfd, address, table_offset, sc->psc_bar[baridx].addr) != 0) warnx("pci_passthru: map_pptdev_mmio failed"); } } table_size = pi->pi_msix.table_offset - table_offset; table_size += pi->pi_msix.table_count * MSIX_TABLE_ENTRY_SIZE; table_size = roundup2(table_size, 4096); remaining = pi->pi_bar[baridx].size - table_offset - table_size; if (remaining > 0) { address += table_offset + table_size; if (!enabled) { if (vm_unmap_pptdev_mmio(ctx, sc->pptfd, address, remaining) != 0) warnx("pci_passthru: unmap_pptdev_mmio failed"); } else { if (vm_map_pptdev_mmio(ctx, sc->pptfd, address, remaining, sc->psc_bar[baridx].addr + table_offset + table_size) != 0) warnx("pci_passthru: map_pptdev_mmio failed"); } } } static void passthru_mmio_addr(struct vmctx *ctx, struct pci_devinst *pi, int baridx, int enabled, uint64_t address) { struct passthru_softc *sc; sc = pi->pi_arg; if (!enabled) { if (vm_unmap_pptdev_mmio(ctx, sc->pptfd, address, sc->psc_bar[baridx].size) != 0) warnx("pci_passthru: unmap_pptdev_mmio failed"); } else { if (vm_map_pptdev_mmio(ctx, sc->pptfd, address, sc->psc_bar[baridx].size, sc->psc_bar[baridx].addr) != 0) warnx("pci_passthru: map_pptdev_mmio failed"); } } static void passthru_addr_rom(struct pci_devinst *const pi, const int idx, const int enabled) { const uint64_t addr = pi->pi_bar[idx].addr; const uint64_t size = pi->pi_bar[idx].size; if (!enabled) { if (vm_munmap_memseg(pi->pi_vmctx, addr, size) != 0) { errx(4, "%s: munmap_memseg @ [%016lx - %016lx] failed", __func__, addr, addr + size); } } else { if (vm_mmap_memseg(pi->pi_vmctx, addr, VM_PCIROM, pi->pi_romoffset, size, PROT_READ | PROT_EXEC) != 0) { errx(4, "%s: mmap_memseg @ [%016lx - %016lx] failed", __func__, addr, addr + size); } } } static void passthru_addr(struct pci_devinst *pi, int baridx, int enabled, uint64_t address) { struct vmctx *ctx = pi->pi_vmctx; switch (pi->pi_bar[baridx].type) { case PCIBAR_IO: /* IO BARs are emulated */ break; case PCIBAR_ROM: passthru_addr_rom(pi, baridx, enabled); break; case PCIBAR_MEM32: case PCIBAR_MEM64: if (baridx == pci_msix_table_bar(pi)) passthru_msix_addr(ctx, pi, baridx, enabled, address); else passthru_mmio_addr(ctx, pi, baridx, enabled, address); break; default: errx(4, "%s: invalid BAR type %d", __func__, pi->pi_bar[baridx].type); } } static const struct pci_devemu passthru = { .pe_emu = "passthru", .pe_init = passthru_init, .pe_legacy_config = passthru_legacy_config, .pe_cfgwrite = passthru_cfgwrite, .pe_cfgread = passthru_cfgread, .pe_barwrite = passthru_write, .pe_barread = passthru_read, .pe_baraddr = passthru_addr, }; PCI_EMUL_SET(passthru); /* * This isn't the right place for these functions which, on FreeBSD, can * read or write from arbitrary devices. They are not supported on illumos; * not least because bhyve is generally run in a non-global zone which doesn't * have access to the devinfo tree. */ uint32_t pci_host_read_config(const struct pcisel *sel __unused, long reg __unused, int width __unused) { return (-1); } void pci_host_write_config(const struct pcisel *sel __unused, long reg __unused, int width __unused, uint32_t data __unused) { errx(4, "pci_host_write_config() unimplemented on illumos"); } /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2020 Beckhoff Automation GmbH & Co. KG * Author: Corvin Khne */ #ifndef _PCI_PASSTHRU_H_ #define _PCI_PASSTHRU_H_ #include #include "pci_emul.h" struct passthru_softc; typedef int (*cfgread_handler)(struct passthru_softc *sc, struct pci_devinst *pi, int coff, int bytes, uint32_t *rv); typedef int (*cfgwrite_handler)(struct passthru_softc *sc, struct pci_devinst *pi, int coff, int bytes, uint32_t val); uint32_t pci_host_read_config(const struct pcisel *sel, long reg, int width); void pci_host_write_config(const struct pcisel *sel, long reg, int width, uint32_t data); int passthru_cfgread_emulate(struct passthru_softc *sc, struct pci_devinst *pi, int coff, int bytes, uint32_t *rv); int passthru_cfgwrite_emulate(struct passthru_softc *sc, struct pci_devinst *pi, int coff, int bytes, uint32_t val); int set_pcir_handler(struct passthru_softc *sc, int reg, int len, cfgread_handler rhandler, cfgwrite_handler whandler); #endif /* _PCI_PASSTHRU_H_ */ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2012 NetApp, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include #include #include "bhyverun.h" #include "config.h" #include "debug.h" #include "pci_emul.h" #include "uart_emul.h" /* * Pick a PCI vid/did of a chip with a single uart at * BAR0, that most versions of FreeBSD can understand: * Siig CyberSerial 1-port. */ #define COM_VENDOR 0x131f #define COM_DEV 0x2000 static void pci_uart_intr_assert(void *arg) { struct pci_devinst *pi = arg; pci_lintr_assert(pi); } static void pci_uart_intr_deassert(void *arg) { struct pci_devinst *pi = arg; pci_lintr_deassert(pi); } static void pci_uart_write(struct pci_devinst *pi, int baridx, uint64_t offset, int size, uint64_t value) { assert(baridx == 0); assert(size == 1); uart_ns16550_write(pi->pi_arg, offset, value); } static uint64_t pci_uart_read(struct pci_devinst *pi, int baridx, uint64_t offset, int size) { uint8_t val; assert(baridx == 0); assert(size == 1); val = uart_ns16550_read(pi->pi_arg, offset); return (val); } static int pci_uart_legacy_config(nvlist_t *nvl, const char *opts) { if (opts != NULL) set_config_value_node(nvl, "path", opts); return (0); } static int pci_uart_init(struct pci_devinst *pi, nvlist_t *nvl) { struct uart_ns16550_softc *sc; const char *device; pci_emul_alloc_bar(pi, 0, PCIBAR_IO, UART_NS16550_IO_BAR_SIZE); pci_lintr_request(pi); /* initialize config space */ pci_set_cfgdata16(pi, PCIR_DEVICE, COM_DEV); pci_set_cfgdata16(pi, PCIR_VENDOR, COM_VENDOR); pci_set_cfgdata8(pi, PCIR_CLASS, PCIC_SIMPLECOMM); pci_set_cfgdata8(pi, PCIR_SUBCLASS, PCIS_SIMPLECOMM_UART); pci_set_cfgdata8(pi, PCIR_PROGIF, PCIP_SIMPLECOMM_UART_16550A); sc = uart_ns16550_init(pci_uart_intr_assert, pci_uart_intr_deassert, pi); pi->pi_arg = sc; device = get_config_value_node(nvl, "path"); if (device != NULL && uart_ns16550_tty_open(sc, device) != 0) { EPRINTLN("Unable to initialize backend '%s' for " "pci uart at %d:%d", device, pi->pi_slot, pi->pi_func); return (-1); } return (0); } static const struct pci_devemu pci_de_com = { .pe_emu = "uart", .pe_init = pci_uart_init, .pe_legacy_config = pci_uart_legacy_config, .pe_barwrite = pci_uart_write, .pe_barread = pci_uart_read }; PCI_EMUL_SET(pci_de_com); /*- * Copyright (c) 2015 iXsystems Inc. * Copyright (c) 2017-2018 Jakub Klama * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer * in this position and unchanged. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * This file and its contents are supplied under the terms of the * Common Development and Distribution License ("CDDL"), version 1.0. * You may only use this file in accordance with the terms of version * 1.0 of the CDDL. * * A full copy of the text of the CDDL should have accompanied this * source. A copy of the CDDL is also available via the Internet at * http://www.illumos.org/license/CDDL. */ /* This file is dual-licensed; see usr/src/contrib/bhyve/LICENSE */ /* * Copyright 2026 Oxide Computer Company */ /* * VirtIO filesystem passthrough using 9p protocol. */ #include #include #include #ifndef __FreeBSD__ #include #endif #ifndef WITHOUT_CAPSICUM #include #endif #include #include #include #include #include #include #include #include #include #include #include "bhyverun.h" #include "config.h" #include "debug.h" #include "pci_emul.h" #include "virtio.h" #ifndef __FreeBSD__ #include "privileges.h" #endif #define VT9P_MAX_IOV 128 #define VT9P_RINGSZ 256 #define VT9P_MAXTAGSZ 256 #define VT9P_CONFIGSPACESZ (VT9P_MAXTAGSZ + sizeof(uint16_t)) static int pci_vt9p_debug; #define DPRINTF(params) if (pci_vt9p_debug) printf params #define WPRINTF(params) printf params /* Capability bits */ #define VT9P_MOUNT_TAG (1 << 0) /* Mount point in config var */ /* * Per-device softc */ struct pci_vt9p_softc { struct virtio_softc vsc_vs; struct vqueue_info vsc_vq; pthread_mutex_t vsc_mtx; uint64_t vsc_cfg; uint64_t vsc_features; char * vsc_rootpath; struct pci_vt9p_config * vsc_config; struct l9p_backend * vsc_fs_backend; struct l9p_server * vsc_server; struct l9p_connection * vsc_conn; }; struct pci_vt9p_request { struct pci_vt9p_softc * vsr_sc; struct iovec * vsr_iov; size_t vsr_niov; size_t vsr_respidx; size_t vsr_iolen; uint16_t vsr_idx; }; struct pci_vt9p_config { uint16_t tag_len; #ifdef __FreeBSD__ char tag[0]; #else char tag[VT9P_MAXTAGSZ]; #endif } __attribute__((packed)); static int pci_vt9p_send(struct l9p_request *, const struct iovec *, const size_t, const size_t, void *); static void pci_vt9p_drop(struct l9p_request *, const struct iovec *, size_t, void *); static void pci_vt9p_reset(void *); static void pci_vt9p_notify(void *, struct vqueue_info *); static int pci_vt9p_cfgread(void *, int, int, uint32_t *); static void pci_vt9p_neg_features(void *, uint64_t *); static virtio_capstr_t vt9p_caps[] = { { VT9P_MOUNT_TAG, "VT9P_MOUNT_TAG" }, }; static struct virtio_consts vt9p_vi_consts = { .vc_name = "vt9p", .vc_nvq = 1, .vc_cfgsize = VT9P_CONFIGSPACESZ, .vc_reset = pci_vt9p_reset, .vc_qnotify = pci_vt9p_notify, .vc_cfgread = pci_vt9p_cfgread, .vc_apply_features = pci_vt9p_neg_features, .vc_hv_caps_legacy = VT9P_MOUNT_TAG, .vc_hv_caps_modern = VT9P_MOUNT_TAG, .vc_capstr = vt9p_caps, .vc_ncapstr = ARRAY_SIZE(vt9p_caps), }; static void pci_vt9p_reset(void *vsc) { struct pci_vt9p_softc *sc; sc = vsc; DPRINTF(("vt9p: device reset requested !\n")); vi_reset_dev(&sc->vsc_vs); } static void pci_vt9p_neg_features(void *vsc, uint64_t *negotiated_features) { struct pci_vt9p_softc *sc = vsc; sc->vsc_features = *negotiated_features; } static int pci_vt9p_cfgread(void *vsc, int offset, int size, uint32_t *retval) { struct pci_vt9p_softc *sc = vsc; void *ptr; ptr = (uint8_t *)sc->vsc_config + offset; memcpy(retval, ptr, size); return (0); } static int pci_vt9p_get_buffer(struct l9p_request *req, struct iovec *iov, size_t *niov, void *arg __unused) { struct pci_vt9p_request *preq = req->lr_aux; size_t n = preq->vsr_niov - preq->vsr_respidx; memcpy(iov, preq->vsr_iov + preq->vsr_respidx, n * sizeof(struct iovec)); *niov = n; return (0); } static int pci_vt9p_send(struct l9p_request *req, const struct iovec *iov __unused, const size_t niov __unused, const size_t iolen, void *arg __unused) { struct pci_vt9p_request *preq = req->lr_aux; struct pci_vt9p_softc *sc = preq->vsr_sc; preq->vsr_iolen = iolen; pthread_mutex_lock(&sc->vsc_mtx); vq_relchain(&sc->vsc_vq, preq->vsr_idx, preq->vsr_iolen); vq_endchains(&sc->vsc_vq, 1); pthread_mutex_unlock(&sc->vsc_mtx); free(preq); return (0); } static void pci_vt9p_drop(struct l9p_request *req, const struct iovec *iov __unused, size_t niov __unused, void *arg __unused) { struct pci_vt9p_request *preq = req->lr_aux; struct pci_vt9p_softc *sc = preq->vsr_sc; pthread_mutex_lock(&sc->vsc_mtx); vq_relchain(&sc->vsc_vq, preq->vsr_idx, 0); vq_endchains(&sc->vsc_vq, 1); pthread_mutex_unlock(&sc->vsc_mtx); free(preq); } static void pci_vt9p_notify(void *vsc, struct vqueue_info *vq) { struct iovec iov[VT9P_MAX_IOV]; struct pci_vt9p_softc *sc; struct pci_vt9p_request *preq; struct vi_req req; int n; sc = vsc; while (vq_has_descs(vq)) { n = vq_getchain(vq, iov, VT9P_MAX_IOV, &req); assert(n >= 1 && n <= VT9P_MAX_IOV); preq = calloc(1, sizeof(struct pci_vt9p_request)); #ifndef __FreeBSD__ if (preq == NULL) { EPRINTLN("virtio-9p: allocation failure: %s", strerror(errno)); break; } #endif preq->vsr_sc = sc; preq->vsr_idx = req.idx; preq->vsr_iov = iov; preq->vsr_niov = n; preq->vsr_respidx = req.readable; for (int i = 0; i < n; i++) { DPRINTF(("vt9p: vt9p_notify(): desc%d base=%p, " "len=%zu\r\n", i, iov[i].iov_base, iov[i].iov_len)); } l9p_connection_recv(sc->vsc_conn, iov, preq->vsr_respidx, preq); } } static int pci_vt9p_legacy_config(nvlist_t *nvl, const char *opts) { char *sharename = NULL, *tofree, *token, *tokens; if (opts == NULL) return (0); tokens = tofree = strdup(opts); while ((token = strsep(&tokens, ",")) != NULL) { if (strchr(token, '=') != NULL) { if (sharename != NULL) { EPRINTLN( "virtio-9p: more than one share name given"); return (-1); } sharename = strsep(&token, "="); set_config_value_node(nvl, "sharename", sharename); set_config_value_node(nvl, "path", token); } else set_config_bool_node(nvl, token, true); } free(tofree); return (0); } static int pci_vt9p_init(struct pci_devinst *pi, nvlist_t *nvl) { struct pci_vt9p_softc *sc; const char *value; const char *sharename; int rootfd; bool ro; #ifndef WITHOUT_CAPSICUM cap_rights_t rootcap; #endif ro = get_config_bool_node_default(nvl, "ro", false); #ifndef __FreeBSD__ illumos_priv_add_min(PRIV_FILE_DAC_READ, "vt9p"); illumos_priv_add_min(PRIV_FILE_DAC_SEARCH, "vt9p"); if (!ro) { illumos_priv_add_min(PRIV_FILE_CHOWN, "vt9p"); illumos_priv_add_min(PRIV_FILE_CHOWN_SELF, "vt9p"); illumos_priv_add_min(PRIV_FILE_WRITE, "vt9p"); illumos_priv_add_min(PRIV_FILE_DAC_WRITE, "vt9p"); illumos_priv_add_min(PRIV_FILE_OWNER, "vt9p"); illumos_priv_add_min(PRIV_FILE_LINK_ANY, "vt9p"); } #endif value = get_config_value_node(nvl, "path"); if (value == NULL) { EPRINTLN("virtio-9p: path required"); return (1); } rootfd = open(value, O_DIRECTORY); if (rootfd < 0) { EPRINTLN("virtio-9p: failed to open '%s': %s", value, strerror(errno)); return (-1); } sharename = get_config_value_node(nvl, "sharename"); if (sharename == NULL) { EPRINTLN("virtio-9p: share name required"); return (1); } if (strlen(sharename) > VT9P_MAXTAGSZ) { EPRINTLN("virtio-9p: share name too long"); return (1); } sc = calloc(1, sizeof(struct pci_vt9p_softc)); #ifdef __FreeBSD__ sc->vsc_config = calloc(1, sizeof(struct pci_vt9p_config) + VT9P_MAXTAGSZ); #else if (sc == NULL) { EPRINTLN("virtio-9p: soft state allocation failure: %s", strerror(errno)); return (1); } sc->vsc_config = calloc(1, sizeof(struct pci_vt9p_config)); if (sc == NULL) { EPRINTLN("virtio-9p: vsc_config allocation failure: %s", strerror(errno)); return (1); } #endif pthread_mutex_init(&sc->vsc_mtx, NULL); #ifndef WITHOUT_CAPSICUM cap_rights_init(&rootcap, CAP_LOOKUP, CAP_ACL_CHECK, CAP_ACL_DELETE, CAP_ACL_GET, CAP_ACL_SET, CAP_READ, CAP_WRITE, CAP_SEEK, CAP_FSTAT, CAP_CREATE, CAP_FCHMODAT, CAP_FCHOWNAT, CAP_FTRUNCATE, CAP_LINKAT_SOURCE, CAP_LINKAT_TARGET, CAP_MKDIRAT, CAP_MKNODAT, CAP_PREAD, CAP_PWRITE, CAP_RENAMEAT_SOURCE, CAP_RENAMEAT_TARGET, CAP_SEEK, CAP_SYMLINKAT, CAP_UNLINKAT, CAP_EXTATTR_DELETE, CAP_EXTATTR_GET, CAP_EXTATTR_LIST, CAP_EXTATTR_SET, CAP_FUTIMES, CAP_FSTATFS, CAP_FSYNC, CAP_FPATHCONF); if (cap_rights_limit(rootfd, &rootcap) != 0) return (1); #endif sc->vsc_config->tag_len = (uint16_t)strlen(sharename); memcpy(sc->vsc_config->tag, sharename, sc->vsc_config->tag_len); if (l9p_backend_fs_init(&sc->vsc_fs_backend, rootfd, ro) != 0) { errno = ENXIO; return (1); } if (l9p_server_init(&sc->vsc_server, sc->vsc_fs_backend) != 0) { errno = ENXIO; return (1); } if (l9p_connection_init(sc->vsc_server, &sc->vsc_conn) != 0) { errno = EIO; return (1); } sc->vsc_conn->lc_msize = L9P_MAX_IOV * PAGE_SIZE; sc->vsc_conn->lc_lt.lt_get_response_buffer = pci_vt9p_get_buffer; sc->vsc_conn->lc_lt.lt_send_response = pci_vt9p_send; sc->vsc_conn->lc_lt.lt_drop_response = pci_vt9p_drop; vi_softc_linkup(&sc->vsc_vs, &vt9p_vi_consts, sc, pi, &sc->vsc_vq); sc->vsc_vs.vs_mtx = &sc->vsc_mtx; sc->vsc_vq.vq_qsize = VT9P_RINGSZ; /* initialize config space */ vi_pci_init(pi, VIRTIO_MODE_TRANSITIONAL, VIRTIO_DEV_9P, VIRTIO_ID_9P, PCIC_STORAGE); if (!vi_intr_init(&sc->vsc_vs, fbsdrun_virtio_msix())) return (1); if (!vi_pcibar_setup(&sc->vsc_vs)) return (1); return (0); } static const struct pci_devemu pci_de_v9p = { .pe_emu = "virtio-9p", .pe_legacy_config = pci_vt9p_legacy_config, .pe_init = pci_vt9p_init, .pe_cfgwrite = vi_pci_cfgwrite, .pe_cfgread = vi_pci_cfgread, .pe_barwrite = vi_pci_write, .pe_barread = vi_pci_read }; PCI_EMUL_SET(pci_de_v9p); /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2011 NetApp, Inc. * All rights reserved. * Copyright 2020-2021 Joyent, Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * This file and its contents are supplied under the terms of the * Common Development and Distribution License ("CDDL"), version 1.0. * You may only use this file in accordance with the terms of version * 1.0 of the CDDL. * * A full copy of the text of the CDDL should have accompanied this * source. A copy of the CDDL is also available via the Internet at * http://www.illumos.org/license/CDDL. */ /* This file is dual-licensed; see usr/src/contrib/bhyve/LICENSE */ /* * Copyright 2014 Pluribus Networks Inc. * Copyright 2026 Oxide Computer Company */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "bhyverun.h" #include "config.h" #include "debug.h" #include "iov.h" #include "pci_emul.h" #include "virtio.h" #include "block_if.h" #define VTBLK_BSIZE 512 #define VTBLK_RINGSZ 128 _Static_assert(VTBLK_RINGSZ <= BLOCKIF_RING_MAX, "Each ring entry must be able to queue a request"); #define VTBLK_S_OK 0 #define VTBLK_S_IOERR 1 #define VTBLK_S_UNSUPP 2 #define VTBLK_BLK_ID_BYTES 20 + 1 /* Capability bits */ #define VTBLK_F_BARRIER (1 << 0) /* Does host support barriers? */ #define VTBLK_F_SIZE_MAX (1 << 1) /* Indicates maximum segment size */ #define VTBLK_F_SEG_MAX (1 << 2) /* Indicates maximum # of segments */ #define VTBLK_F_GEOMETRY (1 << 4) /* Legacy geometry available */ #define VTBLK_F_RO (1 << 5) /* Disk is read-only */ #define VTBLK_F_BLK_SIZE (1 << 6) /* Block size of disk is available */ #define VTBLK_F_SCSI (1 << 7) /* Supports scsi command passthru */ #define VTBLK_F_FLUSH (1 << 9) /* Writeback enabled after reset */ #define VTBLK_F_WCE (1 << 9) /* Legacy alias for FLUSH */ #define VTBLK_F_TOPOLOGY (1 << 10) /* Topology information available */ #define VTBLK_F_CONFIG_WCE (1 << 11) /* Writeback mode avail in config */ #define VTBLK_F_MQ (1 << 12) /* Multi-Queue */ #define VTBLK_F_DISCARD (1 << 13) /* Trim blocks */ #define VTBLK_F_WRITE_ZEROES (1 << 14) /* Write zeros */ /* * Host capabilities */ #define VTBLK_S_HOSTCAPS \ (VTBLK_F_SEG_MAX | \ VTBLK_F_BLK_SIZE | \ VTBLK_F_FLUSH | \ VTBLK_F_TOPOLOGY | \ VIRTIO_RING_F_INDIRECT_DESC) /* indirect descriptors */ /* * The current blockif_delete() interface only allows a single delete * request at a time. */ #define VTBLK_MAX_DISCARD_SEG 1 /* * An arbitrary limit to prevent excessive latency due to large * delete requests. */ #define VTBLK_MAX_DISCARD_SECT ((16 << 20) / VTBLK_BSIZE) /* 16 MiB */ /* * Config space "registers" */ struct vtblk_config { uint64_t vbc_capacity; uint32_t vbc_size_max; uint32_t vbc_seg_max; struct { uint16_t cylinders; uint8_t heads; uint8_t sectors; } vbc_geometry; uint32_t vbc_blk_size; struct { uint8_t physical_block_exp; uint8_t alignment_offset; uint16_t min_io_size; uint32_t opt_io_size; } vbc_topology; uint8_t vbc_writeback; uint8_t unused0[1]; uint16_t num_queues; uint32_t max_discard_sectors; uint32_t max_discard_seg; uint32_t discard_sector_alignment; uint32_t max_write_zeroes_sectors; uint32_t max_write_zeroes_seg; uint8_t write_zeroes_may_unmap; uint8_t unused1[3]; } __packed; /* * Fixed-size block header */ struct virtio_blk_hdr { #define VBH_OP_READ 0 #define VBH_OP_WRITE 1 #define VBH_OP_SCSI_CMD 2 #define VBH_OP_SCSI_CMD_OUT 3 #define VBH_OP_FLUSH 4 #define VBH_OP_FLUSH_OUT 5 #define VBH_OP_IDENT 8 #define VBH_OP_DISCARD 11 #define VBH_OP_WRITE_ZEROES 13 #define VBH_FLAG_BARRIER 0x80000000 /* OR'ed into vbh_type */ uint32_t vbh_type; uint32_t vbh_ioprio; uint64_t vbh_sector; } __packed; /* * Debug printf */ static int pci_vtblk_debug; #define DPRINTF(params) if (pci_vtblk_debug) PRINTLN params #define WPRINTF(params) PRINTLN params struct pci_vtblk_ioreq { struct blockif_req io_req; struct pci_vtblk_softc *io_sc; uint8_t *io_status; uint16_t io_idx; }; struct virtio_blk_discard_write_zeroes { uint64_t sector; uint32_t num_sectors; struct { uint32_t unmap:1; uint32_t reserved:31; } flags; }; /* * Per-device softc */ struct pci_vtblk_softc { struct virtio_softc vbsc_vs; pthread_mutex_t vsc_mtx; struct vqueue_info vbsc_vq; struct vtblk_config vbsc_cfg; struct virtio_consts vbsc_consts; struct blockif_ctxt *bc; #ifndef __FreeBSD__ int vbsc_wce; #endif char vbsc_ident[VTBLK_BLK_ID_BYTES]; struct pci_vtblk_ioreq vbsc_ios[VTBLK_RINGSZ]; }; static void pci_vtblk_reset(void *); static void pci_vtblk_notify(void *, struct vqueue_info *); static int pci_vtblk_cfgread(void *, int, int, uint32_t *); static int pci_vtblk_cfgwrite(void *, int, int, uint32_t); #ifndef __FreeBSD__ static void pci_vtblk_apply_feats(void *, uint64_t *); #endif static virtio_capstr_t vtblk_caps[] = { { VTBLK_F_BARRIER, "VTBLK_F_BARRIER" }, { VTBLK_F_SIZE_MAX, "VTBLK_F_SIZE_MAX" }, { VTBLK_F_SEG_MAX, "VTBLK_F_SEG_MAX" }, { VTBLK_F_GEOMETRY, "VTBLK_F_GEOMETRY" }, { VTBLK_F_RO, "VTBLK_F_RO" }, { VTBLK_F_BLK_SIZE, "VTBLK_F_BLK_SIZE" }, { VTBLK_F_SCSI, "VTBLK_F_SCSI" }, { VTBLK_F_FLUSH, "VTBLK_F_FLUSH" }, { VTBLK_F_WCE, "VTBLK_F_WCE" }, { VTBLK_F_TOPOLOGY, "VTBLK_F_TOPOLOGY" }, { VTBLK_F_CONFIG_WCE, "VTBLK_F_CONFIG_WCE" }, { VTBLK_F_MQ, "VTBLK_F_MQ" }, { VTBLK_F_DISCARD, "VTBLK_F_DISCARD" }, { VTBLK_F_WRITE_ZEROES, "VTBLK_F_WRITE_ZEROES" }, }; static struct virtio_consts vtblk_vi_consts = { .vc_name = "vtblk", .vc_nvq = 1, .vc_cfgsize = sizeof (struct vtblk_config), .vc_reset = pci_vtblk_reset, .vc_qnotify = pci_vtblk_notify, .vc_cfgread = pci_vtblk_cfgread, .vc_cfgwrite = pci_vtblk_cfgwrite, #ifndef __FreeBSD__ .vc_apply_features = pci_vtblk_apply_feats, #else .vc_apply_features = NULL, #endif .vc_hv_caps_legacy = VTBLK_S_HOSTCAPS, .vc_hv_caps_modern = VTBLK_S_HOSTCAPS, .vc_capstr = vtblk_caps, .vc_ncapstr = ARRAY_SIZE(vtblk_caps), }; static void pci_vtblk_reset(void *vsc) { struct pci_vtblk_softc *sc = vsc; DPRINTF(("vtblk: device reset requested !")); vi_reset_dev(&sc->vbsc_vs); #ifndef __FreeBSD__ /* Disable write cache until FLUSH feature is negotiated */ (void) blockif_set_wce(sc->bc, 0); sc->vbsc_wce = 0; #endif } static void pci_vtblk_done_locked(struct pci_vtblk_ioreq *io, int err) { struct pci_vtblk_softc *sc = io->io_sc; /* convert errno into a virtio block error return */ if (err == EOPNOTSUPP || err == ENOSYS) *io->io_status = VTBLK_S_UNSUPP; else if (err != 0) *io->io_status = VTBLK_S_IOERR; else *io->io_status = VTBLK_S_OK; /* * Return the descriptor back to the host. * We wrote 1 byte (our status) to host. */ vq_relchain(&sc->vbsc_vq, io->io_idx, 1); vq_endchains(&sc->vbsc_vq, 0); } static void pci_vtblk_done(struct blockif_req *br, int err) { struct pci_vtblk_ioreq *io = br->br_param; struct pci_vtblk_softc *sc = io->io_sc; pthread_mutex_lock(&sc->vsc_mtx); pci_vtblk_done_locked(io, err); pthread_mutex_unlock(&sc->vsc_mtx); } static void pci_vtblk_proc(struct pci_vtblk_softc *sc, struct vqueue_info *vq) { struct virtio_blk_hdr vbh; struct pci_vtblk_ioreq *io; int niov; int err; bool writeop; int type; struct vi_req req; struct iovec iov[BLOCKIF_IOV_MAX + 2]; struct iovec *siov; iov_bunch_t iob; size_t len; niov = vq_getchain(vq, iov, BLOCKIF_IOV_MAX + 2, &req); /* * As a transitional device we cannot make any assumptions about the * descriptor layout. We know that there will always be at least two * descriptors since every request contains at least one RO and one RW * descriptor but it's perfectly valid (although extremely unlikely) * for a driver to combine things like the last data block in a read * request with the final status byte, or the first data block with * the header in a write. */ if (niov < 2 || niov >= BLOCKIF_IOV_MAX + 2 || req.readable == 0 || req.writable == 0) { EPRINTLN("vioblk: invalid chain niov=0x%x ro=%x rw=%x", niov, req.readable, req.writable); vq_relchain(vq, req.idx, 0); return; } len = iov_bunch_init(&iob, iov, niov); if (!iov_bunch_copy(&iob, &vbh, sizeof (vbh))) { EPRINTLN("vioblk: control header copy failed, chain len 0x%x", len); vq_relchain(vq, req.idx, 0); return; } io = &sc->vbsc_ios[req.idx]; io->io_req.br_offset = vbh.vbh_sector * VTBLK_BSIZE; /* * The IO status byte is the last byte in the last descriptor which we * know is writable having checked above. */ siov = &iov[niov - 1]; io->io_status = (uint8_t *)&siov->iov_base[siov->iov_len - 1]; iob.ib_remain--; /* * The guest should not be setting the BARRIER flag because * we don't advertise the capability. */ type = vbh.vbh_type & ~VBH_FLAG_BARRIER; writeop = (type == VBH_OP_WRITE || type == VBH_OP_DISCARD); switch (type) { case VBH_OP_DISCARD: { struct virtio_blk_discard_write_zeroes discard; /* * We currently only support a single request, as advertised in * the configuration space. If the guest has submitted a * request that doesn't conform to the requirements, we return * a error. */ if (!iov_bunch_copy(&iob, &discard, sizeof (discard)) || iob.ib_remain != 0) { EPRINTLN("vioblk: bad discard message"); pci_vtblk_done_locked(io, EINVAL); return; } /* * virtio v1.1 5.2.6.2: * The device MUST set the status byte to VIRTIO_BLK_S_UNSUPP * for discard and write zeroes commands if any unknown flag is * set. Furthermore, the device MUST set the status byte to * VIRTIO_BLK_S_UNSUPP for discard commands if the unmap flag * is set. * * Currently there are no known flags for a DISCARD request. */ if (discard.flags.unmap != 0 || discard.flags.reserved != 0) { pci_vtblk_done_locked(io, ENOTSUP); return; } /* Make sure the request doesn't exceed our size limit */ if (discard.num_sectors > VTBLK_MAX_DISCARD_SECT) { pci_vtblk_done_locked(io, EINVAL); return; } io->io_req.br_iovcnt = 0; io->io_req.br_offset = discard.sector * VTBLK_BSIZE; io->io_req.br_resid = discard.num_sectors * VTBLK_BSIZE; DPRINTF(("virtio-block: discard op, %zd bytes, offset %ld", io->io_req.br_resid, io->io_req.br_offset)); err = blockif_delete(sc->bc, &io->io_req); return; } case VBH_OP_IDENT: { char *buf; size_t len; int err; len = iob.ib_remain; buf = calloc(len, sizeof (char)); if (buf == NULL) { pci_vtblk_done_locked(io, ENOMEM); return; } len = MIN(len, sizeof (sc->vbsc_ident)); strncpy(buf, sc->vbsc_ident, len); DPRINTF(("virtio-block: ident op, '%.*s'", len, buf)); err = buf_to_iov_bunch(&iob, buf, len) ? 0 : ENOSPC; free(buf); pci_vtblk_done_locked(io, err); return; } default: break; } /* * Accumulate the remainder of the data into the IO request iov. */ io->io_req.br_resid = iob.ib_remain; iov_bunch_to_iov(&iob, (struct iovec *)&io->io_req.br_iov, &io->io_req.br_iovcnt, ARRAY_SIZE(io->io_req.br_iov)); DPRINTF(("virtio-block: %s op, %zd bytes, %d segs, offset %ld", writeop ? "write" : "read", io->io_req.br_resid, io->io_req.br_iovcnt, io->io_req.br_offset)); switch (type) { case VBH_OP_READ: err = blockif_read(sc->bc, &io->io_req); break; case VBH_OP_WRITE: err = blockif_write(sc->bc, &io->io_req); break; case VBH_OP_FLUSH: case VBH_OP_FLUSH_OUT: err = blockif_flush(sc->bc, &io->io_req); break; default: pci_vtblk_done_locked(io, EOPNOTSUPP); return; } assert(err == 0); } static void pci_vtblk_notify(void *vsc, struct vqueue_info *vq) { struct pci_vtblk_softc *sc = vsc; while (vq_has_descs(vq)) pci_vtblk_proc(sc, vq); } static void pci_vtblk_resized(struct blockif_ctxt *bctxt __unused, void *arg, size_t new_size) { struct pci_vtblk_softc *sc; sc = arg; sc->vbsc_cfg.vbc_capacity = new_size / VTBLK_BSIZE; /* 512-byte units */ vq_devcfg_changed(&sc->vbsc_vs); } static int pci_vtblk_init(struct pci_devinst *pi, nvlist_t *nvl) { char bident[sizeof ("XXX:XXX")]; struct blockif_ctxt *bctxt; const char *path, *serial; MD5_CTX mdctx; uchar_t digest[16]; struct pci_vtblk_softc *sc; off_t size; int i, sectsz, sts, sto; /* * The supplied backing file has to exist */ snprintf(bident, sizeof (bident), "%u:%u", pi->pi_slot, pi->pi_func); bctxt = blockif_open(nvl, bident); if (bctxt == NULL) { perror("Could not open backing file"); return (1); } if (blockif_add_boot_device(pi, bctxt)) { perror("Invalid boot device"); return (1); } size = blockif_size(bctxt); sectsz = blockif_sectsz(bctxt); blockif_psectsz(bctxt, &sts, &sto); sc = calloc(1, sizeof (struct pci_vtblk_softc)); sc->bc = bctxt; if (get_config_bool_default("virtio.blk.debug", false)) pci_vtblk_debug = 1; vi_set_debug(&sc->vbsc_vs, pci_vtblk_debug); for (i = 0; i < VTBLK_RINGSZ; i++) { struct pci_vtblk_ioreq *io = &sc->vbsc_ios[i]; io->io_req.br_callback = pci_vtblk_done; io->io_req.br_param = io; io->io_sc = sc; io->io_idx = i; } bcopy(&vtblk_vi_consts, &sc->vbsc_consts, sizeof (vtblk_vi_consts)); if (blockif_candelete(sc->bc)) { sc->vbsc_consts.vc_hv_caps_legacy |= VTBLK_F_DISCARD; sc->vbsc_consts.vc_hv_caps_modern |= VTBLK_F_DISCARD; } #ifndef __FreeBSD__ /* Disable write cache until FLUSH feature is negotiated */ (void) blockif_set_wce(sc->bc, 0); sc->vbsc_wce = 0; #endif pthread_mutex_init(&sc->vsc_mtx, NULL); /* init virtio softc and virtqueues */ vi_softc_linkup(&sc->vbsc_vs, &sc->vbsc_consts, sc, pi, &sc->vbsc_vq); sc->vbsc_vs.vs_mtx = &sc->vsc_mtx; sc->vbsc_vq.vq_qsize = VTBLK_RINGSZ; /* sc->vbsc_vq.vq_notify = we have no per-queue notify */ /* * If an explicit identifier is not given, create an * identifier using parts of the md5 sum of the filename. */ bzero(sc->vbsc_ident, VTBLK_BLK_ID_BYTES); if ((serial = get_config_value_node(nvl, "serial")) != NULL || (serial = get_config_value_node(nvl, "ser")) != NULL) { strlcpy(sc->vbsc_ident, serial, VTBLK_BLK_ID_BYTES); } else { path = get_config_value_node(nvl, "path"); MD5Init(&mdctx); MD5Update(&mdctx, path, strlen(path)); MD5Final(digest, &mdctx); snprintf(sc->vbsc_ident, VTBLK_BLK_ID_BYTES, "BHYVE-%02X%02X-%02X%02X-%02X%02X", digest[0], digest[1], digest[2], digest[3], digest[4], digest[5]); } /* setup virtio block config space */ sc->vbsc_cfg.vbc_capacity = size / VTBLK_BSIZE; /* 512-byte units */ sc->vbsc_cfg.vbc_size_max = 0; /* not negotiated */ /* * If Linux is presented with a seg_max greater than the virtio queue * size, it can stumble into situations where it violates its own * invariants and panics. For safety, we keep seg_max clamped, paying * heed to the two extra descriptors needed for the header and status * of a request. */ sc->vbsc_cfg.vbc_seg_max = MIN(VTBLK_RINGSZ - 2, BLOCKIF_IOV_MAX); sc->vbsc_cfg.vbc_geometry.cylinders = 0; /* no geometry */ sc->vbsc_cfg.vbc_geometry.heads = 0; sc->vbsc_cfg.vbc_geometry.sectors = 0; sc->vbsc_cfg.vbc_blk_size = sectsz; sc->vbsc_cfg.vbc_topology.physical_block_exp = (sts > sectsz) ? (ffsll(sts / sectsz) - 1) : 0; sc->vbsc_cfg.vbc_topology.alignment_offset = (sto != 0) ? ((sts - sto) / sectsz) : 0; sc->vbsc_cfg.vbc_topology.min_io_size = 0; sc->vbsc_cfg.vbc_topology.opt_io_size = 0; sc->vbsc_cfg.vbc_writeback = 0; sc->vbsc_cfg.max_discard_sectors = VTBLK_MAX_DISCARD_SECT; sc->vbsc_cfg.max_discard_seg = VTBLK_MAX_DISCARD_SEG; sc->vbsc_cfg.discard_sector_alignment = MAX(sectsz, sts) / VTBLK_BSIZE; vi_pci_init(pi, VIRTIO_MODE_TRANSITIONAL, VIRTIO_DEV_BLOCK, VIRTIO_ID_BLOCK, PCIC_STORAGE); if (!vi_intr_init(&sc->vbsc_vs, fbsdrun_virtio_msix())) goto fail; if (!vi_pcibar_setup(&sc->vbsc_vs)) goto fail; blockif_register_resize_callback(sc->bc, pci_vtblk_resized, sc); return (0); fail: blockif_close(sc->bc); free(sc); return (1); } static int pci_vtblk_cfgwrite(void *vsc __unused, int offset, int size __unused, uint32_t value __unused) { DPRINTF(("vtblk: write to readonly reg %d", offset)); return (1); } static int pci_vtblk_cfgread(void *vsc, int offset, int size, uint32_t *retval) { struct pci_vtblk_softc *sc = vsc; void *ptr; /* our caller has already verified offset and size */ ptr = (uint8_t *)&sc->vbsc_cfg + offset; memcpy(retval, ptr, size); return (0); } #ifndef __FreeBSD__ void pci_vtblk_apply_feats(void *vsc, uint64_t *caps) { struct pci_vtblk_softc *sc = vsc; const int wce_next = ((*caps & VTBLK_F_FLUSH) != 0) ? 1 : 0; if (sc->vbsc_wce != wce_next) { (void) blockif_set_wce(sc->bc, wce_next); sc->vbsc_wce = wce_next; } } #endif /* __FreeBSD__ */ static const struct pci_devemu pci_de_vblk = { .pe_emu = "virtio-blk", .pe_init = pci_vtblk_init, .pe_legacy_config = blockif_legacy_config, .pe_cfgwrite = vi_pci_cfgwrite, .pe_cfgread = vi_pci_cfgread, .pe_barwrite = vi_pci_write, .pe_barread = vi_pci_read, }; PCI_EMUL_SET(pci_de_vblk); /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2016 iXsystems Inc. * All rights reserved. * * This software was developed by Jakub Klama * under sponsorship from iXsystems Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer * in this position and unchanged. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * This file and its contents are supplied under the terms of the * Common Development and Distribution License ("CDDL"), version 1.0. * You may only use this file in accordance with the terms of version * 1.0 of the CDDL. * * A full copy of the text of the CDDL should have accompanied this * source. A copy of the CDDL is also available via the Internet at * http://www.illumos.org/license/CDDL. */ /* This file is dual-licensed; see usr/src/contrib/bhyve/LICENSE */ /* * Copyright 2018 Joyent, Inc. * Copyright 2025 OmniOS Community Edition (OmniOSce) Association. * Copyright 2026 Oxide Computer Company */ #include #ifndef WITHOUT_CAPSICUM #include #endif #include #include #include #include #include #ifndef __FreeBSD__ #include #include #endif #ifndef WITHOUT_CAPSICUM #include #endif #include #include #include #include #include #include #include #include #include #include #include #include #include "bhyverun.h" #include "config.h" #include "debug.h" #include "pci_emul.h" #include "virtio.h" #include "mevent.h" #include "sockstream.h" #define VTCON_RINGSZ 64 #define VTCON_MAXPORTS 16 #define VTCON_MAXQ (VTCON_MAXPORTS * 2 + 2) #define VTCON_DEVICE_READY 0 #define VTCON_DEVICE_ADD 1 #define VTCON_DEVICE_REMOVE 2 #define VTCON_PORT_READY 3 #define VTCON_CONSOLE_PORT 4 #define VTCON_CONSOLE_RESIZE 5 #define VTCON_PORT_OPEN 6 #define VTCON_PORT_NAME 7 #define VTCON_F_SIZE 0 #define VTCON_F_MULTIPORT 1 #define VTCON_F_EMERG_WRITE 2 #define VTCON_S_HOSTCAPS \ (VTCON_F_SIZE | VTCON_F_MULTIPORT | VTCON_F_EMERG_WRITE) static int pci_vtcon_debug; #define DPRINTF(params) if (pci_vtcon_debug) PRINTLN params #define WPRINTF(params) PRINTLN params struct pci_vtcon_softc; struct pci_vtcon_port; struct pci_vtcon_config; typedef void (pci_vtcon_cb_t)(struct pci_vtcon_port *, void *, struct iovec *, int); struct pci_vtcon_port { struct pci_vtcon_softc * vsp_sc; int vsp_id; const char * vsp_name; bool vsp_enabled; bool vsp_console; bool vsp_rx_ready; bool vsp_open; int vsp_rxq; int vsp_txq; void * vsp_arg; pci_vtcon_cb_t * vsp_cb; }; struct pci_vtcon_sock { struct pci_vtcon_port * vss_port; const char * vss_path; struct mevent * vss_server_evp; struct mevent * vss_conn_evp; int vss_server_fd; int vss_conn_fd; bool vss_open; }; struct pci_vtcon_softc { struct virtio_softc vsc_vs; struct vqueue_info vsc_queues[VTCON_MAXQ]; pthread_mutex_t vsc_mtx; uint64_t vsc_cfg; uint64_t vsc_features; char * vsc_rootdir; int vsc_kq; bool vsc_ready; struct pci_vtcon_port vsc_control_port; struct pci_vtcon_port vsc_ports[VTCON_MAXPORTS]; struct pci_vtcon_config *vsc_config; }; struct pci_vtcon_config { uint16_t cols; uint16_t rows; uint32_t max_nr_ports; uint32_t emerg_wr; } __attribute__((packed)); struct pci_vtcon_control { uint32_t id; uint16_t event; uint16_t value; } __attribute__((packed)); struct pci_vtcon_console_resize { uint16_t cols; uint16_t rows; } __attribute__((packed)); static void pci_vtcon_reset(void *); static void pci_vtcon_notify_rx(void *, struct vqueue_info *); static void pci_vtcon_notify_tx(void *, struct vqueue_info *); static int pci_vtcon_cfgread(void *, int, int, uint32_t *); static int pci_vtcon_cfgwrite(void *, int, int, uint32_t); static void pci_vtcon_neg_features(void *, uint64_t *); static void pci_vtcon_sock_accept(int, enum ev_type, void *); static void pci_vtcon_sock_rx(int, enum ev_type, void *); static void pci_vtcon_sock_tx(struct pci_vtcon_port *, void *, struct iovec *, int); static void pci_vtcon_control_send(struct pci_vtcon_softc *, struct pci_vtcon_control *, const void *, size_t); static void pci_vtcon_announce_port(struct pci_vtcon_port *); static void pci_vtcon_open_port(struct pci_vtcon_port *, bool); static virtio_capstr_t vtcon_caps[] = { { VTCON_F_SIZE, "VTCON_F_SIZE" }, { VTCON_F_MULTIPORT, "VTCON_F_MULTIPORT" }, { VTCON_F_EMERG_WRITE, "VTCON_F_EMERG_WRITE" }, }; static struct virtio_consts vtcon_vi_consts = { .vc_name = "vtcon", .vc_nvq = VTCON_MAXQ, .vc_cfgsize = sizeof(struct pci_vtcon_config), .vc_reset = pci_vtcon_reset, .vc_cfgread = pci_vtcon_cfgread, .vc_cfgwrite = pci_vtcon_cfgwrite, .vc_apply_features = pci_vtcon_neg_features, .vc_hv_caps_legacy = VTCON_S_HOSTCAPS, .vc_hv_caps_modern = VTCON_S_HOSTCAPS, .vc_capstr = vtcon_caps, .vc_ncapstr = ARRAY_SIZE(vtcon_caps), }; static void pci_vtcon_reset(void *vsc) { struct pci_vtcon_softc *sc; sc = vsc; DPRINTF(("vtcon: device reset requested!")); vi_reset_dev(&sc->vsc_vs); } static void pci_vtcon_neg_features(void *vsc, uint64_t *negotiated_features) { struct pci_vtcon_softc *sc = vsc; sc->vsc_features = *negotiated_features; } static int pci_vtcon_cfgread(void *vsc, int offset, int size, uint32_t *retval) { struct pci_vtcon_softc *sc = vsc; void *ptr; ptr = (uint8_t *)sc->vsc_config + offset; memcpy(retval, ptr, size); return (0); } static int pci_vtcon_cfgwrite(void *vsc __unused, int offset __unused, int size __unused, uint32_t val __unused) { return (0); } static inline struct pci_vtcon_port * pci_vtcon_vq_to_port(struct pci_vtcon_softc *sc, struct vqueue_info *vq) { uint16_t num = vq->vq_num; if (num == 0 || num == 1) return (&sc->vsc_ports[0]); if (num == 2 || num == 3) return (&sc->vsc_control_port); return (&sc->vsc_ports[(num / 2) - 1]); } static inline struct vqueue_info * pci_vtcon_port_to_vq(struct pci_vtcon_port *port, bool tx_queue) { int qnum; qnum = tx_queue ? port->vsp_txq : port->vsp_rxq; return (&port->vsp_sc->vsc_queues[qnum]); } static struct pci_vtcon_port * pci_vtcon_port_add(struct pci_vtcon_softc *sc, int port_id, const char *name, pci_vtcon_cb_t *cb, void *arg) { struct pci_vtcon_port *port; port = &sc->vsc_ports[port_id]; if (port->vsp_enabled) { errno = EBUSY; return (NULL); } port->vsp_id = port_id; port->vsp_sc = sc; port->vsp_name = name; port->vsp_cb = cb; port->vsp_arg = arg; if (port->vsp_id == 0) { /* port0 */ port->vsp_txq = 0; port->vsp_rxq = 1; } else { port->vsp_txq = (port_id + 1) * 2; port->vsp_rxq = port->vsp_txq + 1; } port->vsp_enabled = true; return (port); } static int pci_vtcon_sock_add(struct pci_vtcon_softc *sc, const char *port_name, const nvlist_t *nvl) { struct pci_vtcon_sock *sock = NULL; #ifdef __FreeBSD__ struct sockaddr_un sun; #else /* Our compiler #defines 'sun' as '1'. Awesome. */ struct sockaddr_un addr; #endif const char *name, *path; char *cp, *pathcopy; long port; #ifdef __FreeBSD__ int s = -1, fd = -1, error = 0; #else int s = -1, error = 0; #endif #ifndef WITHOUT_CAPSICUM cap_rights_t rights; #endif port = strtol(port_name, &cp, 0); if (*cp != '\0' || port < 0 || port >= VTCON_MAXPORTS) { EPRINTLN("vtcon: Invalid port %s", port_name); error = -1; goto out; } path = get_config_value_node(nvl, "path"); if (path == NULL) { EPRINTLN("vtcon: required path missing for port %ld", port); error = -1; goto out; } sock = calloc(1, sizeof(struct pci_vtcon_sock)); if (sock == NULL) { error = -1; goto out; } s = socket(AF_UNIX, SOCK_STREAM, 0); if (s < 0) { error = -1; goto out; } #ifdef __FreeBSD__ pathcopy = strdup(path); if (pathcopy == NULL) { error = -1; goto out; } fd = open(dirname(pathcopy), O_RDONLY | O_DIRECTORY); if (fd < 0) { free(pathcopy); error = -1; goto out; } sun.sun_family = AF_UNIX; sun.sun_len = sizeof(struct sockaddr_un); strcpy(pathcopy, path); strlcpy(sun.sun_path, basename(pathcopy), sizeof(sun.sun_path)); free(pathcopy); if (bindat(fd, s, (struct sockaddr *)&sun, sun.sun_len) < 0) { error = -1; goto out; } #else /* __FreeBSD__ */ /* Do a simple bind rather than the FreeBSD bindat() */ pathcopy = (char *)path; addr.sun_family = AF_UNIX; (void) strlcpy(addr.sun_path, pathcopy, sizeof (addr.sun_path)); (void) unlink(addr.sun_path); if (bind(s, (struct sockaddr *)&addr, sizeof (addr)) < 0) { error = -1; goto out; } #endif /* __FreeBSD__ */ if (fcntl(s, F_SETFL, O_NONBLOCK) < 0) { error = -1; goto out; } if (listen(s, 1) < 0) { error = -1; goto out; } #ifndef WITHOUT_CAPSICUM cap_rights_init(&rights, CAP_ACCEPT, CAP_EVENT, CAP_READ, CAP_WRITE); if (caph_rights_limit(s, &rights) == -1) errx(EX_OSERR, "Unable to apply rights for sandbox"); #endif name = get_config_value_node(nvl, "name"); if (name == NULL) { EPRINTLN("vtcon: required name missing for port %ld", port); error = -1; goto out; } sock->vss_port = pci_vtcon_port_add(sc, port, name, pci_vtcon_sock_tx, sock); if (sock->vss_port == NULL) { error = -1; goto out; } sock->vss_open = false; sock->vss_conn_fd = -1; sock->vss_server_fd = s; sock->vss_server_evp = mevent_add(s, EVF_READ, pci_vtcon_sock_accept, sock); if (sock->vss_server_evp == NULL) { error = -1; goto out; } out: #ifdef __FreeBSD__ if (fd != -1) close(fd); #endif if (error != 0) { if (s != -1) close(s); free(sock); } return (error); } static void pci_vtcon_sock_accept(int fd __unused, enum ev_type t __unused, void *arg) { struct pci_vtcon_sock *sock = (struct pci_vtcon_sock *)arg; int s; s = accept(sock->vss_server_fd, NULL, NULL); if (s < 0) return; if (sock->vss_open) { close(s); return; } sock->vss_open = true; sock->vss_conn_fd = s; sock->vss_conn_evp = mevent_add(s, EVF_READ, pci_vtcon_sock_rx, sock); pci_vtcon_open_port(sock->vss_port, true); } static void pci_vtcon_sock_rx(int fd __unused, enum ev_type t __unused, void *arg) { struct pci_vtcon_port *port; struct pci_vtcon_sock *sock = (struct pci_vtcon_sock *)arg; struct vqueue_info *vq; struct vi_req req; struct iovec iov; static char dummybuf[2048]; int len, n; port = sock->vss_port; vq = pci_vtcon_port_to_vq(port, true); if (!sock->vss_open || !port->vsp_rx_ready) { len = read(sock->vss_conn_fd, dummybuf, sizeof(dummybuf)); if (len == 0) goto close; return; } if (!vq_has_descs(vq)) { len = read(sock->vss_conn_fd, dummybuf, sizeof(dummybuf)); vq_endchains(vq, 1); if (len == 0) goto close; return; } do { n = vq_getchain(vq, &iov, 1, &req); assert(n == 1); len = readv(sock->vss_conn_fd, &iov, n); if (len == 0 || (len < 0 && errno == EWOULDBLOCK)) { vq_retchains(vq, 1); vq_endchains(vq, 0); if (len == 0) goto close; return; } vq_relchain(vq, req.idx, len); } while (vq_has_descs(vq)); vq_endchains(vq, 1); close: mevent_delete_close(sock->vss_conn_evp); sock->vss_conn_fd = -1; sock->vss_open = false; } static void pci_vtcon_sock_tx(struct pci_vtcon_port *port __unused, void *arg __unused, struct iovec *iov, int niov) { struct pci_vtcon_sock *sock; int i, ret; #ifndef __FreeBSD__ ret = 0; #endif sock = (struct pci_vtcon_sock *)arg; if (sock->vss_conn_fd == -1) return; for (i = 0; i < niov; i++) { ret = stream_write(sock->vss_conn_fd, iov[i].iov_base, iov[i].iov_len); if (ret <= 0) break; } if (ret <= 0) { mevent_delete_close(sock->vss_conn_evp); sock->vss_conn_fd = -1; sock->vss_open = false; } } static void pci_vtcon_control_tx(struct pci_vtcon_port *port, void *arg __unused, struct iovec *iov, int niov) { struct pci_vtcon_softc *sc; struct pci_vtcon_port *tmp; struct pci_vtcon_control resp, *ctrl; int i; assert(niov == 1); sc = port->vsp_sc; ctrl = (struct pci_vtcon_control *)iov->iov_base; switch (ctrl->event) { case VTCON_DEVICE_READY: sc->vsc_ready = true; /* set port ready events for registered ports */ for (i = 0; i < VTCON_MAXPORTS; i++) { tmp = &sc->vsc_ports[i]; if (tmp->vsp_enabled) pci_vtcon_announce_port(tmp); if (tmp->vsp_open) pci_vtcon_open_port(tmp, true); } break; case VTCON_PORT_READY: tmp = &sc->vsc_ports[ctrl->id]; if (ctrl->id >= VTCON_MAXPORTS || !tmp->vsp_enabled) { WPRINTF(("VTCON_PORT_READY event for unknown port %d", ctrl->id)); return; } /* Guest port ready; send additional configuration */ if (ctrl->value == 1 && tmp->vsp_name != NULL) { resp.id = ctrl->id; resp.event = VTCON_PORT_NAME; resp.value = 1; pci_vtcon_control_send(sc, &resp, tmp->vsp_name, strlen(tmp->vsp_name)); } if (tmp->vsp_console) { resp.event = VTCON_CONSOLE_PORT; resp.id = ctrl->id; resp.value = 1; pci_vtcon_control_send(sc, &resp, NULL, 0); } break; } } static void pci_vtcon_announce_port(struct pci_vtcon_port *port) { struct pci_vtcon_control event; event.id = port->vsp_id; event.event = VTCON_DEVICE_ADD; event.value = 1; pci_vtcon_control_send(port->vsp_sc, &event, NULL, 0); } static void pci_vtcon_open_port(struct pci_vtcon_port *port, bool open) { struct pci_vtcon_control event; if (!port->vsp_sc->vsc_ready) { port->vsp_open = true; return; } event.id = port->vsp_id; event.event = VTCON_PORT_OPEN; event.value = (int)open; pci_vtcon_control_send(port->vsp_sc, &event, NULL, 0); } static void pci_vtcon_control_send(struct pci_vtcon_softc *sc, struct pci_vtcon_control *ctrl, const void *payload, size_t len) { struct vqueue_info *vq; struct vi_req req; struct iovec iov; int n; if (len > SIZE_T_MAX - sizeof(struct pci_vtcon_control)) return; vq = pci_vtcon_port_to_vq(&sc->vsc_control_port, true); if (!vq_has_descs(vq)) return; n = vq_getchain(vq, &iov, 1, &req); assert(n == 1); if (iov.iov_len < sizeof(struct pci_vtcon_control) + len) goto out; memcpy(iov.iov_base, ctrl, sizeof(struct pci_vtcon_control)); if (len > 0) memcpy((uint8_t *)iov.iov_base + sizeof(struct pci_vtcon_control), payload, len); out: vq_relchain(vq, req.idx, sizeof(struct pci_vtcon_control) + len); vq_endchains(vq, 1); } static void pci_vtcon_notify_tx(void *vsc, struct vqueue_info *vq) { struct pci_vtcon_softc *sc; struct pci_vtcon_port *port; struct iovec iov[1]; struct vi_req req; int n; sc = vsc; port = pci_vtcon_vq_to_port(sc, vq); while (vq_has_descs(vq)) { n = vq_getchain(vq, iov, 1, &req); assert(n == 1); if (port != NULL) port->vsp_cb(port, port->vsp_arg, iov, 1); /* * Release this chain and handle more */ vq_relchain(vq, req.idx, 0); } vq_endchains(vq, 1); /* Generate interrupt if appropriate. */ } static void pci_vtcon_notify_rx(void *vsc, struct vqueue_info *vq) { struct pci_vtcon_softc *sc; struct pci_vtcon_port *port; sc = vsc; port = pci_vtcon_vq_to_port(sc, vq); if (!port->vsp_rx_ready) { port->vsp_rx_ready = 1; vq_kick_disable(vq); } } /* * Each console device has a "port" node which contains nodes for * each port. Ports are numbered starting at 0. */ static int pci_vtcon_legacy_config_port(nvlist_t *nvl, int port, char *opt) { char *name, *path; char node_name[sizeof("XX")]; nvlist_t *port_nvl; name = strsep(&opt, "="); path = opt; if (path == NULL) { EPRINTLN("vtcon: port %s requires a path", name); return (-1); } if (port >= VTCON_MAXPORTS) { EPRINTLN("vtcon: too many ports"); return (-1); } snprintf(node_name, sizeof(node_name), "%d", port); port_nvl = create_relative_config_node(nvl, node_name); set_config_value_node(port_nvl, "name", name); set_config_value_node(port_nvl, "path", path); return (0); } static int pci_vtcon_legacy_config(nvlist_t *nvl, const char *opts) { char *opt, *str, *tofree; nvlist_t *ports_nvl; int error, port; ports_nvl = create_relative_config_node(nvl, "port"); tofree = str = strdup(opts); error = 0; port = 0; while ((opt = strsep(&str, ",")) != NULL) { error = pci_vtcon_legacy_config_port(ports_nvl, port, opt); if (error) break; port++; } free(tofree); return (error); } static int pci_vtcon_init(struct pci_devinst *pi, nvlist_t *nvl) { struct pci_vtcon_softc *sc; nvlist_t *ports_nvl; int i; sc = calloc(1, sizeof(struct pci_vtcon_softc)); sc->vsc_config = calloc(1, sizeof(struct pci_vtcon_config)); sc->vsc_config->max_nr_ports = VTCON_MAXPORTS; sc->vsc_config->cols = 80; sc->vsc_config->rows = 25; pthread_mutex_init(&sc->vsc_mtx, NULL); vi_softc_linkup(&sc->vsc_vs, &vtcon_vi_consts, sc, pi, sc->vsc_queues); sc->vsc_vs.vs_mtx = &sc->vsc_mtx; for (i = 0; i < VTCON_MAXQ; i++) { sc->vsc_queues[i].vq_qsize = VTCON_RINGSZ; sc->vsc_queues[i].vq_notify = i % 2 == 0 ? pci_vtcon_notify_rx : pci_vtcon_notify_tx; } /* initialize config space */ vi_pci_init(pi, VIRTIO_MODE_TRANSITIONAL, VIRTIO_DEV_CONSOLE, VIRTIO_ID_CONSOLE, PCIC_SIMPLECOMM); if (!vi_intr_init(&sc->vsc_vs, fbsdrun_virtio_msix())) return (1); if (!vi_pcibar_setup(&sc->vsc_vs)) return (1); /* create control port */ sc->vsc_control_port.vsp_sc = sc; sc->vsc_control_port.vsp_txq = 2; sc->vsc_control_port.vsp_rxq = 3; sc->vsc_control_port.vsp_cb = pci_vtcon_control_tx; sc->vsc_control_port.vsp_enabled = true; ports_nvl = find_relative_config_node(nvl, "port"); if (ports_nvl != NULL) { const char *name; void *cookie; int type; cookie = NULL; while ((name = nvlist_next(ports_nvl, &type, &cookie)) != NULL) { if (type != NV_TYPE_NVLIST) continue; if (pci_vtcon_sock_add(sc, name, nvlist_get_nvlist(ports_nvl, name)) < 0) { EPRINTLN("cannot create port %s: %s", name, strerror(errno)); return (1); } } } return (0); } static const struct pci_devemu pci_de_vcon = { .pe_emu = "virtio-console", .pe_init = pci_vtcon_init, .pe_cfgwrite = vi_pci_cfgwrite, .pe_cfgread = vi_pci_cfgread, .pe_barwrite = vi_pci_write, .pe_barread = vi_pci_read, .pe_legacy_config = pci_vtcon_legacy_config, }; PCI_EMUL_SET(pci_de_vcon); /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2021 Beckhoff Automation GmbH & Co. KG * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer * in this position and unchanged. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * This file and its contents are supplied under the terms of the * Common Development and Distribution License ("CDDL"), version 1.0. * You may only use this file in accordance with the terms of version * 1.0 of the CDDL. * * A full copy of the text of the CDDL should have accompanied this * source. A copy of the CDDL is also available via the Internet at * http://www.illumos.org/license/CDDL. */ /* This file is dual-licensed; see usr/src/contrib/bhyve/LICENSE */ /* * Copyright 2026 Oxide Computer Company */ /* * virtio input device emulation. */ #include #ifndef WITHOUT_CAPSICUM #include #include #endif #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "bhyverun.h" #include "config.h" #include "debug.h" #include "mevent.h" #include "pci_emul.h" #include "virtio.h" #define VTINPUT_RINGSZ 64 #define VTINPUT_MAX_PKT_LEN 10 /* * Queue definitions. */ #define VTINPUT_EVENTQ 0 #define VTINPUT_STATUSQ 1 #define VTINPUT_MAXQ 2 static int pci_vtinput_debug; #define DPRINTF(params) \ if (pci_vtinput_debug) \ PRINTLN params #define WPRINTF(params) PRINTLN params enum vtinput_config_select { VTINPUT_CFG_UNSET = 0x00, VTINPUT_CFG_ID_NAME = 0x01, VTINPUT_CFG_ID_SERIAL = 0x02, VTINPUT_CFG_ID_DEVIDS = 0x03, VTINPUT_CFG_PROP_BITS = 0x10, VTINPUT_CFG_EV_BITS = 0x11, VTINPUT_CFG_ABS_INFO = 0x12 }; struct vtinput_absinfo { uint32_t min; uint32_t max; uint32_t fuzz; uint32_t flat; uint32_t res; } __packed; struct vtinput_devids { uint16_t bustype; uint16_t vendor; uint16_t product; uint16_t version; } __packed; struct vtinput_config { uint8_t select; uint8_t subsel; uint8_t size; uint8_t reserved[5]; union { char string[128]; uint8_t bitmap[128]; struct vtinput_absinfo abs; struct vtinput_devids ids; } u; } __packed; struct vtinput_event { uint16_t type; uint16_t code; uint32_t value; } __packed; struct vtinput_event_elem { struct vtinput_event event; struct iovec iov; uint16_t idx; }; struct vtinput_eventqueue { struct vtinput_event_elem *events; uint32_t size; uint32_t idx; }; /* * Per-device softc */ struct pci_vtinput_softc { struct virtio_softc vsc_vs; struct vqueue_info vsc_queues[VTINPUT_MAXQ]; pthread_mutex_t vsc_mtx; const char *vsc_evdev; int vsc_fd; struct vtinput_config vsc_config; int vsc_config_valid; struct mevent *vsc_evp; struct vtinput_eventqueue vsc_eventqueue; }; static void pci_vtinput_reset(void *); static int pci_vtinput_cfgread(void *, int, int, uint32_t *); static int pci_vtinput_cfgwrite(void *, int, int, uint32_t); static struct virtio_consts vtinput_vi_consts = { .vc_name = "vtinput", .vc_nvq = VTINPUT_MAXQ, .vc_cfgsize = sizeof(struct vtinput_config), .vc_reset = pci_vtinput_reset, .vc_cfgread = pci_vtinput_cfgread, .vc_cfgwrite = pci_vtinput_cfgwrite, .vc_hv_caps_legacy = 0, .vc_hv_caps_modern = 0, }; static void pci_vtinput_reset(void *vsc) { struct pci_vtinput_softc *sc = vsc; DPRINTF(("%s: device reset requested", __func__)); vi_reset_dev(&sc->vsc_vs); } static void pci_vtinput_notify_eventq(void *vsc __unused, struct vqueue_info *vq __unused) { DPRINTF(("%s", __func__)); } static void pci_vtinput_notify_statusq(void *vsc, struct vqueue_info *vq) { struct pci_vtinput_softc *sc = vsc; while (vq_has_descs(vq)) { /* get descriptor chain */ struct iovec iov; struct vi_req req; const int n = vq_getchain(vq, &iov, 1, &req); if (n <= 0) { WPRINTF(("%s: invalid descriptor: %d", __func__, n)); return; } /* get event */ struct vtinput_event event; memcpy(&event, iov.iov_base, sizeof(event)); /* * on multi touch devices: * - host send EV_MSC to guest * - guest sends EV_MSC back to host * - host writes EV_MSC to evdev * - evdev saves EV_MSC in it's event buffer * - host receives an extra EV_MSC by reading the evdev event * buffer * - frames become larger and larger * avoid endless loops by ignoring EV_MSC */ if (event.type == EV_MSC) { vq_relchain(vq, req.idx, sizeof(event)); continue; } /* send event to evdev */ struct input_event host_event; host_event.type = event.type; host_event.code = event.code; host_event.value = event.value; if (gettimeofday(&host_event.time, NULL) != 0) { WPRINTF(("%s: failed gettimeofday", __func__)); } if (write(sc->vsc_fd, &host_event, sizeof(host_event)) == -1) { WPRINTF(("%s: failed to write host_event", __func__)); } vq_relchain(vq, req.idx, sizeof(event)); } vq_endchains(vq, 1); } static int pci_vtinput_get_bitmap(struct pci_vtinput_softc *sc, int cmd, int count) { if (count <= 0 || !sc) { return (-1); } /* query bitmap */ memset(sc->vsc_config.u.bitmap, 0, sizeof(sc->vsc_config.u.bitmap)); if (ioctl(sc->vsc_fd, cmd, sc->vsc_config.u.bitmap) < 0) { return (-1); } /* get number of set bytes in bitmap */ for (int i = count - 1; i >= 0; i--) { if (sc->vsc_config.u.bitmap[i]) { return i + 1; } } return (-1); } static int pci_vtinput_read_config_id_name(struct pci_vtinput_softc *sc) { char name[128]; if (ioctl(sc->vsc_fd, EVIOCGNAME(sizeof(name) - 1), name) < 0) { return (1); } memcpy(sc->vsc_config.u.string, name, sizeof(name)); sc->vsc_config.size = strnlen(name, sizeof(name)); return (0); } static int pci_vtinput_read_config_id_serial(struct pci_vtinput_softc *sc) { /* serial isn't supported */ sc->vsc_config.size = 0; return (0); } static int pci_vtinput_read_config_id_devids(struct pci_vtinput_softc *sc) { struct input_id devids; if (ioctl(sc->vsc_fd, EVIOCGID, &devids)) { return (1); } sc->vsc_config.u.ids.bustype = devids.bustype; sc->vsc_config.u.ids.vendor = devids.vendor; sc->vsc_config.u.ids.product = devids.product; sc->vsc_config.u.ids.version = devids.version; sc->vsc_config.size = sizeof(struct vtinput_devids); return (0); } static int pci_vtinput_read_config_prop_bits(struct pci_vtinput_softc *sc) { /* * Evdev bitmap countains 1 bit per count. Additionally evdev bitmaps * are arrays of longs instead of chars. Calculate how many longs are * required for evdev bitmap. Multiply that with sizeof(long) to get the * number of elements. */ const int count = howmany(INPUT_PROP_CNT, sizeof(long) * 8) * sizeof(long); const unsigned int cmd = EVIOCGPROP(count); const int size = pci_vtinput_get_bitmap(sc, cmd, count); if (size <= 0) { return (1); } sc->vsc_config.size = size; return (0); } static int pci_vtinput_read_config_ev_bits(struct pci_vtinput_softc *sc, uint8_t type) { int count; switch (type) { case EV_KEY: count = KEY_CNT; break; case EV_REL: count = REL_CNT; break; case EV_ABS: count = ABS_CNT; break; case EV_MSC: count = MSC_CNT; break; case EV_SW: count = SW_CNT; break; case EV_LED: count = LED_CNT; break; default: return (1); } /* * Evdev bitmap countains 1 bit per count. Additionally evdev bitmaps * are arrays of longs instead of chars. Calculate how many longs are * required for evdev bitmap. Multiply that with sizeof(long) to get the * number of elements. */ count = howmany(count, sizeof(long) * 8) * sizeof(long); const unsigned int cmd = EVIOCGBIT(sc->vsc_config.subsel, count); const int size = pci_vtinput_get_bitmap(sc, cmd, count); if (size <= 0) { return (1); } sc->vsc_config.size = size; return (0); } static int pci_vtinput_read_config_abs_info(struct pci_vtinput_softc *sc) { /* check if evdev has EV_ABS */ if (!pci_vtinput_read_config_ev_bits(sc, EV_ABS)) { return (1); } /* get abs information */ struct input_absinfo abs; if (ioctl(sc->vsc_fd, EVIOCGABS(sc->vsc_config.subsel), &abs) < 0) { return (1); } /* save abs information */ sc->vsc_config.u.abs.min = abs.minimum; sc->vsc_config.u.abs.max = abs.maximum; sc->vsc_config.u.abs.fuzz = abs.fuzz; sc->vsc_config.u.abs.flat = abs.flat; sc->vsc_config.u.abs.res = abs.resolution; sc->vsc_config.size = sizeof(struct vtinput_absinfo); return (0); } static int pci_vtinput_read_config(struct pci_vtinput_softc *sc) { switch (sc->vsc_config.select) { case VTINPUT_CFG_UNSET: return (0); case VTINPUT_CFG_ID_NAME: return pci_vtinput_read_config_id_name(sc); case VTINPUT_CFG_ID_SERIAL: return pci_vtinput_read_config_id_serial(sc); case VTINPUT_CFG_ID_DEVIDS: return pci_vtinput_read_config_id_devids(sc); case VTINPUT_CFG_PROP_BITS: return pci_vtinput_read_config_prop_bits(sc); case VTINPUT_CFG_EV_BITS: return pci_vtinput_read_config_ev_bits( sc, sc->vsc_config.subsel); case VTINPUT_CFG_ABS_INFO: return pci_vtinput_read_config_abs_info(sc); default: return (1); } } static int pci_vtinput_cfgread(void *vsc, int offset, int size, uint32_t *retval) { struct pci_vtinput_softc *sc = vsc; /* check for valid offset and size */ if (offset + size > (int)sizeof(struct vtinput_config)) { WPRINTF(("%s: read to invalid offset/size %d/%d", __func__, offset, size)); memset(retval, 0, size); return (0); } /* read new config values, if select and subsel changed. */ if (!sc->vsc_config_valid) { if (pci_vtinput_read_config(sc) != 0) { DPRINTF(("%s: could not read config %d/%d", __func__, sc->vsc_config.select, sc->vsc_config.subsel)); memset(retval, 0, size); return (0); } sc->vsc_config_valid = 1; } uint8_t *ptr = (uint8_t *)&sc->vsc_config; memcpy(retval, ptr + offset, size); return (0); } static int pci_vtinput_cfgwrite(void *vsc, int offset, int size, uint32_t value) { struct pci_vtinput_softc *sc = vsc; /* guest can only write to select and subsel fields */ if (offset + size > 2) { WPRINTF(("%s: write to readonly reg %d", __func__, offset)); return (1); } /* copy value into config */ uint8_t *ptr = (uint8_t *)&sc->vsc_config; memcpy(ptr + offset, &value, size); /* select/subsel changed, query new config on next cfgread */ sc->vsc_config_valid = 0; /* notify the guest the device configuration has been changed */ vq_devcfg_changed(&sc->vsc_vs); return (0); } static int vtinput_eventqueue_add_event( struct vtinput_eventqueue *queue, struct input_event *e) { /* check if queue is full */ if (queue->idx >= queue->size) { /* alloc new elements for queue */ const uint32_t newSize = queue->idx; void *newPtr = realloc(queue->events, queue->size * sizeof(struct vtinput_event_elem)); if (newPtr == NULL) { WPRINTF(("%s: realloc memory for eventqueue failed!", __func__)); return (1); } queue->events = newPtr; queue->size = newSize; } /* save event */ struct vtinput_event *event = &queue->events[queue->idx].event; event->type = e->type; event->code = e->code; event->value = e->value; queue->idx++; return (0); } static void vtinput_eventqueue_clear(struct vtinput_eventqueue *queue) { /* just reset index to clear queue */ queue->idx = 0; } static void vtinput_eventqueue_send_events( struct vtinput_eventqueue *queue, struct vqueue_info *vq) { /* * First iteration through eventqueue: * Get descriptor chains. */ for (uint32_t i = 0; i < queue->idx; ++i) { /* get descriptor */ if (!vq_has_descs(vq)) { /* * We don't have enough descriptors for all events. * Return chains back to guest. */ vq_retchains(vq, i); WPRINTF(( "%s: not enough available descriptors, dropping %d events", __func__, queue->idx)); goto done; } /* get descriptor chain */ struct iovec iov; struct vi_req req; const int n = vq_getchain(vq, &iov, 1, &req); if (n <= 0) { WPRINTF(("%s: invalid descriptor: %d", __func__, n)); return; } if (n != 1) { WPRINTF( ("%s: invalid number of descriptors in chain: %d", __func__, n)); /* release invalid chain */ vq_relchain(vq, req.idx, 0); return; } if (iov.iov_len < sizeof(struct vtinput_event)) { WPRINTF(("%s: invalid descriptor length: %lu", __func__, iov.iov_len)); /* release invalid chain */ vq_relchain(vq, req.idx, 0); return; } /* save descriptor */ queue->events[i].iov = iov; queue->events[i].idx = req.idx; } /* * Second iteration through eventqueue: * Send events to guest by releasing chains */ for (uint32_t i = 0; i < queue->idx; ++i) { struct vtinput_event_elem event = queue->events[i]; memcpy(event.iov.iov_base, &event.event, sizeof(struct vtinput_event)); vq_relchain(vq, event.idx, sizeof(struct vtinput_event)); } done: /* clear queue and send interrupt to guest */ vtinput_eventqueue_clear(queue); vq_endchains(vq, 1); } static int vtinput_read_event_from_host(int fd, struct input_event *event) { const int len = read(fd, event, sizeof(struct input_event)); if (len != sizeof(struct input_event)) { if (len == -1 && errno != EAGAIN) { WPRINTF(("%s: event read failed! len = %d, errno = %d", __func__, len, errno)); } /* host doesn't have more events for us */ return (1); } return (0); } static void vtinput_read_event(int fd __attribute((unused)), enum ev_type t __attribute__((unused)), void *arg __attribute__((unused))) { struct pci_vtinput_softc *sc = arg; /* skip if driver isn't ready */ if (!(sc->vsc_vs.vs_status & VIRTIO_CONFIG_STATUS_DRIVER_OK)) return; /* read all events from host */ struct input_event event; while (vtinput_read_event_from_host(sc->vsc_fd, &event) == 0) { /* add events to our queue */ vtinput_eventqueue_add_event(&sc->vsc_eventqueue, &event); /* only send events to guest on EV_SYN or SYN_REPORT */ if (event.type != EV_SYN || event.type != SYN_REPORT) { continue; } /* send host events to guest */ vtinput_eventqueue_send_events( &sc->vsc_eventqueue, &sc->vsc_queues[VTINPUT_EVENTQ]); } } static int pci_vtinput_legacy_config(nvlist_t *nvl, const char *opts) { if (opts == NULL) return (-1); /* * parse opts: * virtio-input,/dev/input/eventX */ char *cp = strchr(opts, ','); if (cp == NULL) { set_config_value_node(nvl, "path", opts); return (0); } char *path = strndup(opts, cp - opts); set_config_value_node(nvl, "path", path); free(path); return (pci_parse_legacy_config(nvl, cp + 1)); } static int pci_vtinput_init(struct pci_devinst *pi, nvlist_t *nvl) { struct pci_vtinput_softc *sc; /* * Keep it here. * Else it's possible to access it uninitialized by jumping to failed. */ pthread_mutexattr_t mtx_attr = NULL; sc = calloc(1, sizeof(struct pci_vtinput_softc)); sc->vsc_evdev = get_config_value_node(nvl, "path"); if (sc->vsc_evdev == NULL) { WPRINTF(("%s: missing required path config value", __func__)); goto failed; } /* * open evdev by using non blocking I/O: * read from /dev/input/eventX would block our thread otherwise */ sc->vsc_fd = open(sc->vsc_evdev, O_RDWR | O_NONBLOCK); if (sc->vsc_fd < 0) { WPRINTF(("%s: failed to open %s", __func__, sc->vsc_evdev)); goto failed; } /* check if evdev is really a evdev */ int evversion; int error = ioctl(sc->vsc_fd, EVIOCGVERSION, &evversion); if (error < 0) { WPRINTF(("%s: %s is no evdev", __func__, sc->vsc_evdev)); goto failed; } /* gain exclusive access to evdev */ error = ioctl(sc->vsc_fd, EVIOCGRAB, 1); if (error < 0) { WPRINTF(("%s: failed to grab %s", __func__, sc->vsc_evdev)); goto failed; } if (pthread_mutexattr_init(&mtx_attr)) { WPRINTF(("%s: init mutexattr failed", __func__)); goto failed; } if (pthread_mutexattr_settype(&mtx_attr, PTHREAD_MUTEX_RECURSIVE)) { WPRINTF(("%s: settype mutexattr failed", __func__)); goto failed; } if (pthread_mutex_init(&sc->vsc_mtx, &mtx_attr)) { WPRINTF(("%s: init mutex failed", __func__)); goto failed; } /* init softc */ sc->vsc_eventqueue.idx = 0; sc->vsc_eventqueue.size = VTINPUT_MAX_PKT_LEN; sc->vsc_eventqueue.events = calloc( sc->vsc_eventqueue.size, sizeof(struct vtinput_event_elem)); sc->vsc_config_valid = 0; if (sc->vsc_eventqueue.events == NULL) { WPRINTF(("%s: failed to alloc eventqueue", __func__)); goto failed; } /* register event handler */ sc->vsc_evp = mevent_add(sc->vsc_fd, EVF_READ, vtinput_read_event, sc); if (sc->vsc_evp == NULL) { WPRINTF(("%s: could not register mevent", __func__)); goto failed; } #ifndef WITHOUT_CAPSICUM cap_rights_t rights; cap_rights_init(&rights, CAP_EVENT, CAP_IOCTL, CAP_READ, CAP_WRITE); if (caph_rights_limit(sc->vsc_fd, &rights) == -1) { errx(EX_OSERR, "Unable to apply rights for sandbox"); } #endif /* link virtio to softc */ vi_softc_linkup( &sc->vsc_vs, &vtinput_vi_consts, sc, pi, sc->vsc_queues); sc->vsc_vs.vs_mtx = &sc->vsc_mtx; /* init virtio queues */ sc->vsc_queues[VTINPUT_EVENTQ].vq_qsize = VTINPUT_RINGSZ; sc->vsc_queues[VTINPUT_EVENTQ].vq_notify = pci_vtinput_notify_eventq; sc->vsc_queues[VTINPUT_STATUSQ].vq_qsize = VTINPUT_RINGSZ; sc->vsc_queues[VTINPUT_STATUSQ].vq_notify = pci_vtinput_notify_statusq; /* initialize config space */ vi_pci_init(pi, VIRTIO_MODE_TRANSITIONAL, VIRTIO_DEV_INPUT, VIRTIO_ID_INPUT, PCIC_INPUTDEV); pci_set_cfgdata8(pi, PCIR_SUBCLASS, PCIS_INPUTDEV_OTHER); pci_set_cfgdata8(pi, PCIR_REVID, VIRTIO_REV_INPUT); pci_set_cfgdata16(pi, PCIR_SUBDEV_0, VIRTIO_SUBDEV_INPUT); pci_set_cfgdata16(pi, PCIR_SUBVEND_0, VIRTIO_SUBVEN_INPUT); /* add MSI-X table BAR */ if (!vi_intr_init(&sc->vsc_vs, fbsdrun_virtio_msix())) goto failed; /* add VirtIO BARs */ if (!vi_pcibar_setup(&sc->vsc_vs)) goto failed; return (0); failed: if (sc == NULL) { return (-1); } if (sc->vsc_evp) mevent_delete(sc->vsc_evp); if (sc->vsc_eventqueue.events) free(sc->vsc_eventqueue.events); if (sc->vsc_mtx) pthread_mutex_destroy(&sc->vsc_mtx); if (mtx_attr) pthread_mutexattr_destroy(&mtx_attr); if (sc->vsc_fd) close(sc->vsc_fd); free(sc); return (-1); } static const struct pci_devemu pci_de_vinput = { .pe_emu = "virtio-input", .pe_init = pci_vtinput_init, .pe_legacy_config = pci_vtinput_legacy_config, .pe_barwrite = vi_pci_write, .pe_barread = vi_pci_read, }; PCI_EMUL_SET(pci_de_vinput); /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2011 NetApp, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * This file and its contents are supplied under the terms of the * Common Development and Distribution License ("CDDL"), version 1.0. * You may only use this file in accordance with the terms of version * 1.0 of the CDDL. * * A full copy of the text of the CDDL should have accompanied this * source. A copy of the CDDL is also available via the Internet at * http://www.illumos.org/license/CDDL. */ /* This file is dual-licensed; see usr/src/contrib/bhyve/LICENSE */ /* * Copyright 2026 Oxide Computer Company */ #include #include #include #include #include #include #include /* IFNAMSIZ */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include "bhyverun.h" #include "config.h" #include "debug.h" #include "pci_emul.h" #include "mevent.h" #include "virtio.h" #include "net_utils.h" #include "net_backends.h" #include "iov.h" #define VTNET_RINGSZ 1024 #define VTNET_MAXSEGS 256 #define VTNET_MAX_PKT_LEN (65536 + 64) #define VTNET_MIN_MTU ETHERMIN #define VTNET_MAX_MTU 65535 #define VTNET_S_HOSTCAPS_MODERN \ (VIRTIO_NET_F_MAC | VIRTIO_NET_F_STATUS | \ VIRTIO_RING_F_INDIRECT_DESC | VIRTIO_NET_F_MRG_RXBUF) #define VTNET_S_HOSTCAPS_LEGACY \ (VTNET_S_HOSTCAPS_MODERN | VIRTIO_F_NOTIFY_ON_EMPTY) /* * Queue definitions. */ #define VTNET_RXQ 0 #define VTNET_TXQ 1 #define VTNET_CTLQ 2 /* NB: not yet supported */ #define VTNET_MAXQ 3 /* * Debug printf */ static int pci_vtnet_debug; #define DPRINTF(params) if (pci_vtnet_debug) PRINTLN params #define WPRINTF(params) PRINTLN params /* * Per-device softc */ struct pci_vtnet_softc { struct virtio_softc vsc_vs; struct vqueue_info vsc_queues[VTNET_MAXQ - 1]; pthread_mutex_t vsc_mtx; net_backend_t *vsc_be; bool features_negotiated; /* protected by rx_mtx */ int resetting; /* protected by tx_mtx */ uint64_t vsc_features; /* negotiated features */ pthread_mutex_t rx_mtx; int rx_merge; /* merged rx bufs in use */ pthread_t tx_tid; pthread_mutex_t tx_mtx; pthread_cond_t tx_cond; int tx_in_progress; size_t vhdrlen; size_t be_vhdrlen; struct virtio_net_config vsc_config; struct virtio_consts vsc_consts; }; static void pci_vtnet_reset(void *); /* static void pci_vtnet_notify(void *, struct vqueue_info *); */ static int pci_vtnet_cfgread(void *, int, int, uint32_t *); static int pci_vtnet_cfgwrite(void *, int, int, uint32_t); static void pci_vtnet_neg_features(void *, uint64_t *); static virtio_capstr_t vtnet_caps[] = { { VIRTIO_NET_F_CSUM, "VIRTIO_NET_F_CSUM" }, { VIRTIO_NET_F_GUEST_CSUM, "VIRTIO_NET_F_GUEST_CSUM" }, { VIRTIO_NET_F_MTU, "VIRTIO_NET_F_MTU" }, { VIRTIO_NET_F_MAC, "VIRTIO_NET_F_MAC" }, { VIRTIO_NET_F_GSO_DEPREC, "VIRTIO_NET_F_GSO_DEPREC" }, { VIRTIO_NET_F_GUEST_TSO4, "VIRTIO_NET_F_GUEST_TSO4" }, { VIRTIO_NET_F_GUEST_TSO6, "VIRTIO_NET_F_GUEST_TSO6" }, { VIRTIO_NET_F_GUEST_ECN, "VIRTIO_NET_F_GUEST_ECN" }, { VIRTIO_NET_F_GUEST_UFO, "VIRTIO_NET_F_GUEST_UFO" }, { VIRTIO_NET_F_HOST_TSO4, "VIRTIO_NET_F_HOST_TSO4" }, { VIRTIO_NET_F_HOST_TSO6, "VIRTIO_NET_F_HOST_TSO6" }, { VIRTIO_NET_F_HOST_ECN, "VIRTIO_NET_F_HOST_ECN" }, { VIRTIO_NET_F_HOST_UFO, "VIRTIO_NET_F_HOST_UFO" }, { VIRTIO_NET_F_MRG_RXBUF, "VIRTIO_NET_F_MRG_RXBUF" }, { VIRTIO_NET_F_STATUS, "VIRTIO_NET_F_STATUS" }, { VIRTIO_NET_F_CTRL_VQ, "VIRTIO_NET_F_CTRL_VQ" }, { VIRTIO_NET_F_CTRL_RX, "VIRTIO_NET_F_CTRL_RX" }, { VIRTIO_NET_F_CTRL_VLAN, "VIRTIO_NET_F_CTRL_VLAN" }, { VIRTIO_NET_F_GUEST_ANNOUNCE, "VIRTIO_NET_F_GUEST_ANNOUNCE" }, { VIRTIO_NET_F_MQ, "VIRTIO_NET_F_MQ" }, }; static struct virtio_consts vtnet_vi_consts = { .vc_name = "vtnet", .vc_nvq = VTNET_MAXQ - 1, .vc_cfgsize = sizeof (struct virtio_net_config), .vc_reset = pci_vtnet_reset, .vc_cfgread = pci_vtnet_cfgread, .vc_cfgwrite = pci_vtnet_cfgwrite, .vc_apply_features = pci_vtnet_neg_features, .vc_hv_caps_legacy = VTNET_S_HOSTCAPS_LEGACY, .vc_hv_caps_modern = VTNET_S_HOSTCAPS_MODERN, .vc_capstr = vtnet_caps, .vc_ncapstr = ARRAY_SIZE(vtnet_caps), }; static void pci_vtnet_reset(void *vsc) { struct pci_vtnet_softc *sc = vsc; DPRINTF(("vtnet: device reset requested !")); /* Acquire the RX lock to block RX processing. */ pthread_mutex_lock(&sc->rx_mtx); /* * Make sure receive operation is disabled at least until we * re-negotiate the features, since receive operation depends * on the value of sc->rx_merge and the header length, which * are both set in pci_vtnet_neg_features(). * Receive operation will be enabled again once the guest adds * the first receive buffers and kicks us. */ sc->features_negotiated = false; netbe_rx_disable(sc->vsc_be); /* Set sc->resetting and give a chance to the TX thread to stop. */ pthread_mutex_lock(&sc->tx_mtx); sc->resetting = 1; while (sc->tx_in_progress) { pthread_mutex_unlock(&sc->tx_mtx); usleep(10000); pthread_mutex_lock(&sc->tx_mtx); } /* * Now reset rings, MSI-X vectors, and negotiated capabilities. * Do that with the TX lock held, since we need to reset * sc->resetting. */ vi_reset_dev(&sc->vsc_vs); sc->resetting = 0; pthread_mutex_unlock(&sc->tx_mtx); pthread_mutex_unlock(&sc->rx_mtx); } static __inline struct iovec * iov_trim_hdr(struct iovec *iov, int *iovcnt, unsigned int hlen) { struct iovec *riov; if (iov[0].iov_len < hlen) { /* * Not enough header space in the first fragment. * That's not ok for us. */ return (NULL); } iov[0].iov_len -= hlen; if (iov[0].iov_len == 0) { *iovcnt -= 1; if (*iovcnt == 0) { /* * Only space for the header. That's not * enough for us. */ return (NULL); } riov = &iov[1]; } else { iov[0].iov_base = (void *)((uintptr_t)iov[0].iov_base + hlen); riov = &iov[0]; } return (riov); } struct virtio_mrg_rxbuf_info { uint16_t idx; uint16_t pad; uint32_t len; }; static void pci_vtnet_rx(struct pci_vtnet_softc *sc) { int prepend_hdr_len = sc->vhdrlen - sc->be_vhdrlen; struct virtio_mrg_rxbuf_info info[VTNET_MAXSEGS]; struct iovec iov[VTNET_MAXSEGS + 1]; struct vqueue_info *vq; struct vi_req req; vq = &sc->vsc_queues[VTNET_RXQ]; /* Features must be negotiated */ if (!sc->features_negotiated) { return; } for (;;) { struct virtio_net_rxhdr *hdr; uint32_t riov_bytes; struct iovec *riov; uint32_t ulen; int riov_len; int n_chains; ssize_t rlen; ssize_t plen; plen = netbe_peek_recvlen(sc->vsc_be); if (plen <= 0) { /* * No more packets (plen == 0), or backend errored * (plen < 0). Interrupt if needed and stop. */ vq_endchains(vq, /* used_all_avail= */0); return; } plen += prepend_hdr_len; /* * Get a descriptor chain to store the next ingress * packet. In case of mergeable rx buffers, get as * many chains as necessary in order to make room * for plen bytes. */ riov_bytes = 0; riov_len = 0; riov = iov; n_chains = 0; do { int n = vq_getchain(vq, riov, VTNET_MAXSEGS - riov_len, &req); info[n_chains].idx = req.idx; if (n == 0) { /* * No rx buffers. Enable RX kicks and double * check. */ vq_kick_enable(vq); if (!vq_has_descs(vq)) { /* * Still no buffers. Return the unused * chains (if any), interrupt if needed * (including for NOTIFY_ON_EMPTY), and * disable the backend until the next * kick. */ vq_retchains(vq, n_chains); vq_endchains(vq, /* used_all_avail */1); netbe_rx_disable(sc->vsc_be); return; } /* More rx buffers found, so keep going. */ vq_kick_disable(vq); continue; } #ifndef __FreeBSD__ if (n == -1) { /* * An error from vq_getchain() means that * an invalid descriptor was found. */ vq_retchains(vq, n_chains); vq_endchains(vq, /* used_all_avail= */0); return; } #endif assert(n >= 1 && riov_len + n <= VTNET_MAXSEGS); riov_len += n; if (!sc->rx_merge) { n_chains = 1; break; } #ifndef __FreeBSD__ size_t c = count_iov(riov, n); if (c > UINT32_MAX) { vq_retchains(vq, n_chains); vq_endchains(vq, /* used_all_avail= */0); return; } info[n_chains].len = (uint32_t)c; #else info[n_chains].len = (uint32_t)count_iov(riov, n); #endif riov_bytes += info[n_chains].len; riov += n; n_chains++; } while (riov_bytes < plen && riov_len < VTNET_MAXSEGS); riov = iov; #ifdef __FreeBSD__ hdr = riov[0].iov_base; #else hdr = (struct virtio_net_rxhdr *)riov[0].iov_base; #endif if (prepend_hdr_len > 0) { /* * The frontend uses a virtio-net header, but the * backend does not. We need to prepend a zeroed * header. */ riov = iov_trim_hdr(riov, &riov_len, prepend_hdr_len); if (riov == NULL) { /* * The first collected chain is nonsensical, * as it is not even enough to store the * virtio-net header. Just drop it. */ vq_relchain(vq, info[0].idx, 0); vq_retchains(vq, n_chains - 1); continue; } memset(hdr, 0, prepend_hdr_len); } rlen = netbe_recv(sc->vsc_be, riov, riov_len); if (rlen != plen - prepend_hdr_len) { /* * If this happens it means there is something * wrong with the backend (e.g., some other * process is stealing our packets). */ WPRINTF(( "netbe_recv: expected %zd bytes, got %zd", plen - prepend_hdr_len, rlen)); vq_retchains(vq, n_chains); continue; } ulen = (uint32_t)plen; /* * Publish the used buffers to the guest, reporting the * number of bytes that we wrote. */ if (!sc->rx_merge) { vq_relchain(vq, info[0].idx, ulen); } else { uint32_t iolen; int i = 0; do { iolen = info[i].len; if (iolen > ulen) { iolen = ulen; } vq_relchain_prepare(vq, info[i].idx, iolen); ulen -= iolen; i++; } while (ulen > 0); hdr->vrh_bufs = i; vq_relchain_publish(vq); assert(i == n_chains); } } } /* * Called when there is read activity on the backend file descriptor. * Each buffer posted by the guest is assumed to be able to contain * an entire ethernet frame + rx header. */ static void pci_vtnet_rx_callback(int fd __unused, enum ev_type type __unused, void *param) { struct pci_vtnet_softc *sc = param; pthread_mutex_lock(&sc->rx_mtx); pci_vtnet_rx(sc); pthread_mutex_unlock(&sc->rx_mtx); } /* Called on RX kick. */ static void pci_vtnet_ping_rxq(void *vsc, struct vqueue_info *vq) { struct pci_vtnet_softc *sc = vsc; /* * A qnotify means that the rx process can now begin. * Enable RX only if features are negotiated. */ pthread_mutex_lock(&sc->rx_mtx); if (!sc->features_negotiated) { pthread_mutex_unlock(&sc->rx_mtx); return; } vq_kick_disable(vq); netbe_rx_enable(sc->vsc_be); pthread_mutex_unlock(&sc->rx_mtx); } /* TX virtqueue processing, called by the TX thread. */ static void pci_vtnet_proctx(struct pci_vtnet_softc *sc, struct vqueue_info *vq) { struct iovec iov[VTNET_MAXSEGS + 1]; struct iovec *siov = iov; struct vi_req req; ssize_t len; int n; /* * Obtain chain of descriptors. The first descriptor also * contains the virtio-net header. */ n = vq_getchain(vq, iov, VTNET_MAXSEGS, &req); assert(n >= 1 && n <= VTNET_MAXSEGS); if (sc->vhdrlen != sc->be_vhdrlen) { /* * The frontend uses a virtio-net header, but the backend * does not. We simply strip the header and ignore it, as * it should be zero-filled. */ siov = iov_trim_hdr(siov, &n, sc->vhdrlen); } if (siov == NULL) { /* The chain is nonsensical. Just drop it. */ len = 0; } else { len = netbe_send(sc->vsc_be, siov, n); if (len < 0) { /* * If send failed, report that 0 bytes * were read. */ len = 0; } } /* * Return the processed chain to the guest, reporting * the number of bytes that we read. */ vq_relchain(vq, req.idx, len); } /* Called on TX kick. */ static void pci_vtnet_ping_txq(void *vsc, struct vqueue_info *vq) { struct pci_vtnet_softc *sc = vsc; /* * Any ring entries to process? */ if (!vq_has_descs(vq)) return; /* Signal the tx thread for processing */ pthread_mutex_lock(&sc->tx_mtx); vq_kick_disable(vq); if (sc->tx_in_progress == 0) pthread_cond_signal(&sc->tx_cond); pthread_mutex_unlock(&sc->tx_mtx); } /* * Thread which will handle processing of TX desc */ static void * pci_vtnet_tx_thread(void *param) { struct pci_vtnet_softc *sc = param; struct vqueue_info *vq; int error; vq = &sc->vsc_queues[VTNET_TXQ]; /* * Let us wait till the tx queue pointers get initialised & * first tx signaled */ pthread_mutex_lock(&sc->tx_mtx); error = pthread_cond_wait(&sc->tx_cond, &sc->tx_mtx); assert(error == 0); for (;;) { /* note - tx mutex is locked here */ while (sc->resetting || !vq_has_descs(vq)) { vq_kick_enable(vq); if (!sc->resetting && vq_has_descs(vq)) break; sc->tx_in_progress = 0; error = pthread_cond_wait(&sc->tx_cond, &sc->tx_mtx); assert(error == 0); } vq_kick_disable(vq); sc->tx_in_progress = 1; pthread_mutex_unlock(&sc->tx_mtx); do { /* * Run through entries, placing them into * iovecs and sending when an end-of-packet * is found */ pci_vtnet_proctx(sc, vq); } while (vq_has_descs(vq)); /* * Generate an interrupt if needed. */ vq_endchains(vq, /* used_all_avail= */1); pthread_mutex_lock(&sc->tx_mtx); } #ifndef __FreeBSD__ return (NULL); #endif } #ifdef notyet static void pci_vtnet_ping_ctlq(void *vsc, struct vqueue_info *vq) { DPRINTF(("vtnet: control qnotify!")); } #endif static int pci_vtnet_free_softstate(struct pci_vtnet_softc *sc, int ret) { pthread_mutex_destroy(&sc->vsc_mtx); if (sc->vsc_be != NULL) netbe_cleanup(sc->vsc_be); free(sc); return (ret); } static int pci_vtnet_init(struct pci_devinst *pi, nvlist_t *nvl) { struct pci_vtnet_softc *sc; const char *value; char tname[MAXCOMLEN + 1]; unsigned long mtu = ETHERMTU; int err; /* * Allocate data structures for further virtio initializations. * sc also contains a copy of vtnet_vi_consts, since capabilities * change depending on the backend. */ sc = calloc(1, sizeof (struct pci_vtnet_softc)); if (sc == NULL) return (errno); if (get_config_bool_default("virtio.net.debug", false)) pci_vtnet_debug = 1; vi_set_debug(&sc->vsc_vs, pci_vtnet_debug); sc->vsc_consts = vtnet_vi_consts; pthread_mutex_init(&sc->vsc_mtx, NULL); sc->vsc_queues[VTNET_RXQ].vq_qsize = VTNET_RINGSZ; sc->vsc_queues[VTNET_RXQ].vq_notify = pci_vtnet_ping_rxq; sc->vsc_queues[VTNET_TXQ].vq_qsize = VTNET_RINGSZ; sc->vsc_queues[VTNET_TXQ].vq_notify = pci_vtnet_ping_txq; #ifdef notyet sc->vsc_queues[VTNET_CTLQ].vq_qsize = VTNET_RINGSZ; sc->vsc_queues[VTNET_CTLQ].vq_notify = pci_vtnet_ping_ctlq; #endif value = get_config_value_node(nvl, "mac"); if (value != NULL) { err = net_parsemac(value, sc->vsc_config.vnc_macaddr); if (err != 0) return (pci_vtnet_free_softstate(sc, err)); } else net_genmac(pi, sc->vsc_config.vnc_macaddr); value = get_config_value_node(nvl, "mtu"); if (value != NULL) { err = net_parsemtu(value, &mtu); if (err != 0) return (pci_vtnet_free_softstate(sc, err)); if (mtu < VTNET_MIN_MTU || mtu > VTNET_MAX_MTU) { errno = EINVAL; return (pci_vtnet_free_softstate(sc, errno)); } sc->vsc_consts.vc_hv_caps_legacy |= VIRTIO_NET_F_MTU; sc->vsc_consts.vc_hv_caps_modern |= VIRTIO_NET_F_MTU; } sc->vsc_config.vnc_mtu = mtu; /* Permit interfaces without a configured backend. */ if (get_config_value_node(nvl, "backend") != NULL) { err = netbe_init(&sc->vsc_be, nvl, pci_vtnet_rx_callback, sc); if (err != 0) return (pci_vtnet_free_softstate(sc, err)); #ifndef __FreeBSD__ size_t buflen = sizeof (sc->vsc_config.vnc_macaddr); err = netbe_get_mac(sc->vsc_be, sc->vsc_config.vnc_macaddr, &buflen); if (err != 0) return (pci_vtnet_free_softstate(sc, err)); #endif } sc->vsc_consts.vc_hv_caps_legacy |= netbe_get_cap(sc->vsc_be); sc->vsc_consts.vc_hv_caps_modern |= netbe_get_cap(sc->vsc_be); /* * Since we do not actually support multiqueue, * set the maximum virtqueue pairs to 1. */ sc->vsc_config.vnc_max_qpair = 1; vi_softc_linkup(&sc->vsc_vs, &sc->vsc_consts, sc, pi, sc->vsc_queues); sc->vsc_vs.vs_mtx = &sc->vsc_mtx; /* initialize config space */ vi_pci_init(pi, VIRTIO_MODE_TRANSITIONAL, VIRTIO_DEV_NET, VIRTIO_ID_NETWORK, PCIC_NETWORK); /* Link is always up. */ sc->vsc_config.vnc_status = VIRTIO_NET_S_LINK_UP; /* use BAR 1 to map MSI-X table and PBA, if we're using MSI-X */ if (!vi_intr_init(&sc->vsc_vs, fbsdrun_virtio_msix())) return (pci_vtnet_free_softstate(sc, EIO)); if (!vi_pcibar_setup(&sc->vsc_vs)) return (pci_vtnet_free_softstate(sc, EIO)); sc->resetting = 0; sc->rx_merge = 0; sc->vhdrlen = sizeof (struct virtio_net_rxhdr) - 2; pthread_mutex_init(&sc->rx_mtx, NULL); /* * Initialize tx semaphore & spawn TX processing thread. * As of now, only one thread for TX desc processing is * spawned. */ sc->tx_in_progress = 0; pthread_mutex_init(&sc->tx_mtx, NULL); pthread_cond_init(&sc->tx_cond, NULL); pthread_create(&sc->tx_tid, NULL, pci_vtnet_tx_thread, (void *)sc); snprintf(tname, sizeof (tname), "vtnet-%d:%d tx", pi->pi_slot, pi->pi_func); pthread_set_name_np(sc->tx_tid, tname); return (0); } static int pci_vtnet_cfgwrite(void *vsc, int offset, int size, uint32_t value) { struct pci_vtnet_softc *sc = vsc; void *ptr; if (offset < (int)sizeof (sc->vsc_config.vnc_macaddr)) { assert(offset + size <= (int)sizeof (sc->vsc_config.vnc_macaddr)); /* * The driver is allowed to change the MAC address */ ptr = &sc->vsc_config.vnc_macaddr[offset]; memcpy(ptr, &value, size); vq_devcfg_changed(&sc->vsc_vs); } else { /* silently ignore other writes */ DPRINTF(("vtnet: write to readonly reg %d", offset)); } return (0); } static int pci_vtnet_cfgread(void *vsc, int offset, int size, uint32_t *retval) { struct pci_vtnet_softc *sc = vsc; void *ptr; ptr = (uint8_t *)&sc->vsc_config + offset; memcpy(retval, ptr, size); return (0); } static void pci_vtnet_neg_features(void *vsc, uint64_t *negotiated_features) { struct pci_vtnet_softc *sc = vsc; sc->vsc_features = *negotiated_features; if ((*negotiated_features & VIRTIO_NET_F_MRG_RXBUF) != 0) { sc->vhdrlen = sizeof (struct virtio_net_rxhdr); sc->rx_merge = 1; } else { /* * Without mergeable rx buffers, virtio-net header is 2 * bytes shorter than sizeof (struct virtio_net_rxhdr) unless, * that is, we are operating in modern mode. */ sc->vhdrlen = sizeof (struct virtio_net_rxhdr); if (!vi_is_modern(&sc->vsc_vs)) sc->vhdrlen -= 2; sc->rx_merge = 0; } /* Tell the backend to enable some capabilities it has advertised. */ netbe_set_cap(sc->vsc_be, *negotiated_features, sc->vhdrlen); sc->be_vhdrlen = netbe_get_vnet_hdr_len(sc->vsc_be); assert(sc->be_vhdrlen == 0 || sc->be_vhdrlen == sc->vhdrlen); pthread_mutex_lock(&sc->rx_mtx); sc->features_negotiated = true; pthread_mutex_unlock(&sc->rx_mtx); } static const struct pci_devemu pci_de_vnet = { .pe_emu = "virtio-net", .pe_init = pci_vtnet_init, .pe_legacy_config = netbe_legacy_config, .pe_cfgwrite = vi_pci_cfgwrite, .pe_cfgread = vi_pci_cfgread, .pe_barwrite = vi_pci_write, .pe_barread = vi_pci_read, }; PCI_EMUL_SET(pci_de_vnet); /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2014 Nahanni Systems Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer * in this position and unchanged. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * This file and its contents are supplied under the terms of the * Common Development and Distribution License ("CDDL"), version 1.0. * You may only use this file in accordance with the terms of version * 1.0 of the CDDL. * * A full copy of the text of the CDDL should have accompanied this * source. A copy of the CDDL is also available via the Internet at * http://www.illumos.org/license/CDDL. */ /* This file is dual-licensed; see usr/src/contrib/bhyve/LICENSE */ /* * Copyright 2026 Oxide Computer Company */ /* * virtio entropy device emulation. * Randomness is sourced from /dev/random which does not block * once it has been seeded at bootup. */ #include #ifndef WITHOUT_CAPSICUM #include #endif #include #include #ifndef WITHOUT_CAPSICUM #include #endif #include #include #include #include #include #include #include #include #include #include #include "bhyverun.h" #include "debug.h" #include "pci_emul.h" #include "virtio.h" #define VTRND_RINGSZ 64 static int pci_vtrnd_debug; #define DPRINTF(params) if (pci_vtrnd_debug) PRINTLN params #define WPRINTF(params) PRINTLN params /* * Per-device softc */ struct pci_vtrnd_softc { struct virtio_softc vrsc_vs; struct vqueue_info vrsc_vq; pthread_mutex_t vrsc_mtx; uint64_t vrsc_cfg; int vrsc_fd; }; static void pci_vtrnd_reset(void *); static void pci_vtrnd_notify(void *, struct vqueue_info *); static struct virtio_consts vtrnd_vi_consts = { .vc_name = "vtrnd", .vc_nvq = 1, .vc_cfgsize = 0, .vc_reset = pci_vtrnd_reset, .vc_qnotify = pci_vtrnd_notify, .vc_hv_caps_legacy = 0, .vc_hv_caps_modern = 0, }; static void pci_vtrnd_reset(void *vsc) { struct pci_vtrnd_softc *sc; sc = vsc; DPRINTF(("vtrnd: device reset requested !")); vi_reset_dev(&sc->vrsc_vs); } static void pci_vtrnd_notify(void *vsc, struct vqueue_info *vq) { struct iovec iov; struct pci_vtrnd_softc *sc; struct vi_req req; int len, n; sc = vsc; if (sc->vrsc_fd < 0) { vq_endchains(vq, 0); return; } while (vq_has_descs(vq)) { n = vq_getchain(vq, &iov, 1, &req); assert(n == 1); len = read(sc->vrsc_fd, iov.iov_base, iov.iov_len); DPRINTF(("vtrnd: vtrnd_notify(): %d", len)); /* Catastrophe if unable to read from /dev/random */ assert(len > 0); /* * Release this chain and handle more */ vq_relchain(vq, req.idx, len); } vq_endchains(vq, 1); /* Generate interrupt if appropriate. */ } static int pci_vtrnd_init(struct pci_devinst *pi, nvlist_t *nvl __unused) { struct pci_vtrnd_softc *sc; int fd; int len; uint8_t v; #ifndef WITHOUT_CAPSICUM cap_rights_t rights; #endif /* * Should always be able to open /dev/random. */ fd = open("/dev/random", O_RDONLY | O_NONBLOCK); assert(fd >= 0); #ifndef WITHOUT_CAPSICUM cap_rights_init(&rights, CAP_READ); if (caph_rights_limit(fd, &rights) == -1) errx(EX_OSERR, "Unable to apply rights for sandbox"); #endif /* * Check that device is seeded and non-blocking. */ len = read(fd, &v, sizeof(v)); if (len <= 0) { WPRINTF(("vtrnd: /dev/random not ready, read(): %d", len)); close(fd); return (1); } sc = calloc(1, sizeof(struct pci_vtrnd_softc)); pthread_mutex_init(&sc->vrsc_mtx, NULL); vi_softc_linkup(&sc->vrsc_vs, &vtrnd_vi_consts, sc, pi, &sc->vrsc_vq); sc->vrsc_vs.vs_mtx = &sc->vrsc_mtx; sc->vrsc_vq.vq_qsize = VTRND_RINGSZ; /* keep /dev/random opened while emulating */ sc->vrsc_fd = fd; /* initialize config space */ vi_pci_init(pi, VIRTIO_MODE_TRANSITIONAL, VIRTIO_DEV_RANDOM, VIRTIO_ID_ENTROPY, PCIC_CRYPTO); if (!vi_intr_init(&sc->vrsc_vs, fbsdrun_virtio_msix())) return (1); if (!vi_pcibar_setup(&sc->vrsc_vs)) return (1); return (0); } static const struct pci_devemu pci_de_vrnd = { .pe_emu = "virtio-rnd", .pe_init = pci_vtrnd_init, .pe_cfgwrite = vi_pci_cfgwrite, .pe_cfgread = vi_pci_cfgread, .pe_barwrite = vi_pci_write, .pe_barread = vi_pci_read, }; PCI_EMUL_SET(pci_de_vrnd); /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2016 Jakub Klama . * Copyright (c) 2018 Marcelo Araujo . * Copyright (c) 2026 Hans Rosenfeld * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer * in this position and unchanged. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * This file and its contents are supplied under the terms of the * Common Development and Distribution License ("CDDL"), version 1.0. * You may only use this file in accordance with the terms of version * 1.0 of the CDDL. * * A full copy of the text of the CDDL should have accompanied this * source. A copy of the CDDL is also available via the Internet at * http://www.illumos.org/license/CDDL. */ /* This file is dual-licensed; see usr/src/contrib/bhyve/LICENSE */ /* * Copyright 2026 Oxide Computer Company */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "bhyverun.h" #include "config.h" #include "debug.h" #include "pci_emul.h" #include "virtio.h" #include "iov.h" #include "pci_virtio_scsi.h" enum pci_vtscsi_walk { PCI_VTSCSI_WALK_CONTINUE = 0, PCI_VTSCSI_WALK_STOP, }; typedef enum pci_vtscsi_walk pci_vtscsi_walk_t; typedef pci_vtscsi_walk_t pci_vtscsi_walk_request_queue_cb_t( struct pci_vtscsi_queue *, struct pci_vtscsi_request *, void *); static void pci_vtscsi_print_supported_backends(void); static void *pci_vtscsi_proc(void *); static void pci_vtscsi_reset(void *); static void pci_vtscsi_neg_features(void *, uint64_t *); static int pci_vtscsi_cfgread(void *, int, int, uint32_t *); static int pci_vtscsi_cfgwrite(void *, int, int, uint32_t); static pci_vtscsi_walk_request_queue_cb_t pci_vtscsi_tmf_handle_abort_task; static pci_vtscsi_walk_request_queue_cb_t pci_vtscsi_tmf_handle_abort_task_set; static pci_vtscsi_walk_request_queue_cb_t pci_vtscsi_tmf_handle_clear_aca; static pci_vtscsi_walk_request_queue_cb_t pci_vtscsi_tmf_handle_clear_task_set; static pci_vtscsi_walk_request_queue_cb_t pci_vtscsi_tmf_handle_i_t_nexus_reset; static pci_vtscsi_walk_request_queue_cb_t pci_vtscsi_tmf_handle_lun_reset; static pci_vtscsi_walk_request_queue_cb_t pci_vtscsi_tmf_handle_query_task; static pci_vtscsi_walk_request_queue_cb_t pci_vtscsi_tmf_handle_query_task_set; static pci_vtscsi_walk_t pci_vtscsi_walk_request_queue( struct pci_vtscsi_queue *, pci_vtscsi_walk_request_queue_cb_t *, void *); static void pci_vtscsi_tmf_handle(struct pci_vtscsi_softc *, struct pci_vtscsi_ctrl_tmf *); static void pci_vtscsi_an_handle(struct pci_vtscsi_softc *, struct pci_vtscsi_ctrl_an *); static void pci_vtscsi_control_handle(struct pci_vtscsi_softc *, void *, size_t); static struct pci_vtscsi_request *pci_vtscsi_alloc_request( struct pci_vtscsi_softc *); static void pci_vtscsi_free_request(struct pci_vtscsi_softc *, struct pci_vtscsi_request *); static struct pci_vtscsi_request *pci_vtscsi_get_request( struct pci_vtscsi_req_queue *); static void pci_vtscsi_put_request(struct pci_vtscsi_req_queue *, struct pci_vtscsi_request *); static void pci_vtscsi_queue_request(struct pci_vtscsi_softc *, struct vqueue_info *); static void pci_vtscsi_return_request(struct pci_vtscsi_queue *, struct pci_vtscsi_request *, int); static int pci_vtscsi_request_handle(struct pci_vtscsi_softc *, int, struct pci_vtscsi_request *); static void pci_vtscsi_controlq_notify(void *, struct vqueue_info *); static void pci_vtscsi_eventq_notify(void *, struct vqueue_info *); static void pci_vtscsi_requestq_notify(void *, struct vqueue_info *); static int pci_vtscsi_add_target_config(nvlist_t *, const char *, int); static int pci_vtscsi_init_queue(struct pci_vtscsi_softc *, struct pci_vtscsi_queue *, int); static int pci_vtscsi_init(struct pci_devinst *, nvlist_t *); SET_DECLARE(pci_vtscsi_backend_set, struct pci_vtscsi_backend); static struct virtio_consts vtscsi_vi_consts = { .vc_name = "vtscsi", .vc_nvq = VTSCSI_DEF_REQUESTQ + VIRTIO_SCSI_ADDL_Q, .vc_cfgsize = sizeof(struct pci_vtscsi_config), .vc_reset = pci_vtscsi_reset, .vc_cfgread = pci_vtscsi_cfgread, .vc_cfgwrite = pci_vtscsi_cfgwrite, .vc_apply_features = pci_vtscsi_neg_features, .vc_hv_caps_legacy = VIRTIO_RING_F_INDIRECT_DESC, .vc_hv_caps_modern = VIRTIO_RING_F_INDIRECT_DESC, }; static const struct pci_vtscsi_config vtscsi_config = { .num_queues = VTSCSI_DEF_REQUESTQ, /* Leave room for the request and the response. */ .seg_max = VTSCSI_DEF_MAXSEG - VIRTIO_SCSI_HDR_SEG, .max_sectors = 0, .cmd_per_lun = 1, .event_info_size = sizeof(struct pci_vtscsi_event), .sense_size = 96, .cdb_size = 32, .max_channel = VIRTIO_SCSI_MAX_CHANNEL, .max_target = VIRTIO_SCSI_MAX_TARGET, .max_lun = VIRTIO_SCSI_MAX_LUN }; int pci_vtscsi_debug = 0; static void pci_vtscsi_print_supported_backends(void) { struct pci_vtscsi_backend **vbpp; if (SET_COUNT(pci_vtscsi_backend_set) == 0) { printf("No virtio-scsi backends available"); return; } SET_FOREACH(vbpp, pci_vtscsi_backend_set) { struct pci_vtscsi_backend *vbp = *vbpp; printf("%s\n", vbp->vsb_name); } } static void * pci_vtscsi_proc(void *arg) { struct pci_vtscsi_worker *worker = (struct pci_vtscsi_worker *)arg; struct pci_vtscsi_queue *q = worker->vsw_queue; struct pci_vtscsi_softc *sc = q->vsq_sc; for (;;) { struct pci_vtscsi_request *req; uint8_t target; int iolen; int fd; pthread_mutex_lock(&q->vsq_rmtx); while (STAILQ_EMPTY(&q->vsq_requests) && !worker->vsw_exiting) pthread_cond_wait(&q->vsq_cv, &q->vsq_rmtx); if (worker->vsw_exiting) { pthread_mutex_unlock(&q->vsq_rmtx); return (NULL); } req = pci_vtscsi_get_request(&q->vsq_requests); pthread_mutex_unlock(&q->vsq_rmtx); target = pci_vtscsi_get_target(sc, req->vsr_cmd_rd->lun); fd = sc->vss_targets[target].vst_fd; DPRINTF("I/O request tgt %u, lun %d, data_niov_in %zu, " "data_niov_out %zu", target, pci_vtscsi_get_lun(sc, req->vsr_cmd_rd->lun), req->vsr_data_niov_in, req->vsr_data_niov_out); iolen = pci_vtscsi_request_handle(sc, fd, req); pci_vtscsi_return_request(q, req, iolen); } } static void pci_vtscsi_reset(void *vsc) { struct pci_vtscsi_softc *sc; sc = vsc; DPRINTF("device reset requested"); vi_reset_dev(&sc->vss_vs); /* initialize config structure */ sc->vss_config = sc->vss_default_config; sc->vss_config.max_target = MAX(1, sc->vss_num_target) - 1; sc->vss_backend->vsb_reset(sc); } static void pci_vtscsi_neg_features(void *vsc, uint64_t *negotiated_features) { struct pci_vtscsi_softc *sc = vsc; sc->vss_features = *negotiated_features; } static int pci_vtscsi_cfgread(void *vsc, int offset, int size, uint32_t *retval) { struct pci_vtscsi_softc *sc = vsc; void *ptr; ptr = (uint8_t *)&sc->vss_config + offset; memcpy(retval, ptr, size); return (0); } static int pci_vtscsi_cfgwrite(void *vsc __unused, int offset __unused, int size __unused, uint32_t val __unused) { return (0); } /* * ABORT TASK: Abort the specifed task queued for this LUN. * * We can stop once we have found the specified task queued for this LUN. */ static pci_vtscsi_walk_t pci_vtscsi_tmf_handle_abort_task(struct pci_vtscsi_queue *q, struct pci_vtscsi_request *req, void *arg) { struct pci_vtscsi_ctrl_tmf *tmf = arg; assert(tmf->subtype == VIRTIO_SCSI_T_TMF_ABORT_TASK); if (pci_vtscsi_get_target(q->vsq_sc, tmf->lun) != pci_vtscsi_get_target(q->vsq_sc, req->vsr_cmd_rd->lun)) return (PCI_VTSCSI_WALK_CONTINUE); if (pci_vtscsi_get_lun(q->vsq_sc, tmf->lun) != pci_vtscsi_get_lun(q->vsq_sc, req->vsr_cmd_rd->lun)) return (PCI_VTSCSI_WALK_CONTINUE); if (tmf->id != req->vsr_cmd_rd->id) return (PCI_VTSCSI_WALK_CONTINUE); req->vsr_cmd_wr->response = VIRTIO_SCSI_S_ABORTED; STAILQ_REMOVE(&q->vsq_requests, req, pci_vtscsi_request, vsr_link); pci_vtscsi_return_request(q, req, 0); return (PCI_VTSCSI_WALK_STOP); } /* * ABORT TASK SET: Abort all tasks queued for this LUN. */ static pci_vtscsi_walk_t pci_vtscsi_tmf_handle_abort_task_set(struct pci_vtscsi_queue *q, struct pci_vtscsi_request *req, void *arg) { struct pci_vtscsi_ctrl_tmf *tmf = arg; assert(tmf->subtype == VIRTIO_SCSI_T_TMF_ABORT_TASK_SET); if (pci_vtscsi_get_target(q->vsq_sc, tmf->lun) != pci_vtscsi_get_target(q->vsq_sc, req->vsr_cmd_rd->lun)) return (PCI_VTSCSI_WALK_CONTINUE); if (pci_vtscsi_get_lun(q->vsq_sc, tmf->lun) != pci_vtscsi_get_lun(q->vsq_sc, req->vsr_cmd_rd->lun)) return (PCI_VTSCSI_WALK_CONTINUE); req->vsr_cmd_wr->response = VIRTIO_SCSI_S_ABORTED; STAILQ_REMOVE(&q->vsq_requests, req, pci_vtscsi_request, vsr_link); pci_vtscsi_return_request(q, req, 0); return (PCI_VTSCSI_WALK_CONTINUE); } /* * CLEAR ACA: Clear ACA (auto contingent allegiance) state. */ static pci_vtscsi_walk_t pci_vtscsi_tmf_handle_clear_aca(struct pci_vtscsi_queue *q __unused, struct pci_vtscsi_request *req __unused, void *arg) { struct pci_vtscsi_ctrl_tmf *tmf = arg; assert(tmf->subtype == VIRTIO_SCSI_T_TMF_CLEAR_ACA); /* * We don't implement handling of NACA=1 in the CONTROL byte at all. * * Thus, we probably should start filtering NORMACA in INQUIRY and * reject any command that sets NACA=1. * * In any case, there isn't anything we need to do with our queued * requests, so stop right here. */ return (PCI_VTSCSI_WALK_STOP); } /* * CLEAR TASK SET: Clear all tasks queued for this LUN. * * All tasks in our queue were placed there by us, so there can be no other * I_T nexus involved. Hence, this is handled the same as ABORT TASK SET. */ static pci_vtscsi_walk_t pci_vtscsi_tmf_handle_clear_task_set(struct pci_vtscsi_queue *q, struct pci_vtscsi_request *req, void *arg) { struct pci_vtscsi_ctrl_tmf *tmf = arg; assert(tmf->subtype == VIRTIO_SCSI_T_TMF_CLEAR_TASK_SET); if (pci_vtscsi_get_target(q->vsq_sc, tmf->lun) != pci_vtscsi_get_target(q->vsq_sc, req->vsr_cmd_rd->lun)) return (PCI_VTSCSI_WALK_CONTINUE); if (pci_vtscsi_get_lun(q->vsq_sc, tmf->lun) != pci_vtscsi_get_lun(q->vsq_sc, req->vsr_cmd_rd->lun)) return (PCI_VTSCSI_WALK_CONTINUE); req->vsr_cmd_wr->response = VIRTIO_SCSI_S_ABORTED; STAILQ_REMOVE(&q->vsq_requests, req, pci_vtscsi_request, vsr_link); pci_vtscsi_return_request(q, req, 0); return (PCI_VTSCSI_WALK_CONTINUE); } /* * I_T NEXUS RESET: Abort all tasks queued for any LUN of this target. */ static pci_vtscsi_walk_t pci_vtscsi_tmf_handle_i_t_nexus_reset(struct pci_vtscsi_queue *q, struct pci_vtscsi_request *req, void *arg) { struct pci_vtscsi_ctrl_tmf *tmf = arg; assert(tmf->subtype == VIRTIO_SCSI_T_TMF_I_T_NEXUS_RESET); if (pci_vtscsi_get_target(q->vsq_sc, tmf->lun) != pci_vtscsi_get_target(q->vsq_sc, req->vsr_cmd_rd->lun)) return (PCI_VTSCSI_WALK_CONTINUE); /* * T10 "06-026r4 SAM-4 TASK ABORTED status clarifications" indicates * that we should actually return ABORTED here, but other documents * such as the VirtIO spec suggest RESET. */ req->vsr_cmd_wr->response = VIRTIO_SCSI_S_RESET; STAILQ_REMOVE(&q->vsq_requests, req, pci_vtscsi_request, vsr_link); pci_vtscsi_return_request(q, req, 0); return (PCI_VTSCSI_WALK_CONTINUE); } /* * LOGICAL UNIT RESET: Abort all tasks queued for this LUN. */ static pci_vtscsi_walk_t pci_vtscsi_tmf_handle_lun_reset(struct pci_vtscsi_queue *q, struct pci_vtscsi_request *req, void *arg) { struct pci_vtscsi_ctrl_tmf *tmf = arg; assert(tmf->subtype == VIRTIO_SCSI_T_TMF_LOGICAL_UNIT_RESET); if (pci_vtscsi_get_target(q->vsq_sc, tmf->lun) != pci_vtscsi_get_target(q->vsq_sc, req->vsr_cmd_rd->lun)) return (PCI_VTSCSI_WALK_CONTINUE); if (pci_vtscsi_get_lun(q->vsq_sc, tmf->lun) != pci_vtscsi_get_lun(q->vsq_sc, req->vsr_cmd_rd->lun)) return (PCI_VTSCSI_WALK_CONTINUE); /* * T10 "06-026r4 SAM-4 TASK ABORTED status clarifications" indicates * that we should actually return ABORTED here, but other documents * such as the VirtIO spec suggest RESET. */ req->vsr_cmd_wr->response = VIRTIO_SCSI_S_RESET; STAILQ_REMOVE(&q->vsq_requests, req, pci_vtscsi_request, vsr_link); pci_vtscsi_return_request(q, req, 0); return (PCI_VTSCSI_WALK_CONTINUE); } /* * QUERY TASK: Is the specified task present in this LUN? * * We can stop once we have found the specified task queued for this LUN. */ static pci_vtscsi_walk_t pci_vtscsi_tmf_handle_query_task(struct pci_vtscsi_queue *q, struct pci_vtscsi_request *req, void *arg) { struct pci_vtscsi_ctrl_tmf *tmf = arg; assert(tmf->subtype == VIRTIO_SCSI_T_TMF_QUERY_TASK); if (pci_vtscsi_get_target(q->vsq_sc, tmf->lun) != pci_vtscsi_get_target(q->vsq_sc, req->vsr_cmd_rd->lun)) return (PCI_VTSCSI_WALK_CONTINUE); if (pci_vtscsi_get_lun(q->vsq_sc, tmf->lun) != pci_vtscsi_get_lun(q->vsq_sc, req->vsr_cmd_rd->lun)) return (PCI_VTSCSI_WALK_CONTINUE); if (tmf->id != req->vsr_cmd_rd->id) return (PCI_VTSCSI_WALK_CONTINUE); tmf->response = VIRTIO_SCSI_S_FUNCTION_SUCCEEDED; return (PCI_VTSCSI_WALK_STOP); } /* * QUERY TASK SET: Are there any tasks present in this LUN? * * We can stop as soon as we've found at least one task queued for this LUN. */ static pci_vtscsi_walk_t pci_vtscsi_tmf_handle_query_task_set(struct pci_vtscsi_queue *q, struct pci_vtscsi_request *req, void *arg) { struct pci_vtscsi_ctrl_tmf *tmf = arg; assert(tmf->subtype == VIRTIO_SCSI_T_TMF_QUERY_TASK_SET); if (pci_vtscsi_get_target(q->vsq_sc, tmf->lun) != pci_vtscsi_get_target(q->vsq_sc, req->vsr_cmd_rd->lun)) return (PCI_VTSCSI_WALK_CONTINUE); if (pci_vtscsi_get_lun(q->vsq_sc, tmf->lun) != pci_vtscsi_get_lun(q->vsq_sc, req->vsr_cmd_rd->lun)) return (PCI_VTSCSI_WALK_CONTINUE); tmf->response = VIRTIO_SCSI_S_FUNCTION_SUCCEEDED; return (PCI_VTSCSI_WALK_STOP); } static pci_vtscsi_walk_t pci_vtscsi_walk_request_queue(struct pci_vtscsi_queue *q, pci_vtscsi_walk_request_queue_cb_t cb, void *arg) { struct pci_vtscsi_request *req, *tmp; STAILQ_FOREACH_SAFE(req, &q->vsq_requests, vsr_link, tmp) { if (cb(q, req, arg) == PCI_VTSCSI_WALK_STOP) return (PCI_VTSCSI_WALK_STOP); } return (PCI_VTSCSI_WALK_CONTINUE); } static pci_vtscsi_walk_request_queue_cb_t *pci_vtscsi_tmf_handler_cb[] = { pci_vtscsi_tmf_handle_abort_task, pci_vtscsi_tmf_handle_abort_task_set, pci_vtscsi_tmf_handle_clear_aca, pci_vtscsi_tmf_handle_clear_task_set, pci_vtscsi_tmf_handle_i_t_nexus_reset, pci_vtscsi_tmf_handle_lun_reset, pci_vtscsi_tmf_handle_query_task, pci_vtscsi_tmf_handle_query_task_set }; static void pci_vtscsi_tmf_handle(struct pci_vtscsi_softc *sc, struct pci_vtscsi_ctrl_tmf *tmf) { uint8_t target; int fd; if (tmf->subtype > VIRTIO_SCSI_T_TMF_MAX_FUNC) { WPRINTF("pci_vtscsi_tmf_handle: invalid subtype %u", tmf->subtype); tmf->response = VIRTIO_SCSI_S_FUNCTION_REJECTED; return; } if (pci_vtscsi_check_lun(sc, tmf->lun) == false) { DPRINTF("TMF request to invalid LUN %.2hhx%.2hhx-%.2hhx%.2hhx-" "%.2hhx%.2hhx-%.2hhx%.2hhx", tmf->lun[0], tmf->lun[1], tmf->lun[2], tmf->lun[3], tmf->lun[4], tmf->lun[5], tmf->lun[6], tmf->lun[7]); tmf->response = VIRTIO_SCSI_S_BAD_TARGET; return; } target = pci_vtscsi_get_target(sc, tmf->lun); fd = sc->vss_targets[target].vst_fd; DPRINTF("TMF request tgt %d, lun %d, subtype %d, id %lu", target, pci_vtscsi_get_lun(sc, tmf->lun), tmf->subtype, tmf->id); /* * Lock out all the worker threads from processing any waiting requests * while we're processing the TMF request. This also effectively blocks * pci_vtscsi_requestq_notify() from adding any new requests to the * request queue. This does not prevent any requests currently being * processed by the backend from being completed and returned, which we * must guarantee to adhere to the ordering requirements for any TMF * function which aborts tasks. */ for (uint32_t i = 0; i < sc->vss_config.num_queues; i++) { struct pci_vtscsi_queue *q = &sc->vss_queues[i]; pthread_mutex_lock(&q->vsq_rmtx); } /* * The backend may set response to FAILURE for the TMF request. * * The default response of all TMF functions is FUNCTION COMPLETE if * there was no error, regardless of whether it actually succeeded or * not. The two notable exceptions are QUERY TASK and QUERY TASK SET, * which will explicitly return FUNCTION SUCCEEDED if the specified * task or any task was active in the target/LUN, respectively. * * Thus, we will call the backend first. Only if the response we get * is FUNCTION COMPLETE we'll continue processing the TMF function on * our queues. */ sc->vss_backend->vsb_tmf_hdl(sc, fd, tmf); if (tmf->response != VIRTIO_SCSI_S_FUNCTION_COMPLETE) { /* * If this is either a FAILURE or FUNCTION REJECTED, we must * not continue to process the TMF function on our queued * requests. * * If it is FUNCTION SUCCEEDED, we do not need to process the * TMF function on our queued requests. * * If it is anything else, log a warning, but handle it the * same as above. */ if (tmf->response != VIRTIO_SCSI_S_FAILURE && tmf->response != VIRTIO_SCSI_S_FUNCTION_REJECTED && tmf->response != VIRTIO_SCSI_S_FUNCTION_SUCCEEDED) { WPRINTF("pci_vtscsi_tmf_hdl: unexpected response from " "backend: %d", tmf->response); } } else { pci_vtscsi_walk_t ret = PCI_VTSCSI_WALK_CONTINUE; uint32_t i; for (i = 0; i < sc->vss_config.num_queues; i++) { struct pci_vtscsi_queue *q = &sc->vss_queues[i]; ret = pci_vtscsi_walk_request_queue(q, pci_vtscsi_tmf_handler_cb[tmf->subtype], tmf); if (ret == PCI_VTSCSI_WALK_STOP) break; } } /* Unlock the request queues before we return. */ for (uint32_t i = 0; i < sc->vss_config.num_queues; i++) { struct pci_vtscsi_queue *q = &sc->vss_queues[i]; pthread_mutex_unlock(&q->vsq_rmtx); } } static void pci_vtscsi_an_handle(struct pci_vtscsi_softc *sc, struct pci_vtscsi_ctrl_an *an) { int target; int fd; if (pci_vtscsi_check_lun(sc, an->lun) == false) { DPRINTF("AN request to invalid LUN %.2hhx%.2hhx-%.2hhx%.2hhx-" "%.2hhx%.2hhx-%.2hhx%.2hhx", an->lun[0], an->lun[1], an->lun[2], an->lun[3], an->lun[4], an->lun[5], an->lun[6], an->lun[7]); an->response = VIRTIO_SCSI_S_BAD_TARGET; return; } target = pci_vtscsi_get_target(sc, an->lun); fd = sc->vss_targets[target].vst_fd; DPRINTF("AN request tgt %d, lun %d, event requested %x", target, pci_vtscsi_get_lun(sc, an->lun), an->event_requested); sc->vss_backend->vsb_an_hdl(sc, fd, an); } static void pci_vtscsi_control_handle(struct pci_vtscsi_softc *sc, void *buf, size_t bufsize) { uint32_t type; if (bufsize < sizeof(uint32_t)) { WPRINTF("ignoring truncated control request"); return; } type = *(uint32_t *)buf; if (type == VIRTIO_SCSI_T_TMF) { if (bufsize != sizeof(struct pci_vtscsi_ctrl_tmf)) { WPRINTF("ignoring TMF request with size %zu", bufsize); return; } pci_vtscsi_tmf_handle(sc, buf); } else if (type == VIRTIO_SCSI_T_AN_QUERY) { if (bufsize != sizeof(struct pci_vtscsi_ctrl_an)) { WPRINTF("ignoring AN request with size %zu", bufsize); return; } pci_vtscsi_an_handle(sc, buf); } else { WPRINTF("ignoring unknown control request type = %u", type); } } static struct pci_vtscsi_request * pci_vtscsi_alloc_request(struct pci_vtscsi_softc *sc) { struct pci_vtscsi_request *req; req = calloc(1, sizeof(struct pci_vtscsi_request)); if (req == NULL) goto alloc_fail; req->vsr_iov = calloc(sc->vss_config.seg_max + VIRTIO_SCSI_HDR_SEG + SPLIT_IOV_ADDL_IOV, sizeof(struct iovec)); if (req->vsr_iov == NULL) goto alloc_fail; req->vsr_cmd_rd = calloc(1, VTSCSI_IN_HEADER_LEN(sc)); if (req->vsr_cmd_rd == NULL) goto alloc_fail; req->vsr_cmd_wr = calloc(1, VTSCSI_OUT_HEADER_LEN(sc)); if (req->vsr_cmd_wr == NULL) goto alloc_fail; req->vsr_backend = sc->vss_backend->vsb_req_alloc(sc); if (req->vsr_backend == NULL) goto alloc_fail; return (req); alloc_fail: EPRINTLN("failed to allocate request: %s", strerror(errno)); if (req != NULL) pci_vtscsi_free_request(sc, req); return (NULL); } static void pci_vtscsi_free_request(struct pci_vtscsi_softc *sc, struct pci_vtscsi_request *req) { if (req->vsr_backend != NULL) sc->vss_backend->vsb_req_free(req->vsr_backend); if (req->vsr_cmd_rd != NULL) free(req->vsr_cmd_rd); if (req->vsr_cmd_wr != NULL) free(req->vsr_cmd_wr); if (req->vsr_iov != NULL) free(req->vsr_iov); free(req); } static struct pci_vtscsi_request * pci_vtscsi_get_request(struct pci_vtscsi_req_queue *req_queue) { struct pci_vtscsi_request *req; assert(!STAILQ_EMPTY(req_queue)); req = STAILQ_FIRST(req_queue); STAILQ_REMOVE_HEAD(req_queue, vsr_link); return (req); } static void pci_vtscsi_put_request(struct pci_vtscsi_req_queue *req_queue, struct pci_vtscsi_request *req) { STAILQ_INSERT_TAIL(req_queue, req, vsr_link); } static void pci_vtscsi_queue_request(struct pci_vtscsi_softc *sc, struct vqueue_info *vq) { struct pci_vtscsi_queue *q; struct pci_vtscsi_request *req; struct vi_req vireq; int n, numseg; q = &sc->vss_queues[vq->vq_num - VIRTIO_SCSI_ADDL_Q]; pthread_mutex_lock(&q->vsq_fmtx); req = pci_vtscsi_get_request(&q->vsq_free_requests); assert(req != NULL); pthread_mutex_unlock(&q->vsq_fmtx); numseg = (int)(sc->vss_config.seg_max + VIRTIO_SCSI_HDR_SEG); n = vq_getchain(vq, req->vsr_iov, numseg, &vireq); assert(n >= 1 && n <= numseg); req->vsr_idx = vireq.idx; req->vsr_queue = q; req->vsr_iov_in = &req->vsr_iov[0]; req->vsr_niov_in = vireq.readable; req->vsr_iov_out = &req->vsr_iov[vireq.readable]; req->vsr_niov_out = vireq.writable; /* * Make sure we got at least enough space for the VirtIO-SCSI * command headers. If not, return this request immediately. */ if (check_iov_len(req->vsr_iov_out, req->vsr_niov_out, VTSCSI_OUT_HEADER_LEN(q->vsq_sc)) == false) { WPRINTF("ignoring request with insufficient output"); req->vsr_cmd_wr->response = VIRTIO_SCSI_S_FAILURE; pci_vtscsi_return_request(q, req, 1); return; } if (check_iov_len(req->vsr_iov_in, req->vsr_niov_in, VTSCSI_IN_HEADER_LEN(q->vsq_sc)) == false) { WPRINTF("ignoring request with incomplete header"); req->vsr_cmd_wr->response = VIRTIO_SCSI_S_FAILURE; pci_vtscsi_return_request(q, req, 1); return; } /* * We have to split the iovec array into a header and data portion each * for input and output. * * We need to start with the output section (at the end of iov) in case * the iovec covering the final part of the output header needs to be * split, in which case split_iov() will move all reamaining iovecs up * by one to make room for a new iovec covering the first part of the * output data portion. */ req->vsr_data_iov_out = split_iov(req->vsr_iov_out, &req->vsr_niov_out, VTSCSI_OUT_HEADER_LEN(q->vsq_sc), &req->vsr_data_niov_out); /* * Similarly, to not overwrite the first iovec of the output section, * the 2nd call to split_iov() to split the input section must actually * cover the entire iovec array (both input and the already split output * sections). */ req->vsr_niov_in += req->vsr_niov_out + req->vsr_data_niov_out; req->vsr_data_iov_in = split_iov(req->vsr_iov_in, &req->vsr_niov_in, VTSCSI_IN_HEADER_LEN(q->vsq_sc), &req->vsr_data_niov_in); /* * And of course we now have to adjust data_niov_in accordingly. */ req->vsr_data_niov_in -= req->vsr_niov_out + req->vsr_data_niov_out; /* * iov_to_buf() realloc()s the buffer given as 3rd argument to the * total size of all iovecs it will be copying. Since we've just * truncated it in split_iov(), we know that the size will be * VTSCSI_IN_HEADER_LEN(q->vsq_sc). * * Since we pre-allocated req->vsr_cmd_rd to this size, the realloc() * should never fail. * * This will have to change if we begin allowing config space writes * to change sense size. */ assert(iov_to_buf(req->vsr_iov_in, req->vsr_niov_in, (void **)&req->vsr_cmd_rd) == VTSCSI_IN_HEADER_LEN(q->vsq_sc)); /* Make sure this request addresses a valid LUN. */ if (pci_vtscsi_check_lun(sc, req->vsr_cmd_rd->lun) == false) { DPRINTF("I/O request to invalid LUN " "%.2hhx%.2hhx-%.2hhx%.2hhx-%.2hhx%.2hhx-%.2hhx%.2hhx", req->vsr_cmd_rd->lun[0], req->vsr_cmd_rd->lun[1], req->vsr_cmd_rd->lun[2], req->vsr_cmd_rd->lun[3], req->vsr_cmd_rd->lun[4], req->vsr_cmd_rd->lun[5], req->vsr_cmd_rd->lun[6], req->vsr_cmd_rd->lun[7]); req->vsr_cmd_wr->response = VIRTIO_SCSI_S_BAD_TARGET; pci_vtscsi_return_request(q, req, 1); return; } pthread_mutex_lock(&q->vsq_rmtx); pci_vtscsi_put_request(&q->vsq_requests, req); pthread_cond_signal(&q->vsq_cv); pthread_mutex_unlock(&q->vsq_rmtx); DPRINTF("request enqueued", vireq.idx); } static void pci_vtscsi_return_request(struct pci_vtscsi_queue *q, struct pci_vtscsi_request *req, int iolen) { struct pci_vtscsi_softc *sc = q->vsq_sc; void *iov = req->vsr_iov; void *cmd_rd = req->vsr_cmd_rd; void *cmd_wr = req->vsr_cmd_wr; void *backend = req->vsr_backend; int idx = req->vsr_idx; DPRINTF("request completed, response %d", idx, req->vsr_cmd_wr->response); iolen += buf_to_iov(cmd_wr, VTSCSI_OUT_HEADER_LEN(q->vsq_sc), req->vsr_iov_out, req->vsr_niov_out); sc->vss_backend->vsb_req_clear(backend); memset(iov, 0, sizeof(struct iovec) * (sc->vss_config.seg_max + VIRTIO_SCSI_HDR_SEG + SPLIT_IOV_ADDL_IOV)); memset(cmd_rd, 0, VTSCSI_IN_HEADER_LEN(q->vsq_sc)); memset(cmd_wr, 0, VTSCSI_OUT_HEADER_LEN(q->vsq_sc)); memset(req, 0, sizeof(struct pci_vtscsi_request)); req->vsr_iov = iov; req->vsr_cmd_rd = cmd_rd; req->vsr_cmd_wr = cmd_wr; req->vsr_backend = backend; pthread_mutex_lock(&q->vsq_fmtx); pci_vtscsi_put_request(&q->vsq_free_requests, req); pthread_mutex_unlock(&q->vsq_fmtx); pthread_mutex_lock(&q->vsq_qmtx); vq_relchain(q->vsq_vq, idx, iolen); vq_endchains(q->vsq_vq, 0); pthread_mutex_unlock(&q->vsq_qmtx); } static int pci_vtscsi_request_handle(struct pci_vtscsi_softc *sc, int fd, struct pci_vtscsi_request *req) { return (sc->vss_backend->vsb_req_hdl(sc, fd, req)); } static void pci_vtscsi_controlq_notify(void *vsc, struct vqueue_info *vq) { struct pci_vtscsi_softc *sc = vsc; int numseg = (int)(sc->vss_config.seg_max + VIRTIO_SCSI_HDR_SEG); struct iovec iov[numseg]; struct vi_req req; void *buf = NULL; size_t bufsize; int n; while (vq_has_descs(vq)) { n = vq_getchain(vq, iov, numseg, &req); assert(n >= 1 && n <= numseg); bufsize = iov_to_buf(iov, n, &buf); pci_vtscsi_control_handle(sc, buf, bufsize); buf_to_iov((uint8_t *)buf, bufsize, iov, n); /* * Release this chain and handle more */ vq_relchain(vq, req.idx, bufsize); } vq_endchains(vq, 1); /* Generate interrupt if appropriate. */ free(buf); } static void pci_vtscsi_eventq_notify(void *vsc __unused, struct vqueue_info *vq) { vq_kick_disable(vq); } static void pci_vtscsi_requestq_notify(void *vsc, struct vqueue_info *vq) { while (vq_has_descs(vq)) { pci_vtscsi_queue_request(vsc, vq); } } static int pci_vtscsi_init_queue(struct pci_vtscsi_softc *sc, struct pci_vtscsi_queue *queue, int num) { char tname[MAXCOMLEN + 1]; uint32_t i; queue->vsq_sc = sc; queue->vsq_vq = &sc->vss_vq[num]; pthread_mutex_init(&queue->vsq_rmtx, NULL); pthread_mutex_init(&queue->vsq_fmtx, NULL); pthread_mutex_init(&queue->vsq_qmtx, NULL); pthread_cond_init(&queue->vsq_cv, NULL); STAILQ_INIT(&queue->vsq_requests); STAILQ_INIT(&queue->vsq_free_requests); LIST_INIT(&queue->vsq_workers); for (i = 0; i < sc->vss_req_ringsz; i++) { struct pci_vtscsi_request *req; req = pci_vtscsi_alloc_request(sc); if (req == NULL) return (-1); pci_vtscsi_put_request(&queue->vsq_free_requests, req); } for (i = 0; i < sc->vss_thr_per_q; i++) { struct pci_vtscsi_worker *worker; worker = calloc(1, sizeof(struct pci_vtscsi_worker)); if (worker == NULL) return (-1); worker->vsw_queue = queue; pthread_create(&worker->vsw_thread, NULL, &pci_vtscsi_proc, (void *)worker); snprintf(tname, sizeof(tname), "vtscsi:%d-%d", num, i); pthread_set_name_np(worker->vsw_thread, tname); LIST_INSERT_HEAD(&queue->vsq_workers, worker, vsw_link); } return (0); } /* * Create a target config node, return target id. If the target number isn't * given as part of the path argument, use last_id + 1. */ static int pci_vtscsi_add_target_config(nvlist_t *nvl, const char *path, int last_id) { uint64_t target; char *id; char tmp[4]; if (path == NULL) { EPRINTLN("target path must be specified"); return (-1); } if (path[0] != '/' && (id = strchr(path, ':')) != NULL) { const char *errstr; int len = id - path; id = strndup(path, len); if (id == NULL) { EPRINTLN("failed to get id string: %s", strerror(errno)); return (-1); } target = strtonumx(id, 0, VIRTIO_SCSI_MAX_TARGET, &errstr, 0); if (errstr != NULL) { EPRINTLN("invalid target %s: target ID is %s", id, errstr); free(id); return (-1); } free(id); path += len + 1; } else { target = last_id + 1; if (target > VIRTIO_SCSI_MAX_TARGET) { EPRINTLN("max target (%d) reached, can't add another", VIRTIO_SCSI_MAX_TARGET); return (-1); } } snprintf(tmp, sizeof(tmp), "%lu", target); if (get_config_value_node(nvl, tmp) != NULL) { EPRINTLN("cannot add '%s' as target %s: already exits as '%s'", path, tmp, get_config_value_node(nvl, tmp)); return (-1); } set_config_value_node(nvl, tmp, path); return (target); } /* * The following forms are accepted for legacy config options to configure a * single target: * * (0) -s B:D:F,virtio-scsi * (1) -s B:D:F,virtio-scsi, * (2) -s B:D:F,virtio-scsi,,,... * (3) -s B:D:F,virtio-scsi,,... * (4) -s B:D:F,virtio-scsi, * * To configure multiple targets, the following form is accepted: * (5) -s B:D:F,virtio-scsi,[target=[id:],...] */ static int pci_vtscsi_legacy_config(nvlist_t *nvl, const char *opts) { int last_id = -1; char *config, *tofree, *name, *value; nvlist_t *targets; size_t n; /* Make sure no one accidentally sets "dev" anymore. */ (void) create_relative_config_node(nvl, "dev"); targets = create_relative_config_node(nvl, "target"); /* Legacy form (0) is handled in pci_vtscsi_init(). */ if (opts == NULL) return (0); if (strcmp("help", opts) == 0) { pci_vtscsi_print_supported_backends(); exit(0); } n = strcspn(opts, ",="); /* Handle legacy form (1) and (2). */ if (opts[n] == ',' || opts[n] == '\0') { char *tmp = strndup(opts, n); last_id = pci_vtscsi_add_target_config(targets, tmp, last_id); free(tmp); if (last_id < 0) return (-1); opts += n; if (opts[0] == ',' && opts[1] != '\0') opts++; } /* If this was form (1), we're done. */ if (opts[0] == '\0') return (0); /* * For form (2), (3), (4), and (5), parse the remaining options. * * Contrary to other options, multiple target= options create a new * target for each such option. * * For compatibility reasons we also accept dev= options for * targets. */ config = tofree = strdup(opts); while ((name = strsep(&config, ",")) != NULL) { value = strchr(name, '='); if (value != NULL) *value++ = '\0'; if (strcmp(name, "dev") == 0 || strcmp(name, "target") == 0) { int new_id = pci_vtscsi_add_target_config(targets, value, last_id); if (new_id < 0) { free(tofree); return (-1); } if (new_id > last_id) last_id = new_id; } else if (value != NULL) { set_config_value_node(nvl, name, value); } else { set_config_bool_node(nvl, name, true); } } free(tofree); return (0); } static int pci_vtscsi_count_targets(const char *prefix __unused, const nvlist_t *parent __unused, const char *name, int type, void *arg) { struct pci_vtscsi_softc *sc = arg; const char *errstr; uint64_t target; if (type != NV_TYPE_STRING) { EPRINTLN("invalid target \"%s\" type: not a string", name); errno = EINVAL; return (-1); } target = strtonumx(name, 0, VIRTIO_SCSI_MAX_TARGET, &errstr, 0); if (errstr != NULL) { EPRINTLN("invalid target %s: target ID is %s", name, errstr); return (-1); } if (target >= sc->vss_num_target) sc->vss_num_target = target + 1; return (0); } static int pci_vtscsi_init_target(const char *prefix __unused, const nvlist_t *parent, const char *name, int type, void *arg) { struct pci_vtscsi_softc *sc = arg; const char *value; const char *errstr; uint64_t target; int ret; assert(type == NV_TYPE_STRING); /* * Get the numeric value of the target id from 'name'. */ target = strtonumx(name, 0, sc->vss_num_target, &errstr, 0); assert(errstr == NULL); sc->vss_targets[target].vst_target = target; /* * 'value' contains the backend path. Call the backend to open it. */ value = nvlist_get_string(parent, name); ret = sc->vss_backend->vsb_open(sc, value, target); if (ret != 0) EPRINTLN("cannot open target %lu at %s: %s", target, value, strerror(errno)); return (ret); } static int pci_vtscsi_get_config_num(nvlist_t *nvl, const char *name, uint32_t lim_lo, uint32_t lim_hi, uint32_t *res) { const char *value; const char *errstr; long long val; value = get_config_value_node(nvl, name); if (value == NULL) return (0); val = strtonumx(value, lim_lo, lim_hi, &errstr, 0); if (errstr != NULL) { EPRINTLN("Invalid value for %s: %s", name, value); return (-1); } *res = (uint32_t)val; return (0); } static int pci_vtscsi_init(struct pci_devinst *pi, nvlist_t *nvl) { struct pci_vtscsi_softc *sc; struct pci_vtscsi_backend *backend, **vbpp; const char *value; uint32_t val; int err; sc = calloc(1, sizeof(struct pci_vtscsi_softc)); if (sc == NULL) return (-1); sc->vss_vi_consts = vtscsi_vi_consts; sc->vss_ctl_ringsz = VTSCSI_DEF_RINGSZ; sc->vss_evt_ringsz = VTSCSI_DEF_RINGSZ; sc->vss_req_ringsz = VTSCSI_DEF_RINGSZ; sc->vss_thr_per_q = VTSCSI_DEF_THR_PER_Q; sc->vss_default_config = vtscsi_config; value = get_config_value_node(nvl, "bootindex"); if (value != NULL) { if (pci_emul_add_boot_device(pi, atoi(value))) { EPRINTLN("Invalid bootindex %d", atoi(value)); errno = EINVAL; goto fail; } } val = vtscsi_config.seg_max; if (pci_vtscsi_get_config_num(nvl, "seg_max", VTSCSI_MIN_MAXSEG, VTSCSI_MAX_MAXSEG, &val) != 0) goto fail; sc->vss_default_config.seg_max = val; val = vtscsi_config.num_queues; if (pci_vtscsi_get_config_num(nvl, "num_queues", VTSCSI_MIN_REQUESTQ, VTSCSI_MAX_REQUESTQ, &val) != 0) goto fail; sc->vss_default_config.num_queues = val; /* * num_queues is only the number of request queues, but nvq must * account for the control and event queues. */ sc->vss_vi_consts.vc_nvq = val + VIRTIO_SCSI_ADDL_Q; /* * Allocate queues early, so that they're there for the call to * vi_softc_linkup(). */ sc->vss_vq = calloc(sc->vss_vi_consts.vc_nvq, sizeof(struct vqueue_info)); if (sc->vss_vq == NULL) { EPRINTLN("can't allocate space for %d virtqueues", sc->vss_vi_consts.vc_nvq); goto fail; } sc->vss_queues = calloc(sc->vss_default_config.num_queues, sizeof(struct pci_vtscsi_queue)); if (sc->vss_queues == NULL) { EPRINTLN("can't allocate space for %d request queues", sc->vss_config.num_queues); goto fail; } if (pci_vtscsi_get_config_num(nvl, "ctl_ringsz", VTSCSI_MIN_RINGSZ, VTSCSI_MAX_RINGSZ, &sc->vss_ctl_ringsz) != 0) goto fail; if (pci_vtscsi_get_config_num(nvl, "evt_ringsz", VTSCSI_MIN_RINGSZ, VTSCSI_MAX_RINGSZ, &sc->vss_evt_ringsz) != 0) goto fail; if (pci_vtscsi_get_config_num(nvl, "req_ringsz", VTSCSI_MIN_RINGSZ, VTSCSI_MAX_RINGSZ, &sc->vss_req_ringsz) != 0) goto fail; if (pci_vtscsi_get_config_num(nvl, "thr_per_q", VTSCSI_MIN_THR_PER_Q, VTSCSI_MAX_THR_PER_Q, &sc->vss_thr_per_q) != 0) goto fail; value = get_config_value_node(nvl, "backend"); if (value == NULL) { if (SET_COUNT(pci_vtscsi_backend_set) == 0) { WPRINTF("No virtio-scsi backends available"); errno = EINVAL; goto fail; } backend = SET_ITEM(pci_vtscsi_backend_set, 0); } else { backend = NULL; SET_FOREACH(vbpp, pci_vtscsi_backend_set) { if (strcasecmp(value, (*vbpp)->vsb_name) == 0) { backend = *vbpp; break; } } if (backend == NULL) { WPRINTF("No such virtio-scsi backend: %s", value); errno = EINVAL; goto fail; } } err = backend->vsb_init(sc, backend, nvl); if (err != 0) { errno = EINVAL; goto fail; } nvl = find_relative_config_node(nvl, "target"); if (nvl != NULL) { err = walk_config_nodes("", nvl, sc, pci_vtscsi_count_targets); if (err != 0) goto fail; } if (sc->vss_num_target > 0) { sc->vss_targets = malloc(sc->vss_num_target * sizeof(struct pci_vtscsi_target)); if (sc->vss_targets == NULL) { EPRINTLN("can't allocate space for %lu targets", sc->vss_num_target); goto fail; } memset(sc->vss_targets, -1, sc->vss_num_target * sizeof(struct pci_vtscsi_target)); err = walk_config_nodes("", nvl, sc, pci_vtscsi_init_target); if (err != 0) goto fail; } /* * All targets should be open now and have a valid fd. */ for (size_t i = 0; i < sc->vss_num_target; i++) { if (sc->vss_targets[i].vst_target == i && sc->vss_targets[i].vst_fd < 0) { goto fail; } } pthread_mutex_init(&sc->vss_mtx, NULL); vi_softc_linkup(&sc->vss_vs, &sc->vss_vi_consts, sc, pi, sc->vss_vq); sc->vss_vs.vs_mtx = &sc->vss_mtx; /* * Perform a "reset" before we set up our queues. * * This will write the default config into vss_config, which is used * by the rest of the driver to get the request header size. Note that * if we ever allow the guest to override sense size through config * space writes, pre-allocation of I/O requests will have to change * accordingly. */ pthread_mutex_lock(&sc->vss_mtx); pci_vtscsi_reset(sc); pthread_mutex_unlock(&sc->vss_mtx); /* virtqueue 0: control queue */ sc->vss_vq[0].vq_qsize = sc->vss_ctl_ringsz; sc->vss_vq[0].vq_notify = pci_vtscsi_controlq_notify; /* virtqueue 1: event queue */ sc->vss_vq[1].vq_qsize = sc->vss_evt_ringsz; sc->vss_vq[1].vq_notify = pci_vtscsi_eventq_notify; /* virtqueue 2-n: request queues */ for (int i = VIRTIO_SCSI_ADDL_Q; i < sc->vss_vi_consts.vc_nvq; i++) { int rq = i - VIRTIO_SCSI_ADDL_Q; sc->vss_vq[i].vq_qsize = sc->vss_req_ringsz; sc->vss_vq[i].vq_notify = pci_vtscsi_requestq_notify; err = pci_vtscsi_init_queue(sc, &sc->vss_queues[rq], i); if (err != 0) { free(sc->vss_targets); goto fail; } } /* initialize config space */ vi_pci_init(pi, VIRTIO_MODE_TRANSITIONAL, VIRTIO_DEV_SCSI, VIRTIO_ID_SCSI, PCIC_STORAGE); if (!vi_intr_init(&sc->vss_vs, fbsdrun_virtio_msix())) { free(sc->vss_targets); goto fail; } if (!vi_pcibar_setup(&sc->vss_vs)) { free(sc->vss_targets); goto fail; } return (0); fail: free(sc); return (-1); } static const struct pci_devemu pci_de_vscsi = { .pe_emu = "virtio-scsi", .pe_init = pci_vtscsi_init, .pe_legacy_config = pci_vtscsi_legacy_config, .pe_cfgwrite = vi_pci_cfgwrite, .pe_cfgread = vi_pci_cfgread, .pe_barwrite = vi_pci_write, .pe_barread = vi_pci_read }; PCI_EMUL_SET(pci_de_vscsi); /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2016 Jakub Klama . * Copyright (c) 2018 Marcelo Araujo . * Copyright (c) 2026 Hans Rosenfeld * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer * in this position and unchanged. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef _PCI_VIRTIO_SCSI_H_ #define _PCI_VIRTIO_SCSI_H_ #include "iov.h" extern int pci_vtscsi_debug; #define WPRINTF(msg, params...) PRINTLN("virtio-scsi: " msg, ##params) #define DPRINTF(msg, params...) if (pci_vtscsi_debug) WPRINTF(msg, ##params) /* Absolute limits given by the VirtIO SCSI spec */ #define VIRTIO_SCSI_MAX_CHANNEL 0 #define VIRTIO_SCSI_MAX_TARGET 255 #define VIRTIO_SCSI_MAX_LUN 16383 #define VIRTIO_SCSI_HDR_SEG 2 #define VIRTIO_SCSI_ADDL_Q 2 /* Features specific to VirtIO SCSI, none of which we currently support */ #define VIRTIO_SCSI_F_INOUT (1 << 0) #define VIRTIO_SCSI_F_HOTPLUG (1 << 1) #define VIRTIO_SCSI_F_CHANGE (1 << 2) /* Default limits which we set. All of these are configurable. */ #define VTSCSI_DEF_RINGSZ 64 #define VTSCSI_MIN_RINGSZ 4 #define VTSCSI_MAX_RINGSZ 4096 #define VTSCSI_DEF_THR_PER_Q 16 #define VTSCSI_MIN_THR_PER_Q 1 #define VTSCSI_MAX_THR_PER_Q 256 #define VTSCSI_DEF_MAXSEG 64 #define VTSCSI_MIN_MAXSEG (VIRTIO_SCSI_HDR_SEG + 1) #define VTSCSI_MAX_MAXSEG \ (4096 - VIRTIO_SCSI_HDR_SEG - SPLIT_IOV_ADDL_IOV) #define VTSCSI_DEF_REQUESTQ 1 #define VTSCSI_MIN_REQUESTQ 1 #define VTSCSI_MAX_REQUESTQ (32 - VIRTIO_SCSI_ADDL_Q) /* * Device-specific config space registers * * The guest driver may try to modify cdb_size and sense_size by writing the * respective config space registers. Since we currently ignore all writes to * config space, these macros are essentially constant. */ #define VTSCSI_IN_HEADER_LEN(_sc) \ (sizeof(struct pci_vtscsi_req_cmd_rd) + _sc->vss_config.cdb_size) #define VTSCSI_OUT_HEADER_LEN(_sc) \ (sizeof(struct pci_vtscsi_req_cmd_wr) + _sc->vss_config.sense_size) struct pci_vtscsi_config { uint32_t num_queues; uint32_t seg_max; uint32_t max_sectors; uint32_t cmd_per_lun; uint32_t event_info_size; uint32_t sense_size; uint32_t cdb_size; uint16_t max_channel; uint16_t max_target; uint32_t max_lun; } __attribute__((packed)); /* * I/O request state * * Each pci_vtscsi_queue has configurable number of pci_vtscsi_request * structures pre-allocated on vsq_free_requests. For each I/O request * coming in on the I/O virtqueue, the request queue handler takes * pci_vtscsi_request off vsq_free_requests, fills in the data from the * I/O virtqueue, puts it on vsq_requests, and signals vsq_cv. * * Each pci_vtscsi_queue will have a configurable number of worker threads, * which wait on vsq_cv. When signalled, they repeatedly take a single * pci_vtscsi_request off vsq_requests and hand it to the backend, which * processes it synchronously. After completion, the pci_vtscsi_request * is re-initialized and put back onto vsq_free_requests. * * The worker threads exit when vsq_cv is signalled after vsw_exiting was set. * * The I/O vectors for each request are kept in the preallocated iovec array * vsr_iov, and pointers to the respective header/data in/out portions are set * up to point into the array when the request is queued for processing. * * The number of iovecs preallocated for vsr_iov is derived from the configured * 'seg_max' parameter defined by the virtio spec: * - 'seg_max' parameter specifies the maximum number of I/O data vectors * we support in any request * - we need 2 additional iovecs for the I/O headers (VIRTIO_SCSI_HDR_SEG) * - we need another 2 additional iovecs for split_iov() (SPLIT_IOV_ADDL_IOV) * * The only time we explicitly need the full size of vsr_iov after preallocation * is during re-initialization after completing a request, and implicitly in the * calls to split_iov() the set up the pointers. In all other cases, we use only * 'seg_max' + VIRTIO_SCSI_HDR_SEG, and we advertise only 'seg_max' to the guest * in accordance to the virtio spec. */ STAILQ_HEAD(pci_vtscsi_req_queue, pci_vtscsi_request); struct pci_vtscsi_queue { struct pci_vtscsi_softc *vsq_sc; struct vqueue_info *vsq_vq; pthread_mutex_t vsq_rmtx; pthread_mutex_t vsq_fmtx; pthread_mutex_t vsq_qmtx; pthread_cond_t vsq_cv; struct pci_vtscsi_req_queue vsq_requests; struct pci_vtscsi_req_queue vsq_free_requests; LIST_HEAD(, pci_vtscsi_worker) vsq_workers; }; struct pci_vtscsi_worker { struct pci_vtscsi_queue *vsw_queue; pthread_t vsw_thread; bool vsw_exiting; LIST_ENTRY(pci_vtscsi_worker) vsw_link; }; struct pci_vtscsi_request { struct pci_vtscsi_queue *vsr_queue; struct iovec *vsr_iov; struct iovec *vsr_iov_in; struct iovec *vsr_iov_out; struct iovec *vsr_data_iov_in; struct iovec *vsr_data_iov_out; struct pci_vtscsi_req_cmd_rd *vsr_cmd_rd; struct pci_vtscsi_req_cmd_wr *vsr_cmd_wr; void *vsr_backend; size_t vsr_niov_in; size_t vsr_niov_out; size_t vsr_data_niov_in; size_t vsr_data_niov_out; uint32_t vsr_idx; STAILQ_ENTRY(pci_vtscsi_request) vsr_link; }; /* * Per-target state. */ struct pci_vtscsi_target { uint8_t vst_target; int vst_fd; int vst_max_sectors; }; /* * Per-device softc */ struct pci_vtscsi_softc { struct virtio_softc vss_vs; struct virtio_consts vss_vi_consts; struct vqueue_info *vss_vq; struct pci_vtscsi_queue *vss_queues; pthread_mutex_t vss_mtx; uint32_t vss_features; size_t vss_num_target; uint32_t vss_ctl_ringsz; uint32_t vss_evt_ringsz; uint32_t vss_req_ringsz; uint32_t vss_thr_per_q; struct pci_vtscsi_config vss_default_config; struct pci_vtscsi_config vss_config; struct pci_vtscsi_target *vss_targets; struct pci_vtscsi_backend *vss_backend; }; /* * VirtIO-SCSI Task Management Function control requests */ #define VIRTIO_SCSI_T_TMF 0 #define VIRTIO_SCSI_T_TMF_ABORT_TASK 0 #define VIRTIO_SCSI_T_TMF_ABORT_TASK_SET 1 #define VIRTIO_SCSI_T_TMF_CLEAR_ACA 2 #define VIRTIO_SCSI_T_TMF_CLEAR_TASK_SET 3 #define VIRTIO_SCSI_T_TMF_I_T_NEXUS_RESET 4 #define VIRTIO_SCSI_T_TMF_LOGICAL_UNIT_RESET 5 #define VIRTIO_SCSI_T_TMF_QUERY_TASK 6 #define VIRTIO_SCSI_T_TMF_QUERY_TASK_SET 7 #define VIRTIO_SCSI_T_TMF_MAX_FUNC VIRTIO_SCSI_T_TMF_QUERY_TASK_SET /* command-specific response values */ #define VIRTIO_SCSI_S_FUNCTION_COMPLETE 0 #define VIRTIO_SCSI_S_FUNCTION_SUCCEEDED 10 #define VIRTIO_SCSI_S_FUNCTION_REJECTED 11 struct pci_vtscsi_ctrl_tmf { const uint32_t type; const uint32_t subtype; const uint8_t lun[8]; const uint64_t id; uint8_t response; } __attribute__((packed)); /* * VirtIO-SCSI Asynchronous Notification control requests */ #define VIRTIO_SCSI_T_AN_QUERY 1 #define VIRTIO_SCSI_EVT_ASYNC_OPERATIONAL_CHANGE 2 #define VIRTIO_SCSI_EVT_ASYNC_POWER_MGMT 4 #define VIRTIO_SCSI_EVT_ASYNC_EXTERNAL_REQUEST 8 #define VIRTIO_SCSI_EVT_ASYNC_MEDIA_CHANGE 16 #define VIRTIO_SCSI_EVT_ASYNC_MULTI_HOST 32 #define VIRTIO_SCSI_EVT_ASYNC_DEVICE_BUSY 64 struct pci_vtscsi_ctrl_an { const uint32_t type; const uint8_t lun[8]; const uint32_t event_requested; uint32_t event_actual; uint8_t response; } __attribute__((packed)); /* command-specific response values */ #define VIRTIO_SCSI_S_OK 0 #define VIRTIO_SCSI_S_OVERRUN 1 #define VIRTIO_SCSI_S_ABORTED 2 #define VIRTIO_SCSI_S_BAD_TARGET 3 #define VIRTIO_SCSI_S_RESET 4 #define VIRTIO_SCSI_S_BUSY 5 #define VIRTIO_SCSI_S_TRANSPORT_FAILURE 6 #define VIRTIO_SCSI_S_TARGET_FAILURE 7 #define VIRTIO_SCSI_S_NEXUS_FAILURE 8 #define VIRTIO_SCSI_S_FAILURE 9 #define VIRTIO_SCSI_S_INCORRECT_LUN 12 struct pci_vtscsi_event { uint32_t event; uint8_t lun[8]; uint32_t reason; } __attribute__((packed)); /* * VirtIO-SCSI I/O requests */ struct pci_vtscsi_req_cmd_rd { const uint8_t lun[8]; const uint64_t id; const uint8_t task_attr; const uint8_t prio; const uint8_t crn; const uint8_t cdb[]; } __attribute__((packed)); /* task_attr */ #define VIRTIO_SCSI_S_SIMPLE 0 #define VIRTIO_SCSI_S_ORDERED 1 #define VIRTIO_SCSI_S_HEAD 2 #define VIRTIO_SCSI_S_ACA 3 struct pci_vtscsi_req_cmd_wr { uint32_t sense_len; uint32_t residual; uint16_t status_qualifier; uint8_t status; uint8_t response; uint8_t sense[]; } __attribute__((packed)); /* * Backend interface */ struct pci_vtscsi_backend { const char *vsb_name; int (*vsb_init)(struct pci_vtscsi_softc *, struct pci_vtscsi_backend *, nvlist_t *); int (*vsb_open)(struct pci_vtscsi_softc *, const char *, long); void (*vsb_reset)(struct pci_vtscsi_softc *); void* (*vsb_req_alloc)(struct pci_vtscsi_softc *); void (*vsb_req_clear)(void *); void (*vsb_req_free)(void *); void (*vsb_tmf_hdl)(struct pci_vtscsi_softc *, int, struct pci_vtscsi_ctrl_tmf *); void (*vsb_an_hdl)(struct pci_vtscsi_softc *, int, struct pci_vtscsi_ctrl_an *); int (*vsb_req_hdl)(struct pci_vtscsi_softc *, int, struct pci_vtscsi_request *); }; #define PCI_VTSCSI_BACKEND_SET(x) DATA_SET(pci_vtscsi_backend_set, x) /* * LUN address parsing * * The LUN address consists of 8 bytes. While the spec describes this as 0x01, * followed by the target byte, followed by a "single-level LUN structure", * this is actually the same as a hierarchical LUN address as defined by SAM-5, * consisting of four levels of addressing, where in each level the two MSB of * byte 0 select the address mode used in the remaining bits and bytes. * * * Only the first two levels are acutally used by virtio-scsi: * * Level 1: 0x01, 0xTT: Peripheral Device Addressing: Bus 1, Target 0-255 * Level 2: 0xLL, 0xLL: Peripheral Device Addressing: Bus MBZ, LUN 0-255 * or: Flat Space Addressing: LUN (0-16383) * Level 3 and 4: not used, MBZ * * * Alternatively, the first level may contain an extended LUN address to select * the REPORT_LUNS well-known logical unit: * * Level 1: 0xC1, 0x01: Extended LUN Adressing, Well-Known LUN 1 (REPORT_LUNS) * Level 2, 3, and 4: not used, MBZ * * The virtio spec says that we SHOULD implement the REPORT_LUNS well-known * logical unit but we currently don't. * * According to the virtio spec, these are the only LUNS address formats to be * used with virtio-scsi. */ /* * Check that the given LUN address conforms to the virtio spec, does not * address an unknown target, and especially does not address the REPORT_LUNS * well-known logical unit. */ static inline bool pci_vtscsi_check_lun(struct pci_vtscsi_softc *sc, const uint8_t *lun) { if (lun[0] == 0xC1) return (false); if (lun[0] != 0x01) return (false); if (lun[1] >= sc->vss_num_target) return (false); if (lun[1] != sc->vss_targets[lun[1]].vst_target) return (false); if (sc->vss_targets[lun[1]].vst_fd < 0) return (false); if (lun[2] != 0x00 && (lun[2] & 0xc0) != 0x40) return (false); if (lun[4] != 0 || lun[5] != 0 || lun[6] != 0 || lun[7] != 0) return (false); return (true); } /* * Get the target id from a LUN address. * * Every code path using this function must have called pci_vtscsi_check_lun() * before to make sure the LUN address is valid. */ static inline uint8_t pci_vtscsi_get_target(struct pci_vtscsi_softc *sc, const uint8_t *lun) { assert(lun[0] == 0x01); assert(lun[1] < sc->vss_num_target); assert(lun[1] == sc->vss_targets[lun[1]].vst_target); assert(sc->vss_targets[lun[1]].vst_fd >= 0); assert(lun[2] == 0x00 || (lun[2] & 0xc0) == 0x40); return (lun[1]); } /* * Get the LUN id from a LUN address. * * Every code path using this function must have called pci_vtscsi_check_lun() * before to make sure the LUN address is valid. */ static inline uint16_t pci_vtscsi_get_lun(struct pci_vtscsi_softc *sc, const uint8_t *lun) { assert(lun[0] == 0x01); assert(lun[1] < sc->vss_num_target); assert(lun[1] == sc->vss_targets[lun[1]].vst_target); assert(sc->vss_targets[lun[1]].vst_fd >= 0); assert(lun[2] == 0x00 || (lun[2] & 0xc0) == 0x40); return (((lun[2] << 8) | lun[3]) & 0x3fff); } #endif /* _PCI_VIRTIO_SCSI_H_ */ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2016 Jakub Klama . * Copyright (c) 2018 Marcelo Araujo . * Copyright (c) 2026 Hans Rosenfeld * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer * in this position and unchanged. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "bhyverun.h" #include "config.h" #include "debug.h" #include "pci_emul.h" #include "virtio.h" #include "iov.h" #include "pci_virtio_scsi.h" struct vtscsi_ctl_backend { struct pci_vtscsi_backend vcb_backend; int vcb_iid; }; static int vtscsi_ctl_init(struct pci_vtscsi_softc *, struct pci_vtscsi_backend *, nvlist_t *); static int vtscsi_ctl_open(struct pci_vtscsi_softc *, const char *, long); static void vtscsi_ctl_reset(struct pci_vtscsi_softc *); static void *vtscsi_ctl_req_alloc(struct pci_vtscsi_softc *); static void vtscsi_ctl_req_clear(void *); static void vtscsi_ctl_req_free(void *); static void vtscsi_ctl_tmf_hdl(struct pci_vtscsi_softc *, int, struct pci_vtscsi_ctrl_tmf *); static void vtscsi_ctl_an_hdl(struct pci_vtscsi_softc *, int, struct pci_vtscsi_ctrl_an *); static int vtscsi_ctl_req_hdl(struct pci_vtscsi_softc *, int, struct pci_vtscsi_request *); static int vtscsi_ctl_count_targets(const char *prefix __unused, const nvlist_t *parent __unused, const char *name __unused, int type, void *arg) { int *count = arg; if (type != NV_TYPE_STRING) { EPRINTLN("invalid target \"%s\" type: not a string", name); errno = EINVAL; return (-1); } (*count)++; return (0); } static int vtscsi_ctl_init(struct pci_vtscsi_softc *sc, struct pci_vtscsi_backend *backend, nvlist_t *nvl) { int count = 0; int ret = 0; struct vtscsi_ctl_backend *ctl_backend; const char *value; ctl_backend = calloc(1, sizeof(struct vtscsi_ctl_backend)); if (ctl_backend == NULL) { EPRINTLN("failed to allocate backend data: %s", strerror(errno)); return (-1); } ctl_backend->vcb_backend = *backend; sc->vss_backend = &ctl_backend->vcb_backend; value = get_config_value_node(nvl, "iid"); if (value != NULL) ctl_backend->vcb_iid = strtoul(value, NULL, 10); /* * Count configured targets. If no targets were configured, use * /dev/cam/ctl to remain compatible with previous versions. */ nvl = find_relative_config_node(nvl, "target"); if (nvl != NULL) ret = walk_config_nodes("", nvl, &count, vtscsi_ctl_count_targets); if (ret != 0) return (ret); if (count == 0) set_config_value_node(nvl, "0", "/dev/cam/ctl"); return (0); } static int vtscsi_ctl_open(struct pci_vtscsi_softc *sc __unused, const char *devname, long target) { struct pci_vtscsi_target *tgt = &sc->vss_targets[target]; tgt->vst_fd = open(devname, O_RDWR); if (tgt->vst_fd < 0) return (-1); return (0); } static void vtscsi_ctl_reset(struct pci_vtscsi_softc *sc __unused) { /* * There doesn't seem to be a limit to the maximum number of * sectors CTL can transfer in one request. */ sc->vss_config.max_sectors = INT32_MAX; } static void *vtscsi_ctl_req_alloc(struct pci_vtscsi_softc *sc) { struct vtscsi_ctl_backend *ctl = (struct vtscsi_ctl_backend *)sc; union ctl_io *io = ctl_scsi_alloc_io(ctl->vcb_iid); if (io != NULL) ctl_scsi_zero_io(io); return (io); } static void vtscsi_ctl_req_clear(void *io) { ctl_scsi_zero_io(io); } static void vtscsi_ctl_req_free(void *io) { ctl_scsi_free_io(io); } static void vtscsi_ctl_tmf_hdl(struct pci_vtscsi_softc *sc, int fd, struct pci_vtscsi_ctrl_tmf *tmf) { struct vtscsi_ctl_backend *ctl; union ctl_io *io; int err; ctl = (struct vtscsi_ctl_backend *)sc->vss_backend; io = vtscsi_ctl_req_alloc(sc); if (io == NULL) { tmf->response = VIRTIO_SCSI_S_FAILURE; return; } vtscsi_ctl_req_clear(io); io->io_hdr.io_type = CTL_IO_TASK; io->io_hdr.nexus.initid = ctl->vcb_iid; io->io_hdr.nexus.targ_lun = pci_vtscsi_get_lun(sc, tmf->lun); io->taskio.tag_type = CTL_TAG_SIMPLE; io->taskio.tag_num = tmf->id; io->io_hdr.flags |= CTL_FLAG_USER_TAG; switch (tmf->subtype) { case VIRTIO_SCSI_T_TMF_ABORT_TASK: io->taskio.task_action = CTL_TASK_ABORT_TASK; break; case VIRTIO_SCSI_T_TMF_ABORT_TASK_SET: io->taskio.task_action = CTL_TASK_ABORT_TASK_SET; break; case VIRTIO_SCSI_T_TMF_CLEAR_ACA: io->taskio.task_action = CTL_TASK_CLEAR_ACA; break; case VIRTIO_SCSI_T_TMF_CLEAR_TASK_SET: io->taskio.task_action = CTL_TASK_CLEAR_TASK_SET; break; case VIRTIO_SCSI_T_TMF_I_T_NEXUS_RESET: io->taskio.task_action = CTL_TASK_I_T_NEXUS_RESET; break; case VIRTIO_SCSI_T_TMF_LOGICAL_UNIT_RESET: io->taskio.task_action = CTL_TASK_LUN_RESET; break; case VIRTIO_SCSI_T_TMF_QUERY_TASK: io->taskio.task_action = CTL_TASK_QUERY_TASK; break; case VIRTIO_SCSI_T_TMF_QUERY_TASK_SET: io->taskio.task_action = CTL_TASK_QUERY_TASK_SET; break; } if (pci_vtscsi_debug) { struct sbuf *sb = sbuf_new_auto(); ctl_io_sbuf(io, sb); sbuf_finish(sb); DPRINTF("%s", sbuf_data(sb)); sbuf_delete(sb); } err = ioctl(fd, CTL_IO, io); if (err != 0) { WPRINTF("CTL_IO: err=%d (%s)", errno, strerror(errno)); tmf->response = VIRTIO_SCSI_S_FAILURE; } else { tmf->response = io->taskio.task_status; } vtscsi_ctl_req_free(io); } static void vtscsi_ctl_an_hdl(struct pci_vtscsi_softc *sc __unused, int fd __unused, struct pci_vtscsi_ctrl_an *an) { an->response = VIRTIO_SCSI_S_FAILURE; } static int vtscsi_ctl_req_hdl(struct pci_vtscsi_softc *sc, int fd, struct pci_vtscsi_request *req) { union ctl_io *io = req->vsr_backend; void *ext_data_ptr = NULL; uint32_t ext_data_len = 0, ext_sg_entries = 0; struct vtscsi_ctl_backend *ctl; int err, nxferred; ctl = (struct vtscsi_ctl_backend *)sc->vss_backend; io->io_hdr.nexus.initid = ctl->vcb_iid; io->io_hdr.nexus.targ_lun = pci_vtscsi_get_lun(sc, req->vsr_cmd_rd->lun); io->io_hdr.io_type = CTL_IO_SCSI; if (req->vsr_data_niov_in > 0) { ext_data_ptr = (void *)req->vsr_data_iov_in; ext_sg_entries = req->vsr_data_niov_in; ext_data_len = count_iov(req->vsr_data_iov_in, req->vsr_data_niov_in); io->io_hdr.flags |= CTL_FLAG_DATA_OUT; } else if (req->vsr_data_niov_out > 0) { ext_data_ptr = (void *)req->vsr_data_iov_out; ext_sg_entries = req->vsr_data_niov_out; ext_data_len = count_iov(req->vsr_data_iov_out, req->vsr_data_niov_out); io->io_hdr.flags |= CTL_FLAG_DATA_IN; } io->scsiio.sense_len = sc->vss_config.sense_size; io->scsiio.tag_num = req->vsr_cmd_rd->id; io->io_hdr.flags |= CTL_FLAG_USER_TAG; switch (req->vsr_cmd_rd->task_attr) { case VIRTIO_SCSI_S_ORDERED: io->scsiio.tag_type = CTL_TAG_ORDERED; break; case VIRTIO_SCSI_S_HEAD: io->scsiio.tag_type = CTL_TAG_HEAD_OF_QUEUE; break; case VIRTIO_SCSI_S_ACA: io->scsiio.tag_type = CTL_TAG_ACA; break; case VIRTIO_SCSI_S_SIMPLE: default: io->scsiio.tag_type = CTL_TAG_SIMPLE; break; } io->scsiio.ext_sg_entries = ext_sg_entries; io->scsiio.ext_data_ptr = ext_data_ptr; io->scsiio.ext_data_len = ext_data_len; io->scsiio.ext_data_filled = 0; io->scsiio.cdb_len = sc->vss_config.cdb_size; memcpy(io->scsiio.cdb, req->vsr_cmd_rd->cdb, sc->vss_config.cdb_size); if (pci_vtscsi_debug) { struct sbuf *sb = sbuf_new_auto(); ctl_io_sbuf(io, sb); sbuf_finish(sb); DPRINTF("%s", sbuf_data(sb)); sbuf_delete(sb); } err = ioctl(fd, CTL_IO, io); if (err != 0) { WPRINTF("CTL_IO: err=%d (%s)", errno, strerror(errno)); req->vsr_cmd_wr->response = VIRTIO_SCSI_S_FAILURE; } else { req->vsr_cmd_wr->sense_len = MIN(io->scsiio.sense_len, sc->vss_config.sense_size); req->vsr_cmd_wr->residual = ext_data_len - io->scsiio.ext_data_filled; req->vsr_cmd_wr->status = io->scsiio.scsi_status; req->vsr_cmd_wr->response = VIRTIO_SCSI_S_OK; memcpy(&req->vsr_cmd_wr->sense, &io->scsiio.sense_data, req->vsr_cmd_wr->sense_len); } nxferred = io->scsiio.ext_data_filled; return (nxferred); } static const struct pci_vtscsi_backend vtscsi_ctl_backend = { .vsb_name = "ctl", .vsb_init = vtscsi_ctl_init, .vsb_open = vtscsi_ctl_open, .vsb_reset = vtscsi_ctl_reset, .vsb_req_alloc = vtscsi_ctl_req_alloc, .vsb_req_clear = vtscsi_ctl_req_clear, .vsb_req_free = vtscsi_ctl_req_free, .vsb_tmf_hdl = vtscsi_ctl_tmf_hdl, .vsb_an_hdl = vtscsi_ctl_an_hdl, .vsb_req_hdl = vtscsi_ctl_req_hdl }; PCI_VTSCSI_BACKEND_SET(vtscsi_ctl_backend); /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2016 Jakub Klama . * Copyright (c) 2018 Marcelo Araujo . * Copyright (c) 2026 Hans Rosenfeld * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer * in this position and unchanged. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "bhyverun.h" #include "config.h" #include "debug.h" #include "pci_emul.h" #include "virtio.h" #include "iov.h" #include "privileges.h" #include "pci_virtio_scsi.h" struct vtscsi_uscsi_backend { struct pci_vtscsi_backend vub_backend; libscsi_hdl_t *vub_scsi_hdl; }; static int vtscsi_uscsi_init(struct pci_vtscsi_softc *, struct pci_vtscsi_backend *, nvlist_t *); static int vtscsi_uscsi_open(struct pci_vtscsi_softc *, const char *, long); static void vtscsi_uscsi_reset(struct pci_vtscsi_softc *); static void *vtscsi_uscsi_req_alloc(struct pci_vtscsi_softc *); static void vtscsi_uscsi_req_clear(void *); static void vtscsi_uscsi_req_free(void *); static void vtscsi_uscsi_tmf_hdl(struct pci_vtscsi_softc *, int, struct pci_vtscsi_ctrl_tmf *); static void vtscsi_uscsi_an_hdl(struct pci_vtscsi_softc *, int, struct pci_vtscsi_ctrl_an *); static int vtscsi_uscsi_req_hdl(struct pci_vtscsi_softc *, int, struct pci_vtscsi_request *); static void vtscsi_uscsi_make_check_condition(struct uscsi_cmd *, char, char, char); static void vtscsi_uscsi_filter_post_report_luns(struct uscsi_cmd *); static void vtscsi_uscsi_filter_post(struct uscsi_cmd *); static int vtscsi_uscsi_init(struct pci_vtscsi_softc *sc, struct pci_vtscsi_backend *backend, nvlist_t *nvl __unused) { struct vtscsi_uscsi_backend *uscsi_backend; libscsi_errno_t serr; uscsi_backend = calloc(1, sizeof (struct vtscsi_uscsi_backend)); if (uscsi_backend == NULL) { EPRINTLN("failed to allocate backend data: %s", strerror(errno)); return (-1); } uscsi_backend->vub_backend = *backend; uscsi_backend->vub_scsi_hdl = libscsi_init(LIBSCSI_VERSION, &serr); if (uscsi_backend->vub_scsi_hdl == NULL) { EPRINTLN("failed to initialize libscsi: %s", libscsi_strerror(serr)); free(uscsi_backend); return (-1); } sc->vss_backend = &uscsi_backend->vub_backend; return (0); } static int vtscsi_uscsi_open(struct pci_vtscsi_softc *sc, const char *path, long target) { struct pci_vtscsi_target *tgt = &sc->vss_targets[target]; uscsi_xfer_t maxxfer = 0; /* * Most SCSI target drivers require the SYS_DEVICES privilege to send * USCSI commands. */ illumos_priv_add_min(PRIV_SYS_DEVICES, "scsi"); /* * Open the target. */ tgt->vst_fd = open(path, O_RDWR); if (tgt->vst_fd < 0) return (-1); /* * Get the maximum transfer size of the backend device. */ if (ioctl(tgt->vst_fd, USCSIMAXXFER, &maxxfer) < 0) { int errno_save = errno; if (errno == ENOTTY) { /* * The underlying device doesn't support this ioctl. * Limit max_sectors to 128MB, which is as good as * any other assumption. */ tgt->vst_max_sectors = 128 << (20 - 9); return (0); } WPRINTF("USCSIMAXXFER: unexpected error: errno=%d (%s)", strerrorname_np(errno), strerror(errno)); (void) close(tgt->vst_fd); tgt->vst_fd = -1; errno = errno_save; return (-1); } /* * Even though the virtio spec isn't particularly verbose about what * "max_sectors" actually means and what size a sector is, Linux seems * to treat it as a number of 512b sectors. * * In any case, we need to limit maxxfer such that it fits into a signed * 32bit int. */ if (maxxfer > INT32_MAX) maxxfer = INT32_MAX; tgt->vst_max_sectors = maxxfer >> 9; return (0); } static void vtscsi_uscsi_reset(struct pci_vtscsi_softc *sc) { size_t i; sc->vss_config.max_sectors = INT32_MAX; /* * As we may be configured to use a variety of differing backend devices * with varying maximum transfer sizes but virtio-scsi supports only one * max_sectors limit per instance, we'll use the smallest maximum * transfer size found. */ for (i = 0; i < sc->vss_num_target; i++) { struct pci_vtscsi_target *tgt = &sc->vss_targets[i]; if (tgt->vst_max_sectors < sc->vss_config.max_sectors) sc->vss_config.max_sectors = tgt->vst_max_sectors; } } static void * vtscsi_uscsi_req_alloc(struct pci_vtscsi_softc *sc) { return (calloc(1, sizeof (struct uscsi_cmd))); } static void vtscsi_uscsi_req_clear(void *io) { bzero(io, sizeof (struct uscsi_cmd)); } static void vtscsi_uscsi_req_free(void *io) { free(io); } static void vtscsi_uscsi_tmf_hdl(struct pci_vtscsi_softc *sc __unused, int fd, struct pci_vtscsi_ctrl_tmf *tmf) { struct uscsi_cmd cmd; int err; /* We currently support only LUN 0. */ if (pci_vtscsi_get_lun(sc, tmf->lun) != 0) { tmf->response = VIRTIO_SCSI_S_BAD_TARGET; return; } tmf->response = VIRTIO_SCSI_S_FUNCTION_COMPLETE; memset(&cmd, 0, sizeof (cmd)); cmd.uscsi_status = -1; cmd.uscsi_flags = USCSI_DIAGNOSE | USCSI_SILENT; /* The only TMF requests that we can handle here are RESETs. */ switch (tmf->subtype) { case VIRTIO_SCSI_T_TMF_I_T_NEXUS_RESET: cmd.uscsi_flags |= USCSI_RESET_TARGET; break; case VIRTIO_SCSI_T_TMF_LOGICAL_UNIT_RESET: cmd.uscsi_flags |= USCSI_RESET_LUN; break; default: /* * For all other TMF requests, return FUNCTION COMPLETE as * there is nothing we can or need to do for them. * * See the comments in pci_vtscsi_tmf_handle() for additional * information on how the common code and the backend-specific * code interact for TMF requests. */ tmf->response = VIRTIO_SCSI_S_FUNCTION_COMPLETE; return; } err = ioctl(fd, USCSICMD, &cmd); if (err != 0) { WPRINTF("USCSICMD: unexpected TMF error, errno=%d (%s)", strerrorname_np(errno), strerror(errno)); tmf->response = VIRTIO_SCSI_S_FAILURE; } } static void vtscsi_uscsi_an_hdl(struct pci_vtscsi_softc *sc __unused, int fd __unused, struct pci_vtscsi_ctrl_an *an) { /* We currently support only LUN 0. */ if (pci_vtscsi_get_lun(sc, an->lun) != 0) { an->response = VIRTIO_SCSI_S_BAD_TARGET; return; } an->response = VIRTIO_SCSI_S_FAILURE; } static int vtscsi_uscsi_req_hdl(struct pci_vtscsi_softc *sc, int fd, struct pci_vtscsi_request *req) { struct vtscsi_uscsi_backend *uscsi = (struct vtscsi_uscsi_backend *)sc->vss_backend; struct uscsi_cmd *cmd = req->vsr_backend; void *ext_data = NULL; ssize_t ext_data_len = 0; int nxferred = 0; /* We currently support only LUN 0. */ if (pci_vtscsi_get_lun(sc, req->vsr_cmd_rd->lun) != 0) { req->vsr_cmd_wr->response = VIRTIO_SCSI_S_BAD_TARGET; return (0); } if (req->vsr_data_niov_in > 0) { ext_data_len = iov_to_buf(req->vsr_data_iov_in, req->vsr_data_niov_in, &ext_data); cmd->uscsi_flags |= USCSI_WRITE; } else if (req->vsr_data_niov_out > 0) { ext_data_len = count_iov(req->vsr_data_iov_out, req->vsr_data_niov_out); ext_data = malloc(ext_data_len); cmd->uscsi_flags |= USCSI_READ; } /* Stop here if we failed to allocate ext_data. */ if (ext_data == NULL && ext_data_len != 0) { WPRINTF("failed to allocate buffer for ext_data"); req->vsr_cmd_wr->response = VIRTIO_SCSI_S_FAILURE; return (0); } cmd->uscsi_buflen = ext_data_len; cmd->uscsi_bufaddr = ext_data; cmd->uscsi_cdb = (caddr_t)req->vsr_cmd_rd->cdb; cmd->uscsi_cdblen = libscsi_cmd_cdblen(uscsi->vub_scsi_hdl, req->vsr_cmd_rd->cdb[0]); cmd->uscsi_status = -1; /* * We set an unreasonably large timeout here. The virtio spec doesn't * provide a way for the guest driver to pass a I/O timeout value to * the device, but if our timeout here is larger than any timeout the * guest uses, we can expect them to abort the command before we would. * * INT16_MAX corresponds to a bit over 9 hours, which should be enough. */ cmd->uscsi_timeout = INT16_MAX; cmd->uscsi_flags |= USCSI_DIAGNOSE; cmd->uscsi_rqlen = sc->vss_config.sense_size; cmd->uscsi_rqbuf = (caddr_t)req->vsr_cmd_wr->sense; cmd->uscsi_flags |= USCSI_RQENABLE; switch (req->vsr_cmd_rd->task_attr) { case VIRTIO_SCSI_S_ORDERED: cmd->uscsi_flags |= USCSI_OTAG; break; case VIRTIO_SCSI_S_HEAD: cmd->uscsi_flags |= USCSI_HEAD|USCSI_HTAG; break; case VIRTIO_SCSI_S_SIMPLE: break; case VIRTIO_SCSI_S_ACA: /* * I haven't found any indication in our code that would * suggest that we support ACA in any way in illumos. In * fact, scsi_transport() asserts that NACA isn't set in * a packet, and scsi_uscsi_pktinit() warns about it and * clears the flag if found set. There's a tunable to * override that behaviour (scsi_pkt_allow_naca), but there * really seems to be no code properly handling ACA or * setting the ACA flag. * * I guess this makes sense since we're doing ARQ anyway, * so let's just pretend no target is ever in ACA state * and thus no packet will ever require this. */ default: WPRINTF("USCSICMD: unexpected task attr in request: 0x%x", req->vsr_cmd_rd->task_attr); req->vsr_cmd_wr->response = VIRTIO_SCSI_S_FAILURE; return (0); } errno = 0; (void) ioctl(fd, USCSICMD, cmd); switch (errno) { case EIO: /* * EIO may indicate that a SCSI error occured. If that's the * case, uscsi_status should have been set to a valid value, * and we want to continue to process the request normally. */ if (cmd->uscsi_status == -1) { req->vsr_cmd_wr->response = VIRTIO_SCSI_S_FAILURE; break; } /*FALLTHRU*/ case 0: /* * If the command completed successfully, apply any necessary * post-completion filtering. */ if (cmd->uscsi_status == STATUS_GOOD) vtscsi_uscsi_filter_post(cmd); req->vsr_cmd_wr->sense_len = sc->vss_config.sense_size - cmd->uscsi_rqresid; req->vsr_cmd_wr->residual = cmd->uscsi_resid; req->vsr_cmd_wr->status = cmd->uscsi_status; req->vsr_cmd_wr->response = VIRTIO_SCSI_S_OK; nxferred = ext_data_len - req->vsr_cmd_wr->residual; if (req->vsr_data_niov_out > 0) { (void) buf_to_iov(ext_data, nxferred, req->vsr_data_iov_out, req->vsr_data_niov_out); } break; case EAGAIN: /* * Despite not being documented in uscsi(4I), sd(4D) returns * this when the device is busy formatting. */ req->vsr_cmd_wr->response = VIRTIO_SCSI_S_BUSY; break; case EINVAL: /* * This may happen if packet allocation fails, which in turn * may happen if we didn't honor USCSIMAXXFER. */ req->vsr_cmd_wr->response = VIRTIO_SCSI_S_OVERRUN; break; case EFAULT: /* * EFAULT should never happen as we never send bogus memory * addresses in our USCSI commands. */ case EPERM: /* * EPERM should never happen as we have the SYS_DEVICES * privilege. */ default: WPRINTF("USCSICMD: unexpected I/O error: errno=%d (%s)", strerrorname_np(errno), strerror(errno)); abort(); } free(ext_data); return (nxferred); } /* * Return a CHECK CONDITION and fill in the sense data with the given sense key, * additional sense code, and additional sense qualifier. */ static void vtscsi_uscsi_make_check_condition(struct uscsi_cmd *cmd, char key, char asc, char qual) { cmd->uscsi_status = STATUS_CHECK; cmd->uscsi_resid = cmd->uscsi_buflen; cmd->uscsi_rqstatus = STATUS_GOOD; bzero(cmd->uscsi_rqbuf, cmd->uscsi_rqlen); if (cmd->uscsi_rqlen >= 1) cmd->uscsi_rqbuf[0] = 0x70; if (cmd->uscsi_rqlen >= 3) cmd->uscsi_rqbuf[2] = key; if (cmd->uscsi_rqlen >= 8) cmd->uscsi_rqbuf[7] = cmd->uscsi_rqlen - 8; if (cmd->uscsi_rqlen >= 13) cmd->uscsi_rqbuf[12] = asc; if (cmd->uscsi_rqlen >= 14) cmd->uscsi_rqbuf[13] = qual; } /* * We currently only support LUN 0. Make sure we never report anything else. * * We make no assumption about the buffer size. If it's large enough to hold the * LUN list length, we'll set the LUN list length to 8. The resid is adjusted if * the buffer size is larger than 16 bytes, which is the length needed to hold * one LUN address. */ static void vtscsi_uscsi_filter_post_report_luns(struct uscsi_cmd *cmd) { uint8_t report = (uint8_t)cmd->uscsi_cdb[2]; bzero(cmd->uscsi_bufaddr, cmd->uscsi_buflen); switch (report) { case 0: case 2: /* * We'll overwrite the output from the device to report just one * LUN with an all-zero address: * - LUN list length is 8 * - LUN 1 address is 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 */ if (cmd->uscsi_buflen >= 4) cmd->uscsi_bufaddr[3] = 8; if (cmd->uscsi_buflen >= 16) cmd->uscsi_resid = cmd->uscsi_buflen - 16; break; case 1: /* * We don't report any Well-Known LUNs either, because we have * no way to address them anyway using USCSICMD. */ cmd->uscsi_resid = cmd->uscsi_buflen; break; default: /* * All other values for "select report" are either invalid or * vendor-specific and thus unsupported. Return the command with * CHECK CONDITION, and fill in sense data to report a ILLEGAL * REQUEST with INVALID FIELD IN CDB. */ vtscsi_uscsi_make_check_condition(cmd, KEY_ILLEGAL_REQUEST, 0x24, 0x00); } } static void vtscsi_uscsi_filter_post(struct uscsi_cmd *cmd) { switch ((uint8_t)cmd->uscsi_cdb[0]) { case SCMD_REPORT_LUNS: vtscsi_uscsi_filter_post_report_luns(cmd); break; default: break; } } static const struct pci_vtscsi_backend vtscsi_uscsi_backend = { .vsb_name = "uscsi", .vsb_init = vtscsi_uscsi_init, .vsb_open = vtscsi_uscsi_open, .vsb_reset = vtscsi_uscsi_reset, .vsb_req_alloc = vtscsi_uscsi_req_alloc, .vsb_req_clear = vtscsi_uscsi_req_clear, .vsb_req_free = vtscsi_uscsi_req_free, .vsb_tmf_hdl = vtscsi_uscsi_tmf_hdl, .vsb_an_hdl = vtscsi_uscsi_an_hdl, .vsb_req_hdl = vtscsi_uscsi_req_hdl }; PCI_VTSCSI_BACKEND_SET(vtscsi_uscsi_backend); /* * Copyright (c) 2011 NetApp, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * This file and its contents are supplied under the terms of the * Common Development and Distribution License ("CDDL"), version 1.0. * You may only use this file in accordance with the terms of version * 1.0 of the CDDL. * * A full copy of the text of the CDDL should have accompanied this * source. A copy of the CDDL is also available via the Internet at * http://www.illumos.org/license/CDDL. * * Copyright 2015 Pluribus Networks Inc. * Copyright 2019 Joyent, Inc. * Copyright 2022 OmniOS Community Edition (OmniOSce) Association. * Copyright 2026 Oxide Computer Company */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "bhyverun.h" #include "config.h" #include "debug.h" #include "pci_emul.h" #include "virtio.h" #include "iov.h" #include "virtio_net.h" /* * This is the default number of queues allocated and advertised via the * multi-queue feature. It can be overridden via the `qpair` device option. */ #define VIONA_DEFAULT_MAX_QPAIR 8 /* * These are the default TX and RX ring sizes. They can be overidden with the * `vqsize`, `rxvqsize` and `txvqsize` options. The first sets both values to * the same while the last two allow setting different values for TX and RX. * * It is common to have asymmetry here as the RX queue must absorb host-side * burstiness while some backpressure is desirable on the TX side. */ #define VIONA_DEFAULT_TX_RINGSZ 256 #define VIONA_DEFAULT_RX_RINGSZ 1024 #define VIONA_CTLQ_SIZE 64 #define VIONA_CTLQ_MAXSEGS 32 #define VIONA_DEFAULT_LINK_SPEED 1000 /* In Mb/s, so this is 1Gb/s */ /* * These macros work in terms of TX/RX queues only, which is always what we * need when interfacing with viona since the control queue is implemented * entirely in userspace. */ #define VIONA_P2QS(p) ((p) * 2) #define VIONA_NRINGS(sc) ((sc)->vsc_consts.vc_nvq - 1) #define VIONA_USABLE_RINGS(sc) (VIONA_P2QS((sc)->vsc_vq_usepairs)) #define VIONA_RING_VALID(sc, r) ((r) < VIONA_NRINGS(sc)) #define VIONA_RING(sc, n) (&(sc)->vsc_queues[(n)]) #define VIONA_RXQ(sc, n) (VIONA_RING(sc, (n) * 2)) #define VIONA_TXQ(sc, n) (VIONA_RING(sc, (n) * 2 + 1)) #define IS_RXQ(i) ((i) % 2 == 0) #define IS_TXQ(i) ((i) % 2 != 0) /* * The control queue is always in the last slot of allocated rings, regardless * of how many rings are in use. */ #define VIONA_CTLQ_NUM(sc) (VIONA_NRINGS(sc)) #define VIONA_RING_CTLQ(sc, r) (r == VIONA_CTLQ_NUM(sc)) #define VIONA_CTLQ(sc) (VIONA_RING(sc, VIONA_CTLQ_NUM(sc))) /* * Debug printf */ static volatile int pci_viona_debug; #define DPRINTF(fmt, arg...) \ do { \ if (pci_viona_debug) { \ FPRINTLN(stdout, fmt, ##arg); \ fflush(stdout); \ } \ } while (0) #define WPRINTF(fmt, arg...) FPRINTLN(stderr, fmt, ##arg) /* * Per-device softc */ struct pci_viona_softc { struct virtio_softc vsc_vs; struct virtio_consts vsc_consts; struct vqueue_info *vsc_queues; pthread_mutex_t vsc_mtx; struct virtio_net_config vsc_config; datalink_id_t vsc_linkid; int vsc_vnafd; /* Configurable parameters */ char vsc_linkname[MAXLINKNAMELEN]; uint64_t vsc_feature_mask; uint16_t vsc_vq_usepairs; /* RX/TX pairs in use */ uint16_t vsc_vq_rxsize; /* size of each RX queue */ uint16_t vsc_vq_txsize; /* size of each TX queue */ bool vsc_resetting; bool vsc_msix_active; viona_promisc_t vsc_promisc; /* Current promisc mode */ bool vsc_promisc_promisc; /* PROMISC enabled */ bool vsc_promisc_allmulti; /* ALLMULTI enabled */ bool vsc_promisc_umac; /* unicast MACs sent */ bool vsc_promisc_mmac; /* multicast MACs sent */ }; static int pci_viona_cfgread(void *, int, int, uint32_t *); static int pci_viona_cfgwrite(void *, int, int, uint32_t); static uint64_t pci_viona_get_hv_features(void *, bool); static void pci_viona_set_hv_features(void *, uint64_t *); static void pci_viona_qnotify(void *, struct vqueue_info *); static void pci_viona_ctlqnotify(void *, struct vqueue_info *); static void pci_viona_ring_init(void *, uint64_t, bool); static void pci_viona_reset(void *); static void pci_viona_ring_set_msix(void *, int); static void pci_viona_update_msix(void *, uint64_t); static virtio_capstr_t viona_caps[] = { { VIRTIO_NET_F_CSUM, "VIRTIO_NET_F_CSUM" }, { VIRTIO_NET_F_GUEST_CSUM, "VIRTIO_NET_F_GUEST_CSUM" }, { VIRTIO_NET_F_MTU, "VIRTIO_NET_F_MTU" }, { VIRTIO_NET_F_MAC, "VIRTIO_NET_F_MAC" }, { VIRTIO_NET_F_GSO_DEPREC, "VIRTIO_NET_F_GSO_DEPREC" }, { VIRTIO_NET_F_GUEST_TSO4, "VIRTIO_NET_F_GUEST_TSO4" }, { VIRTIO_NET_F_GUEST_TSO6, "VIRTIO_NET_F_GUEST_TSO6" }, { VIRTIO_NET_F_GUEST_ECN, "VIRTIO_NET_F_GUEST_ECN" }, { VIRTIO_NET_F_GUEST_UFO, "VIRTIO_NET_F_GUEST_UFO" }, { VIRTIO_NET_F_HOST_TSO4, "VIRTIO_NET_F_HOST_TSO4" }, { VIRTIO_NET_F_HOST_TSO6, "VIRTIO_NET_F_HOST_TSO6" }, { VIRTIO_NET_F_HOST_ECN, "VIRTIO_NET_F_HOST_ECN" }, { VIRTIO_NET_F_HOST_UFO, "VIRTIO_NET_F_HOST_UFO" }, { VIRTIO_NET_F_MRG_RXBUF, "VIRTIO_NET_F_MRG_RXBUF" }, { VIRTIO_NET_F_STATUS, "VIRTIO_NET_F_STATUS" }, { VIRTIO_NET_F_CTRL_VQ, "VIRTIO_NET_F_CTRL_VQ" }, { VIRTIO_NET_F_CTRL_RX, "VIRTIO_NET_F_CTRL_RX" }, { VIRTIO_NET_F_CTRL_VLAN, "VIRTIO_NET_F_CTRL_VLAN" }, { VIRTIO_NET_F_GUEST_ANNOUNCE, "VIRTIO_NET_F_GUEST_ANNOUNCE" }, { VIRTIO_NET_F_MQ, "VIRTIO_NET_F_MQ" }, { VIRTIO_F_CTRL_MAC_ADDR, "VIRTIO_F_CTRL_MAC_ADDR" }, { VIRTIO_NET_F_SPEED_DUPLEX, "VIRTIO_NET_F_SPEED_DUPLEX" }, }; static struct virtio_consts viona_vi_consts = { .vc_name = "viona", .vc_nvq = 0, /* set in pci_viona_qalloc() */ .vc_max_nvq = 0, /* set in pci_viona_parse_opts() */ .vc_cfgsize = sizeof (struct virtio_net_config), .vc_cfgread = pci_viona_cfgread, .vc_cfgwrite = pci_viona_cfgwrite, .vc_set_msix = pci_viona_ring_set_msix, .vc_update_msix = pci_viona_update_msix, .vc_reset = pci_viona_reset, .vc_qinit = pci_viona_ring_init, .vc_qnotify = pci_viona_qnotify, .vc_hv_features = pci_viona_get_hv_features, .vc_apply_features = pci_viona_set_hv_features, /* * The following fields are populated using the response from the * viona driver during initialisation, augmented with the additional * capabilities emulated in userspace. */ .vc_hv_caps_legacy = 0, .vc_hv_caps_modern = 0, .vc_capstr = viona_caps, .vc_ncapstr = ARRAY_SIZE(viona_caps), }; static void pci_viona_ring_reset(struct pci_viona_softc *sc, int ring) { DPRINTF("viona: ring reset 0x%x", ring); for (;;) { int res; res = ioctl(sc->vsc_vnafd, VNA_IOC_RING_RESET, ring); if (res == 0) { break; } else if (errno != EINTR) { WPRINTF("ioctl viona ring 0x%x reset failed %d", ring, errno); return; } } } static bool pci_viona_set_usepairs(struct pci_viona_softc *sc, uint16_t pairs) { const uint16_t opairs = sc->vsc_vq_usepairs; DPRINTF("QUSE pairs 0x%x -> 0x%x", opairs, pairs); if (opairs == pairs) return (true); if (ioctl(sc->vsc_vnafd, VNA_IOC_SET_USEPAIRS, pairs) != 0) { WPRINTF("error setting viona use pairs(0x%x): %s", pairs, strerror(errno)); return (false); } sc->vsc_vq_usepairs = pairs; return (true); } static bool pci_viona_qalloc(struct pci_viona_softc *sc, int pairs) { struct virtio_consts *vc = &sc->vsc_consts; struct vqueue_info *queues; int nqueues, oqueues; oqueues = vc->vc_nvq; /* Add one for the control queue */ nqueues = VIONA_P2QS(pairs) + 1; DPRINTF("QALLOC pairs 0x%x (0x%x -> 0x%x)", pairs, oqueues, nqueues); if (oqueues == nqueues) return (true); queues = recallocarray(sc->vsc_queues, oqueues, nqueues, sizeof (struct vqueue_info)); if (queues == NULL) { WPRINTF("Failed to allocate memory changing queues from " "0x%x to 0x%x", oqueues, nqueues); return (false); } sc->vsc_queues = queues; vc->vc_nvq = nqueues; for (uint_t i = 0; i < vc->vc_nvq; i++) { sc->vsc_queues[i].vq_qsize = IS_RXQ(i) ? sc->vsc_vq_rxsize : sc->vsc_vq_txsize; sc->vsc_queues[i].vq_notify = NULL; } VIONA_CTLQ(sc)->vq_qsize = VIONA_CTLQ_SIZE; VIONA_CTLQ(sc)->vq_notify = pci_viona_ctlqnotify; vi_queue_linkup(&sc->vsc_vs, sc->vsc_queues); if (ioctl(sc->vsc_vnafd, VNA_IOC_SET_PAIRS, pairs) != 0) { WPRINTF("error setting viona queue pairs(0x%x): %s", pairs, strerror(errno)); return (false); } return (true); } static void pci_viona_reset(void *vsc) { struct pci_viona_softc *sc = vsc; DPRINTF("viona: device reset requested !"); vi_reset_dev(&sc->vsc_vs); /* Reset all TX/RX rings */ for (uint16_t i = 0; i < VIONA_NRINGS(sc); i++) pci_viona_ring_reset(sc, i); /* Shrink back down to one queue pair */ VERIFY(pci_viona_set_usepairs(sc, 1)); VERIFY(pci_viona_qalloc(sc, 1)); } static const char * pci_viona_promisc_descr(viona_promisc_t mode) { switch (mode) { case VIONA_PROMISC_NONE: return ("none"); case VIONA_PROMISC_MULTI: return ("multicast"); case VIONA_PROMISC_ALL: return ("all"); default: abort(); } } static int pci_viona_eval_promisc(struct pci_viona_softc *sc) { viona_promisc_t mode = VIONA_PROMISC_NONE; int err = 0; /* * If the guest has explicitly requested promiscuous mode or has sent a * non-empty unicast MAC address table, then set viona to promiscuous * mode. Otherwise, if the guest has explicitly requested multicast * promiscuity or has sent a non-empty multicast MAC address table, * then set viona to multicast promiscuous mode. */ if (sc->vsc_promisc_promisc || sc->vsc_promisc_umac) mode = VIONA_PROMISC_ALL; else if (sc->vsc_promisc_allmulti || sc->vsc_promisc_mmac) mode = VIONA_PROMISC_MULTI; if (mode != sc->vsc_promisc) { DPRINTF("viona: setting promiscuous mode to %d (%s)", mode, pci_viona_promisc_descr(mode)); DPRINTF(" promisc=%u, umac=%u, allmulti=%u, mmac=%u", sc->vsc_promisc_promisc, sc->vsc_promisc_umac, sc->vsc_promisc_allmulti, sc->vsc_promisc_mmac); err = ioctl(sc->vsc_vnafd, VNA_IOC_SET_PROMISC, mode); if (err == 0) sc->vsc_promisc = mode; else WPRINTF("ioctl viona set promisc failed %d", errno); } return (err); } static uint8_t pci_viona_control_rx(struct vqueue_info *vq, const virtio_net_ctrl_hdr_t *hdr, iov_bunch_t *iob) { struct pci_viona_softc *sc = (struct pci_viona_softc *)vq->vq_vs; uint8_t v; if (!iov_bunch_copy(iob, &v, sizeof (v))) { EPRINTLN("viona: bad control RX data"); return (VIRTIO_NET_CQ_ERR); } switch (hdr->vnch_command) { case VIRTIO_NET_CTRL_RX_PROMISC: DPRINTF("viona: ctrl RX promisc %d", v); sc->vsc_promisc_promisc = (v != 0); break; case VIRTIO_NET_CTRL_RX_ALLMULTI: DPRINTF("viona: ctrl RX allmulti %d", v); sc->vsc_promisc_allmulti = (v != 0); break; default: /* * VIRTIO_NET_F_CTRL_RX_EXTRA was not offered so no other * commands are expected. */ EPRINTLN("viona: unrecognised RX control cmd %u", hdr->vnch_command); return (VIRTIO_NET_CQ_ERR); } if (pci_viona_eval_promisc(sc) == 0) return (VIRTIO_NET_CQ_OK); return (VIRTIO_NET_CQ_ERR); } static void pci_viona_control_mac_dump(const char *tag, iov_bunch_t *iob, uint32_t cnt) { if (!pci_viona_debug) { (void) iov_bunch_skip(iob, cnt * ETHERADDRL); return; } DPRINTF("-- %s MAC TABLE (entries: %u)", tag, cnt); for (uint32_t i = 0; i < cnt; i++) { ether_addr_t mac; if (!iov_bunch_copy(iob, &mac, sizeof (mac))) return; DPRINTF(" [%2d] %s", i, ether_ntoa((struct ether_addr *)mac)); } } static uint8_t pci_viona_control_mac(struct vqueue_info *vq, const virtio_net_ctrl_hdr_t *hdr, iov_bunch_t *iob) { struct pci_viona_softc *sc = (struct pci_viona_softc *)vq->vq_vs; switch (hdr->vnch_command) { case VIRTIO_NET_CTRL_MAC_TABLE_SET: { virtio_net_ctrl_mac_t table; DPRINTF("viona: ctrl MAC table set"); /* * We advertise VIRTIO_NET_F_CTRL_RX and therefore need to * accept VIRTIO_NET_CTRL_MAC, but we don't support passing * changes in the MAC address lists down to viona. * Instead, we set flags to indicate if the guest has sent * any MAC addresses for each table, and use these to determine * the resulting promiscuous mode, see pci_viona_eval_promisc() * above. */ /* Unicast MAC table */ if (!iov_bunch_copy(iob, &table.vncm_entries, sizeof (table.vncm_entries))) { EPRINTLN("viona: bad control MAC unicast header"); return (VIRTIO_NET_CQ_ERR); } sc->vsc_promisc_umac = (table.vncm_entries != 0); pci_viona_control_mac_dump("UNICAST", iob, table.vncm_entries); /* Multicast MAC table */ if (!iov_bunch_copy(iob, &table.vncm_entries, sizeof (table.vncm_entries))) { EPRINTLN("viona: bad control MAC multicast header"); return (VIRTIO_NET_CQ_ERR); } sc->vsc_promisc_mmac = (table.vncm_entries != 0); pci_viona_control_mac_dump("MULTICAST", iob, table.vncm_entries); break; } case VIRTIO_NET_CTRL_MAC_ADDR_SET: /* disallow setting the primary filter MAC address */ DPRINTF("viona: ctrl MAC addr set with 0x%x bytes", iob->ib_remain); return (VIRTIO_NET_CQ_ERR); default: EPRINTLN("viona: unrecognised MAC control cmd %u", hdr->vnch_command); return (VIRTIO_NET_CQ_ERR); } if (pci_viona_eval_promisc(sc) == 0) return (VIRTIO_NET_CQ_OK); return (VIRTIO_NET_CQ_ERR); } static uint8_t pci_viona_control_mq(struct vqueue_info *vq, const virtio_net_ctrl_hdr_t *hdr, iov_bunch_t *iob) { struct pci_viona_softc *sc = (struct pci_viona_softc *)vq->vq_vs; switch (hdr->vnch_command) { case VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET: { virtio_net_ctrl_mq_t mq; if (!iov_bunch_copy(iob, &mq, sizeof (mq))) { EPRINTLN("viona: bad control MQ data"); return (VIRTIO_NET_CQ_ERR); } DPRINTF("VQ PAIRS SET 0x%x", mq.virtqueue_pairs); if (mq.virtqueue_pairs < 1 || VIONA_P2QS(mq.virtqueue_pairs) > VIONA_NRINGS(sc)) { EPRINTLN("viona: invalid VQ pairs from guest 0x%x", mq.virtqueue_pairs); return (VIRTIO_NET_CQ_ERR); } if (!pci_viona_set_usepairs(sc, mq.virtqueue_pairs)) return (VIRTIO_NET_CQ_ERR); break; } default: EPRINTLN("viona: unrecognised MQ control cmd %u", hdr->vnch_command); return (VIRTIO_NET_CQ_ERR); } return (VIRTIO_NET_CQ_OK); } static void pci_viona_control(struct vqueue_info *vq) { struct iovec iov[VIONA_CTLQ_MAXSEGS]; virtio_net_ctrl_hdr_t hdr; struct vi_req req = { 0 }; iov_bunch_t iob; uint8_t *ackp; uint32_t wlen = 0; size_t len; int niov; niov = vq_getchain(vq, iov, VIONA_CTLQ_MAXSEGS, &req); assert(niov >= 1 && niov <= VIONA_CTLQ_MAXSEGS); /* * Since we support the modern interface we must accept a flexible * layout here. Even with that we can do some basic checks - there have * to be at least two descriptors and only a single writable one sized * for one byte. Check the incoming message to make sure it matches * this layout and drop the entire chain if not. */ if (niov < 2 || req.writable != 1 || req.readable + 1 != niov || iov[req.readable].iov_len != sizeof (uint8_t)) { EPRINTLN("viona: bad control chain, niov=0x%x, w=0x%x, r=0x%x", niov, req.writable, req.readable); goto drop; } len = iov_bunch_init(&iob, iov, niov); if (!iov_bunch_copy(&iob, &hdr, sizeof (hdr))) { EPRINTLN("viona: control header copy failed, len=0x%x", len); goto drop; } /* * Writable iovecs start at iov[req.readable], and we've already * checked that there is only one writable, it's at the end, and the * right size; it's the acknowledgement byte. */ ackp = (uint8_t *)iov[req.readable].iov_base; iob.ib_remain--; switch (hdr.vnch_class) { case VIRTIO_NET_CTRL_RX: *ackp = pci_viona_control_rx(vq, &hdr, &iob); break; case VIRTIO_NET_CTRL_MAC: *ackp = pci_viona_control_mac(vq, &hdr, &iob); break; case VIRTIO_NET_CTRL_MQ: *ackp = pci_viona_control_mq(vq, &hdr, &iob); break; default: EPRINTLN("viona: unrecognised control class %u, cmd %u", hdr.vnch_class, hdr.vnch_command); *ackp = VIRTIO_NET_CQ_ERR; break; } /* We've written the status byte */ wlen++; drop: vq_relchain(vq, req.idx, wlen); } static void pci_viona_process_ctrlq(struct vqueue_info *vq) { for (;;) { vq_kick_disable(vq); while (vq_has_descs(vq)) pci_viona_control(vq); vq_kick_enable(vq); /* * One more check in case a late addition raced with * re-enabling kicks. Note that vq_kick_enable() includes a * memory barrier. */ if (!vq_has_descs(vq)) break; } vq_endchains(vq, /* used_all_avail= */1); } static void * pci_viona_poll_thread(void *param) { struct pci_viona_softc *sc = param; pollfd_t pollset; const int fd = sc->vsc_vnafd; pollset.fd = fd; pollset.events = POLLRDBAND; for (;;) { if (poll(&pollset, 1, -1) < 0) { if (errno == EINTR || errno == EAGAIN) { continue; } else { WPRINTF("pci_viona_poll_thread poll() error %d", errno); break; } } if (pollset.revents & POLLRDBAND) { uint_t i; int entries; bool assert_lintr = false; const bool do_msix = pci_msix_enabled(sc->vsc_vs.vs_pi); vioc_intr_poll_mq_t vipm; vipm.vipm_nrings = VIONA_USABLE_RINGS(sc); entries = ioctl(fd, VNA_IOC_INTR_POLL_MQ, &vipm); for (i = 0; entries > 0 && i < vipm.vipm_nrings && i < VIONA_USABLE_RINGS(sc); i++) { if (!VIONA_INTR_TEST(&vipm, i)) continue; entries--; if (do_msix) { pci_generate_msix(sc->vsc_vs.vs_pi, sc->vsc_queues[i].vq_msix_idx); } else { assert_lintr = true; } if (ioctl(fd, VNA_IOC_RING_INTR_CLR, i) != 0) { WPRINTF("ioctl viona vq %d intr " "clear failed %d", i, errno); } } if (assert_lintr) { pthread_mutex_lock(&sc->vsc_mtx); sc->vsc_vs.vs_isr |= VIRTIO_PCI_ISR_INTR; pci_lintr_assert(sc->vsc_vs.vs_pi); pthread_mutex_unlock(&sc->vsc_mtx); } } } pthread_exit(NULL); } static void pci_viona_ring_init(void *vsc, uint64_t pfn, bool modern) { struct pci_viona_softc *sc = vsc; int qnum = sc->vsc_vs.vs_curq; struct vqueue_info *vq = &sc->vsc_queues[qnum]; vioc_ring_init_modern_t vna_rim = { 0 }; int error; const bool ctlq = VIONA_RING_CTLQ(sc, qnum); DPRINTF("viona: ring init 0x%x", qnum); if (!ctlq && !VIONA_RING_VALID(sc, qnum)) { DPRINTF("viona: bad ring 0x%d", qnum); return; } if (modern) vi_vq_init(&sc->vsc_vs); else vi_legacy_vq_init(&sc->vsc_vs, pfn); if (ctlq) return; vna_rim.rim_index = qnum; vna_rim.rim_qsize = vq->vq_qsize; vna_rim.rim_qaddr_desc = vq->vq_desc_gpa; vna_rim.rim_qaddr_avail = vq->vq_avail_gpa; vna_rim.rim_qaddr_used = vq->vq_used_gpa; error = ioctl(sc->vsc_vnafd, VNA_IOC_RING_INIT_MODERN, &vna_rim); if (error != 0) { WPRINTF("ioctl viona ring 0x%x init failed %d", qnum, errno); } } static int pci_viona_viona_init(struct vmctx *ctx, struct pci_viona_softc *sc) { vioc_create_t vna_create; int error, version; sc->vsc_vnafd = open("/dev/viona", O_RDWR | O_EXCL); if (sc->vsc_vnafd == -1) { WPRINTF("open viona ctl failed: %d", errno); return (-1); } version = ioctl(sc->vsc_vnafd, VNA_IOC_VERSION, 0); if (version != VIONA_CURRENT_INTERFACE_VERSION) { (void) close(sc->vsc_vnafd); WPRINTF( "ioctl interface version %d != expected %d", version, VIONA_CURRENT_INTERFACE_VERSION); return (-1); } vna_create.c_linkid = sc->vsc_linkid; vna_create.c_vmfd = vm_get_device_fd(ctx); error = ioctl(sc->vsc_vnafd, VNA_IOC_CREATE, &vna_create); if (error != 0) { (void) close(sc->vsc_vnafd); WPRINTF("ioctl viona create failed %d", errno); return (-1); } return (0); } static int pci_viona_legacy_config(nvlist_t *nvl, const char *opt) { char *config, *name, *tofree, *value; if (opt == NULL) return (0); config = tofree = strdup(opt); while ((name = strsep(&config, ",")) != NULL) { value = strchr(name, '='); if (value != NULL) { *value++ = '\0'; set_config_value_node(nvl, name, value); } else { set_config_value_node(nvl, "vnic", name); } } free(tofree); return (0); } static int pci_viona_parse_opts(struct pci_viona_softc *sc, nvlist_t *nvl) { const char *value, *errstr; long long num; int err = 0; sc->vsc_config.vnc_max_qpair = VIONA_DEFAULT_MAX_QPAIR; sc->vsc_vq_txsize = VIONA_DEFAULT_TX_RINGSZ; sc->vsc_vq_rxsize = VIONA_DEFAULT_RX_RINGSZ; sc->vsc_config.vnc_speed = VIONA_DEFAULT_LINK_SPEED; sc->vsc_feature_mask = 0; sc->vsc_linkname[0] = '\0'; value = get_config_value_node(nvl, "feature_mask"); if (value != NULL) { num = strtonumx(value, 0, UINT64_MAX, &errstr, 0); if (errstr != NULL) { EPRINTLN("viona: invalid feature_mask '%s': %s", value, errstr); err = -1; } else { sc->vsc_feature_mask = num; } } value = get_config_value_node(nvl, "vqsize"); if (value != NULL) { num = strtonumx(value, 2, 32768, &errstr, 0); if (errstr != NULL) { EPRINTLN("viona: invalid vqsize '%s': %s", value, errstr); err = -1; } else if ((1 << (ffsll(num) - 1)) != num) { EPRINTLN("viona: vqsize '%s' must be power of 2", value); err = -1; } else { sc->vsc_vq_txsize = num; sc->vsc_vq_rxsize = num; } } value = get_config_value_node(nvl, "txvqsize"); if (value != NULL) { num = strtonumx(value, 2, 32768, &errstr, 0); if (errstr != NULL) { EPRINTLN("viona: invalid txvqsize '%s': %s", value, errstr); err = -1; } else if ((1 << (ffsll(num) - 1)) != num) { EPRINTLN("viona: txvqsize '%s' must be power of 2", value); err = -1; } else { sc->vsc_vq_txsize = num; } } value = get_config_value_node(nvl, "rxvqsize"); if (value != NULL) { num = strtonumx(value, 2, 32768, &errstr, 0); if (errstr != NULL) { EPRINTLN("viona: invalid rxvqsize '%s': %s", value, errstr); err = -1; } else if ((1 << (ffsll(num) - 1)) != num) { EPRINTLN("viona: rxvqsize '%s' must be power of 2", value); err = -1; } else { sc->vsc_vq_rxsize = num; } } value = get_config_value_node(nvl, "qpair"); if (value != NULL) { num = strtonumx(value, VIONA_MIN_QPAIR, VIONA_MAX_QPAIR, &errstr, 0); if (errstr != NULL) { EPRINTLN("viona: invalid qpair '%s': %s", value, errstr); err = -1; } else { sc->vsc_config.vnc_max_qpair = num; } } value = get_config_value_node(nvl, "speed"); if (value != NULL) { num = strtonumx(value, 0, INT32_MAX, &errstr, 0); if (errstr != NULL) { EPRINTLN("viona: invalid speed '%s': %s", value, errstr); err = -1; } else { sc->vsc_config.vnc_speed = num; } } value = get_config_value_node(nvl, "vnic"); if (value == NULL) { EPRINTLN("viona: vnic name required"); err = -1; } else { (void) strlcpy(sc->vsc_linkname, value, MAXLINKNAMELEN); } DPRINTF("viona=%p dev=%s vqsize(TX/RX)=0x%x/0x%x qpair=0x%x " "feature_mask=0x%" PRIx64, sc, sc->vsc_linkname, sc->vsc_vq_txsize, sc->vsc_vq_rxsize, sc->vsc_config.vnc_max_qpair, sc->vsc_feature_mask); return (err); } static uint16_t pci_viona_query_mtu(dladm_handle_t handle, datalink_id_t linkid) { char buf[DLADM_PROP_VAL_MAX]; char *propval = buf; uint_t propcnt = 1; if (dladm_get_linkprop(handle, linkid, DLADM_PROP_VAL_CURRENT, "mtu", &propval, &propcnt) == DLADM_STATUS_OK && propcnt == 1) { ulong_t parsed = strtoul(buf, NULL, 10); /* * The virtio spec notes that for devices implementing * VIRTIO_NET_F_MTU, that the noted MTU MUST be between * 68-65535, inclusive. */ if (parsed >= 68 && parsed <= 65535) return (parsed); } /* * Default to 1500 if query is unsuccessful or the result is out of * bounds. */ return (1500); } static int pci_viona_free_softstate(struct pci_viona_softc *sc, int err) { pthread_mutex_destroy(&sc->vsc_mtx); free(sc->vsc_queues); free(sc); return (err); } static int pci_viona_init(struct pci_devinst *pi, nvlist_t *nvl) { dladm_handle_t handle; dladm_status_t status; dladm_vnic_attr_t attr; char errmsg[DLADM_STRSIZE]; char tname[MAXCOMLEN + 1]; int error; struct pci_viona_softc *sc; struct virtio_consts *vc; const char *vnic; pthread_t tid; vnic = get_config_value_node(nvl, "vnic"); if (vnic == NULL) { WPRINTF("virtio-viona: vnic required"); return (1); } sc = calloc(1, sizeof (struct pci_viona_softc)); if (sc == NULL) { WPRINTF("Failed to allocate memory for soft state"); return (1); } vc = &sc->vsc_consts; *vc = viona_vi_consts; pthread_mutex_init(&sc->vsc_mtx, NULL); if (get_config_bool_default("viona.debug", false)) pci_viona_debug = 1; vi_set_debug(&sc->vsc_vs, pci_viona_debug); if (pci_viona_parse_opts(sc, nvl) != 0) return (pci_viona_free_softstate(sc, 1)); /* Add one for the control queue */ vc->vc_max_nvq = VIONA_P2QS(sc->vsc_config.vnc_max_qpair) + 1; if ((status = dladm_open(&handle)) != DLADM_STATUS_OK) { WPRINTF("could not open /dev/dld"); return (pci_viona_free_softstate(sc, 1)); } if ((status = dladm_name2info(handle, sc->vsc_linkname, &sc->vsc_linkid, NULL, NULL, NULL)) != DLADM_STATUS_OK) { WPRINTF("dladm_name2info() for %s failed: %s", vnic, dladm_status2str(status, errmsg)); dladm_close(handle); return (pci_viona_free_softstate(sc, 1)); } if ((status = dladm_vnic_info(handle, sc->vsc_linkid, &attr, DLADM_OPT_ACTIVE)) != DLADM_STATUS_OK) { WPRINTF("dladm_vnic_info() for %s failed: %s", vnic, dladm_status2str(status, errmsg)); dladm_close(handle); return (pci_viona_free_softstate(sc, 1)); } memcpy(sc->vsc_config.vnc_macaddr, attr.va_mac_addr, ETHERADDRL); sc->vsc_config.vnc_status = VIRTIO_NET_S_LINK_UP; /* link always up */ sc->vsc_config.vnc_duplex = VIRTIO_NET_DUPLEX_FULL; sc->vsc_config.vnc_mtu = pci_viona_query_mtu(handle, sc->vsc_linkid); dladm_close(handle); error = pci_viona_viona_init(pi->pi_vmctx, sc); if (error != 0) return (pci_viona_free_softstate(sc, 1)); if (ioctl(sc->vsc_vnafd, VNA_IOC_SET_MTU, sc->vsc_config.vnc_mtu) != 0) { WPRINTF("error setting viona MTU(%u): %s", sc->vsc_config.vnc_mtu, strerror(errno)); } /* link virtio to softc */ vi_softc_linkup(&sc->vsc_vs, vc, sc, pi, sc->vsc_queues); sc->vsc_vs.vs_mtx = &sc->vsc_mtx; /* * We initially need to configure a single queue pair. Some legacy * drivers will set these up before completing feature negotiation, * which is before we know if they support multi-queue. */ if (!pci_viona_qalloc(sc, 1)) return (pci_viona_free_softstate(sc, 1)); /* Until the guest tells us otherwise, we'll only use a single pair */ sc->vsc_vq_usepairs = 1; error = pthread_create(&tid, NULL, pci_viona_poll_thread, sc); assert(error == 0); snprintf(tname, sizeof (tname), "vionapoll:%s", vnic); pthread_set_name_np(tid, tname); /* initialize config space */ vi_pci_init(pi, VIRTIO_MODE_TRANSITIONAL, VIRTIO_DEV_NET, VIRTIO_ID_NETWORK, PCIC_NETWORK); /* * Guests that do not support CTRL_RX_MAC still generally need to * receive multicast packets. Guests that do support this feature will * end up setting this flag indirectly via messages on the control * queue but it does not hurt to default to multicast promiscuity here * and it is what older version of viona did. */ sc->vsc_promisc_mmac = true; pci_viona_eval_promisc(sc); /* Viona always uses MSI-X */ if (!vi_intr_init(&sc->vsc_vs, true)) return (pci_viona_free_softstate(sc, 1)); if (!vi_pcibar_setup(&sc->vsc_vs)) return (pci_viona_free_softstate(sc, 1)); return (0); } static int pci_viona_cfgwrite(void *vsc, int offset, int size, uint32_t value) { struct pci_viona_softc *sc = vsc; void *ptr; /* We will only ever end up here with an 8, 16 or 32-bit size */ ASSERT(size == 1 || size == 2 || size == 4); /* * The driver is allowed to change the MAC address. * vnc_macaddr is the first element of vsc_config */ if (offset < (int)sizeof (sc->vsc_config.vnc_macaddr) && offset + size <= (int)sizeof (sc->vsc_config.vnc_macaddr)) { ptr = &sc->vsc_config.vnc_macaddr[offset]; memcpy(ptr, &value, size); vq_devcfg_changed(&sc->vsc_vs); } else { /* silently ignore other writes */ DPRINTF("viona: write to readonly reg 0x%x", offset); } return (0); } static int pci_viona_cfgread(void *vsc, int offset, int size, uint32_t *retval) { struct pci_viona_softc *sc = vsc; void *ptr; ptr = (uint8_t *)&sc->vsc_config + offset; memcpy(retval, ptr, size); return (0); } static void pci_viona_ring_set_msix(void *vsc, int ring) { struct pci_viona_softc *sc = vsc; struct pci_devinst *pi = sc->vsc_vs.vs_pi; struct msix_table_entry mte; uint16_t tab_index; vioc_ring_msi_t vrm; int res; if (!VIONA_RING_VALID(sc, ring)) return; vrm.rm_index = ring; vrm.rm_addr = 0; vrm.rm_msg = 0; tab_index = sc->vsc_queues[ring].vq_msix_idx; if (tab_index != VIRTIO_MSI_NO_VECTOR && sc->vsc_msix_active) { mte = pi->pi_msix.table[tab_index]; if ((mte.vector_control & PCIM_MSIX_VCTRL_MASK) == 0) { vrm.rm_addr = mte.addr; vrm.rm_msg = mte.msg_data; } } DPRINTF("SET MSI ring=0x%x addr=0x%" PRIx64 " msg=0x%" PRIx64, vrm.rm_index, vrm.rm_addr, vrm.rm_msg); res = ioctl(sc->vsc_vnafd, VNA_IOC_RING_SET_MSI, &vrm); if (res != 0) { WPRINTF("ioctl viona set_msi %d failed %d", ring, errno); } } static void pci_viona_lintrupdate(struct pci_devinst *pi) { struct pci_viona_softc *sc = pi->pi_arg; bool msix_on = false; pthread_mutex_lock(&sc->vsc_mtx); msix_on = pci_msix_enabled(pi) && (pi->pi_msix.function_mask == 0); if ((sc->vsc_msix_active && !msix_on) || (msix_on && !sc->vsc_msix_active)) { DPRINTF("MSIX %u -> %u", sc->vsc_msix_active, msix_on); sc->vsc_msix_active = msix_on; /* Update in-kernel ring configs */ for (uint16_t i = 0; i < VIONA_NRINGS(sc); i++) pci_viona_ring_set_msix(sc, VIONA_RING(sc, i)->vq_num); } pthread_mutex_unlock(&sc->vsc_mtx); } static void pci_viona_update_msix(void *vsc, uint64_t offset) { struct pci_viona_softc *sc = vsc; uint_t tab_index; pthread_mutex_lock(&sc->vsc_mtx); if (!sc->vsc_msix_active) { pthread_mutex_unlock(&sc->vsc_mtx); return; } /* * Rather than update every possible MSI-X vector, cheat and use the * offset to calculate the entry within the table. Since this should * only be called when a write to the table succeeds, the index should * be valid. */ tab_index = offset / MSIX_TABLE_ENTRY_SIZE; for (uint16_t i = 0; i < VIONA_NRINGS(sc); i++) { struct vqueue_info *vq = VIONA_RING(sc, i); if (vq->vq_msix_idx == tab_index) pci_viona_ring_set_msix(vsc, vq->vq_num); } pthread_mutex_unlock(&sc->vsc_mtx); } static void pci_viona_ctlqnotify(void *vsc, struct vqueue_info *vq) { if (vq_has_descs(vq)) pci_viona_process_ctrlq(vq); } static void pci_viona_qnotify(void *vsc, struct vqueue_info *vq) { struct pci_viona_softc *sc = vsc; int ring = vq->vq_num; int error; if (!VIONA_RING_VALID(sc, ring)) return; error = ioctl(sc->vsc_vnafd, VNA_IOC_RING_KICK, ring); if (error != 0) WPRINTF("ioctl viona ring 0x%x kick failed %d", ring, errno); } static void pci_viona_baraddr(struct pci_devinst *pi, int baridx, int enabled, uint64_t address) { struct pci_viona_softc *sc = pi->pi_arg; int err; DPRINTF("BAR%d ADDRESS %" PRIx64 " (%s)", baridx, address, enabled == 1 ? "enable": "disable"); switch (baridx) { case VIRTIO_LEGACY_BAR: { uint64_t ioport; if (enabled == 0) { err = ioctl(sc->vsc_vnafd, VNA_IOC_SET_NOTIFY_IOP, 0); if (err != 0) WPRINTF("Uninstall ioport hook fail %d", errno); break; } /* * Install ioport hook for virtqueue notification. * This is part of the virtio common configuration area so the * address does not change with MSI-X status. */ ioport = address + VIRTIO_PCI_QUEUE_NOTIFY; err = ioctl(sc->vsc_vnafd, VNA_IOC_SET_NOTIFY_IOP, ioport); if (err != 0) { WPRINTF("Install ioport hook at 0x%x failed %d", ioport, errno); } break; } case VIRTIO_MODERN_BAR: { virtio_pci_capcfg_t *cfg; vioc_notify_mmio_t vim; if (enabled == 0) { err = ioctl(sc->vsc_vnafd, VNA_IOC_SET_NOTIFY_MMIO, 0); if (err != 0) WPRINTF("Uninstall MMIO hook fail %d", errno); break; } cfg = vi_pci_cfg_bytype(&sc->vsc_vs, VIRTIO_PCI_CAP_NOTIFY_CFG); if (cfg == NULL) break; vim.vim_address = address + cfg->c_baroff; vim.vim_size = cfg->c_barlen; DPRINTF("MODERN BAR NOTIFY address 0x%" PRIx64 " size 0x%x", vim.vim_address, vim.vim_size); err = ioctl(sc->vsc_vnafd, VNA_IOC_SET_NOTIFY_MMIO, &vim); if (err != 0) { WPRINTF( "Install MMIO hook at 0x%" PRIx64 "+0x%x failed %d", vim.vim_address, vim.vim_size, errno); } break; } default: break; } } static uint64_t pci_viona_get_hv_features(void *vsc, bool modern) { struct pci_viona_softc *sc = vsc; uint64_t value; int err; err = ioctl(sc->vsc_vnafd, VNA_IOC_GET_FEATURES, &value); if (err != 0) WPRINTF("ioctl get host features returned err = %d", errno); /* * Supplementary device capabilities provided in the userspace * component. */ value |= VIRTIO_NET_F_MAC; value |= VIRTIO_NET_F_STATUS; value |= VIRTIO_NET_F_MTU; value |= VIRTIO_NET_F_CTRL_VQ; value |= VIRTIO_NET_F_CTRL_RX; value |= VIRTIO_NET_F_MQ; if (modern) value |= VIRTIO_NET_F_SPEED_DUPLEX; value &= ~sc->vsc_feature_mask; if (modern) { value |= VIRTIO_F_VERSION_1; sc->vsc_consts.vc_hv_caps_modern = value; } else { /* * To be a conforming transitional device we must support * arbitrary descriptor layouts on the legacy interface. The * specification is a little ambiguous as it mandates this but * also provides details on how descriptors should be laid out * by transitional drivers that don't negotiate this. Since we * don't make any assumptions about descriptor layout we may as * well set this. */ value |= VIRTIO_F_ANY_LAYOUT; sc->vsc_consts.vc_hv_caps_legacy = value; } return (value); } static void pci_viona_set_hv_features(void *vsc, uint64_t *value) { struct pci_viona_softc *sc = vsc; int err; *value &= ~sc->vsc_feature_mask; err = ioctl(sc->vsc_vnafd, VNA_IOC_SET_FEATURES, value); if (err != 0) WPRINTF("ioctl feature negotiation returned err = %d", errno); if (*value & VIRTIO_NET_F_MQ) { /* * If multi-queue is negotiated then we need to provision all * of the queues we can support. Even if the guest chooses not * to use all of them it will still set them up. */ DPRINTF("Going MULTIQUEUE with 0x%x pairs!", sc->vsc_config.vnc_max_qpair); if (!pci_viona_qalloc(sc, sc->vsc_config.vnc_max_qpair)) { vi_error(&sc->vsc_vs, "Failed to allocate 0x%x queue pairs", sc->vsc_config.vnc_max_qpair); } } } struct pci_devemu pci_de_viona = { .pe_emu = "virtio-net-viona", .pe_init = pci_viona_init, .pe_legacy_config = pci_viona_legacy_config, .pe_cfgwrite = vi_pci_cfgwrite, .pe_cfgread = vi_pci_cfgread, .pe_barwrite = vi_pci_write, .pe_barread = vi_pci_read, .pe_baraddr = pci_viona_baraddr, .pe_lintrupdate = pci_viona_lintrupdate }; PCI_EMUL_SET(pci_de_viona); /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2014 Leon Dang * Copyright 2018 Joyent, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* XHCI options: -s ,xhci,{devices} devices: tablet USB tablet mouse */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "bhyverun.h" #include "config.h" #include "debug.h" #include "pci_emul.h" #include "pci_xhci.h" #include "usb_emul.h" static int xhci_debug = 0; #define DPRINTF(params) if (xhci_debug) PRINTLN params #define WPRINTF(params) PRINTLN params #define XHCI_NAME "xhci" #define XHCI_MAX_DEVS 8 /* 4 USB3 + 4 USB2 devs */ #define XHCI_MAX_SLOTS 64 /* min allowed by Windows drivers */ /* * XHCI data structures can be up to 64k, but limit paddr_guest2host mapping * to 4k to avoid going over the guest physical memory barrier. */ #define XHCI_PADDR_SZ 4096 /* paddr_guest2host max size */ #define XHCI_ERST_MAX 0 /* max 2^entries event ring seg tbl */ #define XHCI_CAPLEN (4*8) /* offset of op register space */ #define XHCI_HCCPRAMS2 0x1C /* offset of HCCPARAMS2 register */ #define XHCI_PORTREGS_START 0x400 #define XHCI_DOORBELL_MAX 256 #define XHCI_STREAMS_MAX 1 /* 4-15 in XHCI spec */ /* caplength and hci-version registers */ #define XHCI_SET_CAPLEN(x) ((x) & 0xFF) #define XHCI_SET_HCIVERSION(x) (((x) & 0xFFFF) << 16) #define XHCI_GET_HCIVERSION(x) (((x) >> 16) & 0xFFFF) /* hcsparams1 register */ #define XHCI_SET_HCSP1_MAXSLOTS(x) ((x) & 0xFF) #define XHCI_SET_HCSP1_MAXINTR(x) (((x) & 0x7FF) << 8) #define XHCI_SET_HCSP1_MAXPORTS(x) (((x) & 0xFF) << 24) /* hcsparams2 register */ #define XHCI_SET_HCSP2_IST(x) ((x) & 0x0F) #define XHCI_SET_HCSP2_ERSTMAX(x) (((x) & 0x0F) << 4) #define XHCI_SET_HCSP2_MAXSCRATCH_HI(x) (((x) & 0x1F) << 21) #define XHCI_SET_HCSP2_MAXSCRATCH_LO(x) (((x) & 0x1F) << 27) /* hcsparams3 register */ #define XHCI_SET_HCSP3_U1EXITLATENCY(x) ((x) & 0xFF) #define XHCI_SET_HCSP3_U2EXITLATENCY(x) (((x) & 0xFFFF) << 16) /* hccparams1 register */ #define XHCI_SET_HCCP1_AC64(x) ((x) & 0x01) #define XHCI_SET_HCCP1_BNC(x) (((x) & 0x01) << 1) #define XHCI_SET_HCCP1_CSZ(x) (((x) & 0x01) << 2) #define XHCI_SET_HCCP1_PPC(x) (((x) & 0x01) << 3) #define XHCI_SET_HCCP1_PIND(x) (((x) & 0x01) << 4) #define XHCI_SET_HCCP1_LHRC(x) (((x) & 0x01) << 5) #define XHCI_SET_HCCP1_LTC(x) (((x) & 0x01) << 6) #define XHCI_SET_HCCP1_NSS(x) (((x) & 0x01) << 7) #define XHCI_SET_HCCP1_PAE(x) (((x) & 0x01) << 8) #define XHCI_SET_HCCP1_SPC(x) (((x) & 0x01) << 9) #define XHCI_SET_HCCP1_SEC(x) (((x) & 0x01) << 10) #define XHCI_SET_HCCP1_CFC(x) (((x) & 0x01) << 11) #define XHCI_SET_HCCP1_MAXPSA(x) (((x) & 0x0F) << 12) #define XHCI_SET_HCCP1_XECP(x) (((x) & 0xFFFF) << 16) /* hccparams2 register */ #define XHCI_SET_HCCP2_U3C(x) ((x) & 0x01) #define XHCI_SET_HCCP2_CMC(x) (((x) & 0x01) << 1) #define XHCI_SET_HCCP2_FSC(x) (((x) & 0x01) << 2) #define XHCI_SET_HCCP2_CTC(x) (((x) & 0x01) << 3) #define XHCI_SET_HCCP2_LEC(x) (((x) & 0x01) << 4) #define XHCI_SET_HCCP2_CIC(x) (((x) & 0x01) << 5) /* other registers */ #define XHCI_SET_DOORBELL(x) ((x) & ~0x03) #define XHCI_SET_RTSOFFSET(x) ((x) & ~0x0F) /* register masks */ #define XHCI_PS_PLS_MASK (0xF << 5) /* port link state */ #define XHCI_PS_SPEED_MASK (0xF << 10) /* port speed */ #define XHCI_PS_PIC_MASK (0x3 << 14) /* port indicator */ /* port register set */ #define XHCI_PORTREGS_BASE 0x400 /* base offset */ #define XHCI_PORTREGS_PORT0 0x3F0 #define XHCI_PORTREGS_SETSZ 0x10 /* size of a set */ #define MASK_64_HI(x) ((x) & ~0xFFFFFFFFULL) #define MASK_64_LO(x) ((x) & 0xFFFFFFFFULL) #define FIELD_REPLACE(a,b,m,s) (((a) & ~((m) << (s))) | \ (((b) & (m)) << (s))) #define FIELD_COPY(a,b,m,s) (((a) & ~((m) << (s))) | \ (((b) & ((m) << (s))))) struct pci_xhci_trb_ring { uint64_t ringaddr; /* current dequeue guest address */ uint32_t ccs; /* consumer cycle state */ }; /* device endpoint transfer/stream rings */ struct pci_xhci_dev_ep { union { struct xhci_trb *_epu_tr; struct xhci_stream_ctx *_epu_sctx; } _ep_trbsctx; #define ep_tr _ep_trbsctx._epu_tr #define ep_sctx _ep_trbsctx._epu_sctx /* * Caches the value of MaxPStreams from the endpoint context * when an endpoint is initialized and is used to validate the * use of ep_ringaddr vs ep_sctx_trbs[] as well as the length * of ep_sctx_trbs[]. */ uint32_t ep_MaxPStreams; union { struct pci_xhci_trb_ring _epu_trb; struct pci_xhci_trb_ring *_epu_sctx_trbs; } _ep_trb_rings; #define ep_ringaddr _ep_trb_rings._epu_trb.ringaddr #define ep_ccs _ep_trb_rings._epu_trb.ccs #define ep_sctx_trbs _ep_trb_rings._epu_sctx_trbs struct usb_data_xfer *ep_xfer; /* transfer chain */ }; /* device context base address array: maps slot->device context */ struct xhci_dcbaa { uint64_t dcba[USB_MAX_DEVICES+1]; /* xhci_dev_ctx ptrs */ }; /* port status registers */ struct pci_xhci_portregs { uint32_t portsc; /* port status and control */ uint32_t portpmsc; /* port pwr mgmt status & control */ uint32_t portli; /* port link info */ uint32_t porthlpmc; /* port hardware LPM control */ } __packed; #define XHCI_PS_SPEED_SET(x) (((x) & 0xF) << 10) /* xHC operational registers */ struct pci_xhci_opregs { uint32_t usbcmd; /* usb command */ uint32_t usbsts; /* usb status */ uint32_t pgsz; /* page size */ uint32_t dnctrl; /* device notification control */ uint64_t crcr; /* command ring control */ uint64_t dcbaap; /* device ctx base addr array ptr */ uint32_t config; /* configure */ /* guest mapped addresses: */ struct xhci_trb *cr_p; /* crcr dequeue */ struct xhci_dcbaa *dcbaa_p; /* dev ctx array ptr */ }; /* xHC runtime registers */ struct pci_xhci_rtsregs { uint32_t mfindex; /* microframe index */ struct { /* interrupter register set */ uint32_t iman; /* interrupter management */ uint32_t imod; /* interrupter moderation */ uint32_t erstsz; /* event ring segment table size */ uint32_t rsvd; uint64_t erstba; /* event ring seg-tbl base addr */ uint64_t erdp; /* event ring dequeue ptr */ } intrreg __packed; /* guest mapped addresses */ struct xhci_event_ring_seg *erstba_p; struct xhci_trb *erst_p; /* event ring segment tbl */ int er_deq_seg; /* event ring dequeue segment */ int er_enq_idx; /* event ring enqueue index - xHCI */ int er_enq_seg; /* event ring enqueue segment */ uint32_t er_events_cnt; /* number of events in ER */ uint32_t event_pcs; /* producer cycle state flag */ }; struct pci_xhci_softc; /* * USB device emulation container. * This is referenced from usb_hci->hci_sc; 1 pci_xhci_dev_emu for each * emulated device instance. */ struct pci_xhci_dev_emu { struct pci_xhci_softc *xsc; /* XHCI contexts */ struct xhci_dev_ctx *dev_ctx; struct pci_xhci_dev_ep eps[XHCI_MAX_ENDPOINTS]; int dev_slotstate; struct usb_devemu *dev_ue; /* USB emulated dev */ void *dev_sc; /* device's softc */ struct usb_hci hci; }; struct pci_xhci_softc { struct pci_devinst *xsc_pi; pthread_mutex_t mtx; uint32_t caplength; /* caplen & hciversion */ uint32_t hcsparams1; /* structural parameters 1 */ uint32_t hcsparams2; /* structural parameters 2 */ uint32_t hcsparams3; /* structural parameters 3 */ uint32_t hccparams1; /* capability parameters 1 */ uint32_t dboff; /* doorbell offset */ uint32_t rtsoff; /* runtime register space offset */ uint32_t hccparams2; /* capability parameters 2 */ uint32_t regsend; /* end of configuration registers */ struct pci_xhci_opregs opregs; struct pci_xhci_rtsregs rtsregs; struct pci_xhci_portregs *portregs; struct pci_xhci_dev_emu **devices; /* XHCI[port] = device */ struct pci_xhci_dev_emu **slots; /* slots assigned from 1 */ int usb2_port_start; int usb3_port_start; }; /* port and slot numbering start from 1 */ #define XHCI_PORTREG_PTR(x,n) &((x)->portregs[(n) - 1]) #define XHCI_DEVINST_PTR(x,n) ((x)->devices[(n) - 1]) #define XHCI_SLOTDEV_PTR(x,n) ((x)->slots[(n) - 1]) #define XHCI_HALTED(sc) ((sc)->opregs.usbsts & XHCI_STS_HCH) #define XHCI_GADDR(sc,a) paddr_guest2host((sc)->xsc_pi->pi_vmctx, \ (a), \ XHCI_PADDR_SZ - ((a) & (XHCI_PADDR_SZ-1))) static int xhci_in_use; /* map USB errors to XHCI */ static const int xhci_usb_errors[USB_ERR_MAX] = { [USB_ERR_NORMAL_COMPLETION] = XHCI_TRB_ERROR_SUCCESS, [USB_ERR_PENDING_REQUESTS] = XHCI_TRB_ERROR_RESOURCE, [USB_ERR_NOT_STARTED] = XHCI_TRB_ERROR_ENDP_NOT_ON, [USB_ERR_INVAL] = XHCI_TRB_ERROR_INVALID, [USB_ERR_NOMEM] = XHCI_TRB_ERROR_RESOURCE, [USB_ERR_CANCELLED] = XHCI_TRB_ERROR_STOPPED, [USB_ERR_BAD_ADDRESS] = XHCI_TRB_ERROR_PARAMETER, [USB_ERR_BAD_BUFSIZE] = XHCI_TRB_ERROR_PARAMETER, [USB_ERR_BAD_FLAG] = XHCI_TRB_ERROR_PARAMETER, [USB_ERR_NO_CALLBACK] = XHCI_TRB_ERROR_STALL, [USB_ERR_IN_USE] = XHCI_TRB_ERROR_RESOURCE, [USB_ERR_NO_ADDR] = XHCI_TRB_ERROR_RESOURCE, [USB_ERR_NO_PIPE] = XHCI_TRB_ERROR_RESOURCE, [USB_ERR_ZERO_NFRAMES] = XHCI_TRB_ERROR_UNDEFINED, [USB_ERR_ZERO_MAXP] = XHCI_TRB_ERROR_UNDEFINED, [USB_ERR_SET_ADDR_FAILED] = XHCI_TRB_ERROR_RESOURCE, [USB_ERR_NO_POWER] = XHCI_TRB_ERROR_ENDP_NOT_ON, [USB_ERR_TOO_DEEP] = XHCI_TRB_ERROR_RESOURCE, [USB_ERR_IOERROR] = XHCI_TRB_ERROR_TRB, [USB_ERR_NOT_CONFIGURED] = XHCI_TRB_ERROR_ENDP_NOT_ON, [USB_ERR_TIMEOUT] = XHCI_TRB_ERROR_CMD_ABORTED, [USB_ERR_SHORT_XFER] = XHCI_TRB_ERROR_SHORT_PKT, [USB_ERR_STALLED] = XHCI_TRB_ERROR_STALL, [USB_ERR_INTERRUPTED] = XHCI_TRB_ERROR_CMD_ABORTED, [USB_ERR_DMA_LOAD_FAILED] = XHCI_TRB_ERROR_DATA_BUF, [USB_ERR_BAD_CONTEXT] = XHCI_TRB_ERROR_TRB, [USB_ERR_NO_ROOT_HUB] = XHCI_TRB_ERROR_UNDEFINED, [USB_ERR_NO_INTR_THREAD] = XHCI_TRB_ERROR_UNDEFINED, [USB_ERR_NOT_LOCKED] = XHCI_TRB_ERROR_UNDEFINED, }; #define USB_TO_XHCI_ERR(e) ((e) < USB_ERR_MAX ? xhci_usb_errors[(e)] : \ XHCI_TRB_ERROR_INVALID) static int pci_xhci_insert_event(struct pci_xhci_softc *sc, struct xhci_trb *evtrb, int do_intr); static void pci_xhci_dump_trb(struct xhci_trb *trb); static void pci_xhci_assert_interrupt(struct pci_xhci_softc *sc); static void pci_xhci_reset_slot(struct pci_xhci_softc *sc, int slot); static void pci_xhci_reset_port(struct pci_xhci_softc *sc, int portn, int warm); static void pci_xhci_update_ep_ring(struct pci_xhci_softc *sc, struct pci_xhci_dev_emu *dev, struct pci_xhci_dev_ep *devep, struct xhci_endp_ctx *ep_ctx, uint32_t streamid, uint64_t ringaddr, int ccs); static int pci_xhci_validate_slot(uint32_t slot); static void pci_xhci_set_evtrb(struct xhci_trb *evtrb, uint64_t port, uint32_t errcode, uint32_t evtype) { evtrb->qwTrb0 = port << 24; evtrb->dwTrb2 = XHCI_TRB_2_ERROR_SET(errcode); evtrb->dwTrb3 = XHCI_TRB_3_TYPE_SET(evtype); } /* controller reset */ static void pci_xhci_reset(struct pci_xhci_softc *sc) { int i; sc->rtsregs.er_enq_idx = 0; sc->rtsregs.er_events_cnt = 0; sc->rtsregs.event_pcs = 1; for (i = 1; i <= XHCI_MAX_SLOTS; i++) { pci_xhci_reset_slot(sc, i); } } static uint32_t pci_xhci_usbcmd_write(struct pci_xhci_softc *sc, uint32_t cmd) { int do_intr = 0; int i; if (cmd & XHCI_CMD_RS) { do_intr = (sc->opregs.usbcmd & XHCI_CMD_RS) == 0; sc->opregs.usbcmd |= XHCI_CMD_RS; sc->opregs.usbsts &= ~XHCI_STS_HCH; sc->opregs.usbsts |= XHCI_STS_PCD; /* Queue port change event on controller run from stop */ if (do_intr) for (i = 1; i <= XHCI_MAX_DEVS; i++) { struct pci_xhci_dev_emu *dev; struct pci_xhci_portregs *port; struct xhci_trb evtrb; if ((dev = XHCI_DEVINST_PTR(sc, i)) == NULL) continue; port = XHCI_PORTREG_PTR(sc, i); port->portsc |= XHCI_PS_CSC | XHCI_PS_CCS; port->portsc &= ~XHCI_PS_PLS_MASK; /* * XHCI 4.19.3 USB2 RxDetect->Polling, * USB3 Polling->U0 */ if (dev->dev_ue->ue_usbver == 2) port->portsc |= XHCI_PS_PLS_SET(UPS_PORT_LS_POLL); else port->portsc |= XHCI_PS_PLS_SET(UPS_PORT_LS_U0); pci_xhci_set_evtrb(&evtrb, i, XHCI_TRB_ERROR_SUCCESS, XHCI_TRB_EVENT_PORT_STS_CHANGE); if (pci_xhci_insert_event(sc, &evtrb, 0) != XHCI_TRB_ERROR_SUCCESS) break; } } else { sc->opregs.usbcmd &= ~XHCI_CMD_RS; sc->opregs.usbsts |= XHCI_STS_HCH; sc->opregs.usbsts &= ~XHCI_STS_PCD; } /* start execution of schedule; stop when set to 0 */ cmd |= sc->opregs.usbcmd & XHCI_CMD_RS; if (cmd & XHCI_CMD_HCRST) { /* reset controller */ pci_xhci_reset(sc); cmd &= ~XHCI_CMD_HCRST; } cmd &= ~(XHCI_CMD_CSS | XHCI_CMD_CRS); if (do_intr) pci_xhci_assert_interrupt(sc); return (cmd); } static void pci_xhci_portregs_write(struct pci_xhci_softc *sc, uint64_t offset, uint64_t value) { struct xhci_trb evtrb; struct pci_xhci_portregs *p; int port; uint32_t oldpls, newpls; if (sc->portregs == NULL) return; port = (offset - XHCI_PORTREGS_PORT0) / XHCI_PORTREGS_SETSZ; offset = (offset - XHCI_PORTREGS_PORT0) % XHCI_PORTREGS_SETSZ; DPRINTF(("pci_xhci: portregs wr offset 0x%lx, port %u: 0x%lx", offset, port, value)); assert(port >= 0); if (port > XHCI_MAX_DEVS) { DPRINTF(("pci_xhci: portregs_write port %d > ndevices", port)); return; } if (XHCI_DEVINST_PTR(sc, port) == NULL) { DPRINTF(("pci_xhci: portregs_write to unattached port %d", port)); } p = XHCI_PORTREG_PTR(sc, port); switch (offset) { case 0: /* port reset or warm reset */ if (value & (XHCI_PS_PR | XHCI_PS_WPR)) { pci_xhci_reset_port(sc, port, value & XHCI_PS_WPR); break; } if ((p->portsc & XHCI_PS_PP) == 0) { WPRINTF(("pci_xhci: portregs_write to unpowered " "port %d", port)); break; } /* Port status and control register */ oldpls = XHCI_PS_PLS_GET(p->portsc); newpls = XHCI_PS_PLS_GET(value); #ifndef __FreeBSD__ p->portsc &= XHCI_PS_PED | XHCI_PS_PP | XHCI_PS_PLS_MASK | XHCI_PS_SPEED_MASK | XHCI_PS_PIC_MASK; #else p->portsc &= XHCI_PS_PED | XHCI_PS_PLS_MASK | XHCI_PS_SPEED_MASK | XHCI_PS_PIC_MASK; #endif if (XHCI_DEVINST_PTR(sc, port)) p->portsc |= XHCI_PS_CCS; p->portsc |= (value & ~(XHCI_PS_OCA | XHCI_PS_PR | XHCI_PS_PED | XHCI_PS_PLS_MASK | /* link state */ XHCI_PS_SPEED_MASK | XHCI_PS_PIC_MASK | /* port indicator */ XHCI_PS_LWS | XHCI_PS_DR | XHCI_PS_WPR)); /* clear control bits */ p->portsc &= ~(value & (XHCI_PS_CSC | XHCI_PS_PEC | XHCI_PS_WRC | XHCI_PS_OCC | XHCI_PS_PRC | XHCI_PS_PLC | XHCI_PS_CEC | XHCI_PS_CAS)); /* port disable request; for USB3, don't care */ if (value & XHCI_PS_PED) DPRINTF(("Disable port %d request", port)); if (!(value & XHCI_PS_LWS)) break; DPRINTF(("Port new PLS: %d", newpls)); switch (newpls) { case 0: /* U0 */ case 3: /* U3 */ if (oldpls != newpls) { p->portsc &= ~XHCI_PS_PLS_MASK; p->portsc |= XHCI_PS_PLS_SET(newpls) | XHCI_PS_PLC; if (oldpls != 0 && newpls == 0) { pci_xhci_set_evtrb(&evtrb, port, XHCI_TRB_ERROR_SUCCESS, XHCI_TRB_EVENT_PORT_STS_CHANGE); pci_xhci_insert_event(sc, &evtrb, 1); } } break; default: DPRINTF(("Unhandled change port %d PLS %u", port, newpls)); break; } break; case 4: /* Port power management status and control register */ p->portpmsc = value; break; case 8: /* Port link information register */ DPRINTF(("pci_xhci attempted write to PORTLI, port %d", port)); break; case 12: /* * Port hardware LPM control register. * For USB3, this register is reserved. */ p->porthlpmc = value; break; default: DPRINTF(("pci_xhci: unaligned portreg write offset %#lx", offset)); break; } } static struct xhci_dev_ctx * pci_xhci_get_dev_ctx(struct pci_xhci_softc *sc, uint32_t slot) { uint64_t devctx_addr; struct xhci_dev_ctx *devctx; assert(slot > 0 && slot <= XHCI_MAX_SLOTS); assert(XHCI_SLOTDEV_PTR(sc, slot) != NULL); assert(sc->opregs.dcbaa_p != NULL); devctx_addr = sc->opregs.dcbaa_p->dcba[slot]; if (devctx_addr == 0) { DPRINTF(("get_dev_ctx devctx_addr == 0")); return (NULL); } DPRINTF(("pci_xhci: get dev ctx, slot %u devctx addr %016lx", slot, devctx_addr)); devctx = XHCI_GADDR(sc, devctx_addr & ~0x3FUL); return (devctx); } static struct xhci_trb * pci_xhci_trb_next(struct pci_xhci_softc *sc, struct xhci_trb *curtrb, uint64_t *guestaddr) { struct xhci_trb *next; assert(curtrb != NULL); if (XHCI_TRB_3_TYPE_GET(curtrb->dwTrb3) == XHCI_TRB_TYPE_LINK) { if (guestaddr) *guestaddr = curtrb->qwTrb0 & ~0xFUL; next = XHCI_GADDR(sc, curtrb->qwTrb0 & ~0xFUL); } else { if (guestaddr) *guestaddr += sizeof(struct xhci_trb) & ~0xFUL; next = curtrb + 1; } return (next); } static void pci_xhci_assert_interrupt(struct pci_xhci_softc *sc) { sc->rtsregs.intrreg.erdp |= XHCI_ERDP_LO_BUSY; sc->rtsregs.intrreg.iman |= XHCI_IMAN_INTR_PEND; sc->opregs.usbsts |= XHCI_STS_EINT; /* only trigger interrupt if permitted */ if ((sc->opregs.usbcmd & XHCI_CMD_INTE) && (sc->rtsregs.intrreg.iman & XHCI_IMAN_INTR_ENA)) { if (pci_msi_enabled(sc->xsc_pi)) pci_generate_msi(sc->xsc_pi, 0); else pci_lintr_assert(sc->xsc_pi); } } static void pci_xhci_deassert_interrupt(struct pci_xhci_softc *sc) { if (!pci_msi_enabled(sc->xsc_pi)) pci_lintr_assert(sc->xsc_pi); } static void pci_xhci_init_ep(struct pci_xhci_dev_emu *dev, int epid) { struct xhci_dev_ctx *dev_ctx; struct pci_xhci_dev_ep *devep; struct xhci_endp_ctx *ep_ctx; uint32_t i, pstreams; dev_ctx = dev->dev_ctx; ep_ctx = &dev_ctx->ctx_ep[epid]; devep = &dev->eps[epid]; pstreams = XHCI_EPCTX_0_MAXP_STREAMS_GET(ep_ctx->dwEpCtx0); if (pstreams > 0) { DPRINTF(("init_ep %d with pstreams %u", epid, pstreams)); assert(devep->ep_sctx_trbs == NULL); devep->ep_sctx = XHCI_GADDR(dev->xsc, ep_ctx->qwEpCtx2 & XHCI_EPCTX_2_TR_DQ_PTR_MASK); devep->ep_sctx_trbs = calloc(pstreams, sizeof(struct pci_xhci_trb_ring)); for (i = 0; i < pstreams; i++) { devep->ep_sctx_trbs[i].ringaddr = devep->ep_sctx[i].qwSctx0 & XHCI_SCTX_0_TR_DQ_PTR_MASK; devep->ep_sctx_trbs[i].ccs = XHCI_SCTX_0_DCS_GET(devep->ep_sctx[i].qwSctx0); } } else { DPRINTF(("init_ep %d with no pstreams", epid)); devep->ep_ringaddr = ep_ctx->qwEpCtx2 & XHCI_EPCTX_2_TR_DQ_PTR_MASK; devep->ep_ccs = XHCI_EPCTX_2_DCS_GET(ep_ctx->qwEpCtx2); devep->ep_tr = XHCI_GADDR(dev->xsc, devep->ep_ringaddr); DPRINTF(("init_ep tr DCS %x", devep->ep_ccs)); } devep->ep_MaxPStreams = pstreams; if (devep->ep_xfer == NULL) { devep->ep_xfer = malloc(sizeof(struct usb_data_xfer)); USB_DATA_XFER_INIT(devep->ep_xfer); } } static void pci_xhci_disable_ep(struct pci_xhci_dev_emu *dev, int epid) { struct xhci_dev_ctx *dev_ctx; struct pci_xhci_dev_ep *devep; struct xhci_endp_ctx *ep_ctx; DPRINTF(("pci_xhci disable_ep %d", epid)); dev_ctx = dev->dev_ctx; ep_ctx = &dev_ctx->ctx_ep[epid]; ep_ctx->dwEpCtx0 = (ep_ctx->dwEpCtx0 & ~0x7) | XHCI_ST_EPCTX_DISABLED; devep = &dev->eps[epid]; if (devep->ep_MaxPStreams > 0) free(devep->ep_sctx_trbs); if (devep->ep_xfer != NULL) { free(devep->ep_xfer); devep->ep_xfer = NULL; } memset(devep, 0, sizeof(struct pci_xhci_dev_ep)); } /* reset device at slot and data structures related to it */ static void pci_xhci_reset_slot(struct pci_xhci_softc *sc, int slot) { struct pci_xhci_dev_emu *dev; dev = XHCI_SLOTDEV_PTR(sc, slot); if (!dev) { DPRINTF(("xhci reset unassigned slot (%d)?", slot)); } else { dev->dev_slotstate = XHCI_ST_DISABLED; } /* TODO: reset ring buffer pointers */ } static int pci_xhci_insert_event(struct pci_xhci_softc *sc, struct xhci_trb *evtrb, int do_intr) { struct pci_xhci_rtsregs *rts; uint64_t erdp; int erdp_idx; int err; struct xhci_trb *evtrbptr; err = XHCI_TRB_ERROR_SUCCESS; rts = &sc->rtsregs; erdp = rts->intrreg.erdp & ~0xF; erdp_idx = (erdp - rts->erstba_p[rts->er_deq_seg].qwEvrsTablePtr) / sizeof(struct xhci_trb); DPRINTF(("pci_xhci: insert event 0[%lx] 2[%x] 3[%x]", evtrb->qwTrb0, evtrb->dwTrb2, evtrb->dwTrb3)); DPRINTF(("\terdp idx %d/seg %d, enq idx %d/seg %d, pcs %u", erdp_idx, rts->er_deq_seg, rts->er_enq_idx, rts->er_enq_seg, rts->event_pcs)); DPRINTF(("\t(erdp=0x%lx, erst=0x%lx, tblsz=%u, do_intr %d)", erdp, rts->erstba_p->qwEvrsTablePtr, rts->erstba_p->dwEvrsTableSize, do_intr)); evtrbptr = &rts->erst_p[rts->er_enq_idx]; /* TODO: multi-segment table */ if (rts->er_events_cnt >= rts->erstba_p->dwEvrsTableSize) { DPRINTF(("pci_xhci[%d] cannot insert event; ring full", __LINE__)); err = XHCI_TRB_ERROR_EV_RING_FULL; goto done; } if (rts->er_events_cnt == rts->erstba_p->dwEvrsTableSize - 1) { struct xhci_trb errev; if ((evtrbptr->dwTrb3 & 0x1) == (rts->event_pcs & 0x1)) { DPRINTF(("pci_xhci[%d] insert evt err: ring full", __LINE__)); errev.qwTrb0 = 0; errev.dwTrb2 = XHCI_TRB_2_ERROR_SET( XHCI_TRB_ERROR_EV_RING_FULL); errev.dwTrb3 = XHCI_TRB_3_TYPE_SET( XHCI_TRB_EVENT_HOST_CTRL) | rts->event_pcs; rts->er_events_cnt++; memcpy(&rts->erst_p[rts->er_enq_idx], &errev, sizeof(struct xhci_trb)); rts->er_enq_idx = (rts->er_enq_idx + 1) % rts->erstba_p->dwEvrsTableSize; err = XHCI_TRB_ERROR_EV_RING_FULL; do_intr = 1; goto done; } } else { rts->er_events_cnt++; } evtrb->dwTrb3 &= ~XHCI_TRB_3_CYCLE_BIT; evtrb->dwTrb3 |= rts->event_pcs; memcpy(&rts->erst_p[rts->er_enq_idx], evtrb, sizeof(struct xhci_trb)); rts->er_enq_idx = (rts->er_enq_idx + 1) % rts->erstba_p->dwEvrsTableSize; if (rts->er_enq_idx == 0) rts->event_pcs ^= 1; done: if (do_intr) pci_xhci_assert_interrupt(sc); return (err); } static uint32_t pci_xhci_cmd_enable_slot(struct pci_xhci_softc *sc, uint32_t *slot) { struct pci_xhci_dev_emu *dev; uint32_t cmderr; int i; cmderr = XHCI_TRB_ERROR_NO_SLOTS; if (sc->portregs != NULL) for (i = 1; i <= XHCI_MAX_SLOTS; i++) { dev = XHCI_SLOTDEV_PTR(sc, i); if (dev && dev->dev_slotstate == XHCI_ST_DISABLED) { *slot = i; dev->dev_slotstate = XHCI_ST_ENABLED; cmderr = XHCI_TRB_ERROR_SUCCESS; dev->hci.hci_address = i; break; } } DPRINTF(("pci_xhci enable slot (error=%d) slot %u", cmderr != XHCI_TRB_ERROR_SUCCESS, *slot)); return (cmderr); } static uint32_t pci_xhci_cmd_disable_slot(struct pci_xhci_softc *sc, uint32_t slot) { struct pci_xhci_dev_emu *dev; uint32_t cmderr; DPRINTF(("pci_xhci disable slot %u", slot)); if (sc->portregs == NULL) { cmderr = XHCI_TRB_ERROR_NO_SLOTS; goto done; } cmderr = pci_xhci_validate_slot(slot); if (cmderr != XHCI_TRB_ERROR_SUCCESS) goto done; dev = XHCI_SLOTDEV_PTR(sc, slot); if (dev) { if (dev->dev_slotstate == XHCI_ST_DISABLED) { cmderr = XHCI_TRB_ERROR_SLOT_NOT_ON; } else { dev->dev_slotstate = XHCI_ST_DISABLED; /* TODO: reset events and endpoints */ } } else cmderr = XHCI_TRB_ERROR_SLOT_NOT_ON; done: return (cmderr); } static uint32_t pci_xhci_cmd_reset_device(struct pci_xhci_softc *sc, uint32_t slot) { struct pci_xhci_dev_emu *dev; struct xhci_dev_ctx *dev_ctx; struct xhci_endp_ctx *ep_ctx; uint32_t cmderr; int i; if (sc->portregs == NULL) { cmderr = XHCI_TRB_ERROR_NO_SLOTS; goto done; } DPRINTF(("pci_xhci reset device slot %u", slot)); cmderr = pci_xhci_validate_slot(slot); if (cmderr != XHCI_TRB_ERROR_SUCCESS) goto done; dev = XHCI_SLOTDEV_PTR(sc, slot); if (!dev || dev->dev_slotstate == XHCI_ST_DISABLED) cmderr = XHCI_TRB_ERROR_SLOT_NOT_ON; else { dev->dev_slotstate = XHCI_ST_DEFAULT; dev->hci.hci_address = 0; dev_ctx = pci_xhci_get_dev_ctx(sc, slot); if (dev_ctx == NULL) { cmderr = XHCI_TRB_ERROR_PARAMETER; goto done; } /* slot state */ dev_ctx->ctx_slot.dwSctx3 = FIELD_REPLACE( dev_ctx->ctx_slot.dwSctx3, XHCI_ST_SLCTX_DEFAULT, 0x1F, 27); /* number of contexts */ dev_ctx->ctx_slot.dwSctx0 = FIELD_REPLACE( dev_ctx->ctx_slot.dwSctx0, 1, 0x1F, 27); /* reset all eps other than ep-0 */ for (i = 2; i <= 31; i++) { ep_ctx = &dev_ctx->ctx_ep[i]; ep_ctx->dwEpCtx0 = FIELD_REPLACE( ep_ctx->dwEpCtx0, XHCI_ST_EPCTX_DISABLED, 0x7, 0); } } pci_xhci_reset_slot(sc, slot); done: return (cmderr); } static uint32_t pci_xhci_cmd_address_device(struct pci_xhci_softc *sc, uint32_t slot, struct xhci_trb *trb) { struct pci_xhci_dev_emu *dev; struct xhci_input_dev_ctx *input_ctx; struct xhci_slot_ctx *islot_ctx; struct xhci_dev_ctx *dev_ctx; struct xhci_endp_ctx *ep0_ctx; uint32_t cmderr; input_ctx = XHCI_GADDR(sc, trb->qwTrb0 & ~0xFUL); islot_ctx = &input_ctx->ctx_slot; ep0_ctx = &input_ctx->ctx_ep[1]; DPRINTF(("pci_xhci: address device, input ctl: D 0x%08x A 0x%08x,", input_ctx->ctx_input.dwInCtx0, input_ctx->ctx_input.dwInCtx1)); DPRINTF((" slot %08x %08x %08x %08x", islot_ctx->dwSctx0, islot_ctx->dwSctx1, islot_ctx->dwSctx2, islot_ctx->dwSctx3)); DPRINTF((" ep0 %08x %08x %016lx %08x", ep0_ctx->dwEpCtx0, ep0_ctx->dwEpCtx1, ep0_ctx->qwEpCtx2, ep0_ctx->dwEpCtx4)); /* when setting address: drop-ctx=0, add-ctx=slot+ep0 */ if ((input_ctx->ctx_input.dwInCtx0 != 0) || (input_ctx->ctx_input.dwInCtx1 & 0x03) != 0x03) { DPRINTF(("pci_xhci: address device, input ctl invalid")); cmderr = XHCI_TRB_ERROR_TRB; goto done; } cmderr = pci_xhci_validate_slot(slot); if (cmderr != XHCI_TRB_ERROR_SUCCESS) goto done; /* assign address to slot */ dev_ctx = pci_xhci_get_dev_ctx(sc, slot); if (dev_ctx == NULL) { cmderr = XHCI_TRB_ERROR_PARAMETER; goto done; } DPRINTF(("pci_xhci: address device, dev ctx")); DPRINTF((" slot %08x %08x %08x %08x", dev_ctx->ctx_slot.dwSctx0, dev_ctx->ctx_slot.dwSctx1, dev_ctx->ctx_slot.dwSctx2, dev_ctx->ctx_slot.dwSctx3)); dev = XHCI_SLOTDEV_PTR(sc, slot); assert(dev != NULL); dev->hci.hci_address = slot; dev->dev_ctx = dev_ctx; if (dev->dev_ue->ue_reset == NULL || dev->dev_ue->ue_reset(dev->dev_sc) < 0) { cmderr = XHCI_TRB_ERROR_ENDP_NOT_ON; goto done; } memcpy(&dev_ctx->ctx_slot, islot_ctx, sizeof(struct xhci_slot_ctx)); dev_ctx->ctx_slot.dwSctx3 = XHCI_SCTX_3_SLOT_STATE_SET(XHCI_ST_SLCTX_ADDRESSED) | XHCI_SCTX_3_DEV_ADDR_SET(slot); memcpy(&dev_ctx->ctx_ep[1], ep0_ctx, sizeof(struct xhci_endp_ctx)); ep0_ctx = &dev_ctx->ctx_ep[1]; ep0_ctx->dwEpCtx0 = (ep0_ctx->dwEpCtx0 & ~0x7) | XHCI_EPCTX_0_EPSTATE_SET(XHCI_ST_EPCTX_RUNNING); pci_xhci_init_ep(dev, 1); dev->dev_slotstate = XHCI_ST_ADDRESSED; DPRINTF(("pci_xhci: address device, output ctx")); DPRINTF((" slot %08x %08x %08x %08x", dev_ctx->ctx_slot.dwSctx0, dev_ctx->ctx_slot.dwSctx1, dev_ctx->ctx_slot.dwSctx2, dev_ctx->ctx_slot.dwSctx3)); DPRINTF((" ep0 %08x %08x %016lx %08x", ep0_ctx->dwEpCtx0, ep0_ctx->dwEpCtx1, ep0_ctx->qwEpCtx2, ep0_ctx->dwEpCtx4)); done: return (cmderr); } static uint32_t pci_xhci_cmd_config_ep(struct pci_xhci_softc *sc, uint32_t slot, struct xhci_trb *trb) { struct xhci_input_dev_ctx *input_ctx; struct pci_xhci_dev_emu *dev; struct xhci_dev_ctx *dev_ctx; struct xhci_endp_ctx *ep_ctx, *iep_ctx; uint32_t cmderr; int i; DPRINTF(("pci_xhci config_ep slot %u", slot)); cmderr = pci_xhci_validate_slot(slot); if (cmderr != XHCI_TRB_ERROR_SUCCESS) goto done; dev = XHCI_SLOTDEV_PTR(sc, slot); assert(dev != NULL); if ((trb->dwTrb3 & XHCI_TRB_3_DCEP_BIT) != 0) { DPRINTF(("pci_xhci config_ep - deconfigure ep slot %u", slot)); if (dev->dev_ue->ue_stop != NULL) dev->dev_ue->ue_stop(dev->dev_sc); dev->dev_slotstate = XHCI_ST_ADDRESSED; dev->hci.hci_address = 0; dev_ctx = pci_xhci_get_dev_ctx(sc, slot); if (dev_ctx == NULL) { cmderr = XHCI_TRB_ERROR_PARAMETER; goto done; } /* number of contexts */ dev_ctx->ctx_slot.dwSctx0 = FIELD_REPLACE( dev_ctx->ctx_slot.dwSctx0, 1, 0x1F, 27); /* slot state */ dev_ctx->ctx_slot.dwSctx3 = FIELD_REPLACE( dev_ctx->ctx_slot.dwSctx3, XHCI_ST_SLCTX_ADDRESSED, 0x1F, 27); /* disable endpoints */ for (i = 2; i < 32; i++) pci_xhci_disable_ep(dev, i); cmderr = XHCI_TRB_ERROR_SUCCESS; goto done; } if (dev->dev_slotstate < XHCI_ST_ADDRESSED) { DPRINTF(("pci_xhci: config_ep slotstate x%x != addressed", dev->dev_slotstate)); cmderr = XHCI_TRB_ERROR_SLOT_NOT_ON; goto done; } /* In addressed/configured state; * for each drop endpoint ctx flag: * ep->state = DISABLED * for each add endpoint ctx flag: * cp(ep-in, ep-out) * ep->state = RUNNING * for each drop+add endpoint flag: * reset ep resources * cp(ep-in, ep-out) * ep->state = RUNNING * if input->DisabledCtx[2-31] < 30: (at least 1 ep not disabled) * slot->state = configured */ input_ctx = XHCI_GADDR(sc, trb->qwTrb0 & ~0xFUL); dev_ctx = dev->dev_ctx; DPRINTF(("pci_xhci: config_ep inputctx: D:x%08x A:x%08x 7:x%08x", input_ctx->ctx_input.dwInCtx0, input_ctx->ctx_input.dwInCtx1, input_ctx->ctx_input.dwInCtx7)); for (i = 2; i <= 31; i++) { ep_ctx = &dev_ctx->ctx_ep[i]; if (input_ctx->ctx_input.dwInCtx0 & XHCI_INCTX_0_DROP_MASK(i)) { DPRINTF((" config ep - dropping ep %d", i)); pci_xhci_disable_ep(dev, i); } if (input_ctx->ctx_input.dwInCtx1 & XHCI_INCTX_1_ADD_MASK(i)) { iep_ctx = &input_ctx->ctx_ep[i]; DPRINTF((" enable ep[%d] %08x %08x %016lx %08x", i, iep_ctx->dwEpCtx0, iep_ctx->dwEpCtx1, iep_ctx->qwEpCtx2, iep_ctx->dwEpCtx4)); memcpy(ep_ctx, iep_ctx, sizeof(struct xhci_endp_ctx)); pci_xhci_init_ep(dev, i); /* ep state */ ep_ctx->dwEpCtx0 = FIELD_REPLACE( ep_ctx->dwEpCtx0, XHCI_ST_EPCTX_RUNNING, 0x7, 0); } } /* slot state to configured */ dev_ctx->ctx_slot.dwSctx3 = FIELD_REPLACE( dev_ctx->ctx_slot.dwSctx3, XHCI_ST_SLCTX_CONFIGURED, 0x1F, 27); dev_ctx->ctx_slot.dwSctx0 = FIELD_COPY( dev_ctx->ctx_slot.dwSctx0, input_ctx->ctx_slot.dwSctx0, 0x1F, 27); dev->dev_slotstate = XHCI_ST_CONFIGURED; DPRINTF(("EP configured; slot %u [0]=0x%08x [1]=0x%08x [2]=0x%08x " "[3]=0x%08x", slot, dev_ctx->ctx_slot.dwSctx0, dev_ctx->ctx_slot.dwSctx1, dev_ctx->ctx_slot.dwSctx2, dev_ctx->ctx_slot.dwSctx3)); done: return (cmderr); } static uint32_t pci_xhci_cmd_reset_ep(struct pci_xhci_softc *sc, uint32_t slot, struct xhci_trb *trb) { struct pci_xhci_dev_emu *dev; struct pci_xhci_dev_ep *devep; struct xhci_dev_ctx *dev_ctx; struct xhci_endp_ctx *ep_ctx; uint32_t cmderr, epid; uint32_t type; epid = XHCI_TRB_3_EP_GET(trb->dwTrb3); DPRINTF(("pci_xhci: reset ep %u: slot %u", epid, slot)); cmderr = pci_xhci_validate_slot(slot); if (cmderr != XHCI_TRB_ERROR_SUCCESS) goto done; dev = XHCI_SLOTDEV_PTR(sc, slot); assert(dev != NULL); type = XHCI_TRB_3_TYPE_GET(trb->dwTrb3); if (type == XHCI_TRB_TYPE_STOP_EP && (trb->dwTrb3 & XHCI_TRB_3_SUSP_EP_BIT) != 0) { /* XXX suspend endpoint for 10ms */ } if (epid < 1 || epid > 31) { DPRINTF(("pci_xhci: reset ep: invalid epid %u", epid)); cmderr = XHCI_TRB_ERROR_TRB; goto done; } devep = &dev->eps[epid]; if (devep->ep_xfer != NULL) USB_DATA_XFER_RESET(devep->ep_xfer); dev_ctx = dev->dev_ctx; assert(dev_ctx != NULL); ep_ctx = &dev_ctx->ctx_ep[epid]; ep_ctx->dwEpCtx0 = (ep_ctx->dwEpCtx0 & ~0x7) | XHCI_ST_EPCTX_STOPPED; if (devep->ep_MaxPStreams == 0) ep_ctx->qwEpCtx2 = devep->ep_ringaddr | devep->ep_ccs; DPRINTF(("pci_xhci: reset ep[%u] %08x %08x %016lx %08x", epid, ep_ctx->dwEpCtx0, ep_ctx->dwEpCtx1, ep_ctx->qwEpCtx2, ep_ctx->dwEpCtx4)); if (type == XHCI_TRB_TYPE_RESET_EP && (dev->dev_ue->ue_reset == NULL || dev->dev_ue->ue_reset(dev->dev_sc) < 0)) { cmderr = XHCI_TRB_ERROR_ENDP_NOT_ON; goto done; } done: return (cmderr); } static uint32_t pci_xhci_find_stream(struct pci_xhci_softc *sc, struct xhci_endp_ctx *ep, struct pci_xhci_dev_ep *devep, uint32_t streamid) { struct xhci_stream_ctx *sctx; if (devep->ep_MaxPStreams == 0) return (XHCI_TRB_ERROR_TRB); if (devep->ep_MaxPStreams > XHCI_STREAMS_MAX) return (XHCI_TRB_ERROR_INVALID_SID); if (XHCI_EPCTX_0_LSA_GET(ep->dwEpCtx0) == 0) { DPRINTF(("pci_xhci: find_stream; LSA bit not set")); return (XHCI_TRB_ERROR_INVALID_SID); } /* only support primary stream */ if (streamid >= devep->ep_MaxPStreams) return (XHCI_TRB_ERROR_STREAM_TYPE); sctx = (struct xhci_stream_ctx *)XHCI_GADDR(sc, ep->qwEpCtx2 & ~0xFUL) + streamid; if (!XHCI_SCTX_0_SCT_GET(sctx->qwSctx0)) return (XHCI_TRB_ERROR_STREAM_TYPE); return (XHCI_TRB_ERROR_SUCCESS); } static uint32_t pci_xhci_cmd_set_tr(struct pci_xhci_softc *sc, uint32_t slot, struct xhci_trb *trb) { struct pci_xhci_dev_emu *dev; struct pci_xhci_dev_ep *devep; struct xhci_dev_ctx *dev_ctx; struct xhci_endp_ctx *ep_ctx; uint32_t cmderr, epid; uint32_t streamid; cmderr = pci_xhci_validate_slot(slot); if (cmderr != XHCI_TRB_ERROR_SUCCESS) goto done; dev = XHCI_SLOTDEV_PTR(sc, slot); assert(dev != NULL); DPRINTF(("pci_xhci set_tr: new-tr x%016lx, SCT %u DCS %u", (trb->qwTrb0 & ~0xF), (uint32_t)((trb->qwTrb0 >> 1) & 0x7), (uint32_t)(trb->qwTrb0 & 0x1))); DPRINTF((" stream-id %u, slot %u, epid %u, C %u", (trb->dwTrb2 >> 16) & 0xFFFF, XHCI_TRB_3_SLOT_GET(trb->dwTrb3), XHCI_TRB_3_EP_GET(trb->dwTrb3), trb->dwTrb3 & 0x1)); epid = XHCI_TRB_3_EP_GET(trb->dwTrb3); if (epid < 1 || epid > 31) { DPRINTF(("pci_xhci: set_tr_deq: invalid epid %u", epid)); cmderr = XHCI_TRB_ERROR_TRB; goto done; } dev_ctx = dev->dev_ctx; assert(dev_ctx != NULL); ep_ctx = &dev_ctx->ctx_ep[epid]; devep = &dev->eps[epid]; switch (XHCI_EPCTX_0_EPSTATE_GET(ep_ctx->dwEpCtx0)) { case XHCI_ST_EPCTX_STOPPED: case XHCI_ST_EPCTX_ERROR: break; default: DPRINTF(("pci_xhci cmd set_tr invalid state %x", XHCI_EPCTX_0_EPSTATE_GET(ep_ctx->dwEpCtx0))); cmderr = XHCI_TRB_ERROR_CONTEXT_STATE; goto done; } streamid = XHCI_TRB_2_STREAM_GET(trb->dwTrb2); if (devep->ep_MaxPStreams > 0) { cmderr = pci_xhci_find_stream(sc, ep_ctx, devep, streamid); if (cmderr == XHCI_TRB_ERROR_SUCCESS) { assert(devep->ep_sctx != NULL); devep->ep_sctx[streamid].qwSctx0 = trb->qwTrb0; devep->ep_sctx_trbs[streamid].ringaddr = trb->qwTrb0 & ~0xF; devep->ep_sctx_trbs[streamid].ccs = XHCI_EPCTX_2_DCS_GET(trb->qwTrb0); } } else { if (streamid != 0) { DPRINTF(("pci_xhci cmd set_tr streamid %x != 0", streamid)); } ep_ctx->qwEpCtx2 = trb->qwTrb0 & ~0xFUL; devep->ep_ringaddr = ep_ctx->qwEpCtx2 & ~0xFUL; devep->ep_ccs = trb->qwTrb0 & 0x1; devep->ep_tr = XHCI_GADDR(sc, devep->ep_ringaddr); DPRINTF(("pci_xhci set_tr first TRB:")); pci_xhci_dump_trb(devep->ep_tr); } ep_ctx->dwEpCtx0 = (ep_ctx->dwEpCtx0 & ~0x7) | XHCI_ST_EPCTX_STOPPED; done: return (cmderr); } static uint32_t pci_xhci_cmd_eval_ctx(struct pci_xhci_softc *sc, uint32_t slot, struct xhci_trb *trb) { struct xhci_input_dev_ctx *input_ctx; struct xhci_slot_ctx *islot_ctx; struct xhci_dev_ctx *dev_ctx; struct xhci_endp_ctx *ep0_ctx; uint32_t cmderr; input_ctx = XHCI_GADDR(sc, trb->qwTrb0 & ~0xFUL); islot_ctx = &input_ctx->ctx_slot; ep0_ctx = &input_ctx->ctx_ep[1]; DPRINTF(("pci_xhci: eval ctx, input ctl: D 0x%08x A 0x%08x,", input_ctx->ctx_input.dwInCtx0, input_ctx->ctx_input.dwInCtx1)); DPRINTF((" slot %08x %08x %08x %08x", islot_ctx->dwSctx0, islot_ctx->dwSctx1, islot_ctx->dwSctx2, islot_ctx->dwSctx3)); DPRINTF((" ep0 %08x %08x %016lx %08x", ep0_ctx->dwEpCtx0, ep0_ctx->dwEpCtx1, ep0_ctx->qwEpCtx2, ep0_ctx->dwEpCtx4)); /* this command expects drop-ctx=0 & add-ctx=slot+ep0 */ if ((input_ctx->ctx_input.dwInCtx0 != 0) || (input_ctx->ctx_input.dwInCtx1 & 0x03) == 0) { DPRINTF(("pci_xhci: eval ctx, input ctl invalid")); cmderr = XHCI_TRB_ERROR_TRB; goto done; } cmderr = pci_xhci_validate_slot(slot); if (cmderr != XHCI_TRB_ERROR_SUCCESS) goto done; /* assign address to slot; in this emulation, slot_id = address */ dev_ctx = pci_xhci_get_dev_ctx(sc, slot); if (dev_ctx == NULL) { cmderr = XHCI_TRB_ERROR_PARAMETER; goto done; } DPRINTF(("pci_xhci: eval ctx, dev ctx")); DPRINTF((" slot %08x %08x %08x %08x", dev_ctx->ctx_slot.dwSctx0, dev_ctx->ctx_slot.dwSctx1, dev_ctx->ctx_slot.dwSctx2, dev_ctx->ctx_slot.dwSctx3)); if (input_ctx->ctx_input.dwInCtx1 & 0x01) { /* slot ctx */ /* set max exit latency */ dev_ctx->ctx_slot.dwSctx1 = FIELD_COPY( dev_ctx->ctx_slot.dwSctx1, input_ctx->ctx_slot.dwSctx1, 0xFFFF, 0); /* set interrupter target */ dev_ctx->ctx_slot.dwSctx2 = FIELD_COPY( dev_ctx->ctx_slot.dwSctx2, input_ctx->ctx_slot.dwSctx2, 0x3FF, 22); } if (input_ctx->ctx_input.dwInCtx1 & 0x02) { /* control ctx */ /* set max packet size */ dev_ctx->ctx_ep[1].dwEpCtx1 = FIELD_COPY( dev_ctx->ctx_ep[1].dwEpCtx1, ep0_ctx->dwEpCtx1, 0xFFFF, 16); ep0_ctx = &dev_ctx->ctx_ep[1]; } DPRINTF(("pci_xhci: eval ctx, output ctx")); DPRINTF((" slot %08x %08x %08x %08x", dev_ctx->ctx_slot.dwSctx0, dev_ctx->ctx_slot.dwSctx1, dev_ctx->ctx_slot.dwSctx2, dev_ctx->ctx_slot.dwSctx3)); DPRINTF((" ep0 %08x %08x %016lx %08x", ep0_ctx->dwEpCtx0, ep0_ctx->dwEpCtx1, ep0_ctx->qwEpCtx2, ep0_ctx->dwEpCtx4)); done: return (cmderr); } static int pci_xhci_complete_commands(struct pci_xhci_softc *sc) { struct xhci_trb evtrb; struct xhci_trb *trb; uint64_t crcr; uint32_t ccs; /* cycle state (XHCI 4.9.2) */ uint32_t type; uint32_t slot; uint32_t cmderr; int error; error = 0; sc->opregs.crcr |= XHCI_CRCR_LO_CRR; trb = sc->opregs.cr_p; ccs = sc->opregs.crcr & XHCI_CRCR_LO_RCS; crcr = sc->opregs.crcr & ~0xF; while (1) { sc->opregs.cr_p = trb; type = XHCI_TRB_3_TYPE_GET(trb->dwTrb3); if ((trb->dwTrb3 & XHCI_TRB_3_CYCLE_BIT) != (ccs & XHCI_TRB_3_CYCLE_BIT)) break; DPRINTF(("pci_xhci: cmd type 0x%x, Trb0 x%016lx dwTrb2 x%08x" " dwTrb3 x%08x, TRB_CYCLE %u/ccs %u", type, trb->qwTrb0, trb->dwTrb2, trb->dwTrb3, trb->dwTrb3 & XHCI_TRB_3_CYCLE_BIT, ccs)); cmderr = XHCI_TRB_ERROR_SUCCESS; evtrb.dwTrb2 = 0; evtrb.dwTrb3 = (ccs & XHCI_TRB_3_CYCLE_BIT) | XHCI_TRB_3_TYPE_SET(XHCI_TRB_EVENT_CMD_COMPLETE); slot = 0; switch (type) { case XHCI_TRB_TYPE_LINK: /* 0x06 */ if (trb->dwTrb3 & XHCI_TRB_3_TC_BIT) ccs ^= XHCI_CRCR_LO_RCS; break; case XHCI_TRB_TYPE_ENABLE_SLOT: /* 0x09 */ cmderr = pci_xhci_cmd_enable_slot(sc, &slot); break; case XHCI_TRB_TYPE_DISABLE_SLOT: /* 0x0A */ slot = XHCI_TRB_3_SLOT_GET(trb->dwTrb3); cmderr = pci_xhci_cmd_disable_slot(sc, slot); break; case XHCI_TRB_TYPE_ADDRESS_DEVICE: /* 0x0B */ slot = XHCI_TRB_3_SLOT_GET(trb->dwTrb3); cmderr = pci_xhci_cmd_address_device(sc, slot, trb); break; case XHCI_TRB_TYPE_CONFIGURE_EP: /* 0x0C */ slot = XHCI_TRB_3_SLOT_GET(trb->dwTrb3); cmderr = pci_xhci_cmd_config_ep(sc, slot, trb); break; case XHCI_TRB_TYPE_EVALUATE_CTX: /* 0x0D */ slot = XHCI_TRB_3_SLOT_GET(trb->dwTrb3); cmderr = pci_xhci_cmd_eval_ctx(sc, slot, trb); break; case XHCI_TRB_TYPE_RESET_EP: /* 0x0E */ DPRINTF(("Reset Endpoint on slot %d", slot)); slot = XHCI_TRB_3_SLOT_GET(trb->dwTrb3); cmderr = pci_xhci_cmd_reset_ep(sc, slot, trb); break; case XHCI_TRB_TYPE_STOP_EP: /* 0x0F */ DPRINTF(("Stop Endpoint on slot %d", slot)); slot = XHCI_TRB_3_SLOT_GET(trb->dwTrb3); cmderr = pci_xhci_cmd_reset_ep(sc, slot, trb); break; case XHCI_TRB_TYPE_SET_TR_DEQUEUE: /* 0x10 */ slot = XHCI_TRB_3_SLOT_GET(trb->dwTrb3); cmderr = pci_xhci_cmd_set_tr(sc, slot, trb); break; case XHCI_TRB_TYPE_RESET_DEVICE: /* 0x11 */ slot = XHCI_TRB_3_SLOT_GET(trb->dwTrb3); cmderr = pci_xhci_cmd_reset_device(sc, slot); break; case XHCI_TRB_TYPE_FORCE_EVENT: /* 0x12 */ /* TODO: */ break; case XHCI_TRB_TYPE_NEGOTIATE_BW: /* 0x13 */ break; case XHCI_TRB_TYPE_SET_LATENCY_TOL: /* 0x14 */ break; case XHCI_TRB_TYPE_GET_PORT_BW: /* 0x15 */ break; case XHCI_TRB_TYPE_FORCE_HEADER: /* 0x16 */ break; case XHCI_TRB_TYPE_NOOP_CMD: /* 0x17 */ break; default: DPRINTF(("pci_xhci: unsupported cmd %x", type)); break; } if (type != XHCI_TRB_TYPE_LINK) { /* * insert command completion event and assert intr */ evtrb.qwTrb0 = crcr; evtrb.dwTrb2 |= XHCI_TRB_2_ERROR_SET(cmderr); evtrb.dwTrb3 |= XHCI_TRB_3_SLOT_SET(slot); DPRINTF(("pci_xhci: command 0x%x result: 0x%x", type, cmderr)); pci_xhci_insert_event(sc, &evtrb, 1); } trb = pci_xhci_trb_next(sc, trb, &crcr); } sc->opregs.crcr = crcr | (sc->opregs.crcr & XHCI_CRCR_LO_CA) | ccs; sc->opregs.crcr &= ~XHCI_CRCR_LO_CRR; return (error); } static void pci_xhci_dump_trb(struct xhci_trb *trb) { static const char *trbtypes[] = { "RESERVED", "NORMAL", "SETUP_STAGE", "DATA_STAGE", "STATUS_STAGE", "ISOCH", "LINK", "EVENT_DATA", "NOOP", "ENABLE_SLOT", "DISABLE_SLOT", "ADDRESS_DEVICE", "CONFIGURE_EP", "EVALUATE_CTX", "RESET_EP", "STOP_EP", "SET_TR_DEQUEUE", "RESET_DEVICE", "FORCE_EVENT", "NEGOTIATE_BW", "SET_LATENCY_TOL", "GET_PORT_BW", "FORCE_HEADER", "NOOP_CMD" }; uint32_t type; type = XHCI_TRB_3_TYPE_GET(trb->dwTrb3); DPRINTF(("pci_xhci: trb[@%p] type x%02x %s 0:x%016lx 2:x%08x 3:x%08x", trb, type, type <= XHCI_TRB_TYPE_NOOP_CMD ? trbtypes[type] : "INVALID", trb->qwTrb0, trb->dwTrb2, trb->dwTrb3)); } static int pci_xhci_xfer_complete(struct pci_xhci_softc *sc, struct usb_data_xfer *xfer, uint32_t slot, uint32_t epid, int *do_intr) { struct pci_xhci_dev_emu *dev; struct pci_xhci_dev_ep *devep; struct xhci_dev_ctx *dev_ctx; struct xhci_endp_ctx *ep_ctx; struct xhci_trb *trb; struct xhci_trb evtrb; uint32_t trbflags; uint32_t edtla; int i, err; dev = XHCI_SLOTDEV_PTR(sc, slot); devep = &dev->eps[epid]; dev_ctx = pci_xhci_get_dev_ctx(sc, slot); if (dev_ctx == NULL) { return XHCI_TRB_ERROR_PARAMETER; } ep_ctx = &dev_ctx->ctx_ep[epid]; err = XHCI_TRB_ERROR_SUCCESS; *do_intr = 0; edtla = 0; /* go through list of TRBs and insert event(s) */ for (i = xfer->head; xfer->ndata > 0; ) { evtrb.qwTrb0 = (uint64_t)xfer->data[i].hci_data; trb = XHCI_GADDR(sc, evtrb.qwTrb0); trbflags = trb->dwTrb3; DPRINTF(("pci_xhci: xfer[%d] done?%u:%d trb %x %016lx %x " "(err %d) IOC?%d", i, xfer->data[i].processed, xfer->data[i].blen, XHCI_TRB_3_TYPE_GET(trbflags), evtrb.qwTrb0, trbflags, err, trb->dwTrb3 & XHCI_TRB_3_IOC_BIT ? 1 : 0)); if (!xfer->data[i].processed) { xfer->head = i; break; } xfer->ndata--; edtla += xfer->data[i].bdone; trb->dwTrb3 = (trb->dwTrb3 & ~0x1) | (xfer->data[i].ccs); pci_xhci_update_ep_ring(sc, dev, devep, ep_ctx, xfer->data[i].streamid, xfer->data[i].trbnext, xfer->data[i].ccs); /* Only interrupt if IOC or short packet */ if (!(trb->dwTrb3 & XHCI_TRB_3_IOC_BIT) && !((err == XHCI_TRB_ERROR_SHORT_PKT) && (trb->dwTrb3 & XHCI_TRB_3_ISP_BIT))) { i = (i + 1) % USB_MAX_XFER_BLOCKS; continue; } evtrb.dwTrb2 = XHCI_TRB_2_ERROR_SET(err) | XHCI_TRB_2_REM_SET(xfer->data[i].blen); evtrb.dwTrb3 = XHCI_TRB_3_TYPE_SET(XHCI_TRB_EVENT_TRANSFER) | XHCI_TRB_3_SLOT_SET(slot) | XHCI_TRB_3_EP_SET(epid); if (XHCI_TRB_3_TYPE_GET(trbflags) == XHCI_TRB_TYPE_EVENT_DATA) { DPRINTF(("pci_xhci EVENT_DATA edtla %u", edtla)); evtrb.qwTrb0 = trb->qwTrb0; evtrb.dwTrb2 = (edtla & 0xFFFFF) | XHCI_TRB_2_ERROR_SET(err); evtrb.dwTrb3 |= XHCI_TRB_3_ED_BIT; edtla = 0; } *do_intr = 1; err = pci_xhci_insert_event(sc, &evtrb, 0); if (err != XHCI_TRB_ERROR_SUCCESS) { break; } i = (i + 1) % USB_MAX_XFER_BLOCKS; } return (err); } static void pci_xhci_update_ep_ring(struct pci_xhci_softc *sc, struct pci_xhci_dev_emu *dev __unused, struct pci_xhci_dev_ep *devep, struct xhci_endp_ctx *ep_ctx, uint32_t streamid, uint64_t ringaddr, int ccs) { if (devep->ep_MaxPStreams != 0) { devep->ep_sctx[streamid].qwSctx0 = (ringaddr & ~0xFUL) | (ccs & 0x1); devep->ep_sctx_trbs[streamid].ringaddr = ringaddr & ~0xFUL; devep->ep_sctx_trbs[streamid].ccs = ccs & 0x1; ep_ctx->qwEpCtx2 = (ep_ctx->qwEpCtx2 & ~0x1) | (ccs & 0x1); DPRINTF(("xhci update ep-ring stream %d, addr %lx", streamid, devep->ep_sctx[streamid].qwSctx0)); } else { devep->ep_ringaddr = ringaddr & ~0xFUL; devep->ep_ccs = ccs & 0x1; devep->ep_tr = XHCI_GADDR(sc, ringaddr & ~0xFUL); ep_ctx->qwEpCtx2 = (ringaddr & ~0xFUL) | (ccs & 0x1); DPRINTF(("xhci update ep-ring, addr %lx", (devep->ep_ringaddr | devep->ep_ccs))); } } static int pci_xhci_validate_slot(uint32_t slot) { if (slot == 0) return (XHCI_TRB_ERROR_TRB); else if (slot > XHCI_MAX_SLOTS) return (XHCI_TRB_ERROR_SLOT_NOT_ON); else return (XHCI_TRB_ERROR_SUCCESS); } /* * Outstanding transfer still in progress (device NAK'd earlier) so retry * the transfer again to see if it succeeds. */ static int pci_xhci_try_usb_xfer(struct pci_xhci_softc *sc, struct pci_xhci_dev_emu *dev, struct pci_xhci_dev_ep *devep, struct xhci_endp_ctx *ep_ctx, uint32_t slot, uint32_t epid) { struct usb_data_xfer *xfer; int err; int do_intr; ep_ctx->dwEpCtx0 = FIELD_REPLACE( ep_ctx->dwEpCtx0, XHCI_ST_EPCTX_RUNNING, 0x7, 0); err = 0; do_intr = 0; xfer = devep->ep_xfer; #ifdef __FreeBSD__ USB_DATA_XFER_LOCK(xfer); #else /* * At least one caller needs to hold this lock across the call to this * function and other code. To avoid deadlock from a recursive mutex * enter, we ensure that all callers hold this lock. */ assert(USB_DATA_XFER_LOCK_HELD(xfer)); #endif /* outstanding requests queued up */ if (dev->dev_ue->ue_data != NULL) { err = dev->dev_ue->ue_data(dev->dev_sc, xfer, epid & 0x1 ? USB_XFER_IN : USB_XFER_OUT, epid/2); if (err == USB_ERR_CANCELLED) { if (USB_DATA_GET_ERRCODE(&xfer->data[xfer->head]) == USB_NAK) err = XHCI_TRB_ERROR_SUCCESS; } else { err = pci_xhci_xfer_complete(sc, xfer, slot, epid, &do_intr); if (err == XHCI_TRB_ERROR_SUCCESS && do_intr) { pci_xhci_assert_interrupt(sc); } /* XXX should not do it if error? */ USB_DATA_XFER_RESET(xfer); } } #ifdef __FreeBSD__ USB_DATA_XFER_UNLOCK(xfer); #endif return (err); } static int pci_xhci_handle_transfer(struct pci_xhci_softc *sc, struct pci_xhci_dev_emu *dev, struct pci_xhci_dev_ep *devep, struct xhci_endp_ctx *ep_ctx, struct xhci_trb *trb, uint32_t slot, uint32_t epid, uint64_t addr, uint32_t ccs, uint32_t streamid) { struct xhci_trb *setup_trb; struct usb_data_xfer *xfer; struct usb_data_xfer_block *xfer_block; uint64_t val; uint32_t trbflags; int do_intr, err; int do_retry; ep_ctx->dwEpCtx0 = FIELD_REPLACE(ep_ctx->dwEpCtx0, XHCI_ST_EPCTX_RUNNING, 0x7, 0); xfer = devep->ep_xfer; USB_DATA_XFER_LOCK(xfer); DPRINTF(("pci_xhci handle_transfer slot %u", slot)); retry: err = XHCI_TRB_ERROR_INVALID; do_retry = 0; do_intr = 0; setup_trb = NULL; while (1) { pci_xhci_dump_trb(trb); trbflags = trb->dwTrb3; if (XHCI_TRB_3_TYPE_GET(trbflags) != XHCI_TRB_TYPE_LINK && (trbflags & XHCI_TRB_3_CYCLE_BIT) != (ccs & XHCI_TRB_3_CYCLE_BIT)) { DPRINTF(("Cycle-bit changed trbflags %x, ccs %x", trbflags & XHCI_TRB_3_CYCLE_BIT, ccs)); break; } xfer_block = NULL; switch (XHCI_TRB_3_TYPE_GET(trbflags)) { case XHCI_TRB_TYPE_LINK: if (trb->dwTrb3 & XHCI_TRB_3_TC_BIT) ccs ^= 0x1; xfer_block = usb_data_xfer_append(xfer, NULL, 0, (void *)addr, ccs); xfer_block->processed = 1; break; case XHCI_TRB_TYPE_SETUP_STAGE: if ((trbflags & XHCI_TRB_3_IDT_BIT) == 0 || XHCI_TRB_2_BYTES_GET(trb->dwTrb2) != 8) { DPRINTF(("pci_xhci: invalid setup trb")); err = XHCI_TRB_ERROR_TRB; goto errout; } setup_trb = trb; val = trb->qwTrb0; if (!xfer->ureq) xfer->ureq = malloc( sizeof(struct usb_device_request)); memcpy(xfer->ureq, &val, sizeof(struct usb_device_request)); xfer_block = usb_data_xfer_append(xfer, NULL, 0, (void *)addr, ccs); xfer_block->processed = 1; break; case XHCI_TRB_TYPE_NORMAL: case XHCI_TRB_TYPE_ISOCH: if (setup_trb != NULL) { DPRINTF(("pci_xhci: trb not supposed to be in " "ctl scope")); err = XHCI_TRB_ERROR_TRB; goto errout; } /* fall through */ case XHCI_TRB_TYPE_DATA_STAGE: xfer_block = usb_data_xfer_append(xfer, (void *)(trbflags & XHCI_TRB_3_IDT_BIT ? &trb->qwTrb0 : XHCI_GADDR(sc, trb->qwTrb0)), trb->dwTrb2 & 0x1FFFF, (void *)addr, ccs); break; case XHCI_TRB_TYPE_STATUS_STAGE: xfer_block = usb_data_xfer_append(xfer, NULL, 0, (void *)addr, ccs); break; case XHCI_TRB_TYPE_NOOP: xfer_block = usb_data_xfer_append(xfer, NULL, 0, (void *)addr, ccs); xfer_block->processed = 1; break; case XHCI_TRB_TYPE_EVENT_DATA: xfer_block = usb_data_xfer_append(xfer, NULL, 0, (void *)addr, ccs); if ((epid > 1) && (trbflags & XHCI_TRB_3_IOC_BIT)) { xfer_block->processed = 1; } break; default: DPRINTF(("pci_xhci: handle xfer unexpected trb type " "0x%x", XHCI_TRB_3_TYPE_GET(trbflags))); err = XHCI_TRB_ERROR_TRB; goto errout; } trb = pci_xhci_trb_next(sc, trb, &addr); DPRINTF(("pci_xhci: next trb: 0x%lx", (uint64_t)trb)); if (xfer_block) { xfer_block->trbnext = addr; xfer_block->streamid = streamid; } if (!setup_trb && !(trbflags & XHCI_TRB_3_CHAIN_BIT) && XHCI_TRB_3_TYPE_GET(trbflags) != XHCI_TRB_TYPE_LINK) { break; } /* handle current batch that requires interrupt on complete */ if (trbflags & XHCI_TRB_3_IOC_BIT) { DPRINTF(("pci_xhci: trb IOC bit set")); if (epid == 1) do_retry = 1; break; } } DPRINTF(("pci_xhci[%d]: xfer->ndata %u", __LINE__, xfer->ndata)); if (xfer->ndata <= 0) goto errout; if (epid == 1) { int usberr; if (dev->dev_ue->ue_request != NULL) usberr = dev->dev_ue->ue_request(dev->dev_sc, xfer); else usberr = USB_ERR_NOT_STARTED; err = USB_TO_XHCI_ERR(usberr); if (err == XHCI_TRB_ERROR_SUCCESS || err == XHCI_TRB_ERROR_STALL || err == XHCI_TRB_ERROR_SHORT_PKT) { err = pci_xhci_xfer_complete(sc, xfer, slot, epid, &do_intr); if (err != XHCI_TRB_ERROR_SUCCESS) do_retry = 0; } } else { /* handle data transfer */ pci_xhci_try_usb_xfer(sc, dev, devep, ep_ctx, slot, epid); err = XHCI_TRB_ERROR_SUCCESS; } errout: if (err == XHCI_TRB_ERROR_EV_RING_FULL) DPRINTF(("pci_xhci[%d]: event ring full", __LINE__)); if (!do_retry) USB_DATA_XFER_UNLOCK(xfer); if (do_intr) pci_xhci_assert_interrupt(sc); if (do_retry) { USB_DATA_XFER_RESET(xfer); DPRINTF(("pci_xhci[%d]: retry:continuing with next TRBs", __LINE__)); goto retry; } if (epid == 1) USB_DATA_XFER_RESET(xfer); return (err); } static void pci_xhci_device_doorbell(struct pci_xhci_softc *sc, uint32_t slot, uint32_t epid, uint32_t streamid) { struct pci_xhci_dev_emu *dev; struct pci_xhci_dev_ep *devep; struct xhci_dev_ctx *dev_ctx; struct xhci_endp_ctx *ep_ctx; struct pci_xhci_trb_ring *sctx_tr; struct xhci_trb *trb; uint64_t ringaddr; uint32_t ccs; int error; DPRINTF(("pci_xhci doorbell slot %u epid %u stream %u", slot, epid, streamid)); if (slot == 0 || slot > XHCI_MAX_SLOTS) { DPRINTF(("pci_xhci: invalid doorbell slot %u", slot)); return; } if (epid == 0 || epid >= XHCI_MAX_ENDPOINTS) { DPRINTF(("pci_xhci: invalid endpoint %u", epid)); return; } dev = XHCI_SLOTDEV_PTR(sc, slot); devep = &dev->eps[epid]; dev_ctx = pci_xhci_get_dev_ctx(sc, slot); if (!dev_ctx) { return; } ep_ctx = &dev_ctx->ctx_ep[epid]; sctx_tr = NULL; DPRINTF(("pci_xhci: device doorbell ep[%u] %08x %08x %016lx %08x", epid, ep_ctx->dwEpCtx0, ep_ctx->dwEpCtx1, ep_ctx->qwEpCtx2, ep_ctx->dwEpCtx4)); if (ep_ctx->qwEpCtx2 == 0) return; /* handle pending transfers */ if (devep->ep_xfer->ndata > 0) { #ifndef __FreeBSD__ USB_DATA_XFER_LOCK(devep->ep_xfer); #endif pci_xhci_try_usb_xfer(sc, dev, devep, ep_ctx, slot, epid); #ifndef __FreeBSD__ USB_DATA_XFER_UNLOCK(devep->ep_xfer); #endif return; } /* get next trb work item */ if (devep->ep_MaxPStreams != 0) { /* * Stream IDs of 0, 65535 (any stream), and 65534 * (prime) are invalid. */ if (streamid == 0 || streamid == 65534 || streamid == 65535) { DPRINTF(("pci_xhci: invalid stream %u", streamid)); return; } error = pci_xhci_find_stream(sc, ep_ctx, devep, streamid); if (error != XHCI_TRB_ERROR_SUCCESS) { DPRINTF(("pci_xhci: invalid stream %u: %d", streamid, error)); return; } sctx_tr = &devep->ep_sctx_trbs[streamid]; ringaddr = sctx_tr->ringaddr; ccs = sctx_tr->ccs; trb = XHCI_GADDR(sc, sctx_tr->ringaddr & ~0xFUL); DPRINTF(("doorbell, stream %u, ccs %lx, trb ccs %x", streamid, ep_ctx->qwEpCtx2 & XHCI_TRB_3_CYCLE_BIT, trb->dwTrb3 & XHCI_TRB_3_CYCLE_BIT)); } else { if (streamid != 0) { DPRINTF(("pci_xhci: invalid stream %u", streamid)); return; } ringaddr = devep->ep_ringaddr; ccs = devep->ep_ccs; trb = devep->ep_tr; DPRINTF(("doorbell, ccs %lx, trb ccs %x", ep_ctx->qwEpCtx2 & XHCI_TRB_3_CYCLE_BIT, trb->dwTrb3 & XHCI_TRB_3_CYCLE_BIT)); } if (XHCI_TRB_3_TYPE_GET(trb->dwTrb3) == 0) { DPRINTF(("pci_xhci: ring %lx trb[%lx] EP %u is RESERVED?", ep_ctx->qwEpCtx2, devep->ep_ringaddr, epid)); return; } pci_xhci_handle_transfer(sc, dev, devep, ep_ctx, trb, slot, epid, ringaddr, ccs, streamid); } static void pci_xhci_dbregs_write(struct pci_xhci_softc *sc, uint64_t offset, uint64_t value) { offset = (offset - sc->dboff) / sizeof(uint32_t); DPRINTF(("pci_xhci: doorbell write offset 0x%lx: 0x%lx", offset, value)); if (XHCI_HALTED(sc)) { DPRINTF(("pci_xhci: controller halted")); return; } if (offset == 0) pci_xhci_complete_commands(sc); else if (sc->portregs != NULL) pci_xhci_device_doorbell(sc, offset, XHCI_DB_TARGET_GET(value), XHCI_DB_SID_GET(value)); } static void pci_xhci_rtsregs_write(struct pci_xhci_softc *sc, uint64_t offset, uint64_t value) { struct pci_xhci_rtsregs *rts; offset -= sc->rtsoff; if (offset == 0) { DPRINTF(("pci_xhci attempted write to MFINDEX")); return; } DPRINTF(("pci_xhci: runtime regs write offset 0x%lx: 0x%lx", offset, value)); offset -= 0x20; /* start of intrreg */ rts = &sc->rtsregs; switch (offset) { case 0x00: if (value & XHCI_IMAN_INTR_PEND) rts->intrreg.iman &= ~XHCI_IMAN_INTR_PEND; rts->intrreg.iman = (value & XHCI_IMAN_INTR_ENA) | (rts->intrreg.iman & XHCI_IMAN_INTR_PEND); if (!(value & XHCI_IMAN_INTR_ENA)) pci_xhci_deassert_interrupt(sc); break; case 0x04: rts->intrreg.imod = value; break; case 0x08: rts->intrreg.erstsz = value & 0xFFFF; break; case 0x10: /* ERSTBA low bits */ rts->intrreg.erstba = MASK_64_HI(sc->rtsregs.intrreg.erstba) | (value & ~0x3F); break; case 0x14: /* ERSTBA high bits */ rts->intrreg.erstba = (value << 32) | MASK_64_LO(sc->rtsregs.intrreg.erstba); rts->erstba_p = XHCI_GADDR(sc, sc->rtsregs.intrreg.erstba & ~0x3FUL); rts->erst_p = XHCI_GADDR(sc, sc->rtsregs.erstba_p->qwEvrsTablePtr & ~0x3FUL); rts->er_enq_idx = 0; rts->er_events_cnt = 0; DPRINTF(("pci_xhci: wr erstba erst (%p) ptr 0x%lx, sz %u", rts->erstba_p, rts->erstba_p->qwEvrsTablePtr, rts->erstba_p->dwEvrsTableSize)); break; case 0x18: /* ERDP low bits */ rts->intrreg.erdp = MASK_64_HI(sc->rtsregs.intrreg.erdp) | (rts->intrreg.erdp & XHCI_ERDP_LO_BUSY) | (value & ~0xF); if (value & XHCI_ERDP_LO_BUSY) { rts->intrreg.erdp &= ~XHCI_ERDP_LO_BUSY; rts->intrreg.iman &= ~XHCI_IMAN_INTR_PEND; } rts->er_deq_seg = XHCI_ERDP_LO_SINDEX(value); break; case 0x1C: /* ERDP high bits */ rts->intrreg.erdp = (value << 32) | MASK_64_LO(sc->rtsregs.intrreg.erdp); if (rts->er_events_cnt > 0) { uint64_t erdp; int erdp_i; erdp = rts->intrreg.erdp & ~0xF; erdp_i = (erdp - rts->erstba_p->qwEvrsTablePtr) / sizeof(struct xhci_trb); if (erdp_i <= rts->er_enq_idx) rts->er_events_cnt = rts->er_enq_idx - erdp_i; else rts->er_events_cnt = rts->erstba_p->dwEvrsTableSize - (erdp_i - rts->er_enq_idx); DPRINTF(("pci_xhci: erdp 0x%lx, events cnt %u", erdp, rts->er_events_cnt)); } break; default: DPRINTF(("pci_xhci attempted write to RTS offset 0x%lx", offset)); break; } } static uint64_t pci_xhci_portregs_read(struct pci_xhci_softc *sc, uint64_t offset) { struct pci_xhci_portregs *portregs; int port; uint32_t reg; if (sc->portregs == NULL) return (0); port = (offset - XHCI_PORTREGS_PORT0) / XHCI_PORTREGS_SETSZ; offset = (offset - XHCI_PORTREGS_PORT0) % XHCI_PORTREGS_SETSZ; if (port > XHCI_MAX_DEVS) { DPRINTF(("pci_xhci: portregs_read port %d >= XHCI_MAX_DEVS", port)); /* return default value for unused port */ return (XHCI_PS_SPEED_SET(3)); } portregs = XHCI_PORTREG_PTR(sc, port); switch (offset) { case 0: reg = portregs->portsc; break; case 4: reg = portregs->portpmsc; break; case 8: reg = portregs->portli; break; case 12: reg = portregs->porthlpmc; break; default: DPRINTF(("pci_xhci: unaligned portregs read offset %#lx", offset)); reg = 0xffffffff; break; } DPRINTF(("pci_xhci: portregs read offset 0x%lx port %u -> 0x%x", offset, port, reg)); return (reg); } static void pci_xhci_hostop_write(struct pci_xhci_softc *sc, uint64_t offset, uint64_t value) { offset -= XHCI_CAPLEN; if (offset < 0x400) DPRINTF(("pci_xhci: hostop write offset 0x%lx: 0x%lx", offset, value)); switch (offset) { case XHCI_USBCMD: sc->opregs.usbcmd = pci_xhci_usbcmd_write(sc, value & 0x3F0F); break; case XHCI_USBSTS: /* clear bits on write */ sc->opregs.usbsts &= ~(value & (XHCI_STS_HSE|XHCI_STS_EINT|XHCI_STS_PCD|XHCI_STS_SSS| XHCI_STS_RSS|XHCI_STS_SRE|XHCI_STS_CNR)); break; case XHCI_PAGESIZE: /* read only */ break; case XHCI_DNCTRL: sc->opregs.dnctrl = value & 0xFFFF; break; case XHCI_CRCR_LO: if (sc->opregs.crcr & XHCI_CRCR_LO_CRR) { sc->opregs.crcr &= ~(XHCI_CRCR_LO_CS|XHCI_CRCR_LO_CA); sc->opregs.crcr |= value & (XHCI_CRCR_LO_CS|XHCI_CRCR_LO_CA); } else { sc->opregs.crcr = MASK_64_HI(sc->opregs.crcr) | (value & (0xFFFFFFC0 | XHCI_CRCR_LO_RCS)); } break; case XHCI_CRCR_HI: if (!(sc->opregs.crcr & XHCI_CRCR_LO_CRR)) { sc->opregs.crcr = MASK_64_LO(sc->opregs.crcr) | (value << 32); sc->opregs.cr_p = XHCI_GADDR(sc, sc->opregs.crcr & ~0xF); } if (sc->opregs.crcr & XHCI_CRCR_LO_CS) { /* Stop operation of Command Ring */ } if (sc->opregs.crcr & XHCI_CRCR_LO_CA) { /* Abort command */ } break; case XHCI_DCBAAP_LO: sc->opregs.dcbaap = MASK_64_HI(sc->opregs.dcbaap) | (value & 0xFFFFFFC0); break; case XHCI_DCBAAP_HI: sc->opregs.dcbaap = MASK_64_LO(sc->opregs.dcbaap) | (value << 32); sc->opregs.dcbaa_p = XHCI_GADDR(sc, sc->opregs.dcbaap & ~0x3FUL); DPRINTF(("pci_xhci: opregs dcbaap = 0x%lx (vaddr 0x%lx)", sc->opregs.dcbaap, (uint64_t)sc->opregs.dcbaa_p)); break; case XHCI_CONFIG: sc->opregs.config = value & 0x03FF; break; default: if (offset >= 0x400) pci_xhci_portregs_write(sc, offset, value); break; } } static void pci_xhci_write(struct pci_devinst *pi, int baridx, uint64_t offset, int size __unused, uint64_t value) { struct pci_xhci_softc *sc; sc = pi->pi_arg; assert(baridx == 0); pthread_mutex_lock(&sc->mtx); if (offset < XHCI_CAPLEN) /* read only registers */ WPRINTF(("pci_xhci: write RO-CAPs offset %ld", offset)); else if (offset < sc->dboff) pci_xhci_hostop_write(sc, offset, value); else if (offset < sc->rtsoff) pci_xhci_dbregs_write(sc, offset, value); else if (offset < sc->regsend) pci_xhci_rtsregs_write(sc, offset, value); else WPRINTF(("pci_xhci: write invalid offset %ld", offset)); pthread_mutex_unlock(&sc->mtx); } static uint64_t pci_xhci_hostcap_read(struct pci_xhci_softc *sc, uint64_t offset) { uint64_t value; switch (offset) { case XHCI_CAPLENGTH: /* 0x00 */ value = sc->caplength; break; case XHCI_HCSPARAMS1: /* 0x04 */ value = sc->hcsparams1; break; case XHCI_HCSPARAMS2: /* 0x08 */ value = sc->hcsparams2; break; case XHCI_HCSPARAMS3: /* 0x0C */ value = sc->hcsparams3; break; case XHCI_HCSPARAMS0: /* 0x10 */ value = sc->hccparams1; break; case XHCI_DBOFF: /* 0x14 */ value = sc->dboff; break; case XHCI_RTSOFF: /* 0x18 */ value = sc->rtsoff; break; case XHCI_HCCPRAMS2: /* 0x1C */ value = sc->hccparams2; break; default: value = 0; break; } DPRINTF(("pci_xhci: hostcap read offset 0x%lx -> 0x%lx", offset, value)); return (value); } static uint64_t pci_xhci_hostop_read(struct pci_xhci_softc *sc, uint64_t offset) { uint64_t value; offset = (offset - XHCI_CAPLEN); switch (offset) { case XHCI_USBCMD: /* 0x00 */ value = sc->opregs.usbcmd; break; case XHCI_USBSTS: /* 0x04 */ value = sc->opregs.usbsts; break; case XHCI_PAGESIZE: /* 0x08 */ value = sc->opregs.pgsz; break; case XHCI_DNCTRL: /* 0x14 */ value = sc->opregs.dnctrl; break; case XHCI_CRCR_LO: /* 0x18 */ value = sc->opregs.crcr & XHCI_CRCR_LO_CRR; break; case XHCI_CRCR_HI: /* 0x1C */ value = 0; break; case XHCI_DCBAAP_LO: /* 0x30 */ value = sc->opregs.dcbaap & 0xFFFFFFFF; break; case XHCI_DCBAAP_HI: /* 0x34 */ value = (sc->opregs.dcbaap >> 32) & 0xFFFFFFFF; break; case XHCI_CONFIG: /* 0x38 */ value = sc->opregs.config; break; default: if (offset >= 0x400) value = pci_xhci_portregs_read(sc, offset); else value = 0; break; } if (offset < 0x400) DPRINTF(("pci_xhci: hostop read offset 0x%lx -> 0x%lx", offset, value)); return (value); } static uint64_t pci_xhci_dbregs_read(struct pci_xhci_softc *sc __unused, uint64_t offset __unused) { /* read doorbell always returns 0 */ return (0); } static uint64_t pci_xhci_rtsregs_read(struct pci_xhci_softc *sc, uint64_t offset) { uint32_t value; offset -= sc->rtsoff; value = 0; if (offset == XHCI_MFINDEX) { value = sc->rtsregs.mfindex; } else if (offset >= 0x20) { int item; uint32_t *p; offset -= 0x20; item = offset % 32; assert(offset < sizeof(sc->rtsregs.intrreg)); p = &sc->rtsregs.intrreg.iman; p += item / sizeof(uint32_t); value = *p; } DPRINTF(("pci_xhci: rtsregs read offset 0x%lx -> 0x%x", offset, value)); return (value); } static uint64_t pci_xhci_xecp_read(struct pci_xhci_softc *sc, uint64_t offset) { uint32_t value; offset -= sc->regsend; value = 0; switch (offset) { case 0: /* rev major | rev minor | next-cap | cap-id */ value = (0x02 << 24) | (4 << 8) | XHCI_ID_PROTOCOLS; break; case 4: /* name string = "USB" */ value = 0x20425355; break; case 8: /* psic | proto-defined | compat # | compat offset */ value = ((XHCI_MAX_DEVS/2) << 8) | sc->usb2_port_start; break; case 12: break; case 16: /* rev major | rev minor | next-cap | cap-id */ value = (0x03 << 24) | XHCI_ID_PROTOCOLS; break; case 20: /* name string = "USB" */ value = 0x20425355; break; case 24: /* psic | proto-defined | compat # | compat offset */ value = ((XHCI_MAX_DEVS/2) << 8) | sc->usb3_port_start; break; case 28: break; default: DPRINTF(("pci_xhci: xecp invalid offset 0x%lx", offset)); break; } DPRINTF(("pci_xhci: xecp read offset 0x%lx -> 0x%x", offset, value)); return (value); } static uint64_t pci_xhci_read(struct pci_devinst *pi, int baridx, uint64_t offset, int size) { struct pci_xhci_softc *sc; uint32_t value; sc = pi->pi_arg; assert(baridx == 0); pthread_mutex_lock(&sc->mtx); if (offset < XHCI_CAPLEN) value = pci_xhci_hostcap_read(sc, offset); else if (offset < sc->dboff) value = pci_xhci_hostop_read(sc, offset); else if (offset < sc->rtsoff) value = pci_xhci_dbregs_read(sc, offset); else if (offset < sc->regsend) value = pci_xhci_rtsregs_read(sc, offset); else if (offset < (sc->regsend + 4*32)) value = pci_xhci_xecp_read(sc, offset); else { value = 0; WPRINTF(("pci_xhci: read invalid offset %ld", offset)); } pthread_mutex_unlock(&sc->mtx); switch (size) { case 1: value &= 0xFF; break; case 2: value &= 0xFFFF; break; case 4: value &= 0xFFFFFFFF; break; } return (value); } static void pci_xhci_reset_port(struct pci_xhci_softc *sc, int portn, int warm) { struct pci_xhci_portregs *port; struct pci_xhci_dev_emu *dev; struct xhci_trb evtrb; int error; assert(portn <= XHCI_MAX_DEVS); DPRINTF(("xhci reset port %d", portn)); port = XHCI_PORTREG_PTR(sc, portn); dev = XHCI_DEVINST_PTR(sc, portn); if (dev) { port->portsc &= ~(XHCI_PS_PLS_MASK | XHCI_PS_PR | XHCI_PS_PRC); port->portsc |= XHCI_PS_PED | XHCI_PS_SPEED_SET(dev->dev_ue->ue_usbspeed); if (warm && dev->dev_ue->ue_usbver == 3) { port->portsc |= XHCI_PS_WRC; } if ((port->portsc & XHCI_PS_PRC) == 0) { port->portsc |= XHCI_PS_PRC; pci_xhci_set_evtrb(&evtrb, portn, XHCI_TRB_ERROR_SUCCESS, XHCI_TRB_EVENT_PORT_STS_CHANGE); error = pci_xhci_insert_event(sc, &evtrb, 1); if (error != XHCI_TRB_ERROR_SUCCESS) DPRINTF(("xhci reset port insert event " "failed")); } } } static void pci_xhci_init_port(struct pci_xhci_softc *sc, int portn) { struct pci_xhci_portregs *port; struct pci_xhci_dev_emu *dev; port = XHCI_PORTREG_PTR(sc, portn); dev = XHCI_DEVINST_PTR(sc, portn); if (dev) { port->portsc = XHCI_PS_CCS | /* connected */ XHCI_PS_PP; /* port power */ if (dev->dev_ue->ue_usbver == 2) { port->portsc |= XHCI_PS_PLS_SET(UPS_PORT_LS_POLL) | XHCI_PS_SPEED_SET(dev->dev_ue->ue_usbspeed); } else { port->portsc |= XHCI_PS_PLS_SET(UPS_PORT_LS_U0) | XHCI_PS_PED | /* enabled */ XHCI_PS_SPEED_SET(dev->dev_ue->ue_usbspeed); } DPRINTF(("Init port %d 0x%x", portn, port->portsc)); } else { port->portsc = XHCI_PS_PLS_SET(UPS_PORT_LS_RX_DET) | XHCI_PS_PP; DPRINTF(("Init empty port %d 0x%x", portn, port->portsc)); } } static int pci_xhci_dev_intr(struct usb_hci *hci, int epctx) { struct pci_xhci_dev_emu *dev; struct xhci_dev_ctx *dev_ctx; struct xhci_trb evtrb; struct pci_xhci_softc *sc; struct pci_xhci_portregs *p; struct xhci_endp_ctx *ep_ctx; int error = 0; int dir_in; int epid; dir_in = epctx & 0x80; epid = epctx & ~0x80; /* HW endpoint contexts are 0-15; convert to epid based on dir */ epid = (epid * 2) + (dir_in ? 1 : 0); assert(epid >= 1 && epid <= 31); dev = hci->hci_sc; sc = dev->xsc; /* check if device is ready; OS has to initialise it */ if (sc->rtsregs.erstba_p == NULL || (sc->opregs.usbcmd & XHCI_CMD_RS) == 0 || dev->dev_ctx == NULL) return (0); p = XHCI_PORTREG_PTR(sc, hci->hci_port); /* raise event if link U3 (suspended) state */ if (XHCI_PS_PLS_GET(p->portsc) == 3) { p->portsc &= ~XHCI_PS_PLS_MASK; p->portsc |= XHCI_PS_PLS_SET(UPS_PORT_LS_RESUME); if ((p->portsc & XHCI_PS_PLC) != 0) return (0); p->portsc |= XHCI_PS_PLC; pci_xhci_set_evtrb(&evtrb, hci->hci_port, XHCI_TRB_ERROR_SUCCESS, XHCI_TRB_EVENT_PORT_STS_CHANGE); error = pci_xhci_insert_event(sc, &evtrb, 0); if (error != XHCI_TRB_ERROR_SUCCESS) goto done; } dev_ctx = dev->dev_ctx; ep_ctx = &dev_ctx->ctx_ep[epid]; if ((ep_ctx->dwEpCtx0 & 0x7) == XHCI_ST_EPCTX_DISABLED) { DPRINTF(("xhci device interrupt on disabled endpoint %d", epid)); return (0); } DPRINTF(("xhci device interrupt on endpoint %d", epid)); pci_xhci_device_doorbell(sc, hci->hci_port, epid, 0); done: return (error); } static int pci_xhci_dev_event(struct usb_hci *hci, enum hci_usbev evid __unused, void *param __unused) { DPRINTF(("xhci device event port %d", hci->hci_port)); return (0); } /* * Each controller contains a "slot" node which contains a list of * child nodes each of which is a device. Each slot node's name * corresponds to a specific controller slot. These nodes * contain a "device" variable identifying the device model of the * USB device. For example: * * pci.0.1.0 * .device="xhci" * .slot * .1 * .device="tablet" */ static int pci_xhci_legacy_config(nvlist_t *nvl, const char *opts) { char node_name[16]; nvlist_t *slots_nvl, *slot_nvl; char *cp, *opt, *str, *tofree; int slot; if (opts == NULL) return (0); slots_nvl = create_relative_config_node(nvl, "slot"); slot = 1; tofree = str = strdup(opts); while ((opt = strsep(&str, ",")) != NULL) { /* device[=] */ cp = strchr(opt, '='); if (cp != NULL) { *cp = '\0'; cp++; } snprintf(node_name, sizeof(node_name), "%d", slot); slot++; slot_nvl = create_relative_config_node(slots_nvl, node_name); set_config_value_node(slot_nvl, "device", opt); /* * NB: Given that we split on commas above, the legacy * format only supports a single option. */ if (cp != NULL && *cp != '\0') pci_parse_legacy_config(slot_nvl, cp); } free(tofree); return (0); } static int pci_xhci_parse_devices(struct pci_xhci_softc *sc, nvlist_t *nvl) { struct pci_xhci_dev_emu *dev; struct usb_devemu *ue; const nvlist_t *slots_nvl, *slot_nvl; const char *name, *device; char *cp; void *devsc, *cookie; long slot; int type, usb3_port, usb2_port, i, ndevices; usb3_port = sc->usb3_port_start; usb2_port = sc->usb2_port_start; sc->devices = calloc(XHCI_MAX_DEVS, sizeof(struct pci_xhci_dev_emu *)); sc->slots = calloc(XHCI_MAX_SLOTS, sizeof(struct pci_xhci_dev_emu *)); ndevices = 0; slots_nvl = find_relative_config_node(nvl, "slot"); if (slots_nvl == NULL) goto portsfinal; cookie = NULL; while ((name = nvlist_next(slots_nvl, &type, &cookie)) != NULL) { if (usb2_port == ((sc->usb2_port_start) + XHCI_MAX_DEVS/2) || usb3_port == ((sc->usb3_port_start) + XHCI_MAX_DEVS/2)) { WPRINTF(("pci_xhci max number of USB 2 or 3 " "devices reached, max %d", XHCI_MAX_DEVS/2)); goto bad; } if (type != NV_TYPE_NVLIST) { EPRINTLN( "pci_xhci: config variable '%s' under slot node", name); goto bad; } slot = strtol(name, &cp, 0); if (*cp != '\0' || slot <= 0 || slot > XHCI_MAX_SLOTS) { EPRINTLN("pci_xhci: invalid slot '%s'", name); goto bad; } if (XHCI_SLOTDEV_PTR(sc, slot) != NULL) { EPRINTLN("pci_xhci: duplicate slot '%s'", name); goto bad; } slot_nvl = nvlist_get_nvlist(slots_nvl, name); device = get_config_value_node(slot_nvl, "device"); if (device == NULL) { EPRINTLN( "pci_xhci: missing \"device\" value for slot '%s'", name); goto bad; } ue = usb_emu_finddev(device); if (ue == NULL) { EPRINTLN("pci_xhci: unknown device model \"%s\"", device); goto bad; } DPRINTF(("pci_xhci adding device %s", device)); dev = calloc(1, sizeof(struct pci_xhci_dev_emu)); dev->xsc = sc; dev->hci.hci_sc = dev; dev->hci.hci_intr = pci_xhci_dev_intr; dev->hci.hci_event = pci_xhci_dev_event; if (ue->ue_usbver == 2) { if (usb2_port == sc->usb2_port_start + XHCI_MAX_DEVS / 2) { WPRINTF(("pci_xhci max number of USB 2 devices " "reached, max %d", XHCI_MAX_DEVS / 2)); goto bad; } dev->hci.hci_port = usb2_port; usb2_port++; } else { if (usb3_port == sc->usb3_port_start + XHCI_MAX_DEVS / 2) { WPRINTF(("pci_xhci max number of USB 3 devices " "reached, max %d", XHCI_MAX_DEVS / 2)); goto bad; } dev->hci.hci_port = usb3_port; usb3_port++; } XHCI_DEVINST_PTR(sc, dev->hci.hci_port) = dev; dev->hci.hci_address = 0; devsc = ue->ue_init(&dev->hci, nvl); if (devsc == NULL) { goto bad; } dev->dev_ue = ue; dev->dev_sc = devsc; XHCI_SLOTDEV_PTR(sc, slot) = dev; ndevices++; } portsfinal: sc->portregs = calloc(XHCI_MAX_DEVS, sizeof(struct pci_xhci_portregs)); if (ndevices > 0) { for (i = 1; i <= XHCI_MAX_DEVS; i++) { pci_xhci_init_port(sc, i); } } else { WPRINTF(("pci_xhci no USB devices configured")); } return (0); bad: for (i = 1; i <= XHCI_MAX_DEVS; i++) { free(XHCI_DEVINST_PTR(sc, i)); } free(sc->devices); free(sc->slots); return (-1); } static int pci_xhci_init(struct pci_devinst *pi, nvlist_t *nvl) { struct pci_xhci_softc *sc; int error; #ifndef __FreeBSD__ if (get_config_bool_default("xhci.debug", false)) xhci_debug = 1; #endif if (xhci_in_use) { WPRINTF(("pci_xhci controller already defined")); return (-1); } xhci_in_use = 1; sc = calloc(1, sizeof(struct pci_xhci_softc)); pi->pi_arg = sc; sc->xsc_pi = pi; sc->usb2_port_start = (XHCI_MAX_DEVS/2) + 1; sc->usb3_port_start = 1; /* discover devices */ error = pci_xhci_parse_devices(sc, nvl); if (error < 0) goto done; else error = 0; sc->caplength = XHCI_SET_CAPLEN(XHCI_CAPLEN) | XHCI_SET_HCIVERSION(0x0100); sc->hcsparams1 = XHCI_SET_HCSP1_MAXPORTS(XHCI_MAX_DEVS) | XHCI_SET_HCSP1_MAXINTR(1) | /* interrupters */ XHCI_SET_HCSP1_MAXSLOTS(XHCI_MAX_SLOTS); sc->hcsparams2 = XHCI_SET_HCSP2_ERSTMAX(XHCI_ERST_MAX) | XHCI_SET_HCSP2_IST(0x04); sc->hcsparams3 = 0; /* no latency */ sc->hccparams1 = XHCI_SET_HCCP1_AC64(1) | /* 64-bit addrs */ XHCI_SET_HCCP1_NSS(1) | /* no 2nd-streams */ XHCI_SET_HCCP1_SPC(1) | /* short packet */ XHCI_SET_HCCP1_MAXPSA(XHCI_STREAMS_MAX); sc->hccparams2 = XHCI_SET_HCCP2_LEC(1) | XHCI_SET_HCCP2_U3C(1); sc->dboff = XHCI_SET_DOORBELL(XHCI_CAPLEN + XHCI_PORTREGS_START + XHCI_MAX_DEVS * sizeof(struct pci_xhci_portregs)); /* dboff must be 32-bit aligned */ if (sc->dboff & 0x3) sc->dboff = (sc->dboff + 0x3) & ~0x3; /* rtsoff must be 32-bytes aligned */ sc->rtsoff = XHCI_SET_RTSOFFSET(sc->dboff + (XHCI_MAX_SLOTS+1) * 32); if (sc->rtsoff & 0x1F) sc->rtsoff = (sc->rtsoff + 0x1F) & ~0x1F; DPRINTF(("pci_xhci dboff: 0x%x, rtsoff: 0x%x", sc->dboff, sc->rtsoff)); sc->opregs.usbsts = XHCI_STS_HCH; sc->opregs.pgsz = XHCI_PAGESIZE_4K; pci_xhci_reset(sc); sc->regsend = sc->rtsoff + 0x20 + 32; /* only 1 intrpter */ /* * Set extended capabilities pointer to be after regsend; * value of xecp field is 32-bit offset. */ sc->hccparams1 |= XHCI_SET_HCCP1_XECP(sc->regsend/4); pci_set_cfgdata16(pi, PCIR_DEVICE, 0x1E31); pci_set_cfgdata16(pi, PCIR_VENDOR, 0x8086); pci_set_cfgdata8(pi, PCIR_CLASS, PCIC_SERIALBUS); pci_set_cfgdata8(pi, PCIR_SUBCLASS, PCIS_SERIALBUS_USB); pci_set_cfgdata8(pi, PCIR_PROGIF,PCIP_SERIALBUS_USB_XHCI); pci_set_cfgdata8(pi, PCI_USBREV, PCI_USB_REV_3_0); pci_emul_add_msicap(pi, 1); /* regsend + xecp registers */ pci_emul_alloc_bar(pi, 0, PCIBAR_MEM32, sc->regsend + 4*32); DPRINTF(("pci_xhci pci_emu_alloc: %d", sc->regsend + 4*32)); pci_lintr_request(pi); pthread_mutex_init(&sc->mtx, NULL); done: if (error) { free(sc); } return (error); } static const struct pci_devemu pci_de_xhci = { .pe_emu = "xhci", .pe_init = pci_xhci_init, .pe_legacy_config = pci_xhci_legacy_config, .pe_barwrite = pci_xhci_write, .pe_barread = pci_xhci_read }; PCI_EMUL_SET(pci_de_xhci); /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2014 Leon Dang * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef _PCI_XHCI_H_ #define _PCI_XHCI_H_ #define PCI_USBREV 0x60 /* USB protocol revision */ enum { /* dsc_slotstate */ XHCI_ST_DISABLED, XHCI_ST_ENABLED, XHCI_ST_DEFAULT, XHCI_ST_ADDRESSED, XHCI_ST_CONFIGURED, XHCI_ST_MAX }; enum { XHCI_ST_SLCTX_DISABLED, XHCI_ST_SLCTX_DEFAULT, XHCI_ST_SLCTX_ADDRESSED, XHCI_ST_SLCTX_CONFIGURED }; enum { XHCI_ST_EPCTX_DISABLED, XHCI_ST_EPCTX_RUNNING, XHCI_ST_EPCTX_HALTED, XHCI_ST_EPCTX_STOPPED, XHCI_ST_EPCTX_ERROR }; #define XHCI_MAX_DEVICES MIN(USB_MAX_DEVICES, 128) #define XHCI_MAX_ENDPOINTS 32 /* hardcoded - do not change */ #define XHCI_MAX_SCRATCHPADS 32 #define XHCI_MAX_EVENTS (16 * 13) #define XHCI_MAX_COMMANDS (16 * 1) #define XHCI_MAX_RSEG 1 #define XHCI_MAX_TRANSFERS 4 #if USB_MAX_EP_STREAMS == 8 #define XHCI_MAX_STREAMS 8 #define XHCI_MAX_STREAMS_LOG 3 #elif USB_MAX_EP_STREAMS == 1 #define XHCI_MAX_STREAMS 1 #define XHCI_MAX_STREAMS_LOG 0 #else #error "The USB_MAX_EP_STREAMS value is not supported." #endif #define XHCI_DEV_CTX_ADDR_ALIGN 64 /* bytes */ #define XHCI_DEV_CTX_ALIGN 64 /* bytes */ #define XHCI_INPUT_CTX_ALIGN 64 /* bytes */ #define XHCI_SLOT_CTX_ALIGN 32 /* bytes */ #define XHCI_ENDP_CTX_ALIGN 32 /* bytes */ #define XHCI_STREAM_CTX_ALIGN 16 /* bytes */ #define XHCI_TRANS_RING_SEG_ALIGN 16 /* bytes */ #define XHCI_CMD_RING_SEG_ALIGN 64 /* bytes */ #define XHCI_EVENT_RING_SEG_ALIGN 64 /* bytes */ #define XHCI_SCRATCH_BUF_ARRAY_ALIGN 64 /* bytes */ #define XHCI_SCRATCH_BUFFER_ALIGN USB_PAGE_SIZE #define XHCI_TRB_ALIGN 16 /* bytes */ #define XHCI_TD_ALIGN 64 /* bytes */ #define XHCI_PAGE_SIZE 4096 /* bytes */ struct xhci_slot_ctx { uint32_t dwSctx0; #define XHCI_SCTX_0_ROUTE_SET(x) ((x) & 0xFFFFF) #define XHCI_SCTX_0_ROUTE_GET(x) ((x) & 0xFFFFF) #define XHCI_SCTX_0_SPEED_SET(x) (((x) & 0xF) << 20) #define XHCI_SCTX_0_SPEED_GET(x) (((x) >> 20) & 0xF) #define XHCI_SCTX_0_MTT_SET(x) (((x) & 0x1) << 25) #define XHCI_SCTX_0_MTT_GET(x) (((x) >> 25) & 0x1) #define XHCI_SCTX_0_HUB_SET(x) (((x) & 0x1) << 26) #define XHCI_SCTX_0_HUB_GET(x) (((x) >> 26) & 0x1) #define XHCI_SCTX_0_CTX_NUM_SET(x) (((x) & 0x1F) << 27) #define XHCI_SCTX_0_CTX_NUM_GET(x) (((x) >> 27) & 0x1F) uint32_t dwSctx1; #define XHCI_SCTX_1_MAX_EL_SET(x) ((x) & 0xFFFF) #define XHCI_SCTX_1_MAX_EL_GET(x) ((x) & 0xFFFF) #define XHCI_SCTX_1_RH_PORT_SET(x) (((x) & 0xFF) << 16) #define XHCI_SCTX_1_RH_PORT_GET(x) (((x) >> 16) & 0xFF) #define XHCI_SCTX_1_NUM_PORTS_SET(x) (((x) & 0xFF) << 24) #define XHCI_SCTX_1_NUM_PORTS_GET(x) (((x) >> 24) & 0xFF) uint32_t dwSctx2; #define XHCI_SCTX_2_TT_HUB_SID_SET(x) ((x) & 0xFF) #define XHCI_SCTX_2_TT_HUB_SID_GET(x) ((x) & 0xFF) #define XHCI_SCTX_2_TT_PORT_NUM_SET(x) (((x) & 0xFF) << 8) #define XHCI_SCTX_2_TT_PORT_NUM_GET(x) (((x) >> 8) & 0xFF) #define XHCI_SCTX_2_TT_THINK_TIME_SET(x) (((x) & 0x3) << 16) #define XHCI_SCTX_2_TT_THINK_TIME_GET(x) (((x) >> 16) & 0x3) #define XHCI_SCTX_2_IRQ_TARGET_SET(x) (((x) & 0x3FF) << 22) #define XHCI_SCTX_2_IRQ_TARGET_GET(x) (((x) >> 22) & 0x3FF) uint32_t dwSctx3; #define XHCI_SCTX_3_DEV_ADDR_SET(x) ((x) & 0xFF) #define XHCI_SCTX_3_DEV_ADDR_GET(x) ((x) & 0xFF) #define XHCI_SCTX_3_SLOT_STATE_SET(x) (((x) & 0x1F) << 27) #define XHCI_SCTX_3_SLOT_STATE_GET(x) (((x) >> 27) & 0x1F) uint32_t dwSctx4; uint32_t dwSctx5; uint32_t dwSctx6; uint32_t dwSctx7; }; struct xhci_endp_ctx { uint32_t dwEpCtx0; #define XHCI_EPCTX_0_EPSTATE_SET(x) ((x) & 0x7) #define XHCI_EPCTX_0_EPSTATE_GET(x) ((x) & 0x7) #define XHCI_EPCTX_0_MULT_SET(x) (((x) & 0x3) << 8) #define XHCI_EPCTX_0_MULT_GET(x) (((x) >> 8) & 0x3) #define XHCI_EPCTX_0_MAXP_STREAMS_SET(x) (((x) & 0x1F) << 10) #define XHCI_EPCTX_0_MAXP_STREAMS_GET(x) (((x) >> 10) & 0x1F) #define XHCI_EPCTX_0_LSA_SET(x) (((x) & 0x1) << 15) #define XHCI_EPCTX_0_LSA_GET(x) (((x) >> 15) & 0x1) #define XHCI_EPCTX_0_IVAL_SET(x) (((x) & 0xFF) << 16) #define XHCI_EPCTX_0_IVAL_GET(x) (((x) >> 16) & 0xFF) uint32_t dwEpCtx1; #define XHCI_EPCTX_1_CERR_SET(x) (((x) & 0x3) << 1) #define XHCI_EPCTX_1_CERR_GET(x) (((x) >> 1) & 0x3) #define XHCI_EPCTX_1_EPTYPE_SET(x) (((x) & 0x7) << 3) #define XHCI_EPCTX_1_EPTYPE_GET(x) (((x) >> 3) & 0x7) #define XHCI_EPCTX_1_HID_SET(x) (((x) & 0x1) << 7) #define XHCI_EPCTX_1_HID_GET(x) (((x) >> 7) & 0x1) #define XHCI_EPCTX_1_MAXB_SET(x) (((x) & 0xFF) << 8) #define XHCI_EPCTX_1_MAXB_GET(x) (((x) >> 8) & 0xFF) #define XHCI_EPCTX_1_MAXP_SIZE_SET(x) (((x) & 0xFFFF) << 16) #define XHCI_EPCTX_1_MAXP_SIZE_GET(x) (((x) >> 16) & 0xFFFF) uint64_t qwEpCtx2; #define XHCI_EPCTX_2_DCS_SET(x) ((x) & 0x1) #define XHCI_EPCTX_2_DCS_GET(x) ((x) & 0x1) #define XHCI_EPCTX_2_TR_DQ_PTR_MASK 0xFFFFFFFFFFFFFFF0U uint32_t dwEpCtx4; #define XHCI_EPCTX_4_AVG_TRB_LEN_SET(x) ((x) & 0xFFFF) #define XHCI_EPCTX_4_AVG_TRB_LEN_GET(x) ((x) & 0xFFFF) #define XHCI_EPCTX_4_MAX_ESIT_PAYLOAD_SET(x) (((x) & 0xFFFF) << 16) #define XHCI_EPCTX_4_MAX_ESIT_PAYLOAD_GET(x) (((x) >> 16) & 0xFFFF) uint32_t dwEpCtx5; uint32_t dwEpCtx6; uint32_t dwEpCtx7; }; struct xhci_input_ctx { #define XHCI_INCTX_NON_CTRL_MASK 0xFFFFFFFCU uint32_t dwInCtx0; #define XHCI_INCTX_0_DROP_MASK(n) (1U << (n)) uint32_t dwInCtx1; #define XHCI_INCTX_1_ADD_MASK(n) (1U << (n)) uint32_t dwInCtx2; uint32_t dwInCtx3; uint32_t dwInCtx4; uint32_t dwInCtx5; uint32_t dwInCtx6; uint32_t dwInCtx7; }; struct xhci_input_dev_ctx { struct xhci_input_ctx ctx_input; union { struct xhci_slot_ctx u_slot; struct xhci_endp_ctx u_ep[XHCI_MAX_ENDPOINTS]; } ctx_dev_slep; }; struct xhci_dev_ctx { union { struct xhci_slot_ctx u_slot; struct xhci_endp_ctx u_ep[XHCI_MAX_ENDPOINTS]; } ctx_dev_slep; } __aligned(XHCI_DEV_CTX_ALIGN); #define ctx_slot ctx_dev_slep.u_slot #define ctx_ep ctx_dev_slep.u_ep struct xhci_stream_ctx { uint64_t qwSctx0; #define XHCI_SCTX_0_DCS_GET(x) ((x) & 0x1) #define XHCI_SCTX_0_DCS_SET(x) ((x) & 0x1) #define XHCI_SCTX_0_SCT_SET(x) (((x) & 0x7) << 1) #define XHCI_SCTX_0_SCT_GET(x) (((x) >> 1) & 0x7) #define XHCI_SCTX_0_SCT_SEC_TR_RING 0x0 #define XHCI_SCTX_0_SCT_PRIM_TR_RING 0x1 #define XHCI_SCTX_0_SCT_PRIM_SSA_8 0x2 #define XHCI_SCTX_0_SCT_PRIM_SSA_16 0x3 #define XHCI_SCTX_0_SCT_PRIM_SSA_32 0x4 #define XHCI_SCTX_0_SCT_PRIM_SSA_64 0x5 #define XHCI_SCTX_0_SCT_PRIM_SSA_128 0x6 #define XHCI_SCTX_0_SCT_PRIM_SSA_256 0x7 #define XHCI_SCTX_0_TR_DQ_PTR_MASK 0xFFFFFFFFFFFFFFF0U uint32_t dwSctx2; uint32_t dwSctx3; }; struct xhci_trb { uint64_t qwTrb0; #define XHCI_TRB_0_DIR_IN_MASK (0x80ULL << 0) #define XHCI_TRB_0_WLENGTH_MASK (0xFFFFULL << 48) uint32_t dwTrb2; #define XHCI_TRB_2_ERROR_GET(x) (((x) >> 24) & 0xFF) #define XHCI_TRB_2_ERROR_SET(x) (((x) & 0xFF) << 24) #define XHCI_TRB_2_TDSZ_GET(x) (((x) >> 17) & 0x1F) #define XHCI_TRB_2_TDSZ_SET(x) (((x) & 0x1F) << 17) #define XHCI_TRB_2_REM_GET(x) ((x) & 0xFFFFFF) #define XHCI_TRB_2_REM_SET(x) ((x) & 0xFFFFFF) #define XHCI_TRB_2_BYTES_GET(x) ((x) & 0x1FFFF) #define XHCI_TRB_2_BYTES_SET(x) ((x) & 0x1FFFF) #define XHCI_TRB_2_IRQ_GET(x) (((x) >> 22) & 0x3FF) #define XHCI_TRB_2_IRQ_SET(x) (((x) & 0x3FF) << 22) #define XHCI_TRB_2_STREAM_GET(x) (((x) >> 16) & 0xFFFF) #define XHCI_TRB_2_STREAM_SET(x) (((x) & 0xFFFF) << 16) uint32_t dwTrb3; #define XHCI_TRB_3_TYPE_GET(x) (((x) >> 10) & 0x3F) #define XHCI_TRB_3_TYPE_SET(x) (((x) & 0x3F) << 10) #define XHCI_TRB_3_CYCLE_BIT (1U << 0) #define XHCI_TRB_3_TC_BIT (1U << 1) /* command ring only */ #define XHCI_TRB_3_ENT_BIT (1U << 1) /* transfer ring only */ #define XHCI_TRB_3_ISP_BIT (1U << 2) #define XHCI_TRB_3_ED_BIT (1U << 2) #define XHCI_TRB_3_NSNOOP_BIT (1U << 3) #define XHCI_TRB_3_CHAIN_BIT (1U << 4) #define XHCI_TRB_3_IOC_BIT (1U << 5) #define XHCI_TRB_3_IDT_BIT (1U << 6) #define XHCI_TRB_3_TBC_GET(x) (((x) >> 7) & 3) #define XHCI_TRB_3_TBC_SET(x) (((x) & 3) << 7) #define XHCI_TRB_3_BEI_BIT (1U << 9) #define XHCI_TRB_3_DCEP_BIT (1U << 9) #define XHCI_TRB_3_PRSV_BIT (1U << 9) #define XHCI_TRB_3_BSR_BIT (1U << 9) #define XHCI_TRB_3_TRT_MASK (3U << 16) #define XHCI_TRB_3_TRT_NONE (0U << 16) #define XHCI_TRB_3_TRT_OUT (2U << 16) #define XHCI_TRB_3_TRT_IN (3U << 16) #define XHCI_TRB_3_DIR_IN (1U << 16) #define XHCI_TRB_3_TLBPC_GET(x) (((x) >> 16) & 0xF) #define XHCI_TRB_3_TLBPC_SET(x) (((x) & 0xF) << 16) #define XHCI_TRB_3_EP_GET(x) (((x) >> 16) & 0x1F) #define XHCI_TRB_3_EP_SET(x) (((x) & 0x1F) << 16) #define XHCI_TRB_3_FRID_GET(x) (((x) >> 20) & 0x7FF) #define XHCI_TRB_3_FRID_SET(x) (((x) & 0x7FF) << 20) #define XHCI_TRB_3_ISO_SIA_BIT (1U << 31) #define XHCI_TRB_3_SUSP_EP_BIT (1U << 23) #define XHCI_TRB_3_SLOT_GET(x) (((x) >> 24) & 0xFF) #define XHCI_TRB_3_SLOT_SET(x) (((x) & 0xFF) << 24) /* Commands */ #define XHCI_TRB_TYPE_RESERVED 0x00 #define XHCI_TRB_TYPE_NORMAL 0x01 #define XHCI_TRB_TYPE_SETUP_STAGE 0x02 #define XHCI_TRB_TYPE_DATA_STAGE 0x03 #define XHCI_TRB_TYPE_STATUS_STAGE 0x04 #define XHCI_TRB_TYPE_ISOCH 0x05 #define XHCI_TRB_TYPE_LINK 0x06 #define XHCI_TRB_TYPE_EVENT_DATA 0x07 #define XHCI_TRB_TYPE_NOOP 0x08 #define XHCI_TRB_TYPE_ENABLE_SLOT 0x09 #define XHCI_TRB_TYPE_DISABLE_SLOT 0x0A #define XHCI_TRB_TYPE_ADDRESS_DEVICE 0x0B #define XHCI_TRB_TYPE_CONFIGURE_EP 0x0C #define XHCI_TRB_TYPE_EVALUATE_CTX 0x0D #define XHCI_TRB_TYPE_RESET_EP 0x0E #define XHCI_TRB_TYPE_STOP_EP 0x0F #define XHCI_TRB_TYPE_SET_TR_DEQUEUE 0x10 #define XHCI_TRB_TYPE_RESET_DEVICE 0x11 #define XHCI_TRB_TYPE_FORCE_EVENT 0x12 #define XHCI_TRB_TYPE_NEGOTIATE_BW 0x13 #define XHCI_TRB_TYPE_SET_LATENCY_TOL 0x14 #define XHCI_TRB_TYPE_GET_PORT_BW 0x15 #define XHCI_TRB_TYPE_FORCE_HEADER 0x16 #define XHCI_TRB_TYPE_NOOP_CMD 0x17 /* Events */ #define XHCI_TRB_EVENT_TRANSFER 0x20 #define XHCI_TRB_EVENT_CMD_COMPLETE 0x21 #define XHCI_TRB_EVENT_PORT_STS_CHANGE 0x22 #define XHCI_TRB_EVENT_BW_REQUEST 0x23 #define XHCI_TRB_EVENT_DOORBELL 0x24 #define XHCI_TRB_EVENT_HOST_CTRL 0x25 #define XHCI_TRB_EVENT_DEVICE_NOTIFY 0x26 #define XHCI_TRB_EVENT_MFINDEX_WRAP 0x27 /* Error codes */ #define XHCI_TRB_ERROR_INVALID 0x00 #define XHCI_TRB_ERROR_SUCCESS 0x01 #define XHCI_TRB_ERROR_DATA_BUF 0x02 #define XHCI_TRB_ERROR_BABBLE 0x03 #define XHCI_TRB_ERROR_XACT 0x04 #define XHCI_TRB_ERROR_TRB 0x05 #define XHCI_TRB_ERROR_STALL 0x06 #define XHCI_TRB_ERROR_RESOURCE 0x07 #define XHCI_TRB_ERROR_BANDWIDTH 0x08 #define XHCI_TRB_ERROR_NO_SLOTS 0x09 #define XHCI_TRB_ERROR_STREAM_TYPE 0x0A #define XHCI_TRB_ERROR_SLOT_NOT_ON 0x0B #define XHCI_TRB_ERROR_ENDP_NOT_ON 0x0C #define XHCI_TRB_ERROR_SHORT_PKT 0x0D #define XHCI_TRB_ERROR_RING_UNDERRUN 0x0E #define XHCI_TRB_ERROR_RING_OVERRUN 0x0F #define XHCI_TRB_ERROR_VF_RING_FULL 0x10 #define XHCI_TRB_ERROR_PARAMETER 0x11 #define XHCI_TRB_ERROR_BW_OVERRUN 0x12 #define XHCI_TRB_ERROR_CONTEXT_STATE 0x13 #define XHCI_TRB_ERROR_NO_PING_RESP 0x14 #define XHCI_TRB_ERROR_EV_RING_FULL 0x15 #define XHCI_TRB_ERROR_INCOMPAT_DEV 0x16 #define XHCI_TRB_ERROR_MISSED_SERVICE 0x17 #define XHCI_TRB_ERROR_CMD_RING_STOP 0x18 #define XHCI_TRB_ERROR_CMD_ABORTED 0x19 #define XHCI_TRB_ERROR_STOPPED 0x1A #define XHCI_TRB_ERROR_LENGTH 0x1B #define XHCI_TRB_ERROR_BAD_MELAT 0x1D #define XHCI_TRB_ERROR_ISOC_OVERRUN 0x1F #define XHCI_TRB_ERROR_EVENT_LOST 0x20 #define XHCI_TRB_ERROR_UNDEFINED 0x21 #define XHCI_TRB_ERROR_INVALID_SID 0x22 #define XHCI_TRB_ERROR_SEC_BW 0x23 #define XHCI_TRB_ERROR_SPLIT_XACT 0x24 } __aligned(4); struct xhci_dev_endpoint_trbs { struct xhci_trb trb[(XHCI_MAX_STREAMS * XHCI_MAX_TRANSFERS) + XHCI_MAX_STREAMS]; }; struct xhci_event_ring_seg { uint64_t qwEvrsTablePtr; uint32_t dwEvrsTableSize; uint32_t dwEvrsReserved; }; #endif /* _PCI_XHCI_H_ */ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2020 Adam Fenn * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * Emulation of selected legacy test/debug interfaces expected by KVM-unit-tests */ #include #include #include #include #include #include #include #include #include #include "debug.h" #include "inout.h" #include "mem.h" #include "pctestdev.h" #define DEBUGEXIT_BASE 0xf4 #define DEBUGEXIT_LEN 4 #define DEBUGEXIT_NAME "isa-debug-exit" #define IOMEM_BASE 0xff000000 #define IOMEM_LEN 0x10000 #define IOMEM_NAME "pc-testdev-iomem" #define IOPORT_BASE 0xe0 #define IOPORT_LEN 4 #define IOPORT_NAME "pc-testdev-ioport" #define IRQ_BASE 0x2000 #define IRQ_IOAPIC_PINCOUNT_MIN 24 #define IRQ_IOAPIC_PINCOUNT_MAX 32 #define IRQ_NAME "pc-testdev-irq-line" #define PCTESTDEV_NAME "pc-testdev" static bool pctestdev_inited; static uint8_t pctestdev_iomem_buf[IOMEM_LEN]; static uint32_t pctestdev_ioport_data; static int pctestdev_debugexit_io(struct vmctx *ctx, int in, int port, int bytes, uint32_t *eax, void *arg); static int pctestdev_iomem_io(struct vcpu *vcpu, int dir, uint64_t addr, int size, uint64_t *val, void *arg1, long arg2); static int pctestdev_ioport_io(struct vmctx *ctx, int in, int port, int bytes, uint32_t *eax, void *arg); static int pctestdev_irq_io(struct vmctx *ctx, int in, int port, int bytes, uint32_t *eax, void *arg); const char * pctestdev_getname(void) { return (PCTESTDEV_NAME); } int pctestdev_init(struct vmctx *ctx) { struct mem_range iomem; struct inout_port debugexit, ioport, irq; int err, pincount; if (pctestdev_inited) { EPRINTLN("Only one pc-testdev device is allowed."); return (-1); } err = vm_ioapic_pincount(ctx, &pincount); if (err != 0) { EPRINTLN("pc-testdev: Failed to obtain IOAPIC pin count."); return (-1); } if (pincount < IRQ_IOAPIC_PINCOUNT_MIN || pincount > IRQ_IOAPIC_PINCOUNT_MAX) { EPRINTLN("pc-testdev: Unsupported IOAPIC pin count: %d.", pincount); return (-1); } debugexit.name = DEBUGEXIT_NAME; debugexit.port = DEBUGEXIT_BASE; debugexit.size = DEBUGEXIT_LEN; debugexit.flags = IOPORT_F_INOUT; debugexit.handler = pctestdev_debugexit_io; debugexit.arg = NULL; iomem.name = IOMEM_NAME; iomem.flags = MEM_F_RW | MEM_F_IMMUTABLE; iomem.handler = pctestdev_iomem_io; iomem.arg1 = NULL; iomem.arg2 = 0; iomem.base = IOMEM_BASE; iomem.size = IOMEM_LEN; ioport.name = IOPORT_NAME; ioport.port = IOPORT_BASE; ioport.size = IOPORT_LEN; ioport.flags = IOPORT_F_INOUT; ioport.handler = pctestdev_ioport_io; ioport.arg = NULL; irq.name = IRQ_NAME; irq.port = IRQ_BASE; irq.size = pincount; irq.flags = IOPORT_F_INOUT; irq.handler = pctestdev_irq_io; irq.arg = NULL; err = register_inout(&debugexit); if (err != 0) goto fail; err = register_inout(&ioport); if (err != 0) goto fail_after_debugexit_reg; err = register_inout(&irq); if (err != 0) goto fail_after_ioport_reg; err = register_mem(&iomem); if (err != 0) goto fail_after_irq_reg; pctestdev_inited = true; return (0); fail_after_irq_reg: (void)unregister_inout(&irq); fail_after_ioport_reg: (void)unregister_inout(&ioport); fail_after_debugexit_reg: (void)unregister_inout(&debugexit); fail: return (err); } static int pctestdev_debugexit_io(struct vmctx *ctx __unused, int in, int port __unused, int bytes __unused, uint32_t *eax, void *arg __unused) { if (in) *eax = 0; else exit((*eax << 1) | 1); return (0); } static int pctestdev_iomem_io(struct vcpu *vcpu __unused, int dir, uint64_t addr, int size, uint64_t *val, void *arg1 __unused, long arg2 __unused) { uint64_t offset; if (addr + size > IOMEM_BASE + IOMEM_LEN) return (-1); offset = addr - IOMEM_BASE; if (dir == MEM_F_READ) { (void)memcpy(val, pctestdev_iomem_buf + offset, size); } else { assert(dir == MEM_F_WRITE); (void)memcpy(pctestdev_iomem_buf + offset, val, size); } return (0); } static int pctestdev_ioport_io(struct vmctx *ctx __unused, int in, int port, int bytes, uint32_t *eax, void *arg __unused) { uint32_t mask; int lsb; if (port + bytes > IOPORT_BASE + IOPORT_LEN) return (-1); lsb = (port & 0x3) * 8; mask = (-1UL >> (32 - (bytes * 8))) << lsb; if (in) *eax = (pctestdev_ioport_data & mask) >> lsb; else { pctestdev_ioport_data &= ~mask; pctestdev_ioport_data |= *eax << lsb; } return (0); } static int pctestdev_irq_io(struct vmctx *ctx, int in, int port, int bytes, uint32_t *eax, void *arg __unused) { int irq; if (bytes != 1) return (-1); if (in) { *eax = 0; return (0); } else { irq = port - IRQ_BASE; if (irq < 16) { if (*eax) return (vm_isa_assert_irq(ctx, irq, irq)); else return (vm_isa_deassert_irq(ctx, irq, irq)); } else { if (*eax) return (vm_ioapic_assert_irq(ctx, irq)); else return (vm_ioapic_deassert_irq(ctx, irq)); } } } /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2020 Adam Fenn * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * Emulation of selected legacy test/debug interfaces expected by KVM-unit-tests */ #ifndef _PCTESTDEV_H_ #define _PCTESTDEV_H_ struct vmctx; const char *pctestdev_getname(void); int pctestdev_init(struct vmctx *ctx); #endif /* * This file and its contents are supplied under the terms of the * Common Development and Distribution License ("CDDL"), version 1.0. * You may only use this file in accordance with the terms of version * 1.0 of the CDDL. * * A full copy of the text of the CDDL should have accompanied this * source. A copy of the CDDL is also available via the Internet at * http://www.illumos.org/license/CDDL. */ /* * Copyright 2021 OmniOS Community Edition (OmniOSce) Association. */ /* * Two privilege sets are maintained. One which is the minimal privileges * necessary to run bhyve - 'bhyve_priv_min' - and one which is the maximum * privileges required - 'bhyve_priv_max'. This second set starts off identical * to the first, and is added to by modules during initialisation depending on * their requirements and their configuration. * Once all modules are initialised, and before the VM is set running, the * privileges are locked by setting the 'Permitted' privilege set from the * contents of the maximum set, which is now the upper bound, and dropping back * to the minimum set. * * Privileges are process wide, and this framework does not support threaded * access. The ID of the thread that first initialises privileges is recorded, * and all subsequent privilege operations must be done by the same thread. */ #include #include #include #include #include #include #include #include "config.h" #include "debug.h" #include "privileges.h" static pthread_t priv_thread; static priv_set_t *bhyve_priv_init; static priv_set_t *bhyve_priv_min; static priv_set_t *bhyve_priv_max; static bool priv_debug = false; #define DPRINTF(params) \ if (priv_debug) do { PRINTLN params; fflush(stdout); } while (0) static void illumos_priv_printset(const char *tag, priv_set_t *set) { char *s; s = priv_set_to_str(set, ',', PRIV_STR_LIT); if (s == NULL) { warn("priv_set_to_str(%s) failed", tag); return; } DPRINTF((" + %s: %s", tag, s)); free(s); } void illumos_priv_reduce(void) { VERIFY3S(pthread_equal(pthread_self(), priv_thread), !=, 0); DPRINTF((" + Reducing privileges to minimum")); if (setppriv(PRIV_SET, PRIV_EFFECTIVE, bhyve_priv_min) != 0) err(4, "failed to reduce privileges"); } static void illumos_priv_add_set(priv_set_t *set, const char *priv, const char *src) { VERIFY3S(pthread_equal(pthread_self(), priv_thread), !=, 0); DPRINTF((" + Adding privilege %s (%s)", priv, src)); if (!priv_ismember(bhyve_priv_init, priv)) errx(4, "Privilege %s (for %s) not in initial set", priv, src); if (priv_addset(set, priv) != 0) err(4, "Failed to add %s privilege for %s", priv, src); } void illumos_priv_add(const char *priv, const char *src) { illumos_priv_add_set(bhyve_priv_max, priv, src); } void illumos_priv_add_min(const char *priv, const char *src) { illumos_priv_add_set(bhyve_priv_min, priv, src); illumos_priv_add_set(bhyve_priv_max, priv, src); } void illumos_priv_init(void) { priv_debug = get_config_bool_default("privileges.debug", false); DPRINTF((" + Initialising privileges.")); if (priv_debug) (void) setpflags(PRIV_DEBUG, 1); priv_thread = pthread_self(); if ((bhyve_priv_init = priv_allocset()) == NULL) err(4, "failed to allocate memory for initial priv set"); if ((bhyve_priv_min = priv_allocset()) == NULL) err(4, "failed to allocate memory for minimum priv set"); if ((bhyve_priv_max = priv_allocset()) == NULL) err(4, "failed to allocate memory for maximum priv set"); if (getppriv(PRIV_EFFECTIVE, bhyve_priv_init) != 0) err(4, "failed to fetch current privileges"); if (priv_debug) illumos_priv_printset("initial", bhyve_priv_init); /* * file_read is left in the minimum set to allow for lazy library * loading. */ priv_basicset(bhyve_priv_min); VERIFY0(priv_delset(bhyve_priv_min, PRIV_FILE_LINK_ANY)); VERIFY0(priv_delset(bhyve_priv_min, PRIV_FILE_WRITE)); VERIFY0(priv_delset(bhyve_priv_min, PRIV_NET_ACCESS)); VERIFY0(priv_delset(bhyve_priv_min, PRIV_PROC_EXEC)); VERIFY0(priv_delset(bhyve_priv_min, PRIV_PROC_FORK)); VERIFY0(priv_delset(bhyve_priv_min, PRIV_PROC_INFO)); VERIFY0(priv_delset(bhyve_priv_min, PRIV_PROC_SECFLAGS)); VERIFY0(priv_delset(bhyve_priv_min, PRIV_PROC_SESSION)); priv_intersect(bhyve_priv_init, bhyve_priv_min); priv_copyset(bhyve_priv_min, bhyve_priv_max); /* * These are privileges that we know will always be needed. * Other privileges may be added by modules as necessary during * initialisation. */ illumos_priv_add(PRIV_FILE_WRITE, "init"); /* * bhyve can work without proc_clock_highres so don't enforce that * it is present. */ if (priv_ismember(bhyve_priv_init, PRIV_PROC_CLOCK_HIGHRES)) illumos_priv_add_min(PRIV_PROC_CLOCK_HIGHRES, "init"); else warnx("The 'proc_clock_highres' privilege is not available"); } void illumos_priv_lock(void) { VERIFY3S(pthread_equal(pthread_self(), priv_thread), !=, 0); if (bhyve_priv_init == NULL) { const char *msg = "attempted to re-lock privileges"; upanic(msg, strlen(msg)); } priv_intersect(bhyve_priv_init, bhyve_priv_max); if (priv_debug) { DPRINTF((" + Locking privileges")); illumos_priv_printset("min", bhyve_priv_min); illumos_priv_printset("max", bhyve_priv_max); } if (setppriv(PRIV_SET, PRIV_PERMITTED, bhyve_priv_max) != 0) { const char *fail = "failed to reduce permitted privileges"; upanic(fail, strlen(fail)); } if (setppriv(PRIV_SET, PRIV_LIMIT, bhyve_priv_max) != 0) { const char *fail = "failed to reduce limit privileges"; upanic(fail, strlen(fail)); } illumos_priv_reduce(); priv_freeset(bhyve_priv_init); bhyve_priv_init = NULL; } /* * This file and its contents are supplied under the terms of the * Common Development and Distribution License ("CDDL"), version 1.0. * You may only use this file in accordance with the terms of version * 1.0 of the CDDL. * * A full copy of the text of the CDDL should have accompanied this * source. A copy of the CDDL is also available via the Internet at * http://www.illumos.org/license/CDDL. */ /* * Copyright 2021 OmniOS Community Edition (OmniOSce) Association. */ #ifndef _BHYVE_PRIVILEGES_H #define _BHYVE_PRIVILEGES_H #include #include void illumos_priv_init(void); void illumos_priv_lock(void); void illumos_priv_add(const char *, const char *); void illumos_priv_add_min(const char *, const char *); void illumos_priv_reduce(void); #endif /* _BHYVE_PRIVILEGES_H */ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2021 Beckhoff Automation GmbH & Co. KG * Author: Corvin Köhne */ #include #ifdef __FreeBSD__ #include #else #include #endif #include #include #include #include #include #include #include #include #include #include #include "acpi_device.h" #include "bhyverun.h" #include "inout.h" #include "pci_lpc.h" #include "qemu_fwcfg.h" #define QEMU_FWCFG_ACPI_DEVICE_NAME "FWCF" #define QEMU_FWCFG_ACPI_HARDWARE_ID "QEMU0002" #define QEMU_FWCFG_SELECTOR_PORT_NUMBER 0x510 #define QEMU_FWCFG_SELECTOR_PORT_SIZE 1 #define QEMU_FWCFG_SELECTOR_PORT_FLAGS IOPORT_F_INOUT #define QEMU_FWCFG_DATA_PORT_NUMBER 0x511 #define QEMU_FWCFG_DATA_PORT_SIZE 1 #define QEMU_FWCFG_DATA_PORT_FLAGS \ IOPORT_F_INOUT /* QEMU v2.4+ ignores writes */ #define QEMU_FWCFG_ARCHITECTURE_MASK 0x0001 #define QEMU_FWCFG_INDEX_MASK 0x3FFF #define QEMU_FWCFG_SELECT_READ 0 #define QEMU_FWCFG_SELECT_WRITE 1 #define QEMU_FWCFG_ARCHITECTURE_GENERIC 0 #define QEMU_FWCFG_ARCHITECTURE_SPECIFIC 1 #define QEMU_FWCFG_INDEX_SIGNATURE 0x00 #define QEMU_FWCFG_INDEX_ID 0x01 #define QEMU_FWCFG_INDEX_NB_CPUS 0x05 #define QEMU_FWCFG_INDEX_MAX_CPUS 0x0F #define QEMU_FWCFG_INDEX_FILE_DIR 0x19 #define QEMU_FWCFG_FIRST_FILE_INDEX 0x20 #define QEMU_FWCFG_MIN_FILES 10 #pragma pack(1) union qemu_fwcfg_selector { struct { uint16_t index : 14; uint16_t writeable : 1; uint16_t architecture : 1; }; uint16_t bits; }; struct qemu_fwcfg_signature { uint8_t signature[4]; }; struct qemu_fwcfg_id { uint32_t interface : 1; /* always set */ uint32_t DMA : 1; uint32_t reserved : 30; }; struct qemu_fwcfg_file { uint32_t be_size; uint16_t be_selector; uint16_t reserved; uint8_t name[QEMU_FWCFG_MAX_NAME]; }; struct qemu_fwcfg_directory { uint32_t be_count; struct qemu_fwcfg_file files[0]; }; #pragma pack() struct qemu_fwcfg_softc { struct acpi_device *acpi_dev; uint32_t data_offset; union qemu_fwcfg_selector selector; struct qemu_fwcfg_item items[QEMU_FWCFG_MAX_ARCHS] [QEMU_FWCFG_MAX_ENTRIES]; struct qemu_fwcfg_directory *directory; }; static struct qemu_fwcfg_softc fwcfg_sc; struct qemu_fwcfg_user_file { STAILQ_ENTRY(qemu_fwcfg_user_file) chain; uint8_t name[QEMU_FWCFG_MAX_NAME]; uint32_t size; void *data; }; static STAILQ_HEAD(qemu_fwcfg_user_file_list, qemu_fwcfg_user_file) user_files = STAILQ_HEAD_INITIALIZER(user_files); static int qemu_fwcfg_selector_port_handler(struct vmctx *const ctx __unused, const int in, const int port __unused, const int bytes, uint32_t *const eax, void *const arg __unused) { if (bytes != sizeof(uint16_t)) { warnx("%s: invalid size (%d) of IO port access", __func__, bytes); return (-1); } if (in) { *eax = htole16(fwcfg_sc.selector.bits); return (0); } fwcfg_sc.data_offset = 0; fwcfg_sc.selector.bits = le16toh(*eax); return (0); } static int qemu_fwcfg_data_port_handler(struct vmctx *const ctx __unused, const int in, const int port __unused, const int bytes, uint32_t *const eax, void *const arg __unused) { if (bytes != sizeof(uint8_t)) { warnx("%s: invalid size (%d) of IO port access", __func__, bytes); return (-1); } if (!in) { warnx("%s: Writes to qemu fwcfg data port aren't allowed", __func__); return (-1); } /* get fwcfg item */ struct qemu_fwcfg_item *const item = &fwcfg_sc.items[fwcfg_sc.selector.architecture] [fwcfg_sc.selector.index]; if (item->data == NULL) { warnx( "%s: qemu fwcfg item doesn't exist (architecture %s index 0x%x)", __func__, fwcfg_sc.selector.architecture ? "specific" : "generic", fwcfg_sc.selector.index); *eax = 0x00; return (0); } else if (fwcfg_sc.data_offset >= item->size) { warnx( "%s: qemu fwcfg item read exceeds size (architecture %s index 0x%x size 0x%x offset 0x%x)", __func__, fwcfg_sc.selector.architecture ? "specific" : "generic", fwcfg_sc.selector.index, item->size, fwcfg_sc.data_offset); *eax = 0x00; return (0); } /* return item data */ *eax = item->data[fwcfg_sc.data_offset]; fwcfg_sc.data_offset++; return (0); } static int qemu_fwcfg_add_item(const uint16_t architecture, const uint16_t index, const uint32_t size, void *const data) { /* truncate architecture and index to their desired size */ const uint16_t arch = architecture & QEMU_FWCFG_ARCHITECTURE_MASK; const uint16_t idx = index & QEMU_FWCFG_INDEX_MASK; /* get pointer to item specified by selector */ struct qemu_fwcfg_item *const fwcfg_item = &fwcfg_sc.items[arch][idx]; /* check if item is already used */ if (fwcfg_item->data != NULL) { warnx("%s: qemu fwcfg item exists (architecture %s index 0x%x)", __func__, arch ? "specific" : "generic", idx); return (EEXIST); } /* save data of the item */ fwcfg_item->size = size; fwcfg_item->data = data; return (0); } static int qemu_fwcfg_add_item_file_dir(void) { const size_t size = sizeof(struct qemu_fwcfg_directory) + QEMU_FWCFG_MIN_FILES * sizeof(struct qemu_fwcfg_file); #ifdef __FreeBSD__ struct qemu_fwcfg_directory *const fwcfg_directory = calloc(1, size); #else // SMATCH: double check that we're allocating correct size: 4 vs 644 void *ptr = calloc(1, size); struct qemu_fwcfg_directory *const fwcfg_directory = (struct qemu_fwcfg_directory *)ptr; #endif if (fwcfg_directory == NULL) { return (ENOMEM); } fwcfg_sc.directory = fwcfg_directory; return (qemu_fwcfg_add_item(QEMU_FWCFG_ARCHITECTURE_GENERIC, QEMU_FWCFG_INDEX_FILE_DIR, sizeof(struct qemu_fwcfg_directory), (uint8_t *)fwcfg_sc.directory)); } static int qemu_fwcfg_add_item_id(void) { struct qemu_fwcfg_id *const fwcfg_id = calloc(1, sizeof(struct qemu_fwcfg_id)); if (fwcfg_id == NULL) { return (ENOMEM); } fwcfg_id->interface = 1; fwcfg_id->DMA = 0; uint32_t *const le_fwcfg_id_ptr = (uint32_t *)fwcfg_id; *le_fwcfg_id_ptr = htole32(*le_fwcfg_id_ptr); return (qemu_fwcfg_add_item(QEMU_FWCFG_ARCHITECTURE_GENERIC, QEMU_FWCFG_INDEX_ID, sizeof(struct qemu_fwcfg_id), (uint8_t *)fwcfg_id)); } static int qemu_fwcfg_add_item_max_cpus(void) { uint16_t *fwcfg_max_cpus = calloc(1, sizeof(uint16_t)); if (fwcfg_max_cpus == NULL) { return (ENOMEM); } /* * We don't support cpu hotplug yet. For that reason, use guest_ncpus instead * of maxcpus. */ *fwcfg_max_cpus = htole16(guest_ncpus); return (qemu_fwcfg_add_item(QEMU_FWCFG_ARCHITECTURE_GENERIC, QEMU_FWCFG_INDEX_MAX_CPUS, sizeof(uint16_t), fwcfg_max_cpus)); } static int qemu_fwcfg_add_item_nb_cpus(void) { uint16_t *fwcfg_max_cpus = calloc(1, sizeof(uint16_t)); if (fwcfg_max_cpus == NULL) { return (ENOMEM); } *fwcfg_max_cpus = htole16(guest_ncpus); return (qemu_fwcfg_add_item(QEMU_FWCFG_ARCHITECTURE_GENERIC, QEMU_FWCFG_INDEX_NB_CPUS, sizeof(uint16_t), fwcfg_max_cpus)); } static int qemu_fwcfg_add_item_signature(void) { struct qemu_fwcfg_signature *const fwcfg_signature = calloc(1, sizeof(struct qemu_fwcfg_signature)); if (fwcfg_signature == NULL) { return (ENOMEM); } fwcfg_signature->signature[0] = 'Q'; fwcfg_signature->signature[1] = 'E'; fwcfg_signature->signature[2] = 'M'; fwcfg_signature->signature[3] = 'U'; return (qemu_fwcfg_add_item(QEMU_FWCFG_ARCHITECTURE_GENERIC, QEMU_FWCFG_INDEX_SIGNATURE, sizeof(struct qemu_fwcfg_signature), (uint8_t *)fwcfg_signature)); } static int qemu_fwcfg_register_port(const char *const name, const int port, const int size, const int flags, const inout_func_t handler) { struct inout_port iop; bzero(&iop, sizeof(iop)); iop.name = name; iop.port = port; iop.size = size; iop.flags = flags; iop.handler = handler; return (register_inout(&iop)); } int qemu_fwcfg_add_file(const char *name, const uint32_t size, void *const data) { if (strlen(name) >= QEMU_FWCFG_MAX_NAME) return (EINVAL); /* * QEMU specifies count as big endian. * Convert it to host endian to work with it. */ const uint32_t count = be32toh(fwcfg_sc.directory->be_count) + 1; /* add file to items list */ const uint32_t index = QEMU_FWCFG_FIRST_FILE_INDEX + count - 1; const int error = qemu_fwcfg_add_item(QEMU_FWCFG_ARCHITECTURE_GENERIC, index, size, data); if (error != 0) { return (error); } /* * files should be sorted alphabetical, get index for new file */ uint32_t file_index; for (file_index = 0; file_index < count - 1; ++file_index) { #ifdef __FreeBSD__ if (strcmp(name, fwcfg_sc.directory->files[file_index].name) < 0) break; #else if (strcmp((char *)name, (char *)fwcfg_sc.directory->files[file_index].name) < 0) { break; } #endif } if (count > QEMU_FWCFG_MIN_FILES) { /* alloc new file directory */ const uint64_t new_size = sizeof(struct qemu_fwcfg_directory) + count * sizeof(struct qemu_fwcfg_file); struct qemu_fwcfg_directory *const new_directory = calloc(1, new_size); if (new_directory == NULL) { warnx( "%s: Unable to allocate a new qemu fwcfg files directory (count %d)", __func__, count); return (ENOMEM); } /* copy files below file_index to new directory */ memcpy(new_directory->files, fwcfg_sc.directory->files, file_index * sizeof(struct qemu_fwcfg_file)); /* copy files above file_index to directory */ memcpy(&new_directory->files[file_index + 1], &fwcfg_sc.directory->files[file_index], (count - file_index - 1) * sizeof(struct qemu_fwcfg_file)); /* free old directory */ free(fwcfg_sc.directory); /* set directory pointer to new directory */ fwcfg_sc.directory = new_directory; /* adjust directory pointer */ fwcfg_sc.items[0][QEMU_FWCFG_INDEX_FILE_DIR].data = (uint8_t *)fwcfg_sc.directory; } else { /* shift files behind file_index */ for (uint32_t i = QEMU_FWCFG_MIN_FILES - 1; i > file_index; --i) { memcpy(&fwcfg_sc.directory->files[i], &fwcfg_sc.directory->files[i - 1], sizeof(struct qemu_fwcfg_file)); } } /* * QEMU specifies count, size and index as big endian. * Save these values in big endian to simplify guest reads of these * values. */ fwcfg_sc.directory->be_count = htobe32(count); fwcfg_sc.directory->files[file_index].be_size = htobe32(size); fwcfg_sc.directory->files[file_index].be_selector = htobe16(index); #ifdef __FreeBSD__ strcpy(fwcfg_sc.directory->files[file_index].name, name); #else strcpy((char *)fwcfg_sc.directory->files[file_index].name, (char *)name); #endif /* set new size for the fwcfg_file_directory */ fwcfg_sc.items[0][QEMU_FWCFG_INDEX_FILE_DIR].size = sizeof(struct qemu_fwcfg_directory) + count * sizeof(struct qemu_fwcfg_file); return (0); } static int qemu_fwcfg_add_user_files(void) { const struct qemu_fwcfg_user_file *fwcfg_file; int error; STAILQ_FOREACH(fwcfg_file, &user_files, chain) { #ifdef __FreeBSD__ error = qemu_fwcfg_add_file(fwcfg_file->name, fwcfg_file->size, fwcfg_file->data); #else error = qemu_fwcfg_add_file((char *)fwcfg_file->name, fwcfg_file->size, fwcfg_file->data); #endif if (error) return (error); } return (0); } static const struct acpi_device_emul qemu_fwcfg_acpi_device_emul = { .name = QEMU_FWCFG_ACPI_DEVICE_NAME, .hid = QEMU_FWCFG_ACPI_HARDWARE_ID, }; int qemu_fwcfg_init(struct vmctx *const ctx) { int error; /* * Bhyve supports fwctl (bhyve) and fwcfg (qemu) as firmware interfaces. * Both are using the same ports. So, it's not possible to provide both * interfaces at the same time to the guest. Therefore, only create acpi * tables and register io ports for fwcfg, if it's used. */ if (strcmp(lpc_fwcfg(), "qemu") == 0) { error = acpi_device_create(&fwcfg_sc.acpi_dev, &fwcfg_sc, ctx, &qemu_fwcfg_acpi_device_emul); if (error) { warnx("%s: failed to create ACPI device for QEMU FwCfg", __func__); goto done; } error = acpi_device_add_res_fixed_ioport(fwcfg_sc.acpi_dev, QEMU_FWCFG_SELECTOR_PORT_NUMBER, 2); if (error) { warnx("%s: failed to add fixed IO port for QEMU FwCfg", __func__); goto done; } /* add handlers for fwcfg ports */ if ((error = qemu_fwcfg_register_port("qemu_fwcfg_selector", QEMU_FWCFG_SELECTOR_PORT_NUMBER, QEMU_FWCFG_SELECTOR_PORT_SIZE, QEMU_FWCFG_SELECTOR_PORT_FLAGS, qemu_fwcfg_selector_port_handler)) != 0) { warnx( "%s: Unable to register qemu fwcfg selector port 0x%x", __func__, QEMU_FWCFG_SELECTOR_PORT_NUMBER); goto done; } if ((error = qemu_fwcfg_register_port("qemu_fwcfg_data", QEMU_FWCFG_DATA_PORT_NUMBER, QEMU_FWCFG_DATA_PORT_SIZE, QEMU_FWCFG_DATA_PORT_FLAGS, qemu_fwcfg_data_port_handler)) != 0) { warnx( "%s: Unable to register qemu fwcfg data port 0x%x", __func__, QEMU_FWCFG_DATA_PORT_NUMBER); goto done; } } /* add common fwcfg items */ if ((error = qemu_fwcfg_add_item_signature()) != 0) { warnx("%s: Unable to add signature item", __func__); goto done; } if ((error = qemu_fwcfg_add_item_id()) != 0) { warnx("%s: Unable to add id item", __func__); goto done; } if ((error = qemu_fwcfg_add_item_nb_cpus()) != 0) { warnx("%s: Unable to add nb_cpus item", __func__); goto done; } if ((error = qemu_fwcfg_add_item_max_cpus()) != 0) { warnx("%s: Unable to add max_cpus item", __func__); goto done; } if ((error = qemu_fwcfg_add_item_file_dir()) != 0) { warnx("%s: Unable to add file_dir item", __func__); goto done; } /* add user defined fwcfg files */ if ((error = qemu_fwcfg_add_user_files()) != 0) { warnx("%s: Unable to add user files", __func__); goto done; } done: if (error) { acpi_device_destroy(fwcfg_sc.acpi_dev); } return (error); } static void qemu_fwcfg_usage(const char *opt) { warnx("Invalid fw_cfg option \"%s\"", opt); warnx("-f [name=],(string|file)="); } /* * Parses the cmdline argument for user defined fw_cfg items. The cmdline * argument has the format: * "-f [name=],(string|file)=" * * E.g.: "-f opt/com.page/example,string=Hello" */ int qemu_fwcfg_parse_cmdline_arg(const char *opt) { struct qemu_fwcfg_user_file *fwcfg_file; struct stat sb; const char *opt_ptr, *opt_end; ssize_t bytes_read; int fd; fwcfg_file = malloc(sizeof(*fwcfg_file)); if (fwcfg_file == NULL) { warnx("Unable to allocate fw_cfg_user_file"); return (ENOMEM); } /* get pointer to */ opt_ptr = opt; /* If [name=] is specified, skip it */ if (strncmp(opt_ptr, "name=", sizeof("name=") - 1) == 0) { opt_ptr += sizeof("name=") - 1; } /* get the end of */ opt_end = strchr(opt_ptr, ','); if (opt_end == NULL) { qemu_fwcfg_usage(opt); #ifndef __FreeBSD__ free(fwcfg_file); #endif return (EINVAL); } /* check if is too long */ if (opt_end - opt_ptr >= QEMU_FWCFG_MAX_NAME) { warnx("fw_cfg name too long: \"%s\"", opt); #ifndef __FreeBSD__ free(fwcfg_file); #endif return (EINVAL); } /* save */ #ifdef __FreeBSD__ strncpy(fwcfg_file->name, opt_ptr, opt_end - opt_ptr); #else strncpy((char *)fwcfg_file->name, opt_ptr, opt_end - opt_ptr); #endif fwcfg_file->name[opt_end - opt_ptr] = '\0'; /* set opt_ptr and opt_end to */ opt_ptr = opt_end + 1; opt_end = opt_ptr + strlen(opt_ptr); if (strncmp(opt_ptr, "string=", sizeof("string=") - 1) == 0) { opt_ptr += sizeof("string=") - 1; fwcfg_file->data = strdup(opt_ptr); if (fwcfg_file->data == NULL) { warnx("Can't duplicate fw_cfg_user_file string \"%s\"", opt_ptr); return (ENOMEM); } fwcfg_file->size = strlen(opt_ptr) + 1; } else if (strncmp(opt_ptr, "file=", sizeof("file=") - 1) == 0) { opt_ptr += sizeof("file=") - 1; fd = open(opt_ptr, O_RDONLY); if (fd < 0) { warn("Can't open fw_cfg_user_file file \"%s\"", opt_ptr); return (EINVAL); } if (fstat(fd, &sb) < 0) { warn("Unable to get size of file \"%s\"", opt_ptr); close(fd); return (-1); } fwcfg_file->data = malloc(sb.st_size); if (fwcfg_file->data == NULL) { warnx( "Can't allocate fw_cfg_user_file file \"%s\" (size: 0x%16lx)", opt_ptr, sb.st_size); close(fd); return (ENOMEM); } bytes_read = read(fd, fwcfg_file->data, sb.st_size); if (bytes_read < 0 || bytes_read != sb.st_size) { warn("Unable to read file \"%s\"", opt_ptr); free(fwcfg_file->data); close(fd); return (-1); } fwcfg_file->size = bytes_read; close(fd); } else { qemu_fwcfg_usage(opt); return (EINVAL); } STAILQ_INSERT_TAIL(&user_files, fwcfg_file, chain); return (0); } /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2021 Beckhoff Automation GmbH & Co. KG * Author: Corvin Köhne */ #ifndef __QEMU_FWCFG_H__ #define __QEMU_FWCFG_H__ #pragma once #include #define QEMU_FWCFG_MAX_ARCHS 0x2 #define QEMU_FWCFG_MAX_ENTRIES 0x4000 #define QEMU_FWCFG_MAX_NAME 56 #define QEMU_FWCFG_FILE_TABLE_LOADER "etc/table-loader" struct qemu_fwcfg_item { uint32_t size; uint8_t *data; }; int qemu_fwcfg_add_file(const char *name, const uint32_t size, void *const data); int qemu_fwcfg_init(struct vmctx *const ctx); int qemu_fwcfg_parse_cmdline_arg(const char *opt); #endif /* !__QEMU_FWCFG_H__ */ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2022 Beckhoff Automation GmbH & Co. KG * Author: Corvin Köhne */ #include #include #include #include #include #include #include #include #include #include #include #include "qemu_fwcfg.h" #include "qemu_loader.h" #ifndef __FreeBSD__ /* * This is better than the number of ifdef blocks that would be otherwise * required throughout this code. Hopefully upstream will clear up the * char* /uint8_t * confusion. */ #define strlen(x) strlen((char *)(x)) #define strncpy(p, q, n) strncpy((char *)(p), (char *)(q), (n)) #endif struct qemu_loader_entry { uint32_t cmd_le; union { struct { uint8_t name[QEMU_FWCFG_MAX_NAME]; uint32_t alignment_le; uint8_t zone; } alloc; struct { uint8_t dest_name[QEMU_FWCFG_MAX_NAME]; uint8_t src_name[QEMU_FWCFG_MAX_NAME]; uint32_t off_le; uint8_t size; } add_pointer; struct { uint8_t name[QEMU_FWCFG_MAX_NAME]; uint32_t off_le; uint32_t start_le; uint32_t len_le; } add_checksum; struct { uint8_t dest_name[QEMU_FWCFG_MAX_NAME]; uint8_t src_name[QEMU_FWCFG_MAX_NAME]; uint32_t dest_off_le; uint32_t src_off_le; uint8_t size; } write_pointer; /* padding */ uint8_t pad[124]; }; } __packed; enum qemu_loader_command { QEMU_LOADER_CMD_ALLOC = 0x1, QEMU_LOADER_CMD_ADD_POINTER = 0x2, QEMU_LOADER_CMD_ADD_CHECKSUM = 0x3, QEMU_LOADER_CMD_WRITE_POINTER = 0x4, }; struct qemu_loader_element { STAILQ_ENTRY(qemu_loader_element) chain; struct qemu_loader_entry entry; }; struct qemu_loader { uint8_t fwcfg_name[QEMU_FWCFG_MAX_NAME]; STAILQ_HEAD(qemu_loader_list, qemu_loader_element) list; }; int qemu_loader_alloc(struct qemu_loader *const loader, const uint8_t *name, const uint32_t alignment, const enum qemu_loader_zone zone) { struct qemu_loader_element *element; if (strlen(name) >= QEMU_FWCFG_MAX_NAME) return (EINVAL); element = calloc(1, sizeof(struct qemu_loader_element)); if (element == NULL) { warnx("%s: failed to allocate command", __func__); return (ENOMEM); } element->entry.cmd_le = htole32(QEMU_LOADER_CMD_ALLOC); strncpy(element->entry.alloc.name, name, QEMU_FWCFG_MAX_NAME); element->entry.alloc.alignment_le = htole32(alignment); element->entry.alloc.zone = zone; /* * The guest always works on copies of the fwcfg item, which where * loaded into guest memory. Loading a fwcfg item is caused by ALLOC. * For that reason, ALLOC should be scheduled in front of any other * commands. */ STAILQ_INSERT_HEAD(&loader->list, element, chain); return (0); } int qemu_loader_add_checksum(struct qemu_loader *const loader, const uint8_t *name, const uint32_t off, const uint32_t start, const uint32_t len) { struct qemu_loader_element *element; if (strlen(name) >= QEMU_FWCFG_MAX_NAME) return (EINVAL); element = calloc(1, sizeof(struct qemu_loader_element)); if (element == NULL) { warnx("%s: failed to allocate command", __func__); return (ENOMEM); } element->entry.cmd_le = htole32(QEMU_LOADER_CMD_ADD_CHECKSUM); strncpy(element->entry.add_checksum.name, name, QEMU_FWCFG_MAX_NAME); element->entry.add_checksum.off_le = htole32(off); element->entry.add_checksum.start_le = htole32(start); element->entry.add_checksum.len_le = htole32(len); STAILQ_INSERT_TAIL(&loader->list, element, chain); return (0); } int qemu_loader_add_pointer(struct qemu_loader *const loader, const uint8_t *dest_name, const uint8_t *src_name, const uint32_t off, const uint8_t size) { struct qemu_loader_element *element; if (strlen(dest_name) >= QEMU_FWCFG_MAX_NAME || strlen(src_name) >= QEMU_FWCFG_MAX_NAME) return (EINVAL); element = calloc(1, sizeof(struct qemu_loader_element)); if (element == NULL) { warnx("%s: failed to allocate command", __func__); return (ENOMEM); } element->entry.cmd_le = htole32(QEMU_LOADER_CMD_ADD_POINTER); strncpy(element->entry.add_pointer.dest_name, dest_name, QEMU_FWCFG_MAX_NAME); strncpy(element->entry.add_pointer.src_name, src_name, QEMU_FWCFG_MAX_NAME); element->entry.add_pointer.off_le = htole32(off); element->entry.add_pointer.size = size; STAILQ_INSERT_TAIL(&loader->list, element, chain); return (0); } int qemu_loader_create(struct qemu_loader **const new_loader, const uint8_t *fwcfg_name) { struct qemu_loader *loader; if (new_loader == NULL || strlen(fwcfg_name) >= QEMU_FWCFG_MAX_NAME) { return (EINVAL); } loader = calloc(1, sizeof(struct qemu_loader)); if (loader == NULL) { warnx("%s: failed to allocate loader", __func__); return (ENOMEM); } strncpy(loader->fwcfg_name, fwcfg_name, QEMU_FWCFG_MAX_NAME); STAILQ_INIT(&loader->list); *new_loader = loader; return (0); } static const uint8_t * qemu_loader_get_zone_name(const enum qemu_loader_zone zone) { #ifdef __FreeBSD__ switch (zone) { case QEMU_LOADER_ALLOC_HIGH: return ("HIGH"); case QEMU_LOADER_ALLOC_FSEG: return ("FSEG"); default: return ("Unknown"); } #else switch (zone) { case QEMU_LOADER_ALLOC_HIGH: return ((uint8_t *)"HIGH"); case QEMU_LOADER_ALLOC_FSEG: return ((uint8_t *)"FSEG"); default: return ((uint8_t *)"Unknown"); } #endif } static void __unused qemu_loader_dump_entry(const struct qemu_loader_entry *const entry) { switch (le32toh(entry->cmd_le)) { case QEMU_LOADER_CMD_ALLOC: printf("CMD_ALLOC\n\r"); printf(" name : %s\n\r", entry->alloc.name); printf(" alignment: %8x\n\r", le32toh(entry->alloc.alignment_le)); printf(" zone : %s\n\r", qemu_loader_get_zone_name(entry->alloc.zone)); break; case QEMU_LOADER_CMD_ADD_POINTER: printf("CMD_ADD_POINTER\n\r"); printf(" dest_name: %s\n\r", entry->add_pointer.dest_name); printf(" src_name : %s\n\r", entry->add_pointer.src_name); printf(" off : %8x\n\r", le32toh(entry->add_pointer.off_le)); printf(" size : %8x\n\r", entry->add_pointer.size); break; case QEMU_LOADER_CMD_ADD_CHECKSUM: printf("CMD_ADD_CHECKSUM\n\r"); printf(" name : %s\n\r", entry->add_checksum.name); printf(" off : %8x\n\r", le32toh(entry->add_checksum.off_le)); printf(" start : %8x\n\r", le32toh(entry->add_checksum.start_le)); printf(" length : %8x\n\r", le32toh(entry->add_checksum.len_le)); break; case QEMU_LOADER_CMD_WRITE_POINTER: printf("CMD_WRITE_POINTER\n\r"); printf(" dest_name: %s\n\r", entry->write_pointer.dest_name); printf(" src_name : %s\n\r", entry->write_pointer.src_name); printf(" dest_off : %8x\n\r", le32toh(entry->write_pointer.dest_off_le)); printf(" src_off : %8x\n\r", le32toh(entry->write_pointer.src_off_le)); printf(" size : %8x\n\r", entry->write_pointer.size); break; default: printf("UNKNOWN\n\r"); break; } } int qemu_loader_finish(struct qemu_loader *const loader) { struct qemu_loader_element *element; struct qemu_loader_entry *data; size_t len = 0; STAILQ_FOREACH(element, &loader->list, chain) { len += sizeof(struct qemu_loader_entry); } if (len == 0) { warnx("%s: bios loader empty", __func__); return (EFAULT); } data = calloc(1, len); if (data == NULL) { warnx("%s: failed to allocate fwcfg data", __func__); return (ENOMEM); } int i = 0; STAILQ_FOREACH(element, &loader->list, chain) { memcpy(&data[i], &element->entry, sizeof(struct qemu_loader_entry)); ++i; } #ifdef __FreeBSD__ return (qemu_fwcfg_add_file(loader->fwcfg_name, len, data)); #else return (qemu_fwcfg_add_file((const char *)loader->fwcfg_name, len, data)); #endif } /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2022 Beckhoff Automation GmbH & Co. KG * Author: Corvin Köhne */ #pragma once #include "qemu_fwcfg.h" struct qemu_loader; /* * Some guest bios like seabios assume the RSDP to be located in the FSEG. Bhyve * only supports OVMF which has no such requirement. */ enum qemu_loader_zone { QEMU_LOADER_ALLOC_HIGH = 1, QEMU_LOADER_ALLOC_FSEG, /* 0x0F000000 - 0x100000 */ }; /** * Loads a fwcfg item into guest memory. This command has to be issued before * any subsequent command can be used. * * @param loader Qemu loader instance the command should be added to. * @param name Name of the fwcfg item which should be allocated. * @param alignment Alignment required by the data. * @param zone Memory zone in which it should be loaded. */ int qemu_loader_alloc(struct qemu_loader *loader, const uint8_t *name, uint32_t alignment, enum qemu_loader_zone zone); /** * Calculates a checksum for @p name and writes it to @p name + @p off . The * checksum calculation ranges from @p start to @p start + @p len. The checksum * field is always one byte large and all bytes in the specified range, * including the checksum, have to sum up to 0. * * @param loader Qemu loader instance the command should be added to. * @param name Name of the fwcfg item which should be patched. * @param off Offset into @p name . * @param start Start offset of checksum calculation. * @param len Length of the checksum calculation. */ int qemu_loader_add_checksum(struct qemu_loader *loader, const uint8_t *name, uint32_t off, uint32_t start, uint32_t len); /** * Adds the address of @p src_name to the value at @p dest_name + @p off . The * size of the pointer is determined by @p dest_size and should be 1, 2, 4 or 8. * * @param loader Qemu loader instance the command should be added to. * @param dest_name Name of the fwcfg item which should be patched. * @param src_name Name of the fwcfg item which address should be written to * @p dest_name + @p off. * @param off Offset into @p dest_name . * @param size Size of the pointer (1, 2, 4 or 8). */ int qemu_loader_add_pointer(struct qemu_loader *loader, const uint8_t *dest_name, const uint8_t *src_name, uint32_t off, uint8_t size); /** * Creates a qemu loader instance. * * @param new_loader Returns the newly created qemu loader instance. * @param fwcfg_name Name of the FwCfg item which represents the qemu loader */ int qemu_loader_create(struct qemu_loader **new_loader, const uint8_t *fwcfg_name); /** * Signals that all commands are written to the qemu loader. This function * creates a proper FwCfg item and registers it. * * @param loader Qemu loader instance which should be finished. */ int qemu_loader_finish(struct qemu_loader *loader); /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2015 Tycho Nightingale * Copyright (c) 2015 Leon Dang * Copyright 2020 Joyent, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * This file and its contents are supplied under the terms of the * Common Development and Distribution License ("CDDL"), version 1.0. * You may only use this file in accordance with the terms of version * 1.0 of the CDDL. * * A full copy of the text of the CDDL should have accompanied this * source. A copy of the CDDL is also available via the Internet at * http://www.illumos.org/license/CDDL. * * Copyright 2022 OmniOS Community Edition (OmniOSce) Association. */ /* * References to the RFB protocol specification refer to: * - [1] https://github.com/rfbproto/rfbproto/blob/master/rfbproto.rst */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifndef NO_OPENSSL #include #endif #include #include #include #include #include #ifndef WITHOUT_CAPSICUM #include #include #include #endif #include "bhyvegc.h" #include "config.h" #include "debug.h" #include "console.h" #include "rfb.h" #include "rfb_impl.h" #include "sockstream.h" static uint_t rfb_debug = 0; static list_t rfb_list; static id_space_t *rfb_idspace; static bool rfb_sse42; static pthread_once_t rfb_once = PTHREAD_ONCE_INIT; extern int raw_stdio; static void rfb_send_extended_keyevent_update_msg(rfb_client_t *); static void rfb_printf(rfb_client_t *c, rfb_loglevel_t level, const char *fmt, ...) { FILE *fp = stdout; va_list ap; switch (level) { case RFB_LOGDEBUG: if (rfb_debug == 0) return; /* FALLTHROUGH */ case RFB_LOGERR: fp = stderr; /* FALLTHROUGH */ case RFB_LOGWARN: if (c != NULL) (void) fprintf(fp, "rfb%u: ", c->rc_instance); else (void) fprintf(fp, "rfb: "); va_start(ap, fmt); (void) vfprintf(fp, fmt, ap); va_end(ap); if (raw_stdio) (void) fprintf(fp, "\r\n"); else (void) fprintf(fp, "\n"); (void) fflush(fp); } } static void rfb_init_once(void) { uint_t cpu_registers[4], ecx; do_cpuid(1, cpu_registers); ecx = cpu_registers[2]; rfb_sse42 = (ecx & CPUID2_SSE42) != 0; if (rfb_sse42) rfb_printf(NULL, RFB_LOGDEBUG, "enabled fast crc32"); else rfb_printf(NULL, RFB_LOGWARN, "no support for fast crc32"); if (get_config_bool_default("rfb.debug", false)) rfb_debug = 1; list_create(&rfb_list, sizeof (rfb_server_t), offsetof(rfb_server_t, rs_node)); rfb_idspace = id_space_create("rfb", 0, INT32_MAX); } static void rfb_free_client(rfb_client_t *c) { free(c->rc_crc); free(c->rc_crc_tmp); free(c->rc_zbuf); free(c->rc_gci.data); if (c->rc_encodings & RFB_ENCODING_ZLIB) (void) deflateEnd(&c->rc_zstream); if (c->rc_fd != -1) (void) close(c->rc_fd); free(c); } /* * Calculate CRC32 using SSE4.2; Intel or AMD Bulldozer+ CPUs only */ static inline uint32_t fast_crc32(void *buf, int len, uint32_t crcval) { uint32_t q = len / sizeof (uint32_t); uint32_t *p = (uint32_t *)buf; while (q--) { /* BEGIN CSTYLED */ asm volatile ( /* crc32l %ecx,%esi */ ".byte 0xf2, 0xf, 0x38, 0xf1, 0xf1;" :"=S" (crcval) :"0" (crcval), "c" (*p) ); /* END CSTYLED */ p++; } return (crcval); } static void rfb_send_client_status(rfb_client_t *c, uint32_t status, const char *msg) { rfb_printf(c, RFB_LOGDEBUG, "sending client status %u (%s)", status, msg ? msg : "NULL"); status = htonl(status); (void) stream_write(c->rc_fd, &status, sizeof (status)); if (msg != NULL && status != 0 && c->rc_cver == RFB_CVER_3_8) { char buf[4]; rfb_printf(c, RFB_LOGWARN, msg); be32enc(buf, strlen((char *)msg)); (void) stream_write(c->rc_fd, buf, 4); (void) stream_write(c->rc_fd, msg, strlen((char *)msg)); } } static bool rfb_handshake_version(rfb_client_t *c) { unsigned char buf[RFB_VERSION_LEN]; ssize_t l; rfb_printf(c, RFB_LOGDEBUG, "handshake version"); if (stream_write(c->rc_fd, RFB_VERSION, RFB_VERSION_LEN) != RFB_VERSION_LEN) { rfb_printf(c, RFB_LOGWARN, "could not send server version."); return (false); } l = stream_read(c->rc_fd, buf, sizeof (buf)); if (l <= 0) { rfb_printf(c, RFB_LOGWARN, "client version not read"); return (false); } else if (l != RFB_VERSION_LEN) { rfb_printf(c, RFB_LOGWARN, "client sent short version - '%.*s'", l, buf); return (false); } rfb_printf(c, RFB_LOGDEBUG, "version handshake, client ver '%.*s'", l - 1, buf); if (strncmp(RFB_VERSION, (char *)buf, RFB_VERSION_LEN - 2) != 0) { rfb_printf(c, RFB_LOGERR, "bad client version '%.*s'", l, buf); return (false); } switch (buf[RFB_VERSION_LEN - 2]) { case '8': c->rc_cver = RFB_CVER_3_8; break; case '7': c->rc_cver = RFB_CVER_3_7; break; case '5': /* * From the RFB specification[1], section 7.1.1: * "version 3.5 was wrongly reported by some clients, but this * should be interpreted by all servers as 3.3." */ case '3': c->rc_cver = RFB_CVER_3_3; break; default: rfb_printf(c, RFB_LOGERR, "unsupported client version '%.*s'", l - 1, buf); return (false); } return (true); } static bool rfb_handshake_auth(rfb_client_t *c) { unsigned char buf[RFBP_SECURITY_VNC_AUTH_LEN]; int auth_type; rfb_printf(c, RFB_LOGDEBUG, "handshake auth"); auth_type = RFBP_SECURITY_NONE; #ifndef NO_OPENSSL if (c->rc_s->rs_password != NULL) auth_type = RFBP_SECURITY_VNC_AUTH; #endif switch (c->rc_cver) { case RFB_CVER_3_3: /* * RFB specification[1] section 7.1.2: * The server decides the security type and sends a single word. */ be32enc(buf, auth_type); (void) stream_write(c->rc_fd, buf, 4); break; case RFB_CVER_3_7: case RFB_CVER_3_8: /* Send list of supported types. */ buf[0] = 1; /* list length */ buf[1] = auth_type; (void) stream_write(c->rc_fd, buf, 2); /* Read agreed security type. */ if (stream_read(c->rc_fd, buf, 1) != 1) { rfb_printf(c, RFB_LOGWARN, "auth fail, no type from client"); return (false); } if (buf[0] != auth_type) { rfb_send_client_status(c, 1, "Auth failed: authentication type mismatch"); return (false); } break; } if (auth_type == RFBP_SECURITY_NONE) { /* * According to the RFB specification[1], section 7.2.1, for a * security type of 'None', client versions 3.3 and 3.7 expect * to move straight to the ClientInit phase, without the server * sending a response. For version 3.8, a SecurityResult word * needs to be sent indicating success. */ switch (c->rc_cver) { case RFB_CVER_3_3: case RFB_CVER_3_7: break; case RFB_CVER_3_8: rfb_send_client_status(c, 0, NULL); break; } return (true); } /* Perform VNC authentication. */ #ifdef NO_OPENSSL rfb_printf(c, RFB_LOGERR, "Auth not supported, no OpenSSL in your system"); rfb_send_client_status(c, 1, "Auth failed."); return (false); #else unsigned char challenge[RFBP_SECURITY_VNC_AUTH_LEN]; unsigned char keystr[RFBP_SECURITY_VNC_PASSWD_LEN]; unsigned char crypt_expected[RFBP_SECURITY_VNC_AUTH_LEN]; DES_key_schedule ks; /* * The client encrypts the challenge with DES, using a password * supplied by the user as the key. * To form the key, the password is truncated to eight characters, or * padded with null bytes on the right. * The client then sends the resulting 16-bytes response. */ (void) strncpy((char *)keystr, c->rc_s->rs_password, RFBP_SECURITY_VNC_PASSWD_LEN); /* * VNC clients encrypt the challenge with all the bit fields in each * byte of the password mirrored. * Here we flip each byte of the keystr. */ for (uint_t i = 0; i < RFBP_SECURITY_VNC_PASSWD_LEN; i++) { keystr[i] = (keystr[i] & 0xf0) >> 4 | (keystr[i] & 0x0f) << 4; keystr[i] = (keystr[i] & 0xcc) >> 2 | (keystr[i] & 0x33) << 2; keystr[i] = (keystr[i] & 0xaa) >> 1 | (keystr[i] & 0x55) << 1; } /* Initialize a 16-byte random challenge. */ arc4random_buf(challenge, sizeof (challenge)); /* Send the challenge to the client. */ if (stream_write(c->rc_fd, challenge, RFBP_SECURITY_VNC_AUTH_LEN) != RFBP_SECURITY_VNC_AUTH_LEN) { rfb_printf(c, RFB_LOGERR, "failed to send challenge to client"); return (false); } /* Receive the 16-byte challenge response. */ if (stream_read(c->rc_fd, buf, RFBP_SECURITY_VNC_AUTH_LEN) != RFBP_SECURITY_VNC_AUTH_LEN) { rfb_send_client_status(c, 1, "Challenge response read failed"); return (false); } memcpy(crypt_expected, challenge, RFBP_SECURITY_VNC_AUTH_LEN); /* Encrypt the Challenge with DES. */ DES_set_key_unchecked((const_DES_cblock *)keystr, &ks); DES_ecb_encrypt((const_DES_cblock *)challenge, (const_DES_cblock *)crypt_expected, &ks, DES_ENCRYPT); DES_ecb_encrypt( (const_DES_cblock *)(challenge + RFBP_SECURITY_VNC_PASSWD_LEN), (const_DES_cblock *)(crypt_expected + RFBP_SECURITY_VNC_PASSWD_LEN), &ks, DES_ENCRYPT); if (memcmp(crypt_expected, buf, RFBP_SECURITY_VNC_AUTH_LEN) != 0) { rfb_send_client_status(c, 1, "Auth failed: Invalid password."); return (false); } rfb_printf(c, RFB_LOGDEBUG, "authentication succeeded"); rfb_send_client_status(c, 0, NULL); #endif return (true); } static bool rfb_handshake_init_message(rfb_client_t *c) { struct bhyvegc_image *gci; char buf[1]; char *name; rfb_printf(c, RFB_LOGDEBUG, "handshake server init"); /* Read the client init message. */ if (stream_read(c->rc_fd, buf, 1) != 1) { rfb_printf(c, RFB_LOGWARN, "client did not send init"); return (false); } if (buf[0] == 0) { rfb_client_t *oc; rfb_printf(c, RFB_LOGDEBUG, "client requested exclusive access"); pthread_mutex_lock(&c->rc_s->rs_clientlock); c->rc_s->rs_exclusive = true; /* Disconnect all other clients. */ for (oc = list_head(&c->rc_s->rs_clients); oc != NULL; oc = list_next(&c->rc_s->rs_clients, oc)) { if (oc != c) oc->rc_closing = true; } pthread_mutex_unlock(&c->rc_s->rs_clientlock); } else { rfb_printf(c, RFB_LOGDEBUG, "client requested shared access"); pthread_mutex_lock(&c->rc_s->rs_clientlock); if (c->rc_s->rs_exclusive) { rfb_printf(c, RFB_LOGWARN, "deny due to existing exclusive session"); pthread_mutex_unlock(&c->rc_s->rs_clientlock); return (false); } pthread_mutex_unlock(&c->rc_s->rs_clientlock); } gci = console_get_image(); c->rc_sinfo.rsi_width = htons(gci->width); c->rc_sinfo.rsi_height = htons(gci->height); c->rc_width = gci->width; c->rc_height = gci->height; if (c->rc_s->rs_name != NULL) name = (char *)c->rc_s->rs_name; else name = "bhyve"; c->rc_sinfo.rsi_namelen = htonl(strlen(name)); (void) stream_write(c->rc_fd, &c->rc_sinfo, sizeof (c->rc_sinfo)); (void) stream_write(c->rc_fd, name, strlen(name)); return (true); } static bool rfb_handshake(rfb_client_t *c) { if (!rfb_handshake_version(c)) return (false); if (!rfb_handshake_auth(c)) return (false); if (!rfb_handshake_init_message(c)) return (false); return (true); } static void rfb_print_pixfmt(rfb_client_t *c, rfb_pixfmt_t *px, rfb_loglevel_t level) { rfb_printf(c, level, "%20s: %u", "bpp", px->rp_bpp); rfb_printf(c, level, "%20s: %u", "depth", px->rp_depth); rfb_printf(c, level, "%20s: %u", "bigendian", px->rp_bigendian); rfb_printf(c, level, "%20s: %u", "truecolour", px->rp_truecolour); rfb_printf(c, level, "%20s: %u", "r_max", ntohs(px->rp_r_max)); rfb_printf(c, level, "%20s: %u", "g_max", ntohs(px->rp_g_max)); rfb_printf(c, level, "%20s: %u", "b_max", ntohs(px->rp_b_max)); rfb_printf(c, level, "%20s: %u", "r_shift", px->rp_r_shift); rfb_printf(c, level, "%20s: %u", "g_shift", px->rp_g_shift); rfb_printf(c, level, "%20s: %u", "b_shift", px->rp_b_shift); } static bool rfb_recv_set_pixel_format(rfb_client_t *c) { rfb_cs_pixfmt_msg_t msg; rfb_pixfmt_t *newpx = &msg.rp_pixfmt; rfb_pixfmt_t *oldpx = &c->rc_sinfo.rsi_pixfmt; rfb_pixfmt_t *spx = &c->rc_s->rs_pixfmt; rfb_printf(c, RFB_LOGDEBUG, "received pixel format"); if (stream_read(c->rc_fd, &msg, sizeof (msg)) != sizeof (msg)) return (false); /* * The client has sent its desired pixel format. The protocol does not * have a mechanism to reject this, we are supposed to just start using * the requested format from the next update. * * At present, we can only support alternative rgb-shift values and * will accept (and ignore) a new depth value. */ if (oldpx->rp_bpp != newpx->rp_bpp || oldpx->rp_bigendian != newpx->rp_bigendian || oldpx->rp_truecolour != newpx->rp_truecolour || oldpx->rp_r_max != newpx->rp_r_max || oldpx->rp_g_max != newpx->rp_g_max || oldpx->rp_b_max != newpx->rp_b_max) { rfb_printf(c, RFB_LOGWARN, "unsupported pixfmt from client"); rfb_print_pixfmt(c, newpx, RFB_LOGWARN); return (false); } rfb_print_pixfmt(c, newpx, RFB_LOGDEBUG); /* Check if the new shifts match the server's native values. */ if (newpx->rp_r_shift != spx->rp_r_shift || newpx->rp_g_shift != spx->rp_g_shift || newpx->rp_b_shift != spx->rp_b_shift) { c->rc_custom_pixfmt = true; rfb_printf(c, RFB_LOGDEBUG, "Using custom pixfmt"); } else { c->rc_custom_pixfmt = false; rfb_printf(c, RFB_LOGDEBUG, "Using native pixfmt"); } c->rc_sinfo.rsi_pixfmt = msg.rp_pixfmt; c->rc_crc_reset = true; return (true); } static bool rfb_recv_set_encodings(rfb_client_t *c) { rfb_cs_encodings_msg_t msg; rfb_printf(c, RFB_LOGDEBUG, "received encodings"); if (stream_read(c->rc_fd, &msg, sizeof (msg)) != sizeof (msg)) return (false); msg.re_numencs = htons(msg.re_numencs); rfb_printf(c, RFB_LOGDEBUG, "%d values", msg.re_numencs); for (uint_t i = 0; i < msg.re_numencs; i++) { uint32_t enc; if (stream_read(c->rc_fd, &enc, sizeof (enc)) != sizeof (enc)) return (false); enc = htonl(enc); switch (enc) { case RFBP_ENCODING_RAW: rfb_printf(c, RFB_LOGDEBUG, "client supports raw encoding"); c->rc_encodings |= RFB_ENCODING_RAW; break; case RFBP_ENCODING_ZLIB: rfb_printf(c, RFB_LOGDEBUG, "client supports zlib encoding"); if (!(c->rc_encodings & RFB_ENCODING_ZLIB)) { if (deflateInit(&c->rc_zstream, Z_BEST_SPEED) != Z_OK) { return (false); } c->rc_encodings |= RFB_ENCODING_ZLIB; } break; case RFBP_ENCODING_RESIZE: rfb_printf(c, RFB_LOGDEBUG, "client supports resize"); c->rc_encodings |= RFB_ENCODING_RESIZE; break; case RFBP_ENCODING_EXT_KEVENT: rfb_printf(c, RFB_LOGDEBUG, "client supports ext key event"); c->rc_encodings |= RFB_ENCODING_EXT_KEVENT; break; case RFBP_ENCODING_DESKTOP_NAME: rfb_printf(c, RFB_LOGDEBUG, "client supports desktop name"); c->rc_encodings |= RFB_ENCODING_DESKTOP_NAME; break; default: rfb_printf(c, RFB_LOGDEBUG, "client supports encoding %d", (int32_t)enc); } } return (true); } static bool rfb_recv_update(rfb_client_t *c) { rfb_cs_update_msg_t msg; if (stream_read(c->rc_fd, &msg, sizeof (msg)) != sizeof (msg)) return (false); if (!c->rc_keyevent_sent && (c->rc_encodings & RFB_ENCODING_EXT_KEVENT)) { /* * Take this opportunity to tell the client that we * accept QEMU Extended Key Event Pseudo-encoding. */ c->rc_keyevent_sent = true; rfb_send_extended_keyevent_update_msg(c); } c->rc_pending = true; if (msg.rum_incremental == 0) { rfb_printf(c, RFB_LOGDEBUG, "client requested full screen update"); c->rc_send_fullscreen = true; } return (true); } static bool rfb_recv_key_event(rfb_client_t *c) { rfb_cs_key_event_msg_t msg; if (stream_read(c->rc_fd, &msg, sizeof (msg)) != sizeof (msg)) return (false); msg.rke_sym = htonl(msg.rke_sym); rfb_printf(c, RFB_LOGDEBUG, "received key %s %x", msg.rke_down == 0 ? "up" : "down", msg.rke_sym); console_key_event(msg.rke_down, msg.rke_sym, htonl(0)); c->rc_input_detected = true; return (true); } static bool rfb_recv_pointer_event(rfb_client_t *c) { rfb_cs_pointer_event_msg_t msg; if (stream_read(c->rc_fd, &msg, sizeof (msg)) != sizeof (msg)) return (false); msg.rpe_x = htons(msg.rpe_x); msg.rpe_y = htons(msg.rpe_y); if (rfb_debug > 1) { rfb_printf(c, RFB_LOGDEBUG, "received pointer event @ %dx%d", msg.rpe_x, msg.rpe_y); } console_ptr_event(msg.rpe_button, msg.rpe_x, msg.rpe_y); c->rc_input_detected = true; return (true); } static bool rfb_recv_cut_text(rfb_client_t *c) { rfb_cs_cut_text_msg_t msg; unsigned char buf[32]; rfb_printf(c, RFB_LOGDEBUG, "received cut text event"); if (stream_read(c->rc_fd, &msg, sizeof (msg)) != sizeof (msg)) return (false); msg.rct_length = htonl(msg.rct_length); rfb_printf(c, RFB_LOGDEBUG, "%u bytes in buffer", msg.rct_length); /* Consume the buffer */ while (msg.rct_length > 0) { ssize_t l; l = stream_read(c->rc_fd, buf, MIN(sizeof (buf), msg.rct_length)); if (l <= 0) return (false); msg.rct_length -= l; } return (true); } static bool rfb_recv_qemu(rfb_client_t *c) { rfb_cs_qemu_msg_t msg; rfb_printf(c, RFB_LOGDEBUG, "received QEMU event"); if (stream_read(c->rc_fd, &msg, sizeof (msg)) != sizeof (msg)) return (false); switch (msg.rq_subtype) { case RFBP_CS_QEMU_KEVENT: { rfb_cs_qemu_extended_key_msg_t keymsg; if (stream_read(c->rc_fd, &keymsg, sizeof (keymsg)) != sizeof (keymsg)) { return (false); } keymsg.rqek_sym = htonl(keymsg.rqek_sym); keymsg.rqek_code = htonl(keymsg.rqek_code); rfb_printf(c, RFB_LOGDEBUG, "QEMU key %s %x / %x", keymsg.rqek_down == 0 ? "up" : "down", keymsg.rqek_sym, keymsg.rqek_code); console_key_event((int)keymsg.rqek_down, keymsg.rqek_sym, keymsg.rqek_code); c->rc_input_detected = true; break; } default: rfb_printf(c, RFB_LOGWARN, "Unknown QEMU event subtype: %d\n", msg.rq_subtype); return (false); } return (true); } static bool rfb_send_update_header(rfb_client_t *c, int numrects) { rfb_server_update_msg_t msg; msg.rss_type = RFBP_SC_UPDATE; msg.rss_pad = 0; msg.rss_numrects = htons(numrects); return (stream_write(c->rc_fd, &msg, sizeof (msg)) == sizeof (msg)); } static void rfb_send_resize_update_msg(rfb_client_t *c) { rfb_rect_hdr_t rect; rfb_printf(c, RFB_LOGDEBUG, "sending screen resize %dx%d", c->rc_width, c->rc_height); (void) rfb_send_update_header(c, 1); rect.rr_x = htons(0); rect.rr_y = htons(0); rect.rr_width = htons(c->rc_width); rect.rr_height = htons(c->rc_height); rect.rr_encoding = htonl(RFBP_ENCODING_RESIZE); (void) stream_write(c->rc_fd, &rect, sizeof (rect)); } static void rfb_send_extended_keyevent_update_msg(rfb_client_t *c) { rfb_rect_hdr_t rect; rfb_printf(c, RFB_LOGDEBUG, "sending extended keyevent update message"); (void) rfb_send_update_header(c, 1); rect.rr_x = htons(0); rect.rr_y = htons(0); rect.rr_width = htons(c->rc_width); rect.rr_height = htons(c->rc_height); rect.rr_encoding = htonl(RFBP_ENCODING_EXT_KEVENT); (void) stream_write(c->rc_fd, &rect, sizeof (rect)); } static void translate_pixels(rfb_client_t *c, struct bhyvegc_image *gci, int x1, int y1, int x2, int y2) { rfb_pixfmt_t *px = &c->rc_sinfo.rsi_pixfmt; rfb_pixfmt_t *spx = &c->rc_s->rs_pixfmt; int w, h; w = gci->width; h = gci->height; VERIFY3S(gci->width, ==, c->rc_gci.width); VERIFY3S(gci->height, ==, c->rc_gci.height); for (uint_t y = y1; y < h && y < y2; y++) { for (uint_t x = x1; x < w && x < x2; x++) { uint32_t p; p = gci->data[y * w + x]; c->rc_gci.data[y * w + x] = 0xff000000 | ((p >> spx->rp_r_shift) & 0xff) << px->rp_r_shift | ((p >> spx->rp_g_shift) & 0xff) << px->rp_g_shift | ((p >> spx->rp_b_shift) & 0xff) << px->rp_b_shift; } } } static bool rfb_send_rect(rfb_client_t *c, struct bhyvegc_image *gci, int x, int y, int w, int h) { rfb_rect_hdr_t rect; unsigned long zlen; ssize_t nwrite, total; int err; uint32_t *p; uint8_t *zbufp; if (rfb_debug > 1) { rfb_printf(c, RFB_LOGDEBUG, "send rect %dx%d %dx%d", x, y, w, h); } /* Rectangle header. */ rect.rr_x = htons(x); rect.rr_y = htons(y); rect.rr_width = htons(w); rect.rr_height = htons(h); uint32_t *data = gci->data; if (c->rc_custom_pixfmt) { translate_pixels(c, gci, x, y, x + w, y + h); data = c->rc_gci.data; } h = y + h; w *= sizeof (uint32_t); if (c->rc_encodings & RFB_ENCODING_ZLIB) { zbufp = c->rc_zbuf; c->rc_zstream.total_in = 0; c->rc_zstream.total_out = 0; for (p = &data[y * gci->width + x]; y < h; y++) { c->rc_zstream.next_in = (Bytef *)p; c->rc_zstream.avail_in = w; c->rc_zstream.next_out = (Bytef *)zbufp; c->rc_zstream.avail_out = RFB_ZLIB_BUFSZ + 16 - c->rc_zstream.total_out; c->rc_zstream.data_type = Z_BINARY; /* Compress with zlib. */ err = deflate(&c->rc_zstream, Z_SYNC_FLUSH); if (err != Z_OK) { rfb_printf(c, RFB_LOGWARN, "zlib[rect] deflate err: %d", err); goto doraw; } zbufp = c->rc_zbuf + c->rc_zstream.total_out; p += gci->width; } rect.rr_encoding = htonl(RFBP_ENCODING_ZLIB); nwrite = stream_write(c->rc_fd, &rect, sizeof (rect)); if (nwrite <= 0) return (false); zlen = htonl(c->rc_zstream.total_out); nwrite = stream_write(c->rc_fd, &zlen, sizeof (uint32_t)); if (nwrite <= 0) return (false); return (stream_write(c->rc_fd, c->rc_zbuf, c->rc_zstream.total_out) == c->rc_zstream.total_out); } doraw: total = 0; zbufp = c->rc_zbuf; for (p = &data[y * gci->width + x]; y < h; y++) { memcpy(zbufp, p, w); zbufp += w; total += w; p += gci->width; } rect.rr_encoding = htonl(RFBP_ENCODING_RAW); nwrite = stream_write(c->rc_fd, &rect, sizeof (rect)); if (nwrite <= 0) return (false); return (stream_write(c->rc_fd, c->rc_zbuf, total) == total); } static bool rfb_send_all(rfb_client_t *c, struct bhyvegc_image *gci) { rfb_rect_hdr_t rect; ssize_t nwrite; unsigned long zlen; int err; rfb_printf(c, RFB_LOGDEBUG, "send entire screen"); /* Just the one (big) rect. */ if (!rfb_send_update_header(c, 1)) return (false); rect.rr_x = 0; rect.rr_y = 0; rect.rr_width = htons(gci->width); rect.rr_height = htons(gci->height); uint32_t *data = gci->data; if (c->rc_custom_pixfmt) { translate_pixels(c, gci, 0, 0, gci->width, gci->height); data = c->rc_gci.data; } if (c->rc_encodings & RFB_ENCODING_ZLIB) { c->rc_zstream.next_in = (Bytef *)data; c->rc_zstream.avail_in = gci->width * gci->height * sizeof (uint32_t); c->rc_zstream.next_out = (Bytef *)c->rc_zbuf; c->rc_zstream.avail_out = RFB_ZLIB_BUFSZ + 16; c->rc_zstream.data_type = Z_BINARY; c->rc_zstream.total_in = 0; c->rc_zstream.total_out = 0; /* Compress with zlib. */ err = deflate(&c->rc_zstream, Z_SYNC_FLUSH); if (err != Z_OK) { rfb_printf(c, RFB_LOGWARN, "zlib deflate err: %d", err); goto doraw; } rect.rr_encoding = htonl(RFBP_ENCODING_ZLIB); nwrite = stream_write(c->rc_fd, &rect, sizeof (rect)); if (nwrite <= 0) return (false); zlen = htonl(c->rc_zstream.total_out); nwrite = stream_write(c->rc_fd, &zlen, sizeof (uint32_t)); if (nwrite <= 0) return (false); return (stream_write(c->rc_fd, c->rc_zbuf, c->rc_zstream.total_out) == c->rc_zstream.total_out); } doraw: rect.rr_encoding = htonl(RFBP_ENCODING_RAW); nwrite = stream_write(c->rc_fd, &rect, sizeof (rect)); if (nwrite <= 0) return (false); nwrite = gci->width * gci->height * sizeof (uint32_t); return (stream_write(c->rc_fd, data, nwrite) == nwrite); } static bool rfb_send_screen(rfb_client_t *c) { struct bhyvegc_image *gci; bool retval = true; bool sendall = false; int xcells, ycells; int rem_x, rem_y; uint32_t *p, *ncrc, *ocrc; uint_t changes, perc, x, y; /* Updates require a preceding client update request. */ if (atomic_exchange(&c->rc_pending, false) == false) return (true); console_refresh(); gci = console_get_image(); /* * It's helpful if the image size or data address does not change * underneath us. */ pthread_mutex_lock(&gci->mtx); /* Check for screen resolution changes. */ if (c->rc_width != gci->width || c->rc_height != gci->height) { c->rc_width = gci->width; c->rc_height = gci->height; c->rc_crc_reset = true; c->rc_send_fullscreen = true; /* If the client supports it, send a resize event. */ if (c->rc_encodings & RFB_ENCODING_RESIZE) { rfb_send_resize_update_msg(c); /* * A resize message counts as an update in response to * the client's preceding request so rc->pending does * not need to be reset here. */ goto done; } } /* Clear old CRC values. */ if (atomic_exchange(&c->rc_crc_reset, false)) memset(c->rc_crc, '\0', c->rc_cells * sizeof (uint32_t)); if (c->rc_custom_pixfmt && (c->rc_gci.data == NULL || c->rc_gci.width != c->rc_width || c->rc_gci.height != c->rc_height)) { c->rc_gci.data = reallocarray(c->rc_gci.data, c->rc_width * c->rc_height, sizeof (uint32_t)); if (c->rc_gci.data == NULL) { retval = false; goto done; } c->rc_gci.width = c->rc_width; c->rc_gci.height = c->rc_height; } else if (!c->rc_custom_pixfmt && c->rc_gci.data != NULL) { free(c->rc_gci.data); c->rc_gci.data = NULL; } sendall = atomic_exchange(&c->rc_send_fullscreen, false); /* * Calculate a checksum for each 32x32 cell. Send all that have * changed since the last scan. */ xcells = howmany(gci->width, RFB_PIX_PER_CELL); ycells = howmany(gci->height, RFB_PIX_PER_CELL); rem_x = gci->width & RFB_PIXCELL_MASK; rem_y = gci->height & RFB_PIXCELL_MASK; if (rem_y == 0) rem_y = RFB_PIX_PER_CELL; p = gci->data; ncrc = c->rc_crc_tmp - xcells; ocrc = c->rc_crc - xcells; changes = 0; memset(c->rc_crc_tmp, '\0', sizeof (uint32_t) * xcells * ycells); for (y = 0; y < gci->height; y++) { if ((y & RFB_PIXCELL_MASK) == 0) { ncrc += xcells; ocrc += xcells; } for (x = 0; x < xcells; x++) { uint_t cellwidth; if (x == xcells - 1 && rem_x > 0) cellwidth = rem_x; else cellwidth = RFB_PIX_PER_CELL; if (rfb_sse42) { ncrc[x] = fast_crc32(p, cellwidth * sizeof (uint32_t), ncrc[x]); } else { ncrc[x] = (uint32_t)crc32(ncrc[x], (Bytef *)p, cellwidth * sizeof (uint32_t)); } p += cellwidth; /* check for crc delta if last row in cell. */ if ((y & RFB_PIXCELL_MASK) == RFB_PIXCELL_MASK || y == gci->height - 1) { if (ocrc[x] != ncrc[x]) { ocrc[x] = ncrc[x]; ncrc[x] = 1; changes++; } else { ncrc[x] = 0; } } } } perc = (changes * 100) / (xcells * ycells); if (rfb_debug > 1 && changes > 0) { rfb_printf(c, RFB_LOGDEBUG, "scanned and found %u changed cell(s) - %u%%", changes, perc); } /* * If there are no changes, don't send an update. Restore the pending * flag since we still owe the client an update. */ if (!sendall && !changes) { c->rc_pending = true; goto done; } /* If there are a lot of changes, send the whole screen. */ if (perc >= RFB_SENDALL_THRESH) sendall = true; if (sendall) { retval = rfb_send_all(c, gci); goto done; } if (!rfb_send_update_header(c, changes)) { retval = false; goto done; } /* Send the changed cells as separate rects. */ ncrc = c->rc_crc_tmp; for (y = 0; y < gci->height; y += RFB_PIX_PER_CELL) { /* Previous cell's row. */ int celly = (y >> RFB_PIXCELL_SHIFT); /* Delta check crc to previous set. */ for (x = 0; x < xcells; x++) { uint_t cellwidth; if (*ncrc++ == 0) continue; if (x == xcells - 1 && rem_x > 0) cellwidth = rem_x; else cellwidth = RFB_PIX_PER_CELL; if (!rfb_send_rect(c, gci, x * RFB_PIX_PER_CELL, celly * RFB_PIX_PER_CELL, cellwidth, y + RFB_PIX_PER_CELL >= gci->height ? rem_y : RFB_PIX_PER_CELL)) { retval = false; goto done; } } } done: pthread_mutex_unlock(&gci->mtx); return (retval); } static void * rfb_client_rx_thread(void *arg) { rfb_client_t *c = arg; unsigned char cmd; bool ret = true; while (ret && !c->rc_closing && (read(c->rc_fd, &cmd, 1) == 1)) { switch (cmd) { case RFBP_CS_SET_PIXEL_FORMAT: ret = rfb_recv_set_pixel_format(c); break; case RFBP_CS_SET_ENCODINGS: ret = rfb_recv_set_encodings(c); break; case RFBP_CS_UPDATE_REQUEST: ret = rfb_recv_update(c); break; case RFBP_CS_KEY_EVENT: ret = rfb_recv_key_event(c); break; case RFBP_CS_POINTER_EVENT: ret = rfb_recv_pointer_event(c); break; case RFBP_CS_CUT_TEXT: ret = rfb_recv_cut_text(c); break; case RFBP_CS_QEMU: ret = rfb_recv_qemu(c); break; default: rfb_printf(c, RFB_LOGWARN, "unknown cs code %d", cmd & 0xff); ret = false; } } rfb_printf(c, RFB_LOGDEBUG, "client rx thread exiting"); c->rc_closing = true; return (NULL); } static void * rfb_client_tx_thread(void *arg) { rfb_client_t *c = arg; rfb_server_t *s = c->rc_s; char tname[MAXCOMLEN + 1]; uint_t counter = 0; hrtime_t tprev; void *status; int err; (void) snprintf(tname, sizeof (tname), "rfb%u tx", c->rc_instance); (void) pthread_set_name_np(c->rc_tx_tid, tname); c->rc_sinfo.rsi_pixfmt = c->rc_s->rs_pixfmt; c->rc_encodings = RFB_ENCODING_RAW; if (!rfb_handshake(c)) { rfb_printf(c, RFB_LOGWARN, "handshake failure"); goto out; } c->rc_cells = howmany(RFB_MAX_WIDTH * RFB_MAX_HEIGHT, RFB_PIX_PER_CELL); if ((c->rc_crc = calloc(c->rc_cells, sizeof (uint32_t))) == NULL || (c->rc_crc_tmp = calloc(c->rc_cells, sizeof (uint32_t))) == NULL) { perror("calloc crc"); goto out; } err = pthread_create(&c->rc_rx_tid, NULL, rfb_client_rx_thread, c); if (err != 0) { perror("pthread_create client rx thread"); goto out; } (void) snprintf(tname, sizeof (tname), "rfb%u rx", c->rc_instance); (void) pthread_set_name_np(c->rc_rx_tid, tname); tprev = gethrtime(); while (!c->rc_closing) { struct timeval tv; hrtime_t tnow; int64_t tdiff; fd_set rfds; int err; FD_ZERO(&rfds); FD_SET(c->rc_fd, &rfds); tv.tv_sec = 0; tv.tv_usec = RFB_SEL_DELAY_US; err = select(c->rc_fd + 1, &rfds, NULL, NULL, &tv); if (err < 0) break; /* Determine if its time to push the screen; ~24hz. */ tnow = gethrtime(); tdiff = NSEC2USEC(tnow - tprev); if (tdiff >= RFB_SCREEN_POLL_DELAY) { bool input; tprev = tnow; input = atomic_exchange(&c->rc_input_detected, false); /* * Refresh the screen on every second trip through the * loop, or if keyboard/mouse input has been detected. */ if ((++counter & 1) != 0 || input) { if (!rfb_send_screen(c)) break; } } else { (void) usleep(RFB_SCREEN_POLL_DELAY - tdiff); } } out: rfb_printf(c, RFB_LOGWARN, "disconnected"); (void) pthread_join(c->rc_rx_tid, &status); pthread_mutex_lock(&s->rs_clientlock); s->rs_clientcount--; list_remove(&s->rs_clients, c); if (s->rs_exclusive && s->rs_clientcount == 0) s->rs_exclusive = false; id_free(rfb_idspace, c->rc_instance); pthread_mutex_unlock(&s->rs_clientlock); rfb_free_client(c); return (NULL); } static void rfb_accept(int sfd, enum ev_type event, void *arg) { rfb_server_t *s = arg; rfb_client_t *c = NULL; struct sockaddr_storage cliaddr; socklen_t len; char host[NI_MAXHOST], port[NI_MAXSERV]; int cfd, err; uint_t cc; rfb_printf(c, RFB_LOGDEBUG, "incoming connection"); len = sizeof (cliaddr); cfd = accept(sfd, (struct sockaddr *)&cliaddr, &len); if (cfd == -1) { perror("client accept"); return; } *host = *port = '\0'; if (cliaddr.ss_family == AF_UNIX) { rfb_printf(NULL, RFB_LOGDEBUG, "connection on UNIX socket"); (void) strlcpy(host, "", sizeof (host)); } else { err = getnameinfo((struct sockaddr *)&cliaddr, len, host, sizeof (host), port, sizeof (port), NI_NUMERICHOST | NI_NUMERICSERV); if (err != 0) { rfb_printf(NULL, RFB_LOGERR, "getnameinfo: %s", gai_strerror(err)); *host = *port = '\0'; } else { rfb_printf(NULL, RFB_LOGDEBUG, "connection from %s:%s", host, port); } } pthread_mutex_lock(&s->rs_clientlock); cc = s->rs_clientcount; pthread_mutex_unlock(&s->rs_clientlock); if (cc >= RFB_MAX_CLIENTS) { rfb_printf(NULL, RFB_LOGERR, "too many clients, closing connection."); goto fail; } if ((c = calloc(1, sizeof (rfb_client_t))) == NULL) { perror("calloc client"); goto fail; } c->rc_fd = cfd; c->rc_s = s; c->rc_zbuf = malloc(RFB_ZLIB_BUFSZ + 16); if (c->rc_zbuf == NULL) goto fail; pthread_mutex_lock(&s->rs_clientlock); err = pthread_create(&c->rc_tx_tid, NULL, rfb_client_tx_thread, c); if (err != 0) { perror("pthread_create client tx thread"); pthread_mutex_unlock(&s->rs_clientlock); goto fail; } s->rs_clientcount++; list_insert_tail(&s->rs_clients, c); c->rc_instance = id_allocff(rfb_idspace); pthread_mutex_unlock(&s->rs_clientlock); (void) pthread_detach(c->rc_tx_tid); rfb_printf(c, RFB_LOGWARN, "connection from %s", host); return; fail: (void) close(cfd); free(c); } int rfb_init(char *hostname, int port, int wait, const char *password, const char *name) { rfb_server_t *s; #ifndef WITHOUT_CAPSICUM cap_rights_t rights; #endif (void) pthread_once(&rfb_once, rfb_init_once); if (rfb_idspace == NULL) { rfb_printf(NULL, RFB_LOGERR, "rfb_idspace could not be allocated"); return (-1); } if ((s = calloc(1, sizeof (rfb_server_t))) == NULL) { perror("calloc"); return (-1); } s->rs_fd = -1; s->rs_name = name; if (password != NULL && strlen(password) > 0) s->rs_password = password; if (pthread_mutex_init(&s->rs_clientlock, NULL) != 0) { perror("pthread_mutex_init"); free(s); return (-1); } list_create(&s->rs_clients, sizeof (rfb_client_t), offsetof(rfb_client_t, rc_node)); /* Server pixel format. */ s->rs_pixfmt.rp_bpp = RFB_PIX_BPP; s->rs_pixfmt.rp_depth = RFB_PIX_DEPTH; s->rs_pixfmt.rp_bigendian = 0; s->rs_pixfmt.rp_truecolour = 1; s->rs_pixfmt.rp_r_max = htons(RFB_PIX_RMAX); s->rs_pixfmt.rp_g_max = htons(RFB_PIX_GMAX); s->rs_pixfmt.rp_b_max = htons(RFB_PIX_BMAX); s->rs_pixfmt.rp_r_shift = RFB_PIX_RSHIFT; s->rs_pixfmt.rp_g_shift = RFB_PIX_GSHIFT; s->rs_pixfmt.rp_b_shift = RFB_PIX_BSHIFT; /* UNIX socket. */ if (port == -1 && hostname != NULL && *hostname == '/') { struct sockaddr_un sock; s->rs_fd = socket(PF_UNIX, SOCK_STREAM, 0); if (s->rs_fd < 0) { perror("socket"); goto fail; } sock.sun_family = AF_UNIX; if (strlcpy(sock.sun_path, hostname, sizeof (sock.sun_path)) >= sizeof (sock.sun_path)) { rfb_printf(NULL, RFB_LOGERR, "socket path '%s' too long\n", hostname); goto fail; } (void) unlink(hostname); if (bind(s->rs_fd, (struct sockaddr *)&sock, sizeof (sock)) < 0) { perror("bind"); goto fail; } } else { struct addrinfo hints, *ai = NULL; char servname[6]; int e; (void) snprintf(servname, sizeof (servname), "%d", port ? port : RFB_DEFAULT_PORT); if (hostname == NULL || strlen(hostname) == 0) { #if defined(INET) hostname = "127.0.0.1"; #elif defined(INET6) hostname = "[::1]"; #endif } memset(&hints, '\0', sizeof (hints)); hints.ai_family = AF_UNSPEC; hints.ai_socktype = SOCK_STREAM; hints.ai_flags = AI_NUMERICHOST | AI_NUMERICSERV | AI_PASSIVE; if ((e = getaddrinfo(hostname, servname, &hints, &ai)) != 0) { rfb_printf(NULL, RFB_LOGERR, "getaddrinfo: %s", gai_strerror(e)); goto fail; } s->rs_fd = socket(ai->ai_family, ai->ai_socktype, 0); if (s->rs_fd < 0) { perror("socket"); freeaddrinfo(ai); goto fail; } e = 1; (void) setsockopt(s->rs_fd, SOL_SOCKET, SO_REUSEADDR, &e, sizeof (e)); if (bind(s->rs_fd, ai->ai_addr, ai->ai_addrlen) < 0) { perror("bind"); freeaddrinfo(ai); goto fail; } freeaddrinfo(ai); } if (listen(s->rs_fd, 5) < 0) { perror("listen"); goto fail; } #ifndef WITHOUT_CAPSICUM cap_rights_init(&rights, CAP_ACCEPT, CAP_EVENT, CAP_READ, CAP_WRITE); if (caph_rights_limit(s->rs_fd, &rights) == -1) errx(EX_OSERR, "Unable to apply rights for sandbox"); #endif s->rs_connevent = mevent_add(s->rs_fd, EVF_READ, rfb_accept, s); if (s->rs_connevent == NULL) { rfb_printf(NULL, RFB_LOGERR, "Failed to set up rfb connection mevent"); goto fail; } list_insert_tail(&rfb_list, s); /* * Wait for first connection. Since the mevent thread is * not yet running, we can't rely on normal incoming connection * handling. */ if (wait != 0) { fd_set rfds; int e; rfb_printf(NULL, RFB_LOGWARN, "holding boot until first client connection"); for (;;) { FD_ZERO(&rfds); FD_SET(s->rs_fd, &rfds); e = select(s->rs_fd + 1, &rfds, NULL, NULL, NULL); if (e < 0 && errno == EINTR) continue; if (e < 0 || FD_ISSET(s->rs_fd, &rfds)) break; } rfb_printf(NULL, RFB_LOGWARN, "continuing boot"); } return (0); fail: if (s->rs_fd != -1) VERIFY3S(close(s->rs_fd), ==, 0); (void) pthread_mutex_destroy(&s->rs_clientlock); list_destroy(&s->rs_clients); free(s); return (-1); } /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2015 Tycho Nightingale * Copyright 2018 Joyent, Inc. * Copyright 2022 OmniOS Community Edition (OmniOSce) Association. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef _RFB_H_ #define _RFB_H_ #define RFB_DEFAULT_PORT 5900 int rfb_init(char *hostname, int port, int wait, const char *password, const char *name); #endif /* _RFB_H_ */ /* * This file and its contents are supplied under the terms of the * Common Development and Distribution License ("CDDL"), version 1.0. * You may only use this file in accordance with the terms of version * 1.0 of the CDDL. * * A full copy of the text of the CDDL should have accompanied this * source. A copy of the CDDL is also available via the Internet at * http://www.illumos.org/license/CDDL. */ /* * Copyright 2022 OmniOS Community Edition (OmniOSce) Association. */ #ifndef _RFB_IMPL_H #define _RFB_IMPL_H #include #include #include #include #include #include "mevent.h" /* * The ProtocolVersion message consists of 12 bytes interpreted as a string of * ASCII characters in the format "RFB xxx.yyy\n" where xxx and yyy are the * major and minor version numbers, padded with zeros. */ #define RFB_VERSION "RFB 003.008\n" #define RFB_VERSION_LEN (sizeof (RFB_VERSION) - 1) _Static_assert(RFB_VERSION_LEN == 12, "RFB_VERSION length incorrect"); /* Keep synchronised with pci_fbuf.c */ #define RFB_MAX_WIDTH 1920 #define RFB_MAX_HEIGHT 1200 #define RFB_MAX_CLIENTS 10 /* Framebuffer pixel format */ #define RFB_PIX_BPP 32 #define RFB_PIX_DEPTH 24 #define RFB_PIX_RSHIFT 16 #define RFB_PIX_GSHIFT 8 #define RFB_PIX_BSHIFT 0 #define RFB_PIX_RMAX 255 #define RFB_PIX_GMAX 255 #define RFB_PIX_BMAX 255 #define RFB_ZLIB_BUFSZ (RFB_MAX_WIDTH * RFB_MAX_HEIGHT * 4) #define RFB_PIX_PER_CELL 32 #define RFB_PIXCELL_SHIFT 5 #define RFB_PIXCELL_MASK 0x1f #define RFB_SENDALL_THRESH 25 #define RFB_SEL_DELAY_US 10000 #define RFB_SCREEN_REFRESH_DELAY 33300 /* 30 Hz */ #define RFB_SCREEN_POLL_DELAY (RFB_SCREEN_REFRESH_DELAY / 2) /* Client-to-server message types */ #define RFBP_CS_SET_PIXEL_FORMAT 0 #define RFBP_CS_SET_ENCODINGS 2 #define RFBP_CS_UPDATE_REQUEST 3 #define RFBP_CS_KEY_EVENT 4 #define RFBP_CS_POINTER_EVENT 5 #define RFBP_CS_CUT_TEXT 6 #define RFBP_CS_QEMU 255 #define RFBP_CS_QEMU_KEVENT 0 /* Server-to-client message types */ #define RFBP_SC_UPDATE 0 #define RFBP_SC_SET_COLOURMAP_ENTRIES 1 #define RFBP_SC_BELL 2 #define RFBP_SC_CUT_TEXT 3 /* Encodings */ #define RFBP_ENCODING_RAW 0 #define RFBP_ENCODING_ZLIB 6 /* Pseudo-encodings */ #define RFBP_ENCODING_RESIZE -223 #define RFBP_ENCODING_EXT_KEVENT -258 /* QEMU ext. key event */ #define RFBP_ENCODING_DESKTOP_NAME -307 /* Security types */ #define RFBP_SECURITY_INVALID 0 #define RFBP_SECURITY_NONE 1 #define RFBP_SECURITY_VNC_AUTH 2 #define RFBP_SECURITY_VNC_AUTH_LEN 16 #define RFBP_SECURITY_VNC_PASSWD_LEN 8 typedef enum rfb_loglevel { RFB_LOGDEBUG, RFB_LOGWARN, RFB_LOGERR } rfb_loglevel_t; typedef enum rfb_encodings { RFB_ENCODING_RAW = (1ULL << 0), RFB_ENCODING_ZLIB = (1ULL << 1), RFB_ENCODING_RESIZE = (1ULL << 2), RFB_ENCODING_EXT_KEVENT = (1ULL << 3), RFB_ENCODING_DESKTOP_NAME = (1ULL << 4) } rfb_encodings_t; typedef enum rfb_cver { RFB_CVER_3_3, RFB_CVER_3_7, RFB_CVER_3_8 } rfb_cver_t; typedef struct rfb_pixfmt { uint8_t rp_bpp; uint8_t rp_depth; uint8_t rp_bigendian; uint8_t rp_truecolour; uint16_t rp_r_max; uint16_t rp_g_max; uint16_t rp_b_max; uint8_t rp_r_shift; uint8_t rp_g_shift; uint8_t rp_b_shift; uint8_t rp_pad[3]; } __packed rfb_pixfmt_t; /* Server-to-client message formats */ typedef struct rfb_server_info { uint16_t rsi_width; uint16_t rsi_height; rfb_pixfmt_t rsi_pixfmt; uint32_t rsi_namelen; } __packed rfb_server_info_t; typedef struct rfb_server_update_msg { uint8_t rss_type; uint8_t rss_pad; uint16_t rss_numrects; } __packed rfb_server_update_msg_t; typedef struct rfb_rect_hdr { uint16_t rr_x; uint16_t rr_y; uint16_t rr_width; uint16_t rr_height; uint32_t rr_encoding; } __packed rfb_rect_hdr_t; /* Client-to-server message formats */ typedef struct rfb_cs_pixfmt_msg { uint8_t rp_pad[3]; rfb_pixfmt_t rp_pixfmt; } __packed rfb_cs_pixfmt_msg_t; typedef struct rfb_cs_update_msg { uint8_t rum_incremental; uint16_t rum_x; uint16_t rum_y; uint16_t rum_width; uint16_t rum_height; } __packed rfb_cs_update_msg_t; typedef struct rfb_cs_encodings_msg { uint8_t re_pad; uint16_t re_numencs; } __packed rfb_cs_encodings_msg_t; typedef struct rfb_cs_key_event_msg { uint8_t rke_down; uint16_t rke_pad; uint32_t rke_sym; } __packed rfb_cs_key_event_msg_t; typedef struct rfb_cs_pointer_event_msg { uint8_t rpe_button; uint16_t rpe_x; uint16_t rpe_y; } __packed rfb_cs_pointer_event_msg_t; typedef struct rfb_cs_cut_text_msg { uint8_t rct_padding[3]; uint32_t rct_length; } __packed rfb_cs_cut_text_msg_t; typedef struct rfb_cs_qemu_msg { uint8_t rq_subtype; } __packed rfb_cs_qemu_msg_t; typedef struct rfb_cs_qemu_extended_key_msg { uint16_t rqek_down; uint32_t rqek_sym; uint32_t rqek_code; } __packed rfb_cs_qemu_extended_key_msg_t; /* Client/server data structures */ typedef struct rfb_server { list_node_t rs_node; int rs_fd; const char *rs_name; const char *rs_password; struct mevent *rs_connevent; uint_t rs_clientcount; pthread_mutex_t rs_clientlock; list_t rs_clients; bool rs_exclusive; rfb_pixfmt_t rs_pixfmt; } rfb_server_t; typedef struct rfb_client { list_node_t rc_node; uint_t rc_instance; int rc_fd; rfb_server_t *rc_s; rfb_server_info_t rc_sinfo; pthread_t rc_rx_tid; pthread_t rc_tx_tid; int rc_width; int rc_height; size_t rc_cells; uint32_t *rc_crc; uint32_t *rc_crc_tmp; z_stream rc_zstream; uint8_t *rc_zbuf; struct bhyvegc_image rc_gci; rfb_cver_t rc_cver; rfb_encodings_t rc_encodings; atomic_bool rc_closing; atomic_bool rc_pending; atomic_bool rc_input_detected; atomic_bool rc_crc_reset; atomic_bool rc_send_fullscreen; bool rc_custom_pixfmt; bool rc_keyevent_sent; } rfb_client_t; #endif /* _RFB_IMPL_H */ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2014 Tycho Nightingale * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include #include #include #include #include #include #include #include #include #include #include #ifndef __FreeBSD__ #include #endif #include "bhyverun.h" #include "config.h" #include "debug.h" #include "smbiostbl.h" #define MB (1024*1024) #define GB (1024ULL*1024*1024) #define SMBIOS_BASE 0xF1000 #define FIRMWARE_VERSION "14.0" /* The SMBIOS specification defines the date format to be mm/dd/yyyy */ #define FIRMWARE_RELEASE_DATE "10/10/2021" /* BHYVE_ACPI_BASE - SMBIOS_BASE) */ #define SMBIOS_MAX_LENGTH (0xF2400 - 0xF1000) #define SMBIOS_TYPE_BIOS 0 #define SMBIOS_TYPE_SYSTEM 1 #define SMBIOS_TYPE_BOARD 2 #define SMBIOS_TYPE_CHASSIS 3 #define SMBIOS_TYPE_PROCESSOR 4 #define SMBIOS_TYPE_MEMARRAY 16 #define SMBIOS_TYPE_MEMDEVICE 17 #define SMBIOS_TYPE_MEMARRAYMAP 19 #define SMBIOS_TYPE_BOOT 32 #define SMBIOS_TYPE_EOT 127 struct smbios_structure { uint8_t type; uint8_t length; uint16_t handle; } __packed; struct smbios_string { const char *node; const char *value; }; typedef int (*initializer_func_t)(const struct smbios_structure *template_entry, const struct smbios_string *template_strings, char *curaddr, char **endaddr, uint16_t *n); struct smbios_template_entry { const struct smbios_structure *entry; const struct smbios_string *strings; initializer_func_t initializer; }; /* * SMBIOS Structure Table Entry Point */ #define SMBIOS_ENTRY_EANCHOR "_SM_" #define SMBIOS_ENTRY_EANCHORLEN 4 #define SMBIOS_ENTRY_IANCHOR "_DMI_" #define SMBIOS_ENTRY_IANCHORLEN 5 struct smbios_entry_point { char eanchor[4]; /* anchor tag */ uint8_t echecksum; /* checksum of entry point structure */ uint8_t eplen; /* length in bytes of entry point */ uint8_t major; /* major version of the SMBIOS spec */ uint8_t minor; /* minor version of the SMBIOS spec */ uint16_t maxssize; /* maximum size in bytes of a struct */ uint8_t revision; /* entry point structure revision */ uint8_t format[5]; /* entry point rev-specific data */ char ianchor[5]; /* intermediate anchor tag */ uint8_t ichecksum; /* intermediate checksum */ uint16_t stlen; /* len in bytes of structure table */ uint32_t staddr; /* physical addr of structure table */ uint16_t stnum; /* number of structure table entries */ uint8_t bcdrev; /* BCD value representing DMI ver */ } __packed; /* * BIOS Information */ #define SMBIOS_FL_ISA 0x00000010 /* ISA is supported */ #define SMBIOS_FL_PCI 0x00000080 /* PCI is supported */ #define SMBIOS_FL_SHADOW 0x00001000 /* BIOS shadowing is allowed */ #define SMBIOS_FL_CDBOOT 0x00008000 /* Boot from CD is supported */ #define SMBIOS_FL_SELBOOT 0x00010000 /* Selectable Boot supported */ #define SMBIOS_FL_EDD 0x00080000 /* EDD Spec is supported */ #define SMBIOS_XB1_FL_ACPI 0x00000001 /* ACPI is supported */ #define SMBIOS_XB2_FL_BBS 0x00000001 /* BIOS Boot Specification */ #define SMBIOS_XB2_FL_VM 0x00000010 /* Virtual Machine */ struct smbios_table_type0 { struct smbios_structure header; uint8_t vendor; /* vendor string */ uint8_t version; /* version string */ uint16_t segment; /* address segment location */ uint8_t rel_date; /* release date */ uint8_t size; /* rom size */ uint64_t cflags; /* characteristics */ uint8_t xc_bytes[2]; /* characteristics ext bytes */ uint8_t sb_major_rel; /* system bios version */ uint8_t sb_minor_rele; uint8_t ecfw_major_rel; /* embedded ctrl fw version */ uint8_t ecfw_minor_rel; } __packed; /* * System Information */ #define SMBIOS_WAKEUP_SWITCH 0x06 /* power switch */ struct smbios_table_type1 { struct smbios_structure header; uint8_t manufacturer; /* manufacturer string */ uint8_t product; /* product name string */ uint8_t version; /* version string */ uint8_t serial; /* serial number string */ uint8_t uuid[16]; /* uuid byte array */ uint8_t wakeup; /* wake-up event */ uint8_t sku; /* sku number string */ uint8_t family; /* family name string */ } __packed; /* * Baseboard (or Module) Information */ #define SMBIOS_BRF_HOSTING 0x1 #define SMBIOS_BRT_MOTHERBOARD 0xa struct smbios_table_type2 { struct smbios_structure header; uint8_t manufacturer; /* manufacturer string */ uint8_t product; /* product name string */ uint8_t version; /* version string */ uint8_t serial; /* serial number string */ uint8_t asset; /* asset tag string */ uint8_t fflags; /* feature flags */ uint8_t location; /* location in chassis */ uint16_t chandle; /* chassis handle */ uint8_t type; /* board type */ uint8_t n_objs; /* number of contained object handles */ } __packed; /* * System Enclosure or Chassis */ #define SMBIOS_CHT_UNKNOWN 0x02 /* unknown */ #define SMBIOS_CHT_DESKTOP 0x03 /* desktop */ #define SMBIOS_CHST_SAFE 0x03 /* safe */ #define SMBIOS_CHSC_NONE 0x03 /* none */ struct smbios_table_type3 { struct smbios_structure header; uint8_t manufacturer; /* manufacturer string */ uint8_t type; /* type */ uint8_t version; /* version string */ uint8_t serial; /* serial number string */ uint8_t asset; /* asset tag string */ uint8_t bustate; /* boot-up state */ uint8_t psstate; /* power supply state */ uint8_t tstate; /* thermal state */ uint8_t security; /* security status */ uint32_t oemdata; /* OEM-specific data */ uint8_t uheight; /* height in 'u's */ uint8_t cords; /* number of power cords */ uint8_t elems; /* number of element records */ uint8_t elemlen; /* length of records */ uint8_t sku; /* sku number string */ } __packed; /* * Processor Information */ #define SMBIOS_PRT_CENTRAL 0x03 /* central processor */ #define SMBIOS_PRF_OTHER 0x01 /* other */ #define SMBIOS_PRS_PRESENT 0x40 /* socket is populated */ #define SMBIOS_PRS_ENABLED 0x1 /* enabled */ #define SMBIOS_PRU_NONE 0x06 /* none */ #define SMBIOS_PFL_64B 0x04 /* 64-bit capable */ struct smbios_table_type4 { struct smbios_structure header; uint8_t socket; /* socket designation string */ uint8_t type; /* processor type */ uint8_t family; /* processor family */ uint8_t manufacturer; /* manufacturer string */ uint64_t cpuid; /* processor cpuid */ uint8_t version; /* version string */ uint8_t voltage; /* voltage */ uint16_t clkspeed; /* ext clock speed in mhz */ uint16_t maxspeed; /* maximum speed in mhz */ uint16_t curspeed; /* current speed in mhz */ uint8_t status; /* status */ uint8_t upgrade; /* upgrade */ uint16_t l1handle; /* l1 cache handle */ uint16_t l2handle; /* l2 cache handle */ uint16_t l3handle; /* l3 cache handle */ uint8_t serial; /* serial number string */ uint8_t asset; /* asset tag string */ uint8_t part; /* part number string */ uint8_t cores; /* cores per socket */ uint8_t ecores; /* enabled cores */ uint8_t threads; /* threads per socket */ uint16_t cflags; /* processor characteristics */ uint16_t family2; /* processor family 2 */ } __packed; /* * Physical Memory Array */ #define SMBIOS_MAL_SYSMB 0x03 /* system board or motherboard */ #define SMBIOS_MAU_SYSTEM 0x03 /* system memory */ #define SMBIOS_MAE_NONE 0x03 /* none */ struct smbios_table_type16 { struct smbios_structure header; uint8_t location; /* physical device location */ uint8_t use; /* device functional purpose */ uint8_t ecc; /* err detect/correct method */ uint32_t size; /* max mem capacity in kb */ uint16_t errhand; /* handle of error (if any) */ uint16_t ndevs; /* num of slots or sockets */ uint64_t xsize; /* max mem capacity in bytes */ } __packed; /* * Memory Device */ #define SMBIOS_MDFF_UNKNOWN 0x02 /* unknown */ #define SMBIOS_MDT_UNKNOWN 0x02 /* unknown */ #define SMBIOS_MDF_UNKNOWN 0x0004 /* unknown */ struct smbios_table_type17 { struct smbios_structure header; uint16_t arrayhand; /* handle of physl mem array */ uint16_t errhand; /* handle of mem error data */ uint16_t twidth; /* total width in bits */ uint16_t dwidth; /* data width in bits */ uint16_t size; /* size in kb or mb */ uint8_t form; /* form factor */ uint8_t set; /* set */ uint8_t dloc; /* device locator string */ uint8_t bloc; /* phys bank locator string */ uint8_t type; /* memory type */ uint16_t flags; /* memory characteristics */ uint16_t maxspeed; /* maximum speed in mhz */ uint8_t manufacturer; /* manufacturer string */ uint8_t serial; /* serial number string */ uint8_t asset; /* asset tag string */ uint8_t part; /* part number string */ uint8_t attributes; /* attributes */ uint32_t xsize; /* extended size in mb */ uint16_t curspeed; /* current speed in mhz */ uint16_t minvoltage; /* minimum voltage */ uint16_t maxvoltage; /* maximum voltage */ uint16_t curvoltage; /* configured voltage */ } __packed; /* * Memory Array Mapped Address */ struct smbios_table_type19 { struct smbios_structure header; uint32_t saddr; /* start phys addr in kb */ uint32_t eaddr; /* end phys addr in kb */ uint16_t arrayhand; /* physical mem array handle */ uint8_t width; /* num of dev in row */ uint64_t xsaddr; /* start phys addr in bytes */ uint64_t xeaddr; /* end phys addr in bytes */ } __packed; /* * System Boot Information */ #define SMBIOS_BOOT_NORMAL 0 /* no errors detected */ struct smbios_table_type32 { struct smbios_structure header; uint8_t reserved[6]; uint8_t status; /* boot status */ } __packed; /* * End-of-Table */ struct smbios_table_type127 { struct smbios_structure header; } __packed; static const struct smbios_table_type0 smbios_type0_template = { { SMBIOS_TYPE_BIOS, sizeof (struct smbios_table_type0), 0 }, 1, /* bios vendor string */ 2, /* bios version string */ 0xF000, /* bios address segment location */ 3, /* bios release date */ 0x0, /* bios size (64k * (n + 1) is the size in bytes) */ SMBIOS_FL_ISA | SMBIOS_FL_PCI | SMBIOS_FL_SHADOW | SMBIOS_FL_CDBOOT | SMBIOS_FL_EDD, { SMBIOS_XB1_FL_ACPI, SMBIOS_XB2_FL_BBS | SMBIOS_XB2_FL_VM }, 0x0, /* bios major release */ 0x0, /* bios minor release */ 0xff, /* embedded controller firmware major release */ 0xff /* embedded controller firmware minor release */ }; static const struct smbios_string smbios_type0_strings[] = { { "bios.vendor", "BHYVE" }, /* vendor string */ { "bios.version", FIRMWARE_VERSION }, /* bios version string */ { "bios.release_date", FIRMWARE_RELEASE_DATE }, /* bios release date string */ { 0 } }; static const struct smbios_table_type1 smbios_type1_template = { { SMBIOS_TYPE_SYSTEM, sizeof (struct smbios_table_type1), 0 }, 1, /* manufacturer string */ 2, /* product string */ 3, /* version string */ 4, /* serial number string */ { 0 }, SMBIOS_WAKEUP_SWITCH, 5, /* sku string */ 6 /* family string */ }; static int smbios_type1_initializer(const struct smbios_structure *template_entry, const struct smbios_string *template_strings, char *curaddr, char **endaddr, uint16_t *n); static const struct smbios_string smbios_type1_strings[] = { { "system.manufacturer", "illumos" }, /* manufacturer string */ { "system.product_name", "BHYVE" }, /* product string */ { "system.version", "1.0" }, /* version string */ { "system.serial_number", "None" }, /* serial number string */ { "system.sku", "None" }, /* sku string */ { "system.family_name", "Virtual Machine" }, /* family string */ { 0 } }; static const struct smbios_table_type2 smbios_type2_template = { { SMBIOS_TYPE_BOARD, sizeof (struct smbios_table_type2), 0 }, 1, /* manufacturer string */ 2, /* product string */ 3, /* version string */ 4, /* serial number string */ 5, /* asset tag string */ SMBIOS_BRF_HOSTING, /* feature flags */ 6, /* location string */ SMBIOS_CHT_DESKTOP, /* chassis handle */ SMBIOS_BRT_MOTHERBOARD, /* board type */ 0 }; static const struct smbios_string smbios_type2_strings[] = { { "board.manufacturer", "illumos" }, /* manufacturer string */ { "board.product_name", "BHYVE" }, /* product name string */ { "board.version", "1.0" }, /* version string */ { "board.serial_number", "None" }, /* serial number string */ { "board.asset_tag", "None" }, /* asset tag string */ { "board.location", "None" }, /* location string */ { 0 } }; static const struct smbios_table_type3 smbios_type3_template = { { SMBIOS_TYPE_CHASSIS, sizeof (struct smbios_table_type3), 0 }, 1, /* manufacturer string */ SMBIOS_CHT_UNKNOWN, 2, /* version string */ 3, /* serial number string */ 4, /* asset tag string */ SMBIOS_CHST_SAFE, SMBIOS_CHST_SAFE, SMBIOS_CHST_SAFE, SMBIOS_CHSC_NONE, 0, /* OEM specific data, we have none */ 0, /* height in 'u's (0=enclosure height unspecified) */ 0, /* number of power cords (0=number unspecified) */ 0, /* number of contained element records */ 0, /* length of records */ 5 /* sku number string */ }; static const struct smbios_string smbios_type3_strings[] = { { "chassis.manufacturer", "illumos" }, /* manufacturer string */ { "chassis.version", "1.0" }, /* version string */ { "chassis.serial_number", "None" }, /* serial number string */ { "chassis.asset_tag", "None" }, /* asset tag string */ { "chassis.sku", "None" }, /* sku number string */ { 0 } }; static const struct smbios_table_type4 smbios_type4_template = { { SMBIOS_TYPE_PROCESSOR, sizeof (struct smbios_table_type4), 0 }, 1, /* socket designation string */ SMBIOS_PRT_CENTRAL, SMBIOS_PRF_OTHER, 2, /* manufacturer string */ 0, /* cpuid */ 3, /* version string */ 0, /* voltage */ 0, /* external clock frequency in mhz (0=unknown) */ 0, /* maximum frequency in mhz (0=unknown) */ 0, /* current frequency in mhz (0=unknown) */ SMBIOS_PRS_PRESENT | SMBIOS_PRS_ENABLED, SMBIOS_PRU_NONE, -1, /* l1 cache handle */ -1, /* l2 cache handle */ -1, /* l3 cache handle */ 4, /* serial number string */ 5, /* asset tag string */ 6, /* part number string */ 0, /* cores per socket (0=unknown) */ 0, /* enabled cores per socket (0=unknown) */ 0, /* threads per socket (0=unknown) */ SMBIOS_PFL_64B, SMBIOS_PRF_OTHER }; static const struct smbios_string smbios_type4_strings[] = { { NULL, " " }, /* socket designation string */ { NULL, " " }, /* manufacturer string */ { NULL, " " }, /* version string */ { NULL, "None" }, /* serial number string */ { NULL, "None" }, /* asset tag string */ { NULL, "None" }, /* part number string */ { 0 } }; static int smbios_type4_initializer( const struct smbios_structure *template_entry, const struct smbios_string *template_strings, char *curaddr, char **endaddr, uint16_t *n); static const struct smbios_table_type16 smbios_type16_template = { { SMBIOS_TYPE_MEMARRAY, sizeof (struct smbios_table_type16), 0 }, SMBIOS_MAL_SYSMB, SMBIOS_MAU_SYSTEM, SMBIOS_MAE_NONE, 0x80000000, /* max mem capacity in kb (0x80000000=use extended) */ -1, /* handle of error (if any) */ 0, /* number of slots or sockets (TBD) */ 0 /* extended maximum memory capacity in bytes (TBD) */ }; static int smbios_type16_initializer( const struct smbios_structure *template_entry, const struct smbios_string *template_strings, char *curaddr, char **endaddr, uint16_t *n); static const struct smbios_table_type17 smbios_type17_template = { { SMBIOS_TYPE_MEMDEVICE, sizeof (struct smbios_table_type17), 0 }, -1, /* handle of physical memory array */ -1, /* handle of memory error data */ 64, /* total width in bits including ecc */ 64, /* data width in bits */ 0, /* size in kb or mb (0x7fff=use extended)*/ SMBIOS_MDFF_UNKNOWN, 0, /* set (0x00=none, 0xff=unknown) */ 1, /* device locator string */ 2, /* physical bank locator string */ SMBIOS_MDT_UNKNOWN, SMBIOS_MDF_UNKNOWN, 0, /* maximum memory speed in mhz (0=unknown) */ 3, /* manufacturer string */ 4, /* serial number string */ 5, /* asset tag string */ 6, /* part number string */ 0, /* attributes (0=unknown rank information) */ 0, /* extended size in mb (TBD) */ 0, /* current speed in mhz (0=unknown) */ 0, /* minimum voltage in mv (0=unknown) */ 0, /* maximum voltage in mv (0=unknown) */ 0 /* configured voltage in mv (0=unknown) */ }; static const struct smbios_string smbios_type17_strings[] = { { NULL, " " }, /* device locator string */ { NULL, " " }, /* physical bank locator string */ { NULL, " " }, /* manufacturer string */ { NULL, "None" }, /* serial number string */ { NULL, "None" }, /* asset tag string */ { NULL, "None" }, /* part number string */ { 0 } }; static int smbios_type17_initializer( const struct smbios_structure *template_entry, const struct smbios_string *template_strings, char *curaddr, char **endaddr, uint16_t *n); static const struct smbios_table_type19 smbios_type19_template = { { SMBIOS_TYPE_MEMARRAYMAP, sizeof (struct smbios_table_type19), 0 }, 0xffffffff, /* starting phys addr in kb (0xffffffff=use ext) */ 0xffffffff, /* ending phys addr in kb (0xffffffff=use ext) */ -1, /* physical memory array handle */ 1, /* number of devices that form a row */ 0, /* extended starting phys addr in bytes (TDB) */ 0 /* extended ending phys addr in bytes (TDB) */ }; static int smbios_type19_initializer( const struct smbios_structure *template_entry, const struct smbios_string *template_strings, char *curaddr, char **endaddr, uint16_t *n); static struct smbios_table_type32 smbios_type32_template = { { SMBIOS_TYPE_BOOT, sizeof (struct smbios_table_type32), 0 }, { 0, 0, 0, 0, 0, 0 }, SMBIOS_BOOT_NORMAL }; static const struct smbios_table_type127 smbios_type127_template = { { SMBIOS_TYPE_EOT, sizeof (struct smbios_table_type127), 0 } }; static int smbios_generic_initializer( const struct smbios_structure *template_entry, const struct smbios_string *template_strings, char *curaddr, char **endaddr, uint16_t *n); static struct smbios_template_entry smbios_template[] = { { (const struct smbios_structure *)&smbios_type0_template, smbios_type0_strings, smbios_generic_initializer }, { (const struct smbios_structure *)&smbios_type1_template, smbios_type1_strings, smbios_type1_initializer }, { (const struct smbios_structure *)&smbios_type2_template, smbios_type2_strings, smbios_generic_initializer }, { (const struct smbios_structure *)&smbios_type3_template, smbios_type3_strings, smbios_generic_initializer }, { (const struct smbios_structure *)&smbios_type4_template, smbios_type4_strings, smbios_type4_initializer }, { (const struct smbios_structure *)&smbios_type16_template, NULL, smbios_type16_initializer }, { (const struct smbios_structure *)&smbios_type17_template, smbios_type17_strings, smbios_type17_initializer }, { (const struct smbios_structure *)&smbios_type19_template, NULL, smbios_type19_initializer }, { (const struct smbios_structure *)&smbios_type32_template, NULL, smbios_generic_initializer }, { (const struct smbios_structure *)&smbios_type127_template, NULL, smbios_generic_initializer }, { NULL,NULL, NULL } }; static uint64_t guest_lomem, guest_himem, guest_himem_base; static uint16_t type16_handle; static int smbios_generic_initializer(const struct smbios_structure *template_entry, const struct smbios_string *template_strings, char *curaddr, char **endaddr, uint16_t *n) { struct smbios_structure *entry; memcpy(curaddr, template_entry, template_entry->length); entry = (struct smbios_structure *)curaddr; entry->handle = *n + 1; curaddr += entry->length; if (template_strings != NULL) { int i; for (i = 0; template_strings[i].value != NULL; i++) { const char *string; int len; if (template_strings[i].node == NULL) { string = template_strings[i].value; } else { set_config_value_if_unset( template_strings[i].node, template_strings[i].value); string = get_config_value( template_strings[i].node); } len = strlen(string) + 1; memcpy(curaddr, string, len); curaddr += len; } *curaddr = '\0'; curaddr++; } else { /* Minimum string section is double nul */ *curaddr = '\0'; curaddr++; *curaddr = '\0'; curaddr++; } (*n)++; *endaddr = curaddr; return (0); } static int smbios_type1_initializer(const struct smbios_structure *template_entry, const struct smbios_string *template_strings, char *curaddr, char **endaddr, uint16_t *n) { struct smbios_table_type1 *type1; const char *guest_uuid_str; smbios_generic_initializer(template_entry, template_strings, curaddr, endaddr, n); type1 = (struct smbios_table_type1 *)curaddr; guest_uuid_str = get_config_value("uuid"); if (guest_uuid_str != NULL) { uuid_t uuid; uint32_t status; uuid_from_string(guest_uuid_str, &uuid, &status); if (status != uuid_s_ok) { EPRINTLN("Invalid UUID"); return (-1); } uuid_enc_le(&type1->uuid, &uuid); } else { MD5_CTX mdctx; u_char digest[16]; char hostname[MAXHOSTNAMELEN]; const char *vmname; /* * Universally unique and yet reproducible are an * oxymoron, however reproducible is desirable in * this case. */ if (gethostname(hostname, sizeof(hostname))) return (-1); MD5Init(&mdctx); vmname = get_config_value("name"); MD5Update(&mdctx, vmname, strlen(vmname)); MD5Update(&mdctx, hostname, sizeof(hostname)); MD5Final(digest, &mdctx); /* * Set the variant and version number. */ digest[6] &= 0x0F; digest[6] |= 0x30; /* version 3 */ digest[8] &= 0x3F; digest[8] |= 0x80; memcpy(&type1->uuid, digest, sizeof (digest)); } return (0); } static int smbios_type4_initializer(const struct smbios_structure *template_entry, const struct smbios_string *template_strings, char *curaddr, char **endaddr, uint16_t *n) { int i; for (i = 0; i < cpu_sockets; i++) { struct smbios_table_type4 *type4; char *p; int nstrings, len; smbios_generic_initializer(template_entry, template_strings, curaddr, endaddr, n); type4 = (struct smbios_table_type4 *)curaddr; p = curaddr + sizeof (struct smbios_table_type4); nstrings = 0; while (p < *endaddr - 1) { if (*p++ == '\0') nstrings++; } len = sprintf(*endaddr - 1, "CPU #%d", i) + 1; *endaddr += len - 1; *(*endaddr) = '\0'; (*endaddr)++; type4->socket = nstrings + 1; /* Revise cores and threads after update to smbios 3.0 */ if (cpu_cores > 254) type4->cores = 0; else type4->cores = cpu_cores; /* This threads is total threads in a socket */ if (cpu_cores * cpu_threads > 254) type4->threads = 0; else type4->threads = cpu_cores * cpu_threads; curaddr = *endaddr; } return (0); } static int smbios_type16_initializer(const struct smbios_structure *template_entry, const struct smbios_string *template_strings, char *curaddr, char **endaddr, uint16_t *n) { struct smbios_table_type16 *type16; type16_handle = *n; smbios_generic_initializer(template_entry, template_strings, curaddr, endaddr, n); type16 = (struct smbios_table_type16 *)curaddr; type16->xsize = guest_lomem + guest_himem; type16->ndevs = guest_himem > 0 ? 2 : 1; return (0); } static int smbios_type17_initializer(const struct smbios_structure *template_entry, const struct smbios_string *template_strings, char *curaddr, char **endaddr, uint16_t *n) { struct smbios_table_type17 *type17; uint64_t memsize, size_KB, size_MB; smbios_generic_initializer(template_entry, template_strings, curaddr, endaddr, n); type17 = (struct smbios_table_type17 *)curaddr; type17->arrayhand = type16_handle; memsize = guest_lomem + guest_himem; size_KB = memsize / 1024; size_MB = memsize / MB; /* A single Type 17 entry can't represent more than ~2PB RAM */ if (size_MB > 0x7FFFFFFF) { printf("Warning: guest memory too big for SMBIOS Type 17 table: " "%luMB greater than max supported 2147483647MB\n", size_MB); size_MB = 0x7FFFFFFF; } /* See SMBIOS 2.7.0 section 7.18 - Memory Device (Type 17) */ if (size_KB <= 0x7FFF) { /* Can represent up to 32767KB with the top bit set */ type17->size = size_KB | (1 << 15); } else if (size_MB < 0x7FFF) { /* Can represent up to 32766MB with the top bit unset */ type17->size = size_MB & 0x7FFF; } else { type17->size = 0x7FFF; /* * Can represent up to 2147483647MB (~2PB) * The top bit is reserved */ type17->xsize = size_MB & 0x7FFFFFFF; } return (0); } static int smbios_type19_initializer(const struct smbios_structure *template_entry, const struct smbios_string *template_strings, char *curaddr, char **endaddr, uint16_t *n) { struct smbios_table_type19 *type19; smbios_generic_initializer(template_entry, template_strings, curaddr, endaddr, n); type19 = (struct smbios_table_type19 *)curaddr; type19->arrayhand = type16_handle; type19->xsaddr = 0; type19->xeaddr = guest_lomem; if (guest_himem > 0) { curaddr = *endaddr; smbios_generic_initializer(template_entry, template_strings, curaddr, endaddr, n); type19 = (struct smbios_table_type19 *)curaddr; type19->arrayhand = type16_handle; type19->xsaddr = guest_himem_base; type19->xeaddr = guest_himem_base + guest_himem; } return (0); } static void smbios_ep_initializer(struct smbios_entry_point *smbios_ep, uint32_t staddr) { memset(smbios_ep, 0, sizeof(*smbios_ep)); memcpy(smbios_ep->eanchor, SMBIOS_ENTRY_EANCHOR, SMBIOS_ENTRY_EANCHORLEN); smbios_ep->eplen = 0x1F; assert(sizeof (struct smbios_entry_point) == smbios_ep->eplen); smbios_ep->major = 2; smbios_ep->minor = 6; smbios_ep->revision = 0; memcpy(smbios_ep->ianchor, SMBIOS_ENTRY_IANCHOR, SMBIOS_ENTRY_IANCHORLEN); smbios_ep->staddr = staddr; smbios_ep->bcdrev = (smbios_ep->major & 0xf) << 4 | (smbios_ep->minor & 0xf); } static void smbios_ep_finalizer(struct smbios_entry_point *smbios_ep, uint16_t len, uint16_t num, uint16_t maxssize) { uint8_t checksum; int i; smbios_ep->maxssize = maxssize; smbios_ep->stlen = len; smbios_ep->stnum = num; checksum = 0; for (i = 0x10; i < 0x1f; i++) { checksum -= ((uint8_t *)smbios_ep)[i]; } smbios_ep->ichecksum = checksum; checksum = 0; for (i = 0; i < 0x1f; i++) { checksum -= ((uint8_t *)smbios_ep)[i]; } smbios_ep->echecksum = checksum; } #ifndef __FreeBSD__ /* * bhyve on illumos previously used configuration keys starting with 'smbios.' * to control type 1 SMBIOS information. Since these may still be present in * bhyve configuration files, the following table is used to translate them * to their new key names. */ static struct { const char *oldkey; const char *newkey; } smbios_legacy_config_map[] = { { "smbios.manufacturer", "system.manufacturer" }, { "smbios.family", "system.family_name" }, { "smbios.product", "system.product_name" }, { "smbios.serial", "system.serial_number" }, { "smbios.sku", "system.sku" }, { "smbios.version", "system.version" }, }; #endif int smbios_build(struct vmctx *ctx) { struct smbios_entry_point *smbios_ep; uint16_t n; uint16_t maxssize; char *curaddr, *startaddr, *ststartaddr; int i; int err; guest_lomem = vm_get_lowmem_size(ctx); guest_himem = vm_get_highmem_size(ctx); guest_himem_base = vm_get_highmem_base(ctx); startaddr = paddr_guest2host(ctx, SMBIOS_BASE, SMBIOS_MAX_LENGTH); if (startaddr == NULL) { EPRINTLN("smbios table requires mapped mem"); return (ENOMEM); } #ifndef __FreeBSD__ /* Translate legacy illumos configuration keys */ for (uint_t i = 0; i < ARRAY_SIZE(smbios_legacy_config_map); i++) { const char *v; v = get_config_value(smbios_legacy_config_map[i].oldkey); if (v != NULL) { set_config_value_if_unset( smbios_legacy_config_map[i].newkey, v); } } #endif curaddr = startaddr; smbios_ep = (struct smbios_entry_point *)curaddr; smbios_ep_initializer(smbios_ep, SMBIOS_BASE + sizeof(struct smbios_entry_point)); curaddr += sizeof(struct smbios_entry_point); ststartaddr = curaddr; n = 0; maxssize = 0; for (i = 0; smbios_template[i].entry != NULL; i++) { const struct smbios_structure *entry; const struct smbios_string *strings; initializer_func_t initializer; char *endaddr; size_t size; entry = smbios_template[i].entry; strings = smbios_template[i].strings; initializer = smbios_template[i].initializer; err = (*initializer)(entry, strings, curaddr, &endaddr, &n); if (err != 0) return (err); size = endaddr - curaddr; assert(size <= UINT16_MAX); if (size > maxssize) maxssize = (uint16_t)size; curaddr = endaddr; } assert(curaddr - startaddr < SMBIOS_MAX_LENGTH); smbios_ep_finalizer(smbios_ep, curaddr - ststartaddr, n, maxssize); return (0); } #ifndef __FreeBSD__ static struct { uint_t type; const char *key; char *val; } smbios_legacy_map[] = { { 1, "product", "product_name" }, { 1, "serial", "serial_number" }, { 1, "family", "family_name" }, }; static const struct smbios_string *smbios_tbl_map[] = { smbios_type0_strings, smbios_type1_strings, smbios_type2_strings, smbios_type3_strings, }; /* * This function accepts an option of the form * type,[key=value][,key=value]... * and sets smbios data for the given type. Keys for type X are defined in the * smbios_typeX_strings tables above, but for type 1 there are also some * legacy values which were accepted in earlier versions of bhyve on illumos * which need to be mapped. */ int smbios_parse(const char *opts) { char *buf, *lasts, *token, *typekey = NULL; const char *errstr; const struct smbios_string *tbl; nvlist_t *nvl; uint_t i; long type; if ((buf = strdup(opts)) == NULL) { (void) fprintf(stderr, "out of memory\n"); return (-1); } if ((token = strtok_r(buf, ",", &lasts)) == NULL) { (void) fprintf(stderr, "too few fields\n"); goto fail; } type = strtonum(token, 0, 3, &errstr); if (errstr != NULL) { fprintf(stderr, "First token (type) is %s\n", errstr); goto fail; } tbl = smbios_tbl_map[type]; /* Extract the config key for this type */ typekey = strdup(tbl[0].node); if (typekey == NULL) { (void) fprintf(stderr, "out of memory\n"); goto fail; } token = strchr(typekey, '.'); assert(token != NULL); *token = '\0'; nvl = create_config_node(typekey); if (nvl == NULL) { (void) fprintf(stderr, "out of memory\n"); goto fail; } while ((token = strtok_r(NULL, ",", &lasts)) != NULL) { char *val; if ((val = strchr(token, '=')) == NULL || val[1] == '\0') { (void) fprintf(stderr, "invalid key=value: '%s'\n", token); goto fail; } *val++ = '\0'; /* UUID is a top-level config item, but -U takes priority */ if (strcmp(token, "uuid") == 0) { set_config_value_if_unset(token, val); continue; } /* Translate legacy keys */ for (i = 0; i < ARRAY_SIZE(smbios_legacy_map); i++) { if (type == smbios_legacy_map[i].type && strcmp(token, smbios_legacy_map[i].key) == 0) { token = smbios_legacy_map[i].val; break; } } for (i = 0; tbl[i].value != NULL; i++) { if (strcmp(tbl[i].node + strlen(typekey) + 1, token) == 0) { /* Found match */ break; } } if (tbl[i].value == NULL) { (void) fprintf(stderr, "Unknown SMBIOS key %s for type %d\n", token, type); goto fail; } set_config_value_node(nvl, token, val); } free(typekey); return (0); fail: free(buf); free(typekey); return (-1); } #endif /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2014 Tycho Nightingale * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * Copyright 2018 Joyent, Inc. */ #ifndef _SMBIOSTBL_H_ #define _SMBIOSTBL_H_ struct vmctx; int smbios_build(struct vmctx *ctx); #ifndef __FreeBSD__ int smbios_parse(const char *opts); void smbios_apply(void); #endif #endif /* _SMBIOSTBL_H_ */ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2015 Nahanni Systems, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include #include #ifndef __FreeBSD__ #include #endif #include #include "sockstream.h" ssize_t stream_read(int fd, void *buf, ssize_t nbytes) { uint8_t *p; ssize_t len = 0; ssize_t n; p = buf; while (len < nbytes) { n = read(fd, p + len, nbytes - len); if (n == 0) break; if (n < 0) { if (errno == EINTR || errno == EAGAIN) continue; return (n); } len += n; } return (len); } ssize_t stream_write(int fd, const void *buf, ssize_t nbytes) { const uint8_t *p; ssize_t len = 0; ssize_t n; p = buf; while (len < nbytes) { #ifdef __FreeBSD__ n = write(fd, p + len, nbytes - len); #else n = send(fd, p + len, nbytes - len, MSG_NOSIGNAL); #endif if (n == 0) break; if (n < 0) { if (errno == EINTR || errno == EAGAIN) continue; return (n); } len += n; } return (len); } /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2015 Nahanni Systems, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include #include ssize_t stream_read(int fd, void *buf, ssize_t nbytes); ssize_t stream_write(int fd, const void *buf, ssize_t nbytes); /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2023 Beckhoff Automation GmbH & Co. KG * Author: Corvin Köhne */ #include #include #include #include #include #include #ifndef __FreeBSD__ #include #include #endif #include #include "acpi.h" #include "acpi_device.h" #include "config.h" #include "tpm_device.h" #include "tpm_emul.h" #include "tpm_intf.h" #include "tpm_ppi.h" #define TPM_ACPI_DEVICE_NAME "TPM" #define TPM_ACPI_HARDWARE_ID "MSFT0101" SET_DECLARE(tpm_emul_set, struct tpm_emul); SET_DECLARE(tpm_intf_set, struct tpm_intf); SET_DECLARE(tpm_ppi_set, struct tpm_ppi); struct tpm_device { struct vmctx *vm_ctx; struct acpi_device *acpi_dev; struct tpm_emul *emul; void *emul_sc; struct tpm_intf *intf; void *intf_sc; struct tpm_ppi *ppi; void *ppi_sc; }; static int tpm_build_acpi_table(const struct acpi_device *const dev) { const struct tpm_device *const tpm = acpi_device_get_softc(dev); if (tpm->intf->build_acpi_table == NULL) { return (0); } return (tpm->intf->build_acpi_table(tpm->intf_sc, tpm->vm_ctx)); } static int tpm_write_dsdt(const struct acpi_device *const dev) { int error; const struct tpm_device *const tpm = acpi_device_get_softc(dev); const struct tpm_ppi *const ppi = tpm->ppi; /* * packages for returns */ dsdt_line("Name(TPM2, Package(2) {0, 0})"); dsdt_line("Name(TPM3, Package(3) {0, 0, 0})"); if (ppi->write_dsdt_regions) { error = ppi->write_dsdt_regions(tpm->ppi_sc); if (error) { warnx("%s: failed to write ppi dsdt regions\n", __func__); return (error); } } /* * Device Specific Method * Arg0: UUID * Arg1: Revision ID * Arg2: Function Index * Arg3: Arguments */ dsdt_line("Method(_DSM, 4, Serialized)"); dsdt_line("{"); dsdt_indent(1); if (ppi->write_dsdt_dsm) { error = ppi->write_dsdt_dsm(tpm->ppi_sc); if (error) { warnx("%s: failed to write ppi dsdt dsm\n", __func__); return (error); } } dsdt_unindent(1); dsdt_line("}"); return (0); } static const struct acpi_device_emul tpm_acpi_device_emul = { .name = TPM_ACPI_DEVICE_NAME, .hid = TPM_ACPI_HARDWARE_ID, .build_table = tpm_build_acpi_table, .write_dsdt = tpm_write_dsdt, }; void tpm_device_destroy(struct tpm_device *const dev) { if (dev == NULL) return; if (dev->ppi != NULL && dev->ppi->deinit != NULL) dev->ppi->deinit(dev->ppi_sc); if (dev->intf != NULL && dev->intf->deinit != NULL) dev->intf->deinit(dev->intf_sc); if (dev->emul != NULL && dev->emul->deinit != NULL) dev->emul->deinit(dev->emul_sc); acpi_device_destroy(dev->acpi_dev); free(dev); } int tpm_device_create(struct tpm_device **const new_dev, struct vmctx *const vm_ctx, nvlist_t *const nvl) { struct tpm_device *dev = NULL; struct tpm_emul **ppemul; struct tpm_intf **ppintf; struct tpm_ppi **pp_ppi; const char *value; int error; if (new_dev == NULL || vm_ctx == NULL) { error = EINVAL; goto err_out; } set_config_value_node_if_unset(nvl, "intf", "crb"); set_config_value_node_if_unset(nvl, "ppi", "qemu"); value = get_config_value_node(nvl, "version"); assert(value != NULL); if (strcmp(value, "2.0")) { warnx("%s: unsupported tpm version %s", __func__, value); error = EINVAL; goto err_out; } dev = calloc(1, sizeof(*dev)); if (dev == NULL) { error = ENOMEM; goto err_out; } dev->vm_ctx = vm_ctx; error = acpi_device_create(&dev->acpi_dev, dev, dev->vm_ctx, &tpm_acpi_device_emul); if (error) goto err_out; value = get_config_value_node(nvl, "type"); assert(value != NULL); SET_FOREACH(ppemul, tpm_emul_set) { if (strcmp(value, (*ppemul)->name)) continue; dev->emul = *ppemul; break; } if (dev->emul == NULL) { warnx("TPM emulation \"%s\" not found", value); error = EINVAL; goto err_out; } if (dev->emul->init) { error = dev->emul->init(&dev->emul_sc, nvl); if (error) goto err_out; } value = get_config_value_node(nvl, "intf"); SET_FOREACH(ppintf, tpm_intf_set) { if (strcmp(value, (*ppintf)->name)) { continue; } dev->intf = *ppintf; break; } if (dev->intf == NULL) { warnx("TPM interface \"%s\" not found", value); error = EINVAL; goto err_out; } if (dev->intf->init) { error = dev->intf->init(&dev->intf_sc, dev->emul, dev->emul_sc, dev->acpi_dev); if (error) goto err_out; } value = get_config_value_node(nvl, "ppi"); SET_FOREACH(pp_ppi, tpm_ppi_set) { if (strcmp(value, (*pp_ppi)->name)) { continue; } dev->ppi = *pp_ppi; break; } if (dev->ppi == NULL) { warnx("TPM PPI \"%s\" not found\n", value); error = EINVAL; goto err_out; } if (dev->ppi->init) { error = dev->ppi->init(&dev->ppi_sc); if (error) goto err_out; } *new_dev = dev; return (0); err_out: tpm_device_destroy(dev); return (error); } static struct tpm_device *lpc_tpm; int init_tpm(struct vmctx *ctx) { nvlist_t *nvl; int error; nvl = find_config_node("tpm"); if (nvl == NULL) return (0); error = tpm_device_create(&lpc_tpm, ctx, nvl); if (error) { warnx("%s: unable to create a TPM device (%d)", __func__, error); return (error); } return (0); } /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2023 Beckhoff Automation GmbH & Co. KG * Author: Corvin Köhne */ #pragma once #include #include "config.h" struct tpm_device; int tpm_device_create(struct tpm_device **new_dev, struct vmctx *vm_ctx, nvlist_t *nvl); void tpm_device_destroy(struct tpm_device *dev); int init_tpm(struct vmctx *ctx); /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2023 Beckhoff Automation GmbH & Co. KG * Author: Corvin Köhne */ #pragma once #include #include "config.h" struct tpm_device; struct tpm_emul { const char *name; int (*init)(void **sc, nvlist_t *nvl); void (*deinit)(void *sc); int (*execute_cmd)(void *sc, void *cmd, uint32_t cmd_size, void *rsp, uint32_t rsp_size); }; #define TPM_EMUL_SET(x) DATA_SET(tpm_emul_set, x) /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2023 Beckhoff Automation GmbH & Co. KG * Author: Corvin Köhne */ #include #include #include #include #ifdef __FreeBSD__ #include #else #include #endif #include #include #include "config.h" #include "tpm_device.h" #include "tpm_emul.h" struct tpm_passthru { int fd; }; struct tpm_resp_hdr { uint16_t tag; uint32_t len; uint32_t errcode; } __packed; static int tpm_passthru_init(void **sc, nvlist_t *nvl) { #ifndef __FreeBSD__ /* * Until illumos has a TPM 2.0 driver, we can't finish plumbing the TPM * pass-through. */ errx(4, "TPM pass-through devices are not yet supported on illumos"); #else struct tpm_passthru *tpm; const char *path; tpm = calloc(1, sizeof(struct tpm_passthru)); if (tpm == NULL) { warnx("%s: failed to allocate tpm passthru", __func__); return (ENOMEM); } path = get_config_value_node(nvl, "path"); tpm->fd = open(path, O_RDWR); if (tpm->fd < 0) { warnx("%s: unable to open tpm device \"%s\"", __func__, path); return (ENOENT); } *sc = tpm; #endif /* __FreeBSD__ */ return (0); } static int tpm_passthru_execute_cmd(void *sc, void *cmd, uint32_t cmd_size, void *rsp, uint32_t rsp_size) { struct tpm_passthru *tpm; ssize_t len; if (rsp_size < (ssize_t)sizeof(struct tpm_resp_hdr)) { warn("%s: rsp_size of %u is too small", __func__, rsp_size); return (EINVAL); } tpm = sc; len = write(tpm->fd, cmd, cmd_size); if (len != cmd_size) { warn("%s: cmd write failed (bytes written: %zd / %d)", __func__, len, cmd_size); return (EFAULT); } len = read(tpm->fd, rsp, rsp_size); if (len < (ssize_t)sizeof(struct tpm_resp_hdr)) { warn("%s: rsp read failed (bytes read: %zd / %d)", __func__, len, rsp_size); return (EFAULT); } return (0); } static void tpm_passthru_deinit(void *sc) { struct tpm_passthru *tpm; tpm = sc; if (tpm == NULL) return; if (tpm->fd >= 0) close(tpm->fd); free(tpm); } static const struct tpm_emul tpm_emul_passthru = { .name = "passthru", .init = tpm_passthru_init, .deinit = tpm_passthru_deinit, .execute_cmd = tpm_passthru_execute_cmd, }; TPM_EMUL_SET(tpm_emul_passthru); /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2024 Hans Rosenfeld * Author: Hans Rosenfeld */ #include #include #include #include #include #include #ifdef __FreeBSD__ #include #else #include #endif #include #include #include #include #include "config.h" #include "tpm_device.h" #include "tpm_emul.h" struct tpm_swtpm { int fd; }; struct tpm_resp_hdr { uint16_t tag; uint32_t len; uint32_t errcode; } __packed; static int tpm_swtpm_init(void **sc, nvlist_t *nvl) { struct tpm_swtpm *tpm; const char *path; struct sockaddr_un tpm_addr; tpm = calloc(1, sizeof (struct tpm_swtpm)); if (tpm == NULL) { warnx("%s: failed to allocate tpm_swtpm", __func__); return (ENOMEM); } path = get_config_value_node(nvl, "path"); if (path == NULL) { warnx("%s: no socket path specified", __func__); return (ENOENT); } tpm->fd = socket(PF_UNIX, SOCK_STREAM | SOCK_CLOEXEC, 0); if (tpm->fd < 0) { warnx("%s: unable to open tpm socket", __func__); return (ENOENT); } bzero(&tpm_addr, sizeof (tpm_addr)); tpm_addr.sun_family = AF_UNIX; strlcpy(tpm_addr.sun_path, path, sizeof (tpm_addr.sun_path) - 1); if (connect(tpm->fd, (struct sockaddr *)&tpm_addr, sizeof (tpm_addr)) == -1) { warnx("%s: unable to connect to tpm socket \"%s\"", __func__, path); return (ENOENT); } *sc = tpm; return (0); } static int tpm_swtpm_execute_cmd(void *sc, void *cmd, uint32_t cmd_size, void *rsp, uint32_t rsp_size) { struct tpm_swtpm *tpm; ssize_t len; if (rsp_size < (ssize_t)sizeof(struct tpm_resp_hdr)) { warn("%s: rsp_size of %u is too small", __func__, rsp_size); return (EINVAL); } tpm = sc; #ifdef __FreeBSD__ len = send(tpm->fd, cmd, cmd_size, MSG_NOSIGNAL|MSG_DONTWAIT); if (len == -1) err(1, "%s: cmd send failed, is swtpm running?", __func__); if (len != cmd_size) { warn("%s: cmd write failed (bytes written: %zd / %d)", __func__, len, cmd_size); return (EFAULT); } len = recv(tpm->fd, rsp, rsp_size, 0); if (len == -1) err(1, "%s: rsp recv failed, is swtpm running?", __func__); if (len < (ssize_t)sizeof(struct tpm_resp_hdr)) { warn("%s: rsp read failed (bytes read: %zd / %d)", __func__, len, rsp_size); return (EFAULT); } #else while (cmd_size > 0) { len = send(tpm->fd, cmd, cmd_size, MSG_NOSIGNAL|MSG_DONTWAIT); if (len == -1) { if (errno == EINTR) continue; err(1, "%s: cmd send failed, is swtpm running?", __func__); } cmd += len; cmd_size -= len; } size_t buflen = rsp_size; size_t rcvd = 0; while (buflen > 0 && rcvd < sizeof (struct tpm_resp_hdr)) { len = recv(tpm->fd, rsp, buflen, 0); if (len == -1) { if (errno == EINTR) continue; err(1, "%s: rsp recv failed, is swtpm running?", __func__); } if (len == 0) break; rsp += len; buflen -= len; rcvd += len; } #endif return (0); } static void tpm_swtpm_deinit(void *sc) { struct tpm_swtpm *tpm; tpm = sc; if (tpm == NULL) return; if (tpm->fd >= 0) close(tpm->fd); free(tpm); } static const struct tpm_emul tpm_emul_swtpm = { .name = "swtpm", .init = tpm_swtpm_init, .deinit = tpm_swtpm_deinit, .execute_cmd = tpm_swtpm_execute_cmd, }; TPM_EMUL_SET(tpm_emul_swtpm); /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2022 Beckhoff Automation GmbH & Co. KG * Author: Corvin Köhne */ #pragma once #include #include "acpi_device.h" #include "config.h" #include "tpm_device.h" #include "tpm_emul.h" #define TPM_INTF_TYPE_FIFO_PTP 0x0 #define TPM_INTF_TYPE_CRB 0x1 #define TPM_INTF_TYPE_FIFO_TIS 0xF #define TPM_INTF_VERSION_FIFO 0 #define TPM_INTF_VERSION_CRB 1 #define TPM_INTF_CAP_CRB_DATA_XFER_SIZE_4 0 #define TPM_INTF_CAP_CRB_DATA_XFER_SIZE_8 1 #define TPM_INTF_CAP_CRB_DATA_XFER_SIZE_32 2 #define TPM_INTF_CAP_CRB_DATA_XFER_SIZE_64 3 #define TPM_INTF_SELECTOR_FIFO 0 #define TPM_INTF_SELECTOR_CRB 1 struct tpm_intf { const char *name; int (*init)(void **sc, struct tpm_emul *emul, void *emul_sc, struct acpi_device *acpi_dev); void (*deinit)(void *sc); int (*build_acpi_table)(void *sc, struct vmctx *vm_ctx); }; #define TPM_INTF_SET(x) DATA_SET(tpm_intf_set, x) /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2022 Beckhoff Automation GmbH & Co. KG * Author: Corvin Köhne */ #include #include #include #include #include #include #include #include #include #include #include #include #include #ifndef __FreeBSD__ #include #endif #include "basl.h" #include "config.h" #include "mem.h" #include "qemu_fwcfg.h" #include "tpm_device.h" #include "tpm_intf.h" #define TPM_CRB_ADDRESS 0xFED40000 #define TPM_CRB_REGS_SIZE 0x1000 #define TPM_CRB_CONTROL_AREA_ADDRESS \ (TPM_CRB_ADDRESS + offsetof(struct tpm_crb_regs, ctrl_req)) #define TPM_CRB_CONTROL_AREA_SIZE TPM_CRB_REGS_SIZE #define TPM_CRB_DATA_BUFFER_ADDRESS \ (TPM_CRB_ADDRESS + offsetof(struct tpm_crb_regs, data_buffer)) #define TPM_CRB_DATA_BUFFER_SIZE 0xF80 #define TPM_CRB_LOCALITIES_MAX 5 #define TPM_CRB_LOG_AREA_MINIMUM_SIZE (64 * 1024) #define TPM_CRB_LOG_AREA_FWCFG_NAME "etc/tpm/log" #define TPM_CRB_INTF_NAME "crb" struct tpm_crb_regs { union tpm_crb_reg_loc_state { struct { uint32_t tpm_established : 1; uint32_t loc_assigned : 1; uint32_t active_locality : 3; uint32_t _reserved : 2; uint32_t tpm_req_valid_sts : 1; }; uint32_t val; } loc_state; /* 0h */ uint8_t _reserved1[4]; /* 4h */ union tpm_crb_reg_loc_ctrl { struct { uint32_t request_access : 1; uint32_t relinquish : 1; uint32_t seize : 1; uint32_t reset_establishment_bit : 1; }; uint32_t val; } loc_ctrl; /* 8h */ union tpm_crb_reg_loc_sts { struct { uint32_t granted : 1; uint32_t been_seized : 1; }; uint32_t val; } loc_sts; /* Ch */ uint8_t _reserved2[0x20]; /* 10h */ union tpm_crb_reg_intf_id { struct { uint64_t interface_type : 4; uint64_t interface_version : 4; uint64_t cap_locality : 1; uint64_t cap_crb_idle_bypass : 1; uint64_t _reserved1 : 1; uint64_t cap_data_xfer_size_support : 2; uint64_t cap_fifo : 1; uint64_t cap_crb : 1; uint64_t _reserved2 : 2; uint64_t interface_selector : 2; uint64_t intf_sel_lock : 1; uint64_t _reserved3 : 4; uint64_t rid : 8; uint64_t vid : 16; uint64_t did : 16; }; uint64_t val; } intf_id; /* 30h */ union tpm_crb_reg_ctrl_ext { struct { uint32_t clear; uint32_t remaining_bytes; }; uint64_t val; } ctrl_ext; /* 38 */ union tpm_crb_reg_ctrl_req { struct { uint32_t cmd_ready : 1; uint32_t go_idle : 1; }; uint32_t val; } ctrl_req; /* 40h */ union tpm_crb_reg_ctrl_sts { struct { uint32_t tpm_sts : 1; uint32_t tpm_idle : 1; }; uint32_t val; } ctrl_sts; /* 44h */ union tpm_crb_reg_ctrl_cancel { struct { uint32_t cancel : 1; }; uint32_t val; } ctrl_cancel; /* 48h */ union tpm_crb_reg_ctrl_start { struct { uint32_t start : 1; }; uint32_t val; } ctrl_start; /* 4Ch*/ uint32_t int_enable; /* 50h */ uint32_t int_sts; /* 54h */ uint32_t cmd_size; /* 58h */ uint32_t cmd_addr_lo; /* 5Ch */ uint32_t cmd_addr_hi; /* 60h */ uint32_t rsp_size; /* 64h */ uint64_t rsp_addr; /* 68h */ uint8_t _reserved3[0x10]; /* 70h */ uint8_t data_buffer[TPM_CRB_DATA_BUFFER_SIZE]; /* 80h */ } __packed; static_assert(sizeof(struct tpm_crb_regs) == TPM_CRB_REGS_SIZE, "Invalid size of tpm_crb"); #define CRB_CMD_SIZE_READ(regs) (regs.cmd_size) #define CRB_CMD_SIZE_WRITE(regs, val) \ do { \ regs.cmd_size = val; \ } while (0) #define CRB_CMD_ADDR_READ(regs) \ (((uint64_t)regs.cmd_addr_hi << 32) | regs.cmd_addr_lo) #define CRB_CMD_ADDR_WRITE(regs, val) \ do { \ regs.cmd_addr_lo = val & 0xFFFFFFFF; \ regs.cmd_addr_hi = val >> 32; \ } while (0) #define CRB_RSP_SIZE_READ(regs) (regs.rsp_size) #define CRB_RSP_SIZE_WRITE(regs, val) \ do { \ regs.rsp_size = val; \ } while (0) #define CRB_RSP_ADDR_READ(regs) (regs.rsp_addr) #define CRB_RSP_ADDR_WRITE(regs, val) \ do { \ regs.rsp_addr = val; \ } while (0) struct tpm_cmd_hdr { uint16_t tag; uint32_t len; union { uint32_t ordinal; uint32_t errcode; }; } __packed; struct tpm_crb { struct tpm_emul *emul; void *emul_sc; uint8_t tpm_log_area[TPM_CRB_LOG_AREA_MINIMUM_SIZE]; struct tpm_crb_regs regs; pthread_t thread; pthread_mutex_t mutex; pthread_cond_t cond; bool closing; }; static void * tpm_crb_thread(void *const arg) { struct tpm_crb *const crb = arg; pthread_mutex_lock(&crb->mutex); for (;;) { /* * We're releasing the lock after wake up. Therefore, we have to * check the closing condition before and after going to sleep. */ if (crb->closing) break; pthread_cond_wait(&crb->cond, &crb->mutex); if (crb->closing) break; const uint64_t cmd_addr = CRB_CMD_ADDR_READ(crb->regs); const uint64_t rsp_addr = CRB_RSP_ADDR_READ(crb->regs); const uint32_t cmd_size = CRB_CMD_SIZE_READ(crb->regs); const uint32_t rsp_size = CRB_RSP_SIZE_READ(crb->regs); if ((cmd_addr < TPM_CRB_DATA_BUFFER_ADDRESS) || (cmd_size < sizeof (struct tpm_cmd_hdr)) || (cmd_size > TPM_CRB_DATA_BUFFER_SIZE) || (cmd_addr + cmd_size > TPM_CRB_DATA_BUFFER_ADDRESS + TPM_CRB_DATA_BUFFER_SIZE)) { warnx("%s: invalid cmd [%16lx/%8x] outside of TPM " "buffer", __func__, cmd_addr, cmd_size); break; } if ((rsp_addr < TPM_CRB_DATA_BUFFER_ADDRESS) || (rsp_size < sizeof (struct tpm_cmd_hdr)) || (rsp_size > TPM_CRB_DATA_BUFFER_SIZE) || (rsp_addr + rsp_size > TPM_CRB_DATA_BUFFER_ADDRESS + TPM_CRB_DATA_BUFFER_SIZE)) { warnx("%s: invalid rsp [%16lx/%8x] outside of TPM " "buffer", __func__, rsp_addr, rsp_size); break; } const uint64_t cmd_off = cmd_addr - TPM_CRB_DATA_BUFFER_ADDRESS; const uint64_t rsp_off = rsp_addr - TPM_CRB_DATA_BUFFER_ADDRESS; if (cmd_off > TPM_CRB_DATA_BUFFER_SIZE || cmd_off + cmd_size > TPM_CRB_DATA_BUFFER_SIZE || rsp_off > TPM_CRB_DATA_BUFFER_SIZE || rsp_off + rsp_size > TPM_CRB_DATA_BUFFER_SIZE) { warnx( "%s: invalid cmd [%16lx, %16lx] --> [%16lx, %16lx]\n\r", __func__, cmd_addr, cmd_addr + cmd_size, rsp_addr, rsp_addr + rsp_size); break; } uint8_t cmd[TPM_CRB_DATA_BUFFER_SIZE]; memcpy(cmd, crb->regs.data_buffer, TPM_CRB_DATA_BUFFER_SIZE); /* * Do a basic sanity check of the TPM request header. We'll need * the TPM request length for execute_cmd() below. */ struct tpm_cmd_hdr *req = (struct tpm_cmd_hdr *)&cmd[cmd_off]; if (be32toh(req->len) < sizeof (struct tpm_cmd_hdr) || be32toh(req->len) > cmd_size) { warnx("%s: invalid TPM request header", __func__); break; } /* * A TPM command can take multiple seconds to execute. As we've * copied all required values and buffers at this point, we can * release the mutex. */ pthread_mutex_unlock(&crb->mutex); /* * The command response buffer interface uses a single buffer * for sending a command to and receiving a response from the * tpm. To avoid reading old data from the command buffer which * might be a security issue, we zero out the command buffer * before writing the response into it. The rsp_size parameter * is controlled by the guest and it's not guaranteed that the * response has a size of rsp_size (e.g. if the tpm returned an * error, the response would have a different size than * expected). For that reason, use a second buffer for the * response. */ uint8_t rsp[TPM_CRB_DATA_BUFFER_SIZE] = { 0 }; (void) crb->emul->execute_cmd(crb->emul_sc, req, be32toh(req->len), &rsp[rsp_off], rsp_size); pthread_mutex_lock(&crb->mutex); memset(crb->regs.data_buffer, 0, TPM_CRB_DATA_BUFFER_SIZE); memcpy(&crb->regs.data_buffer[rsp_off], &rsp[rsp_off], rsp_size); crb->regs.ctrl_start.start = false; } pthread_mutex_unlock(&crb->mutex); return (NULL); } static int tpm_crb_mmiocpy(void *const dst, void *const src, const int size) { if (!(size == 1 || size == 2 || size == 4 || size == 8)) return (EINVAL); memcpy(dst, src, size); return (0); } static int tpm_crb_mem_handler(struct vcpu *vcpu __unused, const int dir, const uint64_t addr, const int size, uint64_t *const val, void *const arg1, const long arg2 __unused) { struct tpm_crb *crb; uint8_t *ptr; uint64_t off, shift; int error = 0; if ((addr & (size - 1)) != 0) { warnx("%s: unaligned %s access @ %16lx [size = %x]", __func__, (dir == MEM_F_READ) ? "read" : "write", addr, size); return (EINVAL); } crb = arg1; off = addr - TPM_CRB_ADDRESS; if (off > TPM_CRB_REGS_SIZE || off + size >= TPM_CRB_REGS_SIZE) { return (EINVAL); } shift = 8 * (off & 3); ptr = (uint8_t *)&crb->regs + off; if (dir == MEM_F_READ) { error = tpm_crb_mmiocpy(val, ptr, size); if (error) goto err_out; } else { switch (off & ~0x3) { case offsetof(struct tpm_crb_regs, loc_ctrl): { union tpm_crb_reg_loc_ctrl loc_ctrl; if ((size_t)size > sizeof(loc_ctrl)) goto err_out; *val = *val << shift; tpm_crb_mmiocpy(&loc_ctrl, val, size); if (loc_ctrl.relinquish) { crb->regs.loc_sts.granted = false; crb->regs.loc_state.loc_assigned = false; } else if (loc_ctrl.request_access) { crb->regs.loc_sts.granted = true; crb->regs.loc_state.loc_assigned = true; } break; } case offsetof(struct tpm_crb_regs, ctrl_req): { union tpm_crb_reg_ctrl_req req; if ((size_t)size > sizeof(req)) goto err_out; *val = *val << shift; tpm_crb_mmiocpy(&req, val, size); if (req.cmd_ready && !req.go_idle) { crb->regs.ctrl_sts.tpm_idle = false; } else if (!req.cmd_ready && req.go_idle) { crb->regs.ctrl_sts.tpm_idle = true; } break; } case offsetof(struct tpm_crb_regs, ctrl_cancel): { /* TODO: cancel the tpm command */ warnx( "%s: cancelling a TPM command is not implemented yet", __func__); break; } case offsetof(struct tpm_crb_regs, int_enable): /* No interrupt support. Ignore writes to int_enable. */ break; case offsetof(struct tpm_crb_regs, ctrl_start): { union tpm_crb_reg_ctrl_start start; if ((size_t)size > sizeof(start)) goto err_out; *val = *val << shift; pthread_mutex_lock(&crb->mutex); tpm_crb_mmiocpy(&start, val, size); if (!start.start || crb->regs.ctrl_start.start) { pthread_mutex_unlock(&crb->mutex); break; } crb->regs.ctrl_start.start = true; pthread_cond_signal(&crb->cond); pthread_mutex_unlock(&crb->mutex); break; } case offsetof(struct tpm_crb_regs, cmd_size): case offsetof(struct tpm_crb_regs, cmd_addr_lo): case offsetof(struct tpm_crb_regs, cmd_addr_hi): case offsetof(struct tpm_crb_regs, rsp_size): case offsetof(struct tpm_crb_regs, rsp_addr) ... offsetof(struct tpm_crb_regs, rsp_addr) + 4: case offsetof(struct tpm_crb_regs, data_buffer) ... offsetof(struct tpm_crb_regs, data_buffer) + TPM_CRB_DATA_BUFFER_SIZE / 4: /* * Those fields are used to execute a TPM command. The * crb_thread will access them. For that reason, we have * to acquire the crb mutex in order to write them. */ pthread_mutex_lock(&crb->mutex); error = tpm_crb_mmiocpy(ptr, val, size); pthread_mutex_unlock(&crb->mutex); if (error) goto err_out; break; default: /* * The other fields are either readonly or we do not * support writing them. */ error = EINVAL; goto err_out; } } return (0); err_out: warnx("%s: invalid %s @ %16lx [size = %d]", __func__, dir == MEM_F_READ ? "read" : "write", addr, size); return (error); } static int tpm_crb_modify_mmio_registration(const bool registration, void *const arg1) { struct mem_range crb_mmio = { .name = "crb-mmio", .base = TPM_CRB_ADDRESS, .size = TPM_CRB_LOCALITIES_MAX * TPM_CRB_CONTROL_AREA_SIZE, .flags = MEM_F_RW, .arg1 = arg1, .handler = tpm_crb_mem_handler, }; if (registration) return (register_mem(&crb_mmio)); else return (unregister_mem(&crb_mmio)); } static int tpm_crb_init(void **sc, struct tpm_emul *emul, void *emul_sc, struct acpi_device *acpi_dev) { struct tpm_crb *crb = NULL; int error; assert(sc != NULL); assert(emul != NULL); crb = calloc(1, sizeof(struct tpm_crb)); if (crb == NULL) { warnx("%s: failed to allocate tpm crb", __func__); error = ENOMEM; goto err_out; } memset(crb, 0, sizeof(*crb)); crb->emul = emul; crb->emul_sc = emul_sc; crb->regs.loc_state.tpm_req_valid_sts = true; crb->regs.loc_state.tpm_established = true; crb->regs.intf_id.interface_type = TPM_INTF_TYPE_CRB; crb->regs.intf_id.interface_version = TPM_INTF_VERSION_CRB; crb->regs.intf_id.cap_locality = false; crb->regs.intf_id.cap_crb_idle_bypass = false; crb->regs.intf_id.cap_data_xfer_size_support = TPM_INTF_CAP_CRB_DATA_XFER_SIZE_64; crb->regs.intf_id.cap_fifo = false; crb->regs.intf_id.cap_crb = true; crb->regs.intf_id.interface_selector = TPM_INTF_SELECTOR_CRB; crb->regs.intf_id.intf_sel_lock = false; crb->regs.intf_id.rid = 0; crb->regs.intf_id.vid = 0x1014; /* IBM */ crb->regs.intf_id.did = 0x1014; /* IBM */ crb->regs.ctrl_sts.tpm_idle = true; CRB_CMD_SIZE_WRITE(crb->regs, TPM_CRB_DATA_BUFFER_SIZE); CRB_CMD_ADDR_WRITE(crb->regs, TPM_CRB_DATA_BUFFER_ADDRESS); CRB_RSP_SIZE_WRITE(crb->regs, TPM_CRB_DATA_BUFFER_SIZE); CRB_RSP_ADDR_WRITE(crb->regs, TPM_CRB_DATA_BUFFER_ADDRESS); error = qemu_fwcfg_add_file(TPM_CRB_LOG_AREA_FWCFG_NAME, TPM_CRB_LOG_AREA_MINIMUM_SIZE, crb->tpm_log_area); if (error) { warnx("%s: failed to add fwcfg file", __func__); goto err_out; } error = acpi_device_add_res_fixed_memory32(acpi_dev, false, TPM_CRB_ADDRESS, TPM_CRB_CONTROL_AREA_SIZE); if (error) { warnx("%s: failed to add acpi resources\n", __func__); goto err_out; } error = tpm_crb_modify_mmio_registration(true, crb); if (error) { warnx("%s: failed to register crb mmio", __func__); goto err_out; } error = pthread_mutex_init(&crb->mutex, NULL); if (error) { warnc(error, "%s: failed to init mutex", __func__); goto err_out; } error = pthread_cond_init(&crb->cond, NULL); if (error) { warnc(error, "%s: failed to init cond", __func__); goto err_out; } error = pthread_create(&crb->thread, NULL, tpm_crb_thread, crb); if (error) { warnx("%s: failed to create thread\n", __func__); goto err_out; } pthread_set_name_np(crb->thread, "tpm_intf_crb"); *sc = crb; return (0); err_out: free(crb); return (error); } static void tpm_crb_deinit(void *sc) { struct tpm_crb *crb; int error; if (sc == NULL) { return; } crb = sc; crb->closing = true; pthread_cond_signal(&crb->cond); pthread_join(crb->thread, NULL); pthread_cond_destroy(&crb->cond); pthread_mutex_destroy(&crb->mutex); error = tpm_crb_modify_mmio_registration(false, NULL); assert(error == 0); free(crb); } static int tpm_crb_build_acpi_table(void *sc __unused, struct vmctx *vm_ctx) { struct basl_table *table; BASL_EXEC(basl_table_create(&table, vm_ctx, ACPI_SIG_TPM2, BASL_TABLE_ALIGNMENT)); /* Header */ BASL_EXEC(basl_table_append_header(table, ACPI_SIG_TPM2, 4, 1)); /* Platform Class */ BASL_EXEC(basl_table_append_int(table, 0, 2)); /* Reserved */ BASL_EXEC(basl_table_append_int(table, 0, 2)); /* Control Address */ BASL_EXEC( basl_table_append_int(table, TPM_CRB_CONTROL_AREA_ADDRESS, 8)); /* Start Method == (7) Command Response Buffer */ BASL_EXEC(basl_table_append_int(table, 7, 4)); /* Start Method Specific Parameters */ uint8_t parameters[12] = { 0 }; BASL_EXEC(basl_table_append_bytes(table, parameters, 12)); /* Log Area Minimum Length */ BASL_EXEC( basl_table_append_int(table, TPM_CRB_LOG_AREA_MINIMUM_SIZE, 4)); /* Log Area Start Address */ #ifdef __FreeBSD__ BASL_EXEC( basl_table_append_fwcfg(table, TPM_CRB_LOG_AREA_FWCFG_NAME, 1, 8)); #else BASL_EXEC( basl_table_append_fwcfg(table, (const uint8_t *)TPM_CRB_LOG_AREA_FWCFG_NAME, 1, 8)); #endif BASL_EXEC(basl_table_register_to_rsdt(table)); return (0); } static struct tpm_intf tpm_intf_crb = { .name = TPM_CRB_INTF_NAME, .init = tpm_crb_init, .deinit = tpm_crb_deinit, .build_acpi_table = tpm_crb_build_acpi_table, }; TPM_INTF_SET(tpm_intf_crb); /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2022 Beckhoff Automation GmbH & Co. KG * Author: Corvin Köhne */ #pragma once #include "config.h" struct tpm_ppi { const char *name; int (*init)(void **sc); void (*deinit)(void *sc); int (*write_dsdt_regions)(void *sc); int (*write_dsdt_dsm)(void *sc); }; #define TPM_PPI_SET(x) DATA_SET(tpm_ppi_set, x) /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2022 Beckhoff Automation GmbH & Co. KG * Author: Corvin Köhne */ #include #include #include #include #include #include #include #include #include #include "acpi.h" #include "acpi_device.h" #include "config.h" #include "mem.h" #include "qemu_fwcfg.h" #include "tpm_ppi.h" #define TPM_PPI_ADDRESS 0xFED45000 #define TPM_PPI_SIZE 0x400 #define TPM_PPI_FWCFG_FILE "etc/tpm/config" #define TPM_PPI_QEMU_NAME "qemu" struct tpm_ppi_qemu { uint8_t func[256]; // FUNC uint8_t in; // PPIN uint32_t ip; // PPIP uint32_t response; // PPRP uint32_t request; // PPRQ uint32_t request_parameter; // PPRM uint32_t last_request; // LPPR uint32_t func_ret; // FRET uint8_t _reserved1[0x40]; // RES1 uint8_t next_step; // next_step } __packed; static_assert(sizeof(struct tpm_ppi_qemu) <= TPM_PPI_SIZE, "Wrong size of tpm_ppi_qemu"); struct tpm_ppi_fwcfg { uint32_t ppi_address; uint8_t tpm_version; uint8_t ppi_version; } __packed; static int tpm_ppi_mem_handler(struct vcpu *const vcpu __unused, const int dir, const uint64_t addr, const int size, uint64_t *const val, void *const arg1, const long arg2 __unused) { struct tpm_ppi_qemu *ppi; uint8_t *ptr; uint64_t off; if ((addr & (size - 1)) != 0) { warnx("%s: unaligned %s access @ %16lx [size = %x]", __func__, (dir == MEM_F_READ) ? "read" : "write", addr, size); } ppi = arg1; off = addr - TPM_PPI_ADDRESS; ptr = (uint8_t *)ppi + off; if (off > TPM_PPI_SIZE || off + size > TPM_PPI_SIZE) { return (EINVAL); } assert(size == 1 || size == 2 || size == 4 || size == 8); if (dir == MEM_F_READ) { memcpy(val, ptr, size); } else { memcpy(ptr, val, size); } return (0); } static struct mem_range ppi_mmio = { .name = "ppi-mmio", .base = TPM_PPI_ADDRESS, .size = TPM_PPI_SIZE, .flags = MEM_F_RW, .handler = tpm_ppi_mem_handler, }; static int tpm_ppi_init(void **sc) { struct tpm_ppi_qemu *ppi = NULL; struct tpm_ppi_fwcfg *fwcfg = NULL; int error; #ifdef __FreeBSD__ ppi = calloc(1, TPM_PPI_SIZE); #else // SMATCH: double check that we're allocating correct size: 346 vs 1024 void *ptr = calloc(1, TPM_PPI_SIZE); ppi = (struct tpm_ppi_qemu *)ptr; #endif if (ppi == NULL) { warnx("%s: failed to allocate acpi region for ppi", __func__); error = ENOMEM; goto err_out; } fwcfg = calloc(1, sizeof(struct tpm_ppi_fwcfg)); if (fwcfg == NULL) { warnx("%s: failed to allocate fwcfg item", __func__); error = ENOMEM; goto err_out; } fwcfg->ppi_address = htole32(TPM_PPI_ADDRESS); fwcfg->tpm_version = 2; fwcfg->ppi_version = 1; error = qemu_fwcfg_add_file(TPM_PPI_FWCFG_FILE, sizeof(struct tpm_ppi_fwcfg), fwcfg); if (error) { warnx("%s: failed to add fwcfg file", __func__); goto err_out; } /* * We would just need to create some guest memory for the PPI region. * Sadly, bhyve has a strange memory interface. We can't just add more * memory to the VM. So, create a trap instead which reads and writes to * the ppi region. It's very slow but ppi shouldn't be used frequently. */ ppi_mmio.arg1 = ppi; error = register_mem(&ppi_mmio); if (error) { warnx("%s: failed to create trap for ppi accesses", __func__); goto err_out; } *sc = ppi; return (0); err_out: free(fwcfg); free(ppi); return (error); } static void tpm_ppi_deinit(void *sc) { struct tpm_ppi_qemu *ppi; int error; if (sc == NULL) return; ppi = sc; error = unregister_mem(&ppi_mmio); assert(error == 0); free(ppi); } static int tpm_ppi_write_dsdt_regions(void *sc __unused) { /* * struct tpm_ppi_qemu */ /* * According to qemu the Windows ACPI parser has a bug that DerefOf is * broken for SYSTEM_MEMORY. Due to that bug, qemu uses a dynamic * operation region inside a method. */ dsdt_line("Method(TPFN, 1, Serialized)"); dsdt_line("{"); dsdt_line(" If(LGreaterEqual(Arg0, 0x100))"); dsdt_line(" {"); dsdt_line(" Return(Zero)"); dsdt_line(" }"); dsdt_line( " OperationRegion(TPP1, SystemMemory, Add(0x%8x, Arg0), One)", TPM_PPI_ADDRESS); dsdt_line(" Field(TPP1, ByteAcc, NoLock, Preserve)"); dsdt_line(" {"); dsdt_line(" TPPF, 8,"); dsdt_line(" }"); dsdt_line(" Return(TPPF)"); dsdt_line("}"); dsdt_line("OperationRegion(TPP2, SystemMemory, 0x%8x, 0x%x)", TPM_PPI_ADDRESS + 0x100, 0x5A); dsdt_line("Field(TPP2, AnyAcc, NoLock, Preserve)"); dsdt_line("{"); dsdt_line(" PPIN, 8,"); dsdt_line(" PPIP, 32,"); dsdt_line(" PPRP, 32,"); dsdt_line(" PPRQ, 32,"); dsdt_line(" PPRM, 32,"); dsdt_line(" LPPR, 32,"); dsdt_line("}"); /* * Used for TCG Platform Reset Attack Mitigation */ dsdt_line("OperationRegion(TPP3, SystemMemory, 0x%8x, 1)", TPM_PPI_ADDRESS + sizeof(struct tpm_ppi_qemu)); dsdt_line("Field(TPP3, ByteAcc, NoLock, Preserve)"); dsdt_line("{"); dsdt_line(" MOVV, 8,"); dsdt_line("}"); return (0); } static int tpm_ppi_write_dsdt_dsm(void *sc __unused) { /* * Physical Presence Interface */ dsdt_line( "If(LEqual(Arg0, ToUUID(\"3DDDFAA6-361B-4EB4-A424-8D10089D1653\"))) /* UUID */"); dsdt_line("{"); /* * Function 0 - _DSM Query Function * Arguments: * Empty Package * Return: * Buffer - Index field of supported functions */ dsdt_line(" If(LEqual(Arg2, 0)) /* Function */"); dsdt_line(" {"); dsdt_line(" Return(Buffer(0x02)"); dsdt_line(" {"); dsdt_line(" 0xFF, 0x01"); dsdt_line(" })"); dsdt_line(" }"); /* * Function 1 - Get Physical Presence Interface Version * Arguments: * Empty Package * Return: * String - Supported Physical Presence Interface revision */ dsdt_line(" If(LEqual(Arg2, 1)) /* Function */"); dsdt_line(" {"); dsdt_line(" Return(\"1.3\")"); dsdt_line(" }"); /* * Function 2 - Submit TPM Operation Request to Pre-OS Environment * !!!DEPRECATED BUT MANDATORY!!! * Arguments: * Integer - Operation Value of the Request * Return: * Integer - Function Return Code * 0 - Success * 1 - Operation Value of the Request Not Supported * 2 - General Failure */ dsdt_line(" If(LEqual(Arg2, 2)) /* Function */"); dsdt_line(" {"); dsdt_line(" Store(DerefOf(Index(Arg3, 0)), Local0)"); dsdt_line(" Store(TPFN(Local0), Local1)"); dsdt_line(" If (LEqual(And(Local1, 7), 0))"); dsdt_line(" {"); dsdt_line(" Return(1)"); dsdt_line(" }"); dsdt_line(" Store(Local0, PPRQ)"); dsdt_line(" Store(0, PPRM)"); dsdt_line(" Return(0)"); dsdt_line(" }"); /* * Function 3 - Get Pending TPM Operation Request By the OS * Arguments: * Empty Package * Return: * Package * Integer 1 - Function Return Code * 0 - Success * 1 - General Failure * Integer 2 - Pending operation requested by the OS * 0 - None * >0 - Operation Value of the Pending Request * Integer 3 - Optional argument to pending operation requested by * the OS * 0 - None * >0 - Argument of the Pending Request */ dsdt_line(" If(LEqual(Arg2, 3)) /* Function */"); dsdt_line(" {"); dsdt_line(" If(LEqual(Arg1, 1)) /* Revision */"); dsdt_line(" {"); dsdt_line(" Store(PPRQ, Index(TPM2, 1))"); dsdt_line(" Return(TPM2)"); dsdt_line(" }"); dsdt_line(" If(LEqual(Arg1, 2)) /* Revision */"); dsdt_line(" {"); dsdt_line(" Store(PPRQ, Index(TPM3, 1))"); dsdt_line(" Store(PPRM, Index(TPM3, 2))"); dsdt_line(" Return(TPM3)"); dsdt_line(" }"); dsdt_line(" }"); /* * Function 4 - Get Platform-Specific Action to Transition to Pre-OS * Environment * Arguments: * Empty Package * Return: * Integer - Action that the OS should take to transition to the * pre-OS environment for execution of a requested operation * 0 - None * 1 - Shutdown * 2 - Reboot * 3 - OS Vendor-specific */ dsdt_line(" If(LEqual(Arg2, 4)) /* Function */"); dsdt_line(" {"); dsdt_line(" Return(2)"); dsdt_line(" }"); /* * Function 5 - Return TPM Operation Response to OS Environment * Arguments: * Empty Package * Return: * Package * Integer 1 - Function Return Code * 0 - Success * 1 - General Failure * Integer 2 - Most recent operation request * 0 - None * >0 - Operation value of the most recent request * Integer 3 - Response to the most recent operation request * 0 - Success * 0x00000001..0x000000FF - Corresponding TPM error code * 0xFFFFFFF0 - User Abort or timeout of dialog * 0xFFFFFFF1 - firmware failure */ dsdt_line(" If(LEqual(Arg2, 5)) /* Function */"); dsdt_line(" {"); dsdt_line(" Store(LPPR, Index(TPM3, 1))"); dsdt_line(" Store(PPRP, Index(TPM3, 2))"); dsdt_line(" Return(TPM3)"); dsdt_line(" }"); /* * Function 6 - Submit preferred user language * !!!DEPRECATED BUT MANDATORY!!! * Arguments: * Package * String - Preferred language code * Return: * Integer * 3 - Not implemented */ dsdt_line(" If(LEqual(Arg2, 6)) /* Function */"); dsdt_line(" {"); dsdt_line(" Return(3)"); dsdt_line(" }"); /* * Function 7 - Submit TPM Operation Request to Pre-OS Environment 2 * Arguments: * Package * Integer 1 - Operation Value of the Request * Integer 2 - Argument for Operation * Return: * Integer - Function Return Code * 0 - Success * 1 - Not Implemented * 2 - General Failure * 3 - Operation blocked by current firmware settings */ dsdt_line(" If(LEqual(Arg2, 7)) /* Function */"); dsdt_line(" {"); dsdt_line(" Store(DerefOf(Index(Arg3, 0)), Local0)"); dsdt_line(" Store(TPFN(Local0), Local1)"); dsdt_line(" If (LEqual(And(Local1, 7), 0)) /* Not Implemented */"); dsdt_line(" {"); dsdt_line(" Return(1)"); dsdt_line(" }"); dsdt_line(" If (LEqual(And(Local1, 7), 2)) /* Blocked */ "); dsdt_line(" {"); dsdt_line(" Return(3)"); dsdt_line(" }"); dsdt_line(" If(LEqual(Arg1, 1)) /* Revision */"); dsdt_line(" {"); dsdt_line(" Store(Local0, PPRQ)"); dsdt_line(" Store(0, PPRM)"); dsdt_line(" }"); dsdt_line(" If(LEqual(Arg1, 2)) /* Revision */"); dsdt_line(" {"); dsdt_line(" Store(Local0, PPRQ)"); dsdt_line(" Store(DerefOf(Index(Arg3, 1)), PPRM)"); dsdt_line(" }"); dsdt_line(" Return(0)"); dsdt_line(" }"); /* * Function 8 - Get User Confirmation Status for Operation * Arguments: * Package * Integer - Operation Value that may need user confirmation * Return: * Integer - Function Return Code * 0 - Not implemented * 1 - Firmware only * 2 - Blocked for OS by firmware configuration * 3 - Allowed and physically present user required * 4 - Allowed and physically present user not required */ dsdt_line(" If(LEqual(Arg2, 8)) /* Function */"); dsdt_line(" {"); dsdt_line(" Store(DerefOf(Index(Arg3, 0)), Local0)"); dsdt_line(" Store(TPFN(Local0), Local1)"); dsdt_line(" Return(And(Local1, 7))"); dsdt_line(" }"); /* * Unknown function */ dsdt_line(" Return(Buffer(1)"); dsdt_line(" {"); dsdt_line(" 0x00"); dsdt_line(" })"); dsdt_line("}"); /* * TCG Platform Reset Attack Mitigation */ dsdt_line( "If(LEqual(Arg0, ToUUID(\"376054ED-CC13-4675-901C-4756D7F2D45D\"))) /* UUID */"); dsdt_line("{"); /* * Function 0 - _DSM Query Function * Arguments: * Empty Package * Return: * Buffer - Index field of supported functions */ dsdt_line(" If(LEqual(Arg2, 0)) /* Function */"); dsdt_line(" {"); dsdt_line(" Return(Buffer(1)"); dsdt_line(" {"); dsdt_line(" 0x03"); dsdt_line(" })"); dsdt_line(" }"); /* * Function 1 - Memory Clear * Arguments: * Package * Integer - Operation Value of the Request * Return: * Integer - Function Return Code * 0 - Success * 1 - General Failure */ dsdt_line(" If(LEqual(Arg2, 1)) /* Function */"); dsdt_line(" {"); dsdt_line(" Store(DerefOf(Index(Arg3, 0)), Local0)"); dsdt_line(" Store(Local0, MOVV)"); dsdt_line(" Return(0)"); dsdt_line(" }"); dsdt_line("}"); return (0); } static struct tpm_ppi tpm_ppi_qemu = { .name = TPM_PPI_QEMU_NAME, .init = tpm_ppi_init, .deinit = tpm_ppi_deinit, .write_dsdt_regions = tpm_ppi_write_dsdt_regions, .write_dsdt_dsm = tpm_ppi_write_dsdt_dsm, }; TPM_PPI_SET(tpm_ppi_qemu); /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2012 NetApp, Inc. * Copyright (c) 2013 Neel Natu * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include #include #include #ifndef WITHOUT_CAPSICUM #include #endif #include #include #include #include #include #include #include #include #include #ifndef __FreeBSD__ #include #include #endif #include "debug.h" #include "mevent.h" #include "uart_backend.h" #include "uart_emul.h" struct ttyfd { bool opened; int rfd; /* fd for reading */ int wfd; /* fd for writing, may be == rfd */ }; #ifndef __FreeBSD__ struct sockfd { bool sock; int clifd; /* console client unix domain socket */ int servfd; /* console server unix domain socket */ struct mevent *servmev; /* mevent for server socket */ void (*drain)(int, enum ev_type, void *); void *drainarg; }; #endif #define FIFOSZ 16 struct fifo { uint8_t buf[FIFOSZ]; int rindex; /* index to read from */ int windex; /* index to write to */ int num; /* number of characters in the fifo */ int size; /* size of the fifo */ }; struct uart_softc { struct ttyfd tty; #ifndef __FreeBSD__ struct sockfd usc_sock; #endif struct fifo rxfifo; struct mevent *mev; pthread_mutex_t mtx; }; static bool uart_stdio; /* stdio in use for i/o */ static struct termios tio_stdio_orig; static void ttyclose(void) { tcsetattr(STDIN_FILENO, TCSANOW, &tio_stdio_orig); } static void ttyopen(struct ttyfd *tf) { struct termios orig, new; tcgetattr(tf->rfd, &orig); new = orig; cfmakeraw(&new); new.c_cflag |= CLOCAL; tcsetattr(tf->rfd, TCSANOW, &new); if (uart_stdio) { tio_stdio_orig = orig; atexit(ttyclose); } raw_stdio = 1; } static int ttyread(struct ttyfd *tf) { unsigned char rb; if (read(tf->rfd, &rb, 1) == 1) return (rb); else return (-1); } static void ttywrite(struct ttyfd *tf, unsigned char wb) { (void)write(tf->wfd, &wb, 1); } #ifndef __FreeBSD__ static void sockwrite(struct uart_softc *sc, unsigned char wb) { (void) write(sc->usc_sock.clifd, &wb, 1); } #endif static bool rxfifo_available(struct uart_softc *sc) { return (sc->rxfifo.num < sc->rxfifo.size); } int uart_rxfifo_getchar(struct uart_softc *sc) { struct fifo *fifo; int c, error, wasfull; wasfull = 0; fifo = &sc->rxfifo; if (fifo->num > 0) { if (!rxfifo_available(sc)) wasfull = 1; c = fifo->buf[fifo->rindex]; fifo->rindex = (fifo->rindex + 1) % fifo->size; fifo->num--; if (wasfull) { if (sc->tty.opened) { error = mevent_enable(sc->mev); assert(error == 0); } #ifndef __FreeBSD__ if (sc->usc_sock.sock && sc->usc_sock.clifd != -1) { error = mevent_enable(sc->mev); assert(error == 0); } #endif /* __FreeBSD__ */ } return (c); } else return (-1); } int uart_rxfifo_numchars(struct uart_softc *sc) { return (sc->rxfifo.num); } static int rxfifo_putchar(struct uart_softc *sc, uint8_t ch) { struct fifo *fifo; int error; fifo = &sc->rxfifo; if (fifo->num < fifo->size) { fifo->buf[fifo->windex] = ch; fifo->windex = (fifo->windex + 1) % fifo->size; fifo->num++; if (!rxfifo_available(sc)) { if (sc->tty.opened) { /* * Disable mevent callback if the FIFO is full. */ error = mevent_disable(sc->mev); assert(error == 0); } #ifndef __FreeBSD__ if (sc->usc_sock.sock && sc->usc_sock.clifd != -1) { /* * Disable mevent callback if the FIFO is full. */ error = mevent_disable(sc->mev); assert(error == 0); } #endif /* __FreeBSD__ */ } return (0); } else return (-1); } void uart_rxfifo_drain(struct uart_softc *sc, bool loopback) { int ch; if (loopback) { (void)ttyread(&sc->tty); } else { while (rxfifo_available(sc) && ((ch = ttyread(&sc->tty)) != -1)) rxfifo_putchar(sc, ch); } } #ifndef __FreeBSD__ void uart_rxfifo_sock_drain(struct uart_softc *sc, bool loopback) { int ch; if (loopback) { (void) read(sc->usc_sock.clifd, &ch, 1); } else { bool err_close = false; while (rxfifo_available(sc)) { int res; res = read(sc->usc_sock.clifd, &ch, 1); if (res == 0) { err_close = true; break; } else if (res == -1) { if (errno != EAGAIN && errno != EINTR) { err_close = true; } break; } rxfifo_putchar(sc, ch); } if (err_close) { (void) fprintf(stderr, "uart: closing client conn\n"); (void) shutdown(sc->usc_sock.clifd, SHUT_RDWR); mevent_delete_close(sc->mev); sc->mev = NULL; sc->usc_sock.clifd = -1; } } } #endif int uart_rxfifo_putchar(struct uart_softc *sc, uint8_t ch, bool loopback) { if (loopback) { return (rxfifo_putchar(sc, ch)); } else if (sc->tty.opened) { ttywrite(&sc->tty, ch); return (0); #ifndef __FreeBSD__ } else if (sc->usc_sock.sock) { sockwrite(sc, ch); return (0); #endif } else { /* Drop on the floor. */ return (0); } } void uart_rxfifo_reset(struct uart_softc *sc, int size) { char flushbuf[32]; struct fifo *fifo; ssize_t nread; int error; fifo = &sc->rxfifo; bzero(fifo, sizeof(struct fifo)); fifo->size = size; if (sc->tty.opened) { /* * Flush any unread input from the tty buffer. */ while (1) { nread = read(sc->tty.rfd, flushbuf, sizeof(flushbuf)); if (nread != sizeof(flushbuf)) break; } /* * Enable mevent to trigger when new characters are available * on the tty fd. */ error = mevent_enable(sc->mev); assert(error == 0); } #ifndef __FreeBSD__ if (sc->usc_sock.sock && sc->usc_sock.clifd != -1) { /* Flush any unread input from the socket buffer. */ do { nread = read(sc->usc_sock.clifd, flushbuf, sizeof (flushbuf)); } while (nread == sizeof (flushbuf)); /* Enable mevent to trigger when new data available on sock */ error = mevent_enable(sc->mev); assert(error == 0); } #endif /* __FreeBSD__ */ } int uart_rxfifo_size(struct uart_softc *sc __unused) { return (FIFOSZ); } #ifdef BHYVE_SNAPSHOT int uart_rxfifo_snapshot(struct uart_softc *sc, struct vm_snapshot_meta *meta) { int ret; SNAPSHOT_VAR_OR_LEAVE(sc->rxfifo.rindex, meta, ret, done); SNAPSHOT_VAR_OR_LEAVE(sc->rxfifo.windex, meta, ret, done); SNAPSHOT_VAR_OR_LEAVE(sc->rxfifo.num, meta, ret, done); SNAPSHOT_VAR_OR_LEAVE(sc->rxfifo.size, meta, ret, done); SNAPSHOT_BUF_OR_LEAVE(sc->rxfifo.buf, sizeof(sc->rxfifo.buf), meta, ret, done); done: return (ret); } #endif static int uart_stdio_backend(struct uart_softc *sc) { #ifndef WITHOUT_CAPSICUM cap_rights_t rights; cap_ioctl_t cmds[] = { TIOCGETA, TIOCSETA, TIOCGWINSZ }; #endif if (uart_stdio) return (-1); sc->tty.rfd = STDIN_FILENO; sc->tty.wfd = STDOUT_FILENO; sc->tty.opened = true; if (fcntl(sc->tty.rfd, F_SETFL, O_NONBLOCK) != 0) return (-1); if (fcntl(sc->tty.wfd, F_SETFL, O_NONBLOCK) != 0) return (-1); #ifndef WITHOUT_CAPSICUM cap_rights_init(&rights, CAP_EVENT, CAP_IOCTL, CAP_READ); if (caph_rights_limit(sc->tty.rfd, &rights) == -1) errx(EX_OSERR, "Unable to apply rights for sandbox"); if (caph_ioctls_limit(sc->tty.rfd, cmds, nitems(cmds)) == -1) errx(EX_OSERR, "Unable to apply rights for sandbox"); #endif uart_stdio = true; return (0); } static int uart_tty_backend(struct uart_softc *sc, const char *path) { #ifndef WITHOUT_CAPSICUM cap_rights_t rights; cap_ioctl_t cmds[] = { TIOCGETA, TIOCSETA, TIOCGWINSZ }; #endif int fd; fd = open(path, O_RDWR | O_NONBLOCK); if (fd < 0) return (-1); if (!isatty(fd)) { close(fd); return (-1); } sc->tty.rfd = sc->tty.wfd = fd; sc->tty.opened = true; #ifndef WITHOUT_CAPSICUM cap_rights_init(&rights, CAP_EVENT, CAP_IOCTL, CAP_READ, CAP_WRITE); if (caph_rights_limit(fd, &rights) == -1) errx(EX_OSERR, "Unable to apply rights for sandbox"); if (caph_ioctls_limit(fd, cmds, nitems(cmds)) == -1) errx(EX_OSERR, "Unable to apply rights for sandbox"); #endif return (0); } #ifndef __FreeBSD__ static void uart_sock_accept(int fd, enum ev_type ev, void *arg) { struct uart_softc *sc = arg; int connfd; connfd = accept(sc->usc_sock.servfd, NULL, NULL); if (connfd == -1) { return; } /* * Do client connection management under protection of the softc lock * to avoid racing with concurrent UART events. */ pthread_mutex_lock(&sc->mtx); if (sc->usc_sock.clifd != -1) { /* we're already handling a client */ (void) fprintf(stderr, "uart: unexpected client conn\n"); (void) shutdown(connfd, SHUT_RDWR); (void) close(connfd); } else { if (fcntl(connfd, F_SETFL, O_NONBLOCK) < 0) { perror("uart: fcntl(O_NONBLOCK)"); (void) shutdown(connfd, SHUT_RDWR); (void) close(connfd); } else { sc->usc_sock.clifd = connfd; sc->mev = mevent_add(sc->usc_sock.clifd, EVF_READ, sc->usc_sock.drain, sc->usc_sock.drainarg); } } pthread_mutex_unlock(&sc->mtx); } static int init_sock(const char *path) { int servfd; struct sockaddr_un servaddr; bzero(&servaddr, sizeof (servaddr)); servaddr.sun_family = AF_UNIX; if (strlcpy(servaddr.sun_path, path, sizeof (servaddr.sun_path)) >= sizeof (servaddr.sun_path)) { (void) fprintf(stderr, "uart: path '%s' too long\n", path); return (-1); } if ((servfd = socket(AF_UNIX, SOCK_STREAM, 0)) == -1) { (void) fprintf(stderr, "uart: socket() error - %s\n", strerror(errno)); return (-1); } (void) unlink(servaddr.sun_path); if (bind(servfd, (struct sockaddr *)&servaddr, sizeof (servaddr)) == -1) { (void) fprintf(stderr, "uart: bind() error - %s\n", strerror(errno)); goto out; } if (listen(servfd, 1) == -1) { (void) fprintf(stderr, "uart: listen() error - %s\n", strerror(errno)); goto out; } return (servfd); out: (void) unlink(servaddr.sun_path); (void) close(servfd); return (-1); } static int uart_sock_backend(struct uart_softc *sc, const char *inopts, void (*drain)(int, enum ev_type, void *), void *drainarg) { char *opts, *tofree; char *opt; char *nextopt; char *path = NULL; if (strncmp(inopts, "socket,", 7) != 0) { return (-1); } if ((opts = strdup(inopts + 7)) == NULL) { return (-1); } tofree = nextopt = opts; for (opt = strsep(&nextopt, ","); opt != NULL; opt = strsep(&nextopt, ",")) { if (path == NULL && *opt == '/') { path = opt; continue; } /* * XXX check for server and client options here. For now, * everything is a server */ free(tofree); return (-1); } sc->usc_sock.clifd = -1; if ((sc->usc_sock.servfd = init_sock(path)) == -1) { free(tofree); return (-1); } sc->usc_sock.sock = true; sc->tty.rfd = sc->tty.wfd = -1; sc->usc_sock.servmev = mevent_add(sc->usc_sock.servfd, EVF_READ, uart_sock_accept, sc); assert(sc->usc_sock.servmev != NULL); sc->usc_sock.drain = drain; sc->usc_sock.drainarg = drainarg; free(tofree); return (0); } #endif /* not __FreeBSD__ */ struct uart_softc * uart_init(void) { struct uart_softc *sc = calloc(1, sizeof(struct uart_softc)); if (sc == NULL) return (NULL); pthread_mutex_init(&sc->mtx, NULL); return (sc); } int uart_tty_open(struct uart_softc *sc, const char *path, void (*drain)(int, enum ev_type, void *), void *arg) { int retval; #ifndef __FreeBSD__ if (strncmp("socket,", path, 7) == 0) return (uart_sock_backend(sc, path, drain, arg)); #endif if (strcmp("stdio", path) == 0) retval = uart_stdio_backend(sc); else retval = uart_tty_backend(sc, path); if (retval == 0) { ttyopen(&sc->tty); sc->mev = mevent_add(sc->tty.rfd, EVF_READ, drain, arg); assert(sc->mev != NULL); } return (retval); } void uart_softc_lock(struct uart_softc *sc) { pthread_mutex_lock(&sc->mtx); } void uart_softc_unlock(struct uart_softc *sc) { pthread_mutex_unlock(&sc->mtx); } /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2012 NetApp, Inc. * Copyright (c) 2013 Neel Natu * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef _UART_BACKEND_H_ #define _UART_BACKEND_H_ #include #include "mevent.h" struct uart_softc; struct vm_snapshot_meta; void uart_rxfifo_drain(struct uart_softc *sc, bool loopback); #ifndef __FreeBSD__ void uart_rxfifo_sock_drain(struct uart_softc *sc, bool loopback); #endif int uart_rxfifo_getchar(struct uart_softc *sc); int uart_rxfifo_numchars(struct uart_softc *sc); int uart_rxfifo_putchar(struct uart_softc *sc, uint8_t ch, bool loopback); void uart_rxfifo_reset(struct uart_softc *sc, int size); int uart_rxfifo_size(struct uart_softc *sc); struct uart_softc *uart_init(void); int uart_tty_open(struct uart_softc *sc, const char *path, void (*drain)(int, enum ev_type, void *), void *arg); void uart_softc_lock(struct uart_softc *sc); void uart_softc_unlock(struct uart_softc *sc); #endif /* _UART_BACKEND_H_ */ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2012 NetApp, Inc. * Copyright (c) 2013 Neel Natu * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include #include #include #include #include #include #include #include #include #include #include "uart_backend.h" #include "uart_emul.h" #define COM1_BASE 0x3F8 #define COM1_IRQ 4 #define COM2_BASE 0x2F8 #define COM2_IRQ 3 #define COM3_BASE 0x3E8 #define COM3_IRQ 4 #define COM4_BASE 0x2E8 #define COM4_IRQ 3 #define DEFAULT_RCLK 1843200 #define DEFAULT_BAUD 115200 #define FCR_RX_MASK 0xC0 #define MCR_OUT1 0x04 #define MCR_OUT2 0x08 #define MSR_DELTA_MASK 0x0f #ifndef REG_SCR #define REG_SCR com_scr #endif static struct { int baseaddr; int irq; bool inuse; } uart_lres[] = { { COM1_BASE, COM1_IRQ, false}, { COM2_BASE, COM2_IRQ, false}, { COM3_BASE, COM3_IRQ, false}, { COM4_BASE, COM4_IRQ, false}, }; #define UART_NLDEVS (sizeof(uart_lres) / sizeof(uart_lres[0])) struct uart_ns16550_softc { struct uart_softc *backend; uint8_t data; /* Data register (R/W) */ uint8_t ier; /* Interrupt enable register (R/W) */ uint8_t lcr; /* Line control register (R/W) */ uint8_t mcr; /* Modem control register (R/W) */ uint8_t lsr; /* Line status register (R/W) */ uint8_t msr; /* Modem status register (R/W) */ uint8_t fcr; /* FIFO control register (W) */ uint8_t scr; /* Scratch register (R/W) */ uint8_t dll; /* Baudrate divisor latch LSB */ uint8_t dlh; /* Baudrate divisor latch MSB */ bool thre_int_pending; /* THRE interrupt pending */ void *arg; uart_intr_func_t intr_assert; uart_intr_func_t intr_deassert; }; static uint8_t modem_status(uint8_t mcr) { uint8_t msr; if (mcr & MCR_LOOPBACK) { /* * In the loopback mode certain bits from the MCR are * reflected back into MSR. */ msr = 0; if (mcr & MCR_RTS) msr |= MSR_CTS; if (mcr & MCR_DTR) msr |= MSR_DSR; if (mcr & MCR_OUT1) msr |= MSR_RI; if (mcr & MCR_OUT2) msr |= MSR_DCD; } else { /* * Always assert DCD and DSR so tty open doesn't block * even if CLOCAL is turned off. */ msr = MSR_DCD | MSR_DSR; } assert((msr & MSR_DELTA_MASK) == 0); return (msr); } /* * The IIR returns a prioritized interrupt reason: * - receive data available * - transmit holding register empty * - modem status change * * Return an interrupt reason if one is available. */ static int uart_intr_reason(struct uart_ns16550_softc *sc) { if ((sc->lsr & LSR_OE) != 0 && (sc->ier & IER_ERLS) != 0) return (IIR_RLS); else if (uart_rxfifo_numchars(sc->backend) > 0 && (sc->ier & IER_ERXRDY) != 0) return (IIR_RXTOUT); else if (sc->thre_int_pending && (sc->ier & IER_ETXRDY) != 0) return (IIR_TXRDY); else if ((sc->msr & MSR_DELTA_MASK) != 0 && (sc->ier & IER_EMSC) != 0) return (IIR_MLSC); else return (IIR_NOPEND); } static void uart_reset(struct uart_ns16550_softc *sc) { uint16_t divisor; divisor = DEFAULT_RCLK / DEFAULT_BAUD / 16; sc->dll = divisor; #ifdef __FreeBSD__ sc->dlh = divisor >> 16; #else sc->dlh = 0; #endif sc->msr = modem_status(sc->mcr); uart_rxfifo_reset(sc->backend, 1); } /* * Toggle the COM port's intr pin depending on whether or not we have an * interrupt condition to report to the processor. */ static void uart_toggle_intr(struct uart_ns16550_softc *sc) { uint8_t intr_reason; intr_reason = uart_intr_reason(sc); if (intr_reason == IIR_NOPEND) (*sc->intr_deassert)(sc->arg); else (*sc->intr_assert)(sc->arg); } static void uart_drain(int fd __unused, enum ev_type ev, void *arg) { struct uart_ns16550_softc *sc; bool loopback; sc = arg; assert(ev == EVF_READ); /* * This routine is called in the context of the mevent thread * to take out the softc lock to protect against concurrent * access from a vCPU i/o exit */ uart_softc_lock(sc->backend); loopback = (sc->mcr & MCR_LOOPBACK) != 0; uart_rxfifo_drain(sc->backend, loopback); if (!loopback) uart_toggle_intr(sc); uart_softc_unlock(sc->backend); } #ifndef __FreeBSD__ void uart_sock_drain(int fd, enum ev_type ev, void *arg) { struct uart_ns16550_softc *sc; bool loopback; sc = arg; /* * Take the softc lock to protect against concurrent * access from a vCPU i/o exit */ uart_softc_lock(sc->backend); loopback = (sc->mcr & MCR_LOOPBACK) != 0; uart_rxfifo_sock_drain(sc->backend, loopback); if (!loopback) uart_toggle_intr(sc); uart_softc_unlock(sc->backend); } #endif /* !__FreeBSD__ */ void uart_ns16550_write(struct uart_ns16550_softc *sc, int offset, uint8_t value) { int fifosz; uint8_t msr; uart_softc_lock(sc->backend); /* * Take care of the special case DLAB accesses first */ if ((sc->lcr & LCR_DLAB) != 0) { if (offset == REG_DLL) { sc->dll = value; goto done; } if (offset == REG_DLH) { sc->dlh = value; goto done; } } switch (offset) { case REG_DATA: if (uart_rxfifo_putchar(sc->backend, value, (sc->mcr & MCR_LOOPBACK) != 0)) sc->lsr |= LSR_OE; sc->thre_int_pending = true; break; case REG_IER: /* Set pending when IER_ETXRDY is raised (edge-triggered). */ if ((sc->ier & IER_ETXRDY) == 0 && (value & IER_ETXRDY) != 0) sc->thre_int_pending = true; /* * Apply mask so that bits 4-7 are 0 * Also enables bits 0-3 only if they're 1 */ sc->ier = value & 0x0F; break; case REG_FCR: /* * When moving from FIFO and 16450 mode and vice versa, * the FIFO contents are reset. */ if ((sc->fcr & FCR_ENABLE) ^ (value & FCR_ENABLE)) { fifosz = (value & FCR_ENABLE) ? uart_rxfifo_size(sc->backend) : 1; uart_rxfifo_reset(sc->backend, fifosz); } /* * The FCR_ENABLE bit must be '1' for the programming * of other FCR bits to be effective. */ if ((value & FCR_ENABLE) == 0) { sc->fcr = 0; } else { if ((value & FCR_RCV_RST) != 0) uart_rxfifo_reset(sc->backend, uart_rxfifo_size(sc->backend)); sc->fcr = value & (FCR_ENABLE | FCR_DMA | FCR_RX_MASK); } break; case REG_LCR: sc->lcr = value; break; case REG_MCR: /* Apply mask so that bits 5-7 are 0 */ sc->mcr = value & 0x1F; msr = modem_status(sc->mcr); /* * Detect if there has been any change between the * previous and the new value of MSR. If there is * then assert the appropriate MSR delta bit. */ if ((msr & MSR_CTS) ^ (sc->msr & MSR_CTS)) sc->msr |= MSR_DCTS; if ((msr & MSR_DSR) ^ (sc->msr & MSR_DSR)) sc->msr |= MSR_DDSR; if ((msr & MSR_DCD) ^ (sc->msr & MSR_DCD)) sc->msr |= MSR_DDCD; if ((sc->msr & MSR_RI) != 0 && (msr & MSR_RI) == 0) sc->msr |= MSR_TERI; /* * Update the value of MSR while retaining the delta * bits. */ sc->msr &= MSR_DELTA_MASK; sc->msr |= msr; break; case REG_LSR: /* * Line status register is not meant to be written to * during normal operation. */ break; case REG_MSR: /* * As far as I can tell MSR is a read-only register. */ break; case REG_SCR: sc->scr = value; break; default: break; } done: uart_toggle_intr(sc); uart_softc_unlock(sc->backend); } uint8_t uart_ns16550_read(struct uart_ns16550_softc *sc, int offset) { uint8_t iir, intr_reason, reg; uart_softc_lock(sc->backend); /* * Take care of the special case DLAB accesses first */ if ((sc->lcr & LCR_DLAB) != 0) { if (offset == REG_DLL) { reg = sc->dll; goto done; } if (offset == REG_DLH) { reg = sc->dlh; goto done; } } switch (offset) { case REG_DATA: reg = uart_rxfifo_getchar(sc->backend); break; case REG_IER: reg = sc->ier; break; case REG_IIR: iir = (sc->fcr & FCR_ENABLE) ? IIR_FIFO_MASK : 0; intr_reason = uart_intr_reason(sc); /* * Deal with side effects of reading the IIR register */ if (intr_reason == IIR_TXRDY) sc->thre_int_pending = false; iir |= intr_reason; reg = iir; break; case REG_LCR: reg = sc->lcr; break; case REG_MCR: reg = sc->mcr; break; case REG_LSR: /* Transmitter is always ready for more data */ sc->lsr |= LSR_TEMT | LSR_THRE; /* Check for new receive data */ if (uart_rxfifo_numchars(sc->backend) > 0) sc->lsr |= LSR_RXRDY; else sc->lsr &= ~LSR_RXRDY; reg = sc->lsr; /* The LSR_OE bit is cleared on LSR read */ sc->lsr &= ~LSR_OE; break; case REG_MSR: /* * MSR delta bits are cleared on read */ reg = sc->msr; sc->msr &= ~MSR_DELTA_MASK; break; case REG_SCR: reg = sc->scr; break; default: reg = 0xFF; break; } done: uart_toggle_intr(sc); uart_softc_unlock(sc->backend); return (reg); } int uart_legacy_alloc(int which, int *baseaddr, int *irq) { if (which < 0 || which >= (int)UART_NLDEVS || uart_lres[which].inuse) return (-1); uart_lres[which].inuse = true; *baseaddr = uart_lres[which].baseaddr; *irq = uart_lres[which].irq; return (0); } struct uart_ns16550_softc * uart_ns16550_init(uart_intr_func_t intr_assert, uart_intr_func_t intr_deassert, void *arg) { struct uart_ns16550_softc *sc; sc = calloc(1, sizeof(struct uart_ns16550_softc)); sc->arg = arg; sc->intr_assert = intr_assert; sc->intr_deassert = intr_deassert; sc->backend = uart_init(); uart_reset(sc); return (sc); } int uart_ns16550_tty_open(struct uart_ns16550_softc *sc, const char *device) { #ifndef __FreeBSD__ if (strncmp("socket,", device, 7) == 0) { return (uart_tty_open(sc->backend, device, uart_sock_drain, sc)); } #endif return (uart_tty_open(sc->backend, device, uart_drain, sc)); } /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2013 Neel Natu * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef _UART_EMUL_H_ #define _UART_EMUL_H_ #define UART_NS16550_IO_BAR_SIZE 8 struct uart_ns16550_softc; typedef void (*uart_intr_func_t)(void *arg); int uart_legacy_alloc(int unit, int *ioaddr, int *irq); struct uart_ns16550_softc *uart_ns16550_init(uart_intr_func_t intr_assert, uart_intr_func_t intr_deassert, void *arg); uint8_t uart_ns16550_read(struct uart_ns16550_softc *sc, int offset); void uart_ns16550_write(struct uart_ns16550_softc *sc, int offset, uint8_t value); int uart_ns16550_tty_open(struct uart_ns16550_softc *sc, const char *device); #endif /* _UART_EMUL_H_ */ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2014 Nahanni Systems Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include #include #include #include #include #include #include #include "usb_emul.h" SET_DECLARE(usb_emu_set, struct usb_devemu); struct usb_devemu * usb_emu_finddev(const char *name) { struct usb_devemu **udpp, *udp; SET_FOREACH(udpp, usb_emu_set) { udp = *udpp; if (!strcmp(udp->ue_emu, name)) return (udp); } return (NULL); } struct usb_data_xfer_block * usb_data_xfer_append(struct usb_data_xfer *xfer, void *buf, int blen, void *hci_data, int ccs) { struct usb_data_xfer_block *xb; if (xfer->ndata >= USB_MAX_XFER_BLOCKS) return (NULL); xb = &xfer->data[xfer->tail]; xb->buf = buf; xb->blen = blen; xb->hci_data = hci_data; xb->ccs = ccs; xb->processed = 0; xb->bdone = 0; xfer->ndata++; xfer->tail = (xfer->tail + 1) % USB_MAX_XFER_BLOCKS; return (xb); } /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2014 Leon Dang * Copyright 2018 Joyent, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef _USB_EMUL_H_ #define _USB_EMUL_H_ #include #include #include #include #ifndef __FreeBSD__ #include #endif #define USB_MAX_XFER_BLOCKS 8 #define USB_XFER_OUT 0 #define USB_XFER_IN 1 struct usb_hci; struct usb_device_request; struct usb_data_xfer; /* Device emulation handlers */ struct usb_devemu { const char *ue_emu; /* name of device emulation */ int ue_usbver; /* usb version: 2 or 3 */ int ue_usbspeed; /* usb device speed */ /* instance creation */ void *(*ue_init)(struct usb_hci *hci, nvlist_t *nvl); /* handlers */ int (*ue_request)(void *sc, struct usb_data_xfer *xfer); int (*ue_data)(void *sc, struct usb_data_xfer *xfer, int dir, int epctx); int (*ue_reset)(void *sc); int (*ue_remove)(void *sc); int (*ue_stop)(void *sc); }; #define USB_EMUL_SET(x) DATA_SET(usb_emu_set, x) /* * USB device events to notify HCI when state changes */ enum hci_usbev { USBDEV_ATTACH, USBDEV_RESET, USBDEV_STOP, USBDEV_REMOVE, }; /* usb controller, ie xhci, ehci */ struct usb_hci { int (*hci_intr)(struct usb_hci *hci, int epctx); int (*hci_event)(struct usb_hci *hci, enum hci_usbev evid, void *param); void *hci_sc; /* private softc for hci */ /* controller managed fields */ int hci_address; int hci_port; }; /* * Each xfer block is mapped to the hci transfer block. * On input into the device handler, blen is set to the length of buf. * The device handler is to update blen to reflect on the residual size * of the buffer, i.e. len(buf) - len(consumed). */ struct usb_data_xfer_block { void *buf; /* IN or OUT pointer */ int blen; /* in:len(buf), out:len(remaining) */ int bdone; /* bytes transferred */ uint32_t processed; /* device processed this + errcode */ void *hci_data; /* HCI private reference */ int ccs; uint32_t streamid; uint64_t trbnext; /* next TRB guest address */ }; struct usb_data_xfer { struct usb_data_xfer_block data[USB_MAX_XFER_BLOCKS]; struct usb_device_request *ureq; /* setup ctl request */ int ndata; /* # of data items */ int head; int tail; pthread_mutex_t mtx; }; enum USB_ERRCODE { USB_ACK, USB_NAK, USB_STALL, USB_NYET, USB_ERR, USB_SHORT }; #define USB_DATA_GET_ERRCODE(x) (x)->processed >> 8 #define USB_DATA_SET_ERRCODE(x,e) do { \ (x)->processed = ((x)->processed & 0xFF) | (e << 8); \ } while (0) #define USB_DATA_OK(x,i) ((x)->data[(i)].buf != NULL) #define USB_DATA_XFER_INIT(x) do { \ memset((x), 0, sizeof(*(x))); \ pthread_mutex_init(&((x)->mtx), NULL); \ } while (0) #define USB_DATA_XFER_RESET(x) do { \ memset((x)->data, 0, sizeof((x)->data)); \ (x)->ndata = 0; \ (x)->head = (x)->tail = 0; \ } while (0) #define USB_DATA_XFER_LOCK(x) do { \ pthread_mutex_lock(&((x)->mtx)); \ } while (0) #define USB_DATA_XFER_UNLOCK(x) do { \ pthread_mutex_unlock(&((x)->mtx)); \ } while (0) #ifndef __FreeBSD__ #define USB_DATA_XFER_LOCK_HELD(x) MUTEX_HELD(&((x)->mtx)) #endif struct usb_devemu *usb_emu_finddev(const char *name); struct usb_data_xfer_block *usb_data_xfer_append(struct usb_data_xfer *xfer, void *buf, int blen, void *hci_data, int ccs); #endif /* _USB_EMUL_H_ */ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2014 Leon Dang * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include #include #include #include #include #include #include #include "usb_emul.h" #include "console.h" #include "bhyvegc.h" #include "debug.h" static int umouse_debug = 0; #define DPRINTF(params) if (umouse_debug) PRINTLN params #define WPRINTF(params) PRINTLN params /* USB endpoint context (1-15) for reporting mouse data events*/ #define UMOUSE_INTR_ENDPT 1 #define UMOUSE_REPORT_DESC_TYPE 0x22 #define UMOUSE_GET_REPORT 0x01 #define UMOUSE_GET_IDLE 0x02 #define UMOUSE_GET_PROTOCOL 0x03 #define UMOUSE_SET_REPORT 0x09 #define UMOUSE_SET_IDLE 0x0A #define UMOUSE_SET_PROTOCOL 0x0B #define HSETW(ptr, val) ptr = { (uint8_t)(val), (uint8_t)((val) >> 8) } enum { UMSTR_LANG, UMSTR_MANUFACTURER, UMSTR_PRODUCT, UMSTR_SERIAL, UMSTR_CONFIG, UMSTR_MAX }; static const char *umouse_desc_strings[] = { "\x09\x04", "BHYVE", "HID Tablet", "01", "HID Tablet Device", }; struct umouse_hid_descriptor { uint8_t bLength; uint8_t bDescriptorType; uint8_t bcdHID[2]; uint8_t bCountryCode; uint8_t bNumDescriptors; uint8_t bReportDescriptorType; uint8_t wItemLength[2]; } __packed; struct umouse_config_desc { struct usb_config_descriptor confd; struct usb_interface_descriptor ifcd; struct umouse_hid_descriptor hidd; struct usb_endpoint_descriptor endpd; struct usb_endpoint_ss_comp_descriptor sscompd; } __packed; #define MOUSE_MAX_X 0x8000 #define MOUSE_MAX_Y 0x8000 static const uint8_t umouse_report_desc[] = { 0x05, 0x01, /* USAGE_PAGE (Generic Desktop) */ 0x09, 0x02, /* USAGE (Mouse) */ 0xa1, 0x01, /* COLLECTION (Application) */ 0x09, 0x01, /* USAGE (Pointer) */ 0xa1, 0x00, /* COLLECTION (Physical) */ 0x05, 0x09, /* USAGE_PAGE (Button) */ 0x19, 0x01, /* USAGE_MINIMUM (Button 1) */ 0x29, 0x03, /* USAGE_MAXIMUM (Button 3) */ 0x15, 0x00, /* LOGICAL_MINIMUM (0) */ 0x25, 0x01, /* LOGICAL_MAXIMUM (1) */ 0x75, 0x01, /* REPORT_SIZE (1) */ 0x95, 0x03, /* REPORT_COUNT (3) */ 0x81, 0x02, /* INPUT (Data,Var,Abs); 3 buttons */ 0x75, 0x05, /* REPORT_SIZE (5) */ 0x95, 0x01, /* REPORT_COUNT (1) */ 0x81, 0x03, /* INPUT (Cnst,Var,Abs); padding */ 0x05, 0x01, /* USAGE_PAGE (Generic Desktop) */ 0x09, 0x30, /* USAGE (X) */ 0x09, 0x31, /* USAGE (Y) */ 0x35, 0x00, /* PHYSICAL_MINIMUM (0) */ 0x46, 0xff, 0x7f, /* PHYSICAL_MAXIMUM (0x7fff) */ 0x15, 0x00, /* LOGICAL_MINIMUM (0) */ 0x26, 0xff, 0x7f, /* LOGICAL_MAXIMUM (0x7fff) */ 0x75, 0x10, /* REPORT_SIZE (16) */ 0x95, 0x02, /* REPORT_COUNT (2) */ 0x81, 0x02, /* INPUT (Data,Var,Abs) */ 0x05, 0x01, /* USAGE Page (Generic Desktop) */ 0x09, 0x38, /* USAGE (Wheel) */ 0x35, 0x00, /* PHYSICAL_MINIMUM (0) */ 0x45, 0x00, /* PHYSICAL_MAXIMUM (0) */ 0x15, 0x81, /* LOGICAL_MINIMUM (-127) */ 0x25, 0x7f, /* LOGICAL_MAXIMUM (127) */ 0x75, 0x08, /* REPORT_SIZE (8) */ 0x95, 0x01, /* REPORT_COUNT (1) */ 0x81, 0x06, /* INPUT (Data,Var,Rel) */ 0xc0, /* END_COLLECTION */ 0xc0 /* END_COLLECTION */ }; struct umouse_report { uint8_t buttons; /* bits: 0 left, 1 right, 2 middle */ int16_t x; /* x position */ int16_t y; /* y position */ int8_t z; /* z wheel position */ } __packed; #define MSETW(ptr, val) ptr = { (uint8_t)(val), (uint8_t)((val) >> 8) } static struct usb_device_descriptor umouse_dev_desc = { .bLength = sizeof(umouse_dev_desc), .bDescriptorType = UDESC_DEVICE, MSETW(.bcdUSB, UD_USB_3_0), .bMaxPacketSize = 9, /* max pkt size, 2^9 = 512 */ MSETW(.idVendor, 0xFB5D), /* vendor */ MSETW(.idProduct, 0x0001), /* product */ MSETW(.bcdDevice, 0), /* device version */ .iManufacturer = UMSTR_MANUFACTURER, .iProduct = UMSTR_PRODUCT, .iSerialNumber = UMSTR_SERIAL, .bNumConfigurations = 1, }; static struct umouse_config_desc umouse_confd = { .confd = { .bLength = sizeof(umouse_confd.confd), .bDescriptorType = UDESC_CONFIG, .wTotalLength[0] = sizeof(umouse_confd), .bNumInterface = 1, .bConfigurationValue = 1, .iConfiguration = UMSTR_CONFIG, .bmAttributes = UC_BUS_POWERED | UC_REMOTE_WAKEUP, .bMaxPower = 0, }, .ifcd = { .bLength = sizeof(umouse_confd.ifcd), .bDescriptorType = UDESC_INTERFACE, .bNumEndpoints = 1, .bInterfaceClass = UICLASS_HID, .bInterfaceSubClass = UISUBCLASS_BOOT, .bInterfaceProtocol = UIPROTO_MOUSE, }, .hidd = { .bLength = sizeof(umouse_confd.hidd), .bDescriptorType = 0x21, .bcdHID = { 0x01, 0x10 }, .bCountryCode = 0, .bNumDescriptors = 1, .bReportDescriptorType = UMOUSE_REPORT_DESC_TYPE, .wItemLength = { sizeof(umouse_report_desc), 0 }, }, .endpd = { .bLength = sizeof(umouse_confd.endpd), .bDescriptorType = UDESC_ENDPOINT, .bEndpointAddress = UE_DIR_IN | UMOUSE_INTR_ENDPT, .bmAttributes = UE_INTERRUPT, .wMaxPacketSize[0] = 8, .bInterval = 0xA, }, .sscompd = { .bLength = sizeof(umouse_confd.sscompd), .bDescriptorType = UDESC_ENDPOINT_SS_COMP, .bMaxBurst = 0, .bmAttributes = 0, MSETW(.wBytesPerInterval, 0), }, }; struct umouse_bos_desc { struct usb_bos_descriptor bosd; struct usb_devcap_ss_descriptor usbssd; } __packed; static struct umouse_bos_desc umouse_bosd = { .bosd = { .bLength = sizeof(umouse_bosd.bosd), .bDescriptorType = UDESC_BOS, HSETW(.wTotalLength, sizeof(umouse_bosd)), .bNumDeviceCaps = 1, }, .usbssd = { .bLength = sizeof(umouse_bosd.usbssd), .bDescriptorType = UDESC_DEVICE_CAPABILITY, .bDevCapabilityType = 3, .bmAttributes = 0, HSETW(.wSpeedsSupported, 0x08), .bFunctionalitySupport = 3, .bU1DevExitLat = 0xa, /* dummy - not used */ .wU2DevExitLat = { 0x20, 0x00 }, } }; struct umouse_softc { struct usb_hci *hci; struct umouse_report um_report; int newdata; struct { uint8_t idle; uint8_t protocol; uint8_t feature; } hid; pthread_mutex_t mtx; pthread_mutex_t ev_mtx; int polling; struct timeval prev_evt; }; static void umouse_event(uint8_t button, int x, int y, void *arg) { struct umouse_softc *sc; struct bhyvegc_image *gc; gc = console_get_image(); if (gc == NULL) { /* not ready */ return; } sc = arg; pthread_mutex_lock(&sc->mtx); sc->um_report.buttons = 0; sc->um_report.z = 0; if (button & 0x01) sc->um_report.buttons |= 0x01; /* left */ if (button & 0x02) sc->um_report.buttons |= 0x04; /* middle */ if (button & 0x04) sc->um_report.buttons |= 0x02; /* right */ if (button & 0x8) sc->um_report.z = 1; if (button & 0x10) sc->um_report.z = -1; /* scale coords to mouse resolution */ sc->um_report.x = MOUSE_MAX_X * x / gc->width; sc->um_report.y = MOUSE_MAX_Y * y / gc->height; sc->newdata = 1; pthread_mutex_unlock(&sc->mtx); pthread_mutex_lock(&sc->ev_mtx); sc->hci->hci_intr(sc->hci, UE_DIR_IN | UMOUSE_INTR_ENDPT); pthread_mutex_unlock(&sc->ev_mtx); } static void * umouse_init(struct usb_hci *hci, nvlist_t *nvl) { struct umouse_softc *sc; sc = calloc(1, sizeof(struct umouse_softc)); sc->hci = hci; sc->hid.protocol = 1; /* REPORT protocol */ pthread_mutex_init(&sc->mtx, NULL); pthread_mutex_init(&sc->ev_mtx, NULL); console_ptr_register(umouse_event, sc, 10); return (sc); } #define UREQ(x,y) ((x) | ((y) << 8)) static int umouse_request(void *scarg, struct usb_data_xfer *xfer) { struct umouse_softc *sc; struct usb_data_xfer_block *data; const char *str; uint16_t value; uint16_t index; uint16_t len; uint16_t slen; uint8_t *udata; int err; int i, idx; int eshort; sc = scarg; data = NULL; udata = NULL; idx = xfer->head; for (i = 0; i < xfer->ndata; i++) { xfer->data[idx].bdone = 0; if (data == NULL && USB_DATA_OK(xfer,idx)) { data = &xfer->data[idx]; udata = data->buf; } xfer->data[idx].processed = 1; idx = (idx + 1) % USB_MAX_XFER_BLOCKS; } err = USB_ERR_NORMAL_COMPLETION; eshort = 0; if (!xfer->ureq) { DPRINTF(("umouse_request: port %d", sc->hci->hci_port)); goto done; } value = UGETW(xfer->ureq->wValue); index = UGETW(xfer->ureq->wIndex); len = UGETW(xfer->ureq->wLength); DPRINTF(("umouse_request: port %d, type 0x%x, req 0x%x, val 0x%x, " "idx 0x%x, len %u", sc->hci->hci_port, xfer->ureq->bmRequestType, xfer->ureq->bRequest, value, index, len)); switch (UREQ(xfer->ureq->bRequest, xfer->ureq->bmRequestType)) { case UREQ(UR_GET_CONFIG, UT_READ_DEVICE): DPRINTF(("umouse: (UR_GET_CONFIG, UT_READ_DEVICE)")); if (!data) break; *udata = umouse_confd.confd.bConfigurationValue; data->blen = len > 0 ? len - 1 : 0; eshort = data->blen > 0; data->bdone += 1; break; case UREQ(UR_GET_DESCRIPTOR, UT_READ_DEVICE): DPRINTF(("umouse: (UR_GET_DESCRIPTOR, UT_READ_DEVICE) val %x", value >> 8)); if (!data) break; switch (value >> 8) { case UDESC_DEVICE: DPRINTF(("umouse: (->UDESC_DEVICE) len %u ?= " "sizeof(umouse_dev_desc) %lu", len, sizeof(umouse_dev_desc))); if ((value & 0xFF) != 0) { err = USB_ERR_STALLED; goto done; } if (len > sizeof(umouse_dev_desc)) { data->blen = len - sizeof(umouse_dev_desc); len = sizeof(umouse_dev_desc); } else data->blen = 0; memcpy(data->buf, &umouse_dev_desc, len); data->bdone += len; break; case UDESC_CONFIG: DPRINTF(("umouse: (->UDESC_CONFIG)")); if ((value & 0xFF) != 0) { err = USB_ERR_STALLED; goto done; } if (len > sizeof(umouse_confd)) { data->blen = len - sizeof(umouse_confd); len = sizeof(umouse_confd); } else data->blen = 0; memcpy(data->buf, &umouse_confd, len); data->bdone += len; break; case UDESC_STRING: DPRINTF(("umouse: (->UDESC_STRING)")); str = NULL; if ((value & 0xFF) < UMSTR_MAX) str = umouse_desc_strings[value & 0xFF]; else goto done; if ((value & 0xFF) == UMSTR_LANG) { udata[0] = 4; udata[1] = UDESC_STRING; data->blen = len - 2; len -= 2; data->bdone += 2; if (len >= 2) { udata[2] = str[0]; udata[3] = str[1]; data->blen -= 2; data->bdone += 2; } else data->blen = 0; goto done; } slen = 2 + strlen(str) * 2; udata[0] = slen; udata[1] = UDESC_STRING; if (len > slen) { data->blen = len - slen; len = slen; } else data->blen = 0; for (i = 2; i < len; i += 2) { udata[i] = *str++; udata[i+1] = '\0'; } data->bdone += slen; break; case UDESC_BOS: DPRINTF(("umouse: USB3 BOS")); if (len > sizeof(umouse_bosd)) { data->blen = len - sizeof(umouse_bosd); len = sizeof(umouse_bosd); } else data->blen = 0; memcpy(udata, &umouse_bosd, len); data->bdone += len; break; default: DPRINTF(("umouse: unknown(%d)->ERROR", value >> 8)); err = USB_ERR_STALLED; goto done; } eshort = data->blen > 0; break; case UREQ(UR_GET_DESCRIPTOR, UT_READ_INTERFACE): DPRINTF(("umouse: (UR_GET_DESCRIPTOR, UT_READ_INTERFACE) " "0x%x", (value >> 8))); if (!data) break; switch (value >> 8) { case UMOUSE_REPORT_DESC_TYPE: if (len > sizeof(umouse_report_desc)) { data->blen = len - sizeof(umouse_report_desc); len = sizeof(umouse_report_desc); } else data->blen = 0; memcpy(data->buf, umouse_report_desc, len); data->bdone += len; break; default: DPRINTF(("umouse: IO ERROR")); err = USB_ERR_STALLED; goto done; } eshort = data->blen > 0; break; case UREQ(UR_GET_INTERFACE, UT_READ_INTERFACE): DPRINTF(("umouse: (UR_GET_INTERFACE, UT_READ_INTERFACE)")); if (index != 0) { DPRINTF(("umouse get_interface, invalid index %d", index)); err = USB_ERR_STALLED; goto done; } if (!data) break; if (len > 0) { *udata = 0; data->blen = len - 1; } eshort = data->blen > 0; data->bdone += 1; break; case UREQ(UR_GET_STATUS, UT_READ_DEVICE): DPRINTF(("umouse: (UR_GET_STATUS, UT_READ_DEVICE)")); if (data == NULL) break; if (len > 1) { if (sc->hid.feature == UF_DEVICE_REMOTE_WAKEUP) USETW(udata, UDS_REMOTE_WAKEUP); else USETW(udata, 0); data->blen = len - 2; data->bdone += 2; } eshort = data->blen > 0; break; case UREQ(UR_GET_STATUS, UT_READ_INTERFACE): case UREQ(UR_GET_STATUS, UT_READ_ENDPOINT): DPRINTF(("umouse: (UR_GET_STATUS, UT_READ_INTERFACE)")); if (data == NULL) break; if (len > 1) { USETW(udata, 0); data->blen = len - 2; data->bdone += 2; } eshort = data->blen > 0; break; case UREQ(UR_SET_ADDRESS, UT_WRITE_DEVICE): /* XXX Controller should've handled this */ DPRINTF(("umouse set address %u", value)); break; case UREQ(UR_SET_CONFIG, UT_WRITE_DEVICE): DPRINTF(("umouse set config %u", value)); break; case UREQ(UR_SET_DESCRIPTOR, UT_WRITE_DEVICE): DPRINTF(("umouse set descriptor %u", value)); break; case UREQ(UR_CLEAR_FEATURE, UT_WRITE_DEVICE): DPRINTF(("umouse: (UR_SET_FEATURE, UT_WRITE_DEVICE) %x", value)); if (value == UF_DEVICE_REMOTE_WAKEUP) sc->hid.feature = 0; break; case UREQ(UR_SET_FEATURE, UT_WRITE_DEVICE): DPRINTF(("umouse: (UR_SET_FEATURE, UT_WRITE_DEVICE) %x", value)); if (value == UF_DEVICE_REMOTE_WAKEUP) sc->hid.feature = UF_DEVICE_REMOTE_WAKEUP; break; case UREQ(UR_CLEAR_FEATURE, UT_WRITE_INTERFACE): case UREQ(UR_CLEAR_FEATURE, UT_WRITE_ENDPOINT): case UREQ(UR_SET_FEATURE, UT_WRITE_INTERFACE): case UREQ(UR_SET_FEATURE, UT_WRITE_ENDPOINT): DPRINTF(("umouse: (UR_CLEAR_FEATURE, UT_WRITE_INTERFACE)")); err = USB_ERR_STALLED; goto done; case UREQ(UR_SET_INTERFACE, UT_WRITE_INTERFACE): DPRINTF(("umouse set interface %u", value)); break; case UREQ(UR_ISOCH_DELAY, UT_WRITE_DEVICE): DPRINTF(("umouse set isoch delay %u", value)); break; case UREQ(UR_SET_SEL, 0): DPRINTF(("umouse set sel")); break; case UREQ(UR_SYNCH_FRAME, UT_WRITE_ENDPOINT): DPRINTF(("umouse synch frame")); break; /* HID device requests */ case UREQ(UMOUSE_GET_REPORT, UT_READ_CLASS_INTERFACE): DPRINTF(("umouse: (UMOUSE_GET_REPORT, UT_READ_CLASS_INTERFACE) " "0x%x", (value >> 8))); if (!data) break; if ((value >> 8) == 0x01 && len >= sizeof(sc->um_report)) { /* TODO read from backend */ if (len > sizeof(sc->um_report)) { data->blen = len - sizeof(sc->um_report); len = sizeof(sc->um_report); } else data->blen = 0; memcpy(data->buf, &sc->um_report, len); data->bdone += len; } else { err = USB_ERR_STALLED; goto done; } eshort = data->blen > 0; break; case UREQ(UMOUSE_GET_IDLE, UT_READ_CLASS_INTERFACE): if (data == NULL) break; if (len > 0) { *udata = sc->hid.idle; data->blen = len - 1; data->bdone += 1; } eshort = data->blen > 0; break; case UREQ(UMOUSE_GET_PROTOCOL, UT_READ_CLASS_INTERFACE): if (data == NULL) break; if (len > 0) { *udata = sc->hid.protocol; data->blen = len - 1; data->bdone += 1; } eshort = data->blen > 0; break; case UREQ(UMOUSE_SET_REPORT, UT_WRITE_CLASS_INTERFACE): DPRINTF(("umouse: (UMOUSE_SET_REPORT, UT_WRITE_CLASS_INTERFACE) ignored")); break; case UREQ(UMOUSE_SET_IDLE, UT_WRITE_CLASS_INTERFACE): sc->hid.idle = UGETW(xfer->ureq->wValue) >> 8; DPRINTF(("umouse: (UMOUSE_SET_IDLE, UT_WRITE_CLASS_INTERFACE) %x", sc->hid.idle)); break; case UREQ(UMOUSE_SET_PROTOCOL, UT_WRITE_CLASS_INTERFACE): sc->hid.protocol = UGETW(xfer->ureq->wValue) >> 8; DPRINTF(("umouse: (UR_CLEAR_FEATURE, UT_WRITE_CLASS_INTERFACE) %x", sc->hid.protocol)); break; default: DPRINTF(("**** umouse request unhandled")); err = USB_ERR_STALLED; break; } done: /* UT_WRITE is 0, so this is condition is never true. */ #ifdef __FreeBSD__ if (xfer->ureq && (xfer->ureq->bmRequestType & UT_WRITE) && (err == USB_ERR_NORMAL_COMPLETION) && (data != NULL)) data->blen = 0; else if (eshort) err = USB_ERR_SHORT_XFER; #else if (eshort) err = USB_ERR_SHORT_XFER; #endif DPRINTF(("umouse request error code %d (0=ok), blen %u txlen %u", err, (data ? data->blen : 0), (data ? data->bdone : 0))); return (err); } static int umouse_data_handler(void *scarg, struct usb_data_xfer *xfer, int dir, int epctx) { struct umouse_softc *sc; struct usb_data_xfer_block *data; uint8_t *udata; int len, i, idx; int err; DPRINTF(("umouse handle data - DIR=%s|EP=%d, blen %d", dir ? "IN" : "OUT", epctx, xfer->data[0].blen)); /* find buffer to add data */ udata = NULL; err = USB_ERR_NORMAL_COMPLETION; /* handle xfer at first unprocessed item with buffer */ data = NULL; idx = xfer->head; for (i = 0; i < xfer->ndata; i++) { data = &xfer->data[idx]; if (data->buf != NULL && data->blen != 0) { break; } else { data->processed = 1; data = NULL; } idx = (idx + 1) % USB_MAX_XFER_BLOCKS; } if (!data) goto done; udata = data->buf; len = data->blen; if (udata == NULL) { DPRINTF(("umouse no buffer provided for input")); err = USB_ERR_NOMEM; goto done; } sc = scarg; if (dir) { pthread_mutex_lock(&sc->mtx); if (!sc->newdata) { err = USB_ERR_CANCELLED; USB_DATA_SET_ERRCODE(&xfer->data[xfer->head], USB_NAK); pthread_mutex_unlock(&sc->mtx); goto done; } if (sc->polling) { err = USB_ERR_STALLED; USB_DATA_SET_ERRCODE(data, USB_STALL); pthread_mutex_unlock(&sc->mtx); goto done; } sc->polling = 1; if (len > 0) { sc->newdata = 0; data->processed = 1; data->bdone += 6; memcpy(udata, &sc->um_report, 6); data->blen = len - 6; if (data->blen > 0) err = USB_ERR_SHORT_XFER; } sc->polling = 0; pthread_mutex_unlock(&sc->mtx); } else { USB_DATA_SET_ERRCODE(data, USB_STALL); err = USB_ERR_STALLED; } done: return (err); } static int umouse_reset(void *scarg) { struct umouse_softc *sc; sc = scarg; sc->newdata = 0; return (0); } static int umouse_remove(void *scarg) { return (0); } static int umouse_stop(void *scarg) { return (0); } static struct usb_devemu ue_mouse = { .ue_emu = "tablet", .ue_usbver = 3, .ue_usbspeed = USB_SPEED_HIGH, .ue_init = umouse_init, .ue_request = umouse_request, .ue_data = umouse_data_handler, .ue_reset = umouse_reset, .ue_remove = umouse_remove, .ue_stop = umouse_stop }; USB_EMUL_SET(ue_mouse); /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2013 Chris Torek * All rights reserved. * Copyright (c) 2019 Joyent, Inc. * Copyright (c) 2021 The FreeBSD Foundation * * Portions of this software were developed by Ka Ho Ng * under sponsorship of the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * This file and its contents are supplied under the terms of the * Common Development and Distribution License ("CDDL"), version 1.0. * You may only use this file in accordance with the terms of version * 1.0 of the CDDL. * * A full copy of the text of the CDDL should have accompanied this * source. A copy of the CDDL is also available via the Internet at * http://www.illumos.org/license/CDDL. */ /* This file is dual-licensed; see usr/src/contrib/bhyve/LICENSE */ /* * Copyright 2026 Oxide Computer Company */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "bhyverun.h" #include "config.h" #include "debug.h" #include "pci_emul.h" #include "virtio.h" /* * Functions for dealing with generalized "virtual devices" as * defined by * * The reference for the implementation of virtio modern is on * */ #define DPRINTF(vs, fmt, arg...) \ do { \ if ((((vs)->vs_flags) & VIRTIO_DEBUG) != 0) { \ FPRINTLN(stdout, fmt, ##arg); \ fflush(stdout); \ } \ } while (0) #define VQ_NOTIFY_OFF_MULTIPLIER sizeof (uint32_t) /* * In case we decide to relax the "virtio softc comes at the * front of virtio-based device softc" constraint, let's use * this to convert. */ #define DEV_SOFTC(vs) ((void *)(vs)) #define VI_MASK(nbytes) \ (((nbytes) >= 4) ? 0xFFFFFFFFu : (~0u >> (32 - 8 * (nbytes)))) static uint64_t vi_modern_pci_read(struct virtio_softc *, int, uint64_t, int); static void vi_modern_pci_write(struct virtio_softc *, int, uint64_t, int, uint64_t); void vi_queue_linkup(struct virtio_softc *vs, struct vqueue_info *queues) { struct virtio_consts *vc = vs->vs_vc; vs->vs_queues = queues; for (int i = 0; i < vc->vc_nvq; i++) { vs->vs_queues[i].vq_vs = vs; vs->vs_queues[i].vq_num = i; } } /* * Link a virtio_softc to its constants, the device softc, and * the PCI emulation. */ void vi_softc_linkup(struct virtio_softc *vs, struct virtio_consts *vc, void *dev_softc, struct pci_devinst *pi, struct vqueue_info *queues) { /* vs and dev_softc addresses must match */ assert((void *)vs == dev_softc); vs->vs_vc = vc; vs->vs_pi = pi; pi->pi_arg = vs; vi_queue_linkup(vs, queues); } /* * Reset device (device-wide). This erases all queues, i.e., * all the queues become invalid (though we don't wipe out the * internal pointers, we just clear the VQ_ALLOC flag). * * It resets negotiated features to "none". * * If MSI-X is enabled, this also resets all the vectors to NO_VECTOR. */ void vi_reset_dev(struct virtio_softc *vs) { struct vqueue_info *vq; int i, nvq; if (vs->vs_mtx) assert(pthread_mutex_isowned_np(vs->vs_mtx)); nvq = vs->vs_vc->vc_nvq; for (vq = vs->vs_queues, i = 0; i < nvq; vq++, i++) { vq->vq_flags = 0; vq->vq_last_avail = 0; vq->vq_next_used = 0; vq->vq_save_used = 0; vq->vq_pfn = 0; vq->vq_desc_gpa = vq->vq_avail_gpa = vq->vq_used_gpa = 0; vq->vq_msix_idx = VIRTIO_MSI_NO_VECTOR; } vs->vs_negotiated_caps = 0; vs->vs_curq = 0; if (vs->vs_isr != 0) pci_lintr_deassert(vs->vs_pi); vs->vs_isr = 0; vs->vs_msix_cfg_idx = VIRTIO_MSI_NO_VECTOR; } /* * These are the capability bits common to all virtio devices. */ static const virtio_capstr_t virtio_caps[] = { { VIRTIO_F_NOTIFY_ON_EMPTY, "VIRTIO_F_NOTIFY_ON_EMPTY" }, { VIRTIO_F_ANY_LAYOUT, "VIRTIO_F_ANY_LAYOUT" }, { VIRTIO_RING_F_INDIRECT_DESC, "VIRTIO_RING_F_INDIRECT_DESC" }, { VIRTIO_RING_F_EVENT_IDX, "VIRTIO_RING_F_EVENT_IDX" }, { VIRTIO_F_BAD_FEATURE, "VIRTIO_F_BAD_FEATURE" }, { VIRTIO_F_VERSION_1, "VIRTIO_F_VERSION_1" }, }; static void vi_print_caps(struct virtio_softc *vs, uint64_t caps) { struct virtio_consts *vc = vs->vs_vc; if ((vs->vs_flags & VIRTIO_DEBUG) == 0) return; for (size_t i = 0; i < vc->vc_ncapstr; i++) { if ((caps & vc->vc_capstr[i].vp_flag) != 0) FPRINTLN(stdout, " -> %s", vc->vc_capstr[i].vp_name); } for (size_t i = 0; i < ARRAY_SIZE(virtio_caps); i++) { if ((caps & virtio_caps[i].vp_flag) != 0) FPRINTLN(stdout, " -> %s", virtio_caps[i].vp_name); } fflush(stdout); } void vi_set_debug(struct virtio_softc *vs, bool debug) { if (debug) vs->vs_flags |= VIRTIO_DEBUG; else vs->vs_flags &= ~VIRTIO_DEBUG; } bool vi_is_modern(struct virtio_softc *vs) { return (vs->vs_negotiated_caps & VIRTIO_F_VERSION_1) != 0; } void __PRINTFLIKE(2) vi_error(struct virtio_softc *vs, const char *fmt, ...) { va_list ap; va_start(ap, fmt); vfprintf(stderr, fmt, ap); fprintf(stderr, "%s", raw_stdio ? "\r\n" : "\n"); va_end(ap); if (vi_is_modern(vs)) { vs->vs_status |= VTCFG_STATUS_NEEDS_RST; vq_devcfg_changed(vs); } vs->vs_flags |= VIRTIO_BROKEN; } /* * Set I/O BAR (usually 0) to map legacy PCI config registers. */ static bool vi_legacy_iobar_setup(struct virtio_softc *vs, int barnum) { size_t size; /* * We set the size to that which will accommodate the configuration * space with MSI-X enabled, plus the configuration size. */ size = VIRTIO_PCI_CONFIG_OFF(1) + vs->vs_vc->vc_cfgsize; if (pci_emul_alloc_bar(vs->vs_pi, barnum, PCIBAR_IO, size) != 0) return (false); return (true); } virtio_pci_capcfg_t * vi_pci_cfg_bytype(struct virtio_softc *vs, uint8_t cfgtype) { for (uint_t i = 0; i < vs->vs_ncaps; i++) { if (vs->vs_caps[i].c_captype == cfgtype) return (&vs->vs_caps[i]); } return (NULL); } virtio_pci_capcfg_t * vi_pci_cfg_bycapaddr(struct virtio_softc *vs, uint32_t start, uint32_t size) { if (size == 0 || start > UINT32_MAX - size) return (NULL); const uint32_t end = start + size; for (uint_t i = 0; i < vs->vs_ncaps; i++) { virtio_pci_capcfg_t *cfg = &vs->vs_caps[i]; const uint32_t cap_start = cfg->c_capoff; const uint32_t cap_end = cap_start + cfg->c_caplen; if (cap_start <= start && end <= cap_end) return (cfg); } return (NULL); } virtio_pci_capcfg_t * vi_pci_cfg_bybaraddr(struct virtio_softc *vs, uint8_t bar, uint64_t offset, uint32_t size) { /* * We currently don't use the larger capabilities introduced in VirtIO * 1.2 that allow for 64-bit offsets and sizes. */ if (size == 0 || offset > UINT32_MAX - size) return (NULL); const uint32_t end = offset + size; for (uint_t i = 0; i < vs->vs_ncaps; i++) { virtio_pci_capcfg_t *cfg = &vs->vs_caps[i]; if (cfg->c_baridx != bar) continue; const uint32_t bar_start = cfg->c_baroff; const uint32_t bar_end = bar_start + cfg->c_barlen; if (bar_start <= offset && end <= bar_end) return (cfg); } return (NULL); } /* * Add a modern configuration structure capability. */ static bool vi_modern_add_cfg(struct virtio_softc *vs, struct virtio_pci_cap *cap, int barnum, uint32_t baroff, uint32_t barlen, uint8_t caplen, uint8_t cfgtype) { int capoff; cap->cap_vndr = PCIY_VENDOR; cap->cap_len = caplen; cap->cfg_type = cfgtype; cap->bar = barnum; cap->id = 0; cap->offset = baroff; cap->length = barlen; if (pci_emul_add_capability(vs->vs_pi, (u_char *)cap, caplen, &capoff) != 0) { return (false); } vs->vs_caps[vs->vs_ncaps].c_captype = cfgtype; vs->vs_caps[vs->vs_ncaps].c_baridx = cap->bar; vs->vs_caps[vs->vs_ncaps].c_baroff = cap->offset; vs->vs_caps[vs->vs_ncaps].c_barlen = cap->length; vs->vs_caps[vs->vs_ncaps].c_capoff = capoff; vs->vs_caps[vs->vs_ncaps].c_caplen = caplen; vs->vs_ncaps++; VERIFY3U(vs->vs_ncaps, <=, sizeof (vs->vs_caps)); return (true); } /* * Add COMMON_CFG configuration structure capability. */ static bool vi_modern_add_common_cfg(struct virtio_softc *vs, int barnum, uint32_t *offp) { struct virtio_pci_cap cap; uint32_t bardatalen; *offp = roundup2(*offp, VIRTIO_PCI_CAP_COMMON_CFG_ALIGN); /* * We choose to round this BAR area up to a page size in common with * other hypervisors. */ bardatalen = roundup2(sizeof (struct virtio_pci_common_cfg), PAGE_SIZE); memset(&cap, 0, sizeof (cap)); if (vi_modern_add_cfg(vs, &cap, barnum, *offp, bardatalen, sizeof (cap), VIRTIO_PCI_CAP_COMMON_CFG)) { *offp += bardatalen; return (true); } return (false); } /* * Add NOTIFY_CFG configuration structure capability. */ static bool vi_modern_add_notify_cfg(struct virtio_softc *vs, int barnum, uint32_t *offp) { struct virtio_pci_notify_cap cap; struct virtio_consts *vc = vs->vs_vc; int numq = MAX(vc->vc_max_nvq, vc->vc_nvq); uint32_t bardatalen; VERIFY3S(numq, >, 0); VERIFY3S(numq, <=, UINT16_MAX); *offp = roundup2(*offp, VIRTIO_PCI_CAP_NOTIFY_CFG_ALIGN); /* * We choose to round this BAR area up to a page size in common with * other hypervisors. */ uint64_t datalen = (uint64_t)numq * VQ_NOTIFY_OFF_MULTIPLIER; VERIFY3U(datalen, <=, UINT32_MAX - (PAGE_SIZE - 1)); bardatalen = roundup2((uint32_t)datalen, PAGE_SIZE); memset(&cap, 0, sizeof (cap)); cap.notify_off_multiplier = VQ_NOTIFY_OFF_MULTIPLIER; if (vi_modern_add_cfg(vs, &cap.cap, barnum, *offp, bardatalen, sizeof (cap), VIRTIO_PCI_CAP_NOTIFY_CFG)) { *offp += bardatalen; return (true); } return (false); } /* * Add ISR_CFG configuration structure capability. */ static bool vi_modern_add_isr_cfg(struct virtio_softc *vs, int barnum, uint32_t *offp) { struct virtio_pci_cap cap; uint32_t bardatalen; *offp = roundup2(*offp, VIRTIO_PCI_CAP_ISR_CFG_ALIGN); /* * While this capability could point to a single byte in the BAR, we * choose to round up to a page in common with other hypervisors. */ bardatalen = PAGE_SIZE; memset(&cap, 0, sizeof (cap)); if (vi_modern_add_cfg(vs, &cap, barnum, *offp, bardatalen, sizeof (cap), VIRTIO_PCI_CAP_ISR_CFG)) { *offp += bardatalen; return (true); } return (false); } /* * Add DEV_CFG configuration structure capability. */ static bool vi_modern_add_dev_cfg(struct virtio_softc *vs, int barnum, uint32_t *offp) { struct virtio_pci_cap cap; uint32_t bardatalen; *offp = roundup2(*offp, VIRTIO_PCI_CAP_DEVICE_CFG_ALIGN); /* * We choose to round this BAR area up to a page size in common with * other hypervisors. */ bardatalen = PAGE_SIZE; memset(&cap, 0, sizeof (cap)); if (vi_modern_add_cfg(vs, &cap, barnum, *offp, bardatalen, sizeof (cap), VIRTIO_PCI_CAP_DEVICE_CFG)) { *offp += bardatalen; return (true); } return (false); } /* * Add PCI_CFG configuration structure capability. */ static bool vi_modern_add_pci_cfg(struct virtio_softc *vs) { struct virtio_pci_cfg_cap cap; memset(&cap, 0, sizeof (cap)); memset(cap.pci_cfg_data, 0xff, sizeof (cap.pci_cfg_data)); if (vi_modern_add_cfg(vs, &cap.cap, 0, 0, 0, sizeof (cap), VIRTIO_PCI_CAP_PCI_CFG)) { vs->vs_pcicap = &vs->vs_caps[vs->vs_ncaps - 1]; return (true); } return (false); } /* * Set up Virtio modern device pci configuration space */ static bool vi_modern_membar_setup(struct virtio_softc *vs, int barnum) { uint32_t baroff = 0; bool ret = false; ret |= vi_modern_add_common_cfg(vs, barnum, &baroff); ret |= vi_modern_add_notify_cfg(vs, barnum, &baroff); ret |= vi_modern_add_dev_cfg(vs, barnum, &baroff); ret |= vi_modern_add_isr_cfg(vs, barnum, &baroff); ret |= vi_modern_add_pci_cfg(vs); if (!ret) return (false); if (pci_emul_alloc_bar(vs->vs_pi, barnum, PCIBAR_MEM64, baroff) != 0) return (false); return (true); } void vi_pci_init(struct pci_devinst *pi, virtio_mode_t mode, uint16_t legacy, uint16_t device_id, uint8_t class) { struct virtio_softc *vs = pi->pi_arg; DPRINTF(vs, "VIRTIO %s PCI init mode=%x, legacy=0x%x devid=0x%x", vs->vs_vc->vc_name, mode, legacy, device_id); /* * We provide global options to force transitional devices to present * as pure legacy or modern. This is mostly to support testing guest * drivers or bhyve itself. * * TRANSITIONAL mode usually exposes both interfaces * - virtio.legacy=false forces a modern-only device * - virtio.modern=false forces a legacy-only device */ if (mode == VIRTIO_MODE_TRANSITIONAL) { if (!get_config_bool_default("virtio.legacy", true)) mode = VIRTIO_MODE_MODERN; else if (!get_config_bool_default("virtio.modern", true)) mode = VIRTIO_MODE_LEGACY; } vs->vs_mode = mode; pci_set_cfgdata16(pi, PCIR_VENDOR, VIRTIO_VENDOR); pci_set_cfgdata16(pi, PCIR_SUBVEND_0, VIRTIO_VENDOR); pci_set_cfgdata8(pi, PCIR_CLASS, class); if (mode == VIRTIO_MODE_MODERN) { /* * Pure modern / non-transitional device. * * Virtio 1.2, 4.1.2.1: * - PCI Device ID = 0x1040 + virtio device ID * - PCI Revision ID> >= 1 * - PCI Subsystem Device ID >= 0x40 * * `device_id` here is the virtio Device ID from section 5 * [0x0-0x3f]. */ VERIFY3U(device_id, <=, 0x3f); pci_set_cfgdata16(pi, PCIR_DEVICE, VIRTIO_PCI_DEVICEID_MODERN_MIN + device_id); /* * For modern devices the spec only recommends that the * Subsystem Device ID be >= 0x40 to avoid legacy binding. * We choose to mirror the main device ID here so that the * (vendor,device) and (subvendor,subdevice) pairs line up. */ pci_set_cfgdata16(pi, PCIR_SUBDEV_0, VIRTIO_PCI_DEVICEID_MODERN_MIN + device_id); pci_set_cfgdata16(pi, PCIR_REVID, 1); } else { /* * Legacy-only or transitional device. * * For *transitional* devices, virtio 1.2, 4.1.2.3 requires: * - PCI Device ID in [0x1000, 0x103f] * - PCI Revision ID == 0 * - PCI Subsystem Device ID == virtio Device ID * * We rely on the caller to pass: * - `legacy` the 0x1000-0x103f PCI Device ID * - `device_id` the virtio Device ID from section 5 * * For a true legacy-only device this layout is also compatible * with old drivers. */ VERIFY(legacy >= 0x1000 && legacy <= 0x103f); pci_set_cfgdata16(pi, PCIR_DEVICE, legacy); pci_set_cfgdata16(pi, PCIR_SUBDEV_0, device_id); pci_set_cfgdata16(pi, PCIR_REVID, 0); } } /* * Set up Virtio device pci configuration space. */ bool vi_pcibar_setup(struct virtio_softc *vs) { DPRINTF(vs, "VIRTIO %s set up PCI BARs", vs->vs_vc->vc_name); assert(vs->vs_mode != VIRTIO_MODE_UNSET); if (vs->vs_mode == VIRTIO_MODE_LEGACY || vs->vs_mode == VIRTIO_MODE_TRANSITIONAL) { if (!vi_legacy_iobar_setup(vs, VIRTIO_LEGACY_BAR)) return (false); } if (vs->vs_mode == VIRTIO_MODE_MODERN || vs->vs_mode == VIRTIO_MODE_TRANSITIONAL) { if (!vi_modern_membar_setup(vs, VIRTIO_MODERN_BAR)) return (false); } return (true); } /* * Configure interrupt delivery for this VirtIO device. * * If requested, enable MSI-X and allocate one vector per queue plus * a configuration vector. Regardless, always establish the mandatory * legacy (INTx) interrupt as VirtIO devices do not support MSI and * require a fixed interrupt line for compatibility. */ bool vi_intr_init(struct virtio_softc *vs, bool use_msix) { if (use_msix) { struct virtio_consts *vc = vs->vs_vc; int nvec = MIN(MAX(vc->vc_max_nvq, vc->vc_nvq) + 1, MAX_MSIX_TABLE_ENTRIES); vs->vs_flags |= VIRTIO_USE_MSIX; VS_LOCK(vs); vi_reset_dev(vs); /* set all vectors to NO_VECTOR */ VS_UNLOCK(vs); if (pci_emul_add_msixcap(vs->vs_pi, nvec, VIRTIO_MSIX_BAR) != 0) return (false); } else { vs->vs_flags &= ~VIRTIO_USE_MSIX; } /* Legacy interrupts are mandatory for virtio devices */ pci_lintr_request(vs->vs_pi); return (true); } /* * Initialize the currently-selected virtio queue (vs->vs_curq) */ void vi_vq_init(struct virtio_softc *vs) { struct vqueue_info *vq; uint64_t phys; size_t size; char *base; vq = &vs->vs_queues[vs->vs_curq]; phys = vq->vq_desc_gpa; size = vq->vq_qsize * sizeof (struct vring_desc); base = paddr_guest2host(vs->vs_pi->pi_vmctx, phys, size); if (base == NULL) { vi_error(vs, "Could not map queue 0x%x phys 0x%" PRIx64, vq->vq_num, phys); return; } vq->vq_desc = (struct vring_desc *)base; phys = vq->vq_avail_gpa; size = sizeof (struct vring_avail) + sizeof (uint16_t) + vq->vq_qsize * sizeof (uint16_t); base = paddr_guest2host(vs->vs_pi->pi_vmctx, phys, size); if (base == NULL) { vi_error(vs, "Could not map queue 0x%x phys 0x%" PRIx64, vq->vq_num, phys); return; } vq->vq_avail = (struct vring_avail *)base; phys = vq->vq_used_gpa; size = sizeof (struct vring_used) + sizeof (uint16_t) + vq->vq_qsize * sizeof (struct vring_used_elem); base = paddr_guest2host(vs->vs_pi->pi_vmctx, phys, size); if (base == NULL) { vi_error(vs, "Could not map queue 0x%x phys 0x%" PRIx64, vq->vq_num, phys); return; } vq->vq_used = (struct vring_used *)base; /* Mark queue as allocated, and start at 0 when we use it. */ vq->vq_flags = VQ_ALLOC; vq->vq_last_avail = 0; vq->vq_next_used = 0; vq->vq_save_used = 0; } /* * Initialize the currently-selected virtio queue (vs->vs_curq). * The guest just gave us a page frame number, from which we can * calculate the addresses of the queue components. */ void vi_legacy_vq_init(struct virtio_softc *vs, uint32_t pfn) { struct vqueue_info *vq; uint64_t phys; vq = &vs->vs_queues[vs->vs_curq]; vq->vq_pfn = pfn; phys = (uint64_t)pfn << LEGACY_VRING_PFN; /* First page(s) are descriptors... */ vq->vq_desc_gpa = phys; phys += vq->vq_qsize * sizeof (struct vring_desc); /* ... immediately followed by "avail" ring (entirely uint16_t's) */ vq->vq_avail_gpa = phys; phys += sizeof (struct vring_avail) + sizeof (uint16_t) + vq->vq_qsize * sizeof (uint16_t); /* Then it's rounded up to the next page... */ phys = roundup2(phys, LEGACY_VRING_ALIGN); /* ... and the last page(s) are the used ring. */ vq->vq_used_gpa = phys; vi_vq_init(vs); } /* * Helper inline for vq_getchain(): record the i'th "real" * descriptor. */ static inline void _vq_record(struct virtio_softc *vs, int i, struct vring_desc *vd, struct iovec *iov, int n_iov, struct vi_req *reqp) { struct vmctx *ctx; uint32_t len; uint64_t addr; ctx = vs->vs_pi->pi_vmctx; if (i >= n_iov) return; len = atomic_load_32(&vd->len); addr = atomic_load_64(&vd->addr); iov[i].iov_len = len; iov[i].iov_base = paddr_guest2host(ctx, addr, len); if ((vd->flags & VRING_DESC_F_WRITE) == 0) reqp->readable++; else reqp->writable++; } #define VQ_MAX_DESCRIPTORS 512 /* see below */ /* * Examine the chain of descriptors starting at the "next one" to * make sure that they describe a sensible request. If so, return * the number of "real" descriptors that would be needed/used in * acting on this request. This may be smaller than the number of * available descriptors, e.g., if there are two available but * they are two separate requests, this just returns 1. Or, it * may be larger: if there are indirect descriptors involved, * there may only be one descriptor available but it may be an * indirect pointing to eight more. We return 8 in this case, * i.e., we do not count the indirect descriptors, only the "real" * ones. * * Basically, this vets the "flags" and "next" field of each * descriptor and tells you how many are involved. Since some may * be indirect, this also needs the vmctx (in the pci_devinst * at vs->vs_pi) so that it can find indirect descriptors. * * As we process each descriptor, we copy and adjust it (guest to * host address wise, also using the vmtctx) into the given iov[] * array (of the given size). If the array overflows, we stop * placing values into the array but keep processing descriptors, * up to VQ_MAX_DESCRIPTORS, before giving up and returning -1. * So you, the caller, must not assume that iov[] is as big as the * return value (you can process the same thing twice to allocate * a larger iov array if needed, or supply a zero length to find * out how much space is needed). * * If some descriptor(s) are invalid, this prints a diagnostic message * and returns -1. If no descriptors are ready now it simply returns 0. * * You are assumed to have done a vq_ring_ready() if needed (note * that vq_has_descs() does one). */ int vq_getchain(struct vqueue_info *vq, struct iovec *iov, int niov, struct vi_req *reqp) { int i; u_int ndesc, n_indir; u_int idx, next; struct vi_req req; struct vring_desc *vdir, *vindir, *vp; struct vmctx *ctx; struct virtio_softc *vs; const char *name; vs = vq->vq_vs; name = vs->vs_vc->vc_name; memset(&req, 0, sizeof (req)); /* * Note: it's the responsibility of the guest not to * update vq->vq_avail->idx until all of the descriptors * the guest has written are valid (including all their * "next" fields and "flags"). * * Compute (vq_avail->idx - last_avail) in integers mod 2**16. This is * the number of descriptors the device has made available * since the last time we updated vq->vq_last_avail. * * We just need to do the subtraction as an unsigned int, * then trim off excess bits. */ idx = vq->vq_last_avail; ndesc = (uint16_t)((u_int)vq->vq_avail->idx - idx); if (ndesc == 0) return (0); if (ndesc > vq->vq_qsize) { vi_error(vs, "%s: ndesc (%u) out of range, driver confused?", name, (u_int)ndesc); return (-1); } /* * Now count/parse "involved" descriptors starting from * the head of the chain. * * To prevent loops, we could be more complicated and * check whether we're re-visiting a previously visited * index, but we just abort if the count gets excessive. */ ctx = vs->vs_pi->pi_vmctx; req.idx = next = vq->vq_avail->ring[idx & (vq->vq_qsize - 1)]; vq->vq_last_avail++; for (i = 0; i < VQ_MAX_DESCRIPTORS; next = vdir->next) { if (next >= vq->vq_qsize) { vi_error(vs, "%s: descriptor index %u out of range, " "driver confused?", name, next); return (-1); } vdir = &vq->vq_desc[next]; if ((vdir->flags & VRING_DESC_F_INDIRECT) == 0) { _vq_record(vs, i, vdir, iov, niov, &req); i++; } else if ((vs->vs_negotiated_caps & VIRTIO_RING_F_INDIRECT_DESC) == 0) { vi_error(vs, "%s: descriptor has forbidden INDIRECT flag, " "driver confused?", name); return (-1); } else { n_indir = vdir->len / 16; if ((vdir->len & 0xf) || n_indir == 0) { vi_error(vs, "%s: invalid indir len 0x%x, " "driver confused?", name, (u_int)vdir->len); return (-1); } vindir = paddr_guest2host(ctx, vdir->addr, vdir->len); /* * Indirects start at the 0th, then follow * their own embedded "next"s until those run * out. Each one's indirect flag must be off * (we don't really have to check, could just * ignore errors...). */ next = 0; for (;;) { vp = &vindir[next]; if (vp->flags & VRING_DESC_F_INDIRECT) { vi_error(vs, "%s: indirect desc has INDIR flag," " driver confused?", name); return (-1); } _vq_record(vs, i, vp, iov, niov, &req); if (++i > VQ_MAX_DESCRIPTORS) goto loopy; if ((vp->flags & VRING_DESC_F_NEXT) == 0) break; next = vp->next; if (next >= n_indir) { vi_error(vs, "%s: invalid next %u > %u, " "driver confused?", name, (u_int)next, n_indir); return (-1); } } } if ((vdir->flags & VRING_DESC_F_NEXT) == 0) goto done; } loopy: vi_error(vs, "%s: descriptor loop? count > %d - driver confused?", name, i); return (-1); done: *reqp = req; return (i); } /* * Return the first n_chain request chains back to the available queue. * * (These chains are the ones you handled when you called vq_getchain() * and used its positive return value.) */ void vq_retchains(struct vqueue_info *vq, uint16_t n_chains) { vq->vq_last_avail -= n_chains; } void vq_relchain_prepare(struct vqueue_info *vq, uint16_t idx, uint32_t iolen) { struct vring_used *vuh; struct vring_used_elem *vue; uint16_t mask; /* * Notes: * - mask is N-1 where N is a power of 2 so computes x % N * - vuh points to the "used" data shared with guest * - vue points to the "used" ring entry we want to update */ mask = vq->vq_qsize - 1; vuh = vq->vq_used; vue = &vuh->ring[vq->vq_next_used++ & mask]; vue->id = idx; vue->len = iolen; } void vq_relchain_publish(struct vqueue_info *vq) { /* * Ensure the used descriptor is visible before updating the index. * This is necessary on ISAs with memory ordering less strict than x86 * (and even on x86 to act as a compiler barrier). */ atomic_thread_fence_rel(); vq->vq_used->idx = vq->vq_next_used; } /* * Return specified request chain to the guest, setting its I/O length * to the provided value. * * (This chain is the one you handled when you called vq_getchain() * and used its positive return value.) */ void vq_relchain(struct vqueue_info *vq, uint16_t idx, uint32_t iolen) { vq_relchain_prepare(vq, idx, iolen); vq_relchain_publish(vq); } /* * Driver has finished processing "available" chains and calling * vq_relchain on each one. If driver used all the available * chains, used_all should be set. * * If the "used" index moved we may need to inform the guest, i.e., * deliver an interrupt. Even if the used index did NOT move we * may need to deliver an interrupt, if the avail ring is empty and * we are supposed to interrupt on empty. * * Note that used_all_avail is provided by the caller because it's * a snapshot of the ring state when he decided to finish interrupt * processing -- it's possible that descriptors became available after * that point. (It's also typically a constant 1/True as well.) */ void vq_endchains(struct vqueue_info *vq, int used_all_avail) { struct virtio_softc *vs; uint16_t event_idx, new_idx, old_idx; int intr; /* * Interrupt generation: if we're using EVENT_IDX, * interrupt if we've crossed the event threshold. * Otherwise interrupt is generated if we added "used" entries, * but suppressed by VRING_AVAIL_F_NO_INTERRUPT. * * In any case, though, if NOTIFY_ON_EMPTY is set and the * entire avail was processed, we need to interrupt always. */ vs = vq->vq_vs; old_idx = vq->vq_save_used; vq->vq_save_used = new_idx = vq->vq_used->idx; /* * Use full memory barrier between "idx" store from preceding * vq_relchain() call and the loads from VQ_USED_EVENT_IDX() or * "flags" field below. */ atomic_thread_fence_seq_cst(); if (used_all_avail && (vs->vs_negotiated_caps & VIRTIO_F_NOTIFY_ON_EMPTY)) { intr = 1; } else if (vs->vs_negotiated_caps & VIRTIO_RING_F_EVENT_IDX) { event_idx = VQ_USED_EVENT_IDX(vq); /* * This calculation is per docs and the kernel * (see src/sys/dev/virtio/virtio_ring.h). */ intr = (uint16_t)(new_idx - event_idx - 1) < (uint16_t)(new_idx - old_idx); } else { intr = new_idx != old_idx && !(vq->vq_avail->flags & VRING_AVAIL_F_NO_INTERRUPT); } if (intr) vq_interrupt(vs, vq); } /* Note: these are in sorted order to make for a fast search */ static struct config_reg { uint16_t cr_offset; /* register offset */ uint8_t cr_size; /* size (bytes) */ uint8_t cr_ro; /* true => reg is read only */ const char *cr_name; /* name of reg */ } legacy_cfg_regs[] = { { VIRTIO_PCI_HOST_FEATURES, 4, 1, "HOST_FEATURES" }, { VIRTIO_PCI_GUEST_FEATURES, 4, 0, "GUEST_FEATURES" }, { VIRTIO_PCI_QUEUE_PFN, 4, 0, "QUEUE_PFN" }, { VIRTIO_PCI_QUEUE_NUM, 2, 1, "QUEUE_NUM" }, { VIRTIO_PCI_QUEUE_SEL, 2, 0, "QUEUE_SEL" }, { VIRTIO_PCI_QUEUE_NOTIFY, 2, 0, "QUEUE_NOTIFY" }, { VIRTIO_PCI_STATUS, 1, 0, "STATUS" }, { VIRTIO_PCI_ISR, 1, 0, "ISR" }, { VIRTIO_MSI_CONFIG_VECTOR, 2, 0, "CONFIG_VECTOR" }, { VIRTIO_MSI_QUEUE_VECTOR, 2, 0, "QUEUE_VECTOR" }, }, common_cfg_regs[] = { { VIRTIO_PCI_COMMON_DFSELECT, 4, 0, "DFSELECT" }, { VIRTIO_PCI_COMMON_DF, 4, 1, "DF" }, { VIRTIO_PCI_COMMON_GFSELECT, 4, 0, "GFSELECT" }, { VIRTIO_PCI_COMMON_GF, 4, 0, "GF" }, { VIRTIO_PCI_COMMON_MSIX, 2, 0, "MSIX" }, { VIRTIO_PCI_COMMON_NUMQ, 2, 1, "NUMQ" }, { VIRTIO_PCI_COMMON_STATUS, 1, 0, "STATUS" }, { VIRTIO_PCI_COMMON_CFGGENERATION, 1, 1, "CFGGENERATION" }, { VIRTIO_PCI_COMMON_Q_SELECT, 2, 0, "Q_SELECT" }, { VIRTIO_PCI_COMMON_Q_SIZE, 2, 0, "Q_SIZE" }, { VIRTIO_PCI_COMMON_Q_MSIX, 2, 0, "Q_MSIX" }, { VIRTIO_PCI_COMMON_Q_ENABLE, 2, 0, "Q_ENABLE" }, { VIRTIO_PCI_COMMON_Q_NOFF, 2, 1, "Q_NOFF" }, { VIRTIO_PCI_COMMON_Q_DESCLO, 4, 0, "Q_DESCLO" }, { VIRTIO_PCI_COMMON_Q_DESCHI, 4, 0, "Q_DESCHI" }, { VIRTIO_PCI_COMMON_Q_AVAILLO, 4, 0, "Q_AVAILLO" }, { VIRTIO_PCI_COMMON_Q_AVAILHI, 4, 0, "Q_AVAILHI" }, { VIRTIO_PCI_COMMON_Q_USEDLO, 4, 0, "Q_USEDLO" }, { VIRTIO_PCI_COMMON_Q_USEDHI, 4, 0, "Q_USEDHI" }, }; static inline struct config_reg * vi_find_cr(struct config_reg *regstbl, size_t n, int offset) { u_int hi, lo, mid; struct config_reg *cr; lo = 0; hi = n - 1; while (hi >= lo) { mid = (hi + lo) >> 1; cr = ®stbl[mid]; if (cr->cr_offset == offset) return (cr); if (cr->cr_offset < offset) lo = mid + 1; else hi = mid - 1; } return (NULL); } static uint64_t vi_hv_features(struct virtio_softc *vs, bool modern) { return (modern ? vs->vs_vc->vc_hv_caps_modern | VIRTIO_F_VERSION_1 : vs->vs_vc->vc_hv_caps_legacy); } /* * Handle legacy pci config space reads. * * If it's part of the legacy virtio config structure, do that. * Otherwise dispatch to the actual device backend's config read * callback. */ static uint64_t vi_legacy_pci_read(struct virtio_softc *vs, uint64_t offset, int size) { struct virtio_consts *vc; struct config_reg *cr; uint64_t virtio_config_size; const char *name; uint32_t newoff; uint32_t value; int error; /* Checked by caller */ assert(size == 1 || size == 2 || size == 4); vc = vs->vs_vc; name = vc->vc_name; value = VI_MASK(size); virtio_config_size = VIRTIO_PCI_CONFIG_OFF(pci_msix_enabled(vs->vs_pi)); if (offset >= virtio_config_size) { /* * Subtract off the standard size (including MSI-X * registers if enabled) and dispatch to underlying driver. * If that fails, fall into general code. */ newoff = offset - virtio_config_size; if (newoff + size > vc->vc_cfgsize) goto bad; if (vc->vc_cfgread != NULL) { error = (*vc->vc_cfgread)(DEV_SOFTC(vs), newoff, size, &value); } else { error = 0; } if (error == 0) { DPRINTF(vs, "VIRTIO %s LEGACY PCI devcfg read[0x%" PRIx64 "] = 0x%x", name, newoff, value); goto done; } } bad: cr = vi_find_cr(legacy_cfg_regs, nitems(legacy_cfg_regs), offset); if (cr == NULL || cr->cr_size != size) { if (cr != NULL) { /* offset must be OK, so size must be bad */ EPRINTLN( "%s: read from %s: bad size %d", name, cr->cr_name, size); } else { EPRINTLN( "%s: read from bad offset/size %jd/%d", name, (uintmax_t)offset, size); } goto done; } switch (offset) { case VIRTIO_PCI_HOST_FEATURES: /* Caps for legacy PCI configuration layout is only 32bit */ if (vc->vc_hv_features != NULL) value = vc->vc_hv_features(DEV_SOFTC(vs), false); else value = vi_hv_features(vs, false); break; case VIRTIO_PCI_GUEST_FEATURES: value = vs->vs_negotiated_caps; break; case VIRTIO_PCI_QUEUE_PFN: if (!vi_is_modern(vs) && vs->vs_curq < vc->vc_nvq) value = vs->vs_queues[vs->vs_curq].vq_pfn; break; case VIRTIO_PCI_QUEUE_NUM: value = vs->vs_curq < vc->vc_nvq ? vs->vs_queues[vs->vs_curq].vq_qsize : 0; break; case VIRTIO_PCI_QUEUE_SEL: value = vs->vs_curq; break; case VIRTIO_PCI_QUEUE_NOTIFY: value = 0; /* XXX */ break; case VIRTIO_PCI_STATUS: value = vs->vs_status; break; case VIRTIO_PCI_ISR: value = vs->vs_isr; vs->vs_isr = 0; /* a read clears this flag */ if (value != 0) pci_lintr_deassert(vs->vs_pi); break; case VIRTIO_MSI_CONFIG_VECTOR: value = vs->vs_msix_cfg_idx; break; case VIRTIO_MSI_QUEUE_VECTOR: value = vs->vs_curq < vc->vc_nvq ? vs->vs_queues[vs->vs_curq].vq_msix_idx : VIRTIO_MSI_NO_VECTOR; break; } DPRINTF(vs, "VIRTIO %s LEGACY READ %s = 0x%x", name, cr->cr_name, value); switch (offset) { case VIRTIO_PCI_GUEST_FEATURES: case VIRTIO_PCI_HOST_FEATURES: vi_print_caps(vs, value); break; } done: return (value); } /* * Handle legacy pci config space writes. * * If it's part of the legacy virtio config structure, do that. * Otherwise dispatch to the actual device backend's config write * callback. */ static void vi_legacy_pci_write(struct virtio_softc *vs, uint64_t offset, int size, uint64_t value) { struct vqueue_info *vq; struct virtio_consts *vc; struct config_reg *cr; uint64_t virtio_config_size; const char *name; uint32_t newoff; int error; /* Checked by caller */ assert(size == 1 || size == 2 || size == 4); vc = vs->vs_vc; name = vc->vc_name; virtio_config_size = VIRTIO_PCI_CONFIG_OFF(pci_msix_enabled(vs->vs_pi)); if (offset >= virtio_config_size) { /* * Subtract off the standard size (including MSI-X * registers if enabled) and dispatch to underlying driver. */ newoff = offset - virtio_config_size; if (newoff + size > vc->vc_cfgsize) goto bad; if (vc->vc_cfgwrite != NULL) { error = (*vc->vc_cfgwrite)(DEV_SOFTC(vs), newoff, size, value); } else { error = 0; } if (error == 0) { DPRINTF(vs, "VIRTIO %s LEGACY PCI devcfg write[0x%" PRIx64 "] = 0x%x", name, newoff, value); return; } } bad: cr = vi_find_cr(legacy_cfg_regs, nitems(legacy_cfg_regs), offset); if (cr == NULL || cr->cr_size != size || cr->cr_ro) { if (cr != NULL) { /* offset must be OK, wrong size and/or reg is R/O */ if (cr->cr_size != size) EPRINTLN( "%s: write to %s: bad size %d", name, cr->cr_name, size); if (cr->cr_ro) EPRINTLN( "%s: write to read-only reg %s", name, cr->cr_name); } else { EPRINTLN( "%s: write to bad offset/size %jd/%d", name, (uintmax_t)offset, size); } return; } DPRINTF(vs, "VIRTIO %s LEGACY WRITE %s = 0x%x", name, cr->cr_name, value); switch (offset) { case VIRTIO_PCI_GUEST_FEATURES: if (vc->vc_hv_features != NULL) value &= vc->vc_hv_features(DEV_SOFTC(vs), false); else value &= vi_hv_features(vs, false); vs->vs_negotiated_caps = value; if (vc->vc_apply_features != NULL) { (*vc->vc_apply_features)(DEV_SOFTC(vs), &vs->vs_negotiated_caps); } DPRINTF(vs, "NEGOTIATED FEATURES 0x%" PRIx64 " (%s)", vs->vs_negotiated_caps, vi_is_modern(vs) ? "modern" : "legacy"); vi_print_caps(vs, vs->vs_negotiated_caps); break; case VIRTIO_PCI_QUEUE_PFN: if (vs->vs_curq >= vc->vc_nvq) goto bad_qindex; if (vc->vc_qinit != NULL) vc->vc_qinit(DEV_SOFTC(vs), value, false); else vi_legacy_vq_init(vs, value); break; case VIRTIO_PCI_QUEUE_SEL: /* * Note that the guest is allowed to select an * invalid queue; we just need to return a QNUM * of 0 while the bad queue is selected. */ vs->vs_curq = value; break; case VIRTIO_PCI_QUEUE_NOTIFY: if (value >= (unsigned int)vc->vc_nvq) { EPRINTLN("%s: queue %d notify out of range", name, (int)value); break; } if ((vs->vs_flags & VIRTIO_BROKEN) != 0) break; vq = &vs->vs_queues[value]; if (vq->vq_notify != NULL) { (*vq->vq_notify)(DEV_SOFTC(vs), vq); } else if (vc->vc_qnotify != NULL) { (*vc->vc_qnotify)(DEV_SOFTC(vs), vq); } else { EPRINTLN("%s: qnotify queue %d: missing vq/vc notify", name, (int)value); } break; case VIRTIO_PCI_STATUS: vs->vs_status = value; if (value == 0) { DPRINTF(vs, "VIRTIO %s RESET", name); DPRINTF(vs, "**************************************"); vc->vc_reset(DEV_SOFTC(vs)); } break; case VIRTIO_MSI_CONFIG_VECTOR: vs->vs_msix_cfg_idx = value; break; case VIRTIO_MSI_QUEUE_VECTOR: if (vs->vs_curq >= vc->vc_nvq) goto bad_qindex; vq = &vs->vs_queues[vs->vs_curq]; vq->vq_msix_idx = value; if (vc->vc_set_msix != NULL) vc->vc_set_msix(DEV_SOFTC(vs), vs->vs_curq); break; } return; bad_qindex: EPRINTLN( "%s: write config reg %s: curq %d >= max %d", name, cr->cr_name, vs->vs_curq, vc->vc_nvq); } #define VI_HIGH(x) (((x) >> 32) & 0xffffffff) #define VI_LOW(x) ((x) & 0xffffffff) /* * Virtio modern: * Handle pci config space reads to common config structure. */ static uint64_t vi_pci_common_cfg_read(struct virtio_softc *vs, uint64_t offset, int size) { uint64_t value = -1; struct virtio_consts *vc; struct vqueue_info *vq; struct config_reg *cr; const char *name; uint64_t capval = 0; /* Checked by caller */ assert(size == 1 || size == 2 || size == 4); vc = vs->vs_vc; name = vc->vc_name; cr = vi_find_cr(common_cfg_regs, nitems(common_cfg_regs), offset); if (cr == NULL) { EPRINTLN("%s: read from bad offset/size 0x%jx/0x%x", name, (uintmax_t)offset, size); goto done; } /* * We check that the requested size matches the register at this * offset, and refuse to process it if there is a mismatch. */ if (cr->cr_size != size) { EPRINTLN("%s: read from %s: bad size 0x%x", name, cr->cr_name, size); goto done; } vq = (vs->vs_curq < vc->vc_nvq ? &vs->vs_queues[vs->vs_curq] : NULL); switch (offset) { case VIRTIO_PCI_COMMON_DFSELECT: value = vs->vs_dfselect; break; case VIRTIO_PCI_COMMON_DF: if (vc->vc_hv_features != NULL) value = vc->vc_hv_features(DEV_SOFTC(vs), true); else value = vi_hv_features(vs, true); switch (vs->vs_dfselect) { case 0: capval = value = VI_LOW(value); break; case 1: value = VI_HIGH(value); capval = value << 32; break; default: value = capval = 0; break; } /* capval is debug printed below */ break; case VIRTIO_PCI_COMMON_GFSELECT: value = vs->vs_gfselect; break; case VIRTIO_PCI_COMMON_GF: value = vs->vs_negotiated_caps; switch (vs->vs_gfselect) { case 0: capval = value = VI_LOW(value); break; case 1: value = VI_HIGH(value); capval = value << 32; break; default: value = capval = 0; break; } /* capval is debug printed below */ break; case VIRTIO_PCI_COMMON_MSIX: value = vs->vs_msix_cfg_idx; break; case VIRTIO_PCI_COMMON_NUMQ: value = vc->vc_nvq; break; case VIRTIO_PCI_COMMON_STATUS: value = vs->vs_status; break; case VIRTIO_PCI_COMMON_CFGGENERATION: if ((vs->vs_flags & VIRTIO_DEVCFG_CHG) != 0) { vs->vs_devcfg_gen++; vs->vs_flags &= ~VIRTIO_DEVCFG_CHG; } value = vs->vs_devcfg_gen; break; case VIRTIO_PCI_COMMON_Q_SELECT: value = vs->vs_curq; break; case VIRTIO_PCI_COMMON_Q_SIZE: value = vq != NULL ? vq->vq_qsize : 0; break; case VIRTIO_PCI_COMMON_Q_MSIX: if (vq != NULL) value = vq->vq_msix_idx; break; case VIRTIO_PCI_COMMON_Q_ENABLE: value = vq != NULL ? !!(vq->vq_flags & VQ_ENABLED) : 0; break; case VIRTIO_PCI_COMMON_Q_NOFF: /* queue_notify_off is equal to qid for now */ value = vs->vs_curq; break; case VIRTIO_PCI_COMMON_Q_DESCLO: if (vq != NULL) value = VI_LOW(vq->vq_desc_gpa); break; case VIRTIO_PCI_COMMON_Q_DESCHI: if (vq != NULL) value = VI_HIGH(vq->vq_desc_gpa); break; case VIRTIO_PCI_COMMON_Q_AVAILLO: if (vq != NULL) value = VI_LOW(vq->vq_avail_gpa); break; case VIRTIO_PCI_COMMON_Q_AVAILHI: if (vq != NULL) value = VI_HIGH(vq->vq_avail_gpa); break; case VIRTIO_PCI_COMMON_Q_USEDLO: if (vq != NULL) value = VI_LOW(vq->vq_used_gpa); break; case VIRTIO_PCI_COMMON_Q_USEDHI: if (vq != NULL) value = VI_HIGH(vq->vq_used_gpa); break; } done: value &= VI_MASK(size); DPRINTF(vs, "VIRTIO %s COMMON %s read = 0x%x", name, cr->cr_name, value); switch (offset) { case VIRTIO_PCI_COMMON_DF: case VIRTIO_PCI_COMMON_GF: vi_print_caps(vs, capval); break; } return (value); } /* * Virtio modern: * Handle pci config space writes to common config structure. */ static void vi_pci_common_cfg_write(struct virtio_softc *vs, uint64_t offset, int size, uint64_t value) { uint64_t capval = 0; struct virtio_consts *vc; struct vqueue_info *vq; struct config_reg *cr; const char *name; /* Checked by caller */ assert(size == 1 || size == 2 || size == 4); vc = vs->vs_vc; name = vc->vc_name; value &= VI_MASK(size); cr = vi_find_cr(common_cfg_regs, nitems(common_cfg_regs), offset); if (cr == NULL) { EPRINTLN( "%s: write to %s: bad size 0x%x", name, cr->cr_name, size); return; } /* * We check that the requested size matches the register at this * offset, and refuse to process it if there is a mismatch. */ if (cr->cr_size != size) { EPRINTLN("%s: write to bad offset/size 0x%jx/0x%x", name, (uintmax_t)offset, size); return; } DPRINTF(vs, "VIRTIO %s COMMON %s write 0x%x", name, cr->cr_name, value); vq = NULL; switch (offset) { case VIRTIO_PCI_COMMON_Q_SIZE: case VIRTIO_PCI_COMMON_Q_MSIX: case VIRTIO_PCI_COMMON_Q_ENABLE: case VIRTIO_PCI_COMMON_Q_DESCLO: case VIRTIO_PCI_COMMON_Q_DESCHI: case VIRTIO_PCI_COMMON_Q_AVAILLO: case VIRTIO_PCI_COMMON_Q_AVAILHI: case VIRTIO_PCI_COMMON_Q_USEDLO: case VIRTIO_PCI_COMMON_Q_USEDHI: if (vs->vs_curq >= vc->vc_nvq) { EPRINTLN("%s: write queue %d out of range", name, vs->vs_curq); goto bad_write; } vq = &vs->vs_queues[vs->vs_curq]; break; default: break; } switch (offset) { case VIRTIO_PCI_COMMON_DFSELECT: vs->vs_dfselect = value; break; case VIRTIO_PCI_COMMON_GFSELECT: vs->vs_gfselect = value; break; case VIRTIO_PCI_COMMON_GF: switch (vs->vs_gfselect) { case 0: capval = value; vs->vs_negotiated_caps = (VI_HIGH(vs->vs_negotiated_caps) << 32) | value; break; case 1: capval = value << 32; vs->vs_negotiated_caps = capval | VI_LOW(vs->vs_negotiated_caps); break; default: capval = 0; break; } vi_print_caps(vs, capval); uint64_t hvfeat; if (vc->vc_hv_features != NULL) hvfeat = vc->vc_hv_features(DEV_SOFTC(vs), true); else hvfeat = vi_hv_features(vs, true); vs->vs_negotiated_caps &= hvfeat; break; case VIRTIO_PCI_COMMON_MSIX: vs->vs_msix_cfg_idx = value; break; case VIRTIO_PCI_COMMON_STATUS: if (value == 0) { DPRINTF(vs, "VIRTIO %s RESET", name); (*vc->vc_reset)(DEV_SOFTC(vs)); vs->vs_status = value; break; } if ((vs->vs_status & VIRTIO_CONFIG_S_FEATURES_OK) == 0 && (value & VIRTIO_CONFIG_S_FEATURES_OK) != 0) { if (vc->vc_apply_features != NULL) { (*vc->vc_apply_features)(DEV_SOFTC(vs), &vs->vs_negotiated_caps); } DPRINTF(vs, "NEGOTIATED FEATURES 0x%" PRIx64 " (%s)", vs->vs_negotiated_caps, vi_is_modern(vs) ? "modern" : "legacy"); vi_print_caps(vs, vs->vs_negotiated_caps); } vs->vs_status = value; break; case VIRTIO_PCI_COMMON_Q_SELECT: if (value >= vc->vc_nvq) { EPRINTLN("%s: queue select %d out of range", name, (int)value); goto bad_write; } vs->vs_curq = value; break; case VIRTIO_PCI_COMMON_Q_SIZE: /* * If the guest has passed us a queue size that is not a power * of two, something is very wrong. */ if (!ISP2(value)) { vi_error(vs, "Bad queue size 0x%" PRIx64 " for qid 0x%x, not power of 2", value, vq->vq_num); } else { vq->vq_qsize = value; } break; case VIRTIO_PCI_COMMON_Q_MSIX: vq->vq_msix_idx = value; if (vc->vc_set_msix != NULL) vc->vc_set_msix(DEV_SOFTC(vs), vs->vs_curq); break; case VIRTIO_PCI_COMMON_Q_ENABLE: if ((vq->vq_flags & VQ_ENABLED) == 0 && value == 1) { if (vc->vc_qinit != NULL) vc->vc_qinit(DEV_SOFTC(vs), 0, true); else vi_vq_init(vs); vq->vq_flags |= VQ_ENABLED; } else if (value == 0) { /* * The driver is not permitted to write a 0 to this * register. We choose to ignore it rather than fault * the device. */ } break; case VIRTIO_PCI_COMMON_Q_DESCLO: vq->vq_desc_gpa = (VI_HIGH(vq->vq_desc_gpa) << 32) | value; break; case VIRTIO_PCI_COMMON_Q_DESCHI: vq->vq_desc_gpa = (value << 32) | VI_LOW(vq->vq_desc_gpa); break; case VIRTIO_PCI_COMMON_Q_AVAILLO: vq->vq_avail_gpa = (VI_HIGH(vq->vq_avail_gpa) << 32) | value; break; case VIRTIO_PCI_COMMON_Q_AVAILHI: vq->vq_avail_gpa = (value << 32) | VI_LOW(vq->vq_avail_gpa); break; case VIRTIO_PCI_COMMON_Q_USEDLO: vq->vq_used_gpa = (VI_HIGH(vq->vq_used_gpa) << 32) | value; break; case VIRTIO_PCI_COMMON_Q_USEDHI: vq->vq_used_gpa = (value << 32) | VI_LOW(vq->vq_used_gpa); break; default: EPRINTLN("%s: write to bad offset/size %jd/%d", name, (uintmax_t)offset, size); goto bad_write; } return; bad_write: return; } /* * Virtio modern: * Handle pci MMIO reads to the notification structure. * * Reading the structure always returns zero. */ static uint64_t vi_pci_notify_cfg_read(struct virtio_softc *vs, uint64_t offset, int size) { return (0); } /* * Virtio modern: * Handle pci MMIO writes to the notification structure. * * VIRTIO_F_NOTIFICATION_DATA is not a feature that this device advertises * so we only need to consider the simple case where the vq index is written * into the registers. */ static void vi_pci_notify_cfg_write(struct virtio_softc *vs, uint64_t offset, int size, uint64_t value) { struct virtio_consts *vc = vs->vs_vc; const char *name = vc->vc_name; unsigned int qid = value; struct vqueue_info *vq; DPRINTF(vs, "VIRTIO %s notify VQ 0x%x offset 0x%x", name, qid, offset); if (size != 2) { EPRINTLN("%s: bad size 0x%x access at offset 0x%" PRIx64, name, size, offset); return; } if ((vs->vs_status & VIRTIO_CONFIG_STATUS_DRIVER_OK) == 0) { EPRINTLN("%s: attempt to use VQ 0x%x before DRIVER_OK, " "driver confused?", name, qid); return; } if ((vs->vs_flags & VIRTIO_BROKEN) != 0) { EPRINTLN("%s: attempt to use VQ 0x%x while VIRTIO device is " "flagged as broken", name, qid); return; } if (offset != qid * VQ_NOTIFY_OFF_MULTIPLIER) { EPRINTLN( "%s: VQ 0x%x notify does not have matching offset at 0x%" PRIx64, name, qid, offset); return; } if (qid >= vc->vc_nvq) { EPRINTLN("%s: VQ 0x%x notify out of range", name, qid); return; } vq = &vs->vs_queues[qid]; if ((vq->vq_flags & VQ_ENABLED) == 0) return; if (vq->vq_notify != NULL) (*vq->vq_notify)(DEV_SOFTC(vs), vq); else if (vc->vc_qnotify != NULL) (*vc->vc_qnotify)(DEV_SOFTC(vs), vq); else EPRINTLN("%s: qnotify VQ 0x%x: no vq/vc notify", name, qid); } /* * Virtio modern: * Handle pci MMIO reads to ISR structure. * * The ISR structure has a relaxed requirement on alignment. */ static uint64_t vi_pci_isr_cfg_read(struct virtio_softc *vs, uint64_t offset, int size) { uint64_t value; if (offset != 0) return (0); value = vs->vs_isr; vs->vs_isr = 0; if (value != 0) { DPRINTF(vs, "VIRTIO ISR read[0x%" PRIx64 "] = 0x%x", offset, value); pci_lintr_deassert(vs->vs_pi); } return (value); } /* * Virtio modern: * pci MMIO writes to ISR structure are disallowed. */ static void vi_pci_isr_cfg_write(struct virtio_softc *vs, uint64_t offset, int size, uint64_t value) { const char *name = vs->vs_vc->vc_name; EPRINTLN("%s: invalid write into isr cfg", name); } /* * Virtio modern: * Handle pci MMIO reads to device-specific config structure. */ static uint64_t vi_pci_dev_cfg_read(struct virtio_softc *vs, uint64_t offset, int size) { struct virtio_consts *vc = vs->vs_vc; uint32_t value = VI_MASK(size); if (offset + size > vc->vc_cfgsize) return (value); vc->vc_cfgread(DEV_SOFTC(vs), offset, size, &value); DPRINTF(vs, "VIRTIO %s PCI devcfg read[0x%" PRIx64 "] = 0x%x", vs->vs_vc->vc_name, offset, value); return (value); } /* * Virtio modern: * Handle pci MMIO writes to device-specific config structure. */ static void vi_pci_dev_cfg_write(struct virtio_softc *vs, uint64_t offset, int size, uint64_t value) { struct virtio_consts *vc = vs->vs_vc; value &= VI_MASK(size); if (offset + size > vc->vc_cfgsize) return; if (vc->vc_cfgwrite != NULL) vc->vc_cfgwrite(DEV_SOFTC(vs), offset, size, value); DPRINTF(vs, "VIRTIO %s PCI devcfg write[0x%" PRIx64 "] = 0x%x", vs->vs_vc->vc_name, offset, value); } /* * Handle configuration space reads. */ int vi_pci_cfgread(struct pci_devinst *pi, int offset, int bytes, uint32_t *retval) { struct virtio_softc *vs = pi->pi_arg; virtio_pci_capcfg_t *cfg; uint32_t baroff, barlen; int baridx; cfg = vi_pci_cfg_bycapaddr(vs, offset, bytes); /* If this is not a VirtIO cap, use the default cfgspace handler */ if (cfg == NULL) return (PE_CFGRW_DEFAULT); /* Only the PCI cap has special handling */ if (cfg->c_captype != VIRTIO_PCI_CAP_PCI_CFG) return (PE_CFGRW_DEFAULT); /* and then only the data field */ if (offset != vs->vs_pcicap->c_capoff + offsetof(struct virtio_pci_cfg_cap, pci_cfg_data)) { return (PE_CFGRW_DEFAULT); } if (bytes != 1 && bytes != 2 && bytes != 4) return (PE_CFGRW_DROP); if (vs->vs_mtx) pthread_mutex_lock(vs->vs_mtx); baridx = pci_get_cfgdata8(pi, offset + offsetof(struct virtio_pci_cap, bar)); baroff = pci_get_cfgdata32(pi, offset + offsetof(struct virtio_pci_cap, offset)); barlen = pci_get_cfgdata32(pi, offset + offsetof(struct virtio_pci_cap, length)); if (baridx > PCIR_MAX_BAR_0) { *retval = VI_MASK(bytes); goto done; } *retval = vi_modern_pci_read(vs, baridx, baroff, barlen); done: if (vs->vs_mtx) pthread_mutex_unlock(vs->vs_mtx); DPRINTF(vs, "VIRTIO %s PCI READ BAR%u[0x%x+%x] = 0x%x", vs->vs_vc->vc_name, baridx, baroff, barlen, *retval); return (PE_CFGRW_DROP); } /* * Handle configuration space writes. */ int vi_pci_cfgwrite(struct pci_devinst *pi, int offset, int bytes, uint32_t val) { struct virtio_softc *vs = pi->pi_arg; virtio_pci_capcfg_t *cfg; uint32_t baroff, barlen; int baridx; cfg = vi_pci_cfg_bycapaddr(vs, offset, bytes); /* If this is not a VirtIO cap, use the default cfgspace handler */ if (cfg == NULL) return (PE_CFGRW_DEFAULT); /* Only the PCI VirtIO cap can be written to */ if (cfg->c_captype != VIRTIO_PCI_CAP_PCI_CFG) return (PE_CFGRW_DROP); /* and then only the data field needs special handling */ if (offset != vs->vs_pcicap->c_capoff + offsetof(struct virtio_pci_cfg_cap, pci_cfg_data)) { return (PE_CFGRW_DEFAULT); } if (bytes != 1 && bytes != 2 && bytes != 4) return (PE_CFGRW_DROP); if (vs->vs_mtx) pthread_mutex_lock(vs->vs_mtx); baridx = pci_get_cfgdata8(pi, offset + offsetof(struct virtio_pci_cap, bar)); baroff = pci_get_cfgdata32(pi, offset + offsetof(struct virtio_pci_cap, offset)); barlen = pci_get_cfgdata32(pi, offset + offsetof(struct virtio_pci_cap, length)); if (baridx > PCIR_MAX_BAR_0) goto done; vi_modern_pci_write(vs, baridx, baroff, barlen, val); done: if (vs->vs_mtx) pthread_mutex_unlock(vs->vs_mtx); DPRINTF(vs, "VIRTIO %s PCI WRITE BAR%x[0x%x+%x] = 0x%x", vs->vs_vc->vc_name, baridx, baroff, barlen, val); return (PE_CFGRW_DROP); } /* * Handle pci config space reads to virtio-related structures */ static uint64_t vi_modern_pci_read(struct virtio_softc *vs, int baridx, uint64_t offset, int size) { virtio_pci_capcfg_t *cfg; uint64_t value = VI_MASK(size); cfg = vi_pci_cfg_bybaraddr(vs, baridx, offset, size); if (cfg == NULL) return (value); offset -= cfg->c_baroff; switch (cfg->c_captype) { case VIRTIO_PCI_CAP_COMMON_CFG: value = vi_pci_common_cfg_read(vs, offset, size); break; case VIRTIO_PCI_CAP_NOTIFY_CFG: value = vi_pci_notify_cfg_read(vs, offset, size); break; case VIRTIO_PCI_CAP_ISR_CFG: value = vi_pci_isr_cfg_read(vs, offset, size); break; case VIRTIO_PCI_CAP_DEVICE_CFG: value = vi_pci_dev_cfg_read(vs, offset, size); break; default: break; } return (value); } /* * Handle pci config space reads to virtio-related structures */ static void vi_modern_pci_write(struct virtio_softc *vs, int baridx, uint64_t offset, int size, uint64_t value) { virtio_pci_capcfg_t *cfg; cfg = vi_pci_cfg_bybaraddr(vs, baridx, offset, size); if (cfg == NULL) return; offset -= cfg->c_baroff; switch (cfg->c_captype) { case VIRTIO_PCI_CAP_COMMON_CFG: vi_pci_common_cfg_write(vs, offset, size, value); break; case VIRTIO_PCI_CAP_NOTIFY_CFG: vi_pci_notify_cfg_write(vs, offset, size, value); break; case VIRTIO_PCI_CAP_ISR_CFG: vi_pci_isr_cfg_write(vs, offset, size, value); break; case VIRTIO_PCI_CAP_DEVICE_CFG: vi_pci_dev_cfg_write(vs, offset, size, value); break; } } /* * Handle virtio bar reads. * * If it's to the MSI-X info, dispatch the reads to the msix handling code. * Otherwise, dispatch the reads to virtio device code. */ uint64_t vi_pci_read(struct pci_devinst *pi, int baridx, uint64_t offset, int size) { struct virtio_softc *vs = pi->pi_arg; uint64_t value; if ((vs->vs_flags & VIRTIO_USE_MSIX) != 0 && (baridx == pci_msix_table_bar(pi) || baridx == pci_msix_pba_bar(pi))) { return (pci_emul_msix_tread(pi, offset, size)); } if (vs->vs_mtx) pthread_mutex_lock(vs->vs_mtx); value = VI_MASK(size); if (size != 1 && size != 2 && size != 4) goto done; switch (baridx) { case VIRTIO_LEGACY_BAR: value = vi_legacy_pci_read(vs, offset, size); break; case VIRTIO_MODERN_BAR: value = vi_modern_pci_read(vs, baridx, offset, size); break; default: break; } done: if (vs->vs_mtx) pthread_mutex_unlock(vs->vs_mtx); return (value); } /* * Handle virtio bar writes. * * If it's to the MSI-X info, dispatch the writes to the msix handling code. * Otherwise, dispatch the writes to virtio device code. */ void vi_pci_write(struct pci_devinst *pi, int baridx, uint64_t offset, int size, uint64_t value) { struct virtio_softc *vs = pi->pi_arg; struct virtio_consts *vc = vs->vs_vc; if ((vs->vs_flags & VIRTIO_USE_MSIX) != 0 && (baridx == pci_msix_table_bar(pi) || baridx == pci_msix_pba_bar(pi))) { if (pci_emul_msix_twrite(pi, offset, size, value) == 0 && vc->vc_update_msix != NULL) { vc->vc_update_msix(DEV_SOFTC(vs), offset); } return; } if (vs->vs_mtx) pthread_mutex_lock(vs->vs_mtx); if (size != 1 && size != 2 && size != 4) goto done; switch (baridx) { case VIRTIO_LEGACY_BAR: vi_legacy_pci_write(vs, offset, size, value); break; case VIRTIO_MODERN_BAR: vi_modern_pci_write(vs, baridx, offset, size, value); break; default: break; } done: if (vs->vs_mtx) pthread_mutex_unlock(vs->vs_mtx); } /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2013 Chris Torek * All rights reserved. * Copyright (c) 2021 The FreeBSD Foundation * * Portions of this software were developed by Ka Ho Ng * under sponsorship of the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * This file and its contents are supplied under the terms of the * Common Development and Distribution License ("CDDL"), version 1.0. * You may only use this file in accordance with the terms of version * 1.0 of the CDDL. * * A full copy of the text of the CDDL should have accompanied this * source. A copy of the CDDL is also available via the Internet at * http://www.illumos.org/license/CDDL. */ /* This file is dual-licensed; see usr/src/contrib/bhyve/LICENSE */ /* * Copyright 2026 Oxide Computer Company */ #ifndef _BHYVE_VIRTIO_H_ #define _BHYVE_VIRTIO_H_ #include #include #include #include /* * Virtio legacy support is derived from several specifications below: * https://github.com/rustyrussell/virtio-spec * http://people.redhat.com/pbonzini/virtio-spec.pdf * * Virtio modern support is authored with the reference below: * https://docs.oasis-open.org/virtio/virtio/v1.2/virtio-v1.2.pdf */ /* * A virtual device has zero or more "virtual queues" (virtqueue). * For a legacy device, each virtqueue uses at least two 4096-byte pages, laid * out thus: * * +-----------------------------------------------+ * | "desc": descriptors, 16 bytes each | * | ----------------------------------------- | * | "avail": 2 uint16; uint16; 1 uint16 | * | ----------------------------------------- | * | pad to 4k boundary | * +-----------------------------------------------+ * | "used": 2 x uint16; elems; 1 uint16 | * | ----------------------------------------- | * | pad to 4k boundary | * +-----------------------------------------------+ * * The number that appears here is always a power of two and is * limited to no more than 32768 (as it must fit in a 16-bit field). * If is sufficiently large, the above will occupy more than * two pages. In any case, all pages must be physically contiguous * within the guest's physical address space. * * The 16-byte "desc" descriptors consist of a 64-bit guest * physical address , a 32-bit length , a 16-bit * , and a 16-bit field (all in guest byte order). * * There are three flags that may be set : * NEXT descriptor is chained, so use its "next" field * WRITE descriptor is for host to write into guest RAM * (else host is to read from guest RAM) * INDIRECT descriptor address field is (guest physical) * address of a linear array of descriptors * * Unless INDIRECT is set, is the number of bytes that may * be read/written from guest physical address . If * INDIRECT is set, WRITE is ignored and provides the length * of the indirect descriptors (and must be a multiple of * 16). Note that NEXT may still be set in the main descriptor * pointing to the indirect, and should be set in each indirect * descriptor that uses the next descriptor (these should generally * be numbered sequentially). However, INDIRECT must not be set * in the indirect descriptors. Upon reaching an indirect descriptor * without a NEXT bit, control returns to the direct descriptors. * * Except inside an indirect, each value must be in the * range [0 .. N) (i.e., the half-open interval). (Inside an * indirect, each must be in the range [0 .. /16).) * * The "avail" data structures reside in the same pages as the * "desc" structures since both together are used by the device to * pass information to the hypervisor's virtual driver. These * begin with a 16-bit field and 16-bit index , then * have 16-bit values, followed by one final 16-bit * field . The entries are simply indices * into the descriptor ring (and thus must meet the same * constraints as each value). However, is counted * up from 0 (initially) and simply wraps around after 65535; it * is taken mod to find the next available entry. * * The "used" ring occupies a separate page or pages, and contains * values written from the virtual driver back to the guest OS. * This begins with a 16-bit and 16-bit , then there * are "vring_used" elements, followed by a 16-bit . * The "vring_used" elements consist of a 32-bit and a * 32-bit (vu_tlen below). The is simply the index of * the head of a descriptor chain the guest made available * earlier, and the is the number of bytes actually written, * e.g., in the case of a network driver that provided a large * receive buffer but received only a small amount of data. * * The two event fields, and , in the * avail and used rings (respectively -- note the reversal!), are * always provided, but are used only if the virtual device * negotiates the VIRTIO_RING_F_EVENT_IDX feature during feature * negotiation. Similarly, both rings provide a flag -- * VRING_AVAIL_F_NO_INTERRUPT and VRING_USED_F_NO_NOTIFY -- in * their field, indicating that the guest does not need an * interrupt, or that the hypervisor driver does not need a * notify, when descriptors are added to the corresponding ring. * (These are provided only for interrupt optimization and need * not be implemented.) */ #define LEGACY_VRING_ALIGN 4096 /* * Virtio legacy: * * The address of any given virtual queue is determined by a single * Page Frame Number register. The guest writes the PFN into the * PCI config space. However, a device that has two or more * virtqueues can have a different PFN, and size, for each queue. * The number of queues is determinable via the PCI config space * VTCFG_R_QSEL register. Writes to QSEL select the queue: 0 means * queue #0, 1 means queue#1, etc. Once a queue is selected, the * remaining PFN and QNUM registers refer to that queue. * * QNUM is a read-only register containing a nonzero power of two * that indicates the (hypervisor's) queue size. Or, if reading it * produces zero, the hypervisor does not have a corresponding * queue. (The number of possible queues depends on the virtual * device. The block device has just one; the network device * provides either two -- 0 = receive, 1 = transmit -- or three, * with 2 = control.) * * PFN is a read/write register giving the physical page address of * the virtqueue in guest memory (the guest must allocate enough space * based on the hypervisor's provided QNUM). * * QNOTIFY is effectively write-only: when the guest writes a queue * number to the register, the hypervisor should scan the specified * virtqueue. (Reading QNOTIFY currently always gets 0). */ /* * PFN register shift amount */ #define LEGACY_VRING_PFN 12 /* * PCI vendor/device IDs */ #define VIRTIO_VENDOR 0x1AF4 #define VIRTIO_DEV_NET 0x1000 #define VIRTIO_DEV_BLOCK 0x1001 #define VIRTIO_DEV_CONSOLE 0x1003 #define VIRTIO_DEV_SCSI 0x1004 #define VIRTIO_DEV_RANDOM 0x1005 #define VIRTIO_DEV_9P 0x1009 #define VIRTIO_DEV_INPUT 0x1052 /* * PCI revision IDs */ #define VIRTIO_REV_INPUT 1 /* * PCI subvendor IDs */ #define VIRTIO_SUBVEN_INPUT 0x108E /* * PCI subdevice IDs */ #define VIRTIO_SUBDEV_INPUT 0x1100 struct pci_devinst; struct vqueue_info; /* * A virtual device, with some number (possibly 0) of virtual * queues and some size (possibly 0) of configuration-space * registers private to the device. The virtio_softc should come * at the front of each "derived class", so that a pointer to the * virtio_softc is also a pointer to the more specific, derived- * from-virtio driver's softc. * * Note: inside each hypervisor virtio driver, changes to these * data structures must be locked against other threads, if any. * Except for PCI config space register read/write, we assume each * driver does the required locking, but we need a pointer to the * lock (if there is one) for PCI config space read/write ops. * * When the guest reads or writes the device's config space, the * generic layer checks for operations on the special registers * described above. If the offset of the register(s) being read * or written is past the CFG area (CFG0 or CFG1), the request is * passed on to the virtual device, after subtracting off the * generic-layer size. (So, drivers can just use the offset as * an offset into "struct config", for instance.) * * (The virtio layer also makes sure that the read or write is to/ * from a "good" config offset, hence vc_cfgsize, and on BAR #0. * However, the driver must verify the read or write size and offset * and that no one is writing a readonly register.) */ typedef enum virtio_flags { VIRTIO_DEBUG = 1 << 0, VIRTIO_USE_MSIX = 1 << 1, VIRTIO_EVENT_IDX = 1 << 2, /* use the event-index values */ VIRTIO_DEVCFG_CHG = 1 << 3, /* Device configuration changed */ VIRTIO_BROKEN = 1 << 4, } virtio_flags_t; typedef enum virtio_mode { VIRTIO_MODE_UNSET = 0, VIRTIO_MODE_LEGACY, VIRTIO_MODE_TRANSITIONAL, VIRTIO_MODE_MODERN } virtio_mode_t; /* * This describes a Virtio PCI capability in config space. */ typedef struct virtio_pci_capcfg { uint8_t c_captype; /* * The offset and length of the capability in config space. */ uint32_t c_capoff; uint32_t c_caplen; /* * The containing BAR, offset and length of the data to which the * capability points. */ uint32_t c_barlen; uint32_t c_baroff; uint8_t c_baridx; } virtio_pci_capcfg_t; struct virtio_softc { struct virtio_consts *vs_vc; /* constants (see below) */ virtio_flags_t vs_flags; /* VIRTIO_* flags from above */ virtio_mode_t vs_mode; /* VIRTIO_MODE_* values from above */ pthread_mutex_t *vs_mtx; /* POSIX mutex, if any */ struct pci_devinst *vs_pi; /* PCI device instance */ uint64_t vs_negotiated_caps; /* negotiated capabilities */ struct vqueue_info *vs_queues; /* one per vc_nvq */ int vs_curq; /* current queue */ uint8_t vs_status; /* value from last status write */ uint8_t vs_isr; /* ISR flags, if not MSI-X */ uint16_t vs_msix_cfg_idx; /* MSI-X vector for config event */ uint32_t vs_dfselect; /* Current DFSELECT value */ uint32_t vs_gfselect; /* Current GFSELECT value */ uint8_t vs_devcfg_gen; /* Generation of device config space */ virtio_pci_capcfg_t vs_caps[VIRTIO_PCI_CAP_MAX]; /* PCI capabilities */ virtio_pci_capcfg_t *vs_pcicap; /* PCI configuration access cap */ uint_t vs_ncaps; /* Number of PCI capabilities */ }; #define VS_LOCK(vs) \ do { \ if (vs->vs_mtx) \ pthread_mutex_lock(vs->vs_mtx); \ } while (0) #define VS_UNLOCK(vs) \ do { \ if (vs->vs_mtx) \ pthread_mutex_unlock(vs->vs_mtx); \ } while (0) /* * To aid debugging we allow drivers to provide a table to map feature bits to * text. */ typedef struct virtio_capstr { uint64_t vp_flag; const char *vp_name; } virtio_capstr_t; struct virtio_consts { const char *vc_name; /* name of driver (for diagnostics) */ int vc_nvq; /* current number of virtual queues */ int vc_max_nvq; /* max no. queues, for multi-queue */ size_t vc_cfgsize; /* size of dev-specific config regs */ void (*vc_reset)(void *); /* called on virtual device reset */ void (*vc_qinit)(void *, uint64_t, bool); void (*vc_qnotify)(void *, struct vqueue_info *); /* called on QNOTIFY if no VQ notify */ int (*vc_cfgread)(void *, int, int, uint32_t *); /* called to read config regs */ int (*vc_cfgwrite)(void *, int, int, uint32_t); /* called to write config regs */ void (*vc_apply_features)(void *, uint64_t *); /* called to apply negotiated features */ uint64_t (*vc_hv_features)(void *, bool); /* called to read device features */ void (*vc_set_msix)(void *, int); void (*vc_update_msix)(void *, uint64_t); uint64_t vc_hv_caps_legacy; /* hypervisor-provided capabilities (legacy) */ uint64_t vc_hv_caps_modern; /* hypervisor-provided capabilities (modern) */ /* * Optional feature bit map. */ size_t vc_ncapstr; virtio_capstr_t *vc_capstr; }; /* * Data structure allocated (statically) per virtual queue. * * Drivers may change vq_qsize after a reset. When the guest OS * requests a device reset, the hypervisor first calls * vs->vs_vc->vc_reset(); then the data structure below is * reinitialized (for each virtqueue: vs->vs_vc->vc_nvq). * * The remaining fields should only be fussed-with by the generic * code. * * Note: the addresses of vq_desc, vq_avail, and vq_used are all * computable from each other in the legacy interface, but even * there it's a lot simpler if we just keep a pointer to * each one. The event indices are similarly (but more * easily) computable, and this time we'll compute them: * they're just XX_ring[N]. */ #define VQ_ALLOC 0x01 /* set once we have a pfn */ #define VQ_ENABLED 0x02 /* set if the queue was enabled */ struct vqueue_info { uint16_t vq_qsize; /* size of this queue (a power of 2) */ void (*vq_notify)(void *, struct vqueue_info *); /* called instead of vc_notify, if not NULL */ struct virtio_softc *vq_vs; /* backpointer to softc */ uint16_t vq_num; /* we're the num'th queue in the softc */ uint16_t vq_flags; /* flags (see above) */ uint16_t vq_last_avail; /* a recent value of vq_avail->idx */ uint16_t vq_next_used; /* index of the next used slot to be filled */ uint16_t vq_save_used; /* saved vq_used->idx; see vq_endchains */ uint16_t vq_msix_idx; /* MSI-X index, or VIRTIO_MSI_NO_VECTOR */ uint32_t vq_pfn; /* PFN of virt queue (not shifted!) */ uint64_t vq_desc_gpa; /* PA of virtqueue descriptors ring */ uint64_t vq_avail_gpa; /* PA of virtqueue avail ring */ uint64_t vq_used_gpa; /* PA of virtqueue used ring */ struct vring_desc *vq_desc; /* descriptor array */ struct vring_avail *vq_avail; /* the "avail" ring */ struct vring_used *vq_used; /* the "used" ring */ }; /* * As noted above, these are sort of backwards, name-wise. * * Endian helpers must be used when using the following macros. */ #define VQ_AVAIL_EVENT_IDX(vq) \ (*(uint16_t *)&(vq)->vq_used->ring[(vq)->vq_qsize]) #define VQ_USED_EVENT_IDX(vq) \ ((vq)->vq_avail->ring[(vq)->vq_qsize]) /* * Is this ring ready for I/O? */ static inline int vq_ring_ready(struct vqueue_info *vq) { return (vq->vq_flags & VQ_ALLOC); } /* * Are there "available" descriptors? (This does not count * how many, just returns True if there are some.) */ static inline int vq_has_descs(struct vqueue_info *vq) { return (vq_ring_ready(vq) && vq->vq_last_avail != vq->vq_avail->idx); } /* * Deliver an interrupt to the guest for a specific MSI-X queue or * event. */ static inline void vi_interrupt(struct virtio_softc *vs, uint8_t isr, uint16_t msix_idx) { if (!(vs->vs_status & VIRTIO_CONFIG_STATUS_DRIVER_OK)) return; if (pci_msix_enabled(vs->vs_pi)) { pci_generate_msix(vs->vs_pi, msix_idx); } else { vs->vs_isr |= isr; pci_generate_msi(vs->vs_pi, 0); pci_lintr_assert(vs->vs_pi); } } /* * Deliver an interrupt to the guest on the given virtual queue (if * possible, or a generic MSI interrupt if not using MSI-X). */ static inline void vq_interrupt(struct virtio_softc *vs, struct vqueue_info *vq) { vi_interrupt(vs, VIRTIO_PCI_ISR_INTR, vq->vq_msix_idx); } /* * Deliver an interrupt to guest on device-specific configuration changes * (if possible, or a generic MSI interrupt if not using MSI-X). */ static inline void vq_devcfg_changed(struct virtio_softc *vs) { vs->vs_flags |= VIRTIO_DEVCFG_CHG; vi_interrupt(vs, VIRTIO_PCI_ISR_CONFIG, vs->vs_msix_cfg_idx); } static inline void vq_kick_enable(struct vqueue_info *vq) { vq->vq_used->flags &= ~VRING_USED_F_NO_NOTIFY; /* * Full memory barrier to make sure the store to vq_used->flags * happens before the load from vq_avail->idx, which results from a * subsequent call to vq_has_descs(). */ atomic_thread_fence_seq_cst(); } static inline void vq_kick_disable(struct vqueue_info *vq) { vq->vq_used->flags |= VRING_USED_F_NO_NOTIFY; } #define VIRTIO_LEGACY_BAR 0 /* BAR for virtio legacy cfg regs */ #define VIRTIO_MSIX_BAR 1 /* BAR for host MSI-X tables */ #define VIRTIO_MODERN_BAR 2 /* BAR for virtio modern cfg regs */ struct iovec; /* * Request description returned by vq_getchain. * * Writable iovecs start at iov[req.readable]. */ struct vi_req { int readable; /* num of readable iovecs */ int writable; /* num of writable iovecs */ unsigned int idx; /* ring index */ }; void vi_softc_linkup(struct virtio_softc *vs, struct virtio_consts *vc, void *dev_softc, struct pci_devinst *pi, struct vqueue_info *queues); void vi_queue_linkup(struct virtio_softc *vc, struct vqueue_info *queues); bool vi_intr_init(struct virtio_softc *vs, bool use_msix); void vi_pci_init(struct pci_devinst *, virtio_mode_t, uint16_t, uint16_t, uint8_t); bool vi_pcibar_setup(struct virtio_softc *); virtio_pci_capcfg_t *vi_pci_cfg_bytype(struct virtio_softc *, uint8_t); virtio_pci_capcfg_t *vi_pci_cfg_bycapaddr(struct virtio_softc *, uint32_t, uint32_t); virtio_pci_capcfg_t *vi_pci_cfg_bybaraddr(struct virtio_softc *, uint8_t, uint64_t, uint32_t); void vi_reset_dev(struct virtio_softc *); void vi_set_debug(struct virtio_softc *, bool); bool vi_is_modern(struct virtio_softc *); void vi_error(struct virtio_softc *, const char *, ...) __PRINTFLIKE(2); int vq_getchain(struct vqueue_info *vq, struct iovec *iov, int niov, struct vi_req *reqp); void vq_retchains(struct vqueue_info *vq, uint16_t n_chains); void vq_relchain_prepare(struct vqueue_info *vq, uint16_t idx, uint32_t iolen); void vq_relchain_publish(struct vqueue_info *vq); void vq_relchain(struct vqueue_info *vq, uint16_t idx, uint32_t iolen); void vq_endchains(struct vqueue_info *vq, int used_all_avail); int vi_pci_cfgread(struct pci_devinst *pi, int offset, int bytes, uint32_t *retval); int vi_pci_cfgwrite(struct pci_devinst *pi, int offset, int bytes, uint32_t val); uint64_t vi_pci_read(struct pci_devinst *pi, int baridx, uint64_t offset, int size); void vi_pci_write(struct pci_devinst *pi, int baridx, uint64_t offset, int size, uint64_t value); void vi_vq_init(struct virtio_softc *); void vi_legacy_vq_init(struct virtio_softc *, uint32_t); #endif /* _BHYVE_VIRTIO_H_ */ /* * This file and its contents are supplied under the terms of the * Common Development and Distribution License ("CDDL"), version 1.0. * You may only use this file in accordance with the terms of version * 1.0 of the CDDL. * * A full copy of the text of the CDDL should have accompanied this * source. A copy of the CDDL is also available via the Internet at * http://www.illumos.org/license/CDDL. * * Copyright 2022 OmniOS Community Edition (OmniOSce) Association. */ #ifndef _VIRTIO_NET_H_ #define _VIRTIO_NET_H_ #include "mevent.h" #include "net_backends.h" /* * This structure appears at the start of each control virtqueue request. */ typedef struct virtio_net_ctrl_hdr { uint8_t vnch_class; uint8_t vnch_command; } __packed virtio_net_ctrl_hdr_t; /* * This structure is used for the mac address tables associated with the * VIRTIO_NET_CTRL_MAC class. */ typedef struct virtio_net_ctrl_mac { uint32_t vncm_entries; ether_addr_t vncm_mac; } __packed virtio_net_ctrl_mac_t; /* * This structure is used to pass the number of queues. */ typedef struct virtio_net_ctrl_mq { uint16_t virtqueue_pairs; } __packed virtio_net_ctrl_mq_t; /* * Control Queue Classes */ #define VIRTIO_NET_CTRL_RX 0 #define VIRTIO_NET_CTRL_MAC 1 #define VIRTIO_NET_CTRL_VLAN 2 #define VIRTIO_NET_CTRL_ANNOUNCE 3 #define VIRTIO_NET_CTRL_MQ 4 /* * CTRL_RX commands */ #define VIRTIO_NET_CTRL_RX_PROMISC 0 #define VIRTIO_NET_CTRL_RX_ALLMULTI 1 #define VIRTIO_NET_CTRL_RX_ALLUNI 2 #define VIRTIO_NET_CTRL_RX_NOMULTI 3 #define VIRTIO_NET_CTRL_RX_NOUNI 4 #define VIRTIO_NET_CTRL_RX_NOBCAST 5 /* CTRL_MAC commands */ #define VIRTIO_NET_CTRL_MAC_TABLE_SET 0 #define VIRTIO_NET_CTRL_MAC_ADDR_SET 1 /* CTRL_MQ commands */ #define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET 0 #define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN 1 #define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX 0x8000 /* * Control queue ack values */ #define VIRTIO_NET_CQ_OK 0 #define VIRTIO_NET_CQ_ERR 1 #endif /* _VIRTIO_NET_H_ */ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2011 NetApp, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef _VMEXIT_H_ #define _VMEXIT_H_ extern const vmexit_handler_t vmexit_handlers[VM_EXITCODE_MAX]; #ifndef __FreeBSD__ extern int vmentry_init(int); extern struct vm_entry *vmentry_vcpu(int); #endif #endif /* !_VMEXIT_H_ */ /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright 2020 Conrad Meyer . All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "acpi.h" #include "bootrom.h" #include "vmgenc.h" static uint64_t vmgen_gpa; void vmgenc_init(struct vmctx *ctx) { char *region; int error; error = bootrom_alloc(ctx, PAGE_SIZE, PROT_READ, 0, ®ion, &vmgen_gpa); if (error != 0) errx(4, "%s: bootrom_alloc", __func__); /* * It is basically harmless to always generate a random ID when * starting a VM. */ error = getentropy(region, sizeof(struct uuid)); if (error == -1) err(4, "%s: getentropy", __func__); /* XXX When we have suspend/resume/rollback. */ #if 0 acpi_raise_gpe(ctx, GPE_VMGENC); #endif } void vmgenc_write_dsdt(void) { dsdt_line(""); dsdt_indent(1); dsdt_line("Scope (_SB)"); dsdt_line("{"); dsdt_line(" Device (GENC)"); dsdt_line(" {"); dsdt_indent(2); dsdt_line("Name (_CID, \"VM_Gen_Counter\")"); dsdt_line("Method (_HID, 0, NotSerialized)"); dsdt_line("{"); dsdt_line(" Return (\"Bhyve_V_Gen_Counter_V1\")"); dsdt_line("}"); dsdt_line("Name (_UID, 0)"); dsdt_line("Name (_DDN, \"VM_Gen_Counter\")"); dsdt_line("Name (ADDR, Package (0x02)"); dsdt_line("{"); dsdt_line(" 0x%08x,", (uint32_t)vmgen_gpa); dsdt_line(" 0x%08x", (uint32_t)(vmgen_gpa >> 32)); dsdt_line("})"); dsdt_unindent(2); dsdt_line(" }"); /* Device (GENC) */ dsdt_line("}"); /* Scope (_SB) */ dsdt_line(""); dsdt_line("Scope (_GPE)"); dsdt_line("{"); dsdt_line(" Method (_E%02x, 0, NotSerialized)", GPE_VMGENC); dsdt_line(" {"); dsdt_line(" Notify (\\_SB.GENC, 0x80)"); dsdt_line(" }"); dsdt_line("}"); dsdt_unindent(1); } /*- * SPDX-License-Identifier: BSD-2-Clause * * Copyright 2020 Conrad Meyer . All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #pragma once void vmgenc_init(struct vmctx *); void vmgenc_write_dsdt(void);