/* * This file and its contents are supplied under the terms of the * Common Development and Distribution License ("CDDL"), version 1.0. * You may only use this file in accordance with the terms of version * 1.0 of the CDDL. * * A full copy of the text of the CDDL should have accompanied this * source. A copy of the CDDL is also available via the Internet at * http://www.illumos.org/license/CDDL. */ /* * Copyright 2026 Oxide Computer Company */ #include "ena_hw.h" #include "ena.h" /* * Elastic Network Adapter (ENA) Driver * ------------------------------------ * * The ena driver provides support for the AWS ENA device, also * referred to as their "enhanced networking". This device is present * on "Nitro"-based instances. It presents itself with the following * PCI Vendor/Device IDs * * o 1d0f:0ec2 -- ENA PF * o 1d0f:1ec2 -- ENA PF (Reserved) * o 1d0f:ec20 -- ENA VF * o 1d0f:ec21 -- ENA VF (Reserved) * * This driver provides support for only the essential features needed * to drive traffic on an ENA device. Support for the following * features IS NOT currently implemented. * * o Admin Queue Interrupts: queue completion events are always polled * o FMA * o Rx checksum offloads * o Tx checksum offloads * o Tx DMA bind (borrow buffers) * o Rx DMA bind (loaned buffers) * o TSO * o RSS * o Support for different Tx completion policies * o More controlled Tx recycling and Rx refill * * Even without these features the ena driver should perform * reasonably well. * * Driver vs. Hardware Types * ------------------------- * * To properly communicate with the ENA device the driver must * populate memory (registers and buffers) with specific types. These * types are defined by the device and are found under the "common" * (ena_com) code of the AWS Linux and FreeBSD drivers [1]. We have * simplified this a bit by defining all device-specific types in the * ena_hw.h file. Furthermore, all device-specific types are given an * "enahw" prefix. This makes it clear when we are dealing with a * device type and when we are dealing with a driver type. * * [1]: https://github.com/amzn/amzn-drivers * * Groups, Rings (Queues), and Interrupts * -------------------------------------- * * The ENA device presents one mac group. This single mac group * represents the single unicast address that this device represents * in your AWS instance. The ENA device presents no option for * configuring additional MAC addresses, multicast, or promisc mode -- * you receive only what AWS wants you to receive. * * This single mac group may have one or more rings. The ENA driver * refers to rings as queues, for no special reason other than it was * the dominant language in the Linux and FreeBSD drivers, and it * spilled over into this port. The upper bound on number of queues is * presented by the device. However, we don't just go with whatever * number of queues the device reports; but rather we limit the queues * based on other factors such as an absolute maximum, number of * online CPUs, and number of available interrupts. The upper bound is * calculated by ena_set_max_io_queues(), and that is used and * possibly further restricted in ena_attach_intr_alloc(). As this * point, ultimately, it is the number of available interrupts (minus * one for the admin queue) that determines the number of queues: one * Tx and one Rx on each I/O interrupt. * * NOTE: Perhaps it is overly restrictive to limit the number of * queues to the number of I/O interrupts. Something worth considering * on larger instances if they present far less interrupts than they * do queues + CPUs. * * The ENA device presents MSI-X interrupts only. During attach the * driver queries the number of available interrupts and sets aside * one for admin/AENQ (vector 0) and the rest for I/O (vector 1 to N). * This means that a Tx/Rx queue at index 0 will map to vector 1, and * so on. * * NOTE: The ENA driver currently doesn't make full use of the Admin * Queue interrupt. This interrupt is used both to notify the driver * when a command response is ready, and when an async event is posted. * The ENA driver always polls the Admin Queue for responses. * * Tx Queue Workings * ----------------- * * A single Tx queue (ena_txq_t) is made up of one submission queue * (SQ) and its paired completion queue (CQ). These two queues form a * logical descriptor ring which is used to send packets out of the * device -- where each SQ entry describes the packet to be sent * (enahw_tx_desc_t) and each CQ entry describes the result of sending * a packet (enahw_tx_cdesc_t). For this to work the host and device * must agree on which descriptors are currently owned by the host * (free for sending) and which are owned by the device (pending * device completion). This state is tracked on the host side via head * and tail indexes along with a phase value. * * The head and tail values represent the head and tail of the FIFO * queue of pending packets -- the next packet to be sent by the * device is head, and all descriptors up to tail are ready for * sending. The phase allows the host to determine which CQ * descriptors represent completed events when using per-SQ completion * events (as opposed to queue head pointer updates). As the queues * represent a logical ring buffer, the phase must alternate on * wrap-around. The device initializes the phase to zero, and the host * starts with a phase of 1. The first packet descriptor writes, and * their corresponding completions, are indicated with a phase of 1. * * * For example, the diagram below represents the SQ/CQ state after the * first 6 packets have been sent by the host and 2 of them have been * completed by the device (and these completions have been processed * by the driver). In this state the host could send 4 more packets * before needing to wait on completion events. * * * +---+---+---+---+---+---+---+---+ * SQ | 1 | 1 | 1 | 1 | 1 | 1 | 0 | 0 | phase = 1 * +---+---+---+---+---+---+---+---+ * ^ * | * tail * head * | * v * +---+---+---+---+---+---+---+---+ * CQ | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | phase = 1 * +---+---+---+---+---+---+---+---+ * * * The next diagram shows how the state changes as 5 more packets are * sent (for a total of 11) and 7 more are completed (for a total of * 9). Notice that as the SQ and CQ have wrapped around their phases * have been complemented. In this state the host could send 6 more * packets before needing to wait on completion events. * * +---+---+---+---+---+---+---+---+ * SQ | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 | phase = 0 * +---+---+---+---+---+---+---+---+ * ^ * | * tail * head * | * v * +---+---+---+---+---+---+---+---+ * CQ | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 | phase = 0 * +---+---+---+---+---+---+---+---+ * * * Currently, all packets are copied for Tx. At ring start we allocate * a Tx Control Buffer (TCB) for each queue descriptor. Each TCB has * DMA buffer associated with it; and each buffer is large enough to * hold the MTU. Therefore, Tx descriptors and TCBs currently have a * 1:1 mapping. When a packet is sent, the mblk's buffer is copied to * the TCB's DMA buffer, and a new descriptor is written to the SQ * describing said TCB buffer. If and when we add more advanced * features like DMA binding of mblks and TSO, this 1:1 guarantee will * no longer hold. * * Low Latency Queues (LLQ) * ------------------------ * * The ENA device supports two placement policies for Tx submission * queue descriptors: host memory (PLACEMENT_POLICY_HOST) and device * memory (PLACEMENT_POLICY_DEV). With host placement, the driver * writes descriptors to host memory and the device reads them via * DMA. With device placement, the driver writes descriptors directly * to a region of device memory mapped through PCI BAR 2. This is * known as LLQ mode and reduces latency by eliminating the DMA read. * * Some device generations require LLQ. In particular, Nitro v5 and * later instances will not attach without it. When the device * supports LLQ, the driver negotiates LLQ parameters during attach * via the ENA_ADMIN_LLQ feature. If the device does not support LLQ, * the driver uses host placement. If the device supports LLQ but * negotiation fails, the driver fails to attach. * * The following LLQ parameters are negotiated with the device * during attach: * * o Entry size: we honour the device's recommended entry size * if it is supported. If the device has no recommendation we * fall back to the lowest supported size. Each SQ entry is this * size, comprising descriptors followed by inline packet header * data. Larger entries allow more header to be inlined but * reduce the maximum queue depth since the LLQ memory is * fixed-size. * * o Headers inline in the first entry: the packet header is * placed in the first SQ entry immediately after the * descriptors, using the remaining space in the entry. * * o Descriptors before header: the number of descriptors that * precede the inline header in the first entry is negotiated * with the device, preferring 2, then 1, then 4, then 8. This * is the same order as other ena driver common code, with 2 * being preferred over 1 as it maximises inline header space * while still allowing a meta + data descriptor pair. * * o Stride control: prefers multiple descriptors per entry, * falling back to single descriptor per entry. * * The header location is fixed as inline, which is the only mode * current ENA hardware supports. The negotiation fails attach if * the device does not support any of the driver's choices for a * given parameter. * * Accelerated LLQ * --------------- * * Newer devices (Nitro v5/v6) additionally require accelerated LLQ * mode, which the driver enables by setting two flags in the * SET_FEATURE command: * * o DISABLE_META_CACHING: the device will not cache Tx meta * descriptors, so the driver must include a meta descriptor * with every packet. This simplifies device state tracking * at the cost of additional descriptor space per packet. * * o LIMIT_TX_BURST: the device specifies a maximum number of * LLQ entries that may be written between doorbells. The * driver tracks this per-queue and rings the doorbell when * the limit is reached, even if more packets are ready. * * LLQ Tx Path * ----------- * * In LLQ mode, the Tx path differs from the host placement path * described above. Each queue has a bounce buffer -- a single * staging area the size of one LLQ entry. For each packet the * driver composes the complete LLQ entry in the bounce buffer: * * +--------+------//------+---------------------------+ * | meta | data descs | inline packet header | * | 16B | N x 16B | remaining bytes | * +--------+------//------+---------------------------+ * |<------------- entry_size bytes ------------------>| * * The meta descriptor carries per-packet metadata (L3 header * offset, MSS, etc.). The data descriptor(s) are the same as in * host mode; N is the negotiated descs_before_header value. The * inline header is copied from the packet's mblk chain and allows * the device to begin processing before the full DMA payload * arrives. * * Once the bounce buffer is complete, it is written to device * memory using 64-bit stores. The bounce buffer approach ensures * that the device sees a complete, consistent entry rather than * a partial one that could result from writing fields individually. * * The LLQ BAR is mapped write-combining (DDI_STORECACHING_OK_ACC) * so the CPU may merge and reorder stores within an entry. This * is safe because the device does not poll its local memory for * phase bit transitions the way a traditional NIC scans a * host-memory descriptor ring; it only inspects LLQ entries * after receiving a doorbell write. A membar_producer() before * the doorbell guarantees that all prior stores to the LLQ entry * are ordered before the doorbell, so there is no need to write * phase bits in reverse order or otherwise arrange for the phase * to be the last thing written. * * There is no DMA involved for the descriptors themselves; they * reach the device via MMIO writes through the BAR. The packet * payload beyond the inline header is still transferred by DMA * and is synced separately (see ena_tcb_pull()). * * Rx Queue Workings * ----------------- * * In terms of implementing the logical descriptor ring, the Rx queues * are very much like the Tx queues. There is a paired SQ and CQ for * each logical ring. The difference is that in Rx the SQ is for * handing buffers to the device to fill, and the CQ is for describing * the contents of those buffers for a given received frame. At Rx * ring start we allocate a Rx Control Buffer (RCB) for each * descriptor in the ring. Each RCB has a DMA buffer associated with * it; and each buffer is large enough to hold the MTU. For each * received frame we copy the contents out of the RCB and into its own * mblk, immediately returning the RCB for reuse. As with Tx, this * gives us a simple 1:1 mapping currently, but if more advanced * features are implemented later this could change. * * Asynchronous Event Notification Queue (AENQ) * -------------------------------------------- * * Each ENA device comes with a mechanism for sending out-of-band * notifications to the driver. This includes events like link state * changes, fatal errors, and a watchdog/keep alive signal. The AENQ * delivery mechanism is via interrupt, handled by the ena_aenq_work() * function, which dispatches via the eaenq_hdlrs table. If no handler * is registered, the ena_aenq_default_hdlr() handler is used. A given * device may not support all the different event types * (enahw_aenq_groups_t); and the driver may choose to enable a subset * of the supported events. During attach we call ena_aenq_configure() * to negotiate the supported/enabled events. The enabled group is * stored at ena_aenq_enabled_groups. * * Queues and Unsigned Wraparound * ------------------------------ * * All the queues use a uint16_t value as their head/tail values, e.g. * the Rx queue's er_cq_head_idx value. You might notice that we only * ever increment these values, letting them perform implicit unsigned * integer wraparound. This is intended. This is the same behavior as * the common code, and seems to be what the hardware expects. Of * course, when accessing our own descriptor arrays we must make sure * to first perform a modulo of this value or risk running off into * space. * * Watchdog and Device Reset * ------------------------- * * While the device is running, the driver periodically invokes a * watchdog function to check that all is well, and to reset the * device if not. The device will be reset if any of the following is * true: * * o The device's status register fatal error bit is set. A device * in this state will no longer process any queues; * o No asynchronous event keepalives have been received for some * time -- see ENA_DEVICE_KEEPALIVE_TIMEOUT_NS; * o A Tx queue has remained blocked for some time -- see * ENA_TX_STALL_TIMEOUT; * o The device has requested, via an asynchronous event, that we * perform a reset; * o Driver code has detected an error and set the EN_STATE_ERROR * bit in ena_state. * * There is a "fatal error" asynchronous event, but common code does * not use that as a reason to trigger a reset, and so neither do we. * * The global `ena_force_reset` variable can be used as a simple means * to trigger a reset during driver development and testing. If there * are multiple instances, it is likely that only one of them will * reset when this variable is changed to `true`. * * Attach Sequencing * ----------------- * * Most drivers implement their attach/detach/cleanup functions as a * sequential stream of function calls used to allocate and initialize * resources in an order determined by the device's programming manual * combined with any requirements imposed by the kernel and its * relevant modules. These functions can become quite long. It is * often hard to see the order in which steps are taken, and even * harder to tell if detach/cleanup undoes them in the correct order, * or even if it undoes them at all! The only sure way to understand * the flow is to take good notes while closely inspecting each line * of code. Even then, it's easy for attach and detach to get out of * sync. * * Some more recent drivers have improved on this situation by using a * bit vector to track the sequence of events in attach/detach. Each * bit is declared in as an enum value, in the same order it is * expected attach would run, and thus detach would run in the exact * opposite order. This has three main benefits: * * 1. It makes it easier to determine sequence order at a * glance. * * 2. It gives a better idea of what state the device is in during * debugging (the sequence bit vector is kept with the instance * state). * * 3. The detach function can verify that all sequence bits are * cleared, indicating that everything done in attach was * successfully undone. * * These are great improvements. However, the attach/detach functions * can still become unruly, and there is still no guarantee that * detach is done in opposite order of attach (this is not always * strictly required, but is probably the best way to write detach). * There is still a lot of boilerplate and chance for programmer * error. * * The ena driver takes the sequence idea a bit further, creating a * descriptor table of the attach sequence (ena_attach_tbl). This * table is used by attach/detach to generically, declaratively, and * programmatically enforce the precise sequence order and verify that * anything that is done is undone. This provides several benefits: * * o Correct order is enforced implicitly by the descriptor table. * It is impossible for the detach sequence to run in any other * order other than opposite that of attach. * * o It is obvious what the precise attach sequence is. While the * bit vector enum helps a lot with this it doesn't prevent * programmer error. With the sequence defined as a declarative * table it makes it easy for the programmer to see the order and * know it's followed exactly. * * o It is impossible to modify the attach sequence without also * specifying a callback for its dual in the detach sequence. * * o Common and repetitive code like error checking, logging, and bit * vector modification is eliminated and centralized, again * reducing the chance of programmer error. * * The ena attach sequence is defined under ena_attach_seq_t. The * descriptor table is defined under ena_attach_tbl. */ /* * These are some basic data layout invariants on which development * assumptions where made. */ CTASSERT(sizeof (enahw_tx_data_desc_t) == 16); CTASSERT(sizeof (enahw_tx_data_desc_t) == sizeof (enahw_tx_meta_desc_t)); CTASSERT(sizeof (enahw_tx_data_desc_t) == sizeof (enahw_tx_desc_t)); CTASSERT(sizeof (enahw_tx_meta_desc_t) == sizeof (enahw_tx_desc_t)); /* * Amazon does not specify the endianess of the ENA device. We assume * it's the same as the bus, and we assume the CPU/bus is always * little endian. */ #ifdef _BIG_ENDIAN #error "ENA driver is little-endian only" #endif uint64_t ena_admin_cmd_timeout_ns = ENA_ADMIN_CMD_DEF_TIMEOUT_NS; /* * Log an error message. We leave the destination (console or system * log) up to the caller */ void ena_err(const ena_t *ena, const char *fmt, ...) { va_list ap; va_start(ap, fmt); if (ena != NULL && ena->ena_dip != NULL) { vdev_err(ena->ena_dip, CE_WARN, fmt, ap); } else { vcmn_err(CE_WARN, fmt, ap); } va_end(ap); } void ena_panic(const ena_t *ena, const char *fmt, ...) { va_list ap; va_start(ap, fmt); if (ena != NULL && ena->ena_dip != NULL) { vdev_err(ena->ena_dip, CE_PANIC, fmt, ap); } else { vcmn_err(CE_PANIC, fmt, ap); } va_end(ap); } /* * Set this to true to enable debug messages. */ bool ena_debug = false; /* * Log a debug message. We force all debug messages to go to the * system log. */ void ena_dbg(const ena_t *ena, const char *fmt, ...) { va_list ap; if (ena_debug) { char msg[1024]; va_start(ap, fmt); (void) vsnprintf(msg, sizeof (msg), fmt, ap); va_end(ap); if (ena != NULL && ena->ena_dip != NULL) { dev_err(ena->ena_dip, CE_NOTE, "!%s", msg); } else { cmn_err(CE_NOTE, "!%s", msg); } } } void ena_trigger_reset(ena_t *ena, enahw_reset_reason_t reason) { mutex_enter(&ena->ena_lock); ena->ena_reset_reason = reason; mutex_exit(&ena->ena_lock); atomic_or_32(&ena->ena_state, ENA_STATE_ERROR); } /* * Determine if a given feature is available on this device. */ bool ena_is_feat_avail(ena_t *ena, const enahw_feature_id_t feat_id) { VERIFY3U(feat_id, <=, ENAHW_FEAT_NUM); uint32_t mask = 1U << feat_id; /* * The device attributes feature is always supported, as * indicated by the common code. Host attributes must be sent * before querying device attributes, so the supported features * bitmask is not yet populated at that point. */ if (feat_id == ENAHW_FEAT_DEVICE_ATTRIBUTES || feat_id == ENAHW_FEAT_HOST_ATTR_CONFIG) { return (true); } return ((ena->ena_supported_features & mask) != 0); } /* * Determine if a given capability is available on this device. */ bool ena_is_cap_avail(ena_t *ena, const enahw_capability_id_t cap_id) { VERIFY3U(cap_id, <=, ENAHW_CAP_NUM); uint32_t mask = 1U << cap_id; return ((ena->ena_capabilities & mask) != 0); } static bool ena_device_reset(ena_t *ena, enum enahw_reset_reason_types reason) { uint32_t rval, wval, reason_lsb, reason_msb; hrtime_t timeout, expired; rval = ena_hw_bar_read32(ena, ENAHW_REG_DEV_STS); if ((rval & ENAHW_DEV_STS_READY_MASK) == 0) { ena_err(ena, "reset: device is not ready"); return (false); } rval = ena_hw_bar_read32(ena, ENAHW_REG_CAPS); /* * The device stores the reset timeout at 100ms resolution; we * normalize that to nanoseconds. */ timeout = MSEC2NSEC(ENAHW_CAPS_RESET_TIMEOUT(rval) * 100); if (timeout == 0) { ena_err(ena, "device gave invalid (0) reset timeout"); return (false); } expired = gethrtime() + timeout; wval = ENAHW_DEV_CTL_DEV_RESET_MASK; reason_lsb = ENAHW_RESET_REASON_LSB(reason); reason_msb = ENAHW_RESET_REASON_MSB(reason); wval |= (reason_lsb << ENAHW_DEV_CTL_RESET_REASON_SHIFT) & ENAHW_DEV_CTL_RESET_REASON_MASK; if (ena_is_cap_avail(ena, ENAHW_CAP_EXTENDED_RESET_REASONS)) { wval |= (reason_msb << ENAHW_DEV_CTL_RESET_REASON_EXT_SHIFT) & ENAHW_DEV_CTL_RESET_REASON_EXT_MASK; } else if (reason_msb != 0) { /* Fall back to "generic" which we know will fit */ wval = ENAHW_DEV_CTL_DEV_RESET_MASK; wval |= (ENAHW_RESET_GENERIC << ENAHW_DEV_CTL_RESET_REASON_SHIFT) & ENAHW_DEV_CTL_RESET_REASON_MASK; } ena_hw_bar_write32(ena, ENAHW_REG_DEV_CTL, wval); /* * Make sure reset is in progress. */ for (;;) { rval = ena_hw_bar_read32(ena, ENAHW_REG_DEV_STS); if ((rval & ENAHW_DEV_STS_RESET_IN_PROGRESS_MASK) != 0) break; if (gethrtime() > expired) { ena_err(ena, "device reset start timed out"); return (false); } /* Sleep for 100 milliseconds. */ delay(drv_usectohz(100 * 1000)); } /* * Reset the timeout counter for the next device request. */ expired = gethrtime() + timeout; /* * Wait for the device reset to finish. */ ena_hw_bar_write32(ena, ENAHW_REG_DEV_CTL, 0); for (;;) { rval = ena_hw_bar_read32(ena, ENAHW_REG_DEV_STS); if ((rval & ENAHW_DEV_STS_RESET_IN_PROGRESS_MASK) == 0) { break; } if (gethrtime() > expired) { ena_err(ena, "device reset timed out"); return (false); } /* Sleep for 100 milliseconds. */ delay(drv_usectohz(100 * 1000)); } ena_dbg(ena, "device reset succeeded"); return (true); } static bool ena_attach_pci(ena_t *ena) { ddi_acc_handle_t hdl; if (pci_config_setup(ena->ena_dip, &hdl) != 0) { return (false); } ena->ena_pci_hdl = hdl; ena->ena_pci_vid = pci_config_get16(hdl, PCI_CONF_VENID); ena->ena_pci_did = pci_config_get16(hdl, PCI_CONF_DEVID); ena->ena_pci_rev = pci_config_get8(hdl, PCI_CONF_REVID); ena->ena_pci_svid = pci_config_get16(hdl, PCI_CONF_SUBVENID); ena->ena_pci_sdid = pci_config_get16(hdl, PCI_CONF_SUBSYSID); ena_dbg(ena, "vid: 0x%x did: 0x%x rev: 0x%x svid: 0x%x sdid: 0x%x", ena->ena_pci_vid, ena->ena_pci_did, ena->ena_pci_rev, ena->ena_pci_svid, ena->ena_pci_sdid); return (true); } static void ena_cleanup_pci(ena_t *ena, bool resetting) { VERIFY0(resetting); pci_config_teardown(&ena->ena_pci_hdl); } static void ena_cleanup_regs_map(ena_t *ena, bool resetting) { VERIFY0(resetting); ddi_regs_map_free(&ena->ena_reg_hdl); } /* * Map a PCI BAR number (0-5) to a DDI register set number. */ static int ena_bar_to_rnumber(ena_t *ena, uint8_t bar) { pci_regspec_t *regs; uint_t bar_offset, regs_length, rcount; int rnumber = -1; if (bar > 5) return (-1); /* * PCI_CONF_BASE0 is 0x10; each BAR is 4 bytes apart. */ bar_offset = PCI_CONF_BASE0 + sizeof (uint32_t) * bar; if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, ena->ena_dip, DDI_PROP_DONTPASS, "reg", (int **)®s, ®s_length) != DDI_PROP_SUCCESS) { return (-1); } rcount = regs_length * sizeof (int) / sizeof (pci_regspec_t); for (int i = 0; i < rcount; i++) { if (PCI_REG_REG_G(regs[i].pci_phys_hi) == bar_offset) { rnumber = i; break; } } ddi_prop_free(regs); return (rnumber); } static bool ena_attach_regs_map(ena_t *ena) { ddi_device_acc_attr_t attr; int rnumber; int ret = 0; rnumber = ena_bar_to_rnumber(ena, ENA_REG_BAR); if (rnumber == -1) { ena_err(ena, "failed to find rnumber for BAR %d", ENA_REG_BAR); return (false); } if (ddi_dev_regsize(ena->ena_dip, rnumber, &ena->ena_reg_size) != DDI_SUCCESS) { ena_err(ena, "failed to get register set %d size", rnumber); return (false); } ena_dbg(ena, "register size: %ld", ena->ena_reg_size); bzero(&attr, sizeof (attr)); attr.devacc_attr_version = DDI_DEVICE_ATTR_V1; attr.devacc_attr_endian_flags = DDI_NEVERSWAP_ACC; attr.devacc_attr_dataorder = DDI_STRICTORDER_ACC; attr.devacc_attr_access = DDI_DEFAULT_ACC; /* * This function can return several different failure values, * so we make sure to capture its return value for the purpose * of logging. */ ret = ddi_regs_map_setup(ena->ena_dip, rnumber, &ena->ena_reg_base, 0, ena->ena_reg_size, &attr, &ena->ena_reg_hdl); if (ret != DDI_SUCCESS) { ena_err(ena, "failed to map BAR %d (reg %d): %d", ENA_REG_BAR, rnumber, ret); return (false); } ena_dbg(ena, "registers BAR %d mapped to base: 0x%p size: %ld", ENA_REG_BAR, (void *)ena->ena_reg_base, ena->ena_reg_size); return (true); } /* * Free any resources related to the admin submission queue. */ static void ena_admin_sq_free(ena_t *ena) { ena_dma_free(&ena->ena_aq.ea_sq.eas_dma); } /* * Initialize the admin submission queue. */ static bool ena_admin_sq_init(ena_t *ena) { ena_adminq_t *aq = &ena->ena_aq; ena_dma_buf_t *dma = &aq->ea_sq.eas_dma; size_t size = aq->ea_qlen * sizeof (*aq->ea_sq.eas_entries); uint32_t addr_low, addr_high, wval; if (aq->ea_sq.eas_entries == NULL) { ena_dma_conf_t conf = { .edc_size = size, .edc_align = ENAHW_ADMIN_SQ_DESC_BUF_ALIGNMENT, .edc_sgl = 1, .edc_endian = DDI_NEVERSWAP_ACC, .edc_stream = false, }; if (!ena_dma_alloc(ena, dma, &conf, size)) { ena_err(ena, "failed to allocate DMA for Admin SQ"); return (false); } ENA_DMA_VERIFY_ADDR(ena, dma->edb_cookie->dmac_laddress); aq->ea_sq.eas_entries = (void *)dma->edb_va; } else { ena_dma_bzero(dma); } aq->ea_sq.eas_tail = 0; aq->ea_sq.eas_phase = 1; aq->ea_sq.eas_dbaddr = (uint32_t *)(ena->ena_reg_base + ENAHW_REG_ASQ_DB); addr_low = (uint32_t)(dma->edb_cookie->dmac_laddress); addr_high = (uint32_t)(dma->edb_cookie->dmac_laddress >> 32); ena_hw_bar_write32(ena, ENAHW_REG_ASQ_BASE_LO, addr_low); ena_hw_bar_write32(ena, ENAHW_REG_ASQ_BASE_HI, addr_high); wval = ENAHW_ASQ_CAPS_DEPTH(aq->ea_qlen) | ENAHW_ASQ_CAPS_ENTRY_SIZE(sizeof (*aq->ea_sq.eas_entries)); ena_hw_bar_write32(ena, ENAHW_REG_ASQ_CAPS, wval); return (true); } /* * Free any resources related to the admin completion queue. */ static void ena_admin_cq_free(ena_t *ena) { ena_dma_free(&ena->ena_aq.ea_cq.eac_dma); } /* * Initialize the admin completion queue. */ static bool ena_admin_cq_init(ena_t *ena) { ena_adminq_t *aq = &ena->ena_aq; ena_dma_buf_t *dma = &aq->ea_cq.eac_dma; uint32_t addr_low, addr_high, wval; if (aq->ea_cq.eac_entries == NULL) { size_t size = aq->ea_qlen * sizeof (*aq->ea_cq.eac_entries); ena_dma_conf_t conf = { .edc_size = size, .edc_align = ENAHW_ADMIN_CQ_DESC_BUF_ALIGNMENT, .edc_sgl = 1, .edc_endian = DDI_NEVERSWAP_ACC, .edc_stream = false, }; if (!ena_dma_alloc(ena, dma, &conf, size)) { ena_err(ena, "failed to allocate DMA for Admin CQ"); return (false); } ENA_DMA_VERIFY_ADDR(ena, dma->edb_cookie->dmac_laddress); aq->ea_cq.eac_entries = (void *)dma->edb_va; } else { ena_dma_bzero(dma); } aq->ea_cq.eac_head = 0; aq->ea_cq.eac_phase = 1; addr_low = (uint32_t)(dma->edb_cookie->dmac_laddress); addr_high = (uint32_t)(dma->edb_cookie->dmac_laddress >> 32); ena_hw_bar_write32(ena, ENAHW_REG_ACQ_BASE_LO, addr_low); ena_hw_bar_write32(ena, ENAHW_REG_ACQ_BASE_HI, addr_high); wval = ENAHW_ACQ_CAPS_DEPTH(aq->ea_qlen) | ENAHW_ACQ_CAPS_ENTRY_SIZE(sizeof (*aq->ea_cq.eac_entries)); ena_hw_bar_write32(ena, ENAHW_REG_ACQ_CAPS, wval); return (true); } void ena_update_hints(ena_t *ena, enahw_device_hints_t *hints) { ena->ena_device_hints.eh_mmio_read_timeout = hints->edh_mmio_read_timeout; ena->ena_device_hints.eh_keep_alive_timeout = hints->edh_keep_alive_timeout; ena->ena_device_hints.eh_tx_comp_timeout = hints->edh_tx_comp_timeout; ena->ena_device_hints.eh_missed_tx_reset_threshold = hints->edh_missed_tx_reset_threshold; ena->ena_device_hints.eh_admin_comp_timeout = hints->edh_admin_comp_timeout; ena->ena_device_hints.eh_max_tx_sgl = hints->edh_max_tx_sgl; ena->ena_device_hints.eh_max_rx_sgl = hints->edh_max_rx_sgl; } /* * We limit the max number of I/O queues based on several aspects of * the underlying hardware. * * 1. The absolute upper limit is set by ENAHW_MAX_NUM_IO_QUEUES, * which comes from the common code and presumably is based on device * constraints. * * 2. Next we latch the number of I/O queues to the number of online * CPUs. The idea being that each queue is a parallel work stream, * and having more queues than CPUs to flush them will not improve * performance. The number of online CPUs can change dynamically, * and that's okay, everything should still work fine, it just * might not be ideal. * * 3. Next we latch the number of I/O queues to the smallest of the * max Tx queues and max Rx queues. We could probably loosen this * restriction in the future, and have separate max I/O queues for * Tx and Rx. This is what Linux does, and seems like a fine place * to start. */ static void ena_set_max_io_queues(ena_t *ena) { uint32_t max = ENAHW_MAX_NUM_IO_QUEUES; max = MIN(ncpus_online, max); /* * Supposedly a device could present a different number of SQs * and CQs. This driver is designed in a way that requires * each SQ to have a corresponding and dedicated CQ (how would * it work otherwise). Therefore, we must check both values * and find the minimum between them. */ max = MIN(ena->ena_tx_max_sq_num, max); max = MIN(ena->ena_tx_max_cq_num, max); max = MIN(ena->ena_rx_max_sq_num, max); max = MIN(ena->ena_rx_max_cq_num, max); /* This shouldn't happen, but just in case. */ if (max == 0) { max = 1; } ena->ena_max_io_queues = max; } /* * We require that an Rx or Tx buffer be able to hold the maximum MTU * along with the maximum frame header length. In this case we know * ENA is presenting us an Ethernet frame so we add the size of an * Ethernet VLAN header. Rx has the additional requirement of needing * additional margin for the sake of IP header alignment. */ static void ena_update_buf_sizes(ena_t *ena) { ena->ena_max_frame_hdr = sizeof (struct ether_vlan_header); ena->ena_max_frame_total = ena->ena_max_frame_hdr + ena->ena_mtu; ena->ena_tx_buf_sz = P2ROUNDUP_TYPED(ena->ena_max_frame_total, ena->ena_page_sz, uint32_t); ena->ena_rx_buf_sz = P2ROUNDUP_TYPED(ena->ena_max_frame_total + ENA_RX_BUF_IPHDR_ALIGNMENT, ena->ena_page_sz, uint32_t); } static bool ena_get_hints(ena_t *ena) { int ret; enahw_resp_desc_t resp; enahw_device_hints_t *hints = &resp.erd_resp.erd_get_feat.ergf_hints; ena_dbg(ena, "Requesting hints"); bzero(&resp, sizeof (resp)); ret = ena_get_feature(ena, &resp, ENAHW_FEAT_HW_HINTS, ENAHW_FEAT_HW_HINTS_VER); if (ret == ENOTSUP) { /* In this case the device does not support querying hints */ ena_dbg(ena, "Hints are unsupported"); return (true); } else if (ret != 0) { ena_err(ena, "Error getting hints: %d", ret); return (false); } ena_update_hints(ena, hints); return (true); } static bool ena_get_offloads(ena_t *ena) { int ret = 0; enahw_resp_desc_t resp; enahw_feat_offload_t *feat = &resp.erd_resp.erd_get_feat.ergf_offload; ena->ena_tx_l3_ipv4_csum = false; ena->ena_tx_l4_ipv4_part_csum = false; ena->ena_tx_l4_ipv4_full_csum = false; ena->ena_tx_l4_ipv4_lso = false; ena->ena_tx_l4_ipv6_part_csum = false; ena->ena_tx_l4_ipv6_full_csum = false; ena->ena_tx_l4_ipv6_lso = false; ena->ena_rx_l3_ipv4_csum = false; ena->ena_rx_l4_ipv4_csum = false; ena->ena_rx_l4_ipv6_csum = false; ena->ena_rx_hash = false; bzero(&resp, sizeof (resp)); ret = ena_get_feature(ena, &resp, ENAHW_FEAT_STATELESS_OFFLOAD_CONFIG, ENAHW_FEAT_STATELESS_OFFLOAD_CONFIG_VER); if (ret == ENOTSUP) { /* * In this case the device does not support querying * for hardware offloads. We take that as a sign that * the device provides no offloads. */ return (true); } else if (ret != 0) { ena_err(ena, "error getting stateless offload: %d", ret); return (false); } ena->ena_tx_l3_ipv4_csum = ENAHW_FEAT_OFFLOAD_TX_L3_IPV4_CSUM(feat); ena->ena_tx_l4_ipv4_part_csum = ENAHW_FEAT_OFFLOAD_TX_L4_IPV4_CSUM_PART(feat); ena->ena_tx_l4_ipv4_full_csum = ENAHW_FEAT_OFFLOAD_TX_L4_IPV4_CSUM_FULL(feat); ena->ena_tx_l4_ipv4_lso = ENAHW_FEAT_OFFLOAD_TSO_IPV4(feat); ena->ena_tx_l4_ipv6_part_csum = ENAHW_FEAT_OFFLOAD_TX_L4_IPV6_CSUM_PART(feat); ena->ena_tx_l4_ipv6_full_csum = ENAHW_FEAT_OFFLOAD_TX_L4_IPV6_CSUM_FULL(feat); ena->ena_tx_l4_ipv6_lso = ENAHW_FEAT_OFFLOAD_TSO_IPV6(feat); ena->ena_rx_l3_ipv4_csum = ENAHW_FEAT_OFFLOAD_RX_L3_IPV4_CSUM(feat); ena->ena_rx_l4_ipv4_csum = ENAHW_FEAT_OFFLOAD_RX_L4_IPV4_CSUM(feat); ena->ena_rx_l4_ipv6_csum = ENAHW_FEAT_OFFLOAD_RX_L4_IPV6_CSUM(feat); return (true); } static int ena_get_prop(ena_t *ena, char *propname, const int minval, const int maxval, const int defval) { int value = ddi_prop_get_int(DDI_DEV_T_ANY, ena->ena_dip, DDI_PROP_DONTPASS, propname, defval); if (value > maxval) { ena_err(ena, "user value %s=%d exceeded maximum, setting to %d", propname, value, maxval); value = maxval; } if (value < minval) { ena_err(ena, "user value %s=%d below minimum, setting to %d", propname, value, minval); value = minval; } return (value); } static bool ena_set_mtu(ena_t *ena) { int ret = 0; enahw_cmd_desc_t cmd; enahw_feat_mtu_t *feat = &cmd.ecd_cmd.ecd_set_feat.ecsf_feat.ecsf_mtu; enahw_resp_desc_t resp; bzero(&cmd, sizeof (cmd)); bzero(&resp, sizeof (resp)); feat->efm_mtu = ena->ena_mtu; if ((ret = ena_set_feature(ena, &cmd, &resp, ENAHW_FEAT_MTU, ENAHW_FEAT_MTU_VER)) != 0) { ena_err(ena, "failed to set device MTU to %u: %d", ena->ena_mtu, ret); return (false); } return (true); } static void ena_get_link_config(ena_t *ena) { enahw_resp_desc_t resp; enahw_feat_link_conf_t *feat = &resp.erd_resp.erd_get_feat.ergf_link_conf; bool full_duplex; bzero(&resp, sizeof (resp)); if (ena_get_feature(ena, &resp, ENAHW_FEAT_LINK_CONFIG, ENAHW_FEAT_LINK_CONFIG_VER) != 0) { /* * Some ENA devices do not support this feature. In * those cases we report a 1Gbps link, full duplex. * For the most accurate information on bandwidth * limits see the official AWS documentation. */ ena->ena_link_speed_mbits = 1000; ena->ena_link_speeds = ENAHW_LINK_SPEED_1G; ena->ena_link_duplex = LINK_DUPLEX_FULL; ena->ena_link_autoneg = true; return; } ena->ena_link_speed_mbits = feat->eflc_speed; ena->ena_link_speeds = feat->eflc_supported; full_duplex = ENAHW_FEAT_LINK_CONF_FULL_DUPLEX(feat); ena->ena_link_duplex = full_duplex ? LINK_DUPLEX_FULL : LINK_DUPLEX_HALF; ena->ena_link_autoneg = ENAHW_FEAT_LINK_CONF_AUTONEG(feat); } /* * Retrieve all configuration values which are modifiable via * ena.conf, and set ena_t members accordingly. While the conf values * have priority, they may be implicitly modified by the driver to * meet resource constraints on a given platform. If no value is * specified in the conf file, the driver will attempt to use the * largest value supported. While there should be no value large * enough, keep in mind that ena_get_prop() will cast the values to an * int. * * This function should be called after the device is initialized, * admin queue is established, and the hardware features/capabs have * been queried; it should be called before mac registration. */ static bool ena_attach_read_conf(ena_t *ena) { uint32_t gcv; /* Greatest Common Value */ /* * We expect that the queue lengths are the same for both the * CQ and SQ, but technically the device could return * different lengths. For now the driver locks them together. */ gcv = MIN(ena->ena_rx_max_sq_num_descs, ena->ena_rx_max_cq_num_descs); ASSERT3U(gcv, <=, INT_MAX); ena->ena_rxq_num_descs = ena_get_prop(ena, ENA_PROP_RXQ_NUM_DESCS, ENA_PROP_RXQ_NUM_DESCS_MIN, gcv, gcv); ena->ena_rxq_intr_limit = ena_get_prop(ena, ENA_PROP_RXQ_INTR_LIMIT, ENA_PROP_RXQ_INTR_LIMIT_MIN, ENA_PROP_RXQ_INTR_LIMIT_MAX, ENA_PROP_RXQ_INTR_LIMIT_DEF); gcv = MIN(ena->ena_tx_max_sq_num_descs, ena->ena_tx_max_cq_num_descs); ASSERT3U(gcv, <=, INT_MAX); ena->ena_txq_num_descs = ena_get_prop(ena, ENA_PROP_TXQ_NUM_DESCS, ENA_PROP_TXQ_NUM_DESCS_MIN, gcv, gcv); return (true); } /* * Perform any necessary device configuration after the driver.conf * has been read. */ static bool ena_attach_dev_cfg(ena_t *ena) { ASSERT3U(ena->ena_attach_seq, >=, ENA_ATTACH_READ_CONF); if (!ena_set_mtu(ena)) { /* * We don't expect this to fail, but we try a fallback * first before failing the attach sequence. */ ena->ena_mtu = 1500; ena_err(ena, "trying fallback MTU: %u", ena->ena_mtu); if (!ena_set_mtu(ena)) { return (false); } } return (true); } static bool ena_check_versions(ena_t *ena) { uint32_t dev_vsn = ena_hw_bar_read32(ena, ENAHW_REG_VERSION); uint32_t ctrl_vsn = ena_hw_bar_read32(ena, ENAHW_REG_CONTROLLER_VERSION); ena->ena_dev_major_vsn = ENAHW_DEV_MAJOR_VSN(dev_vsn); ena->ena_dev_minor_vsn = ENAHW_DEV_MINOR_VSN(dev_vsn); ena->ena_ctrl_major_vsn = ENAHW_CTRL_MAJOR_VSN(ctrl_vsn); ena->ena_ctrl_minor_vsn = ENAHW_CTRL_MINOR_VSN(ctrl_vsn); ena->ena_ctrl_subminor_vsn = ENAHW_CTRL_SUBMINOR_VSN(ctrl_vsn); ena->ena_ctrl_impl_id = ENAHW_CTRL_IMPL_ID(ctrl_vsn); ena_dbg(ena, "device version: %u.%u", ena->ena_dev_major_vsn, ena->ena_dev_minor_vsn); ena_dbg(ena, "controller version: %u.%u.%u implementation %u", ena->ena_ctrl_major_vsn, ena->ena_ctrl_minor_vsn, ena->ena_ctrl_subminor_vsn, ena->ena_ctrl_impl_id); if (ena->ena_ctrl_subminor_vsn < ENA_CTRL_SUBMINOR_VSN_MIN) { ena_err(ena, "unsupported controller version: %u.%u.%u", ena->ena_ctrl_major_vsn, ena->ena_ctrl_minor_vsn, ena->ena_ctrl_subminor_vsn); return (false); } return (true); } static bool ena_adminq_init(ena_t *ena) { ena_adminq_t *aq = &ena->ena_aq; /* * As we are not using an interrupt for admin queue completion * signaling, we do not need a priority on these mutexes. If * that changes, we will have to rejigger some code to create * the admin queue interrupt before this function. */ mutex_init(&aq->ea_sq_lock, NULL, MUTEX_DRIVER, NULL); mutex_init(&aq->ea_cq_lock, NULL, MUTEX_DRIVER, NULL); mutex_init(&aq->ea_stat_lock, NULL, MUTEX_DRIVER, NULL); aq->ea_qlen = ENA_ADMINQ_DEPTH; aq->ea_pending_cmds = 0; aq->ea_cmd_ctxs = kmem_zalloc(sizeof (ena_cmd_ctx_t) * aq->ea_qlen, KM_SLEEP); list_create(&aq->ea_cmd_ctxs_free, sizeof (ena_cmd_ctx_t), offsetof(ena_cmd_ctx_t, ectx_node)); list_create(&aq->ea_cmd_ctxs_used, sizeof (ena_cmd_ctx_t), offsetof(ena_cmd_ctx_t, ectx_node)); ena_create_cmd_ctx(ena); /* * Start in polling mode until we've determined the number of queues * and are ready to configure and enable interrupts. */ ena_hw_bar_write32(ena, ENAHW_REG_INTERRUPT_MASK, ENAHW_INTR_MASK); aq->ea_poll_mode = true; return (true); } /* * Free all resources allocated as part of ena_device_init(). */ static void ena_cleanup_device_init(ena_t *ena, bool resetting) { ena_adminq_t *aq = &ena->ena_aq; VERIFY0(resetting); if (ena->ena_llq_bar_mapped) { ddi_regs_map_free(&ena->ena_llq_bar_hdl); ena->ena_llq_bar_mapped = false; } ena_free_host_info(ena); mutex_destroy(&aq->ea_sq_lock); mutex_destroy(&aq->ea_cq_lock); mutex_destroy(&aq->ea_stat_lock); list_destroy(&aq->ea_cmd_ctxs_free); list_destroy(&aq->ea_cmd_ctxs_used); kmem_free(aq->ea_cmd_ctxs, sizeof (ena_cmd_ctx_t) * aq->ea_qlen); ena_admin_sq_free(ena); ena_admin_cq_free(ena); ena_aenq_free(ena); ena_stat_device_cleanup(ena); ena_stat_device_basic_cleanup(ena); ena_stat_device_extended_cleanup(ena); ena_stat_aenq_cleanup(ena); } /* * We explicitly disable hardware timestamping as a precaution. * When enabled, the device uses (larger) extended completion descriptors to * incorporate timestamp fields. Our completion processing assumes the base * descriptor sizes. */ static bool ena_disable_hw_timestamp(ena_t *ena) { enahw_cmd_desc_t cmd; enahw_resp_desc_t resp; enahw_feat_hw_timestamp_t *ts = &cmd.ecd_cmd.ecd_set_feat.ecsf_feat.ecsf_hw_timestamp; int ret; if (!ena_is_feat_avail(ena, ENAHW_FEAT_HW_TIMESTAMP)) return (true); ena_dbg(ena, "Disabling HW timestamping"); bzero(&cmd, sizeof (cmd)); ts->efhwts_tx = ENAHW_HW_TIMESTAMP_NONE; ts->efhwts_rx = ENAHW_HW_TIMESTAMP_NONE; ret = ena_set_feature(ena, &cmd, &resp, ENAHW_FEAT_HW_TIMESTAMP, ENAHW_FEAT_HW_TIMESTAMP_VER); if (ret != 0) { ena_err(ena, "failed to disable HW timestamping: %d", ret); return (false); } return (true); } /* * Map the device memory BAR used for LLQ. This must be called before creating * any LLQ submission queues. */ static bool ena_map_llq_mem_bar(ena_t *ena) { ddi_device_acc_attr_t attr; int rnumber, ret; rnumber = ena_bar_to_rnumber(ena, ENA_LLQ_BAR); if (rnumber == -1) { ena_err(ena, "failed to find rnumber for BAR %d", ENA_LLQ_BAR); return (false); } if (ddi_dev_regsize(ena->ena_dip, rnumber, &ena->ena_llq_bar_size) != DDI_SUCCESS) { ena_err(ena, "failed to get mem BAR %d size", ENA_LLQ_BAR); return (false); } bzero(&attr, sizeof (attr)); attr.devacc_attr_version = DDI_DEVICE_ATTR_V1; attr.devacc_attr_endian_flags = DDI_NEVERSWAP_ACC; /* * See the "LLQ Tx Path" section of the big theory statement for why * it is safe to map the LLQ BAR with DDI_STORECACHING_OK_ACC. */ attr.devacc_attr_dataorder = DDI_STORECACHING_OK_ACC; attr.devacc_attr_access = DDI_DEFAULT_ACC; ret = ddi_regs_map_setup(ena->ena_dip, rnumber, &ena->ena_llq_bar_base, 0, ena->ena_llq_bar_size, &attr, &ena->ena_llq_bar_hdl); if (ret != DDI_SUCCESS) { ena_err(ena, "failed to map mem BAR %d (reg %d): %d", ENA_LLQ_BAR, rnumber, ret); return (false); } ena->ena_llq_bar_mapped = true; ena_dbg(ena, "LLQ BAR %d mapped to base: 0x%p size: %ld", ENA_LLQ_BAR, (void *)ena->ena_llq_bar_base, ena->ena_llq_bar_size); return (true); } /* * Negotiate LLQ parameters with the device and send SET_FEATURE to * configure it. If the device does not advertise LLQ support we * silently fall back to host placement. If the device does advertise * LLQ but negotiation fails, we treat that as a fatal error rather * than falling back. For newer instance types (Nitro v5 and later) * it really is fatal, they require LLQ and do not support host * placement at all. */ static bool ena_configure_llq(ena_t *ena) { enahw_resp_desc_t resp; enahw_feat_llq_t *llq_feat; enahw_cmd_desc_t cmd; enahw_feat_llq_t *llq_cmd; enahw_llq_accel_mode_get_t accel_get; uint16_t supported; int ret; /* Query the device's LLQ capabilities */ bzero(&resp, sizeof (resp)); ret = ena_get_feature(ena, &resp, ENAHW_FEAT_LLQ, ENAHW_FEAT_LLQ_VER); if (ret == ENOTSUP) { ena_dbg(ena, "LLQ not supported, using host placement"); return (true); } else if (ret != 0) { ena_err(ena, "failed to query LLQ feature: %d", ret); return (false); } llq_feat = &resp.erd_resp.erd_get_feat.ergf_llq; ena_dbg(ena, "LLQ max_num=%u max_depth=%u", llq_feat->efllq_max_llq_num, llq_feat->efllq_max_llq_depth); ena_dbg(ena, "LLQ header_location supported=0x%x " "entry_size supported=0x%x recommended=%u", llq_feat->efllq_header_location_ctrl_supported, llq_feat->efllq_entry_size_ctrl_supported, llq_feat->efllq_entry_size_recommended); ena_dbg(ena, "LLQ descs_before_header supported=0x%x " "stride supported=0x%x", llq_feat->efllq_desc_num_before_header_supported, llq_feat->efllq_descs_stride_ctrl_supported); ena_dbg(ena, "LLQ accel_mode supported_flags=0x%x " "max_tx_burst_size=%u", llq_feat->efllq_accel_mode.eam_get.eamg_supported_flags, llq_feat->efllq_accel_mode.eam_get.eamg_max_tx_burst_size); /* * Negotiate header location - we currently only support inline header, * which is advertised by all current devices. */ supported = llq_feat->efllq_header_location_ctrl_supported; if (!(supported & ENAHW_LLQ_HEADER_INLINE)) { ena_err(ena, "device does not support inline header " "LLQ (supported: 0x%x)", supported); return (false); } ena->ena_llq_header_location = ENAHW_LLQ_HEADER_INLINE; /* Negotiate stride control for inline header mode */ supported = llq_feat->efllq_descs_stride_ctrl_supported; if (supported & ENAHW_LLQ_MULTIPLE_DESCS_PER_ENTRY) { ena->ena_llq_stride_ctrl = ENAHW_LLQ_MULTIPLE_DESCS_PER_ENTRY; } else if (supported & ENAHW_LLQ_SINGLE_DESC_PER_ENTRY) { ena->ena_llq_stride_ctrl = ENAHW_LLQ_SINGLE_DESC_PER_ENTRY; } else { ena_err(ena, "no supported LLQ stride control (0x%x)", supported); return (false); } /* Negotiate entry size */ supported = llq_feat->efllq_entry_size_ctrl_supported; if (llq_feat->efllq_entry_size_recommended != 0 && (supported & llq_feat->efllq_entry_size_recommended)) { ena->ena_llq_entry_size = llq_feat->efllq_entry_size_recommended; } else if (supported & ENAHW_LLQ_ENTRY_SIZE_128B) { ena->ena_llq_entry_size = ENAHW_LLQ_ENTRY_SIZE_128B; } else if (supported & ENAHW_LLQ_ENTRY_SIZE_192B) { ena->ena_llq_entry_size = ENAHW_LLQ_ENTRY_SIZE_192B; } else if (supported & ENAHW_LLQ_ENTRY_SIZE_256B) { ena->ena_llq_entry_size = ENAHW_LLQ_ENTRY_SIZE_256B; } else { ena_err(ena, "no supported LLQ entry size (0x%x)", supported); return (false); } switch (ena->ena_llq_entry_size) { case ENAHW_LLQ_ENTRY_SIZE_128B: ena->ena_llq_entry_size_bytes = 128; break; case ENAHW_LLQ_ENTRY_SIZE_192B: ena->ena_llq_entry_size_bytes = 192; break; case ENAHW_LLQ_ENTRY_SIZE_256B: ena->ena_llq_entry_size_bytes = 256; break; } /* * Negotiate number of descriptors before header. Since we always * send both a meta and data descriptor in each LLQ entry we cannot * support ENAHW_LLQ_NUM_DESCS_BEFORE_HEADER_1. */ supported = llq_feat->efllq_desc_num_before_header_supported; if (supported & ENAHW_LLQ_NUM_DESCS_BEFORE_HEADER_2) { ena->ena_llq_num_descs_before_header = ENAHW_LLQ_NUM_DESCS_BEFORE_HEADER_2; } else if (supported & ENAHW_LLQ_NUM_DESCS_BEFORE_HEADER_4) { ena->ena_llq_num_descs_before_header = ENAHW_LLQ_NUM_DESCS_BEFORE_HEADER_4; } else if (supported & ENAHW_LLQ_NUM_DESCS_BEFORE_HEADER_8) { ena->ena_llq_num_descs_before_header = ENAHW_LLQ_NUM_DESCS_BEFORE_HEADER_8; } else { ena_err(ena, "no supported descs_before_header (0x%x)", supported); return (false); } /* Check for accelerated LLQ mode support */ accel_get = llq_feat->efllq_accel_mode.eam_get; const bool limit_tx_burst = (accel_get.eamg_supported_flags & ENAHW_LLQ_ACCEL_MODE_LIMIT_TX_BURST); if (limit_tx_burst) { ena->ena_llq_max_tx_burst_size = accel_get.eamg_max_tx_burst_size; } ena_dbg(ena, "LLQ negotiated: header_location=%u entry_size=%uB " "descs_before_header=%u stride=%u max_tx_burst=%u", ena->ena_llq_header_location, ena->ena_llq_entry_size_bytes, ena->ena_llq_num_descs_before_header, ena->ena_llq_stride_ctrl, ena->ena_llq_max_tx_burst_size); /* * When LLQ is active the SQ lives in device memory rather * than host memory so max_tx_sq_depth does not apply; * replace it with max_llq_depth. For 256B entries the depth * is further reduced; the device may report a specific wide * depth, otherwise we halve the standard LLQ depth. */ ena->ena_tx_max_sq_num_descs = llq_feat->efllq_max_llq_depth; if (ena->ena_llq_entry_size == ENAHW_LLQ_ENTRY_SIZE_256B) { if (llq_feat->efllq_max_wide_llq_depth != 0) { ena->ena_tx_max_sq_num_descs = MIN( ena->ena_tx_max_sq_num_descs, llq_feat->efllq_max_wide_llq_depth); } else { ena->ena_tx_max_sq_num_descs /= 2; } } /* Send the LLQ configuration request */ bzero(&cmd, sizeof (cmd)); llq_cmd = &cmd.ecd_cmd.ecd_set_feat.ecsf_feat.ecsf_llq; llq_cmd->efllq_header_location_ctrl_enabled = ena->ena_llq_header_location; llq_cmd->efllq_entry_size_ctrl_enabled = ena->ena_llq_entry_size; llq_cmd->efllq_desc_num_before_header_enabled = ena->ena_llq_num_descs_before_header; llq_cmd->efllq_descs_stride_ctrl_enabled = ena->ena_llq_stride_ctrl; llq_cmd->efllq_accel_mode.eam_set.eams_enabled_flags = ENAHW_LLQ_ACCEL_MODE_DISABLE_META_CACHING | (limit_tx_burst ? ENAHW_LLQ_ACCEL_MODE_LIMIT_TX_BURST : 0); bzero(&resp, sizeof (resp)); ret = ena_set_feature(ena, &cmd, &resp, ENAHW_FEAT_LLQ, ENAHW_FEAT_LLQ_VER); if (ret != 0) { ena_err(ena, "failed to set LLQ feature: %d", ret); return (false); } if (!ena_map_llq_mem_bar(ena)) return (false); ena->ena_llq_enabled = true; ena_dbg(ena, "LLQ enabled"); return (true); } static bool ena_attach_device_init(ena_t *ena) { ena_adminq_t *aq = &ena->ena_aq; uint32_t rval; uint8_t dma_width; hrtime_t cmd_timeout; enahw_resp_desc_t resp; enahw_feat_dev_attr_t *feat = &resp.erd_resp.erd_get_feat.ergf_dev_attr; uint8_t *maddr; uint32_t supported_features; int ret = 0; ena->ena_reset_reason = ENAHW_RESET_NORMAL; if (!ena_device_reset(ena, ena->ena_reset_reason)) return (false); if (!ena_check_versions(ena)) return (false); ena_init_regcache(ena); rval = ena_hw_bar_read32(ena, ENAHW_REG_CAPS); dma_width = ENAHW_CAPS_DMA_ADDR_WIDTH(rval); ena->ena_dma_width = dma_width; /* * The value stored in the device register is in the * resolution of 100 milliseconds. We normalize that to * nanoseconds. */ cmd_timeout = MSEC2NSEC(ENAHW_CAPS_ADMIN_CMD_TIMEOUT(rval) * 100); aq->ea_cmd_timeout_ns = max(cmd_timeout, ena_admin_cmd_timeout_ns); if (aq->ea_cmd_timeout_ns == 0) aq->ea_cmd_timeout_ns = ENA_ADMIN_CMD_DEF_TIMEOUT_NS; if (!ena_adminq_init(ena)) return (false); if (!ena_admin_sq_init(ena)) return (false); if (!ena_admin_cq_init(ena)) return (false); if (!ena_aenq_init(ena)) return (false); /* * The device will change the set of advertised features based on the * host info that we provide. We therefore need to send this here * before querying the features. */ if (!ena_init_host_info(ena)) return (false); bzero(&resp, sizeof (resp)); ret = ena_get_feature(ena, &resp, ENAHW_FEAT_DEVICE_ATTRIBUTES, ENAHW_FEAT_DEVICE_ATTRIBUTES_VER); if (ret != 0) { ena_err(ena, "failed to get device attributes: %d", ret); return (false); } ena_dbg(ena, "impl ID: %u", feat->efda_impl_id); ena_dbg(ena, "device version: %u", feat->efda_device_version); ena_dbg(ena, "supported features: 0x%x", feat->efda_supported_features); ena_dbg(ena, "device capabilities: 0x%x", feat->efda_capabilities); ena_dbg(ena, "phys addr width: %u", feat->efda_phys_addr_width); ena_dbg(ena, "virt addr width: %u", feat->efda_virt_addr_with); maddr = feat->efda_mac_addr; ena_dbg(ena, "mac addr: %x:%x:%x:%x:%x:%x", maddr[0], maddr[1], maddr[2], maddr[3], maddr[4], maddr[5]); ena_dbg(ena, "max MTU: %u", feat->efda_max_mtu); bcopy(maddr, ena->ena_mac_addr, ETHERADDRL); ena->ena_max_mtu = feat->efda_max_mtu; ena->ena_capabilities = feat->efda_capabilities; supported_features = feat->efda_supported_features; ena->ena_supported_features = supported_features; feat = NULL; bzero(&resp, sizeof (resp)); if (ena_is_feat_avail(ena, ENAHW_FEAT_MAX_QUEUES_EXT)) { enahw_feat_max_queue_ext_t *feat_mqe = &resp.erd_resp.erd_get_feat.ergf_max_queue_ext; ret = ena_get_feature(ena, &resp, ENAHW_FEAT_MAX_QUEUES_EXT, ENAHW_FEAT_MAX_QUEUES_EXT_VER); if (ret != 0) { ena_err(ena, "failed to query max queues ext: %d", ret); return (false); } ena->ena_tx_max_sq_num = feat_mqe->efmqe_max_tx_sq_num; ena->ena_tx_max_sq_num_descs = feat_mqe->efmqe_max_tx_sq_depth; ena->ena_tx_max_cq_num = feat_mqe->efmqe_max_tx_cq_num; ena->ena_tx_max_cq_num_descs = feat_mqe->efmqe_max_tx_cq_depth; ena->ena_tx_max_desc_per_pkt = feat_mqe->efmqe_max_per_packet_tx_descs; ena->ena_tx_max_hdr_len = feat_mqe->efmqe_max_tx_header_size; ena->ena_rx_max_sq_num = feat_mqe->efmqe_max_rx_sq_num; ena->ena_rx_max_sq_num_descs = feat_mqe->efmqe_max_rx_sq_depth; ena->ena_rx_max_cq_num = feat_mqe->efmqe_max_rx_cq_num; ena->ena_rx_max_cq_num_descs = feat_mqe->efmqe_max_rx_cq_depth; ena->ena_rx_max_desc_per_pkt = feat_mqe->efmqe_max_per_packet_rx_descs; ena_set_max_io_queues(ena); } else { enahw_feat_max_queue_t *feat_mq = &resp.erd_resp.erd_get_feat.ergf_max_queue; ret = ena_get_feature(ena, &resp, ENAHW_FEAT_MAX_QUEUES_NUM, ENAHW_FEAT_MAX_QUEUES_NUM_VER); if (ret != 0) { ena_err(ena, "failed to query max queues: %d", ret); return (false); } ena->ena_tx_max_sq_num = feat_mq->efmq_max_sq_num; ena->ena_tx_max_sq_num_descs = feat_mq->efmq_max_sq_depth; ena->ena_tx_max_cq_num = feat_mq->efmq_max_cq_num; ena->ena_tx_max_cq_num_descs = feat_mq->efmq_max_cq_depth; ena->ena_tx_max_desc_per_pkt = feat_mq->efmq_max_per_packet_tx_descs; ena->ena_tx_max_hdr_len = feat_mq->efmq_max_header_size; ena->ena_rx_max_sq_num = feat_mq->efmq_max_sq_num; ena->ena_rx_max_sq_num_descs = feat_mq->efmq_max_sq_depth; ena->ena_rx_max_cq_num = feat_mq->efmq_max_cq_num; ena->ena_rx_max_cq_num_descs = feat_mq->efmq_max_cq_depth; ena->ena_rx_max_desc_per_pkt = feat_mq->efmq_max_per_packet_rx_descs; ena_set_max_io_queues(ena); } ena->ena_mtu = ena->ena_max_mtu; ena_update_buf_sizes(ena); if (!ena_get_hints(ena)) return (false); ena->ena_tx_sgl_max_sz = 1; ena->ena_rx_sgl_max_sz = 1; if (ena->ena_device_hints.eh_max_tx_sgl != 0) ena->ena_tx_sgl_max_sz = ena->ena_device_hints.eh_max_tx_sgl; if (ena->ena_device_hints.eh_max_rx_sgl != 0) ena->ena_rx_sgl_max_sz = ena->ena_device_hints.eh_max_rx_sgl; if (!ena_disable_hw_timestamp(ena)) return (false); if (!ena_configure_llq(ena)) return (false); if (!ena_aenq_configure(ena)) return (false); ena_get_link_config(ena); if (!ena_get_offloads(ena)) return (false); if (!ena_stat_device_init(ena)) return (false); if (!ena_stat_device_basic_init(ena)) return (false); if (!ena_stat_device_extended_init(ena)) return (false); if (!ena_stat_aenq_init(ena)) return (false); ena_update_regcache(ena); return (true); } static void ena_cleanup_intr_alloc(ena_t *ena, bool resetting) { VERIFY0(resetting); for (int i = 0; i < ena->ena_num_intrs; i++) { int ret = ddi_intr_free(ena->ena_intr_handles[i]); if (ret != DDI_SUCCESS) { ena_err(ena, "failed to free interrupt %d: %d", i, ret); } } if (ena->ena_intr_handles != NULL) { kmem_free(ena->ena_intr_handles, ena->ena_intr_handles_sz); ena->ena_intr_handles = NULL; ena->ena_intr_handles_sz = 0; } } /* * The Linux driver supports only MSI-X interrupts. We do the same, * with the assumption that it's the only type of interrupt the device * can present. */ static bool ena_attach_intr_alloc(ena_t *ena) { int ret; int types; int min, req, ideal, avail, actual; ret = ddi_intr_get_supported_types(ena->ena_dip, &types); if (ret != DDI_SUCCESS) { ena_err(ena, "failed to get interrupt types: %d", ret); return (false); } ena_dbg(ena, "supported interrupt types: 0x%x", types); if ((types & DDI_INTR_TYPE_MSIX) == 0) { ena_err(ena, "the ena driver only supports MSI-X interrupts"); return (false); } /* One for I/O, one for adminq. */ min = 2; ideal = ena->ena_max_io_queues + 1; ret = ddi_intr_get_nintrs(ena->ena_dip, DDI_INTR_TYPE_MSIX, &avail); if (ret != DDI_SUCCESS) { ena_err(ena, "failed to get number of MSI-X interrupts: %d", ret); return (false); } if (avail < min) { ena_err(ena, "number of MSI-X interrupts is %d, but the driver " "requires a minimum of %d", avail, min); return (false); } ena_dbg(ena, "%d MSI-X interrupts available", avail); ret = ddi_intr_get_navail(ena->ena_dip, DDI_INTR_TYPE_MSIX, &avail); if (ret != DDI_SUCCESS) { ena_err(ena, "failed to get available interrupts: %d", ret); return (false); } if (avail < min) { ena_err(ena, "number of available MSI-X interrupts is %d, " "but the driver requires a minimum of %d", avail, min); return (false); } req = MIN(ideal, avail); ena->ena_intr_handles_sz = req * sizeof (ddi_intr_handle_t); ena->ena_intr_handles = kmem_zalloc(ena->ena_intr_handles_sz, KM_SLEEP); ret = ddi_intr_alloc(ena->ena_dip, ena->ena_intr_handles, DDI_INTR_TYPE_MSIX, 0, req, &actual, DDI_INTR_ALLOC_NORMAL); if (ret != DDI_SUCCESS) { ena_err(ena, "failed to allocate %d MSI-X interrupts: %d", req, ret); return (false); } if (actual < min) { ena_err(ena, "number of allocated interrupts is %d, but the " "driver requires a minimum of %d", actual, min); return (false); } ena->ena_num_intrs = actual; ret = ddi_intr_get_cap(ena->ena_intr_handles[0], &ena->ena_intr_caps); if (ret != DDI_SUCCESS) { ena_err(ena, "failed to get interrupt capability: %d", ret); return (false); } ret = ddi_intr_get_pri(ena->ena_intr_handles[0], &ena->ena_intr_pri); if (ret != DDI_SUCCESS) { ena_err(ena, "failed to get interrupt priority: %d", ret); return (false); } ena_dbg(ena, "MSI-X interrupts allocated: %d, cap: 0x%x, pri: %u", actual, ena->ena_intr_caps, ena->ena_intr_pri); /* * The ena_lock should not be held in the data path, but it is * held as part of the AENQ handler, which runs in interrupt * context. Therefore, we delayed the initialization of this * mutex until after the interrupts are allocated. */ mutex_init(&ena->ena_lock, NULL, MUTEX_DRIVER, DDI_INTR_PRI(ena->ena_intr_pri)); mutex_init(&ena->ena_watchdog_lock, NULL, MUTEX_DRIVER, NULL); return (true); } /* * Allocate the parent Rx queue structures. More importantly, this is * NOT allocating the queue descriptors or data buffers. Those are * allocated on demand as queues are started. */ static bool ena_attach_alloc_rxqs(ena_t *ena) { bool resetting = false; if (ena->ena_rxqs == NULL) { /* * We rely on the interrupt priority for initializing the * mutexes. */ VERIFY3U(ena->ena_attach_seq, >=, ENA_ATTACH_INTR_ALLOC); ena->ena_num_rxqs = ena->ena_num_intrs - 1; ASSERT3U(ena->ena_num_rxqs, >, 0); ena->ena_rxqs = kmem_zalloc( ena->ena_num_rxqs * sizeof (*ena->ena_rxqs), KM_SLEEP); } else { resetting = true; } for (uint_t i = 0; i < ena->ena_num_rxqs; i++) { ena_rxq_t *rxq = &ena->ena_rxqs[i]; rxq->er_rxqs_idx = i; /* The 0th vector is for Admin + AENQ. */ rxq->er_intr_vector = i + 1; rxq->er_mrh = NULL; if (!resetting) { mutex_init(&rxq->er_lock, NULL, MUTEX_DRIVER, DDI_INTR_PRI(ena->ena_intr_pri)); mutex_init(&rxq->er_stat_lock, NULL, MUTEX_DRIVER, DDI_INTR_PRI(ena->ena_intr_pri)); } rxq->er_ena = ena; rxq->er_sq_num_descs = ena->ena_rxq_num_descs; rxq->er_cq_num_descs = ena->ena_rxq_num_descs; if (!ena_stat_rxq_init(rxq)) { return (false); } if (!ena_alloc_rxq(rxq)) { ena_stat_rxq_cleanup(rxq); return (false); } } return (true); } static void ena_cleanup_rxqs(ena_t *ena, bool resetting) { for (uint_t i = 0; i < ena->ena_num_rxqs; i++) { ena_rxq_t *rxq = &ena->ena_rxqs[i]; ena_cleanup_rxq(rxq, resetting); if (!resetting) { mutex_destroy(&rxq->er_lock); mutex_destroy(&rxq->er_stat_lock); } ena_stat_rxq_cleanup(rxq); } if (!resetting) { kmem_free(ena->ena_rxqs, ena->ena_num_rxqs * sizeof (*ena->ena_rxqs)); ena->ena_rxqs = NULL; } } /* * Allocate the parent Tx queue structures. More importantly, this is * NOT allocating the queue descriptors or data buffers. Those are * allocated on demand as a queue is started. */ static bool ena_attach_alloc_txqs(ena_t *ena) { bool resetting = false; if (ena->ena_txqs == NULL) { /* * We rely on the interrupt priority for initializing the * mutexes. */ VERIFY3U(ena->ena_attach_seq, >=, ENA_ATTACH_INTR_ALLOC); ena->ena_num_txqs = ena->ena_num_intrs - 1; ASSERT3U(ena->ena_num_txqs, >, 0); ena->ena_txqs = kmem_zalloc( ena->ena_num_txqs * sizeof (*ena->ena_txqs), KM_SLEEP); } else { resetting = true; } for (uint_t i = 0; i < ena->ena_num_txqs; i++) { ena_txq_t *txq = &ena->ena_txqs[i]; txq->et_txqs_idx = i; /* The 0th vector is for Admin + AENQ. */ txq->et_intr_vector = i + 1; txq->et_mrh = NULL; if (!resetting) { mutex_init(&txq->et_lock, NULL, MUTEX_DRIVER, DDI_INTR_PRI(ena->ena_intr_pri)); mutex_init(&txq->et_stat_lock, NULL, MUTEX_DRIVER, DDI_INTR_PRI(ena->ena_intr_pri)); } txq->et_ena = ena; txq->et_sq_num_descs = ena->ena_txq_num_descs; txq->et_cq_num_descs = ena->ena_txq_num_descs; if (!ena_stat_txq_init(txq)) { return (false); } if (!ena_alloc_txq(txq)) { ena_stat_txq_cleanup(txq); return (false); } } return (true); } static void ena_cleanup_txqs(ena_t *ena, bool resetting) { for (uint_t i = 0; i < ena->ena_num_txqs; i++) { ena_txq_t *txq = &ena->ena_txqs[i]; ena_cleanup_txq(txq, resetting); if (!resetting) { mutex_destroy(&txq->et_lock); mutex_destroy(&txq->et_stat_lock); } ena_stat_txq_cleanup(txq); } if (!resetting) { kmem_free(ena->ena_txqs, ena->ena_num_txqs * sizeof (*ena->ena_txqs)); ena->ena_txqs = NULL; } } /* * To reset the device we need to unwind some of the steps taken during attach * but, since the device could well be in a failed state, we cannot rely on * being able to talk via the admin queue to do things such as explicitly * destroy rings. We call selected cleanup handlers with the second parameter * set to "true" to indicate that we are resetting and should avoid such * communication. * * The existing DMA memory regions for the admin queue, async event queue and * host information are preserved but have their contents zeroed. * Experimentation has shown that the device hangs onto old async event queue * addresses, even through a reset, with surprising results if the addresses * happen to change. * * We clean up all of the Tx and Rx ring descriptors and the TCBs but leave the * allocated memory for the ring data and mutexes intact. Pointers to this * memory have already been provided to MAC, and the mutexes keep the rings * locked until we're ready to start them again. * * To ensure that other driver activity is excluded, we hold the mutexes on the * Tx and Rx rings throughout, and unset the `ENA_STATE_STARTED` bit in the * state, which causes the interrupt handlers to return without doing any work. * The admin interrupt, used for notifications of admin completions or new * asynchronous events, is masked after the device is reset until we're ready * to process them again. */ bool ena_reset(ena_t *ena, const enahw_reset_reason_t reason) { ena_txq_state_t tx_state[ena->ena_num_txqs]; ena_rxq_state_t rx_state[ena->ena_num_rxqs]; bool ret = false; ena_err(ena, "resetting device with reason 0x%x [%s]", reason, enahw_reset_reason(reason)); VERIFY0(ena->ena_state & ENA_STATE_RESETTING); atomic_or_32(&ena->ena_state, ENA_STATE_RESETTING); VERIFY(ena->ena_state & ENA_STATE_STARTED); atomic_and_32(&ena->ena_state, ~ENA_STATE_STARTED); mutex_enter(&ena->ena_lock); ena_update_regcache(ena); for (uint_t i = 0; i < ena->ena_num_txqs; i++) { ena_txq_t *txq = &ena->ena_txqs[i]; mutex_enter(&txq->et_lock); tx_state[i] = txq->et_state; if (txq->et_state & ENA_TXQ_STATE_RUNNING) ena_ring_tx_stop((mac_ring_driver_t)txq); } for (uint_t i = 0; i < ena->ena_num_rxqs; i++) { ena_rxq_t *rxq = &ena->ena_rxqs[i]; mutex_enter(&rxq->er_lock); rx_state[i] = rxq->er_state; if (rxq->er_state & ENA_RXQ_STATE_RUNNING) ena_ring_rx_stop((mac_ring_driver_t)rxq); } if (!ena_device_reset(ena, reason)) { ena_err(ena, "reset: failed to reset device"); goto out; } /* This masks the admin/aenq interrupt */ ena_hw_bar_write32(ena, ENAHW_REG_INTERRUPT_MASK, ENAHW_INTR_MASK); ena_cleanup_txqs(ena, true); ena_cleanup_rxqs(ena, true); ena_release_all_cmd_ctx(ena); if (!ena_admin_cq_init(ena) || !ena_admin_sq_init(ena)) { ena_err(ena, "reset: failed to program admin queues"); goto out; } if (!ena_init_host_info(ena)) { ena_err(ena, "reset: failed to set host info"); goto out; } if (!ena_aenq_init(ena) || !ena_aenq_configure(ena)) { ena_err(ena, "reset: failed to configure aenq"); goto out; } if (!ena_set_mtu(ena)) { ena_err(ena, "reset: failed to set MTU"); goto out; } if (!ena_attach_alloc_txqs(ena) || !ena_attach_alloc_rxqs(ena)) { ena_err(ena, "reset: failed to program IO queues"); goto out; } ena_aenq_enable(ena); ena_hw_bar_write32(ena, ENAHW_REG_INTERRUPT_MASK, ENAHW_INTR_UNMASK); for (uint_t i = 0; i < ena->ena_num_rxqs; i++) { ena_rxq_t *rxq = &ena->ena_rxqs[i]; mutex_exit(&rxq->er_lock); if (rx_state[i] & ENA_RXQ_STATE_RUNNING) { (void) ena_ring_rx_start((mac_ring_driver_t)rxq, rxq->er_m_gen_num); } } for (uint_t i = 0; i < ena->ena_num_txqs; i++) { ena_txq_t *txq = &ena->ena_txqs[i]; mutex_exit(&txq->et_lock); if (tx_state[i] & ENA_TXQ_STATE_RUNNING) { (void) ena_ring_tx_start((mac_ring_driver_t)txq, txq->et_m_gen_num); } } atomic_or_32(&ena->ena_state, ENA_STATE_STARTED); ret = true; out: atomic_and_32(&ena->ena_state, ~ENA_STATE_RESETTING); mutex_exit(&ena->ena_lock); ena_update_regcache(ena); return (ret); } ena_attach_desc_t ena_attach_tbl[ENA_ATTACH_NUM_ENTRIES] = { { .ead_seq = ENA_ATTACH_PCI, .ead_name = "PCI config", .ead_attach_fn = ena_attach_pci, .ead_attach_hard_fail = true, .ead_cleanup_fn = ena_cleanup_pci, }, { .ead_seq = ENA_ATTACH_REGS, .ead_name = "BAR mapping", .ead_attach_fn = ena_attach_regs_map, .ead_attach_hard_fail = true, .ead_cleanup_fn = ena_cleanup_regs_map, }, { .ead_seq = ENA_ATTACH_DEV_INIT, .ead_name = "device initialization", .ead_attach_fn = ena_attach_device_init, .ead_attach_hard_fail = true, .ead_cleanup_fn = ena_cleanup_device_init, }, { .ead_seq = ENA_ATTACH_READ_CONF, .ead_name = "ena.conf", .ead_attach_fn = ena_attach_read_conf, .ead_attach_hard_fail = true, .ead_cleanup_fn = NULL, }, { .ead_seq = ENA_ATTACH_DEV_CFG, .ead_name = "device config", .ead_attach_fn = ena_attach_dev_cfg, .ead_attach_hard_fail = true, .ead_cleanup_fn = NULL, }, { .ead_seq = ENA_ATTACH_INTR_ALLOC, .ead_name = "interrupt allocation", .ead_attach_fn = ena_attach_intr_alloc, .ead_attach_hard_fail = true, .ead_cleanup_fn = ena_cleanup_intr_alloc, }, { .ead_seq = ENA_ATTACH_INTR_HDLRS, .ead_name = "interrupt handlers", .ead_attach_fn = ena_intr_add_handlers, .ead_attach_hard_fail = true, .ead_cleanup_fn = ena_intr_remove_handlers, }, { .ead_seq = ENA_ATTACH_TXQS_ALLOC, .ead_name = "Tx queues", .ead_attach_fn = ena_attach_alloc_txqs, .ead_attach_hard_fail = true, .ead_cleanup_fn = ena_cleanup_txqs, }, { .ead_seq = ENA_ATTACH_RXQS_ALLOC, .ead_name = "Rx queues", .ead_attach_fn = ena_attach_alloc_rxqs, .ead_attach_hard_fail = true, .ead_cleanup_fn = ena_cleanup_rxqs, }, /* * The chance of mac_unregister() failure poses a problem to * cleanup. We address interrupt disablement and mac * unregistration explicitly in the attach/detach routines. */ { .ead_seq = ENA_ATTACH_MAC_REGISTER, .ead_name = "mac registration", .ead_attach_fn = ena_mac_register, .ead_attach_hard_fail = true, .ead_cleanup_fn = NULL, }, { .ead_seq = ENA_ATTACH_INTRS_ENABLE, .ead_name = "enable interrupts", .ead_attach_fn = ena_intrs_enable, .ead_attach_hard_fail = true, .ead_cleanup_fn = NULL, } }; /* * This function undoes any work done by ena_attach(), either in * response to a failed attach or a planned detach. At the end of this * function ena_attach_seq should be zero, otherwise it means * something has not be freed/uninitialized. */ static void ena_cleanup(ena_t *ena) { if (ena == NULL || ena->ena_attach_seq == 0) { return; } /* * We VERIFY this because if the seq is greater than entries * we drift into space and execute god knows what. */ VERIFY3U(ena->ena_attach_seq, <, ENA_ATTACH_NUM_ENTRIES); while (ena->ena_attach_seq > 0) { int idx = ena->ena_attach_seq - 1; ena_attach_desc_t *desc = &ena_attach_tbl[idx]; ena_dbg(ena, "running cleanup sequence: %s (%d)", desc->ead_name, idx); if (desc->ead_cleanup_fn != NULL) desc->ead_cleanup_fn(ena, false); ena->ena_attach_seq--; } ASSERT3U(ena->ena_attach_seq, ==, 0); mutex_destroy(&ena->ena_lock); mutex_destroy(&ena->ena_watchdog_lock); } static int ena_attach(dev_info_t *dip, ddi_attach_cmd_t cmd) { ena_t *ena; if (cmd != DDI_ATTACH) { return (DDI_FAILURE); } ena = kmem_zalloc(sizeof (ena_t), KM_SLEEP); ena->ena_instance = ddi_get_instance(dip); ena->ena_dip = dip; ena->ena_instance = ddi_get_instance(dip); ena->ena_page_sz = ddi_ptob(dip, 1); for (int i = 0; i < ENA_ATTACH_NUM_ENTRIES; i++) { bool success; ena_attach_desc_t *desc = &ena_attach_tbl[i]; ena_dbg(ena, "running attach sequence: %s (%d)", desc->ead_name, i); if (!(success = desc->ead_attach_fn(ena))) { ena_err(ena, "attach sequence failed: %s (%d)", desc->ead_name, i); if (ena->ena_attach_seq == ENA_ATTACH_MAC_REGISTER) { /* * In this specific case * ENA_ATTACH_INTRS_ENABLE has failed, * and we may or may not be able to * unregister the mac, depending on if * something in userspace has created * a client on top. * * NOTE: Something that would be nice * to add to mac is the ability to * register a provider separate from * "publishing" it to the rest of the * system. This would allow a driver * to register its mac, do some * additional work that might fail, * and then unregister if that work * fails without concern for any * chance of failure when calling * unregister. This would remove the * complexity of the situation we are * trying to address here, as we would * know that until the mac has been * "published", there is no chance for * mac_unregister() to fail. */ if (ena_mac_unregister(ena) != 0) { return (DDI_FAILURE); } ena->ena_attach_seq--; } else { /* * Since the ead_seq is predicated on * successful ead_attach_fn we must * run the specific cleanup handler * before calling the global cleanup * routine. This also means that all * cleanup functions must be able to * deal with partial success of the * corresponding ead_attach_fn. */ if (desc->ead_cleanup_fn != NULL) desc->ead_cleanup_fn(ena, false); } ena_cleanup(ena); kmem_free(ena, sizeof (ena_t)); return (DDI_FAILURE); } if (success) { ena_dbg(ena, "attach sequence completed: %s (%d)", desc->ead_name, i); } ena->ena_attach_seq = desc->ead_seq; } /* * Now that interrupts are enabled, unmask the admin interrupt. * Note that this interrupt is generated for both the admin queue and * the AENQ, but this driver always polls the admin queue. The surplus * interrupt for admin command completion triggers a harmless check of * the AENQ. */ ena_hw_bar_write32(ena, ENAHW_REG_INTERRUPT_MASK, ENAHW_INTR_UNMASK); ena_aenq_enable(ena); ddi_set_driver_private(dip, ena); ena_update_regcache(ena); atomic_or_32(&ena->ena_state, ENA_STATE_INITIALIZED); return (DDI_SUCCESS); } static int ena_detach(dev_info_t *dip, ddi_detach_cmd_t cmd) { ena_t *ena = ddi_get_driver_private(dip); if (ena == NULL) { return (DDI_FAILURE); } /* * Before we can proceed to cleanup we have to treat * mac_unregister() explicitly -- if there are still * outstanding clients, then we can't proceed with detach or * cleanup. */ /* * Why this would fail I don't know, but if we proceed to mac * unregister, then there is a good chance we will panic in * the Rx interrupt handler when calling mac_rx_ring() */ if (!ena_intrs_disable(ena)) { return (DDI_FAILURE); } /* We can't detach if clients are actively using the device. */ if (ena_mac_unregister(ena) != 0) { (void) ena_intrs_enable(ena); return (DDI_FAILURE); } /* * At this point we can proceed with the rest of cleanup on a * best-effort basis. */ ena->ena_attach_seq = ENA_ATTACH_RXQS_ALLOC; ena_cleanup(ena); ddi_set_driver_private(dip, NULL); kmem_free(ena, sizeof (ena_t)); return (DDI_SUCCESS); } static struct cb_ops ena_cb_ops = { .cb_open = nodev, .cb_close = nodev, .cb_strategy = nodev, .cb_print = nodev, .cb_dump = nodev, .cb_read = nodev, .cb_write = nodev, .cb_ioctl = nodev, .cb_devmap = nodev, .cb_mmap = nodev, .cb_segmap = nodev, .cb_chpoll = nochpoll, .cb_prop_op = ddi_prop_op, .cb_flag = D_MP, .cb_rev = CB_REV, .cb_aread = nodev, .cb_awrite = nodev }; static struct dev_ops ena_dev_ops = { .devo_rev = DEVO_REV, .devo_refcnt = 0, .devo_getinfo = NULL, .devo_identify = nulldev, .devo_probe = nulldev, .devo_attach = ena_attach, .devo_detach = ena_detach, .devo_reset = nodev, .devo_quiesce = ddi_quiesce_not_supported, .devo_cb_ops = &ena_cb_ops }; static struct modldrv ena_modldrv = { .drv_modops = &mod_driverops, .drv_linkinfo = "AWS ENA Ethernet", .drv_dev_ops = &ena_dev_ops }; static struct modlinkage ena_modlinkage = { .ml_rev = MODREV_1, .ml_linkage = { &ena_modldrv, NULL } }; int _init(void) { int ret; mac_init_ops(&ena_dev_ops, ENA_MODULE_NAME); if ((ret = mod_install(&ena_modlinkage)) != 0) { mac_fini_ops(&ena_dev_ops); return (ret); } return (ret); } int _info(struct modinfo *modinfop) { return (mod_info(&ena_modlinkage, modinfop)); } int _fini(void) { int ret; if ((ret = mod_remove(&ena_modlinkage)) != 0) { return (ret); } mac_fini_ops(&ena_dev_ops); return (ret); }