vioscsi.c - zygaena/hammerhead

root / base / usr / src / uts / common / io / vioscsi / vioscsi.c
vioscsi.c C 1675 lines 40.4 KB
/*
 * This file and its contents are supplied under the terms of the
 * Common Development and Distribution License ("CDDL"), version 1.0.
 * You may only use this file in accordance with the terms of version
 * 1.0 of the CDDL.
 *
 * A full copy of the text of the CDDL should have accompanied this
 * source.  A copy of the CDDL is also available via the Internet at
 * http://www.illumos.org/license/CDDL.
 */

/*
 * Copyright 2019 Nexenta by DDN, Inc. All rights reserved.
 * Copyright 2022 RackTop Systems, Inc.
 * Copyright 2025 Oxide Computer Company
 * Copyright 2026 Hans Rosenfeld
 */

#include "vioscsi.h"

static char vioscsi_ident[] = "VIRTIO SCSI driver";

static uint_t vioscsi_ctl_handler(caddr_t arg1, caddr_t arg2);
static uint_t vioscsi_evt_handler(caddr_t arg1, caddr_t arg2);
static uint_t vioscsi_cmd_handler(caddr_t arg1, caddr_t arg2);

static int vioscsi_tran_getcap(struct scsi_address *, char *, int);
static int vioscsi_tran_setcap(struct scsi_address *, char *, int, int);
static int vioscsi_tran_reset(struct scsi_address *, int);

static int vioscsi_tran_start(struct scsi_address *, struct scsi_pkt *);
static int vioscsi_tran_abort(struct scsi_address *, struct scsi_pkt *);

static int vioscsi_iport_attach(dev_info_t *);
static int vioscsi_iport_detach(dev_info_t *);

static int vioscsi_req_init(vioscsi_softc_t *, vioscsi_request_t *,
    virtio_queue_t *, int);
static void vioscsi_req_fini(vioscsi_request_t *);
static boolean_t vioscsi_req_abort(vioscsi_softc_t *, vioscsi_request_t *);
static void vioscsi_lun_changed(vioscsi_softc_t *sc, uint8_t target);
static void vioscsi_discover(void *);

static void *vioscsi_state;

/*
 * DMA attributes. We support a linked list, but most of our uses require a
 * single aligned buffer.  The HBA buffers will use a copy of this adjusted for
 * the actual virtio limits.
 */
static ddi_dma_attr_t virtio_dma_attr = {
	.dma_attr_version =		DMA_ATTR_V0,
	.dma_attr_addr_lo =		0,
	.dma_attr_addr_hi =		0xFFFFFFFFFFFFFFFFull,
	.dma_attr_count_max =		0x00000000FFFFFFFFull,
	.dma_attr_align =		1,
	.dma_attr_burstsizes =		1,
	.dma_attr_minxfer =		1,
	.dma_attr_maxxfer =		0xFFFFFFFFull,
	.dma_attr_seg =			0xFFFFFFFFFFFFFFFFull,
	.dma_attr_sgllen =		1,
	.dma_attr_granular =		1,
	.dma_attr_flags =		0,
};

/*
 * this avoids calls to drv_usectohz that might be expensive:
 */
static clock_t vioscsi_hz;

static boolean_t
vioscsi_poll_until(vioscsi_softc_t *sc, vioscsi_request_t *req,
    ddi_intr_handler_t func, clock_t until)
{
	until *= 1000000; /* convert to usec */
	while (until > 0) {
		(void) func((caddr_t)sc, NULL);
		if (req->vr_done) {
			return (B_TRUE);
		}
		drv_usecwait(10);
		until -= 10;
	}
	atomic_or_8(&req->vr_expired, 1);
	return (B_FALSE);
}

static boolean_t
vioscsi_tmf(vioscsi_softc_t *sc, uint32_t func, uint8_t target, uint16_t lun,
    vioscsi_request_t *task)
{
	vioscsi_request_t req;
	vioscsi_tmf_res_t *res;
	vioscsi_tmf_req_t *tmf;

	bzero(&req, sizeof (req));

	if (vioscsi_req_init(sc, &req, sc->vs_ctl_vq, KM_NOSLEEP) != 0) {
		return (B_FALSE);
	}

	tmf = &req.vr_req->tmf;
	res = &req.vr_res->tmf;

	tmf->type = VIRTIO_SCSI_T_TMF;
	tmf->subtype = func;
	tmf->lun[0] = 1;
	tmf->lun[1] = target;
	tmf->lun[2] = 0x40 | (lun >> 8);
	tmf->lun[3] = lun & 0xff;
	tmf->tag = (uint64_t)task;

	virtio_chain_clear(req.vr_vic);
	if (virtio_chain_append(req.vr_vic, req.vr_req_pa, sizeof (*tmf),
	    VIRTIO_DIR_DEVICE_READS) != DDI_SUCCESS) {
		virtio_chain_clear(req.vr_vic);
		return (B_FALSE);
	}

	if (virtio_chain_append(req.vr_vic, req.vr_res_pa, sizeof (*res),
	    VIRTIO_DIR_DEVICE_WRITES) != DDI_SUCCESS) {
		virtio_chain_clear(req.vr_vic);
		return (B_FALSE);
	}

	/*
	 * Make sure the device can see our request:
	 */
	virtio_dma_sync(req.vr_dma, DDI_DMA_SYNC_FORDEV);

	/*
	 * Push chain into the queue:
	 */
	virtio_chain_submit(req.vr_vic, B_TRUE);

	/*
	 * Wait for it to complete -- these should always complete in a tiny
	 * amount of time.  Give it 5 seconds to be sure.
	 */
	if (!vioscsi_poll_until(sc, &req,  vioscsi_ctl_handler, 5)) {
		/*
		 * We timed out -- this should *NEVER* happen!
		 * There is no safe way to deal with this if it occurs, so we
		 * just warn and leak the resources.  Plan for a reboot soon.
		 */
		dev_err(sc->vs_dip, CE_WARN,
		    "task mgmt timeout! (target %d lun %d)", target, lun);
		return (B_FALSE);
	}

	vioscsi_req_fini(&req);

	switch (res->response) {
	case VIRTIO_SCSI_S_OK:
	case VIRTIO_SCSI_S_FUNCTION_SUCCEEDED:
		break;
	default:
		return (B_FALSE);
	}
	return (B_TRUE);
}

static boolean_t
vioscsi_lun_reset(vioscsi_softc_t *sc, uint8_t target, uint16_t lun)
{
	return (vioscsi_tmf(sc, VIRTIO_SCSI_T_TMF_LOGICAL_UNIT_RESET,
	    target, lun, NULL));
}

static boolean_t
vioscsi_target_reset(vioscsi_softc_t *sc, uint8_t target)
{
	return (vioscsi_tmf(sc, VIRTIO_SCSI_T_TMF_I_T_NEXUS_RESET,
	    target, 0, NULL));
}

static boolean_t
vioscsi_req_abort(vioscsi_softc_t *sc, vioscsi_request_t *req)
{
	return (vioscsi_tmf(sc, VIRTIO_SCSI_T_TMF_ABORT_TASK,
	    req->vr_target, req->vr_lun, req));
}

static void
vioscsi_dev_abort(vioscsi_dev_t *vd)
{
	vioscsi_request_t *req;
	list_t *l = &vd->vd_reqs;

	mutex_enter(&vd->vd_lock);
	for (req = list_head(l); req != NULL; req = list_next(l, req)) {
		(void) vioscsi_tmf(vd->vd_sc, VIRTIO_SCSI_T_TMF_ABORT_TASK,
		    req->vr_target, req->vr_lun, req);
	}
	mutex_exit(&vd->vd_lock);
}

static void
vioscsi_dev_timeout(void *arg)
{
	vioscsi_dev_t *vd = arg;
	vioscsi_softc_t *sc = vd->vd_sc;
	vioscsi_request_t *req;
	timeout_id_t tid;
	clock_t now;
	list_t *l;

	mutex_enter(&vd->vd_lock);
	if ((tid = vd->vd_timeout) == 0) {
		/*
		 * We are shutting down, stop and do not reschedule.
		 */
		mutex_exit(&vd->vd_lock);
		return;
	}
	vd->vd_timeout = 0;

	now = ddi_get_lbolt();
	l = &vd->vd_reqs;

	for (req = list_head(l); req != NULL; req = list_next(l, req)) {
		/*
		 * The list is sorted by expiration time, so if we reach an
		 * item that hasn't expired yet, we're done.
		 */
		if (now < req->vr_expire) {
			break;
		}
		atomic_or_8(&req->vr_expired, 1);

		/*
		 * This command timed out, so send an abort.
		 */
		dev_err(sc->vs_dip, CE_WARN, "cmd timed out (%ds)",
		    (int)req->vr_time);
		(void) vioscsi_req_abort(sc, req);
	}

	if (!list_is_empty(l)) {
		/*
		 * Check again in a second.
		 * If these wake ups are too expensive, we could
		 * calculate other timeouts, but that would require
		 * doing untimeout if we want to wake up earlier.
		 * This is probably cheaper, and certainly simpler.
		 */
		vd->vd_timeout = timeout(vioscsi_dev_timeout, vd, vioscsi_hz);
	}
	mutex_exit(&vd->vd_lock);
}

static void
vioscsi_poll(vioscsi_softc_t *sc, vioscsi_request_t *req)
{
	if (vioscsi_poll_until(sc, req, vioscsi_cmd_handler, req->vr_time)) {
		return;
	}

	/*
	 * Try a "gentle" task abort -- timeouts may be quasi-normal for some
	 * types of requests and devices.
	 */
	if (vioscsi_req_abort(sc, req) &&
	    vioscsi_poll_until(sc, req, vioscsi_cmd_handler, 1)) {
		return;
	}

	/*
	 * A little more forceful with a lun reset:
	 */
	if (vioscsi_lun_reset(sc, req->vr_target, req->vr_lun) &&
	    vioscsi_poll_until(sc, req, vioscsi_cmd_handler, 1)) {
		return;
	}

	/*
	 * If all else fails, reset the target, and keep trying.
	 * This can wind up blocking forever, but if it does it means we are in
	 * a very bad situation (and the virtio device is busted).
	 * We may also be leaking request structures at this point, but only at
	 * the maximum rate of one per minute.
	 */
	for (;;) {
		dev_err(sc->vs_dip, CE_WARN, "request stuck, resetting target");
		(void) vioscsi_target_reset(sc, req->vr_target);
		if (vioscsi_poll_until(sc, req, vioscsi_cmd_handler, 60)) {
			return;
		}
	}
}

static void
vioscsi_start(vioscsi_softc_t *sc, vioscsi_request_t *req)
{
	vioscsi_cmd_req_t *cmd = &req->vr_req->cmd;

	req->vr_done = 0;
	req->vr_expired = 0;
	cmd->lun[0] = 1;
	cmd->lun[1] = req->vr_target;
	cmd->lun[2] = 0x40 | ((req->vr_lun >> 8) & 0xff);
	cmd->lun[3] = req->vr_lun & 0xff;
	cmd->lun[4] = 0;
	cmd->lun[5] = 0;
	cmd->lun[6] = 0;
	cmd->lun[7] = 0;
	cmd->tag = (uint64_t)req;
	cmd->prio = 0;
	cmd->crn = 0;
	cmd->task_attr = req->vr_task_attr;

	/*
	 * Make sure the device can see our CDB data:
	 */
	virtio_dma_sync(req->vr_dma, DDI_DMA_SYNC_FORDEV);

	/*
	 * Determine whether we expect to poll before submitting (because we
	 * cannot touch the request after submission if we are not polling).
	 */
	if (req->vr_poll) {
		/*
		 * Push chain into the queue:
		 */
		virtio_chain_submit(req->vr_vic, B_TRUE);

		/*
		 * NB: Interrupts may be enabled, or might not be.  It is fine
		 * either way.
		 */
		vioscsi_poll(sc, req);
	} else {
		/*
		 * Push chain into the queue:
		 */
		virtio_chain_submit(req->vr_vic, B_TRUE);
	}
}

static int
vioscsi_tran_start(struct scsi_address *ap, struct scsi_pkt *pkt)
{
	struct scsi_device *sd = scsi_address_device(ap);
	vioscsi_dev_t *vd = scsi_device_hba_private_get(sd);
	vioscsi_request_t *req = pkt->pkt_ha_private;
	virtio_chain_t *vic = req->vr_vic;
	vioscsi_cmd_req_t *cmd = &req->vr_req->cmd;
	vioscsi_cmd_res_t *res = &req->vr_res->cmd;

	if (pkt->pkt_cdbp == NULL) {
		return (TRAN_BADPKT);
	}

	bzero(cmd, sizeof (*cmd));
	bcopy(pkt->pkt_cdbp, cmd->cdb, pkt->pkt_cdblen);

	/*
	 * Default expiration is 10 seconds, clip at an hour.
	 * (order of operations here is to avoid wrapping, if run in a 32-bit
	 * kernel)
	 */
	req->vr_time = min(pkt->pkt_time ? pkt->pkt_time : 10, 3600);
	req->vr_dev = vd;
	req->vr_poll = ((pkt->pkt_flags & FLAG_NOINTR) != 0);
	req->vr_target = vd->vd_target;
	req->vr_lun = vd->vd_lun;
	req->vr_start = ddi_get_lbolt();
	req->vr_expire = req->vr_start + req->vr_time * vioscsi_hz;

	/*
	 * Configure task queuing behavior:
	 */
	if (pkt->pkt_flags & (FLAG_HTAG|FLAG_HEAD)) {
		req->vr_task_attr = VIRTIO_SCSI_S_HEAD;
	} else if (pkt->pkt_flags & FLAG_OTAG) {
		req->vr_task_attr = VIRTIO_SCSI_S_ORDERED;
	} else if (pkt->pkt_flags & FLAG_SENSING) {
		req->vr_task_attr = VIRTIO_SCSI_S_ACA;
	} else { /* FLAG_STAG is also our default */
		req->vr_task_attr = VIRTIO_SCSI_S_SIMPLE;
	}

	/*
	 * Make sure we start with a clear chain:
	 */
	virtio_chain_clear(vic);

	/*
	 * The KVM SCSI emulation requires that all outgoing buffers are added
	 * first with the request header being the first entry.  After the
	 * outgoing have been added then the incoming buffers with the response
	 * buffer being the first of the incoming.  This requirement is
	 * independent of using chained ring entries or one ring entry with
	 * indirect buffers.
	 */

	/*
	 * Add request header:
	 */
	if (virtio_chain_append(vic, req->vr_req_pa, sizeof (*cmd),
	    VIRTIO_DIR_DEVICE_READS) != DDI_SUCCESS)
		goto busy;

	/*
	 * Add write buffers:
	 */
	if (pkt->pkt_dma_flags & DDI_DMA_WRITE) {
		for (int i = 0; i < pkt->pkt_numcookies; i++) {
			if (virtio_chain_append(vic,
			    pkt->pkt_cookies[i].dmac_laddress,
			    pkt->pkt_cookies[i].dmac_size,
			    VIRTIO_DIR_DEVICE_READS) != DDI_SUCCESS)
				goto busy;
		}
	}

	/*
	 * Add response header:
	 */
	if (virtio_chain_append(vic, req->vr_res_pa, sizeof (*res),
	    VIRTIO_DIR_DEVICE_WRITES) != DDI_SUCCESS)
		goto busy;

	/*
	 * Add read buffers:
	 */
	if (pkt->pkt_dma_flags & DDI_DMA_READ) {
		for (int i = 0; i < pkt->pkt_numcookies; i++) {
			if (virtio_chain_append(vic,
			    pkt->pkt_cookies[i].dmac_laddress,
			    pkt->pkt_cookies[i].dmac_size,
			    VIRTIO_DIR_DEVICE_WRITES) != DDI_SUCCESS)
				goto busy;
		}
	}

	/*
	 * Check for queue depth, and add to the timeout list:
	 */
	mutex_enter(&vd->vd_lock);
	if (vd->vd_num_cmd >= vd->vd_max_cmd) {
		mutex_exit(&vd->vd_lock);
		goto busy;
	}

	vd->vd_num_cmd++;

	if (!req->vr_poll) {
		/*
		 * Add the request to the timeout list.
		 *
		 * In order to minimize the work done during timeout handling,
		 * we keep requests sorted.  This assumes that requests mostly
		 * have the same timeout, and requests with long timeouts are
		 * infrequent.
		 */
		list_t *l = &vd->vd_reqs;
		vioscsi_request_t *r;

		for (r = list_tail(l); r != NULL; r = list_prev(l, r)) {
			/*
			 * Avoids wrapping lbolt:
			 */
			if ((req->vr_expire - r->vr_expire) >= 0) {
				list_insert_after(l, r, req);
				break;
			}
		}
		if (r == NULL) {
			/*
			 * List empty, or this one expires before others:
			 */
			list_insert_head(l, req);
		}
		if (vd->vd_timeout == 0) {
			vd->vd_timeout = timeout(vioscsi_dev_timeout, vd,
			    vioscsi_hz);
		}
	}

	mutex_exit(&vd->vd_lock);

	vioscsi_start(vd->vd_sc, req);
	return (TRAN_ACCEPT);

busy:
	virtio_chain_clear(vic);
	return (TRAN_BUSY);
}

static int
vioscsi_tran_abort(struct scsi_address *ap, struct scsi_pkt *pkt)
{
	struct scsi_device *sd;
	vioscsi_dev_t *vd;
	vioscsi_request_t *req;

	if ((ap == NULL) ||
	    ((sd = scsi_address_device(ap)) == NULL) ||
	    ((vd = scsi_device_hba_private_get(sd)) == NULL)) {
		return (0);
	}
	if (pkt == NULL) {
		/*
		 * Abort all requests for the LUN.
		 */
		vioscsi_dev_abort(vd);
		return (1);
	}
	if ((req = pkt->pkt_ha_private) != NULL) {
		return (vioscsi_req_abort(vd->vd_sc, req) ? 1 : 0);
	}

	return (0);
}

static void
vioscsi_req_fini(vioscsi_request_t *req)
{
	if (req->vr_dma != NULL) {
		virtio_dma_free(req->vr_dma);
		req->vr_dma = NULL;
	}
	if (req->vr_vic != NULL) {
		virtio_chain_free(req->vr_vic);
		req->vr_vic = NULL;
	}
}

static int
vioscsi_req_init(vioscsi_softc_t *sc, vioscsi_request_t *req,
    virtio_queue_t *vq, int sleep)
{
	uint64_t pa;

	bzero(req, sizeof (*req));
	list_link_init(&req->vr_node);
	req->vr_vq = vq;
	req->vr_dma = virtio_dma_alloc(sc->vs_virtio, sizeof (vioscsi_op_t),
	    &virtio_dma_attr, DDI_DMA_STREAMING | DDI_DMA_READ | DDI_DMA_WRITE,
	    sleep);
	req->vr_vic = virtio_chain_alloc(vq, sleep);
	if ((req->vr_dma == NULL) || (req->vr_vic == NULL)) {
		return (-1);
	}
	virtio_chain_data_set(req->vr_vic, req);
	req->vr_req = virtio_dma_va(req->vr_dma, VIOSCSI_REQ_OFFSET);
	req->vr_res = virtio_dma_va(req->vr_dma, VIOSCSI_RES_OFFSET);
	pa = virtio_dma_cookie_pa(req->vr_dma, 0);
	req->vr_req_pa = pa + VIOSCSI_REQ_OFFSET;
	req->vr_res_pa = pa + VIOSCSI_RES_OFFSET;
	return (0);
}

static void
vioscsi_tran_pkt_destructor(struct scsi_pkt *pkt, scsi_hba_tran_t *tran)
{
	vioscsi_request_t *req = pkt->pkt_ha_private;

	vioscsi_req_fini(req);
}

static int
vioscsi_tran_pkt_constructor(struct scsi_pkt *pkt, scsi_hba_tran_t *tran,
    int sleep)
{
	vioscsi_softc_t *sc = tran->tran_hba_private;
	vioscsi_request_t *req = pkt->pkt_ha_private;

	if (vioscsi_req_init(sc, req, sc->vs_cmd_vq, sleep) != 0) {
		vioscsi_req_fini(req);
		return (-1);
	}
	req->vr_pkt = pkt;
	return (0);
}

static int
vioscsi_tran_setup_pkt(struct scsi_pkt *pkt, int (*cb)(caddr_t), caddr_t arg)
{
	if ((pkt->pkt_dma_flags & DDI_DMA_RDWR) == DDI_DMA_RDWR) {
		/*
		 * We can do read, or write, but not both.
		 */
		return (-1);
	}

	return (0);
}

static void
vioscsi_tran_teardown_pkt(struct scsi_pkt *pkt)
{
	vioscsi_request_t *req = pkt->pkt_ha_private;
	virtio_chain_t *vic = req->vr_vic;

	virtio_chain_clear(vic);
}

static int
vioscsi_tran_getcap(struct scsi_address *ap, char *cap, int whom)
{
	int rval = 0;
	vioscsi_softc_t *sc = ap->a_hba_tran->tran_hba_private;

	if (cap == NULL)
		return (-1);

	switch (scsi_hba_lookup_capstr(cap)) {
	case SCSI_CAP_CDB_LEN:
		rval = sc->vs_cdb_size;
		break;

	case SCSI_CAP_DMA_MAX:
		rval = sc->vs_tran->tran_dma_attr.dma_attr_maxxfer;
		break;

	case SCSI_CAP_INTERCONNECT_TYPE:
		rval = sc->vs_tran->tran_interconnect_type;
		break;

	case SCSI_CAP_ARQ:
	case SCSI_CAP_LUN_RESET:
	case SCSI_CAP_TAGGED_QING:
	case SCSI_CAP_UNTAGGED_QING:
		rval = 1;
		break;

	default:
		rval = -1;
	}
	return (rval);
}

static int
vioscsi_tran_setcap(struct scsi_address *ap, char *cap, int value, int whom)
{
	int rval = 1;

	if (cap == NULL || whom == 0) {
		return (-1);
	}

	switch (scsi_hba_lookup_capstr(cap)) {
	default:
		rval = 1;
	}
	return (rval);
}

static int
vioscsi_tran_reset(struct scsi_address *ap, int level)
{
	struct scsi_device *sd;
	vioscsi_dev_t *vd;

	if ((ap == NULL) ||
	    ((sd = scsi_address_device(ap)) == NULL) ||
	    ((vd = scsi_device_hba_private_get(sd)) == NULL)) {
		return (0);
	}

	switch (level) {
	case RESET_LUN:
		if (vioscsi_lun_reset(vd->vd_sc, vd->vd_target, vd->vd_lun)) {
			return (1);
		}
		break;
	case RESET_TARGET:
		if (vioscsi_target_reset(vd->vd_sc, vd->vd_target)) {
			return (1);
		}
		break;
	case RESET_ALL:
	default:
		break;
	}
	return (0);
}

static boolean_t
vioscsi_parse_unit_address(const char *ua, int *tgt, int *lun)
{
	long num;
	char *end;

	if ((ddi_strtol(ua, &end, 16, &num) != 0) ||
	    ((*end != ',') && (*end != 0))) {
		return (B_FALSE);
	}
	*tgt = (int)num;
	if (*end == 0) {
		*lun = 0;
		return (B_TRUE);
	}
	end++; /* skip comma */
	if ((ddi_strtol(end, &end, 16, &num) != 0) || (*end != 0)) {
		return (B_FALSE);
	}
	*lun = (int)num;
	return (B_TRUE);
}

uint_t
vioscsi_ctl_handler(caddr_t arg1, caddr_t arg2)
{
	vioscsi_softc_t *sc = (vioscsi_softc_t *)arg1;
	virtio_chain_t *vic;

	while ((vic = virtio_queue_poll(sc->vs_ctl_vq)) != NULL) {
		vioscsi_request_t *req;

		if ((req = virtio_chain_data(vic)) == NULL) {
			dev_err(sc->vs_dip, CE_WARN, "missing ctl chain data");
			continue;
		}
		atomic_or_8(&req->vr_done, 1);
	}
	return (DDI_INTR_CLAIMED);
}

uint_t
vioscsi_evt_handler(caddr_t arg1, caddr_t arg2)
{
	vioscsi_softc_t *sc = (vioscsi_softc_t *)arg1;
	virtio_chain_t *vic;
	boolean_t missed = B_FALSE;

	while ((vic = virtio_queue_poll(sc->vs_evt_vq)) != NULL) {
		vioscsi_evt_t *evt;
		vioscsi_event_t *ve;
		uint8_t target;

		if ((ve = virtio_chain_data(vic)) == NULL) {
			/*
			 * This should never occur, it's a bug if it does.
			 */
			dev_err(sc->vs_dip, CE_WARN, "missing evt chain data");
			continue;
		}
		evt = ve->ve_evt;

		virtio_dma_sync(ve->ve_dma, DDI_DMA_SYNC_FORKERNEL);

		target = evt->lun[1];
		switch (evt->event & 0x7FFFFFFF) {
		case VIRTIO_SCSI_T_TRANSPORT_RESET:
			switch (evt->reason) {
			case VIRTIO_SCSI_EVT_RESET_HARD:
				/*
				 * We could reset-notify, but this doesn't seem
				 * to get fired for targets initiated from
				 * host.
				 */
				break;
			case VIRTIO_SCSI_EVT_RESET_REMOVED:
			case VIRTIO_SCSI_EVT_RESET_RESCAN:
				/*
				 * We can treat these the same for the target,
				 * and not worry about the actual LUN id here.
				 */
				vioscsi_lun_changed(sc, target);
				break;
			default:
				/*
				 * Some other event we don't know about.
				 */
				break;
			}
			break;
		case VIRTIO_SCSI_T_NO_EVENT:
			/*
			 * If this happens, we missed some event(s).
			 */
			missed = B_TRUE;
			break;
		case VIRTIO_SCSI_T_ASYNC_NOTIFY:
			/*
			 * We don't register for these, so we don't expect
			 * them.
			 */
			break;
		}

		if (evt->event & VIRTIO_SCSI_T_EVENTS_MISSED) {
			missed = B_TRUE;
		}

		/*
		 * Resubmit the chain for the next event.
		 */
		virtio_chain_submit(vic, B_TRUE);
	}

	if (missed) {
		(void) ddi_taskq_dispatch(sc->vs_tq, vioscsi_discover, sc,
		    DDI_NOSLEEP);
	}

	return (DDI_INTR_CLAIMED);
}

uint_t
vioscsi_cmd_handler(caddr_t arg1, caddr_t arg2)
{
	vioscsi_softc_t *sc = (vioscsi_softc_t *)arg1;
	virtio_chain_t *vic;

	while ((vic = virtio_queue_poll(sc->vs_cmd_vq)) != NULL) {

		vioscsi_request_t *req;
		vioscsi_dev_t *vd;
		struct scsi_pkt *pkt;
		struct virtio_scsi_cmd_resp *res;

		virtio_chain_clear(vic);

		if ((req = virtio_chain_data(vic)) == NULL) {
			/*
			 * This should never occur, it's a bug if it does.
			 */
			dev_err(sc->vs_dip, CE_WARN, "missing cmd chain data");
			continue;
		}

		virtio_dma_sync(req->vr_dma, DDI_DMA_SYNC_FORKERNEL);
		res = &req->vr_res->cmd;
		pkt = req->vr_pkt;

		if (pkt == NULL) {
			/*
			 * This is an internal request (from discovery), and
			 * doesn't have an associated SCSI pkt structure.  In
			 * this case, the notification we've done is
			 * sufficient, and the submitter will examine the
			 * response field directly.
			 */
			if (req->vr_poll) {
				atomic_or_8(&req->vr_done, 1);
			}
			continue;
		}

		if ((vd = req->vr_dev) != NULL) {
			mutex_enter(&vd->vd_lock);
			vd->vd_num_cmd--;
			list_remove(&vd->vd_reqs, req);
			mutex_exit(&vd->vd_lock);
		}

		pkt->pkt_state = STATE_GOT_BUS;

		switch (res->response) {

		case VIRTIO_SCSI_S_OK:
			/*
			 * Request processed successfully, check SCSI status.
			 */
			pkt->pkt_scbp[0] = res->status;
			pkt->pkt_resid = 0;
			pkt->pkt_reason = CMD_CMPLT;
			pkt->pkt_state |= STATE_GOT_TARGET | STATE_SENT_CMD |
			    STATE_GOT_STATUS;
			if ((pkt->pkt_numcookies > 0) &&
			    (pkt->pkt_cookies[0].dmac_size > 0)) {
				pkt->pkt_state |= STATE_XFERRED_DATA;
			}

			/*
			 * For CHECK_CONDITION, fill out the ARQ details:
			 */
			if (res->status == STATUS_CHECK) {
				/*
				 * ARQ status and arq structure:
				 */
				pkt->pkt_state |= STATE_ARQ_DONE;
				pkt->pkt_scbp[1] = STATUS_GOOD;
				struct scsi_arq_status *ars =
				    (void *)pkt->pkt_scbp;
				ars->sts_rqpkt_reason = CMD_CMPLT;
				ars->sts_rqpkt_resid = 0;
				ars->sts_rqpkt_state =
				    STATE_GOT_BUS | STATE_GOT_TARGET |
				    STATE_GOT_STATUS | STATE_SENT_CMD |
				    STATE_XFERRED_DATA;
				bcopy(res->sense, &ars->sts_sensedata,
				    MIN(sizeof (ars->sts_sensedata),
				    MIN(sc->vs_sense_size, res->sense_len)));
			}
			break;

		case VIRTIO_SCSI_S_BAD_TARGET:
		case VIRTIO_SCSI_S_INCORRECT_LUN:
		case VIRTIO_SCSI_S_TRANSPORT_FAILURE:
		case VIRTIO_SCSI_S_TARGET_FAILURE:
			pkt->pkt_reason = CMD_DEV_GONE;
			break;

		case VIRTIO_SCSI_S_OVERRUN:
			dev_err(sc->vs_dip, CE_WARN, "OVERRUN");
			pkt->pkt_reason = CMD_DATA_OVR;
			pkt->pkt_state |= STATE_GOT_TARGET | STATE_SENT_CMD;
			break;

		case VIRTIO_SCSI_S_RESET:
			pkt->pkt_reason = CMD_RESET;
			pkt->pkt_state |= STATE_GOT_TARGET | STATE_SENT_CMD;
			pkt->pkt_statistics |= STAT_DEV_RESET;
			break;

		case VIRTIO_SCSI_S_ABORTED:
			pkt->pkt_state |= STATE_GOT_TARGET | STATE_SENT_CMD;
			if (req->vr_expired) {
				pkt->pkt_statistics |= STAT_TIMEOUT;
				pkt->pkt_reason = CMD_TIMEOUT;
			} else {
				pkt->pkt_reason = CMD_ABORTED;
				pkt->pkt_statistics |= STAT_ABORTED;
			}
			break;

		case VIRTIO_SCSI_S_BUSY:
			/*
			 * Busy, should have been caught at submission:
			 */
			pkt->pkt_reason = CMD_TRAN_ERR;
			break;

		case VIRTIO_SCSI_S_NEXUS_FAILURE:
		case VIRTIO_SCSI_S_FAILURE:
			pkt->pkt_reason = CMD_TRAN_ERR;
			break;

		default:
			dev_err(sc->vs_dip, CE_WARN, "Unknown response: 0x%x",
			    res->response);
			pkt->pkt_reason = CMD_TRAN_ERR;
			break;
		}

		if (!req->vr_poll) {
			scsi_hba_pkt_comp(pkt);
		} else {
			atomic_or_8(&req->vr_done, 1);
		}
	}
	return (DDI_INTR_CLAIMED);
}

static int
vioscsi_tran_tgt_init(dev_info_t *hdip, dev_info_t *tdip, scsi_hba_tran_t *tran,
    struct scsi_device *sd)
{
	const char *ua;
	vioscsi_softc_t *sc;
	int target;
	int lun;
	vioscsi_dev_t *vd;

	if (scsi_hba_iport_unit_address(hdip) == NULL) {
		return (DDI_FAILURE); /* only iport has targets */
	}
	if ((sc = tran->tran_hba_private) == NULL) {
		return (DDI_FAILURE);
	}

	if (((ua = scsi_device_unit_address(sd)) == NULL) ||
	    (!vioscsi_parse_unit_address(ua, &target, &lun))) {
		return (DDI_FAILURE);
	}

	vd = kmem_zalloc(sizeof (*vd), KM_SLEEP);
	list_create(&vd->vd_reqs, sizeof (vioscsi_request_t),
	    offsetof(vioscsi_request_t, vr_node));
	mutex_init(&vd->vd_lock, NULL, MUTEX_DRIVER,
	    virtio_intr_pri(sc->vs_virtio));

	vd->vd_target = (uint8_t)target;
	vd->vd_lun = (uint16_t)lun;
	vd->vd_sc = sc;
	vd->vd_sd = sd;
	vd->vd_max_cmd = sc->vs_cmd_per_lun;
	vd->vd_num_cmd = 0;

	scsi_device_hba_private_set(sd, vd);

	mutex_enter(&sc->vs_lock);
	list_insert_tail(&sc->vs_devs, vd);
	mutex_exit(&sc->vs_lock);

	return (DDI_SUCCESS);
}

static void
vioscsi_tran_tgt_free(dev_info_t *hdip, dev_info_t *tdip, scsi_hba_tran_t *tran,
    struct scsi_device *sd)
{
	vioscsi_dev_t *vd = scsi_device_hba_private_get(sd);
	vioscsi_softc_t *sc = vd->vd_sc;
	timeout_id_t tid;

	scsi_device_hba_private_set(sd, NULL);

	mutex_enter(&vd->vd_lock);
	tid = vd->vd_timeout;
	vd->vd_timeout = 0;
	mutex_exit(&vd->vd_lock);

	if (tid != 0) {
		(void) untimeout(tid);
	}

	mutex_enter(&sc->vs_lock);
	list_remove(&sc->vs_devs, vd);
	mutex_exit(&sc->vs_lock);

	list_destroy(&vd->vd_reqs);
	mutex_destroy(&vd->vd_lock);
	kmem_free(vd, sizeof (*vd));
}

/*
 * vioscsi_probe_target probes for existence of a valid target (LUN 0).
 * It utilizes the supplied request, and sends TEST UNIT READY.
 * (This command is used because it requires no data.)
 * It returns 1 if the target is found, 0 if not, and -1 on error.
 * It is expected additional LUNs will be discovered by the HBA framework using
 * REPORT LUNS on LUN 0.
 */
static int
vioscsi_probe_target(vioscsi_softc_t *sc, vioscsi_request_t *req,
    uint8_t target)
{
	struct virtio_scsi_cmd_req *cmd = &req->vr_req->cmd;
	struct virtio_scsi_cmd_resp *res = &req->vr_res->cmd;

	bzero(cmd, sizeof (*cmd));
	cmd->cdb[0] = SCMD_TEST_UNIT_READY;

	virtio_chain_clear(req->vr_vic);
	if (virtio_chain_append(req->vr_vic, req->vr_req_pa,
	    sizeof (*cmd), VIRTIO_DIR_DEVICE_READS) != DDI_SUCCESS) {
		return (-1);
	}
	if (virtio_chain_append(req->vr_vic, req->vr_res_pa,
	    sizeof (*res), VIRTIO_DIR_DEVICE_WRITES) != DDI_SUCCESS) {
		return (-1);
	}
	req->vr_poll = B_TRUE;
	req->vr_start = ddi_get_lbolt();
	req->vr_time = 10; /* seconds */
	req->vr_target = target;
	req->vr_lun = 0;
	req->vr_task_attr = VIRTIO_SCSI_S_HEAD;
	vioscsi_start(sc, req);
	switch (res->response) {
	case VIRTIO_SCSI_S_OK:
		return (1);
	case VIRTIO_SCSI_S_INCORRECT_LUN:
	case VIRTIO_SCSI_S_BAD_TARGET:
		return (0);
	default:
		return (-1);
	}
}

static void
vioscsi_rescan_luns(void *arg)
{
	vioscsi_softc_t		*sc = arg;
	vioscsi_dev_t		*vd;
	scsi_hba_tgtmap_t	*tm = sc->vs_tgtmap;
	list_t			*l;
	char			addr[16];

	l = &sc->vs_devs;
	mutex_enter(&sc->vs_lock);
	for (vd = list_head(l); vd != NULL; vd = list_next(l, vd)) {
		if (!vd->vd_rescan) {
			continue;
		}

		vd->vd_rescan = B_FALSE;
		(void) snprintf(addr, sizeof (addr), "%x", vd->vd_target);
		scsi_hba_tgtmap_scan_luns(tm, addr);
	}
	mutex_exit(&sc->vs_lock);
}

static void
vioscsi_lun_changed(vioscsi_softc_t *sc, uint8_t target)
{
	vioscsi_dev_t *vd;
	list_t *l = &sc->vs_devs;
	boolean_t found = B_FALSE;

	mutex_enter(&sc->vs_lock);
	for (vd = list_head(l); vd != NULL; vd = list_next(l, vd)) {
		if ((vd->vd_target == target) && (vd->vd_lun == 0)) {
			vd->vd_rescan = B_TRUE;
			found = B_TRUE;
			break;
		}
	}
	mutex_exit(&sc->vs_lock);

	if (found) {
		/*
		 * We have lun 0 already, so report luns changed:
		 */
		(void) ddi_taskq_dispatch(sc->vs_tq, vioscsi_rescan_luns,
		    sc, DDI_NOSLEEP);
	} else {
		/*
		 * We didn't find lun 0, so issue a new discovery:
		 */
		(void) ddi_taskq_dispatch(sc->vs_tq, vioscsi_discover,
		    sc, DDI_NOSLEEP);
	}
}

/*
 * vioscsi_discover is our task function for performing target and lun
 * discovery.  This is done using active SCSI probes.
 */
static void
vioscsi_discover(void *arg)
{
	vioscsi_softc_t *sc = arg;
	scsi_hba_tgtmap_t *tm = sc->vs_tgtmap;
	vioscsi_request_t req;
	uint32_t target;

	if (vioscsi_req_init(sc, &req, sc->vs_cmd_vq, KM_SLEEP) != 0) {
		vioscsi_req_fini(&req);
		return;
	}

	if (scsi_hba_tgtmap_set_begin(tm) != DDI_SUCCESS) {
		vioscsi_req_fini(&req);
		return;
	}
	for (target = 0;
	    target <= sc->vs_max_target && target < VIOSCSI_MAX_TARGET;
	    target++) {
		char ua[10];
		switch (vioscsi_probe_target(sc, &req, target)) {
		case 1:
			(void) snprintf(ua, sizeof (ua), "%x", target);
			if (scsi_hba_tgtmap_set_add(tm, SCSI_TGT_SCSI_DEVICE,
			    ua, NULL) != DDI_SUCCESS) {
				(void) scsi_hba_tgtmap_set_flush(tm);
				vioscsi_req_fini(&req);
				return;
			}
			break;
		case 0:
			continue;
		case -1:
			(void) scsi_hba_tgtmap_set_flush(tm);
			vioscsi_req_fini(&req);
			return;
		}
	}
	(void) scsi_hba_tgtmap_set_end(tm, 0);
	vioscsi_req_fini(&req);
}

static void
vioscsi_teardown(vioscsi_softc_t *sc, boolean_t failed)
{
	int instance = ddi_get_instance(sc->vs_dip);

	/*
	 * Free up the event resources:
	 */
	for (int i = 0; i < VIOSCSI_NUM_EVENTS; i++) {
		vioscsi_event_t *ve = &sc->vs_events[i];
		if (ve->ve_vic != NULL) {
			virtio_chain_free(ve->ve_vic);
		}
		if (ve->ve_dma != NULL) {
			virtio_dma_free(ve->ve_dma);
		}
	}

	if (sc->vs_virtio != NULL) {
		virtio_fini(sc->vs_virtio, failed);
	}

	if (sc->vs_tran != NULL) {
		scsi_hba_tran_free(sc->vs_tran);
	}
	if (sc->vs_tq != NULL) {
		ddi_taskq_destroy(sc->vs_tq);
	}
	if (sc->vs_intr_pri != NULL) {
		mutex_destroy(&sc->vs_lock);
	}
	ddi_soft_state_free(vioscsi_state, instance);
}

static int
vioscsi_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
{
	scsi_hba_tran_t *tran = NULL;
	vioscsi_softc_t *sc;
	virtio_t *vio;
	ddi_dma_attr_t attr;
	int instance;

	if (cmd != DDI_ATTACH) { /* no suspend/resume support */
		return (DDI_FAILURE);
	}

	if (scsi_hba_iport_unit_address(dip) != NULL) {
		return (vioscsi_iport_attach(dip));
	}

	instance = ddi_get_instance(dip);

	if (ddi_soft_state_zalloc(vioscsi_state, instance) != DDI_SUCCESS)
		return (DDI_FAILURE);

	sc = ddi_get_soft_state(vioscsi_state, instance);
	sc->vs_dip = dip;

	list_create(&sc->vs_devs, sizeof (vioscsi_dev_t),
	    offsetof(vioscsi_dev_t, vd_node));

	tran = scsi_hba_tran_alloc(dip, SCSI_HBA_CANSLEEP);
	sc->vs_tran = tran;

	tran->tran_hba_len = sizeof (vioscsi_request_t);
	tran->tran_hba_private = sc;

	/*
	 * We don't use WWN addressing, so advertise parallel.  The underlying
	 * device might still be using a different transport, even in a
	 * pass-through, but we cannot discriminate that at this layer.
	 */
	tran->tran_interconnect_type = INTERCONNECT_PARALLEL;

	tran->tran_start = vioscsi_tran_start;
	tran->tran_abort = vioscsi_tran_abort;
	tran->tran_reset = vioscsi_tran_reset;
	tran->tran_getcap = vioscsi_tran_getcap;
	tran->tran_setcap = vioscsi_tran_setcap;

	tran->tran_tgt_init = vioscsi_tran_tgt_init;
	tran->tran_tgt_free = vioscsi_tran_tgt_free;

	tran->tran_setup_pkt = vioscsi_tran_setup_pkt;
	tran->tran_teardown_pkt = vioscsi_tran_teardown_pkt;
	tran->tran_pkt_constructor = vioscsi_tran_pkt_constructor;
	tran->tran_pkt_destructor = vioscsi_tran_pkt_destructor;

	/*
	 * We need to determine some device settings here, so we initialize the
	 * virtio in order to access those values.  The rest of the setup we do
	 * in the iport attach.  Note that this driver cannot support
	 * reattaching a child iport once it is removed -- the entire driver
	 * will need to be reset for that.
	 */
	vio = virtio_init(dip);
	if ((sc->vs_virtio = vio) == NULL) {
		dev_err(dip, CE_WARN, "failed to init virtio");
		vioscsi_teardown(sc, B_TRUE);
		return (DDI_FAILURE);
	}
	if (!virtio_init_features(vio, VIOSCSI_WANTED_FEATURES, B_TRUE)) {
		vioscsi_teardown(sc, B_TRUE);
		return (DDI_FAILURE);
	}

	if (!virtio_features_present(vio, VIOSCSI_NEEDED_FEATURES)) {
		dev_err(dip, CE_WARN, "need features %b, have %b",
		    VIOSCSI_NEEDED_FEATURES, VIOSCSI_FEATURE_FORMAT,
		    virtio_features(vio), VIOSCSI_FEATURE_FORMAT);
		vioscsi_teardown(sc, B_TRUE);
		return (DDI_FAILURE);
	}
	/*
	 * Get virtio parameters:
	 */
	sc->vs_num_queues = virtio_dev_get32(vio, VIRTIO_SCSI_CFG_NUM_QUEUES);
	sc->vs_seg_max = virtio_dev_get32(vio, VIRTIO_SCSI_CFG_SEG_MAX);
	sc->vs_max_sectors = virtio_dev_get32(vio, VIRTIO_SCSI_CFG_MAX_SECTORS);
	sc->vs_cmd_per_lun = virtio_dev_get32(vio, VIRTIO_SCSI_CFG_CMD_PER_LUN);
	sc->vs_evi_size = virtio_dev_get32(vio, VIRTIO_SCSI_CFG_EVI_SIZE);
	sc->vs_sense_size = virtio_dev_get32(vio, VIRTIO_SCSI_CFG_SENSE_SIZE);
	sc->vs_cdb_size = virtio_dev_get32(vio, VIRTIO_SCSI_CFG_CDB_SIZE);
	sc->vs_max_channel = virtio_dev_get16(vio, VIRTIO_SCSI_CFG_MAX_CHANNEL);
	sc->vs_max_target = virtio_dev_get16(vio, VIRTIO_SCSI_CFG_MAX_TARGET);
	sc->vs_max_lun = virtio_dev_get32(vio, VIRTIO_SCSI_CFG_MAX_LUN);

	/*
	 * Check virtio parameter sanity.
	 */
#define	VIOSCSI_CHECK(var, op, val)					\
	do {								\
		if (!(sc->vs_ ##var op val)) {				\
			dev_err(dip, CE_WARN,				\
			    "!device config error: "#var" = %lx "	\
			    "(want "#op" %lx)", (uint64_t)sc->vs_ ##var,\
			    (uint64_t)val);				\
			sc->vs_ ##var = val;				\
		}							\
	} while (0)

	/*
	 * max_sectors determines the maxxfer size of our DMA attributes.
	 * This needs to be at least PAGESIZE. The virtio spec doesn't spell
	 * it out explicitly, but max_sectors is in units of 512 bytes.
	 */
	VIOSCSI_CHECK(max_sectors, >=, 1 << (PAGESHIFT - 9));

	/*
	 * seg_max is the maximum number of data segments (aka SGL entries)
	 * the device can handle in a single command. (If we supported INOUT,
	 * each command can have seg_max input segments and seg_max output
	 * segments, but we don't.)
	 *
	 * We cap this at VIOSCSI_MAX_SEGS just in case.
	 *
	 * This does not include the 2 segments needed for input and output
	 * headers, hence we add 2 in the call to virtio_queue_alloc() below.
	 */
	VIOSCSI_CHECK(seg_max, >=, VIOSCSI_MIN_SEGS);
	VIOSCSI_CHECK(seg_max, <=, VIOSCSI_MAX_SEGS);
	VIOSCSI_CHECK(cmd_per_lun, >=, 1);
	VIOSCSI_CHECK(evi_size, <=, sizeof (vioscsi_evt_t));
	VIOSCSI_CHECK(cdb_size, ==, VIRTIO_SCSI_CDB_SIZE);
	VIOSCSI_CHECK(sense_size, ==, VIRTIO_SCSI_SENSE_SIZE);
	VIOSCSI_CHECK(max_channel, ==, 0);
	VIOSCSI_CHECK(max_target, <=, VIOSCSI_MAX_TARGET - 1);
	VIOSCSI_CHECK(max_lun, <=, VIOSCSI_MAX_LUN - 1);
#undef	VIOSCSI_CHECK

	/*
	 * Allocate queues
	 */
	sc->vs_ctl_vq = virtio_queue_alloc(vio, 0, "ctl",
	    vioscsi_ctl_handler, sc, B_FALSE, sc->vs_seg_max + 2);
	sc->vs_evt_vq = virtio_queue_alloc(vio, 1, "evt",
	    vioscsi_evt_handler, sc, B_FALSE, sc->vs_seg_max + 2);
	sc->vs_cmd_vq = virtio_queue_alloc(vio, 2, "cmd",
	    vioscsi_cmd_handler, sc, B_FALSE, sc->vs_seg_max + 2);

	if ((sc->vs_ctl_vq == NULL) || (sc->vs_evt_vq == NULL) ||
	    (sc->vs_cmd_vq == NULL)) {
		dev_err(dip, CE_WARN, "failed allocating queue(s)");
		vioscsi_teardown(sc, B_TRUE);
		return (DDI_FAILURE);
	}

	if (virtio_init_complete(vio, VIRTIO_ANY_INTR_TYPE) != DDI_SUCCESS) {
		dev_err(dip, CE_WARN, "virtio_init_complete failed");
		vioscsi_teardown(sc, B_TRUE);
		return (DDI_FAILURE);
	}

	/*
	 * We cannot initialize this mutex before virtio_init_complete:
	 */
	sc->vs_intr_pri = virtio_intr_pri(vio);
	mutex_init(&sc->vs_lock, NULL, MUTEX_DRIVER, sc->vs_intr_pri);

	/*
	 * Allocate events, but do not submit yet:
	 */
	for (int i = 0; i < VIOSCSI_NUM_EVENTS; i++) {
		vioscsi_event_t *ve = &sc->vs_events[i];
		ve->ve_vic = virtio_chain_alloc(sc->vs_evt_vq, KM_SLEEP);
		ve->ve_dma = virtio_dma_alloc(sc->vs_virtio,
		    sizeof (vioscsi_evt_t), &virtio_dma_attr,
		    DDI_DMA_STREAMING | DDI_DMA_READ, KM_SLEEP);
		if ((ve->ve_vic == NULL) || (ve->ve_dma == NULL)) {
			vioscsi_teardown(sc, B_TRUE);
			return (DDI_FAILURE);
		}
		if (virtio_chain_append(ve->ve_vic,
		    virtio_dma_cookie_pa(ve->ve_dma, 0), sizeof (*ve->ve_evt),
		    VIRTIO_DIR_DEVICE_WRITES) != DDI_SUCCESS) {
			vioscsi_teardown(sc, B_TRUE);
			return (DDI_FAILURE);
		}
		ve->ve_evt = virtio_dma_va(ve->ve_dma, 0);
		virtio_chain_data_set(ve->ve_vic, ve);
	}

	sc->vs_tq = ddi_taskq_create(dip, "task", 1, TASKQ_DEFAULTPRI, 0);
	if (sc->vs_tq == NULL) {
		dev_err(dip, CE_WARN, "failed to create taskq");
		vioscsi_teardown(sc, B_TRUE);
		return (DDI_FAILURE);
	}

	/* Adjust DMA limits. */
	attr = virtio_dma_attr;
	attr.dma_attr_sgllen = sc->vs_seg_max;
	attr.dma_attr_maxxfer = (uint64_t)sc->vs_max_sectors << 9;

	if (scsi_hba_attach_setup(dip, &attr, tran,
	    SCSI_HBA_ADDR_COMPLEX | SCSI_HBA_HBA |
	    SCSI_HBA_TRAN_CDB | SCSI_HBA_TRAN_SCB) !=
	    DDI_SUCCESS) {
		vioscsi_teardown(sc, B_TRUE);
		return (DDI_FAILURE);
	}

	if (scsi_hba_iport_register(dip, "iport0") != DDI_SUCCESS) {
		vioscsi_teardown(sc, B_TRUE);
		return (DDI_FAILURE);
	}

	ddi_report_dev(dip);

	return (DDI_SUCCESS);
}

static void
vioscsi_iport_teardown(vioscsi_softc_t *sc)
{
	/*
	 * Stop the taskq -- ensures we don't try to access resources from a
	 * task while we are tearing down.
	 */
	ddi_taskq_suspend(sc->vs_tq);
	ddi_taskq_wait(sc->vs_tq);

	/*
	 * Shutdown all interrupts and device transfers:
	 */
	virtio_interrupts_disable(sc->vs_virtio);
	virtio_shutdown(sc->vs_virtio);

	/*
	 * Common resources:
	 */
	if (sc->vs_tgtmap != NULL) {
		scsi_hba_tgtmap_destroy(sc->vs_tgtmap);
		sc->vs_tgtmap = NULL;
	}
}

/*
 * vioscsi_iport_attach implements the attach of the iport.  We do the final
 * set up of interrupts, and posting of event buffers here, as we do not want
 * any activity unless the iport is attached.  This matches detach, and makes
 * teardown safer.
 */
static int
vioscsi_iport_attach(dev_info_t *dip)
{
	const char *ua = scsi_hba_iport_unit_address(dip);
	scsi_hba_tran_t *tran;
	vioscsi_softc_t *sc;

	/*
	 * We only support a single iport -- all disks are virtual and all
	 * disks use target/lun addresses.
	 */
	if ((ua == NULL) || (strcmp(ua, "iport0") != 0)) {
		return (DDI_FAILURE);
	}

	/*
	 * Get our parent's tran, and look up the sc from that:
	 */
	tran = ddi_get_driver_private(ddi_get_parent(dip));
	if ((tran == NULL) ||
	    ((sc = tran->tran_hba_private) == NULL)) {
		return (DDI_FAILURE);
	}

	/*
	 * Save a copy of the soft state in our tran private area.
	 * (The framework clears this after cloning from parent.)
	 */
	tran = ddi_get_driver_private(dip);
	tran->tran_hba_private = sc;

	/*
	 * We don't want interrupts on the control queue -- strictly polled
	 * (however if this handler is called from an interrupt, it should
	 * still be absolutely fine).
	 */
	virtio_queue_no_interrupt(sc->vs_ctl_vq, B_TRUE);

	if (scsi_hba_tgtmap_create(dip, SCSI_TM_FULLSET, MICROSEC,
	    2 * MICROSEC, sc, NULL, NULL, &sc->vs_tgtmap) != DDI_SUCCESS) {
		vioscsi_iport_teardown(sc);
		return (DDI_FAILURE);
	}

	/*
	 * Post events:
	 */
	for (int i = 0; i < VIOSCSI_NUM_EVENTS; i++) {
		virtio_chain_submit(sc->vs_events[i].ve_vic, B_FALSE);
	}
	virtio_queue_flush(sc->vs_evt_vq);

	/*
	 * Start interrupts going now:
	 */
	if (virtio_interrupts_enable(sc->vs_virtio) != DDI_SUCCESS) {
		vioscsi_iport_teardown(sc);
		return (DDI_FAILURE);
	}

	/*
	 * Start a discovery:
	 */
	(void) ddi_taskq_dispatch(sc->vs_tq, vioscsi_discover, sc, DDI_SLEEP);

	return (DDI_SUCCESS);
}

static int
vioscsi_quiesce(dev_info_t *dip)
{
	vioscsi_softc_t *sc;
	scsi_hba_tran_t *tran;

	if (((tran = ddi_get_driver_private(dip)) == NULL) ||
	    ((sc = tran->tran_hba_private) == NULL)) {
		return (DDI_FAILURE);
	}
	if (sc->vs_virtio == NULL) {
		return (DDI_SUCCESS); /* not initialized yet */
	}

	return (virtio_quiesce(sc->vs_virtio));
}

/*
 * vioscsi_iport_detach is used to perform the detach of the iport.  It
 * disables interrupts and the device, but does not free resources, other than
 * the target map.  Note that due to lack of a way to start virtio after
 * virtio_shutdown(), it is not possible to reattach the iport after this is
 * called, unless the underlying HBA is also detached and then re-attached.
 */
static int
vioscsi_iport_detach(dev_info_t *dip)
{
	const char *ua = scsi_hba_iport_unit_address(dip);
	vioscsi_softc_t *sc;
	scsi_hba_tran_t *tran;

	if ((ua == NULL) || (strcmp(ua, "iport0") != 0)) {
		return (DDI_FAILURE);
	}

	if (((tran = ddi_get_driver_private(dip)) == NULL) ||
	    ((sc = tran->tran_hba_private) == NULL)) {
		return (DDI_FAILURE);
	}

	mutex_enter(&sc->vs_lock);
	if (!list_is_empty(&sc->vs_devs)) {
		/*
		 * Cannot detach while we have target children.
		 */
		mutex_exit(&sc->vs_lock);
		return (DDI_FAILURE);
	}

	vioscsi_iport_teardown(sc);

	return (DDI_SUCCESS);
}

static int
vioscsi_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
{
	vioscsi_softc_t *sc;
	scsi_hba_tran_t *tran;

	if (cmd != DDI_DETACH)  {
		return (DDI_FAILURE);
	}

	if (scsi_hba_iport_unit_address(dip) != NULL) {
		return (vioscsi_iport_detach(dip));
	}

	if (((tran = ddi_get_driver_private(dip)) == NULL) ||
	    ((sc = tran->tran_hba_private) == NULL)) {
		return (DDI_FAILURE);
	}

	if (scsi_hba_detach(dip) != DDI_SUCCESS) {
		return (DDI_FAILURE);
	}
	vioscsi_teardown(sc, B_FALSE);

	return (DDI_SUCCESS);
}

static struct dev_ops vioscsi_dev_ops = {
	.devo_rev =		DEVO_REV,
	.devo_refcnt =		0,
	.devo_getinfo =		nodev,
	.devo_identify =	nulldev,
	.devo_probe =		nulldev,
	.devo_attach =		vioscsi_attach,
	.devo_detach =		vioscsi_detach,
	.devo_reset =		nodev,
	.devo_cb_ops =		NULL,
	.devo_bus_ops =		NULL,
	.devo_power =		NULL,
	.devo_quiesce =		vioscsi_quiesce,
};

static struct modldrv modldrv = {
	.drv_modops =		&mod_driverops,
	.drv_linkinfo =		vioscsi_ident,
	.drv_dev_ops =		&vioscsi_dev_ops,
};

static struct modlinkage modlinkage = {
	.ml_rev =		MODREV_1,
	.ml_linkage =		{ &modldrv, NULL, },
};


int
_init(void)
{
	int err;

	/*
	 * Initialize this unconditionally:
	 */
	vioscsi_hz = drv_usectohz(1000000);

	err = ddi_soft_state_init(&vioscsi_state, sizeof (vioscsi_softc_t), 1);
	if (err != DDI_SUCCESS)
		return (err);

	if ((err = scsi_hba_init(&modlinkage)) != 0) {
		return (err);
	}

	if ((err = mod_install(&modlinkage)) != 0) {
		scsi_hba_fini(&modlinkage);
		return (err);
	}

	return (err);
}

int
_fini(void)
{
	int err;

	if ((err = mod_remove(&modlinkage)) != 0) {
		return (err);
	}

	scsi_hba_fini(&modlinkage);
	ddi_soft_state_fini(&vioscsi_state);

	return (DDI_SUCCESS);
}

int
_info(struct modinfo *modinfop)
{
	return (mod_info(&modlinkage, modinfop));
}