/* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2009 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "blk_common.h" /* blk interface status */ enum blk_if_state { /* * initial state */ BLK_IF_UNKNOWN = 0, /* * frontend xenbus state changed to XenbusStateConnected, * we finally connect */ BLK_IF_CONNECTED, /* * frontend xenbus state changed to XenbusStateClosed, * interface disconnected */ BLK_IF_DISCONNECTED }; /* backend device status */ enum blk_be_state { /* initial state */ BLK_BE_UNKNOWN = 0, /* backend device is ready (hotplug script finishes successfully) */ BLK_BE_READY }; /* frontend status */ enum blk_fe_state { /* initial state */ BLK_FE_UNKNOWN = 0, /* * frontend's xenbus state has changed to * XenbusStateInitialised, is ready for connecting */ BLK_FE_READY }; typedef struct blk_ring_state_s { kmutex_t rs_mutex; boolean_t rs_sleeping_on_ring; boolean_t rs_ring_up; kcondvar_t rs_cv; } blk_ring_state_t; /* Disk Statistics */ static char *blk_stats[] = { "rd_reqs", "wr_reqs", "br_reqs", "fl_reqs", "oo_reqs" }; typedef struct blk_stats_s { uint64_t bs_req_reads; uint64_t bs_req_writes; uint64_t bs_req_barriers; uint64_t bs_req_flushes; } blk_stats_t; struct blk_ring_s { kmutex_t ri_mutex; dev_info_t *ri_dip; kstat_t *ri_kstats; blk_stats_t ri_stats; blk_intr_t ri_intr; caddr_t ri_intr_arg; blk_ring_cb_t ri_ringup; caddr_t ri_ringup_arg; blk_ring_cb_t ri_ringdown; caddr_t ri_ringdown_arg; /* blk interface, backend, and frontend status */ enum blk_if_state ri_if_status; enum blk_be_state ri_be_status; enum blk_fe_state ri_fe_status; domid_t ri_fe; enum blkif_protocol ri_protocol; size_t ri_nentry; size_t ri_entrysize; xendev_ring_t *ri_ring; blk_ring_state_t ri_state; }; static void blk_oe_state_change(dev_info_t *dip, ddi_eventcookie_t id, void *arg, void *impl_data); static void blk_hp_state_change(dev_info_t *dip, ddi_eventcookie_t id, void *arg, void *impl_data); static int blk_check_state_transition(blk_ring_t ring, XenbusState oestate); static int blk_start_connect(blk_ring_t ring); static void blk_start_disconnect(blk_ring_t ring); static void blk_ring_close(blk_ring_t ring); static int blk_bindto_frontend(blk_ring_t ring); static void blk_unbindfrom_frontend(blk_ring_t ring); static uint_t blk_intr(caddr_t arg); static int blk_kstat_init(blk_ring_t ring); static void blk_kstat_fini(blk_ring_t ring); static int blk_kstat_update(kstat_t *ksp, int flag); static void blk_ring_request_32(blkif_request_t *dst, blkif_x86_32_request_t *src); static void blk_ring_request_64(blkif_request_t *dst, blkif_x86_64_request_t *src); static void blk_ring_response_32(blkif_x86_32_response_t *dst, blkif_response_t *src); static void blk_ring_response_64(blkif_x86_64_response_t *dst, blkif_response_t *src); /* * blk_ring_init() */ int blk_ring_init(blk_ringinit_args_t *args, blk_ring_t *ringp) { blk_ring_t ring; int e; ring = kmem_zalloc(sizeof (struct blk_ring_s), KM_SLEEP); mutex_init(&ring->ri_mutex, NULL, MUTEX_DRIVER, NULL); ring->ri_dip = args->ar_dip; ring->ri_intr = args->ar_intr; ring->ri_intr_arg = args->ar_intr_arg; ring->ri_ringup = args->ar_ringup; ring->ri_ringup_arg = args->ar_ringup_arg; ring->ri_ringdown = args->ar_ringdown; ring->ri_ringdown_arg = args->ar_ringdown_arg; ring->ri_if_status = BLK_IF_UNKNOWN; ring->ri_be_status = BLK_BE_UNKNOWN; ring->ri_fe_status = BLK_FE_UNKNOWN; ring->ri_state.rs_sleeping_on_ring = B_FALSE; ring->ri_state.rs_ring_up = B_FALSE; mutex_init(&ring->ri_state.rs_mutex, NULL, MUTEX_DRIVER, NULL); cv_init(&ring->ri_state.rs_cv, NULL, CV_DRIVER, NULL); e = blk_kstat_init(ring); if (e != DDI_SUCCESS) { goto ringinitfail_kstat; } /* Watch frontend and hotplug state change */ if (xvdi_add_event_handler(ring->ri_dip, XS_OE_STATE, blk_oe_state_change, ring) != DDI_SUCCESS) { goto ringinitfail_oestate; } if (xvdi_add_event_handler(ring->ri_dip, XS_HP_STATE, blk_hp_state_change, ring) != DDI_SUCCESS) { goto ringinitfail_hpstate; } /* * Kick-off hotplug script */ if (xvdi_post_event(ring->ri_dip, XEN_HP_ADD) != DDI_SUCCESS) { cmn_err(CE_WARN, "blk@%s: failed to start hotplug script", ddi_get_name_addr(ring->ri_dip)); goto ringinitfail_postevent; } /* * start waiting for hotplug event and otherend state event * mainly for debugging, frontend will not take any op seeing this */ (void) xvdi_switch_state(ring->ri_dip, XBT_NULL, XenbusStateInitWait); *ringp = ring; return (DDI_SUCCESS); ringinitfail_postevent: xvdi_remove_event_handler(ring->ri_dip, XS_HP_STATE); ringinitfail_hpstate: xvdi_remove_event_handler(ring->ri_dip, XS_OE_STATE); ringinitfail_oestate: blk_kstat_fini(ring); ringinitfail_kstat: cv_destroy(&ring->ri_state.rs_cv); mutex_destroy(&ring->ri_state.rs_mutex); mutex_destroy(&ring->ri_mutex); kmem_free(ring, sizeof (struct blk_ring_s)); return (DDI_FAILURE); } /* * blk_ring_fini() */ void blk_ring_fini(blk_ring_t *ringp) { blk_ring_t ring; ring = *ringp; mutex_enter(&ring->ri_mutex); if (ring->ri_if_status != BLK_IF_DISCONNECTED) { blk_ring_close(ring); } mutex_exit(&ring->ri_mutex); xvdi_remove_event_handler(ring->ri_dip, NULL); blk_kstat_fini(ring); cv_destroy(&ring->ri_state.rs_cv); mutex_destroy(&ring->ri_state.rs_mutex); mutex_destroy(&ring->ri_mutex); kmem_free(ring, sizeof (struct blk_ring_s)); *ringp = NULL; } /* * blk_kstat_init() */ static int blk_kstat_init(blk_ring_t ring) { int nstat = sizeof (blk_stats) / sizeof (blk_stats[0]); char **cp = blk_stats; kstat_named_t *knp; ring->ri_kstats = kstat_create(ddi_get_name(ring->ri_dip), ddi_get_instance(ring->ri_dip), "req_statistics", "block", KSTAT_TYPE_NAMED, nstat, 0); if (ring->ri_kstats == NULL) { return (DDI_FAILURE); } ring->ri_kstats->ks_private = ring; ring->ri_kstats->ks_update = blk_kstat_update; knp = ring->ri_kstats->ks_data; while (nstat > 0) { kstat_named_init(knp, *cp, KSTAT_DATA_UINT64); knp++; cp++; nstat--; } kstat_install(ring->ri_kstats); return (DDI_SUCCESS); } /* * blk_kstat_fini() */ static void blk_kstat_fini(blk_ring_t ring) { kstat_delete(ring->ri_kstats); } /* * blk_kstat_update() */ static int blk_kstat_update(kstat_t *ksp, int flag) { kstat_named_t *knp; blk_stats_t *stats; blk_ring_t ring; if (flag != KSTAT_READ) { return (EACCES); } ring = ksp->ks_private; stats = &ring->ri_stats; knp = ksp->ks_data; /* * Assignment order should match that of the names in * blk_stats. */ (knp++)->value.ui64 = stats->bs_req_reads; (knp++)->value.ui64 = stats->bs_req_writes; (knp++)->value.ui64 = stats->bs_req_barriers; (knp++)->value.ui64 = stats->bs_req_flushes; (knp++)->value.ui64 = 0; /* oo_req */ return (0); } /* * blk_oe_state_change() */ /*ARGSUSED*/ static void blk_oe_state_change(dev_info_t *dip, ddi_eventcookie_t id, void *arg, void *impl_data) { XenbusState new_state; blk_ring_t ring; ring = (blk_ring_t)arg; new_state = *(XenbusState *)impl_data; mutex_enter(&ring->ri_mutex); if (blk_check_state_transition(ring, new_state) == DDI_FAILURE) { mutex_exit(&ring->ri_mutex); return; } switch (new_state) { case XenbusStateInitialised: ASSERT(ring->ri_if_status == BLK_IF_UNKNOWN); /* frontend is ready for connecting */ ring->ri_fe_status = BLK_FE_READY; if (ring->ri_be_status == BLK_BE_READY) { mutex_exit(&ring->ri_mutex); if (blk_start_connect(ring) != DDI_SUCCESS) (void) blk_start_disconnect(ring); mutex_enter(&ring->ri_mutex); } break; case XenbusStateClosing: (void) xvdi_switch_state(dip, XBT_NULL, XenbusStateClosing); break; case XenbusStateClosed: /* clean up */ (void) xvdi_post_event(ring->ri_dip, XEN_HP_REMOVE); if (ring->ri_ringdown != NULL) { (*(ring->ri_ringdown))(ring->ri_ringdown_arg); } blk_ring_close(ring); /* reset state in case of reconnect */ ring->ri_if_status = BLK_IF_UNKNOWN; ring->ri_be_status = BLK_BE_UNKNOWN; ring->ri_fe_status = BLK_FE_UNKNOWN; ring->ri_state.rs_sleeping_on_ring = B_FALSE; ring->ri_state.rs_ring_up = B_FALSE; break; default: ASSERT(0); } mutex_exit(&ring->ri_mutex); } /* * blk_hp_state_change() */ /*ARGSUSED*/ static void blk_hp_state_change(dev_info_t *dip, ddi_eventcookie_t id, void *arg, void *impl_data) { xendev_hotplug_state_t hpstate; blk_ring_t ring; ring = (blk_ring_t)arg; hpstate = *(xendev_hotplug_state_t *)impl_data; mutex_enter(&ring->ri_mutex); if (hpstate == Connected) { /* Hotplug script has completed successfully */ if (ring->ri_be_status == BLK_BE_UNKNOWN) { ring->ri_be_status = BLK_BE_READY; if (ring->ri_fe_status == BLK_FE_READY) { mutex_exit(&ring->ri_mutex); /* try to connect to frontend */ if (blk_start_connect(ring) != DDI_SUCCESS) (void) blk_start_disconnect(ring); mutex_enter(&ring->ri_mutex); } } } mutex_exit(&ring->ri_mutex); } /* * blk_check_state_transition() * check the XenbusState change to see if the change is a valid transition * or not. The new state is written by frontend domain, or by running * xenstore-write to change it manually in dom0. */ static int blk_check_state_transition(blk_ring_t ring, XenbusState oestate) { switch (ring->ri_if_status) { case BLK_IF_UNKNOWN: if (ring->ri_fe_status == BLK_FE_UNKNOWN) { if ((oestate == XenbusStateUnknown) || (oestate == XenbusStateConnected)) goto statechkfail_bug; else if ((oestate == XenbusStateInitialising) || (oestate == XenbusStateInitWait)) goto statechkfail_nop; } else { if ((oestate == XenbusStateUnknown) || (oestate == XenbusStateInitialising) || (oestate == XenbusStateInitWait) || (oestate == XenbusStateConnected)) goto statechkfail_bug; else if (oestate == XenbusStateInitialised) goto statechkfail_nop; } break; case BLK_IF_CONNECTED: if ((oestate == XenbusStateUnknown) || (oestate == XenbusStateInitialising) || (oestate == XenbusStateInitWait) || (oestate == XenbusStateInitialised)) goto statechkfail_bug; else if (oestate == XenbusStateConnected) goto statechkfail_nop; break; case BLK_IF_DISCONNECTED: default: goto statechkfail_bug; } return (DDI_SUCCESS); statechkfail_bug: cmn_err(CE_NOTE, "blk@%s: unexpected otherend " "state change to %d!, when status is %d", ddi_get_name_addr(ring->ri_dip), oestate, ring->ri_if_status); statechkfail_nop: return (DDI_FAILURE); } /* * blk_start_connect() * Kick-off connect process * If ri_fe_status == BLK_FE_READY and ri_be_status == BLK_BE_READY * the ri_if_status will be changed to BLK_IF_CONNECTED on success, * otherwise, ri_if_status will not be changed */ static int blk_start_connect(blk_ring_t ring) { xenbus_transaction_t xbt; dev_info_t *dip; char *barrier; char *xsnode; uint_t len; int e; dip = ring->ri_dip; /* * Start connect to frontend only when backend device are ready * and frontend has moved to XenbusStateInitialised, which means * ready to connect */ ASSERT(ring->ri_fe_status == BLK_FE_READY); ASSERT(ring->ri_be_status == BLK_BE_READY); xsnode = xvdi_get_xsname(dip); if (xsnode == NULL) { goto startconnectfail_get_xsname; } ring->ri_fe = xvdi_get_oeid(dip); if (ring->ri_fe == (domid_t)-1) { goto startconnectfail_get_oeid; } e = xvdi_switch_state(dip, XBT_NULL, XenbusStateInitialised); if (e > 0) { goto startconnectfail_switch_init; } e = blk_bindto_frontend(ring); if (e != DDI_SUCCESS) { goto startconnectfail_bindto_frontend; } ring->ri_if_status = BLK_IF_CONNECTED; e = ddi_add_intr(dip, 0, NULL, NULL, blk_intr, (caddr_t)ring); if (e != DDI_SUCCESS) { goto startconnectfail_add_intr; } trans_retry: e = xenbus_transaction_start(&xbt); if (e != 0) { xvdi_fatal_error(dip, e, "transaction start"); goto startconnectfail_transaction_start; } /* xentop requires the instance in xenstore */ e = xenbus_printf(xbt, xsnode, "instance", "%d", ddi_get_instance(ring->ri_dip)); if (e != 0) { cmn_err(CE_WARN, "xdb@%s: failed to write 'instance'", ddi_get_name_addr(dip)); xvdi_fatal_error(dip, e, "writing 'instance'"); (void) xenbus_transaction_end(xbt, 1); goto startconnectfail_xenbus_printf; } /* If feature-barrier isn't present in xenstore, add it */ e = xenbus_read(xbt, xsnode, "feature-barrier", (void **)&barrier, &len); if (e != 0) { e = xenbus_printf(xbt, xsnode, "feature-barrier", "%d", 1); if (e != 0) { cmn_err(CE_WARN, "xdb@%s: failed to write " "'feature-barrier'", ddi_get_name_addr(dip)); xvdi_fatal_error(dip, e, "writing 'feature-barrier'"); (void) xenbus_transaction_end(xbt, 1); goto startconnectfail_xenbus_printf; } } else { kmem_free(barrier, len); } e = xvdi_switch_state(dip, xbt, XenbusStateConnected); if (e > 0) { xvdi_fatal_error(dip, e, "writing 'state'"); (void) xenbus_transaction_end(xbt, 1); goto startconnectfail_switch_connected; } e = xenbus_transaction_end(xbt, 0); if (e != 0) { if (e == EAGAIN) { /* transaction is ended, don't need to abort it */ goto trans_retry; } xvdi_fatal_error(dip, e, "completing transaction"); goto startconnectfail_transaction_end; } mutex_enter(&ring->ri_state.rs_mutex); ring->ri_state.rs_ring_up = B_TRUE; if (ring->ri_state.rs_sleeping_on_ring) { ring->ri_state.rs_sleeping_on_ring = B_FALSE; cv_signal(&ring->ri_state.rs_cv); } mutex_exit(&ring->ri_state.rs_mutex); if (ring->ri_ringup != NULL) { (*(ring->ri_ringup))(ring->ri_ringup_arg); } return (DDI_SUCCESS); startconnectfail_transaction_end: startconnectfail_switch_connected: startconnectfail_xenbus_printf: startconnectfail_transaction_start: ddi_remove_intr(dip, 0, NULL); startconnectfail_add_intr: blk_unbindfrom_frontend(ring); ring->ri_fe = (domid_t)-1; startconnectfail_bindto_frontend: (void) xvdi_switch_state(dip, XBT_NULL, XenbusStateClosed); startconnectfail_switch_init: startconnectfail_get_oeid: startconnectfail_get_xsname: return (DDI_FAILURE); } /* * blk_start_disconnect() * Kick-off disconnect process. ri_if_status will not be changed */ static void blk_start_disconnect(blk_ring_t ring) { /* Kick-off disconnect process */ (void) xvdi_switch_state(ring->ri_dip, XBT_NULL, XenbusStateClosing); } /* * blk_ring_close() * Disconnect from frontend and close backend device * ifstatus will be changed to BLK_DISCONNECTED * Xenbus state will be changed to XenbusStateClosed */ static void blk_ring_close(blk_ring_t ring) { dev_info_t *dip; /* mutex protect ri_if_status only here */ ASSERT(MUTEX_HELD(&ring->ri_mutex)); dip = ring->ri_dip; if (ring->ri_if_status != BLK_IF_CONNECTED) { return; } ring->ri_if_status = BLK_IF_DISCONNECTED; mutex_exit(&ring->ri_mutex); /* stop accepting I/O request from frontend */ ddi_remove_intr(dip, 0, NULL); blk_unbindfrom_frontend(ring); ring->ri_fe = (domid_t)-1; (void) xvdi_switch_state(dip, XBT_NULL, XenbusStateClosed); mutex_enter(&ring->ri_mutex); } /* * blk_bindto_frontend() */ static int blk_bindto_frontend(blk_ring_t ring) { evtchn_port_t evtchn; char protocol[64]; grant_ref_t gref; dev_info_t *dip; char *oename; int e; dip = ring->ri_dip; protocol[0] = 0x0; /* * Gather info from frontend */ oename = xvdi_get_oename(dip); if (oename == NULL) { return (DDI_FAILURE); } e = xenbus_gather(XBT_NULL, oename, "ring-ref", "%lu", &gref, "event-channel", "%u", &evtchn, NULL); if (e != 0) { xvdi_fatal_error(dip, e, "Getting ring-ref and evtchn from frontend"); return (DDI_FAILURE); } e = xenbus_gather(XBT_NULL, oename, "protocol", "%63s", protocol, NULL); if (e != 0) { (void) strcpy(protocol, "unspecified, assuming native"); } else if (strcmp(protocol, XEN_IO_PROTO_ABI_NATIVE) == 0) { ring->ri_protocol = BLKIF_PROTOCOL_NATIVE; ring->ri_nentry = BLKIF_RING_SIZE; ring->ri_entrysize = sizeof (union blkif_sring_entry); } else if (strcmp(protocol, XEN_IO_PROTO_ABI_X86_32) == 0) { ring->ri_protocol = BLKIF_PROTOCOL_X86_32; ring->ri_nentry = BLKIF_X86_32_RING_SIZE; ring->ri_entrysize = sizeof (union blkif_x86_32_sring_entry); } else if (strcmp(protocol, XEN_IO_PROTO_ABI_X86_64) == 0) { ring->ri_protocol = BLKIF_PROTOCOL_X86_64; ring->ri_nentry = BLKIF_X86_64_RING_SIZE; ring->ri_entrysize = sizeof (union blkif_x86_64_sring_entry); } else { xvdi_fatal_error(dip, e, "unknown fe protocol"); return (DDI_FAILURE); } /* * map and init ring */ e = xvdi_map_ring(dip, ring->ri_nentry, ring->ri_entrysize, gref, &ring->ri_ring); if (e != DDI_SUCCESS) { return (DDI_FAILURE); } /* * bind event channel */ e = xvdi_bind_evtchn(dip, evtchn); if (e != DDI_SUCCESS) { xvdi_unmap_ring(ring->ri_ring); return (DDI_FAILURE); } return (DDI_SUCCESS); } /* * blk_unbindfrom_frontend() */ static void blk_unbindfrom_frontend(blk_ring_t ring) { xvdi_free_evtchn(ring->ri_dip); xvdi_unmap_ring(ring->ri_ring); } /* * blk_intr() */ static uint_t blk_intr(caddr_t arg) { blk_ring_t ring; ring = (blk_ring_t)arg; if (ring->ri_if_status != BLK_IF_CONNECTED) { return (DDI_INTR_CLAIMED); } (void) (*ring->ri_intr)(ring->ri_intr_arg); return (DDI_INTR_CLAIMED); } /* * blk_ring_request_get() */ boolean_t blk_ring_request_get(blk_ring_t ring, blkif_request_t *req) { blkif_request_t *src; blk_stats_t *stats; mutex_enter(&ring->ri_mutex); if (ring->ri_if_status != BLK_IF_CONNECTED) { mutex_exit(&ring->ri_mutex); return (B_FALSE); } src = xvdi_ring_get_request(ring->ri_ring); if (src == NULL) { mutex_exit(&ring->ri_mutex); return (B_FALSE); } switch (ring->ri_protocol) { case BLKIF_PROTOCOL_NATIVE: bcopy(src, req, sizeof (*req)); break; case BLKIF_PROTOCOL_X86_32: blk_ring_request_32(req, (blkif_x86_32_request_t *)src); break; case BLKIF_PROTOCOL_X86_64: blk_ring_request_64(req, (blkif_x86_64_request_t *)src); break; default: cmn_err(CE_WARN, "blkif@%s: unrecognised protocol: %d", ddi_get_name_addr(ring->ri_dip), ring->ri_protocol); } mutex_exit(&ring->ri_mutex); stats = &ring->ri_stats; switch (req->operation) { case BLKIF_OP_READ: stats->bs_req_reads++; break; case BLKIF_OP_WRITE: stats->bs_req_writes++; break; case BLKIF_OP_WRITE_BARRIER: stats->bs_req_barriers++; break; case BLKIF_OP_FLUSH_DISKCACHE: stats->bs_req_flushes++; break; } return (B_TRUE); } /* * blk_ring_request_requeue() * if a request is requeued, caller will have to poll for request * later. */ void blk_ring_request_requeue(blk_ring_t ring) { mutex_enter(&ring->ri_mutex); if (ring->ri_if_status != BLK_IF_CONNECTED) { mutex_exit(&ring->ri_mutex); return; } ring->ri_ring->xr_sring.br.req_cons--; mutex_exit(&ring->ri_mutex); } /* * blk_ring_response_put() */ void blk_ring_response_put(blk_ring_t ring, blkif_response_t *src) { blkif_response_t *rsp; int e; mutex_enter(&ring->ri_mutex); if (ring->ri_if_status != BLK_IF_CONNECTED) { mutex_exit(&ring->ri_mutex); return; } rsp = xvdi_ring_get_response(ring->ri_ring); ASSERT(rsp); switch (ring->ri_protocol) { case BLKIF_PROTOCOL_NATIVE: bcopy(src, rsp, sizeof (*rsp)); break; case BLKIF_PROTOCOL_X86_32: blk_ring_response_32((blkif_x86_32_response_t *)rsp, src); break; case BLKIF_PROTOCOL_X86_64: blk_ring_response_64((blkif_x86_64_response_t *)rsp, src); break; default: cmn_err(CE_WARN, "blk@%s: unrecognised protocol: %d", ddi_get_name_addr(ring->ri_dip), ring->ri_protocol); } e = xvdi_ring_push_response(ring->ri_ring); if (e != 0) { xvdi_notify_oe(ring->ri_dip); } mutex_exit(&ring->ri_mutex); } /* * blk_ring_request_32() */ static void blk_ring_request_32(blkif_request_t *dst, blkif_x86_32_request_t *src) { int i, n = BLKIF_MAX_SEGMENTS_PER_REQUEST; dst->operation = src->operation; dst->nr_segments = src->nr_segments; dst->handle = src->handle; dst->id = src->id; dst->sector_number = src->sector_number; if (n > src->nr_segments) n = src->nr_segments; for (i = 0; i < n; i++) dst->seg[i] = src->seg[i]; } /* * blk_ring_request_64() */ static void blk_ring_request_64(blkif_request_t *dst, blkif_x86_64_request_t *src) { int i, n = BLKIF_MAX_SEGMENTS_PER_REQUEST; dst->operation = src->operation; dst->nr_segments = src->nr_segments; dst->handle = src->handle; dst->id = src->id; dst->sector_number = src->sector_number; if (n > src->nr_segments) n = src->nr_segments; for (i = 0; i < n; i++) dst->seg[i] = src->seg[i]; } /* * blk_ring_response_32() */ static void blk_ring_response_32(blkif_x86_32_response_t *dst, blkif_response_t *src) { dst->id = src->id; dst->operation = src->operation; dst->status = src->status; } /* * blk_ring_response_64() */ static void blk_ring_response_64(blkif_x86_64_response_t *dst, blkif_response_t *src) { dst->id = src->id; dst->operation = src->operation; dst->status = src->status; } /* * blk_ring_request_dump() */ void blk_ring_request_dump(blkif_request_t *req) { int i; /* * Exploit the public interface definitions for BLKIF_OP_READ * etc.. */ char *op_name[] = { "read", "write", "barrier", "flush" }; cmn_err(CE_NOTE, " op=%s", op_name[req->operation]); cmn_err(CE_NOTE, " num of segments=%d", req->nr_segments); cmn_err(CE_NOTE, " handle=%d", req->handle); cmn_err(CE_NOTE, " id=0x%llx", (unsigned long long)req->id); cmn_err(CE_NOTE, " start sector=%llu", (unsigned long long)req->sector_number); for (i = 0; i < req->nr_segments; i++) { cmn_err(CE_NOTE, " gref=%d, first sec=%d," "last sec=%d", req->seg[i].gref, req->seg[i].first_sect, req->seg[i].last_sect); } } /* * blk_ring_response_dump() */ void blk_ring_response_dump(blkif_response_t *resp) { /* * Exploit the public interface definitions for BLKIF_OP_READ * etc.. */ char *op_name[] = { "read", "write", "barrier", "flush" }; cmn_err(CE_NOTE, " op=%d:%s", resp->operation, op_name[resp->operation]); cmn_err(CE_NOTE, " op=%d", resp->operation); cmn_err(CE_NOTE, " status=%d", resp->status); } /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2008 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ #ifndef _SYS_BLK_COMMON_H #define _SYS_BLK_COMMON_H #ifdef __cplusplus extern "C" { #endif #include #include typedef uint_t (*blk_intr_t)(caddr_t arg); typedef void (*blk_ring_cb_t)(caddr_t arg); typedef struct blk_ringinit_args_s { dev_info_t *ar_dip; /* callbacks */ blk_intr_t ar_intr; caddr_t ar_intr_arg; blk_ring_cb_t ar_ringup; caddr_t ar_ringup_arg; blk_ring_cb_t ar_ringdown; caddr_t ar_ringdown_arg; } blk_ringinit_args_t; typedef struct blk_ring_s *blk_ring_t; int blk_ring_init(blk_ringinit_args_t *args, blk_ring_t *ring); void blk_ring_fini(blk_ring_t *ring); boolean_t blk_ring_request_get(blk_ring_t ring, blkif_request_t *req); void blk_ring_request_requeue(blk_ring_t ring); void blk_ring_request_dump(blkif_request_t *req); void blk_ring_response_put(blk_ring_t ring, blkif_response_t *resp); void blk_ring_response_dump(blkif_response_t *req); #ifdef __cplusplus } #endif #endif /* _SYS_BLK_COMMON_H */ /* * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to * deal in the Software without restriction, including without limitation the * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or * sell copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER * DEALINGS IN THE SOFTWARE. */ #ifndef __XEN_BLKIF_H__ #define __XEN_BLKIF_H__ #include #include #include /* Not a real protocol. Used to generate ring structs which contain * the elements common to all protocols only. This way we get a * compiler-checkable way to use common struct elements, so we can * avoid using switch(protocol) in a number of places. */ /* i386 protocol version */ #pragma pack(4) struct blkif_x86_32_request { uint8_t operation; /* BLKIF_OP_??? */ uint8_t nr_segments; /* number of segments */ blkif_vdev_t handle; /* only for read/write requests */ uint64_t id; /* private guest value, echoed in resp */ blkif_sector_t sector_number;/* start sector idx on disk (r/w only) */ struct blkif_request_segment seg[BLKIF_MAX_SEGMENTS_PER_REQUEST]; }; struct blkif_x86_32_response { uint64_t id; /* copied from request */ uint8_t operation; /* copied from request */ int16_t status; /* BLKIF_RSP_??? */ }; typedef struct blkif_x86_32_request blkif_x86_32_request_t; typedef struct blkif_x86_32_response blkif_x86_32_response_t; #pragma pack() /* x86_64 protocol version */ struct blkif_x86_64_request { uint8_t operation; /* BLKIF_OP_??? */ uint8_t nr_segments; /* number of segments */ blkif_vdev_t handle; /* only for read/write requests */ #if defined(__GNUC__) uint64_t __attribute__((__aligned__(8))) id; #else uint8_t pad[4]; uint64_t id; #endif blkif_sector_t sector_number;/* start sector idx on disk (r/w only) */ struct blkif_request_segment seg[BLKIF_MAX_SEGMENTS_PER_REQUEST]; }; struct blkif_x86_64_response { #if defined(__GNUC__) uint64_t __attribute__((__aligned__(8))) id; #else uint64_t id; #endif uint8_t operation; /* copied from request */ int16_t status; /* BLKIF_RSP_??? */ }; typedef struct blkif_x86_64_request blkif_x86_64_request_t; typedef struct blkif_x86_64_response blkif_x86_64_response_t; DEFINE_RING_TYPES(blkif_x86_32, struct blkif_x86_32_request, struct blkif_x86_32_response); DEFINE_RING_TYPES(blkif_x86_64, struct blkif_x86_64_request, struct blkif_x86_64_response); enum blkif_protocol { BLKIF_PROTOCOL_NATIVE = 1, BLKIF_PROTOCOL_X86_32 = 2, BLKIF_PROTOCOL_X86_64 = 3, }; #define BLKIF_RING_SIZE \ __RING_SIZE((blkif_sring_t *)NULL, PAGESIZE) #define BLKIF_X86_32_RING_SIZE \ __RING_SIZE((blkif_x86_32_sring_t *)NULL, PAGESIZE) #define BLKIF_X86_64_RING_SIZE \ __RING_SIZE((blkif_x86_64_sring_t *)NULL, PAGESIZE) #endif /* __XEN_BLKIF_H__ */ /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2009 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. * Copyright 2017 Joyent, Inc. */ /* * evtchn.c * * Driver for receiving and demuxing event-channel signals. * * Copyright (c) 2004-2005, K A Fraser * Multi-process extensions Copyright (c) 2004, Steven Smith * * This file may be distributed separately from the Linux kernel, or * incorporated into other software packages, subject to the following license: * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this source file (the "Software"), to deal in the Software without * restriction, including without limitation the rights to use, copy, modify, * merge, publish, distribute, sublicense, and/or sell copies of the Software, * and to permit persons to whom the Software is furnished to do so, subject to * the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS * IN THE SOFTWARE. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* Some handy macros */ #define EVTCHNDRV_MINOR2INST(minor) ((int)(minor)) #define EVTCHNDRV_DEFAULT_NCLONES 256 #define EVTCHNDRV_INST2SOFTS(inst) \ (ddi_get_soft_state(evtchndrv_statep, (inst))) /* Soft state data structure for evtchn driver */ struct evtsoftdata { dev_info_t *dip; /* Notification ring, accessed via /dev/xen/evtchn. */ #define EVTCHN_RING_SIZE (PAGESIZE / sizeof (evtchn_port_t)) #define EVTCHN_RING_MASK(_i) ((_i) & (EVTCHN_RING_SIZE - 1)) evtchn_port_t *ring; unsigned int ring_cons, ring_prod, ring_overflow; kcondvar_t evtchn_wait; /* Processes wait on this when ring is empty. */ kmutex_t evtchn_lock; pollhead_t evtchn_pollhead; pid_t pid; /* last pid to bind to this event channel. */ processorid_t cpu; /* cpu thread/evtchn is bound to */ }; static void *evtchndrv_statep; int evtchndrv_nclones = EVTCHNDRV_DEFAULT_NCLONES; static int *evtchndrv_clone_tab; static dev_info_t *evtchndrv_dip; static kmutex_t evtchndrv_clone_tab_mutex; static int evtchndrv_detach(dev_info_t *, ddi_detach_cmd_t); /* Who's bound to each port? */ static struct evtsoftdata *port_user[NR_EVENT_CHANNELS]; static kmutex_t port_user_lock; uint_t evtchn_device_upcall(caddr_t arg __unused, caddr_t arg1 __unused) { struct evtsoftdata *ep; int port; /* * This is quite gross, we had to leave the evtchn that led to this * invocation in a per-cpu mailbox, retrieve it now. * We do this because the interface doesn't offer us a way to pass * a dynamic argument up through the generic interrupt service layer. * The mailbox is safe since we either run with interrupts disabled or * non-preemptable till we reach here. */ port = CPU->cpu_m.mcpu_ec_mbox; ASSERT(port != 0); CPU->cpu_m.mcpu_ec_mbox = 0; ec_clear_evtchn(port); mutex_enter(&port_user_lock); if ((ep = port_user[port]) != NULL) { mutex_enter(&ep->evtchn_lock); if ((ep->ring_prod - ep->ring_cons) < EVTCHN_RING_SIZE) { ep->ring[EVTCHN_RING_MASK(ep->ring_prod)] = port; /* * Wake up reader when ring goes non-empty */ if (ep->ring_cons == ep->ring_prod++) { cv_signal(&ep->evtchn_wait); mutex_exit(&ep->evtchn_lock); pollwakeup(&ep->evtchn_pollhead, POLLIN | POLLRDNORM); goto done; } } else { ep->ring_overflow = 1; } mutex_exit(&ep->evtchn_lock); } done: mutex_exit(&port_user_lock); return (DDI_INTR_CLAIMED); } /* ARGSUSED */ static int evtchndrv_read(dev_t dev, struct uio *uio, cred_t *cr) { int rc = 0; ssize_t count; unsigned int c, p, bytes1 = 0, bytes2 = 0; struct evtsoftdata *ep; minor_t minor = getminor(dev); if (secpolicy_xvm_control(cr)) return (EPERM); ep = EVTCHNDRV_INST2SOFTS(EVTCHNDRV_MINOR2INST(minor)); /* Whole number of ports. */ count = uio->uio_resid; count &= ~(sizeof (evtchn_port_t) - 1); if (count == 0) return (0); if (count > PAGESIZE) count = PAGESIZE; mutex_enter(&ep->evtchn_lock); for (;;) { if (ep->ring_overflow) { rc = EFBIG; goto done; } if ((c = ep->ring_cons) != (p = ep->ring_prod)) break; if (uio->uio_fmode & O_NONBLOCK) { rc = EAGAIN; goto done; } if (cv_wait_sig(&ep->evtchn_wait, &ep->evtchn_lock) == 0) { rc = EINTR; goto done; } } /* Byte lengths of two chunks. Chunk split (if any) is at ring wrap. */ if (((c ^ p) & EVTCHN_RING_SIZE) != 0) { bytes1 = (EVTCHN_RING_SIZE - EVTCHN_RING_MASK(c)) * sizeof (evtchn_port_t); bytes2 = EVTCHN_RING_MASK(p) * sizeof (evtchn_port_t); } else { bytes1 = (p - c) * sizeof (evtchn_port_t); bytes2 = 0; } /* Truncate chunks according to caller's maximum byte count. */ if (bytes1 > count) { bytes1 = count; bytes2 = 0; } else if ((bytes1 + bytes2) > count) { bytes2 = count - bytes1; } if (uiomove(&ep->ring[EVTCHN_RING_MASK(c)], bytes1, UIO_READ, uio) || ((bytes2 != 0) && uiomove(&ep->ring[0], bytes2, UIO_READ, uio))) { rc = EFAULT; goto done; } ep->ring_cons += (bytes1 + bytes2) / sizeof (evtchn_port_t); done: mutex_exit(&ep->evtchn_lock); return (rc); } /* ARGSUSED */ static int evtchndrv_write(dev_t dev, struct uio *uio, cred_t *cr) { int rc, i; ssize_t count; evtchn_port_t *kbuf; struct evtsoftdata *ep; ulong_t flags; minor_t minor = getminor(dev); evtchn_port_t sbuf[32]; if (secpolicy_xvm_control(cr)) return (EPERM); ep = EVTCHNDRV_INST2SOFTS(EVTCHNDRV_MINOR2INST(minor)); /* Whole number of ports. */ count = uio->uio_resid; count &= ~(sizeof (evtchn_port_t) - 1); if (count == 0) return (0); if (count > PAGESIZE) count = PAGESIZE; if (count <= sizeof (sbuf)) kbuf = sbuf; else kbuf = kmem_alloc(PAGESIZE, KM_SLEEP); if ((rc = uiomove(kbuf, count, UIO_WRITE, uio)) != 0) goto out; mutex_enter(&port_user_lock); for (i = 0; i < (count / sizeof (evtchn_port_t)); i++) if ((kbuf[i] < NR_EVENT_CHANNELS) && (port_user[kbuf[i]] == ep)) { flags = intr_clear(); ec_unmask_evtchn(kbuf[i]); intr_restore(flags); } mutex_exit(&port_user_lock); out: if (kbuf != sbuf) kmem_free(kbuf, PAGESIZE); return (rc); } static void evtchn_bind_to_user(struct evtsoftdata *u, int port) { ulong_t flags; /* * save away the PID of the last process to bind to this event channel. * Useful for debugging. */ u->pid = ddi_get_pid(); mutex_enter(&port_user_lock); ASSERT(port_user[port] == NULL); port_user[port] = u; ec_irq_add_evtchn(ec_dev_irq, port); flags = intr_clear(); ec_unmask_evtchn(port); intr_restore(flags); mutex_exit(&port_user_lock); } static void evtchndrv_close_evtchn(int port) { struct evtsoftdata *ep; ASSERT(MUTEX_HELD(&port_user_lock)); ep = port_user[port]; ASSERT(ep != NULL); (void) ec_mask_evtchn(port); /* * It is possible the event is in transit to us. * If it is already in the ring buffer, then a client may * get a spurious event notification on the next read of * of the evtchn device. Clients will need to be able to * handle getting a spurious event notification. */ port_user[port] = NULL; /* * The event is masked and should stay so, clean it up. */ ec_irq_rm_evtchn(ec_dev_irq, port); } /* ARGSUSED */ static int evtchndrv_ioctl(dev_t dev, int cmd, intptr_t data, int flag, cred_t *cr, int *rvalp) { int err = 0; struct evtsoftdata *ep; minor_t minor = getminor(dev); if (secpolicy_xvm_control(cr)) return (EPERM); ep = EVTCHNDRV_INST2SOFTS(EVTCHNDRV_MINOR2INST(minor)); *rvalp = 0; switch (cmd) { case IOCTL_EVTCHN_BIND_VIRQ: { struct ioctl_evtchn_bind_virq bind; if (copyin((void *)data, &bind, sizeof (bind))) { err = EFAULT; break; } if ((err = xen_bind_virq(bind.virq, 0, rvalp)) != 0) break; evtchn_bind_to_user(ep, *rvalp); break; } case IOCTL_EVTCHN_BIND_INTERDOMAIN: { struct ioctl_evtchn_bind_interdomain bind; if (copyin((void *)data, &bind, sizeof (bind))) { err = EFAULT; break; } if ((err = xen_bind_interdomain(bind.remote_domain, bind.remote_port, rvalp)) != 0) break; ec_bind_vcpu(*rvalp, 0); evtchn_bind_to_user(ep, *rvalp); break; } case IOCTL_EVTCHN_BIND_UNBOUND_PORT: { struct ioctl_evtchn_bind_unbound_port bind; if (copyin((void *)data, &bind, sizeof (bind))) { err = EFAULT; break; } if ((err = xen_alloc_unbound_evtchn(bind.remote_domain, rvalp)) != 0) break; evtchn_bind_to_user(ep, *rvalp); break; } case IOCTL_EVTCHN_UNBIND: { struct ioctl_evtchn_unbind unbind; if (copyin((void *)data, &unbind, sizeof (unbind))) { err = EFAULT; break; } if (unbind.port >= NR_EVENT_CHANNELS) { err = EFAULT; break; } mutex_enter(&port_user_lock); if (port_user[unbind.port] != ep) { mutex_exit(&port_user_lock); err = ENOTCONN; break; } evtchndrv_close_evtchn(unbind.port); mutex_exit(&port_user_lock); break; } case IOCTL_EVTCHN_NOTIFY: { struct ioctl_evtchn_notify notify; if (copyin((void *)data, ¬ify, sizeof (notify))) { err = EFAULT; break; } if (notify.port >= NR_EVENT_CHANNELS) { err = EINVAL; } else if (port_user[notify.port] != ep) { err = ENOTCONN; } else { ec_notify_via_evtchn(notify.port); } break; } default: err = ENOSYS; } return (err); } static int evtchndrv_poll(dev_t dev, short ev, int anyyet, short *revp, pollhead_t **phpp) { struct evtsoftdata *ep; minor_t minor = getminor(dev); short mask = 0; ep = EVTCHNDRV_INST2SOFTS(EVTCHNDRV_MINOR2INST(minor)); if (ev & POLLOUT) mask |= POLLOUT; if (ep->ring_overflow) mask |= POLLERR; if (ev & (POLLIN | POLLRDNORM)) { mutex_enter(&ep->evtchn_lock); if (ep->ring_cons != ep->ring_prod) { mask |= (POLLIN | POLLRDNORM) & ev; } mutex_exit(&ep->evtchn_lock); } if ((mask == 0 && !anyyet) || (ev & POLLET)) { *phpp = &ep->evtchn_pollhead; } *revp = mask; return (0); } /* ARGSUSED */ static int evtchndrv_open(dev_t *devp, int flag, int otyp, cred_t *credp) { struct evtsoftdata *ep; minor_t minor = getminor(*devp); if (otyp == OTYP_BLK) return (ENXIO); /* * only allow open on minor = 0 - the clone device */ if (minor != 0) return (ENXIO); /* * find a free slot and grab it */ mutex_enter(&evtchndrv_clone_tab_mutex); for (minor = 1; minor < evtchndrv_nclones; minor++) { if (evtchndrv_clone_tab[minor] == 0) { evtchndrv_clone_tab[minor] = 1; break; } } mutex_exit(&evtchndrv_clone_tab_mutex); if (minor == evtchndrv_nclones) return (EAGAIN); /* Allocate softstate structure */ if (ddi_soft_state_zalloc(evtchndrv_statep, EVTCHNDRV_MINOR2INST(minor)) != DDI_SUCCESS) { mutex_enter(&evtchndrv_clone_tab_mutex); evtchndrv_clone_tab[minor] = 0; mutex_exit(&evtchndrv_clone_tab_mutex); return (EAGAIN); } ep = EVTCHNDRV_INST2SOFTS(EVTCHNDRV_MINOR2INST(minor)); /* ... and init it */ ep->dip = evtchndrv_dip; cv_init(&ep->evtchn_wait, NULL, CV_DEFAULT, NULL); mutex_init(&ep->evtchn_lock, NULL, MUTEX_DEFAULT, NULL); ep->ring = kmem_alloc(PAGESIZE, KM_SLEEP); /* clone driver */ *devp = makedevice(getmajor(*devp), minor); return (0); } /* ARGSUSED */ static int evtchndrv_close(dev_t dev, int flag, int otyp, struct cred *credp) { struct evtsoftdata *ep; minor_t minor = getminor(dev); int i; ep = EVTCHNDRV_INST2SOFTS(EVTCHNDRV_MINOR2INST(minor)); if (ep == NULL) return (ENXIO); mutex_enter(&port_user_lock); for (i = 0; i < NR_EVENT_CHANNELS; i++) { if (port_user[i] != ep) continue; evtchndrv_close_evtchn(i); } mutex_exit(&port_user_lock); kmem_free(ep->ring, PAGESIZE); pollhead_clean(&ep->evtchn_pollhead); ddi_soft_state_free(evtchndrv_statep, EVTCHNDRV_MINOR2INST(minor)); /* * free clone tab slot */ mutex_enter(&evtchndrv_clone_tab_mutex); evtchndrv_clone_tab[minor] = 0; mutex_exit(&evtchndrv_clone_tab_mutex); return (0); } /* ARGSUSED */ static int evtchndrv_info(dev_info_t *dip, ddi_info_cmd_t cmd, void *arg, void **result) { dev_t dev = (dev_t)arg; minor_t minor = getminor(dev); int retval; switch (cmd) { case DDI_INFO_DEVT2DEVINFO: if (minor != 0 || evtchndrv_dip == NULL) { *result = (void *)NULL; retval = DDI_FAILURE; } else { *result = (void *)evtchndrv_dip; retval = DDI_SUCCESS; } break; case DDI_INFO_DEVT2INSTANCE: *result = (void *)0; retval = DDI_SUCCESS; break; default: retval = DDI_FAILURE; } return (retval); } static int evtchndrv_attach(dev_info_t *dip, ddi_attach_cmd_t cmd) { int error; int unit = ddi_get_instance(dip); switch (cmd) { case DDI_ATTACH: break; case DDI_RESUME: return (DDI_SUCCESS); default: cmn_err(CE_WARN, "evtchn_attach: unknown cmd 0x%x\n", cmd); return (DDI_FAILURE); } /* DDI_ATTACH */ /* * only one instance - but we clone using the open routine */ if (ddi_get_instance(dip) > 0) return (DDI_FAILURE); mutex_init(&evtchndrv_clone_tab_mutex, NULL, MUTEX_DRIVER, NULL); error = ddi_create_minor_node(dip, "evtchn", S_IFCHR, unit, DDI_PSEUDO, 0); if (error != DDI_SUCCESS) goto fail; /* * save dip for getinfo */ evtchndrv_dip = dip; ddi_report_dev(dip); mutex_init(&port_user_lock, NULL, MUTEX_DRIVER, NULL); (void) memset(port_user, 0, sizeof (port_user)); ec_dev_irq = ec_dev_alloc_irq(); (void) add_avintr(NULL, IPL_EVTCHN, (avfunc)evtchn_device_upcall, "evtchn_driver", ec_dev_irq, NULL, NULL, NULL, dip); return (DDI_SUCCESS); fail: (void) evtchndrv_detach(dip, DDI_DETACH); return (error); } /*ARGSUSED*/ static int evtchndrv_detach(dev_info_t *dip, ddi_detach_cmd_t cmd) { /* * Don't allow detach for now. */ return (DDI_FAILURE); } /* Solaris driver framework */ static struct cb_ops evtchndrv_cb_ops = { evtchndrv_open, /* cb_open */ evtchndrv_close, /* cb_close */ nodev, /* cb_strategy */ nodev, /* cb_print */ nodev, /* cb_dump */ evtchndrv_read, /* cb_read */ evtchndrv_write, /* cb_write */ evtchndrv_ioctl, /* cb_ioctl */ nodev, /* cb_devmap */ nodev, /* cb_mmap */ nodev, /* cb_segmap */ evtchndrv_poll, /* cb_chpoll */ ddi_prop_op, /* cb_prop_op */ 0, /* cb_stream */ D_NEW | D_MP | D_64BIT /* cb_flag */ }; static struct dev_ops evtchndrv_dev_ops = { DEVO_REV, /* devo_rev */ 0, /* devo_refcnt */ evtchndrv_info, /* devo_getinfo */ nulldev, /* devo_identify */ nulldev, /* devo_probe */ evtchndrv_attach, /* devo_attach */ evtchndrv_detach, /* devo_detach */ nodev, /* devo_reset */ &evtchndrv_cb_ops, /* devo_cb_ops */ NULL, /* devo_bus_ops */ NULL, /* power */ ddi_quiesce_not_needed, /* devo_quiesce */ }; static struct modldrv modldrv = { &mod_driverops, /* Type of module. This one is a driver */ "Evtchn driver", /* Name of the module. */ &evtchndrv_dev_ops /* driver ops */ }; static struct modlinkage modlinkage = { MODREV_1, &modldrv, NULL }; int _init(void) { int err; err = ddi_soft_state_init(&evtchndrv_statep, sizeof (struct evtsoftdata), 1); if (err) return (err); err = mod_install(&modlinkage); if (err) ddi_soft_state_fini(&evtchndrv_statep); else evtchndrv_clone_tab = kmem_zalloc( sizeof (int) * evtchndrv_nclones, KM_SLEEP); return (err); } int _fini(void) { int e; e = mod_remove(&modlinkage); if (e) return (e); ddi_soft_state_fini(&evtchndrv_statep); return (0); } int _info(struct modinfo *modinfop) { return (mod_info(&modlinkage, modinfop)); } /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2009 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ /* * Note: This is the backend part of the split PV disk driver. This driver * is not a nexus driver, nor is it a leaf driver(block/char/stream driver). * Currently, it does not create any minor node. So, although, it runs in * backend domain, it will not be used directly from within dom0. * It simply gets block I/O requests issued by frontend from a shared page * (blkif ring buffer - defined by Xen) between backend and frontend domain, * generates a buf, and push it down to underlying disk target driver via * ldi interface. When buf is done, this driver will generate a response * and put it into ring buffer to inform frontend of the status of the I/O * request issued by it. When a new virtual device entry is added in xenstore, * there will be an watch event sent from Xen to xvdi framework, who will, * in turn, create the devinfo node and try to attach this driver * (see xvdi_create_dev). When frontend peer changes its state to * XenbusStateClose, an event will also be sent from Xen to xvdi framework, * who will detach and remove this devinfo node (see i_xvdi_oestate_handler). * I/O requests get from ring buffer and event coming from xenstore cannot be * trusted. We verify them in xdb_get_buf() and xdb_check_state_transition(). * * Virtual device configuration is read/written from/to the database via * xenbus_* interfaces. Driver also use xvdi_* to interact with hypervisor. * There is an on-going effort to make xvdi_* cover all xenbus_*. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include static xdb_t *xdb_statep; static int xdb_debug = 0; static void xdb_close(dev_info_t *); static int xdb_push_response(xdb_t *, uint64_t, uint8_t, uint16_t); static int xdb_get_request(xdb_t *, blkif_request_t *); static void blkif_get_x86_32_req(blkif_request_t *, blkif_x86_32_request_t *); static void blkif_get_x86_64_req(blkif_request_t *, blkif_x86_64_request_t *); static int xdb_biodone(buf_t *); #ifdef DEBUG /* * debug aid functions */ static void logva(xdb_t *vdp, uint64_t va) { uint64_t *page_addrs; int i; page_addrs = vdp->page_addrs; for (i = 0; i < XDB_MAX_IO_PAGES(vdp); i++) { if (page_addrs[i] == va) debug_enter("VA remapping found!"); } for (i = 0; i < XDB_MAX_IO_PAGES(vdp); i++) { if (page_addrs[i] == 0) { page_addrs[i] = va; break; } } ASSERT(i < XDB_MAX_IO_PAGES(vdp)); } static void unlogva(xdb_t *vdp, uint64_t va) { uint64_t *page_addrs; int i; page_addrs = vdp->page_addrs; for (i = 0; i < XDB_MAX_IO_PAGES(vdp); i++) { if (page_addrs[i] == va) { page_addrs[i] = 0; break; } } ASSERT(i < XDB_MAX_IO_PAGES(vdp)); } static void xdb_dump_request_oe(blkif_request_t *req) { int i; /* * Exploit the public interface definitions for BLKIF_OP_READ * etc.. */ char *op_name[] = { "read", "write", "barrier", "flush" }; XDB_DBPRINT(XDB_DBG_IO, (CE_NOTE, "op=%s", op_name[req->operation])); XDB_DBPRINT(XDB_DBG_IO, (CE_NOTE, "num of segments=%d", req->nr_segments)); XDB_DBPRINT(XDB_DBG_IO, (CE_NOTE, "handle=%d", req->handle)); XDB_DBPRINT(XDB_DBG_IO, (CE_NOTE, "id=%llu", (unsigned long long)req->id)); XDB_DBPRINT(XDB_DBG_IO, (CE_NOTE, "start sector=%llu", (unsigned long long)req->sector_number)); for (i = 0; i < req->nr_segments; i++) { XDB_DBPRINT(XDB_DBG_IO, (CE_NOTE, "gref=%d, first sec=%d," "last sec=%d", req->seg[i].gref, req->seg[i].first_sect, req->seg[i].last_sect)); } } #endif /* DEBUG */ /* * Statistics. */ static char *xdb_stats[] = { "rd_reqs", "wr_reqs", "br_reqs", "fl_reqs", "oo_reqs" }; static int xdb_kstat_update(kstat_t *ksp, int flag) { xdb_t *vdp; kstat_named_t *knp; if (flag != KSTAT_READ) return (EACCES); vdp = ksp->ks_private; knp = ksp->ks_data; /* * Assignment order should match that of the names in * xdb_stats. */ (knp++)->value.ui64 = vdp->xs_stat_req_reads; (knp++)->value.ui64 = vdp->xs_stat_req_writes; (knp++)->value.ui64 = vdp->xs_stat_req_barriers; (knp++)->value.ui64 = vdp->xs_stat_req_flushes; (knp++)->value.ui64 = 0; /* oo_req */ return (0); } static boolean_t xdb_kstat_init(xdb_t *vdp) { int nstat = sizeof (xdb_stats) / sizeof (xdb_stats[0]); char **cp = xdb_stats; kstat_named_t *knp; if ((vdp->xs_kstats = kstat_create("xdb", ddi_get_instance(vdp->xs_dip), "req_statistics", "block", KSTAT_TYPE_NAMED, nstat, 0)) == NULL) return (B_FALSE); vdp->xs_kstats->ks_private = vdp; vdp->xs_kstats->ks_update = xdb_kstat_update; knp = vdp->xs_kstats->ks_data; while (nstat > 0) { kstat_named_init(knp, *cp, KSTAT_DATA_UINT64); knp++; cp++; nstat--; } kstat_install(vdp->xs_kstats); return (B_TRUE); } static char * i_pathname(dev_info_t *dip) { char *path, *rv; path = kmem_alloc(MAXPATHLEN, KM_SLEEP); (void) ddi_pathname(dip, path); rv = strdup(path); kmem_free(path, MAXPATHLEN); return (rv); } static buf_t * xdb_get_buf(xdb_t *vdp, blkif_request_t *req, xdb_request_t *xreq) { buf_t *bp; uint8_t segs, curseg; int sectors; int i, err; gnttab_map_grant_ref_t mapops[BLKIF_MAX_SEGMENTS_PER_REQUEST]; ddi_acc_handle_t acchdl; acchdl = vdp->xs_ring_hdl; bp = XDB_XREQ2BP(xreq); curseg = xreq->xr_curseg; /* init a new xdb request */ if (req != NULL) { ASSERT(MUTEX_HELD(&vdp->xs_iomutex)); boolean_t pagemapok = B_TRUE; uint8_t op = ddi_get8(acchdl, &req->operation); xreq->xr_vdp = vdp; xreq->xr_op = op; xreq->xr_id = ddi_get64(acchdl, &req->id); segs = xreq->xr_buf_pages = ddi_get8(acchdl, &req->nr_segments); if (segs == 0) { if (op != BLKIF_OP_FLUSH_DISKCACHE) cmn_err(CE_WARN, "!non-BLKIF_OP_FLUSH_DISKCACHE" " is seen from domain %d with zero " "length data buffer!", vdp->xs_peer); bioinit(bp); bp->b_bcount = 0; bp->b_lblkno = 0; bp->b_un.b_addr = NULL; return (bp); } else if (op == BLKIF_OP_FLUSH_DISKCACHE) { cmn_err(CE_WARN, "!BLKIF_OP_FLUSH_DISKCACHE" " is seen from domain %d with non-zero " "length data buffer!", vdp->xs_peer); } /* * segs should be no bigger than BLKIF_MAX_SEGMENTS_PER_REQUEST * according to the definition of blk interface by Xen * we do sanity check here */ if (segs > BLKIF_MAX_SEGMENTS_PER_REQUEST) segs = xreq->xr_buf_pages = BLKIF_MAX_SEGMENTS_PER_REQUEST; for (i = 0; i < segs; i++) { uint8_t fs, ls; mapops[i].host_addr = (uint64_t)(uintptr_t)XDB_IOPAGE_VA( vdp->xs_iopage_va, xreq->xr_idx, i); mapops[i].dom = vdp->xs_peer; mapops[i].ref = ddi_get32(acchdl, &req->seg[i].gref); mapops[i].flags = GNTMAP_host_map; if (op != BLKIF_OP_READ) mapops[i].flags |= GNTMAP_readonly; fs = ddi_get8(acchdl, &req->seg[i].first_sect); ls = ddi_get8(acchdl, &req->seg[i].last_sect); /* * first_sect should be no bigger than last_sect and * both of them should be no bigger than * XB_LAST_SECTOR_IN_SEG according to definition * of blk interface by Xen, so sanity check again */ if (fs > XB_LAST_SECTOR_IN_SEG) fs = XB_LAST_SECTOR_IN_SEG; if (ls > XB_LAST_SECTOR_IN_SEG) ls = XB_LAST_SECTOR_IN_SEG; if (fs > ls) fs = ls; xreq->xr_segs[i].fs = fs; xreq->xr_segs[i].ls = ls; } /* map in io pages */ err = xen_map_gref(GNTTABOP_map_grant_ref, mapops, i, B_FALSE); if (err != 0) return (NULL); for (i = 0; i < segs; i++) { /* * Although HYPERVISOR_grant_table_op() returned no * error, mapping of each single page can fail. So, * we have to do the check here and handle the error * if needed */ if (mapops[i].status != GNTST_okay) { int j; for (j = 0; j < i; j++) { #ifdef DEBUG unlogva(vdp, mapops[j].host_addr); #endif xen_release_pfn( xreq->xr_plist[j].p_pagenum); } pagemapok = B_FALSE; break; } /* record page mapping handle for unmapping later */ xreq->xr_page_hdls[i] = mapops[i].handle; #ifdef DEBUG logva(vdp, mapops[i].host_addr); #endif /* * Pass the MFNs down using the shadow list (xr_pplist) * * This is pretty ugly since we have implict knowledge * of how the rootnex binds buffers. * The GNTTABOP_map_grant_ref op makes us do some ugly * stuff since we're not allowed to touch these PTEs * from the VM. * * Obviously, these aren't real page_t's. The rootnex * only needs p_pagenum. * Also, don't use btop() here or 32 bit PAE breaks. */ xreq->xr_pplist[i] = &xreq->xr_plist[i]; xreq->xr_plist[i].p_pagenum = xen_assign_pfn(mapops[i].dev_bus_addr >> PAGESHIFT); } /* * not all pages mapped in successfully, unmap those mapped-in * page and return failure */ if (!pagemapok) { gnttab_unmap_grant_ref_t unmapop; for (i = 0; i < segs; i++) { if (mapops[i].status != GNTST_okay) continue; unmapop.host_addr = (uint64_t)(uintptr_t)XDB_IOPAGE_VA( vdp->xs_iopage_va, xreq->xr_idx, i); unmapop.dev_bus_addr = 0; unmapop.handle = mapops[i].handle; (void) HYPERVISOR_grant_table_op( GNTTABOP_unmap_grant_ref, &unmapop, 1); } return (NULL); } bioinit(bp); bp->b_lblkno = ddi_get64(acchdl, &req->sector_number); bp->b_flags = B_BUSY | B_SHADOW | B_PHYS; bp->b_flags |= (ddi_get8(acchdl, &req->operation) == BLKIF_OP_READ) ? B_READ : (B_WRITE | B_ASYNC); } else { uint64_t blkst; int isread; /* reuse this buf */ blkst = bp->b_lblkno + bp->b_bcount / DEV_BSIZE; isread = bp->b_flags & B_READ; bioreset(bp); bp->b_lblkno = blkst; bp->b_flags = B_BUSY | B_SHADOW | B_PHYS; bp->b_flags |= isread ? B_READ : (B_WRITE | B_ASYNC); XDB_DBPRINT(XDB_DBG_IO, (CE_NOTE, "reuse buf, xreq is %d!!", xreq->xr_idx)); } /* form a buf */ bp->b_un.b_addr = XDB_IOPAGE_VA(vdp->xs_iopage_va, xreq->xr_idx, curseg) + xreq->xr_segs[curseg].fs * DEV_BSIZE; bp->b_shadow = &xreq->xr_pplist[curseg]; bp->b_iodone = xdb_biodone; sectors = 0; /* * Run through the segments. There are XB_NUM_SECTORS_PER_SEG sectors * per segment. On some OSes (e.g. Linux), there may be empty gaps * between segments. (i.e. the first segment may end on sector 6 and * the second segment start on sector 4). * * if a segments first sector is not set to 0, and this is not the * first segment in our buf, end this buf now. * * if a segments last sector is not set to XB_LAST_SECTOR_IN_SEG, and * this is not the last segment in the request, add this segment into * the buf, then end this buf (updating the pointer to point to the * next segment next time around). */ for (i = curseg; i < xreq->xr_buf_pages; i++) { if ((xreq->xr_segs[i].fs != 0) && (i != curseg)) { break; } sectors += (xreq->xr_segs[i].ls - xreq->xr_segs[i].fs + 1); if ((xreq->xr_segs[i].ls != XB_LAST_SECTOR_IN_SEG) && (i != (xreq->xr_buf_pages - 1))) { i++; break; } } xreq->xr_curseg = i; bp->b_bcount = sectors * DEV_BSIZE; bp->b_bufsize = bp->b_bcount; return (bp); } static xdb_request_t * xdb_get_req(xdb_t *vdp) { xdb_request_t *req; int idx; ASSERT(MUTEX_HELD(&vdp->xs_iomutex)); ASSERT(vdp->xs_free_req != -1); req = &vdp->xs_req[vdp->xs_free_req]; vdp->xs_free_req = req->xr_next; idx = req->xr_idx; bzero(req, sizeof (xdb_request_t)); req->xr_idx = idx; return (req); } static void xdb_free_req(xdb_request_t *req) { xdb_t *vdp = req->xr_vdp; ASSERT(MUTEX_HELD(&vdp->xs_iomutex)); req->xr_next = vdp->xs_free_req; vdp->xs_free_req = req->xr_idx; } static void xdb_response(xdb_t *vdp, blkif_request_t *req, boolean_t ok) { ddi_acc_handle_t acchdl = vdp->xs_ring_hdl; if (xdb_push_response(vdp, ddi_get64(acchdl, &req->id), ddi_get8(acchdl, &req->operation), ok)) xvdi_notify_oe(vdp->xs_dip); } static void xdb_init_ioreqs(xdb_t *vdp) { int i; ASSERT(vdp->xs_nentry); if (vdp->xs_req == NULL) vdp->xs_req = kmem_alloc(vdp->xs_nentry * sizeof (xdb_request_t), KM_SLEEP); #ifdef DEBUG if (vdp->page_addrs == NULL) vdp->page_addrs = kmem_zalloc(XDB_MAX_IO_PAGES(vdp) * sizeof (uint64_t), KM_SLEEP); #endif for (i = 0; i < vdp->xs_nentry; i++) { vdp->xs_req[i].xr_idx = i; vdp->xs_req[i].xr_next = i + 1; } vdp->xs_req[vdp->xs_nentry - 1].xr_next = -1; vdp->xs_free_req = 0; /* alloc va in host dom for io page mapping */ vdp->xs_iopage_va = vmem_xalloc(heap_arena, XDB_MAX_IO_PAGES(vdp) * PAGESIZE, PAGESIZE, 0, 0, 0, 0, VM_SLEEP); for (i = 0; i < XDB_MAX_IO_PAGES(vdp); i++) hat_prepare_mapping(kas.a_hat, vdp->xs_iopage_va + i * PAGESIZE, NULL); } static void xdb_uninit_ioreqs(xdb_t *vdp) { int i; for (i = 0; i < XDB_MAX_IO_PAGES(vdp); i++) hat_release_mapping(kas.a_hat, vdp->xs_iopage_va + i * PAGESIZE); vmem_xfree(heap_arena, vdp->xs_iopage_va, XDB_MAX_IO_PAGES(vdp) * PAGESIZE); if (vdp->xs_req != NULL) { kmem_free(vdp->xs_req, vdp->xs_nentry * sizeof (xdb_request_t)); vdp->xs_req = NULL; } #ifdef DEBUG if (vdp->page_addrs != NULL) { kmem_free(vdp->page_addrs, XDB_MAX_IO_PAGES(vdp) * sizeof (uint64_t)); vdp->page_addrs = NULL; } #endif } static uint_t xdb_intr(caddr_t arg) { xdb_t *vdp = (xdb_t *)arg; dev_info_t *dip = vdp->xs_dip; blkif_request_t req, *reqp = &req; xdb_request_t *xreq; buf_t *bp; uint8_t op; int ret = DDI_INTR_UNCLAIMED; XDB_DBPRINT(XDB_DBG_IO, (CE_NOTE, "xdb@%s: I/O request received from dom %d", ddi_get_name_addr(dip), vdp->xs_peer)); mutex_enter(&vdp->xs_iomutex); /* shouldn't touch ring buffer if not in connected state */ if (!vdp->xs_if_connected) { mutex_exit(&vdp->xs_iomutex); return (DDI_INTR_UNCLAIMED); } ASSERT(vdp->xs_hp_connected && vdp->xs_fe_initialised); /* * We'll loop till there is no more request in the ring * We won't stuck in this loop for ever since the size of ring buffer * is limited, and frontend will stop pushing requests into it when * the ring buffer is full */ /* req_event will be increased in xvdi_ring_get_request() */ while (xdb_get_request(vdp, reqp)) { ret = DDI_INTR_CLAIMED; op = ddi_get8(vdp->xs_ring_hdl, &reqp->operation); if (op == BLKIF_OP_READ || op == BLKIF_OP_WRITE || op == BLKIF_OP_WRITE_BARRIER || op == BLKIF_OP_FLUSH_DISKCACHE) { #ifdef DEBUG xdb_dump_request_oe(reqp); #endif xreq = xdb_get_req(vdp); ASSERT(xreq); switch (op) { case BLKIF_OP_READ: vdp->xs_stat_req_reads++; break; case BLKIF_OP_WRITE_BARRIER: vdp->xs_stat_req_barriers++; /* FALLTHRU */ case BLKIF_OP_WRITE: vdp->xs_stat_req_writes++; break; case BLKIF_OP_FLUSH_DISKCACHE: vdp->xs_stat_req_flushes++; break; } xreq->xr_curseg = 0; /* start from first segment */ bp = xdb_get_buf(vdp, reqp, xreq); if (bp == NULL) { /* failed to form a buf */ xdb_free_req(xreq); xdb_response(vdp, reqp, B_FALSE); continue; } bp->av_forw = NULL; XDB_DBPRINT(XDB_DBG_IO, (CE_NOTE, " buf %p, blkno %lld, size %lu, addr %p", (void *)bp, (longlong_t)bp->b_blkno, (ulong_t)bp->b_bcount, (void *)bp->b_un.b_addr)); /* send bp to underlying blk driver */ if (vdp->xs_f_iobuf == NULL) { vdp->xs_f_iobuf = vdp->xs_l_iobuf = bp; } else { vdp->xs_l_iobuf->av_forw = bp; vdp->xs_l_iobuf = bp; } } else { xdb_response(vdp, reqp, B_FALSE); XDB_DBPRINT(XDB_DBG_IO, (CE_WARN, "xdb@%s: " "Unsupported cmd received from dom %d", ddi_get_name_addr(dip), vdp->xs_peer)); } } /* notify our taskq to push buf to underlying blk driver */ if (ret == DDI_INTR_CLAIMED) cv_broadcast(&vdp->xs_iocv); mutex_exit(&vdp->xs_iomutex); return (ret); } static int xdb_biodone(buf_t *bp) { int i, err, bioerr; uint8_t segs; gnttab_unmap_grant_ref_t unmapops[BLKIF_MAX_SEGMENTS_PER_REQUEST]; xdb_request_t *xreq = XDB_BP2XREQ(bp); xdb_t *vdp = xreq->xr_vdp; buf_t *nbp; bioerr = geterror(bp); if (bioerr) XDB_DBPRINT(XDB_DBG_IO, (CE_WARN, "xdb@%s: I/O error %d", ddi_get_name_addr(vdp->xs_dip), bioerr)); /* check if we are done w/ this I/O request */ if ((bioerr == 0) && (xreq->xr_curseg < xreq->xr_buf_pages)) { nbp = xdb_get_buf(vdp, NULL, xreq); if (nbp) { err = ldi_strategy(vdp->xs_ldi_hdl, nbp); if (err == 0) { XDB_DBPRINT(XDB_DBG_IO, (CE_NOTE, "sent buf to backend ok")); return (DDI_SUCCESS); } bioerr = EIO; XDB_DBPRINT(XDB_DBG_IO, (CE_WARN, "xdb@%s: " "sent buf to backend dev failed, err=%d", ddi_get_name_addr(vdp->xs_dip), err)); } else { bioerr = EIO; } } /* unmap io pages */ segs = xreq->xr_buf_pages; /* * segs should be no bigger than BLKIF_MAX_SEGMENTS_PER_REQUEST * according to the definition of blk interface by Xen */ ASSERT(segs <= BLKIF_MAX_SEGMENTS_PER_REQUEST); for (i = 0; i < segs; i++) { unmapops[i].host_addr = (uint64_t)(uintptr_t)XDB_IOPAGE_VA( vdp->xs_iopage_va, xreq->xr_idx, i); #ifdef DEBUG mutex_enter(&vdp->xs_iomutex); unlogva(vdp, unmapops[i].host_addr); mutex_exit(&vdp->xs_iomutex); #endif unmapops[i].dev_bus_addr = 0; unmapops[i].handle = xreq->xr_page_hdls[i]; } err = HYPERVISOR_grant_table_op(GNTTABOP_unmap_grant_ref, unmapops, segs); ASSERT(!err); /* * If we have reached a barrier write or a cache flush , then we must * flush all our I/Os. */ if (xreq->xr_op == BLKIF_OP_WRITE_BARRIER || xreq->xr_op == BLKIF_OP_FLUSH_DISKCACHE) { /* * XXX At this point the write did succeed, so I don't * believe we should report an error because the flush * failed. However, this is a debatable point, so * maybe we need to think more carefully about this. * For now, just cast to void. */ (void) ldi_ioctl(vdp->xs_ldi_hdl, DKIOCFLUSHWRITECACHE, 0, FKIOCTL, kcred, NULL); } mutex_enter(&vdp->xs_iomutex); /* send response back to frontend */ if (vdp->xs_if_connected) { ASSERT(vdp->xs_hp_connected && vdp->xs_fe_initialised); if (xdb_push_response(vdp, xreq->xr_id, xreq->xr_op, bioerr)) xvdi_notify_oe(vdp->xs_dip); XDB_DBPRINT(XDB_DBG_IO, (CE_NOTE, "sent resp back to frontend, id=%llu", (unsigned long long)xreq->xr_id)); } /* free io resources */ biofini(bp); xdb_free_req(xreq); vdp->xs_ionum--; if (!vdp->xs_if_connected && (vdp->xs_ionum == 0)) { /* we're closing, someone is waiting for I/O clean-up */ cv_signal(&vdp->xs_ionumcv); } mutex_exit(&vdp->xs_iomutex); return (DDI_SUCCESS); } static int xdb_bindto_frontend(xdb_t *vdp) { int err; char *oename; grant_ref_t gref; evtchn_port_t evtchn; dev_info_t *dip = vdp->xs_dip; char protocol[64] = ""; ASSERT(MUTEX_HELD(&vdp->xs_cbmutex)); /* * Switch to the XenbusStateInitialised state. This let's the * frontend know that we're about to negotiate a connection. */ (void) xvdi_switch_state(dip, XBT_NULL, XenbusStateInitialised); /* * Gather info from frontend */ oename = xvdi_get_oename(dip); if (oename == NULL) return (DDI_FAILURE); err = xenbus_gather(XBT_NULL, oename, XBP_RING_REF, "%lu", &gref, XBP_EVENT_CHAN, "%u", &evtchn, NULL); if (err != 0) { xvdi_dev_error(dip, err, "Getting ring-ref and evtchn from frontend"); return (DDI_FAILURE); } vdp->xs_blk_protocol = BLKIF_PROTOCOL_NATIVE; vdp->xs_nentry = BLKIF_RING_SIZE; vdp->xs_entrysize = sizeof (union blkif_sring_entry); err = xenbus_gather(XBT_NULL, oename, XBP_PROTOCOL, "%63s", protocol, NULL); if (err) (void) strcpy(protocol, "unspecified, assuming native"); else { /* * We must check for NATIVE first, so that the fast path * is taken for copying data from the guest to the host. */ if (strcmp(protocol, XEN_IO_PROTO_ABI_NATIVE) != 0) { if (strcmp(protocol, XEN_IO_PROTO_ABI_X86_32) == 0) { vdp->xs_blk_protocol = BLKIF_PROTOCOL_X86_32; vdp->xs_nentry = BLKIF_X86_32_RING_SIZE; vdp->xs_entrysize = sizeof (union blkif_x86_32_sring_entry); } else if (strcmp(protocol, XEN_IO_PROTO_ABI_X86_64) == 0) { vdp->xs_blk_protocol = BLKIF_PROTOCOL_X86_64; vdp->xs_nentry = BLKIF_X86_64_RING_SIZE; vdp->xs_entrysize = sizeof (union blkif_x86_64_sring_entry); } else { xvdi_fatal_error(dip, err, "unknown protocol"); return (DDI_FAILURE); } } } #ifdef DEBUG cmn_err(CE_NOTE, "!xdb@%s: blkif protocol '%s' ", ddi_get_name_addr(dip), protocol); #endif /* * Map and init ring. The ring parameters must match those which * have been allocated in the front end. */ if (xvdi_map_ring(dip, vdp->xs_nentry, vdp->xs_entrysize, gref, &vdp->xs_ring) != DDI_SUCCESS) return (DDI_FAILURE); /* * This will be removed after we use shadow I/O ring request since * we don't need to access the ring itself directly, thus the access * handle is not needed */ vdp->xs_ring_hdl = vdp->xs_ring->xr_acc_hdl; /* bind event channel */ err = xvdi_bind_evtchn(dip, evtchn); if (err != DDI_SUCCESS) { xvdi_unmap_ring(vdp->xs_ring); return (DDI_FAILURE); } return (DDI_SUCCESS); } static void xdb_unbindfrom_frontend(xdb_t *vdp) { ASSERT(MUTEX_HELD(&vdp->xs_cbmutex)); xvdi_free_evtchn(vdp->xs_dip); xvdi_unmap_ring(vdp->xs_ring); } /* * xdb_params_change() initiates a allows change to the underlying device/file * that the backend is accessing. It does this by disconnecting from the * frontend, closing the old device, clearing a bunch of xenbus parameters, * and switching back to the XenbusStateInitialising state. The frontend * should notice this transition to the XenbusStateInitialising state and * should attempt to reconnect to us (the backend). */ static void xdb_params_change(xdb_t *vdp, char *params, boolean_t update_xs) { xenbus_transaction_t xbt; dev_info_t *dip = vdp->xs_dip; char *xsname; int err; ASSERT(MUTEX_HELD(&vdp->xs_cbmutex)); ASSERT(vdp->xs_params_path != NULL); if ((xsname = xvdi_get_xsname(dip)) == NULL) return; if (strcmp(vdp->xs_params_path, params) == 0) return; /* * Close the device we're currently accessing and update the * path which points to our backend device/file. */ xdb_close(dip); vdp->xs_fe_initialised = B_FALSE; trans_retry: if ((err = xenbus_transaction_start(&xbt)) != 0) { xvdi_dev_error(dip, err, "params change transaction init"); goto errout; } /* * Delete all the xenbus properties that are connection dependant * and go back to the initializing state so that the frontend * driver can re-negotiate a connection. */ if (((err = xenbus_rm(xbt, xsname, XBP_FB)) != 0) || ((err = xenbus_rm(xbt, xsname, XBP_INFO)) != 0) || ((err = xenbus_rm(xbt, xsname, "sector-size")) != 0) || ((err = xenbus_rm(xbt, xsname, XBP_SECTORS)) != 0) || ((err = xenbus_rm(xbt, xsname, "instance")) != 0) || ((err = xenbus_rm(xbt, xsname, "node")) != 0) || (update_xs && ((err = xenbus_printf(xbt, xsname, "params", "%s", params)) != 0)) || ((err = xvdi_switch_state(dip, xbt, XenbusStateInitialising) > 0))) { (void) xenbus_transaction_end(xbt, 1); xvdi_dev_error(dip, err, "params change transaction setup"); goto errout; } if ((err = xenbus_transaction_end(xbt, 0)) != 0) { if (err == EAGAIN) { /* transaction is ended, don't need to abort it */ goto trans_retry; } xvdi_dev_error(dip, err, "params change transaction commit"); goto errout; } /* Change the device that we plan to access */ strfree(vdp->xs_params_path); vdp->xs_params_path = strdup(params); return; errout: (void) xvdi_switch_state(dip, xbt, XenbusStateInitialising); } /* * xdb_watch_params_cb() - This callback is invoked whenever there * is an update to the following xenbus parameter: * /local/domain/0/backend/vbd///params * * This normally happens during xm block-configure operations, which * are used to change CD device images for HVM domUs. */ /*ARGSUSED*/ static void xdb_watch_params_cb(dev_info_t *dip, const char *path, void *arg) { xdb_t *vdp = (xdb_t *)ddi_get_driver_private(dip); char *xsname, *oename, *str, *str2; if (((xsname = xvdi_get_xsname(dip)) == NULL) || ((oename = xvdi_get_oename(dip)) == NULL)) { return; } mutex_enter(&vdp->xs_cbmutex); if (xenbus_read_str(xsname, "params", &str) != 0) { mutex_exit(&vdp->xs_cbmutex); return; } if (strcmp(vdp->xs_params_path, str) == 0) { /* Nothing todo */ mutex_exit(&vdp->xs_cbmutex); strfree(str); return; } /* * If the frontend isn't a cd device, doesn't support media * requests, or has locked the media, then we can't change * the params value. restore the current value. */ str2 = NULL; if (!XDB_IS_FE_CD(vdp) || (xenbus_read_str(oename, XBP_MEDIA_REQ, &str2) != 0) || (strcmp(str2, XBV_MEDIA_REQ_LOCK) == 0)) { if (str2 != NULL) strfree(str2); strfree(str); str = i_pathname(dip); cmn_err(CE_NOTE, "!%s: media locked, ignoring params update", str); strfree(str); mutex_exit(&vdp->xs_cbmutex); return; } XDB_DBPRINT(XDB_DBG_INFO, (CE_NOTE, "block-configure params request: \"%s\"", str)); xdb_params_change(vdp, str, B_FALSE); mutex_exit(&vdp->xs_cbmutex); strfree(str); } /* * xdb_watch_media_req_cb() - This callback is invoked whenever there * is an update to the following xenbus parameter: * /local/domain//device/vbd//media-req * * Media requests are only supported on CD devices and are issued by * the frontend. Currently the only supported media request operaions * are "lock" and "eject". A "lock" prevents the backend from changing * the backing device/file (via xm block-configure). An "eject" requests * tells the backend device that it should disconnect from the frontend * and closing the backing device/file that is currently in use. */ /*ARGSUSED*/ static void xdb_watch_media_req_cb(dev_info_t *dip, const char *path, void *arg) { xdb_t *vdp = (xdb_t *)ddi_get_driver_private(dip); char *oename, *str; mutex_enter(&vdp->xs_cbmutex); if ((oename = xvdi_get_oename(dip)) == NULL) { mutex_exit(&vdp->xs_cbmutex); return; } if (xenbus_read_str(oename, XBP_MEDIA_REQ, &str) != 0) { mutex_exit(&vdp->xs_cbmutex); return; } if (!XDB_IS_FE_CD(vdp)) { xvdi_dev_error(dip, EINVAL, "media-req only supported for cdrom devices"); mutex_exit(&vdp->xs_cbmutex); return; } if (strcmp(str, XBV_MEDIA_REQ_EJECT) != 0) { mutex_exit(&vdp->xs_cbmutex); strfree(str); return; } strfree(str); XDB_DBPRINT(XDB_DBG_INFO, (CE_NOTE, "media eject request")); xdb_params_change(vdp, "", B_TRUE); (void) xenbus_printf(XBT_NULL, oename, XBP_MEDIA_REQ, "%s", XBV_MEDIA_REQ_NONE); mutex_exit(&vdp->xs_cbmutex); } /* * If we're dealing with a cdrom device, let the frontend know that * we support media requests via XBP_MEDIA_REQ_SUP, and setup a watch * to handle those frontend media request changes, which modify the * following xenstore parameter: * /local/domain//device/vbd//media-req */ static boolean_t xdb_media_req_init(xdb_t *vdp) { dev_info_t *dip = vdp->xs_dip; char *xsname, *oename; ASSERT(MUTEX_HELD(&vdp->xs_cbmutex)); if (((xsname = xvdi_get_xsname(dip)) == NULL) || ((oename = xvdi_get_oename(dip)) == NULL)) return (B_FALSE); if (!XDB_IS_FE_CD(vdp)) return (B_TRUE); if (xenbus_printf(XBT_NULL, xsname, XBP_MEDIA_REQ_SUP, "%d", 1) != 0) return (B_FALSE); if (xvdi_add_xb_watch_handler(dip, oename, XBP_MEDIA_REQ, xdb_watch_media_req_cb, NULL) != DDI_SUCCESS) { xvdi_dev_error(dip, EAGAIN, "Failed to register watch for cdrom media requests"); return (B_FALSE); } return (B_TRUE); } /* * Get our params value. Also, if we're using "params" then setup a * watch to handle xm block-configure operations which modify the * following xenstore parameter: * /local/domain/0/backend/vbd///params */ static boolean_t xdb_params_init(xdb_t *vdp) { dev_info_t *dip = vdp->xs_dip; char *str, *xsname; int err; ASSERT(MUTEX_HELD(&vdp->xs_cbmutex)); ASSERT(vdp->xs_params_path == NULL); if ((xsname = xvdi_get_xsname(dip)) == NULL) return (B_FALSE); err = xenbus_read_str(xsname, "params", &str); if (err != 0) { return (B_FALSE); } vdp->xs_params_path = str; if (xvdi_add_xb_watch_handler(dip, xsname, "params", xdb_watch_params_cb, NULL) != DDI_SUCCESS) { strfree(vdp->xs_params_path); vdp->xs_params_path = NULL; return (B_FALSE); } return (B_TRUE); } #define LOFI_CTRL_NODE "/dev/lofictl" #define LOFI_DEV_NODE "/devices/pseudo/lofi@0:" #define LOFI_MODE (FREAD | FWRITE | FEXCL) static int xdb_setup_node(xdb_t *vdp, char *path) { dev_info_t *dip = vdp->xs_dip; char *xsname, *str; ldi_handle_t ldi_hdl; struct lofi_ioctl *li; int minor, err; ASSERT(MUTEX_HELD(&vdp->xs_cbmutex)); if ((xsname = xvdi_get_xsname(dip)) == NULL) return (DDI_FAILURE); if ((err = xenbus_read_str(xsname, "type", &str)) != 0) { xvdi_dev_error(dip, err, "Getting type from backend device"); return (DDI_FAILURE); } if (strcmp(str, "file") == 0) vdp->xs_type |= XDB_DEV_BE_LOFI; strfree(str); if (!XDB_IS_BE_LOFI(vdp)) { (void) strlcpy(path, vdp->xs_params_path, MAXPATHLEN); ASSERT(vdp->xs_lofi_path == NULL); return (DDI_SUCCESS); } do { err = ldi_open_by_name(LOFI_CTRL_NODE, LOFI_MODE, kcred, &ldi_hdl, vdp->xs_ldi_li); } while (err == EBUSY); if (err != 0) { return (DDI_FAILURE); } li = kmem_zalloc(sizeof (*li), KM_SLEEP); (void) strlcpy(li->li_filename, vdp->xs_params_path, sizeof (li->li_filename)); err = ldi_ioctl(ldi_hdl, LOFI_MAP_FILE, (intptr_t)li, LOFI_MODE | FKIOCTL, kcred, &minor); (void) ldi_close(ldi_hdl, LOFI_MODE, kcred); kmem_free(li, sizeof (*li)); if (err != 0) { cmn_err(CE_WARN, "xdb@%s: Failed to create lofi dev for %s", ddi_get_name_addr(dip), vdp->xs_params_path); return (DDI_FAILURE); } /* * return '/devices/...' instead of '/dev/lofi/...' since the * former is available immediately after calling ldi_ioctl */ (void) snprintf(path, MAXPATHLEN, LOFI_DEV_NODE "%d", minor); (void) xenbus_printf(XBT_NULL, xsname, "node", "%s", path); ASSERT(vdp->xs_lofi_path == NULL); vdp->xs_lofi_path = strdup(path); return (DDI_SUCCESS); } static void xdb_teardown_node(xdb_t *vdp) { dev_info_t *dip = vdp->xs_dip; ldi_handle_t ldi_hdl; struct lofi_ioctl *li; int err; ASSERT(MUTEX_HELD(&vdp->xs_cbmutex)); if (!XDB_IS_BE_LOFI(vdp)) return; vdp->xs_type &= ~XDB_DEV_BE_LOFI; ASSERT(vdp->xs_lofi_path != NULL); li = kmem_zalloc(sizeof (*li), KM_SLEEP); (void) strlcpy(li->li_filename, vdp->xs_params_path, sizeof (li->li_filename)); do { err = ldi_open_by_name(LOFI_CTRL_NODE, LOFI_MODE, kcred, &ldi_hdl, vdp->xs_ldi_li); } while (err == EBUSY); if (err != 0) { kmem_free(li, sizeof (*li)); return; } if (ldi_ioctl(ldi_hdl, LOFI_UNMAP_FILE, (intptr_t)li, LOFI_MODE | FKIOCTL, kcred, NULL) != 0) { cmn_err(CE_WARN, "xdb@%s: Failed to delete lofi dev for %s", ddi_get_name_addr(dip), li->li_filename); } (void) ldi_close(ldi_hdl, LOFI_MODE, kcred); kmem_free(li, sizeof (*li)); strfree(vdp->xs_lofi_path); vdp->xs_lofi_path = NULL; } static int xdb_open_device(xdb_t *vdp) { dev_info_t *dip = vdp->xs_dip; uint64_t devsize; int blksize; char *nodepath; char *xsname; char *str; int err; ASSERT(MUTEX_HELD(&vdp->xs_cbmutex)); if (strlen(vdp->xs_params_path) == 0) { /* * it's possible to have no backing device when dealing * with a pv cdrom drive that has no virtual cd associated * with it. */ ASSERT(XDB_IS_FE_CD(vdp)); ASSERT(vdp->xs_sectors == 0); ASSERT(vdp->xs_ldi_li == NULL); ASSERT(vdp->xs_ldi_hdl == NULL); return (DDI_SUCCESS); } /* * after the hotplug scripts have "connected" the device, check to see * if we're using a dynamic device. If so, replace the params path * with the dynamic one. */ xsname = xvdi_get_xsname(dip); err = xenbus_read_str(xsname, "dynamic-device-path", &str); if (err == 0) { strfree(vdp->xs_params_path); vdp->xs_params_path = str; } if (ldi_ident_from_dip(dip, &vdp->xs_ldi_li) != 0) return (DDI_FAILURE); nodepath = kmem_zalloc(MAXPATHLEN, KM_SLEEP); /* try to open backend device */ if (xdb_setup_node(vdp, nodepath) != DDI_SUCCESS) { xvdi_dev_error(dip, ENXIO, "Getting device path of backend device"); ldi_ident_release(vdp->xs_ldi_li); kmem_free(nodepath, MAXPATHLEN); return (DDI_FAILURE); } if (ldi_open_by_name(nodepath, FREAD | (XDB_IS_RO(vdp) ? 0 : FWRITE), kcred, &vdp->xs_ldi_hdl, vdp->xs_ldi_li) != 0) { xdb_teardown_node(vdp); ldi_ident_release(vdp->xs_ldi_li); cmn_err(CE_WARN, "xdb@%s: Failed to open: %s", ddi_get_name_addr(dip), nodepath); kmem_free(nodepath, MAXPATHLEN); return (DDI_FAILURE); } if (ldi_get_size(vdp->xs_ldi_hdl, &devsize) != DDI_SUCCESS) { (void) ldi_close(vdp->xs_ldi_hdl, FREAD | (XDB_IS_RO(vdp) ? 0 : FWRITE), kcred); xdb_teardown_node(vdp); ldi_ident_release(vdp->xs_ldi_li); kmem_free(nodepath, MAXPATHLEN); return (DDI_FAILURE); } blksize = ldi_prop_get_int64(vdp->xs_ldi_hdl, DDI_PROP_DONTPASS | DDI_PROP_NOTPROM, "blksize", DEV_BSIZE); if (blksize == DEV_BSIZE) blksize = ldi_prop_get_int(vdp->xs_ldi_hdl, LDI_DEV_T_ANY | DDI_PROP_DONTPASS | DDI_PROP_NOTPROM, "device-blksize", DEV_BSIZE); vdp->xs_sec_size = blksize; vdp->xs_sectors = devsize / blksize; /* check if the underlying device is a CD/DVD disc */ if (ldi_prop_get_int(vdp->xs_ldi_hdl, LDI_DEV_T_ANY | DDI_PROP_DONTPASS, INQUIRY_DEVICE_TYPE, DTYPE_DIRECT) == DTYPE_RODIRECT) vdp->xs_type |= XDB_DEV_BE_CD; /* check if the underlying device is a removable disk */ if (ldi_prop_exists(vdp->xs_ldi_hdl, LDI_DEV_T_ANY | DDI_PROP_DONTPASS | DDI_PROP_NOTPROM, "removable-media")) vdp->xs_type |= XDB_DEV_BE_RMB; kmem_free(nodepath, MAXPATHLEN); return (DDI_SUCCESS); } static void xdb_close_device(xdb_t *vdp) { ASSERT(MUTEX_HELD(&vdp->xs_cbmutex)); if (strlen(vdp->xs_params_path) == 0) { ASSERT(XDB_IS_FE_CD(vdp)); ASSERT(vdp->xs_sectors == 0); ASSERT(vdp->xs_ldi_li == NULL); ASSERT(vdp->xs_ldi_hdl == NULL); return; } (void) ldi_close(vdp->xs_ldi_hdl, FREAD | (XDB_IS_RO(vdp) ? 0 : FWRITE), kcred); xdb_teardown_node(vdp); ldi_ident_release(vdp->xs_ldi_li); vdp->xs_type &= ~(XDB_DEV_BE_CD | XDB_DEV_BE_RMB); vdp->xs_sectors = 0; vdp->xs_ldi_li = NULL; vdp->xs_ldi_hdl = NULL; } /* * Kick-off connect process * If xs_fe_initialised == B_TRUE and xs_hp_connected == B_TRUE * the xs_if_connected will be changed to B_TRUE on success, */ static void xdb_start_connect(xdb_t *vdp) { xenbus_transaction_t xbt; dev_info_t *dip = vdp->xs_dip; boolean_t fb_exists; int err, instance = ddi_get_instance(dip); uint64_t sectors; uint_t dinfo, ssize; char *xsname; ASSERT(MUTEX_HELD(&vdp->xs_cbmutex)); if (((xsname = xvdi_get_xsname(dip)) == NULL) || ((vdp->xs_peer = xvdi_get_oeid(dip)) == (domid_t)-1)) return; mutex_enter(&vdp->xs_iomutex); /* * if the hotplug scripts haven't run or if the frontend is not * initialized, then we can't try to connect. */ if (!vdp->xs_hp_connected || !vdp->xs_fe_initialised) { ASSERT(!vdp->xs_if_connected); mutex_exit(&vdp->xs_iomutex); return; } /* If we're already connected then there's nothing todo */ if (vdp->xs_if_connected) { mutex_exit(&vdp->xs_iomutex); return; } mutex_exit(&vdp->xs_iomutex); /* * Start connect to frontend only when backend device are ready * and frontend has moved to XenbusStateInitialised, which means * ready to connect. */ XDB_DBPRINT(XDB_DBG_INFO, (CE_NOTE, "xdb@%s: starting connection process", ddi_get_name_addr(dip))); if (xdb_open_device(vdp) != DDI_SUCCESS) return; if (xdb_bindto_frontend(vdp) != DDI_SUCCESS) { xdb_close_device(vdp); return; } /* init i/o requests */ xdb_init_ioreqs(vdp); if (ddi_add_intr(dip, 0, NULL, NULL, xdb_intr, (caddr_t)vdp) != DDI_SUCCESS) { xdb_uninit_ioreqs(vdp); xdb_unbindfrom_frontend(vdp); xdb_close_device(vdp); return; } dinfo = 0; if (XDB_IS_RO(vdp)) dinfo |= VDISK_READONLY; if (XDB_IS_BE_RMB(vdp)) dinfo |= VDISK_REMOVABLE; if (XDB_IS_BE_CD(vdp)) dinfo |= VDISK_CDROM; if (XDB_IS_FE_CD(vdp)) dinfo |= VDISK_REMOVABLE | VDISK_CDROM; /* * we can recieve intr any time from now on * mark that we're ready to take intr */ mutex_enter(&vdp->xs_iomutex); ASSERT(vdp->xs_fe_initialised); vdp->xs_if_connected = B_TRUE; mutex_exit(&vdp->xs_iomutex); trans_retry: /* write into xenstore the info needed by frontend */ if ((err = xenbus_transaction_start(&xbt)) != 0) { xvdi_dev_error(dip, err, "connect transaction init"); goto errout; } /* If feature-barrier isn't present in xenstore, add it. */ fb_exists = xenbus_exists(xsname, XBP_FB); ssize = (vdp->xs_sec_size == 0) ? DEV_BSIZE : vdp->xs_sec_size; sectors = vdp->xs_sectors; if (((!fb_exists && (err = xenbus_printf(xbt, xsname, XBP_FB, "%d", 1)))) || (err = xenbus_printf(xbt, xsname, XBP_INFO, "%u", dinfo)) || (err = xenbus_printf(xbt, xsname, XBP_SECTOR_SIZE, "%u", ssize)) || (err = xenbus_printf(xbt, xsname, XBP_SECTORS, "%"PRIu64, sectors)) || (err = xenbus_printf(xbt, xsname, "instance", "%d", instance)) || ((err = xvdi_switch_state(dip, xbt, XenbusStateConnected)) > 0)) { (void) xenbus_transaction_end(xbt, 1); xvdi_dev_error(dip, err, "connect transaction setup"); goto errout; } if ((err = xenbus_transaction_end(xbt, 0)) != 0) { if (err == EAGAIN) { /* transaction is ended, don't need to abort it */ goto trans_retry; } xvdi_dev_error(dip, err, "connect transaction commit"); goto errout; } return; errout: xdb_close(dip); } /* * Disconnect from frontend and close backend device */ static void xdb_close(dev_info_t *dip) { xdb_t *vdp = (xdb_t *)ddi_get_driver_private(dip); ASSERT(MUTEX_HELD(&vdp->xs_cbmutex)); mutex_enter(&vdp->xs_iomutex); /* * if the hotplug scripts haven't run or if the frontend is not * initialized, then we can't be connected, so there's no * connection to close. */ if (!vdp->xs_hp_connected || !vdp->xs_fe_initialised) { ASSERT(!vdp->xs_if_connected); mutex_exit(&vdp->xs_iomutex); return; } /* if we're not connected, there's nothing to do */ if (!vdp->xs_if_connected) { cv_broadcast(&vdp->xs_iocv); mutex_exit(&vdp->xs_iomutex); return; } XDB_DBPRINT(XDB_DBG_INFO, (CE_NOTE, "closing while connected")); vdp->xs_if_connected = B_FALSE; cv_broadcast(&vdp->xs_iocv); mutex_exit(&vdp->xs_iomutex); /* stop accepting I/O request from frontend */ ddi_remove_intr(dip, 0, NULL); /* clear all on-going I/Os, if any */ mutex_enter(&vdp->xs_iomutex); while (vdp->xs_ionum > 0) cv_wait(&vdp->xs_ionumcv, &vdp->xs_iomutex); mutex_exit(&vdp->xs_iomutex); /* clean up resources and close this interface */ xdb_uninit_ioreqs(vdp); xdb_unbindfrom_frontend(vdp); xdb_close_device(vdp); vdp->xs_peer = (domid_t)-1; } static void xdb_send_buf(void *arg) { xdb_t *vdp = (xdb_t *)arg; buf_t *bp; int err; mutex_enter(&vdp->xs_iomutex); while (vdp->xs_send_buf) { if ((bp = vdp->xs_f_iobuf) == NULL) { /* wait for some io to send */ XDB_DBPRINT(XDB_DBG_IO, (CE_NOTE, "send buf waiting for io")); cv_wait(&vdp->xs_iocv, &vdp->xs_iomutex); continue; } vdp->xs_f_iobuf = bp->av_forw; bp->av_forw = NULL; vdp->xs_ionum++; mutex_exit(&vdp->xs_iomutex); if (bp->b_bcount == 0) { /* no I/O needs to be done */ (void) xdb_biodone(bp); mutex_enter(&vdp->xs_iomutex); continue; } err = EIO; if (vdp->xs_ldi_hdl != NULL) err = ldi_strategy(vdp->xs_ldi_hdl, bp); if (err != 0) { bp->b_flags |= B_ERROR; (void) xdb_biodone(bp); XDB_DBPRINT(XDB_DBG_IO, (CE_WARN, "xdb@%s: sent buf to backend devfailed, err=%d", ddi_get_name_addr(vdp->xs_dip), err)); } else { XDB_DBPRINT(XDB_DBG_IO, (CE_NOTE, "sent buf to backend ok")); } mutex_enter(&vdp->xs_iomutex); } XDB_DBPRINT(XDB_DBG_IO, (CE_NOTE, "send buf finishing")); mutex_exit(&vdp->xs_iomutex); } /*ARGSUSED*/ static void xdb_hp_state_change(dev_info_t *dip, ddi_eventcookie_t id, void *arg, void *impl_data) { xendev_hotplug_state_t state = *(xendev_hotplug_state_t *)impl_data; xdb_t *vdp = (xdb_t *)ddi_get_driver_private(dip); XDB_DBPRINT(XDB_DBG_INFO, (CE_NOTE, "xdb@%s: " "hotplug status change to %d!", ddi_get_name_addr(dip), state)); if (state != Connected) return; mutex_enter(&vdp->xs_cbmutex); /* If hotplug script have already run, there's nothing todo */ if (vdp->xs_hp_connected) { mutex_exit(&vdp->xs_cbmutex); return; } vdp->xs_hp_connected = B_TRUE; xdb_start_connect(vdp); mutex_exit(&vdp->xs_cbmutex); } /*ARGSUSED*/ static void xdb_oe_state_change(dev_info_t *dip, ddi_eventcookie_t id, void *arg, void *impl_data) { XenbusState new_state = *(XenbusState *)impl_data; xdb_t *vdp = (xdb_t *)ddi_get_driver_private(dip); XDB_DBPRINT(XDB_DBG_INFO, (CE_NOTE, "xdb@%s: " "otherend state change to %d!", ddi_get_name_addr(dip), new_state)); mutex_enter(&vdp->xs_cbmutex); /* * Now it'd really be nice if there was a well defined state * transition model for xen frontend drivers, but unfortunatly * there isn't. So we're stuck with assuming that all state * transitions are possible, and we'll just have to deal with * them regardless of what state we're in. */ switch (new_state) { case XenbusStateUnknown: case XenbusStateInitialising: case XenbusStateInitWait: /* tear down our connection to the frontend */ xdb_close(dip); vdp->xs_fe_initialised = B_FALSE; break; case XenbusStateInitialised: /* * If we were conected, then we need to drop the connection * and re-negotiate it. */ xdb_close(dip); vdp->xs_fe_initialised = B_TRUE; xdb_start_connect(vdp); break; case XenbusStateConnected: /* nothing todo here other than congratulate the frontend */ break; case XenbusStateClosing: /* monkey see monkey do */ (void) xvdi_switch_state(dip, XBT_NULL, XenbusStateClosing); break; case XenbusStateClosed: /* tear down our connection to the frontend */ xdb_close(dip); vdp->xs_fe_initialised = B_FALSE; (void) xvdi_switch_state(dip, XBT_NULL, new_state); break; } mutex_exit(&vdp->xs_cbmutex); } static int xdb_attach(dev_info_t *dip, ddi_attach_cmd_t cmd) { ddi_iblock_cookie_t ibc; xdb_t *vdp; int instance = ddi_get_instance(dip); char *xsname, *oename; char *str; switch (cmd) { case DDI_RESUME: return (DDI_FAILURE); case DDI_ATTACH: break; default: return (DDI_FAILURE); } /* DDI_ATTACH */ if (((xsname = xvdi_get_xsname(dip)) == NULL) || ((oename = xvdi_get_oename(dip)) == NULL)) return (DDI_FAILURE); /* * Disable auto-detach. This is necessary so that we don't get * detached while we're disconnected from the front end. */ (void) ddi_prop_update_int(DDI_DEV_T_NONE, dip, DDI_NO_AUTODETACH, 1); if (ddi_get_iblock_cookie(dip, 0, &ibc) != DDI_SUCCESS) return (DDI_FAILURE); if (ddi_soft_state_zalloc(xdb_statep, instance) != DDI_SUCCESS) return (DDI_FAILURE); vdp = ddi_get_soft_state(xdb_statep, instance); vdp->xs_dip = dip; mutex_init(&vdp->xs_iomutex, NULL, MUTEX_DRIVER, (void *)ibc); mutex_init(&vdp->xs_cbmutex, NULL, MUTEX_DRIVER, (void *)ibc); cv_init(&vdp->xs_iocv, NULL, CV_DRIVER, NULL); cv_init(&vdp->xs_ionumcv, NULL, CV_DRIVER, NULL); ddi_set_driver_private(dip, vdp); if (!xdb_kstat_init(vdp)) goto errout1; /* Check if the frontend device is supposed to be a cdrom */ if (xenbus_read_str(oename, XBP_DEV_TYPE, &str) != 0) return (DDI_FAILURE); if (strcmp(str, XBV_DEV_TYPE_CD) == 0) vdp->xs_type |= XDB_DEV_FE_CD; strfree(str); /* Check if the frontend device is supposed to be read only */ if (xenbus_read_str(xsname, "mode", &str) != 0) return (DDI_FAILURE); if ((strcmp(str, "r") == 0) || (strcmp(str, "ro") == 0)) vdp->xs_type |= XDB_DEV_RO; strfree(str); mutex_enter(&vdp->xs_cbmutex); if (!xdb_media_req_init(vdp) || !xdb_params_init(vdp)) { xvdi_remove_xb_watch_handlers(dip); mutex_exit(&vdp->xs_cbmutex); goto errout2; } mutex_exit(&vdp->xs_cbmutex); vdp->xs_send_buf = B_TRUE; vdp->xs_iotaskq = ddi_taskq_create(dip, "xdb_iotask", 1, TASKQ_DEFAULTPRI, 0); (void) ddi_taskq_dispatch(vdp->xs_iotaskq, xdb_send_buf, vdp, DDI_SLEEP); /* Watch frontend and hotplug state change */ if ((xvdi_add_event_handler(dip, XS_OE_STATE, xdb_oe_state_change, NULL) != DDI_SUCCESS) || (xvdi_add_event_handler(dip, XS_HP_STATE, xdb_hp_state_change, NULL) != DDI_SUCCESS)) goto errout3; /* * Kick-off hotplug script */ if (xvdi_post_event(dip, XEN_HP_ADD) != DDI_SUCCESS) { cmn_err(CE_WARN, "xdb@%s: failed to start hotplug script", ddi_get_name_addr(dip)); goto errout3; } /* * start waiting for hotplug event and otherend state event * mainly for debugging, frontend will not take any op seeing this */ (void) xvdi_switch_state(dip, XBT_NULL, XenbusStateInitWait); XDB_DBPRINT(XDB_DBG_INFO, (CE_NOTE, "xdb@%s: attached!", ddi_get_name_addr(dip))); return (DDI_SUCCESS); errout3: ASSERT(vdp->xs_hp_connected && vdp->xs_if_connected); xvdi_remove_event_handler(dip, NULL); /* Disconnect from the backend */ mutex_enter(&vdp->xs_cbmutex); mutex_enter(&vdp->xs_iomutex); vdp->xs_send_buf = B_FALSE; cv_broadcast(&vdp->xs_iocv); mutex_exit(&vdp->xs_iomutex); mutex_exit(&vdp->xs_cbmutex); /* wait for all io to dtrain and destroy io taskq */ ddi_taskq_destroy(vdp->xs_iotaskq); /* tear down block-configure watch */ mutex_enter(&vdp->xs_cbmutex); xvdi_remove_xb_watch_handlers(dip); mutex_exit(&vdp->xs_cbmutex); errout2: /* remove kstats */ kstat_delete(vdp->xs_kstats); errout1: /* free up driver state */ ddi_set_driver_private(dip, NULL); cv_destroy(&vdp->xs_iocv); cv_destroy(&vdp->xs_ionumcv); mutex_destroy(&vdp->xs_cbmutex); mutex_destroy(&vdp->xs_iomutex); ddi_soft_state_free(xdb_statep, instance); return (DDI_FAILURE); } /*ARGSUSED*/ static int xdb_detach(dev_info_t *dip, ddi_detach_cmd_t cmd) { int instance = ddi_get_instance(dip); xdb_t *vdp = XDB_INST2SOFTS(instance); switch (cmd) { case DDI_SUSPEND: return (DDI_FAILURE); case DDI_DETACH: break; default: return (DDI_FAILURE); } /* DDI_DETACH handling */ /* refuse to detach if we're still in use by the frontend */ mutex_enter(&vdp->xs_iomutex); if (vdp->xs_if_connected) { mutex_exit(&vdp->xs_iomutex); return (DDI_FAILURE); } vdp->xs_send_buf = B_FALSE; cv_broadcast(&vdp->xs_iocv); mutex_exit(&vdp->xs_iomutex); xvdi_remove_event_handler(dip, NULL); (void) xvdi_post_event(dip, XEN_HP_REMOVE); ddi_taskq_destroy(vdp->xs_iotaskq); mutex_enter(&vdp->xs_cbmutex); xvdi_remove_xb_watch_handlers(dip); mutex_exit(&vdp->xs_cbmutex); cv_destroy(&vdp->xs_iocv); cv_destroy(&vdp->xs_ionumcv); mutex_destroy(&vdp->xs_cbmutex); mutex_destroy(&vdp->xs_iomutex); kstat_delete(vdp->xs_kstats); ddi_set_driver_private(dip, NULL); ddi_soft_state_free(xdb_statep, instance); XDB_DBPRINT(XDB_DBG_INFO, (CE_NOTE, "xdb@%s: detached!", ddi_get_name_addr(dip))); return (DDI_SUCCESS); } static struct dev_ops xdb_dev_ops = { DEVO_REV, /* devo_rev */ 0, /* devo_refcnt */ ddi_getinfo_1to1, /* devo_getinfo */ nulldev, /* devo_identify */ nulldev, /* devo_probe */ xdb_attach, /* devo_attach */ xdb_detach, /* devo_detach */ nodev, /* devo_reset */ NULL, /* devo_cb_ops */ NULL, /* devo_bus_ops */ NULL, /* power */ ddi_quiesce_not_needed, /* quiesce */ }; /* * Module linkage information for the kernel. */ static struct modldrv modldrv = { &mod_driverops, /* Type of module. */ "vbd backend driver", /* Name of the module */ &xdb_dev_ops /* driver ops */ }; static struct modlinkage xdb_modlinkage = { MODREV_1, &modldrv, NULL }; int _init(void) { int rv; if ((rv = ddi_soft_state_init((void **)&xdb_statep, sizeof (xdb_t), 0)) == 0) if ((rv = mod_install(&xdb_modlinkage)) != 0) ddi_soft_state_fini((void **)&xdb_statep); return (rv); } int _fini(void) { int rv; if ((rv = mod_remove(&xdb_modlinkage)) != 0) return (rv); ddi_soft_state_fini((void **)&xdb_statep); return (rv); } int _info(struct modinfo *modinfop) { return (mod_info(&xdb_modlinkage, modinfop)); } static int xdb_get_request(xdb_t *vdp, blkif_request_t *req) { void *src = xvdi_ring_get_request(vdp->xs_ring); if (src == NULL) return (0); switch (vdp->xs_blk_protocol) { case BLKIF_PROTOCOL_NATIVE: (void) memcpy(req, src, sizeof (*req)); break; case BLKIF_PROTOCOL_X86_32: blkif_get_x86_32_req(req, src); break; case BLKIF_PROTOCOL_X86_64: blkif_get_x86_64_req(req, src); break; default: cmn_err(CE_PANIC, "xdb@%s: unrecognised protocol: %d", ddi_get_name_addr(vdp->xs_dip), vdp->xs_blk_protocol); } return (1); } static int xdb_push_response(xdb_t *vdp, uint64_t id, uint8_t op, uint16_t status) { ddi_acc_handle_t acchdl = vdp->xs_ring_hdl; blkif_response_t *rsp = xvdi_ring_get_response(vdp->xs_ring); blkif_x86_32_response_t *rsp_32 = (blkif_x86_32_response_t *)rsp; blkif_x86_64_response_t *rsp_64 = (blkif_x86_64_response_t *)rsp; ASSERT(rsp); switch (vdp->xs_blk_protocol) { case BLKIF_PROTOCOL_NATIVE: ddi_put64(acchdl, &rsp->id, id); ddi_put8(acchdl, &rsp->operation, op); ddi_put16(acchdl, (uint16_t *)&rsp->status, status == 0 ? BLKIF_RSP_OKAY : BLKIF_RSP_ERROR); break; case BLKIF_PROTOCOL_X86_32: ddi_put64(acchdl, &rsp_32->id, id); ddi_put8(acchdl, &rsp_32->operation, op); ddi_put16(acchdl, (uint16_t *)&rsp_32->status, status == 0 ? BLKIF_RSP_OKAY : BLKIF_RSP_ERROR); break; case BLKIF_PROTOCOL_X86_64: ddi_put64(acchdl, &rsp_64->id, id); ddi_put8(acchdl, &rsp_64->operation, op); ddi_put16(acchdl, (uint16_t *)&rsp_64->status, status == 0 ? BLKIF_RSP_OKAY : BLKIF_RSP_ERROR); break; default: cmn_err(CE_PANIC, "xdb@%s: unrecognised protocol: %d", ddi_get_name_addr(vdp->xs_dip), vdp->xs_blk_protocol); } return (xvdi_ring_push_response(vdp->xs_ring)); } static void blkif_get_x86_32_req(blkif_request_t *dst, blkif_x86_32_request_t *src) { int i, n = BLKIF_MAX_SEGMENTS_PER_REQUEST; dst->operation = src->operation; dst->nr_segments = src->nr_segments; dst->handle = src->handle; dst->id = src->id; dst->sector_number = src->sector_number; if (n > src->nr_segments) n = src->nr_segments; for (i = 0; i < n; i++) dst->seg[i] = src->seg[i]; } static void blkif_get_x86_64_req(blkif_request_t *dst, blkif_x86_64_request_t *src) { int i, n = BLKIF_MAX_SEGMENTS_PER_REQUEST; dst->operation = src->operation; dst->nr_segments = src->nr_segments; dst->handle = src->handle; dst->id = src->id; dst->sector_number = src->sector_number; if (n > src->nr_segments) n = src->nr_segments; for (i = 0; i < n; i++) dst->seg[i] = src->seg[i]; } /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2009 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ #ifndef _SYS_XDB_H #define _SYS_XDB_H #ifdef __cplusplus extern "C" { #endif #define XDB_DBG_ALL 0xf #define XDB_DBG_IO 0x1 #define XDB_DBG_INFO 0x2 #define XDB_DBPRINT(lvl, fmt) { if (xdb_debug & lvl) cmn_err fmt; } /* * Info of the exported blk device */ #define XDB_DEV_RO (1 << 0) /* backend and frontend are read-only */ #define XDB_DEV_BE_LOFI (1 << 1) /* backend device is a lofi device */ #define XDB_DEV_BE_RMB (1 << 2) /* backend device is removable */ #define XDB_DEV_BE_CD (1 << 3) /* backend device is cdrom */ #define XDB_DEV_FE_CD (1 << 4) /* frontend device is cdrom */ #define XDB_IS_RO(vdp) ((vdp)->xs_type & XDB_DEV_RO) #define XDB_IS_BE_LOFI(vdp) ((vdp)->xs_type & XDB_DEV_BE_LOFI) #define XDB_IS_BE_RMB(vdp) ((vdp)->xs_type & XDB_DEV_BE_RMB) #define XDB_IS_BE_CD(vdp) ((vdp)->xs_type & XDB_DEV_BE_CD) #define XDB_IS_FE_CD(vdp) ((vdp)->xs_type & XDB_DEV_FE_CD) /* * Other handy macrosx */ #define XDB_MINOR2INST(m) (int)(m) #define XDB_INST2MINOR(i) (minor_t)(i) #define XDB_INST2SOFTS(instance) \ ((xdb_t *)ddi_get_soft_state(xdb_statep, (instance))) #define XDB_MAX_IO_PAGES(v) ((v)->xs_nentry * BLKIF_MAX_SEGMENTS_PER_REQUEST) /* get kva of a mapped-in page coresponding to (xreq-index, seg) pair */ #define XDB_IOPAGE_VA(_pagebase, _xreqidx, _seg) \ ((_pagebase) + ((_xreqidx) \ * BLKIF_MAX_SEGMENTS_PER_REQUEST \ + (_seg)) * PAGESIZE) #define XDB_XREQ2BP(xreq) (&(xreq)->xr_buf) #define XDB_BP2XREQ(bp) \ ((xdb_request_t *)((char *)(bp) - offsetof(xdb_request_t, xr_buf))) /* describe one blkif segment */ typedef struct xdb_seg { uint8_t fs; /* start sector # within this page (segment) */ uint8_t ls; /* end sector # within this page (segment) */ } xdb_seg_t; typedef struct xdb xdb_t; /* one blkif_request_t matches one xdb_request_t */ typedef struct xdb_request { /* buf associated with this I/O request */ buf_t xr_buf; /* softstate instance associated with this I/O request */ xdb_t *xr_vdp; /* the next segment we're going to process */ int xr_curseg; /* index of this xdb_request_t in vdp->xs_req */ int xr_idx; /* next index for a statical linked list */ int xr_next; /* 'id' copied from blkif_request_t */ uint64_t xr_id; /* 'operation' copied from blkif_request_t */ uint8_t xr_op; /* how many pages(segments) in this I/O request */ uint8_t xr_buf_pages; /* all segments of this I/O request */ xdb_seg_t xr_segs[BLKIF_MAX_SEGMENTS_PER_REQUEST]; /* all grant table handles used in this I/O request */ grant_handle_t xr_page_hdls[BLKIF_MAX_SEGMENTS_PER_REQUEST]; struct page xr_plist[BLKIF_MAX_SEGMENTS_PER_REQUEST]; struct page *xr_pplist[BLKIF_MAX_SEGMENTS_PER_REQUEST]; } xdb_request_t; /* Soft state data structure for each backend vbd */ struct xdb { /* devinfo node pointer of this xdb */ dev_info_t *xs_dip; /* coresponding frontend domain id */ domid_t xs_peer; /* read-only, removable, cdrom? */ uint32_t xs_type; /* # of total sectors */ uint64_t xs_sectors; /* sector size if existed */ uint_t xs_sec_size; /* blkif I/O request ring buffer */ xendev_ring_t *xs_ring; /* handle to access the ring buffer */ ddi_acc_handle_t xs_ring_hdl; ldi_ident_t xs_ldi_li; ldi_handle_t xs_ldi_hdl; /* base kva for mapped-in I/O page from frontend domain */ caddr_t xs_iopage_va; /* mutex lock for I/O related code path */ kmutex_t xs_iomutex; /* * mutex lock for event handling related code path * need to be grabbed before xs_iomutex */ kmutex_t xs_cbmutex; /* # of on-going I/O buf in backend domain */ uint_t xs_ionum; /* task thread for pushing buf to underlying target driver */ ddi_taskq_t *xs_iotaskq; /* cv used in I/O code path, protected by xs_iomutex */ kcondvar_t xs_iocv; kcondvar_t xs_ionumcv; /* * head and tail of linked list for I/O bufs need to be pushed to * underlying target driver */ buf_t *xs_f_iobuf; buf_t *xs_l_iobuf; /* head of free list of xdb_request_t */ int xs_free_req; /* pre-allocated xdb_request_t pool */ xdb_request_t *xs_req; kstat_t *xs_kstats; uint64_t xs_stat_req_reads; uint64_t xs_stat_req_writes; uint64_t xs_stat_req_barriers; uint64_t xs_stat_req_flushes; enum blkif_protocol xs_blk_protocol; size_t xs_nentry; size_t xs_entrysize; /* Protected by xs_cbmutex */ boolean_t xs_hp_connected; /* hot plug scripts have run */ boolean_t xs_fe_initialised; /* frontend is initialized */ char *xs_lofi_path; char *xs_params_path; struct xenbus_watch *xs_watch_params; struct xenbus_watch *xs_watch_media_req; ddi_taskq_t *xs_watch_taskq; int xs_watch_taskq_count; /* Protected by xs_cbmutex and xs_iomutex */ boolean_t xs_if_connected; /* connected to frontend */ /* Protected by xs_iomutex */ boolean_t xs_send_buf; #ifdef DEBUG uint64_t *page_addrs; /* for debug aid */ #endif /* DEBUG */ }; #ifdef __cplusplus } #endif #endif /* _SYS_XDB_H */ /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2009 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ /* * Copyright (c) 2014, 2017 by Delphix. All rights reserved. * Copyright 2017 Nexenta Systems, Inc. */ /* * xdf.c - Xen Virtual Block Device Driver * TODO: * - support alternate block size (currently only DEV_BSIZE supported) * - revalidate geometry for removable devices * * This driver exports disk device nodes, accepts IO requests from those * nodes, and services those requests by talking to a backend device * in another domain. * * Communication with the backend device is done via a ringbuffer (which is * managed via xvdi interfaces) and dma memory (which is managed via ddi * interfaces). * * Communication with the backend device is dependant upon establishing a * connection to the backend device. This connection process involves * reading device configuration information from xenbus and publishing * some frontend runtime configuration parameters via the xenbus (for * consumption by the backend). Once we've published runtime configuration * information via the xenbus, the backend device can enter the connected * state and we'll enter the XD_CONNECTED state. But before we can allow * random IO to begin, we need to do IO to the backend device to determine * the device label and if flush operations are supported. Once this is * done we enter the XD_READY state and can process any IO operations. * * We receive notifications of xenbus state changes for the backend device * (aka, the "other end") via the xdf_oe_change() callback. This callback * is single threaded, meaning that we can't receive new notification of * other end state changes while we're processing an outstanding * notification of an other end state change. There for we can't do any * blocking operations from the xdf_oe_change() callback. This is why we * have a seperate taskq (xdf_ready_tq) which exists to do the necessary * IO to get us from the XD_CONNECTED to the XD_READY state. All IO * generated by the xdf_ready_tq thread (xdf_ready_tq_thread) will go * throught xdf_lb_rdwr(), which is a synchronous IO interface. IOs * generated by the xdf_ready_tq_thread thread have priority over all * other IO requests. * * We also communicate with the backend device via the xenbus "media-req" * (XBP_MEDIA_REQ) property. For more information on this see the * comments in blkif.h. */ #include #include #include #include #include #include #include #ifdef XPV_HVM_DRIVER #include #else /* !XPV_HVM_DRIVER */ #include #endif /* !XPV_HVM_DRIVER */ #include #include #include #include #include #include #include /* * DEBUG_EVAL can be used to include debug only statements without * having to use '#ifdef DEBUG' statements */ #ifdef DEBUG #define DEBUG_EVAL(x) (x) #else /* !DEBUG */ #define DEBUG_EVAL(x) #endif /* !DEBUG */ #define XDF_DRAIN_MSEC_DELAY (50*1000) /* 00.05 sec */ #define XDF_DRAIN_RETRY_COUNT 200 /* 10.00 sec */ #define XDF_STATE_TIMEOUT (30*1000*1000) /* 30.00 sec */ #define INVALID_DOMID ((domid_t)-1) #define FLUSH_DISKCACHE 0x1 #define WRITE_BARRIER 0x2 #define DEFAULT_FLUSH_BLOCK 156 /* block to write to cause a cache flush */ #define USE_WRITE_BARRIER(vdp) \ ((vdp)->xdf_feature_barrier && !(vdp)->xdf_flush_supported) #define USE_FLUSH_DISKCACHE(vdp) \ ((vdp)->xdf_feature_barrier && (vdp)->xdf_flush_supported) #define IS_WRITE_BARRIER(vdp, bp) \ (!IS_READ(bp) && USE_WRITE_BARRIER(vdp) && \ ((bp)->b_un.b_addr == (vdp)->xdf_cache_flush_block)) #define IS_FLUSH_DISKCACHE(bp) \ (!IS_READ(bp) && USE_FLUSH_DISKCACHE(vdp) && ((bp)->b_bcount == 0)) #define VREQ_DONE(vreq) \ VOID2BOOLEAN(((vreq)->v_status == VREQ_DMAWIN_DONE) && \ (((vreq)->v_flush_diskcache == FLUSH_DISKCACHE) || \ (((vreq)->v_dmaw + 1) == (vreq)->v_ndmaws))) #define BP_VREQ(bp) ((v_req_t *)((bp)->av_back)) #define BP_VREQ_SET(bp, vreq) (((bp)->av_back = (buf_t *)(vreq))) extern int do_polled_io; /* run-time tunables that we don't want the compiler to optimize away */ volatile int xdf_debug = 0; volatile boolean_t xdf_barrier_flush_disable = B_FALSE; /* per module globals */ major_t xdf_major; static void *xdf_ssp; static kmem_cache_t *xdf_vreq_cache; static kmem_cache_t *xdf_gs_cache; static int xdf_maxphys = XB_MAXPHYS; static diskaddr_t xdf_flush_block = DEFAULT_FLUSH_BLOCK; static int xdf_fbrewrites; /* flush block re-write count */ /* misc public functions */ int xdf_lb_rdwr(dev_info_t *, uchar_t, void *, diskaddr_t, size_t, void *); int xdf_lb_getinfo(dev_info_t *, int, void *, void *); /* misc private functions */ static void xdf_io_start(xdf_t *); static void xdf_devid_setup(xdf_t *); /* callbacks from commmon label */ static cmlb_tg_ops_t xdf_lb_ops = { TG_DK_OPS_VERSION_1, xdf_lb_rdwr, xdf_lb_getinfo }; /* * I/O buffer DMA attributes * Make sure: one DMA window contains BLKIF_MAX_SEGMENTS_PER_REQUEST at most */ static ddi_dma_attr_t xb_dma_attr = { DMA_ATTR_V0, (uint64_t)0, /* lowest address */ (uint64_t)0xffffffffffffffff, /* highest usable address */ (uint64_t)0xffffff, /* DMA counter limit max */ (uint64_t)XB_BSIZE, /* alignment in bytes */ XB_BSIZE - 1, /* bitmap of burst sizes */ XB_BSIZE, /* min transfer */ (uint64_t)XB_MAX_XFER, /* maximum transfer */ (uint64_t)PAGEOFFSET, /* 1 page segment length */ BLKIF_MAX_SEGMENTS_PER_REQUEST, /* maximum number of segments */ XB_BSIZE, /* granularity */ 0, /* flags (reserved) */ }; static ddi_device_acc_attr_t xc_acc_attr = { DDI_DEVICE_ATTR_V0, DDI_NEVERSWAP_ACC, DDI_STRICTORDER_ACC }; static void xdf_timeout_handler(void *arg) { xdf_t *vdp = arg; mutex_enter(&vdp->xdf_dev_lk); vdp->xdf_timeout_id = 0; mutex_exit(&vdp->xdf_dev_lk); /* new timeout thread could be re-scheduled */ xdf_io_start(vdp); } /* * callback func when DMA/GTE resources is available * * Note: we only register one callback function to grant table subsystem * since we only have one 'struct gnttab_free_callback' in xdf_t. */ static void xdf_gncallback(void *arg) { xdf_t *vdp = arg; ASSERT(vdp != NULL); DPRINTF(DMA_DBG, ("xdf@%s: DMA callback started\n", vdp->xdf_addr)); ddi_trigger_softintr(vdp->xdf_softintr_id); } static int xdf_dmacallback(caddr_t arg) { xdf_gncallback(arg); return (DDI_DMA_CALLBACK_DONE); } static ge_slot_t * gs_get(xdf_t *vdp, int isread) { grant_ref_t gh; ge_slot_t *gs; /* try to alloc GTEs needed in this slot, first */ if (gnttab_alloc_grant_references( BLKIF_MAX_SEGMENTS_PER_REQUEST, &gh) == -1) { if (vdp->xdf_gnt_callback.next == NULL) { SETDMACBON(vdp); gnttab_request_free_callback( &vdp->xdf_gnt_callback, xdf_gncallback, (void *)vdp, BLKIF_MAX_SEGMENTS_PER_REQUEST); } return (NULL); } gs = kmem_cache_alloc(xdf_gs_cache, KM_NOSLEEP); if (gs == NULL) { gnttab_free_grant_references(gh); if (vdp->xdf_timeout_id == 0) /* restart I/O after one second */ vdp->xdf_timeout_id = timeout(xdf_timeout_handler, vdp, hz); return (NULL); } /* init gs_slot */ gs->gs_oeid = vdp->xdf_peer; gs->gs_isread = isread; gs->gs_ghead = gh; gs->gs_ngrefs = 0; return (gs); } static void gs_free(ge_slot_t *gs) { int i; /* release all grant table entry resources used in this slot */ for (i = 0; i < gs->gs_ngrefs; i++) gnttab_end_foreign_access(gs->gs_ge[i], !gs->gs_isread, 0); gnttab_free_grant_references(gs->gs_ghead); list_remove(&gs->gs_vreq->v_gs, gs); kmem_cache_free(xdf_gs_cache, gs); } static grant_ref_t gs_grant(ge_slot_t *gs, mfn_t mfn) { grant_ref_t gr = gnttab_claim_grant_reference(&gs->gs_ghead); ASSERT(gr != -1); ASSERT(gs->gs_ngrefs < BLKIF_MAX_SEGMENTS_PER_REQUEST); gs->gs_ge[gs->gs_ngrefs++] = gr; gnttab_grant_foreign_access_ref(gr, gs->gs_oeid, mfn, !gs->gs_isread); return (gr); } /* * Alloc a vreq for this bp * bp->av_back contains the pointer to the vreq upon return */ static v_req_t * vreq_get(xdf_t *vdp, buf_t *bp) { v_req_t *vreq = NULL; ASSERT(BP_VREQ(bp) == NULL); vreq = kmem_cache_alloc(xdf_vreq_cache, KM_NOSLEEP); if (vreq == NULL) { if (vdp->xdf_timeout_id == 0) /* restart I/O after one second */ vdp->xdf_timeout_id = timeout(xdf_timeout_handler, vdp, hz); return (NULL); } bzero(vreq, sizeof (v_req_t)); list_create(&vreq->v_gs, sizeof (ge_slot_t), offsetof(ge_slot_t, gs_vreq_link)); vreq->v_buf = bp; vreq->v_status = VREQ_INIT; vreq->v_runq = B_FALSE; BP_VREQ_SET(bp, vreq); /* init of other fields in vreq is up to the caller */ list_insert_head(&vdp->xdf_vreq_act, (void *)vreq); return (vreq); } static void vreq_free(xdf_t *vdp, v_req_t *vreq) { buf_t *bp = vreq->v_buf; ASSERT(MUTEX_HELD(&vdp->xdf_dev_lk)); ASSERT(BP_VREQ(bp) == vreq); list_remove(&vdp->xdf_vreq_act, vreq); if (vreq->v_flush_diskcache == FLUSH_DISKCACHE) goto done; switch (vreq->v_status) { case VREQ_DMAWIN_DONE: case VREQ_GS_ALLOCED: case VREQ_DMABUF_BOUND: (void) ddi_dma_unbind_handle(vreq->v_dmahdl); /*FALLTHRU*/ case VREQ_DMAMEM_ALLOCED: if (!ALIGNED_XFER(bp)) { ASSERT(vreq->v_abuf != NULL); if (!IS_ERROR(bp) && IS_READ(bp)) bcopy(vreq->v_abuf, bp->b_un.b_addr, bp->b_bcount); ddi_dma_mem_free(&vreq->v_align); } /*FALLTHRU*/ case VREQ_MEMDMAHDL_ALLOCED: if (!ALIGNED_XFER(bp)) ddi_dma_free_handle(&vreq->v_memdmahdl); /*FALLTHRU*/ case VREQ_DMAHDL_ALLOCED: ddi_dma_free_handle(&vreq->v_dmahdl); break; default: break; } done: ASSERT(!vreq->v_runq); list_destroy(&vreq->v_gs); kmem_cache_free(xdf_vreq_cache, vreq); } /* * Snarf new data if our flush block was re-written */ static void check_fbwrite(xdf_t *vdp, buf_t *bp, daddr_t blkno) { int nblks; boolean_t mapin; if (IS_WRITE_BARRIER(vdp, bp)) return; /* write was a flush write */ mapin = B_FALSE; nblks = bp->b_bcount >> DEV_BSHIFT; if (xdf_flush_block >= blkno && xdf_flush_block < (blkno + nblks)) { xdf_fbrewrites++; if (bp->b_flags & (B_PAGEIO | B_PHYS)) { mapin = B_TRUE; bp_mapin(bp); } bcopy(bp->b_un.b_addr + ((xdf_flush_block - blkno) << DEV_BSHIFT), vdp->xdf_cache_flush_block, DEV_BSIZE); if (mapin) bp_mapout(bp); } } /* * Initalize the DMA and grant table resources for the buf */ static int vreq_setup(xdf_t *vdp, v_req_t *vreq) { int rc; ddi_dma_attr_t dmaattr; uint_t ndcs, ndws; ddi_dma_handle_t dh; ddi_dma_handle_t mdh; ddi_dma_cookie_t dc; ddi_acc_handle_t abh; caddr_t aba; ge_slot_t *gs; size_t bufsz; off_t off; size_t sz; buf_t *bp = vreq->v_buf; int dma_flags = (IS_READ(bp) ? DDI_DMA_READ : DDI_DMA_WRITE) | DDI_DMA_STREAMING | DDI_DMA_PARTIAL; switch (vreq->v_status) { case VREQ_INIT: if (IS_FLUSH_DISKCACHE(bp)) { if ((gs = gs_get(vdp, IS_READ(bp))) == NULL) { DPRINTF(DMA_DBG, ("xdf@%s: " "get ge_slotfailed\n", vdp->xdf_addr)); return (DDI_FAILURE); } vreq->v_blkno = 0; vreq->v_nslots = 1; vreq->v_flush_diskcache = FLUSH_DISKCACHE; vreq->v_status = VREQ_GS_ALLOCED; gs->gs_vreq = vreq; list_insert_head(&vreq->v_gs, gs); return (DDI_SUCCESS); } if (IS_WRITE_BARRIER(vdp, bp)) vreq->v_flush_diskcache = WRITE_BARRIER; vreq->v_blkno = bp->b_blkno + (diskaddr_t)(uintptr_t)bp->b_private; /* See if we wrote new data to our flush block */ if (!IS_READ(bp) && USE_WRITE_BARRIER(vdp)) check_fbwrite(vdp, bp, vreq->v_blkno); vreq->v_status = VREQ_INIT_DONE; /*FALLTHRU*/ case VREQ_INIT_DONE: /* * alloc DMA handle */ rc = ddi_dma_alloc_handle(vdp->xdf_dip, &xb_dma_attr, xdf_dmacallback, (caddr_t)vdp, &dh); if (rc != DDI_SUCCESS) { SETDMACBON(vdp); DPRINTF(DMA_DBG, ("xdf@%s: DMA handle alloc failed\n", vdp->xdf_addr)); return (DDI_FAILURE); } vreq->v_dmahdl = dh; vreq->v_status = VREQ_DMAHDL_ALLOCED; /*FALLTHRU*/ case VREQ_DMAHDL_ALLOCED: /* * alloc dma handle for 512-byte aligned buf */ if (!ALIGNED_XFER(bp)) { /* * XXPV: we need to temporarily enlarge the seg * boundary and s/g length to work round CR6381968 */ dmaattr = xb_dma_attr; dmaattr.dma_attr_seg = (uint64_t)-1; dmaattr.dma_attr_sgllen = INT_MAX; rc = ddi_dma_alloc_handle(vdp->xdf_dip, &dmaattr, xdf_dmacallback, (caddr_t)vdp, &mdh); if (rc != DDI_SUCCESS) { SETDMACBON(vdp); DPRINTF(DMA_DBG, ("xdf@%s: " "unaligned buf DMAhandle alloc failed\n", vdp->xdf_addr)); return (DDI_FAILURE); } vreq->v_memdmahdl = mdh; vreq->v_status = VREQ_MEMDMAHDL_ALLOCED; } /*FALLTHRU*/ case VREQ_MEMDMAHDL_ALLOCED: /* * alloc 512-byte aligned buf */ if (!ALIGNED_XFER(bp)) { if (bp->b_flags & (B_PAGEIO | B_PHYS)) bp_mapin(bp); rc = ddi_dma_mem_alloc(vreq->v_memdmahdl, roundup(bp->b_bcount, XB_BSIZE), &xc_acc_attr, DDI_DMA_STREAMING, xdf_dmacallback, (caddr_t)vdp, &aba, &bufsz, &abh); if (rc != DDI_SUCCESS) { SETDMACBON(vdp); DPRINTF(DMA_DBG, ("xdf@%s: " "DMA mem allocation failed\n", vdp->xdf_addr)); return (DDI_FAILURE); } vreq->v_abuf = aba; vreq->v_align = abh; vreq->v_status = VREQ_DMAMEM_ALLOCED; ASSERT(bufsz >= bp->b_bcount); if (!IS_READ(bp)) bcopy(bp->b_un.b_addr, vreq->v_abuf, bp->b_bcount); } /*FALLTHRU*/ case VREQ_DMAMEM_ALLOCED: /* * dma bind */ if (ALIGNED_XFER(bp)) { rc = ddi_dma_buf_bind_handle(vreq->v_dmahdl, bp, dma_flags, xdf_dmacallback, (caddr_t)vdp, &dc, &ndcs); } else { rc = ddi_dma_addr_bind_handle(vreq->v_dmahdl, NULL, vreq->v_abuf, bp->b_bcount, dma_flags, xdf_dmacallback, (caddr_t)vdp, &dc, &ndcs); } if (rc == DDI_DMA_MAPPED || rc == DDI_DMA_PARTIAL_MAP) { /* get num of dma windows */ if (rc == DDI_DMA_PARTIAL_MAP) { rc = ddi_dma_numwin(vreq->v_dmahdl, &ndws); ASSERT(rc == DDI_SUCCESS); } else { ndws = 1; } } else { SETDMACBON(vdp); DPRINTF(DMA_DBG, ("xdf@%s: DMA bind failed\n", vdp->xdf_addr)); return (DDI_FAILURE); } vreq->v_dmac = dc; vreq->v_dmaw = 0; vreq->v_ndmacs = ndcs; vreq->v_ndmaws = ndws; vreq->v_nslots = ndws; vreq->v_status = VREQ_DMABUF_BOUND; /*FALLTHRU*/ case VREQ_DMABUF_BOUND: /* * get ge_slot, callback is set upon failure from gs_get(), * if not set previously */ if ((gs = gs_get(vdp, IS_READ(bp))) == NULL) { DPRINTF(DMA_DBG, ("xdf@%s: get ge_slot failed\n", vdp->xdf_addr)); return (DDI_FAILURE); } vreq->v_status = VREQ_GS_ALLOCED; gs->gs_vreq = vreq; list_insert_head(&vreq->v_gs, gs); break; case VREQ_GS_ALLOCED: /* nothing need to be done */ break; case VREQ_DMAWIN_DONE: /* * move to the next dma window */ ASSERT((vreq->v_dmaw + 1) < vreq->v_ndmaws); /* get a ge_slot for this DMA window */ if ((gs = gs_get(vdp, IS_READ(bp))) == NULL) { DPRINTF(DMA_DBG, ("xdf@%s: get ge_slot failed\n", vdp->xdf_addr)); return (DDI_FAILURE); } vreq->v_dmaw++; VERIFY(ddi_dma_getwin(vreq->v_dmahdl, vreq->v_dmaw, &off, &sz, &vreq->v_dmac, &vreq->v_ndmacs) == DDI_SUCCESS); vreq->v_status = VREQ_GS_ALLOCED; gs->gs_vreq = vreq; list_insert_head(&vreq->v_gs, gs); break; default: return (DDI_FAILURE); } return (DDI_SUCCESS); } static int xdf_cmlb_attach(xdf_t *vdp) { dev_info_t *dip = vdp->xdf_dip; return (cmlb_attach(dip, &xdf_lb_ops, XD_IS_CD(vdp) ? DTYPE_RODIRECT : DTYPE_DIRECT, XD_IS_RM(vdp), B_TRUE, XD_IS_CD(vdp) ? DDI_NT_CD_XVMD : DDI_NT_BLOCK_XVMD, 0, vdp->xdf_vd_lbl, NULL)); } static void xdf_io_err(buf_t *bp, int err, size_t resid) { bioerror(bp, err); if (resid == 0) bp->b_resid = bp->b_bcount; biodone(bp); } static void xdf_kstat_enter(xdf_t *vdp, buf_t *bp) { v_req_t *vreq = BP_VREQ(bp); ASSERT(MUTEX_HELD(&vdp->xdf_dev_lk)); if (vdp->xdf_xdev_iostat == NULL) return; if ((vreq != NULL) && vreq->v_runq) { kstat_runq_enter(KSTAT_IO_PTR(vdp->xdf_xdev_iostat)); } else { kstat_waitq_enter(KSTAT_IO_PTR(vdp->xdf_xdev_iostat)); } } static void xdf_kstat_exit(xdf_t *vdp, buf_t *bp) { v_req_t *vreq = BP_VREQ(bp); ASSERT(MUTEX_HELD(&vdp->xdf_dev_lk)); if (vdp->xdf_xdev_iostat == NULL) return; if ((vreq != NULL) && vreq->v_runq) { kstat_runq_exit(KSTAT_IO_PTR(vdp->xdf_xdev_iostat)); } else { kstat_waitq_exit(KSTAT_IO_PTR(vdp->xdf_xdev_iostat)); } if (bp->b_flags & B_READ) { KSTAT_IO_PTR(vdp->xdf_xdev_iostat)->reads++; KSTAT_IO_PTR(vdp->xdf_xdev_iostat)->nread += bp->b_bcount; } else if (bp->b_flags & B_WRITE) { KSTAT_IO_PTR(vdp->xdf_xdev_iostat)->writes++; KSTAT_IO_PTR(vdp->xdf_xdev_iostat)->nwritten += bp->b_bcount; } } static void xdf_kstat_waitq_to_runq(xdf_t *vdp, buf_t *bp) { v_req_t *vreq = BP_VREQ(bp); ASSERT(MUTEX_HELD(&vdp->xdf_dev_lk)); ASSERT(!vreq->v_runq); vreq->v_runq = B_TRUE; if (vdp->xdf_xdev_iostat == NULL) return; kstat_waitq_to_runq(KSTAT_IO_PTR(vdp->xdf_xdev_iostat)); } static void xdf_kstat_runq_to_waitq(xdf_t *vdp, buf_t *bp) { v_req_t *vreq = BP_VREQ(bp); ASSERT(MUTEX_HELD(&vdp->xdf_dev_lk)); ASSERT(vreq->v_runq); vreq->v_runq = B_FALSE; if (vdp->xdf_xdev_iostat == NULL) return; kstat_runq_back_to_waitq(KSTAT_IO_PTR(vdp->xdf_xdev_iostat)); } int xdf_kstat_create(dev_info_t *dip) { xdf_t *vdp = (xdf_t *)ddi_get_driver_private(dip); kstat_t *kstat; buf_t *bp; if ((kstat = kstat_create("xdf", ddi_get_instance(dip), NULL, "disk", KSTAT_TYPE_IO, 1, KSTAT_FLAG_PERSISTENT)) == NULL) return (-1); /* See comment about locking in xdf_kstat_delete(). */ mutex_enter(&vdp->xdf_iostat_lk); mutex_enter(&vdp->xdf_dev_lk); /* only one kstat can exist at a time */ if (vdp->xdf_xdev_iostat != NULL) { mutex_exit(&vdp->xdf_dev_lk); mutex_exit(&vdp->xdf_iostat_lk); kstat_delete(kstat); return (-1); } vdp->xdf_xdev_iostat = kstat; vdp->xdf_xdev_iostat->ks_lock = &vdp->xdf_dev_lk; kstat_install(vdp->xdf_xdev_iostat); /* * Now that we've created a kstat, we need to update the waitq and * runq counts for the kstat to reflect our current state. * * For a buf_t structure to be on the runq, it must have a ring * buffer slot associated with it. To get a ring buffer slot the * buf must first have a v_req_t and a ge_slot_t associated with it. * Then when it is granted a ring buffer slot, v_runq will be set to * true. * * For a buf_t structure to be on the waitq, it must not be on the * runq. So to find all the buf_t's that should be on waitq, we * walk the active buf list and add any buf_t's which aren't on the * runq to the waitq. */ bp = vdp->xdf_f_act; while (bp != NULL) { xdf_kstat_enter(vdp, bp); bp = bp->av_forw; } if (vdp->xdf_ready_tq_bp != NULL) xdf_kstat_enter(vdp, vdp->xdf_ready_tq_bp); mutex_exit(&vdp->xdf_dev_lk); mutex_exit(&vdp->xdf_iostat_lk); return (0); } void xdf_kstat_delete(dev_info_t *dip) { xdf_t *vdp = (xdf_t *)ddi_get_driver_private(dip); kstat_t *kstat; buf_t *bp; /* * The locking order here is xdf_iostat_lk and then xdf_dev_lk. * xdf_dev_lk is used to protect the xdf_xdev_iostat pointer * and the contents of the our kstat. xdf_iostat_lk is used * to protect the allocation and freeing of the actual kstat. * xdf_dev_lk can't be used for this purpose because kstat * readers use it to access the contents of the kstat and * hence it can't be held when calling kstat_delete(). */ mutex_enter(&vdp->xdf_iostat_lk); mutex_enter(&vdp->xdf_dev_lk); if (vdp->xdf_xdev_iostat == NULL) { mutex_exit(&vdp->xdf_dev_lk); mutex_exit(&vdp->xdf_iostat_lk); return; } /* * We're about to destroy the kstat structures, so it isn't really * necessary to update the runq and waitq counts. But, since this * isn't a hot code path we can afford to be a little pedantic and * go ahead and decrement the runq and waitq kstat counters to zero * before free'ing them. This helps us ensure that we've gotten all * our accounting correct. * * For an explanation of how we determine which buffers go on the * runq vs which go on the waitq, see the comments in * xdf_kstat_create(). */ bp = vdp->xdf_f_act; while (bp != NULL) { xdf_kstat_exit(vdp, bp); bp = bp->av_forw; } if (vdp->xdf_ready_tq_bp != NULL) xdf_kstat_exit(vdp, vdp->xdf_ready_tq_bp); kstat = vdp->xdf_xdev_iostat; vdp->xdf_xdev_iostat = NULL; mutex_exit(&vdp->xdf_dev_lk); kstat_delete(kstat); mutex_exit(&vdp->xdf_iostat_lk); } /* * Add an IO requests onto the active queue. * * We have to detect IOs generated by xdf_ready_tq_thread. These IOs * are used to establish a connection to the backend, so they receive * priority over all other IOs. Since xdf_ready_tq_thread only does * synchronous IO, there can only be one xdf_ready_tq_thread request at any * given time and we record the buf associated with that request in * xdf_ready_tq_bp. */ static void xdf_bp_push(xdf_t *vdp, buf_t *bp) { ASSERT(MUTEX_HELD(&vdp->xdf_dev_lk)); ASSERT(bp->av_forw == NULL); xdf_kstat_enter(vdp, bp); if (curthread == vdp->xdf_ready_tq_thread) { /* new IO requests from the ready thread */ ASSERT(vdp->xdf_ready_tq_bp == NULL); vdp->xdf_ready_tq_bp = bp; return; } /* this is normal IO request */ ASSERT(bp != vdp->xdf_ready_tq_bp); if (vdp->xdf_f_act == NULL) { /* this is only only IO on the active queue */ ASSERT(vdp->xdf_l_act == NULL); ASSERT(vdp->xdf_i_act == NULL); vdp->xdf_f_act = vdp->xdf_l_act = vdp->xdf_i_act = bp; return; } /* add this IO to the tail of the active queue */ vdp->xdf_l_act->av_forw = bp; vdp->xdf_l_act = bp; if (vdp->xdf_i_act == NULL) vdp->xdf_i_act = bp; } static void xdf_bp_pop(xdf_t *vdp, buf_t *bp) { buf_t *bp_iter; ASSERT(MUTEX_HELD(&vdp->xdf_dev_lk)); ASSERT(VREQ_DONE(BP_VREQ(bp))); if (vdp->xdf_ready_tq_bp == bp) { /* we're done with a ready thread IO request */ ASSERT(bp->av_forw == NULL); vdp->xdf_ready_tq_bp = NULL; return; } /* we're done with a normal IO request */ ASSERT((bp->av_forw != NULL) || (bp == vdp->xdf_l_act)); ASSERT((bp->av_forw == NULL) || (bp != vdp->xdf_l_act)); ASSERT(VREQ_DONE(BP_VREQ(vdp->xdf_f_act))); ASSERT(vdp->xdf_f_act != vdp->xdf_i_act); if (bp == vdp->xdf_f_act) { /* This IO was at the head of our active queue. */ vdp->xdf_f_act = bp->av_forw; if (bp == vdp->xdf_l_act) vdp->xdf_l_act = NULL; } else { /* There IO finished before some other pending IOs. */ bp_iter = vdp->xdf_f_act; while (bp != bp_iter->av_forw) { bp_iter = bp_iter->av_forw; ASSERT(VREQ_DONE(BP_VREQ(bp_iter))); ASSERT(bp_iter != vdp->xdf_i_act); } bp_iter->av_forw = bp->av_forw; if (bp == vdp->xdf_l_act) vdp->xdf_l_act = bp_iter; } bp->av_forw = NULL; } static buf_t * xdf_bp_next(xdf_t *vdp) { v_req_t *vreq; buf_t *bp; if (vdp->xdf_state == XD_CONNECTED) { /* * If we're in the XD_CONNECTED state, we only service IOs * from the xdf_ready_tq_thread thread. */ if ((bp = vdp->xdf_ready_tq_bp) == NULL) return (NULL); if (((vreq = BP_VREQ(bp)) == NULL) || (!VREQ_DONE(vreq))) return (bp); return (NULL); } /* if we're not in the XD_CONNECTED or XD_READY state we can't do IO */ if (vdp->xdf_state != XD_READY) return (NULL); ASSERT(vdp->xdf_ready_tq_bp == NULL); for (;;) { if ((bp = vdp->xdf_i_act) == NULL) return (NULL); if (((vreq = BP_VREQ(bp)) == NULL) || (!VREQ_DONE(vreq))) return (bp); /* advance the active buf index pointer */ vdp->xdf_i_act = bp->av_forw; } } static void xdf_io_fini(xdf_t *vdp, uint64_t id, int bioerr) { ge_slot_t *gs = (ge_slot_t *)(uintptr_t)id; v_req_t *vreq = gs->gs_vreq; buf_t *bp = vreq->v_buf; ASSERT(MUTEX_HELD(&vdp->xdf_dev_lk)); ASSERT(BP_VREQ(bp) == vreq); gs_free(gs); if (bioerr != 0) bioerror(bp, bioerr); ASSERT(vreq->v_nslots > 0); if (--vreq->v_nslots > 0) return; /* remove this IO from our active queue */ xdf_bp_pop(vdp, bp); ASSERT(vreq->v_runq); xdf_kstat_exit(vdp, bp); vreq->v_runq = B_FALSE; vreq_free(vdp, vreq); if (IS_ERROR(bp)) { xdf_io_err(bp, geterror(bp), 0); } else if (bp->b_resid != 0) { /* Partial transfers are an error */ xdf_io_err(bp, EIO, bp->b_resid); } else { biodone(bp); } } /* * xdf interrupt handler */ static uint_t xdf_intr_locked(xdf_t *vdp) { xendev_ring_t *xbr; blkif_response_t *resp; int bioerr; uint64_t id; uint8_t op; uint16_t status; ddi_acc_handle_t acchdl; ASSERT(MUTEX_HELD(&vdp->xdf_dev_lk)); if ((xbr = vdp->xdf_xb_ring) == NULL) return (DDI_INTR_UNCLAIMED); acchdl = vdp->xdf_xb_ring_hdl; /* * complete all requests which have a response */ while (resp = xvdi_ring_get_response(xbr)) { id = ddi_get64(acchdl, &resp->id); op = ddi_get8(acchdl, &resp->operation); status = ddi_get16(acchdl, (uint16_t *)&resp->status); DPRINTF(INTR_DBG, ("resp: op %d id %"PRIu64" status %d\n", op, id, status)); if (status != BLKIF_RSP_OKAY) { DPRINTF(IO_DBG, ("xdf@%s: I/O error while %s", vdp->xdf_addr, (op == BLKIF_OP_READ) ? "reading" : "writing")); bioerr = EIO; } else { bioerr = 0; } xdf_io_fini(vdp, id, bioerr); } return (DDI_INTR_CLAIMED); } /* * xdf_intr runs at PIL 5, so no one else can grab xdf_dev_lk and * block at a lower pil. */ static uint_t xdf_intr(caddr_t arg) { xdf_t *vdp = (xdf_t *)arg; int rv; mutex_enter(&vdp->xdf_dev_lk); rv = xdf_intr_locked(vdp); mutex_exit(&vdp->xdf_dev_lk); if (!do_polled_io) xdf_io_start(vdp); return (rv); } static void xdf_ring_push(xdf_t *vdp) { ASSERT(MUTEX_HELD(&vdp->xdf_dev_lk)); if (vdp->xdf_xb_ring == NULL) return; if (xvdi_ring_push_request(vdp->xdf_xb_ring)) { DPRINTF(IO_DBG, ( "xdf@%s: xdf_ring_push: sent request(s) to backend\n", vdp->xdf_addr)); } if (xvdi_get_evtchn(vdp->xdf_dip) != INVALID_EVTCHN) xvdi_notify_oe(vdp->xdf_dip); } static int xdf_ring_drain_locked(xdf_t *vdp) { int pollc, rv = 0; ASSERT(MUTEX_HELD(&vdp->xdf_dev_lk)); if (xdf_debug & SUSRES_DBG) xen_printf("xdf_ring_drain: start\n"); for (pollc = 0; pollc < XDF_DRAIN_RETRY_COUNT; pollc++) { if (vdp->xdf_xb_ring == NULL) goto out; if (xvdi_ring_has_unconsumed_responses(vdp->xdf_xb_ring)) (void) xdf_intr_locked(vdp); if (!xvdi_ring_has_incomp_request(vdp->xdf_xb_ring)) goto out; xdf_ring_push(vdp); /* file-backed devices can be slow */ mutex_exit(&vdp->xdf_dev_lk); #ifdef XPV_HVM_DRIVER (void) HYPERVISOR_yield(); #endif /* XPV_HVM_DRIVER */ delay(drv_usectohz(XDF_DRAIN_MSEC_DELAY)); mutex_enter(&vdp->xdf_dev_lk); } cmn_err(CE_WARN, "xdf@%s: xdf_ring_drain: timeout", vdp->xdf_addr); out: if (vdp->xdf_xb_ring != NULL) { if (xvdi_ring_has_incomp_request(vdp->xdf_xb_ring) || xvdi_ring_has_unconsumed_responses(vdp->xdf_xb_ring)) rv = EIO; } if (xdf_debug & SUSRES_DBG) xen_printf("xdf@%s: xdf_ring_drain: end, err=%d\n", vdp->xdf_addr, rv); return (rv); } static int xdf_ring_drain(xdf_t *vdp) { int rv; mutex_enter(&vdp->xdf_dev_lk); rv = xdf_ring_drain_locked(vdp); mutex_exit(&vdp->xdf_dev_lk); return (rv); } /* * Destroy all v_req_t, grant table entries, and our ring buffer. */ static void xdf_ring_destroy(xdf_t *vdp) { v_req_t *vreq; buf_t *bp; ge_slot_t *gs; ASSERT(MUTEX_HELD(&vdp->xdf_cb_lk)); ASSERT(MUTEX_HELD(&vdp->xdf_dev_lk)); if ((vdp->xdf_state != XD_INIT) && (vdp->xdf_state != XD_CONNECTED) && (vdp->xdf_state != XD_READY)) { ASSERT(vdp->xdf_xb_ring == NULL); ASSERT(vdp->xdf_xb_ring_hdl == NULL); ASSERT(vdp->xdf_peer == INVALID_DOMID); ASSERT(vdp->xdf_evtchn == INVALID_EVTCHN); ASSERT(list_is_empty(&vdp->xdf_vreq_act)); return; } /* * We don't want to receive async notifications from the backend * when it finishes processing ring entries. */ #ifdef XPV_HVM_DRIVER ec_unbind_evtchn(vdp->xdf_evtchn); #else /* !XPV_HVM_DRIVER */ (void) ddi_remove_intr(vdp->xdf_dip, 0, NULL); #endif /* !XPV_HVM_DRIVER */ /* * Drain any requests in the ring. We need to do this before we * can free grant table entries, because if active ring entries * point to grants, then the backend could be trying to access * those grants. */ (void) xdf_ring_drain_locked(vdp); /* We're done talking to the backend so free up our event channel */ xvdi_free_evtchn(vdp->xdf_dip); vdp->xdf_evtchn = INVALID_EVTCHN; while ((vreq = list_head(&vdp->xdf_vreq_act)) != NULL) { bp = vreq->v_buf; ASSERT(BP_VREQ(bp) == vreq); /* Free up any grant table entries associaed with this IO */ while ((gs = list_head(&vreq->v_gs)) != NULL) gs_free(gs); /* If this IO was on the runq, move it back to the waitq. */ if (vreq->v_runq) xdf_kstat_runq_to_waitq(vdp, bp); /* * Reset any buf IO state since we're going to re-issue the * IO when we reconnect. */ vreq_free(vdp, vreq); BP_VREQ_SET(bp, NULL); bioerror(bp, 0); } /* reset the active queue index pointer */ vdp->xdf_i_act = vdp->xdf_f_act; /* Destroy the ring */ xvdi_free_ring(vdp->xdf_xb_ring); vdp->xdf_xb_ring = NULL; vdp->xdf_xb_ring_hdl = NULL; vdp->xdf_peer = INVALID_DOMID; } void xdfmin(struct buf *bp) { if (bp->b_bcount > xdf_maxphys) bp->b_bcount = xdf_maxphys; } /* * Check if we have a pending "eject" media request. */ static int xdf_eject_pending(xdf_t *vdp) { dev_info_t *dip = vdp->xdf_dip; char *xsname, *str; if (!vdp->xdf_media_req_supported) return (B_FALSE); if (((xsname = xvdi_get_xsname(dip)) == NULL) || (xenbus_read_str(xsname, XBP_MEDIA_REQ, &str) != 0)) return (B_FALSE); if (strcmp(str, XBV_MEDIA_REQ_EJECT) != 0) { strfree(str); return (B_FALSE); } strfree(str); return (B_TRUE); } /* * Generate a media request. */ static int xdf_media_req(xdf_t *vdp, char *req, boolean_t media_required) { dev_info_t *dip = vdp->xdf_dip; char *xsname; /* * we can't be holding xdf_dev_lk because xenbus_printf() can * block while waiting for a PIL 1 interrupt message. this * would cause a deadlock with xdf_intr() which needs to grab * xdf_dev_lk as well and runs at PIL 5. */ ASSERT(MUTEX_HELD(&vdp->xdf_cb_lk)); ASSERT(MUTEX_NOT_HELD(&vdp->xdf_dev_lk)); if ((xsname = xvdi_get_xsname(dip)) == NULL) return (ENXIO); /* Check if we support media requests */ if (!XD_IS_CD(vdp) || !vdp->xdf_media_req_supported) return (ENOTTY); /* If an eject is pending then don't allow any new requests */ if (xdf_eject_pending(vdp)) return (ENXIO); /* Make sure that there is media present */ if (media_required && (vdp->xdf_xdev_nblocks == 0)) return (ENXIO); /* We only allow operations when the device is ready and connected */ if (vdp->xdf_state != XD_READY) return (EIO); if (xenbus_printf(XBT_NULL, xsname, XBP_MEDIA_REQ, "%s", req) != 0) return (EIO); return (0); } /* * populate a single blkif_request_t w/ a buf */ static void xdf_process_rreq(xdf_t *vdp, struct buf *bp, blkif_request_t *rreq) { grant_ref_t gr; uint8_t fsect, lsect; size_t bcnt; paddr_t dma_addr; off_t blk_off; dev_info_t *dip = vdp->xdf_dip; blkif_vdev_t vdev = xvdi_get_vdevnum(dip); v_req_t *vreq = BP_VREQ(bp); uint64_t blkno = vreq->v_blkno; uint_t ndmacs = vreq->v_ndmacs; ddi_acc_handle_t acchdl = vdp->xdf_xb_ring_hdl; int seg = 0; int isread = IS_READ(bp); ge_slot_t *gs = list_head(&vreq->v_gs); ASSERT(MUTEX_HELD(&vdp->xdf_dev_lk)); ASSERT(vreq->v_status == VREQ_GS_ALLOCED); if (isread) ddi_put8(acchdl, &rreq->operation, BLKIF_OP_READ); else { switch (vreq->v_flush_diskcache) { case FLUSH_DISKCACHE: ddi_put8(acchdl, &rreq->operation, BLKIF_OP_FLUSH_DISKCACHE); ddi_put16(acchdl, &rreq->handle, vdev); ddi_put64(acchdl, &rreq->id, (uint64_t)(uintptr_t)(gs)); ddi_put8(acchdl, &rreq->nr_segments, 0); vreq->v_status = VREQ_DMAWIN_DONE; return; case WRITE_BARRIER: ddi_put8(acchdl, &rreq->operation, BLKIF_OP_WRITE_BARRIER); break; default: if (!vdp->xdf_wce) ddi_put8(acchdl, &rreq->operation, BLKIF_OP_WRITE_BARRIER); else ddi_put8(acchdl, &rreq->operation, BLKIF_OP_WRITE); break; } } ddi_put16(acchdl, &rreq->handle, vdev); ddi_put64(acchdl, &rreq->sector_number, blkno); ddi_put64(acchdl, &rreq->id, (uint64_t)(uintptr_t)(gs)); /* * loop until all segments are populated or no more dma cookie in buf */ for (;;) { /* * Each segment of a blkif request can transfer up to * one 4K page of data. */ bcnt = vreq->v_dmac.dmac_size; dma_addr = vreq->v_dmac.dmac_laddress; blk_off = (uint_t)((paddr_t)XB_SEGOFFSET & dma_addr); fsect = blk_off >> XB_BSHIFT; lsect = fsect + (bcnt >> XB_BSHIFT) - 1; ASSERT(bcnt <= PAGESIZE); ASSERT((bcnt % XB_BSIZE) == 0); ASSERT((blk_off & XB_BMASK) == 0); ASSERT(fsect < XB_MAX_SEGLEN / XB_BSIZE && lsect < XB_MAX_SEGLEN / XB_BSIZE); gr = gs_grant(gs, PATOMA(dma_addr) >> PAGESHIFT); ddi_put32(acchdl, &rreq->seg[seg].gref, gr); ddi_put8(acchdl, &rreq->seg[seg].first_sect, fsect); ddi_put8(acchdl, &rreq->seg[seg].last_sect, lsect); DPRINTF(IO_DBG, ( "xdf@%s: seg%d: dmacS %lu blk_off %ld\n", vdp->xdf_addr, seg, vreq->v_dmac.dmac_size, blk_off)); DPRINTF(IO_DBG, ( "xdf@%s: seg%d: fs %d ls %d gr %d dma 0x%"PRIx64"\n", vdp->xdf_addr, seg, fsect, lsect, gr, dma_addr)); blkno += (bcnt >> XB_BSHIFT); seg++; ASSERT(seg <= BLKIF_MAX_SEGMENTS_PER_REQUEST); if (--ndmacs) { ddi_dma_nextcookie(vreq->v_dmahdl, &vreq->v_dmac); continue; } vreq->v_status = VREQ_DMAWIN_DONE; vreq->v_blkno = blkno; break; } ddi_put8(acchdl, &rreq->nr_segments, seg); DPRINTF(IO_DBG, ( "xdf@%s: xdf_process_rreq: request id=%"PRIx64" ready\n", vdp->xdf_addr, rreq->id)); } static void xdf_io_start(xdf_t *vdp) { struct buf *bp; v_req_t *vreq; blkif_request_t *rreq; boolean_t rreqready = B_FALSE; mutex_enter(&vdp->xdf_dev_lk); /* * Populate the ring request(s). Loop until there is no buf to * transfer or no free slot available in I/O ring. */ for (;;) { /* don't start any new IO if we're suspending */ if (vdp->xdf_suspending) break; if ((bp = xdf_bp_next(vdp)) == NULL) break; /* if the buf doesn't already have a vreq, allocate one */ if (((vreq = BP_VREQ(bp)) == NULL) && ((vreq = vreq_get(vdp, bp)) == NULL)) break; /* alloc DMA/GTE resources */ if (vreq_setup(vdp, vreq) != DDI_SUCCESS) break; /* get next blkif_request in the ring */ if ((rreq = xvdi_ring_get_request(vdp->xdf_xb_ring)) == NULL) break; bzero(rreq, sizeof (blkif_request_t)); rreqready = B_TRUE; /* populate blkif_request with this buf */ xdf_process_rreq(vdp, bp, rreq); /* * This buffer/vreq pair is has been allocated a ring buffer * resources, so if it isn't already in our runq, add it. */ if (!vreq->v_runq) xdf_kstat_waitq_to_runq(vdp, bp); } /* Send the request(s) to the backend */ if (rreqready) xdf_ring_push(vdp); mutex_exit(&vdp->xdf_dev_lk); } /* check if partition is open, -1 - check all partitions on the disk */ static boolean_t xdf_isopen(xdf_t *vdp, int partition) { int i; ulong_t parbit; boolean_t rval = B_FALSE; ASSERT((partition == -1) || ((partition >= 0) || (partition < XDF_PEXT))); if (partition == -1) parbit = (ulong_t)-1; else parbit = 1 << partition; for (i = 0; i < OTYPCNT; i++) { if (vdp->xdf_vd_open[i] & parbit) rval = B_TRUE; } return (rval); } /* * The connection should never be closed as long as someone is holding * us open, there is pending IO, or someone is waiting waiting for a * connection. */ static boolean_t xdf_busy(xdf_t *vdp) { ASSERT(MUTEX_HELD(&vdp->xdf_dev_lk)); if ((vdp->xdf_xb_ring != NULL) && xvdi_ring_has_unconsumed_responses(vdp->xdf_xb_ring)) { ASSERT(vdp->xdf_state != XD_CLOSED); return (B_TRUE); } if (!list_is_empty(&vdp->xdf_vreq_act) || (vdp->xdf_f_act != NULL)) { ASSERT(vdp->xdf_state != XD_CLOSED); return (B_TRUE); } if (xdf_isopen(vdp, -1)) { ASSERT(vdp->xdf_state != XD_CLOSED); return (B_TRUE); } if (vdp->xdf_connect_req > 0) { ASSERT(vdp->xdf_state != XD_CLOSED); return (B_TRUE); } return (B_FALSE); } static void xdf_set_state(xdf_t *vdp, xdf_state_t new_state) { ASSERT(MUTEX_HELD(&vdp->xdf_cb_lk)); ASSERT(MUTEX_HELD(&vdp->xdf_dev_lk)); DPRINTF(DDI_DBG, ("xdf@%s: state change %d -> %d\n", vdp->xdf_addr, vdp->xdf_state, new_state)); vdp->xdf_state = new_state; cv_broadcast(&vdp->xdf_dev_cv); } static void xdf_disconnect(xdf_t *vdp, xdf_state_t new_state, boolean_t quiet) { dev_info_t *dip = vdp->xdf_dip; boolean_t busy; ASSERT(MUTEX_HELD(&vdp->xdf_cb_lk)); ASSERT(MUTEX_NOT_HELD(&vdp->xdf_dev_lk)); ASSERT((new_state == XD_UNKNOWN) || (new_state == XD_CLOSED)); /* Check if we're already there. */ if (vdp->xdf_state == new_state) return; mutex_enter(&vdp->xdf_dev_lk); busy = xdf_busy(vdp); /* If we're already closed then there's nothing todo. */ if (vdp->xdf_state == XD_CLOSED) { ASSERT(!busy); xdf_set_state(vdp, new_state); mutex_exit(&vdp->xdf_dev_lk); return; } #ifdef DEBUG /* UhOh. Warn the user that something bad has happened. */ if (!quiet && busy && (vdp->xdf_state == XD_READY) && (vdp->xdf_xdev_nblocks != 0)) { cmn_err(CE_WARN, "xdf@%s: disconnected while in use", vdp->xdf_addr); } #endif /* DEBUG */ xdf_ring_destroy(vdp); /* If we're busy then we can only go into the unknown state */ xdf_set_state(vdp, (busy) ? XD_UNKNOWN : new_state); mutex_exit(&vdp->xdf_dev_lk); /* if we're closed now, let the other end know */ if (vdp->xdf_state == XD_CLOSED) (void) xvdi_switch_state(dip, XBT_NULL, XenbusStateClosed); } /* * Kick-off connect process * Status should be XD_UNKNOWN or XD_CLOSED * On success, status will be changed to XD_INIT * On error, it will be changed to XD_UNKNOWN */ static int xdf_setstate_init(xdf_t *vdp) { dev_info_t *dip = vdp->xdf_dip; xenbus_transaction_t xbt; grant_ref_t gref; char *xsname, *str; int rv; ASSERT(MUTEX_HELD(&vdp->xdf_cb_lk)); ASSERT(MUTEX_NOT_HELD(&vdp->xdf_dev_lk)); ASSERT((vdp->xdf_state == XD_UNKNOWN) || (vdp->xdf_state == XD_CLOSED)); DPRINTF(DDI_DBG, ("xdf@%s: starting connection process\n", vdp->xdf_addr)); /* * If an eject is pending then don't allow a new connection. * (Only the backend can clear media request eject request.) */ if (xdf_eject_pending(vdp)) return (DDI_FAILURE); if ((xsname = xvdi_get_xsname(dip)) == NULL) goto errout; if ((vdp->xdf_peer = xvdi_get_oeid(dip)) == INVALID_DOMID) goto errout; (void) xvdi_switch_state(dip, XBT_NULL, XenbusStateInitialising); /* * Sanity check for the existance of the xenbus device-type property. * This property might not exist if our xenbus device nodes were * force destroyed while we were still connected to the backend. */ if (xenbus_read_str(xsname, XBP_DEV_TYPE, &str) != 0) goto errout; strfree(str); if (xvdi_alloc_evtchn(dip) != DDI_SUCCESS) goto errout; vdp->xdf_evtchn = xvdi_get_evtchn(dip); #ifdef XPV_HVM_DRIVER ec_bind_evtchn_to_handler(vdp->xdf_evtchn, IPL_VBD, xdf_intr, vdp); #else /* !XPV_HVM_DRIVER */ if (ddi_add_intr(dip, 0, NULL, NULL, xdf_intr, (caddr_t)vdp) != DDI_SUCCESS) { cmn_err(CE_WARN, "xdf@%s: xdf_setstate_init: " "failed to add intr handler", vdp->xdf_addr); goto errout1; } #endif /* !XPV_HVM_DRIVER */ if (xvdi_alloc_ring(dip, BLKIF_RING_SIZE, sizeof (union blkif_sring_entry), &gref, &vdp->xdf_xb_ring) != DDI_SUCCESS) { cmn_err(CE_WARN, "xdf@%s: failed to alloc comm ring", vdp->xdf_addr); goto errout2; } vdp->xdf_xb_ring_hdl = vdp->xdf_xb_ring->xr_acc_hdl; /* ugly!! */ /* * Write into xenstore the info needed by backend */ trans_retry: if (xenbus_transaction_start(&xbt)) { cmn_err(CE_WARN, "xdf@%s: failed to start transaction", vdp->xdf_addr); xvdi_fatal_error(dip, EIO, "connect transaction init"); goto fail_trans; } /* * XBP_PROTOCOL is written by the domain builder in the case of PV * domains. However, it is not written for HVM domains, so let's * write it here. */ if (((rv = xenbus_printf(xbt, xsname, XBP_MEDIA_REQ, "%s", XBV_MEDIA_REQ_NONE)) != 0) || ((rv = xenbus_printf(xbt, xsname, XBP_RING_REF, "%u", gref)) != 0) || ((rv = xenbus_printf(xbt, xsname, XBP_EVENT_CHAN, "%u", vdp->xdf_evtchn)) != 0) || ((rv = xenbus_printf(xbt, xsname, XBP_PROTOCOL, "%s", XEN_IO_PROTO_ABI_NATIVE)) != 0) || ((rv = xvdi_switch_state(dip, xbt, XenbusStateInitialised)) > 0)) { (void) xenbus_transaction_end(xbt, 1); xvdi_fatal_error(dip, rv, "connect transaction setup"); goto fail_trans; } /* kick-off connect process */ if (rv = xenbus_transaction_end(xbt, 0)) { if (rv == EAGAIN) goto trans_retry; xvdi_fatal_error(dip, rv, "connect transaction commit"); goto fail_trans; } ASSERT(MUTEX_HELD(&vdp->xdf_cb_lk)); mutex_enter(&vdp->xdf_dev_lk); xdf_set_state(vdp, XD_INIT); mutex_exit(&vdp->xdf_dev_lk); return (DDI_SUCCESS); fail_trans: xvdi_free_ring(vdp->xdf_xb_ring); errout2: #ifdef XPV_HVM_DRIVER ec_unbind_evtchn(vdp->xdf_evtchn); #else /* !XPV_HVM_DRIVER */ (void) ddi_remove_intr(vdp->xdf_dip, 0, NULL); errout1: #endif /* !XPV_HVM_DRIVER */ xvdi_free_evtchn(dip); vdp->xdf_evtchn = INVALID_EVTCHN; errout: xdf_disconnect(vdp, XD_UNKNOWN, B_FALSE); cmn_err(CE_WARN, "xdf@%s: failed to start connection to backend", vdp->xdf_addr); return (DDI_FAILURE); } int xdf_get_flush_block(xdf_t *vdp) { /* * Get a DEV_BSIZE aligned bufer */ vdp->xdf_flush_mem = kmem_alloc(vdp->xdf_xdev_secsize * 2, KM_SLEEP); vdp->xdf_cache_flush_block = (char *)P2ROUNDUP((uintptr_t)(vdp->xdf_flush_mem), (int)vdp->xdf_xdev_secsize); if (xdf_lb_rdwr(vdp->xdf_dip, TG_READ, vdp->xdf_cache_flush_block, xdf_flush_block, vdp->xdf_xdev_secsize, NULL) != 0) return (DDI_FAILURE); return (DDI_SUCCESS); } static void xdf_setstate_ready(void *arg) { xdf_t *vdp = (xdf_t *)arg; dev_info_t *dip = vdp->xdf_dip; vdp->xdf_ready_tq_thread = curthread; /* Create minor nodes now when we are almost connected */ mutex_enter(&vdp->xdf_dev_lk); if (vdp->xdf_cmlb_reattach) { vdp->xdf_cmlb_reattach = B_FALSE; mutex_exit(&vdp->xdf_dev_lk); if (xdf_cmlb_attach(vdp) != 0) { cmn_err(CE_WARN, "xdf@%s: cmlb attach failed", ddi_get_name_addr(dip)); xdf_disconnect(vdp, XD_UNKNOWN, B_FALSE); return; } mutex_enter(&vdp->xdf_dev_lk); } /* If we're not still trying to get to the ready state, then bail. */ if (vdp->xdf_state != XD_CONNECTED) { mutex_exit(&vdp->xdf_dev_lk); return; } mutex_exit(&vdp->xdf_dev_lk); /* * If backend has feature-barrier, see if it supports disk * cache flush op. */ vdp->xdf_flush_supported = B_FALSE; if (vdp->xdf_feature_barrier) { /* * Pretend we already know flush is supported so probe * will attempt the correct op. */ vdp->xdf_flush_supported = B_TRUE; if (xdf_lb_rdwr(vdp->xdf_dip, TG_WRITE, NULL, 0, 0, 0) == 0) { vdp->xdf_flush_supported = B_TRUE; } else { vdp->xdf_flush_supported = B_FALSE; /* * If the other end does not support the cache flush op * then we must use a barrier-write to force disk * cache flushing. Barrier writes require that a data * block actually be written. * Cache a block to barrier-write when we are * asked to perform a flush. * XXX - would it be better to just copy 1 block * (512 bytes) from whatever write we did last * and rewrite that block? */ if (xdf_get_flush_block(vdp) != DDI_SUCCESS) { xdf_disconnect(vdp, XD_UNKNOWN, B_FALSE); return; } } } mutex_enter(&vdp->xdf_cb_lk); mutex_enter(&vdp->xdf_dev_lk); if (vdp->xdf_state == XD_CONNECTED) xdf_set_state(vdp, XD_READY); mutex_exit(&vdp->xdf_dev_lk); /* Restart any currently queued up io */ xdf_io_start(vdp); mutex_exit(&vdp->xdf_cb_lk); } /* * synthetic geometry */ #define XDF_NSECTS 256 #define XDF_NHEADS 16 static void xdf_synthetic_pgeom(dev_info_t *dip, cmlb_geom_t *geomp) { xdf_t *vdp; uint_t ncyl; vdp = ddi_get_soft_state(xdf_ssp, ddi_get_instance(dip)); ncyl = vdp->xdf_xdev_nblocks / (XDF_NHEADS * XDF_NSECTS); bzero(geomp, sizeof (*geomp)); geomp->g_ncyl = ncyl == 0 ? 1 : ncyl; geomp->g_acyl = 0; geomp->g_nhead = XDF_NHEADS; geomp->g_nsect = XDF_NSECTS; geomp->g_secsize = vdp->xdf_xdev_secsize; geomp->g_capacity = vdp->xdf_xdev_nblocks; geomp->g_intrlv = 0; geomp->g_rpm = 7200; } /* * Finish other initialization after we've connected to backend * Status should be XD_INIT before calling this routine * On success, status should be changed to XD_CONNECTED. * On error, status should stay XD_INIT */ static int xdf_setstate_connected(xdf_t *vdp) { dev_info_t *dip = vdp->xdf_dip; cmlb_geom_t pgeom; diskaddr_t nblocks = 0; uint_t secsize = 0; char *oename, *xsname, *str; uint_t dinfo; ASSERT(MUTEX_HELD(&vdp->xdf_cb_lk)); ASSERT(MUTEX_NOT_HELD(&vdp->xdf_dev_lk)); ASSERT(vdp->xdf_state == XD_INIT); if (((xsname = xvdi_get_xsname(dip)) == NULL) || ((oename = xvdi_get_oename(dip)) == NULL)) return (DDI_FAILURE); /* Make sure the other end is XenbusStateConnected */ if (xenbus_read_driver_state(oename) != XenbusStateConnected) return (DDI_FAILURE); /* Determine if feature barrier is supported by backend */ if (!(vdp->xdf_feature_barrier = xenbus_exists(oename, XBP_FB))) cmn_err(CE_NOTE, "!xdf@%s: feature-barrier not supported", vdp->xdf_addr); /* * Probe backend. Read the device size into xdf_xdev_nblocks * and set the VDISK_READONLY, VDISK_CDROM, and VDISK_REMOVABLE * flags in xdf_dinfo. If the emulated device type is "cdrom", * we always set VDISK_CDROM, regardless of if it's present in * the xenbus info parameter. */ if (xenbus_gather(XBT_NULL, oename, XBP_SECTORS, "%"SCNu64, &nblocks, XBP_SECTOR_SIZE, "%u", &secsize, XBP_INFO, "%u", &dinfo, NULL) != 0) { cmn_err(CE_WARN, "xdf@%s: xdf_setstate_connected: " "cannot read backend info", vdp->xdf_addr); return (DDI_FAILURE); } if (xenbus_read_str(xsname, XBP_DEV_TYPE, &str) != 0) { cmn_err(CE_WARN, "xdf@%s: cannot read device-type", vdp->xdf_addr); return (DDI_FAILURE); } if (strcmp(str, XBV_DEV_TYPE_CD) == 0) dinfo |= VDISK_CDROM; strfree(str); if (secsize == 0 || !(ISP2(secsize / DEV_BSIZE))) secsize = DEV_BSIZE; vdp->xdf_xdev_nblocks = nblocks; vdp->xdf_xdev_secsize = secsize; #ifdef _ILP32 if (vdp->xdf_xdev_nblocks > DK_MAX_BLOCKS) { cmn_err(CE_WARN, "xdf@%s: xdf_setstate_connected: " "backend disk device too large with %llu blocks for" " 32-bit kernel", vdp->xdf_addr, vdp->xdf_xdev_nblocks); xvdi_fatal_error(dip, EFBIG, "reading backend info"); return (DDI_FAILURE); } #endif /* * If the physical geometry for a fixed disk has been explicity * set then make sure that the specified physical geometry isn't * larger than the device we connected to. */ if (vdp->xdf_pgeom_fixed && (vdp->xdf_pgeom.g_capacity > vdp->xdf_xdev_nblocks)) { cmn_err(CE_WARN, "xdf@%s: connect failed, fixed geometry too large", vdp->xdf_addr); return (DDI_FAILURE); } vdp->xdf_media_req_supported = xenbus_exists(oename, XBP_MEDIA_REQ_SUP); /* mark vbd is ready for I/O */ mutex_enter(&vdp->xdf_dev_lk); xdf_set_state(vdp, XD_CONNECTED); /* check if the cmlb label should be updated */ xdf_synthetic_pgeom(dip, &pgeom); if ((vdp->xdf_dinfo != dinfo) || (!vdp->xdf_pgeom_fixed && (memcmp(&vdp->xdf_pgeom, &pgeom, sizeof (pgeom)) != 0))) { vdp->xdf_cmlb_reattach = B_TRUE; vdp->xdf_dinfo = dinfo; if (!vdp->xdf_pgeom_fixed) vdp->xdf_pgeom = pgeom; } if (XD_IS_CD(vdp) || XD_IS_RM(vdp)) { if (vdp->xdf_xdev_nblocks == 0) { vdp->xdf_mstate = DKIO_EJECTED; cv_broadcast(&vdp->xdf_mstate_cv); } else { vdp->xdf_mstate = DKIO_INSERTED; cv_broadcast(&vdp->xdf_mstate_cv); } } else { if (vdp->xdf_mstate != DKIO_NONE) { vdp->xdf_mstate = DKIO_NONE; cv_broadcast(&vdp->xdf_mstate_cv); } } mutex_exit(&vdp->xdf_dev_lk); cmn_err(CE_CONT, "?xdf@%s: %"PRIu64" blocks", vdp->xdf_addr, (uint64_t)vdp->xdf_xdev_nblocks); /* Restart any currently queued up io */ xdf_io_start(vdp); /* * To get to the ready state we have to do IO to the backend device, * but we can't initiate IO from the other end change callback thread * (which is the current context we're executing in.) This is because * if the other end disconnects while we're doing IO from the callback * thread, then we can't receive that disconnect event and we hang * waiting for an IO that can never complete. */ (void) ddi_taskq_dispatch(vdp->xdf_ready_tq, xdf_setstate_ready, vdp, DDI_SLEEP); (void) xvdi_switch_state(dip, XBT_NULL, XenbusStateConnected); return (DDI_SUCCESS); } /*ARGSUSED*/ static void xdf_oe_change(dev_info_t *dip, ddi_eventcookie_t id, void *arg, void *impl_data) { XenbusState new_state = *(XenbusState *)impl_data; xdf_t *vdp = (xdf_t *)ddi_get_driver_private(dip); DPRINTF(DDI_DBG, ("xdf@%s: otherend state change to %d!\n", vdp->xdf_addr, new_state)); mutex_enter(&vdp->xdf_cb_lk); /* We assume that this callback is single threaded */ ASSERT(vdp->xdf_oe_change_thread == NULL); DEBUG_EVAL(vdp->xdf_oe_change_thread = curthread); /* ignore any backend state changes if we're suspending/suspended */ if (vdp->xdf_suspending || (vdp->xdf_state == XD_SUSPEND)) { DEBUG_EVAL(vdp->xdf_oe_change_thread = NULL); mutex_exit(&vdp->xdf_cb_lk); return; } switch (new_state) { case XenbusStateUnknown: case XenbusStateInitialising: case XenbusStateInitWait: case XenbusStateInitialised: if (vdp->xdf_state == XD_INIT) break; xdf_disconnect(vdp, XD_UNKNOWN, B_FALSE); if (xdf_setstate_init(vdp) != DDI_SUCCESS) break; ASSERT(vdp->xdf_state == XD_INIT); break; case XenbusStateConnected: if ((vdp->xdf_state == XD_CONNECTED) || (vdp->xdf_state == XD_READY)) break; if (vdp->xdf_state != XD_INIT) { xdf_disconnect(vdp, XD_UNKNOWN, B_FALSE); if (xdf_setstate_init(vdp) != DDI_SUCCESS) break; ASSERT(vdp->xdf_state == XD_INIT); } if (xdf_setstate_connected(vdp) != DDI_SUCCESS) { xdf_disconnect(vdp, XD_UNKNOWN, B_FALSE); break; } ASSERT(vdp->xdf_state == XD_CONNECTED); break; case XenbusStateClosing: if (xdf_isopen(vdp, -1)) { cmn_err(CE_NOTE, "xdf@%s: hot-unplug failed, still in use", vdp->xdf_addr); break; } /*FALLTHROUGH*/ case XenbusStateClosed: xdf_disconnect(vdp, XD_CLOSED, B_FALSE); break; } /* notify anybody waiting for oe state change */ cv_broadcast(&vdp->xdf_dev_cv); DEBUG_EVAL(vdp->xdf_oe_change_thread = NULL); mutex_exit(&vdp->xdf_cb_lk); } static int xdf_connect_locked(xdf_t *vdp, boolean_t wait) { int rv, timeouts = 0, reset = 20; ASSERT(MUTEX_HELD(&vdp->xdf_cb_lk)); ASSERT(MUTEX_HELD(&vdp->xdf_dev_lk)); /* we can't connect once we're in the closed state */ if (vdp->xdf_state == XD_CLOSED) return (XD_CLOSED); vdp->xdf_connect_req++; while (vdp->xdf_state != XD_READY) { mutex_exit(&vdp->xdf_dev_lk); /* only one thread at a time can be the connection thread */ if (vdp->xdf_connect_thread == NULL) vdp->xdf_connect_thread = curthread; if (vdp->xdf_connect_thread == curthread) { if ((timeouts > 0) && ((timeouts % reset) == 0)) { /* * If we haven't establised a connection * within the reset time, then disconnect * so we can try again, and double the reset * time. The reset time starts at 2 sec. */ (void) xdf_disconnect(vdp, XD_UNKNOWN, B_TRUE); reset *= 2; } if (vdp->xdf_state == XD_UNKNOWN) (void) xdf_setstate_init(vdp); if (vdp->xdf_state == XD_INIT) (void) xdf_setstate_connected(vdp); } mutex_enter(&vdp->xdf_dev_lk); if (!wait || (vdp->xdf_state == XD_READY)) goto out; mutex_exit((&vdp->xdf_cb_lk)); if (vdp->xdf_connect_thread != curthread) { rv = cv_wait_sig(&vdp->xdf_dev_cv, &vdp->xdf_dev_lk); } else { /* delay for 0.1 sec */ rv = cv_reltimedwait_sig(&vdp->xdf_dev_cv, &vdp->xdf_dev_lk, drv_usectohz(100*1000), TR_CLOCK_TICK); if (rv == -1) timeouts++; } mutex_exit((&vdp->xdf_dev_lk)); mutex_enter((&vdp->xdf_cb_lk)); mutex_enter((&vdp->xdf_dev_lk)); if (rv == 0) goto out; } out: ASSERT(MUTEX_HELD(&vdp->xdf_cb_lk)); ASSERT(MUTEX_HELD(&vdp->xdf_dev_lk)); if (vdp->xdf_connect_thread == curthread) { /* * wake up someone else so they can become the connection * thread. */ cv_signal(&vdp->xdf_dev_cv); vdp->xdf_connect_thread = NULL; } /* Try to lock the media */ mutex_exit((&vdp->xdf_dev_lk)); (void) xdf_media_req(vdp, XBV_MEDIA_REQ_LOCK, B_TRUE); mutex_enter((&vdp->xdf_dev_lk)); vdp->xdf_connect_req--; return (vdp->xdf_state); } static uint_t xdf_iorestart(caddr_t arg) { xdf_t *vdp = (xdf_t *)arg; ASSERT(vdp != NULL); mutex_enter(&vdp->xdf_dev_lk); ASSERT(ISDMACBON(vdp)); SETDMACBOFF(vdp); mutex_exit(&vdp->xdf_dev_lk); xdf_io_start(vdp); return (DDI_INTR_CLAIMED); } #ifdef XPV_HVM_DRIVER typedef struct xdf_hvm_entry { list_node_t xdf_he_list; char *xdf_he_path; dev_info_t *xdf_he_dip; } xdf_hvm_entry_t; static list_t xdf_hvm_list; static kmutex_t xdf_hvm_list_lock; static xdf_hvm_entry_t * i_xdf_hvm_find(const char *path, dev_info_t *dip) { xdf_hvm_entry_t *i; ASSERT((path != NULL) || (dip != NULL)); ASSERT(MUTEX_HELD(&xdf_hvm_list_lock)); i = list_head(&xdf_hvm_list); while (i != NULL) { if ((path != NULL) && strcmp(i->xdf_he_path, path) != 0) { i = list_next(&xdf_hvm_list, i); continue; } if ((dip != NULL) && (i->xdf_he_dip != dip)) { i = list_next(&xdf_hvm_list, i); continue; } break; } return (i); } dev_info_t * xdf_hvm_hold(const char *path) { xdf_hvm_entry_t *i; dev_info_t *dip; mutex_enter(&xdf_hvm_list_lock); i = i_xdf_hvm_find(path, NULL); if (i == NULL) { mutex_exit(&xdf_hvm_list_lock); return (B_FALSE); } ndi_hold_devi(dip = i->xdf_he_dip); mutex_exit(&xdf_hvm_list_lock); return (dip); } static void xdf_hvm_add(dev_info_t *dip) { xdf_hvm_entry_t *i; char *path; /* figure out the path for the dip */ path = kmem_zalloc(MAXPATHLEN, KM_SLEEP); (void) ddi_pathname(dip, path); i = kmem_alloc(sizeof (*i), KM_SLEEP); i->xdf_he_dip = dip; i->xdf_he_path = i_ddi_strdup(path, KM_SLEEP); mutex_enter(&xdf_hvm_list_lock); ASSERT(i_xdf_hvm_find(path, NULL) == NULL); ASSERT(i_xdf_hvm_find(NULL, dip) == NULL); list_insert_head(&xdf_hvm_list, i); mutex_exit(&xdf_hvm_list_lock); kmem_free(path, MAXPATHLEN); } static void xdf_hvm_rm(dev_info_t *dip) { xdf_hvm_entry_t *i; mutex_enter(&xdf_hvm_list_lock); VERIFY((i = i_xdf_hvm_find(NULL, dip)) != NULL); list_remove(&xdf_hvm_list, i); mutex_exit(&xdf_hvm_list_lock); kmem_free(i->xdf_he_path, strlen(i->xdf_he_path) + 1); kmem_free(i, sizeof (*i)); } static void xdf_hvm_init(void) { list_create(&xdf_hvm_list, sizeof (xdf_hvm_entry_t), offsetof(xdf_hvm_entry_t, xdf_he_list)); mutex_init(&xdf_hvm_list_lock, NULL, MUTEX_DEFAULT, NULL); } static void xdf_hvm_fini(void) { ASSERT(list_head(&xdf_hvm_list) == NULL); list_destroy(&xdf_hvm_list); mutex_destroy(&xdf_hvm_list_lock); } boolean_t xdf_hvm_connect(dev_info_t *dip) { xdf_t *vdp = (xdf_t *)ddi_get_driver_private(dip); char *oename, *str; int rv; mutex_enter(&vdp->xdf_cb_lk); /* * Before try to establish a connection we need to wait for the * backend hotplug scripts to have run. Once they are run the * "/hotplug-status" property will be set to "connected". */ for (;;) { ASSERT(MUTEX_HELD(&vdp->xdf_cb_lk)); /* * Get the xenbus path to the backend device. Note that * we can't cache this path (and we look it up on each pass * through this loop) because it could change during * suspend, resume, and migration operations. */ if ((oename = xvdi_get_oename(dip)) == NULL) { mutex_exit(&vdp->xdf_cb_lk); return (B_FALSE); } str = NULL; if ((xenbus_read_str(oename, XBP_HP_STATUS, &str) == 0) && (strcmp(str, XBV_HP_STATUS_CONN) == 0)) break; if (str != NULL) strfree(str); /* wait for an update to "/hotplug-status" */ if (cv_wait_sig(&vdp->xdf_hp_status_cv, &vdp->xdf_cb_lk) == 0) { /* we got interrupted by a signal */ mutex_exit(&vdp->xdf_cb_lk); return (B_FALSE); } } /* Good news. The backend hotplug scripts have been run. */ ASSERT(MUTEX_HELD(&vdp->xdf_cb_lk)); ASSERT(strcmp(str, XBV_HP_STATUS_CONN) == 0); strfree(str); /* * If we're emulating a cd device and if the backend doesn't support * media request opreations, then we're not going to bother trying * to establish a connection for a couple reasons. First off, media * requests support is required to support operations like eject and * media locking. Second, other backend platforms like Linux don't * support hvm pv cdrom access. They don't even have a backend pv * driver for cdrom device nodes, so we don't want to block forever * waiting for a connection to a backend driver that doesn't exist. */ if (XD_IS_CD(vdp) && !xenbus_exists(oename, XBP_MEDIA_REQ_SUP)) { mutex_exit(&vdp->xdf_cb_lk); return (B_FALSE); } mutex_enter(&vdp->xdf_dev_lk); rv = xdf_connect_locked(vdp, B_TRUE); mutex_exit(&vdp->xdf_dev_lk); mutex_exit(&vdp->xdf_cb_lk); return ((rv == XD_READY) ? B_TRUE : B_FALSE); } int xdf_hvm_setpgeom(dev_info_t *dip, cmlb_geom_t *geomp) { xdf_t *vdp = (xdf_t *)ddi_get_driver_private(dip); /* sanity check the requested physical geometry */ mutex_enter(&vdp->xdf_dev_lk); if ((geomp->g_secsize != XB_BSIZE) || (geomp->g_capacity == 0)) { mutex_exit(&vdp->xdf_dev_lk); return (EINVAL); } /* * If we've already connected to the backend device then make sure * we're not defining a physical geometry larger than our backend * device. */ if ((vdp->xdf_xdev_nblocks != 0) && (geomp->g_capacity > vdp->xdf_xdev_nblocks)) { mutex_exit(&vdp->xdf_dev_lk); return (EINVAL); } bzero(&vdp->xdf_pgeom, sizeof (vdp->xdf_pgeom)); vdp->xdf_pgeom.g_ncyl = geomp->g_ncyl; vdp->xdf_pgeom.g_acyl = geomp->g_acyl; vdp->xdf_pgeom.g_nhead = geomp->g_nhead; vdp->xdf_pgeom.g_nsect = geomp->g_nsect; vdp->xdf_pgeom.g_secsize = geomp->g_secsize; vdp->xdf_pgeom.g_capacity = geomp->g_capacity; vdp->xdf_pgeom.g_intrlv = geomp->g_intrlv; vdp->xdf_pgeom.g_rpm = geomp->g_rpm; vdp->xdf_pgeom_fixed = B_TRUE; mutex_exit(&vdp->xdf_dev_lk); /* force a re-validation */ cmlb_invalidate(vdp->xdf_vd_lbl, NULL); return (0); } boolean_t xdf_is_cd(dev_info_t *dip) { xdf_t *vdp = (xdf_t *)ddi_get_driver_private(dip); boolean_t rv; mutex_enter(&vdp->xdf_cb_lk); rv = XD_IS_CD(vdp); mutex_exit(&vdp->xdf_cb_lk); return (rv); } boolean_t xdf_is_rm(dev_info_t *dip) { xdf_t *vdp = (xdf_t *)ddi_get_driver_private(dip); boolean_t rv; mutex_enter(&vdp->xdf_cb_lk); rv = XD_IS_RM(vdp); mutex_exit(&vdp->xdf_cb_lk); return (rv); } boolean_t xdf_media_req_supported(dev_info_t *dip) { xdf_t *vdp = (xdf_t *)ddi_get_driver_private(dip); boolean_t rv; mutex_enter(&vdp->xdf_cb_lk); rv = vdp->xdf_media_req_supported; mutex_exit(&vdp->xdf_cb_lk); return (rv); } #endif /* XPV_HVM_DRIVER */ static int xdf_lb_getcap(dev_info_t *dip, diskaddr_t *capp) { xdf_t *vdp; vdp = ddi_get_soft_state(xdf_ssp, ddi_get_instance(dip)); if (vdp == NULL) return (ENXIO); mutex_enter(&vdp->xdf_dev_lk); *capp = vdp->xdf_pgeom.g_capacity; DPRINTF(LBL_DBG, ("xdf@%s:capacity %llu\n", vdp->xdf_addr, *capp)); mutex_exit(&vdp->xdf_dev_lk); return (0); } static int xdf_lb_getpgeom(dev_info_t *dip, cmlb_geom_t *geomp) { xdf_t *vdp; if ((vdp = ddi_get_soft_state(xdf_ssp, ddi_get_instance(dip))) == NULL) return (ENXIO); *geomp = vdp->xdf_pgeom; return (0); } /* * No real HBA, no geometry available from it */ /*ARGSUSED*/ static int xdf_lb_getvgeom(dev_info_t *dip, cmlb_geom_t *geomp) { return (EINVAL); } static int xdf_lb_getattribute(dev_info_t *dip, tg_attribute_t *tgattributep) { xdf_t *vdp; if (!(vdp = ddi_get_soft_state(xdf_ssp, ddi_get_instance(dip)))) return (ENXIO); if (XD_IS_RO(vdp)) tgattributep->media_is_writable = 0; else tgattributep->media_is_writable = 1; tgattributep->media_is_rotational = 0; return (0); } /* ARGSUSED3 */ int xdf_lb_getinfo(dev_info_t *dip, int cmd, void *arg, void *tg_cookie) { int instance; xdf_t *vdp; instance = ddi_get_instance(dip); if ((vdp = ddi_get_soft_state(xdf_ssp, instance)) == NULL) return (ENXIO); switch (cmd) { case TG_GETPHYGEOM: return (xdf_lb_getpgeom(dip, (cmlb_geom_t *)arg)); case TG_GETVIRTGEOM: return (xdf_lb_getvgeom(dip, (cmlb_geom_t *)arg)); case TG_GETCAPACITY: return (xdf_lb_getcap(dip, (diskaddr_t *)arg)); case TG_GETBLOCKSIZE: mutex_enter(&vdp->xdf_cb_lk); *(uint32_t *)arg = vdp->xdf_xdev_secsize; mutex_exit(&vdp->xdf_cb_lk); return (0); case TG_GETATTR: return (xdf_lb_getattribute(dip, (tg_attribute_t *)arg)); default: return (ENOTTY); } } /* ARGSUSED5 */ int xdf_lb_rdwr(dev_info_t *dip, uchar_t cmd, void *bufp, diskaddr_t start, size_t reqlen, void *tg_cookie) { xdf_t *vdp; struct buf *bp; int err = 0; vdp = ddi_get_soft_state(xdf_ssp, ddi_get_instance(dip)); /* We don't allow IO from the oe_change callback thread */ ASSERT(curthread != vdp->xdf_oe_change_thread); /* * Having secsize of 0 means that device isn't connected yet. * FIXME This happens for CD devices, and there's nothing we * can do about it at the moment. */ if (vdp->xdf_xdev_secsize == 0) return (EIO); if ((start + ((reqlen / (vdp->xdf_xdev_secsize / DEV_BSIZE)) >> DEV_BSHIFT)) > vdp->xdf_pgeom.g_capacity) return (EINVAL); bp = getrbuf(KM_SLEEP); if (cmd == TG_READ) bp->b_flags = B_BUSY | B_READ; else bp->b_flags = B_BUSY | B_WRITE; bp->b_un.b_addr = bufp; bp->b_bcount = reqlen; bp->b_blkno = start * (vdp->xdf_xdev_secsize / DEV_BSIZE); bp->b_edev = DDI_DEV_T_NONE; /* don't have dev_t */ mutex_enter(&vdp->xdf_dev_lk); xdf_bp_push(vdp, bp); mutex_exit(&vdp->xdf_dev_lk); xdf_io_start(vdp); if (curthread == vdp->xdf_ready_tq_thread) (void) xdf_ring_drain(vdp); err = biowait(bp); ASSERT(bp->b_flags & B_DONE); freerbuf(bp); return (err); } /* * Lock the current media. Set the media state to "lock". * (Media locks are only respected by the backend driver.) */ static int xdf_ioctl_mlock(xdf_t *vdp) { int rv; mutex_enter(&vdp->xdf_cb_lk); rv = xdf_media_req(vdp, XBV_MEDIA_REQ_LOCK, B_TRUE); mutex_exit(&vdp->xdf_cb_lk); return (rv); } /* * Release a media lock. Set the media state to "none". */ static int xdf_ioctl_munlock(xdf_t *vdp) { int rv; mutex_enter(&vdp->xdf_cb_lk); rv = xdf_media_req(vdp, XBV_MEDIA_REQ_NONE, B_TRUE); mutex_exit(&vdp->xdf_cb_lk); return (rv); } /* * Eject the current media. Ignores any media locks. (Media locks * are only for benifit of the the backend.) */ static int xdf_ioctl_eject(xdf_t *vdp) { int rv; mutex_enter(&vdp->xdf_cb_lk); if ((rv = xdf_media_req(vdp, XBV_MEDIA_REQ_EJECT, B_FALSE)) != 0) { mutex_exit(&vdp->xdf_cb_lk); return (rv); } /* * We've set the media requests xenbus parameter to eject, so now * disconnect from the backend, wait for the backend to clear * the media requets xenbus paramter, and then we can reconnect * to the backend. */ (void) xdf_disconnect(vdp, XD_UNKNOWN, B_TRUE); mutex_enter(&vdp->xdf_dev_lk); if (xdf_connect_locked(vdp, B_TRUE) != XD_READY) { mutex_exit(&vdp->xdf_dev_lk); mutex_exit(&vdp->xdf_cb_lk); return (EIO); } mutex_exit(&vdp->xdf_dev_lk); mutex_exit(&vdp->xdf_cb_lk); return (0); } /* * Watch for media state changes. This can be an insertion of a device * (triggered by a 'xm block-configure' request in another domain) or * the ejection of a device (triggered by a local "eject" operation). * For a full description of the DKIOCSTATE ioctl behavior see dkio(4I). */ static int xdf_dkstate(xdf_t *vdp, enum dkio_state mstate) { enum dkio_state prev_state; mutex_enter(&vdp->xdf_cb_lk); prev_state = vdp->xdf_mstate; if (vdp->xdf_mstate == mstate) { while (vdp->xdf_mstate == prev_state) { if (cv_wait_sig(&vdp->xdf_mstate_cv, &vdp->xdf_cb_lk) == 0) { mutex_exit(&vdp->xdf_cb_lk); return (EINTR); } } } if ((prev_state != DKIO_INSERTED) && (vdp->xdf_mstate == DKIO_INSERTED)) { (void) xdf_media_req(vdp, XBV_MEDIA_REQ_LOCK, B_TRUE); mutex_exit(&vdp->xdf_cb_lk); return (0); } mutex_exit(&vdp->xdf_cb_lk); return (0); } /*ARGSUSED*/ static int xdf_ioctl(dev_t dev, int cmd, intptr_t arg, int mode, cred_t *credp, int *rvalp) { minor_t minor = getminor(dev); int part = XDF_PART(minor); xdf_t *vdp; int rv; if (((vdp = ddi_get_soft_state(xdf_ssp, XDF_INST(minor))) == NULL) || (!xdf_isopen(vdp, part))) return (ENXIO); DPRINTF(IOCTL_DBG, ("xdf@%s:ioctl: cmd %d (0x%x)\n", vdp->xdf_addr, cmd, cmd)); switch (cmd) { default: return (ENOTTY); case DKIOCG_PHYGEOM: case DKIOCG_VIRTGEOM: case DKIOCGGEOM: case DKIOCSGEOM: case DKIOCGAPART: case DKIOCSAPART: case DKIOCGVTOC: case DKIOCSVTOC: case DKIOCPARTINFO: case DKIOCGEXTVTOC: case DKIOCSEXTVTOC: case DKIOCEXTPARTINFO: case DKIOCGMBOOT: case DKIOCSMBOOT: case DKIOCGETEFI: case DKIOCSETEFI: case DKIOCSETEXTPART: case DKIOCPARTITION: rv = cmlb_ioctl(vdp->xdf_vd_lbl, dev, cmd, arg, mode, credp, rvalp, NULL); if (rv != 0) return (rv); /* * If we're labelling the disk, we have to update the geometry * in the cmlb data structures, and we also have to write a new * devid to the disk. Note that writing an EFI label currently * requires 4 ioctls, and devid setup will fail on all but the * last. */ if (cmd == DKIOCSEXTVTOC || cmd == DKIOCSVTOC || cmd == DKIOCSETEFI) { rv = cmlb_validate(vdp->xdf_vd_lbl, 0, 0); if (rv == 0) { xdf_devid_setup(vdp); } else { cmn_err(CE_WARN, "xdf@%s, labeling failed on validate", vdp->xdf_addr); } } return (rv); case FDEJECT: case DKIOCEJECT: case CDROMEJECT: return (xdf_ioctl_eject(vdp)); case DKIOCLOCK: return (xdf_ioctl_mlock(vdp)); case DKIOCUNLOCK: return (xdf_ioctl_munlock(vdp)); case CDROMREADOFFSET: { int offset = 0; if (!XD_IS_CD(vdp)) return (ENOTTY); if (ddi_copyout(&offset, (void *)arg, sizeof (int), mode)) return (EFAULT); return (0); } case DKIOCGMEDIAINFO: { struct dk_minfo media_info; media_info.dki_lbsize = vdp->xdf_xdev_secsize; media_info.dki_capacity = vdp->xdf_pgeom.g_capacity; if (XD_IS_CD(vdp)) media_info.dki_media_type = DK_CDROM; else media_info.dki_media_type = DK_FIXED_DISK; if (ddi_copyout(&media_info, (void *)arg, sizeof (struct dk_minfo), mode)) return (EFAULT); return (0); } case DKIOCINFO: { struct dk_cinfo info; /* controller information */ if (XD_IS_CD(vdp)) info.dki_ctype = DKC_CDROM; else info.dki_ctype = DKC_VBD; info.dki_cnum = 0; (void) strncpy((char *)(&info.dki_cname), "xdf", 8); /* unit information */ info.dki_unit = ddi_get_instance(vdp->xdf_dip); (void) strncpy((char *)(&info.dki_dname), "xdf", 8); info.dki_flags = DKI_FMTVOL; info.dki_partition = part; info.dki_maxtransfer = maxphys / DEV_BSIZE; info.dki_addr = 0; info.dki_space = 0; info.dki_prio = 0; info.dki_vec = 0; if (ddi_copyout(&info, (void *)arg, sizeof (info), mode)) return (EFAULT); return (0); } case DKIOCSTATE: { enum dkio_state mstate; if (ddi_copyin((void *)arg, &mstate, sizeof (mstate), mode) != 0) return (EFAULT); if ((rv = xdf_dkstate(vdp, mstate)) != 0) return (rv); mstate = vdp->xdf_mstate; if (ddi_copyout(&mstate, (void *)arg, sizeof (mstate), mode) != 0) return (EFAULT); return (0); } case DKIOCREMOVABLE: { int i = BOOLEAN2VOID(XD_IS_RM(vdp)); if (ddi_copyout(&i, (caddr_t)arg, sizeof (i), mode)) return (EFAULT); return (0); } case DKIOCGETWCE: { int i = BOOLEAN2VOID(XD_IS_RM(vdp)); if (ddi_copyout(&i, (void *)arg, sizeof (i), mode)) return (EFAULT); return (0); } case DKIOCSETWCE: { int i; if (ddi_copyin((void *)arg, &i, sizeof (i), mode)) return (EFAULT); vdp->xdf_wce = VOID2BOOLEAN(i); return (0); } case DKIOCFLUSHWRITECACHE: { struct dk_callback *dkc = (struct dk_callback *)arg; if (vdp->xdf_flush_supported) { rv = xdf_lb_rdwr(vdp->xdf_dip, TG_WRITE, NULL, 0, 0, (void *)dev); } else if (vdp->xdf_feature_barrier && !xdf_barrier_flush_disable) { rv = xdf_lb_rdwr(vdp->xdf_dip, TG_WRITE, vdp->xdf_cache_flush_block, xdf_flush_block, vdp->xdf_xdev_secsize, (void *)dev); } else { return (ENOTTY); } if ((mode & FKIOCTL) && (dkc != NULL) && (dkc->dkc_callback != NULL)) { (*dkc->dkc_callback)(dkc->dkc_cookie, rv); /* need to return 0 after calling callback */ rv = 0; } return (rv); } } /*NOTREACHED*/ } static int xdf_strategy(struct buf *bp) { xdf_t *vdp; minor_t minor; diskaddr_t p_blkct, p_blkst; daddr_t blkno; ulong_t nblks; int part; minor = getminor(bp->b_edev); part = XDF_PART(minor); vdp = ddi_get_soft_state(xdf_ssp, XDF_INST(minor)); mutex_enter(&vdp->xdf_dev_lk); if (!xdf_isopen(vdp, part)) { mutex_exit(&vdp->xdf_dev_lk); xdf_io_err(bp, ENXIO, 0); return (0); } /* We don't allow IO from the oe_change callback thread */ ASSERT(curthread != vdp->xdf_oe_change_thread); /* Check for writes to a read only device */ if (!IS_READ(bp) && XD_IS_RO(vdp)) { mutex_exit(&vdp->xdf_dev_lk); xdf_io_err(bp, EROFS, 0); return (0); } /* Check if this I/O is accessing a partition or the entire disk */ if ((long)bp->b_private == XB_SLICE_NONE) { /* This I/O is using an absolute offset */ p_blkct = vdp->xdf_xdev_nblocks; p_blkst = 0; } else { /* This I/O is using a partition relative offset */ mutex_exit(&vdp->xdf_dev_lk); if (cmlb_partinfo(vdp->xdf_vd_lbl, part, &p_blkct, &p_blkst, NULL, NULL, NULL)) { xdf_io_err(bp, ENXIO, 0); return (0); } mutex_enter(&vdp->xdf_dev_lk); } /* * Adjust the real blkno and bcount according to the underline * physical sector size. */ blkno = bp->b_blkno / (vdp->xdf_xdev_secsize / XB_BSIZE); /* check for a starting block beyond the disk or partition limit */ if (blkno > p_blkct) { DPRINTF(IO_DBG, ("xdf@%s: block %lld exceeds VBD size %"PRIu64, vdp->xdf_addr, (longlong_t)blkno, (uint64_t)p_blkct)); mutex_exit(&vdp->xdf_dev_lk); xdf_io_err(bp, EINVAL, 0); return (0); } /* Legacy: don't set error flag at this case */ if (blkno == p_blkct) { mutex_exit(&vdp->xdf_dev_lk); bp->b_resid = bp->b_bcount; biodone(bp); return (0); } /* sanitize the input buf */ bioerror(bp, 0); bp->b_resid = 0; bp->av_back = bp->av_forw = NULL; /* Adjust for partial transfer, this will result in an error later */ if (vdp->xdf_xdev_secsize != 0 && vdp->xdf_xdev_secsize != XB_BSIZE) { nblks = bp->b_bcount / vdp->xdf_xdev_secsize; } else { nblks = bp->b_bcount >> XB_BSHIFT; } if ((blkno + nblks) > p_blkct) { if (vdp->xdf_xdev_secsize != 0 && vdp->xdf_xdev_secsize != XB_BSIZE) { bp->b_resid = ((blkno + nblks) - p_blkct) * vdp->xdf_xdev_secsize; } else { bp->b_resid = ((blkno + nblks) - p_blkct) << XB_BSHIFT; } bp->b_bcount -= bp->b_resid; } DPRINTF(IO_DBG, ("xdf@%s: strategy blk %lld len %lu\n", vdp->xdf_addr, (longlong_t)blkno, (ulong_t)bp->b_bcount)); /* Fix up the buf struct */ bp->b_flags |= B_BUSY; bp->b_private = (void *)(uintptr_t)p_blkst; xdf_bp_push(vdp, bp); mutex_exit(&vdp->xdf_dev_lk); xdf_io_start(vdp); if (do_polled_io) (void) xdf_ring_drain(vdp); return (0); } /*ARGSUSED*/ static int xdf_read(dev_t dev, struct uio *uiop, cred_t *credp) { xdf_t *vdp; minor_t minor; diskaddr_t p_blkcnt; int part; minor = getminor(dev); if ((vdp = ddi_get_soft_state(xdf_ssp, XDF_INST(minor))) == NULL) return (ENXIO); DPRINTF(IO_DBG, ("xdf@%s: read offset 0x%"PRIx64"\n", vdp->xdf_addr, (int64_t)uiop->uio_offset)); part = XDF_PART(minor); if (!xdf_isopen(vdp, part)) return (ENXIO); if (cmlb_partinfo(vdp->xdf_vd_lbl, part, &p_blkcnt, NULL, NULL, NULL, NULL)) return (ENXIO); if (uiop->uio_loffset >= XB_DTOB(p_blkcnt, vdp)) return (ENOSPC); if (U_INVAL(uiop)) return (EINVAL); return (physio(xdf_strategy, NULL, dev, B_READ, xdfmin, uiop)); } /*ARGSUSED*/ static int xdf_write(dev_t dev, struct uio *uiop, cred_t *credp) { xdf_t *vdp; minor_t minor; diskaddr_t p_blkcnt; int part; minor = getminor(dev); if ((vdp = ddi_get_soft_state(xdf_ssp, XDF_INST(minor))) == NULL) return (ENXIO); DPRINTF(IO_DBG, ("xdf@%s: write offset 0x%"PRIx64"\n", vdp->xdf_addr, (int64_t)uiop->uio_offset)); part = XDF_PART(minor); if (!xdf_isopen(vdp, part)) return (ENXIO); if (cmlb_partinfo(vdp->xdf_vd_lbl, part, &p_blkcnt, NULL, NULL, NULL, NULL)) return (ENXIO); if (uiop->uio_loffset >= XB_DTOB(p_blkcnt, vdp)) return (ENOSPC); if (U_INVAL(uiop)) return (EINVAL); return (physio(xdf_strategy, NULL, dev, B_WRITE, xdfmin, uiop)); } /*ARGSUSED*/ static int xdf_aread(dev_t dev, struct aio_req *aiop, cred_t *credp) { xdf_t *vdp; minor_t minor; struct uio *uiop = aiop->aio_uio; diskaddr_t p_blkcnt; int part; minor = getminor(dev); if ((vdp = ddi_get_soft_state(xdf_ssp, XDF_INST(minor))) == NULL) return (ENXIO); part = XDF_PART(minor); if (!xdf_isopen(vdp, part)) return (ENXIO); if (cmlb_partinfo(vdp->xdf_vd_lbl, part, &p_blkcnt, NULL, NULL, NULL, NULL)) return (ENXIO); if (uiop->uio_loffset >= XB_DTOB(p_blkcnt, vdp)) return (ENOSPC); if (U_INVAL(uiop)) return (EINVAL); return (aphysio(xdf_strategy, anocancel, dev, B_READ, xdfmin, aiop)); } /*ARGSUSED*/ static int xdf_awrite(dev_t dev, struct aio_req *aiop, cred_t *credp) { xdf_t *vdp; minor_t minor; struct uio *uiop = aiop->aio_uio; diskaddr_t p_blkcnt; int part; minor = getminor(dev); if ((vdp = ddi_get_soft_state(xdf_ssp, XDF_INST(minor))) == NULL) return (ENXIO); part = XDF_PART(minor); if (!xdf_isopen(vdp, part)) return (ENXIO); if (cmlb_partinfo(vdp->xdf_vd_lbl, part, &p_blkcnt, NULL, NULL, NULL, NULL)) return (ENXIO); if (uiop->uio_loffset >= XB_DTOB(p_blkcnt, vdp)) return (ENOSPC); if (U_INVAL(uiop)) return (EINVAL); return (aphysio(xdf_strategy, anocancel, dev, B_WRITE, xdfmin, aiop)); } static int xdf_dump(dev_t dev, caddr_t addr, daddr_t blkno, int nblk) { struct buf dumpbuf, *dbp = &dumpbuf; xdf_t *vdp; minor_t minor; int err = 0; int part; diskaddr_t p_blkcnt, p_blkst; minor = getminor(dev); if ((vdp = ddi_get_soft_state(xdf_ssp, XDF_INST(minor))) == NULL) return (ENXIO); DPRINTF(IO_DBG, ("xdf@%s: dump addr (0x%p) blk (%ld) nblks (%d)\n", vdp->xdf_addr, (void *)addr, blkno, nblk)); /* We don't allow IO from the oe_change callback thread */ ASSERT(curthread != vdp->xdf_oe_change_thread); part = XDF_PART(minor); if (!xdf_isopen(vdp, part)) return (ENXIO); if (cmlb_partinfo(vdp->xdf_vd_lbl, part, &p_blkcnt, &p_blkst, NULL, NULL, NULL)) return (ENXIO); if ((blkno + nblk) > (p_blkcnt * (vdp->xdf_xdev_secsize / XB_BSIZE))) { cmn_err(CE_WARN, "xdf@%s: block %ld exceeds VBD size %"PRIu64, vdp->xdf_addr, (daddr_t)((blkno + nblk) / (vdp->xdf_xdev_secsize / XB_BSIZE)), (uint64_t)p_blkcnt); return (EINVAL); } bioinit(dbp); dbp->b_flags = B_BUSY; dbp->b_un.b_addr = addr; dbp->b_bcount = nblk << DEV_BSHIFT; dbp->b_blkno = blkno; dbp->b_edev = dev; dbp->b_private = (void *)(uintptr_t)p_blkst; mutex_enter(&vdp->xdf_dev_lk); xdf_bp_push(vdp, dbp); mutex_exit(&vdp->xdf_dev_lk); xdf_io_start(vdp); err = xdf_ring_drain(vdp); biofini(dbp); return (err); } /*ARGSUSED*/ static int xdf_close(dev_t dev, int flag, int otyp, struct cred *credp) { minor_t minor; xdf_t *vdp; int part; ulong_t parbit; minor = getminor(dev); if ((vdp = ddi_get_soft_state(xdf_ssp, XDF_INST(minor))) == NULL) return (ENXIO); mutex_enter(&vdp->xdf_dev_lk); part = XDF_PART(minor); if (!xdf_isopen(vdp, part)) { mutex_exit(&vdp->xdf_dev_lk); return (ENXIO); } parbit = 1 << part; ASSERT((vdp->xdf_vd_open[otyp] & parbit) != 0); if (otyp == OTYP_LYR) { ASSERT(vdp->xdf_vd_lyropen[part] > 0); if (--vdp->xdf_vd_lyropen[part] == 0) vdp->xdf_vd_open[otyp] &= ~parbit; } else { vdp->xdf_vd_open[otyp] &= ~parbit; } vdp->xdf_vd_exclopen &= ~parbit; mutex_exit(&vdp->xdf_dev_lk); return (0); } static int xdf_open(dev_t *devp, int flag, int otyp, cred_t *credp) { minor_t minor; xdf_t *vdp; int part; ulong_t parbit; diskaddr_t p_blkct = 0; boolean_t firstopen; boolean_t nodelay; minor = getminor(*devp); if ((vdp = ddi_get_soft_state(xdf_ssp, XDF_INST(minor))) == NULL) return (ENXIO); nodelay = (flag & (FNDELAY | FNONBLOCK)); DPRINTF(DDI_DBG, ("xdf@%s: opening\n", vdp->xdf_addr)); /* do cv_wait until connected or failed */ mutex_enter(&vdp->xdf_cb_lk); mutex_enter(&vdp->xdf_dev_lk); if (!nodelay && (xdf_connect_locked(vdp, B_TRUE) != XD_READY)) { mutex_exit(&vdp->xdf_dev_lk); mutex_exit(&vdp->xdf_cb_lk); return (ENXIO); } mutex_exit(&vdp->xdf_cb_lk); if ((flag & FWRITE) && XD_IS_RO(vdp)) { mutex_exit(&vdp->xdf_dev_lk); return (EROFS); } part = XDF_PART(minor); parbit = 1 << part; if ((vdp->xdf_vd_exclopen & parbit) || ((flag & FEXCL) && xdf_isopen(vdp, part))) { mutex_exit(&vdp->xdf_dev_lk); return (EBUSY); } /* are we the first one to open this node? */ firstopen = !xdf_isopen(vdp, -1); if (otyp == OTYP_LYR) vdp->xdf_vd_lyropen[part]++; vdp->xdf_vd_open[otyp] |= parbit; if (flag & FEXCL) vdp->xdf_vd_exclopen |= parbit; mutex_exit(&vdp->xdf_dev_lk); /* force a re-validation */ if (firstopen) cmlb_invalidate(vdp->xdf_vd_lbl, NULL); /* If this is a non-blocking open then we're done */ if (nodelay) return (0); /* * This is a blocking open, so we require: * - that the disk have a valid label on it * - that the size of the partition that we're opening is non-zero */ if ((cmlb_partinfo(vdp->xdf_vd_lbl, part, &p_blkct, NULL, NULL, NULL, NULL) != 0) || (p_blkct == 0)) { (void) xdf_close(*devp, flag, otyp, credp); return (ENXIO); } return (0); } /*ARGSUSED*/ static void xdf_watch_hp_status_cb(dev_info_t *dip, const char *path, void *arg) { xdf_t *vdp = (xdf_t *)ddi_get_driver_private(dip); cv_broadcast(&vdp->xdf_hp_status_cv); } static int xdf_prop_op(dev_t dev, dev_info_t *dip, ddi_prop_op_t prop_op, int flags, char *name, caddr_t valuep, int *lengthp) { xdf_t *vdp = ddi_get_soft_state(xdf_ssp, ddi_get_instance(dip)); /* * Sanity check that if a dev_t or dip were specified that they * correspond to this device driver. On debug kernels we'll * panic and on non-debug kernels we'll return failure. */ ASSERT(ddi_driver_major(dip) == xdf_major); ASSERT((dev == DDI_DEV_T_ANY) || (getmajor(dev) == xdf_major)); if ((ddi_driver_major(dip) != xdf_major) || ((dev != DDI_DEV_T_ANY) && (getmajor(dev) != xdf_major))) return (DDI_PROP_NOT_FOUND); if (vdp == NULL) return (ddi_prop_op(dev, dip, prop_op, flags, name, valuep, lengthp)); return (cmlb_prop_op(vdp->xdf_vd_lbl, dev, dip, prop_op, flags, name, valuep, lengthp, XDF_PART(getminor(dev)), NULL)); } /*ARGSUSED*/ static int xdf_getinfo(dev_info_t *dip, ddi_info_cmd_t cmd, void *arg, void **rp) { int instance = XDF_INST(getminor((dev_t)arg)); xdf_t *vbdp; switch (cmd) { case DDI_INFO_DEVT2DEVINFO: if ((vbdp = ddi_get_soft_state(xdf_ssp, instance)) == NULL) { *rp = NULL; return (DDI_FAILURE); } *rp = vbdp->xdf_dip; return (DDI_SUCCESS); case DDI_INFO_DEVT2INSTANCE: *rp = (void *)(uintptr_t)instance; return (DDI_SUCCESS); default: return (DDI_FAILURE); } } /*ARGSUSED*/ static int xdf_resume(dev_info_t *dip) { xdf_t *vdp; char *oename; if ((vdp = ddi_get_soft_state(xdf_ssp, ddi_get_instance(dip))) == NULL) goto err; if (xdf_debug & SUSRES_DBG) xen_printf("xdf@%s: xdf_resume\n", vdp->xdf_addr); mutex_enter(&vdp->xdf_cb_lk); if (xvdi_resume(dip) != DDI_SUCCESS) { mutex_exit(&vdp->xdf_cb_lk); goto err; } if (((oename = xvdi_get_oename(dip)) == NULL) || (xvdi_add_xb_watch_handler(dip, oename, XBP_HP_STATUS, xdf_watch_hp_status_cb, NULL) != DDI_SUCCESS)) { mutex_exit(&vdp->xdf_cb_lk); goto err; } mutex_enter(&vdp->xdf_dev_lk); ASSERT(vdp->xdf_state != XD_READY); xdf_set_state(vdp, XD_UNKNOWN); mutex_exit(&vdp->xdf_dev_lk); if (xdf_setstate_init(vdp) != DDI_SUCCESS) { mutex_exit(&vdp->xdf_cb_lk); goto err; } mutex_exit(&vdp->xdf_cb_lk); if (xdf_debug & SUSRES_DBG) xen_printf("xdf@%s: xdf_resume: done\n", vdp->xdf_addr); return (DDI_SUCCESS); err: if (xdf_debug & SUSRES_DBG) xen_printf("xdf@%s: xdf_resume: fail\n", vdp->xdf_addr); return (DDI_FAILURE); } /* * Uses the in-memory devid if one exists. * * Create a devid and write it on the first block of the last track of * the last cylinder. * Return DDI_SUCCESS or DDI_FAILURE. */ static int xdf_devid_fabricate(xdf_t *vdp) { ddi_devid_t devid = vdp->xdf_tgt_devid; /* null if no devid */ struct dk_devid *dkdevidp = NULL; /* devid struct stored on disk */ diskaddr_t blk; uint_t *ip, chksum; int i, devid_size; if (cmlb_get_devid_block(vdp->xdf_vd_lbl, &blk, NULL) != 0) goto err; if (devid == NULL && ddi_devid_init(vdp->xdf_dip, DEVID_FAB, 0, NULL, &devid) != DDI_SUCCESS) goto err; /* allocate a buffer */ dkdevidp = (struct dk_devid *)kmem_zalloc(NBPSCTR, KM_SLEEP); /* Fill in the revision */ dkdevidp->dkd_rev_hi = DK_DEVID_REV_MSB; dkdevidp->dkd_rev_lo = DK_DEVID_REV_LSB; /* Copy in the device id */ devid_size = ddi_devid_sizeof(devid); if (devid_size > DK_DEVID_SIZE) goto err; bcopy(devid, dkdevidp->dkd_devid, devid_size); /* Calculate the chksum */ chksum = 0; ip = (uint_t *)dkdevidp; for (i = 0; i < (NBPSCTR / sizeof (int)) - 1; i++) chksum ^= ip[i]; /* Fill in the checksum */ DKD_FORMCHKSUM(chksum, dkdevidp); if (xdf_lb_rdwr(vdp->xdf_dip, TG_WRITE, dkdevidp, blk, NBPSCTR, NULL) != 0) goto err; kmem_free(dkdevidp, NBPSCTR); vdp->xdf_tgt_devid = devid; return (DDI_SUCCESS); err: if (dkdevidp != NULL) kmem_free(dkdevidp, NBPSCTR); if (devid != NULL && vdp->xdf_tgt_devid == NULL) ddi_devid_free(devid); return (DDI_FAILURE); } /* * xdf_devid_read() is a local copy of xdfs_devid_read(), modified to use xdf * functions. * * Read a devid from on the first block of the last track of * the last cylinder. Make sure what we read is a valid devid. * Return DDI_SUCCESS or DDI_FAILURE. */ static int xdf_devid_read(xdf_t *vdp) { diskaddr_t blk; struct dk_devid *dkdevidp; uint_t *ip, chksum; int i; if (cmlb_get_devid_block(vdp->xdf_vd_lbl, &blk, NULL) != 0) return (DDI_FAILURE); dkdevidp = kmem_zalloc(NBPSCTR, KM_SLEEP); if (xdf_lb_rdwr(vdp->xdf_dip, TG_READ, dkdevidp, blk, NBPSCTR, NULL) != 0) goto err; /* Validate the revision */ if ((dkdevidp->dkd_rev_hi != DK_DEVID_REV_MSB) || (dkdevidp->dkd_rev_lo != DK_DEVID_REV_LSB)) goto err; /* Calculate the checksum */ chksum = 0; ip = (uint_t *)dkdevidp; for (i = 0; i < (NBPSCTR / sizeof (int)) - 1; i++) chksum ^= ip[i]; if (DKD_GETCHKSUM(dkdevidp) != chksum) goto err; /* Validate the device id */ if (ddi_devid_valid((ddi_devid_t)dkdevidp->dkd_devid) != DDI_SUCCESS) goto err; /* keep a copy of the device id */ i = ddi_devid_sizeof((ddi_devid_t)dkdevidp->dkd_devid); vdp->xdf_tgt_devid = kmem_alloc(i, KM_SLEEP); bcopy(dkdevidp->dkd_devid, vdp->xdf_tgt_devid, i); kmem_free(dkdevidp, NBPSCTR); return (DDI_SUCCESS); err: kmem_free(dkdevidp, NBPSCTR); return (DDI_FAILURE); } /* * xdf_devid_setup() is a modified copy of cmdk_devid_setup(). * * This function creates a devid if we don't already have one, and * registers it. If we already have one, we make sure that it can be * read from the disk, otherwise we write it to the disk ourselves. If * we didn't already have a devid, and we create one, we also need to * register it. */ void xdf_devid_setup(xdf_t *vdp) { int rc; boolean_t existed = vdp->xdf_tgt_devid != NULL; /* Read devid from the disk, if present */ rc = xdf_devid_read(vdp); /* Otherwise write a devid (which we create if necessary) on the disk */ if (rc != DDI_SUCCESS) rc = xdf_devid_fabricate(vdp); /* If we created a devid or found it on the disk, register it */ if (rc == DDI_SUCCESS && !existed) (void) ddi_devid_register(vdp->xdf_dip, vdp->xdf_tgt_devid); } static int xdf_attach(dev_info_t *dip, ddi_attach_cmd_t cmd) { int n, instance = ddi_get_instance(dip); ddi_iblock_cookie_t ibc, softibc; boolean_t dev_iscd = B_FALSE; xdf_t *vdp; char *oename, *xsname, *str; clock_t timeout; int err = 0; if ((n = ddi_prop_get_int(DDI_DEV_T_ANY, dip, DDI_PROP_NOTPROM, "xdf_debug", 0)) != 0) xdf_debug = n; switch (cmd) { case DDI_RESUME: return (xdf_resume(dip)); case DDI_ATTACH: break; default: return (DDI_FAILURE); } /* DDI_ATTACH */ if ((xsname = xvdi_get_xsname(dip)) == NULL || (oename = xvdi_get_oename(dip)) == NULL) return (DDI_FAILURE); /* * Disable auto-detach. This is necessary so that we don't get * detached while we're disconnected from the back end. */ if ((ddi_prop_update_int(DDI_DEV_T_NONE, dip, DDI_NO_AUTODETACH, 1) != DDI_PROP_SUCCESS)) return (DDI_FAILURE); /* driver handles kernel-issued IOCTLs */ if (ddi_prop_create(DDI_DEV_T_NONE, dip, DDI_PROP_CANSLEEP, DDI_KERNEL_IOCTL, NULL, 0) != DDI_PROP_SUCCESS) return (DDI_FAILURE); if (ddi_get_iblock_cookie(dip, 0, &ibc) != DDI_SUCCESS) return (DDI_FAILURE); if (ddi_get_soft_iblock_cookie(dip, DDI_SOFTINT_LOW, &softibc) != DDI_SUCCESS) return (DDI_FAILURE); if (xenbus_read_str(xsname, XBP_DEV_TYPE, &str) != 0) { cmn_err(CE_WARN, "xdf@%s: cannot read device-type", ddi_get_name_addr(dip)); return (DDI_FAILURE); } if (strcmp(str, XBV_DEV_TYPE_CD) == 0) dev_iscd = B_TRUE; strfree(str); if (ddi_soft_state_zalloc(xdf_ssp, instance) != DDI_SUCCESS) return (DDI_FAILURE); DPRINTF(DDI_DBG, ("xdf@%s: attaching\n", ddi_get_name_addr(dip))); vdp = ddi_get_soft_state(xdf_ssp, instance); ddi_set_driver_private(dip, vdp); vdp->xdf_dip = dip; vdp->xdf_addr = ddi_get_name_addr(dip); vdp->xdf_suspending = B_FALSE; vdp->xdf_media_req_supported = B_FALSE; vdp->xdf_peer = INVALID_DOMID; vdp->xdf_evtchn = INVALID_EVTCHN; list_create(&vdp->xdf_vreq_act, sizeof (v_req_t), offsetof(v_req_t, v_link)); cv_init(&vdp->xdf_dev_cv, NULL, CV_DEFAULT, NULL); cv_init(&vdp->xdf_hp_status_cv, NULL, CV_DEFAULT, NULL); cv_init(&vdp->xdf_mstate_cv, NULL, CV_DEFAULT, NULL); mutex_init(&vdp->xdf_dev_lk, NULL, MUTEX_DRIVER, (void *)ibc); mutex_init(&vdp->xdf_cb_lk, NULL, MUTEX_DRIVER, (void *)ibc); mutex_init(&vdp->xdf_iostat_lk, NULL, MUTEX_DRIVER, (void *)ibc); vdp->xdf_cmlb_reattach = B_TRUE; if (dev_iscd) { vdp->xdf_dinfo |= VDISK_CDROM; vdp->xdf_mstate = DKIO_EJECTED; } else { vdp->xdf_mstate = DKIO_NONE; } if ((vdp->xdf_ready_tq = ddi_taskq_create(dip, "xdf_ready_tq", 1, TASKQ_DEFAULTPRI, 0)) == NULL) goto errout0; if (xvdi_add_xb_watch_handler(dip, oename, XBP_HP_STATUS, xdf_watch_hp_status_cb, NULL) != DDI_SUCCESS) goto errout0; if (ddi_add_softintr(dip, DDI_SOFTINT_LOW, &vdp->xdf_softintr_id, &softibc, NULL, xdf_iorestart, (caddr_t)vdp) != DDI_SUCCESS) { cmn_err(CE_WARN, "xdf@%s: failed to add softintr", ddi_get_name_addr(dip)); goto errout0; } /* * Initialize the physical geometry stucture. Note that currently * we don't know the size of the backend device so the number * of blocks on the device will be initialized to zero. Once * we connect to the backend device we'll update the physical * geometry to reflect the real size of the device. */ xdf_synthetic_pgeom(dip, &vdp->xdf_pgeom); vdp->xdf_pgeom_fixed = B_FALSE; /* * Allocate the cmlb handle, minor nodes will be created once * the device is connected with backend. */ cmlb_alloc_handle(&vdp->xdf_vd_lbl); /* We ship with cache-enabled disks */ vdp->xdf_wce = B_TRUE; mutex_enter(&vdp->xdf_cb_lk); /* Watch backend XenbusState change */ if (xvdi_add_event_handler(dip, XS_OE_STATE, xdf_oe_change, NULL) != DDI_SUCCESS) { mutex_exit(&vdp->xdf_cb_lk); goto errout0; } if (xdf_setstate_init(vdp) != DDI_SUCCESS) { cmn_err(CE_WARN, "xdf@%s: start connection failed", ddi_get_name_addr(dip)); mutex_exit(&vdp->xdf_cb_lk); goto errout1; } /* Nothing else to do for CD devices */ if (dev_iscd) { mutex_exit(&vdp->xdf_cb_lk); goto done; } /* * In order to do cmlb_validate, we have to wait for the disk to * acknowledge the attach, so we can query the backend for the disk * geometry (see xdf_setstate_connected). * * We only wait 30 seconds; if this is the root disk, the boot * will fail, but it would fail anyway if the device never * connected. If this is a non-boot disk, that disk will fail * to connect, but again, it would fail anyway. */ timeout = ddi_get_lbolt() + drv_usectohz(XDF_STATE_TIMEOUT); while (vdp->xdf_state != XD_CONNECTED && vdp->xdf_state != XD_READY) { if (cv_timedwait(&vdp->xdf_dev_cv, &vdp->xdf_cb_lk, timeout) < 0) { cmn_err(CE_WARN, "xdf@%s: disk failed to connect", ddi_get_name_addr(dip)); mutex_exit(&vdp->xdf_cb_lk); goto errout1; } } mutex_exit(&vdp->xdf_cb_lk); /* * We call cmlb_validate so that the geometry information in * vdp->xdf_vd_lbl is correct; this fills out the number of * alternate cylinders so that we have a place to write the * devid. */ if ((err = cmlb_validate(vdp->xdf_vd_lbl, 0, NULL)) != 0) { cmn_err(CE_NOTE, "xdf@%s: cmlb_validate failed: %d", ddi_get_name_addr(dip), err); /* * We can carry on even if cmlb_validate() returns EINVAL here, * as we'll rewrite the disk label anyway. */ if (err != EINVAL) goto errout1; } /* * xdf_devid_setup will only write a devid if one isn't * already present. If it fails to find or create one, we * create one in-memory so that when we label the disk later, * it will have a devid to use. This is helpful to deal with * cases where people use the devids of their disks before * labelling them; note that this does cause problems if * people rely on the devids of unlabelled disks to persist * across reboot. */ xdf_devid_setup(vdp); if (vdp->xdf_tgt_devid == NULL) { if (ddi_devid_init(vdp->xdf_dip, DEVID_FAB, 0, NULL, &vdp->xdf_tgt_devid) != DDI_SUCCESS) { cmn_err(CE_WARN, "xdf@%s_ attach failed, devid_init failed", ddi_get_name_addr(dip)); goto errout1; } else { (void) ddi_devid_register(vdp->xdf_dip, vdp->xdf_tgt_devid); } } done: #ifdef XPV_HVM_DRIVER xdf_hvm_add(dip); /* Report our version to dom0 */ (void) xenbus_printf(XBT_NULL, "guest/xdf", "version", "%d", HVMPV_XDF_VERS); #endif /* XPV_HVM_DRIVER */ /* Create kstat for iostat(8) */ if (xdf_kstat_create(dip) != 0) { cmn_err(CE_WARN, "xdf@%s: failed to create kstat", ddi_get_name_addr(dip)); goto errout1; } /* * Don't bother with getting real device identification * strings (is it even possible?), they are unlikely to * change often (if at all). */ (void) ndi_prop_update_string(DDI_DEV_T_NONE, dip, INQUIRY_VENDOR_ID, "Xen"); (void) ndi_prop_update_string(DDI_DEV_T_NONE, dip, INQUIRY_PRODUCT_ID, dev_iscd ? "Virtual CD" : "Virtual disk"); (void) ndi_prop_update_string(DDI_DEV_T_NONE, dip, INQUIRY_REVISION_ID, "1.0"); ddi_report_dev(dip); DPRINTF(DDI_DBG, ("xdf@%s: attached\n", vdp->xdf_addr)); return (DDI_SUCCESS); errout1: (void) xvdi_switch_state(vdp->xdf_dip, XBT_NULL, XenbusStateClosed); xvdi_remove_event_handler(dip, XS_OE_STATE); errout0: if (vdp->xdf_vd_lbl != NULL) { cmlb_free_handle(&vdp->xdf_vd_lbl); vdp->xdf_vd_lbl = NULL; } if (vdp->xdf_softintr_id != NULL) ddi_remove_softintr(vdp->xdf_softintr_id); xvdi_remove_xb_watch_handlers(dip); if (vdp->xdf_ready_tq != NULL) ddi_taskq_destroy(vdp->xdf_ready_tq); mutex_destroy(&vdp->xdf_cb_lk); mutex_destroy(&vdp->xdf_dev_lk); cv_destroy(&vdp->xdf_dev_cv); cv_destroy(&vdp->xdf_hp_status_cv); ddi_soft_state_free(xdf_ssp, instance); ddi_set_driver_private(dip, NULL); ddi_prop_remove_all(dip); cmn_err(CE_WARN, "xdf@%s: attach failed", ddi_get_name_addr(dip)); return (DDI_FAILURE); } static int xdf_suspend(dev_info_t *dip) { int instance = ddi_get_instance(dip); xdf_t *vdp; if ((vdp = ddi_get_soft_state(xdf_ssp, instance)) == NULL) return (DDI_FAILURE); if (xdf_debug & SUSRES_DBG) xen_printf("xdf@%s: xdf_suspend\n", vdp->xdf_addr); xvdi_suspend(dip); mutex_enter(&vdp->xdf_cb_lk); mutex_enter(&vdp->xdf_dev_lk); vdp->xdf_suspending = B_TRUE; xdf_ring_destroy(vdp); xdf_set_state(vdp, XD_SUSPEND); vdp->xdf_suspending = B_FALSE; mutex_exit(&vdp->xdf_dev_lk); mutex_exit(&vdp->xdf_cb_lk); if (xdf_debug & SUSRES_DBG) xen_printf("xdf@%s: xdf_suspend: done\n", vdp->xdf_addr); return (DDI_SUCCESS); } static int xdf_detach(dev_info_t *dip, ddi_detach_cmd_t cmd) { xdf_t *vdp; int instance; switch (cmd) { case DDI_PM_SUSPEND: break; case DDI_SUSPEND: return (xdf_suspend(dip)); case DDI_DETACH: break; default: return (DDI_FAILURE); } instance = ddi_get_instance(dip); DPRINTF(DDI_DBG, ("xdf@%s: detaching\n", ddi_get_name_addr(dip))); vdp = ddi_get_soft_state(xdf_ssp, instance); if (vdp == NULL) return (DDI_FAILURE); mutex_enter(&vdp->xdf_cb_lk); xdf_disconnect(vdp, XD_CLOSED, B_FALSE); if (vdp->xdf_state != XD_CLOSED) { mutex_exit(&vdp->xdf_cb_lk); return (DDI_FAILURE); } mutex_exit(&vdp->xdf_cb_lk); ASSERT(!ISDMACBON(vdp)); #ifdef XPV_HVM_DRIVER xdf_hvm_rm(dip); #endif /* XPV_HVM_DRIVER */ if (vdp->xdf_timeout_id != 0) (void) untimeout(vdp->xdf_timeout_id); xvdi_remove_event_handler(dip, XS_OE_STATE); ddi_taskq_destroy(vdp->xdf_ready_tq); cmlb_detach(vdp->xdf_vd_lbl, NULL); cmlb_free_handle(&vdp->xdf_vd_lbl); /* we'll support backend running in domU later */ #ifdef DOMU_BACKEND (void) xvdi_post_event(dip, XEN_HP_REMOVE); #endif list_destroy(&vdp->xdf_vreq_act); ddi_prop_remove_all(dip); xdf_kstat_delete(dip); ddi_remove_softintr(vdp->xdf_softintr_id); xvdi_remove_xb_watch_handlers(dip); ddi_set_driver_private(dip, NULL); cv_destroy(&vdp->xdf_dev_cv); mutex_destroy(&vdp->xdf_cb_lk); mutex_destroy(&vdp->xdf_dev_lk); if (vdp->xdf_cache_flush_block != NULL) kmem_free(vdp->xdf_flush_mem, 2 * vdp->xdf_xdev_secsize); ddi_soft_state_free(xdf_ssp, instance); return (DDI_SUCCESS); } /* * Driver linkage structures. */ static struct cb_ops xdf_cbops = { xdf_open, xdf_close, xdf_strategy, nodev, xdf_dump, xdf_read, xdf_write, xdf_ioctl, nodev, nodev, nodev, nochpoll, xdf_prop_op, NULL, D_MP | D_NEW | D_64BIT, CB_REV, xdf_aread, xdf_awrite }; struct dev_ops xdf_devops = { DEVO_REV, /* devo_rev */ 0, /* devo_refcnt */ xdf_getinfo, /* devo_getinfo */ nulldev, /* devo_identify */ nulldev, /* devo_probe */ xdf_attach, /* devo_attach */ xdf_detach, /* devo_detach */ nodev, /* devo_reset */ &xdf_cbops, /* devo_cb_ops */ NULL, /* devo_bus_ops */ NULL, /* devo_power */ ddi_quiesce_not_supported, /* devo_quiesce */ }; /* * Module linkage structures. */ static struct modldrv modldrv = { &mod_driverops, /* Type of module. This one is a driver */ "virtual block driver", /* short description */ &xdf_devops /* driver specific ops */ }; static struct modlinkage xdf_modlinkage = { MODREV_1, (void *)&modldrv, NULL }; /* * standard module entry points */ int _init(void) { int rc; xdf_major = ddi_name_to_major("xdf"); if (xdf_major == (major_t)-1) return (EINVAL); if ((rc = ddi_soft_state_init(&xdf_ssp, sizeof (xdf_t), 0)) != 0) return (rc); xdf_vreq_cache = kmem_cache_create("xdf_vreq_cache", sizeof (v_req_t), 0, NULL, NULL, NULL, NULL, NULL, 0); xdf_gs_cache = kmem_cache_create("xdf_gs_cache", sizeof (ge_slot_t), 0, NULL, NULL, NULL, NULL, NULL, 0); #ifdef XPV_HVM_DRIVER xdf_hvm_init(); #endif /* XPV_HVM_DRIVER */ if ((rc = mod_install(&xdf_modlinkage)) != 0) { #ifdef XPV_HVM_DRIVER xdf_hvm_fini(); #endif /* XPV_HVM_DRIVER */ kmem_cache_destroy(xdf_vreq_cache); kmem_cache_destroy(xdf_gs_cache); ddi_soft_state_fini(&xdf_ssp); return (rc); } return (rc); } int _fini(void) { int err; if ((err = mod_remove(&xdf_modlinkage)) != 0) return (err); #ifdef XPV_HVM_DRIVER xdf_hvm_fini(); #endif /* XPV_HVM_DRIVER */ kmem_cache_destroy(xdf_vreq_cache); kmem_cache_destroy(xdf_gs_cache); ddi_soft_state_fini(&xdf_ssp); return (0); } int _info(struct modinfo *modinfop) { return (mod_info(&xdf_modlinkage, modinfop)); } /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2009 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ /* * Copyright (c) 2014 by Delphix. All rights reserved. * Copyright 2017 Nexenta Systems, Inc. */ #ifndef _SYS_XDF_H #define _SYS_XDF_H #include #include #include #include #include #include #ifdef __cplusplus extern "C" { #endif /* * VBDs have standard 512 byte blocks * A single blkif_request can transfer up to 11 pages of data, 1 page/segment */ #define XB_BSIZE DEV_BSIZE #define XB_BMASK (XB_BSIZE - 1) #define XB_BSHIFT 9 #define XB_DTOB(bn, vdp) ((bn) * (vdp)->xdf_xdev_secsize) #define XB_MAX_SEGLEN (8 * XB_BSIZE) #define XB_SEGOFFSET (XB_MAX_SEGLEN - 1) #define XB_MAX_XFER (XB_MAX_SEGLEN * BLKIF_MAX_SEGMENTS_PER_REQUEST) #define XB_MAXPHYS (XB_MAX_XFER * BLKIF_RING_SIZE) /* Number of sectors per segement */ #define XB_NUM_SECTORS_PER_SEG (PAGESIZE / XB_BSIZE) /* sectors are number 0 through XB_NUM_SECTORS_PER_SEG - 1 */ #define XB_LAST_SECTOR_IN_SEG (XB_NUM_SECTORS_PER_SEG - 1) /* * Slice for absolute disk transaction. * * Hack Alert. XB_SLICE_NONE is a magic value that can be written into the * b_private field of buf structures passed to xdf_strategy(). When present * it indicates that the I/O is using an absolute offset. (ie, the I/O is * not bound to any one partition.) This magic value is currently used by * the pv_cmdk driver. This hack is shamelessly stolen from the sun4v vdc * driver, another virtual disk device driver. (Although in the case of * vdc the hack is less egregious since it is self contained within the * vdc driver, where as here it is used as an interface between the pv_cmdk * driver and the xdf driver.) */ #define XB_SLICE_NONE 0xFF /* * blkif status */ typedef enum xdf_state { /* * initial state */ XD_UNKNOWN = 0, /* * ring and evtchn alloced, xenbus state changed to * XenbusStateInitialised, wait for backend to connect */ XD_INIT = 1, /* * backend and frontend xenbus state has changed to * XenbusStateConnected. IO is now allowed, but we are not still * fully initialized. */ XD_CONNECTED = 2, /* * We're fully initialized and allowing regular IO. */ XD_READY = 3, /* * vbd interface close request received from backend, no more I/O * requestis allowed to be put into ring buffer, while interrupt handler * is allowed to run to finish any outstanding I/O request, disconnect * process is kicked off by changing xenbus state to XenbusStateClosed */ XD_CLOSING = 4, /* * disconnection process finished, both backend and frontend's * xenbus state has been changed to XenbusStateClosed, can be detached */ XD_CLOSED = 5, /* * We're either being suspended or resuming from a suspend. If we're * in the process of suspending, we block all new IO, but but allow * existing IO to drain. */ XD_SUSPEND = 6 } xdf_state_t; /* * 16 partitions + fdisk */ #define XDF_PSHIFT 6 #define XDF_PMASK ((1 << XDF_PSHIFT) - 1) #define XDF_PEXT (1 << XDF_PSHIFT) #define XDF_MINOR(i, m) (((i) << XDF_PSHIFT) | (m)) #define XDF_INST(m) ((m) >> XDF_PSHIFT) #define XDF_PART(m) ((m) & XDF_PMASK) /* * one blkif_request_t will have one corresponding ge_slot_t * where we save those grant table refs used in this blkif_request_t * * the id of this ge_slot_t will also be put into 'id' field in * each blkif_request_t when sent out to the ring buffer. */ typedef struct ge_slot { list_node_t gs_vreq_link; struct v_req *gs_vreq; domid_t gs_oeid; int gs_isread; grant_ref_t gs_ghead; int gs_ngrefs; grant_ref_t gs_ge[BLKIF_MAX_SEGMENTS_PER_REQUEST]; } ge_slot_t; /* * vbd I/O request * * An instance of this structure is bound to each buf passed to * the driver's strategy by setting the pointer into bp->av_back. * The id of this vreq will also be put into 'id' field in each * blkif_request_t when sent out to the ring buffer for one DMA * window of this buf. * * Vreq mainly contains DMA information for this buf. In one vreq/buf, * there could be more than one DMA window, each of which will be * mapped to one blkif_request_t/ge_slot_t. Ge_slot_t contains all grant * table entry information for this buf. The ge_slot_t for current DMA * window is pointed to by v_gs in vreq. * * So, grant table entries will only be alloc'ed when the DMA window is * about to be transferred via blkif_request_t to the ring buffer. And * they will be freed right after the blkif_response_t is seen. By this * means, we can make use of grant table entries more efficiently. */ typedef struct v_req { list_node_t v_link; list_t v_gs; int v_status; buf_t *v_buf; uint_t v_ndmacs; uint_t v_dmaw; uint_t v_ndmaws; uint_t v_nslots; uint64_t v_blkno; ddi_dma_handle_t v_memdmahdl; ddi_acc_handle_t v_align; ddi_dma_handle_t v_dmahdl; ddi_dma_cookie_t v_dmac; caddr_t v_abuf; uint8_t v_flush_diskcache; boolean_t v_runq; } v_req_t; /* * Status set and checked in vreq->v_status by vreq_setup() * * These flags will help us to continue the vreq setup work from last failure * point, instead of starting from scratch after each failure. */ #define VREQ_INIT 0x0 #define VREQ_INIT_DONE 0x1 #define VREQ_DMAHDL_ALLOCED 0x2 #define VREQ_MEMDMAHDL_ALLOCED 0x3 #define VREQ_DMAMEM_ALLOCED 0x4 #define VREQ_DMABUF_BOUND 0x5 #define VREQ_GS_ALLOCED 0x6 #define VREQ_DMAWIN_DONE 0x7 /* * virtual block device per-instance softstate */ typedef struct xdf { dev_info_t *xdf_dip; char *xdf_addr; ddi_iblock_cookie_t xdf_ibc; /* mutex iblock cookie */ domid_t xdf_peer; /* otherend's dom ID */ xendev_ring_t *xdf_xb_ring; /* I/O ring buffer */ ddi_acc_handle_t xdf_xb_ring_hdl; /* access handler for ring buffer */ list_t xdf_vreq_act; /* active vreq list */ buf_t *xdf_f_act; /* active buf list head */ buf_t *xdf_l_act; /* active buf list tail */ buf_t *xdf_i_act; /* active buf list index */ xdf_state_t xdf_state; /* status of this virtual disk */ boolean_t xdf_suspending; ulong_t xdf_vd_open[OTYPCNT]; ulong_t xdf_vd_lyropen[XDF_PEXT]; ulong_t xdf_connect_req; kthread_t *xdf_connect_thread; ulong_t xdf_vd_exclopen; kmutex_t xdf_iostat_lk; /* muxes lock for the iostat ptr */ kmutex_t xdf_dev_lk; /* mutex lock for I/O path */ kmutex_t xdf_cb_lk; /* mutex lock for event handling path */ kcondvar_t xdf_dev_cv; /* cv used in I/O path */ uint_t xdf_dinfo; /* disk info from backend xenstore */ diskaddr_t xdf_xdev_nblocks; /* total size in block */ uint_t xdf_xdev_secsize; /* disk blksize from backend */ cmlb_geom_t xdf_pgeom; boolean_t xdf_pgeom_set; boolean_t xdf_pgeom_fixed; kstat_t *xdf_xdev_iostat; cmlb_handle_t xdf_vd_lbl; ddi_softintr_t xdf_softintr_id; timeout_id_t xdf_timeout_id; struct gnttab_free_callback xdf_gnt_callback; boolean_t xdf_feature_barrier; boolean_t xdf_flush_supported; boolean_t xdf_media_req_supported; boolean_t xdf_wce; boolean_t xdf_cmlb_reattach; char *xdf_flush_mem; char *xdf_cache_flush_block; int xdf_evtchn; enum dkio_state xdf_mstate; kcondvar_t xdf_mstate_cv; kcondvar_t xdf_hp_status_cv; struct buf *xdf_ready_bp; ddi_taskq_t *xdf_ready_tq; kthread_t *xdf_ready_tq_thread; struct buf *xdf_ready_tq_bp; ddi_devid_t xdf_tgt_devid; #ifdef DEBUG int xdf_dmacallback_num; kthread_t *xdf_oe_change_thread; #endif } xdf_t; /* * VBD I/O requests must be aligned on a 512-byte boundary and specify * a transfer size which is a mutiple of 512-bytes */ #define ALIGNED_XFER(bp) \ ((((uintptr_t)((bp)->b_un.b_addr) & XB_BMASK) == 0) && \ (((bp)->b_bcount & XB_BMASK) == 0)) #define U_INVAL(u) (((u)->uio_loffset & (offset_t)(XB_BMASK)) || \ ((u)->uio_iov->iov_len & (offset_t)(XB_BMASK))) /* wrap pa_to_ma() for xdf to run in dom0 */ #define PATOMA(addr) (DOMAIN_IS_INITDOMAIN(xen_info) ? addr : pa_to_ma(addr)) #define XD_IS_RO(vbd) VOID2BOOLEAN((vbd)->xdf_dinfo & VDISK_READONLY) #define XD_IS_CD(vbd) VOID2BOOLEAN((vbd)->xdf_dinfo & VDISK_CDROM) #define XD_IS_RM(vbd) VOID2BOOLEAN((vbd)->xdf_dinfo & VDISK_REMOVABLE) #define IS_READ(bp) VOID2BOOLEAN((bp)->b_flags & B_READ) #define IS_ERROR(bp) VOID2BOOLEAN((bp)->b_flags & B_ERROR) #define XDF_UPDATE_IO_STAT(vdp, bp) \ { \ kstat_io_t *kip = KSTAT_IO_PTR((vdp)->xdf_xdev_iostat); \ size_t n_done = (bp)->b_bcount - (bp)->b_resid; \ if ((bp)->b_flags & B_READ) { \ kip->reads++; \ kip->nread += n_done; \ } else { \ kip->writes++; \ kip->nwritten += n_done; \ } \ } #ifdef DEBUG #define DPRINTF(flag, args) {if (xdf_debug & (flag)) prom_printf args; } #define SETDMACBON(vbd) {(vbd)->xdf_dmacallback_num++; } #define SETDMACBOFF(vbd) {(vbd)->xdf_dmacallback_num--; } #define ISDMACBON(vbd) ((vbd)->xdf_dmacallback_num > 0) #else #define DPRINTF(flag, args) #define SETDMACBON(vbd) #define SETDMACBOFF(vbd) #define ISDMACBON(vbd) #endif /* DEBUG */ #define DDI_DBG 0x1 #define DMA_DBG 0x2 #define INTR_DBG 0x8 #define IO_DBG 0x10 #define IOCTL_DBG 0x20 #define SUSRES_DBG 0x40 #define LBL_DBG 0x80 #ifdef XPV_HVM_DRIVER extern int xdf_lb_getinfo(dev_info_t *, int, void *, void *); extern int xdf_lb_rdwr(dev_info_t *, uchar_t, void *, diskaddr_t, size_t, void *); extern void xdfmin(struct buf *bp); extern dev_info_t *xdf_hvm_hold(const char *); extern boolean_t xdf_hvm_connect(dev_info_t *); extern int xdf_hvm_setpgeom(dev_info_t *, cmlb_geom_t *); extern boolean_t xdf_is_cd(dev_info_t *); extern boolean_t xdf_is_rm(dev_info_t *); extern boolean_t xdf_media_req_supported(dev_info_t *); #endif /* XPV_HVM_DRIVER */ #ifdef __cplusplus } #endif #endif /* _SYS_XDF_H */ /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2007 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ /* * Client-facing interface for the Xenbus driver. In other words, the * interface between the Xenbus and the device-specific code, be it the * frontend or the backend of that driver. * * Copyright (C) 2005 XenSource Ltd * * This file may be distributed separately from the Linux kernel, or * incorporated into other software packages, subject to the following license: * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this source file (the "Software"), to deal in the Software without * restriction, including without limitation the rights to use, copy, modify, * merge, publish, distribute, sublicense, and/or sell copies of the Software, * and to permit persons to whom the Software is furnished to do so, subject to * the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS * IN THE SOFTWARE. */ #ifdef XPV_HVM_DRIVER #include #include #else #include #include #include #endif #include #include #include int xenbus_watch_path(struct xenbus_device *dev, const char *path, struct xenbus_watch *watch, void (*callback)(struct xenbus_watch *, const char **, unsigned int)) { int err; watch->node = path; watch->callback = callback; err = register_xenbus_watch(watch); if (err) { watch->node = NULL; watch->callback = NULL; xenbus_dev_fatal(dev, err, "adding watch on %s", path); } return (err); } int xenbus_watch_path2(struct xenbus_device *dev, const char *path, const char *path2, struct xenbus_watch *watch, void (*callback)(struct xenbus_watch *, const char **, unsigned int)) { int err; char *state; state = kmem_alloc(strlen(path) + 1 + strlen(path2) + 1, KM_SLEEP); (void) strcpy(state, path); (void) strcat(state, "/"); (void) strcat(state, path2); err = xenbus_watch_path(dev, state, watch, callback); if (err) kmem_free(state, strlen(state) + 1); return (err); } /* * Returns 0 on success, -1 if no change was made, or an errno on failure. We * check whether the state is currently set to the given value, and if not, * then the state is set. We don't want to unconditionally write the given * state, because we don't want to fire watches unnecessarily. Furthermore, if * the node has gone, we don't write to it, as the device will be tearing down, * and we don't want to resurrect that directory. * * XXPV: not clear that this is still safe if two threads are racing to update * the state? */ int xenbus_switch_state(struct xenbus_device *dev, xenbus_transaction_t xbt, XenbusState state) { int current_state; int err; err = xenbus_scanf(xbt, dev->nodename, "state", "%d", ¤t_state); /* XXPV: is this really the right thing to do? */ if (err == ENOENT) return (0); if (err) return (err); err = -1; if ((XenbusState)current_state != state) { err = xenbus_printf(xbt, dev->nodename, "state", "%d", state); if (err) xenbus_dev_fatal(dev, err, "writing new state"); } return (err); } /* * Return the path to the error node for the given device, or NULL on failure. * If the value returned is non-NULL, then it is the caller's to kmem_free. */ static char * error_path(struct xenbus_device *dev) { char *path_buffer; path_buffer = kmem_alloc(strlen("error/") + strlen(dev->nodename) + 1, KM_SLEEP); (void) strcpy(path_buffer, "error/"); (void) strcpy(path_buffer + strlen("error/"), dev->nodename); return (path_buffer); } static void common_dev_error(struct xenbus_device *dev, int err, const char *fmt, va_list ap) { int ret; unsigned int len; char *printf_buffer = NULL, *path_buffer = NULL; #define PRINTF_BUFFER_SIZE 4096 printf_buffer = kmem_alloc(PRINTF_BUFFER_SIZE, KM_SLEEP); (void) snprintf(printf_buffer, PRINTF_BUFFER_SIZE, "%d ", err); len = strlen(printf_buffer); ret = vsnprintf(printf_buffer+len, PRINTF_BUFFER_SIZE-len, fmt, ap); ASSERT(len + ret <= PRINTF_BUFFER_SIZE-1); dev->has_error = 1; path_buffer = error_path(dev); if (path_buffer == NULL) { printf("xenbus: failed to write error node for %s (%s)\n", dev->nodename, printf_buffer); goto fail; } if (xenbus_write(0, path_buffer, "error", printf_buffer) != 0) { printf("xenbus: failed to write error node for %s (%s)\n", dev->nodename, printf_buffer); goto fail; } fail: if (printf_buffer) kmem_free(printf_buffer, PRINTF_BUFFER_SIZE); if (path_buffer) kmem_free(path_buffer, strlen(path_buffer) + 1); } void xenbus_dev_error(struct xenbus_device *dev, int err, const char *fmt, ...) { va_list ap; va_start(ap, fmt); common_dev_error(dev, err, fmt, ap); va_end(ap); } void xenbus_dev_fatal(struct xenbus_device *dev, int err, const char *fmt, ...) { va_list ap; va_start(ap, fmt); common_dev_error(dev, err, fmt, ap); va_end(ap); (void) xenbus_switch_state(dev, XBT_NULL, XenbusStateClosing); } /* Clear any error. */ void xenbus_dev_ok(struct xenbus_device *dev) { if (dev->has_error) { if (xenbus_rm(0, dev->nodename, "error") != 0) printf("xenbus: failed to clear error node for %s\n", dev->nodename); else dev->has_error = 0; } } int xenbus_grant_ring(struct xenbus_device *dev, unsigned long ring_mfn) { int err = gnttab_grant_foreign_access(dev->otherend_id, ring_mfn, 0); if (err < 0) xenbus_dev_fatal(dev, err, "granting access to ring page"); return (err); } int xenbus_alloc_evtchn(struct xenbus_device *dev, int *port) { int err; err = xen_alloc_unbound_evtchn(dev->otherend_id, port); if (err) xenbus_dev_fatal(dev, err, "allocating event channel"); return (err); } XenbusState xenbus_read_driver_state(const char *path) { XenbusState result; int err = xenbus_gather(XBT_NULL, path, "state", "%d", &result, NULL); if (err) result = XenbusStateClosed; return (result); } /* * Local variables: * c-file-style: "solaris" * indent-tabs-mode: t * c-indent-level: 8 * c-basic-offset: 8 * tab-width: 8 * End: */ /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2008 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ /* * * xenbus_comms.c * * Low level code to talks to Xen Store: ringbuffer and event channel. * * Copyright (C) 2005 Rusty Russell, IBM Corporation * * This file may be distributed separately from the Linux kernel, or * incorporated into other software packages, subject to the following license: * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this source file (the "Software"), to deal in the Software without * restriction, including without limitation the rights to use, copy, modify, * merge, publish, distribute, sublicense, and/or sell copies of the Software, * and to permit persons to whom the Software is furnished to do so, subject to * the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS * IN THE SOFTWARE. */ #include #include #include #include #include #ifdef XPV_HVM_DRIVER #include #include #include #else #include #include #include #include #endif #include #include #include #include #include #include #include #include #include #ifndef XPV_HVM_DRIVER static int xenbus_irq; #endif static ddi_umem_cookie_t xb_cookie; /* cookie for xenbus comm page */ extern caddr_t xb_addr; /* va of xenbus comm page */ static kcondvar_t xb_wait_cv; static kmutex_t xb_wait_lock; #define xs_domain_interface(ra) ((struct xenstore_domain_interface *)(ra)) static uint_t xenbus_intr(caddr_t arg __unused, caddr_t arg1 __unused) { mutex_enter(&xb_wait_lock); cv_broadcast(&xb_wait_cv); mutex_exit(&xb_wait_lock); return (DDI_INTR_CLAIMED); } static int check_indexes(XENSTORE_RING_IDX cons, XENSTORE_RING_IDX prod) { return ((prod - cons) <= XENSTORE_RING_SIZE); } static void * get_output_chunk(XENSTORE_RING_IDX cons, XENSTORE_RING_IDX prod, char *buf, uint32_t *len) { *len = XENSTORE_RING_SIZE - MASK_XENSTORE_IDX(prod); if ((XENSTORE_RING_SIZE - (prod - cons)) < *len) *len = XENSTORE_RING_SIZE - (prod - cons); return ((void *)(buf + MASK_XENSTORE_IDX(prod))); } static const void * get_input_chunk(XENSTORE_RING_IDX cons, XENSTORE_RING_IDX prod, const char *buf, uint32_t *len) { *len = XENSTORE_RING_SIZE - MASK_XENSTORE_IDX(cons); if ((prod - cons) < *len) *len = prod - cons; return ((void *)(buf + MASK_XENSTORE_IDX(cons))); } int xb_write(const void *data, unsigned len) { volatile struct xenstore_domain_interface *intf = xs_domain_interface(xb_addr); XENSTORE_RING_IDX cons, prod; extern int do_polled_io; while (len != 0) { void *dst; unsigned int avail; mutex_enter(&xb_wait_lock); while ((intf->req_prod - intf->req_cons) == XENSTORE_RING_SIZE) { if (interrupts_unleashed && !do_polled_io) { if (cv_wait_sig(&xb_wait_cv, &xb_wait_lock) == 0) { mutex_exit(&xb_wait_lock); return (EINTR); } } else { /* polled mode needed for early probes */ (void) HYPERVISOR_yield(); } } mutex_exit(&xb_wait_lock); /* Read indexes, then verify. */ cons = intf->req_cons; prod = intf->req_prod; membar_enter(); if (!check_indexes(cons, prod)) return (EIO); dst = get_output_chunk(cons, prod, (char *)intf->req, &avail); if (avail == 0) continue; if (avail > len) avail = len; (void) memcpy(dst, data, avail); data = (void *)((uintptr_t)data + avail); len -= avail; /* Other side must not see new header until data is there. */ membar_producer(); intf->req_prod += avail; /* This implies mb() before other side sees interrupt. */ ec_notify_via_evtchn(xen_info->store_evtchn); } return (0); } int xb_read(void *data, unsigned len) { volatile struct xenstore_domain_interface *intf = xs_domain_interface(xb_addr); XENSTORE_RING_IDX cons, prod; extern int do_polled_io; while (len != 0) { unsigned int avail; const char *src; mutex_enter(&xb_wait_lock); while (intf->rsp_cons == intf->rsp_prod) { if (interrupts_unleashed && !do_polled_io) { if (cv_wait_sig(&xb_wait_cv, &xb_wait_lock) == 0) { mutex_exit(&xb_wait_lock); return (EINTR); } } else { /* polled mode needed for early probes */ (void) HYPERVISOR_yield(); } } mutex_exit(&xb_wait_lock); /* Read indexes, then verify. */ cons = intf->rsp_cons; prod = intf->rsp_prod; membar_enter(); if (!check_indexes(cons, prod)) return (EIO); src = get_input_chunk(cons, prod, (char *)intf->rsp, &avail); if (avail == 0) continue; if (avail > len) avail = len; /* We must read header before we read data. */ membar_consumer(); (void) memcpy(data, src, avail); data = (void *)((uintptr_t)data + avail); len -= avail; /* Other side must not see free space until we've copied out */ membar_enter(); intf->rsp_cons += avail; /* Implies mb(): they will see new header. */ ec_notify_via_evtchn(xen_info->store_evtchn); } return (0); } void xb_suspend(void) { #ifdef XPV_HVM_DRIVER ec_unbind_evtchn(xen_info->store_evtchn); #else rem_avintr(NULL, IPL_XENBUS, xenbus_intr, xenbus_irq); #endif } void xb_setup_intr(void) { #ifdef XPV_HVM_DRIVER ec_bind_evtchn_to_handler(xen_info->store_evtchn, IPL_XENBUS, xenbus_intr, NULL); #else xenbus_irq = ec_bind_evtchn_to_irq(xen_info->store_evtchn); if (xenbus_irq < 0) { cmn_err(CE_WARN, "Couldn't bind xenbus event channel"); return; } if (!add_avintr(NULL, IPL_XENBUS, xenbus_intr, "xenbus", xenbus_irq, NULL, NULL, NULL, NULL)) cmn_err(CE_WARN, "XENBUS add intr failed\n"); #endif } /* * Set up our xenstore page and event channel. Domain 0 needs to allocate a * page and event channel; other domains use what we are told. */ void xb_init(void) { int err; if (DOMAIN_IS_INITDOMAIN(xen_info)) { if (xb_addr != NULL) return; xb_addr = ddi_umem_alloc(PAGESIZE, DDI_UMEM_SLEEP, &xb_cookie); xen_info->store_mfn = pfn_to_mfn(hat_getpfnum(kas.a_hat, xb_addr)); err = xen_alloc_unbound_evtchn(0, (int *)&xen_info->store_evtchn); ASSERT(err == 0); } else { /* * This is harmless on first boot, but needed for resume and * migrate. We use kbm_map_ma() as a shortcut instead of * directly using HYPERVISOR_update_va_mapping(). */ ASSERT(xb_addr != NULL); kbm_map_ma(mfn_to_ma(xen_info->store_mfn), (uintptr_t)xb_addr, 0); } ASSERT(xen_info->store_evtchn); } void * xb_xenstore_cookie(void) { ASSERT(DOMAIN_IS_INITDOMAIN(xen_info)); return (xb_cookie); } /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2008 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ /* * xenbus_dev.c * * Driver giving user-space access to the kernel's xenbus connection * to xenstore. * * Copyright (c) 2005, Christian Limpach * Copyright (c) 2005, Rusty Russell, IBM Corporation * * This file may be distributed separately from the Linux kernel, or * incorporated into other software packages, subject to the following license: * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this source file (the "Software"), to deal in the Software without * restriction, including without limitation the rights to use, copy, modify, * merge, publish, distribute, sublicense, and/or sell copies of the Software, * and to permit persons to whom the Software is furnished to do so, subject to * the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS * IN THE SOFTWARE. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef XPV_HVM_DRIVER #include #include #include #endif #include #include #include #include #include #ifdef DEBUG #define XENBUSDRV_DBPRINT(fmt) { if (xenbusdrv_debug) cmn_err fmt; } #else #define XENBUSDRV_DBPRINT(fmt) #endif /* ifdef DEBUG */ /* Some handy macros */ #define XENBUSDRV_MASK_READ_IDX(idx) ((idx) & (PAGESIZE - 1)) #define XENBUSDRV_MINOR2INST(minor) ((int)(minor)) #define XENBUSDRV_NCLONES 256 #define XENBUSDRV_INST2SOFTS(instance) \ ((xenbus_dev_t *)ddi_get_soft_state(xenbusdrv_statep, (instance))) static int xenbusdrv_debug = 0; static int xenbusdrv_clone_tab[XENBUSDRV_NCLONES]; static dev_info_t *xenbusdrv_dip; static kmutex_t xenbusdrv_clone_tab_mutex; struct xenbus_dev_transaction { list_t list; xenbus_transaction_t handle; }; /* Soft state data structure for xenbus driver */ struct xenbus_dev_data { dev_info_t *dip; /* In-progress transaction. */ list_t transactions; /* Partial request. */ unsigned int len; union { struct xsd_sockmsg msg; char buffer[MMU_PAGESIZE]; } u; /* Response queue. */ char read_buffer[MMU_PAGESIZE]; unsigned int read_cons, read_prod; kcondvar_t read_cv; kmutex_t read_mutex; int xenstore_inst; }; typedef struct xenbus_dev_data xenbus_dev_t; static void *xenbusdrv_statep; static int xenbusdrv_info(dev_info_t *, ddi_info_cmd_t, void *, void **); static int xenbusdrv_attach(dev_info_t *, ddi_attach_cmd_t); static int xenbusdrv_detach(dev_info_t *, ddi_detach_cmd_t); static int xenbusdrv_open(dev_t *, int, int, cred_t *); static int xenbusdrv_close(dev_t, int, int, cred_t *); static int xenbusdrv_read(dev_t, struct uio *, cred_t *); static int xenbusdrv_write(dev_t, struct uio *, cred_t *); static int xenbusdrv_devmap(dev_t, devmap_cookie_t, offset_t, size_t, size_t *, uint_t); static int xenbusdrv_segmap(dev_t, off_t, ddi_as_handle_t, caddr_t *, off_t, uint_t, uint_t, uint_t, cred_t *); static int xenbusdrv_ioctl(dev_t, int, intptr_t, int, cred_t *, int *); static int xenbusdrv_queue_reply(xenbus_dev_t *, const struct xsd_sockmsg *, const char *); /* Solaris driver framework */ static struct cb_ops xenbusdrv_cb_ops = { xenbusdrv_open, /* cb_open */ xenbusdrv_close, /* cb_close */ nodev, /* cb_strategy */ nodev, /* cb_print */ nodev, /* cb_dump */ xenbusdrv_read, /* cb_read */ xenbusdrv_write, /* cb_write */ xenbusdrv_ioctl, /* cb_ioctl */ xenbusdrv_devmap, /* cb_devmap */ NULL, /* cb_mmap */ xenbusdrv_segmap, /* cb_segmap */ nochpoll, /* cb_chpoll */ ddi_prop_op, /* cb_prop_op */ 0, /* cb_stream */ D_DEVMAP | D_NEW | D_MP, /* cb_flag */ CB_REV }; static struct dev_ops xenbusdrv_dev_ops = { DEVO_REV, /* devo_rev */ 0, /* devo_refcnt */ xenbusdrv_info, /* devo_getinfo */ nulldev, /* devo_identify */ nulldev, /* devo_probe */ xenbusdrv_attach, /* devo_attach */ xenbusdrv_detach, /* devo_detach */ nodev, /* devo_reset */ &xenbusdrv_cb_ops, /* devo_cb_ops */ NULL, /* devo_bus_ops */ NULL, /* devo_power */ ddi_quiesce_not_needed, /* devo_quiesce */ }; static struct modldrv modldrv = { &mod_driverops, /* Type of module. This one is a driver */ "virtual bus driver", /* Name of the module. */ &xenbusdrv_dev_ops /* driver ops */ }; static struct modlinkage modlinkage = { MODREV_1, &modldrv, NULL }; int _init(void) { int e; e = ddi_soft_state_init(&xenbusdrv_statep, sizeof (xenbus_dev_t), 1); if (e) return (e); e = mod_install(&modlinkage); if (e) ddi_soft_state_fini(&xenbusdrv_statep); return (e); } int _fini(void) { int e; e = mod_remove(&modlinkage); if (e) return (e); ddi_soft_state_fini(&xenbusdrv_statep); return (0); } int _info(struct modinfo *modinfop) { return (mod_info(&modlinkage, modinfop)); } /* ARGSUSED */ static int xenbusdrv_info(dev_info_t *dip, ddi_info_cmd_t cmd, void *arg, void **result) { dev_t dev = (dev_t)arg; minor_t minor = getminor(dev); int retval; switch (cmd) { case DDI_INFO_DEVT2DEVINFO: if (minor != 0 || xenbusdrv_dip == NULL) { *result = (void *)NULL; retval = DDI_FAILURE; } else { *result = (void *)xenbusdrv_dip; retval = DDI_SUCCESS; } break; case DDI_INFO_DEVT2INSTANCE: *result = (void *)0; retval = DDI_SUCCESS; break; default: retval = DDI_FAILURE; } return (retval); } static int xenbusdrv_attach(dev_info_t *dip, ddi_attach_cmd_t cmd) { int error; int unit = ddi_get_instance(dip); switch (cmd) { case DDI_ATTACH: break; case DDI_RESUME: return (DDI_SUCCESS); default: cmn_err(CE_WARN, "xenbus_attach: unknown cmd 0x%x\n", cmd); return (DDI_FAILURE); } /* DDI_ATTACH */ /* * only one instance - but we clone using the open routine */ if (ddi_get_instance(dip) > 0) return (DDI_FAILURE); mutex_init(&xenbusdrv_clone_tab_mutex, NULL, MUTEX_DRIVER, NULL); error = ddi_create_minor_node(dip, "xenbus", S_IFCHR, unit, DDI_PSEUDO, 0); if (error != DDI_SUCCESS) goto fail; /* * save dip for getinfo */ xenbusdrv_dip = dip; ddi_report_dev(dip); #ifndef XPV_HVM_DRIVER if (DOMAIN_IS_INITDOMAIN(xen_info)) xs_dom0_init(); #endif return (DDI_SUCCESS); fail: (void) xenbusdrv_detach(dip, DDI_DETACH); return (error); } static int xenbusdrv_detach(dev_info_t *dip, ddi_detach_cmd_t cmd) { /* * again, only one instance */ if (ddi_get_instance(dip) > 0) return (DDI_FAILURE); switch (cmd) { case DDI_DETACH: ddi_remove_minor_node(dip, NULL); mutex_destroy(&xenbusdrv_clone_tab_mutex); xenbusdrv_dip = NULL; return (DDI_SUCCESS); case DDI_SUSPEND: return (DDI_SUCCESS); default: cmn_err(CE_WARN, "xenbus_detach: unknown cmd 0x%x\n", cmd); return (DDI_FAILURE); } } /* ARGSUSED */ static int xenbusdrv_open(dev_t *devp, int flag, int otyp, cred_t *cr) { xenbus_dev_t *xbs; minor_t minor = getminor(*devp); if (otyp == OTYP_BLK) return (ENXIO); /* * only allow open on minor = 0 - the clone device */ if (minor != 0) return (ENXIO); /* * find a free slot and grab it */ mutex_enter(&xenbusdrv_clone_tab_mutex); for (minor = 1; minor < XENBUSDRV_NCLONES; minor++) { if (xenbusdrv_clone_tab[minor] == 0) { xenbusdrv_clone_tab[minor] = 1; break; } } mutex_exit(&xenbusdrv_clone_tab_mutex); if (minor == XENBUSDRV_NCLONES) return (EAGAIN); /* Allocate softstate structure */ if (ddi_soft_state_zalloc(xenbusdrv_statep, XENBUSDRV_MINOR2INST(minor)) != DDI_SUCCESS) { mutex_enter(&xenbusdrv_clone_tab_mutex); xenbusdrv_clone_tab[minor] = 0; mutex_exit(&xenbusdrv_clone_tab_mutex); return (EAGAIN); } xbs = XENBUSDRV_INST2SOFTS(XENBUSDRV_MINOR2INST(minor)); /* ... and init it */ xbs->dip = xenbusdrv_dip; mutex_init(&xbs->read_mutex, NULL, MUTEX_DRIVER, NULL); cv_init(&xbs->read_cv, NULL, CV_DEFAULT, NULL); list_create(&xbs->transactions, sizeof (struct xenbus_dev_transaction), offsetof(struct xenbus_dev_transaction, list)); /* clone driver */ *devp = makedevice(getmajor(*devp), minor); XENBUSDRV_DBPRINT((CE_NOTE, "Xenbus drv open succeeded, minor=%d", minor)); return (0); } /* ARGSUSED */ static int xenbusdrv_close(dev_t dev, int flag, int otyp, struct cred *cr) { xenbus_dev_t *xbs; minor_t minor = getminor(dev); struct xenbus_dev_transaction *trans; xbs = XENBUSDRV_INST2SOFTS(XENBUSDRV_MINOR2INST(minor)); if (xbs == NULL) return (ENXIO); #ifdef notyet /* * XXPV - would like to be able to notify xenstore down here, but * as the daemon is currently written, it doesn't leave the device * open after initial setup, so we have no way of knowing if it has * gone away. */ if (xbs->xenstore_inst) xs_notify_xenstore_down(); #endif /* free pending transaction */ while (trans = (struct xenbus_dev_transaction *) list_head(&xbs->transactions)) { (void) xenbus_transaction_end(trans->handle, 1); list_remove(&xbs->transactions, (void *)trans); kmem_free(trans, sizeof (*trans)); } mutex_destroy(&xbs->read_mutex); cv_destroy(&xbs->read_cv); ddi_soft_state_free(xenbusdrv_statep, XENBUSDRV_MINOR2INST(minor)); /* * free clone tab slot */ mutex_enter(&xenbusdrv_clone_tab_mutex); xenbusdrv_clone_tab[minor] = 0; mutex_exit(&xenbusdrv_clone_tab_mutex); XENBUSDRV_DBPRINT((CE_NOTE, "Xenbus drv close succeeded, minor=%d", minor)); return (0); } /* ARGSUSED */ static int xenbusdrv_read(dev_t dev, struct uio *uiop, cred_t *cr) { xenbus_dev_t *xbs; size_t len; int res, ret; int idx; XENBUSDRV_DBPRINT((CE_NOTE, "xenbusdrv_read called")); if (secpolicy_xvm_control(cr)) return (EPERM); xbs = XENBUSDRV_INST2SOFTS(XENBUSDRV_MINOR2INST(getminor(dev))); mutex_enter(&xbs->read_mutex); /* check if we have something to read */ while (xbs->read_prod == xbs->read_cons) { if (cv_wait_sig(&xbs->read_cv, &xbs->read_mutex) == 0) { mutex_exit(&xbs->read_mutex); return (EINTR); } } idx = XENBUSDRV_MASK_READ_IDX(xbs->read_cons); res = uiop->uio_resid; len = xbs->read_prod - xbs->read_cons; if (len > (sizeof (xbs->read_buffer) - idx)) len = sizeof (xbs->read_buffer) - idx; if (len > res) len = res; ret = uiomove(xbs->read_buffer + idx, len, UIO_READ, uiop); xbs->read_cons += res - uiop->uio_resid; mutex_exit(&xbs->read_mutex); return (ret); } /* * prepare data for xenbusdrv_read() */ static int xenbusdrv_queue_reply(xenbus_dev_t *xbs, const struct xsd_sockmsg *msg, const char *reply) { int i; int remaining; XENBUSDRV_DBPRINT((CE_NOTE, "xenbusdrv_queue_reply called")); mutex_enter(&xbs->read_mutex); remaining = sizeof (xbs->read_buffer) - (xbs->read_prod - xbs->read_cons); if (sizeof (*msg) + msg->len > remaining) { mutex_exit(&xbs->read_mutex); return (EOVERFLOW); } for (i = 0; i < sizeof (*msg); i++, xbs->read_prod++) { xbs->read_buffer[XENBUSDRV_MASK_READ_IDX(xbs->read_prod)] = ((char *)msg)[i]; } for (i = 0; i < msg->len; i++, xbs->read_prod++) { xbs->read_buffer[XENBUSDRV_MASK_READ_IDX(xbs->read_prod)] = reply[i]; } cv_broadcast(&xbs->read_cv); mutex_exit(&xbs->read_mutex); XENBUSDRV_DBPRINT((CE_NOTE, "xenbusdrv_queue_reply exited")); return (0); } /* ARGSUSED */ static int xenbusdrv_write(dev_t dev, struct uio *uiop, cred_t *cr) { xenbus_dev_t *xbs; struct xenbus_dev_transaction *trans; void *reply; size_t len; int rc = 0; XENBUSDRV_DBPRINT((CE_NOTE, "xenbusdrv_write called")); if (secpolicy_xvm_control(cr)) return (EPERM); xbs = XENBUSDRV_INST2SOFTS(XENBUSDRV_MINOR2INST(getminor(dev))); len = uiop->uio_resid; if ((len + xbs->len) > sizeof (xbs->u.buffer)) { XENBUSDRV_DBPRINT((CE_WARN, "Request is too big")); rc = EINVAL; goto out; } if (uiomove(xbs->u.buffer + xbs->len, len, UIO_WRITE, uiop) != 0) { XENBUSDRV_DBPRINT((CE_WARN, "Uiomove failed")); rc = EFAULT; goto out; } xbs->len += len; if (xbs->len < (sizeof (xbs->u.msg)) || xbs->len < (sizeof (xbs->u.msg) + xbs->u.msg.len)) { XENBUSDRV_DBPRINT((CE_NOTE, "Partial request")); return (0); } switch (xbs->u.msg.type) { case XS_TRANSACTION_START: case XS_TRANSACTION_END: case XS_DIRECTORY: case XS_READ: case XS_GET_PERMS: case XS_RELEASE: case XS_GET_DOMAIN_PATH: case XS_WRITE: case XS_MKDIR: case XS_RM: case XS_SET_PERMS: /* send the request to xenstore and get feedback */ rc = xenbus_dev_request_and_reply(&xbs->u.msg, &reply); if (rc) { XENBUSDRV_DBPRINT((CE_WARN, "xenbus_dev_request_and_reply failed")); goto out; } /* handle transaction start/end */ if (xbs->u.msg.type == XS_TRANSACTION_START) { trans = kmem_alloc(sizeof (*trans), KM_SLEEP); (void) ddi_strtoul((char *)reply, NULL, 0, (unsigned long *)&trans->handle); list_insert_tail(&xbs->transactions, (void *)trans); } else if (xbs->u.msg.type == XS_TRANSACTION_END) { /* try to find out the ending transaction */ for (trans = (struct xenbus_dev_transaction *) list_head(&xbs->transactions); trans; trans = (struct xenbus_dev_transaction *) list_next(&xbs->transactions, (void *)trans)) if (trans->handle == (xenbus_transaction_t) xbs->u.msg.tx_id) break; ASSERT(trans); /* free it, if we find it */ list_remove(&xbs->transactions, (void *)trans); kmem_free(trans, sizeof (*trans)); } /* prepare data for xenbusdrv_read() to get */ rc = xenbusdrv_queue_reply(xbs, &xbs->u.msg, reply); kmem_free(reply, xbs->u.msg.len + 1); break; default: rc = EINVAL; } out: xbs->len = 0; return (rc); } /*ARGSUSED*/ static int xenbusdrv_devmap(dev_t dev, devmap_cookie_t dhp, offset_t off, size_t len, size_t *maplen, uint_t model) { xenbus_dev_t *xbs; int err; xbs = XENBUSDRV_INST2SOFTS(XENBUSDRV_MINOR2INST(getminor(dev))); if (off != 0 || len != PAGESIZE) return (-1); if (!DOMAIN_IS_INITDOMAIN(xen_info)) return (-1); err = devmap_umem_setup(dhp, xbs->dip, NULL, xb_xenstore_cookie(), 0, PAGESIZE, PROT_READ | PROT_WRITE | PROT_USER, 0, NULL); if (err) return (err); *maplen = PAGESIZE; return (0); } static int xenbusdrv_segmap(dev_t dev, off_t off, ddi_as_handle_t as, caddr_t *addrp, off_t len, uint_t prot, uint_t maxprot, uint_t flags, cred_t *cr) { if (secpolicy_xvm_control(cr)) return (EPERM); return (ddi_devmap_segmap(dev, off, as, addrp, len, prot, maxprot, flags, cr)); } /*ARGSUSED*/ static int xenbusdrv_ioctl(dev_t dev, int cmd, intptr_t arg, int mode, cred_t *cr, int *rvalp) { xenbus_dev_t *xbs; if (secpolicy_xvm_control(cr)) return (EPERM); xbs = XENBUSDRV_INST2SOFTS(XENBUSDRV_MINOR2INST(getminor(dev))); switch (cmd) { case IOCTL_XENBUS_XENSTORE_EVTCHN: *rvalp = xen_info->store_evtchn; break; case IOCTL_XENBUS_NOTIFY_UP: xs_notify_xenstore_up(); xbs->xenstore_inst = 1; break; default: return (EINVAL); } return (0); } /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2009 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ /* * Talks to Xen Store to figure out what devices we have. * * Copyright (C) 2005 Rusty Russell, IBM Corporation * Copyright (C) 2005 Mike Wray, Hewlett-Packard * Copyright (C) 2005 XenSource Ltd * * This file may be distributed separately from the Linux kernel, or * incorporated into other software packages, subject to the following license: * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this source file (the "Software"), to deal in the Software without * restriction, including without limitation the rights to use, copy, modify, * merge, publish, distribute, sublicense, and/or sell copies of the Software, * and to permit persons to whom the Software is furnished to do so, subject to * the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS * IN THE SOFTWARE. */ #ifdef XPV_HVM_DRIVER #include #endif #include #include #include #include static int read_otherend_details(struct xenbus_device *xendev, char *id_node, char *path_node) { int err = xenbus_gather(XBT_NULL, xendev->nodename, id_node, "%i", &xendev->otherend_id, path_node, NULL, &xendev->otherend, NULL); if (err) { xenbus_dev_fatal(xendev, err, "reading other end details from %s", xendev->nodename); return (err); } if (strlen(xendev->otherend) == 0 || !xenbus_exists_dir(xendev->otherend, "")) { xenbus_dev_fatal(xendev, X_ENOENT, "missing other end from %s", xendev->nodename); kmem_free((void *)xendev->otherend, strlen(xendev->otherend) + 1); xendev->otherend = NULL; return (X_ENOENT); } return (0); } static int read_backend_details(struct xenbus_device *xendev) { return (read_otherend_details(xendev, "backend-id", "backend")); } static int read_frontend_details(struct xenbus_device *xendev) { return (read_otherend_details(xendev, "frontend-id", "frontend")); } static void free_otherend_details(struct xenbus_device *dev) { if (dev->otherend != NULL) { kmem_free((void *)dev->otherend, strlen(dev->otherend) + 1); dev->otherend = NULL; } } static void free_otherend_watch(struct xenbus_device *dev) { if (dev->otherend_watch.node) { unregister_xenbus_watch(&dev->otherend_watch); kmem_free((void *)dev->otherend_watch.node, strlen(dev->otherend_watch.node) + 1); dev->otherend_watch.node = NULL; } } /*ARGSUSED2*/ static void otherend_changed(struct xenbus_watch *watch, const char **vec, unsigned int len) { struct xenbus_device *dev = watch->dev; XenbusState state; /* * Protect us against watches firing on old details when the otherend * details change, say immediately after a resume. */ if (!dev->otherend || strncmp(dev->otherend, vec[XS_WATCH_PATH], strlen(dev->otherend))) { #if 0 printf("Ignoring watch at %s", vec[XS_WATCH_PATH]); #endif return; } state = xenbus_read_driver_state(dev->otherend); #if 0 printf("state is %d, %s, %s", state, dev->otherend_watch.node, vec[XS_WATCH_PATH]); #endif if (dev->otherend_changed) dev->otherend_changed(dev, state); } int talk_to_otherend(struct xenbus_device *dev) { int err; free_otherend_watch(dev); free_otherend_details(dev); if (dev->frontend) err = read_backend_details(dev); else err = read_frontend_details(dev); if (err) return (err); dev->otherend_watch.dev = dev; return (xenbus_watch_path2(dev, dev->otherend, "state", &dev->otherend_watch, otherend_changed)); } /* * Local variables: * c-file-style: "solaris" * indent-tabs-mode: t * c-indent-level: 8 * c-basic-offset: 8 * tab-width: 8 * End: */ /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2009 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ /* * * xenbus_xs.c * * This is the kernel equivalent of the "xs" library. We don't need everything * and we use xenbus_comms for communication. * * Copyright (C) 2005 Rusty Russell, IBM Corporation * * This file may be distributed separately from the Linux kernel, or * incorporated into other software packages, subject to the following license: * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this source file (the "Software"), to deal in the Software without * restriction, including without limitation the rights to use, copy, modify, * merge, publish, distribute, sublicense, and/or sell copies of the Software, * and to permit persons to whom the Software is furnished to do so, subject to * the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS * IN THE SOFTWARE. */ /* * NOTE: To future maintainers of the Solaris version of this file: * I found the Linux version of this code to be very disgusting in * overloading pointers and error codes into void * return values. * The main difference you will find is that all such usage is changed * to pass pointers to void* to be filled in with return values and * the functions return error codes. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define _XSD_ERRORS_DEFINED #ifdef XPV_HVM_DRIVER #include #endif #include #include #include #include #include #include #include #define streq(a, b) (strcmp((a), (b)) == 0) #define list_empty(list) (list_head(list) == NULL) struct xs_stored_msg { list_node_t list; struct xsd_sockmsg hdr; union { /* Queued replies. */ struct { char *body; } reply; /* Queued watch events. */ struct { struct xenbus_watch *handle; char **vec; unsigned int vec_size; } watch; } un; }; static struct xs_handle { /* A list of replies. Currently only one will ever be outstanding. */ list_t reply_list; kmutex_t reply_lock; kcondvar_t reply_cv; /* One request at a time. */ kmutex_t request_mutex; /* Protect transactions against save/restore. */ krwlock_t suspend_lock; } xs_state; static int last_req_id; /* * List of clients wanting a xenstore up notification, and a lock to protect it */ static boolean_t xenstore_up; static list_t notify_list; static kmutex_t notify_list_lock; static taskq_t *xenbus_taskq; /* List of registered watches, and a lock to protect it. */ static list_t watches; static kmutex_t watches_lock; /* List of pending watch callback events, and a lock to protect it. */ static list_t watch_events; static kmutex_t watch_events_lock; /* * Details of the xenwatch callback kernel thread. The thread waits on the * watch_events_cv for work to do (queued on watch_events list). When it * wakes up it acquires the xenwatch_mutex before reading the list and * carrying out work. */ static kmutex_t xenwatch_mutex; static kcondvar_t watch_events_cv; static int process_msg(void); static int get_error(const char *errorstring) { unsigned int i; for (i = 0; !streq(errorstring, xsd_errors[i].errstring); i++) { if (i == (sizeof (xsd_errors) / sizeof (xsd_errors[0])) - 1) { cmn_err(CE_WARN, "XENBUS xen store gave: unknown error %s", errorstring); return (EINVAL); } } return (xsd_errors[i].errnum); } /* * Read a synchronous reply from xenstore. Since we can return early before * reading a relevant reply, we discard any messages not matching the request * ID. Caller must free returned message on success. */ static int read_reply(struct xsd_sockmsg *req_hdr, struct xs_stored_msg **reply) { extern int do_polled_io; mutex_enter(&xs_state.reply_lock); for (;;) { while (list_empty(&xs_state.reply_list)) { if (interrupts_unleashed && !do_polled_io) { if (cv_wait_sig(&xs_state.reply_cv, &xs_state.reply_lock) == 0) { mutex_exit(&xs_state.reply_lock); *reply = NULL; return (EINTR); } } else { /* polled mode needed for early probes */ mutex_exit(&xs_state.reply_lock); (void) HYPERVISOR_yield(); (void) process_msg(); mutex_enter(&xs_state.reply_lock); } } *reply = list_head(&xs_state.reply_list); list_remove(&xs_state.reply_list, *reply); if ((*reply)->hdr.req_id == req_hdr->req_id) break; } mutex_exit(&xs_state.reply_lock); return (0); } /* Emergency write. */ void xenbus_debug_write(const char *str, unsigned int count) { struct xsd_sockmsg msg = { 0 }; msg.type = XS_DEBUG; msg.len = sizeof ("print") + count + 1; mutex_enter(&xs_state.request_mutex); (void) xb_write(&msg, sizeof (msg)); (void) xb_write("print", sizeof ("print")); (void) xb_write(str, count); (void) xb_write("", 1); mutex_exit(&xs_state.request_mutex); } /* * This is pretty unpleasant. First off, there's the horrible logic around * suspend_lock and transactions. Also, we can be interrupted either before we * write a message, or before we receive a reply. A client that wants to * survive this can't know which case happened. Luckily all clients don't care * about signals currently, and the alternative (a hard wait on a userspace * daemon) isn't exactly preferable. Caller must free 'reply' on success. */ int xenbus_dev_request_and_reply(struct xsd_sockmsg *msg, void **reply) { struct xsd_sockmsg req_msg = *msg; struct xs_stored_msg *reply_msg = NULL; int err; if (req_msg.type == XS_TRANSACTION_START) rw_enter(&xs_state.suspend_lock, RW_READER); mutex_enter(&xs_state.request_mutex); msg->req_id = last_req_id++; err = xb_write(msg, sizeof (*msg) + msg->len); if (err) { if (req_msg.type == XS_TRANSACTION_START) rw_exit(&xs_state.suspend_lock); msg->type = XS_ERROR; *reply = NULL; goto out; } err = read_reply(msg, &reply_msg); if (err) { if (msg->type == XS_TRANSACTION_START) rw_exit(&xs_state.suspend_lock); *reply = NULL; goto out; } *reply = reply_msg->un.reply.body; *msg = reply_msg->hdr; if (reply_msg->hdr.type == XS_TRANSACTION_END) rw_exit(&xs_state.suspend_lock); out: if (reply_msg != NULL) kmem_free(reply_msg, sizeof (*reply_msg)); mutex_exit(&xs_state.request_mutex); return (err); } /* * Send message to xs, return errcode, rval filled in with pointer * to kmem_alloc'ed reply. */ static int xs_talkv(xenbus_transaction_t t, enum xsd_sockmsg_type type, const iovec_t *iovec, unsigned int num_vecs, void **rval, unsigned int *len) { struct xsd_sockmsg msg; struct xs_stored_msg *reply_msg; char *reply; unsigned int i; int err; msg.tx_id = (uint32_t)(unsigned long)t; msg.type = type; msg.len = 0; for (i = 0; i < num_vecs; i++) msg.len += iovec[i].iov_len; mutex_enter(&xs_state.request_mutex); msg.req_id = last_req_id++; err = xb_write(&msg, sizeof (msg)); if (err) { mutex_exit(&xs_state.request_mutex); return (err); } for (i = 0; i < num_vecs; i++) { err = xb_write(iovec[i].iov_base, iovec[i].iov_len); if (err) { mutex_exit(&xs_state.request_mutex); return (err); } } err = read_reply(&msg, &reply_msg); mutex_exit(&xs_state.request_mutex); if (err) return (err); reply = reply_msg->un.reply.body; if (reply_msg->hdr.type == XS_ERROR) { err = get_error(reply); kmem_free(reply, reply_msg->hdr.len + 1); goto out; } if (len != NULL) *len = reply_msg->hdr.len + 1; ASSERT(reply_msg->hdr.type == type); if (rval != NULL) *rval = reply; else kmem_free(reply, reply_msg->hdr.len + 1); out: kmem_free(reply_msg, sizeof (*reply_msg)); return (err); } /* Simplified version of xs_talkv: single message. */ static int xs_single(xenbus_transaction_t t, enum xsd_sockmsg_type type, const char *string, void **ret, unsigned int *len) { iovec_t iovec; iovec.iov_base = (char *)string; iovec.iov_len = strlen(string) + 1; return (xs_talkv(t, type, &iovec, 1, ret, len)); } static unsigned int count_strings(const char *strings, unsigned int len) { unsigned int num; const char *p; for (p = strings, num = 0; p < strings + len; p += strlen(p) + 1) num++; return (num); } /* Return the path to dir with /name appended. Buffer must be kmem_free()'ed */ static char * join(const char *dir, const char *name) { char *buffer; size_t slashlen; slashlen = streq(name, "") ? 0 : 1; buffer = kmem_alloc(strlen(dir) + slashlen + strlen(name) + 1, KM_SLEEP); (void) strcpy(buffer, dir); if (slashlen != 0) { (void) strcat(buffer, "/"); (void) strcat(buffer, name); } return (buffer); } static char ** split(char *strings, unsigned int len, unsigned int *num) { char *p, **ret; /* Count the strings. */ if ((*num = count_strings(strings, len - 1)) == 0) return (NULL); /* Transfer to one big alloc for easy freeing. */ ret = kmem_alloc(*num * sizeof (char *) + (len - 1), KM_SLEEP); (void) memcpy(&ret[*num], strings, len - 1); kmem_free(strings, len); strings = (char *)&ret[*num]; for (p = strings, *num = 0; p < strings + (len - 1); p += strlen(p) + 1) { ret[(*num)++] = p; } return (ret); } char ** xenbus_directory(xenbus_transaction_t t, const char *dir, const char *node, unsigned int *num) { char *strings, *path; unsigned int len; int err; path = join(dir, node); err = xs_single(t, XS_DIRECTORY, path, (void **)&strings, &len); kmem_free(path, strlen(path) + 1); if (err != 0 || strings == NULL) { /* sigh, we lose error code info here */ *num = 0; return (NULL); } return (split(strings, len, num)); } /* Check if a path exists. */ boolean_t xenbus_exists(const char *dir, const char *node) { void *p; uint_t n; if (xenbus_read(XBT_NULL, dir, node, &p, &n) != 0) return (B_FALSE); kmem_free(p, n); return (B_TRUE); } /* Check if a directory path exists. */ boolean_t xenbus_exists_dir(const char *dir, const char *node) { char **d; unsigned int dir_n; int i, len; d = xenbus_directory(XBT_NULL, dir, node, &dir_n); if (d == NULL) return (B_FALSE); for (i = 0, len = 0; i < dir_n; i++) len += strlen(d[i]) + 1 + sizeof (char *); kmem_free(d, len); return (B_TRUE); } /* * Get the value of a single file. * Returns a kmem_alloced value in retp: call kmem_free() on it after use. * len indicates length in bytes. */ int xenbus_read(xenbus_transaction_t t, const char *dir, const char *node, void **retp, unsigned int *len) { char *path; int err; path = join(dir, node); err = xs_single(t, XS_READ, path, retp, len); kmem_free(path, strlen(path) + 1); return (err); } int xenbus_read_str(const char *dir, const char *node, char **retp) { uint_t n; int err; char *str; /* * Since we access the xenbus value immediatly we can't be * part of a transaction. */ if ((err = xenbus_read(XBT_NULL, dir, node, (void **)&str, &n)) != 0) return (err); ASSERT((str != NULL) && (n > 0)); /* * Why bother with this? Because xenbus is truly annoying in the * fact that when it returns a string, it doesn't guarantee that * the memory that holds the string is of size strlen() + 1. * This forces callers to keep track of the size of the memory * containing the string. Ugh. We'll work around this by * re-allocate strings to always be of size strlen() + 1. */ *retp = strdup(str); kmem_free(str, n); return (0); } /* * Write the value of a single file. * Returns err on failure. */ int xenbus_write(xenbus_transaction_t t, const char *dir, const char *node, const char *string) { char *path; iovec_t iovec[2]; int ret; path = join(dir, node); iovec[0].iov_base = (void *)path; iovec[0].iov_len = strlen(path) + 1; iovec[1].iov_base = (void *)string; iovec[1].iov_len = strlen(string); ret = xs_talkv(t, XS_WRITE, iovec, 2, NULL, NULL); kmem_free(path, iovec[0].iov_len); return (ret); } /* Create a new directory. */ int xenbus_mkdir(xenbus_transaction_t t, const char *dir, const char *node) { char *path; int ret; path = join(dir, node); ret = xs_single(t, XS_MKDIR, path, NULL, NULL); kmem_free(path, strlen(path) + 1); return (ret); } /* Destroy a file or directory (directories must be empty). */ int xenbus_rm(xenbus_transaction_t t, const char *dir, const char *node) { char *path; int ret; path = join(dir, node); ret = xs_single(t, XS_RM, path, NULL, NULL); kmem_free(path, strlen(path) + 1); return (ret); } /* * Start a transaction: changes by others will not be seen during this * transaction, and changes will not be visible to others until end. */ int xenbus_transaction_start(xenbus_transaction_t *t) { void *id_str; unsigned long id; int err; unsigned int len; rw_enter(&xs_state.suspend_lock, RW_READER); err = xs_single(XBT_NULL, XS_TRANSACTION_START, "", &id_str, &len); if (err) { rw_exit(&xs_state.suspend_lock); return (err); } (void) ddi_strtoul((char *)id_str, NULL, 0, &id); *t = (xenbus_transaction_t)id; kmem_free(id_str, len); return (0); } /* * End a transaction. * If abandon is true, transaction is discarded instead of committed. */ int xenbus_transaction_end(xenbus_transaction_t t, int abort) { char abortstr[2]; int err; if (abort) (void) strcpy(abortstr, "F"); else (void) strcpy(abortstr, "T"); err = xs_single(t, XS_TRANSACTION_END, abortstr, NULL, NULL); rw_exit(&xs_state.suspend_lock); return (err); } /* * Single read and scanf: returns errno or 0. This can only handle a single * conversion specifier. */ /* SCANFLIKE4 */ int xenbus_scanf(xenbus_transaction_t t, const char *dir, const char *node, const char *fmt, ...) { va_list ap; int ret; char *val; unsigned int len; ret = xenbus_read(t, dir, node, (void **)&val, &len); if (ret) return (ret); va_start(ap, fmt); if (vsscanf(val, fmt, ap) != 1) ret = ERANGE; va_end(ap); kmem_free(val, len); return (ret); } /* Single printf and write: returns errno or 0. */ /* PRINTFLIKE4 */ int xenbus_printf(xenbus_transaction_t t, const char *dir, const char *node, const char *fmt, ...) { va_list ap; int ret; #define PRINTF_BUFFER_SIZE 4096 char *printf_buffer; printf_buffer = kmem_alloc(PRINTF_BUFFER_SIZE, KM_SLEEP); va_start(ap, fmt); ret = vsnprintf(printf_buffer, PRINTF_BUFFER_SIZE, fmt, ap); va_end(ap); ASSERT(ret <= PRINTF_BUFFER_SIZE-1); ret = xenbus_write(t, dir, node, printf_buffer); kmem_free(printf_buffer, PRINTF_BUFFER_SIZE); return (ret); } /* Takes tuples of names, scanf-style args, and void **, NULL terminated. */ int xenbus_gather(xenbus_transaction_t t, const char *dir, ...) { va_list ap; const char *name; int ret = 0; unsigned int len; va_start(ap, dir); while (ret == 0 && (name = va_arg(ap, char *)) != NULL) { const char *fmt = va_arg(ap, char *); void *result = va_arg(ap, void *); char *p; ret = xenbus_read(t, dir, name, (void **)&p, &len); if (ret) break; if (fmt) { ASSERT(result != NULL); if (sscanf(p, fmt, result) != 1) ret = EINVAL; kmem_free(p, len); } else *(char **)result = p; } va_end(ap); return (ret); } static int xs_watch(const char *path, const char *token) { iovec_t iov[2]; iov[0].iov_base = (void *)path; iov[0].iov_len = strlen(path) + 1; iov[1].iov_base = (void *)token; iov[1].iov_len = strlen(token) + 1; return (xs_talkv(XBT_NULL, XS_WATCH, iov, 2, NULL, NULL)); } static int xs_unwatch(const char *path, const char *token) { iovec_t iov[2]; iov[0].iov_base = (char *)path; iov[0].iov_len = strlen(path) + 1; iov[1].iov_base = (char *)token; iov[1].iov_len = strlen(token) + 1; return (xs_talkv(XBT_NULL, XS_UNWATCH, iov, 2, NULL, NULL)); } static struct xenbus_watch * find_watch(const char *token) { struct xenbus_watch *i, *cmp; (void) ddi_strtoul(token, NULL, 16, (unsigned long *)&cmp); for (i = list_head(&watches); i != NULL; i = list_next(&watches, i)) if (i == cmp) break; return (i); } /* Register a xenstore state notify callback */ int xs_register_xenbus_callback(void (*callback)(int)) { struct xenbus_notify *xbn, *xnp; xbn = kmem_alloc(sizeof (struct xenbus_notify), KM_SLEEP); xbn->notify_func = callback; mutex_enter(¬ify_list_lock); /* * Make sure not already on the list */ xnp = list_head(¬ify_list); for (; xnp != NULL; xnp = list_next(¬ify_list, xnp)) { if (xnp->notify_func == callback) { kmem_free(xbn, sizeof (struct xenbus_notify)); mutex_exit(¬ify_list_lock); return (EEXIST); } } xnp = xbn; list_insert_tail(¬ify_list, xbn); if (xenstore_up) xnp->notify_func(XENSTORE_UP); mutex_exit(¬ify_list_lock); return (0); } /* * Notify clients of xenstore state */ static void do_notify_callbacks(void *arg) { struct xenbus_notify *xnp; mutex_enter(¬ify_list_lock); xnp = list_head(¬ify_list); for (; xnp != NULL; xnp = list_next(¬ify_list, xnp)) { xnp->notify_func((int)((uintptr_t)arg)); } mutex_exit(¬ify_list_lock); } void xs_notify_xenstore_up(void) { xenstore_up = B_TRUE; (void) taskq_dispatch(xenbus_taskq, do_notify_callbacks, (void *)XENSTORE_UP, 0); } void xs_notify_xenstore_down(void) { xenstore_up = B_FALSE; (void) taskq_dispatch(xenbus_taskq, do_notify_callbacks, (void *)XENSTORE_DOWN, 0); } /* Register callback to watch this node. */ int register_xenbus_watch(struct xenbus_watch *watch) { /* Pointer in ascii is the token. */ char token[sizeof (watch) * 2 + 1]; int err; ASSERT(xenstore_up); (void) snprintf(token, sizeof (token), "%lX", (long)watch); rw_enter(&xs_state.suspend_lock, RW_READER); mutex_enter(&watches_lock); /* * May be re-registering a watch if xenstore daemon was restarted */ if (find_watch(token) == NULL) list_insert_tail(&watches, watch); mutex_exit(&watches_lock); DTRACE_XPV3(xenbus__register__watch, const char *, watch->node, uintptr_t, watch->callback, struct xenbus_watch *, watch); err = xs_watch(watch->node, token); /* Ignore errors due to multiple registration. */ if ((err != 0) && (err != EEXIST)) { mutex_enter(&watches_lock); list_remove(&watches, watch); mutex_exit(&watches_lock); } rw_exit(&xs_state.suspend_lock); return (err); } static void free_stored_msg(struct xs_stored_msg *msg) { int i, len = 0; for (i = 0; i < msg->un.watch.vec_size; i++) len += strlen(msg->un.watch.vec[i]) + 1 + sizeof (char *); kmem_free(msg->un.watch.vec, len); kmem_free(msg, sizeof (*msg)); } void unregister_xenbus_watch(struct xenbus_watch *watch) { struct xs_stored_msg *msg; char token[sizeof (watch) * 2 + 1]; int err; (void) snprintf(token, sizeof (token), "%lX", (long)watch); rw_enter(&xs_state.suspend_lock, RW_READER); mutex_enter(&watches_lock); ASSERT(find_watch(token)); list_remove(&watches, watch); mutex_exit(&watches_lock); DTRACE_XPV3(xenbus__unregister__watch, const char *, watch->node, uintptr_t, watch->callback, struct xenbus_watch *, watch); err = xs_unwatch(watch->node, token); if (err) cmn_err(CE_WARN, "XENBUS Failed to release watch %s: %d", watch->node, err); rw_exit(&xs_state.suspend_lock); /* Cancel pending watch events. */ mutex_enter(&watch_events_lock); msg = list_head(&watch_events); while (msg != NULL) { struct xs_stored_msg *tmp = list_next(&watch_events, msg); if (msg->un.watch.handle == watch) { list_remove(&watch_events, msg); free_stored_msg(msg); } msg = tmp; } mutex_exit(&watch_events_lock); /* Flush any currently-executing callback, unless we are it. :-) */ if (mutex_owner(&xenwatch_mutex) != curthread) { mutex_enter(&xenwatch_mutex); mutex_exit(&xenwatch_mutex); } } void xenbus_suspend(void) { rw_enter(&xs_state.suspend_lock, RW_WRITER); mutex_enter(&xs_state.request_mutex); xb_suspend(); } void xenbus_resume(void) { struct xenbus_watch *watch; char token[sizeof (watch) * 2 + 1]; mutex_exit(&xs_state.request_mutex); xb_init(); xb_setup_intr(); /* No need for watches_lock: the suspend_lock is sufficient. */ for (watch = list_head(&watches); watch != NULL; watch = list_next(&watches, watch)) { (void) snprintf(token, sizeof (token), "%lX", (long)watch); (void) xs_watch(watch->node, token); } rw_exit(&xs_state.suspend_lock); } static void xenwatch_thread(void) { struct xs_stored_msg *msg; struct xenbus_watch *watch; for (;;) { mutex_enter(&watch_events_lock); while (list_empty(&watch_events)) cv_wait(&watch_events_cv, &watch_events_lock); msg = list_head(&watch_events); ASSERT(msg != NULL); list_remove(&watch_events, msg); watch = msg->un.watch.handle; mutex_exit(&watch_events_lock); mutex_enter(&xenwatch_mutex); DTRACE_XPV4(xenbus__fire__watch, const char *, watch->node, uintptr_t, watch->callback, struct xenbus_watch *, watch, const char *, msg->un.watch.vec[XS_WATCH_PATH]); watch->callback(watch, (const char **)msg->un.watch.vec, msg->un.watch.vec_size); free_stored_msg(msg); mutex_exit(&xenwatch_mutex); } } static int process_msg(void) { struct xs_stored_msg *msg; char *body; int err, mlen; msg = kmem_alloc(sizeof (*msg), KM_SLEEP); err = xb_read(&msg->hdr, sizeof (msg->hdr)); if (err) { kmem_free(msg, sizeof (*msg)); return (err); } mlen = msg->hdr.len + 1; body = kmem_alloc(mlen, KM_SLEEP); err = xb_read(body, msg->hdr.len); if (err) { kmem_free(body, mlen); kmem_free(msg, sizeof (*msg)); return (err); } body[mlen - 1] = '\0'; if (msg->hdr.type == XS_WATCH_EVENT) { const char *token; msg->un.watch.vec = split(body, msg->hdr.len + 1, &msg->un.watch.vec_size); if (msg->un.watch.vec == NULL) { kmem_free(msg, sizeof (*msg)); return (EIO); } mutex_enter(&watches_lock); token = msg->un.watch.vec[XS_WATCH_TOKEN]; if ((msg->un.watch.handle = find_watch(token)) != NULL) { mutex_enter(&watch_events_lock); DTRACE_XPV4(xenbus__enqueue__watch, const char *, msg->un.watch.handle->node, uintptr_t, msg->un.watch.handle->callback, struct xenbus_watch *, msg->un.watch.handle, const char *, msg->un.watch.vec[XS_WATCH_PATH]); list_insert_tail(&watch_events, msg); cv_broadcast(&watch_events_cv); mutex_exit(&watch_events_lock); } else { free_stored_msg(msg); } mutex_exit(&watches_lock); } else { msg->un.reply.body = body; mutex_enter(&xs_state.reply_lock); list_insert_tail(&xs_state.reply_list, msg); mutex_exit(&xs_state.reply_lock); cv_signal(&xs_state.reply_cv); } return (0); } static void xenbus_thread(void) { int err; /* * We have to wait for interrupts to be ready, so we don't clash * with the polled-IO code in read_reply(). */ while (!interrupts_unleashed) delay(10); for (;;) { err = process_msg(); if (err) cmn_err(CE_WARN, "XENBUS error %d while reading " "message", err); } } /* * When setting up xenbus, dom0 and domU have to take different paths, which * makes this code a little confusing. For dom0: * * xs_early_init - mutex init only * xs_dom0_init - called on xenbus dev attach: set up our xenstore page and * event channel; start xenbus threads for responding to interrupts. * * And for domU: * * xs_early_init - mutex init; set up our xenstore page and event channel * xs_domu_init - installation of IRQ handler; start xenbus threads. * * We need an early init on domU so we can use xenbus in polled mode to * discover devices, VCPUs etc. * * On resume, we use xb_init() and xb_setup_intr() to restore xenbus to a * working state. */ void xs_early_init(void) { list_create(&xs_state.reply_list, sizeof (struct xs_stored_msg), offsetof(struct xs_stored_msg, list)); list_create(&watch_events, sizeof (struct xs_stored_msg), offsetof(struct xs_stored_msg, list)); list_create(&watches, sizeof (struct xenbus_watch), offsetof(struct xenbus_watch, list)); list_create(¬ify_list, sizeof (struct xenbus_notify), offsetof(struct xenbus_notify, list)); mutex_init(&xs_state.reply_lock, NULL, MUTEX_DEFAULT, NULL); mutex_init(&xs_state.request_mutex, NULL, MUTEX_DEFAULT, NULL); mutex_init(¬ify_list_lock, NULL, MUTEX_DEFAULT, NULL); rw_init(&xs_state.suspend_lock, NULL, RW_DEFAULT, NULL); cv_init(&xs_state.reply_cv, NULL, CV_DEFAULT, NULL); if (DOMAIN_IS_INITDOMAIN(xen_info)) return; xb_init(); xenstore_up = B_TRUE; } static void xs_thread_init(void) { (void) thread_create(NULL, 0, xenwatch_thread, NULL, 0, &p0, TS_RUN, minclsyspri); (void) thread_create(NULL, 0, xenbus_thread, NULL, 0, &p0, TS_RUN, minclsyspri); xenbus_taskq = taskq_create("xenbus_taskq", 1, maxclsyspri - 1, 1, 1, TASKQ_PREPOPULATE); ASSERT(xenbus_taskq != NULL); } void xs_domu_init(void) { if (DOMAIN_IS_INITDOMAIN(xen_info)) return; /* * Add interrupt handler for xenbus now, must wait till after * psm module is loaded. All use of xenbus is in polled mode * until xs_init is called since it is what kicks off the xs * server threads. */ xs_thread_init(); xb_setup_intr(); } void xs_dom0_init(void) { static boolean_t initialized = B_FALSE; ASSERT(DOMAIN_IS_INITDOMAIN(xen_info)); /* * The xenbus driver might be re-attaching. */ if (initialized) return; xb_init(); xs_thread_init(); xb_setup_intr(); initialized = B_TRUE; } /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* Copyright (c) 1990, 1991 UNIX System Laboratories, Inc. */ /* Copyright (c) 1984, 1986, 1987, 1988, 1989, 1990 AT&T */ /* All Rights Reserved */ /* * Copyright 2008 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ /* * * Copyright (c) 2004 Christian Limpach. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. This section intentionally left blank. * 4. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ /* * Section 3 of the above license was updated in response to bug 6379571. */ /* * Hypervisor virtual console driver */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef DEBUG #include #endif #include #include #include #include #include #include #include #include #include "xencons.h" #include #include #include #include #ifdef DEBUG #define XENCONS_DEBUG_INIT 0x0001 /* msgs during driver initialization. */ #define XENCONS_DEBUG_INPUT 0x0002 /* characters received during int. */ #define XENCONS_DEBUG_EOT 0x0004 /* msgs when wait for xmit to finish. */ #define XENCONS_DEBUG_CLOSE 0x0008 /* msgs when driver open/close called */ #define XENCONS_DEBUG_PROCS 0x0020 /* each proc name as it is entered. */ #define XENCONS_DEBUG_OUT 0x0100 /* msgs about output events. */ #define XENCONS_DEBUG_BUSY 0x0200 /* msgs when xmit is enabled/disabled */ #define XENCONS_DEBUG_MODEM 0x0400 /* msgs about modem status & control. */ #define XENCONS_DEBUG_MODM2 0x0800 /* msgs about modem status & control. */ #define XENCONS_DEBUG_IOCTL 0x1000 /* Output msgs about ioctl messages. */ #define XENCONS_DEBUG_CHIP 0x2000 /* msgs about chip identification. */ #define XENCONS_DEBUG_SFLOW 0x4000 /* msgs when S/W flowcontrol active */ #define XENCONS_DEBUG(x) (debug & (x)) static int debug = 0; #else #define XENCONS_DEBUG(x) B_FALSE #endif #define XENCONS_WBUFSIZE 4096 static boolean_t abort_charseq_recognize(uchar_t); /* The async interrupt entry points */ static void xcasync_ioctl(struct asyncline *, queue_t *, mblk_t *); static void xcasync_reioctl(void *); static void xcasync_start(struct asyncline *); static void xenconsputchar(cons_polledio_arg_t, uchar_t); static int xenconsgetchar(cons_polledio_arg_t); static boolean_t xenconsischar(cons_polledio_arg_t); static uint_t xenconsintr(caddr_t); static uint_t xenconsintr_priv(caddr_t, caddr_t); /*PRINTFLIKE2*/ static void xenconserror(int, const char *, ...) __KPRINTFLIKE(2); static void xencons_soft_state_free(struct xencons *); static boolean_t xcasync_flowcontrol_sw_input(struct xencons *, async_flowc_action, int); static void xcasync_flowcontrol_sw_output(struct xencons *, async_flowc_action); void *xencons_soft_state; char *xencons_wbuf; struct xencons *xencons_console; static void xenconssetup_avintr(struct xencons *xcp, int attach) { /* * On xen, CPU 0 always exists and can't be taken offline, * so binding this thread to it should always succeed. */ mutex_enter(&cpu_lock); thread_affinity_set(curthread, 0); mutex_exit(&cpu_lock); if (attach) { /* Setup our interrupt binding. */ (void) add_avintr(NULL, IPL_CONS, xenconsintr_priv, "xencons", xcp->console_irq, (caddr_t)xcp, NULL, NULL, xcp->dip); } else { /* * Cleanup interrupt configuration. Note that the framework * _should_ ensure that when rem_avintr() returns the interrupt * service routine is not currently executing and that it won't * be invoked again. */ (void) rem_avintr(NULL, IPL_CONS, xenconsintr_priv, xcp->console_irq); } /* Notify our caller that we're done. */ mutex_enter(&xcp->excl); cv_signal(&xcp->excl_cv); mutex_exit(&xcp->excl); /* Clear our binding to CPU 0 */ thread_affinity_clear(curthread); } static void xenconssetup_add_avintr(struct xencons *xcp) { xenconssetup_avintr(xcp, B_TRUE); } static void xenconssetup_rem_avintr(struct xencons *xcp) { xenconssetup_avintr(xcp, B_FALSE); } static int xenconsdetach(dev_info_t *devi, ddi_detach_cmd_t cmd) { int instance; struct xencons *xcp; if (cmd != DDI_DETACH && cmd != DDI_SUSPEND) return (DDI_FAILURE); if (cmd == DDI_SUSPEND) { ddi_remove_intr(devi, 0, NULL); return (DDI_SUCCESS); } /* * We should never try to detach the console driver on a domU * because it should always be held open */ ASSERT(DOMAIN_IS_INITDOMAIN(xen_info)); if (!DOMAIN_IS_INITDOMAIN(xen_info)) return (DDI_FAILURE); instance = ddi_get_instance(devi); /* find out which unit */ xcp = ddi_get_soft_state(xencons_soft_state, instance); if (xcp == NULL) return (DDI_FAILURE); /* * Cleanup our interrupt bindings. For more info on why we * do this in a seperate thread, see the comments for when we * setup the interrupt bindings. */ xencons_console = NULL; mutex_enter(&xcp->excl); (void) taskq_dispatch(system_taskq, (void (*)(void *))xenconssetup_rem_avintr, xcp, TQ_SLEEP); cv_wait(&xcp->excl_cv, &xcp->excl); mutex_exit(&xcp->excl); /* remove all minor device node(s) for this device */ ddi_remove_minor_node(devi, NULL); /* free up state */ xencons_soft_state_free(xcp); kmem_free(xencons_wbuf, XENCONS_WBUFSIZE); DEBUGNOTE1(XENCONS_DEBUG_INIT, "xencons%d: shutdown complete", instance); return (DDI_SUCCESS); } static void xenconssetup(struct xencons *xcp) { xcp->ifp = (volatile struct xencons_interface *)HYPERVISOR_console_page; if (DOMAIN_IS_INITDOMAIN(xen_info)) { xencons_wbuf = kmem_alloc(XENCONS_WBUFSIZE, KM_SLEEP); /* * Activate the xen console virq. Note that xen requires * that VIRQs be bound to CPU 0 when first created. */ xcp->console_irq = ec_bind_virq_to_irq(VIRQ_CONSOLE, 0); /* * Ok. This is kinda ugly. We want to register an * interrupt handler for the xen console virq, but * virq's are xen sepcific and currently the DDI doesn't * support binding to them. So instead we need to use * add_avintr(). So to make things more complicated, * we already had to bind the xen console VIRQ to CPU 0, * and add_avintr() needs to be invoked on the same CPU * where the VIRQ is bound, in this case on CPU 0. We * could just temporarily bind ourselves to CPU 0, but * we don't want to do that since this attach thread * could have been invoked in a user thread context, * in which case this thread could already have some * pre-existing cpu binding. So to avoid changing our * cpu binding we're going to use a taskq thread that * will bind to CPU 0 and register our interrupts * handler for us. */ mutex_enter(&xcp->excl); (void) taskq_dispatch(system_taskq, (void (*)(void *))xenconssetup_add_avintr, xcp, TQ_SLEEP); cv_wait(&xcp->excl_cv, &xcp->excl); mutex_exit(&xcp->excl); } else { (void) xvdi_alloc_evtchn(xcp->dip); xcp->evtchn = xvdi_get_evtchn(xcp->dip); (void) ddi_add_intr(xcp->dip, 0, NULL, NULL, xenconsintr, (caddr_t)xcp); } } static int xenconsattach(dev_info_t *devi, ddi_attach_cmd_t cmd) { int instance = ddi_get_instance(devi); struct xencons *xcp; int ret; /* There can be only one. */ if (instance != 0) return (DDI_FAILURE); switch (cmd) { case DDI_RESUME: xcp = xencons_console; xenconssetup(xcp); return (DDI_SUCCESS); case DDI_ATTACH: break; default: return (DDI_FAILURE); } ret = ddi_soft_state_zalloc(xencons_soft_state, instance); if (ret != DDI_SUCCESS) return (DDI_FAILURE); xcp = ddi_get_soft_state(xencons_soft_state, instance); ASSERT(xcp != NULL); /* can't fail - we only just allocated it */ /* * Set up the other components of the xencons structure for this port. */ xcp->unit = instance; xcp->dip = devi; /* Fill in the polled I/O structure. */ xcp->polledio.cons_polledio_version = CONSPOLLEDIO_V0; xcp->polledio.cons_polledio_argument = (cons_polledio_arg_t)xcp; xcp->polledio.cons_polledio_putchar = xenconsputchar; xcp->polledio.cons_polledio_getchar = xenconsgetchar; xcp->polledio.cons_polledio_ischar = xenconsischar; xcp->polledio.cons_polledio_enter = NULL; xcp->polledio.cons_polledio_exit = NULL; /* * Initializes the asyncline structure which has TTY protocol-private * data before enabling interrupts. */ xcp->priv = kmem_zalloc(sizeof (struct asyncline), KM_SLEEP); xcp->priv->async_common = xcp; cv_init(&xcp->priv->async_flags_cv, NULL, CV_DRIVER, NULL); /* Initialize mutexes before accessing the interface. */ mutex_init(&xcp->excl, NULL, MUTEX_DRIVER, NULL); cv_init(&xcp->excl_cv, NULL, CV_DEFAULT, NULL); /* create minor device node for this device */ ret = ddi_create_minor_node(devi, "xencons", S_IFCHR, instance, DDI_NT_SERIAL, 0); if (ret != DDI_SUCCESS) { ddi_remove_minor_node(devi, NULL); xencons_soft_state_free(xcp); return (DDI_FAILURE); } ddi_report_dev(devi); xencons_console = xcp; xenconssetup(xcp); DEBUGCONT1(XENCONS_DEBUG_INIT, "xencons%dattach: done\n", instance); return (DDI_SUCCESS); } /*ARGSUSED*/ static int xenconsinfo(dev_info_t *dip, ddi_info_cmd_t infocmd, void *arg, void **result) { dev_t dev = (dev_t)arg; int instance, error; struct xencons *xcp; instance = getminor(dev); xcp = ddi_get_soft_state(xencons_soft_state, instance); if (xcp == NULL) return (DDI_FAILURE); switch (infocmd) { case DDI_INFO_DEVT2DEVINFO: if (xcp->dip == NULL) error = DDI_FAILURE; else { *result = (void *) xcp->dip; error = DDI_SUCCESS; } break; case DDI_INFO_DEVT2INSTANCE: *result = (void *)(intptr_t)instance; error = DDI_SUCCESS; break; default: error = DDI_FAILURE; } return (error); } /* xencons_soft_state_free - local wrapper for ddi_soft_state_free(9F) */ static void xencons_soft_state_free(struct xencons *xcp) { mutex_destroy(&xcp->excl); cv_destroy(&xcp->excl_cv); kmem_free(xcp->priv, sizeof (struct asyncline)); ddi_soft_state_free(xencons_soft_state, xcp->unit); } /*ARGSUSED*/ static int xenconsopen(queue_t *rq, dev_t *dev, int flag, int sflag, cred_t *cr) { struct xencons *xcp; struct asyncline *async; int unit; unit = getminor(*dev); DEBUGCONT1(XENCONS_DEBUG_CLOSE, "xencons%dopen\n", unit); xcp = ddi_get_soft_state(xencons_soft_state, unit); if (xcp == NULL) return (ENXIO); /* unit not configured */ async = xcp->priv; mutex_enter(&xcp->excl); if ((async->async_flags & ASYNC_ISOPEN) == 0) { async->async_ttycommon.t_iflag = 0; async->async_ttycommon.t_iocpending = NULL; async->async_ttycommon.t_size.ws_row = 0; async->async_ttycommon.t_size.ws_col = 0; async->async_ttycommon.t_size.ws_xpixel = 0; async->async_ttycommon.t_size.ws_ypixel = 0; async->async_dev = *dev; async->async_wbufcid = 0; async->async_startc = CSTART; async->async_stopc = CSTOP; } else if ((async->async_ttycommon.t_flags & TS_XCLUDE) && secpolicy_excl_open(cr) != 0) { mutex_exit(&xcp->excl); return (EBUSY); } async->async_ttycommon.t_flags |= TS_SOFTCAR; async->async_ttycommon.t_readq = rq; async->async_ttycommon.t_writeq = WR(rq); rq->q_ptr = WR(rq)->q_ptr = (caddr_t)async; mutex_exit(&xcp->excl); /* * Caution here -- qprocson sets the pointers that are used by canput * called by xencons_rxint. ASYNC_ISOPEN must *not* be set until those * pointers are valid. */ qprocson(rq); async->async_flags |= ASYNC_ISOPEN; DEBUGCONT1(XENCONS_DEBUG_INIT, "asy%dopen: done\n", unit); return (0); } /* * Close routine. */ /*ARGSUSED*/ static int xenconsclose(queue_t *q, int flag, cred_t *credp) { struct asyncline *async; struct xencons *xcp; #ifdef DEBUG int instance; #endif async = (struct asyncline *)q->q_ptr; ASSERT(async != NULL); xcp = async->async_common; #ifdef DEBUG instance = xcp->unit; DEBUGCONT1(XENCONS_DEBUG_CLOSE, "xencons%dclose\n", instance); #endif mutex_enter(&xcp->excl); async->async_flags |= ASYNC_CLOSING; async->async_ocnt = 0; if (async->async_xmitblk != NULL) freeb(async->async_xmitblk); async->async_xmitblk = NULL; ttycommon_close(&async->async_ttycommon); /* * Cancel outstanding "bufcall" request. */ if (async->async_wbufcid != 0) { unbufcall(async->async_wbufcid); async->async_wbufcid = 0; } /* Note that qprocsoff can't be done until after interrupts are off */ qprocsoff(q); q->q_ptr = WR(q)->q_ptr = NULL; async->async_ttycommon.t_readq = NULL; async->async_ttycommon.t_writeq = NULL; /* * Clear out device state, except persistant device property flags. */ async->async_flags = 0; cv_broadcast(&async->async_flags_cv); mutex_exit(&xcp->excl); DEBUGCONT1(XENCONS_DEBUG_CLOSE, "xencons%dclose: done\n", instance); return (0); } #define INBUF_IX(ix, ifp) (DOMAIN_IS_INITDOMAIN(xen_info) ? \ (ix) : MASK_XENCONS_IDX((ix), (ifp)->in)) /* * Handle a xen console rx interrupt. */ /*ARGSUSED*/ static void xencons_rxint(struct xencons *xcp) { struct asyncline *async; short cc; mblk_t *bp; queue_t *q; uchar_t c, buf[16]; uchar_t *cp; tty_common_t *tp; int instance; volatile struct xencons_interface *ifp; XENCONS_RING_IDX cons, prod; DEBUGCONT0(XENCONS_DEBUG_PROCS, "xencons_rxint\n"); loop: mutex_enter(&xcp->excl); instance = xcp->unit; /* sanity check if we should bail */ if (xencons_console == NULL) { mutex_exit(&xcp->excl); DEBUGCONT1(XENCONS_DEBUG_PROCS, "xencons%d_rxint: xencons_console is NULL\n", instance); goto out; } async = xcp->priv; ifp = xcp->ifp; tp = &async->async_ttycommon; q = tp->t_readq; if (async->async_flags & ASYNC_OUT_FLW_RESUME) { xcasync_start(async); async->async_flags &= ~ASYNC_OUT_FLW_RESUME; } /* * If data is available, send it up the stream if there's * somebody listening. */ if (!(async->async_flags & ASYNC_ISOPEN)) { mutex_exit(&xcp->excl); goto out; } if (DOMAIN_IS_INITDOMAIN(xen_info)) { cc = HYPERVISOR_console_io(CONSOLEIO_read, 16, (char *)buf); cp = buf; cons = 0; } else { cons = ifp->in_cons; prod = ifp->in_prod; cc = prod - cons; cp = (uchar_t *)ifp->in; } if (cc <= 0) { mutex_exit(&xcp->excl); goto out; } /* * Check for character break sequence. * * Note that normally asy drivers only check for a character sequence * if abort_enable == KIOCABORTALTERNATE and otherwise use a break * sensed on the line to do an abort_sequence_enter. Since the * hypervisor does not use a real chip for the console we default to * using the alternate sequence. */ if ((abort_enable == KIOCABORTENABLE) && (xcp->flags & ASY_CONSOLE)) { XENCONS_RING_IDX i; for (i = 0; i < cc; i++) { c = cp[INBUF_IX(cons + i, ifp)]; if (abort_charseq_recognize(c)) { /* * Eat abort seg, it's not a valid debugger * command. */ if (!DOMAIN_IS_INITDOMAIN(xen_info)) { membar_producer(); ifp->in_cons = cons + i; } else { cons += i; } abort_sequence_enter((char *)NULL); /* * Back from debugger, resume normal processing */ mutex_exit(&xcp->excl); goto loop; } } } if (!canput(q)) { if (!(async->async_inflow_source & IN_FLOW_STREAMS)) { (void) xcasync_flowcontrol_sw_input(xcp, FLOW_STOP, IN_FLOW_STREAMS); } mutex_exit(&xcp->excl); goto out; } if (async->async_inflow_source & IN_FLOW_STREAMS) { (void) xcasync_flowcontrol_sw_input(xcp, FLOW_START, IN_FLOW_STREAMS); } DEBUGCONT2(XENCONS_DEBUG_INPUT, "xencons%d_rxint: %d char(s) in queue.\n", instance, cc); if (!(bp = allocb(cc, BPRI_MED))) { mutex_exit(&xcp->excl); ttycommon_qfull(&async->async_ttycommon, q); goto out; } do { c = cp[INBUF_IX(cons++, ifp)]; /* * We handle XON/XOFF char if IXON is set, * but if received char is _POSIX_VDISABLE, * we left it to the up level module. */ if (tp->t_iflag & IXON) { if ((c == async->async_stopc) && (c != _POSIX_VDISABLE)) { xcasync_flowcontrol_sw_output(xcp, FLOW_STOP); continue; } else if ((c == async->async_startc) && (c != _POSIX_VDISABLE)) { xcasync_flowcontrol_sw_output(xcp, FLOW_START); continue; } if ((tp->t_iflag & IXANY) && (async->async_flags & ASYNC_SW_OUT_FLW)) { xcasync_flowcontrol_sw_output(xcp, FLOW_START); } } *bp->b_wptr++ = c; } while (--cc); membar_producer(); if (!DOMAIN_IS_INITDOMAIN(xen_info)) ifp->in_cons = cons; mutex_exit(&xcp->excl); if (bp->b_wptr > bp->b_rptr) { if (!canput(q)) { xenconserror(CE_NOTE, "xencons%d: local queue full", instance); freemsg(bp); } else (void) putq(q, bp); } else freemsg(bp); if (DOMAIN_IS_INITDOMAIN(xen_info)) goto loop; out: DEBUGCONT1(XENCONS_DEBUG_PROCS, "xencons%d_rxint: done\n", instance); if (!DOMAIN_IS_INITDOMAIN(xen_info)) ec_notify_via_evtchn(xcp->evtchn); } /* * Handle a xen console tx interrupt. */ /*ARGSUSED*/ static void xencons_txint(struct xencons *xcp) { struct asyncline *async; DEBUGCONT0(XENCONS_DEBUG_PROCS, "xencons_txint\n"); /* * prevent recursive entry */ if (mutex_owner(&xcp->excl) == curthread) { goto out; } mutex_enter(&xcp->excl); if (xencons_console == NULL) { mutex_exit(&xcp->excl); goto out; } /* make sure the device is open */ async = xcp->priv; if ((async->async_flags & ASYNC_ISOPEN) != 0) xcasync_start(async); mutex_exit(&xcp->excl); out: DEBUGCONT0(XENCONS_DEBUG_PROCS, "xencons_txint: done\n"); } /* * Get an event when input ring becomes not empty or output ring becomes not * full. */ static uint_t xenconsintr(caddr_t arg) { struct xencons *xcp = (struct xencons *)arg; volatile struct xencons_interface *ifp = xcp->ifp; if (ifp->in_prod != ifp->in_cons) xencons_rxint(xcp); if (ifp->out_prod - ifp->out_cons < sizeof (ifp->out)) xencons_txint(xcp); return (DDI_INTR_CLAIMED); } /* * Console interrupt routine for priviliged domains */ static uint_t xenconsintr_priv(caddr_t arg, caddr_t arg1 __unused) { struct xencons *xcp = (struct xencons *)arg; xencons_rxint(xcp); xencons_txint(xcp); return (DDI_INTR_CLAIMED); } /* * Start output on a line, unless it's busy, frozen, or otherwise. */ /*ARGSUSED*/ static void xcasync_start(struct asyncline *async) { struct xencons *xcp = async->async_common; int cc; queue_t *q; mblk_t *bp; int len, space, blen; mblk_t *nbp; #ifdef DEBUG int instance = xcp->unit; DEBUGCONT1(XENCONS_DEBUG_PROCS, "async%d_nstart\n", instance); #endif ASSERT(mutex_owned(&xcp->excl)); /* * Check only pended sw input flow control. */ domore: (void) xcasync_flowcontrol_sw_input(xcp, FLOW_CHECK, IN_FLOW_NULL); if ((q = async->async_ttycommon.t_writeq) == NULL) { return; /* not attached to a stream */ } for (;;) { if ((bp = getq(q)) == NULL) return; /* no data to transmit */ /* * We have a message block to work on. * Check whether it's a break, a delay, or an ioctl (the latter * occurs if the ioctl in question was waiting for the output * to drain). If it's one of those, process it immediately. */ switch (bp->b_datap->db_type) { case M_IOCTL: /* * This ioctl was waiting for the output ahead of * it to drain; obviously, it has. Do it, and * then grab the next message after it. */ mutex_exit(&xcp->excl); xcasync_ioctl(async, q, bp); mutex_enter(&xcp->excl); continue; } while (bp != NULL && (cc = bp->b_wptr - bp->b_rptr) == 0) { nbp = bp->b_cont; freeb(bp); bp = nbp; } if (bp != NULL) break; } /* * We have data to transmit. If output is stopped, put * it back and try again later. */ if (async->async_flags & (ASYNC_SW_OUT_FLW | ASYNC_STOPPED)) { (void) putbq(q, bp); return; } if (DOMAIN_IS_INITDOMAIN(xen_info)) { len = 0; space = XENCONS_WBUFSIZE; while (bp != NULL && space) { blen = bp->b_wptr - bp->b_rptr; cc = min(blen, space); bcopy(bp->b_rptr, &xencons_wbuf[len], cc); bp->b_rptr += cc; if (cc == blen) { nbp = bp->b_cont; freeb(bp); bp = nbp; } space -= cc; len += cc; } mutex_exit(&xcp->excl); (void) HYPERVISOR_console_io(CONSOLEIO_write, len, xencons_wbuf); mutex_enter(&xcp->excl); if (bp != NULL) (void) putbq(q, bp); /* not done with this msg yet */ /* * There are no completion interrupts when using the * HYPERVISOR_console_io call to write console data * so we loop here till we have sent all the data to the * hypervisor. */ goto domore; } else { volatile struct xencons_interface *ifp = xcp->ifp; XENCONS_RING_IDX cons, prod; cons = ifp->out_cons; prod = ifp->out_prod; membar_enter(); while (bp != NULL && ((prod - cons) < sizeof (ifp->out))) { ifp->out[MASK_XENCONS_IDX(prod++, ifp->out)] = *bp->b_rptr++; if (bp->b_rptr == bp->b_wptr) { nbp = bp->b_cont; freeb(bp); bp = nbp; } } membar_producer(); ifp->out_prod = prod; ec_notify_via_evtchn(xcp->evtchn); if (bp != NULL) (void) putbq(q, bp); /* not done with this msg yet */ } } /* * Process an "ioctl" message sent down to us. * Note that we don't need to get any locks until we are ready to access * the hardware. Nothing we access until then is going to be altered * outside of the STREAMS framework, so we should be safe. */ static void xcasync_ioctl(struct asyncline *async, queue_t *wq, mblk_t *mp) { struct xencons *xcp = async->async_common; tty_common_t *tp = &async->async_ttycommon; struct iocblk *iocp; unsigned datasize; int error = 0; #ifdef DEBUG int instance = xcp->unit; DEBUGCONT1(XENCONS_DEBUG_PROCS, "async%d_ioctl\n", instance); #endif if (tp->t_iocpending != NULL) { /* * We were holding an "ioctl" response pending the * availability of an "mblk" to hold data to be passed up; * another "ioctl" came through, which means that "ioctl" * must have timed out or been aborted. */ freemsg(async->async_ttycommon.t_iocpending); async->async_ttycommon.t_iocpending = NULL; } iocp = (struct iocblk *)mp->b_rptr; /* * For TIOCMGET and the PPS ioctls, do NOT call ttycommon_ioctl() * because this function frees up the message block (mp->b_cont) that * contains the user location where we pass back the results. * * Similarly, CONSOPENPOLLEDIO needs ioc_count, which ttycommon_ioctl * zaps. We know that ttycommon_ioctl doesn't know any CONS* * ioctls, so keep the others safe too. */ DEBUGCONT2(XENCONS_DEBUG_IOCTL, "async%d_ioctl: %s\n", instance, iocp->ioc_cmd == TIOCMGET ? "TIOCMGET" : iocp->ioc_cmd == TIOCMSET ? "TIOCMSET" : iocp->ioc_cmd == TIOCMBIS ? "TIOCMBIS" : iocp->ioc_cmd == TIOCMBIC ? "TIOCMBIC" : "other"); switch (iocp->ioc_cmd) { case TIOCMGET: case TIOCGPPS: case TIOCSPPS: case TIOCGPPSEV: case CONSOPENPOLLEDIO: case CONSCLOSEPOLLEDIO: case CONSSETABORTENABLE: case CONSGETABORTENABLE: error = -1; /* Do Nothing */ break; default: /* * The only way in which "ttycommon_ioctl" can fail is if the * "ioctl" requires a response containing data to be returned * to the user, and no mblk could be allocated for the data. * No such "ioctl" alters our state. Thus, we always go ahead * and do any state-changes the "ioctl" calls for. If we * couldn't allocate the data, "ttycommon_ioctl" has stashed * the "ioctl" away safely, so we just call "bufcall" to * request that we be called back when we stand a better * chance of allocating the data. */ if ((datasize = ttycommon_ioctl(tp, wq, mp, &error)) != 0) { if (async->async_wbufcid) unbufcall(async->async_wbufcid); async->async_wbufcid = bufcall(datasize, BPRI_HI, (void (*)(void *)) xcasync_reioctl, (void *)(intptr_t)async->async_common->unit); return; } } mutex_enter(&xcp->excl); if (error == 0) { /* * "ttycommon_ioctl" did most of the work; we just use the * data it set up. */ switch (iocp->ioc_cmd) { case TCSETS: case TCSETSF: case TCSETSW: case TCSETA: case TCSETAW: case TCSETAF: break; } } else if (error < 0) { /* * "ttycommon_ioctl" didn't do anything; we process it here. */ error = 0; switch (iocp->ioc_cmd) { case TCSBRK: error = miocpullup(mp, sizeof (int)); break; case TIOCSBRK: mioc2ack(mp, NULL, 0, 0); break; case TIOCCBRK: mioc2ack(mp, NULL, 0, 0); break; case CONSOPENPOLLEDIO: error = miocpullup(mp, sizeof (cons_polledio_arg_t)); if (error != 0) break; *(cons_polledio_arg_t *)mp->b_cont->b_rptr = (cons_polledio_arg_t)&xcp->polledio; mp->b_datap->db_type = M_IOCACK; break; case CONSCLOSEPOLLEDIO: mp->b_datap->db_type = M_IOCACK; iocp->ioc_error = 0; iocp->ioc_rval = 0; break; case CONSSETABORTENABLE: error = secpolicy_console(iocp->ioc_cr); if (error != 0) break; if (iocp->ioc_count != TRANSPARENT) { error = EINVAL; break; } if (*(intptr_t *)mp->b_cont->b_rptr) xcp->flags |= ASY_CONSOLE; else xcp->flags &= ~ASY_CONSOLE; mp->b_datap->db_type = M_IOCACK; iocp->ioc_error = 0; iocp->ioc_rval = 0; break; case CONSGETABORTENABLE: /*CONSTANTCONDITION*/ ASSERT(sizeof (boolean_t) <= sizeof (boolean_t *)); /* * Store the return value right in the payload * we were passed. Crude. */ mcopyout(mp, NULL, sizeof (boolean_t), NULL, NULL); *(boolean_t *)mp->b_cont->b_rptr = (xcp->flags & ASY_CONSOLE) != 0; break; default: /* * If we don't understand it, it's an error. NAK it. */ error = EINVAL; break; } } if (error != 0) { iocp->ioc_error = error; mp->b_datap->db_type = M_IOCNAK; } mutex_exit(&xcp->excl); qreply(wq, mp); DEBUGCONT1(XENCONS_DEBUG_PROCS, "async%d_ioctl: done\n", instance); } static int xenconsrsrv(queue_t *q) { mblk_t *bp; while (canputnext(q) && (bp = getq(q))) putnext(q, bp); return (0); } /* * Put procedure for write queue. * Respond to M_STOP, M_START, M_IOCTL, and M_FLUSH messages here; * set the flow control character for M_STOPI and M_STARTI messages; * queue up M_BREAK, M_DELAY, and M_DATA messages for processing * by the start routine, and then call the start routine; discard * everything else. Note that this driver does not incorporate any * mechanism to negotiate to handle the canonicalization process. * It expects that these functions are handled in upper module(s), * as we do in ldterm. */ static int xenconswput(queue_t *q, mblk_t *mp) { struct asyncline *async; struct xencons *xcp; async = (struct asyncline *)q->q_ptr; xcp = async->async_common; switch (mp->b_datap->db_type) { case M_STOP: mutex_enter(&xcp->excl); async->async_flags |= ASYNC_STOPPED; mutex_exit(&xcp->excl); freemsg(mp); break; case M_START: mutex_enter(&xcp->excl); if (async->async_flags & ASYNC_STOPPED) { async->async_flags &= ~ASYNC_STOPPED; xcasync_start(async); } mutex_exit(&xcp->excl); freemsg(mp); break; case M_IOCTL: switch (((struct iocblk *)mp->b_rptr)->ioc_cmd) { case TCSETSW: case TCSETSF: case TCSETAW: case TCSETAF: /* * The changes do not take effect until all * output queued before them is drained. * Put this message on the queue, so that * "xcasync_start" will see it when it's done * with the output before it. Poke the * start routine, just in case. */ (void) putq(q, mp); mutex_enter(&xcp->excl); xcasync_start(async); mutex_exit(&xcp->excl); break; default: /* * Do it now. */ xcasync_ioctl(async, q, mp); break; } break; case M_FLUSH: if (*mp->b_rptr & FLUSHW) { mutex_enter(&xcp->excl); /* * Flush our write queue. */ flushq(q, FLUSHDATA); /* XXX doesn't flush M_DELAY */ if (async->async_xmitblk != NULL) { freeb(async->async_xmitblk); async->async_xmitblk = NULL; } mutex_exit(&xcp->excl); *mp->b_rptr &= ~FLUSHW; /* it has been flushed */ } if (*mp->b_rptr & FLUSHR) { flushq(RD(q), FLUSHDATA); qreply(q, mp); /* give the read queues a crack at it */ } else { freemsg(mp); } /* * We must make sure we process messages that survive the * write-side flush. */ mutex_enter(&xcp->excl); xcasync_start(async); mutex_exit(&xcp->excl); break; case M_BREAK: case M_DELAY: case M_DATA: /* * Queue the message up to be transmitted, * and poke the start routine. */ (void) putq(q, mp); mutex_enter(&xcp->excl); xcasync_start(async); mutex_exit(&xcp->excl); break; case M_STOPI: mutex_enter(&xcp->excl); mutex_enter(&xcp->excl); if (!(async->async_inflow_source & IN_FLOW_USER)) { (void) xcasync_flowcontrol_sw_input(xcp, FLOW_STOP, IN_FLOW_USER); } mutex_exit(&xcp->excl); mutex_exit(&xcp->excl); freemsg(mp); break; case M_STARTI: mutex_enter(&xcp->excl); mutex_enter(&xcp->excl); if (async->async_inflow_source & IN_FLOW_USER) { (void) xcasync_flowcontrol_sw_input(xcp, FLOW_START, IN_FLOW_USER); } mutex_exit(&xcp->excl); mutex_exit(&xcp->excl); freemsg(mp); break; case M_CTL: if (MBLKL(mp) >= sizeof (struct iocblk) && ((struct iocblk *)mp->b_rptr)->ioc_cmd == MC_POSIXQUERY) { ((struct iocblk *)mp->b_rptr)->ioc_cmd = MC_HAS_POSIX; qreply(q, mp); } else { freemsg(mp); } break; default: freemsg(mp); break; } return (0); } /* * Retry an "ioctl", now that "bufcall" claims we may be able to allocate * the buffer we need. */ static void xcasync_reioctl(void *unit) { int instance = (uintptr_t)unit; struct asyncline *async; struct xencons *xcp; queue_t *q; mblk_t *mp; xcp = ddi_get_soft_state(xencons_soft_state, instance); ASSERT(xcp != NULL); async = xcp->priv; /* * The bufcall is no longer pending. */ mutex_enter(&xcp->excl); async->async_wbufcid = 0; if ((q = async->async_ttycommon.t_writeq) == NULL) { mutex_exit(&xcp->excl); return; } if ((mp = async->async_ttycommon.t_iocpending) != NULL) { /* not pending any more */ async->async_ttycommon.t_iocpending = NULL; mutex_exit(&xcp->excl); xcasync_ioctl(async, q, mp); } else mutex_exit(&xcp->excl); } /* * debugger/console support routines. */ /* * put a character out * Do not use interrupts. If char is LF, put out CR, LF. */ /*ARGSUSED*/ static void xenconsputchar(cons_polledio_arg_t arg, uchar_t c) { struct xencons *xcp = xencons_console; volatile struct xencons_interface *ifp = xcp->ifp; XENCONS_RING_IDX prod; if (c == '\n') xenconsputchar(arg, '\r'); /* * domain 0 can use the console I/O... */ if (DOMAIN_IS_INITDOMAIN(xen_info)) { char buffer[1]; buffer[0] = c; (void) HYPERVISOR_console_io(CONSOLEIO_write, 1, buffer); return; } /* * domU has to go through dom0 virtual console. */ while (ifp->out_prod - ifp->out_cons == sizeof (ifp->out)) (void) HYPERVISOR_yield(); prod = ifp->out_prod; ifp->out[MASK_XENCONS_IDX(prod++, ifp->out)] = c; membar_producer(); ifp->out_prod = prod; ec_notify_via_evtchn(xcp->evtchn); } /* * See if there's a character available. If no character is * available, return 0. Run in polled mode, no interrupts. */ static boolean_t xenconsischar(cons_polledio_arg_t arg) { struct xencons *xcp = (struct xencons *)arg; volatile struct xencons_interface *ifp = xcp->ifp; if (xcp->polldix < xcp->polllen) return (B_TRUE); /* * domain 0 can use the console I/O... */ xcp->polldix = 0; xcp->polllen = 0; if (DOMAIN_IS_INITDOMAIN(xen_info)) { xcp->polllen = HYPERVISOR_console_io(CONSOLEIO_read, 1, (char *)xcp->pollbuf); return (xcp->polllen != 0); } /* * domU has to go through virtual console device. */ if (ifp->in_prod != ifp->in_cons) { XENCONS_RING_IDX cons; cons = ifp->in_cons; membar_enter(); xcp->pollbuf[0] = ifp->in[MASK_XENCONS_IDX(cons++, ifp->in)]; membar_producer(); ifp->in_cons = cons; xcp->polllen = 1; } return (xcp->polllen != 0); } /* * Get a character. Run in polled mode, no interrupts. */ static int xenconsgetchar(cons_polledio_arg_t arg) { struct xencons *xcp = (struct xencons *)arg; ec_wait_on_evtchn(xcp->evtchn, (int (*)(void *))xenconsischar, arg); return (xcp->pollbuf[xcp->polldix++]); } static void xenconserror(int level, const char *fmt, ...) { va_list adx; static time_t last; static const char *lastfmt; time_t now; /* * Don't print the same error message too often. * Print the message only if we have not printed the * message within the last second. * Note: that fmt cannot be a pointer to a string * stored on the stack. The fmt pointer * must be in the data segment otherwise lastfmt would point * to non-sense. */ now = gethrestime_sec(); if (last == now && lastfmt == fmt) return; last = now; lastfmt = fmt; va_start(adx, fmt); vcmn_err(level, fmt, adx); va_end(adx); } /* * Check for abort character sequence */ static boolean_t abort_charseq_recognize(uchar_t ch) { static int state = 0; #define CNTRL(c) ((c)&037) static char sequence[] = { '\r', '~', CNTRL('b') }; if (ch == sequence[state]) { if (++state >= sizeof (sequence)) { state = 0; return (B_TRUE); } } else { state = (ch == sequence[0]) ? 1 : 0; } return (B_FALSE); } /* * Flow control functions */ /* * Software output flow control * This function can be executed sucessfully at any situation. * It does not handle HW, and just change the SW output flow control flag. * INPUT VALUE of onoff: * FLOW_START means to clear SW output flow control flag, * also set ASYNC_OUT_FLW_RESUME. * FLOW_STOP means to set SW output flow control flag, * also clear ASYNC_OUT_FLW_RESUME. */ static void xcasync_flowcontrol_sw_output(struct xencons *xcp, async_flowc_action onoff) { struct asyncline *async = xcp->priv; int instance = xcp->unit; ASSERT(mutex_owned(&xcp->excl)); if (!(async->async_ttycommon.t_iflag & IXON)) return; switch (onoff) { case FLOW_STOP: async->async_flags |= ASYNC_SW_OUT_FLW; async->async_flags &= ~ASYNC_OUT_FLW_RESUME; DEBUGCONT1(XENCONS_DEBUG_SFLOW, "xencons%d: output sflow stop\n", instance); break; case FLOW_START: async->async_flags &= ~ASYNC_SW_OUT_FLW; async->async_flags |= ASYNC_OUT_FLW_RESUME; DEBUGCONT1(XENCONS_DEBUG_SFLOW, "xencons%d: output sflow start\n", instance); break; default: break; } } /* * Software input flow control * This function can execute software input flow control * INPUT VALUE of onoff: * FLOW_START means to send out a XON char * and clear SW input flow control flag. * FLOW_STOP means to send out a XOFF char * and set SW input flow control flag. * FLOW_CHECK means to check whether there is pending XON/XOFF * if it is true, send it out. * INPUT VALUE of type: * IN_FLOW_STREAMS means flow control is due to STREAMS * IN_FLOW_USER means flow control is due to user's commands * RETURN VALUE: B_FALSE means no flow control char is sent * B_TRUE means one flow control char is sent */ static boolean_t xcasync_flowcontrol_sw_input(struct xencons *xcp, async_flowc_action onoff, int type) { struct asyncline *async = xcp->priv; int instance = xcp->unit; int rval = B_FALSE; ASSERT(mutex_owned(&xcp->excl)); if (!(async->async_ttycommon.t_iflag & IXOFF)) return (rval); /* * If we get this far, then we know IXOFF is set. */ switch (onoff) { case FLOW_STOP: async->async_inflow_source |= type; /* * We'll send an XOFF character for each of up to * three different input flow control attempts to stop input. * If we already send out one XOFF, but FLOW_STOP comes again, * it seems that input flow control becomes more serious, * then send XOFF again. */ if (async->async_inflow_source & (IN_FLOW_STREAMS | IN_FLOW_USER)) async->async_flags |= ASYNC_SW_IN_FLOW | ASYNC_SW_IN_NEEDED; DEBUGCONT2(XENCONS_DEBUG_SFLOW, "xencons%d: input sflow stop, " "type = %x\n", instance, async->async_inflow_source); break; case FLOW_START: async->async_inflow_source &= ~type; if (async->async_inflow_source == 0) { async->async_flags = (async->async_flags & ~ASYNC_SW_IN_FLOW) | ASYNC_SW_IN_NEEDED; DEBUGCONT1(XENCONS_DEBUG_SFLOW, "xencons%d: " "input sflow start\n", instance); } break; default: break; } if (async->async_flags & ASYNC_SW_IN_NEEDED) { /* * If we get this far, then we know we need to send out * XON or XOFF char. */ char c; rval = B_TRUE; c = (async->async_flags & ASYNC_SW_IN_FLOW) ? async->async_stopc : async->async_startc; if (DOMAIN_IS_INITDOMAIN(xen_info)) { (void) HYPERVISOR_console_io(CONSOLEIO_write, 1, &c); async->async_flags &= ~ASYNC_SW_IN_NEEDED; return (rval); } else { xenconsputchar(NULL, c); } } return (rval); } struct module_info xencons_info = { 0, "xencons", 0, INFPSZ, 4096, 128 }; static struct qinit xencons_rint = { putq, xenconsrsrv, xenconsopen, xenconsclose, NULL, &xencons_info, NULL }; static struct qinit xencons_wint = { xenconswput, NULL, NULL, NULL, NULL, &xencons_info, NULL }; struct streamtab xencons_str_info = { &xencons_rint, &xencons_wint, NULL, NULL }; static struct cb_ops cb_xencons_ops = { nodev, /* cb_open */ nodev, /* cb_close */ nodev, /* cb_strategy */ nodev, /* cb_print */ nodev, /* cb_dump */ nodev, /* cb_read */ nodev, /* cb_write */ nodev, /* cb_ioctl */ nodev, /* cb_devmap */ nodev, /* cb_mmap */ nodev, /* cb_segmap */ nochpoll, /* cb_chpoll */ ddi_prop_op, /* cb_prop_op */ &xencons_str_info, /* cb_stream */ D_MP /* cb_flag */ }; struct dev_ops xencons_ops = { DEVO_REV, /* devo_rev */ 0, /* devo_refcnt */ xenconsinfo, /* devo_getinfo */ nulldev, /* devo_identify */ nulldev, /* devo_probe */ xenconsattach, /* devo_attach */ xenconsdetach, /* devo_detach */ nodev, /* devo_reset */ &cb_xencons_ops, /* devo_cb_ops */ NULL, /* devo_bus_ops */ NULL, /* devo_power */ ddi_quiesce_not_needed, /* devo_quiesce */ }; static struct modldrv modldrv = { &mod_driverops, /* Type of module. This one is a driver */ "virtual console driver", &xencons_ops, /* driver ops */ }; static struct modlinkage modlinkage = { MODREV_1, (void *)&modldrv, NULL }; int _init(void) { int rv; if ((rv = ddi_soft_state_init(&xencons_soft_state, sizeof (struct xencons), 1)) != 0) return (rv); if ((rv = mod_install(&modlinkage)) != 0) { ddi_soft_state_fini(&xencons_soft_state); return (rv); } DEBUGCONT2(XENCONS_DEBUG_INIT, "%s, debug = %x\n", modldrv.drv_linkinfo, debug); return (0); } int _fini(void) { int rv; if ((rv = mod_remove(&modlinkage)) != 0) return (rv); ddi_soft_state_fini(&xencons_soft_state); return (0); } int _info(struct modinfo *modinfop) { return (mod_info(&modlinkage, modinfop)); } # # CDDL HEADER START # # The contents of this file are subject to the terms of the # Common Development and Distribution License (the "License"). # You may not use this file except in compliance with the License. # # You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE # or http://www.opensolaris.org/os/licensing. # See the License for the specific language governing permissions # and limitations under the License. # # When distributing Covered Code, include this CDDL HEADER in each # file and include the License file at usr/src/OPENSOLARIS.LICENSE. # If applicable, add the following below this CDDL HEADER, with the # fields enclosed by brackets "[]" replaced with your own identifying # information: Portions Copyright [yyyy] [name of copyright owner] # # CDDL HEADER END # # # Copyright 2007 Sun Microsystems, Inc. All rights reserved. # Use is subject to license terms. # /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* Copyright (c) 1990, 1991 UNIX System Laboratories, Inc. */ /* Copyright (c) 1984, 1986, 1987, 1988, 1989, 1990 AT&T */ /* All Rights Reserved */ /* * Copyright 2007 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ #ifndef _SYS_XENCONS_H #define _SYS_XENCONS_H #ifdef __cplusplus extern "C" { #endif #include #include #include #include /* * Xencons tracing macros. These are a similar to some macros in sys/vtrace.h. * * XXX - Needs review: would it be better to use the macros in sys/vtrace.h ? */ #ifdef DEBUG #define DEBUGWARN0(fac, format) \ if (debug & (fac)) \ cmn_err(CE_WARN, format) #define DEBUGNOTE0(fac, format) \ if (debug & (fac)) \ cmn_err(CE_NOTE, format) #define DEBUGNOTE1(fac, format, arg1) \ if (debug & (fac)) \ cmn_err(CE_NOTE, format, arg1) #define DEBUGNOTE2(fac, format, arg1, arg2) \ if (debug & (fac)) \ cmn_err(CE_NOTE, format, arg1, arg2) #define DEBUGNOTE3(fac, format, arg1, arg2, arg3) \ if (debug & (fac)) \ cmn_err(CE_NOTE, format, arg1, arg2, arg3) #define DEBUGCONT0(fac, format) \ if (debug & (fac)) \ cmn_err(CE_CONT, format) #define DEBUGCONT1(fac, format, arg1) \ if (debug & (fac)) \ cmn_err(CE_CONT, format, arg1) #define DEBUGCONT2(fac, format, arg1, arg2) \ if (debug & (fac)) \ cmn_err(CE_CONT, format, arg1, arg2) #define DEBUGCONT3(fac, format, arg1, arg2, arg3) \ if (debug & (fac)) \ cmn_err(CE_CONT, format, arg1, arg2, arg3) #define DEBUGCONT4(fac, format, arg1, arg2, arg3, arg4) \ if (debug & (fac)) \ cmn_err(CE_CONT, format, arg1, arg2, arg3, arg4) #define DEBUGCONT10(fac, format, \ arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, arg9, arg10) \ if (debug & (fac)) \ cmn_err(CE_CONT, format, \ arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, arg9, arg10) #else #define DEBUGWARN0(fac, format) #define DEBUGNOTE0(fac, format) #define DEBUGNOTE1(fac, format, arg1) #define DEBUGNOTE2(fac, format, arg1, arg2) #define DEBUGNOTE3(fac, format, arg1, arg2, arg3) #define DEBUGCONT0(fac, format) #define DEBUGCONT1(fac, format, arg1) #define DEBUGCONT2(fac, format, arg1, arg2) #define DEBUGCONT3(fac, format, arg1, arg2, arg3) #define DEBUGCONT4(fac, format, arg1, arg2, arg3, arg4) #define DEBUGCONT10(fac, format, \ arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, arg9, arg10) #endif /* enum value for sw and hw flow control action */ typedef enum { FLOW_CHECK, FLOW_STOP, FLOW_START } async_flowc_action; #define async_stopc async_ttycommon.t_stopc #define async_startc async_ttycommon.t_startc /* * Console instance data. * Each of the fields in this structure is required to be protected by a * mutex lock at the highest priority at which it can be altered. */ struct xencons { int flags; /* random flags */ struct asyncline *priv; /* protocol private data -- asyncline */ dev_info_t *dip; /* dev_info */ int unit; /* which port */ kmutex_t excl; /* adaptive mutex */ kcondvar_t excl_cv; /* condition variable */ struct cons_polledio polledio; /* polled I/O functions */ unsigned char pollbuf[60]; /* polled I/O data */ int polldix; /* polled data buffer index */ int polllen; /* polled data buffer length */ volatile struct xencons_interface *ifp; /* console ring buffers */ int console_irq; /* dom0 console interrupt */ int evtchn; /* console event channel */ }; /* * Asychronous protocol private data structure for ASY. * Each of the fields in the structure is required to be protected by * the lower priority lock except the fields that are set only at * base level but cleared (with out lock) at interrupt level. */ struct asyncline { int async_flags; /* random flags */ kcondvar_t async_flags_cv; /* condition variable for flags */ dev_t async_dev; /* device major/minor numbers */ mblk_t *async_xmitblk; /* transmit: active msg block */ struct xencons *async_common; /* device common data */ tty_common_t async_ttycommon; /* tty driver common data */ bufcall_id_t async_wbufcid; /* id for pending write-side bufcall */ timeout_id_t async_polltid; /* softint poll timeout id */ timeout_id_t async_dtrtid; /* delaying DTR turn on */ timeout_id_t async_utbrktid; /* hold minimum untimed break time id */ /* * The following fields are protected by the excl_hi lock. * Some, such as async_flowc, are set only at the base level and * cleared (without the lock) only by the interrupt level. */ uchar_t *async_optr; /* output pointer */ int async_ocnt; /* output count */ ushort_t async_rput; /* producing pointer for input */ ushort_t async_rget; /* consuming pointer for input */ int async_inflow_source; /* input flow control type */ union { struct { uchar_t _hw; /* overrun (hw) */ uchar_t _sw; /* overrun (sw) */ } _a; ushort_t uover_overrun; } async_uover; #define async_overrun async_uover._a.uover_overrun #define async_hw_overrun async_uover._a._hw #define async_sw_overrun async_uover._a._sw short async_ext; /* modem status change count */ short async_work; /* work to do flag */ }; /* definitions for async_flags field */ #define ASYNC_EXCL_OPEN 0x10000000 /* exclusive open */ #define ASYNC_WOPEN 0x00000001 /* waiting for open to complete */ #define ASYNC_ISOPEN 0x00000002 /* open is complete */ #define ASYNC_STOPPED 0x00000010 /* output is stopped */ #define ASYNC_PROGRESS 0x00001000 /* made progress on output effort */ #define ASYNC_CLOSING 0x00002000 /* processing close on stream */ #define ASYNC_SW_IN_FLOW 0x00020000 /* sw input flow control in effect */ #define ASYNC_SW_OUT_FLW 0x00040000 /* sw output flow control in effect */ #define ASYNC_SW_IN_NEEDED 0x00080000 /* sw input flow control char is */ /* needed to be sent */ #define ASYNC_OUT_FLW_RESUME 0x00100000 /* output need to be resumed */ /* because of transition of flow */ /* control from stop to start */ /* definitions for asy_flags field */ #define ASY_CONSOLE 0x00000080 /* definitions for async_inflow_source field in struct asyncline */ #define IN_FLOW_NULL 0x00000000 #define IN_FLOW_STREAMS 0x00000002 #define IN_FLOW_USER 0x00000004 #define XENCONS_BURST 128 /* burst size for console writes */ #ifdef __cplusplus } #endif #endif /* _SYS_XENCONS_H */ /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2010 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. * Copyright 2018 Joyent, Inc. */ #ifdef DEBUG #define XNB_DEBUG 1 #endif /* DEBUG */ #include "xnb.h" #include #include #include #include #include #include /* For mac_fix_cksum(). */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* * The terms "transmit" and "receive" are used in alignment with domU, * which means that packets originating from the peer domU are "transmitted" * to other parts of the system and packets are "received" from them. */ /* * Should we allow guests to manipulate multicast group membership? */ static boolean_t xnb_multicast_control = B_TRUE; static boolean_t xnb_connect_rings(dev_info_t *); static void xnb_disconnect_rings(dev_info_t *); static void xnb_oe_state_change(dev_info_t *, ddi_eventcookie_t, void *, void *); static void xnb_hp_state_change(dev_info_t *, ddi_eventcookie_t, void *, void *); static int xnb_txbuf_constructor(void *, void *, int); static void xnb_txbuf_destructor(void *, void *); static void xnb_tx_notify_peer(xnb_t *, boolean_t); static void xnb_tx_mark_complete(xnb_t *, RING_IDX, int16_t); mblk_t *xnb_to_peer(xnb_t *, mblk_t *); mblk_t *xnb_copy_to_peer(xnb_t *, mblk_t *); static void setup_gop(xnb_t *, gnttab_copy_t *, uchar_t *, size_t, size_t, size_t, grant_ref_t); static boolean_t is_foreign(void *); #define INVALID_GRANT_HANDLE ((grant_handle_t)-1) #define INVALID_GRANT_REF ((grant_ref_t)-1) static kmutex_t xnb_alloc_page_lock; /* * On a 32 bit PAE system physical and machine addresses are larger * than 32 bits. ddi_btop() on such systems take an unsigned long * argument, and so addresses above 4G are truncated before ddi_btop() * gets to see them. To avoid this, code the shift operation here. */ #define xnb_btop(addr) ((addr) >> PAGESHIFT) /* DMA attributes for transmit and receive data */ static ddi_dma_attr_t buf_dma_attr = { DMA_ATTR_V0, /* version of this structure */ 0, /* lowest usable address */ 0xffffffffffffffffULL, /* highest usable address */ 0x7fffffff, /* maximum DMAable byte count */ MMU_PAGESIZE, /* alignment in bytes */ 0x7ff, /* bitmap of burst sizes */ 1, /* minimum transfer */ 0xffffffffU, /* maximum transfer */ 0xffffffffffffffffULL, /* maximum segment length */ 1, /* maximum number of segments */ 1, /* granularity */ 0, /* flags (reserved) */ }; /* DMA access attributes for data: NOT to be byte swapped. */ static ddi_device_acc_attr_t data_accattr = { DDI_DEVICE_ATTR_V0, DDI_NEVERSWAP_ACC, DDI_STRICTORDER_ACC }; /* * Statistics. */ static const char * const aux_statistics[] = { "rx_cksum_deferred", "tx_cksum_no_need", "rx_rsp_notok", "tx_notify_deferred", "tx_notify_sent", "rx_notify_deferred", "rx_notify_sent", "tx_too_early", "rx_too_early", "rx_allocb_failed", "tx_allocb_failed", "rx_foreign_page", "mac_full", "spurious_intr", "allocation_success", "allocation_failure", "small_allocation_success", "small_allocation_failure", "other_allocation_failure", "rx_pageboundary_crossed", "rx_cpoparea_grown", "csum_hardware", "csum_software", "tx_overflow_page", "tx_unexpected_flags", }; static int xnb_ks_aux_update(kstat_t *ksp, int flag) { xnb_t *xnbp; kstat_named_t *knp; if (flag != KSTAT_READ) return (EACCES); xnbp = ksp->ks_private; knp = ksp->ks_data; /* * Assignment order should match that of the names in * aux_statistics. */ (knp++)->value.ui64 = xnbp->xnb_stat_rx_cksum_deferred; (knp++)->value.ui64 = xnbp->xnb_stat_tx_cksum_no_need; (knp++)->value.ui64 = xnbp->xnb_stat_rx_rsp_notok; (knp++)->value.ui64 = xnbp->xnb_stat_tx_notify_deferred; (knp++)->value.ui64 = xnbp->xnb_stat_tx_notify_sent; (knp++)->value.ui64 = xnbp->xnb_stat_rx_notify_deferred; (knp++)->value.ui64 = xnbp->xnb_stat_rx_notify_sent; (knp++)->value.ui64 = xnbp->xnb_stat_tx_too_early; (knp++)->value.ui64 = xnbp->xnb_stat_rx_too_early; (knp++)->value.ui64 = xnbp->xnb_stat_rx_allocb_failed; (knp++)->value.ui64 = xnbp->xnb_stat_tx_allocb_failed; (knp++)->value.ui64 = xnbp->xnb_stat_rx_foreign_page; (knp++)->value.ui64 = xnbp->xnb_stat_mac_full; (knp++)->value.ui64 = xnbp->xnb_stat_spurious_intr; (knp++)->value.ui64 = xnbp->xnb_stat_allocation_success; (knp++)->value.ui64 = xnbp->xnb_stat_allocation_failure; (knp++)->value.ui64 = xnbp->xnb_stat_small_allocation_success; (knp++)->value.ui64 = xnbp->xnb_stat_small_allocation_failure; (knp++)->value.ui64 = xnbp->xnb_stat_other_allocation_failure; (knp++)->value.ui64 = xnbp->xnb_stat_rx_pagebndry_crossed; (knp++)->value.ui64 = xnbp->xnb_stat_rx_cpoparea_grown; (knp++)->value.ui64 = xnbp->xnb_stat_csum_hardware; (knp++)->value.ui64 = xnbp->xnb_stat_csum_software; (knp++)->value.ui64 = xnbp->xnb_stat_tx_overflow_page; (knp++)->value.ui64 = xnbp->xnb_stat_tx_unexpected_flags; return (0); } static boolean_t xnb_ks_init(xnb_t *xnbp) { int nstat = sizeof (aux_statistics) / sizeof (aux_statistics[0]); const char * const *cp = aux_statistics; kstat_named_t *knp; /* * Create and initialise kstats. */ xnbp->xnb_kstat_aux = kstat_create(ddi_driver_name(xnbp->xnb_devinfo), ddi_get_instance(xnbp->xnb_devinfo), "aux_statistics", "net", KSTAT_TYPE_NAMED, nstat, 0); if (xnbp->xnb_kstat_aux == NULL) return (B_FALSE); xnbp->xnb_kstat_aux->ks_private = xnbp; xnbp->xnb_kstat_aux->ks_update = xnb_ks_aux_update; knp = xnbp->xnb_kstat_aux->ks_data; while (nstat > 0) { kstat_named_init(knp, *cp, KSTAT_DATA_UINT64); knp++; cp++; nstat--; } kstat_install(xnbp->xnb_kstat_aux); return (B_TRUE); } static void xnb_ks_free(xnb_t *xnbp) { kstat_delete(xnbp->xnb_kstat_aux); } /* * Calculate and insert the transport checksum for an arbitrary packet. */ static mblk_t * xnb_software_csum(xnb_t *xnbp, mblk_t *mp) { _NOTE(ARGUNUSED(xnbp)); /* * XXPV dme: shouldn't rely on mac_fix_cksum(), not least * because it doesn't cover all of the interesting cases :-( */ mac_hcksum_set(mp, 0, 0, 0, 0, HCK_FULLCKSUM); mac_hw_emul(&mp, NULL, NULL, MAC_HWCKSUM_EMUL); return (mp); } mblk_t * xnb_process_cksum_flags(xnb_t *xnbp, mblk_t *mp, uint32_t capab) { struct ether_header *ehp; uint16_t sap; uint32_t offset; ipha_t *ipha; ASSERT(mp->b_next == NULL); /* * Check that the packet is contained in a single mblk. In * the "from peer" path this is true today, but may change * when scatter gather support is added. In the "to peer" * path we cannot be sure, but in most cases it will be true * (in the xnbo case the packet has come from a MAC device * which is unlikely to split packets). */ if (mp->b_cont != NULL) goto software; /* * If the MAC has no hardware capability don't do any further * checking. */ if (capab == 0) goto software; ASSERT(MBLKL(mp) >= sizeof (struct ether_header)); ehp = (struct ether_header *)mp->b_rptr; if (ntohs(ehp->ether_type) == VLAN_TPID) { struct ether_vlan_header *evhp; ASSERT(MBLKL(mp) >= sizeof (struct ether_vlan_header)); evhp = (struct ether_vlan_header *)mp->b_rptr; sap = ntohs(evhp->ether_type); offset = sizeof (struct ether_vlan_header); } else { sap = ntohs(ehp->ether_type); offset = sizeof (struct ether_header); } /* * We only attempt to do IPv4 packets in hardware. */ if (sap != ETHERTYPE_IP) goto software; /* * We know that this is an IPv4 packet. */ ipha = (ipha_t *)(mp->b_rptr + offset); switch (ipha->ipha_protocol) { case IPPROTO_TCP: case IPPROTO_UDP: { uint32_t start, length, stuff, cksum; uint16_t *stuffp; /* * This is a TCP/IPv4 or UDP/IPv4 packet, for which we * can use full IPv4 and partial checksum offload. */ if ((capab & (HCKSUM_INET_FULL_V4|HCKSUM_INET_PARTIAL)) == 0) break; start = IP_SIMPLE_HDR_LENGTH; length = ntohs(ipha->ipha_length); if (ipha->ipha_protocol == IPPROTO_TCP) { stuff = start + TCP_CHECKSUM_OFFSET; cksum = IP_TCP_CSUM_COMP; } else { stuff = start + UDP_CHECKSUM_OFFSET; cksum = IP_UDP_CSUM_COMP; } stuffp = (uint16_t *)(mp->b_rptr + offset + stuff); if (capab & HCKSUM_INET_FULL_V4) { /* * Some devices require that the checksum * field of the packet is zero for full * offload. */ *stuffp = 0; mac_hcksum_set(mp, 0, 0, 0, 0, HCK_FULLCKSUM); xnbp->xnb_stat_csum_hardware++; return (mp); } if (capab & HCKSUM_INET_PARTIAL) { if (*stuffp == 0) { ipaddr_t src, dst; /* * Older Solaris guests don't insert * the pseudo-header checksum, so we * calculate it here. */ src = ipha->ipha_src; dst = ipha->ipha_dst; cksum += (dst >> 16) + (dst & 0xFFFF); cksum += (src >> 16) + (src & 0xFFFF); cksum += length - IP_SIMPLE_HDR_LENGTH; cksum = (cksum >> 16) + (cksum & 0xFFFF); cksum = (cksum >> 16) + (cksum & 0xFFFF); ASSERT(cksum <= 0xFFFF); *stuffp = (uint16_t)(cksum ? cksum : ~cksum); } mac_hcksum_set(mp, start, stuff, length, 0, HCK_PARTIALCKSUM); xnbp->xnb_stat_csum_hardware++; return (mp); } /* NOTREACHED */ break; } default: /* Use software. */ break; } software: /* * We are not able to use any offload so do the whole thing in * software. */ xnbp->xnb_stat_csum_software++; return (xnb_software_csum(xnbp, mp)); } int xnb_attach(dev_info_t *dip, xnb_flavour_t *flavour, void *flavour_data) { xnb_t *xnbp; char *xsname; char cachename[32]; xnbp = kmem_zalloc(sizeof (*xnbp), KM_SLEEP); xnbp->xnb_flavour = flavour; xnbp->xnb_flavour_data = flavour_data; xnbp->xnb_devinfo = dip; xnbp->xnb_evtchn = INVALID_EVTCHN; xnbp->xnb_irq = B_FALSE; xnbp->xnb_tx_ring_handle = INVALID_GRANT_HANDLE; xnbp->xnb_rx_ring_handle = INVALID_GRANT_HANDLE; xnbp->xnb_connected = B_FALSE; xnbp->xnb_hotplugged = B_FALSE; xnbp->xnb_detachable = B_FALSE; xnbp->xnb_peer = xvdi_get_oeid(dip); xnbp->xnb_be_status = XNB_STATE_INIT; xnbp->xnb_fe_status = XNB_STATE_INIT; xnbp->xnb_tx_buf_count = 0; xnbp->xnb_rx_hv_copy = B_FALSE; xnbp->xnb_multicast_control = B_FALSE; xnbp->xnb_rx_va = vmem_alloc(heap_arena, PAGESIZE, VM_SLEEP); ASSERT(xnbp->xnb_rx_va != NULL); if (ddi_get_iblock_cookie(dip, 0, &xnbp->xnb_icookie) != DDI_SUCCESS) goto failure; /* Allocated on demand, when/if we enter xnb_copy_to_peer(). */ xnbp->xnb_rx_cpop = NULL; xnbp->xnb_rx_cpop_count = 0; mutex_init(&xnbp->xnb_tx_lock, NULL, MUTEX_DRIVER, xnbp->xnb_icookie); mutex_init(&xnbp->xnb_rx_lock, NULL, MUTEX_DRIVER, xnbp->xnb_icookie); mutex_init(&xnbp->xnb_state_lock, NULL, MUTEX_DRIVER, xnbp->xnb_icookie); /* Set driver private pointer now. */ ddi_set_driver_private(dip, xnbp); (void) sprintf(cachename, "xnb_tx_buf_cache_%d", ddi_get_instance(dip)); xnbp->xnb_tx_buf_cache = kmem_cache_create(cachename, sizeof (xnb_txbuf_t), 0, xnb_txbuf_constructor, xnb_txbuf_destructor, NULL, xnbp, NULL, 0); if (xnbp->xnb_tx_buf_cache == NULL) goto failure_0; if (!xnb_ks_init(xnbp)) goto failure_1; /* * Receive notification of changes in the state of the * driver in the guest domain. */ if (xvdi_add_event_handler(dip, XS_OE_STATE, xnb_oe_state_change, NULL) != DDI_SUCCESS) goto failure_2; /* * Receive notification of hotplug events. */ if (xvdi_add_event_handler(dip, XS_HP_STATE, xnb_hp_state_change, NULL) != DDI_SUCCESS) goto failure_2; xsname = xvdi_get_xsname(dip); if (xenbus_printf(XBT_NULL, xsname, "feature-multicast-control", "%d", xnb_multicast_control ? 1 : 0) != 0) goto failure_3; if (xenbus_printf(XBT_NULL, xsname, "feature-rx-copy", "%d", 1) != 0) goto failure_3; /* * Linux domUs seem to depend on "feature-rx-flip" being 0 * in addition to "feature-rx-copy" being 1. It seems strange * to use four possible states to describe a binary decision, * but we might as well play nice. */ if (xenbus_printf(XBT_NULL, xsname, "feature-rx-flip", "%d", 0) != 0) goto failure_3; (void) xvdi_switch_state(dip, XBT_NULL, XenbusStateInitWait); (void) xvdi_post_event(dip, XEN_HP_ADD); return (DDI_SUCCESS); failure_3: xvdi_remove_event_handler(dip, NULL); failure_2: xnb_ks_free(xnbp); failure_1: kmem_cache_destroy(xnbp->xnb_tx_buf_cache); failure_0: mutex_destroy(&xnbp->xnb_state_lock); mutex_destroy(&xnbp->xnb_rx_lock); mutex_destroy(&xnbp->xnb_tx_lock); failure: vmem_free(heap_arena, xnbp->xnb_rx_va, PAGESIZE); kmem_free(xnbp, sizeof (*xnbp)); return (DDI_FAILURE); } void xnb_detach(dev_info_t *dip) { xnb_t *xnbp = ddi_get_driver_private(dip); ASSERT(xnbp != NULL); ASSERT(!xnbp->xnb_connected); ASSERT(xnbp->xnb_tx_buf_count == 0); xnb_disconnect_rings(dip); xvdi_remove_event_handler(dip, NULL); xnb_ks_free(xnbp); kmem_cache_destroy(xnbp->xnb_tx_buf_cache); ddi_set_driver_private(dip, NULL); mutex_destroy(&xnbp->xnb_state_lock); mutex_destroy(&xnbp->xnb_rx_lock); mutex_destroy(&xnbp->xnb_tx_lock); if (xnbp->xnb_rx_cpop_count > 0) kmem_free(xnbp->xnb_rx_cpop, sizeof (xnbp->xnb_rx_cpop[0]) * xnbp->xnb_rx_cpop_count); ASSERT(xnbp->xnb_rx_va != NULL); vmem_free(heap_arena, xnbp->xnb_rx_va, PAGESIZE); kmem_free(xnbp, sizeof (*xnbp)); } /* * Allocate a page from the hypervisor to be flipped to the peer. * * Try to get pages in batches to reduce the overhead of calls into * the balloon driver. */ static mfn_t xnb_alloc_page(xnb_t *xnbp) { #define WARNING_RATE_LIMIT 100 #define BATCH_SIZE 256 static mfn_t mfns[BATCH_SIZE]; /* common across all instances */ static int nth = BATCH_SIZE; mfn_t mfn; mutex_enter(&xnb_alloc_page_lock); if (nth == BATCH_SIZE) { if (balloon_alloc_pages(BATCH_SIZE, mfns) != BATCH_SIZE) { xnbp->xnb_stat_allocation_failure++; mutex_exit(&xnb_alloc_page_lock); /* * Try for a single page in low memory situations. */ if (balloon_alloc_pages(1, &mfn) != 1) { if ((xnbp->xnb_stat_small_allocation_failure++ % WARNING_RATE_LIMIT) == 0) cmn_err(CE_WARN, "xnb_alloc_page: " "Cannot allocate memory to " "transfer packets to peer."); return (0); } else { xnbp->xnb_stat_small_allocation_success++; return (mfn); } } nth = 0; xnbp->xnb_stat_allocation_success++; } mfn = mfns[nth++]; mutex_exit(&xnb_alloc_page_lock); ASSERT(mfn != 0); return (mfn); #undef BATCH_SIZE #undef WARNING_RATE_LIMIT } /* * Free a page back to the hypervisor. * * This happens only in the error path, so batching is not worth the * complication. */ static void xnb_free_page(xnb_t *xnbp, mfn_t mfn) { _NOTE(ARGUNUSED(xnbp)); int r; pfn_t pfn; pfn = xen_assign_pfn(mfn); pfnzero(pfn, 0, PAGESIZE); xen_release_pfn(pfn); if ((r = balloon_free_pages(1, &mfn, NULL, NULL)) != 1) { cmn_err(CE_WARN, "free_page: cannot decrease memory " "reservation (%d): page kept but unusable (mfn = 0x%lx).", r, mfn); } } /* * Similar to RING_HAS_UNCONSUMED_REQUESTS(&xnbp->rx_ring) but using * local variables. Used in both xnb_to_peer() and xnb_copy_to_peer(). */ #define XNB_RING_HAS_UNCONSUMED_REQUESTS(_r) \ ((((_r)->sring->req_prod - loop) < \ (RING_SIZE(_r) - (loop - prod))) ? \ ((_r)->sring->req_prod - loop) : \ (RING_SIZE(_r) - (loop - prod))) /* * Pass packets to the peer using page flipping. */ mblk_t * xnb_to_peer(xnb_t *xnbp, mblk_t *mp) { mblk_t *free = mp, *prev = NULL; size_t len; gnttab_transfer_t *gop; boolean_t notify; RING_IDX loop, prod, end; /* * For each packet the sequence of operations is: * * 1. get a new page from the hypervisor. * 2. get a request slot from the ring. * 3. copy the data into the new page. * 4. transfer the page to the peer. * 5. update the request slot. * 6. kick the peer. * 7. free mp. * * In order to reduce the number of hypercalls, we prepare * several packets for the peer and perform a single hypercall * to transfer them. */ len = 0; mutex_enter(&xnbp->xnb_rx_lock); /* * If we are not connected to the peer or have not yet * finished hotplug it is too early to pass packets to the * peer. */ if (!(xnbp->xnb_connected && xnbp->xnb_hotplugged)) { mutex_exit(&xnbp->xnb_rx_lock); DTRACE_PROBE(flip_rx_too_early); xnbp->xnb_stat_rx_too_early++; return (mp); } loop = xnbp->xnb_rx_ring.req_cons; prod = xnbp->xnb_rx_ring.rsp_prod_pvt; gop = xnbp->xnb_rx_top; while ((mp != NULL) && XNB_RING_HAS_UNCONSUMED_REQUESTS(&xnbp->xnb_rx_ring)) { mfn_t mfn; pfn_t pfn; netif_rx_request_t *rxreq; netif_rx_response_t *rxresp; char *valoop; mblk_t *ml; uint16_t cksum_flags; /* 1 */ if ((mfn = xnb_alloc_page(xnbp)) == 0) { xnbp->xnb_stat_rx_defer++; break; } /* 2 */ rxreq = RING_GET_REQUEST(&xnbp->xnb_rx_ring, loop); #ifdef XNB_DEBUG if (!(rxreq->id < NET_RX_RING_SIZE)) cmn_err(CE_PANIC, "xnb_to_peer: " "id %d out of range in request 0x%p", rxreq->id, (void *)rxreq); #endif /* XNB_DEBUG */ /* Assign a pfn and map the new page at the allocated va. */ pfn = xen_assign_pfn(mfn); hat_devload(kas.a_hat, xnbp->xnb_rx_va, PAGESIZE, pfn, PROT_READ | PROT_WRITE, HAT_LOAD); /* 3 */ len = 0; valoop = xnbp->xnb_rx_va; for (ml = mp; ml != NULL; ml = ml->b_cont) { size_t chunk = ml->b_wptr - ml->b_rptr; bcopy(ml->b_rptr, valoop, chunk); valoop += chunk; len += chunk; } ASSERT(len < PAGESIZE); /* Release the pfn. */ hat_unload(kas.a_hat, xnbp->xnb_rx_va, PAGESIZE, HAT_UNLOAD_UNMAP); xen_release_pfn(pfn); /* 4 */ gop->mfn = mfn; gop->domid = xnbp->xnb_peer; gop->ref = rxreq->gref; /* 5.1 */ rxresp = RING_GET_RESPONSE(&xnbp->xnb_rx_ring, prod); rxresp->offset = 0; rxresp->flags = 0; cksum_flags = xnbp->xnb_flavour->xf_cksum_to_peer(xnbp, mp); if (cksum_flags != 0) xnbp->xnb_stat_rx_cksum_deferred++; rxresp->flags |= cksum_flags; rxresp->id = RING_GET_REQUEST(&xnbp->xnb_rx_ring, prod)->id; rxresp->status = len; loop++; prod++; gop++; prev = mp; mp = mp->b_next; } /* * Did we actually do anything? */ if (loop == xnbp->xnb_rx_ring.req_cons) { mutex_exit(&xnbp->xnb_rx_lock); return (mp); } end = loop; /* * Unlink the end of the 'done' list from the remainder. */ ASSERT(prev != NULL); prev->b_next = NULL; if (HYPERVISOR_grant_table_op(GNTTABOP_transfer, xnbp->xnb_rx_top, loop - xnbp->xnb_rx_ring.req_cons) != 0) { cmn_err(CE_WARN, "xnb_to_peer: transfer operation failed"); } loop = xnbp->xnb_rx_ring.req_cons; prod = xnbp->xnb_rx_ring.rsp_prod_pvt; gop = xnbp->xnb_rx_top; while (loop < end) { int16_t status = NETIF_RSP_OKAY; if (gop->status != 0) { status = NETIF_RSP_ERROR; /* * If the status is anything other than * GNTST_bad_page then we don't own the page * any more, so don't try to give it back. */ if (gop->status != GNTST_bad_page) gop->mfn = 0; } else { /* The page is no longer ours. */ gop->mfn = 0; } if (gop->mfn != 0) /* * Give back the page, as we won't be using * it. */ xnb_free_page(xnbp, gop->mfn); else /* * We gave away a page, update our accounting * now. */ balloon_drv_subtracted(1); /* 5.2 */ if (status != NETIF_RSP_OKAY) { RING_GET_RESPONSE(&xnbp->xnb_rx_ring, prod)->status = status; } else { xnbp->xnb_stat_ipackets++; xnbp->xnb_stat_rbytes += len; } loop++; prod++; gop++; } xnbp->xnb_rx_ring.req_cons = loop; xnbp->xnb_rx_ring.rsp_prod_pvt = prod; /* 6 */ /* LINTED: constant in conditional context */ RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(&xnbp->xnb_rx_ring, notify); if (notify) { ec_notify_via_evtchn(xnbp->xnb_evtchn); xnbp->xnb_stat_rx_notify_sent++; } else { xnbp->xnb_stat_rx_notify_deferred++; } if (mp != NULL) xnbp->xnb_stat_rx_defer++; mutex_exit(&xnbp->xnb_rx_lock); /* Free mblk_t's that we consumed. */ freemsgchain(free); return (mp); } /* Helper functions for xnb_copy_to_peer(). */ /* * Grow the array of copy operation descriptors. */ static boolean_t grow_cpop_area(xnb_t *xnbp) { size_t count; gnttab_copy_t *new; ASSERT(MUTEX_HELD(&xnbp->xnb_rx_lock)); count = xnbp->xnb_rx_cpop_count + CPOP_DEFCNT; if ((new = kmem_alloc(sizeof (new[0]) * count, KM_NOSLEEP)) == NULL) { xnbp->xnb_stat_other_allocation_failure++; return (B_FALSE); } bcopy(xnbp->xnb_rx_cpop, new, sizeof (xnbp->xnb_rx_cpop[0]) * xnbp->xnb_rx_cpop_count); kmem_free(xnbp->xnb_rx_cpop, sizeof (xnbp->xnb_rx_cpop[0]) * xnbp->xnb_rx_cpop_count); xnbp->xnb_rx_cpop = new; xnbp->xnb_rx_cpop_count = count; xnbp->xnb_stat_rx_cpoparea_grown++; return (B_TRUE); } /* * Check whether an address is on a page that's foreign to this domain. */ static boolean_t is_foreign(void *addr) { pfn_t pfn = hat_getpfnum(kas.a_hat, addr); return ((pfn & PFN_IS_FOREIGN_MFN) == PFN_IS_FOREIGN_MFN); } /* * Insert a newly allocated mblk into a chain, replacing the old one. */ static mblk_t * replace_msg(mblk_t *mp, size_t len, mblk_t *mp_prev, mblk_t *ml_prev) { uint32_t start, stuff, end, value, flags; mblk_t *new_mp; new_mp = copyb(mp); if (new_mp == NULL) { cmn_err(CE_PANIC, "replace_msg: cannot alloc new message" "for %p, len %lu", (void *) mp, len); } mac_hcksum_get(mp, &start, &stuff, &end, &value, &flags); mac_hcksum_set(new_mp, start, stuff, end, value, flags); new_mp->b_next = mp->b_next; new_mp->b_prev = mp->b_prev; new_mp->b_cont = mp->b_cont; /* Make sure we only overwrite pointers to the mblk being replaced. */ if (mp_prev != NULL && mp_prev->b_next == mp) mp_prev->b_next = new_mp; if (ml_prev != NULL && ml_prev->b_cont == mp) ml_prev->b_cont = new_mp; mp->b_next = mp->b_prev = mp->b_cont = NULL; freemsg(mp); return (new_mp); } /* * Set all the fields in a gnttab_copy_t. */ static void setup_gop(xnb_t *xnbp, gnttab_copy_t *gp, uchar_t *rptr, size_t s_off, size_t d_off, size_t len, grant_ref_t d_ref) { ASSERT(xnbp != NULL && gp != NULL); gp->source.offset = s_off; gp->source.u.gmfn = pfn_to_mfn(hat_getpfnum(kas.a_hat, (caddr_t)rptr)); gp->source.domid = DOMID_SELF; gp->len = (uint16_t)len; gp->flags = GNTCOPY_dest_gref; gp->status = 0; gp->dest.u.ref = d_ref; gp->dest.offset = d_off; gp->dest.domid = xnbp->xnb_peer; } /* * Pass packets to the peer using hypervisor copy operations. */ mblk_t * xnb_copy_to_peer(xnb_t *xnbp, mblk_t *mp) { mblk_t *free = mp, *mp_prev = NULL, *saved_mp = mp; mblk_t *ml, *ml_prev; boolean_t notify; RING_IDX loop, prod; int i; /* * If the peer does not pre-post buffers for received packets, * use page flipping to pass packets to it. */ if (!xnbp->xnb_rx_hv_copy) return (xnb_to_peer(xnbp, mp)); /* * For each packet the sequence of operations is: * * 1. get a request slot from the ring. * 2. set up data for hypercall (see NOTE below) * 3. have the hypervisore copy the data * 4. update the request slot. * 5. kick the peer. * * NOTE ad 2. * In order to reduce the number of hypercalls, we prepare * several mblks (mp->b_cont != NULL) for the peer and * perform a single hypercall to transfer them. We also have * to set up a seperate copy operation for every page. * * If we have more than one packet (mp->b_next != NULL), we do * this whole dance repeatedly. */ mutex_enter(&xnbp->xnb_rx_lock); if (!(xnbp->xnb_connected && xnbp->xnb_hotplugged)) { mutex_exit(&xnbp->xnb_rx_lock); DTRACE_PROBE(copy_rx_too_early); xnbp->xnb_stat_rx_too_early++; return (mp); } loop = xnbp->xnb_rx_ring.req_cons; prod = xnbp->xnb_rx_ring.rsp_prod_pvt; while ((mp != NULL) && XNB_RING_HAS_UNCONSUMED_REQUESTS(&xnbp->xnb_rx_ring)) { netif_rx_request_t *rxreq; size_t d_offset, len; int item_count; gnttab_copy_t *gop_cp; netif_rx_response_t *rxresp; uint16_t cksum_flags; int16_t status = NETIF_RSP_OKAY; /* 1 */ rxreq = RING_GET_REQUEST(&xnbp->xnb_rx_ring, loop); #ifdef XNB_DEBUG if (!(rxreq->id < NET_RX_RING_SIZE)) cmn_err(CE_PANIC, "xnb_copy_to_peer: " "id %d out of range in request 0x%p", rxreq->id, (void *)rxreq); #endif /* XNB_DEBUG */ /* 2 */ d_offset = 0; len = 0; item_count = 0; gop_cp = xnbp->xnb_rx_cpop; /* * We walk the b_cont pointers and set up a * gnttab_copy_t for each sub-page chunk in each data * block. */ /* 2a */ for (ml = mp, ml_prev = NULL; ml != NULL; ml = ml->b_cont) { size_t chunk = ml->b_wptr - ml->b_rptr; uchar_t *r_tmp, *rpt_align; size_t r_offset; /* * The hypervisor will not allow us to * reference a foreign page (e.g. one * belonging to another domain) by mfn in the * copy operation. If the data in this mblk is * on such a page we must copy the data into a * local page before initiating the hypervisor * copy operation. */ if (is_foreign(ml->b_rptr) || is_foreign(ml->b_wptr)) { mblk_t *ml_new = replace_msg(ml, chunk, mp_prev, ml_prev); /* We can still use old ml, but not *ml! */ if (free == ml) free = ml_new; if (mp == ml) mp = ml_new; ml = ml_new; xnbp->xnb_stat_rx_foreign_page++; } rpt_align = (uchar_t *)ALIGN2PAGE(ml->b_rptr); r_offset = (uint16_t)(ml->b_rptr - rpt_align); r_tmp = ml->b_rptr; if (d_offset + chunk > PAGESIZE) cmn_err(CE_PANIC, "xnb_copy_to_peer: mp %p " "(svd: %p), ml %p,rpt_alg. %p, d_offset " "(%lu) + chunk (%lu) > PAGESIZE %d!", (void *)mp, (void *)saved_mp, (void *)ml, (void *)rpt_align, d_offset, chunk, (int)PAGESIZE); while (chunk > 0) { size_t part_len; if (item_count == xnbp->xnb_rx_cpop_count) { if (!grow_cpop_area(xnbp)) goto failure; gop_cp = &xnbp->xnb_rx_cpop[item_count]; } /* * If our mblk crosses a page boundary, we need * to do a seperate copy for each page. */ if (r_offset + chunk > PAGESIZE) { part_len = PAGESIZE - r_offset; DTRACE_PROBE3(mblk_page_crossed, (mblk_t *), ml, int, chunk, int, (int)r_offset); xnbp->xnb_stat_rx_pagebndry_crossed++; } else { part_len = chunk; } setup_gop(xnbp, gop_cp, r_tmp, r_offset, d_offset, part_len, rxreq->gref); chunk -= part_len; len += part_len; d_offset += part_len; r_tmp += part_len; /* * The 2nd, 3rd ... last copies will always * start at r_tmp, therefore r_offset is 0. */ r_offset = 0; gop_cp++; item_count++; } ml_prev = ml; DTRACE_PROBE4(mblk_loop_end, (mblk_t *), ml, int, chunk, int, len, int, item_count); } /* 3 */ if (HYPERVISOR_grant_table_op(GNTTABOP_copy, xnbp->xnb_rx_cpop, item_count) != 0) { cmn_err(CE_WARN, "xnb_copy_to_peer: copy op. failed"); DTRACE_PROBE(HV_granttableopfailed); } /* 4 */ rxresp = RING_GET_RESPONSE(&xnbp->xnb_rx_ring, prod); rxresp->offset = 0; rxresp->flags = 0; DTRACE_PROBE4(got_RX_rsp, int, (int)rxresp->id, int, (int)rxresp->offset, int, (int)rxresp->flags, int, (int)rxresp->status); cksum_flags = xnbp->xnb_flavour->xf_cksum_to_peer(xnbp, mp); if (cksum_flags != 0) xnbp->xnb_stat_rx_cksum_deferred++; rxresp->flags |= cksum_flags; rxresp->id = RING_GET_REQUEST(&xnbp->xnb_rx_ring, prod)->id; rxresp->status = len; DTRACE_PROBE4(RX_rsp_set, int, (int)rxresp->id, int, (int)rxresp->offset, int, (int)rxresp->flags, int, (int)rxresp->status); for (i = 0; i < item_count; i++) { if (xnbp->xnb_rx_cpop[i].status != 0) { DTRACE_PROBE2(cpop_status_nonnull, int, (int)xnbp->xnb_rx_cpop[i].status, int, i); status = NETIF_RSP_ERROR; } } /* 5.2 */ if (status != NETIF_RSP_OKAY) { RING_GET_RESPONSE(&xnbp->xnb_rx_ring, prod)->status = status; xnbp->xnb_stat_rx_rsp_notok++; } else { xnbp->xnb_stat_ipackets++; xnbp->xnb_stat_rbytes += len; } loop++; prod++; mp_prev = mp; mp = mp->b_next; } failure: /* * Did we actually do anything? */ if (loop == xnbp->xnb_rx_ring.req_cons) { mutex_exit(&xnbp->xnb_rx_lock); return (mp); } /* * Unlink the end of the 'done' list from the remainder. */ ASSERT(mp_prev != NULL); mp_prev->b_next = NULL; xnbp->xnb_rx_ring.req_cons = loop; xnbp->xnb_rx_ring.rsp_prod_pvt = prod; /* 6 */ /* LINTED: constant in conditional context */ RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(&xnbp->xnb_rx_ring, notify); if (notify) { ec_notify_via_evtchn(xnbp->xnb_evtchn); xnbp->xnb_stat_rx_notify_sent++; } else { xnbp->xnb_stat_rx_notify_deferred++; } if (mp != NULL) xnbp->xnb_stat_rx_defer++; mutex_exit(&xnbp->xnb_rx_lock); /* Free mblk_t structs we have consumed. */ freemsgchain(free); return (mp); } static void xnb_tx_notify_peer(xnb_t *xnbp, boolean_t force) { boolean_t notify; ASSERT(MUTEX_HELD(&xnbp->xnb_tx_lock)); /* LINTED: constant in conditional context */ RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(&xnbp->xnb_tx_ring, notify); if (notify || force) { ec_notify_via_evtchn(xnbp->xnb_evtchn); xnbp->xnb_stat_tx_notify_sent++; } else { xnbp->xnb_stat_tx_notify_deferred++; } } static void xnb_tx_mark_complete(xnb_t *xnbp, RING_IDX id, int16_t status) { RING_IDX i; netif_tx_response_t *txresp; ASSERT(MUTEX_HELD(&xnbp->xnb_tx_lock)); i = xnbp->xnb_tx_ring.rsp_prod_pvt; txresp = RING_GET_RESPONSE(&xnbp->xnb_tx_ring, i); txresp->id = id; txresp->status = status; xnbp->xnb_tx_ring.rsp_prod_pvt = i + 1; /* * Note that we don't push the change to the peer here - that * is the callers responsibility. */ } static void xnb_txbuf_recycle(xnb_txbuf_t *txp) { xnb_t *xnbp = txp->xt_xnbp; kmem_cache_free(xnbp->xnb_tx_buf_cache, txp); xnbp->xnb_tx_buf_outstanding--; } static int xnb_txbuf_constructor(void *buf, void *arg, int kmflag) { _NOTE(ARGUNUSED(kmflag)); xnb_txbuf_t *txp = buf; xnb_t *xnbp = arg; size_t len; ddi_dma_cookie_t dma_cookie; uint_t ncookies; txp->xt_free_rtn.free_func = xnb_txbuf_recycle; txp->xt_free_rtn.free_arg = (caddr_t)txp; txp->xt_xnbp = xnbp; txp->xt_next = NULL; if (ddi_dma_alloc_handle(xnbp->xnb_devinfo, &buf_dma_attr, 0, 0, &txp->xt_dma_handle) != DDI_SUCCESS) goto failure; if (ddi_dma_mem_alloc(txp->xt_dma_handle, PAGESIZE, &data_accattr, DDI_DMA_STREAMING, 0, 0, &txp->xt_buf, &len, &txp->xt_acc_handle) != DDI_SUCCESS) goto failure_1; if (ddi_dma_addr_bind_handle(txp->xt_dma_handle, NULL, txp->xt_buf, len, DDI_DMA_RDWR | DDI_DMA_STREAMING, DDI_DMA_DONTWAIT, 0, &dma_cookie, &ncookies) != DDI_DMA_MAPPED) goto failure_2; ASSERT(ncookies == 1); txp->xt_mfn = xnb_btop(dma_cookie.dmac_laddress); txp->xt_buflen = dma_cookie.dmac_size; DTRACE_PROBE(txbuf_allocated); atomic_inc_32(&xnbp->xnb_tx_buf_count); xnbp->xnb_tx_buf_outstanding++; return (0); failure_2: ddi_dma_mem_free(&txp->xt_acc_handle); failure_1: ddi_dma_free_handle(&txp->xt_dma_handle); failure: return (-1); } static void xnb_txbuf_destructor(void *buf, void *arg) { xnb_txbuf_t *txp = buf; xnb_t *xnbp = arg; (void) ddi_dma_unbind_handle(txp->xt_dma_handle); ddi_dma_mem_free(&txp->xt_acc_handle); ddi_dma_free_handle(&txp->xt_dma_handle); atomic_dec_32(&xnbp->xnb_tx_buf_count); } /* * Take packets from the peer and deliver them onward. */ static mblk_t * xnb_from_peer(xnb_t *xnbp) { RING_IDX start, end, loop; gnttab_copy_t *cop; xnb_txbuf_t **txpp; netif_tx_request_t *txreq; boolean_t work_to_do, need_notify = B_FALSE; mblk_t *head, *tail; int n_data_req, i; ASSERT(MUTEX_HELD(&xnbp->xnb_tx_lock)); head = tail = NULL; around: /* LINTED: constant in conditional context */ RING_FINAL_CHECK_FOR_REQUESTS(&xnbp->xnb_tx_ring, work_to_do); if (!work_to_do) { finished: xnb_tx_notify_peer(xnbp, need_notify); return (head); } start = xnbp->xnb_tx_ring.req_cons; end = xnbp->xnb_tx_ring.sring->req_prod; if ((end - start) > NET_TX_RING_SIZE) { /* * This usually indicates that the frontend driver is * misbehaving, as it's not possible to have more than * NET_TX_RING_SIZE ring elements in play at any one * time. * * We reset the ring pointers to the state declared by * the frontend and try to carry on. */ cmn_err(CE_WARN, "xnb_from_peer: domain %d tried to give us %u " "items in the ring, resetting and trying to recover.", xnbp->xnb_peer, (end - start)); /* LINTED: constant in conditional context */ BACK_RING_ATTACH(&xnbp->xnb_tx_ring, (netif_tx_sring_t *)xnbp->xnb_tx_ring_addr, PAGESIZE); goto around; } loop = start; cop = xnbp->xnb_tx_cop; txpp = xnbp->xnb_tx_bufp; n_data_req = 0; while (loop < end) { static const uint16_t acceptable_flags = NETTXF_csum_blank | NETTXF_data_validated | NETTXF_extra_info; uint16_t unexpected_flags; txreq = RING_GET_REQUEST(&xnbp->xnb_tx_ring, loop); unexpected_flags = txreq->flags & ~acceptable_flags; if (unexpected_flags != 0) { /* * The peer used flag bits that we do not * recognize. */ cmn_err(CE_WARN, "xnb_from_peer: " "unexpected flag bits (0x%x) from peer " "in transmit request", unexpected_flags); xnbp->xnb_stat_tx_unexpected_flags++; /* Mark this entry as failed. */ xnb_tx_mark_complete(xnbp, txreq->id, NETIF_RSP_ERROR); need_notify = B_TRUE; } else if (txreq->flags & NETTXF_extra_info) { struct netif_extra_info *erp; boolean_t status; loop++; /* Consume another slot in the ring. */ ASSERT(loop <= end); erp = (struct netif_extra_info *) RING_GET_REQUEST(&xnbp->xnb_tx_ring, loop); switch (erp->type) { case XEN_NETIF_EXTRA_TYPE_MCAST_ADD: ASSERT(xnbp->xnb_multicast_control); status = xnbp->xnb_flavour->xf_mcast_add(xnbp, &erp->u.mcast.addr); break; case XEN_NETIF_EXTRA_TYPE_MCAST_DEL: ASSERT(xnbp->xnb_multicast_control); status = xnbp->xnb_flavour->xf_mcast_del(xnbp, &erp->u.mcast.addr); break; default: status = B_FALSE; cmn_err(CE_WARN, "xnb_from_peer: " "unknown extra type %d", erp->type); break; } xnb_tx_mark_complete(xnbp, txreq->id, status ? NETIF_RSP_OKAY : NETIF_RSP_ERROR); need_notify = B_TRUE; } else if ((txreq->offset > PAGESIZE) || (txreq->offset + txreq->size > PAGESIZE)) { /* * Peer attempted to refer to data beyond the * end of the granted page. */ cmn_err(CE_WARN, "xnb_from_peer: " "attempt to refer beyond the end of granted " "page in txreq (offset %d, size %d).", txreq->offset, txreq->size); xnbp->xnb_stat_tx_overflow_page++; /* Mark this entry as failed. */ xnb_tx_mark_complete(xnbp, txreq->id, NETIF_RSP_ERROR); need_notify = B_TRUE; } else { xnb_txbuf_t *txp; txp = kmem_cache_alloc(xnbp->xnb_tx_buf_cache, KM_NOSLEEP); if (txp == NULL) break; txp->xt_mblk = desballoc((unsigned char *)txp->xt_buf, txp->xt_buflen, 0, &txp->xt_free_rtn); if (txp->xt_mblk == NULL) { kmem_cache_free(xnbp->xnb_tx_buf_cache, txp); break; } txp->xt_idx = loop; txp->xt_id = txreq->id; cop->source.u.ref = txreq->gref; cop->source.domid = xnbp->xnb_peer; cop->source.offset = txreq->offset; cop->dest.u.gmfn = txp->xt_mfn; cop->dest.domid = DOMID_SELF; cop->dest.offset = 0; cop->len = txreq->size; cop->flags = GNTCOPY_source_gref; cop->status = 0; *txpp = txp; txpp++; cop++; n_data_req++; ASSERT(n_data_req <= NET_TX_RING_SIZE); } loop++; } xnbp->xnb_tx_ring.req_cons = loop; if (n_data_req == 0) goto around; if (HYPERVISOR_grant_table_op(GNTTABOP_copy, xnbp->xnb_tx_cop, n_data_req) != 0) { cmn_err(CE_WARN, "xnb_from_peer: copy operation failed"); txpp = xnbp->xnb_tx_bufp; i = n_data_req; while (i > 0) { kmem_cache_free(xnbp->xnb_tx_buf_cache, *txpp); txpp++; i--; } goto finished; } txpp = xnbp->xnb_tx_bufp; cop = xnbp->xnb_tx_cop; i = n_data_req; while (i > 0) { xnb_txbuf_t *txp = *txpp; txreq = RING_GET_REQUEST(&xnbp->xnb_tx_ring, txp->xt_idx); if (cop->status != 0) { #ifdef XNB_DEBUG cmn_err(CE_WARN, "xnb_from_peer: " "txpp 0x%p failed (%d)", (void *)*txpp, cop->status); #endif /* XNB_DEBUG */ xnb_tx_mark_complete(xnbp, txp->xt_id, NETIF_RSP_ERROR); freemsg(txp->xt_mblk); } else { mblk_t *mp; mp = txp->xt_mblk; mp->b_rptr = mp->b_wptr = (unsigned char *)txp->xt_buf; mp->b_wptr += txreq->size; mp->b_next = NULL; /* * If there are checksum flags, process them * appropriately. */ if ((txreq->flags & (NETTXF_csum_blank | NETTXF_data_validated)) != 0) { mp = xnbp->xnb_flavour->xf_cksum_from_peer(xnbp, mp, txreq->flags); xnbp->xnb_stat_tx_cksum_no_need++; txp->xt_mblk = mp; } if (head == NULL) { ASSERT(tail == NULL); head = mp; } else { ASSERT(tail != NULL); tail->b_next = mp; } tail = mp; xnbp->xnb_stat_opackets++; xnbp->xnb_stat_obytes += txreq->size; xnb_tx_mark_complete(xnbp, txp->xt_id, NETIF_RSP_OKAY); } txpp++; cop++; i--; } goto around; /* NOTREACHED */ } static uint_t xnb_intr(caddr_t arg) { xnb_t *xnbp = (xnb_t *)arg; mblk_t *mp; xnbp->xnb_stat_intr++; mutex_enter(&xnbp->xnb_tx_lock); ASSERT(xnbp->xnb_connected); mp = xnb_from_peer(xnbp); mutex_exit(&xnbp->xnb_tx_lock); if (!xnbp->xnb_hotplugged) { xnbp->xnb_stat_tx_too_early++; goto fail; } if (mp == NULL) { xnbp->xnb_stat_spurious_intr++; goto fail; } xnbp->xnb_flavour->xf_from_peer(xnbp, mp); return (DDI_INTR_CLAIMED); fail: freemsgchain(mp); return (DDI_INTR_CLAIMED); } /* * Read our configuration from xenstore. */ boolean_t xnb_read_xs_config(xnb_t *xnbp) { char *xsname; char mac[ETHERADDRL * 3]; xsname = xvdi_get_xsname(xnbp->xnb_devinfo); if (xenbus_scanf(XBT_NULL, xsname, "mac", "%s", mac) != 0) { cmn_err(CE_WARN, "xnb_attach: " "cannot read mac address from %s", xsname); return (B_FALSE); } if (ether_aton(mac, xnbp->xnb_mac_addr) != ETHERADDRL) { cmn_err(CE_WARN, "xnb_attach: cannot parse mac address %s", mac); return (B_FALSE); } return (B_TRUE); } /* * Read the configuration of the peer from xenstore. */ boolean_t xnb_read_oe_config(xnb_t *xnbp) { char *oename; int i; oename = xvdi_get_oename(xnbp->xnb_devinfo); if (xenbus_gather(XBT_NULL, oename, "event-channel", "%u", &xnbp->xnb_fe_evtchn, "tx-ring-ref", "%lu", &xnbp->xnb_tx_ring_ref, "rx-ring-ref", "%lu", &xnbp->xnb_rx_ring_ref, NULL) != 0) { cmn_err(CE_WARN, "xnb_read_oe_config: " "cannot read other-end details from %s", oename); return (B_FALSE); } /* * Check whether our peer requests receive side hypervisor * copy. */ if (xenbus_scanf(XBT_NULL, oename, "request-rx-copy", "%d", &i) != 0) i = 0; if (i != 0) xnbp->xnb_rx_hv_copy = B_TRUE; /* * Check whether our peer requests multicast_control. */ if (xenbus_scanf(XBT_NULL, oename, "request-multicast-control", "%d", &i) != 0) i = 0; if (i != 0) xnbp->xnb_multicast_control = B_TRUE; /* * The Linux backend driver here checks to see if the peer has * set 'feature-no-csum-offload'. This is used to indicate * that the guest cannot handle receiving packets without a * valid checksum. We don't check here, because packets passed * to the peer _always_ have a valid checksum. * * There are three cases: * * - the NIC is dedicated: packets from the wire should always * have a valid checksum. If the hardware validates the * checksum then the relevant bit will be set in the packet * attributes and we will inform the peer. It can choose to * ignore the hardware verification. * * - the NIC is shared (VNIC) and a packet originates from the * wire: this is the same as the case above - the packets * will have a valid checksum. * * - the NIC is shared (VNIC) and a packet originates from the * host: the MAC layer ensures that all such packets have a * valid checksum by calculating one if the stack did not. */ return (B_TRUE); } void xnb_start_connect(xnb_t *xnbp) { dev_info_t *dip = xnbp->xnb_devinfo; if (!xnb_connect_rings(dip)) { cmn_err(CE_WARN, "xnb_start_connect: " "cannot connect rings"); goto failed; } if (!xnbp->xnb_flavour->xf_start_connect(xnbp)) { cmn_err(CE_WARN, "xnb_start_connect: " "flavour failed to connect"); goto failed; } (void) xvdi_switch_state(dip, XBT_NULL, XenbusStateConnected); return; failed: xnbp->xnb_flavour->xf_peer_disconnected(xnbp); xnb_disconnect_rings(dip); (void) xvdi_switch_state(dip, XBT_NULL, XenbusStateClosed); (void) xvdi_post_event(dip, XEN_HP_REMOVE); } static boolean_t xnb_connect_rings(dev_info_t *dip) { xnb_t *xnbp = ddi_get_driver_private(dip); struct gnttab_map_grant_ref map_op; /* * Cannot attempt to connect the rings if already connected. */ ASSERT(!xnbp->xnb_connected); /* * 1. allocate a vaddr for the tx page, one for the rx page. * 2. call GNTTABOP_map_grant_ref to map the relevant pages * into the allocated vaddr (one for tx, one for rx). * 3. call EVTCHNOP_bind_interdomain to have the event channel * bound to this domain. * 4. associate the event channel with an interrupt. * 5. enable the interrupt. */ /* 1.tx */ xnbp->xnb_tx_ring_addr = vmem_xalloc(heap_arena, PAGESIZE, PAGESIZE, 0, 0, 0, 0, VM_SLEEP); ASSERT(xnbp->xnb_tx_ring_addr != NULL); /* 2.tx */ map_op.host_addr = (uint64_t)((long)xnbp->xnb_tx_ring_addr); map_op.flags = GNTMAP_host_map; map_op.ref = xnbp->xnb_tx_ring_ref; map_op.dom = xnbp->xnb_peer; hat_prepare_mapping(kas.a_hat, xnbp->xnb_tx_ring_addr, NULL); if (xen_map_gref(GNTTABOP_map_grant_ref, &map_op, 1, B_FALSE) != 0 || map_op.status != 0) { cmn_err(CE_WARN, "xnb_connect_rings: cannot map tx-ring page."); goto fail; } xnbp->xnb_tx_ring_handle = map_op.handle; /* LINTED: constant in conditional context */ BACK_RING_INIT(&xnbp->xnb_tx_ring, (netif_tx_sring_t *)xnbp->xnb_tx_ring_addr, PAGESIZE); /* 1.rx */ xnbp->xnb_rx_ring_addr = vmem_xalloc(heap_arena, PAGESIZE, PAGESIZE, 0, 0, 0, 0, VM_SLEEP); ASSERT(xnbp->xnb_rx_ring_addr != NULL); /* 2.rx */ map_op.host_addr = (uint64_t)((long)xnbp->xnb_rx_ring_addr); map_op.flags = GNTMAP_host_map; map_op.ref = xnbp->xnb_rx_ring_ref; map_op.dom = xnbp->xnb_peer; hat_prepare_mapping(kas.a_hat, xnbp->xnb_rx_ring_addr, NULL); if (xen_map_gref(GNTTABOP_map_grant_ref, &map_op, 1, B_FALSE) != 0 || map_op.status != 0) { cmn_err(CE_WARN, "xnb_connect_rings: cannot map rx-ring page."); goto fail; } xnbp->xnb_rx_ring_handle = map_op.handle; /* LINTED: constant in conditional context */ BACK_RING_INIT(&xnbp->xnb_rx_ring, (netif_rx_sring_t *)xnbp->xnb_rx_ring_addr, PAGESIZE); /* 3 */ if (xvdi_bind_evtchn(dip, xnbp->xnb_fe_evtchn) != DDI_SUCCESS) { cmn_err(CE_WARN, "xnb_connect_rings: " "cannot bind event channel %d", xnbp->xnb_evtchn); xnbp->xnb_evtchn = INVALID_EVTCHN; goto fail; } xnbp->xnb_evtchn = xvdi_get_evtchn(dip); /* * It would be good to set the state to XenbusStateConnected * here as well, but then what if ddi_add_intr() failed? * Changing the state in the store will be noticed by the peer * and cannot be "taken back". */ mutex_enter(&xnbp->xnb_tx_lock); mutex_enter(&xnbp->xnb_rx_lock); xnbp->xnb_connected = B_TRUE; mutex_exit(&xnbp->xnb_rx_lock); mutex_exit(&xnbp->xnb_tx_lock); /* 4, 5 */ if (ddi_add_intr(dip, 0, NULL, NULL, xnb_intr, (caddr_t)xnbp) != DDI_SUCCESS) { cmn_err(CE_WARN, "xnb_connect_rings: cannot add interrupt"); goto fail; } xnbp->xnb_irq = B_TRUE; return (B_TRUE); fail: mutex_enter(&xnbp->xnb_tx_lock); mutex_enter(&xnbp->xnb_rx_lock); xnbp->xnb_connected = B_FALSE; mutex_exit(&xnbp->xnb_rx_lock); mutex_exit(&xnbp->xnb_tx_lock); return (B_FALSE); } static void xnb_disconnect_rings(dev_info_t *dip) { xnb_t *xnbp = ddi_get_driver_private(dip); if (xnbp->xnb_irq) { ddi_remove_intr(dip, 0, NULL); xnbp->xnb_irq = B_FALSE; } if (xnbp->xnb_evtchn != INVALID_EVTCHN) { xvdi_free_evtchn(dip); xnbp->xnb_evtchn = INVALID_EVTCHN; } if (xnbp->xnb_rx_ring_handle != INVALID_GRANT_HANDLE) { struct gnttab_unmap_grant_ref unmap_op; unmap_op.host_addr = (uint64_t)(uintptr_t) xnbp->xnb_rx_ring_addr; unmap_op.dev_bus_addr = 0; unmap_op.handle = xnbp->xnb_rx_ring_handle; if (HYPERVISOR_grant_table_op(GNTTABOP_unmap_grant_ref, &unmap_op, 1) != 0) cmn_err(CE_WARN, "xnb_disconnect_rings: " "cannot unmap rx-ring page (%d)", unmap_op.status); xnbp->xnb_rx_ring_handle = INVALID_GRANT_HANDLE; } if (xnbp->xnb_rx_ring_addr != NULL) { hat_release_mapping(kas.a_hat, xnbp->xnb_rx_ring_addr); vmem_free(heap_arena, xnbp->xnb_rx_ring_addr, PAGESIZE); xnbp->xnb_rx_ring_addr = NULL; } if (xnbp->xnb_tx_ring_handle != INVALID_GRANT_HANDLE) { struct gnttab_unmap_grant_ref unmap_op; unmap_op.host_addr = (uint64_t)(uintptr_t) xnbp->xnb_tx_ring_addr; unmap_op.dev_bus_addr = 0; unmap_op.handle = xnbp->xnb_tx_ring_handle; if (HYPERVISOR_grant_table_op(GNTTABOP_unmap_grant_ref, &unmap_op, 1) != 0) cmn_err(CE_WARN, "xnb_disconnect_rings: " "cannot unmap tx-ring page (%d)", unmap_op.status); xnbp->xnb_tx_ring_handle = INVALID_GRANT_HANDLE; } if (xnbp->xnb_tx_ring_addr != NULL) { hat_release_mapping(kas.a_hat, xnbp->xnb_tx_ring_addr); vmem_free(heap_arena, xnbp->xnb_tx_ring_addr, PAGESIZE); xnbp->xnb_tx_ring_addr = NULL; } } static void xnb_oe_state_change(dev_info_t *dip, ddi_eventcookie_t id, void *arg, void *impl_data) { _NOTE(ARGUNUSED(id, arg)); xnb_t *xnbp = ddi_get_driver_private(dip); XenbusState new_state = *(XenbusState *)impl_data; ASSERT(xnbp != NULL); switch (new_state) { case XenbusStateConnected: /* spurious state change */ if (xnbp->xnb_connected) return; if (!xnb_read_oe_config(xnbp) || !xnbp->xnb_flavour->xf_peer_connected(xnbp)) { cmn_err(CE_WARN, "xnb_oe_state_change: " "read otherend config error"); (void) xvdi_switch_state(dip, XBT_NULL, XenbusStateClosed); (void) xvdi_post_event(dip, XEN_HP_REMOVE); break; } mutex_enter(&xnbp->xnb_state_lock); xnbp->xnb_fe_status = XNB_STATE_READY; if (xnbp->xnb_be_status == XNB_STATE_READY) xnb_start_connect(xnbp); mutex_exit(&xnbp->xnb_state_lock); /* * Now that we've attempted to connect it's reasonable * to allow an attempt to detach. */ xnbp->xnb_detachable = B_TRUE; break; case XenbusStateClosing: (void) xvdi_switch_state(dip, XBT_NULL, XenbusStateClosing); break; case XenbusStateClosed: xnbp->xnb_flavour->xf_peer_disconnected(xnbp); mutex_enter(&xnbp->xnb_tx_lock); mutex_enter(&xnbp->xnb_rx_lock); xnb_disconnect_rings(dip); xnbp->xnb_connected = B_FALSE; mutex_exit(&xnbp->xnb_rx_lock); mutex_exit(&xnbp->xnb_tx_lock); (void) xvdi_switch_state(dip, XBT_NULL, XenbusStateClosed); (void) xvdi_post_event(dip, XEN_HP_REMOVE); /* * In all likelyhood this is already set (in the above * case), but if the peer never attempted to connect * and the domain is destroyed we get here without * having been through the case above, so we set it to * be sure. */ xnbp->xnb_detachable = B_TRUE; break; default: break; } } static void xnb_hp_state_change(dev_info_t *dip, ddi_eventcookie_t id, void *arg, void *impl_data) { _NOTE(ARGUNUSED(id, arg)); xnb_t *xnbp = ddi_get_driver_private(dip); xendev_hotplug_state_t state = *(xendev_hotplug_state_t *)impl_data; ASSERT(xnbp != NULL); switch (state) { case Connected: /* spurious hotplug event */ if (xnbp->xnb_hotplugged) break; if (!xnb_read_xs_config(xnbp)) break; if (!xnbp->xnb_flavour->xf_hotplug_connected(xnbp)) break; mutex_enter(&xnbp->xnb_tx_lock); mutex_enter(&xnbp->xnb_rx_lock); xnbp->xnb_hotplugged = B_TRUE; mutex_exit(&xnbp->xnb_rx_lock); mutex_exit(&xnbp->xnb_tx_lock); mutex_enter(&xnbp->xnb_state_lock); xnbp->xnb_be_status = XNB_STATE_READY; if (xnbp->xnb_fe_status == XNB_STATE_READY) xnb_start_connect(xnbp); mutex_exit(&xnbp->xnb_state_lock); break; default: break; } } static struct modldrv modldrv = { &mod_miscops, "xnb", }; static struct modlinkage modlinkage = { MODREV_1, &modldrv, NULL }; int _init(void) { int i; mutex_init(&xnb_alloc_page_lock, NULL, MUTEX_DRIVER, NULL); i = mod_install(&modlinkage); if (i != DDI_SUCCESS) mutex_destroy(&xnb_alloc_page_lock); return (i); } int _info(struct modinfo *modinfop) { return (mod_info(&modlinkage, modinfop)); } int _fini(void) { int i; i = mod_remove(&modlinkage); if (i == DDI_SUCCESS) mutex_destroy(&xnb_alloc_page_lock); return (i); } /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2010 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. * * xnb.h - definitions for Xen dom0 network driver */ #ifndef _SYS_XNB_H #define _SYS_XNB_H #include #include #include #include #include #include #include #ifdef __cplusplus extern "C" { #endif #define NET_TX_RING_SIZE __CONST_RING_SIZE(netif_tx, PAGESIZE) #define NET_RX_RING_SIZE __CONST_RING_SIZE(netif_rx, PAGESIZE) #define XNBMAXPKT 1500 /* MTU size */ /* DEBUG flags */ #define XNBDDI 0x01 #define XNBTRACE 0x02 #define XNBSEND 0x04 #define XNBRECV 0x08 #define XNBINTR 0x10 #define XNBRING 0x20 #define XNBCKSUM 0x40 #define XNB_STATE_INIT 0x01 #define XNB_STATE_READY 0x02 typedef struct xnb xnb_t; /* * The xnb module provides core inter-domain network protocol functionality. * It is connected to the rest of Solaris in two ways: * - as a GLDv3 driver (with xnbu), * - as a GLDv3 consumer (with xnbo). * * The different modes of operation are termed "flavours" and each * instance of an xnb based driver operates in one and only one mode. * The common xnb driver exports a set of functions to these drivers * (declarations at the foot of this file) and calls back into the * drivers via the xnb_flavour_t structure. */ typedef struct xnb_flavour { void (*xf_from_peer)(xnb_t *, mblk_t *); boolean_t (*xf_peer_connected)(xnb_t *); void (*xf_peer_disconnected)(xnb_t *); boolean_t (*xf_hotplug_connected)(xnb_t *); boolean_t (*xf_start_connect)(xnb_t *); mblk_t *(*xf_cksum_from_peer)(xnb_t *, mblk_t *, uint16_t); uint16_t (*xf_cksum_to_peer)(xnb_t *, mblk_t *); boolean_t (*xf_mcast_add)(xnb_t *, ether_addr_t *); boolean_t (*xf_mcast_del)(xnb_t *, ether_addr_t *); } xnb_flavour_t; typedef struct xnb_txbuf { frtn_t xt_free_rtn; xnb_t *xt_xnbp; struct xnb_txbuf *xt_next; RING_IDX xt_id; RING_IDX xt_idx; uint16_t xt_status; ddi_dma_handle_t xt_dma_handle; ddi_acc_handle_t xt_acc_handle; caddr_t xt_buf; size_t xt_buflen; mfn_t xt_mfn; mblk_t *xt_mblk; unsigned int xt_flags; #define XNB_TXBUF_INUSE 0x01 } xnb_txbuf_t; /* Per network-interface-controller driver private structure */ struct xnb { /* most interesting stuff first to assist debugging */ dev_info_t *xnb_devinfo; /* System per-device info. */ xnb_flavour_t *xnb_flavour; void *xnb_flavour_data; boolean_t xnb_irq; unsigned char xnb_mac_addr[ETHERADDRL]; uint64_t xnb_stat_ipackets; uint64_t xnb_stat_opackets; uint64_t xnb_stat_rbytes; uint64_t xnb_stat_obytes; uint64_t xnb_stat_intr; uint64_t xnb_stat_rx_defer; uint64_t xnb_stat_rx_cksum_deferred; uint64_t xnb_stat_tx_cksum_no_need; uint64_t xnb_stat_rx_rsp_notok; uint64_t xnb_stat_tx_notify_sent; uint64_t xnb_stat_tx_notify_deferred; uint64_t xnb_stat_rx_notify_sent; uint64_t xnb_stat_rx_notify_deferred; uint64_t xnb_stat_tx_too_early; uint64_t xnb_stat_rx_too_early; uint64_t xnb_stat_rx_allocb_failed; uint64_t xnb_stat_tx_allocb_failed; uint64_t xnb_stat_rx_foreign_page; uint64_t xnb_stat_tx_overflow_page; uint64_t xnb_stat_tx_unexpected_flags; uint64_t xnb_stat_mac_full; uint64_t xnb_stat_spurious_intr; uint64_t xnb_stat_allocation_success; uint64_t xnb_stat_allocation_failure; uint64_t xnb_stat_small_allocation_success; uint64_t xnb_stat_small_allocation_failure; uint64_t xnb_stat_other_allocation_failure; uint64_t xnb_stat_rx_pagebndry_crossed; uint64_t xnb_stat_rx_cpoparea_grown; uint64_t xnb_stat_csum_hardware; uint64_t xnb_stat_csum_software; kstat_t *xnb_kstat_aux; ddi_iblock_cookie_t xnb_icookie; kmutex_t xnb_rx_lock; kmutex_t xnb_tx_lock; kmutex_t xnb_state_lock; int xnb_be_status; int xnb_fe_status; kmem_cache_t *xnb_tx_buf_cache; uint32_t xnb_tx_buf_count; int xnb_tx_buf_outstanding; netif_rx_back_ring_t xnb_rx_ring; /* rx interface struct ptr */ void *xnb_rx_ring_addr; grant_ref_t xnb_rx_ring_ref; grant_handle_t xnb_rx_ring_handle; netif_tx_back_ring_t xnb_tx_ring; /* tx interface struct ptr */ void *xnb_tx_ring_addr; grant_ref_t xnb_tx_ring_ref; grant_handle_t xnb_tx_ring_handle; boolean_t xnb_connected; boolean_t xnb_hotplugged; boolean_t xnb_detachable; int xnb_evtchn; /* channel to front end */ evtchn_port_t xnb_fe_evtchn; domid_t xnb_peer; xnb_txbuf_t *xnb_tx_bufp[NET_TX_RING_SIZE]; gnttab_copy_t xnb_tx_cop[NET_TX_RING_SIZE]; caddr_t xnb_rx_va; gnttab_transfer_t xnb_rx_top[NET_RX_RING_SIZE]; boolean_t xnb_rx_hv_copy; boolean_t xnb_multicast_control; boolean_t xnb_no_csum_offload; gnttab_copy_t *xnb_rx_cpop; #define CPOP_DEFCNT 8 size_t xnb_rx_cpop_count; /* in elements */ }; extern int xnb_attach(dev_info_t *, xnb_flavour_t *, void *); extern void xnb_detach(dev_info_t *); extern mblk_t *xnb_copy_to_peer(xnb_t *, mblk_t *); extern mblk_t *xnb_process_cksum_flags(xnb_t *, mblk_t *, uint32_t); #ifdef __cplusplus } #endif #endif /* _SYS_XNB_H */ /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2008 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ /* * Xen network backend - ioemu version. * * HVM guest domains use an emulated network device (typically the * rtl8139) to access the physical network via IO emulation running in * a backend domain (generally domain 0). * * The IO emulation code sends and receives packets using DLPI, usually * through a virtual NIC (vnic). * * The creation of the relevant vnic to correspond to the network interface * in the guest domain requires the use of 'hotplug' scripts in the backend * domain. This driver ensures that the hotplug scripts are run when * such guest domains are created. * * It is used as a result of the 'compatible' property associated with * IO emulated devices. See /etc/driver_aliases and common/xen/os/xvdi.c. */ #ifdef DEBUG #define XNBE_DEBUG 1 #endif /* DEBUG */ #include #include #include #include #include #ifdef XNBE_DEBUG #include #endif /* XNBE_DEBUG */ #ifdef XNBE_DEBUG int xnbe_debug = 0; #endif /* XNBE_DEBUG */ static int xnbe_attach(dev_info_t *dip, ddi_attach_cmd_t cmd) { #ifdef XNBE_DEBUG if (xnbe_debug > 0) cmn_err(CE_NOTE, "xnbe_attach: dip 0x%p, cmd %d", (void *)dip, cmd); #endif /* XNBE_DEBUG */ switch (cmd) { case DDI_ATTACH: break; case DDI_RESUME: return (DDI_SUCCESS); default: return (DDI_FAILURE); } (void) xvdi_post_event(dip, XEN_HP_ADD); return (DDI_SUCCESS); } static int xnbe_detach(dev_info_t *dip, ddi_detach_cmd_t cmd) { #ifdef XNBE_DEBUG if (xnbe_debug > 0) cmn_err(CE_NOTE, "detach: dip 0x%p, cmd %d", (void *)dip, cmd); #endif /* XNBE_DEBUG */ switch (cmd) { case DDI_DETACH: return (DDI_SUCCESS); case DDI_SUSPEND: return (DDI_SUCCESS); default: return (DDI_FAILURE); } } static struct cb_ops cb_ops = { nulldev, /* open */ nulldev, /* close */ nodev, /* strategy */ nodev, /* print */ nodev, /* dump */ nodev, /* read */ nodev, /* write */ nodev, /* ioctl */ nodev, /* devmap */ nodev, /* mmap */ nodev, /* segmap */ nochpoll, /* poll */ ddi_prop_op, /* cb_prop_op */ 0, /* streamtab */ D_NEW | D_MP | D_64BIT /* Driver compatibility flag */ }; static struct dev_ops ops = { DEVO_REV, /* devo_rev */ 0, /* devo_refcnt */ nulldev, /* devo_getinfo */ nulldev, /* devo_identify */ nulldev, /* devo_probe */ xnbe_attach, /* devo_attach */ xnbe_detach, /* devo_detach */ nodev, /* devo_reset */ &cb_ops, /* devo_cb_ops */ (struct bus_ops *)0, /* devo_bus_ops */ NULL, /* devo_power */ ddi_quiesce_not_needed, /* devo_quiesce */ }; static struct modldrv modldrv = { &mod_driverops, "xnbe driver", &ops, }; static struct modlinkage modlinkage = { MODREV_1, &modldrv, NULL }; int _init(void) { return (mod_install(&modlinkage)); } int _info(struct modinfo *modinfop) { return (mod_info(&modlinkage, modinfop)); } int _fini(void) { return (mod_remove(&modlinkage)); } /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2010 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ /* * Xen network backend - mac client edition. * * A driver that sits above an existing GLDv3/Nemo MAC driver and * relays packets to/from that driver from/to a guest domain. */ #ifdef DEBUG #define XNBO_DEBUG 1 #endif /* DEBUG */ #include "xnb.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef XNBO_DEBUG boolean_t xnbo_cksum_offload_to_peer = B_TRUE; boolean_t xnbo_cksum_offload_from_peer = B_TRUE; #endif /* XNBO_DEBUG */ /* Track multicast addresses. */ typedef struct xmca { struct xmca *next; ether_addr_t addr; } xmca_t; /* State about this device instance. */ typedef struct xnbo { mac_handle_t o_mh; mac_client_handle_t o_mch; mac_unicast_handle_t o_mah; mac_promisc_handle_t o_mphp; boolean_t o_running; boolean_t o_promiscuous; uint32_t o_hcksum_capab; xmca_t *o_mca; char o_link_name[LIFNAMSIZ]; boolean_t o_need_rx_filter; boolean_t o_need_setphysaddr; boolean_t o_multicast_control; } xnbo_t; static void xnbo_close_mac(xnb_t *); static void i_xnbo_close_mac(xnb_t *, boolean_t); /* * Packets from the peer come here. We pass them to the mac device. */ static void xnbo_to_mac(xnb_t *xnbp, mblk_t *mp) { xnbo_t *xnbop = xnbp->xnb_flavour_data; ASSERT(mp != NULL); if (!xnbop->o_running) { xnbp->xnb_stat_tx_too_early++; goto fail; } if (mac_tx(xnbop->o_mch, mp, 0, MAC_DROP_ON_NO_DESC, NULL) != (mac_tx_cookie_t)NULL) { xnbp->xnb_stat_mac_full++; } return; fail: freemsgchain(mp); } /* * Process the checksum flags `flags' provided by the peer for the * packet `mp'. */ static mblk_t * xnbo_cksum_from_peer(xnb_t *xnbp, mblk_t *mp, uint16_t flags) { xnbo_t *xnbop = xnbp->xnb_flavour_data; ASSERT(mp->b_next == NULL); if ((flags & NETTXF_csum_blank) != 0) { uint32_t capab = xnbop->o_hcksum_capab; #ifdef XNBO_DEBUG if (!xnbo_cksum_offload_from_peer) capab = 0; #endif /* XNBO_DEBUG */ /* * The checksum in the packet is blank. Determine * whether we can do hardware offload and, if so, * update the flags on the mblk according. If not, * calculate and insert the checksum using software. */ mp = xnb_process_cksum_flags(xnbp, mp, capab); } return (mp); } /* * Calculate the checksum flags to be relayed to the peer for the * packet `mp'. */ static uint16_t xnbo_cksum_to_peer(xnb_t *xnbp, mblk_t *mp) { _NOTE(ARGUNUSED(xnbp)); uint16_t r = 0; uint32_t pflags, csum; #ifdef XNBO_DEBUG if (!xnbo_cksum_offload_to_peer) return (0); #endif /* XNBO_DEBUG */ /* * We might also check for HCK_PARTIALCKSUM here and, * providing that the partial checksum covers the TCP/UDP * payload, return NETRXF_data_validated. * * It seems that it's probably not worthwhile, as even MAC * devices which advertise HCKSUM_INET_PARTIAL in their * capabilities tend to use HCK_FULLCKSUM on the receive side * - they are actually saying that in the output path the * caller must use HCK_PARTIALCKSUM. * * Then again, if a NIC supports HCK_PARTIALCKSUM in its' * output path, the host IP stack will use it. If such packets * are destined for the peer (i.e. looped around) we would * gain some advantage. */ mac_hcksum_get(mp, NULL, NULL, NULL, &csum, &pflags); /* * If the MAC driver has asserted that the checksum is * good, let the peer know. */ if (((pflags & HCK_FULLCKSUM) != 0) && (((pflags & HCK_FULLCKSUM_OK) != 0) || (csum == 0xffff))) r |= NETRXF_data_validated; return (r); } /* * Packets from the mac device come here. We pass them to the peer. */ /*ARGSUSED*/ static void xnbo_from_mac(void *arg, mac_resource_handle_t mrh, mblk_t *mp, boolean_t loopback) { xnb_t *xnbp = arg; mp = xnb_copy_to_peer(xnbp, mp); if (mp != NULL) freemsgchain(mp); } /* * Packets from the mac device come here. We pass them to the peer if * the destination mac address matches or it's a multicast/broadcast * address. */ static void xnbo_from_mac_filter(void *arg, mac_resource_handle_t mrh, mblk_t *mp, boolean_t loopback) { _NOTE(ARGUNUSED(loopback)); xnb_t *xnbp = arg; xnbo_t *xnbop = xnbp->xnb_flavour_data; mblk_t *next, *keep, *keep_head, *free, *free_head; keep = keep_head = free = free_head = NULL; #define ADD(list, bp) \ if (list != NULL) \ list->b_next = bp; \ else \ list##_head = bp; \ list = bp; for (; mp != NULL; mp = next) { mac_header_info_t hdr_info; next = mp->b_next; mp->b_next = NULL; if (mac_header_info(xnbop->o_mh, mp, &hdr_info) != 0) { ADD(free, mp); continue; } if ((hdr_info.mhi_dsttype == MAC_ADDRTYPE_BROADCAST) || (hdr_info.mhi_dsttype == MAC_ADDRTYPE_MULTICAST)) { ADD(keep, mp); continue; } if (bcmp(hdr_info.mhi_daddr, xnbp->xnb_mac_addr, sizeof (xnbp->xnb_mac_addr)) == 0) { ADD(keep, mp); continue; } ADD(free, mp); } #undef ADD if (keep_head != NULL) xnbo_from_mac(xnbp, mrh, keep_head, B_FALSE); if (free_head != NULL) freemsgchain(free_head); } static boolean_t xnbo_open_mac(xnb_t *xnbp, char *mac) { xnbo_t *xnbop = xnbp->xnb_flavour_data; int err; const mac_info_t *mi; void (*rx_fn)(void *, mac_resource_handle_t, mblk_t *, boolean_t); struct ether_addr ea; uint_t max_sdu; mac_diag_t diag; if ((err = mac_open_by_linkname(mac, &xnbop->o_mh)) != 0) { cmn_err(CE_WARN, "xnbo_open_mac: " "cannot open mac for link %s (%d)", mac, err); return (B_FALSE); } ASSERT(xnbop->o_mh != NULL); mi = mac_info(xnbop->o_mh); ASSERT(mi != NULL); if (mi->mi_media != DL_ETHER) { cmn_err(CE_WARN, "xnbo_open_mac: " "device is not DL_ETHER (%d)", mi->mi_media); i_xnbo_close_mac(xnbp, B_TRUE); return (B_FALSE); } if (mi->mi_media != mi->mi_nativemedia) { cmn_err(CE_WARN, "xnbo_open_mac: " "device media and native media mismatch (%d != %d)", mi->mi_media, mi->mi_nativemedia); i_xnbo_close_mac(xnbp, B_TRUE); return (B_FALSE); } mac_sdu_get(xnbop->o_mh, NULL, &max_sdu); if (max_sdu > XNBMAXPKT) { cmn_err(CE_WARN, "xnbo_open_mac: mac device SDU too big (%d)", max_sdu); i_xnbo_close_mac(xnbp, B_TRUE); return (B_FALSE); } /* * MAC_OPEN_FLAGS_MULTI_PRIMARY is relevant when we are migrating a * guest on the localhost itself. In this case we would have the MAC * client open for the guest being migrated *and* also for the * migrated guest (i.e. the former will be active till the migration * is complete when the latter will be activated). This flag states * that it is OK for mac_unicast_add to add the primary MAC unicast * address multiple times. */ if (mac_client_open(xnbop->o_mh, &xnbop->o_mch, NULL, MAC_OPEN_FLAGS_USE_DATALINK_NAME | MAC_OPEN_FLAGS_MULTI_PRIMARY) != 0) { cmn_err(CE_WARN, "xnbo_open_mac: " "error (%d) opening mac client", err); i_xnbo_close_mac(xnbp, B_TRUE); return (B_FALSE); } if (xnbop->o_need_rx_filter) rx_fn = xnbo_from_mac_filter; else rx_fn = xnbo_from_mac; err = mac_unicast_add_set_rx(xnbop->o_mch, NULL, MAC_UNICAST_PRIMARY, &xnbop->o_mah, 0, &diag, xnbop->o_multicast_control ? rx_fn : NULL, xnbp); if (err != 0) { cmn_err(CE_WARN, "xnbo_open_mac: failed to get the primary " "MAC address of %s: %d", mac, err); i_xnbo_close_mac(xnbp, B_TRUE); return (B_FALSE); } if (!xnbop->o_multicast_control) { err = mac_promisc_add(xnbop->o_mch, MAC_CLIENT_PROMISC_ALL, rx_fn, xnbp, &xnbop->o_mphp, MAC_PROMISC_FLAGS_NO_TX_LOOP | MAC_PROMISC_FLAGS_VLAN_TAG_STRIP); if (err != 0) { cmn_err(CE_WARN, "xnbo_open_mac: " "cannot enable promiscuous mode of %s: %d", mac, err); i_xnbo_close_mac(xnbp, B_TRUE); return (B_FALSE); } xnbop->o_promiscuous = B_TRUE; } if (xnbop->o_need_setphysaddr) { err = mac_unicast_primary_set(xnbop->o_mh, xnbp->xnb_mac_addr); /* Warn, but continue on. */ if (err != 0) { bcopy(xnbp->xnb_mac_addr, ea.ether_addr_octet, ETHERADDRL); cmn_err(CE_WARN, "xnbo_open_mac: " "cannot set MAC address of %s to " "%s: %d", mac, ether_sprintf(&ea), err); } } if (!mac_capab_get(xnbop->o_mh, MAC_CAPAB_HCKSUM, &xnbop->o_hcksum_capab)) xnbop->o_hcksum_capab = 0; xnbop->o_running = B_TRUE; return (B_TRUE); } static void xnbo_close_mac(xnb_t *xnbp) { i_xnbo_close_mac(xnbp, B_FALSE); } static void i_xnbo_close_mac(xnb_t *xnbp, boolean_t locked) { xnbo_t *xnbop = xnbp->xnb_flavour_data; xmca_t *loop; ASSERT(!locked || MUTEX_HELD(&xnbp->xnb_state_lock)); if (xnbop->o_mh == NULL) return; if (xnbop->o_running) xnbop->o_running = B_FALSE; if (!locked) mutex_enter(&xnbp->xnb_state_lock); loop = xnbop->o_mca; xnbop->o_mca = NULL; if (!locked) mutex_exit(&xnbp->xnb_state_lock); while (loop != NULL) { xmca_t *next = loop->next; DTRACE_PROBE3(mcast_remove, (char *), "close", (void *), xnbp, (etheraddr_t *), loop->addr); (void) mac_multicast_remove(xnbop->o_mch, loop->addr); kmem_free(loop, sizeof (*loop)); loop = next; } if (xnbop->o_promiscuous) { if (xnbop->o_mphp != NULL) { mac_promisc_remove(xnbop->o_mphp); xnbop->o_mphp = NULL; } xnbop->o_promiscuous = B_FALSE; } else { if (xnbop->o_mch != NULL) mac_rx_clear(xnbop->o_mch); } if (xnbop->o_mah != NULL) { (void) mac_unicast_remove(xnbop->o_mch, xnbop->o_mah); xnbop->o_mah = NULL; } if (xnbop->o_mch != NULL) { mac_client_close(xnbop->o_mch, 0); xnbop->o_mch = NULL; } mac_close(xnbop->o_mh); xnbop->o_mh = NULL; } /* * Hotplug has completed and we are connected to the peer. We have all * the information we need to exchange traffic, so open the MAC device * and configure it appropriately. */ static boolean_t xnbo_start_connect(xnb_t *xnbp) { xnbo_t *xnbop = xnbp->xnb_flavour_data; return (xnbo_open_mac(xnbp, xnbop->o_link_name)); } /* * The guest has successfully synchronize with this instance. We read * the configuration of the guest from xenstore to check whether the * guest requests multicast control. If not (the default) we make a * note that the MAC device needs to be used in promiscious mode. */ static boolean_t xnbo_peer_connected(xnb_t *xnbp) { char *oename; int request; xnbo_t *xnbop = xnbp->xnb_flavour_data; oename = xvdi_get_oename(xnbp->xnb_devinfo); if (xenbus_scanf(XBT_NULL, oename, "request-multicast-control", "%d", &request) != 0) request = 0; xnbop->o_multicast_control = (request > 0); return (B_TRUE); } /* * The guest domain has closed down the inter-domain connection. We * close the underlying MAC device. */ static void xnbo_peer_disconnected(xnb_t *xnbp) { xnbo_close_mac(xnbp); } /* * The hotplug script has completed. We read information from xenstore * about our configuration, most notably the name of the MAC device we * should use. */ static boolean_t xnbo_hotplug_connected(xnb_t *xnbp) { char *xsname; xnbo_t *xnbop = xnbp->xnb_flavour_data; int need; xsname = xvdi_get_xsname(xnbp->xnb_devinfo); if (xenbus_scanf(XBT_NULL, xsname, "nic", "%s", xnbop->o_link_name) != 0) { cmn_err(CE_WARN, "xnbo_connect: " "cannot read nic name from %s", xsname); return (B_FALSE); } if (xenbus_scanf(XBT_NULL, xsname, "SUNW-need-rx-filter", "%d", &need) != 0) need = 0; xnbop->o_need_rx_filter = (need > 0); if (xenbus_scanf(XBT_NULL, xsname, "SUNW-need-set-physaddr", "%d", &need) != 0) need = 0; xnbop->o_need_setphysaddr = (need > 0); return (B_TRUE); } /* * Find the multicast address `addr', return B_TRUE if it is one that * we receive. If `remove', remove it from the set received. */ static boolean_t xnbo_mcast_find(xnb_t *xnbp, ether_addr_t *addr, boolean_t remove) { xnbo_t *xnbop = xnbp->xnb_flavour_data; xmca_t *prev, *del, *this; ASSERT(MUTEX_HELD(&xnbp->xnb_state_lock)); ASSERT(xnbop->o_promiscuous == B_FALSE); prev = del = NULL; this = xnbop->o_mca; while (this != NULL) { if (bcmp(&this->addr, addr, sizeof (this->addr)) == 0) { del = this; if (remove) { if (prev == NULL) xnbop->o_mca = this->next; else prev->next = this->next; } break; } prev = this; this = this->next; } if (del == NULL) return (B_FALSE); if (remove) { DTRACE_PROBE3(mcast_remove, (char *), "remove", (void *), xnbp, (etheraddr_t *), del->addr); mac_multicast_remove(xnbop->o_mch, del->addr); kmem_free(del, sizeof (*del)); } return (B_TRUE); } /* * Add the multicast address `addr' to the set received. */ static boolean_t xnbo_mcast_add(xnb_t *xnbp, ether_addr_t *addr) { xnbo_t *xnbop = xnbp->xnb_flavour_data; boolean_t r = B_FALSE; ASSERT(xnbop->o_promiscuous == B_FALSE); mutex_enter(&xnbp->xnb_state_lock); if (xnbo_mcast_find(xnbp, addr, B_FALSE)) { r = B_TRUE; } else if (mac_multicast_add(xnbop->o_mch, (const uint8_t *)addr) == 0) { xmca_t *mca; DTRACE_PROBE3(mcast_add, (char *), "add", (void *), xnbp, (etheraddr_t *), addr); mca = kmem_alloc(sizeof (*mca), KM_SLEEP); bcopy(addr, &mca->addr, sizeof (mca->addr)); mca->next = xnbop->o_mca; xnbop->o_mca = mca; r = B_TRUE; } mutex_exit(&xnbp->xnb_state_lock); return (r); } /* * Remove the multicast address `addr' from the set received. */ static boolean_t xnbo_mcast_del(xnb_t *xnbp, ether_addr_t *addr) { boolean_t r; mutex_enter(&xnbp->xnb_state_lock); r = xnbo_mcast_find(xnbp, addr, B_TRUE); mutex_exit(&xnbp->xnb_state_lock); return (r); } static int xnbo_attach(dev_info_t *dip, ddi_attach_cmd_t cmd) { static xnb_flavour_t flavour = { xnbo_to_mac, xnbo_peer_connected, xnbo_peer_disconnected, xnbo_hotplug_connected, xnbo_start_connect, xnbo_cksum_from_peer, xnbo_cksum_to_peer, xnbo_mcast_add, xnbo_mcast_del, }; xnbo_t *xnbop; switch (cmd) { case DDI_ATTACH: break; case DDI_RESUME: return (DDI_SUCCESS); default: return (DDI_FAILURE); } xnbop = kmem_zalloc(sizeof (*xnbop), KM_SLEEP); if (xnb_attach(dip, &flavour, xnbop) != DDI_SUCCESS) { kmem_free(xnbop, sizeof (*xnbop)); return (DDI_FAILURE); } return (DDI_SUCCESS); } static int xnbo_detach(dev_info_t *dip, ddi_detach_cmd_t cmd) { xnb_t *xnbp = ddi_get_driver_private(dip); xnbo_t *xnbop = xnbp->xnb_flavour_data; switch (cmd) { case DDI_DETACH: break; case DDI_SUSPEND: return (DDI_SUCCESS); default: return (DDI_FAILURE); } mutex_enter(&xnbp->xnb_tx_lock); mutex_enter(&xnbp->xnb_rx_lock); if (!xnbp->xnb_detachable || xnbp->xnb_connected || (xnbp->xnb_tx_buf_count > 0)) { mutex_exit(&xnbp->xnb_rx_lock); mutex_exit(&xnbp->xnb_tx_lock); return (DDI_FAILURE); } mutex_exit(&xnbp->xnb_rx_lock); mutex_exit(&xnbp->xnb_tx_lock); xnbo_close_mac(xnbp); kmem_free(xnbop, sizeof (*xnbop)); xnb_detach(dip); return (DDI_SUCCESS); } static struct cb_ops cb_ops = { nulldev, /* open */ nulldev, /* close */ nodev, /* strategy */ nodev, /* print */ nodev, /* dump */ nodev, /* read */ nodev, /* write */ nodev, /* ioctl */ nodev, /* devmap */ nodev, /* mmap */ nodev, /* segmap */ nochpoll, /* poll */ ddi_prop_op, /* cb_prop_op */ 0, /* streamtab */ D_NEW | D_MP | D_64BIT /* Driver compatibility flag */ }; static struct dev_ops ops = { DEVO_REV, /* devo_rev */ 0, /* devo_refcnt */ nulldev, /* devo_getinfo */ nulldev, /* devo_identify */ nulldev, /* devo_probe */ xnbo_attach, /* devo_attach */ xnbo_detach, /* devo_detach */ nodev, /* devo_reset */ &cb_ops, /* devo_cb_ops */ (struct bus_ops *)0, /* devo_bus_ops */ NULL, /* devo_power */ ddi_quiesce_not_needed, /* devo_quiesce */ }; static struct modldrv modldrv = { &mod_driverops, "xnbo driver", &ops, }; static struct modlinkage modlinkage = { MODREV_1, &modldrv, NULL }; int _init(void) { return (mod_install(&modlinkage)); } int _info(struct modinfo *modinfop) { return (mod_info(&modlinkage, modinfop)); } int _fini(void) { return (mod_remove(&modlinkage)); } /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2010 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ /* * Xen inter-domain backend - GLDv3 driver edition. * * A traditional GLDv3 driver used to communicate with a guest * domain. This driver is typically plumbed underneath the IP stack * or a software ethernet bridge. */ #include "xnb.h" #include #include #include #include #include #include #include #include #include #include /* Required driver entry points for GLDv3 */ static int xnbu_m_start(void *); static void xnbu_m_stop(void *); static int xnbu_m_set_mac_addr(void *, const uint8_t *); static int xnbu_m_set_multicast(void *, boolean_t, const uint8_t *); static int xnbu_m_set_promiscuous(void *, boolean_t); static int xnbu_m_stat(void *, uint_t, uint64_t *); static boolean_t xnbu_m_getcapab(void *, mac_capab_t, void *); static mblk_t *xnbu_m_send(void *, mblk_t *); typedef struct xnbu { mac_handle_t u_mh; boolean_t u_need_sched; } xnbu_t; static mac_callbacks_t xnbu_callbacks = { MC_GETCAPAB, xnbu_m_stat, xnbu_m_start, xnbu_m_stop, xnbu_m_set_promiscuous, xnbu_m_set_multicast, xnbu_m_set_mac_addr, xnbu_m_send, NULL, NULL, xnbu_m_getcapab }; static void xnbu_to_host(xnb_t *xnbp, mblk_t *mp) { xnbu_t *xnbup = xnbp->xnb_flavour_data; boolean_t sched = B_FALSE; ASSERT(mp != NULL); mac_rx(xnbup->u_mh, NULL, mp); mutex_enter(&xnbp->xnb_rx_lock); /* * If a transmit attempt failed because we ran out of ring * space and there is now some space, re-enable the transmit * path. */ if (xnbup->u_need_sched && RING_HAS_UNCONSUMED_REQUESTS(&xnbp->xnb_rx_ring)) { sched = B_TRUE; xnbup->u_need_sched = B_FALSE; } mutex_exit(&xnbp->xnb_rx_lock); if (sched) mac_tx_update(xnbup->u_mh); } static mblk_t * xnbu_cksum_from_peer(xnb_t *xnbp, mblk_t *mp, uint16_t flags) { /* * Take a conservative approach - if the checksum is blank * then we fill it in. * * If the consumer of the packet is IP then we might actually * only need fill it in if the data is not validated, but how * do we know who might end up with the packet? */ if ((flags & NETTXF_csum_blank) != 0) { /* * The checksum is blank. We must fill it in here. */ mp = xnb_process_cksum_flags(xnbp, mp, 0); /* * Because we calculated the checksum ourselves we * know that it must be good, so we assert this. */ flags |= NETTXF_data_validated; } if ((flags & NETTXF_data_validated) != 0) { /* * The checksum is asserted valid. */ mac_hcksum_set(mp, 0, 0, 0, 0, HCK_FULLCKSUM_OK); } return (mp); } static uint16_t xnbu_cksum_to_peer(xnb_t *xnbp, mblk_t *mp) { _NOTE(ARGUNUSED(xnbp)); uint16_t r = 0; uint32_t pflags; mac_hcksum_get(mp, NULL, NULL, NULL, NULL, &pflags); /* * If the protocol stack has requested checksum * offload, inform the peer that we have not * calculated the checksum. */ if ((pflags & HCK_FULLCKSUM) != 0) r |= NETRXF_csum_blank; return (r); } static boolean_t xnbu_start_connect(xnb_t *xnbp) { xnbu_t *xnbup = xnbp->xnb_flavour_data; mac_link_update(xnbup->u_mh, LINK_STATE_UP); /* * We are able to send packets now - bring them on. */ mac_tx_update(xnbup->u_mh); return (B_TRUE); } static boolean_t xnbu_peer_connected(xnb_t *xnbp) { _NOTE(ARGUNUSED(xnbp)); return (B_TRUE); } static void xnbu_peer_disconnected(xnb_t *xnbp) { xnbu_t *xnbup = xnbp->xnb_flavour_data; mac_link_update(xnbup->u_mh, LINK_STATE_DOWN); } /*ARGSUSED*/ static boolean_t xnbu_hotplug_connected(xnb_t *xnbp) { return (B_TRUE); } static mblk_t * xnbu_m_send(void *arg, mblk_t *mp) { xnb_t *xnbp = arg; xnbu_t *xnbup = xnbp->xnb_flavour_data; boolean_t sched = B_FALSE; mp = xnb_copy_to_peer(arg, mp); mutex_enter(&xnbp->xnb_rx_lock); /* * If we consumed all of the mblk_t's offered, perhaps we need * to indicate that we can accept more. Otherwise we are full * and need to wait for space. */ if (mp == NULL) { sched = xnbup->u_need_sched; xnbup->u_need_sched = B_FALSE; } else { xnbup->u_need_sched = B_TRUE; } mutex_exit(&xnbp->xnb_rx_lock); /* * If a previous transmit attempt failed because the ring * was full, try again now. */ if (sched) mac_tx_update(xnbup->u_mh); return (mp); } /* * xnbu_m_set_mac_addr() -- set the physical network address on the board */ /* ARGSUSED */ static int xnbu_m_set_mac_addr(void *arg, const uint8_t *macaddr) { xnb_t *xnbp = arg; xnbu_t *xnbup = xnbp->xnb_flavour_data; bcopy(macaddr, xnbp->xnb_mac_addr, ETHERADDRL); mac_unicst_update(xnbup->u_mh, xnbp->xnb_mac_addr); return (0); } /* * xnbu_m_set_multicast() -- set (enable) or disable a multicast address */ /*ARGSUSED*/ static int xnbu_m_set_multicast(void *arg, boolean_t add, const uint8_t *mca) { /* * We always accept all packets from the peer, so nothing to * do for enable or disable. */ return (0); } /* * xnbu_m_set_promiscuous() -- set or reset promiscuous mode on the board */ /* ARGSUSED */ static int xnbu_m_set_promiscuous(void *arg, boolean_t on) { /* * We always accept all packets from the peer, so nothing to * do for enable or disable. */ return (0); } /* * xnbu_m_start() -- start the board receiving and enable interrupts. */ /*ARGSUSED*/ static int xnbu_m_start(void *arg) { return (0); } /* * xnbu_m_stop() - disable hardware */ /*ARGSUSED*/ static void xnbu_m_stop(void *arg) { } static int xnbu_m_stat(void *arg, uint_t stat, uint64_t *val) { xnb_t *xnbp = arg; mutex_enter(&xnbp->xnb_tx_lock); mutex_enter(&xnbp->xnb_rx_lock); #define map_stat(q, r) \ case (MAC_STAT_##q): \ *val = xnbp->xnb_stat_##r; \ break switch (stat) { map_stat(IPACKETS, opackets); map_stat(OPACKETS, ipackets); map_stat(RBYTES, obytes); map_stat(OBYTES, rbytes); default: mutex_exit(&xnbp->xnb_rx_lock); mutex_exit(&xnbp->xnb_tx_lock); return (ENOTSUP); } #undef map_stat mutex_exit(&xnbp->xnb_rx_lock); mutex_exit(&xnbp->xnb_tx_lock); return (0); } static boolean_t xnbu_m_getcapab(void *arg, mac_capab_t cap, void *cap_data) { _NOTE(ARGUNUSED(arg)); switch (cap) { case MAC_CAPAB_HCKSUM: { uint32_t *capab = cap_data; *capab = HCKSUM_INET_PARTIAL; break; } default: return (B_FALSE); } return (B_TRUE); } /* * All packets are passed to the peer, so adding and removing * multicast addresses is meaningless. */ static boolean_t xnbu_mcast_add(xnb_t *xnbp, ether_addr_t *addr) { _NOTE(ARGUNUSED(xnbp, addr)); return (B_TRUE); } static boolean_t xnbu_mcast_del(xnb_t *xnbp, ether_addr_t *addr) { _NOTE(ARGUNUSED(xnbp, addr)); return (B_TRUE); } static int xnbu_attach(dev_info_t *dip, ddi_attach_cmd_t cmd) { static xnb_flavour_t flavour = { xnbu_to_host, xnbu_peer_connected, xnbu_peer_disconnected, xnbu_hotplug_connected, xnbu_start_connect, xnbu_cksum_from_peer, xnbu_cksum_to_peer, xnbu_mcast_add, xnbu_mcast_del, }; xnbu_t *xnbup; xnb_t *xnbp; mac_register_t *mr; int err; switch (cmd) { case DDI_ATTACH: break; case DDI_RESUME: return (DDI_SUCCESS); default: return (DDI_FAILURE); } xnbup = kmem_zalloc(sizeof (*xnbup), KM_SLEEP); if ((mr = mac_alloc(MAC_VERSION)) == NULL) { kmem_free(xnbup, sizeof (*xnbup)); return (DDI_FAILURE); } if (xnb_attach(dip, &flavour, xnbup) != DDI_SUCCESS) { mac_free(mr); kmem_free(xnbup, sizeof (*xnbup)); return (DDI_FAILURE); } xnbp = ddi_get_driver_private(dip); ASSERT(xnbp != NULL); mr->m_dip = dip; mr->m_driver = xnbp; /* * Initialize pointers to device specific functions which will be * used by the generic layer. */ mr->m_type_ident = MAC_PLUGIN_IDENT_ETHER; mr->m_src_addr = xnbp->xnb_mac_addr; mr->m_callbacks = &xnbu_callbacks; mr->m_min_sdu = 0; mr->m_max_sdu = XNBMAXPKT; /* * xnbu is a virtual device, and it is not associated with any * physical device. Its margin size is determined by the maximum * packet size it can handle, which is PAGESIZE. */ mr->m_margin = PAGESIZE - XNBMAXPKT - sizeof (struct ether_header); (void) memset(xnbp->xnb_mac_addr, 0xff, ETHERADDRL); xnbp->xnb_mac_addr[0] &= 0xfe; xnbup->u_need_sched = B_FALSE; /* * Register ourselves with the GLDv3 interface. */ err = mac_register(mr, &xnbup->u_mh); mac_free(mr); if (err != 0) { xnb_detach(dip); kmem_free(xnbup, sizeof (*xnbup)); return (DDI_FAILURE); } mac_link_update(xnbup->u_mh, LINK_STATE_DOWN); return (DDI_SUCCESS); } /*ARGSUSED*/ int xnbu_detach(dev_info_t *dip, ddi_detach_cmd_t cmd) { xnb_t *xnbp = ddi_get_driver_private(dip); xnbu_t *xnbup = xnbp->xnb_flavour_data; switch (cmd) { case DDI_DETACH: break; case DDI_SUSPEND: return (DDI_SUCCESS); default: return (DDI_FAILURE); } ASSERT(xnbp != NULL); ASSERT(xnbup != NULL); mutex_enter(&xnbp->xnb_tx_lock); mutex_enter(&xnbp->xnb_rx_lock); if (!xnbp->xnb_detachable || xnbp->xnb_connected || (xnbp->xnb_tx_buf_count > 0)) { mutex_exit(&xnbp->xnb_rx_lock); mutex_exit(&xnbp->xnb_tx_lock); return (DDI_FAILURE); } mutex_exit(&xnbp->xnb_rx_lock); mutex_exit(&xnbp->xnb_tx_lock); /* * Attempt to unregister the mac. */ if ((xnbup->u_mh != NULL) && (mac_unregister(xnbup->u_mh) != 0)) return (DDI_FAILURE); kmem_free(xnbup, sizeof (*xnbup)); xnb_detach(dip); return (DDI_SUCCESS); } DDI_DEFINE_STREAM_OPS(ops, nulldev, nulldev, xnbu_attach, xnbu_detach, nodev, NULL, D_MP, NULL, ddi_quiesce_not_supported); static struct modldrv modldrv = { &mod_driverops, "xnbu driver", &ops }; static struct modlinkage modlinkage = { MODREV_1, &modldrv, NULL }; int _init(void) { int i; mac_init_ops(&ops, "xnbu"); i = mod_install(&modlinkage); if (i != DDI_SUCCESS) mac_fini_ops(&ops); return (i); } int _fini(void) { int i; i = mod_remove(&modlinkage); if (i == DDI_SUCCESS) mac_fini_ops(&ops); return (i); } int _info(struct modinfo *modinfop) { return (mod_info(&modlinkage, modinfop)); } /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2010 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ /* * Copyright (c) 2014, 2017 by Delphix. All rights reserved. * Copyright 2020 RackTop Systems, Inc. */ /* * * Copyright (c) 2004 Christian Limpach. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. This section intentionally left blank. * 4. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ /* * Section 3 of the above license was updated in response to bug 6379571. */ /* * xnf.c - GLDv3 network driver for domU. */ /* * This driver uses four per-instance locks: * * xnf_gref_lock: * * Protects access to the grant reference list stored in * xnf_gref_head. Grant references should be acquired and released * using gref_get() and gref_put() respectively. * * xnf_schedlock: * * Protects: * xnf_need_sched - used to record that a previous transmit attempt * failed (and consequently it will be necessary to call * mac_tx_update() when transmit resources are available). * xnf_pending_multicast - the number of multicast requests that * have been submitted to the backend for which we have not * processed responses. * * xnf_txlock: * * Protects the transmit ring (xnf_tx_ring) and associated * structures (notably xnf_tx_pkt_id and xnf_tx_pkt_id_head). * * xnf_rxlock: * * Protects the receive ring (xnf_rx_ring) and associated * structures (notably xnf_rx_pkt_info). * * If driver-global state that affects both the transmit and receive * rings is manipulated, both xnf_txlock and xnf_rxlock should be * held, in that order. * * xnf_schedlock is acquired both whilst holding xnf_txlock and * without. It should always be acquired after xnf_txlock if both are * held. * * Notes: * - atomic_add_64() is used to manipulate counters where we require * accuracy. For counters intended only for observation by humans, * post increment/decrement are used instead. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef XPV_HVM_DRIVER #include #include #else #include #include #include #endif #include #include #include #include #include #include #include /* * On a 32 bit PAE system physical and machine addresses are larger * than 32 bits. ddi_btop() on such systems take an unsigned long * argument, and so addresses above 4G are truncated before ddi_btop() * gets to see them. To avoid this, code the shift operation here. */ #define xnf_btop(addr) ((addr) >> PAGESHIFT) /* * The parameters below should only be changed in /etc/system, never in mdb. */ /* * Should we use the multicast control feature if the backend provides * it? */ boolean_t xnf_multicast_control = B_TRUE; /* * Should we allow scatter-gather for tx if backend allows it? */ boolean_t xnf_enable_tx_sg = B_TRUE; /* * Should we allow scatter-gather for rx if backend allows it? */ boolean_t xnf_enable_rx_sg = B_TRUE; /* * Should we allow lso for tx sends if backend allows it? * Requires xnf_enable_tx_sg to be also set to TRUE. */ boolean_t xnf_enable_lso = B_TRUE; /* * Should we allow lro on rx if backend supports it? * Requires xnf_enable_rx_sg to be also set to TRUE. * * !! WARNING !! * LRO is not yet supported in the OS so this should be left as FALSE. * !! WARNING !! */ boolean_t xnf_enable_lro = B_FALSE; /* * Received packets below this size are copied to a new streams buffer * rather than being desballoc'ed. * * This value is chosen to accommodate traffic where there are a large * number of small packets. For data showing a typical distribution, * see: * * Sinha07a: * Rishi Sinha, Christos Papadopoulos, and John * Heidemann. Internet Packet Size Distributions: Some * Observations. Technical Report ISI-TR-2007-643, * USC/Information Sciences Institute, May, 2007. Orignally * released October 2005 as web page * http://netweb.usc.edu/~sinha/pkt-sizes/. * . */ size_t xnf_rx_copy_limit = 64; #define INVALID_GRANT_HANDLE ((grant_handle_t)-1) #define INVALID_GRANT_REF ((grant_ref_t)-1) #define INVALID_TX_ID ((uint16_t)-1) #define TX_ID_TO_TXID(p, id) (&((p)->xnf_tx_pkt_id[(id)])) #define TX_ID_VALID(i) \ (((i) != INVALID_TX_ID) && ((i) < NET_TX_RING_SIZE)) /* * calculate how many pages are spanned by an mblk fragment */ #define xnf_mblk_pages(mp) (MBLKL(mp) == 0 ? 0 : \ xnf_btop((uintptr_t)mp->b_wptr - 1) - xnf_btop((uintptr_t)mp->b_rptr) + 1) /* Required system entry points */ static int xnf_attach(dev_info_t *, ddi_attach_cmd_t); static int xnf_detach(dev_info_t *, ddi_detach_cmd_t); /* Required driver entry points for Nemo */ static int xnf_start(void *); static void xnf_stop(void *); static int xnf_set_mac_addr(void *, const uint8_t *); static int xnf_set_multicast(void *, boolean_t, const uint8_t *); static int xnf_set_promiscuous(void *, boolean_t); static mblk_t *xnf_send(void *, mblk_t *); static uint_t xnf_intr(caddr_t); static int xnf_stat(void *, uint_t, uint64_t *); static boolean_t xnf_getcapab(void *, mac_capab_t, void *); static int xnf_getprop(void *, const char *, mac_prop_id_t, uint_t, void *); static int xnf_setprop(void *, const char *, mac_prop_id_t, uint_t, const void *); static void xnf_propinfo(void *, const char *, mac_prop_id_t, mac_prop_info_handle_t); /* Driver private functions */ static int xnf_alloc_dma_resources(xnf_t *); static void xnf_release_dma_resources(xnf_t *); static void xnf_release_mblks(xnf_t *); static int xnf_buf_constructor(void *, void *, int); static void xnf_buf_destructor(void *, void *); static xnf_buf_t *xnf_buf_get(xnf_t *, int, boolean_t); static void xnf_buf_put(xnf_t *, xnf_buf_t *, boolean_t); static void xnf_buf_refresh(xnf_buf_t *); static void xnf_buf_recycle(xnf_buf_t *); static int xnf_tx_buf_constructor(void *, void *, int); static void xnf_tx_buf_destructor(void *, void *); static grant_ref_t xnf_gref_get(xnf_t *); static void xnf_gref_put(xnf_t *, grant_ref_t); static xnf_txid_t *xnf_txid_get(xnf_t *); static void xnf_txid_put(xnf_t *, xnf_txid_t *); static void xnf_rxbuf_hang(xnf_t *, xnf_buf_t *); static int xnf_tx_clean_ring(xnf_t *); static void oe_state_change(dev_info_t *, ddi_eventcookie_t, void *, void *); static boolean_t xnf_kstat_init(xnf_t *); static void xnf_rx_collect(xnf_t *); #define XNF_CALLBACK_FLAGS (MC_GETCAPAB | MC_PROPERTIES) static mac_callbacks_t xnf_callbacks = { .mc_callbacks = XNF_CALLBACK_FLAGS, .mc_getstat = xnf_stat, .mc_start = xnf_start, .mc_stop = xnf_stop, .mc_setpromisc = xnf_set_promiscuous, .mc_multicst = xnf_set_multicast, .mc_unicst = xnf_set_mac_addr, .mc_tx = xnf_send, .mc_getcapab = xnf_getcapab, .mc_setprop = xnf_setprop, .mc_getprop = xnf_getprop, .mc_propinfo = xnf_propinfo, }; /* DMA attributes for network ring buffer */ static ddi_dma_attr_t ringbuf_dma_attr = { .dma_attr_version = DMA_ATTR_V0, .dma_attr_addr_lo = 0, .dma_attr_addr_hi = 0xffffffffffffffffULL, .dma_attr_count_max = 0x7fffffff, .dma_attr_align = MMU_PAGESIZE, .dma_attr_burstsizes = 0x7ff, .dma_attr_minxfer = 1, .dma_attr_maxxfer = 0xffffffffU, .dma_attr_seg = 0xffffffffffffffffULL, .dma_attr_sgllen = 1, .dma_attr_granular = 1, .dma_attr_flags = 0 }; /* DMA attributes for receive data */ static ddi_dma_attr_t rx_buf_dma_attr = { .dma_attr_version = DMA_ATTR_V0, .dma_attr_addr_lo = 0, .dma_attr_addr_hi = 0xffffffffffffffffULL, .dma_attr_count_max = MMU_PAGEOFFSET, .dma_attr_align = MMU_PAGESIZE, /* allocation alignment */ .dma_attr_burstsizes = 0x7ff, .dma_attr_minxfer = 1, .dma_attr_maxxfer = 0xffffffffU, .dma_attr_seg = 0xffffffffffffffffULL, .dma_attr_sgllen = 1, .dma_attr_granular = 1, .dma_attr_flags = 0 }; /* DMA attributes for transmit data */ static ddi_dma_attr_t tx_buf_dma_attr = { .dma_attr_version = DMA_ATTR_V0, .dma_attr_addr_lo = 0, .dma_attr_addr_hi = 0xffffffffffffffffULL, .dma_attr_count_max = MMU_PAGEOFFSET, .dma_attr_align = 1, .dma_attr_burstsizes = 0x7ff, .dma_attr_minxfer = 1, .dma_attr_maxxfer = 0xffffffffU, .dma_attr_seg = XEN_DATA_BOUNDARY - 1, /* segment boundary */ .dma_attr_sgllen = XEN_MAX_TX_DATA_PAGES, /* max number of segments */ .dma_attr_granular = 1, .dma_attr_flags = 0 }; /* DMA access attributes for registers and descriptors */ static ddi_device_acc_attr_t accattr = { DDI_DEVICE_ATTR_V0, DDI_STRUCTURE_LE_ACC, /* This is a little-endian device */ DDI_STRICTORDER_ACC }; /* DMA access attributes for data: NOT to be byte swapped. */ static ddi_device_acc_attr_t data_accattr = { DDI_DEVICE_ATTR_V0, DDI_NEVERSWAP_ACC, DDI_STRICTORDER_ACC }; DDI_DEFINE_STREAM_OPS(xnf_dev_ops, nulldev, nulldev, xnf_attach, xnf_detach, nodev, NULL, D_MP, NULL, ddi_quiesce_not_supported); static struct modldrv xnf_modldrv = { &mod_driverops, "Virtual Ethernet driver", &xnf_dev_ops }; static struct modlinkage modlinkage = { MODREV_1, &xnf_modldrv, NULL }; int _init(void) { int r; mac_init_ops(&xnf_dev_ops, "xnf"); r = mod_install(&modlinkage); if (r != DDI_SUCCESS) mac_fini_ops(&xnf_dev_ops); return (r); } int _fini(void) { return (EBUSY); /* XXPV should be removable */ } int _info(struct modinfo *modinfop) { return (mod_info(&modlinkage, modinfop)); } /* * Acquire a grant reference. */ static grant_ref_t xnf_gref_get(xnf_t *xnfp) { grant_ref_t gref; mutex_enter(&xnfp->xnf_gref_lock); do { gref = gnttab_claim_grant_reference(&xnfp->xnf_gref_head); } while ((gref == INVALID_GRANT_REF) && (gnttab_alloc_grant_references(16, &xnfp->xnf_gref_head) == 0)); mutex_exit(&xnfp->xnf_gref_lock); if (gref == INVALID_GRANT_REF) { xnfp->xnf_stat_gref_failure++; } else { atomic_inc_64(&xnfp->xnf_stat_gref_outstanding); if (xnfp->xnf_stat_gref_outstanding > xnfp->xnf_stat_gref_peak) xnfp->xnf_stat_gref_peak = xnfp->xnf_stat_gref_outstanding; } return (gref); } /* * Release a grant reference. */ static void xnf_gref_put(xnf_t *xnfp, grant_ref_t gref) { ASSERT(gref != INVALID_GRANT_REF); mutex_enter(&xnfp->xnf_gref_lock); gnttab_release_grant_reference(&xnfp->xnf_gref_head, gref); mutex_exit(&xnfp->xnf_gref_lock); atomic_dec_64(&xnfp->xnf_stat_gref_outstanding); } /* * Acquire a transmit id. */ static xnf_txid_t * xnf_txid_get(xnf_t *xnfp) { xnf_txid_t *tidp; ASSERT(MUTEX_HELD(&xnfp->xnf_txlock)); if (xnfp->xnf_tx_pkt_id_head == INVALID_TX_ID) return (NULL); ASSERT(TX_ID_VALID(xnfp->xnf_tx_pkt_id_head)); tidp = TX_ID_TO_TXID(xnfp, xnfp->xnf_tx_pkt_id_head); xnfp->xnf_tx_pkt_id_head = tidp->next; tidp->next = INVALID_TX_ID; ASSERT(tidp->txbuf == NULL); return (tidp); } /* * Release a transmit id. */ static void xnf_txid_put(xnf_t *xnfp, xnf_txid_t *tidp) { ASSERT(MUTEX_HELD(&xnfp->xnf_txlock)); ASSERT(TX_ID_VALID(tidp->id)); ASSERT(tidp->next == INVALID_TX_ID); tidp->txbuf = NULL; tidp->next = xnfp->xnf_tx_pkt_id_head; xnfp->xnf_tx_pkt_id_head = tidp->id; } static void xnf_data_txbuf_free(xnf_t *xnfp, xnf_txbuf_t *txp) { ASSERT3U(txp->tx_type, ==, TX_DATA); /* * We are either using a lookaside buffer or we are mapping existing * buffers. */ if (txp->tx_bdesc != NULL) { ASSERT(!txp->tx_handle_bound); xnf_buf_put(xnfp, txp->tx_bdesc, B_TRUE); } else { if (txp->tx_txreq.gref != INVALID_GRANT_REF) { if (gnttab_query_foreign_access(txp->tx_txreq.gref) != 0) { cmn_err(CE_PANIC, "tx grant %d still in use by " "backend domain", txp->tx_txreq.gref); } (void) gnttab_end_foreign_access_ref( txp->tx_txreq.gref, 1); xnf_gref_put(xnfp, txp->tx_txreq.gref); } if (txp->tx_handle_bound) (void) ddi_dma_unbind_handle(txp->tx_dma_handle); } if (txp->tx_mp != NULL) freemsg(txp->tx_mp); if (txp->tx_prev != NULL) { ASSERT3P(txp->tx_prev->tx_next, ==, txp); txp->tx_prev->tx_next = NULL; } if (txp->tx_txreq.id != INVALID_TX_ID) { /* * This should be only possible when resuming from a suspend. */ ASSERT(!xnfp->xnf_connected); xnf_txid_put(xnfp, TX_ID_TO_TXID(xnfp, txp->tx_txreq.id)); txp->tx_txreq.id = INVALID_TX_ID; } kmem_cache_free(xnfp->xnf_tx_buf_cache, txp); } static void xnf_data_txbuf_free_chain(xnf_t *xnfp, xnf_txbuf_t *txp) { if (txp == NULL) return; while (txp->tx_next != NULL) txp = txp->tx_next; /* * We free the chain in reverse order so that grants can be released * for all dma chunks before unbinding the dma handles. The mblk is * freed last, after all its fragments' dma handles are unbound. */ xnf_txbuf_t *prev; for (; txp != NULL; txp = prev) { prev = txp->tx_prev; xnf_data_txbuf_free(xnfp, txp); } } static xnf_txbuf_t * xnf_data_txbuf_alloc(xnf_t *xnfp, int flag) { xnf_txbuf_t *txp; if ((txp = kmem_cache_alloc(xnfp->xnf_tx_buf_cache, flag)) == NULL) { return (NULL); } txp->tx_type = TX_DATA; txp->tx_next = NULL; txp->tx_prev = NULL; txp->tx_head = txp; txp->tx_frags_to_ack = 0; txp->tx_mp = NULL; txp->tx_bdesc = NULL; txp->tx_handle_bound = B_FALSE; txp->tx_txreq.gref = INVALID_GRANT_REF; txp->tx_txreq.id = INVALID_TX_ID; return (txp); } /* * Get `wanted' slots in the transmit ring, waiting for at least that * number if `wait' is B_TRUE. Force the ring to be cleaned by setting * `wanted' to zero. * * Return the number of slots available. */ static int xnf_tx_slots_get(xnf_t *xnfp, int wanted, boolean_t wait) { int slotsfree; boolean_t forced_clean = (wanted == 0); ASSERT(MUTEX_HELD(&xnfp->xnf_txlock)); /* LINTED: constant in conditional context */ while (B_TRUE) { slotsfree = RING_FREE_REQUESTS(&xnfp->xnf_tx_ring); if ((slotsfree < wanted) || forced_clean) slotsfree = xnf_tx_clean_ring(xnfp); /* * If there are more than we need free, tell other * people to come looking again. We hold txlock, so we * are able to take our slots before anyone else runs. */ if (slotsfree > wanted) cv_broadcast(&xnfp->xnf_cv_tx_slots); if (slotsfree >= wanted) break; if (!wait) break; cv_wait(&xnfp->xnf_cv_tx_slots, &xnfp->xnf_txlock); } ASSERT(slotsfree <= RING_SIZE(&(xnfp->xnf_tx_ring))); return (slotsfree); } static int xnf_setup_rings(xnf_t *xnfp) { domid_t oeid; struct xenbus_device *xsd; RING_IDX i; int err; xnf_txid_t *tidp; xnf_buf_t **bdescp; oeid = xvdi_get_oeid(xnfp->xnf_devinfo); xsd = xvdi_get_xsd(xnfp->xnf_devinfo); if (xnfp->xnf_tx_ring_ref != INVALID_GRANT_REF) gnttab_end_foreign_access(xnfp->xnf_tx_ring_ref, 0, 0); err = gnttab_grant_foreign_access(oeid, xnf_btop(pa_to_ma(xnfp->xnf_tx_ring_phys_addr)), 0); if (err <= 0) { err = -err; xenbus_dev_error(xsd, err, "granting access to tx ring page"); goto out; } xnfp->xnf_tx_ring_ref = (grant_ref_t)err; if (xnfp->xnf_rx_ring_ref != INVALID_GRANT_REF) gnttab_end_foreign_access(xnfp->xnf_rx_ring_ref, 0, 0); err = gnttab_grant_foreign_access(oeid, xnf_btop(pa_to_ma(xnfp->xnf_rx_ring_phys_addr)), 0); if (err <= 0) { err = -err; xenbus_dev_error(xsd, err, "granting access to rx ring page"); goto out; } xnfp->xnf_rx_ring_ref = (grant_ref_t)err; mutex_enter(&xnfp->xnf_txlock); /* * We first cleanup the TX ring in case we are doing a resume. * Note that this can lose packets, but we expect to stagger on. */ xnfp->xnf_tx_pkt_id_head = INVALID_TX_ID; /* I.e. emtpy list. */ for (i = 0, tidp = &xnfp->xnf_tx_pkt_id[0]; i < NET_TX_RING_SIZE; i++, tidp++) { xnf_txbuf_t *txp = tidp->txbuf; if (txp == NULL) continue; switch (txp->tx_type) { case TX_DATA: /* * txid_put() will be called for each txbuf's txid in * the chain which will result in clearing tidp->txbuf. */ xnf_data_txbuf_free_chain(xnfp, txp); break; case TX_MCAST_REQ: txp->tx_type = TX_MCAST_RSP; txp->tx_status = NETIF_RSP_DROPPED; cv_broadcast(&xnfp->xnf_cv_multicast); /* * The request consumed two slots in the ring, * yet only a single xnf_txid_t is used. Step * over the empty slot. */ i++; ASSERT3U(i, <, NET_TX_RING_SIZE); break; case TX_MCAST_RSP: break; } } /* * Now purge old list and add each txid to the new free list. */ xnfp->xnf_tx_pkt_id_head = INVALID_TX_ID; /* I.e. emtpy list. */ for (i = 0, tidp = &xnfp->xnf_tx_pkt_id[0]; i < NET_TX_RING_SIZE; i++, tidp++) { tidp->id = i; ASSERT3P(tidp->txbuf, ==, NULL); tidp->next = INVALID_TX_ID; /* Appease txid_put(). */ xnf_txid_put(xnfp, tidp); } /* LINTED: constant in conditional context */ SHARED_RING_INIT(xnfp->xnf_tx_ring.sring); /* LINTED: constant in conditional context */ FRONT_RING_INIT(&xnfp->xnf_tx_ring, xnfp->xnf_tx_ring.sring, PAGESIZE); mutex_exit(&xnfp->xnf_txlock); mutex_enter(&xnfp->xnf_rxlock); /* * Clean out any buffers currently posted to the receive ring * before we reset it. */ for (i = 0, bdescp = &xnfp->xnf_rx_pkt_info[0]; i < NET_RX_RING_SIZE; i++, bdescp++) { if (*bdescp != NULL) { xnf_buf_put(xnfp, *bdescp, B_FALSE); *bdescp = NULL; } } /* LINTED: constant in conditional context */ SHARED_RING_INIT(xnfp->xnf_rx_ring.sring); /* LINTED: constant in conditional context */ FRONT_RING_INIT(&xnfp->xnf_rx_ring, xnfp->xnf_rx_ring.sring, PAGESIZE); /* * Fill the ring with buffers. */ for (i = 0; i < NET_RX_RING_SIZE; i++) { xnf_buf_t *bdesc; bdesc = xnf_buf_get(xnfp, KM_SLEEP, B_FALSE); VERIFY(bdesc != NULL); xnf_rxbuf_hang(xnfp, bdesc); } /* LINTED: constant in conditional context */ RING_PUSH_REQUESTS(&xnfp->xnf_rx_ring); mutex_exit(&xnfp->xnf_rxlock); return (0); out: if (xnfp->xnf_tx_ring_ref != INVALID_GRANT_REF) gnttab_end_foreign_access(xnfp->xnf_tx_ring_ref, 0, 0); xnfp->xnf_tx_ring_ref = INVALID_GRANT_REF; if (xnfp->xnf_rx_ring_ref != INVALID_GRANT_REF) gnttab_end_foreign_access(xnfp->xnf_rx_ring_ref, 0, 0); xnfp->xnf_rx_ring_ref = INVALID_GRANT_REF; return (err); } /* * Connect driver to back end, called to set up communication with * back end driver both initially and on resume after restore/migrate. */ void xnf_be_connect(xnf_t *xnfp) { const char *message; xenbus_transaction_t xbt; struct xenbus_device *xsd; char *xsname; int err; ASSERT(!xnfp->xnf_connected); xsd = xvdi_get_xsd(xnfp->xnf_devinfo); xsname = xvdi_get_xsname(xnfp->xnf_devinfo); err = xnf_setup_rings(xnfp); if (err != 0) { cmn_err(CE_WARN, "failed to set up tx/rx rings"); xenbus_dev_error(xsd, err, "setting up ring"); return; } again: err = xenbus_transaction_start(&xbt); if (err != 0) { xenbus_dev_error(xsd, EIO, "starting transaction"); return; } err = xenbus_printf(xbt, xsname, "tx-ring-ref", "%u", xnfp->xnf_tx_ring_ref); if (err != 0) { message = "writing tx ring-ref"; goto abort_transaction; } err = xenbus_printf(xbt, xsname, "rx-ring-ref", "%u", xnfp->xnf_rx_ring_ref); if (err != 0) { message = "writing rx ring-ref"; goto abort_transaction; } err = xenbus_printf(xbt, xsname, "event-channel", "%u", xnfp->xnf_evtchn); if (err != 0) { message = "writing event-channel"; goto abort_transaction; } err = xenbus_printf(xbt, xsname, "feature-rx-notify", "%d", 1); if (err != 0) { message = "writing feature-rx-notify"; goto abort_transaction; } err = xenbus_printf(xbt, xsname, "request-rx-copy", "%d", 1); if (err != 0) { message = "writing request-rx-copy"; goto abort_transaction; } if (xnfp->xnf_be_mcast_control) { err = xenbus_printf(xbt, xsname, "request-multicast-control", "%d", 1); if (err != 0) { message = "writing request-multicast-control"; goto abort_transaction; } } /* * Tell backend if we support scatter-gather lists on the rx side. */ err = xenbus_printf(xbt, xsname, "feature-sg", "%d", xnf_enable_rx_sg ? 1 : 0); if (err != 0) { message = "writing feature-sg"; goto abort_transaction; } /* * Tell backend if we support LRO for IPv4. Scatter-gather on rx is * a prerequisite. */ err = xenbus_printf(xbt, xsname, "feature-gso-tcpv4", "%d", (xnf_enable_rx_sg && xnf_enable_lro) ? 1 : 0); if (err != 0) { message = "writing feature-gso-tcpv4"; goto abort_transaction; } err = xvdi_switch_state(xnfp->xnf_devinfo, xbt, XenbusStateConnected); if (err != 0) { message = "switching state to XenbusStateConnected"; goto abort_transaction; } err = xenbus_transaction_end(xbt, 0); if (err != 0) { if (err == EAGAIN) goto again; xenbus_dev_error(xsd, err, "completing transaction"); } return; abort_transaction: (void) xenbus_transaction_end(xbt, 1); xenbus_dev_error(xsd, err, "%s", message); } /* * Read configuration information from xenstore. */ void xnf_read_config(xnf_t *xnfp) { int err, be_cap; char mac[ETHERADDRL * 3]; char *oename = xvdi_get_oename(xnfp->xnf_devinfo); err = xenbus_scanf(XBT_NULL, oename, "mac", "%s", (char *)&mac[0]); if (err != 0) { /* * bad: we're supposed to be set up with a proper mac * addr. at this point */ cmn_err(CE_WARN, "%s%d: no mac address", ddi_driver_name(xnfp->xnf_devinfo), ddi_get_instance(xnfp->xnf_devinfo)); return; } if (ether_aton(mac, xnfp->xnf_mac_addr) != ETHERADDRL) { err = ENOENT; xenbus_dev_error(xvdi_get_xsd(xnfp->xnf_devinfo), ENOENT, "parsing %s/mac", xvdi_get_xsname(xnfp->xnf_devinfo)); return; } err = xenbus_scanf(XBT_NULL, oename, "feature-rx-copy", "%d", &be_cap); /* * If we fail to read the store we assume that the key is * absent, implying an older domain at the far end. Older * domains cannot do HV copy. */ if (err != 0) be_cap = 0; xnfp->xnf_be_rx_copy = (be_cap != 0); err = xenbus_scanf(XBT_NULL, oename, "feature-multicast-control", "%d", &be_cap); /* * If we fail to read the store we assume that the key is * absent, implying an older domain at the far end. Older * domains do not support multicast control. */ if (err != 0) be_cap = 0; xnfp->xnf_be_mcast_control = (be_cap != 0) && xnf_multicast_control; /* * See if back-end supports scatter-gather for transmits. If not, * we will not support LSO and limit the mtu to 1500. */ err = xenbus_scanf(XBT_NULL, oename, "feature-sg", "%d", &be_cap); if (err != 0) { be_cap = 0; dev_err(xnfp->xnf_devinfo, CE_WARN, "error reading " "'feature-sg' from backend driver"); } if (be_cap == 0) { dev_err(xnfp->xnf_devinfo, CE_WARN, "scatter-gather is not " "supported for transmits in the backend driver. LSO is " "disabled and MTU is restricted to 1500 bytes."); } xnfp->xnf_be_tx_sg = (be_cap != 0) && xnf_enable_tx_sg; if (xnfp->xnf_be_tx_sg) { /* * Check if LSO is supported. Currently we only check for * IPv4 as Illumos doesn't support LSO for IPv6. */ err = xenbus_scanf(XBT_NULL, oename, "feature-gso-tcpv4", "%d", &be_cap); if (err != 0) { be_cap = 0; dev_err(xnfp->xnf_devinfo, CE_WARN, "error reading " "'feature-gso-tcpv4' from backend driver"); } if (be_cap == 0) { dev_err(xnfp->xnf_devinfo, CE_WARN, "LSO is not " "supported by the backend driver. Performance " "will be affected."); } xnfp->xnf_be_lso = (be_cap != 0) && xnf_enable_lso; } } /* * attach(9E) -- Attach a device to the system */ static int xnf_attach(dev_info_t *devinfo, ddi_attach_cmd_t cmd) { mac_register_t *macp; xnf_t *xnfp; int err; char cachename[32]; switch (cmd) { case DDI_RESUME: xnfp = ddi_get_driver_private(devinfo); xnfp->xnf_gen++; (void) xvdi_resume(devinfo); (void) xvdi_alloc_evtchn(devinfo); xnfp->xnf_evtchn = xvdi_get_evtchn(devinfo); #ifdef XPV_HVM_DRIVER ec_bind_evtchn_to_handler(xnfp->xnf_evtchn, IPL_VIF, xnf_intr, xnfp); #else (void) ddi_add_intr(devinfo, 0, NULL, NULL, xnf_intr, (caddr_t)xnfp); #endif return (DDI_SUCCESS); case DDI_ATTACH: break; default: return (DDI_FAILURE); } /* * Allocate gld_mac_info_t and xnf_instance structures */ macp = mac_alloc(MAC_VERSION); if (macp == NULL) return (DDI_FAILURE); xnfp = kmem_zalloc(sizeof (*xnfp), KM_SLEEP); xnfp->xnf_tx_pkt_id = kmem_zalloc(sizeof (xnf_txid_t) * NET_TX_RING_SIZE, KM_SLEEP); xnfp->xnf_rx_pkt_info = kmem_zalloc(sizeof (xnf_buf_t *) * NET_RX_RING_SIZE, KM_SLEEP); macp->m_dip = devinfo; macp->m_driver = xnfp; xnfp->xnf_devinfo = devinfo; macp->m_type_ident = MAC_PLUGIN_IDENT_ETHER; macp->m_src_addr = xnfp->xnf_mac_addr; macp->m_callbacks = &xnf_callbacks; macp->m_min_sdu = 0; xnfp->xnf_mtu = ETHERMTU; macp->m_max_sdu = xnfp->xnf_mtu; xnfp->xnf_running = B_FALSE; xnfp->xnf_connected = B_FALSE; xnfp->xnf_be_rx_copy = B_FALSE; xnfp->xnf_be_mcast_control = B_FALSE; xnfp->xnf_need_sched = B_FALSE; xnfp->xnf_rx_head = NULL; xnfp->xnf_rx_tail = NULL; xnfp->xnf_rx_new_buffers_posted = B_FALSE; #ifdef XPV_HVM_DRIVER /* Report our version to dom0 */ (void) xenbus_printf(XBT_NULL, "guest/xnf", "version", "%d", HVMPV_XNF_VERS); #endif /* * Get the iblock cookie with which to initialize the mutexes. */ if (ddi_get_iblock_cookie(devinfo, 0, &xnfp->xnf_icookie) != DDI_SUCCESS) goto failure; mutex_init(&xnfp->xnf_txlock, NULL, MUTEX_DRIVER, xnfp->xnf_icookie); mutex_init(&xnfp->xnf_rxlock, NULL, MUTEX_DRIVER, xnfp->xnf_icookie); mutex_init(&xnfp->xnf_schedlock, NULL, MUTEX_DRIVER, xnfp->xnf_icookie); mutex_init(&xnfp->xnf_gref_lock, NULL, MUTEX_DRIVER, xnfp->xnf_icookie); cv_init(&xnfp->xnf_cv_state, NULL, CV_DEFAULT, NULL); cv_init(&xnfp->xnf_cv_multicast, NULL, CV_DEFAULT, NULL); cv_init(&xnfp->xnf_cv_tx_slots, NULL, CV_DEFAULT, NULL); (void) sprintf(cachename, "xnf_buf_cache_%d", ddi_get_instance(devinfo)); xnfp->xnf_buf_cache = kmem_cache_create(cachename, sizeof (xnf_buf_t), 0, xnf_buf_constructor, xnf_buf_destructor, NULL, xnfp, NULL, 0); if (xnfp->xnf_buf_cache == NULL) goto failure_0; (void) sprintf(cachename, "xnf_tx_buf_cache_%d", ddi_get_instance(devinfo)); xnfp->xnf_tx_buf_cache = kmem_cache_create(cachename, sizeof (xnf_txbuf_t), 0, xnf_tx_buf_constructor, xnf_tx_buf_destructor, NULL, xnfp, NULL, 0); if (xnfp->xnf_tx_buf_cache == NULL) goto failure_1; xnfp->xnf_gref_head = INVALID_GRANT_REF; if (xnf_alloc_dma_resources(xnfp) == DDI_FAILURE) { cmn_err(CE_WARN, "xnf%d: failed to allocate and initialize " "driver data structures", ddi_get_instance(xnfp->xnf_devinfo)); goto failure_2; } xnfp->xnf_rx_ring.sring->rsp_event = xnfp->xnf_tx_ring.sring->rsp_event = 1; xnfp->xnf_tx_ring_ref = INVALID_GRANT_REF; xnfp->xnf_rx_ring_ref = INVALID_GRANT_REF; /* set driver private pointer now */ ddi_set_driver_private(devinfo, xnfp); if (!xnf_kstat_init(xnfp)) goto failure_3; /* * Allocate an event channel, add the interrupt handler and * bind it to the event channel. */ (void) xvdi_alloc_evtchn(devinfo); xnfp->xnf_evtchn = xvdi_get_evtchn(devinfo); #ifdef XPV_HVM_DRIVER ec_bind_evtchn_to_handler(xnfp->xnf_evtchn, IPL_VIF, xnf_intr, xnfp); #else (void) ddi_add_intr(devinfo, 0, NULL, NULL, xnf_intr, (caddr_t)xnfp); #endif err = mac_register(macp, &xnfp->xnf_mh); mac_free(macp); macp = NULL; if (err != 0) goto failure_4; if (xvdi_add_event_handler(devinfo, XS_OE_STATE, oe_state_change, NULL) != DDI_SUCCESS) goto failure_5; #ifdef XPV_HVM_DRIVER /* * In the HVM case, this driver essentially replaces a driver for * a 'real' PCI NIC. Without the "model" property set to * "Ethernet controller", like the PCI code does, netbooting does * not work correctly, as strplumb_get_netdev_path() will not find * this interface. */ (void) ndi_prop_update_string(DDI_DEV_T_NONE, devinfo, "model", "Ethernet controller"); #endif return (DDI_SUCCESS); failure_5: (void) mac_unregister(xnfp->xnf_mh); failure_4: #ifdef XPV_HVM_DRIVER ec_unbind_evtchn(xnfp->xnf_evtchn); xvdi_free_evtchn(devinfo); #else ddi_remove_intr(devinfo, 0, xnfp->xnf_icookie); #endif xnfp->xnf_evtchn = INVALID_EVTCHN; kstat_delete(xnfp->xnf_kstat_aux); failure_3: xnf_release_dma_resources(xnfp); failure_2: kmem_cache_destroy(xnfp->xnf_tx_buf_cache); failure_1: kmem_cache_destroy(xnfp->xnf_buf_cache); failure_0: cv_destroy(&xnfp->xnf_cv_tx_slots); cv_destroy(&xnfp->xnf_cv_multicast); cv_destroy(&xnfp->xnf_cv_state); mutex_destroy(&xnfp->xnf_gref_lock); mutex_destroy(&xnfp->xnf_schedlock); mutex_destroy(&xnfp->xnf_rxlock); mutex_destroy(&xnfp->xnf_txlock); failure: kmem_free(xnfp, sizeof (*xnfp)); if (macp != NULL) mac_free(macp); return (DDI_FAILURE); } /* detach(9E) -- Detach a device from the system */ static int xnf_detach(dev_info_t *devinfo, ddi_detach_cmd_t cmd) { xnf_t *xnfp; /* Our private device info */ xnfp = ddi_get_driver_private(devinfo); switch (cmd) { case DDI_SUSPEND: #ifdef XPV_HVM_DRIVER ec_unbind_evtchn(xnfp->xnf_evtchn); xvdi_free_evtchn(devinfo); #else ddi_remove_intr(devinfo, 0, xnfp->xnf_icookie); #endif xvdi_suspend(devinfo); mutex_enter(&xnfp->xnf_rxlock); mutex_enter(&xnfp->xnf_txlock); xnfp->xnf_evtchn = INVALID_EVTCHN; xnfp->xnf_connected = B_FALSE; mutex_exit(&xnfp->xnf_txlock); mutex_exit(&xnfp->xnf_rxlock); /* claim link to be down after disconnect */ mac_link_update(xnfp->xnf_mh, LINK_STATE_DOWN); return (DDI_SUCCESS); case DDI_DETACH: break; default: return (DDI_FAILURE); } if (xnfp->xnf_connected) return (DDI_FAILURE); /* * Cannot detach if we have xnf_buf_t outstanding. */ if (xnfp->xnf_stat_buf_allocated > 0) return (DDI_FAILURE); if (mac_unregister(xnfp->xnf_mh) != 0) return (DDI_FAILURE); kstat_delete(xnfp->xnf_kstat_aux); /* Stop the receiver */ xnf_stop(xnfp); xvdi_remove_event_handler(devinfo, XS_OE_STATE); /* Remove the interrupt */ #ifdef XPV_HVM_DRIVER ec_unbind_evtchn(xnfp->xnf_evtchn); xvdi_free_evtchn(devinfo); #else ddi_remove_intr(devinfo, 0, xnfp->xnf_icookie); #endif /* Release any pending xmit mblks */ xnf_release_mblks(xnfp); /* Release all DMA resources */ xnf_release_dma_resources(xnfp); cv_destroy(&xnfp->xnf_cv_tx_slots); cv_destroy(&xnfp->xnf_cv_multicast); cv_destroy(&xnfp->xnf_cv_state); kmem_cache_destroy(xnfp->xnf_tx_buf_cache); kmem_cache_destroy(xnfp->xnf_buf_cache); mutex_destroy(&xnfp->xnf_gref_lock); mutex_destroy(&xnfp->xnf_schedlock); mutex_destroy(&xnfp->xnf_rxlock); mutex_destroy(&xnfp->xnf_txlock); kmem_free(xnfp, sizeof (*xnfp)); return (DDI_SUCCESS); } /* * xnf_set_mac_addr() -- set the physical network address on the board. */ static int xnf_set_mac_addr(void *arg, const uint8_t *macaddr) { _NOTE(ARGUNUSED(arg, macaddr)); /* * We can't set our macaddr. */ return (ENOTSUP); } /* * xnf_set_multicast() -- set (enable) or disable a multicast address. * * Program the hardware to enable/disable the multicast address * in "mca". Enable if "add" is true, disable if false. */ static int xnf_set_multicast(void *arg, boolean_t add, const uint8_t *mca) { xnf_t *xnfp = arg; xnf_txbuf_t *txp; int n_slots; RING_IDX slot; xnf_txid_t *tidp; netif_tx_request_t *txrp; struct netif_extra_info *erp; boolean_t notify, result; /* * If the backend does not support multicast control then we * must assume that the right packets will just arrive. */ if (!xnfp->xnf_be_mcast_control) return (0); txp = kmem_cache_alloc(xnfp->xnf_tx_buf_cache, KM_SLEEP); mutex_enter(&xnfp->xnf_txlock); /* * If we're not yet connected then claim success. This is * acceptable because we refresh the entire set of multicast * addresses when we get connected. * * We can't wait around here because the MAC layer expects * this to be a non-blocking operation - waiting ends up * causing a deadlock during resume. */ if (!xnfp->xnf_connected) { mutex_exit(&xnfp->xnf_txlock); return (0); } /* * 1. Acquire two slots in the ring. * 2. Fill in the slots. * 3. Request notification when the operation is done. * 4. Kick the peer. * 5. Wait for the response via xnf_tx_clean_ring(). */ n_slots = xnf_tx_slots_get(xnfp, 2, B_TRUE); ASSERT(n_slots >= 2); slot = xnfp->xnf_tx_ring.req_prod_pvt; tidp = xnf_txid_get(xnfp); VERIFY(tidp != NULL); txp->tx_type = TX_MCAST_REQ; txp->tx_slot = slot; txrp = RING_GET_REQUEST(&xnfp->xnf_tx_ring, slot); erp = (struct netif_extra_info *) RING_GET_REQUEST(&xnfp->xnf_tx_ring, slot + 1); txrp->gref = 0; txrp->size = 0; txrp->offset = 0; /* Set tx_txreq.id to appease xnf_tx_clean_ring(). */ txrp->id = txp->tx_txreq.id = tidp->id; txrp->flags = NETTXF_extra_info; erp->type = add ? XEN_NETIF_EXTRA_TYPE_MCAST_ADD : XEN_NETIF_EXTRA_TYPE_MCAST_DEL; bcopy((void *)mca, &erp->u.mcast.addr, ETHERADDRL); tidp->txbuf = txp; xnfp->xnf_tx_ring.req_prod_pvt = slot + 2; mutex_enter(&xnfp->xnf_schedlock); xnfp->xnf_pending_multicast++; mutex_exit(&xnfp->xnf_schedlock); /* LINTED: constant in conditional context */ RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&xnfp->xnf_tx_ring, notify); if (notify) ec_notify_via_evtchn(xnfp->xnf_evtchn); while (txp->tx_type == TX_MCAST_REQ) cv_wait(&xnfp->xnf_cv_multicast, &xnfp->xnf_txlock); ASSERT3U(txp->tx_type, ==, TX_MCAST_RSP); mutex_enter(&xnfp->xnf_schedlock); xnfp->xnf_pending_multicast--; mutex_exit(&xnfp->xnf_schedlock); result = (txp->tx_status == NETIF_RSP_OKAY); xnf_txid_put(xnfp, tidp); mutex_exit(&xnfp->xnf_txlock); kmem_cache_free(xnfp->xnf_tx_buf_cache, txp); return (result ? 0 : 1); } /* * xnf_set_promiscuous() -- set or reset promiscuous mode on the board * * Program the hardware to enable/disable promiscuous mode. */ static int xnf_set_promiscuous(void *arg, boolean_t on) { _NOTE(ARGUNUSED(arg, on)); /* * We can't really do this, but we pretend that we can in * order that snoop will work. */ return (0); } /* * Clean buffers that we have responses for from the transmit ring. */ static int xnf_tx_clean_ring(xnf_t *xnfp) { boolean_t work_to_do; ASSERT(MUTEX_HELD(&xnfp->xnf_txlock)); loop: while (RING_HAS_UNCONSUMED_RESPONSES(&xnfp->xnf_tx_ring)) { RING_IDX cons, prod, i; cons = xnfp->xnf_tx_ring.rsp_cons; prod = xnfp->xnf_tx_ring.sring->rsp_prod; membar_consumer(); /* * Clean tx requests from ring that we have responses * for. */ DTRACE_PROBE2(xnf_tx_clean_range, int, cons, int, prod); for (i = cons; i != prod; i++) { netif_tx_response_t *trp; xnf_txid_t *tidp; xnf_txbuf_t *txp; trp = RING_GET_RESPONSE(&xnfp->xnf_tx_ring, i); /* * if this slot was occupied by netif_extra_info_t, * then the response will be NETIF_RSP_NULL. In this * case there are no resources to clean up. */ if (trp->status == NETIF_RSP_NULL) continue; ASSERT(TX_ID_VALID(trp->id)); tidp = TX_ID_TO_TXID(xnfp, trp->id); ASSERT3U(tidp->id, ==, trp->id); ASSERT3U(tidp->next, ==, INVALID_TX_ID); txp = tidp->txbuf; ASSERT(txp != NULL); ASSERT3U(txp->tx_txreq.id, ==, trp->id); switch (txp->tx_type) { case TX_DATA: /* * We must put the txid for each response we * acknowledge to make sure that we never have * more free slots than txids. Because of this * we do it here instead of waiting for it to * be done in xnf_data_txbuf_free_chain(). */ xnf_txid_put(xnfp, tidp); txp->tx_txreq.id = INVALID_TX_ID; ASSERT3S(txp->tx_head->tx_frags_to_ack, >, 0); txp->tx_head->tx_frags_to_ack--; /* * We clean the whole chain once we got a * response for each fragment. */ if (txp->tx_head->tx_frags_to_ack == 0) xnf_data_txbuf_free_chain(xnfp, txp); break; case TX_MCAST_REQ: txp->tx_type = TX_MCAST_RSP; txp->tx_status = trp->status; cv_broadcast(&xnfp->xnf_cv_multicast); break; default: cmn_err(CE_PANIC, "xnf_tx_clean_ring: " "invalid xnf_txbuf_t type: %d", txp->tx_type); break; } } /* * Record the last response we dealt with so that we * know where to start next time around. */ xnfp->xnf_tx_ring.rsp_cons = prod; membar_enter(); } /* LINTED: constant in conditional context */ RING_FINAL_CHECK_FOR_RESPONSES(&xnfp->xnf_tx_ring, work_to_do); if (work_to_do) goto loop; return (RING_FREE_REQUESTS(&xnfp->xnf_tx_ring)); } /* * Allocate and fill in a look-aside buffer for the packet `mp'. Used * to ensure that the packet is physically contiguous and contained * within a single page. */ static xnf_buf_t * xnf_tx_get_lookaside(xnf_t *xnfp, mblk_t *mp, size_t *plen) { xnf_buf_t *bd; caddr_t bp; if ((bd = xnf_buf_get(xnfp, KM_NOSLEEP, B_TRUE)) == NULL) { return (NULL); } bp = bd->buf; while (mp != NULL) { size_t len = MBLKL(mp); bcopy(mp->b_rptr, bp, len); bp += len; mp = mp->b_cont; } *plen = bp - bd->buf; ASSERT3U(*plen, <=, PAGESIZE); xnfp->xnf_stat_tx_lookaside++; return (bd); } /* * Insert the pseudo-header checksum into the packet. * Assumes packet is IPv4, TCP/UDP since we only advertised support for * HCKSUM_INET_FULL_V4. */ int xnf_pseudo_cksum(mblk_t *mp) { struct ether_header *ehp; uint16_t sap, iplen, *stuff; uint32_t cksum; size_t len; ipha_t *ipha; ipaddr_t src, dst; uchar_t *ptr; ptr = mp->b_rptr; len = MBLKL(mp); /* Each header must fit completely in an mblk. */ ASSERT3U(len, >=, sizeof (*ehp)); ehp = (struct ether_header *)ptr; if (ntohs(ehp->ether_type) == VLAN_TPID) { struct ether_vlan_header *evhp; ASSERT3U(len, >=, sizeof (*evhp)); evhp = (struct ether_vlan_header *)ptr; sap = ntohs(evhp->ether_type); ptr += sizeof (*evhp); len -= sizeof (*evhp); } else { sap = ntohs(ehp->ether_type); ptr += sizeof (*ehp); len -= sizeof (*ehp); } ASSERT3U(sap, ==, ETHERTYPE_IP); /* * Ethernet and IP headers may be in different mblks. */ ASSERT3P(ptr, <=, mp->b_wptr); if (ptr == mp->b_wptr) { mp = mp->b_cont; ptr = mp->b_rptr; len = MBLKL(mp); } ASSERT3U(len, >=, sizeof (ipha_t)); ipha = (ipha_t *)ptr; /* * We assume the IP header has no options. (This is enforced in * ire_send_wire_v4() -- search for IXAF_NO_HW_CKSUM). */ ASSERT3U(IPH_HDR_LENGTH(ipha), ==, IP_SIMPLE_HDR_LENGTH); iplen = ntohs(ipha->ipha_length) - IP_SIMPLE_HDR_LENGTH; ptr += IP_SIMPLE_HDR_LENGTH; len -= IP_SIMPLE_HDR_LENGTH; /* * IP and L4 headers may be in different mblks. */ ASSERT3P(ptr, <=, mp->b_wptr); if (ptr == mp->b_wptr) { mp = mp->b_cont; ptr = mp->b_rptr; len = MBLKL(mp); } switch (ipha->ipha_protocol) { case IPPROTO_TCP: ASSERT3U(len, >=, sizeof (tcph_t)); stuff = (uint16_t *)(ptr + TCP_CHECKSUM_OFFSET); cksum = IP_TCP_CSUM_COMP; break; case IPPROTO_UDP: ASSERT3U(len, >=, sizeof (struct udphdr)); stuff = (uint16_t *)(ptr + UDP_CHECKSUM_OFFSET); cksum = IP_UDP_CSUM_COMP; break; default: cmn_err(CE_WARN, "xnf_pseudo_cksum: unexpected protocol %d", ipha->ipha_protocol); return (EINVAL); } src = ipha->ipha_src; dst = ipha->ipha_dst; cksum += (dst >> 16) + (dst & 0xFFFF); cksum += (src >> 16) + (src & 0xFFFF); cksum += htons(iplen); cksum = (cksum >> 16) + (cksum & 0xFFFF); cksum = (cksum >> 16) + (cksum & 0xFFFF); ASSERT(cksum <= 0xFFFF); *stuff = (uint16_t)(cksum ? cksum : ~cksum); return (0); } /* * Push a packet into the transmit ring. * * Note: the format of a tx packet that spans multiple slots is similar to * what is described in xnf_rx_one_packet(). */ static void xnf_tx_push_packet(xnf_t *xnfp, xnf_txbuf_t *head) { int nslots = 0; int extras = 0; RING_IDX slot; boolean_t notify; ASSERT(MUTEX_HELD(&xnfp->xnf_txlock)); ASSERT(xnfp->xnf_running); slot = xnfp->xnf_tx_ring.req_prod_pvt; /* * The caller has already checked that we have enough slots to proceed. */ for (xnf_txbuf_t *txp = head; txp != NULL; txp = txp->tx_next) { xnf_txid_t *tidp; netif_tx_request_t *txrp; tidp = xnf_txid_get(xnfp); VERIFY(tidp != NULL); txrp = RING_GET_REQUEST(&xnfp->xnf_tx_ring, slot); txp->tx_slot = slot; txp->tx_txreq.id = tidp->id; *txrp = txp->tx_txreq; tidp->txbuf = txp; slot++; nslots++; /* * When present, LSO info is placed in a slot after the first * data segment, and doesn't require a txid. */ if (txp->tx_txreq.flags & NETTXF_extra_info) { netif_extra_info_t *extra; ASSERT3U(nslots, ==, 1); extra = (netif_extra_info_t *) RING_GET_REQUEST(&xnfp->xnf_tx_ring, slot); *extra = txp->tx_extra; slot++; nslots++; extras = 1; } } ASSERT3U(nslots, <=, XEN_MAX_SLOTS_PER_TX); /* * Store the number of data fragments. */ head->tx_frags_to_ack = nslots - extras; xnfp->xnf_tx_ring.req_prod_pvt = slot; /* * Tell the peer that we sent something, if it cares. */ /* LINTED: constant in conditional context */ RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&xnfp->xnf_tx_ring, notify); if (notify) ec_notify_via_evtchn(xnfp->xnf_evtchn); } static xnf_txbuf_t * xnf_mblk_copy(xnf_t *xnfp, mblk_t *mp) { xnf_txbuf_t *txp; size_t length; if ((txp = xnf_data_txbuf_alloc(xnfp, KM_NOSLEEP)) == NULL) { return (NULL); } txp->tx_bdesc = xnf_tx_get_lookaside(xnfp, mp, &length); if (txp->tx_bdesc == NULL) { xnf_data_txbuf_free(xnfp, txp); return (NULL); } txp->tx_mfn = txp->tx_bdesc->buf_mfn; txp->tx_txreq.gref = txp->tx_bdesc->grant_ref; txp->tx_txreq.size = length; txp->tx_txreq.offset = (uintptr_t)txp->tx_bdesc->buf & PAGEOFFSET; txp->tx_txreq.flags = 0; return (txp); } static xnf_txbuf_t * xnf_mblk_map(xnf_t *xnfp, mblk_t *mp, int *countp) { xnf_txbuf_t *head = NULL; xnf_txbuf_t *tail = NULL; domid_t oeid; int nsegs = 0; oeid = xvdi_get_oeid(xnfp->xnf_devinfo); for (mblk_t *ml = mp; ml != NULL; ml = ml->b_cont) { ddi_dma_handle_t dma_handle; const ddi_dma_cookie_t *dma_cookie, *dma_cookie_prev; xnf_txbuf_t *txp; if (MBLKL(ml) == 0) continue; if ((txp = xnf_data_txbuf_alloc(xnfp, KM_NOSLEEP)) == NULL) { goto error; } if (head == NULL) { head = txp; } else { ASSERT(tail != NULL); TXBUF_SETNEXT(tail, txp); txp->tx_head = head; } /* * The necessary segmentation rules (e.g. not crossing a page * boundary) are enforced by the dma attributes of the handle. */ dma_handle = txp->tx_dma_handle; int ret = ddi_dma_addr_bind_handle(dma_handle, NULL, (char *)ml->b_rptr, MBLKL(ml), DDI_DMA_WRITE | DDI_DMA_STREAMING, DDI_DMA_DONTWAIT, 0, NULL, NULL); if (ret != DDI_DMA_MAPPED) { if (ret != DDI_DMA_NORESOURCES) { dev_err(xnfp->xnf_devinfo, CE_WARN, "ddi_dma_addr_bind_handle() failed " "[dma_error=%d]", ret); } goto error; } txp->tx_handle_bound = B_TRUE; dma_cookie_prev = NULL; while ((dma_cookie = ddi_dma_cookie_iter(dma_handle, dma_cookie_prev)) != NULL) { if (nsegs == XEN_MAX_TX_DATA_PAGES) { dev_err(xnfp->xnf_devinfo, CE_WARN, "xnf_dmamap_alloc() failed: " "too many segments"); goto error; } if (dma_cookie_prev != NULL) { if ((txp = xnf_data_txbuf_alloc(xnfp, KM_NOSLEEP)) == NULL) { goto error; } ASSERT(tail != NULL); TXBUF_SETNEXT(tail, txp); txp->tx_head = head; } txp->tx_mfn = xnf_btop(pa_to_ma(dma_cookie->dmac_laddress)); txp->tx_txreq.gref = xnf_gref_get(xnfp); if (txp->tx_txreq.gref == INVALID_GRANT_REF) { dev_err(xnfp->xnf_devinfo, CE_WARN, "xnf_dmamap_alloc() failed: " "invalid grant ref"); goto error; } gnttab_grant_foreign_access_ref(txp->tx_txreq.gref, oeid, txp->tx_mfn, 1); txp->tx_txreq.offset = dma_cookie->dmac_laddress & PAGEOFFSET; txp->tx_txreq.size = dma_cookie->dmac_size; txp->tx_txreq.flags = 0; nsegs++; if (tail != NULL) tail->tx_txreq.flags = NETTXF_more_data; tail = txp; dma_cookie_prev = dma_cookie; } } *countp = nsegs; return (head); error: xnf_data_txbuf_free_chain(xnfp, head); return (NULL); } static void xnf_tx_setup_offload(xnf_t *xnfp, xnf_txbuf_t *head, uint32_t cksum_flags, uint32_t lso_flags, uint32_t mss) { if (lso_flags != 0) { ASSERT3U(lso_flags, ==, HW_LSO); ASSERT3P(head->tx_bdesc, ==, NULL); head->tx_txreq.flags |= NETTXF_extra_info; netif_extra_info_t *extra = &head->tx_extra; extra->type = XEN_NETIF_EXTRA_TYPE_GSO; extra->flags = 0; extra->u.gso.type = XEN_NETIF_GSO_TYPE_TCPV4; extra->u.gso.size = mss; extra->u.gso.features = 0; extra->u.gso.pad = 0; } else if (cksum_flags != 0) { ASSERT3U(cksum_flags, ==, HCK_FULLCKSUM); /* * If the local protocol stack requests checksum * offload we set the 'checksum blank' flag, * indicating to the peer that we need the checksum * calculated for us. * * We _don't_ set the validated flag, because we haven't * validated that the data and the checksum match. * * Note: we already called xnf_pseudo_cksum() in * xnf_send(), so we just set the txreq flag here. */ head->tx_txreq.flags |= NETTXF_csum_blank; xnfp->xnf_stat_tx_cksum_deferred++; } } /* * Send packet mp. Called by the MAC framework. */ static mblk_t * xnf_send(void *arg, mblk_t *mp) { xnf_t *xnfp = arg; xnf_txbuf_t *head; mblk_t *ml; int length; int pages, chunks, slots, slots_free; uint32_t cksum_flags, lso_flags, mss; boolean_t pulledup = B_FALSE; boolean_t force_copy = B_FALSE; ASSERT3P(mp->b_next, ==, NULL); mutex_enter(&xnfp->xnf_txlock); /* * Wait until we are connected to the backend. */ while (!xnfp->xnf_connected) cv_wait(&xnfp->xnf_cv_state, &xnfp->xnf_txlock); /* * To simplify logic and be in sync with the rescheduling mechanism, * we require the maximum amount of slots that could be used by a * transaction to be free before proceeding. The only downside of doing * this is that it slightly reduces the effective size of the ring. */ slots_free = xnf_tx_slots_get(xnfp, XEN_MAX_SLOTS_PER_TX, B_FALSE); if (slots_free < XEN_MAX_SLOTS_PER_TX) { /* * We need to ask for a re-schedule later as the ring is full. */ mutex_enter(&xnfp->xnf_schedlock); xnfp->xnf_need_sched = B_TRUE; mutex_exit(&xnfp->xnf_schedlock); xnfp->xnf_stat_tx_defer++; mutex_exit(&xnfp->xnf_txlock); return (mp); } /* * Get hw offload parameters. * This must be done before pulling up the mp as those parameters * are not copied over. */ mac_hcksum_get(mp, NULL, NULL, NULL, NULL, &cksum_flags); mac_lso_get(mp, &mss, &lso_flags); /* * XXX: fix MAC framework so that we can advertise support for * partial checksum for IPv4 only. This way we won't need to calculate * the pseudo header checksum ourselves. */ if (cksum_flags != 0) { ASSERT3U(cksum_flags, ==, HCK_FULLCKSUM); (void) xnf_pseudo_cksum(mp); } pulledup: for (ml = mp, pages = 0, chunks = 0, length = 0; ml != NULL; ml = ml->b_cont, chunks++) { pages += xnf_mblk_pages(ml); length += MBLKL(ml); } DTRACE_PROBE3(packet, int, length, int, chunks, int, pages); DTRACE_PROBE3(lso, int, length, uint32_t, lso_flags, uint32_t, mss); /* * If the ethernet header crosses a page boundary the packet * will be dropped by the backend. In practice it seems like * this happens fairly rarely so we'll do nothing unless the * packet is small enough to fit in a look-aside buffer. */ if (((uintptr_t)mp->b_rptr & PAGEOFFSET) + sizeof (struct ether_header) > PAGESIZE) { xnfp->xnf_stat_tx_eth_hdr_split++; if (length <= PAGESIZE) force_copy = B_TRUE; } if (force_copy || (pages > 1 && !xnfp->xnf_be_tx_sg)) { /* * If the packet spans several pages and scatter-gather is not * supported then use a look-aside buffer. */ ASSERT3U(length, <=, PAGESIZE); head = xnf_mblk_copy(xnfp, mp); if (head == NULL) { dev_err(xnfp->xnf_devinfo, CE_WARN, "xnf_mblk_copy() failed"); goto drop; } } else { /* * There's a limit for how many pages can be passed to the * backend. If we pass that limit, the packet will be dropped * and some backend implementations (e.g. Linux) could even * offline the interface. */ if (pages > XEN_MAX_TX_DATA_PAGES) { if (pulledup) { dev_err(xnfp->xnf_devinfo, CE_WARN, "too many pages, even after pullup: %d.", pages); goto drop; } /* * Defragment packet if it spans too many pages. */ mblk_t *newmp = msgpullup(mp, -1); if (newmp == NULL) { dev_err(xnfp->xnf_devinfo, CE_WARN, "msgpullup() failed"); goto drop; } freemsg(mp); mp = newmp; xnfp->xnf_stat_tx_pullup++; pulledup = B_TRUE; goto pulledup; } head = xnf_mblk_map(xnfp, mp, &slots); if (head == NULL) goto drop; IMPLY(slots > 1, xnfp->xnf_be_tx_sg); } /* * Set tx_mp so that mblk is freed when the txbuf chain is freed. */ head->tx_mp = mp; xnf_tx_setup_offload(xnfp, head, cksum_flags, lso_flags, mss); /* * The first request must store the total length of the packet. */ head->tx_txreq.size = length; /* * Push the packet we have prepared into the ring. */ xnf_tx_push_packet(xnfp, head); xnfp->xnf_stat_opackets++; xnfp->xnf_stat_obytes += length; mutex_exit(&xnfp->xnf_txlock); return (NULL); drop: freemsg(mp); xnfp->xnf_stat_tx_drop++; mutex_exit(&xnfp->xnf_txlock); return (NULL); } /* * Notification of RX packets. Currently no TX-complete interrupt is * used, as we clean the TX ring lazily. */ static uint_t xnf_intr(caddr_t arg) { xnf_t *xnfp = (xnf_t *)arg; mblk_t *mp; boolean_t need_sched, clean_ring; mutex_enter(&xnfp->xnf_rxlock); /* * Interrupts before we are connected are spurious. */ if (!xnfp->xnf_connected) { mutex_exit(&xnfp->xnf_rxlock); xnfp->xnf_stat_unclaimed_interrupts++; return (DDI_INTR_UNCLAIMED); } /* * Receive side processing. */ do { /* * Collect buffers from the ring. */ xnf_rx_collect(xnfp); /* * Interrupt me when the next receive buffer is consumed. */ xnfp->xnf_rx_ring.sring->rsp_event = xnfp->xnf_rx_ring.rsp_cons + 1; xen_mb(); } while (RING_HAS_UNCONSUMED_RESPONSES(&xnfp->xnf_rx_ring)); if (xnfp->xnf_rx_new_buffers_posted) { boolean_t notify; /* * Indicate to the peer that we have re-filled the * receive ring, if it cares. */ /* LINTED: constant in conditional context */ RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&xnfp->xnf_rx_ring, notify); if (notify) ec_notify_via_evtchn(xnfp->xnf_evtchn); xnfp->xnf_rx_new_buffers_posted = B_FALSE; } mp = xnfp->xnf_rx_head; xnfp->xnf_rx_head = xnfp->xnf_rx_tail = NULL; xnfp->xnf_stat_interrupts++; mutex_exit(&xnfp->xnf_rxlock); if (mp != NULL) mac_rx(xnfp->xnf_mh, NULL, mp); /* * Transmit side processing. * * If a previous transmit attempt failed or we have pending * multicast requests, clean the ring. * * If we previously stalled transmission and cleaning produces * some free slots, tell upstream to attempt sending again. * * The odd style is to avoid acquiring xnf_txlock unless we * will actually look inside the tx machinery. */ mutex_enter(&xnfp->xnf_schedlock); need_sched = xnfp->xnf_need_sched; clean_ring = need_sched || (xnfp->xnf_pending_multicast > 0); mutex_exit(&xnfp->xnf_schedlock); if (clean_ring) { int free_slots; mutex_enter(&xnfp->xnf_txlock); free_slots = xnf_tx_slots_get(xnfp, 0, B_FALSE); if (need_sched && (free_slots >= XEN_MAX_SLOTS_PER_TX)) { mutex_enter(&xnfp->xnf_schedlock); xnfp->xnf_need_sched = B_FALSE; mutex_exit(&xnfp->xnf_schedlock); mac_tx_update(xnfp->xnf_mh); } mutex_exit(&xnfp->xnf_txlock); } return (DDI_INTR_CLAIMED); } /* * xnf_start() -- start the board receiving and enable interrupts. */ static int xnf_start(void *arg) { xnf_t *xnfp = arg; mutex_enter(&xnfp->xnf_rxlock); mutex_enter(&xnfp->xnf_txlock); /* Accept packets from above. */ xnfp->xnf_running = B_TRUE; mutex_exit(&xnfp->xnf_txlock); mutex_exit(&xnfp->xnf_rxlock); return (0); } /* xnf_stop() - disable hardware */ static void xnf_stop(void *arg) { xnf_t *xnfp = arg; mutex_enter(&xnfp->xnf_rxlock); mutex_enter(&xnfp->xnf_txlock); xnfp->xnf_running = B_FALSE; mutex_exit(&xnfp->xnf_txlock); mutex_exit(&xnfp->xnf_rxlock); } /* * Hang buffer `bdesc' on the RX ring. */ static void xnf_rxbuf_hang(xnf_t *xnfp, xnf_buf_t *bdesc) { netif_rx_request_t *reqp; RING_IDX hang_ix; ASSERT(MUTEX_HELD(&xnfp->xnf_rxlock)); reqp = RING_GET_REQUEST(&xnfp->xnf_rx_ring, xnfp->xnf_rx_ring.req_prod_pvt); hang_ix = (RING_IDX) (reqp - RING_GET_REQUEST(&xnfp->xnf_rx_ring, 0)); ASSERT(xnfp->xnf_rx_pkt_info[hang_ix] == NULL); reqp->id = bdesc->id = hang_ix; reqp->gref = bdesc->grant_ref; xnfp->xnf_rx_pkt_info[hang_ix] = bdesc; xnfp->xnf_rx_ring.req_prod_pvt++; xnfp->xnf_rx_new_buffers_posted = B_TRUE; } /* * Receive an entire packet from the ring, starting from slot *consp. * prod indicates the slot of the latest response. * On return, *consp will point to the head of the next packet. * * Note: If slot prod was reached before we could gather a full packet, we will * drop the partial packet; this would most likely indicate a bug in either * the front-end or the back-end driver. * * An rx packet can consist of several fragments and thus span multiple slots. * Each fragment can contain up to 4k of data. * * A typical 9000 MTU packet with look like this: * +------+---------------------+-------------------+-----------------------+ * | SLOT | TYPE | CONTENTS | FLAGS | * +------+---------------------+-------------------+-----------------------+ * | 1 | netif_rx_response_t | 1st data fragment | more_data | * +------+---------------------+-------------------+-----------------------+ * | 2 | netif_rx_response_t | 2nd data fragment | more_data | * +------+---------------------+-------------------+-----------------------+ * | 3 | netif_rx_response_t | 3rd data fragment | [none] | * +------+---------------------+-------------------+-----------------------+ * * Fragments are chained by setting NETRXF_more_data in the previous * response's flags. If there are additional flags, such as * NETRXF_data_validated or NETRXF_extra_info, those should be set on the * first fragment. * * Sometimes extra info can be present. If so, it will follow the first * fragment, and NETRXF_extra_info flag will be set on the first response. * If LRO is set on a packet, it will be stored in the extra info. Conforming * to the spec, extra info can also be chained, but must all be present right * after the first fragment. * * Example of a packet with 2 extra infos: * +------+---------------------+-------------------+-----------------------+ * | SLOT | TYPE | CONTENTS | FLAGS | * +------+---------------------+-------------------+-----------------------+ * | 1 | netif_rx_response_t | 1st data fragment | extra_info, more_data | * +------+---------------------+-------------------+-----------------------+ * | 2 | netif_extra_info_t | 1st extra info | EXTRA_FLAG_MORE | * +------+---------------------+-------------------+-----------------------+ * | 3 | netif_extra_info_t | 2nd extra info | [none] | * +------+---------------------+-------------------+-----------------------+ * | 4 | netif_rx_response_t | 2nd data fragment | more_data | * +------+---------------------+-------------------+-----------------------+ * | 5 | netif_rx_response_t | 3rd data fragment | more_data | * +------+---------------------+-------------------+-----------------------+ * | 6 | netif_rx_response_t | 4th data fragment | [none] | * +------+---------------------+-------------------+-----------------------+ * * In practice, the only extra we expect is for LRO, but only if we advertise * that we support it to the backend (xnf_enable_lro == TRUE). */ static int xnf_rx_one_packet(xnf_t *xnfp, RING_IDX prod, RING_IDX *consp, mblk_t **mpp) { mblk_t *head = NULL; mblk_t *tail = NULL; mblk_t *mp; int error = 0; RING_IDX cons = *consp; netif_extra_info_t lro; boolean_t is_lro = B_FALSE; boolean_t is_extra = B_FALSE; netif_rx_response_t rsp = *RING_GET_RESPONSE(&xnfp->xnf_rx_ring, cons); boolean_t hwcsum = (rsp.flags & NETRXF_data_validated) != 0; boolean_t more_data = (rsp.flags & NETRXF_more_data) != 0; boolean_t more_extra = (rsp.flags & NETRXF_extra_info) != 0; IMPLY(more_data, xnf_enable_rx_sg); while (cons != prod) { xnf_buf_t *bdesc; int len, off; int rxidx = cons & (NET_RX_RING_SIZE - 1); bdesc = xnfp->xnf_rx_pkt_info[rxidx]; xnfp->xnf_rx_pkt_info[rxidx] = NULL; if (is_extra) { netif_extra_info_t *extra = (netif_extra_info_t *)&rsp; /* * The only extra we expect is for LRO, and it should * only be present once. */ if (extra->type == XEN_NETIF_EXTRA_TYPE_GSO && !is_lro) { ASSERT(xnf_enable_lro); lro = *extra; is_lro = B_TRUE; DTRACE_PROBE1(lro, netif_extra_info_t *, &lro); } else { dev_err(xnfp->xnf_devinfo, CE_WARN, "rx packet " "contains unexpected extra info of type %d", extra->type); error = EINVAL; } more_extra = (extra->flags & XEN_NETIF_EXTRA_FLAG_MORE) != 0; goto hang_buf; } ASSERT3U(bdesc->id, ==, rsp.id); /* * status stores packet length when >= 0, or errors when < 0. */ len = rsp.status; off = rsp.offset; more_data = (rsp.flags & NETRXF_more_data) != 0; /* * sanity checks. */ if (!xnfp->xnf_running) { error = EBUSY; } else if (len <= 0) { xnfp->xnf_stat_errrx++; switch (len) { case 0: xnfp->xnf_stat_runt++; break; case NETIF_RSP_ERROR: xnfp->xnf_stat_mac_rcv_error++; break; case NETIF_RSP_DROPPED: xnfp->xnf_stat_norxbuf++; break; } error = EINVAL; } else if (bdesc->grant_ref == INVALID_GRANT_REF) { dev_err(xnfp->xnf_devinfo, CE_WARN, "Bad rx grant reference, rsp id %d", rsp.id); error = EINVAL; } else if ((off + len) > PAGESIZE) { dev_err(xnfp->xnf_devinfo, CE_WARN, "Rx packet crosses " "page boundary (offset %d, length %d)", off, len); error = EINVAL; } if (error != 0) { /* * If an error has been detected, we do not attempt * to read the data but we still need to replace * the rx bufs. */ goto hang_buf; } xnf_buf_t *nbuf = NULL; /* * If the packet is below a pre-determined size we will * copy data out of the buf rather than replace it. */ if (len > xnf_rx_copy_limit) nbuf = xnf_buf_get(xnfp, KM_NOSLEEP, B_FALSE); if (nbuf != NULL) { mp = desballoc((unsigned char *)bdesc->buf, bdesc->len, 0, &bdesc->free_rtn); if (mp == NULL) { xnfp->xnf_stat_rx_desballoc_fail++; xnfp->xnf_stat_norxbuf++; error = ENOMEM; /* * we free the buf we just allocated as we * will re-hang the old buf. */ xnf_buf_put(xnfp, nbuf, B_FALSE); goto hang_buf; } mp->b_rptr = mp->b_rptr + off; mp->b_wptr = mp->b_rptr + len; /* * Release the grant as the backend doesn't need to * access this buffer anymore and grants are scarce. */ (void) gnttab_end_foreign_access_ref(bdesc->grant_ref, 0); xnf_gref_put(xnfp, bdesc->grant_ref); bdesc->grant_ref = INVALID_GRANT_REF; bdesc = nbuf; } else { /* * We failed to allocate a new buf or decided to reuse * the old one. In either case we copy the data off it * and put it back into the ring. */ mp = allocb(len, 0); if (mp == NULL) { xnfp->xnf_stat_rx_allocb_fail++; xnfp->xnf_stat_norxbuf++; error = ENOMEM; goto hang_buf; } bcopy(bdesc->buf + off, mp->b_wptr, len); mp->b_wptr += len; } if (head == NULL) head = mp; else tail->b_cont = mp; tail = mp; hang_buf: /* * No matter what happens, for each response we need to hang * a new buf on the rx ring. Put either the old one, or a new * one if the old one is borrowed by the kernel via desballoc(). */ xnf_rxbuf_hang(xnfp, bdesc); cons++; /* next response is an extra */ is_extra = more_extra; if (!more_data && !more_extra) break; /* * Note that since requests and responses are union'd on the * same ring, we copy the response to a local variable instead * of keeping a pointer. Otherwise xnf_rxbuf_hang() would have * overwritten contents of rsp. */ rsp = *RING_GET_RESPONSE(&xnfp->xnf_rx_ring, cons); } /* * Check that we do not get stuck in a loop. */ ASSERT3U(*consp, !=, cons); *consp = cons; /* * We ran out of responses but the flags indicate there is more data. */ if (more_data) { dev_err(xnfp->xnf_devinfo, CE_WARN, "rx: need more fragments."); error = EINVAL; } if (more_extra) { dev_err(xnfp->xnf_devinfo, CE_WARN, "rx: need more fragments " "(extras)."); error = EINVAL; } /* * An error means the packet must be dropped. If we have already formed * a partial packet, then discard it. */ if (error != 0) { if (head != NULL) freemsg(head); xnfp->xnf_stat_rx_drop++; return (error); } ASSERT(head != NULL); if (hwcsum) { /* * If the peer says that the data has been validated then we * declare that the full checksum has been verified. * * We don't look at the "checksum blank" flag, and hence could * have a packet here that we are asserting is good with * a blank checksum. */ mac_hcksum_set(head, 0, 0, 0, 0, HCK_FULLCKSUM_OK); xnfp->xnf_stat_rx_cksum_no_need++; } /* XXX: set lro info for packet once LRO is supported in OS. */ *mpp = head; return (0); } /* * Collect packets from the RX ring, storing them in `xnfp' for later use. */ static void xnf_rx_collect(xnf_t *xnfp) { RING_IDX prod; ASSERT(MUTEX_HELD(&xnfp->xnf_rxlock)); prod = xnfp->xnf_rx_ring.sring->rsp_prod; /* * Ensure we see queued responses up to 'prod'. */ membar_consumer(); while (xnfp->xnf_rx_ring.rsp_cons != prod) { mblk_t *mp; /* * Collect a packet. * rsp_cons is updated inside xnf_rx_one_packet(). */ int error = xnf_rx_one_packet(xnfp, prod, &xnfp->xnf_rx_ring.rsp_cons, &mp); if (error == 0) { xnfp->xnf_stat_ipackets++; xnfp->xnf_stat_rbytes += xmsgsize(mp); /* * Append the mblk to the rx list. */ if (xnfp->xnf_rx_head == NULL) { ASSERT3P(xnfp->xnf_rx_tail, ==, NULL); xnfp->xnf_rx_head = mp; } else { ASSERT(xnfp->xnf_rx_tail != NULL); xnfp->xnf_rx_tail->b_next = mp; } xnfp->xnf_rx_tail = mp; } } } /* * xnf_alloc_dma_resources() -- initialize the drivers structures */ static int xnf_alloc_dma_resources(xnf_t *xnfp) { dev_info_t *devinfo = xnfp->xnf_devinfo; size_t len; ddi_dma_cookie_t dma_cookie; uint_t ncookies; int rc; caddr_t rptr; /* * The code below allocates all the DMA data structures that * need to be released when the driver is detached. * * Allocate page for the transmit descriptor ring. */ if (ddi_dma_alloc_handle(devinfo, &ringbuf_dma_attr, DDI_DMA_SLEEP, 0, &xnfp->xnf_tx_ring_dma_handle) != DDI_SUCCESS) goto alloc_error; if (ddi_dma_mem_alloc(xnfp->xnf_tx_ring_dma_handle, PAGESIZE, &accattr, DDI_DMA_CONSISTENT, DDI_DMA_SLEEP, 0, &rptr, &len, &xnfp->xnf_tx_ring_dma_acchandle) != DDI_SUCCESS) { ddi_dma_free_handle(&xnfp->xnf_tx_ring_dma_handle); xnfp->xnf_tx_ring_dma_handle = NULL; goto alloc_error; } if ((rc = ddi_dma_addr_bind_handle(xnfp->xnf_tx_ring_dma_handle, NULL, rptr, PAGESIZE, DDI_DMA_RDWR | DDI_DMA_CONSISTENT, DDI_DMA_SLEEP, 0, &dma_cookie, &ncookies)) != DDI_DMA_MAPPED) { ddi_dma_mem_free(&xnfp->xnf_tx_ring_dma_acchandle); ddi_dma_free_handle(&xnfp->xnf_tx_ring_dma_handle); xnfp->xnf_tx_ring_dma_handle = NULL; xnfp->xnf_tx_ring_dma_acchandle = NULL; if (rc == DDI_DMA_NORESOURCES) goto alloc_error; else goto error; } ASSERT(ncookies == 1); bzero(rptr, PAGESIZE); /* LINTED: constant in conditional context */ SHARED_RING_INIT((netif_tx_sring_t *)rptr); /* LINTED: constant in conditional context */ FRONT_RING_INIT(&xnfp->xnf_tx_ring, (netif_tx_sring_t *)rptr, PAGESIZE); xnfp->xnf_tx_ring_phys_addr = dma_cookie.dmac_laddress; /* * Allocate page for the receive descriptor ring. */ if (ddi_dma_alloc_handle(devinfo, &ringbuf_dma_attr, DDI_DMA_SLEEP, 0, &xnfp->xnf_rx_ring_dma_handle) != DDI_SUCCESS) goto alloc_error; if (ddi_dma_mem_alloc(xnfp->xnf_rx_ring_dma_handle, PAGESIZE, &accattr, DDI_DMA_CONSISTENT, DDI_DMA_SLEEP, 0, &rptr, &len, &xnfp->xnf_rx_ring_dma_acchandle) != DDI_SUCCESS) { ddi_dma_free_handle(&xnfp->xnf_rx_ring_dma_handle); xnfp->xnf_rx_ring_dma_handle = NULL; goto alloc_error; } if ((rc = ddi_dma_addr_bind_handle(xnfp->xnf_rx_ring_dma_handle, NULL, rptr, PAGESIZE, DDI_DMA_RDWR | DDI_DMA_CONSISTENT, DDI_DMA_SLEEP, 0, &dma_cookie, &ncookies)) != DDI_DMA_MAPPED) { ddi_dma_mem_free(&xnfp->xnf_rx_ring_dma_acchandle); ddi_dma_free_handle(&xnfp->xnf_rx_ring_dma_handle); xnfp->xnf_rx_ring_dma_handle = NULL; xnfp->xnf_rx_ring_dma_acchandle = NULL; if (rc == DDI_DMA_NORESOURCES) goto alloc_error; else goto error; } ASSERT(ncookies == 1); bzero(rptr, PAGESIZE); /* LINTED: constant in conditional context */ SHARED_RING_INIT((netif_rx_sring_t *)rptr); /* LINTED: constant in conditional context */ FRONT_RING_INIT(&xnfp->xnf_rx_ring, (netif_rx_sring_t *)rptr, PAGESIZE); xnfp->xnf_rx_ring_phys_addr = dma_cookie.dmac_laddress; return (DDI_SUCCESS); alloc_error: cmn_err(CE_WARN, "xnf%d: could not allocate enough DMA memory", ddi_get_instance(xnfp->xnf_devinfo)); error: xnf_release_dma_resources(xnfp); return (DDI_FAILURE); } /* * Release all DMA resources in the opposite order from acquisition */ static void xnf_release_dma_resources(xnf_t *xnfp) { int i; /* * Free receive buffers which are currently associated with * descriptors. */ mutex_enter(&xnfp->xnf_rxlock); for (i = 0; i < NET_RX_RING_SIZE; i++) { xnf_buf_t *bp; if ((bp = xnfp->xnf_rx_pkt_info[i]) == NULL) continue; xnfp->xnf_rx_pkt_info[i] = NULL; xnf_buf_put(xnfp, bp, B_FALSE); } mutex_exit(&xnfp->xnf_rxlock); /* Free the receive ring buffer. */ if (xnfp->xnf_rx_ring_dma_acchandle != NULL) { (void) ddi_dma_unbind_handle(xnfp->xnf_rx_ring_dma_handle); ddi_dma_mem_free(&xnfp->xnf_rx_ring_dma_acchandle); ddi_dma_free_handle(&xnfp->xnf_rx_ring_dma_handle); xnfp->xnf_rx_ring_dma_acchandle = NULL; } /* Free the transmit ring buffer. */ if (xnfp->xnf_tx_ring_dma_acchandle != NULL) { (void) ddi_dma_unbind_handle(xnfp->xnf_tx_ring_dma_handle); ddi_dma_mem_free(&xnfp->xnf_tx_ring_dma_acchandle); ddi_dma_free_handle(&xnfp->xnf_tx_ring_dma_handle); xnfp->xnf_tx_ring_dma_acchandle = NULL; } } /* * Release any packets and associated structures used by the TX ring. */ static void xnf_release_mblks(xnf_t *xnfp) { RING_IDX i; xnf_txid_t *tidp; for (i = 0, tidp = &xnfp->xnf_tx_pkt_id[0]; i < NET_TX_RING_SIZE; i++, tidp++) { xnf_txbuf_t *txp = tidp->txbuf; if (txp != NULL) { ASSERT(txp->tx_mp != NULL); freemsg(txp->tx_mp); xnf_txid_put(xnfp, tidp); kmem_cache_free(xnfp->xnf_tx_buf_cache, txp); } } } static int xnf_buf_constructor(void *buf, void *arg, int kmflag) { int (*ddiflags)(caddr_t) = DDI_DMA_SLEEP; xnf_buf_t *bdesc = buf; xnf_t *xnfp = arg; ddi_dma_cookie_t dma_cookie; uint_t ncookies; size_t len; if (kmflag & KM_NOSLEEP) ddiflags = DDI_DMA_DONTWAIT; /* Allocate a DMA access handle for the buffer. */ if (ddi_dma_alloc_handle(xnfp->xnf_devinfo, &rx_buf_dma_attr, ddiflags, 0, &bdesc->dma_handle) != DDI_SUCCESS) goto failure; /* Allocate DMA-able memory for buffer. */ if (ddi_dma_mem_alloc(bdesc->dma_handle, PAGESIZE, &data_accattr, DDI_DMA_STREAMING, ddiflags, 0, &bdesc->buf, &len, &bdesc->acc_handle) != DDI_SUCCESS) goto failure_1; /* Bind to virtual address of buffer to get physical address. */ if (ddi_dma_addr_bind_handle(bdesc->dma_handle, NULL, bdesc->buf, len, DDI_DMA_RDWR | DDI_DMA_STREAMING, ddiflags, 0, &dma_cookie, &ncookies) != DDI_DMA_MAPPED) goto failure_2; ASSERT(ncookies == 1); bdesc->free_rtn.free_func = xnf_buf_recycle; bdesc->free_rtn.free_arg = (caddr_t)bdesc; bdesc->xnfp = xnfp; bdesc->buf_phys = dma_cookie.dmac_laddress; bdesc->buf_mfn = pfn_to_mfn(xnf_btop(bdesc->buf_phys)); bdesc->len = dma_cookie.dmac_size; bdesc->grant_ref = INVALID_GRANT_REF; bdesc->gen = xnfp->xnf_gen; atomic_inc_64(&xnfp->xnf_stat_buf_allocated); return (0); failure_2: ddi_dma_mem_free(&bdesc->acc_handle); failure_1: ddi_dma_free_handle(&bdesc->dma_handle); failure: ASSERT(kmflag & KM_NOSLEEP); /* Cannot fail for KM_SLEEP. */ return (-1); } static void xnf_buf_destructor(void *buf, void *arg) { xnf_buf_t *bdesc = buf; xnf_t *xnfp = arg; (void) ddi_dma_unbind_handle(bdesc->dma_handle); ddi_dma_mem_free(&bdesc->acc_handle); ddi_dma_free_handle(&bdesc->dma_handle); atomic_dec_64(&xnfp->xnf_stat_buf_allocated); } static xnf_buf_t * xnf_buf_get(xnf_t *xnfp, int flags, boolean_t readonly) { grant_ref_t gref; xnf_buf_t *bufp; /* * Usually grant references are more scarce than memory, so we * attempt to acquire a grant reference first. */ gref = xnf_gref_get(xnfp); if (gref == INVALID_GRANT_REF) return (NULL); bufp = kmem_cache_alloc(xnfp->xnf_buf_cache, flags); if (bufp == NULL) { xnf_gref_put(xnfp, gref); return (NULL); } ASSERT3U(bufp->grant_ref, ==, INVALID_GRANT_REF); bufp->grant_ref = gref; if (bufp->gen != xnfp->xnf_gen) xnf_buf_refresh(bufp); gnttab_grant_foreign_access_ref(bufp->grant_ref, xvdi_get_oeid(bufp->xnfp->xnf_devinfo), bufp->buf_mfn, readonly ? 1 : 0); atomic_inc_64(&xnfp->xnf_stat_buf_outstanding); return (bufp); } static void xnf_buf_put(xnf_t *xnfp, xnf_buf_t *bufp, boolean_t readonly) { if (bufp->grant_ref != INVALID_GRANT_REF) { (void) gnttab_end_foreign_access_ref( bufp->grant_ref, readonly ? 1 : 0); xnf_gref_put(xnfp, bufp->grant_ref); bufp->grant_ref = INVALID_GRANT_REF; } kmem_cache_free(xnfp->xnf_buf_cache, bufp); atomic_dec_64(&xnfp->xnf_stat_buf_outstanding); } /* * Refresh any cached data about a buffer after resume. */ static void xnf_buf_refresh(xnf_buf_t *bdesc) { bdesc->buf_mfn = pfn_to_mfn(xnf_btop(bdesc->buf_phys)); bdesc->gen = bdesc->xnfp->xnf_gen; } /* * Streams `freeb' routine for `xnf_buf_t' when used as transmit * look-aside buffers. */ static void xnf_buf_recycle(xnf_buf_t *bdesc) { xnf_t *xnfp = bdesc->xnfp; xnf_buf_put(xnfp, bdesc, B_TRUE); } static int xnf_tx_buf_constructor(void *buf, void *arg, int kmflag) { int (*ddiflags)(caddr_t) = DDI_DMA_SLEEP; xnf_txbuf_t *txp = buf; xnf_t *xnfp = arg; if (kmflag & KM_NOSLEEP) ddiflags = DDI_DMA_DONTWAIT; if (ddi_dma_alloc_handle(xnfp->xnf_devinfo, &tx_buf_dma_attr, ddiflags, 0, &txp->tx_dma_handle) != DDI_SUCCESS) { ASSERT(kmflag & KM_NOSLEEP); /* Cannot fail for KM_SLEEP. */ return (-1); } return (0); } static void xnf_tx_buf_destructor(void *buf, void *arg) { _NOTE(ARGUNUSED(arg)); xnf_txbuf_t *txp = buf; ddi_dma_free_handle(&txp->tx_dma_handle); } /* * Statistics. */ static char *xnf_aux_statistics[] = { "tx_cksum_deferred", "rx_cksum_no_need", "interrupts", "unclaimed_interrupts", "tx_pullup", "tx_lookaside", "tx_drop", "tx_eth_hdr_split", "buf_allocated", "buf_outstanding", "gref_outstanding", "gref_failure", "gref_peak", "rx_allocb_fail", "rx_desballoc_fail", }; static int xnf_kstat_aux_update(kstat_t *ksp, int flag) { xnf_t *xnfp; kstat_named_t *knp; if (flag != KSTAT_READ) return (EACCES); xnfp = ksp->ks_private; knp = ksp->ks_data; /* * Assignment order must match that of the names in * xnf_aux_statistics. */ (knp++)->value.ui64 = xnfp->xnf_stat_tx_cksum_deferred; (knp++)->value.ui64 = xnfp->xnf_stat_rx_cksum_no_need; (knp++)->value.ui64 = xnfp->xnf_stat_interrupts; (knp++)->value.ui64 = xnfp->xnf_stat_unclaimed_interrupts; (knp++)->value.ui64 = xnfp->xnf_stat_tx_pullup; (knp++)->value.ui64 = xnfp->xnf_stat_tx_lookaside; (knp++)->value.ui64 = xnfp->xnf_stat_tx_drop; (knp++)->value.ui64 = xnfp->xnf_stat_tx_eth_hdr_split; (knp++)->value.ui64 = xnfp->xnf_stat_buf_allocated; (knp++)->value.ui64 = xnfp->xnf_stat_buf_outstanding; (knp++)->value.ui64 = xnfp->xnf_stat_gref_outstanding; (knp++)->value.ui64 = xnfp->xnf_stat_gref_failure; (knp++)->value.ui64 = xnfp->xnf_stat_gref_peak; (knp++)->value.ui64 = xnfp->xnf_stat_rx_allocb_fail; (knp++)->value.ui64 = xnfp->xnf_stat_rx_desballoc_fail; return (0); } static boolean_t xnf_kstat_init(xnf_t *xnfp) { int nstat = sizeof (xnf_aux_statistics) / sizeof (xnf_aux_statistics[0]); char **cp = xnf_aux_statistics; kstat_named_t *knp; /* * Create and initialise kstats. */ if ((xnfp->xnf_kstat_aux = kstat_create("xnf", ddi_get_instance(xnfp->xnf_devinfo), "aux_statistics", "net", KSTAT_TYPE_NAMED, nstat, 0)) == NULL) return (B_FALSE); xnfp->xnf_kstat_aux->ks_private = xnfp; xnfp->xnf_kstat_aux->ks_update = xnf_kstat_aux_update; knp = xnfp->xnf_kstat_aux->ks_data; while (nstat > 0) { kstat_named_init(knp, *cp, KSTAT_DATA_UINT64); knp++; cp++; nstat--; } kstat_install(xnfp->xnf_kstat_aux); return (B_TRUE); } static int xnf_stat(void *arg, uint_t stat, uint64_t *val) { xnf_t *xnfp = arg; mutex_enter(&xnfp->xnf_rxlock); mutex_enter(&xnfp->xnf_txlock); #define mac_stat(q, r) \ case (MAC_STAT_##q): \ *val = xnfp->xnf_stat_##r; \ break #define ether_stat(q, r) \ case (ETHER_STAT_##q): \ *val = xnfp->xnf_stat_##r; \ break switch (stat) { mac_stat(IPACKETS, ipackets); mac_stat(OPACKETS, opackets); mac_stat(RBYTES, rbytes); mac_stat(OBYTES, obytes); mac_stat(NORCVBUF, norxbuf); mac_stat(IERRORS, errrx); mac_stat(NOXMTBUF, tx_defer); ether_stat(MACRCV_ERRORS, mac_rcv_error); ether_stat(TOOSHORT_ERRORS, runt); /* always claim to be in full duplex mode */ case ETHER_STAT_LINK_DUPLEX: *val = LINK_DUPLEX_FULL; break; /* always claim to be at 1Gb/s link speed */ case MAC_STAT_IFSPEED: *val = 1000000000ull; break; default: mutex_exit(&xnfp->xnf_txlock); mutex_exit(&xnfp->xnf_rxlock); return (ENOTSUP); } #undef mac_stat #undef ether_stat mutex_exit(&xnfp->xnf_txlock); mutex_exit(&xnfp->xnf_rxlock); return (0); } static int xnf_change_mtu(xnf_t *xnfp, uint32_t mtu) { if (mtu > ETHERMTU) { if (!xnf_enable_tx_sg) { dev_err(xnfp->xnf_devinfo, CE_WARN, "MTU limited to %d " "because scatter-gather is disabled for transmit " "in driver settings", ETHERMTU); return (EINVAL); } else if (!xnf_enable_rx_sg) { dev_err(xnfp->xnf_devinfo, CE_WARN, "MTU limited to %d " "because scatter-gather is disabled for receive " "in driver settings", ETHERMTU); return (EINVAL); } else if (!xnfp->xnf_be_tx_sg) { dev_err(xnfp->xnf_devinfo, CE_WARN, "MTU limited to %d " "because backend doesn't support scatter-gather", ETHERMTU); return (EINVAL); } if (mtu > XNF_MAXPKT) return (EINVAL); } int error = mac_maxsdu_update(xnfp->xnf_mh, mtu); if (error == 0) xnfp->xnf_mtu = mtu; return (error); } /*ARGSUSED*/ static int xnf_getprop(void *data, const char *prop_name, mac_prop_id_t prop_id, uint_t prop_val_size, void *prop_val) { xnf_t *xnfp = data; switch (prop_id) { case MAC_PROP_MTU: ASSERT(prop_val_size >= sizeof (uint32_t)); bcopy(&xnfp->xnf_mtu, prop_val, sizeof (uint32_t)); break; default: return (ENOTSUP); } return (0); } /*ARGSUSED*/ static int xnf_setprop(void *data, const char *prop_name, mac_prop_id_t prop_id, uint_t prop_val_size, const void *prop_val) { xnf_t *xnfp = data; uint32_t new_mtu; int error; switch (prop_id) { case MAC_PROP_MTU: ASSERT(prop_val_size >= sizeof (uint32_t)); bcopy(prop_val, &new_mtu, sizeof (new_mtu)); error = xnf_change_mtu(xnfp, new_mtu); break; default: return (ENOTSUP); } return (error); } /*ARGSUSED*/ static void xnf_propinfo(void *data, const char *prop_name, mac_prop_id_t prop_id, mac_prop_info_handle_t prop_handle) { switch (prop_id) { case MAC_PROP_MTU: mac_prop_info_set_range_uint32(prop_handle, 0, XNF_MAXPKT); break; default: break; } } static boolean_t xnf_getcapab(void *arg, mac_capab_t cap, void *cap_data) { xnf_t *xnfp = arg; switch (cap) { case MAC_CAPAB_HCKSUM: { uint32_t *capab = cap_data; /* * Whilst the flag used to communicate with the IO * domain is called "NETTXF_csum_blank", the checksum * in the packet must contain the pseudo-header * checksum and not zero. * * To help out the IO domain, we might use * HCKSUM_INET_PARTIAL. Unfortunately our stack will * then use checksum offload for IPv6 packets, which * the IO domain can't handle. * * As a result, we declare outselves capable of * HCKSUM_INET_FULL_V4. This means that we receive * IPv4 packets from the stack with a blank checksum * field and must insert the pseudo-header checksum * before passing the packet to the IO domain. */ *capab = HCKSUM_INET_FULL_V4; /* * TODO: query the "feature-ipv6-csum-offload" capability. * If enabled, that could allow us to use HCKSUM_INET_PARTIAL. */ break; } case MAC_CAPAB_LSO: { if (!xnfp->xnf_be_lso) return (B_FALSE); mac_capab_lso_t *lso = cap_data; lso->lso_flags = LSO_TX_BASIC_TCP_IPV4; lso->lso_basic_tcp_ipv4.lso_max = IP_MAXPACKET; break; } default: return (B_FALSE); } return (B_TRUE); } /* * The state of the peer has changed - react accordingly. */ static void oe_state_change(dev_info_t *dip, ddi_eventcookie_t id, void *arg, void *impl_data) { _NOTE(ARGUNUSED(id, arg)); xnf_t *xnfp = ddi_get_driver_private(dip); XenbusState new_state = *(XenbusState *)impl_data; ASSERT(xnfp != NULL); switch (new_state) { case XenbusStateUnknown: case XenbusStateInitialising: case XenbusStateInitialised: case XenbusStateClosing: case XenbusStateClosed: case XenbusStateReconfiguring: case XenbusStateReconfigured: break; case XenbusStateInitWait: xnf_read_config(xnfp); if (!xnfp->xnf_be_rx_copy) { cmn_err(CE_WARN, "The xnf driver requires a dom0 that " "supports 'feature-rx-copy'."); (void) xvdi_switch_state(xnfp->xnf_devinfo, XBT_NULL, XenbusStateClosed); break; } /* * Connect to the backend. */ xnf_be_connect(xnfp); /* * Our MAC address as discovered by xnf_read_config(). */ mac_unicst_update(xnfp->xnf_mh, xnfp->xnf_mac_addr); /* * We do not know if some features such as LSO are supported * until we connect to the backend. We request the MAC layer * to poll our capabilities again. */ mac_capab_update(xnfp->xnf_mh); break; case XenbusStateConnected: mutex_enter(&xnfp->xnf_rxlock); mutex_enter(&xnfp->xnf_txlock); xnfp->xnf_connected = B_TRUE; /* * Wake up any threads waiting to send data to * backend. */ cv_broadcast(&xnfp->xnf_cv_state); mutex_exit(&xnfp->xnf_txlock); mutex_exit(&xnfp->xnf_rxlock); /* * Kick the peer in case it missed any transmits * request in the TX ring. */ ec_notify_via_evtchn(xnfp->xnf_evtchn); /* * There may already be completed receive requests in * the ring sent by backend after it gets connected * but before we see its state change here, so we call * xnf_intr() to handle them, if any. */ (void) xnf_intr((caddr_t)xnfp); /* * Mark the link up now that we are connected. */ mac_link_update(xnfp->xnf_mh, LINK_STATE_UP); /* * Tell the backend about the multicast addresses in * which we are interested. */ mac_multicast_refresh(xnfp->xnf_mh, NULL, xnfp, B_TRUE); break; default: break; } } /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2009 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ /* * Copyright (c) 2014, 2017 by Delphix. All rights reserved. */ #ifndef _SYS_XNF_H #define _SYS_XNF_H #ifdef __cplusplus extern "C" { #endif /* * As of April 2017, TX and RX ring sizes are fixed to 1 page in * size and Xen doesn't support changing it. * This represents 256 entries. */ #define NET_TX_RING_SIZE __CONST_RING_SIZE(netif_tx, PAGESIZE) #define NET_RX_RING_SIZE __CONST_RING_SIZE(netif_rx, PAGESIZE) /* * There is no MTU limit, however for all practical purposes hardware won't * support anything much larger than 9k. We put an arbitrary 16k limit. */ #define XNF_MAXPKT 16384 #define XNF_FRAMESIZE 1514 /* frame size including MAC header */ /* * Based on XEN_NETIF_NR_SLOTS_MIN in Linux. Packets that span more pages * than this must be defragmented or dropped. */ #define XEN_MAX_TX_DATA_PAGES 18 /* * We keep one extra slot for LSO */ #define XEN_MAX_SLOTS_PER_TX (XEN_MAX_TX_DATA_PAGES + 1) #define XEN_DATA_BOUNDARY 0x1000 /* * Information about each receive buffer and any transmit look-aside * buffers. */ typedef struct xnf_buf { frtn_t free_rtn; struct xnf *xnfp; ddi_dma_handle_t dma_handle; caddr_t buf; /* DMA-able data buffer */ paddr_t buf_phys; mfn_t buf_mfn; size_t len; struct xnf_buf *next; /* For linking into free list */ ddi_acc_handle_t acc_handle; grant_ref_t grant_ref; /* grant table reference */ uint16_t id; /* buffer id */ unsigned int gen; } xnf_buf_t; /* * Information about each transmit buffer. */ typedef enum xnf_txbuf_type { TX_DATA = 1, TX_MCAST_REQ, TX_MCAST_RSP } xnf_txbuf_type_t; /* * A xnf_txbuf is used to store ancillary data for a netif_tx_request_t. * A tx packet can span multiple xnf_txbuf's, linked together through tx_next * and tx_prev; tx_head points to the head of the chain. */ typedef struct xnf_txbuf { struct xnf_txbuf *tx_next; struct xnf_txbuf *tx_prev; struct xnf_txbuf *tx_head; xnf_txbuf_type_t tx_type; netif_tx_request_t tx_txreq; netif_extra_info_t tx_extra; /* Used for TX_DATA types */ ddi_dma_handle_t tx_dma_handle; boolean_t tx_handle_bound; mblk_t *tx_mp; xnf_buf_t *tx_bdesc; /* Look-aside buffer, if used. */ int tx_frags_to_ack; /* Used for TX_MCAST types */ int16_t tx_status; /* Used for debugging */ mfn_t tx_mfn; RING_IDX tx_slot; } xnf_txbuf_t; #define TXBUF_SETNEXT(head, next) \ head->tx_next = next; \ next->tx_prev = head; /* * Information about each outstanding transmit operation. */ typedef struct xnf_txid { uint16_t id; /* Id of this transmit buffer. */ uint16_t next; /* Freelist of ids. */ xnf_txbuf_t *txbuf; /* Buffer details. */ } xnf_txid_t; /* * Per-instance data. */ typedef struct xnf { /* most interesting stuff first to assist debugging */ dev_info_t *xnf_devinfo; mac_handle_t xnf_mh; unsigned char xnf_mac_addr[ETHERADDRL]; uint32_t xnf_mtu; unsigned int xnf_gen; /* Increments on resume. */ boolean_t xnf_connected; boolean_t xnf_running; boolean_t xnf_be_rx_copy; boolean_t xnf_be_mcast_control; boolean_t xnf_be_tx_sg; boolean_t xnf_be_lso; uint64_t xnf_stat_interrupts; uint64_t xnf_stat_unclaimed_interrupts; uint64_t xnf_stat_norxbuf; uint64_t xnf_stat_rx_drop; uint64_t xnf_stat_errrx; uint64_t xnf_stat_tx_pullup; uint64_t xnf_stat_tx_lookaside; uint64_t xnf_stat_tx_defer; uint64_t xnf_stat_tx_drop; uint64_t xnf_stat_tx_eth_hdr_split; uint64_t xnf_stat_mac_rcv_error; uint64_t xnf_stat_runt; uint64_t xnf_stat_ipackets; uint64_t xnf_stat_opackets; uint64_t xnf_stat_rbytes; uint64_t xnf_stat_obytes; uint64_t xnf_stat_tx_cksum_deferred; uint64_t xnf_stat_rx_cksum_no_need; uint64_t xnf_stat_buf_allocated; uint64_t xnf_stat_buf_outstanding; uint64_t xnf_stat_gref_outstanding; uint64_t xnf_stat_gref_failure; uint64_t xnf_stat_gref_peak; uint64_t xnf_stat_rx_allocb_fail; uint64_t xnf_stat_rx_desballoc_fail; kstat_t *xnf_kstat_aux; ddi_iblock_cookie_t xnf_icookie; netif_tx_front_ring_t xnf_tx_ring; ddi_dma_handle_t xnf_tx_ring_dma_handle; ddi_acc_handle_t xnf_tx_ring_dma_acchandle; paddr_t xnf_tx_ring_phys_addr; grant_ref_t xnf_tx_ring_ref; xnf_txid_t *xnf_tx_pkt_id; uint16_t xnf_tx_pkt_id_head; kmutex_t xnf_txlock; kmutex_t xnf_schedlock; boolean_t xnf_need_sched; kcondvar_t xnf_cv_tx_slots; kmem_cache_t *xnf_tx_buf_cache; netif_rx_front_ring_t xnf_rx_ring; ddi_dma_handle_t xnf_rx_ring_dma_handle; ddi_acc_handle_t xnf_rx_ring_dma_acchandle; paddr_t xnf_rx_ring_phys_addr; grant_ref_t xnf_rx_ring_ref; xnf_buf_t **xnf_rx_pkt_info; kmutex_t xnf_rxlock; mblk_t *xnf_rx_head; mblk_t *xnf_rx_tail; boolean_t xnf_rx_new_buffers_posted; kmem_cache_t *xnf_buf_cache; uint16_t xnf_evtchn; kmutex_t xnf_gref_lock; grant_ref_t xnf_gref_head; kcondvar_t xnf_cv_state; kcondvar_t xnf_cv_multicast; uint_t xnf_pending_multicast; } xnf_t; #ifdef __cplusplus } #endif #endif /* _SYS_XNF_H */ /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2008 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include extern int xen_boot_debug; /* * Internal structures and functions */ int xendev_nounload = 0; void xendev_enumerate(int); /* * Interface routines */ static struct modlmisc modlmisc = { &mod_miscops, "virtual device probe" }; static struct modlinkage modlinkage = { MODREV_1, (void *)&modlmisc, NULL }; int _init(void) { int err; if ((err = mod_install(&modlinkage)) != 0) return (err); impl_bus_add_probe(xendev_enumerate); return (0); } int _fini(void) { int err; if (xendev_nounload) return (EBUSY); if ((err = mod_remove(&modlinkage)) != 0) return (err); impl_bus_delete_probe(xendev_enumerate); return (0); } int _info(struct modinfo *modinfop) { return (mod_info(&modlinkage, modinfop)); } /* * This functions is invoked twice, first time with reprogram=0 to * set up the xpvd portion of the device tree. The second time is * ignored. */ void xendev_enumerate(int reprogram) { dev_info_t *dip; if (reprogram != 0) return; ndi_devi_alloc_sleep(ddi_root_node(), "xpvd", (pnode_t)DEVI_SID_NODEID, &dip); (void) ndi_devi_bind_driver(dip, 0); /* * Too early to enumerate split device drivers in domU * since we need to create taskq thread during enumeration. * So, we only enumerate softdevs and console here. */ xendev_enum_all(dip, B_TRUE); } /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2009 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ /* * Copyright 2012 Garrett D'Amore . All rights reserved. * Copyright (c) 2014 by Delphix. All rights reserved. * Copyright 2017 Nexenta Systems, Inc. * Copyright 2023 Oxide Computer Company */ /* * Host to hypervisor virtual devices nexus driver * * TODO: * - Add watchpoints on vbd/vif and enumerate/offline on watch callback * - Add DR IOCTLs * - Filter/restrict property lookups into xenstore */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef XPV_HVM_DRIVER #include #include #include #include #include #include #include #else #include #include #include #endif #include #include /* * DDI dev_ops entrypoints */ static int xpvd_info(dev_info_t *, ddi_info_cmd_t, void *, void **); static int xpvd_attach(dev_info_t *devi, ddi_attach_cmd_t cmd); static int xpvd_detach(dev_info_t *devi, ddi_detach_cmd_t cmd); /* * NDI bus_ops entrypoints */ static int xpvd_ctlops(dev_info_t *, dev_info_t *, ddi_ctl_enum_t, void *, void *); static int xpvd_intr_ops(dev_info_t *, dev_info_t *, ddi_intr_op_t, ddi_intr_handle_impl_t *, void *); static int xpvd_prop_op(dev_t, dev_info_t *, dev_info_t *, ddi_prop_op_t, int, char *, caddr_t, int *); static int xpvd_bus_config(dev_info_t *, uint_t, ddi_bus_config_op_t, void *, dev_info_t **); static int xpvd_bus_unconfig(dev_info_t *, uint_t, ddi_bus_config_op_t, void *); static int xpvd_get_eventcookie(dev_info_t *, dev_info_t *, char *, ddi_eventcookie_t *); static int xpvd_add_eventcall(dev_info_t *, dev_info_t *, ddi_eventcookie_t, void (*)(dev_info_t *, ddi_eventcookie_t, void *, void *), void *, ddi_callback_id_t *); static int xpvd_remove_eventcall(dev_info_t *, ddi_callback_id_t); static int xpvd_post_event(dev_info_t *, dev_info_t *, ddi_eventcookie_t, void *); /* * misc functions */ static int xpvd_enable_intr(dev_info_t *, ddi_intr_handle_impl_t *, int); static void xpvd_disable_intr(dev_info_t *, ddi_intr_handle_impl_t *, int); static int xpvd_removechild(dev_info_t *); static int xpvd_initchild(dev_info_t *); static int xpvd_name_child(dev_info_t *, char *, int); static boolean_t i_xpvd_parse_devname(char *, xendev_devclass_t *, domid_t *, int *); /* Extern declarations */ extern int (*psm_intr_ops)(dev_info_t *, ddi_intr_handle_impl_t *, psm_intr_op_t, int *); struct bus_ops xpvd_bus_ops = { BUSO_REV, i_ddi_bus_map, NULL, NULL, NULL, i_ddi_map_fault, NULL, ddi_dma_allochdl, ddi_dma_freehdl, ddi_dma_bindhdl, ddi_dma_unbindhdl, ddi_dma_flush, ddi_dma_win, ddi_dma_mctl, xpvd_ctlops, xpvd_prop_op, xpvd_get_eventcookie, xpvd_add_eventcall, xpvd_remove_eventcall, xpvd_post_event, 0, /* (*bus_intr_ctl)(); */ xpvd_bus_config, xpvd_bus_unconfig, NULL, /* (*bus_fm_init)(); */ NULL, /* (*bus_fm_fini)(); */ NULL, /* (*bus_fm_access_enter)(); */ NULL, /* (*bus_fm_access_exit)(); */ NULL, /* (*bus_power)(); */ xpvd_intr_ops /* (*bus_intr_op)(); */ }; struct dev_ops xpvd_ops = { DEVO_REV, /* devo_rev */ 0, /* refcnt */ xpvd_info, /* info */ nulldev, /* identify */ nulldev, /* probe */ xpvd_attach, /* attach */ xpvd_detach, /* detach */ nulldev, /* reset */ (struct cb_ops *)0, /* driver operations */ &xpvd_bus_ops, /* bus operations */ NULL, /* power */ ddi_quiesce_not_needed, /* quiesce */ }; dev_info_t *xpvd_dip; #define CF_DBG 0x1 #define ALL_DBG 0xff static ndi_event_definition_t xpvd_ndi_event_defs[] = { { 0, XS_OE_STATE, EPL_KERNEL, NDI_EVENT_POST_TO_TGT }, { 1, XS_HP_STATE, EPL_KERNEL, NDI_EVENT_POST_TO_TGT }, }; #define XENDEV_N_NDI_EVENTS \ (sizeof (xpvd_ndi_event_defs) / sizeof (xpvd_ndi_event_defs[0])) static ndi_event_set_t xpvd_ndi_events = { NDI_EVENTS_REV1, XENDEV_N_NDI_EVENTS, xpvd_ndi_event_defs }; static ndi_event_hdl_t xpvd_ndi_event_handle; /* * Hypervisor interrupt capabilities */ #define XENDEV_INTR_CAPABILITIES \ (DDI_INTR_FLAG_EDGE | DDI_INTR_FLAG_MASKABLE | DDI_INTR_FLAG_PENDING) /* * Module linkage information for the kernel. */ static struct modldrv modldrv = { &mod_driverops, /* Type of module */ "virtual device nexus driver", &xpvd_ops, /* driver ops */ }; static struct modlinkage modlinkage = { MODREV_1, (void *)&modldrv, NULL }; int _init(void) { return (mod_install(&modlinkage)); } int _fini(void) { return (mod_remove(&modlinkage)); } int _info(struct modinfo *modinfop) { return (mod_info(&modlinkage, modinfop)); } /* ARGSUSED */ static int xpvd_info(dev_info_t *dip, ddi_info_cmd_t cmd, void *arg, void **result) { switch (cmd) { default: return (DDI_FAILURE); case DDI_INFO_DEVT2INSTANCE: *result = (void *)0; return (DDI_SUCCESS); case DDI_INFO_DEVT2DEVINFO: *result = (void *)xpvd_dip; return (DDI_SUCCESS); } } /*ARGSUSED*/ static int xpvd_attach(dev_info_t *devi, ddi_attach_cmd_t cmd) { extern void xvdi_watch_devices(int); #ifdef XPV_HVM_DRIVER extern dev_info_t *xpv_dip; if (xpv_dip == NULL) { if (ddi_hold_installed_driver(ddi_name_to_major("xpv")) == NULL) { cmn_err(CE_WARN, "Couldn't initialize xpv framework"); return (DDI_FAILURE); } } #endif /* XPV_HVM_DRIVER */ if (ndi_event_alloc_hdl(devi, 0, &xpvd_ndi_event_handle, NDI_SLEEP) != NDI_SUCCESS) { xpvd_dip = NULL; return (DDI_FAILURE); } if (ndi_event_bind_set(xpvd_ndi_event_handle, &xpvd_ndi_events, NDI_SLEEP) != NDI_SUCCESS) { (void) ndi_event_free_hdl(xpvd_ndi_event_handle); xpvd_dip = NULL; return (DDI_FAILURE); } if (ddi_create_minor_node(devi, "devctl", S_IFCHR, ddi_get_instance(devi), DDI_PSEUDO, 0) != DDI_SUCCESS) { (void) ndi_event_unbind_set(xpvd_ndi_event_handle, &xpvd_ndi_events, NDI_SLEEP); (void) ndi_event_free_hdl(xpvd_ndi_event_handle); xpvd_dip = NULL; return (DDI_FAILURE); } #ifdef XPV_HVM_DRIVER (void) ddi_prop_update_int(DDI_DEV_T_NONE, devi, DDI_NO_AUTODETACH, 1); /* Report our version to dom0 */ (void) xenbus_printf(XBT_NULL, "guest/xpvd", "version", "%d", HVMPV_XPVD_VERS); #endif /* XPV_HVM_DRIVER */ /* watch both frontend and backend for new devices */ if (DOMAIN_IS_INITDOMAIN(xen_info)) (void) xs_register_xenbus_callback(xvdi_watch_devices); else xvdi_watch_devices(XENSTORE_UP); xpvd_dip = devi; ddi_report_dev(devi); return (DDI_SUCCESS); } /*ARGSUSED*/ static int xpvd_detach(dev_info_t *devi, ddi_detach_cmd_t cmd) { return (DDI_FAILURE); } /* * xpvd_prop_op() * * Query xenstore for the value of properties if DDI_PROP_NOTPROM * is not set. Xenstore property values are represented as ascii strings. */ static int xpvd_prop_op(dev_t dev, dev_info_t *dip, dev_info_t *ch_dip, ddi_prop_op_t prop_op, int mod_flags, char *name, caddr_t valuep, int *lengthp) { caddr_t buff; struct xendev_ppd *pdp; void *prop_str; size_t prop_len; unsigned int len; int rv; pdp = (struct xendev_ppd *)ddi_get_parent_data(ch_dip); if ((pdp == NULL) || !(mod_flags & (DDI_PROP_CANSLEEP)) || (mod_flags & DDI_PROP_NOTPROM) || (pdp->xd_xsdev.nodename == NULL)) goto toss_off; /* * First try reading the property off the the frontend. if that * fails, try and read it from the backend node. If that * also fails, pass the request on the DDI framework */ prop_str = NULL; if ((xenbus_read(XBT_NULL, pdp->xd_xsdev.nodename, name, &prop_str, &len) == 0) && (prop_str != NULL) && (strlen(prop_str) != 0)) goto got_xs_prop; prop_str = NULL; if ((pdp->xd_xsdev.otherend != NULL) && (xenbus_read(XBT_NULL, pdp->xd_xsdev.otherend, name, &prop_str, &len) == 0) && (prop_str != NULL) && (strlen(prop_str) != 0)) goto got_xs_prop; toss_off: return (ddi_bus_prop_op(dev, dip, ch_dip, prop_op, mod_flags | DDI_PROP_NOTPROM, name, valuep, lengthp)); got_xs_prop: prop_len = strlen(prop_str) + 1; rv = DDI_PROP_SUCCESS; switch (prop_op) { case PROP_LEN: *lengthp = prop_len; break; case PROP_LEN_AND_VAL_ALLOC: buff = kmem_alloc((size_t)prop_len, KM_SLEEP); *(caddr_t *)valuep = (caddr_t)buff; break; case PROP_LEN_AND_VAL_BUF: buff = (caddr_t)valuep; if (*lengthp < prop_len) rv = DDI_PROP_BUF_TOO_SMALL; break; default: rv = DDI_PROP_INVAL_ARG; break; } if (rv == DDI_PROP_SUCCESS) { if (prop_op != PROP_LEN) { bcopy(prop_str, buff, prop_len); *lengthp = prop_len; } } kmem_free(prop_str, len); return (rv); } /* * return address of the device's interrupt spec structure. */ /*ARGSUSED*/ struct intrspec * xpvd_get_ispec(dev_info_t *rdip, uint_t inumber) { struct xendev_ppd *pdp; ASSERT(inumber == 0); if ((pdp = ddi_get_parent_data(rdip)) == NULL) return (NULL); return (&pdp->xd_ispec); } /* * return (and determine) the interrupt priority of the device. */ /*ARGSUSED*/ static int xpvd_get_priority(dev_info_t *dip, int inum, int *pri) { struct xendev_ppd *pdp; struct intrspec *ispec; int *intpriorities; uint_t num_intpriorities; DDI_INTR_NEXDBG((CE_CONT, "xpvd_get_priority: dip = 0x%p\n", (void *)dip)); ASSERT(inum == 0); if ((pdp = ddi_get_parent_data(dip)) == NULL) return (DDI_FAILURE); ispec = &pdp->xd_ispec; /* * Set the default priority based on the device class. The * "interrupt-priorities" property can be used to override * the default. */ if (ispec->intrspec_pri == 0) { ispec->intrspec_pri = xendev_devclass_ipl(pdp->xd_devclass); if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, dip, DDI_PROP_NOTPROM | DDI_PROP_DONTPASS, "interrupt-priorities", &intpriorities, &num_intpriorities) == DDI_PROP_SUCCESS) { ispec->intrspec_pri = intpriorities[0]; ddi_prop_free(intpriorities); } } *pri = ispec->intrspec_pri; return (DDI_SUCCESS); } /* * xpvd_intr_ops: bus_intr_op() function for interrupt support */ /* ARGSUSED */ static int xpvd_intr_ops(dev_info_t *pdip, dev_info_t *rdip, ddi_intr_op_t intr_op, ddi_intr_handle_impl_t *hdlp, void *result) { int priority = 0; struct intrspec *ispec; struct xendev_ppd *pdp; DDI_INTR_NEXDBG((CE_CONT, "xpvd_intr_ops: pdip 0x%p, rdip 0x%p, op %x handle 0x%p\n", (void *)pdip, (void *)rdip, intr_op, (void *)hdlp)); /* Process the request */ switch (intr_op) { case DDI_INTROP_SUPPORTED_TYPES: /* Fixed supported by default */ *(int *)result = DDI_INTR_TYPE_FIXED; break; case DDI_INTROP_NINTRS: *(int *)result = 1; break; case DDI_INTROP_ALLOC: /* * FIXED interrupts: just return available interrupts */ if (hdlp->ih_type == DDI_INTR_TYPE_FIXED) { /* * event channels are edge-triggered, maskable, * and support int pending. */ hdlp->ih_cap |= XENDEV_INTR_CAPABILITIES; *(int *)result = 1; /* DDI_INTR_TYPE_FIXED */ } else { return (DDI_FAILURE); } break; case DDI_INTROP_FREE: ispec = xpvd_get_ispec(rdip, (int)hdlp->ih_inum); if (ispec == NULL) return (DDI_FAILURE); ispec->intrspec_pri = 0; /* mark as un-initialized */ break; case DDI_INTROP_GETPRI: if (xpvd_get_priority(rdip, hdlp->ih_inum, &priority) != DDI_SUCCESS) return (DDI_FAILURE); DDI_INTR_NEXDBG((CE_CONT, "xpvd_intr_ops: priority = 0x%x\n", priority)); *(int *)result = priority; break; case DDI_INTROP_SETPRI: /* Validate the interrupt priority passed */ if (*(int *)result > LOCK_LEVEL) return (DDI_FAILURE); /* Ensure that PSM is all initialized */ if (psm_intr_ops == NULL) return (DDI_FAILURE); /* Change the priority */ if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_PRI, result) == PSM_FAILURE) return (DDI_FAILURE); ispec = xpvd_get_ispec(rdip, (int)hdlp->ih_inum); if (ispec == NULL) return (DDI_FAILURE); ispec->intrspec_pri = *(int *)result; break; case DDI_INTROP_ADDISR: /* update ispec */ ispec = xpvd_get_ispec(rdip, (int)hdlp->ih_inum); if (ispec == NULL) return (DDI_FAILURE); ispec->intrspec_func = hdlp->ih_cb_func; break; case DDI_INTROP_REMISR: ispec = xpvd_get_ispec(rdip, (int)hdlp->ih_inum); pdp = (struct xendev_ppd *)ddi_get_parent_data(rdip); ASSERT(pdp != NULL); ASSERT(pdp->xd_evtchn != INVALID_EVTCHN); if (ispec) { ispec->intrspec_vec = 0; ispec->intrspec_func = (uint_t (*)()) 0; } pdp->xd_evtchn = INVALID_EVTCHN; break; case DDI_INTROP_GETCAP: if (hdlp->ih_type == DDI_INTR_TYPE_FIXED) { /* * event channels are edge-triggered, maskable, * and support int pending. */ *(int *)result = XENDEV_INTR_CAPABILITIES; } else { *(int *)result = 0; return (DDI_FAILURE); } DDI_INTR_NEXDBG((CE_CONT, "xpvd: GETCAP returned = %x\n", *(int *)result)); break; case DDI_INTROP_SETCAP: DDI_INTR_NEXDBG((CE_CONT, "xpvd_intr_ops: SETCAP cap=0x%x\n", *(int *)result)); if (psm_intr_ops == NULL) return (DDI_FAILURE); if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_CAP, result)) { DDI_INTR_NEXDBG((CE_CONT, "GETCAP: psm_intr_ops" " returned failure\n")); return (DDI_FAILURE); } break; case DDI_INTROP_ENABLE: if (psm_intr_ops == NULL) return (DDI_FAILURE); if (xpvd_enable_intr(rdip, hdlp, (int)hdlp->ih_inum) != DDI_SUCCESS) return (DDI_FAILURE); DDI_INTR_NEXDBG((CE_CONT, "xpvd_intr_ops: ENABLE vec=0x%x\n", hdlp->ih_vector)); break; case DDI_INTROP_DISABLE: if (psm_intr_ops == NULL) return (DDI_FAILURE); xpvd_disable_intr(rdip, hdlp, hdlp->ih_inum); DDI_INTR_NEXDBG((CE_CONT, "xpvd_intr_ops: DISABLE vec = %x\n", hdlp->ih_vector)); break; case DDI_INTROP_BLOCKENABLE: case DDI_INTROP_BLOCKDISABLE: return (DDI_FAILURE); case DDI_INTROP_SETMASK: case DDI_INTROP_CLRMASK: #ifdef XPV_HVM_DRIVER return (DDI_ENOTSUP); #else /* * Handle this here */ if (hdlp->ih_type != DDI_INTR_TYPE_FIXED) return (DDI_FAILURE); if (intr_op == DDI_INTROP_SETMASK) { ec_disable_irq(hdlp->ih_vector); } else { ec_enable_irq(hdlp->ih_vector); } break; #endif case DDI_INTROP_GETPENDING: #ifdef XPV_HVM_DRIVER return (DDI_ENOTSUP); #else if (hdlp->ih_type != DDI_INTR_TYPE_FIXED) return (DDI_FAILURE); *(int *)result = ec_pending_irq(hdlp->ih_vector); DDI_INTR_NEXDBG((CE_CONT, "xpvd: GETPENDING returned = %x\n", *(int *)result)); break; #endif case DDI_INTROP_NAVAIL: *(int *)result = 1; DDI_INTR_NEXDBG((CE_CONT, "xpvd: NAVAIL returned = %x\n", *(int *)result)); break; default: return (i_ddi_intr_ops(pdip, rdip, intr_op, hdlp, result)); } return (DDI_SUCCESS); } static int xpvd_enable_intr(dev_info_t *rdip, ddi_intr_handle_impl_t *hdlp, int inum) { int vector; ihdl_plat_t *ihdl_plat_datap = (ihdl_plat_t *)hdlp->ih_private; DDI_INTR_NEXDBG((CE_CONT, "xpvd_enable_intr: hdlp %p inum %x\n", (void *)hdlp, inum)); ihdl_plat_datap->ip_ispecp = xpvd_get_ispec(rdip, inum); if (ihdl_plat_datap->ip_ispecp == NULL) return (DDI_FAILURE); /* translate the interrupt if needed */ (void) (*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_XLATE_VECTOR, &vector); DDI_INTR_NEXDBG((CE_CONT, "xpvd_enable_intr: priority=%x vector=%x\n", hdlp->ih_pri, vector)); /* Add the interrupt handler */ if (!add_avintr((void *)hdlp, hdlp->ih_pri, hdlp->ih_cb_func, DEVI(rdip)->devi_name, vector, hdlp->ih_cb_arg1, hdlp->ih_cb_arg2, NULL, rdip)) return (DDI_FAILURE); /* Note this really is an irq. */ hdlp->ih_vector = (ushort_t)vector; return (DDI_SUCCESS); } static void xpvd_disable_intr(dev_info_t *rdip, ddi_intr_handle_impl_t *hdlp, int inum) { int vector; ihdl_plat_t *ihdl_plat_datap = (ihdl_plat_t *)hdlp->ih_private; DDI_INTR_NEXDBG((CE_CONT, "xpvd_disable_intr: \n")); ihdl_plat_datap->ip_ispecp = xpvd_get_ispec(rdip, inum); if (ihdl_plat_datap->ip_ispecp == NULL) return; /* translate the interrupt if needed */ (void) (*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_XLATE_VECTOR, &vector); /* Disable the interrupt handler */ rem_avintr((void *)hdlp, hdlp->ih_pri, hdlp->ih_cb_func, vector); ihdl_plat_datap->ip_ispecp = NULL; } /*ARGSUSED*/ static int xpvd_ctlops(dev_info_t *dip, dev_info_t *rdip, ddi_ctl_enum_t ctlop, void *arg, void *result) { switch (ctlop) { case DDI_CTLOPS_REPORTDEV: if (rdip == (dev_info_t *)0) return (DDI_FAILURE); cmn_err(CE_CONT, "?%s@%s, %s%d\n", ddi_node_name(rdip), ddi_get_name_addr(rdip), ddi_driver_name(rdip), ddi_get_instance(rdip)); return (DDI_SUCCESS); case DDI_CTLOPS_INITCHILD: return (xpvd_initchild((dev_info_t *)arg)); case DDI_CTLOPS_UNINITCHILD: return (xpvd_removechild((dev_info_t *)arg)); case DDI_CTLOPS_SIDDEV: return (DDI_SUCCESS); case DDI_CTLOPS_REGSIZE: case DDI_CTLOPS_NREGS: return (DDI_FAILURE); case DDI_CTLOPS_POWER: { return (ddi_ctlops(dip, rdip, ctlop, arg, result)); } default: return (ddi_ctlops(dip, rdip, ctlop, arg, result)); } /* NOTREACHED */ } /* * Assign the address portion of the node name */ static int xpvd_name_child(dev_info_t *child, char *addr, int addrlen) { int *domain, *vdev; uint_t ndomain, nvdev; char *prop_str; /* * i_xpvd_parse_devname() knows the formats used by this * routine. If this code changes, so must that. */ if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, child, DDI_PROP_DONTPASS, "domain", &domain, &ndomain) != DDI_PROP_SUCCESS) return (DDI_FAILURE); ASSERT(ndomain == 1); /* * Use "domain" and "vdev" properties (backend drivers). */ if (*domain != DOMID_SELF) { if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, child, DDI_PROP_DONTPASS, "vdev", &vdev, &nvdev) != DDI_PROP_SUCCESS) { ddi_prop_free(domain); return (DDI_FAILURE); } ASSERT(nvdev == 1); (void) snprintf(addr, addrlen, "%d,%d", domain[0], vdev[0]); ddi_prop_free(vdev); ddi_prop_free(domain); return (DDI_SUCCESS); } ddi_prop_free(domain); /* * Use "vdev" and "unit-address" properties (frontend/softdev drivers). * At boot time, only the vdev property is available on xdf disks. */ if (ddi_prop_lookup_string(DDI_DEV_T_ANY, child, DDI_PROP_DONTPASS, "unit-address", &prop_str) != DDI_PROP_SUCCESS) { if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, child, DDI_PROP_DONTPASS, "vdev", &vdev, &nvdev) != DDI_PROP_SUCCESS) return (DDI_FAILURE); ASSERT(nvdev == 1); (void) snprintf(addr, addrlen, "%d", vdev[0]); ddi_prop_free(vdev); return (DDI_SUCCESS); } (void) strlcpy(addr, prop_str, addrlen); ddi_prop_free(prop_str); return (DDI_SUCCESS); } static int xpvd_initchild(dev_info_t *child) { char addr[80]; /* * Pseudo nodes indicate a prototype node with per-instance * properties to be merged into the real h/w device node. */ if (ndi_dev_is_persistent_node(child) == 0) { ddi_set_parent_data(child, NULL); if (xpvd_name_child(child, addr, sizeof (addr)) != DDI_SUCCESS) return (DDI_FAILURE); ddi_set_name_addr(child, addr); /* * Try to merge the properties from this prototype * node into real h/w nodes. */ if (ndi_merge_node(child, xpvd_name_child) == DDI_SUCCESS) { /* * Merged ok - return failure to remove the node. */ ddi_set_name_addr(child, NULL); return (DDI_FAILURE); } /* * The child was not merged into a h/w node, * but there's not much we can do with it other * than return failure to cause the node to be removed. */ cmn_err(CE_WARN, "!%s@%s: %s.conf properties not merged", ddi_get_name(child), ddi_get_name_addr(child), ddi_get_name(child)); ddi_set_name_addr(child, NULL); return (DDI_NOT_WELL_FORMED); } if (xvdi_init_dev(child) != DDI_SUCCESS) return (DDI_FAILURE); if (xpvd_name_child(child, addr, sizeof (addr)) != DDI_SUCCESS) { xvdi_uninit_dev(child); return (DDI_FAILURE); } ddi_set_name_addr(child, addr); return (DDI_SUCCESS); } static int xpvd_removechild(dev_info_t *dip) { xvdi_uninit_dev(dip); ddi_set_name_addr(dip, NULL); /* * Strip the node to properly convert it back to prototype * form. */ ddi_remove_minor_node(dip, NULL); return (DDI_SUCCESS); } static int xpvd_bus_unconfig(dev_info_t *parent, uint_t flag, ddi_bus_config_op_t op, void *device_name) { return (ndi_busop_bus_unconfig(parent, flag, op, device_name)); } /* * Given the name of a child of xpvd, determine the device class, * domain and vdevnum to which it refers. */ static boolean_t i_xpvd_parse_devname(char *name, xendev_devclass_t *devclassp, domid_t *domp, int *vdevp) { int len = strlen(name) + 1; char *device_name = i_ddi_strdup(name, KM_SLEEP); char *cname = NULL, *caddr = NULL; boolean_t ret = B_FALSE; i_ddi_parse_name(device_name, &cname, &caddr, NULL); if ((cname == NULL) || (strlen(cname) == 0) || (caddr == NULL) || (strlen(caddr) == 0)) { ret = B_FALSE; goto done; } *devclassp = xendev_nodename_to_devclass(cname); if (*devclassp < 0) { ret = B_FALSE; goto done; } /* * Parsing the address component requires knowledge of how * xpvd_name_child() works. If that code changes, so must * this. */ /* Backend format is ",". */ if (sscanf(caddr, "%hu,%d", domp, vdevp) == 2) { ret = B_TRUE; goto done; } /* Frontend format is "". */ *domp = DOMID_SELF; if (sscanf(caddr, "%d", vdevp) == 1) ret = B_TRUE; done: kmem_free(device_name, len); return (ret); } /* * xpvd_bus_config() * * BUS_CONFIG_ONE: * Enumerate the exact instance of a driver. * * BUS_CONFIG_ALL: * Enumerate all the instances of all the possible children (seen before * and never seen before). * * BUS_CONFIG_DRIVER: * Enumerate all the instances of a particular driver. */ static int xpvd_bus_config(dev_info_t *parent, uint_t flag, ddi_bus_config_op_t op, void *arg, dev_info_t **childp) { char *cname = NULL; ndi_devi_enter(parent); switch (op) { case BUS_CONFIG_ONE: { xendev_devclass_t devclass; domid_t dom; int vdev; if (!i_xpvd_parse_devname(arg, &devclass, &dom, &vdev)) { ndi_devi_exit(parent); return (NDI_FAILURE); } *childp = xvdi_find_dev(parent, devclass, dom, vdev); if (*childp == NULL) *childp = xvdi_create_dev(parent, devclass, dom, vdev); ndi_devi_exit(parent); if (*childp == NULL) return (NDI_FAILURE); else return (ndi_busop_bus_config(parent, flag, op, arg, childp, 0)); } case BUS_CONFIG_DRIVER: { xendev_devclass_t devclass = XEN_INVAL; cname = ddi_major_to_name((major_t)(uintptr_t)arg); if (cname != NULL) devclass = xendev_nodename_to_devclass(cname); if (devclass == XEN_INVAL) { ndi_devi_exit(parent); return (NDI_FAILURE); } else { xendev_enum_class(parent, devclass); ndi_devi_exit(parent); return (ndi_busop_bus_config(parent, flag, op, arg, childp, 0)); } /* NOTREACHED */ } case BUS_CONFIG_ALL: xendev_enum_all(parent, B_FALSE); ndi_devi_exit(parent); return (ndi_busop_bus_config(parent, flag, op, arg, childp, 0)); default: ndi_devi_exit(parent); return (NDI_FAILURE); } } /*ARGSUSED*/ static int xpvd_get_eventcookie(dev_info_t *dip, dev_info_t *rdip, char *eventname, ddi_eventcookie_t *cookie) { return (ndi_event_retrieve_cookie(xpvd_ndi_event_handle, rdip, eventname, cookie, NDI_EVENT_NOPASS)); } /*ARGSUSED*/ static int xpvd_add_eventcall(dev_info_t *dip, dev_info_t *rdip, ddi_eventcookie_t cookie, void (*callback)(dev_info_t *dip, ddi_eventcookie_t cookie, void *arg, void *bus_impldata), void *arg, ddi_callback_id_t *cb_id) { return (ndi_event_add_callback(xpvd_ndi_event_handle, rdip, cookie, callback, arg, NDI_SLEEP, cb_id)); } /*ARGSUSED*/ static int xpvd_remove_eventcall(dev_info_t *dip, ddi_callback_id_t cb_id) { return (ndi_event_remove_callback(xpvd_ndi_event_handle, cb_id)); } /*ARGSUSED*/ static int xpvd_post_event(dev_info_t *dip, dev_info_t *rdip, ddi_eventcookie_t cookie, void *bus_impldata) { return (ndi_event_run_callbacks(xpvd_ndi_event_handle, rdip, cookie, bus_impldata)); } # # CDDL HEADER START # # The contents of this file are subject to the terms of the # Common Development and Distribution License (the "License"). # You may not use this file except in compliance with the License. # # You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE # or http://www.opensolaris.org/os/licensing. # See the License for the specific language governing permissions # and limitations under the License. # # When distributing Covered Code, include this CDDL HEADER in each # file and include the License file at usr/src/OPENSOLARIS.LICENSE. # If applicable, add the following below this CDDL HEADER, with the # fields enclosed by brackets "[]" replaced with your own identifying # information: Portions Copyright [yyyy] [name of copyright owner] # # CDDL HEADER END # # # Copyright 2007 Sun Microsystems, Inc. All rights reserved. # Use is subject to license terms. # # ident "%Z%%M% %I% %E% SMI" name="xpvd" class="root"; /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2009 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. * Copyright 2017 Joyent, Inc. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include static int xpvtap_open(dev_t *devp, int flag, int otyp, cred_t *cred); static int xpvtap_close(dev_t devp, int flag, int otyp, cred_t *cred); static int xpvtap_ioctl(dev_t dev, int cmd, intptr_t arg, int mode, cred_t *cred, int *rval); static int xpvtap_devmap(dev_t dev, devmap_cookie_t dhp, offset_t off, size_t len, size_t *maplen, uint_t model); static int xpvtap_segmap(dev_t dev, off_t off, struct as *asp, caddr_t *addrp, off_t len, unsigned int prot, unsigned int maxprot, unsigned int flags, cred_t *cred_p); static int xpvtap_chpoll(dev_t dev, short events, int anyyet, short *reventsp, struct pollhead **phpp); static struct cb_ops xpvtap_cb_ops = { xpvtap_open, /* cb_open */ xpvtap_close, /* cb_close */ nodev, /* cb_strategy */ nodev, /* cb_print */ nodev, /* cb_dump */ nodev, /* cb_read */ nodev, /* cb_write */ xpvtap_ioctl, /* cb_ioctl */ xpvtap_devmap, /* cb_devmap */ nodev, /* cb_mmap */ xpvtap_segmap, /* cb_segmap */ xpvtap_chpoll, /* cb_chpoll */ ddi_prop_op, /* cb_prop_op */ NULL, /* cb_stream */ D_NEW | D_MP | D_64BIT | D_DEVMAP, /* cb_flag */ CB_REV }; static int xpvtap_getinfo(dev_info_t *dip, ddi_info_cmd_t cmd, void *arg, void **result); static int xpvtap_attach(dev_info_t *devi, ddi_attach_cmd_t cmd); static int xpvtap_detach(dev_info_t *devi, ddi_detach_cmd_t cmd); static struct dev_ops xpvtap_dev_ops = { DEVO_REV, /* devo_rev */ 0, /* devo_refcnt */ xpvtap_getinfo, /* devo_getinfo */ nulldev, /* devo_identify */ nulldev, /* devo_probe */ xpvtap_attach, /* devo_attach */ xpvtap_detach, /* devo_detach */ nodev, /* devo_reset */ &xpvtap_cb_ops, /* devo_cb_ops */ NULL, /* devo_bus_ops */ NULL /* power */ }; static struct modldrv xpvtap_modldrv = { &mod_driverops, /* Type of module. This one is a driver */ "xpvtap driver", /* Name of the module. */ &xpvtap_dev_ops, /* driver ops */ }; static struct modlinkage xpvtap_modlinkage = { MODREV_1, (void *) &xpvtap_modldrv, NULL }; void *xpvtap_statep; static xpvtap_state_t *xpvtap_drv_init(int instance); static void xpvtap_drv_fini(xpvtap_state_t *state); static uint_t xpvtap_intr(caddr_t arg); typedef void (*xpvtap_rs_cleanup_t)(xpvtap_state_t *state, uint_t rs); static void xpvtap_rs_init(uint_t min_val, uint_t max_val, xpvtap_rs_hdl_t *handle); static void xpvtap_rs_fini(xpvtap_rs_hdl_t *handle); static int xpvtap_rs_alloc(xpvtap_rs_hdl_t handle, uint_t *rs); static void xpvtap_rs_free(xpvtap_rs_hdl_t handle, uint_t rs); static void xpvtap_rs_flush(xpvtap_rs_hdl_t handle, xpvtap_rs_cleanup_t callback, void *arg); static int xpvtap_segmf_register(xpvtap_state_t *state); static void xpvtap_segmf_unregister(struct as *as, void *arg, uint_t event); static int xpvtap_user_init(xpvtap_state_t *state); static void xpvtap_user_fini(xpvtap_state_t *state); static int xpvtap_user_ring_init(xpvtap_state_t *state); static void xpvtap_user_ring_fini(xpvtap_state_t *state); static int xpvtap_user_thread_init(xpvtap_state_t *state); static void xpvtap_user_thread_fini(xpvtap_state_t *state); static void xpvtap_user_thread_start(caddr_t arg); static void xpvtap_user_thread_stop(xpvtap_state_t *state); static void xpvtap_user_thread(void *arg); static void xpvtap_user_app_stop(caddr_t arg); static int xpvtap_user_request_map(xpvtap_state_t *state, blkif_request_t *req, uint_t *uid); static int xpvtap_user_request_push(xpvtap_state_t *state, blkif_request_t *req, uint_t uid); static int xpvtap_user_response_get(xpvtap_state_t *state, blkif_response_t *resp, uint_t *uid); static void xpvtap_user_request_unmap(xpvtap_state_t *state, uint_t uid); /* * _init() */ int _init(void) { int e; e = ddi_soft_state_init(&xpvtap_statep, sizeof (xpvtap_state_t), 1); if (e != 0) { return (e); } e = mod_install(&xpvtap_modlinkage); if (e != 0) { ddi_soft_state_fini(&xpvtap_statep); return (e); } return (0); } /* * _info() */ int _info(struct modinfo *modinfop) { return (mod_info(&xpvtap_modlinkage, modinfop)); } /* * _fini() */ int _fini(void) { int e; e = mod_remove(&xpvtap_modlinkage); if (e != 0) { return (e); } ddi_soft_state_fini(&xpvtap_statep); return (0); } /* * xpvtap_attach() */ static int xpvtap_attach(dev_info_t *dip, ddi_attach_cmd_t cmd) { blk_ringinit_args_t args; xpvtap_state_t *state; int instance; int e; switch (cmd) { case DDI_ATTACH: break; case DDI_RESUME: return (DDI_SUCCESS); default: return (DDI_FAILURE); } /* initialize our state info */ instance = ddi_get_instance(dip); state = xpvtap_drv_init(instance); if (state == NULL) { return (DDI_FAILURE); } state->bt_dip = dip; /* Initialize the guest ring */ args.ar_dip = state->bt_dip; args.ar_intr = xpvtap_intr; args.ar_intr_arg = (caddr_t)state; args.ar_ringup = xpvtap_user_thread_start; args.ar_ringup_arg = (caddr_t)state; args.ar_ringdown = xpvtap_user_app_stop; args.ar_ringdown_arg = (caddr_t)state; e = blk_ring_init(&args, &state->bt_guest_ring); if (e != DDI_SUCCESS) { goto attachfail_ringinit; } /* create the minor node (for ioctl/mmap) */ e = ddi_create_minor_node(dip, "xpvtap", S_IFCHR, instance, DDI_PSEUDO, 0); if (e != DDI_SUCCESS) { goto attachfail_minor_node; } /* Report that driver was loaded */ ddi_report_dev(dip); return (DDI_SUCCESS); attachfail_minor_node: blk_ring_fini(&state->bt_guest_ring); attachfail_ringinit: xpvtap_drv_fini(state); return (DDI_FAILURE); } /* * xpvtap_detach() */ static int xpvtap_detach(dev_info_t *dip, ddi_detach_cmd_t cmd) { xpvtap_state_t *state; int instance; instance = ddi_get_instance(dip); state = ddi_get_soft_state(xpvtap_statep, instance); if (state == NULL) { return (DDI_FAILURE); } switch (cmd) { case DDI_DETACH: break; case DDI_SUSPEND: default: return (DDI_FAILURE); } xpvtap_user_thread_stop(state); blk_ring_fini(&state->bt_guest_ring); xpvtap_drv_fini(state); ddi_remove_minor_node(dip, NULL); return (DDI_SUCCESS); } /* * xpvtap_getinfo() */ /*ARGSUSED*/ static int xpvtap_getinfo(dev_info_t *dip, ddi_info_cmd_t cmd, void *arg, void **result) { xpvtap_state_t *state; int instance; dev_t dev; int e; dev = (dev_t)arg; instance = getminor(dev); switch (cmd) { case DDI_INFO_DEVT2DEVINFO: state = ddi_get_soft_state(xpvtap_statep, instance); if (state == NULL) { return (DDI_FAILURE); } *result = (void *)state->bt_dip; e = DDI_SUCCESS; break; case DDI_INFO_DEVT2INSTANCE: *result = (void *)(uintptr_t)instance; e = DDI_SUCCESS; break; default: e = DDI_FAILURE; break; } return (e); } /* * xpvtap_open() */ /*ARGSUSED*/ static int xpvtap_open(dev_t *devp, int flag, int otyp, cred_t *cred) { xpvtap_state_t *state; int instance; if (secpolicy_xvm_control(cred)) { return (EPERM); } instance = getminor(*devp); state = ddi_get_soft_state(xpvtap_statep, instance); if (state == NULL) { return (ENXIO); } /* we should only be opened once */ mutex_enter(&state->bt_open.bo_mutex); if (state->bt_open.bo_opened) { mutex_exit(&state->bt_open.bo_mutex); return (EBUSY); } state->bt_open.bo_opened = B_TRUE; mutex_exit(&state->bt_open.bo_mutex); /* * save the apps address space. need it for mapping/unmapping grefs * since will be doing it in a separate kernel thread. */ state->bt_map.um_as = curproc->p_as; return (0); } /* * xpvtap_close() */ /*ARGSUSED*/ static int xpvtap_close(dev_t devp, int flag, int otyp, cred_t *cred) { xpvtap_state_t *state; int instance; instance = getminor(devp); state = ddi_get_soft_state(xpvtap_statep, instance); if (state == NULL) { return (ENXIO); } /* * wake thread so it can cleanup and wait for it to exit so we can * be sure it's not in the middle of processing a request/response. */ mutex_enter(&state->bt_thread.ut_mutex); state->bt_thread.ut_wake = B_TRUE; state->bt_thread.ut_exit = B_TRUE; cv_signal(&state->bt_thread.ut_wake_cv); if (!state->bt_thread.ut_exit_done) { cv_wait(&state->bt_thread.ut_exit_done_cv, &state->bt_thread.ut_mutex); } ASSERT(state->bt_thread.ut_exit_done); mutex_exit(&state->bt_thread.ut_mutex); state->bt_map.um_as = NULL; state->bt_map.um_guest_pages = NULL; /* * when the ring is brought down, a userland hotplug script is run * which tries to bring the userland app down. We'll wait for a bit * for the user app to exit. Notify the thread waiting that the app * has closed the driver. */ mutex_enter(&state->bt_open.bo_mutex); ASSERT(state->bt_open.bo_opened); state->bt_open.bo_opened = B_FALSE; cv_signal(&state->bt_open.bo_exit_cv); mutex_exit(&state->bt_open.bo_mutex); return (0); } /* * xpvtap_ioctl() */ /*ARGSUSED*/ static int xpvtap_ioctl(dev_t dev, int cmd, intptr_t arg, int mode, cred_t *cred, int *rval) { xpvtap_state_t *state; int instance; if (secpolicy_xvm_control(cred)) { return (EPERM); } instance = getminor(dev); if (instance == -1) { return (EBADF); } state = ddi_get_soft_state(xpvtap_statep, instance); if (state == NULL) { return (EBADF); } switch (cmd) { case XPVTAP_IOCTL_RESP_PUSH: /* * wake thread, thread handles guest requests and user app * responses. */ mutex_enter(&state->bt_thread.ut_mutex); state->bt_thread.ut_wake = B_TRUE; cv_signal(&state->bt_thread.ut_wake_cv); mutex_exit(&state->bt_thread.ut_mutex); break; default: cmn_err(CE_WARN, "ioctl(%d) not supported\n", cmd); return (ENXIO); } return (0); } /* * xpvtap_segmap() */ /*ARGSUSED*/ static int xpvtap_segmap(dev_t dev, off_t off, struct as *asp, caddr_t *addrp, off_t len, unsigned int prot, unsigned int maxprot, unsigned int flags, cred_t *cred_p) { struct segmf_crargs a; xpvtap_state_t *state; int instance; int e; if (secpolicy_xvm_control(cred_p)) { return (EPERM); } instance = getminor(dev); state = ddi_get_soft_state(xpvtap_statep, instance); if (state == NULL) { return (EBADF); } /* the user app should be doing a MAP_SHARED mapping */ if ((flags & MAP_TYPE) != MAP_SHARED) { return (EINVAL); } /* * if this is the user ring (offset = 0), devmap it (which ends up in * xpvtap_devmap). devmap will alloc and map the ring into the * app's VA space. */ if (off == 0) { e = devmap_setup(dev, (offset_t)off, asp, addrp, (size_t)len, prot, maxprot, flags, cred_p); return (e); } /* this should be the mmap for the gref pages (offset = PAGESIZE) */ if (off != PAGESIZE) { return (EINVAL); } /* make sure we get the size we're expecting */ if (len != XPVTAP_GREF_BUFSIZE) { return (EINVAL); } /* * reserve user app VA space for the gref pages and use segmf to * manage the backing store for the physical memory. segmf will * map in/out the grefs and fault them in/out. */ ASSERT(asp == state->bt_map.um_as); as_rangelock(asp); if ((flags & MAP_FIXED) == 0) { map_addr(addrp, len, 0, 0, flags); if (*addrp == NULL) { as_rangeunlock(asp); return (ENOMEM); } } else { /* User specified address */ (void) as_unmap(asp, *addrp, len); } a.dev = dev; a.prot = (uchar_t)prot; a.maxprot = (uchar_t)maxprot; e = as_map(asp, *addrp, len, segmf_create, &a); if (e != 0) { as_rangeunlock(asp); return (e); } as_rangeunlock(asp); /* * Stash user base address, and compute address where the request * array will end up. */ state->bt_map.um_guest_pages = (caddr_t)*addrp; state->bt_map.um_guest_size = (size_t)len; /* register an as callback so we can cleanup when the app goes away */ e = as_add_callback(asp, xpvtap_segmf_unregister, state, AS_UNMAP_EVENT, *addrp, len, KM_SLEEP); if (e != 0) { (void) as_unmap(asp, *addrp, len); return (EINVAL); } /* wake thread to see if there are requests already queued up */ mutex_enter(&state->bt_thread.ut_mutex); state->bt_thread.ut_wake = B_TRUE; cv_signal(&state->bt_thread.ut_wake_cv); mutex_exit(&state->bt_thread.ut_mutex); return (0); } /* * xpvtap_devmap() */ /*ARGSUSED*/ static int xpvtap_devmap(dev_t dev, devmap_cookie_t dhp, offset_t off, size_t len, size_t *maplen, uint_t model) { xpvtap_user_ring_t *usring; xpvtap_state_t *state; int instance; int e; instance = getminor(dev); state = ddi_get_soft_state(xpvtap_statep, instance); if (state == NULL) { return (EBADF); } /* we should only get here if the offset was == 0 */ if (off != 0) { return (EINVAL); } /* we should only be mapping in one page */ if (len != PAGESIZE) { return (EINVAL); } /* * we already allocated the user ring during driver attach, all we * need to do is map it into the user app's VA. */ usring = &state->bt_user_ring; e = devmap_umem_setup(dhp, state->bt_dip, NULL, usring->ur_cookie, 0, PAGESIZE, PROT_ALL, DEVMAP_DEFAULTS, NULL); if (e < 0) { return (e); } /* return the size to compete the devmap */ *maplen = PAGESIZE; return (0); } /* * xpvtap_chpoll() */ static int xpvtap_chpoll(dev_t dev, short events, int anyyet, short *reventsp, struct pollhead **phpp) { xpvtap_user_ring_t *usring; xpvtap_state_t *state; int instance; instance = getminor(dev); if (instance == -1) { return (EBADF); } state = ddi_get_soft_state(xpvtap_statep, instance); if (state == NULL) { return (EBADF); } if (((events & (POLLIN | POLLRDNORM)) == 0) && !anyyet) { return (EINVAL); } /* * if we pushed requests on the user ring since the last poll, wakeup * the user app */ *reventsp = 0; usring = &state->bt_user_ring; if (usring->ur_prod_polled != usring->ur_ring.req_prod_pvt) { /* * XXX - is this faster here or xpvtap_user_request_push?? * prelim data says here. Because less membars or because * user thread will spin in poll requests before getting to * responses? */ RING_PUSH_REQUESTS(&usring->ur_ring); usring->ur_prod_polled = usring->ur_ring.sring->req_prod; *reventsp = POLLIN | POLLRDNORM; } if ((*reventsp == 0 && !anyyet) || (events & POLLET)) { *phpp = &state->bt_pollhead; } return (0); } /* * xpvtap_drv_init() */ static xpvtap_state_t * xpvtap_drv_init(int instance) { xpvtap_state_t *state; int e; e = ddi_soft_state_zalloc(xpvtap_statep, instance); if (e != DDI_SUCCESS) { return (NULL); } state = ddi_get_soft_state(xpvtap_statep, instance); if (state == NULL) { goto drvinitfail_get_soft_state; } state->bt_instance = instance; mutex_init(&state->bt_open.bo_mutex, NULL, MUTEX_DRIVER, NULL); cv_init(&state->bt_open.bo_exit_cv, NULL, CV_DRIVER, NULL); state->bt_open.bo_opened = B_FALSE; state->bt_map.um_registered = B_FALSE; /* initialize user ring, thread, mapping state */ e = xpvtap_user_init(state); if (e != DDI_SUCCESS) { goto drvinitfail_userinit; } return (state); drvinitfail_userinit: cv_destroy(&state->bt_open.bo_exit_cv); mutex_destroy(&state->bt_open.bo_mutex); drvinitfail_get_soft_state: (void) ddi_soft_state_free(xpvtap_statep, instance); return (NULL); } /* * xpvtap_drv_fini() */ static void xpvtap_drv_fini(xpvtap_state_t *state) { xpvtap_user_fini(state); cv_destroy(&state->bt_open.bo_exit_cv); mutex_destroy(&state->bt_open.bo_mutex); (void) ddi_soft_state_free(xpvtap_statep, state->bt_instance); } /* * xpvtap_intr() * this routine will be called when we have a request on the guest ring. */ static uint_t xpvtap_intr(caddr_t arg) { xpvtap_state_t *state; state = (xpvtap_state_t *)arg; /* wake thread, thread handles guest requests and user app responses */ mutex_enter(&state->bt_thread.ut_mutex); state->bt_thread.ut_wake = B_TRUE; cv_signal(&state->bt_thread.ut_wake_cv); mutex_exit(&state->bt_thread.ut_mutex); return (DDI_INTR_CLAIMED); } /* * xpvtap_segmf_register() */ static int xpvtap_segmf_register(xpvtap_state_t *state) { struct seg *seg; uint64_t pte_ma; struct as *as; caddr_t uaddr; uint_t pgcnt; int i; as = state->bt_map.um_as; pgcnt = btopr(state->bt_map.um_guest_size); uaddr = state->bt_map.um_guest_pages; if (pgcnt == 0) { return (DDI_FAILURE); } AS_LOCK_ENTER(as, RW_READER); seg = as_findseg(as, state->bt_map.um_guest_pages, 0); if ((seg == NULL) || ((uaddr + state->bt_map.um_guest_size) > (seg->s_base + seg->s_size))) { AS_LOCK_EXIT(as); return (DDI_FAILURE); } /* * lock down the htables so the HAT can't steal them. Register the * PTE MA's for each gref page with seg_mf so we can do user space * gref mappings. */ for (i = 0; i < pgcnt; i++) { hat_prepare_mapping(as->a_hat, uaddr, &pte_ma); hat_devload(as->a_hat, uaddr, PAGESIZE, (pfn_t)0, PROT_READ | PROT_WRITE | PROT_USER | HAT_UNORDERED_OK, HAT_LOAD_NOCONSIST | HAT_LOAD_LOCK); hat_release_mapping(as->a_hat, uaddr); segmf_add_gref_pte(seg, uaddr, pte_ma); uaddr += PAGESIZE; } state->bt_map.um_registered = B_TRUE; AS_LOCK_EXIT(as); return (DDI_SUCCESS); } /* * xpvtap_segmf_unregister() * as_callback routine */ /*ARGSUSED*/ static void xpvtap_segmf_unregister(struct as *as, void *arg, uint_t event) { xpvtap_state_t *state; caddr_t uaddr; uint_t pgcnt; int i; state = (xpvtap_state_t *)arg; if (!state->bt_map.um_registered) { /* remove the callback (which is this routine) */ (void) as_delete_callback(as, arg); return; } pgcnt = btopr(state->bt_map.um_guest_size); uaddr = state->bt_map.um_guest_pages; /* unmap any outstanding req's grefs */ xpvtap_rs_flush(state->bt_map.um_rs, xpvtap_user_request_unmap, state); /* Unlock the gref pages */ for (i = 0; i < pgcnt; i++) { AS_LOCK_ENTER(as, RW_WRITER); hat_prepare_mapping(as->a_hat, uaddr, NULL); hat_unload(as->a_hat, uaddr, PAGESIZE, HAT_UNLOAD_UNLOCK); hat_release_mapping(as->a_hat, uaddr); AS_LOCK_EXIT(as); uaddr += PAGESIZE; } /* remove the callback (which is this routine) */ (void) as_delete_callback(as, arg); state->bt_map.um_registered = B_FALSE; } /* * xpvtap_user_init() */ static int xpvtap_user_init(xpvtap_state_t *state) { xpvtap_user_map_t *map; int e; map = &state->bt_map; /* Setup the ring between the driver and user app */ e = xpvtap_user_ring_init(state); if (e != DDI_SUCCESS) { return (DDI_FAILURE); } /* * the user ring can handle BLKIF_RING_SIZE outstanding requests. This * is the same number of requests as the guest ring. Initialize the * state we use to track request IDs to the user app. These IDs will * also identify which group of gref pages correspond with the * request. */ xpvtap_rs_init(0, (BLKIF_RING_SIZE - 1), &map->um_rs); /* * allocate the space to store a copy of each outstanding requests. We * will need to reference the ID and the number of segments when we * get the response from the user app. */ map->um_outstanding_reqs = kmem_zalloc( sizeof (*map->um_outstanding_reqs) * BLKIF_RING_SIZE, KM_SLEEP); /* * initialize the thread we use to process guest requests and user * responses. */ e = xpvtap_user_thread_init(state); if (e != DDI_SUCCESS) { goto userinitfail_user_thread_init; } return (DDI_SUCCESS); userinitfail_user_thread_init: xpvtap_rs_fini(&map->um_rs); kmem_free(map->um_outstanding_reqs, sizeof (*map->um_outstanding_reqs) * BLKIF_RING_SIZE); xpvtap_user_ring_fini(state); return (DDI_FAILURE); } /* * xpvtap_user_ring_init() */ static int xpvtap_user_ring_init(xpvtap_state_t *state) { xpvtap_user_ring_t *usring; usring = &state->bt_user_ring; /* alocate and initialize the page for the shared user ring */ usring->ur_sring = (blkif_sring_t *)ddi_umem_alloc(PAGESIZE, DDI_UMEM_SLEEP, &usring->ur_cookie); SHARED_RING_INIT(usring->ur_sring); FRONT_RING_INIT(&usring->ur_ring, usring->ur_sring, PAGESIZE); usring->ur_prod_polled = 0; return (DDI_SUCCESS); } /* * xpvtap_user_thread_init() */ static int xpvtap_user_thread_init(xpvtap_state_t *state) { xpvtap_user_thread_t *thread; char taskqname[32]; thread = &state->bt_thread; mutex_init(&thread->ut_mutex, NULL, MUTEX_DRIVER, NULL); cv_init(&thread->ut_wake_cv, NULL, CV_DRIVER, NULL); cv_init(&thread->ut_exit_done_cv, NULL, CV_DRIVER, NULL); thread->ut_wake = B_FALSE; thread->ut_exit = B_FALSE; thread->ut_exit_done = B_TRUE; /* create but don't start the user thread */ (void) sprintf(taskqname, "xvptap_%d", state->bt_instance); thread->ut_taskq = ddi_taskq_create(state->bt_dip, taskqname, 1, TASKQ_DEFAULTPRI, 0); if (thread->ut_taskq == NULL) { goto userinitthrfail_taskq_create; } return (DDI_SUCCESS); userinitthrfail_taskq_create: cv_destroy(&thread->ut_exit_done_cv); cv_destroy(&thread->ut_wake_cv); mutex_destroy(&thread->ut_mutex); return (DDI_FAILURE); } /* * xpvtap_user_thread_start() */ static void xpvtap_user_thread_start(caddr_t arg) { xpvtap_user_thread_t *thread; xpvtap_state_t *state; int e; state = (xpvtap_state_t *)arg; thread = &state->bt_thread; /* start the user thread */ thread->ut_exit_done = B_FALSE; e = ddi_taskq_dispatch(thread->ut_taskq, xpvtap_user_thread, state, DDI_SLEEP); if (e != DDI_SUCCESS) { thread->ut_exit_done = B_TRUE; cmn_err(CE_WARN, "Unable to start user thread\n"); } } /* * xpvtap_user_thread_stop() */ static void xpvtap_user_thread_stop(xpvtap_state_t *state) { /* wake thread so it can exit */ mutex_enter(&state->bt_thread.ut_mutex); state->bt_thread.ut_wake = B_TRUE; state->bt_thread.ut_exit = B_TRUE; cv_signal(&state->bt_thread.ut_wake_cv); if (!state->bt_thread.ut_exit_done) { cv_wait(&state->bt_thread.ut_exit_done_cv, &state->bt_thread.ut_mutex); } mutex_exit(&state->bt_thread.ut_mutex); ASSERT(state->bt_thread.ut_exit_done); } /* * xpvtap_user_fini() */ static void xpvtap_user_fini(xpvtap_state_t *state) { xpvtap_user_map_t *map; map = &state->bt_map; xpvtap_user_thread_fini(state); xpvtap_rs_fini(&map->um_rs); kmem_free(map->um_outstanding_reqs, sizeof (*map->um_outstanding_reqs) * BLKIF_RING_SIZE); xpvtap_user_ring_fini(state); } /* * xpvtap_user_ring_fini() */ static void xpvtap_user_ring_fini(xpvtap_state_t *state) { ddi_umem_free(state->bt_user_ring.ur_cookie); } /* * xpvtap_user_thread_fini() */ static void xpvtap_user_thread_fini(xpvtap_state_t *state) { ddi_taskq_destroy(state->bt_thread.ut_taskq); cv_destroy(&state->bt_thread.ut_exit_done_cv); cv_destroy(&state->bt_thread.ut_wake_cv); mutex_destroy(&state->bt_thread.ut_mutex); } /* * xpvtap_user_thread() */ static void xpvtap_user_thread(void *arg) { xpvtap_user_thread_t *thread; blkif_response_t resp; xpvtap_state_t *state; blkif_request_t req; boolean_t b; uint_t uid; int e; state = (xpvtap_state_t *)arg; thread = &state->bt_thread; xpvtap_thread_start: /* See if we are supposed to exit */ mutex_enter(&thread->ut_mutex); if (thread->ut_exit) { thread->ut_exit_done = B_TRUE; cv_signal(&state->bt_thread.ut_exit_done_cv); mutex_exit(&thread->ut_mutex); return; } /* * if we aren't supposed to be awake, wait until someone wakes us. * when we wake up, check for a kill or someone telling us to exit. */ if (!thread->ut_wake) { e = cv_wait_sig(&thread->ut_wake_cv, &thread->ut_mutex); if ((e == 0) || (thread->ut_exit)) { thread->ut_exit = B_TRUE; mutex_exit(&thread->ut_mutex); goto xpvtap_thread_start; } } /* if someone didn't wake us, go back to the start of the thread */ if (!thread->ut_wake) { mutex_exit(&thread->ut_mutex); goto xpvtap_thread_start; } /* we are awake */ thread->ut_wake = B_FALSE; mutex_exit(&thread->ut_mutex); /* process requests from the guest */ do { /* * check for requests from the guest. if we don't have any, * break out of the loop. */ e = blk_ring_request_get(state->bt_guest_ring, &req); if (e == B_FALSE) { break; } /* we got a request, map the grefs into the user app's VA */ e = xpvtap_user_request_map(state, &req, &uid); if (e != DDI_SUCCESS) { /* * If we couldn't map the request (e.g. user app hasn't * opened the device yet), requeue it and try again * later */ blk_ring_request_requeue(state->bt_guest_ring); break; } /* push the request to the user app */ e = xpvtap_user_request_push(state, &req, uid); if (e != DDI_SUCCESS) { resp.id = req.id; resp.operation = req.operation; resp.status = BLKIF_RSP_ERROR; blk_ring_response_put(state->bt_guest_ring, &resp); } } while (!thread->ut_exit); /* process reponses from the user app */ do { /* * check for responses from the user app. if we don't have any, * break out of the loop. */ b = xpvtap_user_response_get(state, &resp, &uid); if (b != B_TRUE) { break; } /* * if we got a response, unmap the grefs from the matching * request. */ xpvtap_user_request_unmap(state, uid); /* push the response to the guest */ blk_ring_response_put(state->bt_guest_ring, &resp); } while (!thread->ut_exit); goto xpvtap_thread_start; } /* * xpvtap_user_request_map() */ static int xpvtap_user_request_map(xpvtap_state_t *state, blkif_request_t *req, uint_t *uid) { grant_ref_t gref[BLKIF_MAX_SEGMENTS_PER_REQUEST]; struct seg *seg; struct as *as; domid_t domid; caddr_t uaddr; uint_t flags; int i; int e; domid = xvdi_get_oeid(state->bt_dip); as = state->bt_map.um_as; if ((as == NULL) || (state->bt_map.um_guest_pages == NULL)) { return (DDI_FAILURE); } /* has to happen after segmap returns */ if (!state->bt_map.um_registered) { /* register the pte's with segmf */ e = xpvtap_segmf_register(state); if (e != DDI_SUCCESS) { return (DDI_FAILURE); } } /* alloc an ID for the user ring */ e = xpvtap_rs_alloc(state->bt_map.um_rs, uid); if (e != DDI_SUCCESS) { return (DDI_FAILURE); } /* if we don't have any segments to map, we're done */ if ((req->operation == BLKIF_OP_WRITE_BARRIER) || (req->operation == BLKIF_OP_FLUSH_DISKCACHE) || (req->nr_segments == 0)) { return (DDI_SUCCESS); } /* get the apps gref address */ uaddr = XPVTAP_GREF_REQADDR(state->bt_map.um_guest_pages, *uid); AS_LOCK_ENTER(as, RW_READER); seg = as_findseg(as, state->bt_map.um_guest_pages, 0); if ((seg == NULL) || ((uaddr + mmu_ptob(req->nr_segments)) > (seg->s_base + seg->s_size))) { AS_LOCK_EXIT(as); return (DDI_FAILURE); } /* if we are reading from disk, we are writing into memory */ flags = 0; if (req->operation == BLKIF_OP_READ) { flags |= SEGMF_GREF_WR; } /* Load the grefs into seg_mf */ for (i = 0; i < req->nr_segments; i++) { gref[i] = req->seg[i].gref; } (void) segmf_add_grefs(seg, uaddr, flags, gref, req->nr_segments, domid); AS_LOCK_EXIT(as); return (DDI_SUCCESS); } /* * xpvtap_user_request_push() */ static int xpvtap_user_request_push(xpvtap_state_t *state, blkif_request_t *req, uint_t uid) { blkif_request_t *outstanding_req; blkif_front_ring_t *uring; blkif_request_t *target; xpvtap_user_map_t *map; uring = &state->bt_user_ring.ur_ring; map = &state->bt_map; target = RING_GET_REQUEST(uring, uring->req_prod_pvt); /* * Save request from the frontend. used for ID mapping and unmap * on response/cleanup */ outstanding_req = &map->um_outstanding_reqs[uid]; bcopy(req, outstanding_req, sizeof (*outstanding_req)); /* put the request on the user ring */ bcopy(req, target, sizeof (*req)); target->id = (uint64_t)uid; uring->req_prod_pvt++; pollwakeup(&state->bt_pollhead, POLLIN | POLLRDNORM); return (DDI_SUCCESS); } static void xpvtap_user_request_unmap(xpvtap_state_t *state, uint_t uid) { blkif_request_t *req; struct seg *seg; struct as *as; caddr_t uaddr; int e; as = state->bt_map.um_as; if (as == NULL) { return; } /* get a copy of the original request */ req = &state->bt_map.um_outstanding_reqs[uid]; /* unmap the grefs for this request */ if ((req->operation != BLKIF_OP_WRITE_BARRIER) && (req->operation != BLKIF_OP_FLUSH_DISKCACHE) && (req->nr_segments != 0)) { uaddr = XPVTAP_GREF_REQADDR(state->bt_map.um_guest_pages, uid); AS_LOCK_ENTER(as, RW_READER); seg = as_findseg(as, state->bt_map.um_guest_pages, 0); if ((seg == NULL) || ((uaddr + mmu_ptob(req->nr_segments)) > (seg->s_base + seg->s_size))) { AS_LOCK_EXIT(as); xpvtap_rs_free(state->bt_map.um_rs, uid); return; } e = segmf_release_grefs(seg, uaddr, req->nr_segments); if (e != 0) { cmn_err(CE_WARN, "unable to release grefs"); } AS_LOCK_EXIT(as); } /* free up the user ring id */ xpvtap_rs_free(state->bt_map.um_rs, uid); } static int xpvtap_user_response_get(xpvtap_state_t *state, blkif_response_t *resp, uint_t *uid) { blkif_front_ring_t *uring; blkif_response_t *target; uring = &state->bt_user_ring.ur_ring; if (!RING_HAS_UNCONSUMED_RESPONSES(uring)) { return (B_FALSE); } target = NULL; target = RING_GET_RESPONSE(uring, uring->rsp_cons); if (target == NULL) { return (B_FALSE); } /* copy out the user app response */ bcopy(target, resp, sizeof (*resp)); uring->rsp_cons++; /* restore the quests id from the original request */ *uid = (uint_t)resp->id; resp->id = state->bt_map.um_outstanding_reqs[*uid].id; return (B_TRUE); } /* * xpvtap_user_app_stop() */ static void xpvtap_user_app_stop(caddr_t arg) { xpvtap_state_t *state; clock_t rc; state = (xpvtap_state_t *)arg; /* * Give the app 10 secs to exit. If it doesn't exit, it's not a serious * problem, we just won't auto-detach the driver. */ mutex_enter(&state->bt_open.bo_mutex); if (state->bt_open.bo_opened) { rc = cv_reltimedwait(&state->bt_open.bo_exit_cv, &state->bt_open.bo_mutex, drv_usectohz(10000000), TR_CLOCK_TICK); if (rc <= 0) { cmn_err(CE_NOTE, "!user process still has driver open, " "deferring detach\n"); } } mutex_exit(&state->bt_open.bo_mutex); } /* * xpvtap_rs_init() * Initialize the resource structure. init() returns a handle to be used * for the rest of the resource functions. This code is written assuming * that min_val will be close to 0. Therefore, we will allocate the free * buffer only taking max_val into account. */ static void xpvtap_rs_init(uint_t min_val, uint_t max_val, xpvtap_rs_hdl_t *handle) { xpvtap_rs_t *rstruct; uint_t array_size; uint_t index; ASSERT(handle != NULL); ASSERT(min_val < max_val); /* alloc space for resource structure */ rstruct = kmem_alloc(sizeof (xpvtap_rs_t), KM_SLEEP); /* * Test to see if the max value is 64-bit aligned. If so, we don't need * to allocate an extra 64-bit word. alloc space for free buffer * (8 bytes per uint64_t). */ if ((max_val & 0x3F) == 0) { rstruct->rs_free_size = (max_val >> 6) * 8; } else { rstruct->rs_free_size = ((max_val >> 6) + 1) * 8; } rstruct->rs_free = kmem_alloc(rstruct->rs_free_size, KM_SLEEP); /* Initialize resource structure */ rstruct->rs_min = min_val; rstruct->rs_last = min_val; rstruct->rs_max = max_val; mutex_init(&rstruct->rs_mutex, NULL, MUTEX_DRIVER, NULL); rstruct->rs_flushing = B_FALSE; /* Mark all resources as free */ array_size = rstruct->rs_free_size >> 3; for (index = 0; index < array_size; index++) { rstruct->rs_free[index] = (uint64_t)0xFFFFFFFFFFFFFFFF; } /* setup handle which is returned from this function */ *handle = rstruct; } /* * xpvtap_rs_fini() * Frees up the space allocated in init(). Notice that a pointer to the * handle is used for the parameter. fini() will set the handle to NULL * before returning. */ static void xpvtap_rs_fini(xpvtap_rs_hdl_t *handle) { xpvtap_rs_t *rstruct; ASSERT(handle != NULL); rstruct = (xpvtap_rs_t *)*handle; mutex_destroy(&rstruct->rs_mutex); kmem_free(rstruct->rs_free, rstruct->rs_free_size); kmem_free(rstruct, sizeof (xpvtap_rs_t)); /* set handle to null. This helps catch bugs. */ *handle = NULL; } /* * xpvtap_rs_alloc() * alloc a resource. If alloc fails, we are out of resources. */ static int xpvtap_rs_alloc(xpvtap_rs_hdl_t handle, uint_t *resource) { xpvtap_rs_t *rstruct; uint_t array_idx; uint64_t free; uint_t index; uint_t last; uint_t min; uint_t max; ASSERT(handle != NULL); ASSERT(resource != NULL); rstruct = (xpvtap_rs_t *)handle; mutex_enter(&rstruct->rs_mutex); min = rstruct->rs_min; max = rstruct->rs_max; /* * Find a free resource. This will return out of the loop once it finds * a free resource. There are a total of 'max'-'min'+1 resources. * Performs a round robin allocation. */ for (index = min; index <= max; index++) { array_idx = rstruct->rs_last >> 6; free = rstruct->rs_free[array_idx]; last = rstruct->rs_last & 0x3F; /* if the next resource to check is free */ if ((free & ((uint64_t)1 << last)) != 0) { /* we are using this resource */ *resource = rstruct->rs_last; /* take it out of the free list */ rstruct->rs_free[array_idx] &= ~((uint64_t)1 << last); /* * increment the last count so we start checking the * next resource on the next alloc(). Note the rollover * at 'max'+1. */ rstruct->rs_last++; if (rstruct->rs_last > max) { rstruct->rs_last = rstruct->rs_min; } /* unlock the resource structure */ mutex_exit(&rstruct->rs_mutex); return (DDI_SUCCESS); } /* * This resource is not free, lets go to the next one. Note the * rollover at 'max'. */ rstruct->rs_last++; if (rstruct->rs_last > max) { rstruct->rs_last = rstruct->rs_min; } } mutex_exit(&rstruct->rs_mutex); return (DDI_FAILURE); } /* * xpvtap_rs_free() * Free the previously alloc'd resource. Once a resource has been free'd, * it can be used again when alloc is called. */ static void xpvtap_rs_free(xpvtap_rs_hdl_t handle, uint_t resource) { xpvtap_rs_t *rstruct; uint_t array_idx; uint_t offset; ASSERT(handle != NULL); rstruct = (xpvtap_rs_t *)handle; ASSERT(resource >= rstruct->rs_min); ASSERT(resource <= rstruct->rs_max); if (!rstruct->rs_flushing) { mutex_enter(&rstruct->rs_mutex); } /* Put the resource back in the free list */ array_idx = resource >> 6; offset = resource & 0x3F; rstruct->rs_free[array_idx] |= ((uint64_t)1 << offset); if (!rstruct->rs_flushing) { mutex_exit(&rstruct->rs_mutex); } } /* * xpvtap_rs_flush() */ static void xpvtap_rs_flush(xpvtap_rs_hdl_t handle, xpvtap_rs_cleanup_t callback, void *arg) { xpvtap_rs_t *rstruct; uint_t array_idx; uint64_t free; uint_t index; uint_t last; uint_t min; uint_t max; ASSERT(handle != NULL); rstruct = (xpvtap_rs_t *)handle; mutex_enter(&rstruct->rs_mutex); min = rstruct->rs_min; max = rstruct->rs_max; rstruct->rs_flushing = B_TRUE; /* * for all resources not free, call the callback routine to clean it * up. */ for (index = min; index <= max; index++) { array_idx = rstruct->rs_last >> 6; free = rstruct->rs_free[array_idx]; last = rstruct->rs_last & 0x3F; /* if the next resource to check is not free */ if ((free & ((uint64_t)1 << last)) == 0) { /* call the callback to cleanup */ (*callback)(arg, rstruct->rs_last); /* put it back in the free list */ rstruct->rs_free[array_idx] |= ((uint64_t)1 << last); } /* go to the next one. Note the rollover at 'max' */ rstruct->rs_last++; if (rstruct->rs_last > max) { rstruct->rs_last = rstruct->rs_min; } } mutex_exit(&rstruct->rs_mutex); } /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2008 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ #ifndef _SYS_XPVTAP_H #define _SYS_XPVTAP_H #ifdef __cplusplus extern "C" { #endif #include /* Notification from user app that it has pushed responses */ #define XPVTAP_IOCTL_RESP_PUSH 1 /* Number of bytes the user app should mmap for the gref pages */ #define XPVTAP_GREF_BUFSIZE \ (BLKIF_RING_SIZE * BLKIF_MAX_SEGMENTS_PER_REQUEST * PAGESIZE) #ifdef _KERNEL #include #define XPVTAP_GREF_REQADDR(base, id) (caddr_t) \ ((uintptr_t)base + (id * BLKIF_MAX_SEGMENTS_PER_REQUEST * PAGESIZE)) /* structure used to keep track of resources */ typedef struct xpvtap_rs_s { /* * Bounds of resource allocation. We will start allocating at rs_min * and rollover at rs_max+1 (rs_max is included). e.g. for rs_min=0 * and rs_max=7, we will have 8 total resources which can be alloced. */ uint_t rs_min; uint_t rs_max; /* * rs_free points to an array of 64-bit values used to track resource * allocation. rs_free_size is the free buffer size in bytes. */ uint64_t *rs_free; uint_t rs_free_size; /* * last tracks the last alloc'd resource. This allows us to do a round * robin allocation. */ uint_t rs_last; /* * set when flushing all allocated resources. We'll know the lock * is held. */ boolean_t rs_flushing; kmutex_t rs_mutex; } xpvtap_rs_t; typedef struct xpvtap_rs_s *xpvtap_rs_hdl_t; /* track if user app has the device open, and sleep waiting for close */ typedef struct xpvtap_open_s { kmutex_t bo_mutex; boolean_t bo_opened; kcondvar_t bo_exit_cv; } xpvtap_open_t; /* * ring between driver and user app. requests are forwared from the * guest to the user app on this ring. reponses from the user app come in * on this ring are then are forwarded to the guest. */ typedef struct xpvtap_user_ring_s { /* ring state */ blkif_front_ring_t ur_ring; /* * pointer to allocated memory for the ring which is shared between * the driver and the app. */ blkif_sring_t *ur_sring; /* umem cookie for free'ing up the umem */ ddi_umem_cookie_t ur_cookie; RING_IDX ur_prod_polled; } xpvtap_user_ring_t; /* * track the requests that come in from the guest. we need to track the * requests for two reasons. first, we need to know how many grefs we need * to unmap when the app sends the response. second, since we use the ID in * the request to index into um_guest_pages (tells the app where the segments * are mapped), we need to have a mapping between the the ID we sent in the * request to the app and the ID we got from the guest request. The response * to the guest needs to have the later. */ typedef struct xpvtap_user_map_s { /* address space of the user app. grab this in open */ struct as *um_as; /* state to track request IDs we can send to the user app */ xpvtap_rs_hdl_t um_rs; /* * base user app VA of the mapped grefs. this VA space is large enough * to map the max pages per request * max outstanding requests. */ caddr_t um_guest_pages; size_t um_guest_size; /* * have we locked down the gref buffer's ptes and registered * them with segmf. This needs to happen after the user app * has mmaped the gref buf. */ boolean_t um_registered; /* * array of outstanding requests to the user app. Index into this * array using the ID in the user app request. */ blkif_request_t *um_outstanding_reqs; } xpvtap_user_map_t; /* thread start, wake, exit state */ typedef struct xpvtap_user_thread_s { kmutex_t ut_mutex; kcondvar_t ut_wake_cv; volatile boolean_t ut_wake; volatile boolean_t ut_exit; kcondvar_t ut_exit_done_cv; volatile boolean_t ut_exit_done; ddi_taskq_t *ut_taskq; } xpvtap_user_thread_t; /* driver state */ typedef struct xpvtap_state_s { dev_info_t *bt_dip; int bt_instance; /* ring between the guest and xpvtap */ blk_ring_t bt_guest_ring; /* ring between xpvtap and the user app */ xpvtap_user_ring_t bt_user_ring; xpvtap_user_map_t bt_map; xpvtap_user_thread_t bt_thread; struct pollhead bt_pollhead; xpvtap_open_t bt_open; } xpvtap_state_t; #endif /* _KERNEL */ #ifdef __cplusplus } #endif #endif /* _SYS_XPVTAP_H */