xen - zygaena/hammerhead

root / base / usr / src / uts / common / xen
xen Plain Text 37304 lines 999.2 KB
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

/*
 * Xen event provider for DTrace
 *
 * NOTE: This provider is PRIVATE. It is intended as a short-term solution and
 * may disappear or be re-implemented at anytime.
 *
 * This provider isn't suitable as a general-purpose solution for a number of
 * reasons. First and foremost, we rely on the Xen tracing mechanism and don't
 * have any way to gather data other than that collected by the Xen trace
 * buffers. Further, it does not fit into the DTrace model (see "Interacting
 * with DTrace" below.)
 *
 *
 * Tracing in Xen
 * --------------
 *
 * Xen implements a tracing facility for generating and collecting execution
 * event traces from the hypervisor. When tracing is enabled, compiled in
 * probes record events in contiguous per-CPU trace buffers.
 *
 *               +---------+
 * +------+      |         |
 * | CPUn |----> | BUFFERn |
 * +------+      |         |
 *               +---------+- tbuf.va + (tbuf.size * n)
 *               :         :
 *               +---------+
 * +------+      |         |
 * | CPU1 |----> | BUFFER1 |
 * +------+      |         |
 *               +---------+- tbuf.va + tbuf.size
 * +------+      |         |
 * | CPU0 |----> | BUFFER0 |
 * +------+      |         |
 *               +---------+- tbuf.va
 *
 * Each CPU buffer consists of a metadata header followed by the trace records.
 * The metadata consists of a producer/consumer pair of pointers into the buffer
 * that point to the next record to be written and the next record to be read
 * respectively.
 *
 * A trace record can be in one of two forms, depending on if the TSC is
 * included. The record header indicates whether or not the TSC field is
 * present.
 *
 * 1. Trace record without TSC:
 * +------------------------------------------------------------+
 * | HEADER(uint32_t) |            DATA FIELDS                  |
 * +------------------------------------------------------------+
 *
 * 2. Trace record with TSC:
 * +--------------------------------------------------------------------------+
 * | HEADER(uint32_t) | TSC(uint64_t) |              DATA FIELDS              |
 * +--------------------------------------------------------------------------+
 *
 * Where,
 *
 * HEADER bit field:
 * +--------------------------------------------------------------------------+
 * | C |  NDATA  |                        EVENT                               |
 * +--------------------------------------------------------------------------+
 *  31  30     28 27                                                         0
 *
 * EVENT: Event ID.
 * NDATA: Number of populated data fields.
 *     C: TSC included.
 *
 * DATA FIELDS:
 * +--------------------------------------------------------------------------+
 * | D1(uint32_t) | D2(uint32_t) | D3(uint32_t) |     . . .    | D7(uint32_t) |
 * +--------------------------------------------------------------------------+
 *
 *
 * Interacting with DTrace
 * -----------------------
 *
 * Every xdt_poll_nsec nano-seconds we poll the trace buffers for data and feed
 * each entry into dtrace_probe() with the corresponding probe ID for the event.
 * As a result of this periodic collection implementation probe firings are
 * asynchronous. This is the only sensible way to implement this form of
 * provider, but because of its asynchronous nature asking things like
 * "current CPU" and, more importantly, arbitrary questions about the context
 * surrounding the probe firing are not meaningful. So, consumers should not
 * attempt to infer anything beyond what is supplied via the probe arguments.
 */

#include <sys/xpv_user.h>

#include <sys/types.h>
#include <sys/sysmacros.h>
#include <sys/modctl.h>
#include <sys/sunddi.h>
#include <sys/ddi.h>
#include <sys/conf.h>
#include <sys/devops.h>
#include <sys/stat.h>
#include <sys/cmn_err.h>
#include <sys/dtrace.h>
#include <sys/sdt.h>
#include <sys/cyclic.h>
#include <vm/seg_kmem.h>
#include <vm/hat_i86.h>

#include <sys/hypervisor.h>
#include <xen/public/trace.h>
#include <xen/public/sched.h>

#define	XDT_POLL_DEFAULT	100000000	/* default poll interval (ns) */
#define	XDT_POLL_MIN		10000000	/* min poll interval (ns) */
#define	XDT_TBUF_RETRY		50		/* tbuf disable retry count */

/*
 * The domid must match IDLE_DOMAIN_ID in xen.hg/xen/include/xen/sched.h
 * in the xVM gate.
 */
#define	IS_IDLE_DOM(domid)	(domid == 0x7FFFU)

/* Macros to extract the domid and cpuid from a HVM trace data field */
#define	HVM_DOMID(d)		(d >> 16)
#define	HVM_VCPUID(d)		(d & 0xFFFF)

/* Flags for shadow page table events */
#define	SH_GUEST_32	0x000
#define	SH_GUEST_PAE	0x100
#define	SH_GUEST_64	0x200

#define	XDT_PROBE5(event, arg0, arg1, arg2, arg3, arg4) {		\
	dtrace_id_t id = xdt_probemap[event];				\
	if (id)								\
		dtrace_probe(id, arg0, arg1, arg2, arg3, arg4);		\
}									\

#define	XDT_PROBE4(event, arg0, arg1, arg2, arg3) \
	XDT_PROBE5(event, arg0, arg1, arg2, arg3, 0)

#define	XDT_PROBE3(event, arg0, arg1, arg2) \
	XDT_PROBE5(event, arg0, arg1, arg2, 0, 0)

#define	XDT_PROBE2(event, arg0, arg1) \
	XDT_PROBE5(event, arg0, arg1, 0, 0, 0)

#define	XDT_PROBE1(event, arg0) \
	XDT_PROBE5(event, arg0, 0, 0, 0, 0)

#define	XDT_PROBE0(event) \
	XDT_PROBE5(event, 0, 0, 0, 0, 0)

/* Probe classes */
#define	XDT_SCHED			0
#define	XDT_MEM				1
#define	XDT_HVM				2
#define	XDT_GEN				3
#define	XDT_PV				4
#define	XDT_SHADOW			5
#define	XDT_PM				6
#define	XDT_NCLASSES			7

/* Probe events */
#define	XDT_EVT_INVALID			(-(int)1)
#define	XDT_SCHED_OFF_CPU		0
#define	XDT_SCHED_ON_CPU		1
#define	XDT_SCHED_IDLE_OFF_CPU		2
#define	XDT_SCHED_IDLE_ON_CPU		3
#define	XDT_SCHED_BLOCK			4
#define	XDT_SCHED_SLEEP			5
#define	XDT_SCHED_WAKE			6
#define	XDT_SCHED_YIELD			7
#define	XDT_SCHED_SHUTDOWN_POWEROFF	8
#define	XDT_SCHED_SHUTDOWN_REBOOT	9
#define	XDT_SCHED_SHUTDOWN_SUSPEND	10
#define	XDT_SCHED_SHUTDOWN_CRASH	11
#define	XDT_MEM_PAGE_GRANT_MAP		12
#define	XDT_MEM_PAGE_GRANT_UNMAP	13
#define	XDT_MEM_PAGE_GRANT_TRANSFER	14
#define	XDT_HVM_VMENTRY			15
#define	XDT_HVM_VMEXIT			16
#define	XDT_TRC_LOST_RECORDS		17
#define	XDT_SCHED_ADD_VCPU		18
#define	XDT_SCHED_REM_VCPU		19	/* unused */
#define	XDT_SCHED_CTL			20	/* unused */
#define	XDT_SCHED_ADJDOM		21
#define	XDT_SCHED_S_TIMER_FN		22	/* unused */
#define	XDT_SCHED_T_TIMER_FN		23	/* unused */
#define	XDT_SCHED_DOM_TIMER_FN		24	/* unused */
#define	XDT_PV_HYPERCALL		25
#define	XDT_PV_TRAP			26
#define	XDT_PV_PAGE_FAULT		27
#define	XDT_PV_FORCED_INVALID_OP	28
#define	XDT_PV_EMULATE_PRIVOP		29
#define	XDT_PV_EMULATE_4GB		30	/* unused (32-bit HV only ) */
#define	XDT_PV_MATH_STATE_RESTORE	31
#define	XDT_PV_PAGING_FIXUP		32
#define	XDT_PV_DT_MAPPING_FAULT		33
#define	XDT_PV_PTWR_EMULATION		34
#define	XDT_HVM_PF_XEN			35
#define	XDT_HVM_PF_INJECT		36
#define	XDT_HVM_EXC_INJECT		37
#define	XDT_HVM_VIRQ_INJECT		38
#define	XDT_HVM_VIRQ_REINJECT		39
#define	XDT_HVM_IO_READ			40	/* unused */
#define	XDT_HVM_IO_WRITE		41	/* unused */
#define	XDT_HVM_CR_READ			42
#define	XDT_HVM_CR_WRITE		43
#define	XDT_HVM_DR_READ			44	/* unused */
#define	XDT_HVM_DR_WRITE		45	/* unused */
#define	XDT_HVM_MSR_READ		46
#define	XDT_HVM_MSR_WRITE		47
#define	XDT_HVM_CPUID			48
#define	XDT_HVM_INTR			49
#define	XDT_HVM_INTR_WINDOW		50
#define	XDT_HVM_NMI			51
#define	XDT_HVM_SMI			52
#define	XDT_HVM_VMMCALL			53
#define	XDT_HVM_HLT			54
#define	XDT_HVM_INVLPG			55
#define	XDT_HVM_MCE			56
#define	XDT_HVM_IOPORT_READ		57
#define	XDT_HVM_IOPORT_WRITE		58
#define	XDT_HVM_CLTS			59
#define	XDT_HVM_LMSW			60
#define	XDT_HVM_IOMEM_READ		61
#define	XDT_HVM_IOMEM_WRITE		62
#define	XDT_SHADOW_NOT_SHADOW			63
#define	XDT_SHADOW_FAST_PROPAGATE		64
#define	XDT_SHADOW_FAST_MMIO			65
#define	XDT_SHADOW_FALSE_FAST_PATH		66
#define	XDT_SHADOW_MMIO				67
#define	XDT_SHADOW_FIXUP			68
#define	XDT_SHADOW_DOMF_DYING			69
#define	XDT_SHADOW_EMULATE			70
#define	XDT_SHADOW_EMULATE_UNSHADOW_USER	71
#define	XDT_SHADOW_EMULATE_UNSHADOW_EVTINJ	72
#define	XDT_SHADOW_EMULATE_UNSHADOW_UNHANDLED	73
#define	XDT_SHADOW_WRMAP_BF			74
#define	XDT_SHADOW_PREALLOC_UNPIN		75
#define	XDT_SHADOW_RESYNC_FULL			76
#define	XDT_SHADOW_RESYNC_ONLY			77
#define	XDT_PM_FREQ_CHANGE		78
#define	XDT_PM_IDLE_ENTRY		79
#define	XDT_PM_IDLE_EXIT		80
#define	XDT_SCHED_RUNSTATE_CHANGE	81
#define	XDT_SCHED_CONTINUE_RUNNING	82
#define	XDT_NEVENTS			83

typedef struct {
	const char	*pr_mod;	/* probe module */
	const char	*pr_name;	/* probe name */
	int		evt_id;		/* event id */
	uint_t		class;		/* probe class */
} xdt_probe_t;

typedef struct {
	uint32_t	trc_mask;	/* trace mask */
	uint32_t	cnt;		/* num enabled probes in class */
} xdt_classinfo_t;

typedef struct {
	ulong_t prev_domid;		/* previous dom executed */
	ulong_t prev_vcpuid;		/* previous vcpu executed */
	ulong_t prev_ctime;		/* time spent on cpu */
	ulong_t next_domid;		/* next dom to be scheduled */
	ulong_t next_vcpuid;		/* next vcpu to be scheduled */
	ulong_t next_wtime;		/* time spent waiting to get on cpu */
	ulong_t next_ts;		/* allocated time slice */
	ulong_t cur_domid;		/* current dom */
	ulong_t cur_vcpuid;		/* current vcpuid */
	int curinfo_valid;		/* info is valid */
} xdt_schedinfo_t;

static struct {
	uint_t cnt;			/* total num of trace buffers */
	size_t size;			/* size of each cpu buffer */
	mfn_t start_mfn;		/* starting mfn of buffers */
	caddr_t va;			/* va buffers are mapped into */

	/* per-cpu buffers */
	struct t_buf **meta;		/* buffer metadata */
	struct t_rec **data;		/* buffer data records */

	/* statistics */
	uint64_t stat_dropped_recs;	/* records dropped */
	uint64_t stat_spurious_cpu;	/* recs with garbage cpuids */
	uint64_t stat_spurious_switch;	/* inconsistent vcpu switches */
	uint64_t stat_unknown_shutdown;	/* unknown shutdown code */
	uint64_t stat_unknown_recs;	/* unknown records */
} tbuf;

static size_t tbuf_data_size;

static char *xdt_stats[] = {
	"dropped_recs",
};

/*
 * Tunable variables
 *
 * The following may be tuned by adding a line to /etc/system that
 * includes both the name of the module ("xdt") and the name of the variable.
 * For example:
 *     set xdt:xdt_tbuf_pages = 40
 */
uint_t xdt_tbuf_pages = 20;			/* pages to alloc per-cpu buf */

/*
 * The following may be tuned by adding a line to
 * /platform/i86xpv/kernel/drv/xdt.conf.
 * For example:
 *     xdt_poll_nsec = 200000000;
 */
static hrtime_t xdt_poll_nsec;			/* trace buffer poll interval */

/*
 * Another tunable variable: the maximum number of records to process
 * in one scan. If it is 0 (e.g. not set in /etc/system), it will
 * be set to ncpu * (bufsize / max_rec_size).
 *
 * Having an upper limit avoids a situation where the scan would loop
 * endlessly in case the hypervisor adds records quicker than we
 * can process them. It's better to drop records than to loop, obviously.
 */
uint_t xdt_max_recs = 0;

/*
 * Internal variables
 */
static dev_info_t *xdt_devi;
static dtrace_provider_id_t xdt_id;
static uint_t xdt_ncpus;			/* total number of phys CPUs */
static uint32_t cur_trace_mask;			/* current trace mask */
static xdt_schedinfo_t *xdt_cpu_schedinfo;	/* per-cpu sched info */
dtrace_id_t xdt_probemap[XDT_NEVENTS];		/* map of enabled probes */
dtrace_id_t xdt_prid[XDT_NEVENTS];		/* IDs of registered events */
static cyclic_id_t xdt_cyclic = CYCLIC_NONE;
static kstat_t *xdt_kstats;
static xdt_classinfo_t xdt_classinfo[XDT_NCLASSES];

/*
 * These provide context when probes fire. They can be accessed
 * from xdt dtrace probe (as `xdt_curdom, etc). It's ok for these
 * to be global, and not per-cpu, as probes are run strictly in sequence
 * as the trace buffers are
 */
uint_t xdt_curdom, xdt_curvcpu, xdt_curpcpu;
uint64_t xdt_timestamp;

static xdt_probe_t xdt_probe[] = {
	/* Sched probes */
	{ "sched", "off-cpu", XDT_SCHED_OFF_CPU, XDT_SCHED },
	{ "sched", "on-cpu", XDT_SCHED_ON_CPU, XDT_SCHED },
	{ "sched", "idle-off-cpu", XDT_SCHED_IDLE_OFF_CPU, XDT_SCHED },
	{ "sched", "idle-on-cpu", XDT_SCHED_IDLE_ON_CPU, XDT_SCHED },
	{ "sched", "block", XDT_SCHED_BLOCK, XDT_SCHED },
	{ "sched", "sleep", XDT_SCHED_SLEEP, XDT_SCHED },
	{ "sched", "wake", XDT_SCHED_WAKE, XDT_SCHED },
	{ "sched", "yield", XDT_SCHED_YIELD, XDT_SCHED },
	{ "sched", "shutdown-poweroff", XDT_SCHED_SHUTDOWN_POWEROFF,
		XDT_SCHED },
	{ "sched", "shutdown-reboot", XDT_SCHED_SHUTDOWN_REBOOT, XDT_SCHED },
	{ "sched", "shutdown-suspend", XDT_SCHED_SHUTDOWN_SUSPEND, XDT_SCHED },
	{ "sched", "shutdown-crash", XDT_SCHED_SHUTDOWN_CRASH, XDT_SCHED },
	{ "sched", "add", XDT_SCHED_ADD_VCPU, XDT_SCHED },
	{ "sched", "runstate-change", XDT_SCHED_RUNSTATE_CHANGE, XDT_SCHED },
	{ "sched", "continue-running", XDT_SCHED_CONTINUE_RUNNING, XDT_SCHED },

	/* Memory probes */
	{ "mem", "page-grant-map", XDT_MEM_PAGE_GRANT_MAP, XDT_MEM },
	{ "mem", "page-grant-unmap", XDT_MEM_PAGE_GRANT_UNMAP, XDT_MEM },
	{ "mem", "page-grant-transfer", XDT_MEM_PAGE_GRANT_TRANSFER, XDT_MEM },

	{"pv", "hypercall", XDT_PV_HYPERCALL, XDT_PV },
	{"pv", "trap", XDT_PV_TRAP, XDT_PV },
	{"pv", "page-fault", XDT_PV_PAGE_FAULT, XDT_PV },
	{"pv", "forced-invalid-op", XDT_PV_FORCED_INVALID_OP, XDT_PV },
	{"pv", "emulate-priv-op", XDT_PV_EMULATE_PRIVOP, XDT_PV },
	{"pv", "math-state-restore", XDT_PV_MATH_STATE_RESTORE, XDT_PV },
	{"pv", "paging-fixup", XDT_PV_PAGING_FIXUP, XDT_PV },
	{"pv", "dt-mapping-fault", XDT_PV_DT_MAPPING_FAULT, XDT_PV },
	{"pv", "pte-write-emul", XDT_PV_PTWR_EMULATION, XDT_PV },

	/* HVM probes */
	{ "hvm", "vmentry", XDT_HVM_VMENTRY, XDT_HVM },
	{ "hvm", "vmexit", XDT_HVM_VMEXIT, XDT_HVM },
	{ "hvm", "pagefault-xen", XDT_HVM_PF_XEN, XDT_HVM },
	{ "hvm", "pagefault-inject", XDT_HVM_PF_INJECT, XDT_HVM },
	{ "hvm", "exception-inject", XDT_HVM_EXC_INJECT, XDT_HVM },
	{ "hvm", "virq-inject", XDT_HVM_VIRQ_INJECT, XDT_HVM },
	{ "hvm", "cr-read", XDT_HVM_CR_READ, XDT_HVM },
	{ "hvm", "cr-write", XDT_HVM_CR_WRITE, XDT_HVM },
	{ "hvm", "msr-read", XDT_HVM_MSR_READ, XDT_HVM },
	{ "hvm", "msr-write", XDT_HVM_MSR_WRITE, XDT_HVM },
	{ "hvm", "cpuid", XDT_HVM_CPUID, XDT_HVM },
	{ "hvm", "intr", XDT_HVM_INTR, XDT_HVM },
	{ "hvm", "intr-window", XDT_HVM_INTR_WINDOW, XDT_HVM },
	{ "hvm", "nmi", XDT_HVM_NMI, XDT_HVM },
	{ "hvm", "smi", XDT_HVM_SMI, XDT_HVM },
	{ "hvm", "vmmcall", XDT_HVM_VMMCALL, XDT_HVM },
	{ "hvm", "hlt", XDT_HVM_HLT, XDT_HVM },
	{ "hvm", "invlpg", XDT_HVM_INVLPG, XDT_HVM },
	{ "hvm", "mce", XDT_HVM_MCE, XDT_HVM },
	{ "hvm", "pio-read", XDT_HVM_IOPORT_READ, XDT_HVM },
	{ "hvm", "pio-write", XDT_HVM_IOPORT_WRITE, XDT_HVM },
	{ "hvm", "mmio-read", XDT_HVM_IOMEM_READ, XDT_HVM },
	{ "hvm", "mmio-write", XDT_HVM_IOMEM_WRITE, XDT_HVM },
	{ "hvm", "clts", XDT_HVM_CLTS, XDT_HVM },
	{ "hvm", "lmsw", XDT_HVM_LMSW, XDT_HVM },

	{ "shadow", "fault-not-shadow", XDT_SHADOW_NOT_SHADOW, XDT_SHADOW },
	{ "shadow", "fast-propagate", XDT_SHADOW_FAST_PROPAGATE, XDT_SHADOW },
	{ "shadow", "fast-mmio", XDT_SHADOW_FAST_MMIO, XDT_SHADOW },
	{ "shadow", "false-fast-path", XDT_SHADOW_FALSE_FAST_PATH,
	    XDT_SHADOW },
	{ "shadow", "mmio", XDT_SHADOW_MMIO, XDT_SHADOW },
	{ "shadow", "fixup", XDT_SHADOW_FIXUP, XDT_SHADOW },
	{ "shadow", "domf-dying", XDT_SHADOW_DOMF_DYING, XDT_SHADOW },
	{ "shadow", "emulate", XDT_SHADOW_EMULATE, XDT_SHADOW },
	{ "shadow", "emulate-unshadow-user", XDT_SHADOW_EMULATE_UNSHADOW_USER,
	    XDT_SHADOW },
	{ "shadow", "emulate-unshadow-evtinj",
	    XDT_SHADOW_EMULATE_UNSHADOW_EVTINJ, XDT_SHADOW },
	{ "shadow", "emulate-unshadow-unhandled",
	    XDT_SHADOW_EMULATE_UNSHADOW_UNHANDLED, XDT_SHADOW },
	{ "shadow", "wrmap-bf", XDT_SHADOW_WRMAP_BF, XDT_SHADOW },
	{ "shadow", "prealloc-unpin", XDT_SHADOW_PREALLOC_UNPIN, XDT_SHADOW },
	{ "shadow", "resync-full", XDT_SHADOW_RESYNC_FULL, XDT_SHADOW },
	{ "shadow", "resync-only", XDT_SHADOW_RESYNC_ONLY, XDT_SHADOW },

	{ "pm", "freq-change", XDT_PM_FREQ_CHANGE, XDT_PM },
	{ "pm", "idle-entry", XDT_PM_IDLE_ENTRY, XDT_PM },
	{ "pm", "idle-exit", XDT_PM_IDLE_EXIT, XDT_PM },

	/* Trace buffer related probes */
	{ "trace", "records-lost", XDT_TRC_LOST_RECORDS, XDT_GEN },

	{ NULL }
};

static inline uint32_t
xdt_nr_active_probes()
{
	int i;
	uint32_t tot = 0;

	for (i = 0; i < XDT_NCLASSES; i++)
		tot += xdt_classinfo[i].cnt;

	return (tot);
}

static void
xdt_init_trace_masks(void)
{
	xdt_classinfo[XDT_SCHED].trc_mask = TRC_SCHED;
	xdt_classinfo[XDT_MEM].trc_mask = TRC_MEM;
	xdt_classinfo[XDT_HVM].trc_mask = TRC_HVM;
	xdt_classinfo[XDT_GEN].trc_mask = TRC_GEN;
	xdt_classinfo[XDT_PV].trc_mask = TRC_PV;
	xdt_classinfo[XDT_SHADOW].trc_mask = TRC_SHADOW;
	xdt_classinfo[XDT_PM].trc_mask = TRC_PM;
}

static int
xdt_kstat_update(kstat_t *ksp, int flag)
{
	kstat_named_t *knp;

	if (flag != KSTAT_READ)
		return (EACCES);

	knp = ksp->ks_data;

	/*
	 * Assignment order should match that of the names in
	 * xdt_stats.
	 */
	(knp++)->value.ui64 = tbuf.stat_dropped_recs;

	return (0);
}

static void
xdt_kstat_init(void)
{
	int nstats = sizeof (xdt_stats) / sizeof (xdt_stats[0]);
	char **cp = xdt_stats;
	kstat_named_t *knp;

	if ((xdt_kstats = kstat_create("xdt", 0, "trace_statistics", "misc",
	    KSTAT_TYPE_NAMED, nstats, 0)) == NULL)
		return;

	xdt_kstats->ks_update = xdt_kstat_update;

	knp = xdt_kstats->ks_data;
	while (nstats > 0) {
		kstat_named_init(knp, *cp, KSTAT_DATA_UINT64);
		knp++;
		cp++;
		nstats--;
	}

	kstat_install(xdt_kstats);
}

static int
xdt_sysctl_tbuf(xen_sysctl_tbuf_op_t *tbuf_op)
{
	xen_sysctl_t op;
	int xerr;

	op.cmd = XEN_SYSCTL_tbuf_op;
	op.interface_version = XEN_SYSCTL_INTERFACE_VERSION;
	op.u.tbuf_op = *tbuf_op;

	if ((xerr = HYPERVISOR_sysctl(&op)) != 0)
		return (xen_xlate_errcode(xerr));

	*tbuf_op = op.u.tbuf_op;
	return (0);
}

static int
xdt_map_trace_buffers(mfn_t mfn, caddr_t va, size_t len)
{
	x86pte_t pte;
	caddr_t const sva = va;
	caddr_t const eva = va + len;
	int xerr;

	ASSERT(mfn != MFN_INVALID);
	ASSERT(va != NULL);
	ASSERT(IS_PAGEALIGNED(len));

	for (; va < eva; va += MMU_PAGESIZE) {
		/*
		 * Ask the HAT to load a throwaway mapping to page zero, then
		 * overwrite it with the hypervisor mapping. It gets removed
		 * later via hat_unload().
		 */
		hat_devload(kas.a_hat, va, MMU_PAGESIZE, (pfn_t)0,
		    PROT_READ | HAT_UNORDERED_OK,
		    HAT_LOAD_NOCONSIST | HAT_LOAD);

		pte = mmu_ptob((x86pte_t)mfn) | PT_VALID | PT_USER
		    | PT_FOREIGN | PT_WRITABLE;

		xerr = HYPERVISOR_update_va_mapping_otherdomain((ulong_t)va,
		    pte, UVMF_INVLPG | UVMF_LOCAL, DOMID_XEN);

		if (xerr != 0) {
			/* unmap pages loaded so far */
			size_t ulen = (uintptr_t)(va + MMU_PAGESIZE) -
			    (uintptr_t)sva;
			hat_unload(kas.a_hat, sva, ulen, HAT_UNLOAD_UNMAP);
			return (xen_xlate_errcode(xerr));
		}

		mfn++;
	}

	return (0);
}

static int
xdt_attach_trace_buffers(void)
{
	xen_sysctl_tbuf_op_t tbuf_op;
	size_t len;
	int err;
	uint_t i;

	/*
	 * Xen does not support trace buffer re-sizing. If the buffers
	 * have already been allocated we just use them as is.
	 */
	tbuf_op.cmd  = XEN_SYSCTL_TBUFOP_get_info;
	if ((err = xdt_sysctl_tbuf(&tbuf_op)) != 0)
		return (err);

	if (tbuf_op.size == 0) {
		/* set trace buffer size */
		tbuf_op.cmd  = XEN_SYSCTL_TBUFOP_set_size;
		tbuf_op.size = xdt_tbuf_pages;
		(void) xdt_sysctl_tbuf(&tbuf_op);

		/* get trace buffer info */
		tbuf_op.cmd  = XEN_SYSCTL_TBUFOP_get_info;
		if ((err = xdt_sysctl_tbuf(&tbuf_op)) != 0)
			return (err);

		if (tbuf_op.size == 0) {
			cmn_err(CE_NOTE, "Couldn't allocate trace buffers.");
			return (ENOBUFS);
		}
	}

	tbuf.size = tbuf_op.size;
	tbuf.start_mfn = (mfn_t)tbuf_op.buffer_mfn;
	tbuf.cnt = xdt_ncpus;

	ASSERT(tbuf.start_mfn != MFN_INVALID);
	ASSERT(tbuf.cnt > 0);

	len = tbuf.size * tbuf.cnt;
	tbuf.va = vmem_alloc(heap_arena, len, VM_SLEEP);

	if ((err = xdt_map_trace_buffers(tbuf.start_mfn, tbuf.va, len)) != 0) {
		vmem_free(heap_arena, tbuf.va, len);
		tbuf.va = NULL;
		return (err);
	}

	tbuf.meta = (struct t_buf **)kmem_alloc(tbuf.cnt * sizeof (*tbuf.meta),
	    KM_SLEEP);
	tbuf.data = (struct t_rec **)kmem_alloc(tbuf.cnt * sizeof (*tbuf.data),
	    KM_SLEEP);

	for (i = 0; i < tbuf.cnt; i++) {
		void *cpu_buf = (void *)(tbuf.va + (tbuf.size * i));
		tbuf.meta[i] = cpu_buf;
		tbuf.data[i] = (struct t_rec *)((uintptr_t)cpu_buf +
		    sizeof (struct t_buf));

		/* throw away stale trace records */
		tbuf.meta[i]->cons = tbuf.meta[i]->prod;
	}

	tbuf_data_size = tbuf.size - sizeof (struct t_buf);
	if (xdt_max_recs == 0)
		xdt_max_recs = (xdt_ncpus * tbuf_data_size)
		    / sizeof (struct t_rec);

	return (0);
}

static void
xdt_detach_trace_buffers(void)
{
	size_t len = tbuf.size * tbuf.cnt;

	ASSERT(tbuf.va != NULL);

	hat_unload(kas.a_hat, tbuf.va, len,
	    HAT_UNLOAD_UNMAP | HAT_UNLOAD_UNLOCK);
	vmem_free(heap_arena, tbuf.va, len);
	kmem_free(tbuf.meta, tbuf.cnt * sizeof (*tbuf.meta));
	kmem_free(tbuf.data, tbuf.cnt * sizeof (*tbuf.data));
}

static void
xdt_update_sched_context(uint_t cpuid, uint_t dom, uint_t vcpu)
{
	xdt_schedinfo_t *sp = &xdt_cpu_schedinfo[cpuid];

	sp->cur_domid = dom;
	sp->cur_vcpuid = vcpu;
	sp->curinfo_valid = 1;
}

static void
xdt_update_domain_context(uint_t dom, uint_t vcpu)
{
	xdt_curdom = dom;
	xdt_curvcpu = vcpu;
}

static size_t
xdt_process_rec(uint_t cpuid, struct t_rec *rec)
{
	xdt_schedinfo_t *sp = &xdt_cpu_schedinfo[cpuid];
	uint_t dom, vcpu;
	int eid;
	uint32_t *data;
	uint64_t tsc, addr64, rip64, val64, pte64;
	size_t rec_size;

	ASSERT(rec != NULL);
	ASSERT(xdt_ncpus == xpv_nr_phys_cpus());

	eid = 0;
	if (cpuid >= xdt_ncpus) {
		tbuf.stat_spurious_cpu++;
		goto done;
	}

	/*
	 * If our current state isn't valid, and if this is not
	 * an event that will update our state, skip it.
	 */

	if (!sp->curinfo_valid &&
	    rec->event != TRC_SCHED_SWITCH &&
	    rec->event != TRC_LOST_RECORDS)
		goto done;

	if (rec->cycles_included) {
		data = rec->u.cycles.extra_u32;
		tsc = (((uint64_t)rec->u.cycles.cycles_hi) << 32)
		    | rec->u.cycles.cycles_lo;
	} else {
		data = rec->u.nocycles.extra_u32;
		tsc = 0;
	}

	xdt_timestamp = tsc;

	switch (rec->event) {
	/*
	 * Sched probes
	 */
	case TRC_SCHED_SWITCH_INFPREV:
		/*
		 * Info on vCPU being de-scheduled
		 *
		 * data[0] = prev domid
		 * data[1] = time spent on pcpu
		 */
		sp->prev_domid = data[0];
		sp->prev_ctime = data[1];
		break;

	case TRC_SCHED_SWITCH_INFNEXT:
		/*
		 * Info on next vCPU to be scheduled
		 *
		 * data[0] = next domid
		 * data[1] = time spent waiting to get on cpu
		 * data[2] = time slice
		 */
		sp->next_domid = data[0];
		sp->next_wtime = data[1];
		sp->next_ts = data[2];
		break;

	case TRC_SCHED_SWITCH:
		/*
		 * vCPU switch
		 *
		 * data[0] = prev domid
		 * data[1] = prev vcpuid
		 * data[2] = next domid
		 * data[3] = next vcpuid
		 */

		/*
		 * Provide valid context for this probe if there
		 * wasn't one.
		 */
		if (!sp->curinfo_valid)
			xdt_update_domain_context(data[0], data[1]);

		xdt_update_sched_context(cpuid, data[0], data[1]);

		if (data[0] != sp->prev_domid &&
		    data[2] != sp->next_domid) {
			/* prev and next info don't match doms being sched'd */
			tbuf.stat_spurious_switch++;
			goto switchdone;
		}

		sp->prev_vcpuid = data[1];
		sp->next_vcpuid = data[3];

		XDT_PROBE3(IS_IDLE_DOM(sp->prev_domid)?
		    XDT_SCHED_IDLE_OFF_CPU:XDT_SCHED_OFF_CPU,
		    sp->prev_domid, sp->prev_vcpuid, sp->prev_ctime);

		XDT_PROBE4(IS_IDLE_DOM(sp->next_domid)?
		    XDT_SCHED_IDLE_ON_CPU:XDT_SCHED_ON_CPU,
		    sp->next_domid, sp->next_vcpuid, sp->next_wtime,
		    sp->next_ts);
switchdone:
		xdt_update_sched_context(cpuid, data[2], data[3]);
		xdt_update_domain_context(data[2], data[3]);

		break;

	case TRC_SCHED_BLOCK:
		/*
		 * vCPU blocked
		 *
		 * data[0] = domid
		 * data[1] = vcpuid
		 */
		XDT_PROBE2(XDT_SCHED_BLOCK, data[0], data[1]);
		break;

	case TRC_SCHED_SLEEP:
		/*
		 * Put vCPU to sleep
		 *
		 * data[0] = domid
		 * data[1] = vcpuid
		 */
		XDT_PROBE2(XDT_SCHED_SLEEP, data[0], data[1]);
		break;

	case TRC_SCHED_WAKE:
		/*
		 * Wake up vCPU
		 *
		 * data[0] = domid
		 * data[1] = vcpuid
		 */
		XDT_PROBE2(XDT_SCHED_WAKE, data[0], data[1]);
		break;

	case TRC_SCHED_YIELD:
		/*
		 * vCPU yielded
		 *
		 * data[0] = domid
		 * data[1] = vcpuid
		 */
		XDT_PROBE2(XDT_SCHED_YIELD, data[0], data[1]);
		break;

	case TRC_SCHED_SHUTDOWN:
		/*
		 * Guest shutting down
		 *
		 * data[0] = domid
		 * data[1] = initiating vcpu
		 * data[2] = shutdown code
		 */
		switch (data[2]) {
		case SHUTDOWN_poweroff:
			eid = XDT_SCHED_SHUTDOWN_POWEROFF;
			break;
		case SHUTDOWN_reboot:
			eid = XDT_SCHED_SHUTDOWN_REBOOT;
			break;
		case SHUTDOWN_suspend:
			eid = XDT_SCHED_SHUTDOWN_SUSPEND;
			break;
		case SHUTDOWN_crash:
			eid = XDT_SCHED_SHUTDOWN_CRASH;
			break;
		default:
			tbuf.stat_unknown_shutdown++;
			goto done;
		}

		XDT_PROBE2(eid, data[0], data[1]);
		break;

	case TRC_SCHED_DOM_REM:
	case TRC_SCHED_CTL:
	case TRC_SCHED_S_TIMER_FN:
	case TRC_SCHED_T_TIMER_FN:
	case TRC_SCHED_DOM_TIMER_FN:
		/* unused */
		break;
	case TRC_SCHED_DOM_ADD:
		/*
		 * Add vcpu to a guest.
		 *
		 * data[0] = domid
		 * data[1] = vcpu
		 */
		XDT_PROBE2(XDT_SCHED_ADD_VCPU, data[0], data[1]);
		break;
	case TRC_SCHED_ADJDOM:
		/*
		 * Scheduling parameters for a guest
		 * were modified.
		 *
		 * data[0] = domid;
		 */
		XDT_PROBE1(XDT_SCHED_ADJDOM, data[1]);
		break;
	case TRC_SCHED_RUNSTATE_CHANGE:
		/*
		 * Runstate change for a VCPU.
		 *
		 * data[0] = (domain << 16) | vcpu;
		 * data[1] = oldstate;
		 * data[2] = newstate;
		 */
		XDT_PROBE4(XDT_SCHED_RUNSTATE_CHANGE, data[0] >> 16,
		    data[0] & 0xffff, data[1], data[2]);
		break;
	case TRC_SCHED_CONTINUE_RUNNING:
		/*
		 * VCPU is back on a physical CPU that it previously
		 * was also running this VCPU.
		 *
		 * data[0] = (domain << 16) | vcpu;
		 */
		XDT_PROBE2(XDT_SCHED_CONTINUE_RUNNING, data[0] >> 16,
		    data[0] & 0xffff);
		break;
	/*
	 * Mem probes
	 */
	case TRC_MEM_PAGE_GRANT_MAP:
		/*
		 * Guest mapped page grant
		 *
		 * data[0] = target domid
		 */
		XDT_PROBE1(XDT_MEM_PAGE_GRANT_MAP, data[0]);
		break;

	case TRC_MEM_PAGE_GRANT_UNMAP:
		/*
		 * Guest unmapped page grant
		 *
		 * data[0] = target domid
		 */
		XDT_PROBE1(XDT_MEM_PAGE_GRANT_UNMAP, data[0]);
		break;

	case TRC_MEM_PAGE_GRANT_TRANSFER:
		/*
		 * Page grant is being transferred
		 *
		 * data[0] = target domid
		 */
		XDT_PROBE1(XDT_MEM_PAGE_GRANT_TRANSFER, data[0]);
		break;

	/*
	 * Probes for PV domains.
	 */
	case TRC_PV_HYPERCALL:
		/*
		 * Hypercall from a 32-bit PV domain.
		 *
		 * data[0] = eip
		 * data[1] = eax
		 */
		XDT_PROBE2(XDT_PV_HYPERCALL, data[0], data[1]);
		break;
	case TRC_PV_HYPERCALL | TRC_64_FLAG:
		/*
		 * Hypercall from a 64-bit PV domain.
		 *
		 * data[0] = rip(0:31)
		 * data[1] = rip(32:63)
		 * data[2] = eax;
		 */
		rip64 = (((uint64_t)data[1]) << 32) | data[0];
		XDT_PROBE2(XDT_PV_HYPERCALL, rip64, data[2]);
		break;
	case TRC_PV_TRAP:
		/*
		 * Trap in a 32-bit PV domain.
		 *
		 * data[0] = eip
		 * data[1] = trapnr | (error_code_valid << 15)
		 *	| (error_code << 16);
		 */
		XDT_PROBE4(XDT_PV_TRAP, data[0], data[1] & 0x7fff,
		    (data[1] >> 15) & 1, data[1] >> 16);
		break;
	case TRC_PV_TRAP | TRC_64_FLAG:
		/*
		 * Trap in a 64-bit PV domain.
		 *
		 * data[0] = rip(0:31)
		 * data[1] = rip(32:63)
		 * data[2] = trapnr | (error_code_valid << 15)
		 *	| (error_code << 16);
		 */
		rip64 = (((uint64_t)data[1]) << 32) | data[2];
		XDT_PROBE4(XDT_PV_TRAP, rip64, data[2] & 0x7fff,
		    (data[2] >> 15) & 1, data[2] >> 16);
		break;
	case TRC_PV_PAGE_FAULT:
		/*
		 * Page fault in a 32-bit PV domain.
		 *
		 * data[0] = eip
		 * data[1] = vaddr
		 * data[2] = error code
		 */
		XDT_PROBE3(XDT_PV_PAGE_FAULT, data[0], data[1], data[2]);
		break;
	case TRC_PV_PAGE_FAULT | TRC_64_FLAG:
		/*
		 * Page fault in a 32-bit PV domain.
		 *
		 * data[0] = rip(0:31)
		 * data[1] = rip(31:63)
		 * data[2] = vaddr(0:31)
		 * data[3] = vaddr(31:63)
		 * data[4] = error code
		 */
		rip64 = (((uint64_t)data[1]) << 32) | data[0];
		addr64 = (((uint64_t)data[3]) << 32) | data[2];
		XDT_PROBE3(XDT_PV_PAGE_FAULT, rip64, addr64, data[4]);
		break;
	case TRC_PV_FORCED_INVALID_OP:
		/*
		 * Hypervisor emulated a forced invalid op (ud2)
		 * in a 32-bit PV domain.
		 *
		 * data[1] = eip
		 */
		XDT_PROBE1(XDT_PV_FORCED_INVALID_OP, data[1]);
		break;
	case TRC_PV_FORCED_INVALID_OP | TRC_64_FLAG:
		/*
		 * Hypervisor emulated a forced invalid op (ud2)
		 * in a 64-bit PV domain.
		 *
		 * data[1] = rip(0:31)
		 * data[2] = rip(31:63)
		 *
		 */
		rip64 = (((uint64_t)data[2]) << 32) | data[1];
		XDT_PROBE1(XDT_PV_FORCED_INVALID_OP, rip64);
		break;
	case TRC_PV_EMULATE_PRIVOP:
		/*
		 * Hypervisor emulated a privileged operation
		 * in a 32-bit PV domain.
		 *
		 * data[0] = eip
		 */
		XDT_PROBE1(XDT_PV_EMULATE_PRIVOP, data[0]);
		break;
	case TRC_PV_EMULATE_PRIVOP | TRC_64_FLAG:
		/*
		 * Hypervisor emulated a privileged operation
		 * in a 64-bit PV domain.
		 *
		 * data[0] = rip(0:31)
		 * data[1] = rip(31:63)
		 */
		rip64 = (((uint64_t)data[1]) << 32) | data[0];
		XDT_PROBE1(XDT_PV_EMULATE_PRIVOP, rip64);
		break;
	case TRC_PV_EMULATE_4GB:
		/* unused, 32-bit hypervisor only */
		break;
	case TRC_PV_MATH_STATE_RESTORE:
		/*
		 * Hypervisor restores math state after FP DNA trap.
		 *
		 * No arguments.
		 */
		XDT_PROBE0(XDT_PV_MATH_STATE_RESTORE);
		break;
	case TRC_PV_PAGING_FIXUP:
		/*
		 * Hypervisor fixed up a page fault (e.g. it was
		 * a side-effect of hypervisor guest page table
		 * bookkeeping, and not propagated to the guest).
		 *
		 * data[0] = eip
		 * data[1] = vaddr
		 */
		XDT_PROBE2(XDT_PV_PAGING_FIXUP, data[0], data[2]);
		break;
	case TRC_PV_PAGING_FIXUP | TRC_64_FLAG:
		/*
		 * Hypervisor fixed up a page fault (e.g. it was
		 * a side-effect of hypervisor guest page table
		 * bookkeeping, and not propagated to the guest).
		 *
		 * data[0] = eip(0:31)
		 * data[1] = eip(31:63)
		 * data[2] = vaddr(0:31)
		 * data[3] = vaddr(31:63)
		 */
		rip64 = (((uint64_t)data[1]) << 32) | data[0];
		addr64 = (((uint64_t)data[3]) << 32) | data[2];
		XDT_PROBE2(XDT_PV_PAGING_FIXUP, rip64, addr64);
		break;
	case TRC_PV_GDT_LDT_MAPPING_FAULT:
		/*
		 * Descriptor table mapping fault in a 32-bit PV domain.
		 * data[0] = eip
		 * data[1] = offset
		 */
		XDT_PROBE2(XDT_PV_DT_MAPPING_FAULT, data[0], data[1]);
		break;
	case TRC_PV_GDT_LDT_MAPPING_FAULT | TRC_64_FLAG:
		/*
		 * Descriptor table mapping fault in a 64-bit PV domain.
		 *
		 * data[0] = eip(0:31)
		 * data[1] = eip(31:63)
		 * data[2] = offset(0:31)
		 * data[3] = offset(31:63)
		 */
		rip64 = (((uint64_t)data[1]) << 32) | data[0];
		val64 = (((uint64_t)data[3]) << 32) | data[2];
		XDT_PROBE2(XDT_PV_DT_MAPPING_FAULT, rip64, val64);
		break;
	case TRC_PV_PTWR_EMULATION:
	case TRC_PV_PTWR_EMULATION_PAE | TRC_64_FLAG:
		/*
		 * Should only happen on 32-bit hypervisor; unused.
		 */
		break;
	case TRC_PV_PTWR_EMULATION_PAE:
		/*
		 * PTE write emulation for a 32-bit PV domain.
		 *
		 * data[0] = pte
		 * data[1] = addr
		 * data[2] = eip
		 */
		XDT_PROBE3(XDT_PV_PTWR_EMULATION, data[0], data[1], data[2]);
		break;
	case TRC_PV_PTWR_EMULATION | TRC_64_FLAG:
		/*
		 * PTE write emulation for a 64-bit PV domain.
		 *
		 * data[0] = pte(0:31)
		 * data[1] = pte(32:63)
		 * data[2] = addr(0:31)
		 * data[3] = addr(32:63)
		 * data[4] = rip(0:31)
		 * data[5] = rip(32:63)
		 */
		pte64 = (((uint64_t)data[1]) << 32) | data[0];
		addr64 = (((uint64_t)data[3]) << 32) | data[2];
		rip64 = (((uint64_t)data[5]) << 32) | data[4];
		XDT_PROBE3(XDT_PV_PTWR_EMULATION, pte64, addr64, rip64);
		break;

	/*
	 * HVM probes
	 */
	case TRC_HVM_VMENTRY:
		/*
		 * Return to guest via vmx_launch/vmrun
		 *
		 */
		XDT_PROBE0(XDT_HVM_VMENTRY);
		break;

	case TRC_HVM_VMEXIT:
		/*
		 * Entry into VMEXIT handler from 32-bit HVM domain
		 *
		 * data[0] = cpu vendor specific exit code
		 * data[1] = guest eip
		 */
		XDT_PROBE2(XDT_HVM_VMEXIT, data[0], data[1]);
		break;
	case TRC_HVM_VMEXIT64:
		/*
		 * Entry into VMEXIT handler from 64-bit HVM domain
		 *
		 * data[0] = cpu vendor specific exit code
		 * data[1] = guest rip(0:31)
		 * data[2] = guest rip(32:64)
		 */
		rip64 = (((uint64_t)data[2]) << 32) | data[1];
		XDT_PROBE2(XDT_HVM_VMEXIT, data[0], rip64);
		break;

	case TRC_HVM_PF_XEN64:
		/*
		 * Pagefault in a guest that is a Xen (e.g. shadow)
		 * artifact, and is not injected back into the guest.
		 *
		 * data[0] = error code
		 * data[1] = guest VA(0:31)
		 * data[2] = guest VA(32:64)
		 */
		addr64 = (((uint64_t)data[2]) << 32) | data[1];
		XDT_PROBE2(XDT_HVM_PF_XEN, data[0], addr64);
		break;

	case TRC_HVM_PF_XEN:
		/*
		 * Same as above, but for a 32-bit HVM domain.
		 *
		 * data[0] = error code
		 * data[1] = guest VA
		 */
		XDT_PROBE2(XDT_HVM_PF_XEN, data[0], data[1]);
		break;

	case TRC_HVM_PF_INJECT:
		/*
		 * 32-bit Xen only.
		 */
		break;
	case TRC_HVM_PF_INJECT64:
		/*
		 * Pagefault injected back into a guest (e.g. the shadow
		 * code found no mapping).
		 *
		 * data[0] = error code
		 * data[1] = guest VA(0:31)
		 * data[2] = guest VA(32:64)
		 */
		addr64 = (((uint64_t)data[2]) << 32) | data[1];
		XDT_PROBE2(XDT_HVM_PF_INJECT, data[0], addr64);
		break;

	case TRC_HVM_INJ_EXC:
		/*
		 * Exception injected into an HVM guest.
		 *
		 * data[0] = trap
		 * data[1] = error code
		 */
		XDT_PROBE2(XDT_HVM_EXC_INJECT, data[0], data[1]);
		break;
	case TRC_HVM_INJ_VIRQ:
		/*
		 * Interrupt inject into an HVM guest.
		 *
		 * data[0] = vector
		 */
		XDT_PROBE1(XDT_HVM_VIRQ_INJECT, data[0]);
		break;
	case TRC_HVM_REINJ_VIRQ:
	case TRC_HVM_IO_READ:
	case TRC_HVM_IO_WRITE:
		/* unused */
		break;
	case TRC_HVM_CR_READ64:
		/*
		 * Control register read. Intel VMX only.
		 *
		 * data[0] = control register #
		 * data[1] = value(0:31)
		 * data[2] = value(32:63)
		 */
		val64 = (((uint64_t)data[2]) << 32) | data[1];
		XDT_PROBE2(XDT_HVM_CR_READ, data[0], val64);
		break;
	case TRC_HVM_CR_READ:
		/*
		 * unused (32-bit Xen only)
		 */
		break;
	case TRC_HVM_CR_WRITE64:
		/*
		 * Control register write. Intel VMX only.
		 *
		 * data[0] = control register #
		 * data[1] = value(0:31)
		 * data[2] = value(32:63)
		 */
		val64 = (((uint64_t)data[2]) << 32) | data[1];
		XDT_PROBE2(XDT_HVM_CR_READ, data[0], val64);
		break;
	case TRC_HVM_CR_WRITE:
		/*
		 * unused (32-bit Xen only)
		 */
		break;
	case TRC_HVM_DR_READ:
		/*
		 * unused.
		 *
		 * data[0] = (domid<<16 + vcpuid)
		 */
		break;
	case TRC_HVM_DR_WRITE:
		/*
		 * Debug register write. Not too useful; no values,
		 * so we ignore this.
		 *
		 * data[0] = (domid<<16 + vcpuid)
		 */
		break;
	case TRC_HVM_MSR_READ:
		/*
		 * MSR read.
		 *
		 * data[0] = MSR
		 * data[1] = value(0:31)
		 * data[2] = value(32:63)
		 */
		val64 = (((uint64_t)data[3]) << 32) | data[2];
		XDT_PROBE2(XDT_HVM_MSR_READ, data[0], val64);
		break;
	case TRC_HVM_MSR_WRITE:
		/*
		 * MSR write.
		 *
		 * data[0] = MSR;
		 * data[1] = value(0:31)
		 * data[2] = value(32:63)
		 */
		val64 = (((uint64_t)data[2]) << 32) | data[1];
		XDT_PROBE2(XDT_HVM_MSR_WRITE, data[0], val64);
		break;
	case TRC_HVM_CPUID:
		/*
		 * CPUID insn.
		 *
		 * data[0] = %eax (input)
		 * data[1] = %eax
		 * data[2] = %ebx
		 * data[3] = %ecx
		 * data[4] = %edx
		 */
		XDT_PROBE5(XDT_HVM_CPUID, data[0], data[1], data[2], data[3],
		    data[4]);
		break;
	case TRC_HVM_INTR:
		/*
		 * VMEXIT because of an interrupt.
		 */
		XDT_PROBE0(XDT_HVM_INTR);
		break;
	case TRC_HVM_INTR_WINDOW:
		/*
		 * VMEXIT because of an interrupt window (an interrupt
		 * can't be delivered immediately to a HVM guest and must
		 * be delayed).
		 *
		 * data[0] = vector
		 * data[1] = source
		 * data[2] = info
		 */
		XDT_PROBE3(XDT_HVM_INTR_WINDOW, data[0], data[1], data[2]);
		break;
	case TRC_HVM_NMI:
		/*
		 * VMEXIT because of an NMI.
		 */
		XDT_PROBE0(XDT_HVM_NMI);
		break;
	case TRC_HVM_SMI:
		/*
		 * VMEXIT because of an SMI
		 */
		XDT_PROBE0(XDT_HVM_SMI);
		break;
	case TRC_HVM_VMMCALL:
		/*
		 * VMMCALL insn.
		 *
		 * data[0] = %eax
		 */
		XDT_PROBE1(XDT_HVM_VMMCALL, data[0]);
		break;
	case TRC_HVM_HLT:
		/*
		 * HLT insn.
		 *
		 * data[0] = 1 if VCPU runnable, 0 if not
		 */
		XDT_PROBE1(XDT_HVM_HLT, data[0]);
		break;
	case TRC_HVM_INVLPG64:
		/*
		 *
		 * data[0] = INVLPGA ? 1 : 0
		 * data[1] = vaddr(0:31)
		 * data[2] = vaddr(32:63)
		 */
		addr64 = (((uint64_t)data[2]) << 32) | data[1];
		XDT_PROBE2(XDT_HVM_INVLPG, data[0], addr64);
		break;
	case TRC_HVM_INVLPG:
		/*
		 * unused (32-bit Xen only)
		 *
		 * data[0] = (domid<<16 + vcpuid)
		 */
		break;
	case TRC_HVM_MCE:
		/*
		 * #MCE VMEXIT
		 *
		 */
		XDT_PROBE0(XDT_HVM_MCE);
		break;
	case TRC_HVM_IOPORT_READ:
	case TRC_HVM_IOPORT_WRITE:
	case TRC_HVM_IOMEM_READ:
	case TRC_HVM_IOMEM_WRITE:
		/*
		 * data[0] = addr(0:31)
		 * data[1] = addr(32:63)
		 * data[2] = count
		 * data[3] = size
		 */
		switch (rec->event) {
		case TRC_HVM_IOPORT_READ:
			eid = XDT_HVM_IOPORT_READ;
			break;
		case TRC_HVM_IOPORT_WRITE:
			eid = XDT_HVM_IOPORT_WRITE;
			break;
		case TRC_HVM_IOMEM_READ:
			eid = XDT_HVM_IOMEM_READ;
			break;
		case TRC_HVM_IOMEM_WRITE:
			eid = XDT_HVM_IOMEM_WRITE;
			break;
		}
		addr64 = (((uint64_t)data[1]) << 32) | data[0];
		XDT_PROBE3(eid, addr64, data[2], data[3]);
		break;
	case TRC_HVM_CLTS:
		/*
		 * CLTS insn (Intel VMX only)
		 */
		XDT_PROBE0(XDT_HVM_CLTS);
		break;
	case TRC_HVM_LMSW64:
		/*
		 * LMSW insn.
		 *
		 * data[0] = value(0:31)
		 * data[1] = value(32:63)
		 */
		val64 = (((uint64_t)data[1]) << 32) | data[0];
		XDT_PROBE1(XDT_HVM_LMSW, val64);
		break;
	case TRC_HVM_LMSW:
		/*
		 * unused (32-bit Xen only)
		 */
		break;

	/*
	 * Shadow page table probes (mainly used for HVM domains
	 * without hardware paging support).
	 */
	case TRC_SHADOW_NOT_SHADOW | SH_GUEST_32:
		/*
		 * data[0] = pte(0:31)
		 * data[1] = pte(32:63)
		 * data[2] = va
		 * data[3] = flags
		 */
		pte64 = ((uint64_t)data[1] << 32) | data[0];
		XDT_PROBE3(XDT_SHADOW_NOT_SHADOW, pte64, data[2], data[3]);
		break;
	case TRC_SHADOW_NOT_SHADOW | SH_GUEST_PAE:
	case TRC_SHADOW_NOT_SHADOW | SH_GUEST_64:
		/*
		 * data[0] = pte(0:31)
		 * data[1] = pte(32:63)
		 * data[2] = va(0:31)
		 * data[3] = va(32:63)
		 * data[4] = flags
		 */
		addr64 = ((uint64_t)data[2] << 32) | data[3];
		pte64 = ((uint64_t)data[1] << 32) | data[0];
		XDT_PROBE3(XDT_SHADOW_NOT_SHADOW, pte64, addr64, data[4]);
		break;
	case TRC_SHADOW_FAST_PROPAGATE | SH_GUEST_32:
		/*
		 * data[0] = va
		 */
		XDT_PROBE1(XDT_SHADOW_FAST_PROPAGATE, data[0]);
		break;
	case TRC_SHADOW_FAST_PROPAGATE | SH_GUEST_PAE:
	case TRC_SHADOW_FAST_PROPAGATE | SH_GUEST_64:
		/*
		 * data[0] = va(0:31)
		 * data[1] = va(32:63)
		 */
		addr64 = ((uint64_t)data[1] << 32) | data[0];
		XDT_PROBE1(XDT_SHADOW_FAST_PROPAGATE, addr64);
		break;
	case TRC_SHADOW_FAST_MMIO | SH_GUEST_32:
		/*
		 * data[0] = va
		 */
		XDT_PROBE1(XDT_SHADOW_FAST_MMIO, data[0]);
		break;
	case TRC_SHADOW_FAST_MMIO | SH_GUEST_PAE:
	case TRC_SHADOW_FAST_MMIO | SH_GUEST_64:
		/*
		 * data[0] = va(0:31)
		 * data[1] = va(32:63)
		 */
		addr64 = ((uint64_t)data[1] << 32) | data[0];
		XDT_PROBE1(XDT_SHADOW_FAST_MMIO, addr64);
		break;
	case TRC_SHADOW_FALSE_FAST_PATH | SH_GUEST_32:
		/*
		 * data[0] = va
		 */
		XDT_PROBE1(XDT_SHADOW_FALSE_FAST_PATH, data[0]);
		break;
	case TRC_SHADOW_FALSE_FAST_PATH | SH_GUEST_PAE:
	case TRC_SHADOW_FALSE_FAST_PATH | SH_GUEST_64:
		/*
		 * data[0] = va(0:31)
		 * data[1] = va(32:63)
		 */
		addr64 = ((uint64_t)data[1] << 32) | data[0];
		XDT_PROBE1(XDT_SHADOW_FALSE_FAST_PATH, addr64);
		break;
	case TRC_SHADOW_MMIO | SH_GUEST_32:
		/*
		 * data[0] = va
		 */
		XDT_PROBE1(XDT_SHADOW_MMIO, data[0]);
		break;
	case TRC_SHADOW_MMIO | SH_GUEST_PAE:
	case TRC_SHADOW_MMIO | SH_GUEST_64:
		/*
		 * data[0] = va(0:31)
		 * data[1] = va(32:63)
		 */
		addr64 = ((uint64_t)data[1] << 32) | data[0];
		XDT_PROBE1(XDT_SHADOW_MMIO, addr64);
		break;
	case TRC_SHADOW_FIXUP | SH_GUEST_32:
		/*
		 * data[0] = pte(0:31)
		 * data[1] = pte(32:63)
		 * data[2] = va
		 * data[3] = flags
		 */
		pte64 = ((uint64_t)data[1] << 32) | data[0];
		XDT_PROBE3(XDT_SHADOW_FIXUP, pte64, data[2], data[3]);
		break;
	case TRC_SHADOW_FIXUP | SH_GUEST_64:
	case TRC_SHADOW_FIXUP | SH_GUEST_PAE:
		/*
		 * data[0] = pte(0:31)
		 * data[1] = pte(32:63)
		 * data[2] = va(0:31)
		 * data[3] = va(32:63)
		 * data[4] = flags
		 */
		addr64 = ((uint64_t)data[2] << 32) | data[3];
		pte64 = ((uint64_t)data[1] << 32) | data[0];
		XDT_PROBE3(XDT_SHADOW_FIXUP, pte64, addr64, data[4]);
		break;
	case TRC_SHADOW_DOMF_DYING | SH_GUEST_32:
		/*
		 * data[0] = va
		 */
		XDT_PROBE1(XDT_SHADOW_DOMF_DYING, data[0]);
		break;
	case TRC_SHADOW_DOMF_DYING | SH_GUEST_PAE:
	case TRC_SHADOW_DOMF_DYING | SH_GUEST_64:
		/*
		 * data[0] = va(0:31)
		 * data[1] = va(32:63)
		 */
		addr64 = ((uint64_t)data[1] << 32) | data[0];
		XDT_PROBE1(XDT_SHADOW_DOMF_DYING, addr64);
		break;
	case TRC_SHADOW_EMULATE | SH_GUEST_32:
		/*
		 * data[0] = pte(0:31)
		 * data[1] = pte(32:63)
		 * data[2] = val(0:31)
		 * data[3] = val(32:63)
		 * data[4] = addr
		 * data[5] = flags
		 */
		pte64 = ((uint64_t)data[1] << 32) | data[0];
		val64 = ((uint64_t)data[3] << 32) | data[2];
		XDT_PROBE5(XDT_SHADOW_EMULATE, pte64, val64, data[4],
		    data[5] & 0x7fffffff, data[5] >> 29);
		break;
	case TRC_SHADOW_EMULATE | SH_GUEST_PAE:
	case TRC_SHADOW_EMULATE | SH_GUEST_64:
		/*
		 * data[0] = pte(0:31)
		 * data[1] = pte(32:63)
		 * data[2] = val(0:31)
		 * data[3] = val(32:63)
		 * data[4] = addr(0:31)
		 * data[5] = addr(32:63)
		 * data[6] = flags
		 */
		pte64 = ((uint64_t)data[1] << 32) | data[0];
		val64 = ((uint64_t)data[3] << 32) | data[2];
		addr64 = ((uint64_t)data[5] << 32) | data[4];
		XDT_PROBE5(XDT_SHADOW_EMULATE, pte64, val64, data[4],
		    data[6] & 0x7fffffff, data[6] >> 29);
		break;
	case TRC_SHADOW_EMULATE_UNSHADOW_USER | SH_GUEST_32:
		/*
		 * data[0] = gfn
		 * data[1] = vaddr
		 */
		XDT_PROBE2(XDT_SHADOW_EMULATE_UNSHADOW_USER, data[0], data[1]);
		break;
	case TRC_SHADOW_EMULATE_UNSHADOW_USER | SH_GUEST_PAE:
	case TRC_SHADOW_EMULATE_UNSHADOW_USER | SH_GUEST_64:
		/*
		 * data[0] = gfn(0:31)
		 * data[1] = gfn(32:63)
		 * data[2] = vaddr(0:31)
		 * data[3] = vaddr(32:63)
		 */
		val64 = ((uint64_t)data[1] << 32) | data[0];
		addr64 = ((uint64_t)data[3] << 32) | data[2];
		XDT_PROBE2(XDT_SHADOW_EMULATE_UNSHADOW_USER, val64, addr64);
		break;
	case TRC_SHADOW_EMULATE_UNSHADOW_EVTINJ | SH_GUEST_32:
		/*
		 * data[0] = gfn
		 * data[1] = vaddr
		 */
		XDT_PROBE2(XDT_SHADOW_EMULATE_UNSHADOW_EVTINJ, data[0],
		    data[1]);
		break;
	case TRC_SHADOW_EMULATE_UNSHADOW_EVTINJ | SH_GUEST_PAE:
	case TRC_SHADOW_EMULATE_UNSHADOW_EVTINJ | SH_GUEST_64:
		/*
		 * data[0] = gfn(0:31)
		 * data[1] = gfn(32:63)
		 * data[2] = vaddr(0:31)
		 * data[3] = vaddr(32:63)
		 */
		val64 = ((uint64_t)data[1] << 32) | data[0];
		addr64 = ((uint64_t)data[3] << 32) | data[2];
		XDT_PROBE2(XDT_SHADOW_EMULATE_UNSHADOW_EVTINJ, val64, addr64);
		break;
	case TRC_SHADOW_EMULATE_UNSHADOW_UNHANDLED | SH_GUEST_32:
		/*
		 * data[0] = gfn
		 * data[1] = vaddr
		 */
		XDT_PROBE2(XDT_SHADOW_EMULATE_UNSHADOW_UNHANDLED, data[0],
		    data[1]);
		break;
	case TRC_SHADOW_EMULATE_UNSHADOW_UNHANDLED | SH_GUEST_PAE:
	case TRC_SHADOW_EMULATE_UNSHADOW_UNHANDLED | SH_GUEST_64:
		/*
		 * data[0] = gfn(0:31)
		 * data[1] = gfn(32:63)
		 * data[2] = vaddr(0:31)
		 * data[3] = vaddr(32:63)
		 */
		val64 = ((uint64_t)data[1] << 32) | data[0];
		addr64 = ((uint64_t)data[3] << 32) | data[2];
		XDT_PROBE2(XDT_SHADOW_EMULATE_UNSHADOW_UNHANDLED, val64,
		    addr64);
		break;
	case TRC_SHADOW_WRMAP_BF:
		/*
		 * data[0] = gfn(0:31)
		 * data[1] = gfn(32:63)
		 */
		val64 = ((uint64_t)data[1] << 32) | data[0];
		XDT_PROBE1(XDT_SHADOW_WRMAP_BF, val64);
		break;
	case TRC_SHADOW_PREALLOC_UNPIN:
		/*
		 * data[0] = gfn(0:31)
		 * data[1] = gfn(32:63)
		 */
		val64 = ((uint64_t)data[1] << 32) | data[0];
		XDT_PROBE1(XDT_SHADOW_PREALLOC_UNPIN, val64);
		break;
	case TRC_SHADOW_RESYNC_FULL:
		/*
		 * data[0] = gmfn(0:31)
		 * data[1] = gmfn(32:63)
		 */
		val64 = ((uint64_t)data[1] << 32) | data[0];
		XDT_PROBE1(XDT_SHADOW_RESYNC_FULL, val64);
		break;
	case TRC_SHADOW_RESYNC_ONLY:
		/*
		 * data[0] = gmfn(0:31)
		 * data[1] = gmfn(32:63)
		 */
		val64 = ((uint64_t)data[1] << 32) | data[0];
		XDT_PROBE1(XDT_SHADOW_RESYNC_ONLY, val64);
		break;

	/*
	 * Power management probes.
	 */
	case TRC_PM_FREQ_CHANGE:
		/*
		 * data[0] = old freq
		 * data[1] = new freq
		 */
		XDT_PROBE2(XDT_PM_FREQ_CHANGE, data[0], data[1]);
		break;
	case TRC_PM_IDLE_ENTRY:
		/*
		 * data[0] = C-state
		 * data[1] = time
		 */
		XDT_PROBE2(XDT_PM_IDLE_ENTRY, data[0], data[1]);
		break;
	case TRC_PM_IDLE_EXIT:
		/*
		 * data[0] = C-state
		 * data[1] = time
		 */
		XDT_PROBE2(XDT_PM_IDLE_EXIT, data[0], data[1]);
		break;
	case TRC_LOST_RECORDS:
		vcpu = data[1] >> 16;
		dom = data[1] & 0xffff;
		xdt_update_sched_context(cpuid, dom, vcpu);
		xdt_update_domain_context(dom, vcpu);
		XDT_PROBE1(XDT_TRC_LOST_RECORDS, cpuid);
		tbuf.stat_dropped_recs++;
		break;

	default:
		tbuf.stat_unknown_recs++;
		break;
	}

done:
	rec_size = 4 + (rec->cycles_included ? 8 : 0) + (rec->extra_u32 * 4);
	return (rec_size);
}

/*
 * Scan all CPU buffers for the record with the lowest timestamp so
 * that the probes will fire in order.
 */
static int
xdt_get_first_rec(uint_t *cpuidp, struct t_rec **recp, uint32_t *consp)
{
	uint_t cpuid;
	uint32_t prod, cons, offset;
	struct t_rec *rec;
	uint64_t minstamp = ~0ULL, stamp;
	uintptr_t data;

	for (cpuid = 0; cpuid < tbuf.cnt; cpuid++) {
		cons = tbuf.meta[cpuid]->cons;
		prod = tbuf.meta[cpuid]->prod;
		membar_consumer();
		if (prod == cons)
			continue;

		offset = cons % tbuf_data_size;
		data = (uintptr_t)tbuf.data[cpuid] + offset;
		rec = (struct t_rec *)data;
		ASSERT((caddr_t)rec < tbuf.va + (tbuf.size * (cpuid + 1)));

		/*
		 * All records that we know about have time cycles included.
		 * If this record doesn't have them, assume it's a type
		 * that we don't handle. Use a 0 time value, which will make
		 * it get handled first (it will be thrown away).
		 */
		if (rec->cycles_included)
			stamp = (((uint64_t)rec->u.cycles.cycles_hi) << 32)
			    | rec->u.cycles.cycles_lo;
		else
			stamp = 0;

		if (stamp < minstamp) {
			minstamp = stamp;
			*cpuidp = cpuid;
			*recp = rec;
			*consp = cons;
		}
	}

	if (minstamp != ~0ULL)
		return (1);

	return (0);
}

/*ARGSUSED*/
static void
xdt_tbuf_scan(void *arg)
{
	uint32_t bytes_done, cons;
	struct t_rec *rec;
	xdt_schedinfo_t *sp;
	uint_t nrecs, cpuid;

	for (nrecs = 0;
	    nrecs < xdt_max_recs && xdt_get_first_rec(&cpuid, &rec, &cons) > 0;
	    nrecs++) {
		xdt_curpcpu = cpuid;
		sp = &xdt_cpu_schedinfo[cpuid];
		if (sp->curinfo_valid)
			xdt_update_domain_context(sp->cur_domid,
			    sp->cur_vcpuid);

		bytes_done = xdt_process_rec(cpuid, rec);
		cons += bytes_done;
		/*
		 * cons and prod are incremented modulo (2 * tbuf_data_size).
		 * See <xen/public/trace.h>.
		 */
		if (cons >= 2 * tbuf_data_size)
			cons -= 2 * tbuf_data_size;
		membar_exit();
		tbuf.meta[cpuid]->cons = cons;
	}
}

static void
xdt_cyclic_enable(void)
{
	cyc_handler_t hdlr;
	cyc_time_t when;

	ASSERT(MUTEX_HELD(&cpu_lock));

	hdlr.cyh_func = xdt_tbuf_scan;
	hdlr.cyh_arg = NULL;
	hdlr.cyh_level = CY_LOW_LEVEL;

	when.cyt_interval = xdt_poll_nsec;
	when.cyt_when = dtrace_gethrtime() + when.cyt_interval;

	xdt_cyclic = cyclic_add(&hdlr, &when);
}

static void
xdt_probe_create(xdt_probe_t *p)
{
	ASSERT(p != NULL && p->pr_mod != NULL);

	if (dtrace_probe_lookup(xdt_id, p->pr_mod, NULL, p->pr_name) != 0)
		return;

	xdt_prid[p->evt_id] = dtrace_probe_create(xdt_id, p->pr_mod, NULL,
	    p->pr_name, dtrace_mach_aframes(), p);
}

/*ARGSUSED*/
static void
xdt_provide(void *arg, const dtrace_probedesc_t *desc)
{
	const char *mod, *name;
	int i;

	if (desc == NULL) {
		for (i = 0; xdt_probe[i].pr_mod != NULL; i++) {
			xdt_probe_create(&xdt_probe[i]);
		}
	} else {
		mod = desc->dtpd_mod;
		name = desc->dtpd_name;
		for (i = 0; xdt_probe[i].pr_mod != NULL; i++) {
			int l1 = strlen(xdt_probe[i].pr_name);
			int l2 = strlen(xdt_probe[i].pr_mod);
			if (strncmp(name, xdt_probe[i].pr_name, l1) == 0 &&
			    strncmp(mod, xdt_probe[i].pr_mod, l2) == 0)
				break;
		}

		if (xdt_probe[i].pr_mod == NULL)
			return;
		xdt_probe_create(&xdt_probe[i]);
	}

}

/*ARGSUSED*/
static void
xdt_destroy(void *arg, dtrace_id_t id, void *parg)
{
	xdt_probe_t *p = parg;
	xdt_prid[p->evt_id] = 0;
}

static void
xdt_set_trace_mask(uint32_t mask)
{
	xen_sysctl_tbuf_op_t tbuf_op;

	/* Always need to trace scheduling, for context */
	if (mask != 0)
		mask |= TRC_SCHED;
	tbuf_op.evt_mask = mask;
	tbuf_op.cmd  = XEN_SYSCTL_TBUFOP_set_evt_mask;
	(void) xdt_sysctl_tbuf(&tbuf_op);
}

/*ARGSUSED*/
static int
xdt_enable(void *arg, dtrace_id_t id, void *parg)
{
	xdt_probe_t *p = parg;
	xen_sysctl_tbuf_op_t tbuf_op;

	ASSERT(MUTEX_HELD(&cpu_lock));
	ASSERT(xdt_prid[p->evt_id] != 0);

	xdt_probemap[p->evt_id] = xdt_prid[p->evt_id];
	xdt_classinfo[p->class].cnt++;

	if (xdt_classinfo[p->class].cnt == 1) {
		/* set the trace mask for this class */
		cur_trace_mask |= xdt_classinfo[p->class].trc_mask;
		xdt_set_trace_mask(cur_trace_mask);
	}

	if (xdt_cyclic == CYCLIC_NONE) {
		tbuf_op.cmd = XEN_SYSCTL_TBUFOP_enable;
		if (xdt_sysctl_tbuf(&tbuf_op) != 0) {
			cmn_err(CE_NOTE, "Couldn't enable hypervisor tracing.");
			return (-1);
		}

		xdt_cyclic_enable();
	}
	return (0);
}

/*ARGSUSED*/
static void
xdt_disable(void *arg, dtrace_id_t id, void *parg)
{
	xdt_probe_t *p = parg;
	xen_sysctl_tbuf_op_t tbuf_op;
	int i, err;

	ASSERT(MUTEX_HELD(&cpu_lock));
	ASSERT(xdt_probemap[p->evt_id] != 0);
	ASSERT(xdt_probemap[p->evt_id] == xdt_prid[p->evt_id]);
	ASSERT(xdt_classinfo[p->class].cnt > 0);

	/*
	 * We could be here in the slight window between the cyclic firing and
	 * a call to dtrace_probe() occurring. We need to be careful if we tear
	 * down any shared state.
	 */

	xdt_probemap[p->evt_id] = 0;
	xdt_classinfo[p->class].cnt--;

	if (xdt_nr_active_probes() == 0) {
		cur_trace_mask = 0;

		if (xdt_cyclic == CYCLIC_NONE)
			return;

		for (i = 0; i < xdt_ncpus; i++)
			xdt_cpu_schedinfo[i].curinfo_valid = 0;

		/*
		 * We will try to disable the trace buffers. If we fail for some
		 * reason we will try again, up to a count of XDT_TBUF_RETRY.
		 * If we still aren't successful we try to set the trace mask
		 * to 0 in order to prevent trace records from being written.
		 */
		tbuf_op.cmd = XEN_SYSCTL_TBUFOP_disable;
		i = 0;
		do {
			err = xdt_sysctl_tbuf(&tbuf_op);
		} while ((err != 0) && (++i < XDT_TBUF_RETRY));

		if (err != 0) {
			cmn_err(CE_NOTE,
			    "Couldn't disable hypervisor tracing.");
			xdt_set_trace_mask(0);
		} else {
			cyclic_remove(xdt_cyclic);
			xdt_cyclic = CYCLIC_NONE;
			/*
			 * We don't bother making the hypercall to set
			 * the trace mask, since it will be reset when
			 * tracing is re-enabled.
			 */
		}
	} else if (xdt_classinfo[p->class].cnt == 0) {
		cur_trace_mask ^= xdt_classinfo[p->class].trc_mask;
		/* other probes are enabled, so add the sub-class mask back */
		cur_trace_mask |= 0xF000;
		xdt_set_trace_mask(cur_trace_mask);
	}
}

static dtrace_pattr_t xdt_attr = {
{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_PLATFORM },
{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_PLATFORM },
{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_PLATFORM },
{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_PLATFORM },
};

static dtrace_pops_t xdt_pops = {
	xdt_provide,		/* dtps_provide() */
	NULL,			/* dtps_provide_module() */
	xdt_enable,		/* dtps_enable() */
	xdt_disable,		/* dtps_disable() */
	NULL,			/* dtps_suspend() */
	NULL,			/* dtps_resume() */
	NULL,			/* dtps_getargdesc() */
	NULL,			/* dtps_getargval() */
	NULL,			/* dtps_usermode() */
	xdt_destroy		/* dtps_destroy() */
};

static int
xdt_attach(dev_info_t *devi, ddi_attach_cmd_t cmd)
{
	int val;

	if (!DOMAIN_IS_INITDOMAIN(xen_info))
		return (DDI_FAILURE);

	switch (cmd) {
	case DDI_ATTACH:
		break;

	case DDI_RESUME:
		/*
		 * We might support proper suspend/resume in the future, so,
		 * return DDI_FAILURE for now.
		 */
		return (DDI_FAILURE);

	default:
		return (DDI_FAILURE);
	}

	xdt_ncpus = xpv_nr_phys_cpus();
	ASSERT(xdt_ncpus > 0);

	if (ddi_create_minor_node(devi, "xdt", S_IFCHR, 0, DDI_PSEUDO, 0) ==
	    DDI_FAILURE || xdt_attach_trace_buffers() != 0 ||
	    dtrace_register("xdt", &xdt_attr, DTRACE_PRIV_KERNEL, NULL,
	    &xdt_pops, NULL, &xdt_id) != 0) {
		if (tbuf.va != NULL)
			xdt_detach_trace_buffers();
		ddi_remove_minor_node(devi, NULL);
		return (DDI_FAILURE);
	}

	val = ddi_getprop(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS,
	    "xdt_poll_nsec", XDT_POLL_DEFAULT);
	xdt_poll_nsec = MAX(val, XDT_POLL_MIN);

	xdt_cpu_schedinfo = (xdt_schedinfo_t *)kmem_zalloc(xdt_ncpus *
	    sizeof (xdt_schedinfo_t), KM_SLEEP);
	xdt_init_trace_masks();
	xdt_kstat_init();

	xdt_devi = devi;
	ddi_report_dev(devi);
	return (DDI_SUCCESS);
}

static int
xdt_detach(dev_info_t *devi, ddi_detach_cmd_t cmd)
{
	switch (cmd) {
	case DDI_DETACH:
		break;

	case DDI_SUSPEND:
		/*
		 * We might support proper suspend/resume in the future. So
		 * return DDI_FAILURE for now.
		 */
		return (DDI_FAILURE);

	default:
		return (DDI_FAILURE);
	}

	if (dtrace_unregister(xdt_id) != 0)
		return (DDI_FAILURE);

	xdt_detach_trace_buffers();
	kmem_free(xdt_cpu_schedinfo, xdt_ncpus * sizeof (xdt_schedinfo_t));
	if (xdt_cyclic != CYCLIC_NONE)
		cyclic_remove(xdt_cyclic);
	if (xdt_kstats != NULL)
		kstat_delete(xdt_kstats);
	xdt_devi = (void *)0;
	ddi_remove_minor_node(devi, NULL);

	return (DDI_SUCCESS);
}

/*ARGSUSED*/
static int
xdt_info(dev_info_t *devi, ddi_info_cmd_t infocmd, void *arg, void **result)
{
	int error;

	switch (infocmd) {
	case DDI_INFO_DEVT2DEVINFO:
		*result = xdt_devi;
		error = DDI_SUCCESS;
		break;
	case DDI_INFO_DEVT2INSTANCE:
		*result = (void *)0;
		error = DDI_SUCCESS;
		break;
	default:
		error = DDI_FAILURE;
	}
	return (error);
}

static struct cb_ops xdt_cb_ops = {
	nulldev,		/* open(9E) */
	nodev,			/* close(9E) */
	nodev,			/* strategy(9E) */
	nodev,			/* print(9E) */
	nodev,			/* dump(9E) */
	nodev,			/* read(9E) */
	nodev,			/* write(9E) */
	nodev,			/* ioctl(9E) */
	nodev,			/* devmap(9E) */
	nodev,			/* mmap(9E) */
	nodev,			/* segmap(9E) */
	nochpoll,		/* chpoll(9E) */
	ddi_prop_op,		/* prop_op(9E) */
	NULL,			/* streamtab(9S) */
	D_MP | D_64BIT | D_NEW	/* cb_flag */
};

static struct dev_ops xdt_ops = {
	DEVO_REV,		/* devo_rev */
	0,			/* devo_refcnt */
	xdt_info,		/* getinfo(9E) */
	nulldev,		/* identify(9E) */
	nulldev,		/* probe(9E) */
	xdt_attach,		/* attach(9E) */
	xdt_detach,		/* detach(9E) */
	nulldev,		/* devo_reset */
	&xdt_cb_ops,		/* devo_cb_ops */
	NULL,			/* devo_bus_ops */
	NULL,			/* power(9E) */
	ddi_quiesce_not_needed,	/* devo_quiesce */
};


static struct modldrv modldrv = {
	&mod_driverops,
	"Hypervisor event tracing",
	&xdt_ops
};

static struct modlinkage modlinkage = {
	MODREV_1,
	&modldrv,
	NULL
};

int
_init(void)
{
	return (mod_install(&modlinkage));
}

int
_fini(void)
{
	return (mod_remove(&modlinkage));
}

int
_info(struct modinfo *modinfop)
{
	return (mod_info(&modlinkage, modinfop));
}
#
# CDDL HEADER START
#
# The contents of this file are subject to the terms of the
# Common Development and Distribution License (the "License").
# You may not use this file except in compliance with the License.
#
# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
# or http://www.opensolaris.org/os/licensing.
# See the License for the specific language governing permissions
# and limitations under the License.
#
# When distributing Covered Code, include this CDDL HEADER in each
# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
# If applicable, add the following below this CDDL HEADER, with the
# fields enclosed by brackets "[]" replaced with your own identifying
# information: Portions Copyright [yyyy] [name of copyright owner]
#
# CDDL HEADER END
#

#
# Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
# Use is subject to license terms.
#
# ident	"%Z%%M%	%I%	%E% SMI"
#

#
# Tunable Properties
# xdt_poll_nsec: interval in nano-seconds to poll trace buffers 
#
name="xdt" parent="pseudo" instance=0;
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */


#include <sys/errno.h>
#include <sys/types.h>
#include <sys/conf.h>
#include <sys/kmem.h>
#include <sys/ddi.h>
#include <sys/stat.h>
#include <sys/sunddi.h>
#include <sys/file.h>
#include <sys/open.h>
#include <sys/modctl.h>
#include <sys/ddi_impldefs.h>
#include <sys/sysmacros.h>
#include <sys/ddidevmap.h>
#include <sys/xendev.h>
#include <public/io/protocols.h>
#include <xen/io/blkif_impl.h>

#include "blk_common.h"


/* blk interface status */
enum blk_if_state {
	/*
	 * initial state
	 */
	BLK_IF_UNKNOWN = 0,
	/*
	 * frontend xenbus state changed to XenbusStateConnected,
	 * we finally connect
	 */
	BLK_IF_CONNECTED,
	/*
	 * frontend xenbus state changed to XenbusStateClosed,
	 * interface disconnected
	 */
	BLK_IF_DISCONNECTED
};

/* backend device status */
enum blk_be_state {
	/* initial state */
	BLK_BE_UNKNOWN = 0,
	/* backend device is ready (hotplug script finishes successfully) */
	BLK_BE_READY
};

/* frontend status */
enum blk_fe_state {
	/* initial state */
	BLK_FE_UNKNOWN = 0,
	/*
	 * frontend's xenbus state has changed to
	 * XenbusStateInitialised, is ready for connecting
	 */
	BLK_FE_READY
};

typedef struct blk_ring_state_s {
	kmutex_t		rs_mutex;
	boolean_t		rs_sleeping_on_ring;
	boolean_t		rs_ring_up;
	kcondvar_t		rs_cv;
} blk_ring_state_t;

/* Disk Statistics */
static char *blk_stats[] = {
	"rd_reqs",
	"wr_reqs",
	"br_reqs",
	"fl_reqs",
	"oo_reqs"
};

typedef struct blk_stats_s {
	uint64_t bs_req_reads;
	uint64_t bs_req_writes;
	uint64_t bs_req_barriers;
	uint64_t bs_req_flushes;
} blk_stats_t;

struct blk_ring_s {
	kmutex_t		ri_mutex;
	dev_info_t		*ri_dip;

	kstat_t			*ri_kstats;
	blk_stats_t		ri_stats;

	blk_intr_t		ri_intr;
	caddr_t			ri_intr_arg;
	blk_ring_cb_t		ri_ringup;
	caddr_t			ri_ringup_arg;
	blk_ring_cb_t		ri_ringdown;
	caddr_t			ri_ringdown_arg;

	/* blk interface, backend, and frontend status */
	enum blk_if_state	ri_if_status;
	enum blk_be_state	ri_be_status;
	enum blk_fe_state	ri_fe_status;

	domid_t			ri_fe;

	enum blkif_protocol	ri_protocol;
	size_t			ri_nentry;
	size_t			ri_entrysize;

	xendev_ring_t		*ri_ring;
	blk_ring_state_t	ri_state;
};


static void blk_oe_state_change(dev_info_t *dip, ddi_eventcookie_t id,
    void *arg, void *impl_data);
static void blk_hp_state_change(dev_info_t *dip, ddi_eventcookie_t id,
    void *arg, void *impl_data);
static int blk_check_state_transition(blk_ring_t ring, XenbusState oestate);
static int blk_start_connect(blk_ring_t ring);
static void blk_start_disconnect(blk_ring_t ring);
static void blk_ring_close(blk_ring_t ring);
static int blk_bindto_frontend(blk_ring_t ring);
static void blk_unbindfrom_frontend(blk_ring_t ring);
static uint_t blk_intr(caddr_t arg);

static int blk_kstat_init(blk_ring_t ring);
static void blk_kstat_fini(blk_ring_t ring);
static int blk_kstat_update(kstat_t *ksp, int flag);

static void blk_ring_request_32(blkif_request_t *dst,
    blkif_x86_32_request_t *src);
static void blk_ring_request_64(blkif_request_t *dst,
    blkif_x86_64_request_t *src);

static void blk_ring_response_32(blkif_x86_32_response_t *dst,
    blkif_response_t *src);
static void blk_ring_response_64(blkif_x86_64_response_t *dst,
    blkif_response_t *src);


/*
 * blk_ring_init()
 */
int
blk_ring_init(blk_ringinit_args_t *args, blk_ring_t *ringp)
{
	blk_ring_t ring;
	int e;


	ring = kmem_zalloc(sizeof (struct blk_ring_s), KM_SLEEP);
	mutex_init(&ring->ri_mutex, NULL, MUTEX_DRIVER, NULL);
	ring->ri_dip = args->ar_dip;
	ring->ri_intr = args->ar_intr;
	ring->ri_intr_arg = args->ar_intr_arg;
	ring->ri_ringup = args->ar_ringup;
	ring->ri_ringup_arg = args->ar_ringup_arg;
	ring->ri_ringdown = args->ar_ringdown;
	ring->ri_ringdown_arg = args->ar_ringdown_arg;

	ring->ri_if_status = BLK_IF_UNKNOWN;
	ring->ri_be_status = BLK_BE_UNKNOWN;
	ring->ri_fe_status = BLK_FE_UNKNOWN;
	ring->ri_state.rs_sleeping_on_ring = B_FALSE;
	ring->ri_state.rs_ring_up = B_FALSE;

	mutex_init(&ring->ri_state.rs_mutex, NULL, MUTEX_DRIVER, NULL);
	cv_init(&ring->ri_state.rs_cv, NULL, CV_DRIVER, NULL);

	e = blk_kstat_init(ring);
	if (e != DDI_SUCCESS) {
		goto ringinitfail_kstat;
	}

	/* Watch frontend and hotplug state change */
	if (xvdi_add_event_handler(ring->ri_dip, XS_OE_STATE,
	    blk_oe_state_change, ring) != DDI_SUCCESS) {
		goto ringinitfail_oestate;
	}
	if (xvdi_add_event_handler(ring->ri_dip, XS_HP_STATE,
	    blk_hp_state_change, ring) != DDI_SUCCESS) {
		goto ringinitfail_hpstate;
	}

	/*
	 * Kick-off hotplug script
	 */
	if (xvdi_post_event(ring->ri_dip, XEN_HP_ADD) != DDI_SUCCESS) {
		cmn_err(CE_WARN, "blk@%s: failed to start hotplug script",
		    ddi_get_name_addr(ring->ri_dip));
		goto ringinitfail_postevent;
	}

	/*
	 * start waiting for hotplug event and otherend state event
	 * mainly for debugging, frontend will not take any op seeing this
	 */
	(void) xvdi_switch_state(ring->ri_dip, XBT_NULL, XenbusStateInitWait);

	*ringp = ring;
	return (DDI_SUCCESS);

ringinitfail_postevent:
	xvdi_remove_event_handler(ring->ri_dip, XS_HP_STATE);
ringinitfail_hpstate:
	xvdi_remove_event_handler(ring->ri_dip, XS_OE_STATE);
ringinitfail_oestate:
	blk_kstat_fini(ring);
ringinitfail_kstat:
	cv_destroy(&ring->ri_state.rs_cv);
	mutex_destroy(&ring->ri_state.rs_mutex);
	mutex_destroy(&ring->ri_mutex);
	kmem_free(ring, sizeof (struct blk_ring_s));
	return (DDI_FAILURE);
}


/*
 * blk_ring_fini()
 */
void
blk_ring_fini(blk_ring_t *ringp)
{
	blk_ring_t ring;


	ring = *ringp;

	mutex_enter(&ring->ri_mutex);
	if (ring->ri_if_status != BLK_IF_DISCONNECTED) {
		blk_ring_close(ring);
	}
	mutex_exit(&ring->ri_mutex);

	xvdi_remove_event_handler(ring->ri_dip, NULL);
	blk_kstat_fini(ring);
	cv_destroy(&ring->ri_state.rs_cv);
	mutex_destroy(&ring->ri_state.rs_mutex);
	mutex_destroy(&ring->ri_mutex);
	kmem_free(ring, sizeof (struct blk_ring_s));

	*ringp = NULL;
}


/*
 * blk_kstat_init()
 */
static int
blk_kstat_init(blk_ring_t ring)
{
	int nstat = sizeof (blk_stats) / sizeof (blk_stats[0]);
	char **cp = blk_stats;
	kstat_named_t *knp;

	ring->ri_kstats = kstat_create(ddi_get_name(ring->ri_dip),
	    ddi_get_instance(ring->ri_dip), "req_statistics", "block",
	    KSTAT_TYPE_NAMED, nstat, 0);
	if (ring->ri_kstats == NULL) {
		return (DDI_FAILURE);
	}

	ring->ri_kstats->ks_private = ring;
	ring->ri_kstats->ks_update = blk_kstat_update;

	knp = ring->ri_kstats->ks_data;
	while (nstat > 0) {
		kstat_named_init(knp, *cp, KSTAT_DATA_UINT64);
		knp++;
		cp++;
		nstat--;
	}

	kstat_install(ring->ri_kstats);

	return (DDI_SUCCESS);
}


/*
 * blk_kstat_fini()
 */
static void
blk_kstat_fini(blk_ring_t ring)
{
	kstat_delete(ring->ri_kstats);
}


/*
 * blk_kstat_update()
 */
static int
blk_kstat_update(kstat_t *ksp, int flag)
{
	kstat_named_t *knp;
	blk_stats_t *stats;
	blk_ring_t ring;


	if (flag != KSTAT_READ) {
		return (EACCES);
	}

	ring = ksp->ks_private;
	stats = &ring->ri_stats;
	knp = ksp->ks_data;

	/*
	 * Assignment order should match that of the names in
	 * blk_stats.
	 */
	(knp++)->value.ui64 = stats->bs_req_reads;
	(knp++)->value.ui64 = stats->bs_req_writes;
	(knp++)->value.ui64 = stats->bs_req_barriers;
	(knp++)->value.ui64 = stats->bs_req_flushes;
	(knp++)->value.ui64 = 0; /* oo_req */

	return (0);
}


/*
 * blk_oe_state_change()
 */
/*ARGSUSED*/
static void
blk_oe_state_change(dev_info_t *dip, ddi_eventcookie_t id, void *arg,
    void *impl_data)
{
	XenbusState new_state;
	blk_ring_t ring;


	ring = (blk_ring_t)arg;
	new_state = *(XenbusState *)impl_data;

	mutex_enter(&ring->ri_mutex);

	if (blk_check_state_transition(ring, new_state) == DDI_FAILURE) {
		mutex_exit(&ring->ri_mutex);
		return;
	}

	switch (new_state) {
	case XenbusStateInitialised:
		ASSERT(ring->ri_if_status == BLK_IF_UNKNOWN);

		/* frontend is ready for connecting */
		ring->ri_fe_status = BLK_FE_READY;

		if (ring->ri_be_status == BLK_BE_READY) {
			mutex_exit(&ring->ri_mutex);
			if (blk_start_connect(ring) != DDI_SUCCESS)
				(void) blk_start_disconnect(ring);
			mutex_enter(&ring->ri_mutex);
		}
		break;
	case XenbusStateClosing:
		(void) xvdi_switch_state(dip, XBT_NULL, XenbusStateClosing);
		break;
	case XenbusStateClosed:
		/* clean up */
		(void) xvdi_post_event(ring->ri_dip, XEN_HP_REMOVE);
		if (ring->ri_ringdown != NULL) {
			(*(ring->ri_ringdown))(ring->ri_ringdown_arg);
		}
		blk_ring_close(ring);

		/* reset state in case of reconnect */
		ring->ri_if_status = BLK_IF_UNKNOWN;
		ring->ri_be_status = BLK_BE_UNKNOWN;
		ring->ri_fe_status = BLK_FE_UNKNOWN;
		ring->ri_state.rs_sleeping_on_ring = B_FALSE;
		ring->ri_state.rs_ring_up = B_FALSE;

		break;
	default:
		ASSERT(0);
	}

	mutex_exit(&ring->ri_mutex);
}


/*
 * blk_hp_state_change()
 */
/*ARGSUSED*/
static void
blk_hp_state_change(dev_info_t *dip, ddi_eventcookie_t id, void *arg,
    void *impl_data)
{
	xendev_hotplug_state_t hpstate;
	blk_ring_t ring;


	ring = (blk_ring_t)arg;
	hpstate = *(xendev_hotplug_state_t *)impl_data;

	mutex_enter(&ring->ri_mutex);
	if (hpstate == Connected) {
		/* Hotplug script has completed successfully */
		if (ring->ri_be_status == BLK_BE_UNKNOWN) {
			ring->ri_be_status = BLK_BE_READY;
			if (ring->ri_fe_status == BLK_FE_READY) {
				mutex_exit(&ring->ri_mutex);
				/* try to connect to frontend */
				if (blk_start_connect(ring) != DDI_SUCCESS)
					(void) blk_start_disconnect(ring);
				mutex_enter(&ring->ri_mutex);
			}
		}
	}
	mutex_exit(&ring->ri_mutex);
}


/*
 * blk_check_state_transition()
 *    check the XenbusState change to see if the change is a valid transition
 *    or not. The new state is written by frontend domain, or by running
 *    xenstore-write to change it manually in dom0.
 */
static int
blk_check_state_transition(blk_ring_t ring, XenbusState oestate)
{
	switch (ring->ri_if_status) {
	case BLK_IF_UNKNOWN:
		if (ring->ri_fe_status == BLK_FE_UNKNOWN) {
			if ((oestate == XenbusStateUnknown)		||
			    (oestate == XenbusStateConnected))
				goto statechkfail_bug;
			else if ((oestate == XenbusStateInitialising)	||
			    (oestate == XenbusStateInitWait))
				goto statechkfail_nop;
		} else {
			if ((oestate == XenbusStateUnknown)		||
			    (oestate == XenbusStateInitialising)	||
			    (oestate == XenbusStateInitWait)		||
			    (oestate == XenbusStateConnected))
				goto statechkfail_bug;
			else if (oestate == XenbusStateInitialised)
				goto statechkfail_nop;
		}
		break;

	case BLK_IF_CONNECTED:
		if ((oestate == XenbusStateUnknown)		||
		    (oestate == XenbusStateInitialising)	||
		    (oestate == XenbusStateInitWait)		||
		    (oestate == XenbusStateInitialised))
			goto statechkfail_bug;
		else if (oestate == XenbusStateConnected)
			goto statechkfail_nop;
		break;

	case BLK_IF_DISCONNECTED:
	default:
		goto statechkfail_bug;
	}

	return (DDI_SUCCESS);

statechkfail_bug:
	cmn_err(CE_NOTE, "blk@%s: unexpected otherend "
	    "state change to %d!, when status is %d",
	    ddi_get_name_addr(ring->ri_dip), oestate,
	    ring->ri_if_status);

statechkfail_nop:
	return (DDI_FAILURE);
}


/*
 * blk_start_connect()
 *    Kick-off connect process
 *    If ri_fe_status == BLK_FE_READY and ri_be_status == BLK_BE_READY
 *    the ri_if_status will be changed to BLK_IF_CONNECTED on success,
 *    otherwise, ri_if_status will not be changed
 */
static int
blk_start_connect(blk_ring_t ring)
{
	xenbus_transaction_t xbt;
	dev_info_t *dip;
	char *barrier;
	char *xsnode;
	uint_t len;
	int e;


	dip = ring->ri_dip;

	/*
	 * Start connect to frontend only when backend device are ready
	 * and frontend has moved to XenbusStateInitialised, which means
	 * ready to connect
	 */
	ASSERT(ring->ri_fe_status == BLK_FE_READY);
	ASSERT(ring->ri_be_status == BLK_BE_READY);

	xsnode = xvdi_get_xsname(dip);
	if (xsnode == NULL) {
		goto startconnectfail_get_xsname;
	}

	ring->ri_fe = xvdi_get_oeid(dip);
	if (ring->ri_fe == (domid_t)-1) {
		goto startconnectfail_get_oeid;
	}

	e =  xvdi_switch_state(dip, XBT_NULL, XenbusStateInitialised);
	if (e > 0) {
		goto startconnectfail_switch_init;
	}

	e = blk_bindto_frontend(ring);
	if (e != DDI_SUCCESS) {
		goto startconnectfail_bindto_frontend;
	}
	ring->ri_if_status = BLK_IF_CONNECTED;

	e = ddi_add_intr(dip, 0, NULL, NULL, blk_intr, (caddr_t)ring);
	if (e != DDI_SUCCESS) {
		goto startconnectfail_add_intr;
	}

trans_retry:
	e = xenbus_transaction_start(&xbt);
	if (e != 0) {
		xvdi_fatal_error(dip, e, "transaction start");
		goto startconnectfail_transaction_start;
	}

	/* xentop requires the instance in xenstore */
	e = xenbus_printf(xbt, xsnode, "instance", "%d",
	    ddi_get_instance(ring->ri_dip));
	if (e != 0) {
		cmn_err(CE_WARN, "xdb@%s: failed to write 'instance'",
		    ddi_get_name_addr(dip));
		xvdi_fatal_error(dip, e, "writing 'instance'");
		(void) xenbus_transaction_end(xbt, 1);
		goto startconnectfail_xenbus_printf;
	}

	/* If feature-barrier isn't present in xenstore, add it */
	e = xenbus_read(xbt, xsnode, "feature-barrier", (void **)&barrier,
	    &len);
	if (e != 0) {
		e = xenbus_printf(xbt, xsnode, "feature-barrier", "%d", 1);
		if (e != 0) {
			cmn_err(CE_WARN, "xdb@%s: failed to write "
			    "'feature-barrier'", ddi_get_name_addr(dip));
			xvdi_fatal_error(dip, e, "writing 'feature-barrier'");
			(void) xenbus_transaction_end(xbt, 1);
			goto startconnectfail_xenbus_printf;
		}
	} else {
		kmem_free(barrier, len);
	}

	e = xvdi_switch_state(dip, xbt, XenbusStateConnected);
	if (e > 0) {
		xvdi_fatal_error(dip, e, "writing 'state'");
		(void) xenbus_transaction_end(xbt, 1);
		goto startconnectfail_switch_connected;
	}

	e = xenbus_transaction_end(xbt, 0);
	if (e != 0) {
		if (e == EAGAIN) {
			/* transaction is ended, don't need to abort it */
			goto trans_retry;
		}
		xvdi_fatal_error(dip, e, "completing transaction");
		goto startconnectfail_transaction_end;
	}

	mutex_enter(&ring->ri_state.rs_mutex);
	ring->ri_state.rs_ring_up = B_TRUE;
	if (ring->ri_state.rs_sleeping_on_ring) {
		ring->ri_state.rs_sleeping_on_ring = B_FALSE;
		cv_signal(&ring->ri_state.rs_cv);
	}
	mutex_exit(&ring->ri_state.rs_mutex);

	if (ring->ri_ringup != NULL) {
		(*(ring->ri_ringup))(ring->ri_ringup_arg);
	}

	return (DDI_SUCCESS);


startconnectfail_transaction_end:
startconnectfail_switch_connected:
startconnectfail_xenbus_printf:
startconnectfail_transaction_start:
	ddi_remove_intr(dip, 0, NULL);
startconnectfail_add_intr:
	blk_unbindfrom_frontend(ring);
	ring->ri_fe = (domid_t)-1;
startconnectfail_bindto_frontend:
	(void) xvdi_switch_state(dip, XBT_NULL, XenbusStateClosed);
startconnectfail_switch_init:
startconnectfail_get_oeid:
startconnectfail_get_xsname:
	return (DDI_FAILURE);
}


/*
 * blk_start_disconnect()
 *    Kick-off disconnect process. ri_if_status will not be changed
 */
static void
blk_start_disconnect(blk_ring_t ring)
{
	/* Kick-off disconnect process */
	(void) xvdi_switch_state(ring->ri_dip, XBT_NULL, XenbusStateClosing);
}


/*
 * blk_ring_close()
 *    Disconnect from frontend and close backend device
 *    ifstatus will be changed to BLK_DISCONNECTED
 *    Xenbus state will be changed to XenbusStateClosed
 */
static void
blk_ring_close(blk_ring_t ring)
{
	dev_info_t *dip;


	/* mutex protect ri_if_status only here */
	ASSERT(MUTEX_HELD(&ring->ri_mutex));

	dip = ring->ri_dip;

	if (ring->ri_if_status != BLK_IF_CONNECTED) {
		return;
	}

	ring->ri_if_status = BLK_IF_DISCONNECTED;
	mutex_exit(&ring->ri_mutex);

	/* stop accepting I/O request from frontend */
	ddi_remove_intr(dip, 0, NULL);

	blk_unbindfrom_frontend(ring);
	ring->ri_fe = (domid_t)-1;
	(void) xvdi_switch_state(dip, XBT_NULL, XenbusStateClosed);
	mutex_enter(&ring->ri_mutex);
}


/*
 * blk_bindto_frontend()
 */
static int
blk_bindto_frontend(blk_ring_t ring)
{
	evtchn_port_t evtchn;
	char protocol[64];
	grant_ref_t gref;
	dev_info_t *dip;
	char *oename;
	int e;


	dip = ring->ri_dip;
	protocol[0] = 0x0;

	/*
	 * Gather info from frontend
	 */
	oename = xvdi_get_oename(dip);
	if (oename == NULL) {
		return (DDI_FAILURE);
	}

	e = xenbus_gather(XBT_NULL, oename, "ring-ref", "%lu", &gref,
	    "event-channel", "%u", &evtchn, NULL);
	if (e != 0) {
		xvdi_fatal_error(dip, e,
		    "Getting ring-ref and evtchn from frontend");
		return (DDI_FAILURE);
	}

	e = xenbus_gather(XBT_NULL, oename, "protocol", "%63s",
	    protocol, NULL);
	if (e != 0) {
		(void) strcpy(protocol, "unspecified, assuming native");
	} else if (strcmp(protocol, XEN_IO_PROTO_ABI_NATIVE) == 0) {
		ring->ri_protocol = BLKIF_PROTOCOL_NATIVE;
		ring->ri_nentry = BLKIF_RING_SIZE;
		ring->ri_entrysize = sizeof (union blkif_sring_entry);
	} else if (strcmp(protocol, XEN_IO_PROTO_ABI_X86_32) == 0) {
		ring->ri_protocol = BLKIF_PROTOCOL_X86_32;
		ring->ri_nentry = BLKIF_X86_32_RING_SIZE;
		ring->ri_entrysize = sizeof (union blkif_x86_32_sring_entry);
	} else if (strcmp(protocol, XEN_IO_PROTO_ABI_X86_64) == 0) {
		ring->ri_protocol = BLKIF_PROTOCOL_X86_64;
		ring->ri_nentry = BLKIF_X86_64_RING_SIZE;
		ring->ri_entrysize = sizeof (union blkif_x86_64_sring_entry);
	} else {
		xvdi_fatal_error(dip, e, "unknown fe protocol");
		return (DDI_FAILURE);
	}

	/*
	 * map and init ring
	 */
	e = xvdi_map_ring(dip, ring->ri_nentry, ring->ri_entrysize, gref,
	    &ring->ri_ring);
	if (e != DDI_SUCCESS) {
		return (DDI_FAILURE);
	}

	/*
	 * bind event channel
	 */
	e = xvdi_bind_evtchn(dip, evtchn);
	if (e != DDI_SUCCESS) {
		xvdi_unmap_ring(ring->ri_ring);
		return (DDI_FAILURE);
	}


	return (DDI_SUCCESS);
}


/*
 * blk_unbindfrom_frontend()
 */
static void
blk_unbindfrom_frontend(blk_ring_t ring)
{
	xvdi_free_evtchn(ring->ri_dip);
	xvdi_unmap_ring(ring->ri_ring);
}


/*
 * blk_intr()
 */
static uint_t
blk_intr(caddr_t arg)
{
	blk_ring_t ring;

	ring = (blk_ring_t)arg;
	if (ring->ri_if_status != BLK_IF_CONNECTED) {
		return (DDI_INTR_CLAIMED);
	}

	(void) (*ring->ri_intr)(ring->ri_intr_arg);
	return (DDI_INTR_CLAIMED);
}


/*
 * blk_ring_request_get()
 */
boolean_t
blk_ring_request_get(blk_ring_t ring, blkif_request_t *req)
{
	blkif_request_t *src;
	blk_stats_t *stats;


	mutex_enter(&ring->ri_mutex);

	if (ring->ri_if_status != BLK_IF_CONNECTED) {
		mutex_exit(&ring->ri_mutex);
		return (B_FALSE);
	}

	src = xvdi_ring_get_request(ring->ri_ring);
	if (src == NULL) {
		mutex_exit(&ring->ri_mutex);
		return (B_FALSE);
	}

	switch (ring->ri_protocol) {
	case BLKIF_PROTOCOL_NATIVE:
		bcopy(src, req, sizeof (*req));
		break;
	case BLKIF_PROTOCOL_X86_32:
		blk_ring_request_32(req, (blkif_x86_32_request_t *)src);
		break;
	case BLKIF_PROTOCOL_X86_64:
		blk_ring_request_64(req, (blkif_x86_64_request_t *)src);
		break;
	default:
		cmn_err(CE_WARN, "blkif@%s: unrecognised protocol: %d",
		    ddi_get_name_addr(ring->ri_dip),
		    ring->ri_protocol);
	}
	mutex_exit(&ring->ri_mutex);

	stats = &ring->ri_stats;
	switch (req->operation) {
	case BLKIF_OP_READ:
		stats->bs_req_reads++;
		break;
	case BLKIF_OP_WRITE:
		stats->bs_req_writes++;
		break;
	case BLKIF_OP_WRITE_BARRIER:
		stats->bs_req_barriers++;
		break;
	case BLKIF_OP_FLUSH_DISKCACHE:
		stats->bs_req_flushes++;
		break;
	}

	return (B_TRUE);
}


/*
 * blk_ring_request_requeue()
 *    if a request is requeued, caller will have to poll for request
 *    later.
 */
void
blk_ring_request_requeue(blk_ring_t ring)
{
	mutex_enter(&ring->ri_mutex);

	if (ring->ri_if_status != BLK_IF_CONNECTED) {
		mutex_exit(&ring->ri_mutex);
		return;
	}

	ring->ri_ring->xr_sring.br.req_cons--;

	mutex_exit(&ring->ri_mutex);
}


/*
 * blk_ring_response_put()
 */
void
blk_ring_response_put(blk_ring_t ring, blkif_response_t *src)
{
	blkif_response_t *rsp;
	int e;


	mutex_enter(&ring->ri_mutex);

	if (ring->ri_if_status != BLK_IF_CONNECTED) {
		mutex_exit(&ring->ri_mutex);
		return;
	}

	rsp = xvdi_ring_get_response(ring->ri_ring);
	ASSERT(rsp);

	switch (ring->ri_protocol) {
	case BLKIF_PROTOCOL_NATIVE:
		bcopy(src, rsp, sizeof (*rsp));
		break;
	case BLKIF_PROTOCOL_X86_32:
		blk_ring_response_32((blkif_x86_32_response_t *)rsp, src);
		break;
	case BLKIF_PROTOCOL_X86_64:
		blk_ring_response_64((blkif_x86_64_response_t *)rsp, src);
		break;
	default:
		cmn_err(CE_WARN, "blk@%s: unrecognised protocol: %d",
		    ddi_get_name_addr(ring->ri_dip),
		    ring->ri_protocol);
	}

	e = xvdi_ring_push_response(ring->ri_ring);
	if (e != 0) {
		xvdi_notify_oe(ring->ri_dip);
	}

	mutex_exit(&ring->ri_mutex);
}


/*
 * blk_ring_request_32()
 */
static void
blk_ring_request_32(blkif_request_t *dst, blkif_x86_32_request_t *src)
{
	int i, n = BLKIF_MAX_SEGMENTS_PER_REQUEST;
	dst->operation = src->operation;
	dst->nr_segments = src->nr_segments;
	dst->handle = src->handle;
	dst->id = src->id;
	dst->sector_number = src->sector_number;
	if (n > src->nr_segments)
		n = src->nr_segments;
	for (i = 0; i < n; i++)
		dst->seg[i] = src->seg[i];
}


/*
 * blk_ring_request_64()
 */
static void
blk_ring_request_64(blkif_request_t *dst, blkif_x86_64_request_t *src)
{
	int i, n = BLKIF_MAX_SEGMENTS_PER_REQUEST;
	dst->operation = src->operation;
	dst->nr_segments = src->nr_segments;
	dst->handle = src->handle;
	dst->id = src->id;
	dst->sector_number = src->sector_number;
	if (n > src->nr_segments)
		n = src->nr_segments;
	for (i = 0; i < n; i++)
		dst->seg[i] = src->seg[i];
}


/*
 * blk_ring_response_32()
 */
static void
blk_ring_response_32(blkif_x86_32_response_t *dst, blkif_response_t *src)
{
	dst->id = src->id;
	dst->operation = src->operation;
	dst->status = src->status;
}


/*
 * blk_ring_response_64()
 */
static void
blk_ring_response_64(blkif_x86_64_response_t *dst, blkif_response_t *src)
{
	dst->id = src->id;
	dst->operation = src->operation;
	dst->status = src->status;
}


/*
 * blk_ring_request_dump()
 */
void
blk_ring_request_dump(blkif_request_t *req)
{
	int i;

	/*
	 * Exploit the public interface definitions for BLKIF_OP_READ
	 * etc..
	 */
	char *op_name[] = { "read", "write", "barrier", "flush" };

	cmn_err(CE_NOTE, "   op=%s", op_name[req->operation]);
	cmn_err(CE_NOTE, "   num of segments=%d", req->nr_segments);
	cmn_err(CE_NOTE, "   handle=%d", req->handle);
	cmn_err(CE_NOTE, "   id=0x%llx", (unsigned long long)req->id);
	cmn_err(CE_NOTE, "   start sector=%llu",
	    (unsigned long long)req->sector_number);
	for (i = 0; i < req->nr_segments; i++) {
		cmn_err(CE_NOTE, "   gref=%d, first sec=%d,"
		    "last sec=%d", req->seg[i].gref, req->seg[i].first_sect,
		    req->seg[i].last_sect);
	}
}


/*
 * blk_ring_response_dump()
 */
void
blk_ring_response_dump(blkif_response_t *resp)
{
	/*
	 * Exploit the public interface definitions for BLKIF_OP_READ
	 * etc..
	 */
	char *op_name[] = { "read", "write", "barrier", "flush" };

	cmn_err(CE_NOTE, "   op=%d:%s", resp->operation,
	    op_name[resp->operation]);
	cmn_err(CE_NOTE, "   op=%d", resp->operation);
	cmn_err(CE_NOTE, "   status=%d", resp->status);
}
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#ifndef _SYS_BLK_COMMON_H
#define	_SYS_BLK_COMMON_H


#ifdef __cplusplus
extern "C" {
#endif

#include <sys/types.h>
#include <sys/xendev.h>


typedef uint_t (*blk_intr_t)(caddr_t arg);
typedef void (*blk_ring_cb_t)(caddr_t arg);

typedef struct blk_ringinit_args_s {
	dev_info_t	*ar_dip;

	/* callbacks */
	blk_intr_t	ar_intr;
	caddr_t		ar_intr_arg;
	blk_ring_cb_t	ar_ringup;
	caddr_t		ar_ringup_arg;
	blk_ring_cb_t	ar_ringdown;
	caddr_t		ar_ringdown_arg;
} blk_ringinit_args_t;

typedef struct blk_ring_s *blk_ring_t;

int blk_ring_init(blk_ringinit_args_t *args, blk_ring_t *ring);
void blk_ring_fini(blk_ring_t *ring);

boolean_t blk_ring_request_get(blk_ring_t ring, blkif_request_t *req);
void blk_ring_request_requeue(blk_ring_t ring);
void blk_ring_request_dump(blkif_request_t *req);

void blk_ring_response_put(blk_ring_t ring, blkif_response_t *resp);
void blk_ring_response_dump(blkif_response_t *req);

#ifdef __cplusplus
}
#endif

#endif /* _SYS_BLK_COMMON_H */
/*
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to
 * deal in the Software without restriction, including without limitation the
 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
 * sell copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 * DEALINGS IN THE SOFTWARE.
 */
#ifndef __XEN_BLKIF_H__
#define __XEN_BLKIF_H__

#include <public/io/ring.h>
#include <public/io/blkif.h>
#include <public/io/protocols.h>

/* Not a real protocol.  Used to generate ring structs which contain
 * the elements common to all protocols only.  This way we get a
 * compiler-checkable way to use common struct elements, so we can
 * avoid using switch(protocol) in a number of places.  */

/* i386 protocol version */

#pragma pack(4)

struct blkif_x86_32_request {
	uint8_t        operation;    /* BLKIF_OP_???                         */
	uint8_t        nr_segments;  /* number of segments                   */
	blkif_vdev_t   handle;       /* only for read/write requests         */
	uint64_t       id;           /* private guest value, echoed in resp  */
	blkif_sector_t sector_number;/* start sector idx on disk (r/w only)  */
	struct blkif_request_segment seg[BLKIF_MAX_SEGMENTS_PER_REQUEST];
};
struct blkif_x86_32_response {
	uint64_t        id;              /* copied from request */
	uint8_t         operation;       /* copied from request */
	int16_t         status;          /* BLKIF_RSP_???       */
};
typedef struct blkif_x86_32_request blkif_x86_32_request_t;
typedef struct blkif_x86_32_response blkif_x86_32_response_t;

#pragma pack()

/* x86_64 protocol version */
struct blkif_x86_64_request {
	uint8_t        operation;    /* BLKIF_OP_???                         */
	uint8_t        nr_segments;  /* number of segments                   */
	blkif_vdev_t   handle;       /* only for read/write requests         */
#if defined(__GNUC__)
	uint64_t       __attribute__((__aligned__(8))) id;
#else
	uint8_t        pad[4];
	uint64_t       id;
#endif
	blkif_sector_t sector_number;/* start sector idx on disk (r/w only)  */
	struct blkif_request_segment seg[BLKIF_MAX_SEGMENTS_PER_REQUEST];
};
struct blkif_x86_64_response {
#if defined(__GNUC__)
	uint64_t       __attribute__((__aligned__(8))) id;
#else
	uint64_t       id;
#endif
	uint8_t         operation;       /* copied from request */
	int16_t         status;          /* BLKIF_RSP_???       */
};
typedef struct blkif_x86_64_request blkif_x86_64_request_t;
typedef struct blkif_x86_64_response blkif_x86_64_response_t;

DEFINE_RING_TYPES(blkif_x86_32, struct blkif_x86_32_request, struct blkif_x86_32_response);
DEFINE_RING_TYPES(blkif_x86_64, struct blkif_x86_64_request, struct blkif_x86_64_response);

enum blkif_protocol {
	BLKIF_PROTOCOL_NATIVE = 1,
	BLKIF_PROTOCOL_X86_32 = 2,
	BLKIF_PROTOCOL_X86_64 = 3,
};

#define	BLKIF_RING_SIZE	\
	__RING_SIZE((blkif_sring_t *)NULL, PAGESIZE)
#define	BLKIF_X86_32_RING_SIZE \
	__RING_SIZE((blkif_x86_32_sring_t *)NULL, PAGESIZE)
#define	BLKIF_X86_64_RING_SIZE \
	__RING_SIZE((blkif_x86_64_sring_t *)NULL, PAGESIZE)

#endif /* __XEN_BLKIF_H__ */
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 * Copyright 2017 Joyent, Inc.
 */


/*
 * evtchn.c
 *
 * Driver for receiving and demuxing event-channel signals.
 *
 * Copyright (c) 2004-2005, K A Fraser
 * Multi-process extensions Copyright (c) 2004, Steven Smith
 *
 * This file may be distributed separately from the Linux kernel, or
 * incorporated into other software packages, subject to the following license:
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this source file (the "Software"), to deal in the Software without
 * restriction, including without limitation the rights to use, copy, modify,
 * merge, publish, distribute, sublicense, and/or sell copies of the Software,
 * and to permit persons to whom the Software is furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 */

#include <sys/types.h>
#include <sys/hypervisor.h>
#include <sys/machsystm.h>
#include <sys/mutex.h>
#include <sys/evtchn_impl.h>
#include <sys/ddi_impldefs.h>
#include <sys/avintr.h>
#include <sys/cpuvar.h>
#include <sys/smp_impldefs.h>
#include <sys/archsystm.h>
#include <sys/sysmacros.h>
#include <sys/fcntl.h>
#include <sys/open.h>
#include <sys/stat.h>
#include <sys/psm.h>
#include <sys/cpu.h>
#include <sys/cmn_err.h>
#include <sys/xen_errno.h>
#include <sys/policy.h>
#include <xen/sys/evtchn.h>

/* Some handy macros */
#define	EVTCHNDRV_MINOR2INST(minor)	((int)(minor))
#define	EVTCHNDRV_DEFAULT_NCLONES	256
#define	EVTCHNDRV_INST2SOFTS(inst)	\
	(ddi_get_soft_state(evtchndrv_statep, (inst)))

/* Soft state data structure for evtchn driver */
struct evtsoftdata {
	dev_info_t *dip;
	/* Notification ring, accessed via /dev/xen/evtchn. */
#define	EVTCHN_RING_SIZE	(PAGESIZE / sizeof (evtchn_port_t))
#define	EVTCHN_RING_MASK(_i)	((_i) & (EVTCHN_RING_SIZE - 1))
	evtchn_port_t *ring;
	unsigned int ring_cons, ring_prod, ring_overflow;

	kcondvar_t evtchn_wait; /* Processes wait on this when ring is empty. */
	kmutex_t evtchn_lock;
	pollhead_t evtchn_pollhead;

	pid_t pid;		/* last pid to bind to this event channel. */
	processorid_t cpu;	/* cpu thread/evtchn is bound to */
};

static void *evtchndrv_statep;
int evtchndrv_nclones = EVTCHNDRV_DEFAULT_NCLONES;
static int *evtchndrv_clone_tab;
static dev_info_t *evtchndrv_dip;
static kmutex_t evtchndrv_clone_tab_mutex;

static int evtchndrv_detach(dev_info_t *, ddi_detach_cmd_t);

/* Who's bound to each port? */
static struct evtsoftdata *port_user[NR_EVENT_CHANNELS];
static kmutex_t port_user_lock;

uint_t
evtchn_device_upcall(caddr_t arg __unused, caddr_t arg1 __unused)
{
	struct evtsoftdata *ep;
	int port;

	/*
	 * This is quite gross, we had to leave the evtchn that led to this
	 * invocation in a per-cpu mailbox, retrieve it now.
	 * We do this because the interface doesn't offer us a way to pass
	 * a dynamic argument up through the generic interrupt service layer.
	 * The mailbox is safe since we either run with interrupts disabled or
	 * non-preemptable till we reach here.
	 */
	port = CPU->cpu_m.mcpu_ec_mbox;
	ASSERT(port != 0);
	CPU->cpu_m.mcpu_ec_mbox = 0;
	ec_clear_evtchn(port);
	mutex_enter(&port_user_lock);

	if ((ep = port_user[port]) != NULL) {
		mutex_enter(&ep->evtchn_lock);
		if ((ep->ring_prod - ep->ring_cons) < EVTCHN_RING_SIZE) {
			ep->ring[EVTCHN_RING_MASK(ep->ring_prod)] = port;
			/*
			 * Wake up reader when ring goes non-empty
			 */
			if (ep->ring_cons == ep->ring_prod++) {
				cv_signal(&ep->evtchn_wait);
				mutex_exit(&ep->evtchn_lock);
				pollwakeup(&ep->evtchn_pollhead,
				    POLLIN | POLLRDNORM);
				goto done;
			}
		} else {
			ep->ring_overflow = 1;
		}
		mutex_exit(&ep->evtchn_lock);
	}

done:
	mutex_exit(&port_user_lock);
	return (DDI_INTR_CLAIMED);
}

/* ARGSUSED */
static int
evtchndrv_read(dev_t dev, struct uio *uio, cred_t *cr)
{
	int rc = 0;
	ssize_t count;
	unsigned int c, p, bytes1 = 0, bytes2 = 0;
	struct evtsoftdata *ep;
	minor_t minor = getminor(dev);

	if (secpolicy_xvm_control(cr))
		return (EPERM);

	ep = EVTCHNDRV_INST2SOFTS(EVTCHNDRV_MINOR2INST(minor));

	/* Whole number of ports. */
	count = uio->uio_resid;
	count &= ~(sizeof (evtchn_port_t) - 1);

	if (count == 0)
		return (0);

	if (count > PAGESIZE)
		count = PAGESIZE;

	mutex_enter(&ep->evtchn_lock);
	for (;;) {
		if (ep->ring_overflow) {
			rc = EFBIG;
			goto done;
		}

		if ((c = ep->ring_cons) != (p = ep->ring_prod))
			break;

		if (uio->uio_fmode & O_NONBLOCK) {
			rc = EAGAIN;
			goto done;
		}

		if (cv_wait_sig(&ep->evtchn_wait, &ep->evtchn_lock) == 0) {
			rc = EINTR;
			goto done;
		}
	}

	/* Byte lengths of two chunks. Chunk split (if any) is at ring wrap. */
	if (((c ^ p) & EVTCHN_RING_SIZE) != 0) {
		bytes1 = (EVTCHN_RING_SIZE - EVTCHN_RING_MASK(c)) *
		    sizeof (evtchn_port_t);
		bytes2 = EVTCHN_RING_MASK(p) * sizeof (evtchn_port_t);
	} else {
		bytes1 = (p - c) * sizeof (evtchn_port_t);
		bytes2 = 0;
	}

	/* Truncate chunks according to caller's maximum byte count. */
	if (bytes1 > count) {
		bytes1 = count;
		bytes2 = 0;
	} else if ((bytes1 + bytes2) > count) {
		bytes2 = count - bytes1;
	}

	if (uiomove(&ep->ring[EVTCHN_RING_MASK(c)], bytes1, UIO_READ, uio) ||
	    ((bytes2 != 0) && uiomove(&ep->ring[0], bytes2, UIO_READ, uio))) {
		rc = EFAULT;
		goto done;
	}

	ep->ring_cons += (bytes1 + bytes2) / sizeof (evtchn_port_t);
done:
	mutex_exit(&ep->evtchn_lock);
	return (rc);
}

/* ARGSUSED */
static int
evtchndrv_write(dev_t dev, struct uio *uio, cred_t *cr)
{
	int  rc, i;
	ssize_t count;
	evtchn_port_t *kbuf;
	struct evtsoftdata *ep;
	ulong_t flags;
	minor_t minor = getminor(dev);
	evtchn_port_t sbuf[32];

	if (secpolicy_xvm_control(cr))
		return (EPERM);

	ep = EVTCHNDRV_INST2SOFTS(EVTCHNDRV_MINOR2INST(minor));


	/* Whole number of ports. */
	count = uio->uio_resid;
	count &= ~(sizeof (evtchn_port_t) - 1);

	if (count == 0)
		return (0);

	if (count > PAGESIZE)
		count = PAGESIZE;

	if (count <= sizeof (sbuf))
		kbuf = sbuf;
	else
		kbuf = kmem_alloc(PAGESIZE, KM_SLEEP);
	if ((rc = uiomove(kbuf, count, UIO_WRITE, uio)) != 0)
		goto out;

	mutex_enter(&port_user_lock);
	for (i = 0; i < (count / sizeof (evtchn_port_t)); i++)
		if ((kbuf[i] < NR_EVENT_CHANNELS) &&
		    (port_user[kbuf[i]] == ep)) {
			flags = intr_clear();
			ec_unmask_evtchn(kbuf[i]);
			intr_restore(flags);
		}
	mutex_exit(&port_user_lock);

out:
	if (kbuf != sbuf)
		kmem_free(kbuf, PAGESIZE);
	return (rc);
}

static void
evtchn_bind_to_user(struct evtsoftdata *u, int port)
{
	ulong_t flags;

	/*
	 * save away the PID of the last process to bind to this event channel.
	 * Useful for debugging.
	 */
	u->pid = ddi_get_pid();

	mutex_enter(&port_user_lock);
	ASSERT(port_user[port] == NULL);
	port_user[port] = u;
	ec_irq_add_evtchn(ec_dev_irq, port);
	flags = intr_clear();
	ec_unmask_evtchn(port);
	intr_restore(flags);
	mutex_exit(&port_user_lock);
}

static void
evtchndrv_close_evtchn(int port)
{
	struct evtsoftdata *ep;

	ASSERT(MUTEX_HELD(&port_user_lock));
	ep = port_user[port];
	ASSERT(ep != NULL);
	(void) ec_mask_evtchn(port);
	/*
	 * It is possible the event is in transit to us.
	 * If it is already in the ring buffer, then a client may
	 * get a spurious event notification on the next read of
	 * of the evtchn device.  Clients will need to be able to
	 * handle getting a spurious event notification.
	 */
	port_user[port] = NULL;
	/*
	 * The event is masked and should stay so, clean it up.
	 */
	ec_irq_rm_evtchn(ec_dev_irq, port);
}

/* ARGSUSED */
static int
evtchndrv_ioctl(dev_t dev, int cmd, intptr_t data, int flag, cred_t *cr,
    int *rvalp)
{
	int err = 0;
	struct evtsoftdata *ep;
	minor_t minor = getminor(dev);

	if (secpolicy_xvm_control(cr))
		return (EPERM);

	ep = EVTCHNDRV_INST2SOFTS(EVTCHNDRV_MINOR2INST(minor));

	*rvalp = 0;

	switch (cmd) {
	case IOCTL_EVTCHN_BIND_VIRQ: {
		struct ioctl_evtchn_bind_virq bind;

		if (copyin((void *)data, &bind, sizeof (bind))) {
			err = EFAULT;
			break;
		}

		if ((err = xen_bind_virq(bind.virq, 0, rvalp)) != 0)
			break;

		evtchn_bind_to_user(ep, *rvalp);
		break;
	}

	case IOCTL_EVTCHN_BIND_INTERDOMAIN: {
		struct ioctl_evtchn_bind_interdomain bind;

		if (copyin((void *)data, &bind, sizeof (bind))) {
			err = EFAULT;
			break;
		}

		if ((err = xen_bind_interdomain(bind.remote_domain,
		    bind.remote_port, rvalp)) != 0)
			break;

		ec_bind_vcpu(*rvalp, 0);
		evtchn_bind_to_user(ep, *rvalp);
		break;
	}

	case IOCTL_EVTCHN_BIND_UNBOUND_PORT: {
		struct ioctl_evtchn_bind_unbound_port bind;

		if (copyin((void *)data, &bind, sizeof (bind))) {
			err = EFAULT;
			break;
		}

		if ((err = xen_alloc_unbound_evtchn(bind.remote_domain,
		    rvalp)) != 0)
			break;

		evtchn_bind_to_user(ep, *rvalp);
		break;
	}

	case IOCTL_EVTCHN_UNBIND: {
		struct ioctl_evtchn_unbind unbind;

		if (copyin((void *)data, &unbind, sizeof (unbind))) {
			err = EFAULT;
			break;
		}

		if (unbind.port >= NR_EVENT_CHANNELS) {
			err = EFAULT;
			break;
		}

		mutex_enter(&port_user_lock);

		if (port_user[unbind.port] != ep) {
			mutex_exit(&port_user_lock);
			err = ENOTCONN;
			break;
		}

		evtchndrv_close_evtchn(unbind.port);
		mutex_exit(&port_user_lock);
		break;
	}

	case IOCTL_EVTCHN_NOTIFY: {
		struct ioctl_evtchn_notify notify;

		if (copyin((void *)data, &notify, sizeof (notify))) {
			err = EFAULT;
			break;
		}

		if (notify.port >= NR_EVENT_CHANNELS) {
			err = EINVAL;
		} else if (port_user[notify.port] != ep) {
			err = ENOTCONN;
		} else {
			ec_notify_via_evtchn(notify.port);
		}
		break;
	}

	default:
		err = ENOSYS;
	}

	return (err);
}

static int
evtchndrv_poll(dev_t dev, short ev, int anyyet, short *revp, pollhead_t **phpp)
{
	struct evtsoftdata *ep;
	minor_t minor = getminor(dev);
	short mask = 0;

	ep = EVTCHNDRV_INST2SOFTS(EVTCHNDRV_MINOR2INST(minor));

	if (ev & POLLOUT)
		mask |= POLLOUT;
	if (ep->ring_overflow)
		mask |= POLLERR;
	if (ev & (POLLIN | POLLRDNORM)) {
		mutex_enter(&ep->evtchn_lock);
		if (ep->ring_cons != ep->ring_prod) {
			mask |= (POLLIN | POLLRDNORM) & ev;
		}
		mutex_exit(&ep->evtchn_lock);
	}
	if ((mask == 0 && !anyyet) || (ev & POLLET)) {
		*phpp = &ep->evtchn_pollhead;
	}
	*revp = mask;
	return (0);
}


/* ARGSUSED */
static int
evtchndrv_open(dev_t *devp, int flag, int otyp, cred_t *credp)
{
	struct evtsoftdata *ep;
	minor_t minor = getminor(*devp);

	if (otyp == OTYP_BLK)
		return (ENXIO);

	/*
	 * only allow open on minor = 0 - the clone device
	 */
	if (minor != 0)
		return (ENXIO);

	/*
	 * find a free slot and grab it
	 */
	mutex_enter(&evtchndrv_clone_tab_mutex);
	for (minor = 1; minor < evtchndrv_nclones; minor++) {
		if (evtchndrv_clone_tab[minor] == 0) {
			evtchndrv_clone_tab[minor] = 1;
			break;
		}
	}
	mutex_exit(&evtchndrv_clone_tab_mutex);
	if (minor == evtchndrv_nclones)
		return (EAGAIN);

	/* Allocate softstate structure */
	if (ddi_soft_state_zalloc(evtchndrv_statep,
	    EVTCHNDRV_MINOR2INST(minor)) != DDI_SUCCESS) {
		mutex_enter(&evtchndrv_clone_tab_mutex);
		evtchndrv_clone_tab[minor] = 0;
		mutex_exit(&evtchndrv_clone_tab_mutex);
		return (EAGAIN);
	}
	ep = EVTCHNDRV_INST2SOFTS(EVTCHNDRV_MINOR2INST(minor));

	/* ... and init it */
	ep->dip = evtchndrv_dip;

	cv_init(&ep->evtchn_wait, NULL, CV_DEFAULT, NULL);
	mutex_init(&ep->evtchn_lock, NULL, MUTEX_DEFAULT, NULL);

	ep->ring = kmem_alloc(PAGESIZE, KM_SLEEP);

	/* clone driver */
	*devp = makedevice(getmajor(*devp), minor);

	return (0);
}

/* ARGSUSED */
static int
evtchndrv_close(dev_t dev, int flag, int otyp, struct cred *credp)
{
	struct evtsoftdata *ep;
	minor_t minor = getminor(dev);
	int i;

	ep = EVTCHNDRV_INST2SOFTS(EVTCHNDRV_MINOR2INST(minor));
	if (ep == NULL)
		return (ENXIO);

	mutex_enter(&port_user_lock);


	for (i = 0; i < NR_EVENT_CHANNELS; i++) {
		if (port_user[i] != ep)
			continue;

		evtchndrv_close_evtchn(i);
	}

	mutex_exit(&port_user_lock);

	kmem_free(ep->ring, PAGESIZE);
	pollhead_clean(&ep->evtchn_pollhead);
	ddi_soft_state_free(evtchndrv_statep, EVTCHNDRV_MINOR2INST(minor));

	/*
	 * free clone tab slot
	 */
	mutex_enter(&evtchndrv_clone_tab_mutex);
	evtchndrv_clone_tab[minor] = 0;
	mutex_exit(&evtchndrv_clone_tab_mutex);

	return (0);
}

/* ARGSUSED */
static int
evtchndrv_info(dev_info_t *dip, ddi_info_cmd_t cmd, void *arg, void **result)
{
	dev_t	dev = (dev_t)arg;
	minor_t	minor = getminor(dev);
	int	retval;

	switch (cmd) {
	case DDI_INFO_DEVT2DEVINFO:
		if (minor != 0 || evtchndrv_dip == NULL) {
			*result = (void *)NULL;
			retval = DDI_FAILURE;
		} else {
			*result = (void *)evtchndrv_dip;
			retval = DDI_SUCCESS;
		}
		break;
	case DDI_INFO_DEVT2INSTANCE:
		*result = (void *)0;
		retval = DDI_SUCCESS;
		break;
	default:
		retval = DDI_FAILURE;
	}
	return (retval);
}


static int
evtchndrv_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
{
	int	error;
	int	unit = ddi_get_instance(dip);


	switch (cmd) {
	case DDI_ATTACH:
		break;
	case DDI_RESUME:
		return (DDI_SUCCESS);
	default:
		cmn_err(CE_WARN, "evtchn_attach: unknown cmd 0x%x\n", cmd);
		return (DDI_FAILURE);
	}

	/* DDI_ATTACH */

	/*
	 * only one instance - but we clone using the open routine
	 */
	if (ddi_get_instance(dip) > 0)
		return (DDI_FAILURE);

	mutex_init(&evtchndrv_clone_tab_mutex, NULL, MUTEX_DRIVER,
	    NULL);

	error = ddi_create_minor_node(dip, "evtchn", S_IFCHR, unit,
	    DDI_PSEUDO, 0);
	if (error != DDI_SUCCESS)
		goto fail;

	/*
	 * save dip for getinfo
	 */
	evtchndrv_dip = dip;
	ddi_report_dev(dip);

	mutex_init(&port_user_lock, NULL, MUTEX_DRIVER, NULL);
	(void) memset(port_user, 0, sizeof (port_user));

	ec_dev_irq = ec_dev_alloc_irq();
	(void) add_avintr(NULL, IPL_EVTCHN, (avfunc)evtchn_device_upcall,
	    "evtchn_driver", ec_dev_irq, NULL, NULL, NULL, dip);

	return (DDI_SUCCESS);

fail:
	(void) evtchndrv_detach(dip, DDI_DETACH);
	return (error);
}

/*ARGSUSED*/
static int
evtchndrv_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
{
	/*
	 * Don't allow detach for now.
	 */
	return (DDI_FAILURE);
}

/* Solaris driver framework */

static struct cb_ops evtchndrv_cb_ops = {
	evtchndrv_open,		/* cb_open */
	evtchndrv_close,	/* cb_close */
	nodev,			/* cb_strategy */
	nodev,			/* cb_print */
	nodev,			/* cb_dump */
	evtchndrv_read,		/* cb_read */
	evtchndrv_write,	/* cb_write */
	evtchndrv_ioctl,	/* cb_ioctl */
	nodev,			/* cb_devmap */
	nodev,			/* cb_mmap */
	nodev,			/* cb_segmap */
	evtchndrv_poll,		/* cb_chpoll */
	ddi_prop_op,		/* cb_prop_op */
	0,			/* cb_stream */
	D_NEW | D_MP | D_64BIT	/* cb_flag */
};

static struct dev_ops evtchndrv_dev_ops = {
	DEVO_REV,		/* devo_rev */
	0,			/* devo_refcnt */
	evtchndrv_info,		/* devo_getinfo */
	nulldev,		/* devo_identify */
	nulldev,		/* devo_probe */
	evtchndrv_attach,	/* devo_attach */
	evtchndrv_detach,	/* devo_detach */
	nodev,			/* devo_reset */
	&evtchndrv_cb_ops,	/* devo_cb_ops */
	NULL,			/* devo_bus_ops */
	NULL,			/* power */
	ddi_quiesce_not_needed,		/* devo_quiesce */
};

static struct modldrv modldrv = {
	&mod_driverops,		/* Type of module.  This one is a driver */
	"Evtchn driver",	/* Name of the module. */
	&evtchndrv_dev_ops	/* driver ops */
};

static struct modlinkage modlinkage = {
	MODREV_1,
	&modldrv,
	NULL
};

int
_init(void)
{
	int err;

	err = ddi_soft_state_init(&evtchndrv_statep,
	    sizeof (struct evtsoftdata), 1);
	if (err)
		return (err);

	err = mod_install(&modlinkage);
	if (err)
		ddi_soft_state_fini(&evtchndrv_statep);
	else
		evtchndrv_clone_tab = kmem_zalloc(
		    sizeof (int) * evtchndrv_nclones, KM_SLEEP);
	return (err);
}

int
_fini(void)
{
	int e;

	e = mod_remove(&modlinkage);
	if (e)
		return (e);

	ddi_soft_state_fini(&evtchndrv_statep);

	return (0);
}

int
_info(struct modinfo *modinfop)
{
	return (mod_info(&modlinkage, modinfop));
}
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

/*
 * Note: This is the backend part of the split PV disk driver. This driver
 * is not a nexus driver, nor is it a leaf driver(block/char/stream driver).
 * Currently, it does not create any minor node. So, although, it runs in
 * backend domain, it will not be used directly from within dom0.
 * It simply gets block I/O requests issued by frontend from a shared page
 * (blkif ring buffer - defined by Xen) between backend and frontend domain,
 * generates a buf, and push it down to underlying disk target driver via
 * ldi interface. When buf is done, this driver will generate a response
 * and put it into ring buffer to inform frontend of the status of the I/O
 * request issued by it. When a new virtual device entry is added in xenstore,
 * there will be an watch event sent from Xen to xvdi framework, who will,
 * in turn, create the devinfo node and try to attach this driver
 * (see xvdi_create_dev). When frontend peer changes its state to
 * XenbusStateClose, an event will also be sent from Xen to xvdi framework,
 * who will detach and remove this devinfo node (see i_xvdi_oestate_handler).
 * I/O requests get from ring buffer and event coming from xenstore cannot be
 * trusted. We verify them in xdb_get_buf() and xdb_check_state_transition().
 *
 * Virtual device configuration is read/written from/to the database via
 * xenbus_* interfaces. Driver also use xvdi_* to interact with hypervisor.
 * There is an on-going effort to make xvdi_* cover all xenbus_*.
 */

#include <sys/types.h>
#include <sys/conf.h>
#include <sys/ddi.h>
#include <sys/dditypes.h>
#include <sys/sunddi.h>
#include <sys/list.h>
#include <sys/dkio.h>
#include <sys/cmlb.h>
#include <sys/vtoc.h>
#include <sys/modctl.h>
#include <sys/bootconf.h>
#include <sys/promif.h>
#include <sys/sysmacros.h>
#include <public/io/xenbus.h>
#include <public/io/xs_wire.h>
#include <xen/sys/xenbus_impl.h>
#include <xen/sys/xendev.h>
#include <sys/gnttab.h>
#include <sys/scsi/generic/inquiry.h>
#include <vm/seg_kmem.h>
#include <vm/hat_i86.h>
#include <sys/gnttab.h>
#include <sys/lofi.h>
#include <io/xdf.h>
#include <xen/io/blkif_impl.h>
#include <io/xdb.h>

static xdb_t *xdb_statep;
static int xdb_debug = 0;

static void xdb_close(dev_info_t *);
static int xdb_push_response(xdb_t *, uint64_t, uint8_t, uint16_t);
static int xdb_get_request(xdb_t *, blkif_request_t *);
static void blkif_get_x86_32_req(blkif_request_t *, blkif_x86_32_request_t *);
static void blkif_get_x86_64_req(blkif_request_t *, blkif_x86_64_request_t *);
static int xdb_biodone(buf_t *);


#ifdef DEBUG
/*
 * debug aid functions
 */

static void
logva(xdb_t *vdp, uint64_t va)
{
	uint64_t *page_addrs;
	int i;

	page_addrs = vdp->page_addrs;
	for (i = 0; i < XDB_MAX_IO_PAGES(vdp); i++) {
		if (page_addrs[i] == va)
			debug_enter("VA remapping found!");
	}

	for (i = 0; i < XDB_MAX_IO_PAGES(vdp); i++) {
		if (page_addrs[i] == 0) {
			page_addrs[i] = va;
			break;
		}
	}
	ASSERT(i < XDB_MAX_IO_PAGES(vdp));
}

static void
unlogva(xdb_t *vdp, uint64_t va)
{
	uint64_t *page_addrs;
	int i;

	page_addrs = vdp->page_addrs;
	for (i = 0; i < XDB_MAX_IO_PAGES(vdp); i++) {
		if (page_addrs[i] == va) {
			page_addrs[i] = 0;
			break;
		}
	}
	ASSERT(i < XDB_MAX_IO_PAGES(vdp));
}

static void
xdb_dump_request_oe(blkif_request_t *req)
{
	int i;

	/*
	 * Exploit the public interface definitions for BLKIF_OP_READ
	 * etc..
	 */
	char *op_name[] = { "read", "write", "barrier", "flush" };

	XDB_DBPRINT(XDB_DBG_IO, (CE_NOTE, "op=%s", op_name[req->operation]));
	XDB_DBPRINT(XDB_DBG_IO, (CE_NOTE, "num of segments=%d",
	    req->nr_segments));
	XDB_DBPRINT(XDB_DBG_IO, (CE_NOTE, "handle=%d", req->handle));
	XDB_DBPRINT(XDB_DBG_IO, (CE_NOTE, "id=%llu",
	    (unsigned long long)req->id));
	XDB_DBPRINT(XDB_DBG_IO, (CE_NOTE, "start sector=%llu",
	    (unsigned long long)req->sector_number));
	for (i = 0; i < req->nr_segments; i++) {
		XDB_DBPRINT(XDB_DBG_IO, (CE_NOTE, "gref=%d, first sec=%d,"
		    "last sec=%d", req->seg[i].gref, req->seg[i].first_sect,
		    req->seg[i].last_sect));
	}
}
#endif /* DEBUG */

/*
 * Statistics.
 */
static char *xdb_stats[] = {
	"rd_reqs",
	"wr_reqs",
	"br_reqs",
	"fl_reqs",
	"oo_reqs"
};

static int
xdb_kstat_update(kstat_t *ksp, int flag)
{
	xdb_t *vdp;
	kstat_named_t *knp;

	if (flag != KSTAT_READ)
		return (EACCES);

	vdp = ksp->ks_private;
	knp = ksp->ks_data;

	/*
	 * Assignment order should match that of the names in
	 * xdb_stats.
	 */
	(knp++)->value.ui64 = vdp->xs_stat_req_reads;
	(knp++)->value.ui64 = vdp->xs_stat_req_writes;
	(knp++)->value.ui64 = vdp->xs_stat_req_barriers;
	(knp++)->value.ui64 = vdp->xs_stat_req_flushes;
	(knp++)->value.ui64 = 0; /* oo_req */

	return (0);
}

static boolean_t
xdb_kstat_init(xdb_t *vdp)
{
	int nstat = sizeof (xdb_stats) / sizeof (xdb_stats[0]);
	char **cp = xdb_stats;
	kstat_named_t *knp;

	if ((vdp->xs_kstats = kstat_create("xdb",
	    ddi_get_instance(vdp->xs_dip),
	    "req_statistics", "block", KSTAT_TYPE_NAMED,
	    nstat, 0)) == NULL)
		return (B_FALSE);

	vdp->xs_kstats->ks_private = vdp;
	vdp->xs_kstats->ks_update = xdb_kstat_update;

	knp = vdp->xs_kstats->ks_data;
	while (nstat > 0) {
		kstat_named_init(knp, *cp, KSTAT_DATA_UINT64);
		knp++;
		cp++;
		nstat--;
	}

	kstat_install(vdp->xs_kstats);

	return (B_TRUE);
}

static char *
i_pathname(dev_info_t *dip)
{
	char *path, *rv;

	path = kmem_alloc(MAXPATHLEN, KM_SLEEP);
	(void) ddi_pathname(dip, path);
	rv = strdup(path);
	kmem_free(path, MAXPATHLEN);

	return (rv);
}

static buf_t *
xdb_get_buf(xdb_t *vdp, blkif_request_t *req, xdb_request_t *xreq)
{
	buf_t *bp;
	uint8_t segs, curseg;
	int sectors;
	int i, err;
	gnttab_map_grant_ref_t mapops[BLKIF_MAX_SEGMENTS_PER_REQUEST];
	ddi_acc_handle_t acchdl;

	acchdl = vdp->xs_ring_hdl;
	bp = XDB_XREQ2BP(xreq);
	curseg = xreq->xr_curseg;
	/* init a new xdb request */
	if (req != NULL) {
		ASSERT(MUTEX_HELD(&vdp->xs_iomutex));
		boolean_t pagemapok = B_TRUE;
		uint8_t op = ddi_get8(acchdl, &req->operation);

		xreq->xr_vdp = vdp;
		xreq->xr_op = op;
		xreq->xr_id = ddi_get64(acchdl, &req->id);
		segs = xreq->xr_buf_pages = ddi_get8(acchdl, &req->nr_segments);
		if (segs == 0) {
			if (op != BLKIF_OP_FLUSH_DISKCACHE)
				cmn_err(CE_WARN, "!non-BLKIF_OP_FLUSH_DISKCACHE"
				    " is seen from domain %d with zero "
				    "length data buffer!", vdp->xs_peer);
			bioinit(bp);
			bp->b_bcount = 0;
			bp->b_lblkno = 0;
			bp->b_un.b_addr = NULL;
			return (bp);
		} else if (op == BLKIF_OP_FLUSH_DISKCACHE) {
			cmn_err(CE_WARN, "!BLKIF_OP_FLUSH_DISKCACHE"
			    " is seen from domain %d with non-zero "
			    "length data buffer!", vdp->xs_peer);
		}

		/*
		 * segs should be no bigger than BLKIF_MAX_SEGMENTS_PER_REQUEST
		 * according to the definition of blk interface by Xen
		 * we do sanity check here
		 */
		if (segs > BLKIF_MAX_SEGMENTS_PER_REQUEST)
			segs = xreq->xr_buf_pages =
			    BLKIF_MAX_SEGMENTS_PER_REQUEST;

		for (i = 0; i < segs; i++) {
			uint8_t fs, ls;

			mapops[i].host_addr =
			    (uint64_t)(uintptr_t)XDB_IOPAGE_VA(
			    vdp->xs_iopage_va, xreq->xr_idx, i);
			mapops[i].dom = vdp->xs_peer;
			mapops[i].ref = ddi_get32(acchdl, &req->seg[i].gref);
			mapops[i].flags = GNTMAP_host_map;
			if (op != BLKIF_OP_READ)
				mapops[i].flags |= GNTMAP_readonly;

			fs = ddi_get8(acchdl, &req->seg[i].first_sect);
			ls = ddi_get8(acchdl, &req->seg[i].last_sect);

			/*
			 * first_sect should be no bigger than last_sect and
			 * both of them should be no bigger than
			 * XB_LAST_SECTOR_IN_SEG according to definition
			 * of blk interface by Xen, so sanity check again
			 */
			if (fs > XB_LAST_SECTOR_IN_SEG)
				fs = XB_LAST_SECTOR_IN_SEG;
			if (ls > XB_LAST_SECTOR_IN_SEG)
				ls = XB_LAST_SECTOR_IN_SEG;
			if (fs > ls)
				fs = ls;

			xreq->xr_segs[i].fs = fs;
			xreq->xr_segs[i].ls = ls;
		}

		/* map in io pages */
		err = xen_map_gref(GNTTABOP_map_grant_ref, mapops, i, B_FALSE);
		if (err != 0)
			return (NULL);
		for (i = 0; i < segs; i++) {
			/*
			 * Although HYPERVISOR_grant_table_op() returned no
			 * error, mapping of each single page can fail. So,
			 * we have to do the check here and handle the error
			 * if needed
			 */
			if (mapops[i].status != GNTST_okay) {
				int j;
				for (j = 0; j < i; j++) {
#ifdef DEBUG
					unlogva(vdp, mapops[j].host_addr);
#endif
					xen_release_pfn(
					    xreq->xr_plist[j].p_pagenum);
				}
				pagemapok = B_FALSE;
				break;
			}
			/* record page mapping handle for unmapping later */
			xreq->xr_page_hdls[i] = mapops[i].handle;
#ifdef DEBUG
			logva(vdp, mapops[i].host_addr);
#endif
			/*
			 * Pass the MFNs down using the shadow list (xr_pplist)
			 *
			 * This is pretty ugly since we have implict knowledge
			 * of how the rootnex binds buffers.
			 * The GNTTABOP_map_grant_ref op makes us do some ugly
			 * stuff since we're not allowed to touch these PTEs
			 * from the VM.
			 *
			 * Obviously, these aren't real page_t's. The rootnex
			 * only needs p_pagenum.
			 * Also, don't use btop() here or 32 bit PAE breaks.
			 */
			xreq->xr_pplist[i] = &xreq->xr_plist[i];
			xreq->xr_plist[i].p_pagenum =
			    xen_assign_pfn(mapops[i].dev_bus_addr >> PAGESHIFT);
		}

		/*
		 * not all pages mapped in successfully, unmap those mapped-in
		 * page and return failure
		 */
		if (!pagemapok) {
			gnttab_unmap_grant_ref_t unmapop;

			for (i = 0; i < segs; i++) {
				if (mapops[i].status != GNTST_okay)
					continue;
				unmapop.host_addr =
				    (uint64_t)(uintptr_t)XDB_IOPAGE_VA(
				    vdp->xs_iopage_va, xreq->xr_idx, i);
				unmapop.dev_bus_addr = 0;
				unmapop.handle = mapops[i].handle;
				(void) HYPERVISOR_grant_table_op(
				    GNTTABOP_unmap_grant_ref, &unmapop, 1);
			}

			return (NULL);
		}
		bioinit(bp);
		bp->b_lblkno = ddi_get64(acchdl, &req->sector_number);
		bp->b_flags = B_BUSY | B_SHADOW | B_PHYS;
		bp->b_flags |= (ddi_get8(acchdl, &req->operation) ==
		    BLKIF_OP_READ) ? B_READ : (B_WRITE | B_ASYNC);
	} else {
		uint64_t blkst;
		int isread;

		/* reuse this buf */
		blkst = bp->b_lblkno + bp->b_bcount / DEV_BSIZE;
		isread = bp->b_flags & B_READ;
		bioreset(bp);
		bp->b_lblkno = blkst;
		bp->b_flags = B_BUSY | B_SHADOW | B_PHYS;
		bp->b_flags |= isread ? B_READ : (B_WRITE | B_ASYNC);
		XDB_DBPRINT(XDB_DBG_IO, (CE_NOTE, "reuse buf, xreq is %d!!",
		    xreq->xr_idx));
	}

	/* form a buf */
	bp->b_un.b_addr = XDB_IOPAGE_VA(vdp->xs_iopage_va, xreq->xr_idx,
	    curseg) + xreq->xr_segs[curseg].fs * DEV_BSIZE;
	bp->b_shadow = &xreq->xr_pplist[curseg];
	bp->b_iodone = xdb_biodone;
	sectors = 0;

	/*
	 * Run through the segments. There are XB_NUM_SECTORS_PER_SEG sectors
	 * per segment. On some OSes (e.g. Linux), there may be empty gaps
	 * between segments. (i.e. the first segment may end on sector 6 and
	 * the second segment start on sector 4).
	 *
	 * if a segments first sector is not set to 0, and this is not the
	 * first segment in our buf, end this buf now.
	 *
	 * if a segments last sector is not set to XB_LAST_SECTOR_IN_SEG, and
	 * this is not the last segment in the request, add this segment into
	 * the buf, then end this buf (updating the pointer to point to the
	 * next segment next time around).
	 */
	for (i = curseg; i < xreq->xr_buf_pages; i++) {
		if ((xreq->xr_segs[i].fs != 0) && (i != curseg)) {
			break;
		}
		sectors += (xreq->xr_segs[i].ls - xreq->xr_segs[i].fs + 1);
		if ((xreq->xr_segs[i].ls != XB_LAST_SECTOR_IN_SEG) &&
		    (i != (xreq->xr_buf_pages - 1))) {
			i++;
			break;
		}
	}
	xreq->xr_curseg = i;
	bp->b_bcount = sectors * DEV_BSIZE;
	bp->b_bufsize = bp->b_bcount;

	return (bp);
}

static xdb_request_t *
xdb_get_req(xdb_t *vdp)
{
	xdb_request_t *req;
	int idx;

	ASSERT(MUTEX_HELD(&vdp->xs_iomutex));
	ASSERT(vdp->xs_free_req != -1);
	req = &vdp->xs_req[vdp->xs_free_req];
	vdp->xs_free_req = req->xr_next;
	idx = req->xr_idx;
	bzero(req, sizeof (xdb_request_t));
	req->xr_idx = idx;
	return (req);
}

static void
xdb_free_req(xdb_request_t *req)
{
	xdb_t *vdp = req->xr_vdp;

	ASSERT(MUTEX_HELD(&vdp->xs_iomutex));
	req->xr_next = vdp->xs_free_req;
	vdp->xs_free_req = req->xr_idx;
}

static void
xdb_response(xdb_t *vdp, blkif_request_t *req, boolean_t ok)
{
	ddi_acc_handle_t acchdl = vdp->xs_ring_hdl;

	if (xdb_push_response(vdp, ddi_get64(acchdl, &req->id),
	    ddi_get8(acchdl, &req->operation), ok))
		xvdi_notify_oe(vdp->xs_dip);
}

static void
xdb_init_ioreqs(xdb_t *vdp)
{
	int i;

	ASSERT(vdp->xs_nentry);

	if (vdp->xs_req == NULL)
		vdp->xs_req = kmem_alloc(vdp->xs_nentry *
		    sizeof (xdb_request_t), KM_SLEEP);
#ifdef DEBUG
	if (vdp->page_addrs == NULL)
		vdp->page_addrs = kmem_zalloc(XDB_MAX_IO_PAGES(vdp) *
		    sizeof (uint64_t), KM_SLEEP);
#endif
	for (i = 0; i < vdp->xs_nentry; i++) {
		vdp->xs_req[i].xr_idx = i;
		vdp->xs_req[i].xr_next = i + 1;
	}
	vdp->xs_req[vdp->xs_nentry - 1].xr_next = -1;
	vdp->xs_free_req = 0;

	/* alloc va in host dom for io page mapping */
	vdp->xs_iopage_va = vmem_xalloc(heap_arena,
	    XDB_MAX_IO_PAGES(vdp) * PAGESIZE, PAGESIZE, 0, 0, 0, 0,
	    VM_SLEEP);
	for (i = 0; i < XDB_MAX_IO_PAGES(vdp); i++)
		hat_prepare_mapping(kas.a_hat,
		    vdp->xs_iopage_va + i * PAGESIZE, NULL);
}

static void
xdb_uninit_ioreqs(xdb_t *vdp)
{
	int i;

	for (i = 0; i < XDB_MAX_IO_PAGES(vdp); i++)
		hat_release_mapping(kas.a_hat,
		    vdp->xs_iopage_va + i * PAGESIZE);
	vmem_xfree(heap_arena, vdp->xs_iopage_va,
	    XDB_MAX_IO_PAGES(vdp) * PAGESIZE);
	if (vdp->xs_req != NULL) {
		kmem_free(vdp->xs_req, vdp->xs_nentry * sizeof (xdb_request_t));
		vdp->xs_req = NULL;
	}
#ifdef DEBUG
	if (vdp->page_addrs != NULL) {
		kmem_free(vdp->page_addrs, XDB_MAX_IO_PAGES(vdp) *
		    sizeof (uint64_t));
		vdp->page_addrs = NULL;
	}
#endif
}

static uint_t
xdb_intr(caddr_t arg)
{
	xdb_t		*vdp = (xdb_t *)arg;
	dev_info_t	*dip = vdp->xs_dip;
	blkif_request_t	req, *reqp = &req;
	xdb_request_t	*xreq;
	buf_t		*bp;
	uint8_t		op;
	int		ret = DDI_INTR_UNCLAIMED;

	XDB_DBPRINT(XDB_DBG_IO, (CE_NOTE,
	    "xdb@%s: I/O request received from dom %d",
	    ddi_get_name_addr(dip), vdp->xs_peer));

	mutex_enter(&vdp->xs_iomutex);

	/* shouldn't touch ring buffer if not in connected state */
	if (!vdp->xs_if_connected) {
		mutex_exit(&vdp->xs_iomutex);
		return (DDI_INTR_UNCLAIMED);
	}
	ASSERT(vdp->xs_hp_connected && vdp->xs_fe_initialised);

	/*
	 * We'll loop till there is no more request in the ring
	 * We won't stuck in this loop for ever since the size of ring buffer
	 * is limited, and frontend will stop pushing requests into it when
	 * the ring buffer is full
	 */

	/* req_event will be increased in xvdi_ring_get_request() */
	while (xdb_get_request(vdp, reqp)) {
		ret = DDI_INTR_CLAIMED;

		op = ddi_get8(vdp->xs_ring_hdl, &reqp->operation);
		if (op == BLKIF_OP_READ			||
		    op == BLKIF_OP_WRITE		||
		    op == BLKIF_OP_WRITE_BARRIER	||
		    op == BLKIF_OP_FLUSH_DISKCACHE) {
#ifdef DEBUG
			xdb_dump_request_oe(reqp);
#endif
			xreq = xdb_get_req(vdp);
			ASSERT(xreq);
			switch (op) {
			case BLKIF_OP_READ:
				vdp->xs_stat_req_reads++;
				break;
			case BLKIF_OP_WRITE_BARRIER:
				vdp->xs_stat_req_barriers++;
				/* FALLTHRU */
			case BLKIF_OP_WRITE:
				vdp->xs_stat_req_writes++;
				break;
			case BLKIF_OP_FLUSH_DISKCACHE:
				vdp->xs_stat_req_flushes++;
				break;
			}

			xreq->xr_curseg = 0; /* start from first segment */
			bp = xdb_get_buf(vdp, reqp, xreq);
			if (bp == NULL) {
				/* failed to form a buf */
				xdb_free_req(xreq);
				xdb_response(vdp, reqp, B_FALSE);
				continue;
			}
			bp->av_forw = NULL;

			XDB_DBPRINT(XDB_DBG_IO, (CE_NOTE,
			    " buf %p, blkno %lld, size %lu, addr %p",
			    (void *)bp, (longlong_t)bp->b_blkno,
			    (ulong_t)bp->b_bcount, (void *)bp->b_un.b_addr));

			/* send bp to underlying blk driver */
			if (vdp->xs_f_iobuf == NULL) {
				vdp->xs_f_iobuf = vdp->xs_l_iobuf = bp;
			} else {
				vdp->xs_l_iobuf->av_forw = bp;
				vdp->xs_l_iobuf = bp;
			}
		} else {
			xdb_response(vdp, reqp, B_FALSE);
			XDB_DBPRINT(XDB_DBG_IO, (CE_WARN, "xdb@%s: "
			    "Unsupported cmd received from dom %d",
			    ddi_get_name_addr(dip), vdp->xs_peer));
		}
	}
	/* notify our taskq to push buf to underlying blk driver */
	if (ret == DDI_INTR_CLAIMED)
		cv_broadcast(&vdp->xs_iocv);

	mutex_exit(&vdp->xs_iomutex);

	return (ret);
}

static int
xdb_biodone(buf_t *bp)
{
	int i, err, bioerr;
	uint8_t segs;
	gnttab_unmap_grant_ref_t unmapops[BLKIF_MAX_SEGMENTS_PER_REQUEST];
	xdb_request_t *xreq = XDB_BP2XREQ(bp);
	xdb_t *vdp = xreq->xr_vdp;
	buf_t *nbp;

	bioerr = geterror(bp);
	if (bioerr)
		XDB_DBPRINT(XDB_DBG_IO, (CE_WARN, "xdb@%s: I/O error %d",
		    ddi_get_name_addr(vdp->xs_dip), bioerr));

	/* check if we are done w/ this I/O request */
	if ((bioerr == 0) && (xreq->xr_curseg < xreq->xr_buf_pages)) {
		nbp = xdb_get_buf(vdp, NULL, xreq);
		if (nbp) {
			err = ldi_strategy(vdp->xs_ldi_hdl, nbp);
			if (err == 0) {
				XDB_DBPRINT(XDB_DBG_IO, (CE_NOTE,
				    "sent buf to backend ok"));
				return (DDI_SUCCESS);
			}
			bioerr = EIO;
			XDB_DBPRINT(XDB_DBG_IO, (CE_WARN, "xdb@%s: "
			    "sent buf to backend dev failed, err=%d",
			    ddi_get_name_addr(vdp->xs_dip), err));
		} else {
			bioerr = EIO;
		}
	}

	/* unmap io pages */
	segs = xreq->xr_buf_pages;
	/*
	 * segs should be no bigger than BLKIF_MAX_SEGMENTS_PER_REQUEST
	 * according to the definition of blk interface by Xen
	 */
	ASSERT(segs <= BLKIF_MAX_SEGMENTS_PER_REQUEST);
	for (i = 0; i < segs; i++) {
		unmapops[i].host_addr = (uint64_t)(uintptr_t)XDB_IOPAGE_VA(
		    vdp->xs_iopage_va, xreq->xr_idx, i);
#ifdef DEBUG
		mutex_enter(&vdp->xs_iomutex);
		unlogva(vdp, unmapops[i].host_addr);
		mutex_exit(&vdp->xs_iomutex);
#endif
		unmapops[i].dev_bus_addr = 0;
		unmapops[i].handle = xreq->xr_page_hdls[i];
	}
	err = HYPERVISOR_grant_table_op(GNTTABOP_unmap_grant_ref,
	    unmapops, segs);
	ASSERT(!err);

	/*
	 * If we have reached a barrier write or a cache flush , then we must
	 * flush all our I/Os.
	 */
	if (xreq->xr_op == BLKIF_OP_WRITE_BARRIER ||
	    xreq->xr_op == BLKIF_OP_FLUSH_DISKCACHE) {
		/*
		 * XXX At this point the write did succeed, so I don't
		 * believe we should report an error because the flush
		 * failed. However, this is a debatable point, so
		 * maybe we need to think more carefully about this.
		 * For now, just cast to void.
		 */
		(void) ldi_ioctl(vdp->xs_ldi_hdl,
		    DKIOCFLUSHWRITECACHE, 0, FKIOCTL, kcred, NULL);
	}

	mutex_enter(&vdp->xs_iomutex);

	/* send response back to frontend */
	if (vdp->xs_if_connected) {
		ASSERT(vdp->xs_hp_connected && vdp->xs_fe_initialised);
		if (xdb_push_response(vdp, xreq->xr_id, xreq->xr_op, bioerr))
			xvdi_notify_oe(vdp->xs_dip);
		XDB_DBPRINT(XDB_DBG_IO, (CE_NOTE,
		    "sent resp back to frontend, id=%llu",
		    (unsigned long long)xreq->xr_id));
	}
	/* free io resources */
	biofini(bp);
	xdb_free_req(xreq);

	vdp->xs_ionum--;
	if (!vdp->xs_if_connected && (vdp->xs_ionum == 0)) {
		/* we're closing, someone is waiting for I/O clean-up */
		cv_signal(&vdp->xs_ionumcv);
	}

	mutex_exit(&vdp->xs_iomutex);

	return (DDI_SUCCESS);
}

static int
xdb_bindto_frontend(xdb_t *vdp)
{
	int err;
	char *oename;
	grant_ref_t gref;
	evtchn_port_t evtchn;
	dev_info_t *dip = vdp->xs_dip;
	char protocol[64] = "";

	ASSERT(MUTEX_HELD(&vdp->xs_cbmutex));

	/*
	 * Switch to the XenbusStateInitialised state.  This let's the
	 * frontend know that we're about to negotiate a connection.
	 */
	(void) xvdi_switch_state(dip, XBT_NULL, XenbusStateInitialised);

	/*
	 * Gather info from frontend
	 */
	oename = xvdi_get_oename(dip);
	if (oename == NULL)
		return (DDI_FAILURE);

	err = xenbus_gather(XBT_NULL, oename,
	    XBP_RING_REF, "%lu", &gref,
	    XBP_EVENT_CHAN, "%u", &evtchn,
	    NULL);
	if (err != 0) {
		xvdi_dev_error(dip, err,
		    "Getting ring-ref and evtchn from frontend");
		return (DDI_FAILURE);
	}

	vdp->xs_blk_protocol = BLKIF_PROTOCOL_NATIVE;
	vdp->xs_nentry = BLKIF_RING_SIZE;
	vdp->xs_entrysize = sizeof (union blkif_sring_entry);

	err = xenbus_gather(XBT_NULL, oename,
	    XBP_PROTOCOL, "%63s", protocol, NULL);
	if (err)
		(void) strcpy(protocol, "unspecified, assuming native");
	else {
		/*
		 * We must check for NATIVE first, so that the fast path
		 * is taken for copying data from the guest to the host.
		 */
		if (strcmp(protocol, XEN_IO_PROTO_ABI_NATIVE) != 0) {
			if (strcmp(protocol, XEN_IO_PROTO_ABI_X86_32) == 0) {
				vdp->xs_blk_protocol = BLKIF_PROTOCOL_X86_32;
				vdp->xs_nentry = BLKIF_X86_32_RING_SIZE;
				vdp->xs_entrysize =
				    sizeof (union blkif_x86_32_sring_entry);
			} else if (strcmp(protocol, XEN_IO_PROTO_ABI_X86_64) ==
			    0) {
				vdp->xs_blk_protocol = BLKIF_PROTOCOL_X86_64;
				vdp->xs_nentry = BLKIF_X86_64_RING_SIZE;
				vdp->xs_entrysize =
				    sizeof (union blkif_x86_64_sring_entry);
			} else {
				xvdi_fatal_error(dip, err, "unknown protocol");
				return (DDI_FAILURE);
			}
		}
	}
#ifdef DEBUG
	cmn_err(CE_NOTE, "!xdb@%s: blkif protocol '%s' ",
	    ddi_get_name_addr(dip), protocol);
#endif

	/*
	 * Map and init ring.  The ring parameters must match those which
	 * have been allocated in the front end.
	 */
	if (xvdi_map_ring(dip, vdp->xs_nentry, vdp->xs_entrysize,
	    gref, &vdp->xs_ring) != DDI_SUCCESS)
		return (DDI_FAILURE);

	/*
	 * This will be removed after we use shadow I/O ring request since
	 * we don't need to access the ring itself directly, thus the access
	 * handle is not needed
	 */
	vdp->xs_ring_hdl = vdp->xs_ring->xr_acc_hdl;

	/* bind event channel */
	err = xvdi_bind_evtchn(dip, evtchn);
	if (err != DDI_SUCCESS) {
		xvdi_unmap_ring(vdp->xs_ring);
		return (DDI_FAILURE);
	}

	return (DDI_SUCCESS);
}

static void
xdb_unbindfrom_frontend(xdb_t *vdp)
{
	ASSERT(MUTEX_HELD(&vdp->xs_cbmutex));

	xvdi_free_evtchn(vdp->xs_dip);
	xvdi_unmap_ring(vdp->xs_ring);
}

/*
 * xdb_params_change() initiates a allows change to the underlying device/file
 * that the backend is accessing.  It does this by disconnecting from the
 * frontend, closing the old device, clearing a bunch of xenbus parameters,
 * and switching back to the XenbusStateInitialising state.  The frontend
 * should notice this transition to the XenbusStateInitialising state and
 * should attempt to reconnect to us (the backend).
 */
static void
xdb_params_change(xdb_t *vdp, char *params, boolean_t update_xs)
{
	xenbus_transaction_t	xbt;
	dev_info_t		*dip = vdp->xs_dip;
	char			*xsname;
	int			err;

	ASSERT(MUTEX_HELD(&vdp->xs_cbmutex));
	ASSERT(vdp->xs_params_path != NULL);

	if ((xsname = xvdi_get_xsname(dip)) == NULL)
		return;
	if (strcmp(vdp->xs_params_path, params) == 0)
		return;

	/*
	 * Close the device we're currently accessing and update the
	 * path which points to our backend device/file.
	 */
	xdb_close(dip);
	vdp->xs_fe_initialised = B_FALSE;

trans_retry:
	if ((err = xenbus_transaction_start(&xbt)) != 0) {
		xvdi_dev_error(dip, err, "params change transaction init");
		goto errout;
	}

	/*
	 * Delete all the xenbus properties that are connection dependant
	 * and go back to the initializing state so that the frontend
	 * driver can re-negotiate a connection.
	 */
	if (((err = xenbus_rm(xbt, xsname, XBP_FB)) != 0) ||
	    ((err = xenbus_rm(xbt, xsname, XBP_INFO)) != 0) ||
	    ((err = xenbus_rm(xbt, xsname, "sector-size")) != 0) ||
	    ((err = xenbus_rm(xbt, xsname, XBP_SECTORS)) != 0) ||
	    ((err = xenbus_rm(xbt, xsname, "instance")) != 0) ||
	    ((err = xenbus_rm(xbt, xsname, "node")) != 0) ||
	    (update_xs && ((err = xenbus_printf(xbt, xsname,
	    "params", "%s", params)) != 0)) ||
	    ((err = xvdi_switch_state(dip,
	    xbt, XenbusStateInitialising) > 0))) {
		(void) xenbus_transaction_end(xbt, 1);
		xvdi_dev_error(dip, err, "params change transaction setup");
		goto errout;
	}

	if ((err = xenbus_transaction_end(xbt, 0)) != 0) {
		if (err == EAGAIN) {
			/* transaction is ended, don't need to abort it */
			goto trans_retry;
		}
		xvdi_dev_error(dip, err, "params change transaction commit");
		goto errout;
	}

	/* Change the device that we plan to access */
	strfree(vdp->xs_params_path);
	vdp->xs_params_path = strdup(params);
	return;

errout:
	(void) xvdi_switch_state(dip, xbt, XenbusStateInitialising);
}

/*
 * xdb_watch_params_cb() - This callback is invoked whenever there
 * is an update to the following xenbus parameter:
 *     /local/domain/0/backend/vbd/<domU_id>/<domU_dev>/params
 *
 * This normally happens during xm block-configure operations, which
 * are used to change CD device images for HVM domUs.
 */
/*ARGSUSED*/
static void
xdb_watch_params_cb(dev_info_t *dip, const char *path, void *arg)
{
	xdb_t			*vdp = (xdb_t *)ddi_get_driver_private(dip);
	char			*xsname, *oename, *str, *str2;

	if (((xsname = xvdi_get_xsname(dip)) == NULL) ||
	    ((oename = xvdi_get_oename(dip)) == NULL)) {
		return;
	}

	mutex_enter(&vdp->xs_cbmutex);

	if (xenbus_read_str(xsname, "params", &str) != 0) {
		mutex_exit(&vdp->xs_cbmutex);
		return;
	}

	if (strcmp(vdp->xs_params_path, str) == 0) {
		/* Nothing todo */
		mutex_exit(&vdp->xs_cbmutex);
		strfree(str);
		return;
	}

	/*
	 * If the frontend isn't a cd device, doesn't support media
	 * requests, or has locked the media, then we can't change
	 * the params value.  restore the current value.
	 */
	str2 = NULL;
	if (!XDB_IS_FE_CD(vdp) ||
	    (xenbus_read_str(oename, XBP_MEDIA_REQ, &str2) != 0) ||
	    (strcmp(str2, XBV_MEDIA_REQ_LOCK) == 0)) {
		if (str2 != NULL)
			strfree(str2);
		strfree(str);

		str = i_pathname(dip);
		cmn_err(CE_NOTE,
		    "!%s: media locked, ignoring params update", str);
		strfree(str);

		mutex_exit(&vdp->xs_cbmutex);
		return;
	}

	XDB_DBPRINT(XDB_DBG_INFO, (CE_NOTE,
	    "block-configure params request: \"%s\"", str));

	xdb_params_change(vdp, str, B_FALSE);
	mutex_exit(&vdp->xs_cbmutex);
	strfree(str);
}

/*
 * xdb_watch_media_req_cb() - This callback is invoked whenever there
 * is an update to the following xenbus parameter:
 *     /local/domain/<domU_id>/device/vbd/<domU_dev>/media-req
 *
 * Media requests are only supported on CD devices and are issued by
 * the frontend.  Currently the only supported media request operaions
 * are "lock" and "eject".  A "lock" prevents the backend from changing
 * the backing device/file (via xm block-configure).  An "eject" requests
 * tells the backend device that it should disconnect from the frontend
 * and closing the backing device/file that is currently in use.
 */
/*ARGSUSED*/
static void
xdb_watch_media_req_cb(dev_info_t *dip, const char *path, void *arg)
{
	xdb_t			*vdp = (xdb_t *)ddi_get_driver_private(dip);
	char			*oename, *str;

	mutex_enter(&vdp->xs_cbmutex);

	if ((oename = xvdi_get_oename(dip)) == NULL) {
		mutex_exit(&vdp->xs_cbmutex);
		return;
	}

	if (xenbus_read_str(oename, XBP_MEDIA_REQ, &str) != 0) {
		mutex_exit(&vdp->xs_cbmutex);
		return;
	}

	if (!XDB_IS_FE_CD(vdp)) {
		xvdi_dev_error(dip, EINVAL,
		    "media-req only supported for cdrom devices");
		mutex_exit(&vdp->xs_cbmutex);
		return;
	}

	if (strcmp(str, XBV_MEDIA_REQ_EJECT) != 0) {
		mutex_exit(&vdp->xs_cbmutex);
		strfree(str);
		return;
	}
	strfree(str);

	XDB_DBPRINT(XDB_DBG_INFO, (CE_NOTE, "media eject request"));

	xdb_params_change(vdp, "", B_TRUE);
	(void) xenbus_printf(XBT_NULL, oename,
	    XBP_MEDIA_REQ, "%s", XBV_MEDIA_REQ_NONE);
	mutex_exit(&vdp->xs_cbmutex);
}

/*
 * If we're dealing with a cdrom device, let the frontend know that
 * we support media requests via XBP_MEDIA_REQ_SUP, and setup a watch
 * to handle those frontend media request changes, which modify the
 * following xenstore parameter:
 *	/local/domain/<domU_id>/device/vbd/<domU_dev>/media-req
 */
static boolean_t
xdb_media_req_init(xdb_t *vdp)
{
	dev_info_t		*dip = vdp->xs_dip;
	char			*xsname, *oename;

	ASSERT(MUTEX_HELD(&vdp->xs_cbmutex));

	if (((xsname = xvdi_get_xsname(dip)) == NULL) ||
	    ((oename = xvdi_get_oename(dip)) == NULL))
		return (B_FALSE);

	if (!XDB_IS_FE_CD(vdp))
		return (B_TRUE);

	if (xenbus_printf(XBT_NULL, xsname, XBP_MEDIA_REQ_SUP, "%d", 1) != 0)
		return (B_FALSE);

	if (xvdi_add_xb_watch_handler(dip, oename,
	    XBP_MEDIA_REQ, xdb_watch_media_req_cb, NULL) != DDI_SUCCESS) {
		xvdi_dev_error(dip, EAGAIN,
		    "Failed to register watch for cdrom media requests");
		return (B_FALSE);
	}

	return (B_TRUE);
}

/*
 * Get our params value.  Also, if we're using "params" then setup a
 * watch to handle xm block-configure operations which modify the
 * following xenstore parameter:
 *	/local/domain/0/backend/vbd/<domU_id>/<domU_dev>/params
 */
static boolean_t
xdb_params_init(xdb_t *vdp)
{
	dev_info_t		*dip = vdp->xs_dip;
	char			*str, *xsname;
	int			err;

	ASSERT(MUTEX_HELD(&vdp->xs_cbmutex));
	ASSERT(vdp->xs_params_path == NULL);

	if ((xsname = xvdi_get_xsname(dip)) == NULL)
		return (B_FALSE);

	err = xenbus_read_str(xsname, "params", &str);
	if (err != 0) {
		return (B_FALSE);
	}
	vdp->xs_params_path = str;

	if (xvdi_add_xb_watch_handler(dip, xsname, "params",
	    xdb_watch_params_cb, NULL) != DDI_SUCCESS) {
		strfree(vdp->xs_params_path);
		vdp->xs_params_path = NULL;
		return (B_FALSE);
	}

	return (B_TRUE);
}

#define	LOFI_CTRL_NODE	"/dev/lofictl"
#define	LOFI_DEV_NODE	"/devices/pseudo/lofi@0:"
#define	LOFI_MODE	(FREAD | FWRITE | FEXCL)

static int
xdb_setup_node(xdb_t *vdp, char *path)
{
	dev_info_t		*dip = vdp->xs_dip;
	char			*xsname, *str;
	ldi_handle_t		ldi_hdl;
	struct lofi_ioctl	*li;
	int			minor, err;

	ASSERT(MUTEX_HELD(&vdp->xs_cbmutex));

	if ((xsname = xvdi_get_xsname(dip)) == NULL)
		return (DDI_FAILURE);

	if ((err = xenbus_read_str(xsname, "type", &str)) != 0) {
		xvdi_dev_error(dip, err, "Getting type from backend device");
		return (DDI_FAILURE);
	}
	if (strcmp(str, "file") == 0)
		vdp->xs_type |= XDB_DEV_BE_LOFI;
	strfree(str);

	if (!XDB_IS_BE_LOFI(vdp)) {
		(void) strlcpy(path, vdp->xs_params_path, MAXPATHLEN);
		ASSERT(vdp->xs_lofi_path == NULL);
		return (DDI_SUCCESS);
	}

	do {
		err = ldi_open_by_name(LOFI_CTRL_NODE, LOFI_MODE, kcred,
		    &ldi_hdl, vdp->xs_ldi_li);
	} while (err == EBUSY);
	if (err != 0) {
		return (DDI_FAILURE);
	}

	li = kmem_zalloc(sizeof (*li), KM_SLEEP);
	(void) strlcpy(li->li_filename, vdp->xs_params_path,
	    sizeof (li->li_filename));
	err = ldi_ioctl(ldi_hdl, LOFI_MAP_FILE, (intptr_t)li,
	    LOFI_MODE | FKIOCTL, kcred, &minor);
	(void) ldi_close(ldi_hdl, LOFI_MODE, kcred);
	kmem_free(li, sizeof (*li));

	if (err != 0) {
		cmn_err(CE_WARN, "xdb@%s: Failed to create lofi dev for %s",
		    ddi_get_name_addr(dip), vdp->xs_params_path);
		return (DDI_FAILURE);
	}

	/*
	 * return '/devices/...' instead of '/dev/lofi/...' since the
	 * former is available immediately after calling ldi_ioctl
	 */
	(void) snprintf(path, MAXPATHLEN, LOFI_DEV_NODE "%d", minor);
	(void) xenbus_printf(XBT_NULL, xsname, "node", "%s", path);

	ASSERT(vdp->xs_lofi_path == NULL);
	vdp->xs_lofi_path = strdup(path);

	return (DDI_SUCCESS);
}

static void
xdb_teardown_node(xdb_t *vdp)
{
	dev_info_t *dip = vdp->xs_dip;
	ldi_handle_t ldi_hdl;
	struct lofi_ioctl *li;
	int err;

	ASSERT(MUTEX_HELD(&vdp->xs_cbmutex));

	if (!XDB_IS_BE_LOFI(vdp))
		return;

	vdp->xs_type &= ~XDB_DEV_BE_LOFI;
	ASSERT(vdp->xs_lofi_path != NULL);

	li = kmem_zalloc(sizeof (*li), KM_SLEEP);
	(void) strlcpy(li->li_filename, vdp->xs_params_path,
	    sizeof (li->li_filename));

	do {
		err = ldi_open_by_name(LOFI_CTRL_NODE, LOFI_MODE, kcred,
		    &ldi_hdl, vdp->xs_ldi_li);
	} while (err == EBUSY);

	if (err != 0) {
		kmem_free(li, sizeof (*li));
		return;
	}

	if (ldi_ioctl(ldi_hdl, LOFI_UNMAP_FILE, (intptr_t)li,
	    LOFI_MODE | FKIOCTL, kcred, NULL) != 0) {
		cmn_err(CE_WARN, "xdb@%s: Failed to delete lofi dev for %s",
		    ddi_get_name_addr(dip), li->li_filename);
	}

	(void) ldi_close(ldi_hdl, LOFI_MODE, kcred);
	kmem_free(li, sizeof (*li));

	strfree(vdp->xs_lofi_path);
	vdp->xs_lofi_path = NULL;
}

static int
xdb_open_device(xdb_t *vdp)
{
	dev_info_t *dip = vdp->xs_dip;
	uint64_t devsize;
	int blksize;
	char *nodepath;
	char *xsname;
	char *str;
	int err;

	ASSERT(MUTEX_HELD(&vdp->xs_cbmutex));

	if (strlen(vdp->xs_params_path) == 0) {
		/*
		 * it's possible to have no backing device when dealing
		 * with a pv cdrom drive that has no virtual cd associated
		 * with it.
		 */
		ASSERT(XDB_IS_FE_CD(vdp));
		ASSERT(vdp->xs_sectors == 0);
		ASSERT(vdp->xs_ldi_li == NULL);
		ASSERT(vdp->xs_ldi_hdl == NULL);
		return (DDI_SUCCESS);
	}

	/*
	 * after the hotplug scripts have "connected" the device, check to see
	 * if we're using a dynamic device.  If so, replace the params path
	 * with the dynamic one.
	 */
	xsname = xvdi_get_xsname(dip);
	err = xenbus_read_str(xsname, "dynamic-device-path", &str);
	if (err == 0) {
		strfree(vdp->xs_params_path);
		vdp->xs_params_path = str;
	}

	if (ldi_ident_from_dip(dip, &vdp->xs_ldi_li) != 0)
		return (DDI_FAILURE);

	nodepath = kmem_zalloc(MAXPATHLEN, KM_SLEEP);

	/* try to open backend device */
	if (xdb_setup_node(vdp, nodepath) != DDI_SUCCESS) {
		xvdi_dev_error(dip, ENXIO,
		    "Getting device path of backend device");
		ldi_ident_release(vdp->xs_ldi_li);
		kmem_free(nodepath, MAXPATHLEN);
		return (DDI_FAILURE);
	}

	if (ldi_open_by_name(nodepath,
	    FREAD | (XDB_IS_RO(vdp) ? 0 : FWRITE),
	    kcred, &vdp->xs_ldi_hdl, vdp->xs_ldi_li) != 0) {
		xdb_teardown_node(vdp);
		ldi_ident_release(vdp->xs_ldi_li);
		cmn_err(CE_WARN, "xdb@%s: Failed to open: %s",
		    ddi_get_name_addr(dip), nodepath);
		kmem_free(nodepath, MAXPATHLEN);
		return (DDI_FAILURE);
	}

	if (ldi_get_size(vdp->xs_ldi_hdl, &devsize) != DDI_SUCCESS) {
		(void) ldi_close(vdp->xs_ldi_hdl,
		    FREAD | (XDB_IS_RO(vdp) ? 0 : FWRITE), kcred);
		xdb_teardown_node(vdp);
		ldi_ident_release(vdp->xs_ldi_li);
		kmem_free(nodepath, MAXPATHLEN);
		return (DDI_FAILURE);
	}

	blksize = ldi_prop_get_int64(vdp->xs_ldi_hdl,
	    DDI_PROP_DONTPASS | DDI_PROP_NOTPROM,
	    "blksize", DEV_BSIZE);
	if (blksize == DEV_BSIZE)
		blksize = ldi_prop_get_int(vdp->xs_ldi_hdl,
		    LDI_DEV_T_ANY | DDI_PROP_DONTPASS |
		    DDI_PROP_NOTPROM, "device-blksize", DEV_BSIZE);

	vdp->xs_sec_size = blksize;
	vdp->xs_sectors = devsize / blksize;

	/* check if the underlying device is a CD/DVD disc */
	if (ldi_prop_get_int(vdp->xs_ldi_hdl, LDI_DEV_T_ANY | DDI_PROP_DONTPASS,
	    INQUIRY_DEVICE_TYPE, DTYPE_DIRECT) == DTYPE_RODIRECT)
		vdp->xs_type |= XDB_DEV_BE_CD;

	/* check if the underlying device is a removable disk */
	if (ldi_prop_exists(vdp->xs_ldi_hdl,
	    LDI_DEV_T_ANY | DDI_PROP_DONTPASS | DDI_PROP_NOTPROM,
	    "removable-media"))
		vdp->xs_type |= XDB_DEV_BE_RMB;

	kmem_free(nodepath, MAXPATHLEN);
	return (DDI_SUCCESS);
}

static void
xdb_close_device(xdb_t *vdp)
{
	ASSERT(MUTEX_HELD(&vdp->xs_cbmutex));

	if (strlen(vdp->xs_params_path) == 0) {
		ASSERT(XDB_IS_FE_CD(vdp));
		ASSERT(vdp->xs_sectors == 0);
		ASSERT(vdp->xs_ldi_li == NULL);
		ASSERT(vdp->xs_ldi_hdl == NULL);
		return;
	}

	(void) ldi_close(vdp->xs_ldi_hdl,
	    FREAD | (XDB_IS_RO(vdp) ? 0 : FWRITE), kcred);
	xdb_teardown_node(vdp);
	ldi_ident_release(vdp->xs_ldi_li);
	vdp->xs_type &= ~(XDB_DEV_BE_CD | XDB_DEV_BE_RMB);
	vdp->xs_sectors = 0;
	vdp->xs_ldi_li = NULL;
	vdp->xs_ldi_hdl = NULL;
}

/*
 * Kick-off connect process
 * If xs_fe_initialised == B_TRUE and xs_hp_connected == B_TRUE
 * the xs_if_connected will be changed to B_TRUE on success,
 */
static void
xdb_start_connect(xdb_t *vdp)
{
	xenbus_transaction_t	xbt;
	dev_info_t		*dip = vdp->xs_dip;
	boolean_t		fb_exists;
	int			err, instance = ddi_get_instance(dip);
	uint64_t		sectors;
	uint_t			dinfo, ssize;
	char			*xsname;

	ASSERT(MUTEX_HELD(&vdp->xs_cbmutex));

	if (((xsname = xvdi_get_xsname(dip)) == NULL) ||
	    ((vdp->xs_peer = xvdi_get_oeid(dip)) == (domid_t)-1))
		return;

	mutex_enter(&vdp->xs_iomutex);
	/*
	 * if the hotplug scripts haven't run or if the frontend is not
	 * initialized, then we can't try to connect.
	 */
	if (!vdp->xs_hp_connected || !vdp->xs_fe_initialised) {
		ASSERT(!vdp->xs_if_connected);
		mutex_exit(&vdp->xs_iomutex);
		return;
	}

	/* If we're already connected then there's nothing todo */
	if (vdp->xs_if_connected) {
		mutex_exit(&vdp->xs_iomutex);
		return;
	}
	mutex_exit(&vdp->xs_iomutex);

	/*
	 * Start connect to frontend only when backend device are ready
	 * and frontend has moved to XenbusStateInitialised, which means
	 * ready to connect.
	 */
	XDB_DBPRINT(XDB_DBG_INFO, (CE_NOTE,
	    "xdb@%s: starting connection process", ddi_get_name_addr(dip)));

	if (xdb_open_device(vdp) != DDI_SUCCESS)
		return;

	if (xdb_bindto_frontend(vdp) != DDI_SUCCESS) {
		xdb_close_device(vdp);
		return;
	}

	/* init i/o requests */
	xdb_init_ioreqs(vdp);

	if (ddi_add_intr(dip, 0, NULL, NULL, xdb_intr, (caddr_t)vdp)
	    != DDI_SUCCESS) {
		xdb_uninit_ioreqs(vdp);
		xdb_unbindfrom_frontend(vdp);
		xdb_close_device(vdp);
		return;
	}

	dinfo = 0;
	if (XDB_IS_RO(vdp))
		dinfo |= VDISK_READONLY;
	if (XDB_IS_BE_RMB(vdp))
		dinfo |= VDISK_REMOVABLE;
	if (XDB_IS_BE_CD(vdp))
		dinfo |= VDISK_CDROM;
	if (XDB_IS_FE_CD(vdp))
		dinfo |= VDISK_REMOVABLE | VDISK_CDROM;

	/*
	 * we can recieve intr any time from now on
	 * mark that we're ready to take intr
	 */
	mutex_enter(&vdp->xs_iomutex);
	ASSERT(vdp->xs_fe_initialised);
	vdp->xs_if_connected = B_TRUE;
	mutex_exit(&vdp->xs_iomutex);

trans_retry:
	/* write into xenstore the info needed by frontend */
	if ((err = xenbus_transaction_start(&xbt)) != 0) {
		xvdi_dev_error(dip, err, "connect transaction init");
		goto errout;
	}

	/* If feature-barrier isn't present in xenstore, add it.  */
	fb_exists = xenbus_exists(xsname, XBP_FB);

	ssize = (vdp->xs_sec_size == 0) ? DEV_BSIZE : vdp->xs_sec_size;
	sectors = vdp->xs_sectors;
	if (((!fb_exists &&
	    (err = xenbus_printf(xbt, xsname, XBP_FB, "%d", 1)))) ||
	    (err = xenbus_printf(xbt, xsname, XBP_INFO, "%u", dinfo)) ||
	    (err = xenbus_printf(xbt, xsname, XBP_SECTOR_SIZE, "%u", ssize)) ||
	    (err = xenbus_printf(xbt, xsname,
	    XBP_SECTORS, "%"PRIu64, sectors)) ||
	    (err = xenbus_printf(xbt, xsname, "instance", "%d", instance)) ||
	    ((err = xvdi_switch_state(dip, xbt, XenbusStateConnected)) > 0)) {
		(void) xenbus_transaction_end(xbt, 1);
		xvdi_dev_error(dip, err, "connect transaction setup");
		goto errout;
	}

	if ((err = xenbus_transaction_end(xbt, 0)) != 0) {
		if (err == EAGAIN) {
			/* transaction is ended, don't need to abort it */
			goto trans_retry;
		}
		xvdi_dev_error(dip, err, "connect transaction commit");
		goto errout;
	}

	return;

errout:
	xdb_close(dip);
}

/*
 * Disconnect from frontend and close backend device
 */
static void
xdb_close(dev_info_t *dip)
{
	xdb_t *vdp = (xdb_t *)ddi_get_driver_private(dip);

	ASSERT(MUTEX_HELD(&vdp->xs_cbmutex));
	mutex_enter(&vdp->xs_iomutex);

	/*
	 * if the hotplug scripts haven't run or if the frontend is not
	 * initialized, then we can't be connected, so there's no
	 * connection to close.
	 */
	if (!vdp->xs_hp_connected || !vdp->xs_fe_initialised) {
		ASSERT(!vdp->xs_if_connected);
		mutex_exit(&vdp->xs_iomutex);
		return;
	}

	/* if we're not connected, there's nothing to do */
	if (!vdp->xs_if_connected) {
		cv_broadcast(&vdp->xs_iocv);
		mutex_exit(&vdp->xs_iomutex);
		return;
	}

	XDB_DBPRINT(XDB_DBG_INFO, (CE_NOTE, "closing while connected"));

	vdp->xs_if_connected = B_FALSE;
	cv_broadcast(&vdp->xs_iocv);

	mutex_exit(&vdp->xs_iomutex);

	/* stop accepting I/O request from frontend */
	ddi_remove_intr(dip, 0, NULL);

	/* clear all on-going I/Os, if any */
	mutex_enter(&vdp->xs_iomutex);
	while (vdp->xs_ionum > 0)
		cv_wait(&vdp->xs_ionumcv, &vdp->xs_iomutex);
	mutex_exit(&vdp->xs_iomutex);

	/* clean up resources and close this interface */
	xdb_uninit_ioreqs(vdp);
	xdb_unbindfrom_frontend(vdp);
	xdb_close_device(vdp);
	vdp->xs_peer = (domid_t)-1;
}

static void
xdb_send_buf(void *arg)
{
	xdb_t	*vdp = (xdb_t *)arg;
	buf_t	*bp;
	int	err;

	mutex_enter(&vdp->xs_iomutex);
	while (vdp->xs_send_buf) {
		if ((bp = vdp->xs_f_iobuf) == NULL) {
			/* wait for some io to send */
			XDB_DBPRINT(XDB_DBG_IO, (CE_NOTE,
			    "send buf waiting for io"));
			cv_wait(&vdp->xs_iocv, &vdp->xs_iomutex);
			continue;
		}

		vdp->xs_f_iobuf = bp->av_forw;
		bp->av_forw = NULL;
		vdp->xs_ionum++;

		mutex_exit(&vdp->xs_iomutex);
		if (bp->b_bcount == 0) {
			/* no I/O needs to be done */
			(void) xdb_biodone(bp);
			mutex_enter(&vdp->xs_iomutex);
			continue;
		}

		err = EIO;
		if (vdp->xs_ldi_hdl != NULL)
			err = ldi_strategy(vdp->xs_ldi_hdl, bp);
		if (err != 0) {
			bp->b_flags |= B_ERROR;
			(void) xdb_biodone(bp);
			XDB_DBPRINT(XDB_DBG_IO, (CE_WARN,
			    "xdb@%s: sent buf to backend devfailed, err=%d",
			    ddi_get_name_addr(vdp->xs_dip), err));
		} else {
			XDB_DBPRINT(XDB_DBG_IO, (CE_NOTE,
			    "sent buf to backend ok"));
		}
		mutex_enter(&vdp->xs_iomutex);
	}
	XDB_DBPRINT(XDB_DBG_IO, (CE_NOTE, "send buf finishing"));
	mutex_exit(&vdp->xs_iomutex);
}

/*ARGSUSED*/
static void
xdb_hp_state_change(dev_info_t *dip, ddi_eventcookie_t id, void *arg,
    void *impl_data)
{
	xendev_hotplug_state_t state = *(xendev_hotplug_state_t *)impl_data;
	xdb_t *vdp = (xdb_t *)ddi_get_driver_private(dip);

	XDB_DBPRINT(XDB_DBG_INFO, (CE_NOTE, "xdb@%s: "
	    "hotplug status change to %d!", ddi_get_name_addr(dip), state));

	if (state != Connected)
		return;

	mutex_enter(&vdp->xs_cbmutex);

	/* If hotplug script have already run, there's nothing todo */
	if (vdp->xs_hp_connected) {
		mutex_exit(&vdp->xs_cbmutex);
		return;
	}

	vdp->xs_hp_connected = B_TRUE;
	xdb_start_connect(vdp);
	mutex_exit(&vdp->xs_cbmutex);
}

/*ARGSUSED*/
static void
xdb_oe_state_change(dev_info_t *dip, ddi_eventcookie_t id, void *arg,
    void *impl_data)
{
	XenbusState new_state = *(XenbusState *)impl_data;
	xdb_t *vdp = (xdb_t *)ddi_get_driver_private(dip);

	XDB_DBPRINT(XDB_DBG_INFO, (CE_NOTE, "xdb@%s: "
	    "otherend state change to %d!", ddi_get_name_addr(dip), new_state));

	mutex_enter(&vdp->xs_cbmutex);

	/*
	 * Now it'd really be nice if there was a well defined state
	 * transition model for xen frontend drivers, but unfortunatly
	 * there isn't.  So we're stuck with assuming that all state
	 * transitions are possible, and we'll just have to deal with
	 * them regardless of what state we're in.
	 */
	switch (new_state) {
	case XenbusStateUnknown:
	case XenbusStateInitialising:
	case XenbusStateInitWait:
		/* tear down our connection to the frontend */
		xdb_close(dip);
		vdp->xs_fe_initialised = B_FALSE;
		break;

	case XenbusStateInitialised:
		/*
		 * If we were conected, then we need to drop the connection
		 * and re-negotiate it.
		 */
		xdb_close(dip);
		vdp->xs_fe_initialised = B_TRUE;
		xdb_start_connect(vdp);
		break;

	case XenbusStateConnected:
		/* nothing todo here other than congratulate the frontend */
		break;

	case XenbusStateClosing:
		/* monkey see monkey do */
		(void) xvdi_switch_state(dip, XBT_NULL, XenbusStateClosing);
		break;

	case XenbusStateClosed:
		/* tear down our connection to the frontend */
		xdb_close(dip);
		vdp->xs_fe_initialised = B_FALSE;
		(void) xvdi_switch_state(dip, XBT_NULL, new_state);
		break;
	}

	mutex_exit(&vdp->xs_cbmutex);
}

static int
xdb_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
{
	ddi_iblock_cookie_t	ibc;
	xdb_t			*vdp;
	int			instance = ddi_get_instance(dip);
	char			*xsname, *oename;
	char			*str;

	switch (cmd) {
	case DDI_RESUME:
		return (DDI_FAILURE);
	case DDI_ATTACH:
		break;
	default:
		return (DDI_FAILURE);
	}
	/* DDI_ATTACH */

	if (((xsname = xvdi_get_xsname(dip)) == NULL) ||
	    ((oename = xvdi_get_oename(dip)) == NULL))
		return (DDI_FAILURE);

	/*
	 * Disable auto-detach.  This is necessary so that we don't get
	 * detached while we're disconnected from the front end.
	 */
	(void) ddi_prop_update_int(DDI_DEV_T_NONE, dip, DDI_NO_AUTODETACH, 1);

	if (ddi_get_iblock_cookie(dip, 0, &ibc) != DDI_SUCCESS)
		return (DDI_FAILURE);

	if (ddi_soft_state_zalloc(xdb_statep, instance) != DDI_SUCCESS)
		return (DDI_FAILURE);

	vdp = ddi_get_soft_state(xdb_statep, instance);
	vdp->xs_dip = dip;
	mutex_init(&vdp->xs_iomutex, NULL, MUTEX_DRIVER, (void *)ibc);
	mutex_init(&vdp->xs_cbmutex, NULL, MUTEX_DRIVER, (void *)ibc);
	cv_init(&vdp->xs_iocv, NULL, CV_DRIVER, NULL);
	cv_init(&vdp->xs_ionumcv, NULL, CV_DRIVER, NULL);
	ddi_set_driver_private(dip, vdp);

	if (!xdb_kstat_init(vdp))
		goto errout1;

	/* Check if the frontend device is supposed to be a cdrom */
	if (xenbus_read_str(oename, XBP_DEV_TYPE, &str) != 0)
		return (DDI_FAILURE);
	if (strcmp(str, XBV_DEV_TYPE_CD) == 0)
		vdp->xs_type |= XDB_DEV_FE_CD;
	strfree(str);

	/* Check if the frontend device is supposed to be read only */
	if (xenbus_read_str(xsname, "mode", &str) != 0)
		return (DDI_FAILURE);
	if ((strcmp(str, "r") == 0) || (strcmp(str, "ro") == 0))
		vdp->xs_type |= XDB_DEV_RO;
	strfree(str);

	mutex_enter(&vdp->xs_cbmutex);
	if (!xdb_media_req_init(vdp) || !xdb_params_init(vdp)) {
		xvdi_remove_xb_watch_handlers(dip);
		mutex_exit(&vdp->xs_cbmutex);
		goto errout2;
	}
	mutex_exit(&vdp->xs_cbmutex);

	vdp->xs_send_buf = B_TRUE;
	vdp->xs_iotaskq = ddi_taskq_create(dip, "xdb_iotask", 1,
	    TASKQ_DEFAULTPRI, 0);
	(void) ddi_taskq_dispatch(vdp->xs_iotaskq, xdb_send_buf, vdp,
	    DDI_SLEEP);

	/* Watch frontend and hotplug state change */
	if ((xvdi_add_event_handler(dip, XS_OE_STATE, xdb_oe_state_change,
	    NULL) != DDI_SUCCESS) ||
	    (xvdi_add_event_handler(dip, XS_HP_STATE, xdb_hp_state_change,
	    NULL) != DDI_SUCCESS))
		goto errout3;

	/*
	 * Kick-off hotplug script
	 */
	if (xvdi_post_event(dip, XEN_HP_ADD) != DDI_SUCCESS) {
		cmn_err(CE_WARN, "xdb@%s: failed to start hotplug script",
		    ddi_get_name_addr(dip));
		goto errout3;
	}

	/*
	 * start waiting for hotplug event and otherend state event
	 * mainly for debugging, frontend will not take any op seeing this
	 */
	(void) xvdi_switch_state(dip, XBT_NULL, XenbusStateInitWait);

	XDB_DBPRINT(XDB_DBG_INFO, (CE_NOTE, "xdb@%s: attached!",
	    ddi_get_name_addr(dip)));
	return (DDI_SUCCESS);

errout3:
	ASSERT(vdp->xs_hp_connected && vdp->xs_if_connected);

	xvdi_remove_event_handler(dip, NULL);

	/* Disconnect from the backend */
	mutex_enter(&vdp->xs_cbmutex);
	mutex_enter(&vdp->xs_iomutex);
	vdp->xs_send_buf = B_FALSE;
	cv_broadcast(&vdp->xs_iocv);
	mutex_exit(&vdp->xs_iomutex);
	mutex_exit(&vdp->xs_cbmutex);

	/* wait for all io to dtrain and destroy io taskq */
	ddi_taskq_destroy(vdp->xs_iotaskq);

	/* tear down block-configure watch */
	mutex_enter(&vdp->xs_cbmutex);
	xvdi_remove_xb_watch_handlers(dip);
	mutex_exit(&vdp->xs_cbmutex);

errout2:
	/* remove kstats */
	kstat_delete(vdp->xs_kstats);

errout1:
	/* free up driver state */
	ddi_set_driver_private(dip, NULL);
	cv_destroy(&vdp->xs_iocv);
	cv_destroy(&vdp->xs_ionumcv);
	mutex_destroy(&vdp->xs_cbmutex);
	mutex_destroy(&vdp->xs_iomutex);
	ddi_soft_state_free(xdb_statep, instance);

	return (DDI_FAILURE);
}

/*ARGSUSED*/
static int
xdb_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
{
	int instance = ddi_get_instance(dip);
	xdb_t *vdp = XDB_INST2SOFTS(instance);

	switch (cmd) {
	case DDI_SUSPEND:
		return (DDI_FAILURE);
	case DDI_DETACH:
		break;
	default:
		return (DDI_FAILURE);
	}

	/* DDI_DETACH handling */

	/* refuse to detach if we're still in use by the frontend */
	mutex_enter(&vdp->xs_iomutex);
	if (vdp->xs_if_connected) {
		mutex_exit(&vdp->xs_iomutex);
		return (DDI_FAILURE);
	}
	vdp->xs_send_buf = B_FALSE;
	cv_broadcast(&vdp->xs_iocv);
	mutex_exit(&vdp->xs_iomutex);

	xvdi_remove_event_handler(dip, NULL);
	(void) xvdi_post_event(dip, XEN_HP_REMOVE);

	ddi_taskq_destroy(vdp->xs_iotaskq);

	mutex_enter(&vdp->xs_cbmutex);
	xvdi_remove_xb_watch_handlers(dip);
	mutex_exit(&vdp->xs_cbmutex);

	cv_destroy(&vdp->xs_iocv);
	cv_destroy(&vdp->xs_ionumcv);
	mutex_destroy(&vdp->xs_cbmutex);
	mutex_destroy(&vdp->xs_iomutex);
	kstat_delete(vdp->xs_kstats);
	ddi_set_driver_private(dip, NULL);
	ddi_soft_state_free(xdb_statep, instance);

	XDB_DBPRINT(XDB_DBG_INFO, (CE_NOTE, "xdb@%s: detached!",
	    ddi_get_name_addr(dip)));
	return (DDI_SUCCESS);
}

static struct dev_ops xdb_dev_ops = {
	DEVO_REV,	/* devo_rev */
	0,		/* devo_refcnt */
	ddi_getinfo_1to1, /* devo_getinfo */
	nulldev,	/* devo_identify */
	nulldev,	/* devo_probe */
	xdb_attach,	/* devo_attach */
	xdb_detach,	/* devo_detach */
	nodev,		/* devo_reset */
	NULL,		/* devo_cb_ops */
	NULL,		/* devo_bus_ops */
	NULL,		/* power */
	ddi_quiesce_not_needed, /* quiesce */
};

/*
 * Module linkage information for the kernel.
 */
static struct modldrv modldrv = {
	&mod_driverops,			/* Type of module. */
	"vbd backend driver",		/* Name of the module */
	&xdb_dev_ops			/* driver ops */
};

static struct modlinkage xdb_modlinkage = {
	MODREV_1,
	&modldrv,
	NULL
};

int
_init(void)
{
	int rv;

	if ((rv = ddi_soft_state_init((void **)&xdb_statep,
	    sizeof (xdb_t), 0)) == 0)
		if ((rv = mod_install(&xdb_modlinkage)) != 0)
			ddi_soft_state_fini((void **)&xdb_statep);
	return (rv);
}

int
_fini(void)
{
	int rv;

	if ((rv = mod_remove(&xdb_modlinkage)) != 0)
		return (rv);
	ddi_soft_state_fini((void **)&xdb_statep);
	return (rv);
}

int
_info(struct modinfo *modinfop)
{
	return (mod_info(&xdb_modlinkage, modinfop));
}

static int
xdb_get_request(xdb_t *vdp, blkif_request_t *req)
{
	void *src = xvdi_ring_get_request(vdp->xs_ring);

	if (src == NULL)
		return (0);

	switch (vdp->xs_blk_protocol) {
	case BLKIF_PROTOCOL_NATIVE:
		(void) memcpy(req, src, sizeof (*req));
		break;
	case BLKIF_PROTOCOL_X86_32:
		blkif_get_x86_32_req(req, src);
		break;
	case BLKIF_PROTOCOL_X86_64:
		blkif_get_x86_64_req(req, src);
		break;
	default:
		cmn_err(CE_PANIC, "xdb@%s: unrecognised protocol: %d",
		    ddi_get_name_addr(vdp->xs_dip),
		    vdp->xs_blk_protocol);
	}
	return (1);
}

static int
xdb_push_response(xdb_t *vdp, uint64_t id, uint8_t op, uint16_t status)
{
	ddi_acc_handle_t acchdl = vdp->xs_ring_hdl;
	blkif_response_t *rsp = xvdi_ring_get_response(vdp->xs_ring);
	blkif_x86_32_response_t *rsp_32 = (blkif_x86_32_response_t *)rsp;
	blkif_x86_64_response_t *rsp_64 = (blkif_x86_64_response_t *)rsp;

	ASSERT(rsp);

	switch (vdp->xs_blk_protocol) {
	case BLKIF_PROTOCOL_NATIVE:
		ddi_put64(acchdl, &rsp->id, id);
		ddi_put8(acchdl, &rsp->operation, op);
		ddi_put16(acchdl, (uint16_t *)&rsp->status,
		    status == 0 ? BLKIF_RSP_OKAY : BLKIF_RSP_ERROR);
		break;
	case BLKIF_PROTOCOL_X86_32:
		ddi_put64(acchdl, &rsp_32->id, id);
		ddi_put8(acchdl, &rsp_32->operation, op);
		ddi_put16(acchdl, (uint16_t *)&rsp_32->status,
		    status == 0 ? BLKIF_RSP_OKAY : BLKIF_RSP_ERROR);
		break;
	case BLKIF_PROTOCOL_X86_64:
		ddi_put64(acchdl, &rsp_64->id, id);
		ddi_put8(acchdl, &rsp_64->operation, op);
		ddi_put16(acchdl, (uint16_t *)&rsp_64->status,
		    status == 0 ? BLKIF_RSP_OKAY : BLKIF_RSP_ERROR);
		break;
	default:
		cmn_err(CE_PANIC, "xdb@%s: unrecognised protocol: %d",
		    ddi_get_name_addr(vdp->xs_dip),
		    vdp->xs_blk_protocol);
	}

	return (xvdi_ring_push_response(vdp->xs_ring));
}

static void
blkif_get_x86_32_req(blkif_request_t *dst, blkif_x86_32_request_t *src)
{
	int i, n = BLKIF_MAX_SEGMENTS_PER_REQUEST;
	dst->operation = src->operation;
	dst->nr_segments = src->nr_segments;
	dst->handle = src->handle;
	dst->id = src->id;
	dst->sector_number = src->sector_number;
	if (n > src->nr_segments)
		n = src->nr_segments;
	for (i = 0; i < n; i++)
		dst->seg[i] = src->seg[i];
}

static void
blkif_get_x86_64_req(blkif_request_t *dst, blkif_x86_64_request_t *src)
{
	int i, n = BLKIF_MAX_SEGMENTS_PER_REQUEST;
	dst->operation = src->operation;
	dst->nr_segments = src->nr_segments;
	dst->handle = src->handle;
	dst->id = src->id;
	dst->sector_number = src->sector_number;
	if (n > src->nr_segments)
		n = src->nr_segments;
	for (i = 0; i < n; i++)
		dst->seg[i] = src->seg[i];
}
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */


#ifndef _SYS_XDB_H
#define	_SYS_XDB_H

#ifdef __cplusplus
extern "C" {
#endif

#define	XDB_DBG_ALL	0xf
#define	XDB_DBG_IO	0x1
#define	XDB_DBG_INFO	0x2
#define	XDB_DBPRINT(lvl, fmt) { if (xdb_debug & lvl) cmn_err fmt; }

/*
 * Info of the exported blk device
 */
#define	XDB_DEV_RO	(1 << 0) /* backend and frontend are read-only */
#define	XDB_DEV_BE_LOFI	(1 << 1) /* backend device is a lofi device */
#define	XDB_DEV_BE_RMB	(1 << 2) /* backend device is removable */
#define	XDB_DEV_BE_CD	(1 << 3) /* backend device is cdrom */
#define	XDB_DEV_FE_CD	(1 << 4) /* frontend device is cdrom */

#define	XDB_IS_RO(vdp)		((vdp)->xs_type & XDB_DEV_RO)
#define	XDB_IS_BE_LOFI(vdp)	((vdp)->xs_type & XDB_DEV_BE_LOFI)
#define	XDB_IS_BE_RMB(vdp)	((vdp)->xs_type & XDB_DEV_BE_RMB)
#define	XDB_IS_BE_CD(vdp)	((vdp)->xs_type & XDB_DEV_BE_CD)
#define	XDB_IS_FE_CD(vdp)	((vdp)->xs_type & XDB_DEV_FE_CD)

/*
 * Other handy macrosx
 */
#define	XDB_MINOR2INST(m)	(int)(m)
#define	XDB_INST2MINOR(i)	(minor_t)(i)
#define	XDB_INST2SOFTS(instance)			\
	((xdb_t *)ddi_get_soft_state(xdb_statep, (instance)))
#define	XDB_MAX_IO_PAGES(v) ((v)->xs_nentry * BLKIF_MAX_SEGMENTS_PER_REQUEST)
/* get kva of a mapped-in page coresponding to (xreq-index, seg) pair */
#define	XDB_IOPAGE_VA(_pagebase, _xreqidx, _seg)	\
	((_pagebase) + ((_xreqidx)			\
	* BLKIF_MAX_SEGMENTS_PER_REQUEST		\
	+ (_seg)) * PAGESIZE)
#define	XDB_XREQ2BP(xreq) (&(xreq)->xr_buf)
#define	XDB_BP2XREQ(bp) \
	((xdb_request_t *)((char *)(bp) - offsetof(xdb_request_t, xr_buf)))

/* describe one blkif segment */
typedef struct xdb_seg {
	uint8_t fs; /* start sector # within this page (segment) */
	uint8_t ls; /* end sector # within this page (segment) */
} xdb_seg_t;

typedef struct xdb xdb_t;

/* one blkif_request_t matches one xdb_request_t */
typedef struct xdb_request {
	/* buf associated with this I/O request */
	buf_t		xr_buf;
	/* softstate instance associated with this I/O request */
	xdb_t		*xr_vdp;
	/* the next segment we're going to process */
	int		xr_curseg;
	/* index of this xdb_request_t in vdp->xs_req */
	int		xr_idx;
	/* next index for a statical linked list */
	int		xr_next;
	/* 'id' copied from blkif_request_t */
	uint64_t	xr_id;
	/* 'operation' copied from blkif_request_t */
	uint8_t		xr_op;
	/* how many pages(segments) in this I/O request */
	uint8_t		xr_buf_pages;
	/* all segments of this I/O request */
	xdb_seg_t	xr_segs[BLKIF_MAX_SEGMENTS_PER_REQUEST];
	/* all grant table handles used in this I/O request */
	grant_handle_t	xr_page_hdls[BLKIF_MAX_SEGMENTS_PER_REQUEST];
	struct page	xr_plist[BLKIF_MAX_SEGMENTS_PER_REQUEST];
	struct page	*xr_pplist[BLKIF_MAX_SEGMENTS_PER_REQUEST];
} xdb_request_t;

/* Soft state data structure for each backend vbd */
struct xdb {
	/* devinfo node pointer of this xdb */
	dev_info_t	*xs_dip;
	/* coresponding frontend domain id */
	domid_t		xs_peer;
	/* read-only, removable, cdrom? */
	uint32_t	xs_type;
	/* # of total sectors */
	uint64_t	xs_sectors;
	/* sector size if existed */
	uint_t		xs_sec_size;
	/* blkif I/O request ring buffer */
	xendev_ring_t	*xs_ring;
	/* handle to access the ring buffer */
	ddi_acc_handle_t xs_ring_hdl;
	ldi_ident_t	xs_ldi_li;
	ldi_handle_t	xs_ldi_hdl;
	/* base kva for mapped-in I/O page from frontend domain */
	caddr_t		xs_iopage_va;
	/* mutex lock for I/O related code path */
	kmutex_t	xs_iomutex;
	/*
	 * mutex lock for event handling related code path
	 * need to be grabbed before xs_iomutex
	 */
	kmutex_t	xs_cbmutex;
	/* # of on-going I/O buf in backend domain */
	uint_t		xs_ionum;
	/* task thread for pushing buf to underlying target driver */
	ddi_taskq_t	*xs_iotaskq;
	/* cv used in I/O code path, protected by xs_iomutex */
	kcondvar_t	xs_iocv;
	kcondvar_t	xs_ionumcv;
	/*
	 * head and tail of linked list for I/O bufs need to be pushed to
	 * underlying target driver
	 */
	buf_t		*xs_f_iobuf;
	buf_t		*xs_l_iobuf;
	/* head of free list of xdb_request_t */
	int		xs_free_req;
	/* pre-allocated xdb_request_t pool */
	xdb_request_t	*xs_req;
	kstat_t		*xs_kstats;
	uint64_t	xs_stat_req_reads;
	uint64_t	xs_stat_req_writes;
	uint64_t	xs_stat_req_barriers;
	uint64_t	xs_stat_req_flushes;
	enum blkif_protocol xs_blk_protocol;
	size_t		xs_nentry;
	size_t		xs_entrysize;

	/* Protected by xs_cbmutex */
	boolean_t	xs_hp_connected;	/* hot plug scripts have run */
	boolean_t	xs_fe_initialised;	/* frontend is initialized */
	char			*xs_lofi_path;
	char			*xs_params_path;
	struct xenbus_watch	*xs_watch_params;
	struct xenbus_watch	*xs_watch_media_req;
	ddi_taskq_t		*xs_watch_taskq;
	int			xs_watch_taskq_count;

	/* Protected by xs_cbmutex and xs_iomutex */
	boolean_t	xs_if_connected;	/* connected to frontend */

	/* Protected by xs_iomutex */
	boolean_t	xs_send_buf;

#ifdef DEBUG
	uint64_t *page_addrs; /* for debug aid */
#endif /* DEBUG */
};

#ifdef __cplusplus
}
#endif

#endif	/* _SYS_XDB_H */
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

/*
 * Copyright (c) 2014, 2017 by Delphix. All rights reserved.
 * Copyright 2017 Nexenta Systems, Inc.
 */

/*
 * xdf.c - Xen Virtual Block Device Driver
 * TODO:
 *	- support alternate block size (currently only DEV_BSIZE supported)
 *	- revalidate geometry for removable devices
 *
 * This driver exports disk device nodes, accepts IO requests from those
 * nodes, and services those requests by talking to a backend device
 * in another domain.
 *
 * Communication with the backend device is done via a ringbuffer (which is
 * managed via xvdi interfaces) and dma memory (which is managed via ddi
 * interfaces).
 *
 * Communication with the backend device is dependant upon establishing a
 * connection to the backend device.  This connection process involves
 * reading device configuration information from xenbus and publishing
 * some frontend runtime configuration parameters via the xenbus (for
 * consumption by the backend).  Once we've published runtime configuration
 * information via the xenbus, the backend device can enter the connected
 * state and we'll enter the XD_CONNECTED state.  But before we can allow
 * random IO to begin, we need to do IO to the backend device to determine
 * the device label and if flush operations are supported.  Once this is
 * done we enter the XD_READY state and can process any IO operations.
 *
 * We receive notifications of xenbus state changes for the backend device
 * (aka, the "other end") via the xdf_oe_change() callback.  This callback
 * is single threaded, meaning that we can't receive new notification of
 * other end state changes while we're processing an outstanding
 * notification of an other end state change.  There for we can't do any
 * blocking operations from the xdf_oe_change() callback.  This is why we
 * have a seperate taskq (xdf_ready_tq) which exists to do the necessary
 * IO to get us from the XD_CONNECTED to the XD_READY state.  All IO
 * generated by the xdf_ready_tq thread (xdf_ready_tq_thread) will go
 * throught xdf_lb_rdwr(), which is a synchronous IO interface.  IOs
 * generated by the xdf_ready_tq_thread thread have priority over all
 * other IO requests.
 *
 * We also communicate with the backend device via the xenbus "media-req"
 * (XBP_MEDIA_REQ) property.  For more information on this see the
 * comments in blkif.h.
 */

#include <io/xdf.h>

#include <sys/conf.h>
#include <sys/dkio.h>
#include <sys/promif.h>
#include <sys/sysmacros.h>
#include <sys/kstat.h>
#include <sys/mach_mmu.h>
#ifdef XPV_HVM_DRIVER
#include <sys/xpv_support.h>
#else /* !XPV_HVM_DRIVER */
#include <sys/evtchn_impl.h>
#endif /* !XPV_HVM_DRIVER */
#include <sys/sunndi.h>
#include <public/io/xenbus.h>
#include <xen/sys/xenbus_impl.h>
#include <sys/scsi/generic/inquiry.h>
#include <xen/io/blkif_impl.h>
#include <sys/fdio.h>
#include <sys/cdio.h>

/*
 * DEBUG_EVAL can be used to include debug only statements without
 * having to use '#ifdef DEBUG' statements
 */
#ifdef DEBUG
#define	DEBUG_EVAL(x)	(x)
#else /* !DEBUG */
#define	DEBUG_EVAL(x)
#endif /* !DEBUG */

#define	XDF_DRAIN_MSEC_DELAY		(50*1000)	/* 00.05 sec */
#define	XDF_DRAIN_RETRY_COUNT		200		/* 10.00 sec */
#define	XDF_STATE_TIMEOUT		(30*1000*1000)	/* 30.00 sec */

#define	INVALID_DOMID	((domid_t)-1)
#define	FLUSH_DISKCACHE	0x1
#define	WRITE_BARRIER	0x2
#define	DEFAULT_FLUSH_BLOCK	156 /* block to write to cause a cache flush */
#define	USE_WRITE_BARRIER(vdp)						\
	((vdp)->xdf_feature_barrier && !(vdp)->xdf_flush_supported)
#define	USE_FLUSH_DISKCACHE(vdp)					\
	((vdp)->xdf_feature_barrier && (vdp)->xdf_flush_supported)
#define	IS_WRITE_BARRIER(vdp, bp)					\
	(!IS_READ(bp) && USE_WRITE_BARRIER(vdp) &&			\
	((bp)->b_un.b_addr == (vdp)->xdf_cache_flush_block))
#define	IS_FLUSH_DISKCACHE(bp)						\
	(!IS_READ(bp) && USE_FLUSH_DISKCACHE(vdp) && ((bp)->b_bcount == 0))

#define	VREQ_DONE(vreq)							\
	VOID2BOOLEAN(((vreq)->v_status == VREQ_DMAWIN_DONE) &&		\
	    (((vreq)->v_flush_diskcache == FLUSH_DISKCACHE) ||		\
	    (((vreq)->v_dmaw + 1) == (vreq)->v_ndmaws)))

#define	BP_VREQ(bp)		((v_req_t *)((bp)->av_back))
#define	BP_VREQ_SET(bp, vreq)	(((bp)->av_back = (buf_t *)(vreq)))

extern int		do_polled_io;

/* run-time tunables that we don't want the compiler to optimize away */
volatile int		xdf_debug = 0;
volatile boolean_t	xdf_barrier_flush_disable = B_FALSE;

/* per module globals */
major_t			xdf_major;
static void		*xdf_ssp;
static kmem_cache_t	*xdf_vreq_cache;
static kmem_cache_t	*xdf_gs_cache;
static int		xdf_maxphys = XB_MAXPHYS;
static diskaddr_t	xdf_flush_block = DEFAULT_FLUSH_BLOCK;
static int		xdf_fbrewrites;	/* flush block re-write count */

/* misc public functions */
int xdf_lb_rdwr(dev_info_t *, uchar_t, void *, diskaddr_t, size_t, void *);
int xdf_lb_getinfo(dev_info_t *, int, void *, void *);

/*  misc private functions */
static void xdf_io_start(xdf_t *);
static void xdf_devid_setup(xdf_t *);

/* callbacks from commmon label */
static cmlb_tg_ops_t xdf_lb_ops = {
	TG_DK_OPS_VERSION_1,
	xdf_lb_rdwr,
	xdf_lb_getinfo
};

/*
 * I/O buffer DMA attributes
 * Make sure: one DMA window contains BLKIF_MAX_SEGMENTS_PER_REQUEST at most
 */
static ddi_dma_attr_t xb_dma_attr = {
	DMA_ATTR_V0,
	(uint64_t)0,			/* lowest address */
	(uint64_t)0xffffffffffffffff,	/* highest usable address */
	(uint64_t)0xffffff,		/* DMA counter limit max */
	(uint64_t)XB_BSIZE,		/* alignment in bytes */
	XB_BSIZE - 1,			/* bitmap of burst sizes */
	XB_BSIZE,			/* min transfer */
	(uint64_t)XB_MAX_XFER,		/* maximum transfer */
	(uint64_t)PAGEOFFSET,		/* 1 page segment length  */
	BLKIF_MAX_SEGMENTS_PER_REQUEST,	/* maximum number of segments */
	XB_BSIZE,			/* granularity */
	0,				/* flags (reserved) */
};

static ddi_device_acc_attr_t xc_acc_attr = {
	DDI_DEVICE_ATTR_V0,
	DDI_NEVERSWAP_ACC,
	DDI_STRICTORDER_ACC
};

static void
xdf_timeout_handler(void *arg)
{
	xdf_t *vdp = arg;

	mutex_enter(&vdp->xdf_dev_lk);
	vdp->xdf_timeout_id = 0;
	mutex_exit(&vdp->xdf_dev_lk);

	/* new timeout thread could be re-scheduled */
	xdf_io_start(vdp);
}

/*
 * callback func when DMA/GTE resources is available
 *
 * Note: we only register one callback function to grant table subsystem
 * since we only have one 'struct gnttab_free_callback' in xdf_t.
 */
static void
xdf_gncallback(void *arg)
{
	xdf_t *vdp = arg;
	ASSERT(vdp != NULL);

	DPRINTF(DMA_DBG, ("xdf@%s: DMA callback started\n",
	    vdp->xdf_addr));

	ddi_trigger_softintr(vdp->xdf_softintr_id);
}

static int
xdf_dmacallback(caddr_t arg)
{
	xdf_gncallback(arg);
	return (DDI_DMA_CALLBACK_DONE);
}

static ge_slot_t *
gs_get(xdf_t *vdp, int isread)
{
	grant_ref_t gh;
	ge_slot_t *gs;

	/* try to alloc GTEs needed in this slot, first */
	if (gnttab_alloc_grant_references(
	    BLKIF_MAX_SEGMENTS_PER_REQUEST, &gh) == -1) {
		if (vdp->xdf_gnt_callback.next == NULL) {
			SETDMACBON(vdp);
			gnttab_request_free_callback(
			    &vdp->xdf_gnt_callback,
			    xdf_gncallback,
			    (void *)vdp,
			    BLKIF_MAX_SEGMENTS_PER_REQUEST);
		}
		return (NULL);
	}

	gs = kmem_cache_alloc(xdf_gs_cache, KM_NOSLEEP);
	if (gs == NULL) {
		gnttab_free_grant_references(gh);
		if (vdp->xdf_timeout_id == 0)
			/* restart I/O after one second */
			vdp->xdf_timeout_id =
			    timeout(xdf_timeout_handler, vdp, hz);
		return (NULL);
	}

	/* init gs_slot */
	gs->gs_oeid = vdp->xdf_peer;
	gs->gs_isread = isread;
	gs->gs_ghead = gh;
	gs->gs_ngrefs = 0;

	return (gs);
}

static void
gs_free(ge_slot_t *gs)
{
	int		i;

	/* release all grant table entry resources used in this slot */
	for (i = 0; i < gs->gs_ngrefs; i++)
		gnttab_end_foreign_access(gs->gs_ge[i], !gs->gs_isread, 0);
	gnttab_free_grant_references(gs->gs_ghead);
	list_remove(&gs->gs_vreq->v_gs, gs);
	kmem_cache_free(xdf_gs_cache, gs);
}

static grant_ref_t
gs_grant(ge_slot_t *gs, mfn_t mfn)
{
	grant_ref_t gr = gnttab_claim_grant_reference(&gs->gs_ghead);

	ASSERT(gr != -1);
	ASSERT(gs->gs_ngrefs < BLKIF_MAX_SEGMENTS_PER_REQUEST);
	gs->gs_ge[gs->gs_ngrefs++] = gr;
	gnttab_grant_foreign_access_ref(gr, gs->gs_oeid, mfn, !gs->gs_isread);

	return (gr);
}

/*
 * Alloc a vreq for this bp
 * bp->av_back contains the pointer to the vreq upon return
 */
static v_req_t *
vreq_get(xdf_t *vdp, buf_t *bp)
{
	v_req_t *vreq = NULL;

	ASSERT(BP_VREQ(bp) == NULL);

	vreq = kmem_cache_alloc(xdf_vreq_cache, KM_NOSLEEP);
	if (vreq == NULL) {
		if (vdp->xdf_timeout_id == 0)
			/* restart I/O after one second */
			vdp->xdf_timeout_id =
			    timeout(xdf_timeout_handler, vdp, hz);
		return (NULL);
	}
	bzero(vreq, sizeof (v_req_t));
	list_create(&vreq->v_gs, sizeof (ge_slot_t),
	    offsetof(ge_slot_t, gs_vreq_link));
	vreq->v_buf = bp;
	vreq->v_status = VREQ_INIT;
	vreq->v_runq = B_FALSE;
	BP_VREQ_SET(bp, vreq);
	/* init of other fields in vreq is up to the caller */

	list_insert_head(&vdp->xdf_vreq_act, (void *)vreq);

	return (vreq);
}

static void
vreq_free(xdf_t *vdp, v_req_t *vreq)
{
	buf_t	*bp = vreq->v_buf;

	ASSERT(MUTEX_HELD(&vdp->xdf_dev_lk));
	ASSERT(BP_VREQ(bp) == vreq);

	list_remove(&vdp->xdf_vreq_act, vreq);

	if (vreq->v_flush_diskcache == FLUSH_DISKCACHE)
		goto done;

	switch (vreq->v_status) {
	case VREQ_DMAWIN_DONE:
	case VREQ_GS_ALLOCED:
	case VREQ_DMABUF_BOUND:
		(void) ddi_dma_unbind_handle(vreq->v_dmahdl);
		/*FALLTHRU*/
	case VREQ_DMAMEM_ALLOCED:
		if (!ALIGNED_XFER(bp)) {
			ASSERT(vreq->v_abuf != NULL);
			if (!IS_ERROR(bp) && IS_READ(bp))
				bcopy(vreq->v_abuf, bp->b_un.b_addr,
				    bp->b_bcount);
			ddi_dma_mem_free(&vreq->v_align);
		}
		/*FALLTHRU*/
	case VREQ_MEMDMAHDL_ALLOCED:
		if (!ALIGNED_XFER(bp))
			ddi_dma_free_handle(&vreq->v_memdmahdl);
		/*FALLTHRU*/
	case VREQ_DMAHDL_ALLOCED:
		ddi_dma_free_handle(&vreq->v_dmahdl);
		break;
	default:
		break;
	}
done:
	ASSERT(!vreq->v_runq);
	list_destroy(&vreq->v_gs);
	kmem_cache_free(xdf_vreq_cache, vreq);
}

/*
 * Snarf new data if our flush block was re-written
 */
static void
check_fbwrite(xdf_t *vdp, buf_t *bp, daddr_t blkno)
{
	int nblks;
	boolean_t mapin;

	if (IS_WRITE_BARRIER(vdp, bp))
		return; /* write was a flush write */

	mapin = B_FALSE;
	nblks = bp->b_bcount >> DEV_BSHIFT;
	if (xdf_flush_block >= blkno && xdf_flush_block < (blkno + nblks)) {
		xdf_fbrewrites++;
		if (bp->b_flags & (B_PAGEIO | B_PHYS)) {
			mapin = B_TRUE;
			bp_mapin(bp);
		}
		bcopy(bp->b_un.b_addr +
		    ((xdf_flush_block - blkno) << DEV_BSHIFT),
		    vdp->xdf_cache_flush_block, DEV_BSIZE);
		if (mapin)
			bp_mapout(bp);
	}
}

/*
 * Initalize the DMA and grant table resources for the buf
 */
static int
vreq_setup(xdf_t *vdp, v_req_t *vreq)
{
	int rc;
	ddi_dma_attr_t dmaattr;
	uint_t ndcs, ndws;
	ddi_dma_handle_t dh;
	ddi_dma_handle_t mdh;
	ddi_dma_cookie_t dc;
	ddi_acc_handle_t abh;
	caddr_t	aba;
	ge_slot_t *gs;
	size_t bufsz;
	off_t off;
	size_t sz;
	buf_t *bp = vreq->v_buf;
	int dma_flags = (IS_READ(bp) ? DDI_DMA_READ : DDI_DMA_WRITE) |
	    DDI_DMA_STREAMING | DDI_DMA_PARTIAL;

	switch (vreq->v_status) {
	case VREQ_INIT:
		if (IS_FLUSH_DISKCACHE(bp)) {
			if ((gs = gs_get(vdp, IS_READ(bp))) == NULL) {
				DPRINTF(DMA_DBG, ("xdf@%s: "
				    "get ge_slotfailed\n", vdp->xdf_addr));
				return (DDI_FAILURE);
			}
			vreq->v_blkno = 0;
			vreq->v_nslots = 1;
			vreq->v_flush_diskcache = FLUSH_DISKCACHE;
			vreq->v_status = VREQ_GS_ALLOCED;
			gs->gs_vreq = vreq;
			list_insert_head(&vreq->v_gs, gs);
			return (DDI_SUCCESS);
		}

		if (IS_WRITE_BARRIER(vdp, bp))
			vreq->v_flush_diskcache = WRITE_BARRIER;
		vreq->v_blkno = bp->b_blkno +
		    (diskaddr_t)(uintptr_t)bp->b_private;
		/* See if we wrote new data to our flush block */
		if (!IS_READ(bp) && USE_WRITE_BARRIER(vdp))
			check_fbwrite(vdp, bp, vreq->v_blkno);
		vreq->v_status = VREQ_INIT_DONE;
		/*FALLTHRU*/

	case VREQ_INIT_DONE:
		/*
		 * alloc DMA handle
		 */
		rc = ddi_dma_alloc_handle(vdp->xdf_dip, &xb_dma_attr,
		    xdf_dmacallback, (caddr_t)vdp, &dh);
		if (rc != DDI_SUCCESS) {
			SETDMACBON(vdp);
			DPRINTF(DMA_DBG, ("xdf@%s: DMA handle alloc failed\n",
			    vdp->xdf_addr));
			return (DDI_FAILURE);
		}

		vreq->v_dmahdl = dh;
		vreq->v_status = VREQ_DMAHDL_ALLOCED;
		/*FALLTHRU*/

	case VREQ_DMAHDL_ALLOCED:
		/*
		 * alloc dma handle for 512-byte aligned buf
		 */
		if (!ALIGNED_XFER(bp)) {
			/*
			 * XXPV: we need to temporarily enlarge the seg
			 * boundary and s/g length to work round CR6381968
			 */
			dmaattr = xb_dma_attr;
			dmaattr.dma_attr_seg = (uint64_t)-1;
			dmaattr.dma_attr_sgllen = INT_MAX;
			rc = ddi_dma_alloc_handle(vdp->xdf_dip, &dmaattr,
			    xdf_dmacallback, (caddr_t)vdp, &mdh);
			if (rc != DDI_SUCCESS) {
				SETDMACBON(vdp);
				DPRINTF(DMA_DBG, ("xdf@%s: "
				    "unaligned buf DMAhandle alloc failed\n",
				    vdp->xdf_addr));
				return (DDI_FAILURE);
			}
			vreq->v_memdmahdl = mdh;
			vreq->v_status = VREQ_MEMDMAHDL_ALLOCED;
		}
		/*FALLTHRU*/

	case VREQ_MEMDMAHDL_ALLOCED:
		/*
		 * alloc 512-byte aligned buf
		 */
		if (!ALIGNED_XFER(bp)) {
			if (bp->b_flags & (B_PAGEIO | B_PHYS))
				bp_mapin(bp);
			rc = ddi_dma_mem_alloc(vreq->v_memdmahdl,
			    roundup(bp->b_bcount, XB_BSIZE), &xc_acc_attr,
			    DDI_DMA_STREAMING, xdf_dmacallback, (caddr_t)vdp,
			    &aba, &bufsz, &abh);
			if (rc != DDI_SUCCESS) {
				SETDMACBON(vdp);
				DPRINTF(DMA_DBG, ("xdf@%s: "
				    "DMA mem allocation failed\n",
				    vdp->xdf_addr));
				return (DDI_FAILURE);
			}

			vreq->v_abuf = aba;
			vreq->v_align = abh;
			vreq->v_status = VREQ_DMAMEM_ALLOCED;

			ASSERT(bufsz >= bp->b_bcount);
			if (!IS_READ(bp))
				bcopy(bp->b_un.b_addr, vreq->v_abuf,
				    bp->b_bcount);
		}
		/*FALLTHRU*/

	case VREQ_DMAMEM_ALLOCED:
		/*
		 * dma bind
		 */
		if (ALIGNED_XFER(bp)) {
			rc = ddi_dma_buf_bind_handle(vreq->v_dmahdl, bp,
			    dma_flags, xdf_dmacallback, (caddr_t)vdp,
			    &dc, &ndcs);
		} else {
			rc = ddi_dma_addr_bind_handle(vreq->v_dmahdl,
			    NULL, vreq->v_abuf, bp->b_bcount, dma_flags,
			    xdf_dmacallback, (caddr_t)vdp, &dc, &ndcs);
		}
		if (rc == DDI_DMA_MAPPED || rc == DDI_DMA_PARTIAL_MAP) {
			/* get num of dma windows */
			if (rc == DDI_DMA_PARTIAL_MAP) {
				rc = ddi_dma_numwin(vreq->v_dmahdl, &ndws);
				ASSERT(rc == DDI_SUCCESS);
			} else {
				ndws = 1;
			}
		} else {
			SETDMACBON(vdp);
			DPRINTF(DMA_DBG, ("xdf@%s: DMA bind failed\n",
			    vdp->xdf_addr));
			return (DDI_FAILURE);
		}

		vreq->v_dmac = dc;
		vreq->v_dmaw = 0;
		vreq->v_ndmacs = ndcs;
		vreq->v_ndmaws = ndws;
		vreq->v_nslots = ndws;
		vreq->v_status = VREQ_DMABUF_BOUND;
		/*FALLTHRU*/

	case VREQ_DMABUF_BOUND:
		/*
		 * get ge_slot, callback is set upon failure from gs_get(),
		 * if not set previously
		 */
		if ((gs = gs_get(vdp, IS_READ(bp))) == NULL) {
			DPRINTF(DMA_DBG, ("xdf@%s: get ge_slot failed\n",
			    vdp->xdf_addr));
			return (DDI_FAILURE);
		}

		vreq->v_status = VREQ_GS_ALLOCED;
		gs->gs_vreq = vreq;
		list_insert_head(&vreq->v_gs, gs);
		break;

	case VREQ_GS_ALLOCED:
		/* nothing need to be done */
		break;

	case VREQ_DMAWIN_DONE:
		/*
		 * move to the next dma window
		 */
		ASSERT((vreq->v_dmaw + 1) < vreq->v_ndmaws);

		/* get a ge_slot for this DMA window */
		if ((gs = gs_get(vdp, IS_READ(bp))) == NULL) {
			DPRINTF(DMA_DBG, ("xdf@%s: get ge_slot failed\n",
			    vdp->xdf_addr));
			return (DDI_FAILURE);
		}

		vreq->v_dmaw++;
		VERIFY(ddi_dma_getwin(vreq->v_dmahdl, vreq->v_dmaw, &off, &sz,
		    &vreq->v_dmac, &vreq->v_ndmacs) == DDI_SUCCESS);
		vreq->v_status = VREQ_GS_ALLOCED;
		gs->gs_vreq = vreq;
		list_insert_head(&vreq->v_gs, gs);
		break;

	default:
		return (DDI_FAILURE);
	}

	return (DDI_SUCCESS);
}

static int
xdf_cmlb_attach(xdf_t *vdp)
{
	dev_info_t	*dip = vdp->xdf_dip;

	return (cmlb_attach(dip, &xdf_lb_ops,
	    XD_IS_CD(vdp) ? DTYPE_RODIRECT : DTYPE_DIRECT,
	    XD_IS_RM(vdp), B_TRUE,
	    XD_IS_CD(vdp) ? DDI_NT_CD_XVMD : DDI_NT_BLOCK_XVMD,
	    0, vdp->xdf_vd_lbl, NULL));
}

static void
xdf_io_err(buf_t *bp, int err, size_t resid)
{
	bioerror(bp, err);
	if (resid == 0)
		bp->b_resid = bp->b_bcount;
	biodone(bp);
}

static void
xdf_kstat_enter(xdf_t *vdp, buf_t *bp)
{
	v_req_t *vreq = BP_VREQ(bp);

	ASSERT(MUTEX_HELD(&vdp->xdf_dev_lk));

	if (vdp->xdf_xdev_iostat == NULL)
		return;
	if ((vreq != NULL) && vreq->v_runq) {
		kstat_runq_enter(KSTAT_IO_PTR(vdp->xdf_xdev_iostat));
	} else {
		kstat_waitq_enter(KSTAT_IO_PTR(vdp->xdf_xdev_iostat));
	}
}

static void
xdf_kstat_exit(xdf_t *vdp, buf_t *bp)
{
	v_req_t *vreq = BP_VREQ(bp);

	ASSERT(MUTEX_HELD(&vdp->xdf_dev_lk));

	if (vdp->xdf_xdev_iostat == NULL)
		return;

	if ((vreq != NULL) && vreq->v_runq) {
		kstat_runq_exit(KSTAT_IO_PTR(vdp->xdf_xdev_iostat));
	} else {
		kstat_waitq_exit(KSTAT_IO_PTR(vdp->xdf_xdev_iostat));
	}

	if (bp->b_flags & B_READ) {
		KSTAT_IO_PTR(vdp->xdf_xdev_iostat)->reads++;
		KSTAT_IO_PTR(vdp->xdf_xdev_iostat)->nread += bp->b_bcount;
	} else if (bp->b_flags & B_WRITE) {
		KSTAT_IO_PTR(vdp->xdf_xdev_iostat)->writes++;
		KSTAT_IO_PTR(vdp->xdf_xdev_iostat)->nwritten += bp->b_bcount;
	}
}

static void
xdf_kstat_waitq_to_runq(xdf_t *vdp, buf_t *bp)
{
	v_req_t *vreq = BP_VREQ(bp);

	ASSERT(MUTEX_HELD(&vdp->xdf_dev_lk));
	ASSERT(!vreq->v_runq);

	vreq->v_runq = B_TRUE;
	if (vdp->xdf_xdev_iostat == NULL)
		return;
	kstat_waitq_to_runq(KSTAT_IO_PTR(vdp->xdf_xdev_iostat));
}

static void
xdf_kstat_runq_to_waitq(xdf_t *vdp, buf_t *bp)
{
	v_req_t *vreq = BP_VREQ(bp);

	ASSERT(MUTEX_HELD(&vdp->xdf_dev_lk));
	ASSERT(vreq->v_runq);

	vreq->v_runq = B_FALSE;
	if (vdp->xdf_xdev_iostat == NULL)
		return;
	kstat_runq_back_to_waitq(KSTAT_IO_PTR(vdp->xdf_xdev_iostat));
}

int
xdf_kstat_create(dev_info_t *dip)
{
	xdf_t		*vdp = (xdf_t *)ddi_get_driver_private(dip);
	kstat_t		*kstat;
	buf_t		*bp;

	if ((kstat = kstat_create("xdf", ddi_get_instance(dip), NULL, "disk",
	    KSTAT_TYPE_IO, 1, KSTAT_FLAG_PERSISTENT)) == NULL)
		return (-1);

	/* See comment about locking in xdf_kstat_delete(). */
	mutex_enter(&vdp->xdf_iostat_lk);
	mutex_enter(&vdp->xdf_dev_lk);

	/* only one kstat can exist at a time */
	if (vdp->xdf_xdev_iostat != NULL) {
		mutex_exit(&vdp->xdf_dev_lk);
		mutex_exit(&vdp->xdf_iostat_lk);
		kstat_delete(kstat);
		return (-1);
	}

	vdp->xdf_xdev_iostat = kstat;
	vdp->xdf_xdev_iostat->ks_lock = &vdp->xdf_dev_lk;
	kstat_install(vdp->xdf_xdev_iostat);

	/*
	 * Now that we've created a kstat, we need to update the waitq and
	 * runq counts for the kstat to reflect our current state.
	 *
	 * For a buf_t structure to be on the runq, it must have a ring
	 * buffer slot associated with it.  To get a ring buffer slot the
	 * buf must first have a v_req_t and a ge_slot_t associated with it.
	 * Then when it is granted a ring buffer slot, v_runq will be set to
	 * true.
	 *
	 * For a buf_t structure to be on the waitq, it must not be on the
	 * runq.  So to find all the buf_t's that should be on waitq, we
	 * walk the active buf list and add any buf_t's which aren't on the
	 * runq to the waitq.
	 */
	bp = vdp->xdf_f_act;
	while (bp != NULL) {
		xdf_kstat_enter(vdp, bp);
		bp = bp->av_forw;
	}
	if (vdp->xdf_ready_tq_bp != NULL)
		xdf_kstat_enter(vdp, vdp->xdf_ready_tq_bp);

	mutex_exit(&vdp->xdf_dev_lk);
	mutex_exit(&vdp->xdf_iostat_lk);
	return (0);
}

void
xdf_kstat_delete(dev_info_t *dip)
{
	xdf_t		*vdp = (xdf_t *)ddi_get_driver_private(dip);
	kstat_t		*kstat;
	buf_t		*bp;

	/*
	 * The locking order here is xdf_iostat_lk and then xdf_dev_lk.
	 * xdf_dev_lk is used to protect the xdf_xdev_iostat pointer
	 * and the contents of the our kstat.  xdf_iostat_lk is used
	 * to protect the allocation and freeing of the actual kstat.
	 * xdf_dev_lk can't be used for this purpose because kstat
	 * readers use it to access the contents of the kstat and
	 * hence it can't be held when calling kstat_delete().
	 */
	mutex_enter(&vdp->xdf_iostat_lk);
	mutex_enter(&vdp->xdf_dev_lk);

	if (vdp->xdf_xdev_iostat == NULL) {
		mutex_exit(&vdp->xdf_dev_lk);
		mutex_exit(&vdp->xdf_iostat_lk);
		return;
	}

	/*
	 * We're about to destroy the kstat structures, so it isn't really
	 * necessary to update the runq and waitq counts.  But, since this
	 * isn't a hot code path we can afford to be a little pedantic and
	 * go ahead and decrement the runq and waitq kstat counters to zero
	 * before free'ing them.  This helps us ensure that we've gotten all
	 * our accounting correct.
	 *
	 * For an explanation of how we determine which buffers go on the
	 * runq vs which go on the waitq, see the comments in
	 * xdf_kstat_create().
	 */
	bp = vdp->xdf_f_act;
	while (bp != NULL) {
		xdf_kstat_exit(vdp, bp);
		bp = bp->av_forw;
	}
	if (vdp->xdf_ready_tq_bp != NULL)
		xdf_kstat_exit(vdp, vdp->xdf_ready_tq_bp);

	kstat = vdp->xdf_xdev_iostat;
	vdp->xdf_xdev_iostat = NULL;
	mutex_exit(&vdp->xdf_dev_lk);
	kstat_delete(kstat);
	mutex_exit(&vdp->xdf_iostat_lk);
}

/*
 * Add an IO requests onto the active queue.
 *
 * We have to detect IOs generated by xdf_ready_tq_thread.  These IOs
 * are used to establish a connection to the backend, so they receive
 * priority over all other IOs.  Since xdf_ready_tq_thread only does
 * synchronous IO, there can only be one xdf_ready_tq_thread request at any
 * given time and we record the buf associated with that request in
 * xdf_ready_tq_bp.
 */
static void
xdf_bp_push(xdf_t *vdp, buf_t *bp)
{
	ASSERT(MUTEX_HELD(&vdp->xdf_dev_lk));
	ASSERT(bp->av_forw == NULL);

	xdf_kstat_enter(vdp, bp);

	if (curthread == vdp->xdf_ready_tq_thread) {
		/* new IO requests from the ready thread */
		ASSERT(vdp->xdf_ready_tq_bp == NULL);
		vdp->xdf_ready_tq_bp = bp;
		return;
	}

	/* this is normal IO request */
	ASSERT(bp != vdp->xdf_ready_tq_bp);

	if (vdp->xdf_f_act == NULL) {
		/* this is only only IO on the active queue */
		ASSERT(vdp->xdf_l_act == NULL);
		ASSERT(vdp->xdf_i_act == NULL);
		vdp->xdf_f_act = vdp->xdf_l_act = vdp->xdf_i_act = bp;
		return;
	}

	/* add this IO to the tail of the active queue */
	vdp->xdf_l_act->av_forw = bp;
	vdp->xdf_l_act = bp;
	if (vdp->xdf_i_act == NULL)
		vdp->xdf_i_act = bp;
}

static void
xdf_bp_pop(xdf_t *vdp, buf_t *bp)
{
	buf_t	*bp_iter;

	ASSERT(MUTEX_HELD(&vdp->xdf_dev_lk));
	ASSERT(VREQ_DONE(BP_VREQ(bp)));

	if (vdp->xdf_ready_tq_bp == bp) {
		/* we're done with a ready thread IO request */
		ASSERT(bp->av_forw == NULL);
		vdp->xdf_ready_tq_bp = NULL;
		return;
	}

	/* we're done with a normal IO request */
	ASSERT((bp->av_forw != NULL) || (bp == vdp->xdf_l_act));
	ASSERT((bp->av_forw == NULL) || (bp != vdp->xdf_l_act));
	ASSERT(VREQ_DONE(BP_VREQ(vdp->xdf_f_act)));
	ASSERT(vdp->xdf_f_act != vdp->xdf_i_act);

	if (bp == vdp->xdf_f_act) {
		/* This IO was at the head of our active queue. */
		vdp->xdf_f_act = bp->av_forw;
		if (bp == vdp->xdf_l_act)
			vdp->xdf_l_act = NULL;
	} else {
		/* There IO finished before some other pending IOs. */
		bp_iter = vdp->xdf_f_act;
		while (bp != bp_iter->av_forw) {
			bp_iter = bp_iter->av_forw;
			ASSERT(VREQ_DONE(BP_VREQ(bp_iter)));
			ASSERT(bp_iter != vdp->xdf_i_act);
		}
		bp_iter->av_forw = bp->av_forw;
		if (bp == vdp->xdf_l_act)
			vdp->xdf_l_act = bp_iter;
	}
	bp->av_forw = NULL;
}

static buf_t *
xdf_bp_next(xdf_t *vdp)
{
	v_req_t	*vreq;
	buf_t	*bp;

	if (vdp->xdf_state == XD_CONNECTED) {
		/*
		 * If we're in the XD_CONNECTED state, we only service IOs
		 * from the xdf_ready_tq_thread thread.
		 */
		if ((bp = vdp->xdf_ready_tq_bp) == NULL)
			return (NULL);
		if (((vreq = BP_VREQ(bp)) == NULL) || (!VREQ_DONE(vreq)))
			return (bp);
		return (NULL);
	}

	/* if we're not in the XD_CONNECTED or XD_READY state we can't do IO */
	if (vdp->xdf_state != XD_READY)
		return (NULL);

	ASSERT(vdp->xdf_ready_tq_bp == NULL);
	for (;;) {
		if ((bp = vdp->xdf_i_act) == NULL)
			return (NULL);
		if (((vreq = BP_VREQ(bp)) == NULL) || (!VREQ_DONE(vreq)))
			return (bp);

		/* advance the active buf index pointer */
		vdp->xdf_i_act = bp->av_forw;
	}
}

static void
xdf_io_fini(xdf_t *vdp, uint64_t id, int bioerr)
{
	ge_slot_t	*gs = (ge_slot_t *)(uintptr_t)id;
	v_req_t		*vreq = gs->gs_vreq;
	buf_t		*bp = vreq->v_buf;

	ASSERT(MUTEX_HELD(&vdp->xdf_dev_lk));
	ASSERT(BP_VREQ(bp) == vreq);

	gs_free(gs);

	if (bioerr != 0)
		bioerror(bp, bioerr);
	ASSERT(vreq->v_nslots > 0);
	if (--vreq->v_nslots > 0)
		return;

	/* remove this IO from our active queue */
	xdf_bp_pop(vdp, bp);

	ASSERT(vreq->v_runq);
	xdf_kstat_exit(vdp, bp);
	vreq->v_runq = B_FALSE;
	vreq_free(vdp, vreq);

	if (IS_ERROR(bp)) {
		xdf_io_err(bp, geterror(bp), 0);
	} else if (bp->b_resid != 0) {
		/* Partial transfers are an error */
		xdf_io_err(bp, EIO, bp->b_resid);
	} else {
		biodone(bp);
	}
}

/*
 * xdf interrupt handler
 */
static uint_t
xdf_intr_locked(xdf_t *vdp)
{
	xendev_ring_t *xbr;
	blkif_response_t *resp;
	int bioerr;
	uint64_t id;
	uint8_t op;
	uint16_t status;
	ddi_acc_handle_t acchdl;

	ASSERT(MUTEX_HELD(&vdp->xdf_dev_lk));

	if ((xbr = vdp->xdf_xb_ring) == NULL)
		return (DDI_INTR_UNCLAIMED);

	acchdl = vdp->xdf_xb_ring_hdl;

	/*
	 * complete all requests which have a response
	 */
	while (resp = xvdi_ring_get_response(xbr)) {
		id = ddi_get64(acchdl, &resp->id);
		op = ddi_get8(acchdl, &resp->operation);
		status = ddi_get16(acchdl, (uint16_t *)&resp->status);
		DPRINTF(INTR_DBG, ("resp: op %d id %"PRIu64" status %d\n",
		    op, id, status));

		if (status != BLKIF_RSP_OKAY) {
			DPRINTF(IO_DBG, ("xdf@%s: I/O error while %s",
			    vdp->xdf_addr,
			    (op == BLKIF_OP_READ) ? "reading" : "writing"));
			bioerr = EIO;
		} else {
			bioerr = 0;
		}

		xdf_io_fini(vdp, id, bioerr);
	}
	return (DDI_INTR_CLAIMED);
}

/*
 * xdf_intr runs at PIL 5, so no one else can grab xdf_dev_lk and
 * block at a lower pil.
 */
static uint_t
xdf_intr(caddr_t arg)
{
	xdf_t *vdp = (xdf_t *)arg;
	int rv;

	mutex_enter(&vdp->xdf_dev_lk);
	rv = xdf_intr_locked(vdp);
	mutex_exit(&vdp->xdf_dev_lk);

	if (!do_polled_io)
		xdf_io_start(vdp);

	return (rv);
}

static void
xdf_ring_push(xdf_t *vdp)
{
	ASSERT(MUTEX_HELD(&vdp->xdf_dev_lk));

	if (vdp->xdf_xb_ring == NULL)
		return;

	if (xvdi_ring_push_request(vdp->xdf_xb_ring)) {
		DPRINTF(IO_DBG, (
		    "xdf@%s: xdf_ring_push: sent request(s) to backend\n",
		    vdp->xdf_addr));
	}

	if (xvdi_get_evtchn(vdp->xdf_dip) != INVALID_EVTCHN)
		xvdi_notify_oe(vdp->xdf_dip);
}

static int
xdf_ring_drain_locked(xdf_t *vdp)
{
	int		pollc, rv = 0;

	ASSERT(MUTEX_HELD(&vdp->xdf_dev_lk));

	if (xdf_debug & SUSRES_DBG)
		xen_printf("xdf_ring_drain: start\n");

	for (pollc = 0; pollc < XDF_DRAIN_RETRY_COUNT; pollc++) {
		if (vdp->xdf_xb_ring == NULL)
			goto out;

		if (xvdi_ring_has_unconsumed_responses(vdp->xdf_xb_ring))
			(void) xdf_intr_locked(vdp);
		if (!xvdi_ring_has_incomp_request(vdp->xdf_xb_ring))
			goto out;
		xdf_ring_push(vdp);

		/* file-backed devices can be slow */
		mutex_exit(&vdp->xdf_dev_lk);
#ifdef XPV_HVM_DRIVER
		(void) HYPERVISOR_yield();
#endif /* XPV_HVM_DRIVER */
		delay(drv_usectohz(XDF_DRAIN_MSEC_DELAY));
		mutex_enter(&vdp->xdf_dev_lk);
	}
	cmn_err(CE_WARN, "xdf@%s: xdf_ring_drain: timeout", vdp->xdf_addr);

out:
	if (vdp->xdf_xb_ring != NULL) {
		if (xvdi_ring_has_incomp_request(vdp->xdf_xb_ring) ||
		    xvdi_ring_has_unconsumed_responses(vdp->xdf_xb_ring))
			rv = EIO;
	}
	if (xdf_debug & SUSRES_DBG)
		xen_printf("xdf@%s: xdf_ring_drain: end, err=%d\n",
		    vdp->xdf_addr, rv);
	return (rv);
}

static int
xdf_ring_drain(xdf_t *vdp)
{
	int rv;
	mutex_enter(&vdp->xdf_dev_lk);
	rv = xdf_ring_drain_locked(vdp);
	mutex_exit(&vdp->xdf_dev_lk);
	return (rv);
}

/*
 * Destroy all v_req_t, grant table entries, and our ring buffer.
 */
static void
xdf_ring_destroy(xdf_t *vdp)
{
	v_req_t		*vreq;
	buf_t		*bp;
	ge_slot_t	*gs;

	ASSERT(MUTEX_HELD(&vdp->xdf_cb_lk));
	ASSERT(MUTEX_HELD(&vdp->xdf_dev_lk));

	if ((vdp->xdf_state != XD_INIT) &&
	    (vdp->xdf_state != XD_CONNECTED) &&
	    (vdp->xdf_state != XD_READY)) {
		ASSERT(vdp->xdf_xb_ring == NULL);
		ASSERT(vdp->xdf_xb_ring_hdl == NULL);
		ASSERT(vdp->xdf_peer == INVALID_DOMID);
		ASSERT(vdp->xdf_evtchn == INVALID_EVTCHN);
		ASSERT(list_is_empty(&vdp->xdf_vreq_act));
		return;
	}

	/*
	 * We don't want to receive async notifications from the backend
	 * when it finishes processing ring entries.
	 */
#ifdef XPV_HVM_DRIVER
	ec_unbind_evtchn(vdp->xdf_evtchn);
#else /* !XPV_HVM_DRIVER */
	(void) ddi_remove_intr(vdp->xdf_dip, 0, NULL);
#endif /* !XPV_HVM_DRIVER */

	/*
	 * Drain any requests in the ring.  We need to do this before we
	 * can free grant table entries, because if active ring entries
	 * point to grants, then the backend could be trying to access
	 * those grants.
	 */
	(void) xdf_ring_drain_locked(vdp);

	/* We're done talking to the backend so free up our event channel */
	xvdi_free_evtchn(vdp->xdf_dip);
	vdp->xdf_evtchn = INVALID_EVTCHN;

	while ((vreq = list_head(&vdp->xdf_vreq_act)) != NULL) {
		bp = vreq->v_buf;
		ASSERT(BP_VREQ(bp) == vreq);

		/* Free up any grant table entries associaed with this IO */
		while ((gs = list_head(&vreq->v_gs)) != NULL)
			gs_free(gs);

		/* If this IO was on the runq, move it back to the waitq. */
		if (vreq->v_runq)
			xdf_kstat_runq_to_waitq(vdp, bp);

		/*
		 * Reset any buf IO state since we're going to re-issue the
		 * IO when we reconnect.
		 */
		vreq_free(vdp, vreq);
		BP_VREQ_SET(bp, NULL);
		bioerror(bp, 0);
	}

	/* reset the active queue index pointer */
	vdp->xdf_i_act = vdp->xdf_f_act;

	/* Destroy the ring */
	xvdi_free_ring(vdp->xdf_xb_ring);
	vdp->xdf_xb_ring = NULL;
	vdp->xdf_xb_ring_hdl = NULL;
	vdp->xdf_peer = INVALID_DOMID;
}

void
xdfmin(struct buf *bp)
{
	if (bp->b_bcount > xdf_maxphys)
		bp->b_bcount = xdf_maxphys;
}

/*
 * Check if we have a pending "eject" media request.
 */
static int
xdf_eject_pending(xdf_t *vdp)
{
	dev_info_t	*dip = vdp->xdf_dip;
	char		*xsname, *str;

	if (!vdp->xdf_media_req_supported)
		return (B_FALSE);

	if (((xsname = xvdi_get_xsname(dip)) == NULL) ||
	    (xenbus_read_str(xsname, XBP_MEDIA_REQ, &str) != 0))
		return (B_FALSE);

	if (strcmp(str, XBV_MEDIA_REQ_EJECT) != 0) {
		strfree(str);
		return (B_FALSE);
	}
	strfree(str);
	return (B_TRUE);
}

/*
 * Generate a media request.
 */
static int
xdf_media_req(xdf_t *vdp, char *req, boolean_t media_required)
{
	dev_info_t	*dip = vdp->xdf_dip;
	char		*xsname;

	/*
	 * we can't be holding xdf_dev_lk because xenbus_printf() can
	 * block while waiting for a PIL 1 interrupt message.  this
	 * would cause a deadlock with xdf_intr() which needs to grab
	 * xdf_dev_lk as well and runs at PIL 5.
	 */
	ASSERT(MUTEX_HELD(&vdp->xdf_cb_lk));
	ASSERT(MUTEX_NOT_HELD(&vdp->xdf_dev_lk));

	if ((xsname = xvdi_get_xsname(dip)) == NULL)
		return (ENXIO);

	/* Check if we support media requests */
	if (!XD_IS_CD(vdp) || !vdp->xdf_media_req_supported)
		return (ENOTTY);

	/* If an eject is pending then don't allow any new requests */
	if (xdf_eject_pending(vdp))
		return (ENXIO);

	/* Make sure that there is media present */
	if (media_required && (vdp->xdf_xdev_nblocks == 0))
		return (ENXIO);

	/* We only allow operations when the device is ready and connected */
	if (vdp->xdf_state != XD_READY)
		return (EIO);

	if (xenbus_printf(XBT_NULL, xsname, XBP_MEDIA_REQ, "%s", req) != 0)
		return (EIO);

	return (0);
}

/*
 * populate a single blkif_request_t w/ a buf
 */
static void
xdf_process_rreq(xdf_t *vdp, struct buf *bp, blkif_request_t *rreq)
{
	grant_ref_t	gr;
	uint8_t		fsect, lsect;
	size_t		bcnt;
	paddr_t		dma_addr;
	off_t		blk_off;
	dev_info_t	*dip = vdp->xdf_dip;
	blkif_vdev_t	vdev = xvdi_get_vdevnum(dip);
	v_req_t		*vreq = BP_VREQ(bp);
	uint64_t	blkno = vreq->v_blkno;
	uint_t		ndmacs = vreq->v_ndmacs;
	ddi_acc_handle_t acchdl = vdp->xdf_xb_ring_hdl;
	int		seg = 0;
	int		isread = IS_READ(bp);
	ge_slot_t	*gs = list_head(&vreq->v_gs);

	ASSERT(MUTEX_HELD(&vdp->xdf_dev_lk));
	ASSERT(vreq->v_status == VREQ_GS_ALLOCED);

	if (isread)
		ddi_put8(acchdl, &rreq->operation, BLKIF_OP_READ);
	else {
		switch (vreq->v_flush_diskcache) {
		case FLUSH_DISKCACHE:
			ddi_put8(acchdl, &rreq->operation,
			    BLKIF_OP_FLUSH_DISKCACHE);
			ddi_put16(acchdl, &rreq->handle, vdev);
			ddi_put64(acchdl, &rreq->id,
			    (uint64_t)(uintptr_t)(gs));
			ddi_put8(acchdl, &rreq->nr_segments, 0);
			vreq->v_status = VREQ_DMAWIN_DONE;
			return;
		case WRITE_BARRIER:
			ddi_put8(acchdl, &rreq->operation,
			    BLKIF_OP_WRITE_BARRIER);
			break;
		default:
			if (!vdp->xdf_wce)
				ddi_put8(acchdl, &rreq->operation,
				    BLKIF_OP_WRITE_BARRIER);
			else
				ddi_put8(acchdl, &rreq->operation,
				    BLKIF_OP_WRITE);
			break;
		}
	}

	ddi_put16(acchdl, &rreq->handle, vdev);
	ddi_put64(acchdl, &rreq->sector_number, blkno);
	ddi_put64(acchdl, &rreq->id, (uint64_t)(uintptr_t)(gs));

	/*
	 * loop until all segments are populated or no more dma cookie in buf
	 */
	for (;;) {
		/*
		 * Each segment of a blkif request can transfer up to
		 * one 4K page of data.
		 */
		bcnt = vreq->v_dmac.dmac_size;
		dma_addr = vreq->v_dmac.dmac_laddress;
		blk_off = (uint_t)((paddr_t)XB_SEGOFFSET & dma_addr);
		fsect = blk_off >> XB_BSHIFT;
		lsect = fsect + (bcnt >> XB_BSHIFT) - 1;

		ASSERT(bcnt <= PAGESIZE);
		ASSERT((bcnt % XB_BSIZE) == 0);
		ASSERT((blk_off & XB_BMASK) == 0);
		ASSERT(fsect < XB_MAX_SEGLEN / XB_BSIZE &&
		    lsect < XB_MAX_SEGLEN / XB_BSIZE);

		gr = gs_grant(gs, PATOMA(dma_addr) >> PAGESHIFT);
		ddi_put32(acchdl, &rreq->seg[seg].gref, gr);
		ddi_put8(acchdl, &rreq->seg[seg].first_sect, fsect);
		ddi_put8(acchdl, &rreq->seg[seg].last_sect, lsect);

		DPRINTF(IO_DBG, (
		    "xdf@%s: seg%d: dmacS %lu blk_off %ld\n",
		    vdp->xdf_addr, seg, vreq->v_dmac.dmac_size, blk_off));
		DPRINTF(IO_DBG, (
		    "xdf@%s: seg%d: fs %d ls %d gr %d dma 0x%"PRIx64"\n",
		    vdp->xdf_addr, seg, fsect, lsect, gr, dma_addr));

		blkno += (bcnt >> XB_BSHIFT);
		seg++;
		ASSERT(seg <= BLKIF_MAX_SEGMENTS_PER_REQUEST);
		if (--ndmacs) {
			ddi_dma_nextcookie(vreq->v_dmahdl, &vreq->v_dmac);
			continue;
		}

		vreq->v_status = VREQ_DMAWIN_DONE;
		vreq->v_blkno = blkno;
		break;
	}
	ddi_put8(acchdl,  &rreq->nr_segments, seg);
	DPRINTF(IO_DBG, (
	    "xdf@%s: xdf_process_rreq: request id=%"PRIx64" ready\n",
	    vdp->xdf_addr, rreq->id));
}

static void
xdf_io_start(xdf_t *vdp)
{
	struct buf	*bp;
	v_req_t		*vreq;
	blkif_request_t	*rreq;
	boolean_t	rreqready = B_FALSE;

	mutex_enter(&vdp->xdf_dev_lk);

	/*
	 * Populate the ring request(s).  Loop until there is no buf to
	 * transfer or no free slot available in I/O ring.
	 */
	for (;;) {
		/* don't start any new IO if we're suspending */
		if (vdp->xdf_suspending)
			break;
		if ((bp = xdf_bp_next(vdp)) == NULL)
			break;

		/* if the buf doesn't already have a vreq, allocate one */
		if (((vreq = BP_VREQ(bp)) == NULL) &&
		    ((vreq = vreq_get(vdp, bp)) == NULL))
			break;

		/* alloc DMA/GTE resources */
		if (vreq_setup(vdp, vreq) != DDI_SUCCESS)
			break;

		/* get next blkif_request in the ring */
		if ((rreq = xvdi_ring_get_request(vdp->xdf_xb_ring)) == NULL)
			break;
		bzero(rreq, sizeof (blkif_request_t));
		rreqready = B_TRUE;

		/* populate blkif_request with this buf */
		xdf_process_rreq(vdp, bp, rreq);

		/*
		 * This buffer/vreq pair is has been allocated a ring buffer
		 * resources, so if it isn't already in our runq, add it.
		 */
		if (!vreq->v_runq)
			xdf_kstat_waitq_to_runq(vdp, bp);
	}

	/* Send the request(s) to the backend */
	if (rreqready)
		xdf_ring_push(vdp);

	mutex_exit(&vdp->xdf_dev_lk);
}


/* check if partition is open, -1 - check all partitions on the disk */
static boolean_t
xdf_isopen(xdf_t *vdp, int partition)
{
	int i;
	ulong_t parbit;
	boolean_t rval = B_FALSE;

	ASSERT((partition == -1) ||
	    ((partition >= 0) || (partition < XDF_PEXT)));

	if (partition == -1)
		parbit = (ulong_t)-1;
	else
		parbit = 1 << partition;

	for (i = 0; i < OTYPCNT; i++) {
		if (vdp->xdf_vd_open[i] & parbit)
			rval = B_TRUE;
	}

	return (rval);
}

/*
 * The connection should never be closed as long as someone is holding
 * us open, there is pending IO, or someone is waiting waiting for a
 * connection.
 */
static boolean_t
xdf_busy(xdf_t *vdp)
{
	ASSERT(MUTEX_HELD(&vdp->xdf_dev_lk));

	if ((vdp->xdf_xb_ring != NULL) &&
	    xvdi_ring_has_unconsumed_responses(vdp->xdf_xb_ring)) {
		ASSERT(vdp->xdf_state != XD_CLOSED);
		return (B_TRUE);
	}

	if (!list_is_empty(&vdp->xdf_vreq_act) || (vdp->xdf_f_act != NULL)) {
		ASSERT(vdp->xdf_state != XD_CLOSED);
		return (B_TRUE);
	}

	if (xdf_isopen(vdp, -1)) {
		ASSERT(vdp->xdf_state != XD_CLOSED);
		return (B_TRUE);
	}

	if (vdp->xdf_connect_req > 0) {
		ASSERT(vdp->xdf_state != XD_CLOSED);
		return (B_TRUE);
	}

	return (B_FALSE);
}

static void
xdf_set_state(xdf_t *vdp, xdf_state_t new_state)
{
	ASSERT(MUTEX_HELD(&vdp->xdf_cb_lk));
	ASSERT(MUTEX_HELD(&vdp->xdf_dev_lk));
	DPRINTF(DDI_DBG, ("xdf@%s: state change %d -> %d\n",
	    vdp->xdf_addr, vdp->xdf_state, new_state));
	vdp->xdf_state = new_state;
	cv_broadcast(&vdp->xdf_dev_cv);
}

static void
xdf_disconnect(xdf_t *vdp, xdf_state_t new_state, boolean_t quiet)
{
	dev_info_t	*dip = vdp->xdf_dip;
	boolean_t	busy;

	ASSERT(MUTEX_HELD(&vdp->xdf_cb_lk));
	ASSERT(MUTEX_NOT_HELD(&vdp->xdf_dev_lk));
	ASSERT((new_state == XD_UNKNOWN) || (new_state == XD_CLOSED));

	/* Check if we're already there. */
	if (vdp->xdf_state == new_state)
		return;

	mutex_enter(&vdp->xdf_dev_lk);
	busy = xdf_busy(vdp);

	/* If we're already closed then there's nothing todo. */
	if (vdp->xdf_state == XD_CLOSED) {
		ASSERT(!busy);
		xdf_set_state(vdp, new_state);
		mutex_exit(&vdp->xdf_dev_lk);
		return;
	}

#ifdef DEBUG
	/* UhOh.  Warn the user that something bad has happened. */
	if (!quiet && busy && (vdp->xdf_state == XD_READY) &&
	    (vdp->xdf_xdev_nblocks != 0)) {
		cmn_err(CE_WARN, "xdf@%s: disconnected while in use",
		    vdp->xdf_addr);
	}
#endif /* DEBUG */

	xdf_ring_destroy(vdp);

	/* If we're busy then we can only go into the unknown state */
	xdf_set_state(vdp, (busy) ? XD_UNKNOWN : new_state);
	mutex_exit(&vdp->xdf_dev_lk);

	/* if we're closed now, let the other end know */
	if (vdp->xdf_state == XD_CLOSED)
		(void) xvdi_switch_state(dip, XBT_NULL, XenbusStateClosed);
}


/*
 * Kick-off connect process
 * Status should be XD_UNKNOWN or XD_CLOSED
 * On success, status will be changed to XD_INIT
 * On error, it will be changed to XD_UNKNOWN
 */
static int
xdf_setstate_init(xdf_t *vdp)
{
	dev_info_t		*dip = vdp->xdf_dip;
	xenbus_transaction_t	xbt;
	grant_ref_t		gref;
	char			*xsname, *str;
	int			rv;

	ASSERT(MUTEX_HELD(&vdp->xdf_cb_lk));
	ASSERT(MUTEX_NOT_HELD(&vdp->xdf_dev_lk));
	ASSERT((vdp->xdf_state == XD_UNKNOWN) ||
	    (vdp->xdf_state == XD_CLOSED));

	DPRINTF(DDI_DBG,
	    ("xdf@%s: starting connection process\n", vdp->xdf_addr));

	/*
	 * If an eject is pending then don't allow a new connection.
	 * (Only the backend can clear media request eject request.)
	 */
	if (xdf_eject_pending(vdp))
		return (DDI_FAILURE);

	if ((xsname = xvdi_get_xsname(dip)) == NULL)
		goto errout;

	if ((vdp->xdf_peer = xvdi_get_oeid(dip)) == INVALID_DOMID)
		goto errout;

	(void) xvdi_switch_state(dip, XBT_NULL, XenbusStateInitialising);

	/*
	 * Sanity check for the existance of the xenbus device-type property.
	 * This property might not exist if our xenbus device nodes were
	 * force destroyed while we were still connected to the backend.
	 */
	if (xenbus_read_str(xsname, XBP_DEV_TYPE, &str) != 0)
		goto errout;
	strfree(str);

	if (xvdi_alloc_evtchn(dip) != DDI_SUCCESS)
		goto errout;

	vdp->xdf_evtchn = xvdi_get_evtchn(dip);
#ifdef XPV_HVM_DRIVER
	ec_bind_evtchn_to_handler(vdp->xdf_evtchn, IPL_VBD, xdf_intr, vdp);
#else /* !XPV_HVM_DRIVER */
	if (ddi_add_intr(dip, 0, NULL, NULL, xdf_intr, (caddr_t)vdp) !=
	    DDI_SUCCESS) {
		cmn_err(CE_WARN, "xdf@%s: xdf_setstate_init: "
		    "failed to add intr handler", vdp->xdf_addr);
		goto errout1;
	}
#endif /* !XPV_HVM_DRIVER */

	if (xvdi_alloc_ring(dip, BLKIF_RING_SIZE,
	    sizeof (union blkif_sring_entry), &gref, &vdp->xdf_xb_ring) !=
	    DDI_SUCCESS) {
		cmn_err(CE_WARN, "xdf@%s: failed to alloc comm ring",
		    vdp->xdf_addr);
		goto errout2;
	}
	vdp->xdf_xb_ring_hdl = vdp->xdf_xb_ring->xr_acc_hdl; /* ugly!! */

	/*
	 * Write into xenstore the info needed by backend
	 */
trans_retry:
	if (xenbus_transaction_start(&xbt)) {
		cmn_err(CE_WARN, "xdf@%s: failed to start transaction",
		    vdp->xdf_addr);
		xvdi_fatal_error(dip, EIO, "connect transaction init");
		goto fail_trans;
	}

	/*
	 * XBP_PROTOCOL is written by the domain builder in the case of PV
	 * domains. However, it is not written for HVM domains, so let's
	 * write it here.
	 */
	if (((rv = xenbus_printf(xbt, xsname,
	    XBP_MEDIA_REQ, "%s", XBV_MEDIA_REQ_NONE)) != 0) ||
	    ((rv = xenbus_printf(xbt, xsname,
	    XBP_RING_REF, "%u", gref)) != 0) ||
	    ((rv = xenbus_printf(xbt, xsname,
	    XBP_EVENT_CHAN, "%u", vdp->xdf_evtchn)) != 0) ||
	    ((rv = xenbus_printf(xbt, xsname,
	    XBP_PROTOCOL, "%s", XEN_IO_PROTO_ABI_NATIVE)) != 0) ||
	    ((rv = xvdi_switch_state(dip, xbt, XenbusStateInitialised)) > 0)) {
		(void) xenbus_transaction_end(xbt, 1);
		xvdi_fatal_error(dip, rv, "connect transaction setup");
		goto fail_trans;
	}

	/* kick-off connect process */
	if (rv = xenbus_transaction_end(xbt, 0)) {
		if (rv == EAGAIN)
			goto trans_retry;
		xvdi_fatal_error(dip, rv, "connect transaction commit");
		goto fail_trans;
	}

	ASSERT(MUTEX_HELD(&vdp->xdf_cb_lk));
	mutex_enter(&vdp->xdf_dev_lk);
	xdf_set_state(vdp, XD_INIT);
	mutex_exit(&vdp->xdf_dev_lk);

	return (DDI_SUCCESS);

fail_trans:
	xvdi_free_ring(vdp->xdf_xb_ring);
errout2:
#ifdef XPV_HVM_DRIVER
	ec_unbind_evtchn(vdp->xdf_evtchn);
#else /* !XPV_HVM_DRIVER */
	(void) ddi_remove_intr(vdp->xdf_dip, 0, NULL);
errout1:
#endif /* !XPV_HVM_DRIVER */
	xvdi_free_evtchn(dip);
	vdp->xdf_evtchn = INVALID_EVTCHN;
errout:
	xdf_disconnect(vdp, XD_UNKNOWN, B_FALSE);
	cmn_err(CE_WARN, "xdf@%s: failed to start connection to backend",
	    vdp->xdf_addr);
	return (DDI_FAILURE);
}

int
xdf_get_flush_block(xdf_t *vdp)
{
	/*
	 * Get a DEV_BSIZE aligned bufer
	 */
	vdp->xdf_flush_mem = kmem_alloc(vdp->xdf_xdev_secsize * 2, KM_SLEEP);
	vdp->xdf_cache_flush_block =
	    (char *)P2ROUNDUP((uintptr_t)(vdp->xdf_flush_mem),
	    (int)vdp->xdf_xdev_secsize);

	if (xdf_lb_rdwr(vdp->xdf_dip, TG_READ, vdp->xdf_cache_flush_block,
	    xdf_flush_block, vdp->xdf_xdev_secsize, NULL) != 0)
		return (DDI_FAILURE);
	return (DDI_SUCCESS);
}

static void
xdf_setstate_ready(void *arg)
{
	xdf_t		*vdp = (xdf_t *)arg;
	dev_info_t	*dip = vdp->xdf_dip;

	vdp->xdf_ready_tq_thread = curthread;

	/* Create minor nodes now when we are almost connected */
	mutex_enter(&vdp->xdf_dev_lk);
	if (vdp->xdf_cmlb_reattach) {
		vdp->xdf_cmlb_reattach = B_FALSE;
		mutex_exit(&vdp->xdf_dev_lk);
		if (xdf_cmlb_attach(vdp) != 0) {
			cmn_err(CE_WARN,
			    "xdf@%s: cmlb attach failed",
			    ddi_get_name_addr(dip));
			xdf_disconnect(vdp, XD_UNKNOWN, B_FALSE);
			return;
		}
		mutex_enter(&vdp->xdf_dev_lk);
	}

	/* If we're not still trying to get to the ready state, then bail. */
	if (vdp->xdf_state != XD_CONNECTED) {
		mutex_exit(&vdp->xdf_dev_lk);
		return;
	}
	mutex_exit(&vdp->xdf_dev_lk);

	/*
	 * If backend has feature-barrier, see if it supports disk
	 * cache flush op.
	 */
	vdp->xdf_flush_supported = B_FALSE;
	if (vdp->xdf_feature_barrier) {
		/*
		 * Pretend we already know flush is supported so probe
		 * will attempt the correct op.
		 */
		vdp->xdf_flush_supported = B_TRUE;
		if (xdf_lb_rdwr(vdp->xdf_dip, TG_WRITE, NULL, 0, 0, 0) == 0) {
			vdp->xdf_flush_supported = B_TRUE;
		} else {
			vdp->xdf_flush_supported = B_FALSE;
			/*
			 * If the other end does not support the cache flush op
			 * then we must use a barrier-write to force disk
			 * cache flushing.  Barrier writes require that a data
			 * block actually be written.
			 * Cache a block to barrier-write when we are
			 * asked to perform a flush.
			 * XXX - would it be better to just copy 1 block
			 * (512 bytes) from whatever write we did last
			 * and rewrite that block?
			 */
			if (xdf_get_flush_block(vdp) != DDI_SUCCESS) {
				xdf_disconnect(vdp, XD_UNKNOWN, B_FALSE);
				return;
			}
		}
	}

	mutex_enter(&vdp->xdf_cb_lk);
	mutex_enter(&vdp->xdf_dev_lk);
	if (vdp->xdf_state == XD_CONNECTED)
		xdf_set_state(vdp, XD_READY);
	mutex_exit(&vdp->xdf_dev_lk);

	/* Restart any currently queued up io */
	xdf_io_start(vdp);

	mutex_exit(&vdp->xdf_cb_lk);
}

/*
 * synthetic geometry
 */
#define	XDF_NSECTS	256
#define	XDF_NHEADS	16

static void
xdf_synthetic_pgeom(dev_info_t *dip, cmlb_geom_t *geomp)
{
	xdf_t *vdp;
	uint_t ncyl;

	vdp = ddi_get_soft_state(xdf_ssp, ddi_get_instance(dip));

	ncyl = vdp->xdf_xdev_nblocks / (XDF_NHEADS * XDF_NSECTS);

	bzero(geomp, sizeof (*geomp));
	geomp->g_ncyl = ncyl == 0 ? 1 : ncyl;
	geomp->g_acyl = 0;
	geomp->g_nhead = XDF_NHEADS;
	geomp->g_nsect = XDF_NSECTS;
	geomp->g_secsize = vdp->xdf_xdev_secsize;
	geomp->g_capacity = vdp->xdf_xdev_nblocks;
	geomp->g_intrlv = 0;
	geomp->g_rpm = 7200;
}

/*
 * Finish other initialization after we've connected to backend
 * Status should be XD_INIT before calling this routine
 * On success, status should be changed to XD_CONNECTED.
 * On error, status should stay XD_INIT
 */
static int
xdf_setstate_connected(xdf_t *vdp)
{
	dev_info_t	*dip = vdp->xdf_dip;
	cmlb_geom_t	pgeom;
	diskaddr_t	nblocks = 0;
	uint_t		secsize = 0;
	char		*oename, *xsname, *str;
	uint_t		dinfo;

	ASSERT(MUTEX_HELD(&vdp->xdf_cb_lk));
	ASSERT(MUTEX_NOT_HELD(&vdp->xdf_dev_lk));
	ASSERT(vdp->xdf_state == XD_INIT);

	if (((xsname = xvdi_get_xsname(dip)) == NULL) ||
	    ((oename = xvdi_get_oename(dip)) == NULL))
		return (DDI_FAILURE);

	/* Make sure the other end is XenbusStateConnected */
	if (xenbus_read_driver_state(oename) != XenbusStateConnected)
		return (DDI_FAILURE);

	/* Determine if feature barrier is supported by backend */
	if (!(vdp->xdf_feature_barrier = xenbus_exists(oename, XBP_FB)))
		cmn_err(CE_NOTE, "!xdf@%s: feature-barrier not supported",
		    vdp->xdf_addr);

	/*
	 * Probe backend.  Read the device size into xdf_xdev_nblocks
	 * and set the VDISK_READONLY, VDISK_CDROM, and VDISK_REMOVABLE
	 * flags in xdf_dinfo.  If the emulated device type is "cdrom",
	 * we always set VDISK_CDROM, regardless of if it's present in
	 * the xenbus info parameter.
	 */
	if (xenbus_gather(XBT_NULL, oename,
	    XBP_SECTORS, "%"SCNu64, &nblocks,
	    XBP_SECTOR_SIZE, "%u", &secsize,
	    XBP_INFO, "%u", &dinfo,
	    NULL) != 0) {
		cmn_err(CE_WARN, "xdf@%s: xdf_setstate_connected: "
		    "cannot read backend info", vdp->xdf_addr);
		return (DDI_FAILURE);
	}
	if (xenbus_read_str(xsname, XBP_DEV_TYPE, &str) != 0) {
		cmn_err(CE_WARN, "xdf@%s: cannot read device-type",
		    vdp->xdf_addr);
		return (DDI_FAILURE);
	}
	if (strcmp(str, XBV_DEV_TYPE_CD) == 0)
		dinfo |= VDISK_CDROM;
	strfree(str);

	if (secsize == 0 || !(ISP2(secsize / DEV_BSIZE)))
		secsize = DEV_BSIZE;
	vdp->xdf_xdev_nblocks = nblocks;
	vdp->xdf_xdev_secsize = secsize;
#ifdef _ILP32
	if (vdp->xdf_xdev_nblocks > DK_MAX_BLOCKS) {
		cmn_err(CE_WARN, "xdf@%s: xdf_setstate_connected: "
		    "backend disk device too large with %llu blocks for"
		    " 32-bit kernel", vdp->xdf_addr, vdp->xdf_xdev_nblocks);
		xvdi_fatal_error(dip, EFBIG, "reading backend info");
		return (DDI_FAILURE);
	}
#endif

	/*
	 * If the physical geometry for a fixed disk has been explicity
	 * set then make sure that the specified physical geometry isn't
	 * larger than the device we connected to.
	 */
	if (vdp->xdf_pgeom_fixed &&
	    (vdp->xdf_pgeom.g_capacity > vdp->xdf_xdev_nblocks)) {
		cmn_err(CE_WARN,
		    "xdf@%s: connect failed, fixed geometry too large",
		    vdp->xdf_addr);
		return (DDI_FAILURE);
	}

	vdp->xdf_media_req_supported = xenbus_exists(oename, XBP_MEDIA_REQ_SUP);

	/* mark vbd is ready for I/O */
	mutex_enter(&vdp->xdf_dev_lk);
	xdf_set_state(vdp, XD_CONNECTED);

	/* check if the cmlb label should be updated */
	xdf_synthetic_pgeom(dip, &pgeom);
	if ((vdp->xdf_dinfo != dinfo) ||
	    (!vdp->xdf_pgeom_fixed &&
	    (memcmp(&vdp->xdf_pgeom, &pgeom, sizeof (pgeom)) != 0))) {
		vdp->xdf_cmlb_reattach = B_TRUE;

		vdp->xdf_dinfo = dinfo;
		if (!vdp->xdf_pgeom_fixed)
			vdp->xdf_pgeom = pgeom;
	}

	if (XD_IS_CD(vdp) || XD_IS_RM(vdp)) {
		if (vdp->xdf_xdev_nblocks == 0) {
			vdp->xdf_mstate = DKIO_EJECTED;
			cv_broadcast(&vdp->xdf_mstate_cv);
		} else {
			vdp->xdf_mstate = DKIO_INSERTED;
			cv_broadcast(&vdp->xdf_mstate_cv);
		}
	} else {
		if (vdp->xdf_mstate != DKIO_NONE) {
			vdp->xdf_mstate = DKIO_NONE;
			cv_broadcast(&vdp->xdf_mstate_cv);
		}
	}

	mutex_exit(&vdp->xdf_dev_lk);

	cmn_err(CE_CONT, "?xdf@%s: %"PRIu64" blocks", vdp->xdf_addr,
	    (uint64_t)vdp->xdf_xdev_nblocks);

	/* Restart any currently queued up io */
	xdf_io_start(vdp);

	/*
	 * To get to the ready state we have to do IO to the backend device,
	 * but we can't initiate IO from the other end change callback thread
	 * (which is the current context we're executing in.)  This is because
	 * if the other end disconnects while we're doing IO from the callback
	 * thread, then we can't receive that disconnect event and we hang
	 * waiting for an IO that can never complete.
	 */
	(void) ddi_taskq_dispatch(vdp->xdf_ready_tq, xdf_setstate_ready, vdp,
	    DDI_SLEEP);

	(void) xvdi_switch_state(dip, XBT_NULL, XenbusStateConnected);
	return (DDI_SUCCESS);
}

/*ARGSUSED*/
static void
xdf_oe_change(dev_info_t *dip, ddi_eventcookie_t id, void *arg, void *impl_data)
{
	XenbusState new_state = *(XenbusState *)impl_data;
	xdf_t *vdp = (xdf_t *)ddi_get_driver_private(dip);

	DPRINTF(DDI_DBG, ("xdf@%s: otherend state change to %d!\n",
	    vdp->xdf_addr, new_state));

	mutex_enter(&vdp->xdf_cb_lk);

	/* We assume that this callback is single threaded */
	ASSERT(vdp->xdf_oe_change_thread == NULL);
	DEBUG_EVAL(vdp->xdf_oe_change_thread = curthread);

	/* ignore any backend state changes if we're suspending/suspended */
	if (vdp->xdf_suspending || (vdp->xdf_state == XD_SUSPEND)) {
		DEBUG_EVAL(vdp->xdf_oe_change_thread = NULL);
		mutex_exit(&vdp->xdf_cb_lk);
		return;
	}

	switch (new_state) {
	case XenbusStateUnknown:
	case XenbusStateInitialising:
	case XenbusStateInitWait:
	case XenbusStateInitialised:
		if (vdp->xdf_state == XD_INIT)
			break;

		xdf_disconnect(vdp, XD_UNKNOWN, B_FALSE);
		if (xdf_setstate_init(vdp) != DDI_SUCCESS)
			break;
		ASSERT(vdp->xdf_state == XD_INIT);
		break;

	case XenbusStateConnected:
		if ((vdp->xdf_state == XD_CONNECTED) ||
		    (vdp->xdf_state == XD_READY))
			break;

		if (vdp->xdf_state != XD_INIT) {
			xdf_disconnect(vdp, XD_UNKNOWN, B_FALSE);
			if (xdf_setstate_init(vdp) != DDI_SUCCESS)
				break;
			ASSERT(vdp->xdf_state == XD_INIT);
		}

		if (xdf_setstate_connected(vdp) != DDI_SUCCESS) {
			xdf_disconnect(vdp, XD_UNKNOWN, B_FALSE);
			break;
		}
		ASSERT(vdp->xdf_state == XD_CONNECTED);
		break;

	case XenbusStateClosing:
		if (xdf_isopen(vdp, -1)) {
			cmn_err(CE_NOTE,
			    "xdf@%s: hot-unplug failed, still in use",
			    vdp->xdf_addr);
			break;
		}
		/*FALLTHROUGH*/
	case XenbusStateClosed:
		xdf_disconnect(vdp, XD_CLOSED, B_FALSE);
		break;
	}

	/* notify anybody waiting for oe state change */
	cv_broadcast(&vdp->xdf_dev_cv);
	DEBUG_EVAL(vdp->xdf_oe_change_thread = NULL);
	mutex_exit(&vdp->xdf_cb_lk);
}

static int
xdf_connect_locked(xdf_t *vdp, boolean_t wait)
{
	int	rv, timeouts = 0, reset = 20;

	ASSERT(MUTEX_HELD(&vdp->xdf_cb_lk));
	ASSERT(MUTEX_HELD(&vdp->xdf_dev_lk));

	/* we can't connect once we're in the closed state */
	if (vdp->xdf_state == XD_CLOSED)
		return (XD_CLOSED);

	vdp->xdf_connect_req++;
	while (vdp->xdf_state != XD_READY) {
		mutex_exit(&vdp->xdf_dev_lk);

		/* only one thread at a time can be the connection thread */
		if (vdp->xdf_connect_thread == NULL)
			vdp->xdf_connect_thread = curthread;

		if (vdp->xdf_connect_thread == curthread) {
			if ((timeouts > 0) && ((timeouts % reset) == 0)) {
				/*
				 * If we haven't establised a connection
				 * within the reset time, then disconnect
				 * so we can try again, and double the reset
				 * time.  The reset time starts at 2 sec.
				 */
				(void) xdf_disconnect(vdp, XD_UNKNOWN, B_TRUE);
				reset *= 2;
			}
			if (vdp->xdf_state == XD_UNKNOWN)
				(void) xdf_setstate_init(vdp);
			if (vdp->xdf_state == XD_INIT)
				(void) xdf_setstate_connected(vdp);
		}

		mutex_enter(&vdp->xdf_dev_lk);
		if (!wait || (vdp->xdf_state == XD_READY))
			goto out;

		mutex_exit((&vdp->xdf_cb_lk));
		if (vdp->xdf_connect_thread != curthread) {
			rv = cv_wait_sig(&vdp->xdf_dev_cv, &vdp->xdf_dev_lk);
		} else {
			/* delay for 0.1 sec */
			rv = cv_reltimedwait_sig(&vdp->xdf_dev_cv,
			    &vdp->xdf_dev_lk, drv_usectohz(100*1000),
			    TR_CLOCK_TICK);
			if (rv == -1)
				timeouts++;
		}
		mutex_exit((&vdp->xdf_dev_lk));
		mutex_enter((&vdp->xdf_cb_lk));
		mutex_enter((&vdp->xdf_dev_lk));
		if (rv == 0)
			goto out;
	}

out:
	ASSERT(MUTEX_HELD(&vdp->xdf_cb_lk));
	ASSERT(MUTEX_HELD(&vdp->xdf_dev_lk));

	if (vdp->xdf_connect_thread == curthread) {
		/*
		 * wake up someone else so they can become the connection
		 * thread.
		 */
		cv_signal(&vdp->xdf_dev_cv);
		vdp->xdf_connect_thread = NULL;
	}

	/* Try to lock the media */
	mutex_exit((&vdp->xdf_dev_lk));
	(void) xdf_media_req(vdp, XBV_MEDIA_REQ_LOCK, B_TRUE);
	mutex_enter((&vdp->xdf_dev_lk));

	vdp->xdf_connect_req--;
	return (vdp->xdf_state);
}

static uint_t
xdf_iorestart(caddr_t arg)
{
	xdf_t *vdp = (xdf_t *)arg;

	ASSERT(vdp != NULL);

	mutex_enter(&vdp->xdf_dev_lk);
	ASSERT(ISDMACBON(vdp));
	SETDMACBOFF(vdp);
	mutex_exit(&vdp->xdf_dev_lk);

	xdf_io_start(vdp);

	return (DDI_INTR_CLAIMED);
}

#ifdef XPV_HVM_DRIVER

typedef struct xdf_hvm_entry {
	list_node_t	xdf_he_list;
	char		*xdf_he_path;
	dev_info_t	*xdf_he_dip;
} xdf_hvm_entry_t;

static list_t xdf_hvm_list;
static kmutex_t xdf_hvm_list_lock;

static xdf_hvm_entry_t *
i_xdf_hvm_find(const char *path, dev_info_t *dip)
{
	xdf_hvm_entry_t	*i;

	ASSERT((path != NULL) || (dip != NULL));
	ASSERT(MUTEX_HELD(&xdf_hvm_list_lock));

	i = list_head(&xdf_hvm_list);
	while (i != NULL) {
		if ((path != NULL) && strcmp(i->xdf_he_path, path) != 0) {
			i = list_next(&xdf_hvm_list, i);
			continue;
		}
		if ((dip != NULL) && (i->xdf_he_dip != dip)) {
			i = list_next(&xdf_hvm_list, i);
			continue;
		}
		break;
	}
	return (i);
}

dev_info_t *
xdf_hvm_hold(const char *path)
{
	xdf_hvm_entry_t	*i;
	dev_info_t	*dip;

	mutex_enter(&xdf_hvm_list_lock);
	i = i_xdf_hvm_find(path, NULL);
	if (i == NULL) {
		mutex_exit(&xdf_hvm_list_lock);
		return (B_FALSE);
	}
	ndi_hold_devi(dip = i->xdf_he_dip);
	mutex_exit(&xdf_hvm_list_lock);
	return (dip);
}

static void
xdf_hvm_add(dev_info_t *dip)
{
	xdf_hvm_entry_t	*i;
	char		*path;

	/* figure out the path for the dip */
	path = kmem_zalloc(MAXPATHLEN, KM_SLEEP);
	(void) ddi_pathname(dip, path);

	i = kmem_alloc(sizeof (*i), KM_SLEEP);
	i->xdf_he_dip = dip;
	i->xdf_he_path = i_ddi_strdup(path, KM_SLEEP);

	mutex_enter(&xdf_hvm_list_lock);
	ASSERT(i_xdf_hvm_find(path, NULL) == NULL);
	ASSERT(i_xdf_hvm_find(NULL, dip) == NULL);
	list_insert_head(&xdf_hvm_list, i);
	mutex_exit(&xdf_hvm_list_lock);

	kmem_free(path, MAXPATHLEN);
}

static void
xdf_hvm_rm(dev_info_t *dip)
{
	xdf_hvm_entry_t	*i;

	mutex_enter(&xdf_hvm_list_lock);
	VERIFY((i = i_xdf_hvm_find(NULL, dip)) != NULL);
	list_remove(&xdf_hvm_list, i);
	mutex_exit(&xdf_hvm_list_lock);

	kmem_free(i->xdf_he_path, strlen(i->xdf_he_path) + 1);
	kmem_free(i, sizeof (*i));
}

static void
xdf_hvm_init(void)
{
	list_create(&xdf_hvm_list, sizeof (xdf_hvm_entry_t),
	    offsetof(xdf_hvm_entry_t, xdf_he_list));
	mutex_init(&xdf_hvm_list_lock, NULL, MUTEX_DEFAULT, NULL);
}

static void
xdf_hvm_fini(void)
{
	ASSERT(list_head(&xdf_hvm_list) == NULL);
	list_destroy(&xdf_hvm_list);
	mutex_destroy(&xdf_hvm_list_lock);
}

boolean_t
xdf_hvm_connect(dev_info_t *dip)
{
	xdf_t	*vdp = (xdf_t *)ddi_get_driver_private(dip);
	char	*oename, *str;
	int	rv;

	mutex_enter(&vdp->xdf_cb_lk);

	/*
	 * Before try to establish a connection we need to wait for the
	 * backend hotplug scripts to have run.  Once they are run the
	 * "<oename>/hotplug-status" property will be set to "connected".
	 */
	for (;;) {
		ASSERT(MUTEX_HELD(&vdp->xdf_cb_lk));

		/*
		 * Get the xenbus path to the backend device.  Note that
		 * we can't cache this path (and we look it up on each pass
		 * through this loop) because it could change during
		 * suspend, resume, and migration operations.
		 */
		if ((oename = xvdi_get_oename(dip)) == NULL) {
			mutex_exit(&vdp->xdf_cb_lk);
			return (B_FALSE);
		}

		str = NULL;
		if ((xenbus_read_str(oename, XBP_HP_STATUS, &str) == 0) &&
		    (strcmp(str, XBV_HP_STATUS_CONN) == 0))
			break;

		if (str != NULL)
			strfree(str);

		/* wait for an update to "<oename>/hotplug-status" */
		if (cv_wait_sig(&vdp->xdf_hp_status_cv, &vdp->xdf_cb_lk) == 0) {
			/* we got interrupted by a signal */
			mutex_exit(&vdp->xdf_cb_lk);
			return (B_FALSE);
		}
	}

	/* Good news.  The backend hotplug scripts have been run. */
	ASSERT(MUTEX_HELD(&vdp->xdf_cb_lk));
	ASSERT(strcmp(str, XBV_HP_STATUS_CONN) == 0);
	strfree(str);

	/*
	 * If we're emulating a cd device and if the backend doesn't support
	 * media request opreations, then we're not going to bother trying
	 * to establish a connection for a couple reasons.  First off, media
	 * requests support is required to support operations like eject and
	 * media locking.  Second, other backend platforms like Linux don't
	 * support hvm pv cdrom access.  They don't even have a backend pv
	 * driver for cdrom device nodes, so we don't want to block forever
	 * waiting for a connection to a backend driver that doesn't exist.
	 */
	if (XD_IS_CD(vdp) && !xenbus_exists(oename, XBP_MEDIA_REQ_SUP)) {
		mutex_exit(&vdp->xdf_cb_lk);
		return (B_FALSE);
	}

	mutex_enter(&vdp->xdf_dev_lk);
	rv = xdf_connect_locked(vdp, B_TRUE);
	mutex_exit(&vdp->xdf_dev_lk);
	mutex_exit(&vdp->xdf_cb_lk);

	return ((rv == XD_READY) ? B_TRUE : B_FALSE);
}

int
xdf_hvm_setpgeom(dev_info_t *dip, cmlb_geom_t *geomp)
{
	xdf_t	*vdp = (xdf_t *)ddi_get_driver_private(dip);

	/* sanity check the requested physical geometry */
	mutex_enter(&vdp->xdf_dev_lk);
	if ((geomp->g_secsize != XB_BSIZE) ||
	    (geomp->g_capacity == 0)) {
		mutex_exit(&vdp->xdf_dev_lk);
		return (EINVAL);
	}

	/*
	 * If we've already connected to the backend device then make sure
	 * we're not defining a physical geometry larger than our backend
	 * device.
	 */
	if ((vdp->xdf_xdev_nblocks != 0) &&
	    (geomp->g_capacity > vdp->xdf_xdev_nblocks)) {
		mutex_exit(&vdp->xdf_dev_lk);
		return (EINVAL);
	}

	bzero(&vdp->xdf_pgeom, sizeof (vdp->xdf_pgeom));
	vdp->xdf_pgeom.g_ncyl = geomp->g_ncyl;
	vdp->xdf_pgeom.g_acyl = geomp->g_acyl;
	vdp->xdf_pgeom.g_nhead = geomp->g_nhead;
	vdp->xdf_pgeom.g_nsect = geomp->g_nsect;
	vdp->xdf_pgeom.g_secsize = geomp->g_secsize;
	vdp->xdf_pgeom.g_capacity = geomp->g_capacity;
	vdp->xdf_pgeom.g_intrlv = geomp->g_intrlv;
	vdp->xdf_pgeom.g_rpm = geomp->g_rpm;

	vdp->xdf_pgeom_fixed = B_TRUE;
	mutex_exit(&vdp->xdf_dev_lk);

	/* force a re-validation */
	cmlb_invalidate(vdp->xdf_vd_lbl, NULL);

	return (0);
}

boolean_t
xdf_is_cd(dev_info_t *dip)
{
	xdf_t		*vdp = (xdf_t *)ddi_get_driver_private(dip);
	boolean_t	rv;

	mutex_enter(&vdp->xdf_cb_lk);
	rv = XD_IS_CD(vdp);
	mutex_exit(&vdp->xdf_cb_lk);
	return (rv);
}

boolean_t
xdf_is_rm(dev_info_t *dip)
{
	xdf_t		*vdp = (xdf_t *)ddi_get_driver_private(dip);
	boolean_t	rv;

	mutex_enter(&vdp->xdf_cb_lk);
	rv = XD_IS_RM(vdp);
	mutex_exit(&vdp->xdf_cb_lk);
	return (rv);
}

boolean_t
xdf_media_req_supported(dev_info_t *dip)
{
	xdf_t		*vdp = (xdf_t *)ddi_get_driver_private(dip);
	boolean_t	rv;

	mutex_enter(&vdp->xdf_cb_lk);
	rv = vdp->xdf_media_req_supported;
	mutex_exit(&vdp->xdf_cb_lk);
	return (rv);
}

#endif /* XPV_HVM_DRIVER */

static int
xdf_lb_getcap(dev_info_t *dip, diskaddr_t *capp)
{
	xdf_t *vdp;
	vdp = ddi_get_soft_state(xdf_ssp, ddi_get_instance(dip));

	if (vdp == NULL)
		return (ENXIO);

	mutex_enter(&vdp->xdf_dev_lk);
	*capp = vdp->xdf_pgeom.g_capacity;
	DPRINTF(LBL_DBG, ("xdf@%s:capacity %llu\n", vdp->xdf_addr, *capp));
	mutex_exit(&vdp->xdf_dev_lk);
	return (0);
}

static int
xdf_lb_getpgeom(dev_info_t *dip, cmlb_geom_t *geomp)
{
	xdf_t *vdp;

	if ((vdp = ddi_get_soft_state(xdf_ssp, ddi_get_instance(dip))) == NULL)
		return (ENXIO);
	*geomp = vdp->xdf_pgeom;
	return (0);
}

/*
 * No real HBA, no geometry available from it
 */
/*ARGSUSED*/
static int
xdf_lb_getvgeom(dev_info_t *dip, cmlb_geom_t *geomp)
{
	return (EINVAL);
}

static int
xdf_lb_getattribute(dev_info_t *dip, tg_attribute_t *tgattributep)
{
	xdf_t *vdp;

	if (!(vdp = ddi_get_soft_state(xdf_ssp, ddi_get_instance(dip))))
		return (ENXIO);

	if (XD_IS_RO(vdp))
		tgattributep->media_is_writable = 0;
	else
		tgattributep->media_is_writable = 1;
	tgattributep->media_is_rotational = 0;
	return (0);
}

/* ARGSUSED3 */
int
xdf_lb_getinfo(dev_info_t *dip, int cmd, void *arg, void *tg_cookie)
{
	int instance;
	xdf_t   *vdp;

	instance = ddi_get_instance(dip);

	if ((vdp = ddi_get_soft_state(xdf_ssp, instance)) == NULL)
		return (ENXIO);

	switch (cmd) {
	case TG_GETPHYGEOM:
		return (xdf_lb_getpgeom(dip, (cmlb_geom_t *)arg));
	case TG_GETVIRTGEOM:
		return (xdf_lb_getvgeom(dip, (cmlb_geom_t *)arg));
	case TG_GETCAPACITY:
		return (xdf_lb_getcap(dip, (diskaddr_t *)arg));
	case TG_GETBLOCKSIZE:
		mutex_enter(&vdp->xdf_cb_lk);
		*(uint32_t *)arg = vdp->xdf_xdev_secsize;
		mutex_exit(&vdp->xdf_cb_lk);
		return (0);
	case TG_GETATTR:
		return (xdf_lb_getattribute(dip, (tg_attribute_t *)arg));
	default:
		return (ENOTTY);
	}
}

/* ARGSUSED5 */
int
xdf_lb_rdwr(dev_info_t *dip, uchar_t cmd, void *bufp,
    diskaddr_t start, size_t reqlen, void *tg_cookie)
{
	xdf_t *vdp;
	struct buf *bp;
	int err = 0;

	vdp = ddi_get_soft_state(xdf_ssp, ddi_get_instance(dip));

	/* We don't allow IO from the oe_change callback thread */
	ASSERT(curthread != vdp->xdf_oe_change_thread);

	/*
	 * Having secsize of 0 means that device isn't connected yet.
	 * FIXME This happens for CD devices, and there's nothing we
	 * can do about it at the moment.
	 */
	if (vdp->xdf_xdev_secsize == 0)
		return (EIO);

	if ((start + ((reqlen / (vdp->xdf_xdev_secsize / DEV_BSIZE))
	    >> DEV_BSHIFT)) > vdp->xdf_pgeom.g_capacity)
		return (EINVAL);

	bp = getrbuf(KM_SLEEP);
	if (cmd == TG_READ)
		bp->b_flags = B_BUSY | B_READ;
	else
		bp->b_flags = B_BUSY | B_WRITE;

	bp->b_un.b_addr = bufp;
	bp->b_bcount = reqlen;
	bp->b_blkno = start * (vdp->xdf_xdev_secsize / DEV_BSIZE);
	bp->b_edev = DDI_DEV_T_NONE; /* don't have dev_t */

	mutex_enter(&vdp->xdf_dev_lk);
	xdf_bp_push(vdp, bp);
	mutex_exit(&vdp->xdf_dev_lk);
	xdf_io_start(vdp);
	if (curthread == vdp->xdf_ready_tq_thread)
		(void) xdf_ring_drain(vdp);
	err = biowait(bp);
	ASSERT(bp->b_flags & B_DONE);
	freerbuf(bp);
	return (err);
}

/*
 * Lock the current media.  Set the media state to "lock".
 * (Media locks are only respected by the backend driver.)
 */
static int
xdf_ioctl_mlock(xdf_t *vdp)
{
	int rv;
	mutex_enter(&vdp->xdf_cb_lk);
	rv = xdf_media_req(vdp, XBV_MEDIA_REQ_LOCK, B_TRUE);
	mutex_exit(&vdp->xdf_cb_lk);
	return (rv);
}

/*
 * Release a media lock.  Set the media state to "none".
 */
static int
xdf_ioctl_munlock(xdf_t *vdp)
{
	int rv;
	mutex_enter(&vdp->xdf_cb_lk);
	rv = xdf_media_req(vdp, XBV_MEDIA_REQ_NONE, B_TRUE);
	mutex_exit(&vdp->xdf_cb_lk);
	return (rv);
}

/*
 * Eject the current media.  Ignores any media locks.  (Media locks
 * are only for benifit of the the backend.)
 */
static int
xdf_ioctl_eject(xdf_t *vdp)
{
	int rv;

	mutex_enter(&vdp->xdf_cb_lk);
	if ((rv = xdf_media_req(vdp, XBV_MEDIA_REQ_EJECT, B_FALSE)) != 0) {
		mutex_exit(&vdp->xdf_cb_lk);
		return (rv);
	}

	/*
	 * We've set the media requests xenbus parameter to eject, so now
	 * disconnect from the backend, wait for the backend to clear
	 * the media requets xenbus paramter, and then we can reconnect
	 * to the backend.
	 */
	(void) xdf_disconnect(vdp, XD_UNKNOWN, B_TRUE);
	mutex_enter(&vdp->xdf_dev_lk);
	if (xdf_connect_locked(vdp, B_TRUE) != XD_READY) {
		mutex_exit(&vdp->xdf_dev_lk);
		mutex_exit(&vdp->xdf_cb_lk);
		return (EIO);
	}
	mutex_exit(&vdp->xdf_dev_lk);
	mutex_exit(&vdp->xdf_cb_lk);
	return (0);
}

/*
 * Watch for media state changes.  This can be an insertion of a device
 * (triggered by a 'xm block-configure' request in another domain) or
 * the ejection of a device (triggered by a local "eject" operation).
 * For a full description of the DKIOCSTATE ioctl behavior see dkio(4I).
 */
static int
xdf_dkstate(xdf_t *vdp, enum dkio_state mstate)
{
	enum dkio_state		prev_state;

	mutex_enter(&vdp->xdf_cb_lk);
	prev_state = vdp->xdf_mstate;

	if (vdp->xdf_mstate == mstate) {
		while (vdp->xdf_mstate == prev_state) {
			if (cv_wait_sig(&vdp->xdf_mstate_cv,
			    &vdp->xdf_cb_lk) == 0) {
				mutex_exit(&vdp->xdf_cb_lk);
				return (EINTR);
			}
		}
	}

	if ((prev_state != DKIO_INSERTED) &&
	    (vdp->xdf_mstate == DKIO_INSERTED)) {
		(void) xdf_media_req(vdp, XBV_MEDIA_REQ_LOCK, B_TRUE);
		mutex_exit(&vdp->xdf_cb_lk);
		return (0);
	}

	mutex_exit(&vdp->xdf_cb_lk);
	return (0);
}

/*ARGSUSED*/
static int
xdf_ioctl(dev_t dev, int cmd, intptr_t arg, int mode, cred_t *credp,
    int *rvalp)
{
	minor_t		minor = getminor(dev);
	int		part = XDF_PART(minor);
	xdf_t		*vdp;
	int		rv;

	if (((vdp = ddi_get_soft_state(xdf_ssp, XDF_INST(minor))) == NULL) ||
	    (!xdf_isopen(vdp, part)))
		return (ENXIO);

	DPRINTF(IOCTL_DBG, ("xdf@%s:ioctl: cmd %d (0x%x)\n",
	    vdp->xdf_addr, cmd, cmd));

	switch (cmd) {
	default:
		return (ENOTTY);
	case DKIOCG_PHYGEOM:
	case DKIOCG_VIRTGEOM:
	case DKIOCGGEOM:
	case DKIOCSGEOM:
	case DKIOCGAPART:
	case DKIOCSAPART:
	case DKIOCGVTOC:
	case DKIOCSVTOC:
	case DKIOCPARTINFO:
	case DKIOCGEXTVTOC:
	case DKIOCSEXTVTOC:
	case DKIOCEXTPARTINFO:
	case DKIOCGMBOOT:
	case DKIOCSMBOOT:
	case DKIOCGETEFI:
	case DKIOCSETEFI:
	case DKIOCSETEXTPART:
	case DKIOCPARTITION:
		rv = cmlb_ioctl(vdp->xdf_vd_lbl, dev, cmd, arg, mode, credp,
		    rvalp, NULL);
		if (rv != 0)
			return (rv);
		/*
		 * If we're labelling the disk, we have to update the geometry
		 * in the cmlb data structures, and we also have to write a new
		 * devid to the disk.  Note that writing an EFI label currently
		 * requires 4 ioctls, and devid setup will fail on all but the
		 * last.
		 */
		if (cmd == DKIOCSEXTVTOC || cmd == DKIOCSVTOC ||
		    cmd == DKIOCSETEFI) {
			rv = cmlb_validate(vdp->xdf_vd_lbl, 0, 0);
			if (rv == 0) {
				xdf_devid_setup(vdp);
			} else {
				cmn_err(CE_WARN,
				    "xdf@%s, labeling failed on validate",
				    vdp->xdf_addr);
			}
		}
		return (rv);
	case FDEJECT:
	case DKIOCEJECT:
	case CDROMEJECT:
		return (xdf_ioctl_eject(vdp));
	case DKIOCLOCK:
		return (xdf_ioctl_mlock(vdp));
	case DKIOCUNLOCK:
		return (xdf_ioctl_munlock(vdp));
	case CDROMREADOFFSET: {
		int offset = 0;
		if (!XD_IS_CD(vdp))
			return (ENOTTY);
		if (ddi_copyout(&offset, (void *)arg, sizeof (int), mode))
			return (EFAULT);
		return (0);
	}
	case DKIOCGMEDIAINFO: {
		struct dk_minfo media_info;

		media_info.dki_lbsize = vdp->xdf_xdev_secsize;
		media_info.dki_capacity = vdp->xdf_pgeom.g_capacity;
		if (XD_IS_CD(vdp))
			media_info.dki_media_type = DK_CDROM;
		else
			media_info.dki_media_type = DK_FIXED_DISK;

		if (ddi_copyout(&media_info, (void *)arg,
		    sizeof (struct dk_minfo), mode))
			return (EFAULT);
		return (0);
	}
	case DKIOCINFO: {
		struct dk_cinfo info;

		/* controller information */
		if (XD_IS_CD(vdp))
			info.dki_ctype = DKC_CDROM;
		else
			info.dki_ctype = DKC_VBD;

		info.dki_cnum = 0;
		(void) strncpy((char *)(&info.dki_cname), "xdf", 8);

		/* unit information */
		info.dki_unit = ddi_get_instance(vdp->xdf_dip);
		(void) strncpy((char *)(&info.dki_dname), "xdf", 8);
		info.dki_flags = DKI_FMTVOL;
		info.dki_partition = part;
		info.dki_maxtransfer = maxphys / DEV_BSIZE;
		info.dki_addr = 0;
		info.dki_space = 0;
		info.dki_prio = 0;
		info.dki_vec = 0;

		if (ddi_copyout(&info, (void *)arg, sizeof (info), mode))
			return (EFAULT);
		return (0);
	}
	case DKIOCSTATE: {
		enum dkio_state mstate;

		if (ddi_copyin((void *)arg, &mstate,
		    sizeof (mstate), mode) != 0)
			return (EFAULT);
		if ((rv = xdf_dkstate(vdp, mstate)) != 0)
			return (rv);
		mstate = vdp->xdf_mstate;
		if (ddi_copyout(&mstate, (void *)arg,
		    sizeof (mstate), mode) != 0)
			return (EFAULT);
		return (0);
	}
	case DKIOCREMOVABLE: {
		int i = BOOLEAN2VOID(XD_IS_RM(vdp));
		if (ddi_copyout(&i, (caddr_t)arg, sizeof (i), mode))
			return (EFAULT);
		return (0);
	}
	case DKIOCGETWCE: {
		int i = BOOLEAN2VOID(XD_IS_RM(vdp));
		if (ddi_copyout(&i, (void *)arg, sizeof (i), mode))
			return (EFAULT);
		return (0);
	}
	case DKIOCSETWCE: {
		int i;
		if (ddi_copyin((void *)arg, &i, sizeof (i), mode))
			return (EFAULT);
		vdp->xdf_wce = VOID2BOOLEAN(i);
		return (0);
	}
	case DKIOCFLUSHWRITECACHE: {
		struct dk_callback *dkc = (struct dk_callback *)arg;

		if (vdp->xdf_flush_supported) {
			rv = xdf_lb_rdwr(vdp->xdf_dip, TG_WRITE,
			    NULL, 0, 0, (void *)dev);
		} else if (vdp->xdf_feature_barrier &&
		    !xdf_barrier_flush_disable) {
			rv = xdf_lb_rdwr(vdp->xdf_dip, TG_WRITE,
			    vdp->xdf_cache_flush_block, xdf_flush_block,
			    vdp->xdf_xdev_secsize, (void *)dev);
		} else {
			return (ENOTTY);
		}
		if ((mode & FKIOCTL) && (dkc != NULL) &&
		    (dkc->dkc_callback != NULL)) {
			(*dkc->dkc_callback)(dkc->dkc_cookie, rv);
			/* need to return 0 after calling callback */
			rv = 0;
		}
		return (rv);
	}
	}
	/*NOTREACHED*/
}

static int
xdf_strategy(struct buf *bp)
{
	xdf_t	*vdp;
	minor_t minor;
	diskaddr_t p_blkct, p_blkst;
	daddr_t blkno;
	ulong_t nblks;
	int part;

	minor = getminor(bp->b_edev);
	part = XDF_PART(minor);
	vdp = ddi_get_soft_state(xdf_ssp, XDF_INST(minor));

	mutex_enter(&vdp->xdf_dev_lk);
	if (!xdf_isopen(vdp, part)) {
		mutex_exit(&vdp->xdf_dev_lk);
		xdf_io_err(bp, ENXIO, 0);
		return (0);
	}

	/* We don't allow IO from the oe_change callback thread */
	ASSERT(curthread != vdp->xdf_oe_change_thread);

	/* Check for writes to a read only device */
	if (!IS_READ(bp) && XD_IS_RO(vdp)) {
		mutex_exit(&vdp->xdf_dev_lk);
		xdf_io_err(bp, EROFS, 0);
		return (0);
	}

	/* Check if this I/O is accessing a partition or the entire disk */
	if ((long)bp->b_private == XB_SLICE_NONE) {
		/* This I/O is using an absolute offset */
		p_blkct = vdp->xdf_xdev_nblocks;
		p_blkst = 0;
	} else {
		/* This I/O is using a partition relative offset */
		mutex_exit(&vdp->xdf_dev_lk);
		if (cmlb_partinfo(vdp->xdf_vd_lbl, part, &p_blkct,
		    &p_blkst, NULL, NULL, NULL)) {
			xdf_io_err(bp, ENXIO, 0);
			return (0);
		}
		mutex_enter(&vdp->xdf_dev_lk);
	}

	/*
	 * Adjust the real blkno and bcount according to the underline
	 * physical sector size.
	 */
	blkno = bp->b_blkno / (vdp->xdf_xdev_secsize / XB_BSIZE);

	/* check for a starting block beyond the disk or partition limit */
	if (blkno > p_blkct) {
		DPRINTF(IO_DBG, ("xdf@%s: block %lld exceeds VBD size %"PRIu64,
		    vdp->xdf_addr, (longlong_t)blkno, (uint64_t)p_blkct));
		mutex_exit(&vdp->xdf_dev_lk);
		xdf_io_err(bp, EINVAL, 0);
		return (0);
	}

	/* Legacy: don't set error flag at this case */
	if (blkno == p_blkct) {
		mutex_exit(&vdp->xdf_dev_lk);
		bp->b_resid = bp->b_bcount;
		biodone(bp);
		return (0);
	}

	/* sanitize the input buf */
	bioerror(bp, 0);
	bp->b_resid = 0;
	bp->av_back = bp->av_forw = NULL;

	/* Adjust for partial transfer, this will result in an error later */
	if (vdp->xdf_xdev_secsize != 0 &&
	    vdp->xdf_xdev_secsize != XB_BSIZE) {
		nblks = bp->b_bcount / vdp->xdf_xdev_secsize;
	} else {
		nblks = bp->b_bcount >> XB_BSHIFT;
	}

	if ((blkno + nblks) > p_blkct) {
		if (vdp->xdf_xdev_secsize != 0 &&
		    vdp->xdf_xdev_secsize != XB_BSIZE) {
			bp->b_resid =
			    ((blkno + nblks) - p_blkct) *
			    vdp->xdf_xdev_secsize;
		} else {
			bp->b_resid =
			    ((blkno + nblks) - p_blkct) <<
			    XB_BSHIFT;
		}
		bp->b_bcount -= bp->b_resid;
	}

	DPRINTF(IO_DBG, ("xdf@%s: strategy blk %lld len %lu\n",
	    vdp->xdf_addr, (longlong_t)blkno, (ulong_t)bp->b_bcount));

	/* Fix up the buf struct */
	bp->b_flags |= B_BUSY;
	bp->b_private = (void *)(uintptr_t)p_blkst;

	xdf_bp_push(vdp, bp);
	mutex_exit(&vdp->xdf_dev_lk);
	xdf_io_start(vdp);
	if (do_polled_io)
		(void) xdf_ring_drain(vdp);
	return (0);
}

/*ARGSUSED*/
static int
xdf_read(dev_t dev, struct uio *uiop, cred_t *credp)
{
	xdf_t	*vdp;
	minor_t minor;
	diskaddr_t p_blkcnt;
	int part;

	minor = getminor(dev);
	if ((vdp = ddi_get_soft_state(xdf_ssp, XDF_INST(minor))) == NULL)
		return (ENXIO);

	DPRINTF(IO_DBG, ("xdf@%s: read offset 0x%"PRIx64"\n",
	    vdp->xdf_addr, (int64_t)uiop->uio_offset));

	part = XDF_PART(minor);
	if (!xdf_isopen(vdp, part))
		return (ENXIO);

	if (cmlb_partinfo(vdp->xdf_vd_lbl, part, &p_blkcnt,
	    NULL, NULL, NULL, NULL))
		return (ENXIO);

	if (uiop->uio_loffset >= XB_DTOB(p_blkcnt, vdp))
		return (ENOSPC);

	if (U_INVAL(uiop))
		return (EINVAL);

	return (physio(xdf_strategy, NULL, dev, B_READ, xdfmin, uiop));
}

/*ARGSUSED*/
static int
xdf_write(dev_t dev, struct uio *uiop, cred_t *credp)
{
	xdf_t *vdp;
	minor_t minor;
	diskaddr_t p_blkcnt;
	int part;

	minor = getminor(dev);
	if ((vdp = ddi_get_soft_state(xdf_ssp, XDF_INST(minor))) == NULL)
		return (ENXIO);

	DPRINTF(IO_DBG, ("xdf@%s: write offset 0x%"PRIx64"\n",
	    vdp->xdf_addr, (int64_t)uiop->uio_offset));

	part = XDF_PART(minor);
	if (!xdf_isopen(vdp, part))
		return (ENXIO);

	if (cmlb_partinfo(vdp->xdf_vd_lbl, part, &p_blkcnt,
	    NULL, NULL, NULL, NULL))
		return (ENXIO);

	if (uiop->uio_loffset >= XB_DTOB(p_blkcnt, vdp))
		return (ENOSPC);

	if (U_INVAL(uiop))
		return (EINVAL);

	return (physio(xdf_strategy, NULL, dev, B_WRITE, xdfmin, uiop));
}

/*ARGSUSED*/
static int
xdf_aread(dev_t dev, struct aio_req *aiop, cred_t *credp)
{
	xdf_t	*vdp;
	minor_t minor;
	struct uio *uiop = aiop->aio_uio;
	diskaddr_t p_blkcnt;
	int part;

	minor = getminor(dev);
	if ((vdp = ddi_get_soft_state(xdf_ssp, XDF_INST(minor))) == NULL)
		return (ENXIO);

	part = XDF_PART(minor);
	if (!xdf_isopen(vdp, part))
		return (ENXIO);

	if (cmlb_partinfo(vdp->xdf_vd_lbl, part, &p_blkcnt,
	    NULL, NULL, NULL, NULL))
		return (ENXIO);

	if (uiop->uio_loffset >= XB_DTOB(p_blkcnt, vdp))
		return (ENOSPC);

	if (U_INVAL(uiop))
		return (EINVAL);

	return (aphysio(xdf_strategy, anocancel, dev, B_READ, xdfmin, aiop));
}

/*ARGSUSED*/
static int
xdf_awrite(dev_t dev, struct aio_req *aiop, cred_t *credp)
{
	xdf_t *vdp;
	minor_t minor;
	struct uio *uiop = aiop->aio_uio;
	diskaddr_t p_blkcnt;
	int part;

	minor = getminor(dev);
	if ((vdp = ddi_get_soft_state(xdf_ssp, XDF_INST(minor))) == NULL)
		return (ENXIO);

	part = XDF_PART(minor);
	if (!xdf_isopen(vdp, part))
		return (ENXIO);

	if (cmlb_partinfo(vdp->xdf_vd_lbl, part, &p_blkcnt,
	    NULL, NULL, NULL, NULL))
		return (ENXIO);

	if (uiop->uio_loffset >= XB_DTOB(p_blkcnt, vdp))
		return (ENOSPC);

	if (U_INVAL(uiop))
		return (EINVAL);

	return (aphysio(xdf_strategy, anocancel, dev, B_WRITE, xdfmin, aiop));
}

static int
xdf_dump(dev_t dev, caddr_t addr, daddr_t blkno, int nblk)
{
	struct buf dumpbuf, *dbp = &dumpbuf;
	xdf_t	*vdp;
	minor_t minor;
	int err = 0;
	int part;
	diskaddr_t p_blkcnt, p_blkst;

	minor = getminor(dev);
	if ((vdp = ddi_get_soft_state(xdf_ssp, XDF_INST(minor))) == NULL)
		return (ENXIO);

	DPRINTF(IO_DBG, ("xdf@%s: dump addr (0x%p) blk (%ld) nblks (%d)\n",
	    vdp->xdf_addr, (void *)addr, blkno, nblk));

	/* We don't allow IO from the oe_change callback thread */
	ASSERT(curthread != vdp->xdf_oe_change_thread);

	part = XDF_PART(minor);
	if (!xdf_isopen(vdp, part))
		return (ENXIO);

	if (cmlb_partinfo(vdp->xdf_vd_lbl, part, &p_blkcnt, &p_blkst,
	    NULL, NULL, NULL))
		return (ENXIO);

	if ((blkno + nblk) >
	    (p_blkcnt * (vdp->xdf_xdev_secsize / XB_BSIZE))) {
		cmn_err(CE_WARN, "xdf@%s: block %ld exceeds VBD size %"PRIu64,
		    vdp->xdf_addr, (daddr_t)((blkno + nblk) /
		    (vdp->xdf_xdev_secsize / XB_BSIZE)), (uint64_t)p_blkcnt);
		return (EINVAL);
	}

	bioinit(dbp);
	dbp->b_flags = B_BUSY;
	dbp->b_un.b_addr = addr;
	dbp->b_bcount = nblk << DEV_BSHIFT;
	dbp->b_blkno = blkno;
	dbp->b_edev = dev;
	dbp->b_private = (void *)(uintptr_t)p_blkst;

	mutex_enter(&vdp->xdf_dev_lk);
	xdf_bp_push(vdp, dbp);
	mutex_exit(&vdp->xdf_dev_lk);
	xdf_io_start(vdp);
	err = xdf_ring_drain(vdp);
	biofini(dbp);
	return (err);
}

/*ARGSUSED*/
static int
xdf_close(dev_t dev, int flag, int otyp, struct cred *credp)
{
	minor_t	minor;
	xdf_t	*vdp;
	int part;
	ulong_t parbit;

	minor = getminor(dev);
	if ((vdp = ddi_get_soft_state(xdf_ssp, XDF_INST(minor))) == NULL)
		return (ENXIO);

	mutex_enter(&vdp->xdf_dev_lk);
	part = XDF_PART(minor);
	if (!xdf_isopen(vdp, part)) {
		mutex_exit(&vdp->xdf_dev_lk);
		return (ENXIO);
	}
	parbit = 1 << part;

	ASSERT((vdp->xdf_vd_open[otyp] & parbit) != 0);
	if (otyp == OTYP_LYR) {
		ASSERT(vdp->xdf_vd_lyropen[part] > 0);
		if (--vdp->xdf_vd_lyropen[part] == 0)
			vdp->xdf_vd_open[otyp] &= ~parbit;
	} else {
		vdp->xdf_vd_open[otyp] &= ~parbit;
	}
	vdp->xdf_vd_exclopen &= ~parbit;

	mutex_exit(&vdp->xdf_dev_lk);
	return (0);
}

static int
xdf_open(dev_t *devp, int flag, int otyp, cred_t *credp)
{
	minor_t	minor;
	xdf_t	*vdp;
	int part;
	ulong_t parbit;
	diskaddr_t p_blkct = 0;
	boolean_t firstopen;
	boolean_t nodelay;

	minor = getminor(*devp);
	if ((vdp = ddi_get_soft_state(xdf_ssp, XDF_INST(minor))) == NULL)
		return (ENXIO);

	nodelay = (flag & (FNDELAY | FNONBLOCK));

	DPRINTF(DDI_DBG, ("xdf@%s: opening\n", vdp->xdf_addr));

	/* do cv_wait until connected or failed */
	mutex_enter(&vdp->xdf_cb_lk);
	mutex_enter(&vdp->xdf_dev_lk);
	if (!nodelay && (xdf_connect_locked(vdp, B_TRUE) != XD_READY)) {
		mutex_exit(&vdp->xdf_dev_lk);
		mutex_exit(&vdp->xdf_cb_lk);
		return (ENXIO);
	}
	mutex_exit(&vdp->xdf_cb_lk);

	if ((flag & FWRITE) && XD_IS_RO(vdp)) {
		mutex_exit(&vdp->xdf_dev_lk);
		return (EROFS);
	}

	part = XDF_PART(minor);
	parbit = 1 << part;
	if ((vdp->xdf_vd_exclopen & parbit) ||
	    ((flag & FEXCL) && xdf_isopen(vdp, part))) {
		mutex_exit(&vdp->xdf_dev_lk);
		return (EBUSY);
	}

	/* are we the first one to open this node? */
	firstopen = !xdf_isopen(vdp, -1);

	if (otyp == OTYP_LYR)
		vdp->xdf_vd_lyropen[part]++;

	vdp->xdf_vd_open[otyp] |= parbit;

	if (flag & FEXCL)
		vdp->xdf_vd_exclopen |= parbit;

	mutex_exit(&vdp->xdf_dev_lk);

	/* force a re-validation */
	if (firstopen)
		cmlb_invalidate(vdp->xdf_vd_lbl, NULL);

	/* If this is a non-blocking open then we're done */
	if (nodelay)
		return (0);

	/*
	 * This is a blocking open, so we require:
	 * - that the disk have a valid label on it
	 * - that the size of the partition that we're opening is non-zero
	 */
	if ((cmlb_partinfo(vdp->xdf_vd_lbl, part, &p_blkct,
	    NULL, NULL, NULL, NULL) != 0) || (p_blkct == 0)) {
		(void) xdf_close(*devp, flag, otyp, credp);
		return (ENXIO);
	}

	return (0);
}

/*ARGSUSED*/
static void
xdf_watch_hp_status_cb(dev_info_t *dip, const char *path, void *arg)
{
	xdf_t *vdp = (xdf_t *)ddi_get_driver_private(dip);
	cv_broadcast(&vdp->xdf_hp_status_cv);
}

static int
xdf_prop_op(dev_t dev, dev_info_t *dip, ddi_prop_op_t prop_op, int flags,
    char *name, caddr_t valuep, int *lengthp)
{
	xdf_t	*vdp = ddi_get_soft_state(xdf_ssp, ddi_get_instance(dip));

	/*
	 * Sanity check that if a dev_t or dip were specified that they
	 * correspond to this device driver.  On debug kernels we'll
	 * panic and on non-debug kernels we'll return failure.
	 */
	ASSERT(ddi_driver_major(dip) == xdf_major);
	ASSERT((dev == DDI_DEV_T_ANY) || (getmajor(dev) == xdf_major));
	if ((ddi_driver_major(dip) != xdf_major) ||
	    ((dev != DDI_DEV_T_ANY) && (getmajor(dev) != xdf_major)))
		return (DDI_PROP_NOT_FOUND);

	if (vdp == NULL)
		return (ddi_prop_op(dev, dip, prop_op, flags,
		    name, valuep, lengthp));

	return (cmlb_prop_op(vdp->xdf_vd_lbl,
	    dev, dip, prop_op, flags, name, valuep, lengthp,
	    XDF_PART(getminor(dev)), NULL));
}

/*ARGSUSED*/
static int
xdf_getinfo(dev_info_t *dip, ddi_info_cmd_t cmd, void *arg, void **rp)
{
	int	instance = XDF_INST(getminor((dev_t)arg));
	xdf_t	*vbdp;

	switch (cmd) {
	case DDI_INFO_DEVT2DEVINFO:
		if ((vbdp = ddi_get_soft_state(xdf_ssp, instance)) == NULL) {
			*rp = NULL;
			return (DDI_FAILURE);
		}
		*rp = vbdp->xdf_dip;
		return (DDI_SUCCESS);

	case DDI_INFO_DEVT2INSTANCE:
		*rp = (void *)(uintptr_t)instance;
		return (DDI_SUCCESS);

	default:
		return (DDI_FAILURE);
	}
}

/*ARGSUSED*/
static int
xdf_resume(dev_info_t *dip)
{
	xdf_t	*vdp;
	char	*oename;

	if ((vdp = ddi_get_soft_state(xdf_ssp, ddi_get_instance(dip))) == NULL)
		goto err;

	if (xdf_debug & SUSRES_DBG)
		xen_printf("xdf@%s: xdf_resume\n", vdp->xdf_addr);

	mutex_enter(&vdp->xdf_cb_lk);

	if (xvdi_resume(dip) != DDI_SUCCESS) {
		mutex_exit(&vdp->xdf_cb_lk);
		goto err;
	}

	if (((oename = xvdi_get_oename(dip)) == NULL) ||
	    (xvdi_add_xb_watch_handler(dip, oename, XBP_HP_STATUS,
	    xdf_watch_hp_status_cb, NULL) != DDI_SUCCESS)) {
		mutex_exit(&vdp->xdf_cb_lk);
		goto err;
	}

	mutex_enter(&vdp->xdf_dev_lk);
	ASSERT(vdp->xdf_state != XD_READY);
	xdf_set_state(vdp, XD_UNKNOWN);
	mutex_exit(&vdp->xdf_dev_lk);

	if (xdf_setstate_init(vdp) != DDI_SUCCESS) {
		mutex_exit(&vdp->xdf_cb_lk);
		goto err;
	}

	mutex_exit(&vdp->xdf_cb_lk);

	if (xdf_debug & SUSRES_DBG)
		xen_printf("xdf@%s: xdf_resume: done\n", vdp->xdf_addr);
	return (DDI_SUCCESS);
err:
	if (xdf_debug & SUSRES_DBG)
		xen_printf("xdf@%s: xdf_resume: fail\n", vdp->xdf_addr);
	return (DDI_FAILURE);
}

/*
 * Uses the in-memory devid if one exists.
 *
 * Create a devid and write it on the first block of the last track of
 * the last cylinder.
 * Return DDI_SUCCESS or DDI_FAILURE.
 */
static int
xdf_devid_fabricate(xdf_t *vdp)
{
	ddi_devid_t	devid = vdp->xdf_tgt_devid; /* null if no devid */
	struct dk_devid *dkdevidp = NULL; /* devid struct stored on disk */
	diskaddr_t	blk;
	uint_t		*ip, chksum;
	int		i, devid_size;

	if (cmlb_get_devid_block(vdp->xdf_vd_lbl, &blk, NULL) != 0)
		goto err;

	if (devid == NULL && ddi_devid_init(vdp->xdf_dip, DEVID_FAB, 0,
	    NULL, &devid) != DDI_SUCCESS)
		goto err;

	/* allocate a buffer */
	dkdevidp = (struct dk_devid *)kmem_zalloc(NBPSCTR, KM_SLEEP);

	/* Fill in the revision */
	dkdevidp->dkd_rev_hi = DK_DEVID_REV_MSB;
	dkdevidp->dkd_rev_lo = DK_DEVID_REV_LSB;

	/* Copy in the device id */
	devid_size = ddi_devid_sizeof(devid);
	if (devid_size > DK_DEVID_SIZE)
		goto err;
	bcopy(devid, dkdevidp->dkd_devid, devid_size);

	/* Calculate the chksum */
	chksum = 0;
	ip = (uint_t *)dkdevidp;
	for (i = 0; i < (NBPSCTR / sizeof (int)) - 1; i++)
		chksum ^= ip[i];

	/* Fill in the checksum */
	DKD_FORMCHKSUM(chksum, dkdevidp);

	if (xdf_lb_rdwr(vdp->xdf_dip, TG_WRITE, dkdevidp, blk,
	    NBPSCTR, NULL) != 0)
		goto err;

	kmem_free(dkdevidp, NBPSCTR);

	vdp->xdf_tgt_devid = devid;
	return (DDI_SUCCESS);

err:
	if (dkdevidp != NULL)
		kmem_free(dkdevidp, NBPSCTR);
	if (devid != NULL && vdp->xdf_tgt_devid == NULL)
		ddi_devid_free(devid);
	return (DDI_FAILURE);
}

/*
 * xdf_devid_read() is a local copy of xdfs_devid_read(), modified to use xdf
 * functions.
 *
 * Read a devid from on the first block of the last track of
 * the last cylinder.  Make sure what we read is a valid devid.
 * Return DDI_SUCCESS or DDI_FAILURE.
 */
static int
xdf_devid_read(xdf_t *vdp)
{
	diskaddr_t	blk;
	struct dk_devid *dkdevidp;
	uint_t		*ip, chksum;
	int		i;

	if (cmlb_get_devid_block(vdp->xdf_vd_lbl, &blk, NULL) != 0)
		return (DDI_FAILURE);

	dkdevidp = kmem_zalloc(NBPSCTR, KM_SLEEP);
	if (xdf_lb_rdwr(vdp->xdf_dip, TG_READ, dkdevidp, blk,
	    NBPSCTR, NULL) != 0)
		goto err;

	/* Validate the revision */
	if ((dkdevidp->dkd_rev_hi != DK_DEVID_REV_MSB) ||
	    (dkdevidp->dkd_rev_lo != DK_DEVID_REV_LSB))
		goto err;

	/* Calculate the checksum */
	chksum = 0;
	ip = (uint_t *)dkdevidp;
	for (i = 0; i < (NBPSCTR / sizeof (int)) - 1; i++)
		chksum ^= ip[i];
	if (DKD_GETCHKSUM(dkdevidp) != chksum)
		goto err;

	/* Validate the device id */
	if (ddi_devid_valid((ddi_devid_t)dkdevidp->dkd_devid) != DDI_SUCCESS)
		goto err;

	/* keep a copy of the device id */
	i = ddi_devid_sizeof((ddi_devid_t)dkdevidp->dkd_devid);
	vdp->xdf_tgt_devid = kmem_alloc(i, KM_SLEEP);
	bcopy(dkdevidp->dkd_devid, vdp->xdf_tgt_devid, i);
	kmem_free(dkdevidp, NBPSCTR);
	return (DDI_SUCCESS);

err:
	kmem_free(dkdevidp, NBPSCTR);
	return (DDI_FAILURE);
}

/*
 * xdf_devid_setup() is a modified copy of cmdk_devid_setup().
 *
 * This function creates a devid if we don't already have one, and
 * registers it.  If we already have one, we make sure that it can be
 * read from the disk, otherwise we write it to the disk ourselves.  If
 * we didn't already have a devid, and we create one, we also need to
 * register it.
 */
void
xdf_devid_setup(xdf_t *vdp)
{
	int rc;
	boolean_t existed = vdp->xdf_tgt_devid != NULL;

	/* Read devid from the disk, if present */
	rc = xdf_devid_read(vdp);

	/* Otherwise write a devid (which we create if necessary) on the disk */
	if (rc != DDI_SUCCESS)
		rc = xdf_devid_fabricate(vdp);

	/* If we created a devid or found it on the disk, register it */
	if (rc == DDI_SUCCESS && !existed)
		(void) ddi_devid_register(vdp->xdf_dip, vdp->xdf_tgt_devid);
}

static int
xdf_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
{
	int			n, instance = ddi_get_instance(dip);
	ddi_iblock_cookie_t	ibc, softibc;
	boolean_t		dev_iscd = B_FALSE;
	xdf_t			*vdp;
	char			*oename, *xsname, *str;
	clock_t			timeout;
	int			err = 0;

	if ((n = ddi_prop_get_int(DDI_DEV_T_ANY, dip, DDI_PROP_NOTPROM,
	    "xdf_debug", 0)) != 0)
		xdf_debug = n;

	switch (cmd) {
	case DDI_RESUME:
		return (xdf_resume(dip));
	case DDI_ATTACH:
		break;
	default:
		return (DDI_FAILURE);
	}
	/* DDI_ATTACH */

	if ((xsname = xvdi_get_xsname(dip)) == NULL ||
	    (oename = xvdi_get_oename(dip)) == NULL)
		return (DDI_FAILURE);

	/*
	 * Disable auto-detach.  This is necessary so that we don't get
	 * detached while we're disconnected from the back end.
	 */
	if ((ddi_prop_update_int(DDI_DEV_T_NONE, dip,
	    DDI_NO_AUTODETACH, 1) != DDI_PROP_SUCCESS))
		return (DDI_FAILURE);

	/* driver handles kernel-issued IOCTLs */
	if (ddi_prop_create(DDI_DEV_T_NONE, dip,
	    DDI_PROP_CANSLEEP, DDI_KERNEL_IOCTL, NULL, 0) != DDI_PROP_SUCCESS)
		return (DDI_FAILURE);

	if (ddi_get_iblock_cookie(dip, 0, &ibc) != DDI_SUCCESS)
		return (DDI_FAILURE);

	if (ddi_get_soft_iblock_cookie(dip,
	    DDI_SOFTINT_LOW, &softibc) != DDI_SUCCESS)
		return (DDI_FAILURE);

	if (xenbus_read_str(xsname, XBP_DEV_TYPE, &str) != 0) {
		cmn_err(CE_WARN, "xdf@%s: cannot read device-type",
		    ddi_get_name_addr(dip));
		return (DDI_FAILURE);
	}
	if (strcmp(str, XBV_DEV_TYPE_CD) == 0)
		dev_iscd = B_TRUE;
	strfree(str);

	if (ddi_soft_state_zalloc(xdf_ssp, instance) != DDI_SUCCESS)
		return (DDI_FAILURE);

	DPRINTF(DDI_DBG, ("xdf@%s: attaching\n", ddi_get_name_addr(dip)));
	vdp = ddi_get_soft_state(xdf_ssp, instance);
	ddi_set_driver_private(dip, vdp);
	vdp->xdf_dip = dip;
	vdp->xdf_addr = ddi_get_name_addr(dip);
	vdp->xdf_suspending = B_FALSE;
	vdp->xdf_media_req_supported = B_FALSE;
	vdp->xdf_peer = INVALID_DOMID;
	vdp->xdf_evtchn = INVALID_EVTCHN;
	list_create(&vdp->xdf_vreq_act, sizeof (v_req_t),
	    offsetof(v_req_t, v_link));
	cv_init(&vdp->xdf_dev_cv, NULL, CV_DEFAULT, NULL);
	cv_init(&vdp->xdf_hp_status_cv, NULL, CV_DEFAULT, NULL);
	cv_init(&vdp->xdf_mstate_cv, NULL, CV_DEFAULT, NULL);
	mutex_init(&vdp->xdf_dev_lk, NULL, MUTEX_DRIVER, (void *)ibc);
	mutex_init(&vdp->xdf_cb_lk, NULL, MUTEX_DRIVER, (void *)ibc);
	mutex_init(&vdp->xdf_iostat_lk, NULL, MUTEX_DRIVER, (void *)ibc);
	vdp->xdf_cmlb_reattach = B_TRUE;
	if (dev_iscd) {
		vdp->xdf_dinfo |= VDISK_CDROM;
		vdp->xdf_mstate = DKIO_EJECTED;
	} else {
		vdp->xdf_mstate = DKIO_NONE;
	}

	if ((vdp->xdf_ready_tq = ddi_taskq_create(dip, "xdf_ready_tq",
	    1, TASKQ_DEFAULTPRI, 0)) == NULL)
		goto errout0;

	if (xvdi_add_xb_watch_handler(dip, oename, XBP_HP_STATUS,
	    xdf_watch_hp_status_cb, NULL) != DDI_SUCCESS)
		goto errout0;

	if (ddi_add_softintr(dip, DDI_SOFTINT_LOW, &vdp->xdf_softintr_id,
	    &softibc, NULL, xdf_iorestart, (caddr_t)vdp) != DDI_SUCCESS) {
		cmn_err(CE_WARN, "xdf@%s: failed to add softintr",
		    ddi_get_name_addr(dip));
		goto errout0;
	}

	/*
	 * Initialize the physical geometry stucture.  Note that currently
	 * we don't know the size of the backend device so the number
	 * of blocks on the device will be initialized to zero.  Once
	 * we connect to the backend device we'll update the physical
	 * geometry to reflect the real size of the device.
	 */
	xdf_synthetic_pgeom(dip, &vdp->xdf_pgeom);
	vdp->xdf_pgeom_fixed = B_FALSE;

	/*
	 * Allocate the cmlb handle, minor nodes will be created once
	 * the device is connected with backend.
	 */
	cmlb_alloc_handle(&vdp->xdf_vd_lbl);

	/* We ship with cache-enabled disks */
	vdp->xdf_wce = B_TRUE;

	mutex_enter(&vdp->xdf_cb_lk);
	/* Watch backend XenbusState change */
	if (xvdi_add_event_handler(dip,
	    XS_OE_STATE, xdf_oe_change, NULL) != DDI_SUCCESS) {
		mutex_exit(&vdp->xdf_cb_lk);
		goto errout0;
	}

	if (xdf_setstate_init(vdp) != DDI_SUCCESS) {
		cmn_err(CE_WARN, "xdf@%s: start connection failed",
		    ddi_get_name_addr(dip));
		mutex_exit(&vdp->xdf_cb_lk);
		goto errout1;
	}

	/* Nothing else to do for CD devices */
	if (dev_iscd) {
		mutex_exit(&vdp->xdf_cb_lk);
		goto done;
	}

	/*
	 * In order to do cmlb_validate, we have to wait for the disk to
	 * acknowledge the attach, so we can query the backend for the disk
	 * geometry (see xdf_setstate_connected).
	 *
	 * We only wait 30 seconds; if this is the root disk, the boot
	 * will fail, but it would fail anyway if the device never
	 * connected.  If this is a non-boot disk, that disk will fail
	 * to connect, but again, it would fail anyway.
	 */
	timeout = ddi_get_lbolt() + drv_usectohz(XDF_STATE_TIMEOUT);
	while (vdp->xdf_state != XD_CONNECTED && vdp->xdf_state != XD_READY) {
		if (cv_timedwait(&vdp->xdf_dev_cv, &vdp->xdf_cb_lk,
		    timeout) < 0) {
			cmn_err(CE_WARN, "xdf@%s: disk failed to connect",
			    ddi_get_name_addr(dip));
			mutex_exit(&vdp->xdf_cb_lk);
			goto errout1;
		}
	}
	mutex_exit(&vdp->xdf_cb_lk);

	/*
	 * We call cmlb_validate so that the geometry information in
	 * vdp->xdf_vd_lbl is correct; this fills out the number of
	 * alternate cylinders so that we have a place to write the
	 * devid.
	 */
	if ((err = cmlb_validate(vdp->xdf_vd_lbl, 0, NULL)) != 0) {
		cmn_err(CE_NOTE,
		    "xdf@%s: cmlb_validate failed: %d",
		    ddi_get_name_addr(dip), err);
		/*
		 * We can carry on even if cmlb_validate() returns EINVAL here,
		 * as we'll rewrite the disk label anyway.
		 */
		if (err != EINVAL)
			goto errout1;
	}

	/*
	 * xdf_devid_setup will only write a devid if one isn't
	 * already present.  If it fails to find or create one, we
	 * create one in-memory so that when we label the disk later,
	 * it will have a devid to use.  This is helpful to deal with
	 * cases where people use the devids of their disks before
	 * labelling them; note that this does cause problems if
	 * people rely on the devids of unlabelled disks to persist
	 * across reboot.
	 */
	xdf_devid_setup(vdp);
	if (vdp->xdf_tgt_devid == NULL) {
		if (ddi_devid_init(vdp->xdf_dip, DEVID_FAB, 0, NULL,
		    &vdp->xdf_tgt_devid) != DDI_SUCCESS) {
			cmn_err(CE_WARN,
			    "xdf@%s_ attach failed, devid_init failed",
			    ddi_get_name_addr(dip));
			goto errout1;
		} else {
			(void) ddi_devid_register(vdp->xdf_dip,
			    vdp->xdf_tgt_devid);
		}
	}

done:
#ifdef XPV_HVM_DRIVER
	xdf_hvm_add(dip);

	/* Report our version to dom0 */
	(void) xenbus_printf(XBT_NULL, "guest/xdf", "version", "%d",
	    HVMPV_XDF_VERS);
#endif /* XPV_HVM_DRIVER */

	/* Create kstat for iostat(8) */
	if (xdf_kstat_create(dip) != 0) {
		cmn_err(CE_WARN, "xdf@%s: failed to create kstat",
		    ddi_get_name_addr(dip));
		goto errout1;
	}

	/*
	 * Don't bother with getting real device identification
	 * strings (is it even possible?), they are unlikely to
	 * change often (if at all).
	 */
	(void) ndi_prop_update_string(DDI_DEV_T_NONE, dip, INQUIRY_VENDOR_ID,
	    "Xen");
	(void) ndi_prop_update_string(DDI_DEV_T_NONE, dip, INQUIRY_PRODUCT_ID,
	    dev_iscd ? "Virtual CD" : "Virtual disk");
	(void) ndi_prop_update_string(DDI_DEV_T_NONE, dip, INQUIRY_REVISION_ID,
	    "1.0");

	ddi_report_dev(dip);
	DPRINTF(DDI_DBG, ("xdf@%s: attached\n", vdp->xdf_addr));
	return (DDI_SUCCESS);

errout1:
	(void) xvdi_switch_state(vdp->xdf_dip, XBT_NULL, XenbusStateClosed);
	xvdi_remove_event_handler(dip, XS_OE_STATE);
errout0:
	if (vdp->xdf_vd_lbl != NULL) {
		cmlb_free_handle(&vdp->xdf_vd_lbl);
		vdp->xdf_vd_lbl = NULL;
	}
	if (vdp->xdf_softintr_id != NULL)
		ddi_remove_softintr(vdp->xdf_softintr_id);
	xvdi_remove_xb_watch_handlers(dip);
	if (vdp->xdf_ready_tq != NULL)
		ddi_taskq_destroy(vdp->xdf_ready_tq);
	mutex_destroy(&vdp->xdf_cb_lk);
	mutex_destroy(&vdp->xdf_dev_lk);
	cv_destroy(&vdp->xdf_dev_cv);
	cv_destroy(&vdp->xdf_hp_status_cv);
	ddi_soft_state_free(xdf_ssp, instance);
	ddi_set_driver_private(dip, NULL);
	ddi_prop_remove_all(dip);
	cmn_err(CE_WARN, "xdf@%s: attach failed", ddi_get_name_addr(dip));
	return (DDI_FAILURE);
}

static int
xdf_suspend(dev_info_t *dip)
{
	int		instance = ddi_get_instance(dip);
	xdf_t		*vdp;

	if ((vdp = ddi_get_soft_state(xdf_ssp, instance)) == NULL)
		return (DDI_FAILURE);

	if (xdf_debug & SUSRES_DBG)
		xen_printf("xdf@%s: xdf_suspend\n", vdp->xdf_addr);

	xvdi_suspend(dip);

	mutex_enter(&vdp->xdf_cb_lk);
	mutex_enter(&vdp->xdf_dev_lk);

	vdp->xdf_suspending = B_TRUE;
	xdf_ring_destroy(vdp);
	xdf_set_state(vdp, XD_SUSPEND);
	vdp->xdf_suspending = B_FALSE;

	mutex_exit(&vdp->xdf_dev_lk);
	mutex_exit(&vdp->xdf_cb_lk);

	if (xdf_debug & SUSRES_DBG)
		xen_printf("xdf@%s: xdf_suspend: done\n", vdp->xdf_addr);

	return (DDI_SUCCESS);
}

static int
xdf_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
{
	xdf_t *vdp;
	int instance;

	switch (cmd) {

	case DDI_PM_SUSPEND:
		break;

	case DDI_SUSPEND:
		return (xdf_suspend(dip));

	case DDI_DETACH:
		break;

	default:
		return (DDI_FAILURE);
	}

	instance = ddi_get_instance(dip);
	DPRINTF(DDI_DBG, ("xdf@%s: detaching\n", ddi_get_name_addr(dip)));
	vdp = ddi_get_soft_state(xdf_ssp, instance);

	if (vdp == NULL)
		return (DDI_FAILURE);

	mutex_enter(&vdp->xdf_cb_lk);
	xdf_disconnect(vdp, XD_CLOSED, B_FALSE);
	if (vdp->xdf_state != XD_CLOSED) {
		mutex_exit(&vdp->xdf_cb_lk);
		return (DDI_FAILURE);
	}
	mutex_exit(&vdp->xdf_cb_lk);

	ASSERT(!ISDMACBON(vdp));

#ifdef XPV_HVM_DRIVER
	xdf_hvm_rm(dip);
#endif /* XPV_HVM_DRIVER */

	if (vdp->xdf_timeout_id != 0)
		(void) untimeout(vdp->xdf_timeout_id);

	xvdi_remove_event_handler(dip, XS_OE_STATE);
	ddi_taskq_destroy(vdp->xdf_ready_tq);

	cmlb_detach(vdp->xdf_vd_lbl, NULL);
	cmlb_free_handle(&vdp->xdf_vd_lbl);

	/* we'll support backend running in domU later */
#ifdef	DOMU_BACKEND
	(void) xvdi_post_event(dip, XEN_HP_REMOVE);
#endif

	list_destroy(&vdp->xdf_vreq_act);
	ddi_prop_remove_all(dip);
	xdf_kstat_delete(dip);
	ddi_remove_softintr(vdp->xdf_softintr_id);
	xvdi_remove_xb_watch_handlers(dip);
	ddi_set_driver_private(dip, NULL);
	cv_destroy(&vdp->xdf_dev_cv);
	mutex_destroy(&vdp->xdf_cb_lk);
	mutex_destroy(&vdp->xdf_dev_lk);
	if (vdp->xdf_cache_flush_block != NULL)
		kmem_free(vdp->xdf_flush_mem, 2 * vdp->xdf_xdev_secsize);
	ddi_soft_state_free(xdf_ssp, instance);
	return (DDI_SUCCESS);
}

/*
 * Driver linkage structures.
 */
static struct cb_ops xdf_cbops = {
	xdf_open,
	xdf_close,
	xdf_strategy,
	nodev,
	xdf_dump,
	xdf_read,
	xdf_write,
	xdf_ioctl,
	nodev,
	nodev,
	nodev,
	nochpoll,
	xdf_prop_op,
	NULL,
	D_MP | D_NEW | D_64BIT,
	CB_REV,
	xdf_aread,
	xdf_awrite
};

struct dev_ops xdf_devops = {
	DEVO_REV,		/* devo_rev */
	0,			/* devo_refcnt */
	xdf_getinfo,		/* devo_getinfo */
	nulldev,		/* devo_identify */
	nulldev,		/* devo_probe */
	xdf_attach,		/* devo_attach */
	xdf_detach,		/* devo_detach */
	nodev,			/* devo_reset */
	&xdf_cbops,		/* devo_cb_ops */
	NULL,			/* devo_bus_ops */
	NULL,			/* devo_power */
	ddi_quiesce_not_supported, /* devo_quiesce */
};

/*
 * Module linkage structures.
 */
static struct modldrv modldrv = {
	&mod_driverops,		/* Type of module.  This one is a driver */
	"virtual block driver",	/* short description */
	&xdf_devops		/* driver specific ops */
};

static struct modlinkage xdf_modlinkage = {
	MODREV_1, (void *)&modldrv, NULL
};

/*
 * standard module entry points
 */
int
_init(void)
{
	int rc;

	xdf_major = ddi_name_to_major("xdf");
	if (xdf_major == (major_t)-1)
		return (EINVAL);

	if ((rc = ddi_soft_state_init(&xdf_ssp, sizeof (xdf_t), 0)) != 0)
		return (rc);

	xdf_vreq_cache = kmem_cache_create("xdf_vreq_cache",
	    sizeof (v_req_t), 0, NULL, NULL, NULL, NULL, NULL, 0);
	xdf_gs_cache = kmem_cache_create("xdf_gs_cache",
	    sizeof (ge_slot_t), 0, NULL, NULL, NULL, NULL, NULL, 0);

#ifdef XPV_HVM_DRIVER
	xdf_hvm_init();
#endif /* XPV_HVM_DRIVER */

	if ((rc = mod_install(&xdf_modlinkage)) != 0) {
#ifdef XPV_HVM_DRIVER
		xdf_hvm_fini();
#endif /* XPV_HVM_DRIVER */
		kmem_cache_destroy(xdf_vreq_cache);
		kmem_cache_destroy(xdf_gs_cache);
		ddi_soft_state_fini(&xdf_ssp);
		return (rc);
	}

	return (rc);
}

int
_fini(void)
{
	int err;
	if ((err = mod_remove(&xdf_modlinkage)) != 0)
		return (err);

#ifdef XPV_HVM_DRIVER
	xdf_hvm_fini();
#endif /* XPV_HVM_DRIVER */

	kmem_cache_destroy(xdf_vreq_cache);
	kmem_cache_destroy(xdf_gs_cache);
	ddi_soft_state_fini(&xdf_ssp);

	return (0);
}

int
_info(struct modinfo *modinfop)
{
	return (mod_info(&xdf_modlinkage, modinfop));
}
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

/*
 * Copyright (c) 2014 by Delphix. All rights reserved.
 * Copyright 2017 Nexenta Systems, Inc.
 */

#ifndef _SYS_XDF_H
#define	_SYS_XDF_H

#include <sys/ddi.h>
#include <sys/sunddi.h>
#include <sys/cmlb.h>
#include <sys/dkio.h>

#include <sys/gnttab.h>
#include <xen/sys/xendev.h>

#ifdef __cplusplus
extern "C" {
#endif


/*
 * VBDs have standard 512 byte blocks
 * A single blkif_request can transfer up to 11 pages of data, 1 page/segment
 */
#define	XB_BSIZE	DEV_BSIZE
#define	XB_BMASK	(XB_BSIZE - 1)
#define	XB_BSHIFT	9
#define	XB_DTOB(bn, vdp)	((bn) * (vdp)->xdf_xdev_secsize)

#define	XB_MAX_SEGLEN	(8 * XB_BSIZE)
#define	XB_SEGOFFSET	(XB_MAX_SEGLEN - 1)
#define	XB_MAX_XFER	(XB_MAX_SEGLEN * BLKIF_MAX_SEGMENTS_PER_REQUEST)
#define	XB_MAXPHYS	(XB_MAX_XFER * BLKIF_RING_SIZE)

/* Number of sectors per segement */
#define	XB_NUM_SECTORS_PER_SEG	(PAGESIZE / XB_BSIZE)
/* sectors are number 0 through XB_NUM_SECTORS_PER_SEG - 1 */
#define	XB_LAST_SECTOR_IN_SEG	(XB_NUM_SECTORS_PER_SEG - 1)


/*
 * Slice for absolute disk transaction.
 *
 * Hack Alert.  XB_SLICE_NONE is a magic value that can be written into the
 * b_private field of buf structures passed to xdf_strategy().  When present
 * it indicates that the I/O is using an absolute offset.  (ie, the I/O is
 * not bound to any one partition.)  This magic value is currently used by
 * the pv_cmdk driver.  This hack is shamelessly stolen from the sun4v vdc
 * driver, another virtual disk device driver.  (Although in the case of
 * vdc the hack is less egregious since it is self contained within the
 * vdc driver, where as here it is used as an interface between the pv_cmdk
 * driver and the xdf driver.)
 */
#define	XB_SLICE_NONE		0xFF

/*
 * blkif status
 */
typedef enum xdf_state {
	/*
	 * initial state
	 */
	XD_UNKNOWN = 0,
	/*
	 * ring and evtchn alloced, xenbus state changed to
	 * XenbusStateInitialised, wait for backend to connect
	 */
	XD_INIT = 1,
	/*
	 * backend and frontend xenbus state has changed to
	 * XenbusStateConnected.  IO is now allowed, but we are not still
	 * fully initialized.
	 */
	XD_CONNECTED = 2,
	/*
	 * We're fully initialized and allowing regular IO.
	 */
	XD_READY = 3,
	/*
	 * vbd interface close request received from backend, no more I/O
	 * requestis allowed to be put into ring buffer, while interrupt handler
	 * is allowed to run to finish any outstanding I/O request, disconnect
	 * process is kicked off by changing xenbus state to XenbusStateClosed
	 */
	XD_CLOSING = 4,
	/*
	 * disconnection process finished, both backend and frontend's
	 * xenbus state has been changed to XenbusStateClosed, can be detached
	 */
	XD_CLOSED = 5,
	/*
	 * We're either being suspended or resuming from a suspend.  If we're
	 * in the process of suspending, we block all new IO, but but allow
	 * existing IO to drain.
	 */
	XD_SUSPEND = 6
} xdf_state_t;

/*
 * 16 partitions + fdisk
 */
#define	XDF_PSHIFT	6
#define	XDF_PMASK	((1 << XDF_PSHIFT) - 1)
#define	XDF_PEXT	(1 << XDF_PSHIFT)
#define	XDF_MINOR(i, m) (((i) << XDF_PSHIFT) | (m))
#define	XDF_INST(m)	((m) >> XDF_PSHIFT)
#define	XDF_PART(m)	((m) & XDF_PMASK)

/*
 * one blkif_request_t will have one corresponding ge_slot_t
 * where we save those grant table refs used in this blkif_request_t
 *
 * the id of this ge_slot_t will also be put into 'id' field in
 * each blkif_request_t when sent out to the ring buffer.
 */
typedef struct ge_slot {
	list_node_t	gs_vreq_link;
	struct v_req	*gs_vreq;
	domid_t		gs_oeid;
	int		gs_isread;
	grant_ref_t	gs_ghead;
	int		gs_ngrefs;
	grant_ref_t	gs_ge[BLKIF_MAX_SEGMENTS_PER_REQUEST];
} ge_slot_t;

/*
 * vbd I/O request
 *
 * An instance of this structure is bound to each buf passed to
 * the driver's strategy by setting the pointer into bp->av_back.
 * The id of this vreq will also be put into 'id' field in each
 * blkif_request_t when sent out to the ring buffer for one DMA
 * window of this buf.
 *
 * Vreq mainly contains DMA information for this buf. In one vreq/buf,
 * there could be more than one DMA window, each of which will be
 * mapped to one blkif_request_t/ge_slot_t. Ge_slot_t contains all grant
 * table entry information for this buf. The ge_slot_t for current DMA
 * window is pointed to by v_gs in vreq.
 *
 * So, grant table entries will only be alloc'ed when the DMA window is
 * about to be transferred via blkif_request_t to the ring buffer. And
 * they will be freed right after the blkif_response_t is seen. By this
 * means, we can make use of grant table entries more efficiently.
 */
typedef struct v_req {
	list_node_t	v_link;
	list_t		v_gs;
	int		v_status;
	buf_t		*v_buf;
	uint_t		v_ndmacs;
	uint_t		v_dmaw;
	uint_t		v_ndmaws;
	uint_t		v_nslots;
	uint64_t	v_blkno;
	ddi_dma_handle_t v_memdmahdl;
	ddi_acc_handle_t v_align;
	ddi_dma_handle_t v_dmahdl;
	ddi_dma_cookie_t v_dmac;
	caddr_t		v_abuf;
	uint8_t		v_flush_diskcache;
	boolean_t	v_runq;
} v_req_t;

/*
 * Status set and checked in vreq->v_status by vreq_setup()
 *
 * These flags will help us to continue the vreq setup work from last failure
 * point, instead of starting from scratch after each failure.
 */
#define	VREQ_INIT		0x0
#define	VREQ_INIT_DONE		0x1
#define	VREQ_DMAHDL_ALLOCED	0x2
#define	VREQ_MEMDMAHDL_ALLOCED	0x3
#define	VREQ_DMAMEM_ALLOCED	0x4
#define	VREQ_DMABUF_BOUND	0x5
#define	VREQ_GS_ALLOCED		0x6
#define	VREQ_DMAWIN_DONE	0x7

/*
 * virtual block device per-instance softstate
 */
typedef struct xdf {
	dev_info_t	*xdf_dip;
	char		*xdf_addr;
	ddi_iblock_cookie_t xdf_ibc; /* mutex iblock cookie */
	domid_t		xdf_peer; /* otherend's dom ID */
	xendev_ring_t	*xdf_xb_ring; /* I/O ring buffer */
	ddi_acc_handle_t xdf_xb_ring_hdl; /* access handler for ring buffer */
	list_t		xdf_vreq_act; /* active vreq list */
	buf_t		*xdf_f_act; /* active buf list head */
	buf_t		*xdf_l_act; /* active buf list tail */
	buf_t		*xdf_i_act; /* active buf list index */
	xdf_state_t	xdf_state; /* status of this virtual disk */
	boolean_t	xdf_suspending;
	ulong_t		xdf_vd_open[OTYPCNT];
	ulong_t		xdf_vd_lyropen[XDF_PEXT];
	ulong_t		xdf_connect_req;
	kthread_t	*xdf_connect_thread;
	ulong_t		xdf_vd_exclopen;
	kmutex_t	xdf_iostat_lk; /* muxes lock for the iostat ptr */
	kmutex_t	xdf_dev_lk; /* mutex lock for I/O path */
	kmutex_t	xdf_cb_lk; /* mutex lock for event handling path */
	kcondvar_t	xdf_dev_cv; /* cv used in I/O path */
	uint_t		xdf_dinfo; /* disk info from backend xenstore */
	diskaddr_t	xdf_xdev_nblocks; /* total size in block */
	uint_t		xdf_xdev_secsize; /* disk blksize from backend */
	cmlb_geom_t	xdf_pgeom;
	boolean_t	xdf_pgeom_set;
	boolean_t	xdf_pgeom_fixed;
	kstat_t		*xdf_xdev_iostat;
	cmlb_handle_t	xdf_vd_lbl;
	ddi_softintr_t	xdf_softintr_id;
	timeout_id_t	xdf_timeout_id;
	struct gnttab_free_callback xdf_gnt_callback;
	boolean_t	xdf_feature_barrier;
	boolean_t	xdf_flush_supported;
	boolean_t	xdf_media_req_supported;
	boolean_t	xdf_wce;
	boolean_t	xdf_cmlb_reattach;
	char		*xdf_flush_mem;
	char		*xdf_cache_flush_block;
	int		xdf_evtchn;
	enum dkio_state	xdf_mstate;
	kcondvar_t	xdf_mstate_cv;
	kcondvar_t	xdf_hp_status_cv;
	struct buf	*xdf_ready_bp;
	ddi_taskq_t	*xdf_ready_tq;
	kthread_t	*xdf_ready_tq_thread;
	struct buf	*xdf_ready_tq_bp;
	ddi_devid_t	xdf_tgt_devid;
#ifdef	DEBUG
	int		xdf_dmacallback_num;
	kthread_t	*xdf_oe_change_thread;
#endif
} xdf_t;

/*
 * VBD I/O requests must be aligned on a 512-byte boundary and specify
 * a transfer size which is a mutiple of 512-bytes
 */
#define	ALIGNED_XFER(bp) \
	((((uintptr_t)((bp)->b_un.b_addr) & XB_BMASK) == 0) && \
	(((bp)->b_bcount & XB_BMASK) == 0))

#define	U_INVAL(u)	(((u)->uio_loffset & (offset_t)(XB_BMASK)) || \
	((u)->uio_iov->iov_len & (offset_t)(XB_BMASK)))

/* wrap pa_to_ma() for xdf to run in dom0 */
#define	PATOMA(addr)	(DOMAIN_IS_INITDOMAIN(xen_info) ? addr : pa_to_ma(addr))

#define	XD_IS_RO(vbd)	VOID2BOOLEAN((vbd)->xdf_dinfo & VDISK_READONLY)
#define	XD_IS_CD(vbd)	VOID2BOOLEAN((vbd)->xdf_dinfo & VDISK_CDROM)
#define	XD_IS_RM(vbd)	VOID2BOOLEAN((vbd)->xdf_dinfo & VDISK_REMOVABLE)
#define	IS_READ(bp)	VOID2BOOLEAN((bp)->b_flags & B_READ)
#define	IS_ERROR(bp)	VOID2BOOLEAN((bp)->b_flags & B_ERROR)

#define	XDF_UPDATE_IO_STAT(vdp, bp)					\
	{								\
		kstat_io_t *kip = KSTAT_IO_PTR((vdp)->xdf_xdev_iostat);	\
		size_t n_done = (bp)->b_bcount - (bp)->b_resid;		\
		if ((bp)->b_flags & B_READ) {				\
			kip->reads++;					\
			kip->nread += n_done;				\
		} else {                                                \
			kip->writes++;					\
			kip->nwritten += n_done;			\
		}							\
	}

#ifdef DEBUG
#define	DPRINTF(flag, args)	{if (xdf_debug & (flag)) prom_printf args; }
#define	SETDMACBON(vbd)		{(vbd)->xdf_dmacallback_num++; }
#define	SETDMACBOFF(vbd)	{(vbd)->xdf_dmacallback_num--; }
#define	ISDMACBON(vbd)		((vbd)->xdf_dmacallback_num > 0)
#else
#define	DPRINTF(flag, args)
#define	SETDMACBON(vbd)
#define	SETDMACBOFF(vbd)
#define	ISDMACBON(vbd)
#endif /* DEBUG */

#define	DDI_DBG		0x1
#define	DMA_DBG		0x2
#define	INTR_DBG	0x8
#define	IO_DBG		0x10
#define	IOCTL_DBG	0x20
#define	SUSRES_DBG	0x40
#define	LBL_DBG		0x80

#ifdef XPV_HVM_DRIVER
extern int xdf_lb_getinfo(dev_info_t *, int, void *, void *);
extern int xdf_lb_rdwr(dev_info_t *, uchar_t, void *, diskaddr_t, size_t,
    void *);
extern void xdfmin(struct buf *bp);
extern dev_info_t *xdf_hvm_hold(const char *);
extern boolean_t xdf_hvm_connect(dev_info_t *);
extern int xdf_hvm_setpgeom(dev_info_t *, cmlb_geom_t *);
extern boolean_t xdf_is_cd(dev_info_t *);
extern boolean_t xdf_is_rm(dev_info_t *);
extern boolean_t xdf_media_req_supported(dev_info_t *);
#endif /* XPV_HVM_DRIVER */

#ifdef __cplusplus
}
#endif

#endif	/* _SYS_XDF_H */
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

/*
 * Client-facing interface for the Xenbus driver.  In other words, the
 * interface between the Xenbus and the device-specific code, be it the
 * frontend or the backend of that driver.
 *
 * Copyright (C) 2005 XenSource Ltd
 *
 * This file may be distributed separately from the Linux kernel, or
 * incorporated into other software packages, subject to the following license:
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this source file (the "Software"), to deal in the Software without
 * restriction, including without limitation the rights to use, copy, modify,
 * merge, publish, distribute, sublicense, and/or sell copies of the Software,
 * and to permit persons to whom the Software is furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 */

#ifdef XPV_HVM_DRIVER
#include <sys/xpv_support.h>
#include <sys/hypervisor.h>
#else
#include <sys/hypervisor.h>
#include <sys/xen_mmu.h>
#include <sys/evtchn_impl.h>
#endif
#include <sys/gnttab.h>
#include <xen/sys/xenbus_impl.h>
#include <sys/cmn_err.h>


int
xenbus_watch_path(struct xenbus_device *dev, const char *path,
    struct xenbus_watch *watch,
    void (*callback)(struct xenbus_watch *,
    const char **, unsigned int))
{
	int err;

	watch->node = path;
	watch->callback = callback;

	err = register_xenbus_watch(watch);

	if (err) {
		watch->node = NULL;
		watch->callback = NULL;
		xenbus_dev_fatal(dev, err, "adding watch on %s", path);
	}

	return (err);
}


int
xenbus_watch_path2(struct xenbus_device *dev, const char *path,
    const char *path2, struct xenbus_watch *watch,
    void (*callback)(struct xenbus_watch *,
    const char **, unsigned int))
{
	int err;
	char *state;

	state = kmem_alloc(strlen(path) + 1 + strlen(path2) + 1, KM_SLEEP);
	(void) strcpy(state, path);
	(void) strcat(state, "/");
	(void) strcat(state, path2);

	err = xenbus_watch_path(dev, state, watch, callback);
	if (err)
		kmem_free(state, strlen(state) + 1);
	return (err);
}

/*
 * Returns 0 on success, -1 if no change was made, or an errno on failure.  We
 * check whether the state is currently set to the given value, and if not,
 * then the state is set.  We don't want to unconditionally write the given
 * state, because we don't want to fire watches unnecessarily.  Furthermore, if
 * the node has gone, we don't write to it, as the device will be tearing down,
 * and we don't want to resurrect that directory.
 *
 * XXPV: not clear that this is still safe if two threads are racing to update
 * the state?
 */
int
xenbus_switch_state(struct xenbus_device *dev, xenbus_transaction_t xbt,
    XenbusState state)
{
	int current_state;
	int err;

	err = xenbus_scanf(xbt, dev->nodename, "state", "%d", &current_state);

	/* XXPV: is this really the right thing to do? */
	if (err == ENOENT)
		return (0);
	if (err)
		return (err);

	err = -1;

	if ((XenbusState)current_state != state) {
		err = xenbus_printf(xbt, dev->nodename, "state", "%d", state);
		if (err)
			xenbus_dev_fatal(dev, err, "writing new state");
	}

	return (err);
}


/*
 * Return the path to the error node for the given device, or NULL on failure.
 * If the value returned is non-NULL, then it is the caller's to kmem_free.
 */
static char *
error_path(struct xenbus_device *dev)
{
	char *path_buffer;

	path_buffer = kmem_alloc(strlen("error/") + strlen(dev->nodename) +
	    1, KM_SLEEP);

	(void) strcpy(path_buffer, "error/");
	(void) strcpy(path_buffer + strlen("error/"), dev->nodename);

	return (path_buffer);
}

static void
common_dev_error(struct xenbus_device *dev, int err, const char *fmt,
    va_list ap)
{
	int ret;
	unsigned int len;
	char *printf_buffer = NULL, *path_buffer = NULL;

#define	PRINTF_BUFFER_SIZE 4096
	printf_buffer = kmem_alloc(PRINTF_BUFFER_SIZE, KM_SLEEP);

	(void) snprintf(printf_buffer, PRINTF_BUFFER_SIZE, "%d ", err);
	len = strlen(printf_buffer);
	ret = vsnprintf(printf_buffer+len, PRINTF_BUFFER_SIZE-len, fmt, ap);

	ASSERT(len + ret <= PRINTF_BUFFER_SIZE-1);
	dev->has_error = 1;

	path_buffer = error_path(dev);

	if (path_buffer == NULL) {
		printf("xenbus: failed to write error node for %s (%s)\n",
		    dev->nodename, printf_buffer);
		goto fail;
	}

	if (xenbus_write(0, path_buffer, "error", printf_buffer) != 0) {
		printf("xenbus: failed to write error node for %s (%s)\n",
		    dev->nodename, printf_buffer);
		goto fail;
	}

fail:
	if (printf_buffer)
		kmem_free(printf_buffer, PRINTF_BUFFER_SIZE);
	if (path_buffer)
		kmem_free(path_buffer, strlen(path_buffer) + 1);
}


void
xenbus_dev_error(struct xenbus_device *dev, int err, const char *fmt, ...)
{
	va_list ap;

	va_start(ap, fmt);
	common_dev_error(dev, err, fmt, ap);
	va_end(ap);
}


void
xenbus_dev_fatal(struct xenbus_device *dev, int err, const char *fmt, ...)
{
	va_list ap;

	va_start(ap, fmt);
	common_dev_error(dev, err, fmt, ap);
	va_end(ap);

	(void) xenbus_switch_state(dev, XBT_NULL, XenbusStateClosing);
}

/* Clear any error. */
void
xenbus_dev_ok(struct xenbus_device *dev)
{
	if (dev->has_error) {
		if (xenbus_rm(0, dev->nodename, "error") != 0)
			printf("xenbus: failed to clear error node for %s\n",
			    dev->nodename);
		else
			dev->has_error = 0;
	}
}

int
xenbus_grant_ring(struct xenbus_device *dev, unsigned long ring_mfn)
{
	int err = gnttab_grant_foreign_access(dev->otherend_id, ring_mfn, 0);
	if (err < 0)
		xenbus_dev_fatal(dev, err, "granting access to ring page");
	return (err);
}


int
xenbus_alloc_evtchn(struct xenbus_device *dev, int *port)
{
	int err;

	err = xen_alloc_unbound_evtchn(dev->otherend_id, port);
	if (err)
		xenbus_dev_fatal(dev, err, "allocating event channel");
	return (err);
}


XenbusState
xenbus_read_driver_state(const char *path)
{
	XenbusState result;

	int err = xenbus_gather(XBT_NULL, path, "state", "%d", &result, NULL);
	if (err)
		result = XenbusStateClosed;

	return (result);
}


/*
 * Local variables:
 *  c-file-style: "solaris"
 *  indent-tabs-mode: t
 *  c-indent-level: 8
 *  c-basic-offset: 8
 *  tab-width: 8
 * End:
 */
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

/*
 *
 * xenbus_comms.c
 *
 * Low level code to talks to Xen Store: ringbuffer and event channel.
 *
 * Copyright (C) 2005 Rusty Russell, IBM Corporation
 *
 * This file may be distributed separately from the Linux kernel, or
 * incorporated into other software packages, subject to the following license:
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this source file (the "Software"), to deal in the Software without
 * restriction, including without limitation the rights to use, copy, modify,
 * merge, publish, distribute, sublicense, and/or sell copies of the Software,
 * and to permit persons to whom the Software is furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 */

#include <sys/types.h>
#include <vm/hat.h>
#include <vm/as.h>
#include <sys/bootconf.h>
#include <vm/seg_kmem.h>
#ifdef XPV_HVM_DRIVER
#include <sys/pc_mmu.h>
#include <sys/xpv_support.h>
#include <sys/hypervisor.h>
#else
#include <vm/kboot_mmu.h>
#include <sys/bootinfo.h>
#include <sys/hypervisor.h>
#include <sys/evtchn_impl.h>
#endif
#include <sys/condvar.h>
#include <sys/mutex.h>
#include <sys/atomic.h>
#include <sys/mman.h>
#include <sys/errno.h>
#include <sys/cmn_err.h>
#include <sys/avintr.h>
#include <xen/sys/xenbus_comms.h>
#include <xen/public/io/xs_wire.h>

#ifndef XPV_HVM_DRIVER
static int xenbus_irq;
#endif
static ddi_umem_cookie_t xb_cookie; /* cookie for xenbus comm page */
extern caddr_t xb_addr;	/* va of xenbus comm page */

static kcondvar_t xb_wait_cv;
static kmutex_t xb_wait_lock;

#define	xs_domain_interface(ra) ((struct xenstore_domain_interface *)(ra))

static uint_t
xenbus_intr(caddr_t arg __unused, caddr_t arg1 __unused)
{
	mutex_enter(&xb_wait_lock);
	cv_broadcast(&xb_wait_cv);
	mutex_exit(&xb_wait_lock);
	return (DDI_INTR_CLAIMED);
}

static int
check_indexes(XENSTORE_RING_IDX cons, XENSTORE_RING_IDX prod)
{
	return ((prod - cons) <= XENSTORE_RING_SIZE);
}

static void *
get_output_chunk(XENSTORE_RING_IDX cons, XENSTORE_RING_IDX prod,
    char *buf, uint32_t *len)
{
	*len = XENSTORE_RING_SIZE - MASK_XENSTORE_IDX(prod);
	if ((XENSTORE_RING_SIZE - (prod - cons)) < *len)
		*len = XENSTORE_RING_SIZE - (prod - cons);
	return ((void *)(buf + MASK_XENSTORE_IDX(prod)));
}

static const void *
get_input_chunk(XENSTORE_RING_IDX cons, XENSTORE_RING_IDX prod,
    const char *buf, uint32_t *len)
{
	*len = XENSTORE_RING_SIZE - MASK_XENSTORE_IDX(cons);
	if ((prod - cons) < *len)
		*len = prod - cons;
	return ((void *)(buf + MASK_XENSTORE_IDX(cons)));
}


int
xb_write(const void *data, unsigned len)
{
	volatile struct xenstore_domain_interface *intf =
	    xs_domain_interface(xb_addr);
	XENSTORE_RING_IDX cons, prod;
	extern int do_polled_io;

	while (len != 0) {
		void *dst;
		unsigned int avail;

		mutex_enter(&xb_wait_lock);
		while ((intf->req_prod - intf->req_cons) ==
		    XENSTORE_RING_SIZE) {
			if (interrupts_unleashed && !do_polled_io) {
				if (cv_wait_sig(&xb_wait_cv,
				    &xb_wait_lock) == 0) {
					mutex_exit(&xb_wait_lock);
					return (EINTR);
				}
			} else { /* polled mode needed for early probes */
				(void) HYPERVISOR_yield();
			}
		}
		mutex_exit(&xb_wait_lock);
		/* Read indexes, then verify. */
		cons = intf->req_cons;
		prod = intf->req_prod;
		membar_enter();
		if (!check_indexes(cons, prod))
			return (EIO);

		dst = get_output_chunk(cons, prod, (char *)intf->req, &avail);
		if (avail == 0)
			continue;
		if (avail > len)
			avail = len;

		(void) memcpy(dst, data, avail);
		data = (void *)((uintptr_t)data + avail);
		len -= avail;

		/* Other side must not see new header until data is there. */
		membar_producer();
		intf->req_prod += avail;

		/* This implies mb() before other side sees interrupt. */
		ec_notify_via_evtchn(xen_info->store_evtchn);
	}

	return (0);
}

int
xb_read(void *data, unsigned len)
{
	volatile struct xenstore_domain_interface *intf =
	    xs_domain_interface(xb_addr);
	XENSTORE_RING_IDX cons, prod;
	extern int do_polled_io;

	while (len != 0) {
		unsigned int avail;
		const char *src;

		mutex_enter(&xb_wait_lock);
		while (intf->rsp_cons == intf->rsp_prod) {
			if (interrupts_unleashed && !do_polled_io) {
				if (cv_wait_sig(&xb_wait_cv,
				    &xb_wait_lock) == 0) {
					mutex_exit(&xb_wait_lock);
					return (EINTR);
				}
			} else { /* polled mode needed for early probes */
				(void) HYPERVISOR_yield();
			}
		}
		mutex_exit(&xb_wait_lock);
		/* Read indexes, then verify. */
		cons = intf->rsp_cons;
		prod = intf->rsp_prod;
		membar_enter();
		if (!check_indexes(cons, prod))
			return (EIO);

		src = get_input_chunk(cons, prod, (char *)intf->rsp, &avail);
		if (avail == 0)
			continue;
		if (avail > len)
			avail = len;

		/* We must read header before we read data. */
		membar_consumer();

		(void) memcpy(data, src, avail);
		data = (void *)((uintptr_t)data + avail);
		len -= avail;

		/* Other side must not see free space until we've copied out */
		membar_enter();
		intf->rsp_cons += avail;

		/* Implies mb(): they will see new header. */
		ec_notify_via_evtchn(xen_info->store_evtchn);
	}

	return (0);
}

void
xb_suspend(void)
{
#ifdef XPV_HVM_DRIVER
	ec_unbind_evtchn(xen_info->store_evtchn);
#else
	rem_avintr(NULL, IPL_XENBUS, xenbus_intr, xenbus_irq);
#endif
}

void
xb_setup_intr(void)
{
#ifdef XPV_HVM_DRIVER
	ec_bind_evtchn_to_handler(xen_info->store_evtchn, IPL_XENBUS,
	    xenbus_intr, NULL);
#else
	xenbus_irq = ec_bind_evtchn_to_irq(xen_info->store_evtchn);
	if (xenbus_irq < 0) {
		cmn_err(CE_WARN, "Couldn't bind xenbus event channel");
		return;
	}
	if (!add_avintr(NULL, IPL_XENBUS, xenbus_intr, "xenbus",
	    xenbus_irq, NULL, NULL, NULL, NULL))
		cmn_err(CE_WARN, "XENBUS add intr failed\n");
#endif
}

/*
 * Set up our xenstore page and event channel. Domain 0 needs to allocate a
 * page and event channel; other domains use what we are told.
 */
void
xb_init(void)
{
	int err;

	if (DOMAIN_IS_INITDOMAIN(xen_info)) {

		if (xb_addr != NULL)
			return;

		xb_addr = ddi_umem_alloc(PAGESIZE, DDI_UMEM_SLEEP,
		    &xb_cookie);
		xen_info->store_mfn = pfn_to_mfn(hat_getpfnum(kas.a_hat,
		    xb_addr));

		err = xen_alloc_unbound_evtchn(0,
		    (int *)&xen_info->store_evtchn);
		ASSERT(err == 0);
	} else {
		/*
		 * This is harmless on first boot, but needed for resume and
		 * migrate. We use kbm_map_ma() as a shortcut instead of
		 * directly using HYPERVISOR_update_va_mapping().
		 */
		ASSERT(xb_addr != NULL);
		kbm_map_ma(mfn_to_ma(xen_info->store_mfn),
		    (uintptr_t)xb_addr, 0);
	}

	ASSERT(xen_info->store_evtchn);
}

void *
xb_xenstore_cookie(void)
{
	ASSERT(DOMAIN_IS_INITDOMAIN(xen_info));
	return (xb_cookie);
}
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

/*
 * xenbus_dev.c
 *
 * Driver giving user-space access to the kernel's xenbus connection
 * to xenstore.
 *
 * Copyright (c) 2005, Christian Limpach
 * Copyright (c) 2005, Rusty Russell, IBM Corporation
 *
 * This file may be distributed separately from the Linux kernel, or
 * incorporated into other software packages, subject to the following license:
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this source file (the "Software"), to deal in the Software without
 * restriction, including without limitation the rights to use, copy, modify,
 * merge, publish, distribute, sublicense, and/or sell copies of the Software,
 * and to permit persons to whom the Software is furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 */


#include <sys/types.h>
#include <sys/sysmacros.h>
#include <sys/conf.h>
#include <sys/stat.h>
#include <sys/modctl.h>
#include <sys/uio.h>
#include <sys/list.h>
#include <sys/file.h>
#include <sys/errno.h>
#include <sys/open.h>
#include <sys/cred.h>
#include <sys/condvar.h>
#include <sys/ddi.h>
#include <sys/sunddi.h>
#include <sys/policy.h>

#ifdef XPV_HVM_DRIVER
#include <public/io/xenbus.h>
#include <public/io/xs_wire.h>
#include <sys/xpv_support.h>
#endif
#include <sys/hypervisor.h>
#include <xen/sys/xenbus.h>
#include <xen/sys/xenbus_comms.h>
#include <xen/sys/xenbus_impl.h>
#include <xen/public/io/xs_wire.h>

#ifdef DEBUG
#define	XENBUSDRV_DBPRINT(fmt) { if (xenbusdrv_debug) cmn_err fmt; }
#else
#define	XENBUSDRV_DBPRINT(fmt)
#endif /* ifdef DEBUG */

/* Some handy macros */
#define	XENBUSDRV_MASK_READ_IDX(idx)	((idx) & (PAGESIZE - 1))
#define	XENBUSDRV_MINOR2INST(minor)	((int)(minor))
#define	XENBUSDRV_NCLONES		256
#define	XENBUSDRV_INST2SOFTS(instance)	\
	((xenbus_dev_t *)ddi_get_soft_state(xenbusdrv_statep, (instance)))

static int xenbusdrv_debug = 0;
static int xenbusdrv_clone_tab[XENBUSDRV_NCLONES];
static dev_info_t *xenbusdrv_dip;
static kmutex_t xenbusdrv_clone_tab_mutex;

struct xenbus_dev_transaction {
	list_t list;
	xenbus_transaction_t handle;
};

/* Soft state data structure for xenbus driver */
struct xenbus_dev_data {
	dev_info_t *dip;

	/* In-progress transaction. */
	list_t transactions;

	/* Partial request. */
	unsigned int len;
	union {
		struct xsd_sockmsg msg;
		char buffer[MMU_PAGESIZE];
	} u;

	/* Response queue. */
	char read_buffer[MMU_PAGESIZE];
	unsigned int read_cons, read_prod;
	kcondvar_t read_cv;
	kmutex_t read_mutex;
	int xenstore_inst;
};
typedef struct xenbus_dev_data xenbus_dev_t;
static void *xenbusdrv_statep;

static int xenbusdrv_info(dev_info_t *, ddi_info_cmd_t, void *, void **);
static int xenbusdrv_attach(dev_info_t *, ddi_attach_cmd_t);
static int xenbusdrv_detach(dev_info_t *, ddi_detach_cmd_t);
static int xenbusdrv_open(dev_t *, int, int, cred_t *);
static int xenbusdrv_close(dev_t, int, int, cred_t *);
static int xenbusdrv_read(dev_t, struct uio *, cred_t *);
static int xenbusdrv_write(dev_t, struct uio *, cred_t *);
static int xenbusdrv_devmap(dev_t, devmap_cookie_t, offset_t, size_t, size_t *,
    uint_t);
static int xenbusdrv_segmap(dev_t, off_t, ddi_as_handle_t, caddr_t *, off_t,
    uint_t, uint_t, uint_t, cred_t *);
static int xenbusdrv_ioctl(dev_t, int, intptr_t, int, cred_t *, int *);
static int xenbusdrv_queue_reply(xenbus_dev_t *, const struct xsd_sockmsg *,
    const char *);

/* Solaris driver framework */

static struct cb_ops xenbusdrv_cb_ops = {
	xenbusdrv_open,			/* cb_open */
	xenbusdrv_close,		/* cb_close */
	nodev,				/* cb_strategy */
	nodev,				/* cb_print */
	nodev,				/* cb_dump */
	xenbusdrv_read,			/* cb_read */
	xenbusdrv_write,		/* cb_write */
	xenbusdrv_ioctl,		/* cb_ioctl */
	xenbusdrv_devmap,		/* cb_devmap */
	NULL,				/* cb_mmap */
	xenbusdrv_segmap,		/* cb_segmap */
	nochpoll,			/* cb_chpoll */
	ddi_prop_op,			/* cb_prop_op */
	0,				/* cb_stream */
	D_DEVMAP | D_NEW | D_MP,	/* cb_flag */
	CB_REV
};

static struct dev_ops xenbusdrv_dev_ops = {
	DEVO_REV,		/* devo_rev */
	0,			/* devo_refcnt */
	xenbusdrv_info,		/* devo_getinfo */
	nulldev,		/* devo_identify */
	nulldev,		/* devo_probe */
	xenbusdrv_attach,	/* devo_attach */
	xenbusdrv_detach,	/* devo_detach */
	nodev,			/* devo_reset */
	&xenbusdrv_cb_ops,	/* devo_cb_ops */
	NULL,			/* devo_bus_ops */
	NULL,			/* devo_power */
	ddi_quiesce_not_needed,		/* devo_quiesce */
};

static struct modldrv modldrv = {
	&mod_driverops,		/* Type of module.  This one is a driver */
	"virtual bus driver",	/* Name of the module. */
	&xenbusdrv_dev_ops	/* driver ops */
};

static struct modlinkage modlinkage = {
	MODREV_1,
	&modldrv,
	NULL
};

int
_init(void)
{
	int e;

	e = ddi_soft_state_init(&xenbusdrv_statep, sizeof (xenbus_dev_t), 1);
	if (e)
		return (e);

	e = mod_install(&modlinkage);
	if (e)
		ddi_soft_state_fini(&xenbusdrv_statep);

	return (e);
}

int
_fini(void)
{
	int e;

	e = mod_remove(&modlinkage);
	if (e)
		return (e);

	ddi_soft_state_fini(&xenbusdrv_statep);

	return (0);
}

int
_info(struct modinfo *modinfop)
{
	return (mod_info(&modlinkage, modinfop));
}

/* ARGSUSED */
static int
xenbusdrv_info(dev_info_t *dip, ddi_info_cmd_t cmd, void *arg, void **result)
{
	dev_t	dev = (dev_t)arg;
	minor_t	minor = getminor(dev);
	int	retval;

	switch (cmd) {
	case DDI_INFO_DEVT2DEVINFO:
		if (minor != 0 || xenbusdrv_dip == NULL) {
			*result = (void *)NULL;
			retval = DDI_FAILURE;
		} else {
			*result = (void *)xenbusdrv_dip;
			retval = DDI_SUCCESS;
		}
		break;
	case DDI_INFO_DEVT2INSTANCE:
		*result = (void *)0;
		retval = DDI_SUCCESS;
		break;
	default:
		retval = DDI_FAILURE;
	}
	return (retval);
}

static int
xenbusdrv_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
{
	int	error;
	int	unit = ddi_get_instance(dip);


	switch (cmd) {
	case DDI_ATTACH:
		break;
	case DDI_RESUME:
		return (DDI_SUCCESS);
	default:
		cmn_err(CE_WARN, "xenbus_attach: unknown cmd 0x%x\n", cmd);
		return (DDI_FAILURE);
	}

	/* DDI_ATTACH */

	/*
	 * only one instance - but we clone using the open routine
	 */
	if (ddi_get_instance(dip) > 0)
		return (DDI_FAILURE);

	mutex_init(&xenbusdrv_clone_tab_mutex, NULL, MUTEX_DRIVER,
	    NULL);

	error = ddi_create_minor_node(dip, "xenbus", S_IFCHR, unit,
	    DDI_PSEUDO, 0);
	if (error != DDI_SUCCESS)
		goto fail;

	/*
	 * save dip for getinfo
	 */
	xenbusdrv_dip = dip;
	ddi_report_dev(dip);

#ifndef XPV_HVM_DRIVER
	if (DOMAIN_IS_INITDOMAIN(xen_info))
		xs_dom0_init();
#endif

	return (DDI_SUCCESS);

fail:
	(void) xenbusdrv_detach(dip, DDI_DETACH);
	return (error);
}

static int
xenbusdrv_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
{
	/*
	 * again, only one instance
	 */
	if (ddi_get_instance(dip) > 0)
		return (DDI_FAILURE);

	switch (cmd) {
	case DDI_DETACH:
		ddi_remove_minor_node(dip, NULL);
		mutex_destroy(&xenbusdrv_clone_tab_mutex);
		xenbusdrv_dip = NULL;
		return (DDI_SUCCESS);
	case DDI_SUSPEND:
		return (DDI_SUCCESS);
	default:
		cmn_err(CE_WARN, "xenbus_detach: unknown cmd 0x%x\n", cmd);
		return (DDI_FAILURE);
	}
}

/* ARGSUSED */
static int
xenbusdrv_open(dev_t *devp, int flag, int otyp, cred_t *cr)
{
	xenbus_dev_t *xbs;
	minor_t minor = getminor(*devp);

	if (otyp == OTYP_BLK)
		return (ENXIO);

	/*
	 * only allow open on minor = 0 - the clone device
	 */
	if (minor != 0)
		return (ENXIO);

	/*
	 * find a free slot and grab it
	 */
	mutex_enter(&xenbusdrv_clone_tab_mutex);
	for (minor = 1; minor < XENBUSDRV_NCLONES; minor++) {
		if (xenbusdrv_clone_tab[minor] == 0) {
			xenbusdrv_clone_tab[minor] = 1;
			break;
		}
	}
	mutex_exit(&xenbusdrv_clone_tab_mutex);
	if (minor == XENBUSDRV_NCLONES)
		return (EAGAIN);

	/* Allocate softstate structure */
	if (ddi_soft_state_zalloc(xenbusdrv_statep,
	    XENBUSDRV_MINOR2INST(minor)) != DDI_SUCCESS) {
		mutex_enter(&xenbusdrv_clone_tab_mutex);
		xenbusdrv_clone_tab[minor] = 0;
		mutex_exit(&xenbusdrv_clone_tab_mutex);
		return (EAGAIN);
	}
	xbs = XENBUSDRV_INST2SOFTS(XENBUSDRV_MINOR2INST(minor));

	/* ... and init it */
	xbs->dip = xenbusdrv_dip;
	mutex_init(&xbs->read_mutex, NULL, MUTEX_DRIVER, NULL);
	cv_init(&xbs->read_cv, NULL, CV_DEFAULT, NULL);
	list_create(&xbs->transactions, sizeof (struct xenbus_dev_transaction),
	    offsetof(struct xenbus_dev_transaction, list));

	/* clone driver */
	*devp = makedevice(getmajor(*devp), minor);
	XENBUSDRV_DBPRINT((CE_NOTE, "Xenbus drv open succeeded, minor=%d",
	    minor));

	return (0);
}

/* ARGSUSED */
static int
xenbusdrv_close(dev_t dev, int flag, int otyp, struct cred *cr)
{
	xenbus_dev_t *xbs;
	minor_t minor = getminor(dev);
	struct xenbus_dev_transaction *trans;

	xbs = XENBUSDRV_INST2SOFTS(XENBUSDRV_MINOR2INST(minor));
	if (xbs == NULL)
		return (ENXIO);

#ifdef notyet
	/*
	 * XXPV - would like to be able to notify xenstore down here, but
	 * as the daemon is currently written, it doesn't leave the device
	 * open after initial setup, so we have no way of knowing if it has
	 * gone away.
	 */
	if (xbs->xenstore_inst)
		xs_notify_xenstore_down();
#endif
	/* free pending transaction */
	while (trans = (struct xenbus_dev_transaction *)
	    list_head(&xbs->transactions)) {
		(void) xenbus_transaction_end(trans->handle, 1);
		list_remove(&xbs->transactions, (void *)trans);
		kmem_free(trans, sizeof (*trans));
	}

	mutex_destroy(&xbs->read_mutex);
	cv_destroy(&xbs->read_cv);
	ddi_soft_state_free(xenbusdrv_statep, XENBUSDRV_MINOR2INST(minor));

	/*
	 * free clone tab slot
	 */
	mutex_enter(&xenbusdrv_clone_tab_mutex);
	xenbusdrv_clone_tab[minor] = 0;
	mutex_exit(&xenbusdrv_clone_tab_mutex);

	XENBUSDRV_DBPRINT((CE_NOTE, "Xenbus drv close succeeded, minor=%d",
	    minor));

	return (0);
}

/* ARGSUSED */
static int
xenbusdrv_read(dev_t dev, struct uio *uiop, cred_t *cr)
{
	xenbus_dev_t *xbs;
	size_t len;
	int res, ret;
	int idx;

	XENBUSDRV_DBPRINT((CE_NOTE, "xenbusdrv_read called"));

	if (secpolicy_xvm_control(cr))
		return (EPERM);

	xbs = XENBUSDRV_INST2SOFTS(XENBUSDRV_MINOR2INST(getminor(dev)));

	mutex_enter(&xbs->read_mutex);

	/* check if we have something to read */
	while (xbs->read_prod == xbs->read_cons) {
		if (cv_wait_sig(&xbs->read_cv, &xbs->read_mutex) == 0) {
			mutex_exit(&xbs->read_mutex);
			return (EINTR);
		}
	}

	idx = XENBUSDRV_MASK_READ_IDX(xbs->read_cons);
	res = uiop->uio_resid;

	len = xbs->read_prod - xbs->read_cons;

	if (len > (sizeof (xbs->read_buffer) - idx))
		len = sizeof (xbs->read_buffer) - idx;
	if (len > res)
		len = res;

	ret = uiomove(xbs->read_buffer + idx, len, UIO_READ, uiop);
	xbs->read_cons += res - uiop->uio_resid;
	mutex_exit(&xbs->read_mutex);

	return (ret);
}

/*
 * prepare data for xenbusdrv_read()
 */
static int
xenbusdrv_queue_reply(xenbus_dev_t *xbs, const struct xsd_sockmsg *msg,
    const char *reply)
{
	int i;
	int remaining;

	XENBUSDRV_DBPRINT((CE_NOTE, "xenbusdrv_queue_reply called"));

	mutex_enter(&xbs->read_mutex);

	remaining = sizeof (xbs->read_buffer) -
	    (xbs->read_prod - xbs->read_cons);

	if (sizeof (*msg) + msg->len > remaining) {
		mutex_exit(&xbs->read_mutex);
		return (EOVERFLOW);
	}

	for (i = 0; i < sizeof (*msg); i++, xbs->read_prod++) {
		xbs->read_buffer[XENBUSDRV_MASK_READ_IDX(xbs->read_prod)] =
		    ((char *)msg)[i];
	}

	for (i = 0; i < msg->len; i++, xbs->read_prod++) {
		xbs->read_buffer[XENBUSDRV_MASK_READ_IDX(xbs->read_prod)] =
		    reply[i];
	}

	cv_broadcast(&xbs->read_cv);

	mutex_exit(&xbs->read_mutex);

	XENBUSDRV_DBPRINT((CE_NOTE, "xenbusdrv_queue_reply exited"));

	return (0);
}

/* ARGSUSED */
static int
xenbusdrv_write(dev_t dev, struct uio *uiop, cred_t *cr)
{
	xenbus_dev_t *xbs;
	struct xenbus_dev_transaction *trans;
	void *reply;
	size_t len;
	int rc = 0;

	XENBUSDRV_DBPRINT((CE_NOTE, "xenbusdrv_write called"));

	if (secpolicy_xvm_control(cr))
		return (EPERM);

	xbs = XENBUSDRV_INST2SOFTS(XENBUSDRV_MINOR2INST(getminor(dev)));
	len = uiop->uio_resid;

	if ((len + xbs->len) > sizeof (xbs->u.buffer)) {
		XENBUSDRV_DBPRINT((CE_WARN, "Request is too big"));
		rc = EINVAL;
		goto out;
	}

	if (uiomove(xbs->u.buffer + xbs->len, len, UIO_WRITE, uiop) != 0) {
		XENBUSDRV_DBPRINT((CE_WARN, "Uiomove failed"));
		rc = EFAULT;
		goto out;
	}

	xbs->len += len;

	if (xbs->len < (sizeof (xbs->u.msg)) ||
	    xbs->len < (sizeof (xbs->u.msg) + xbs->u.msg.len)) {
		XENBUSDRV_DBPRINT((CE_NOTE, "Partial request"));
		return (0);
	}

	switch (xbs->u.msg.type) {
	case XS_TRANSACTION_START:
	case XS_TRANSACTION_END:
	case XS_DIRECTORY:
	case XS_READ:
	case XS_GET_PERMS:
	case XS_RELEASE:
	case XS_GET_DOMAIN_PATH:
	case XS_WRITE:
	case XS_MKDIR:
	case XS_RM:
	case XS_SET_PERMS:
		/* send the request to xenstore and get feedback */
		rc = xenbus_dev_request_and_reply(&xbs->u.msg, &reply);
		if (rc) {
			XENBUSDRV_DBPRINT((CE_WARN,
			    "xenbus_dev_request_and_reply failed"));
			goto out;
		}

		/* handle transaction start/end */
		if (xbs->u.msg.type == XS_TRANSACTION_START) {
			trans = kmem_alloc(sizeof (*trans), KM_SLEEP);
			(void) ddi_strtoul((char *)reply, NULL, 0,
			    (unsigned long *)&trans->handle);
			list_insert_tail(&xbs->transactions, (void *)trans);
		} else if (xbs->u.msg.type == XS_TRANSACTION_END) {
			/* try to find out the ending transaction */
			for (trans = (struct xenbus_dev_transaction *)
			    list_head(&xbs->transactions); trans;
			    trans = (struct xenbus_dev_transaction *)
			    list_next(&xbs->transactions, (void *)trans))
				if (trans->handle ==
				    (xenbus_transaction_t)
				    xbs->u.msg.tx_id)
					break;
			ASSERT(trans);
			/* free it, if we find it */
			list_remove(&xbs->transactions, (void *)trans);
			kmem_free(trans, sizeof (*trans));
		}

		/* prepare data for xenbusdrv_read() to get */
		rc = xenbusdrv_queue_reply(xbs, &xbs->u.msg, reply);

		kmem_free(reply, xbs->u.msg.len + 1);
		break;
	default:
		rc = EINVAL;
	}

out:
	xbs->len = 0;
	return (rc);
}

/*ARGSUSED*/
static int
xenbusdrv_devmap(dev_t dev, devmap_cookie_t dhp, offset_t off, size_t len,
    size_t *maplen, uint_t model)
{
	xenbus_dev_t *xbs;
	int err;

	xbs = XENBUSDRV_INST2SOFTS(XENBUSDRV_MINOR2INST(getminor(dev)));

	if (off != 0 || len != PAGESIZE)
		return (-1);

	if (!DOMAIN_IS_INITDOMAIN(xen_info))
		return (-1);

	err = devmap_umem_setup(dhp, xbs->dip, NULL, xb_xenstore_cookie(),
	    0, PAGESIZE, PROT_READ | PROT_WRITE | PROT_USER, 0, NULL);

	if (err)
		return (err);

	*maplen = PAGESIZE;

	return (0);
}

static int
xenbusdrv_segmap(dev_t dev, off_t off, ddi_as_handle_t as, caddr_t *addrp,
    off_t len, uint_t prot, uint_t maxprot, uint_t flags, cred_t *cr)
{

	if (secpolicy_xvm_control(cr))
		return (EPERM);

	return (ddi_devmap_segmap(dev, off, as, addrp, len, prot,
	    maxprot, flags, cr));
}

/*ARGSUSED*/
static int
xenbusdrv_ioctl(dev_t dev, int cmd, intptr_t arg, int mode, cred_t *cr,
    int *rvalp)
{
	xenbus_dev_t *xbs;

	if (secpolicy_xvm_control(cr))
		return (EPERM);

	xbs = XENBUSDRV_INST2SOFTS(XENBUSDRV_MINOR2INST(getminor(dev)));
	switch (cmd) {
	case IOCTL_XENBUS_XENSTORE_EVTCHN:
		*rvalp = xen_info->store_evtchn;
		break;
	case IOCTL_XENBUS_NOTIFY_UP:
		xs_notify_xenstore_up();
		xbs->xenstore_inst = 1;
		break;
	default:
		return (EINVAL);
	}

	return (0);
}
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

/*
 * Talks to Xen Store to figure out what devices we have.
 *
 * Copyright (C) 2005 Rusty Russell, IBM Corporation
 * Copyright (C) 2005 Mike Wray, Hewlett-Packard
 * Copyright (C) 2005 XenSource Ltd
 *
 * This file may be distributed separately from the Linux kernel, or
 * incorporated into other software packages, subject to the following license:
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this source file (the "Software"), to deal in the Software without
 * restriction, including without limitation the rights to use, copy, modify,
 * merge, publish, distribute, sublicense, and/or sell copies of the Software,
 * and to permit persons to whom the Software is furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 */

#ifdef XPV_HVM_DRIVER
#include <sys/xpv_support.h>
#endif
#include <sys/hypervisor.h>
#include <xen/sys/xenbus_impl.h>
#include <xen/sys/xenbus_comms.h>
#include <xen/public/io/xs_wire.h>

static int
read_otherend_details(struct xenbus_device *xendev,
			char *id_node, char *path_node)
{
	int err = xenbus_gather(XBT_NULL, xendev->nodename,
	    id_node, "%i", &xendev->otherend_id, path_node, NULL,
	    &xendev->otherend, NULL);
	if (err) {
		xenbus_dev_fatal(xendev, err,
		    "reading other end details from %s", xendev->nodename);
		return (err);
	}
	if (strlen(xendev->otherend) == 0 ||
	    !xenbus_exists_dir(xendev->otherend, "")) {
		xenbus_dev_fatal(xendev, X_ENOENT, "missing other end from %s",
		    xendev->nodename);
		kmem_free((void *)xendev->otherend,
		    strlen(xendev->otherend) + 1);
		xendev->otherend = NULL;
		return (X_ENOENT);
	}

	return (0);
}


static int
read_backend_details(struct xenbus_device *xendev)
{
	return (read_otherend_details(xendev, "backend-id", "backend"));
}


static int
read_frontend_details(struct xenbus_device *xendev)
{
	return (read_otherend_details(xendev, "frontend-id", "frontend"));
}


static void
free_otherend_details(struct xenbus_device *dev)
{
	if (dev->otherend != NULL) {
		kmem_free((void *)dev->otherend, strlen(dev->otherend) + 1);
		dev->otherend = NULL;
	}
}


static void
free_otherend_watch(struct xenbus_device *dev)
{
	if (dev->otherend_watch.node) {
		unregister_xenbus_watch(&dev->otherend_watch);
		kmem_free((void *)dev->otherend_watch.node,
		    strlen(dev->otherend_watch.node) + 1);
		dev->otherend_watch.node = NULL;
	}
}


/*ARGSUSED2*/
static void
otherend_changed(struct xenbus_watch *watch, const char **vec, unsigned int len)
{
	struct xenbus_device *dev = watch->dev;
	XenbusState state;

	/*
	 * Protect us against watches firing on old details when the otherend
	 * details change, say immediately after a resume.
	 */
	if (!dev->otherend ||
	    strncmp(dev->otherend, vec[XS_WATCH_PATH], strlen(dev->otherend))) {
#if 0
		printf("Ignoring watch at %s", vec[XS_WATCH_PATH]);
#endif
		return;
	}

	state = xenbus_read_driver_state(dev->otherend);

#if 0
	printf("state is %d, %s, %s",
	    state, dev->otherend_watch.node, vec[XS_WATCH_PATH]);
#endif
	if (dev->otherend_changed)
		dev->otherend_changed(dev, state);
}


int
talk_to_otherend(struct xenbus_device *dev)
{
	int err;

	free_otherend_watch(dev);
	free_otherend_details(dev);

	if (dev->frontend)
		err = read_backend_details(dev);
	else
		err = read_frontend_details(dev);
	if (err)
		return (err);

	dev->otherend_watch.dev = dev;
	return (xenbus_watch_path2(dev, dev->otherend, "state",
	    &dev->otherend_watch, otherend_changed));
}


/*
 * Local variables:
 *  c-file-style: "solaris"
 *  indent-tabs-mode: t
 *  c-indent-level: 8
 *  c-basic-offset: 8
 *  tab-width: 8
 * End:
 */
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

/*
 *
 * xenbus_xs.c
 *
 * This is the kernel equivalent of the "xs" library.  We don't need everything
 * and we use xenbus_comms for communication.
 *
 * Copyright (C) 2005 Rusty Russell, IBM Corporation
 *
 * This file may be distributed separately from the Linux kernel, or
 * incorporated into other software packages, subject to the following license:
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this source file (the "Software"), to deal in the Software without
 * restriction, including without limitation the rights to use, copy, modify,
 * merge, publish, distribute, sublicense, and/or sell copies of the Software,
 * and to permit persons to whom the Software is furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 */

/*
 * NOTE: To future maintainers of the Solaris version of this file:
 * I found the Linux version of this code to be very disgusting in
 * overloading pointers and error codes into void * return values.
 * The main difference you will find is that all such usage is changed
 * to pass pointers to void* to be filled in with return values and
 * the functions return error codes.
 */

#include <sys/errno.h>
#include <sys/types.h>
#include <sys/sysmacros.h>
#include <sys/uio.h>
#include <sys/mutex.h>
#include <sys/condvar.h>
#include <sys/rwlock.h>
#include <sys/disp.h>
#include <sys/ddi.h>
#include <sys/sunddi.h>
#include <sys/avintr.h>
#include <sys/cmn_err.h>
#include <sys/mach_mmu.h>
#include <util/sscanf.h>
#define	_XSD_ERRORS_DEFINED
#ifdef XPV_HVM_DRIVER
#include <sys/xpv_support.h>
#endif
#include <sys/hypervisor.h>
#include <sys/taskq.h>
#include <sys/sdt.h>
#include <xen/sys/xenbus_impl.h>
#include <xen/sys/xenbus_comms.h>
#include <xen/sys/xendev.h>
#include <xen/public/io/xs_wire.h>

#define	streq(a, b) (strcmp((a), (b)) == 0)

#define	list_empty(list) (list_head(list) == NULL)

struct xs_stored_msg {
	list_node_t list;

	struct xsd_sockmsg hdr;

	union {
		/* Queued replies. */
		struct {
			char *body;
		} reply;

		/* Queued watch events. */
		struct {
			struct xenbus_watch *handle;
			char **vec;
			unsigned int vec_size;
		} watch;
	} un;
};

static struct xs_handle {
	/* A list of replies. Currently only one will ever be outstanding. */
	list_t reply_list;
	kmutex_t reply_lock;
	kcondvar_t reply_cv;

	/* One request at a time. */
	kmutex_t request_mutex;

	/* Protect transactions against save/restore. */
	krwlock_t suspend_lock;
} xs_state;

static int last_req_id;

/*
 * List of clients wanting a xenstore up notification, and a lock to protect it
 */
static boolean_t xenstore_up;
static list_t notify_list;
static kmutex_t notify_list_lock;
static taskq_t *xenbus_taskq;

/* List of registered watches, and a lock to protect it. */
static list_t watches;
static kmutex_t watches_lock;

/* List of pending watch callback events, and a lock to protect it. */
static list_t watch_events;
static kmutex_t watch_events_lock;

/*
 * Details of the xenwatch callback kernel thread. The thread waits on the
 * watch_events_cv for work to do (queued on watch_events list). When it
 * wakes up it acquires the xenwatch_mutex before reading the list and
 * carrying out work.
 */
static kmutex_t xenwatch_mutex;
static kcondvar_t watch_events_cv;

static int process_msg(void);

static int
get_error(const char *errorstring)
{
	unsigned int i;

	for (i = 0; !streq(errorstring, xsd_errors[i].errstring); i++) {
		if (i == (sizeof (xsd_errors) / sizeof (xsd_errors[0])) - 1) {
			cmn_err(CE_WARN,
			    "XENBUS xen store gave: unknown error %s",
			    errorstring);
			return (EINVAL);
		}
	}
	return (xsd_errors[i].errnum);
}

/*
 * Read a synchronous reply from xenstore.  Since we can return early before
 * reading a relevant reply, we discard any messages not matching the request
 * ID.  Caller must free returned message on success.
 */
static int
read_reply(struct xsd_sockmsg *req_hdr, struct xs_stored_msg **reply)
{
	extern int do_polled_io;

	mutex_enter(&xs_state.reply_lock);

	for (;;) {
		while (list_empty(&xs_state.reply_list)) {
			if (interrupts_unleashed && !do_polled_io) {
				if (cv_wait_sig(&xs_state.reply_cv,
				    &xs_state.reply_lock) == 0) {
					mutex_exit(&xs_state.reply_lock);
					*reply = NULL;
					return (EINTR);
				}
			} else { /* polled mode needed for early probes */
				mutex_exit(&xs_state.reply_lock);
				(void) HYPERVISOR_yield();
				(void) process_msg();
				mutex_enter(&xs_state.reply_lock);
			}
		}

		*reply = list_head(&xs_state.reply_list);
		list_remove(&xs_state.reply_list, *reply);

		if ((*reply)->hdr.req_id == req_hdr->req_id)
			break;
	}

	mutex_exit(&xs_state.reply_lock);
	return (0);
}

/* Emergency write. */
void
xenbus_debug_write(const char *str, unsigned int count)
{
	struct xsd_sockmsg msg = { 0 };

	msg.type = XS_DEBUG;
	msg.len = sizeof ("print") + count + 1;

	mutex_enter(&xs_state.request_mutex);
	(void) xb_write(&msg, sizeof (msg));
	(void) xb_write("print", sizeof ("print"));
	(void) xb_write(str, count);
	(void) xb_write("", 1);
	mutex_exit(&xs_state.request_mutex);
}

/*
 * This is pretty unpleasant.  First off, there's the horrible logic around
 * suspend_lock and transactions.  Also, we can be interrupted either before we
 * write a message, or before we receive a reply.  A client that wants to
 * survive this can't know which case happened.  Luckily all clients don't care
 * about signals currently, and the alternative (a hard wait on a userspace
 * daemon) isn't exactly preferable.  Caller must free 'reply' on success.
 */
int
xenbus_dev_request_and_reply(struct xsd_sockmsg *msg, void **reply)
{
	struct xsd_sockmsg req_msg = *msg;
	struct xs_stored_msg *reply_msg = NULL;
	int err;

	if (req_msg.type == XS_TRANSACTION_START)
		rw_enter(&xs_state.suspend_lock, RW_READER);

	mutex_enter(&xs_state.request_mutex);

	msg->req_id = last_req_id++;

	err = xb_write(msg, sizeof (*msg) + msg->len);
	if (err) {
		if (req_msg.type == XS_TRANSACTION_START)
			rw_exit(&xs_state.suspend_lock);
		msg->type = XS_ERROR;
		*reply = NULL;
		goto out;
	}

	err = read_reply(msg, &reply_msg);

	if (err) {
		if (msg->type == XS_TRANSACTION_START)
			rw_exit(&xs_state.suspend_lock);
		*reply = NULL;
		goto out;
	}

	*reply = reply_msg->un.reply.body;
	*msg = reply_msg->hdr;

	if (reply_msg->hdr.type == XS_TRANSACTION_END)
		rw_exit(&xs_state.suspend_lock);

out:
	if (reply_msg != NULL)
		kmem_free(reply_msg, sizeof (*reply_msg));

	mutex_exit(&xs_state.request_mutex);
	return (err);
}

/*
 * Send message to xs, return errcode, rval filled in with pointer
 * to kmem_alloc'ed reply.
 */
static int
xs_talkv(xenbus_transaction_t t,
		    enum xsd_sockmsg_type type,
		    const iovec_t *iovec,
		    unsigned int num_vecs,
		    void **rval,
		    unsigned int *len)
{
	struct xsd_sockmsg msg;
	struct xs_stored_msg *reply_msg;
	char *reply;
	unsigned int i;
	int err;

	msg.tx_id = (uint32_t)(unsigned long)t;
	msg.type = type;
	msg.len = 0;
	for (i = 0; i < num_vecs; i++)
		msg.len += iovec[i].iov_len;

	mutex_enter(&xs_state.request_mutex);

	msg.req_id = last_req_id++;

	err = xb_write(&msg, sizeof (msg));
	if (err) {
		mutex_exit(&xs_state.request_mutex);
		return (err);
	}

	for (i = 0; i < num_vecs; i++) {
		err = xb_write(iovec[i].iov_base, iovec[i].iov_len);
		if (err) {
			mutex_exit(&xs_state.request_mutex);
			return (err);
		}
	}

	err = read_reply(&msg, &reply_msg);

	mutex_exit(&xs_state.request_mutex);

	if (err)
		return (err);

	reply = reply_msg->un.reply.body;

	if (reply_msg->hdr.type == XS_ERROR) {
		err = get_error(reply);
		kmem_free(reply, reply_msg->hdr.len + 1);
		goto out;
	}

	if (len != NULL)
		*len = reply_msg->hdr.len + 1;

	ASSERT(reply_msg->hdr.type == type);

	if (rval != NULL)
		*rval = reply;
	else
		kmem_free(reply, reply_msg->hdr.len + 1);

out:
	kmem_free(reply_msg, sizeof (*reply_msg));
	return (err);
}

/* Simplified version of xs_talkv: single message. */
static int
xs_single(xenbus_transaction_t t,
			enum xsd_sockmsg_type type,
			const char *string, void **ret,
			unsigned int *len)
{
	iovec_t iovec;

	iovec.iov_base = (char *)string;
	iovec.iov_len = strlen(string) + 1;
	return (xs_talkv(t, type, &iovec, 1, ret, len));
}

static unsigned int
count_strings(const char *strings, unsigned int len)
{
	unsigned int num;
	const char *p;

	for (p = strings, num = 0; p < strings + len; p += strlen(p) + 1)
		num++;

	return (num);
}

/* Return the path to dir with /name appended. Buffer must be kmem_free()'ed */
static char *
join(const char *dir, const char *name)
{
	char *buffer;
	size_t slashlen;

	slashlen = streq(name, "") ? 0 : 1;
	buffer = kmem_alloc(strlen(dir) + slashlen + strlen(name) + 1,
	    KM_SLEEP);

	(void) strcpy(buffer, dir);
	if (slashlen != 0) {
		(void) strcat(buffer, "/");
		(void) strcat(buffer, name);
	}
	return (buffer);
}

static char **
split(char *strings, unsigned int len, unsigned int *num)
{
	char *p, **ret;

	/* Count the strings. */
	if ((*num = count_strings(strings, len - 1)) == 0)
		return (NULL);

	/* Transfer to one big alloc for easy freeing. */
	ret = kmem_alloc(*num * sizeof (char *) + (len - 1), KM_SLEEP);
	(void) memcpy(&ret[*num], strings, len - 1);
	kmem_free(strings, len);

	strings = (char *)&ret[*num];
	for (p = strings, *num = 0; p < strings + (len - 1);
	    p += strlen(p) + 1) {
		ret[(*num)++] = p;
	}

	return (ret);
}

char **
xenbus_directory(xenbus_transaction_t t,
			const char *dir, const char *node, unsigned int *num)
{
	char *strings, *path;
	unsigned int len;
	int err;

	path = join(dir, node);
	err = xs_single(t, XS_DIRECTORY, path, (void **)&strings, &len);
	kmem_free(path, strlen(path) + 1);
	if (err != 0 || strings == NULL) {
		/* sigh, we lose error code info here */
		*num = 0;
		return (NULL);
	}

	return (split(strings, len, num));
}

/* Check if a path exists. */
boolean_t
xenbus_exists(const char *dir, const char *node)
{
	void	*p;
	uint_t	n;

	if (xenbus_read(XBT_NULL, dir, node, &p, &n) != 0)
		return (B_FALSE);
	kmem_free(p, n);
	return (B_TRUE);
}

/* Check if a directory path exists. */
boolean_t
xenbus_exists_dir(const char *dir, const char *node)
{
	char **d;
	unsigned int dir_n;
	int i, len;

	d = xenbus_directory(XBT_NULL, dir, node, &dir_n);
	if (d == NULL)
		return (B_FALSE);
	for (i = 0, len = 0; i < dir_n; i++)
		len += strlen(d[i]) + 1 + sizeof (char *);
	kmem_free(d, len);
	return (B_TRUE);
}

/*
 * Get the value of a single file.
 * Returns a kmem_alloced value in retp: call kmem_free() on it after use.
 * len indicates length in bytes.
 */
int
xenbus_read(xenbus_transaction_t t,
	    const char *dir, const char *node, void **retp, unsigned int *len)
{
	char *path;
	int err;

	path = join(dir, node);
	err = xs_single(t, XS_READ, path, retp, len);
	kmem_free(path, strlen(path) + 1);
	return (err);
}

int
xenbus_read_str(const char *dir, const char *node, char **retp)
{
	uint_t	n;
	int	err;
	char	*str;

	/*
	 * Since we access the xenbus value immediatly we can't be
	 * part of a transaction.
	 */
	if ((err = xenbus_read(XBT_NULL, dir, node, (void **)&str, &n)) != 0)
		return (err);
	ASSERT((str != NULL) && (n > 0));

	/*
	 * Why bother with this?  Because xenbus is truly annoying in the
	 * fact that when it returns a string, it doesn't guarantee that
	 * the memory that holds the string is of size strlen() + 1.
	 * This forces callers to keep track of the size of the memory
	 * containing the string.  Ugh.  We'll work around this by
	 * re-allocate strings to always be of size strlen() + 1.
	 */
	*retp = strdup(str);
	kmem_free(str, n);
	return (0);
}

/*
 * Write the value of a single file.
 * Returns err on failure.
 */
int
xenbus_write(xenbus_transaction_t t,
		const char *dir, const char *node, const char *string)
{
	char *path;
	iovec_t iovec[2];
	int ret;

	path = join(dir, node);

	iovec[0].iov_base = (void *)path;
	iovec[0].iov_len = strlen(path) + 1;
	iovec[1].iov_base = (void *)string;
	iovec[1].iov_len = strlen(string);

	ret = xs_talkv(t, XS_WRITE, iovec, 2, NULL, NULL);
	kmem_free(path, iovec[0].iov_len);
	return (ret);
}

/* Create a new directory. */
int
xenbus_mkdir(xenbus_transaction_t t, const char *dir, const char *node)
{
	char *path;
	int ret;

	path = join(dir, node);
	ret = xs_single(t, XS_MKDIR, path, NULL, NULL);
	kmem_free(path, strlen(path) + 1);
	return (ret);
}

/* Destroy a file or directory (directories must be empty). */
int
xenbus_rm(xenbus_transaction_t t, const char *dir, const char *node)
{
	char *path;
	int ret;

	path = join(dir, node);
	ret = xs_single(t, XS_RM, path, NULL, NULL);
	kmem_free(path, strlen(path) + 1);
	return (ret);
}

/*
 * Start a transaction: changes by others will not be seen during this
 * transaction, and changes will not be visible to others until end.
 */
int
xenbus_transaction_start(xenbus_transaction_t *t)
{
	void *id_str;
	unsigned long id;
	int err;
	unsigned int len;

	rw_enter(&xs_state.suspend_lock, RW_READER);

	err = xs_single(XBT_NULL, XS_TRANSACTION_START, "", &id_str, &len);
	if (err) {
		rw_exit(&xs_state.suspend_lock);
		return (err);
	}

	(void) ddi_strtoul((char *)id_str, NULL, 0, &id);
	*t = (xenbus_transaction_t)id;
	kmem_free(id_str, len);

	return (0);
}

/*
 * End a transaction.
 * If abandon is true, transaction is discarded instead of committed.
 */
int
xenbus_transaction_end(xenbus_transaction_t t, int abort)
{
	char abortstr[2];
	int err;

	if (abort)
		(void) strcpy(abortstr, "F");
	else
		(void) strcpy(abortstr, "T");

	err = xs_single(t, XS_TRANSACTION_END, abortstr, NULL, NULL);

	rw_exit(&xs_state.suspend_lock);

	return (err);
}

/*
 * Single read and scanf: returns errno or 0.  This can only handle a single
 * conversion specifier.
 */
/* SCANFLIKE4 */
int
xenbus_scanf(xenbus_transaction_t t,
		const char *dir, const char *node, const char *fmt, ...)
{
	va_list ap;
	int ret;
	char *val;
	unsigned int len;

	ret = xenbus_read(t, dir, node, (void **)&val, &len);
	if (ret)
		return (ret);

	va_start(ap, fmt);
	if (vsscanf(val, fmt, ap) != 1)
		ret = ERANGE;
	va_end(ap);
	kmem_free(val, len);
	return (ret);
}

/* Single printf and write: returns errno or 0. */
/* PRINTFLIKE4 */
int
xenbus_printf(xenbus_transaction_t t,
		const char *dir, const char *node, const char *fmt, ...)
{
	va_list ap;
	int ret;
#define	PRINTF_BUFFER_SIZE 4096
	char *printf_buffer;

	printf_buffer = kmem_alloc(PRINTF_BUFFER_SIZE, KM_SLEEP);

	va_start(ap, fmt);
	ret = vsnprintf(printf_buffer, PRINTF_BUFFER_SIZE, fmt, ap);
	va_end(ap);

	ASSERT(ret <= PRINTF_BUFFER_SIZE-1);
	ret = xenbus_write(t, dir, node, printf_buffer);

	kmem_free(printf_buffer, PRINTF_BUFFER_SIZE);

	return (ret);
}


/* Takes tuples of names, scanf-style args, and void **, NULL terminated. */
int
xenbus_gather(xenbus_transaction_t t, const char *dir, ...)
{
	va_list ap;
	const char *name;
	int ret = 0;
	unsigned int len;

	va_start(ap, dir);
	while (ret == 0 && (name = va_arg(ap, char *)) != NULL) {
		const char *fmt = va_arg(ap, char *);
		void *result = va_arg(ap, void *);
		char *p;

		ret = xenbus_read(t, dir, name, (void **)&p, &len);
		if (ret)
			break;
		if (fmt) {
			ASSERT(result != NULL);
			if (sscanf(p, fmt, result) != 1)
				ret = EINVAL;
			kmem_free(p, len);
		} else
			*(char **)result = p;
	}
	va_end(ap);
	return (ret);
}

static int
xs_watch(const char *path, const char *token)
{
	iovec_t iov[2];

	iov[0].iov_base = (void *)path;
	iov[0].iov_len = strlen(path) + 1;
	iov[1].iov_base = (void *)token;
	iov[1].iov_len = strlen(token) + 1;

	return (xs_talkv(XBT_NULL, XS_WATCH, iov, 2, NULL, NULL));
}

static int
xs_unwatch(const char *path, const char *token)
{
	iovec_t iov[2];

	iov[0].iov_base = (char *)path;
	iov[0].iov_len = strlen(path) + 1;
	iov[1].iov_base = (char *)token;
	iov[1].iov_len = strlen(token) + 1;

	return (xs_talkv(XBT_NULL, XS_UNWATCH, iov, 2, NULL, NULL));
}

static struct xenbus_watch *
find_watch(const char *token)
{
	struct xenbus_watch *i, *cmp;

	(void) ddi_strtoul(token, NULL, 16, (unsigned long *)&cmp);

	for (i = list_head(&watches); i != NULL; i = list_next(&watches, i))
		if (i == cmp)
			break;

	return (i);
}

/* Register a xenstore state notify callback */
int
xs_register_xenbus_callback(void (*callback)(int))
{
	struct xenbus_notify *xbn, *xnp;

	xbn = kmem_alloc(sizeof (struct xenbus_notify), KM_SLEEP);
	xbn->notify_func = callback;
	mutex_enter(&notify_list_lock);
	/*
	 * Make sure not already on the list
	 */
	xnp = list_head(&notify_list);
	for (; xnp != NULL; xnp = list_next(&notify_list, xnp)) {
		if (xnp->notify_func == callback) {
			kmem_free(xbn, sizeof (struct xenbus_notify));
			mutex_exit(&notify_list_lock);
			return (EEXIST);
		}
	}
	xnp = xbn;
	list_insert_tail(&notify_list, xbn);
	if (xenstore_up)
		xnp->notify_func(XENSTORE_UP);
	mutex_exit(&notify_list_lock);
	return (0);
}

/*
 * Notify clients of xenstore state
 */
static void
do_notify_callbacks(void *arg)
{
	struct xenbus_notify *xnp;

	mutex_enter(&notify_list_lock);
	xnp = list_head(&notify_list);
	for (; xnp != NULL; xnp = list_next(&notify_list, xnp)) {
		xnp->notify_func((int)((uintptr_t)arg));
	}
	mutex_exit(&notify_list_lock);
}

void
xs_notify_xenstore_up(void)
{
	xenstore_up = B_TRUE;
	(void) taskq_dispatch(xenbus_taskq, do_notify_callbacks,
	    (void *)XENSTORE_UP, 0);
}

void
xs_notify_xenstore_down(void)
{
	xenstore_up = B_FALSE;
	(void) taskq_dispatch(xenbus_taskq, do_notify_callbacks,
	    (void *)XENSTORE_DOWN, 0);
}

/* Register callback to watch this node. */
int
register_xenbus_watch(struct xenbus_watch *watch)
{
	/* Pointer in ascii is the token. */
	char token[sizeof (watch) * 2 + 1];
	int err;

	ASSERT(xenstore_up);
	(void) snprintf(token, sizeof (token), "%lX", (long)watch);

	rw_enter(&xs_state.suspend_lock, RW_READER);

	mutex_enter(&watches_lock);
	/*
	 * May be re-registering a watch if xenstore daemon was restarted
	 */
	if (find_watch(token) == NULL)
		list_insert_tail(&watches, watch);
	mutex_exit(&watches_lock);

	DTRACE_XPV3(xenbus__register__watch, const char *, watch->node,
	    uintptr_t, watch->callback, struct xenbus_watch *, watch);

	err = xs_watch(watch->node, token);

	/* Ignore errors due to multiple registration. */
	if ((err != 0) && (err != EEXIST)) {
		mutex_enter(&watches_lock);
		list_remove(&watches, watch);
		mutex_exit(&watches_lock);
	}

	rw_exit(&xs_state.suspend_lock);

	return (err);
}

static void
free_stored_msg(struct xs_stored_msg *msg)
{
	int i, len = 0;

	for (i = 0; i < msg->un.watch.vec_size; i++)
		len += strlen(msg->un.watch.vec[i]) + 1 + sizeof (char *);
	kmem_free(msg->un.watch.vec, len);
	kmem_free(msg, sizeof (*msg));
}

void
unregister_xenbus_watch(struct xenbus_watch *watch)
{
	struct xs_stored_msg *msg;
	char token[sizeof (watch) * 2 + 1];
	int err;

	(void) snprintf(token, sizeof (token), "%lX", (long)watch);

	rw_enter(&xs_state.suspend_lock, RW_READER);

	mutex_enter(&watches_lock);
	ASSERT(find_watch(token));
	list_remove(&watches, watch);
	mutex_exit(&watches_lock);

	DTRACE_XPV3(xenbus__unregister__watch, const char *, watch->node,
	    uintptr_t, watch->callback, struct xenbus_watch *, watch);

	err = xs_unwatch(watch->node, token);
	if (err)
		cmn_err(CE_WARN, "XENBUS Failed to release watch %s: %d",
		    watch->node, err);

	rw_exit(&xs_state.suspend_lock);

	/* Cancel pending watch events. */
	mutex_enter(&watch_events_lock);
	msg = list_head(&watch_events);

	while (msg != NULL) {
		struct xs_stored_msg *tmp = list_next(&watch_events, msg);
		if (msg->un.watch.handle == watch) {
			list_remove(&watch_events, msg);
			free_stored_msg(msg);
		}
		msg = tmp;
	}

	mutex_exit(&watch_events_lock);

	/* Flush any currently-executing callback, unless we are it. :-) */
	if (mutex_owner(&xenwatch_mutex) != curthread) {
		mutex_enter(&xenwatch_mutex);
		mutex_exit(&xenwatch_mutex);
	}
}

void
xenbus_suspend(void)
{
	rw_enter(&xs_state.suspend_lock, RW_WRITER);
	mutex_enter(&xs_state.request_mutex);

	xb_suspend();
}

void
xenbus_resume(void)
{
	struct xenbus_watch *watch;
	char token[sizeof (watch) * 2 + 1];

	mutex_exit(&xs_state.request_mutex);

	xb_init();
	xb_setup_intr();

	/* No need for watches_lock: the suspend_lock is sufficient. */
	for (watch = list_head(&watches); watch != NULL;
	    watch = list_next(&watches, watch)) {
		(void) snprintf(token, sizeof (token), "%lX", (long)watch);
		(void) xs_watch(watch->node, token);
	}

	rw_exit(&xs_state.suspend_lock);
}

static void
xenwatch_thread(void)
{
	struct xs_stored_msg *msg;
	struct xenbus_watch *watch;

	for (;;) {
		mutex_enter(&watch_events_lock);
		while (list_empty(&watch_events))
			cv_wait(&watch_events_cv, &watch_events_lock);
		msg = list_head(&watch_events);
		ASSERT(msg != NULL);
		list_remove(&watch_events, msg);
		watch = msg->un.watch.handle;
		mutex_exit(&watch_events_lock);

		mutex_enter(&xenwatch_mutex);

		DTRACE_XPV4(xenbus__fire__watch,
		    const char *, watch->node,
		    uintptr_t, watch->callback,
		    struct xenbus_watch *, watch,
		    const char *, msg->un.watch.vec[XS_WATCH_PATH]);

		watch->callback(watch, (const char **)msg->un.watch.vec,
		    msg->un.watch.vec_size);

		free_stored_msg(msg);
		mutex_exit(&xenwatch_mutex);
	}
}

static int
process_msg(void)
{
	struct xs_stored_msg *msg;
	char *body;
	int err, mlen;

	msg = kmem_alloc(sizeof (*msg), KM_SLEEP);

	err = xb_read(&msg->hdr, sizeof (msg->hdr));
	if (err) {
		kmem_free(msg, sizeof (*msg));
		return (err);
	}

	mlen = msg->hdr.len + 1;
	body = kmem_alloc(mlen, KM_SLEEP);

	err = xb_read(body, msg->hdr.len);
	if (err) {
		kmem_free(body, mlen);
		kmem_free(msg, sizeof (*msg));
		return (err);
	}

	body[mlen - 1] = '\0';

	if (msg->hdr.type == XS_WATCH_EVENT) {
		const char *token;
		msg->un.watch.vec = split(body, msg->hdr.len + 1,
		    &msg->un.watch.vec_size);
		if (msg->un.watch.vec == NULL) {
			kmem_free(msg, sizeof (*msg));
			return (EIO);
		}

		mutex_enter(&watches_lock);
		token = msg->un.watch.vec[XS_WATCH_TOKEN];
		if ((msg->un.watch.handle = find_watch(token)) != NULL) {
			mutex_enter(&watch_events_lock);

			DTRACE_XPV4(xenbus__enqueue__watch,
			    const char *, msg->un.watch.handle->node,
			    uintptr_t, msg->un.watch.handle->callback,
			    struct xenbus_watch *, msg->un.watch.handle,
			    const char *, msg->un.watch.vec[XS_WATCH_PATH]);

			list_insert_tail(&watch_events, msg);
			cv_broadcast(&watch_events_cv);
			mutex_exit(&watch_events_lock);
		} else {
			free_stored_msg(msg);
		}
		mutex_exit(&watches_lock);
	} else {
		msg->un.reply.body = body;
		mutex_enter(&xs_state.reply_lock);
		list_insert_tail(&xs_state.reply_list, msg);
		mutex_exit(&xs_state.reply_lock);
		cv_signal(&xs_state.reply_cv);
	}

	return (0);
}

static void
xenbus_thread(void)
{
	int err;

	/*
	 * We have to wait for interrupts to be ready, so we don't clash
	 * with the polled-IO code in read_reply().
	 */
	while (!interrupts_unleashed)
		delay(10);

	for (;;) {
		err = process_msg();
		if (err)
			cmn_err(CE_WARN, "XENBUS error %d while reading "
			    "message", err);
	}
}

/*
 * When setting up xenbus, dom0 and domU have to take different paths, which
 * makes this code a little confusing. For dom0:
 *
 * xs_early_init - mutex init only
 * xs_dom0_init - called on xenbus dev attach: set up our xenstore page and
 * event channel; start xenbus threads for responding to interrupts.
 *
 * And for domU:
 *
 * xs_early_init - mutex init; set up our xenstore page and event channel
 * xs_domu_init - installation of IRQ handler; start xenbus threads.
 *
 * We need an early init on domU so we can use xenbus in polled mode to
 * discover devices, VCPUs etc.
 *
 * On resume, we use xb_init() and xb_setup_intr() to restore xenbus to a
 * working state.
 */

void
xs_early_init(void)
{
	list_create(&xs_state.reply_list, sizeof (struct xs_stored_msg),
	    offsetof(struct xs_stored_msg, list));
	list_create(&watch_events, sizeof (struct xs_stored_msg),
	    offsetof(struct xs_stored_msg, list));
	list_create(&watches, sizeof (struct xenbus_watch),
	    offsetof(struct xenbus_watch, list));
	list_create(&notify_list, sizeof (struct xenbus_notify),
	    offsetof(struct xenbus_notify, list));
	mutex_init(&xs_state.reply_lock, NULL, MUTEX_DEFAULT, NULL);
	mutex_init(&xs_state.request_mutex, NULL, MUTEX_DEFAULT, NULL);
	mutex_init(&notify_list_lock, NULL, MUTEX_DEFAULT, NULL);
	rw_init(&xs_state.suspend_lock, NULL, RW_DEFAULT, NULL);
	cv_init(&xs_state.reply_cv, NULL, CV_DEFAULT, NULL);

	if (DOMAIN_IS_INITDOMAIN(xen_info))
		return;

	xb_init();
	xenstore_up = B_TRUE;
}

static void
xs_thread_init(void)
{
	(void) thread_create(NULL, 0, xenwatch_thread, NULL, 0, &p0,
	    TS_RUN, minclsyspri);
	(void) thread_create(NULL, 0, xenbus_thread, NULL, 0, &p0,
	    TS_RUN, minclsyspri);
	xenbus_taskq = taskq_create("xenbus_taskq", 1,
	    maxclsyspri - 1, 1, 1, TASKQ_PREPOPULATE);
	ASSERT(xenbus_taskq != NULL);
}

void
xs_domu_init(void)
{
	if (DOMAIN_IS_INITDOMAIN(xen_info))
		return;

	/*
	 * Add interrupt handler for xenbus now, must wait till after
	 * psm module is loaded.  All use of xenbus is in polled mode
	 * until xs_init is called since it is what kicks off the xs
	 * server threads.
	 */
	xs_thread_init();
	xb_setup_intr();
}


void
xs_dom0_init(void)
{
	static boolean_t initialized = B_FALSE;

	ASSERT(DOMAIN_IS_INITDOMAIN(xen_info));

	/*
	 * The xenbus driver might be re-attaching.
	 */
	if (initialized)
		return;

	xb_init();
	xs_thread_init();
	xb_setup_intr();

	initialized = B_TRUE;
}
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*	Copyright (c) 1990, 1991 UNIX System Laboratories, Inc.	*/
/*	Copyright (c) 1984, 1986, 1987, 1988, 1989, 1990 AT&T	*/
/*	  All Rights Reserved					*/

/*
 * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */


/*
 *
 * Copyright (c) 2004 Christian Limpach.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. This section intentionally left blank.
 * 4. The name of the author may not be used to endorse or promote products
 *    derived from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */
/*
 * Section 3 of the above license was updated in response to bug 6379571.
 */

/*
 * Hypervisor virtual console driver
 */

#include <sys/param.h>
#include <sys/types.h>
#include <sys/signal.h>
#include <sys/stream.h>
#include <sys/termio.h>
#include <sys/errno.h>
#include <sys/file.h>
#include <sys/cmn_err.h>
#include <sys/stropts.h>
#include <sys/strsubr.h>
#include <sys/strtty.h>
#include <sys/debug.h>
#include <sys/kbio.h>
#include <sys/cred.h>
#include <sys/stat.h>
#include <sys/consdev.h>
#include <sys/mkdev.h>
#include <sys/kmem.h>
#include <sys/cred.h>
#include <sys/strsun.h>
#ifdef DEBUG
#include <sys/promif.h>
#endif
#include <sys/modctl.h>
#include <sys/ddi.h>
#include <sys/sunddi.h>
#include <sys/sunndi.h>
#include <sys/policy.h>
#include <sys/atomic.h>
#include <sys/psm.h>
#include <xen/public/io/console.h>

#include "xencons.h"

#include <sys/hypervisor.h>
#include <sys/evtchn_impl.h>
#include <xen/sys/xenbus_impl.h>
#include <xen/sys/xendev.h>

#ifdef DEBUG
#define	XENCONS_DEBUG_INIT	0x0001	/* msgs during driver initialization. */
#define	XENCONS_DEBUG_INPUT	0x0002	/* characters received during int. */
#define	XENCONS_DEBUG_EOT	0x0004	/* msgs when wait for xmit to finish. */
#define	XENCONS_DEBUG_CLOSE	0x0008	/* msgs when driver open/close called */
#define	XENCONS_DEBUG_PROCS	0x0020	/* each proc name as it is entered. */
#define	XENCONS_DEBUG_OUT	0x0100	/* msgs about output events. */
#define	XENCONS_DEBUG_BUSY	0x0200	/* msgs when xmit is enabled/disabled */
#define	XENCONS_DEBUG_MODEM	0x0400	/* msgs about modem status & control. */
#define	XENCONS_DEBUG_MODM2	0x0800	/* msgs about modem status & control. */
#define	XENCONS_DEBUG_IOCTL	0x1000	/* Output msgs about ioctl messages. */
#define	XENCONS_DEBUG_CHIP	0x2000	/* msgs about chip identification. */
#define	XENCONS_DEBUG_SFLOW	0x4000	/* msgs when S/W flowcontrol active */
#define	XENCONS_DEBUG(x) (debug & (x))
static int debug  = 0;
#else
#define	XENCONS_DEBUG(x) B_FALSE
#endif

#define	XENCONS_WBUFSIZE	4096

static boolean_t abort_charseq_recognize(uchar_t);

/* The async interrupt entry points */
static void	xcasync_ioctl(struct asyncline *, queue_t *, mblk_t *);
static void	xcasync_reioctl(void *);
static void	xcasync_start(struct asyncline *);
static void	xenconsputchar(cons_polledio_arg_t, uchar_t);
static int	xenconsgetchar(cons_polledio_arg_t);
static boolean_t	xenconsischar(cons_polledio_arg_t);

static uint_t	xenconsintr(caddr_t);
static uint_t	xenconsintr_priv(caddr_t, caddr_t);
/*PRINTFLIKE2*/
static void	xenconserror(int, const char *, ...) __KPRINTFLIKE(2);
static void	xencons_soft_state_free(struct xencons *);
static boolean_t
xcasync_flowcontrol_sw_input(struct xencons *, async_flowc_action, int);
static void
xcasync_flowcontrol_sw_output(struct xencons *, async_flowc_action);

void		*xencons_soft_state;
char		*xencons_wbuf;
struct xencons	*xencons_console;

static void
xenconssetup_avintr(struct xencons *xcp, int attach)
{
	/*
	 * On xen, CPU 0 always exists and can't be taken offline,
	 * so binding this thread to it should always succeed.
	 */
	mutex_enter(&cpu_lock);
	thread_affinity_set(curthread, 0);
	mutex_exit(&cpu_lock);

	if (attach) {
		/* Setup our interrupt binding. */
		(void) add_avintr(NULL, IPL_CONS, xenconsintr_priv,
		    "xencons", xcp->console_irq, (caddr_t)xcp, NULL, NULL,
		    xcp->dip);
	} else {
		/*
		 * Cleanup interrupt configuration.  Note that the framework
		 * _should_ ensure that when rem_avintr() returns the interrupt
		 * service routine is not currently executing and that it won't
		 * be invoked again.
		 */
		(void) rem_avintr(NULL, IPL_CONS, xenconsintr_priv,
		    xcp->console_irq);
	}

	/* Notify our caller that we're done. */
	mutex_enter(&xcp->excl);
	cv_signal(&xcp->excl_cv);
	mutex_exit(&xcp->excl);

	/* Clear our binding to CPU 0 */
	thread_affinity_clear(curthread);

}

static void
xenconssetup_add_avintr(struct xencons *xcp)
{
	xenconssetup_avintr(xcp, B_TRUE);
}

static void
xenconssetup_rem_avintr(struct xencons *xcp)
{
	xenconssetup_avintr(xcp, B_FALSE);
}

static int
xenconsdetach(dev_info_t *devi, ddi_detach_cmd_t cmd)
{
	int instance;
	struct xencons *xcp;

	if (cmd != DDI_DETACH && cmd != DDI_SUSPEND)
		return (DDI_FAILURE);

	if (cmd == DDI_SUSPEND) {
		ddi_remove_intr(devi, 0, NULL);
		return (DDI_SUCCESS);
	}

	/*
	 * We should never try to detach the console driver on a domU
	 * because it should always be held open
	 */
	ASSERT(DOMAIN_IS_INITDOMAIN(xen_info));
	if (!DOMAIN_IS_INITDOMAIN(xen_info))
		return (DDI_FAILURE);

	instance = ddi_get_instance(devi);	/* find out which unit */

	xcp = ddi_get_soft_state(xencons_soft_state, instance);
	if (xcp == NULL)
		return (DDI_FAILURE);

	/*
	 * Cleanup our interrupt bindings.  For more info on why we
	 * do this in a seperate thread, see the comments for when we
	 * setup the interrupt bindings.
	 */
	xencons_console = NULL;
	mutex_enter(&xcp->excl);
	(void) taskq_dispatch(system_taskq,
	    (void (*)(void *))xenconssetup_rem_avintr, xcp, TQ_SLEEP);
	cv_wait(&xcp->excl_cv, &xcp->excl);
	mutex_exit(&xcp->excl);

	/* remove all minor device node(s) for this device */
	ddi_remove_minor_node(devi, NULL);

	/* free up state */
	xencons_soft_state_free(xcp);
	kmem_free(xencons_wbuf, XENCONS_WBUFSIZE);

	DEBUGNOTE1(XENCONS_DEBUG_INIT, "xencons%d: shutdown complete",
	    instance);
	return (DDI_SUCCESS);
}

static void
xenconssetup(struct xencons *xcp)
{
	xcp->ifp = (volatile struct xencons_interface *)HYPERVISOR_console_page;

	if (DOMAIN_IS_INITDOMAIN(xen_info)) {
		xencons_wbuf = kmem_alloc(XENCONS_WBUFSIZE, KM_SLEEP);

		/*
		 * Activate the xen console virq.  Note that xen requires
		 * that VIRQs be bound to CPU 0 when first created.
		 */
		xcp->console_irq = ec_bind_virq_to_irq(VIRQ_CONSOLE, 0);

		/*
		 * Ok.  This is kinda ugly.  We want to register an
		 * interrupt handler for the xen console virq, but
		 * virq's are xen sepcific and currently the DDI doesn't
		 * support binding to them.  So instead we need to use
		 * add_avintr().  So to make things more complicated,
		 * we already had to bind the xen console VIRQ to CPU 0,
		 * and add_avintr() needs to be invoked on the same CPU
		 * where the VIRQ is bound, in this case on CPU 0.  We
		 * could just temporarily bind ourselves to CPU 0, but
		 * we don't want to do that since this attach thread
		 * could have been invoked in a user thread context,
		 * in which case this thread could already have some
		 * pre-existing cpu binding.  So to avoid changing our
		 * cpu binding we're going to use a taskq thread that
		 * will bind to CPU 0 and register our interrupts
		 * handler for us.
		 */
		mutex_enter(&xcp->excl);
		(void) taskq_dispatch(system_taskq,
		    (void (*)(void *))xenconssetup_add_avintr, xcp, TQ_SLEEP);
		cv_wait(&xcp->excl_cv, &xcp->excl);
		mutex_exit(&xcp->excl);
	} else {
		(void) xvdi_alloc_evtchn(xcp->dip);
		xcp->evtchn = xvdi_get_evtchn(xcp->dip);
		(void) ddi_add_intr(xcp->dip, 0, NULL, NULL, xenconsintr,
		    (caddr_t)xcp);
	}
}

static int
xenconsattach(dev_info_t *devi, ddi_attach_cmd_t cmd)
{
	int instance = ddi_get_instance(devi);
	struct xencons *xcp;
	int ret;

	/* There can be only one. */
	if (instance != 0)
		return (DDI_FAILURE);

	switch (cmd) {
	case DDI_RESUME:
		xcp = xencons_console;
		xenconssetup(xcp);
		return (DDI_SUCCESS);
	case DDI_ATTACH:
		break;
	default:
		return (DDI_FAILURE);
	}

	ret = ddi_soft_state_zalloc(xencons_soft_state, instance);
	if (ret != DDI_SUCCESS)
		return (DDI_FAILURE);
	xcp = ddi_get_soft_state(xencons_soft_state, instance);
	ASSERT(xcp != NULL);	/* can't fail - we only just allocated it */

	/*
	 * Set up the other components of the xencons structure for this port.
	 */
	xcp->unit = instance;
	xcp->dip = devi;

	/* Fill in the polled I/O structure. */
	xcp->polledio.cons_polledio_version = CONSPOLLEDIO_V0;
	xcp->polledio.cons_polledio_argument = (cons_polledio_arg_t)xcp;
	xcp->polledio.cons_polledio_putchar = xenconsputchar;
	xcp->polledio.cons_polledio_getchar = xenconsgetchar;
	xcp->polledio.cons_polledio_ischar = xenconsischar;
	xcp->polledio.cons_polledio_enter = NULL;
	xcp->polledio.cons_polledio_exit = NULL;

	/*
	 * Initializes the asyncline structure which has TTY protocol-private
	 * data before enabling interrupts.
	 */
	xcp->priv = kmem_zalloc(sizeof (struct asyncline), KM_SLEEP);
	xcp->priv->async_common = xcp;
	cv_init(&xcp->priv->async_flags_cv, NULL, CV_DRIVER, NULL);

	/* Initialize mutexes before accessing the interface. */
	mutex_init(&xcp->excl, NULL, MUTEX_DRIVER, NULL);
	cv_init(&xcp->excl_cv, NULL, CV_DEFAULT, NULL);

	/* create minor device node for this device */
	ret = ddi_create_minor_node(devi, "xencons", S_IFCHR, instance,
	    DDI_NT_SERIAL, 0);
	if (ret != DDI_SUCCESS) {
		ddi_remove_minor_node(devi, NULL);
		xencons_soft_state_free(xcp);
		return (DDI_FAILURE);
	}

	ddi_report_dev(devi);
	xencons_console = xcp;
	xenconssetup(xcp);
	DEBUGCONT1(XENCONS_DEBUG_INIT, "xencons%dattach: done\n", instance);
	return (DDI_SUCCESS);
}

/*ARGSUSED*/
static int
xenconsinfo(dev_info_t *dip, ddi_info_cmd_t infocmd, void *arg,
    void **result)
{
	dev_t dev = (dev_t)arg;
	int instance, error;
	struct xencons *xcp;

	instance = getminor(dev);
	xcp = ddi_get_soft_state(xencons_soft_state, instance);
	if (xcp == NULL)
		return (DDI_FAILURE);

	switch (infocmd) {
	case DDI_INFO_DEVT2DEVINFO:
		if (xcp->dip == NULL)
			error = DDI_FAILURE;
		else {
			*result = (void *) xcp->dip;
			error = DDI_SUCCESS;
		}
		break;
	case DDI_INFO_DEVT2INSTANCE:
		*result = (void *)(intptr_t)instance;
		error = DDI_SUCCESS;
		break;
	default:
		error = DDI_FAILURE;
	}
	return (error);
}

/* xencons_soft_state_free - local wrapper for ddi_soft_state_free(9F) */

static void
xencons_soft_state_free(struct xencons *xcp)
{
	mutex_destroy(&xcp->excl);
	cv_destroy(&xcp->excl_cv);
	kmem_free(xcp->priv, sizeof (struct asyncline));
	ddi_soft_state_free(xencons_soft_state, xcp->unit);
}

/*ARGSUSED*/
static int
xenconsopen(queue_t *rq, dev_t *dev, int flag, int sflag, cred_t *cr)
{
	struct xencons	*xcp;
	struct asyncline *async;
	int		unit;

	unit = getminor(*dev);
	DEBUGCONT1(XENCONS_DEBUG_CLOSE, "xencons%dopen\n", unit);
	xcp = ddi_get_soft_state(xencons_soft_state, unit);
	if (xcp == NULL)
		return (ENXIO);		/* unit not configured */
	async = xcp->priv;
	mutex_enter(&xcp->excl);

	if ((async->async_flags & ASYNC_ISOPEN) == 0) {
		async->async_ttycommon.t_iflag = 0;
		async->async_ttycommon.t_iocpending = NULL;
		async->async_ttycommon.t_size.ws_row = 0;
		async->async_ttycommon.t_size.ws_col = 0;
		async->async_ttycommon.t_size.ws_xpixel = 0;
		async->async_ttycommon.t_size.ws_ypixel = 0;
		async->async_dev = *dev;
		async->async_wbufcid = 0;

		async->async_startc = CSTART;
		async->async_stopc = CSTOP;
	} else if ((async->async_ttycommon.t_flags & TS_XCLUDE) &&
	    secpolicy_excl_open(cr) != 0) {
		mutex_exit(&xcp->excl);
		return (EBUSY);
	}

	async->async_ttycommon.t_flags |= TS_SOFTCAR;

	async->async_ttycommon.t_readq = rq;
	async->async_ttycommon.t_writeq = WR(rq);
	rq->q_ptr = WR(rq)->q_ptr = (caddr_t)async;
	mutex_exit(&xcp->excl);
	/*
	 * Caution here -- qprocson sets the pointers that are used by canput
	 * called by xencons_rxint.  ASYNC_ISOPEN must *not* be set until those
	 * pointers are valid.
	 */
	qprocson(rq);
	async->async_flags |= ASYNC_ISOPEN;
	DEBUGCONT1(XENCONS_DEBUG_INIT, "asy%dopen: done\n", unit);
	return (0);
}


/*
 * Close routine.
 */
/*ARGSUSED*/
static int
xenconsclose(queue_t *q, int flag, cred_t *credp)
{
	struct asyncline *async;
	struct xencons	 *xcp;
#ifdef DEBUG
	int instance;
#endif

	async = (struct asyncline *)q->q_ptr;
	ASSERT(async != NULL);
	xcp = async->async_common;
#ifdef DEBUG
	instance = xcp->unit;
	DEBUGCONT1(XENCONS_DEBUG_CLOSE, "xencons%dclose\n", instance);
#endif

	mutex_enter(&xcp->excl);
	async->async_flags |= ASYNC_CLOSING;

	async->async_ocnt = 0;
	if (async->async_xmitblk != NULL)
		freeb(async->async_xmitblk);
	async->async_xmitblk = NULL;

	ttycommon_close(&async->async_ttycommon);

	/*
	 * Cancel outstanding "bufcall" request.
	 */
	if (async->async_wbufcid != 0) {
		unbufcall(async->async_wbufcid);
		async->async_wbufcid = 0;
	}

	/* Note that qprocsoff can't be done until after interrupts are off */
	qprocsoff(q);
	q->q_ptr = WR(q)->q_ptr = NULL;
	async->async_ttycommon.t_readq = NULL;
	async->async_ttycommon.t_writeq = NULL;

	/*
	 * Clear out device state, except persistant device property flags.
	 */
	async->async_flags = 0;
	cv_broadcast(&async->async_flags_cv);
	mutex_exit(&xcp->excl);

	DEBUGCONT1(XENCONS_DEBUG_CLOSE, "xencons%dclose: done\n", instance);
	return (0);
}

#define	INBUF_IX(ix, ifp)	(DOMAIN_IS_INITDOMAIN(xen_info) ? \
	(ix) : MASK_XENCONS_IDX((ix), (ifp)->in))

/*
 * Handle a xen console rx interrupt.
 */
/*ARGSUSED*/
static void
xencons_rxint(struct xencons *xcp)
{
	struct asyncline *async;
	short	cc;
	mblk_t	*bp;
	queue_t	*q;
	uchar_t	c, buf[16];
	uchar_t	*cp;
	tty_common_t	*tp;
	int instance;
	volatile struct xencons_interface *ifp;
	XENCONS_RING_IDX cons, prod;

	DEBUGCONT0(XENCONS_DEBUG_PROCS, "xencons_rxint\n");

loop:
	mutex_enter(&xcp->excl);

	instance = xcp->unit;

	/* sanity check if we should bail */
	if (xencons_console == NULL) {
		mutex_exit(&xcp->excl);
		DEBUGCONT1(XENCONS_DEBUG_PROCS,
		    "xencons%d_rxint: xencons_console is NULL\n",
		    instance);
		goto out;
	}

	async = xcp->priv;
	ifp = xcp->ifp;
	tp = &async->async_ttycommon;
	q = tp->t_readq;

	if (async->async_flags & ASYNC_OUT_FLW_RESUME) {
		xcasync_start(async);
		async->async_flags &= ~ASYNC_OUT_FLW_RESUME;
	}

	/*
	 * If data is available, send it up the stream if there's
	 * somebody listening.
	 */
	if (!(async->async_flags & ASYNC_ISOPEN)) {
		mutex_exit(&xcp->excl);
		goto out;
	}
	if (DOMAIN_IS_INITDOMAIN(xen_info)) {
		cc = HYPERVISOR_console_io(CONSOLEIO_read, 16, (char *)buf);
		cp = buf;
		cons = 0;
	} else {
		cons = ifp->in_cons;
		prod = ifp->in_prod;

		cc = prod - cons;
		cp = (uchar_t *)ifp->in;
	}
	if (cc <= 0) {
		mutex_exit(&xcp->excl);
		goto out;
	}

	/*
	 * Check for character break sequence.
	 *
	 * Note that normally asy drivers only check for a character sequence
	 * if abort_enable == KIOCABORTALTERNATE and otherwise use a break
	 * sensed on the line to do an abort_sequence_enter.  Since the
	 * hypervisor does not use a real chip for the console we default to
	 * using the alternate sequence.
	 */
	if ((abort_enable == KIOCABORTENABLE) && (xcp->flags & ASY_CONSOLE)) {
		XENCONS_RING_IDX i;

		for (i = 0; i < cc; i++) {
			c = cp[INBUF_IX(cons + i, ifp)];
			if (abort_charseq_recognize(c)) {
				/*
				 * Eat abort seg, it's not a valid debugger
				 * command.
				 */
				if (!DOMAIN_IS_INITDOMAIN(xen_info)) {
					membar_producer();
					ifp->in_cons = cons + i;
				} else {
					cons += i;
				}
				abort_sequence_enter((char *)NULL);
				/*
				 * Back from debugger, resume normal processing
				 */
				mutex_exit(&xcp->excl);
				goto loop;
			}
		}
	}

	if (!canput(q)) {
		if (!(async->async_inflow_source & IN_FLOW_STREAMS)) {
			(void) xcasync_flowcontrol_sw_input(xcp, FLOW_STOP,
			    IN_FLOW_STREAMS);
		}
		mutex_exit(&xcp->excl);
		goto out;
	}
	if (async->async_inflow_source & IN_FLOW_STREAMS) {
		(void) xcasync_flowcontrol_sw_input(xcp, FLOW_START,
		    IN_FLOW_STREAMS);
	}
	DEBUGCONT2(XENCONS_DEBUG_INPUT,
	    "xencons%d_rxint: %d char(s) in queue.\n", instance, cc);
	if (!(bp = allocb(cc, BPRI_MED))) {
		mutex_exit(&xcp->excl);
		ttycommon_qfull(&async->async_ttycommon, q);
		goto out;
	}
	do {
		c = cp[INBUF_IX(cons++, ifp)];
		/*
		 * We handle XON/XOFF char if IXON is set,
		 * but if received char is _POSIX_VDISABLE,
		 * we left it to the up level module.
		 */
		if (tp->t_iflag & IXON) {
			if ((c == async->async_stopc) &&
			    (c != _POSIX_VDISABLE)) {
				xcasync_flowcontrol_sw_output(xcp, FLOW_STOP);
				continue;
			} else if ((c == async->async_startc) &&
			    (c != _POSIX_VDISABLE)) {
				xcasync_flowcontrol_sw_output(xcp, FLOW_START);
				continue;
			}
			if ((tp->t_iflag & IXANY) &&
			    (async->async_flags & ASYNC_SW_OUT_FLW)) {
				xcasync_flowcontrol_sw_output(xcp, FLOW_START);
			}
		}
		*bp->b_wptr++ = c;
	} while (--cc);
	membar_producer();
	if (!DOMAIN_IS_INITDOMAIN(xen_info))
		ifp->in_cons = cons;
	mutex_exit(&xcp->excl);
	if (bp->b_wptr > bp->b_rptr) {
		if (!canput(q)) {
			xenconserror(CE_NOTE, "xencons%d: local queue full",
			    instance);
			freemsg(bp);
		} else
			(void) putq(q, bp);
	} else
		freemsg(bp);
	if (DOMAIN_IS_INITDOMAIN(xen_info))
		goto loop;
out:
	DEBUGCONT1(XENCONS_DEBUG_PROCS, "xencons%d_rxint: done\n", instance);
	if (!DOMAIN_IS_INITDOMAIN(xen_info))
		ec_notify_via_evtchn(xcp->evtchn);
}


/*
 * Handle a xen console tx interrupt.
 */
/*ARGSUSED*/
static void
xencons_txint(struct xencons *xcp)
{
	struct asyncline *async;

	DEBUGCONT0(XENCONS_DEBUG_PROCS, "xencons_txint\n");

	/*
	 * prevent recursive entry
	 */
	if (mutex_owner(&xcp->excl) == curthread) {
		goto out;
	}

	mutex_enter(&xcp->excl);
	if (xencons_console == NULL) {
		mutex_exit(&xcp->excl);
		goto out;
	}

	/* make sure the device is open */
	async = xcp->priv;
	if ((async->async_flags & ASYNC_ISOPEN) != 0)
		xcasync_start(async);

	mutex_exit(&xcp->excl);
out:
	DEBUGCONT0(XENCONS_DEBUG_PROCS, "xencons_txint: done\n");
}


/*
 * Get an event when input ring becomes not empty or output ring becomes not
 * full.
 */
static uint_t
xenconsintr(caddr_t arg)
{
	struct xencons *xcp = (struct xencons *)arg;
	volatile struct xencons_interface *ifp = xcp->ifp;

	if (ifp->in_prod != ifp->in_cons)
		xencons_rxint(xcp);
	if (ifp->out_prod - ifp->out_cons < sizeof (ifp->out))
		xencons_txint(xcp);
	return (DDI_INTR_CLAIMED);
}

/*
 * Console interrupt routine for priviliged domains
 */
static uint_t
xenconsintr_priv(caddr_t arg, caddr_t arg1 __unused)
{
	struct xencons *xcp = (struct xencons *)arg;

	xencons_rxint(xcp);
	xencons_txint(xcp);
	return (DDI_INTR_CLAIMED);
}

/*
 * Start output on a line, unless it's busy, frozen, or otherwise.
 */
/*ARGSUSED*/
static void
xcasync_start(struct asyncline *async)
{
	struct xencons *xcp = async->async_common;
	int cc;
	queue_t *q;
	mblk_t *bp;
	int	len, space, blen;
	mblk_t *nbp;

#ifdef DEBUG
	int instance = xcp->unit;

	DEBUGCONT1(XENCONS_DEBUG_PROCS, "async%d_nstart\n", instance);
#endif
	ASSERT(mutex_owned(&xcp->excl));

	/*
	 * Check only pended sw input flow control.
	 */
domore:
	(void) xcasync_flowcontrol_sw_input(xcp, FLOW_CHECK, IN_FLOW_NULL);

	if ((q = async->async_ttycommon.t_writeq) == NULL) {
		return;	/* not attached to a stream */
	}

	for (;;) {
		if ((bp = getq(q)) == NULL)
			return;	/* no data to transmit */

		/*
		 * We have a message block to work on.
		 * Check whether it's a break, a delay, or an ioctl (the latter
		 * occurs if the ioctl in question was waiting for the output
		 * to drain).  If it's one of those, process it immediately.
		 */
		switch (bp->b_datap->db_type) {

		case M_IOCTL:
			/*
			 * This ioctl was waiting for the output ahead of
			 * it to drain; obviously, it has.  Do it, and
			 * then grab the next message after it.
			 */
			mutex_exit(&xcp->excl);
			xcasync_ioctl(async, q, bp);
			mutex_enter(&xcp->excl);
			continue;
		}

		while (bp != NULL && (cc = bp->b_wptr - bp->b_rptr) == 0) {
			nbp = bp->b_cont;
			freeb(bp);
			bp = nbp;
		}
		if (bp != NULL)
			break;
	}

	/*
	 * We have data to transmit.  If output is stopped, put
	 * it back and try again later.
	 */
	if (async->async_flags & (ASYNC_SW_OUT_FLW | ASYNC_STOPPED)) {
		(void) putbq(q, bp);
		return;
	}


	if (DOMAIN_IS_INITDOMAIN(xen_info)) {
		len = 0;
		space = XENCONS_WBUFSIZE;
		while (bp != NULL && space) {
			blen = bp->b_wptr - bp->b_rptr;
			cc = min(blen, space);
			bcopy(bp->b_rptr, &xencons_wbuf[len], cc);
			bp->b_rptr += cc;
			if (cc == blen) {
				nbp = bp->b_cont;
				freeb(bp);
				bp = nbp;
			}
			space -= cc;
			len += cc;
		}
		mutex_exit(&xcp->excl);
		(void) HYPERVISOR_console_io(CONSOLEIO_write, len,
		    xencons_wbuf);
		mutex_enter(&xcp->excl);
		if (bp != NULL)
			(void) putbq(q, bp); /* not done with this msg yet */
		/*
		 * There are no completion interrupts when using the
		 * HYPERVISOR_console_io call to write console data
		 * so we loop here till we have sent all the data to the
		 * hypervisor.
		 */
		goto domore;
	} else {
		volatile struct xencons_interface *ifp = xcp->ifp;
		XENCONS_RING_IDX cons, prod;

		cons = ifp->out_cons;
		prod = ifp->out_prod;
		membar_enter();
		while (bp != NULL && ((prod - cons) < sizeof (ifp->out))) {
			ifp->out[MASK_XENCONS_IDX(prod++, ifp->out)] =
			    *bp->b_rptr++;
			if (bp->b_rptr == bp->b_wptr) {
				nbp = bp->b_cont;
				freeb(bp);
				bp = nbp;
			}
		}
		membar_producer();
		ifp->out_prod = prod;
		ec_notify_via_evtchn(xcp->evtchn);
		if (bp != NULL)
			(void) putbq(q, bp); /* not done with this msg yet */
	}
}


/*
 * Process an "ioctl" message sent down to us.
 * Note that we don't need to get any locks until we are ready to access
 * the hardware.  Nothing we access until then is going to be altered
 * outside of the STREAMS framework, so we should be safe.
 */
static void
xcasync_ioctl(struct asyncline *async, queue_t *wq, mblk_t *mp)
{
	struct xencons *xcp = async->async_common;
	tty_common_t  *tp = &async->async_ttycommon;
	struct iocblk *iocp;
	unsigned datasize;
	int error = 0;

#ifdef DEBUG
	int instance = xcp->unit;

	DEBUGCONT1(XENCONS_DEBUG_PROCS, "async%d_ioctl\n", instance);
#endif

	if (tp->t_iocpending != NULL) {
		/*
		 * We were holding an "ioctl" response pending the
		 * availability of an "mblk" to hold data to be passed up;
		 * another "ioctl" came through, which means that "ioctl"
		 * must have timed out or been aborted.
		 */
		freemsg(async->async_ttycommon.t_iocpending);
		async->async_ttycommon.t_iocpending = NULL;
	}

	iocp = (struct iocblk *)mp->b_rptr;

	/*
	 * For TIOCMGET and the PPS ioctls, do NOT call ttycommon_ioctl()
	 * because this function frees up the message block (mp->b_cont) that
	 * contains the user location where we pass back the results.
	 *
	 * Similarly, CONSOPENPOLLEDIO needs ioc_count, which ttycommon_ioctl
	 * zaps.  We know that ttycommon_ioctl doesn't know any CONS*
	 * ioctls, so keep the others safe too.
	 */
	DEBUGCONT2(XENCONS_DEBUG_IOCTL, "async%d_ioctl: %s\n",
	    instance,
	    iocp->ioc_cmd == TIOCMGET ? "TIOCMGET" :
	    iocp->ioc_cmd == TIOCMSET ? "TIOCMSET" :
	    iocp->ioc_cmd == TIOCMBIS ? "TIOCMBIS" :
	    iocp->ioc_cmd == TIOCMBIC ? "TIOCMBIC" : "other");

	switch (iocp->ioc_cmd) {
	case TIOCMGET:
	case TIOCGPPS:
	case TIOCSPPS:
	case TIOCGPPSEV:
	case CONSOPENPOLLEDIO:
	case CONSCLOSEPOLLEDIO:
	case CONSSETABORTENABLE:
	case CONSGETABORTENABLE:
		error = -1; /* Do Nothing */
		break;
	default:

		/*
		 * The only way in which "ttycommon_ioctl" can fail is if the
		 * "ioctl" requires a response containing data to be returned
		 * to the user, and no mblk could be allocated for the data.
		 * No such "ioctl" alters our state.  Thus, we always go ahead
		 * and do any state-changes the "ioctl" calls for.  If we
		 * couldn't allocate the data, "ttycommon_ioctl" has stashed
		 * the "ioctl" away safely, so we just call "bufcall" to
		 * request that we be called back when we stand a better
		 * chance of allocating the data.
		 */
		if ((datasize = ttycommon_ioctl(tp, wq, mp, &error)) != 0) {
			if (async->async_wbufcid)
				unbufcall(async->async_wbufcid);
			async->async_wbufcid = bufcall(datasize, BPRI_HI,
			    (void (*)(void *)) xcasync_reioctl,
			    (void *)(intptr_t)async->async_common->unit);
			return;
		}
	}

	mutex_enter(&xcp->excl);

	if (error == 0) {
		/*
		 * "ttycommon_ioctl" did most of the work; we just use the
		 * data it set up.
		 */
		switch (iocp->ioc_cmd) {

		case TCSETS:
		case TCSETSF:
		case TCSETSW:
		case TCSETA:
		case TCSETAW:
		case TCSETAF:
			break;
		}
	} else if (error < 0) {
		/*
		 * "ttycommon_ioctl" didn't do anything; we process it here.
		 */
		error = 0;
		switch (iocp->ioc_cmd) {

		case TCSBRK:
			error = miocpullup(mp, sizeof (int));
			break;

		case TIOCSBRK:
			mioc2ack(mp, NULL, 0, 0);
			break;

		case TIOCCBRK:
			mioc2ack(mp, NULL, 0, 0);
			break;

		case CONSOPENPOLLEDIO:
			error = miocpullup(mp, sizeof (cons_polledio_arg_t));
			if (error != 0)
				break;

			*(cons_polledio_arg_t *)mp->b_cont->b_rptr =
			    (cons_polledio_arg_t)&xcp->polledio;

			mp->b_datap->db_type = M_IOCACK;
			break;

		case CONSCLOSEPOLLEDIO:
			mp->b_datap->db_type = M_IOCACK;
			iocp->ioc_error = 0;
			iocp->ioc_rval = 0;
			break;

		case CONSSETABORTENABLE:
			error = secpolicy_console(iocp->ioc_cr);
			if (error != 0)
				break;

			if (iocp->ioc_count != TRANSPARENT) {
				error = EINVAL;
				break;
			}

			if (*(intptr_t *)mp->b_cont->b_rptr)
				xcp->flags |= ASY_CONSOLE;
			else
				xcp->flags &= ~ASY_CONSOLE;

			mp->b_datap->db_type = M_IOCACK;
			iocp->ioc_error = 0;
			iocp->ioc_rval = 0;
			break;

		case CONSGETABORTENABLE:
			/*CONSTANTCONDITION*/
			ASSERT(sizeof (boolean_t) <= sizeof (boolean_t *));
			/*
			 * Store the return value right in the payload
			 * we were passed.  Crude.
			 */
			mcopyout(mp, NULL, sizeof (boolean_t), NULL, NULL);
			*(boolean_t *)mp->b_cont->b_rptr =
			    (xcp->flags & ASY_CONSOLE) != 0;
			break;

		default:
			/*
			 * If we don't understand it, it's an error.  NAK it.
			 */
			error = EINVAL;
			break;
		}
	}
	if (error != 0) {
		iocp->ioc_error = error;
		mp->b_datap->db_type = M_IOCNAK;
	}
	mutex_exit(&xcp->excl);
	qreply(wq, mp);
	DEBUGCONT1(XENCONS_DEBUG_PROCS, "async%d_ioctl: done\n", instance);
}

static int
xenconsrsrv(queue_t *q)
{
	mblk_t *bp;

	while (canputnext(q) && (bp = getq(q)))
		putnext(q, bp);
	return (0);
}

/*
 * Put procedure for write queue.
 * Respond to M_STOP, M_START, M_IOCTL, and M_FLUSH messages here;
 * set the flow control character for M_STOPI and M_STARTI messages;
 * queue up M_BREAK, M_DELAY, and M_DATA messages for processing
 * by the start routine, and then call the start routine; discard
 * everything else.  Note that this driver does not incorporate any
 * mechanism to negotiate to handle the canonicalization process.
 * It expects that these functions are handled in upper module(s),
 * as we do in ldterm.
 */
static int
xenconswput(queue_t *q, mblk_t *mp)
{
	struct asyncline *async;
	struct xencons *xcp;

	async = (struct asyncline *)q->q_ptr;
	xcp = async->async_common;

	switch (mp->b_datap->db_type) {

	case M_STOP:
		mutex_enter(&xcp->excl);
		async->async_flags |= ASYNC_STOPPED;
		mutex_exit(&xcp->excl);
		freemsg(mp);
		break;

	case M_START:
		mutex_enter(&xcp->excl);
		if (async->async_flags & ASYNC_STOPPED) {
			async->async_flags &= ~ASYNC_STOPPED;
			xcasync_start(async);
		}
		mutex_exit(&xcp->excl);
		freemsg(mp);
		break;

	case M_IOCTL:
		switch (((struct iocblk *)mp->b_rptr)->ioc_cmd) {

		case TCSETSW:
		case TCSETSF:
		case TCSETAW:
		case TCSETAF:
			/*
			 * The changes do not take effect until all
			 * output queued before them is drained.
			 * Put this message on the queue, so that
			 * "xcasync_start" will see it when it's done
			 * with the output before it.  Poke the
			 * start routine, just in case.
			 */
			(void) putq(q, mp);
			mutex_enter(&xcp->excl);
			xcasync_start(async);
			mutex_exit(&xcp->excl);
			break;

		default:
			/*
			 * Do it now.
			 */
			xcasync_ioctl(async, q, mp);
			break;
		}
		break;

	case M_FLUSH:
		if (*mp->b_rptr & FLUSHW) {
			mutex_enter(&xcp->excl);
			/*
			 * Flush our write queue.
			 */
			flushq(q, FLUSHDATA);	/* XXX doesn't flush M_DELAY */
			if (async->async_xmitblk != NULL) {
				freeb(async->async_xmitblk);
				async->async_xmitblk = NULL;
			}
			mutex_exit(&xcp->excl);
			*mp->b_rptr &= ~FLUSHW;	/* it has been flushed */
		}
		if (*mp->b_rptr & FLUSHR) {
			flushq(RD(q), FLUSHDATA);
			qreply(q, mp);	/* give the read queues a crack at it */
		} else {
			freemsg(mp);
		}

		/*
		 * We must make sure we process messages that survive the
		 * write-side flush.
		 */
		mutex_enter(&xcp->excl);
		xcasync_start(async);
		mutex_exit(&xcp->excl);
		break;

	case M_BREAK:
	case M_DELAY:
	case M_DATA:
		/*
		 * Queue the message up to be transmitted,
		 * and poke the start routine.
		 */
		(void) putq(q, mp);
		mutex_enter(&xcp->excl);
		xcasync_start(async);
		mutex_exit(&xcp->excl);
		break;

	case M_STOPI:
		mutex_enter(&xcp->excl);
		mutex_enter(&xcp->excl);
		if (!(async->async_inflow_source & IN_FLOW_USER)) {
			(void) xcasync_flowcontrol_sw_input(xcp, FLOW_STOP,
			    IN_FLOW_USER);
		}
		mutex_exit(&xcp->excl);
		mutex_exit(&xcp->excl);
		freemsg(mp);
		break;

	case M_STARTI:
		mutex_enter(&xcp->excl);
		mutex_enter(&xcp->excl);
		if (async->async_inflow_source & IN_FLOW_USER) {
			(void) xcasync_flowcontrol_sw_input(xcp, FLOW_START,
			    IN_FLOW_USER);
		}
		mutex_exit(&xcp->excl);
		mutex_exit(&xcp->excl);
		freemsg(mp);
		break;

	case M_CTL:
		if (MBLKL(mp) >= sizeof (struct iocblk) &&
		    ((struct iocblk *)mp->b_rptr)->ioc_cmd == MC_POSIXQUERY) {
			((struct iocblk *)mp->b_rptr)->ioc_cmd = MC_HAS_POSIX;
			qreply(q, mp);
		} else {
			freemsg(mp);
		}
		break;

	default:
		freemsg(mp);
		break;
	}
	return (0);
}

/*
 * Retry an "ioctl", now that "bufcall" claims we may be able to allocate
 * the buffer we need.
 */
static void
xcasync_reioctl(void *unit)
{
	int instance = (uintptr_t)unit;
	struct asyncline *async;
	struct xencons *xcp;
	queue_t	*q;
	mblk_t	*mp;

	xcp = ddi_get_soft_state(xencons_soft_state, instance);
	ASSERT(xcp != NULL);
	async = xcp->priv;

	/*
	 * The bufcall is no longer pending.
	 */
	mutex_enter(&xcp->excl);
	async->async_wbufcid = 0;
	if ((q = async->async_ttycommon.t_writeq) == NULL) {
		mutex_exit(&xcp->excl);
		return;
	}
	if ((mp = async->async_ttycommon.t_iocpending) != NULL) {
		/* not pending any more */
		async->async_ttycommon.t_iocpending = NULL;
		mutex_exit(&xcp->excl);
		xcasync_ioctl(async, q, mp);
	} else
		mutex_exit(&xcp->excl);
}


/*
 * debugger/console support routines.
 */

/*
 * put a character out
 * Do not use interrupts.  If char is LF, put out CR, LF.
 */
/*ARGSUSED*/
static void
xenconsputchar(cons_polledio_arg_t arg, uchar_t c)
{
	struct xencons *xcp = xencons_console;
	volatile struct xencons_interface *ifp = xcp->ifp;
	XENCONS_RING_IDX prod;

	if (c == '\n')
		xenconsputchar(arg, '\r');

	/*
	 * domain 0 can use the console I/O...
	 */
	if (DOMAIN_IS_INITDOMAIN(xen_info)) {
		char	buffer[1];

		buffer[0] = c;
		(void) HYPERVISOR_console_io(CONSOLEIO_write, 1, buffer);
		return;
	}

	/*
	 * domU has to go through dom0 virtual console.
	 */
	while (ifp->out_prod - ifp->out_cons == sizeof (ifp->out))
		(void) HYPERVISOR_yield();

	prod = ifp->out_prod;
	ifp->out[MASK_XENCONS_IDX(prod++, ifp->out)] = c;
	membar_producer();
	ifp->out_prod = prod;
	ec_notify_via_evtchn(xcp->evtchn);
}

/*
 * See if there's a character available. If no character is
 * available, return 0. Run in polled mode, no interrupts.
 */
static boolean_t
xenconsischar(cons_polledio_arg_t arg)
{
	struct xencons *xcp = (struct xencons *)arg;
	volatile struct xencons_interface *ifp = xcp->ifp;

	if (xcp->polldix < xcp->polllen)
		return (B_TRUE);
	/*
	 * domain 0 can use the console I/O...
	 */
	xcp->polldix = 0;
	xcp->polllen = 0;
	if (DOMAIN_IS_INITDOMAIN(xen_info)) {
		xcp->polllen = HYPERVISOR_console_io(CONSOLEIO_read, 1,
		    (char *)xcp->pollbuf);
		return (xcp->polllen != 0);
	}

	/*
	 * domU has to go through virtual console device.
	 */
	if (ifp->in_prod != ifp->in_cons) {
		XENCONS_RING_IDX cons;

		cons = ifp->in_cons;
		membar_enter();
		xcp->pollbuf[0] = ifp->in[MASK_XENCONS_IDX(cons++, ifp->in)];
		membar_producer();
		ifp->in_cons = cons;
		xcp->polllen = 1;
	}
	return (xcp->polllen != 0);
}

/*
 * Get a character. Run in polled mode, no interrupts.
 */
static int
xenconsgetchar(cons_polledio_arg_t arg)
{
	struct xencons *xcp = (struct xencons *)arg;

	ec_wait_on_evtchn(xcp->evtchn, (int (*)(void *))xenconsischar, arg);

	return (xcp->pollbuf[xcp->polldix++]);
}

static void
xenconserror(int level, const char *fmt, ...)
{
	va_list adx;
	static time_t	last;
	static const char *lastfmt;
	time_t now;

	/*
	 * Don't print the same error message too often.
	 * Print the message only if we have not printed the
	 * message within the last second.
	 * Note: that fmt cannot be a pointer to a string
	 * stored on the stack. The fmt pointer
	 * must be in the data segment otherwise lastfmt would point
	 * to non-sense.
	 */
	now = gethrestime_sec();
	if (last == now && lastfmt == fmt)
		return;

	last = now;
	lastfmt = fmt;

	va_start(adx, fmt);
	vcmn_err(level, fmt, adx);
	va_end(adx);
}


/*
 * Check for abort character sequence
 */
static boolean_t
abort_charseq_recognize(uchar_t ch)
{
	static int state = 0;
#define	CNTRL(c) ((c)&037)
	static char sequence[] = { '\r', '~', CNTRL('b') };

	if (ch == sequence[state]) {
		if (++state >= sizeof (sequence)) {
			state = 0;
			return (B_TRUE);
		}
	} else {
		state = (ch == sequence[0]) ? 1 : 0;
	}
	return (B_FALSE);
}

/*
 * Flow control functions
 */

/*
 * Software output flow control
 * This function can be executed sucessfully at any situation.
 * It does not handle HW, and just change the SW output flow control flag.
 * INPUT VALUE of onoff:
 *                 FLOW_START means to clear SW output flow control flag,
 *			also set ASYNC_OUT_FLW_RESUME.
 *                 FLOW_STOP means to set SW output flow control flag,
 *			also clear ASYNC_OUT_FLW_RESUME.
 */
static void
xcasync_flowcontrol_sw_output(struct xencons *xcp, async_flowc_action onoff)
{
	struct asyncline *async = xcp->priv;
	int instance = xcp->unit;

	ASSERT(mutex_owned(&xcp->excl));

	if (!(async->async_ttycommon.t_iflag & IXON))
		return;

	switch (onoff) {
	case FLOW_STOP:
		async->async_flags |= ASYNC_SW_OUT_FLW;
		async->async_flags &= ~ASYNC_OUT_FLW_RESUME;
		DEBUGCONT1(XENCONS_DEBUG_SFLOW,
		    "xencons%d: output sflow stop\n", instance);
		break;
	case FLOW_START:
		async->async_flags &= ~ASYNC_SW_OUT_FLW;
		async->async_flags |= ASYNC_OUT_FLW_RESUME;
		DEBUGCONT1(XENCONS_DEBUG_SFLOW,
		    "xencons%d: output sflow start\n", instance);
		break;
	default:
		break;
	}
}

/*
 * Software input flow control
 * This function can execute software input flow control
 * INPUT VALUE of onoff:
 *               FLOW_START means to send out a XON char
 *                          and clear SW input flow control flag.
 *               FLOW_STOP means to send out a XOFF char
 *                          and set SW input flow control flag.
 *               FLOW_CHECK means to check whether there is pending XON/XOFF
 *                          if it is true, send it out.
 * INPUT VALUE of type:
 *		 IN_FLOW_STREAMS means flow control is due to STREAMS
 *		 IN_FLOW_USER means flow control is due to user's commands
 * RETURN VALUE: B_FALSE means no flow control char is sent
 *               B_TRUE means one flow control char is sent
 */
static boolean_t
xcasync_flowcontrol_sw_input(struct xencons *xcp, async_flowc_action onoff,
    int type)
{
	struct asyncline *async = xcp->priv;
	int instance = xcp->unit;
	int rval = B_FALSE;

	ASSERT(mutex_owned(&xcp->excl));

	if (!(async->async_ttycommon.t_iflag & IXOFF))
		return (rval);

	/*
	 * If we get this far, then we know IXOFF is set.
	 */
	switch (onoff) {
	case FLOW_STOP:
		async->async_inflow_source |= type;

		/*
		 * We'll send an XOFF character for each of up to
		 * three different input flow control attempts to stop input.
		 * If we already send out one XOFF, but FLOW_STOP comes again,
		 * it seems that input flow control becomes more serious,
		 * then send XOFF again.
		 */
		if (async->async_inflow_source & (IN_FLOW_STREAMS |
		    IN_FLOW_USER))
			async->async_flags |= ASYNC_SW_IN_FLOW |
			    ASYNC_SW_IN_NEEDED;
		DEBUGCONT2(XENCONS_DEBUG_SFLOW, "xencons%d: input sflow stop, "
		    "type = %x\n", instance, async->async_inflow_source);
		break;
	case FLOW_START:
		async->async_inflow_source &= ~type;
		if (async->async_inflow_source == 0) {
			async->async_flags = (async->async_flags &
			    ~ASYNC_SW_IN_FLOW) | ASYNC_SW_IN_NEEDED;
			DEBUGCONT1(XENCONS_DEBUG_SFLOW, "xencons%d: "
			    "input sflow start\n", instance);
		}
		break;
	default:
		break;
	}

	if (async->async_flags & ASYNC_SW_IN_NEEDED) {
		/*
		 * If we get this far, then we know we need to send out
		 * XON or XOFF char.
		 */
		char c;

		rval = B_TRUE;
		c = (async->async_flags & ASYNC_SW_IN_FLOW) ?
		    async->async_stopc : async->async_startc;
		if (DOMAIN_IS_INITDOMAIN(xen_info)) {
			(void) HYPERVISOR_console_io(CONSOLEIO_write, 1, &c);
			async->async_flags &= ~ASYNC_SW_IN_NEEDED;
			return (rval);
		} else {
			xenconsputchar(NULL, c);
		}
	}
	return (rval);
}

struct module_info xencons_info = {
	0,
	"xencons",
	0,
	INFPSZ,
	4096,
	128
};

static struct qinit xencons_rint = {
	putq,
	xenconsrsrv,
	xenconsopen,
	xenconsclose,
	NULL,
	&xencons_info,
	NULL
};

static struct qinit xencons_wint = {
	xenconswput,
	NULL,
	NULL,
	NULL,
	NULL,
	&xencons_info,
	NULL
};

struct streamtab xencons_str_info = {
	&xencons_rint,
	&xencons_wint,
	NULL,
	NULL
};

static struct cb_ops cb_xencons_ops = {
	nodev,			/* cb_open */
	nodev,			/* cb_close */
	nodev,			/* cb_strategy */
	nodev,			/* cb_print */
	nodev,			/* cb_dump */
	nodev,			/* cb_read */
	nodev,			/* cb_write */
	nodev,			/* cb_ioctl */
	nodev,			/* cb_devmap */
	nodev,			/* cb_mmap */
	nodev,			/* cb_segmap */
	nochpoll,		/* cb_chpoll */
	ddi_prop_op,		/* cb_prop_op */
	&xencons_str_info,		/* cb_stream */
	D_MP			/* cb_flag */
};

struct dev_ops xencons_ops = {
	DEVO_REV,		/* devo_rev */
	0,			/* devo_refcnt */
	xenconsinfo,		/* devo_getinfo */
	nulldev,		/* devo_identify */
	nulldev,		/* devo_probe */
	xenconsattach,		/* devo_attach */
	xenconsdetach,		/* devo_detach */
	nodev,			/* devo_reset */
	&cb_xencons_ops,	/* devo_cb_ops */
	NULL,			/* devo_bus_ops */
	NULL,			/* devo_power */
	ddi_quiesce_not_needed,		/* devo_quiesce */
};

static struct modldrv modldrv = {
	&mod_driverops, /* Type of module.  This one is a driver */
	"virtual console driver",
	&xencons_ops,	/* driver ops */
};

static struct modlinkage modlinkage = {
	MODREV_1,
	(void *)&modldrv,
	NULL
};

int
_init(void)
{
	int rv;

	if ((rv = ddi_soft_state_init(&xencons_soft_state,
	    sizeof (struct xencons), 1)) != 0)
		return (rv);
	if ((rv = mod_install(&modlinkage)) != 0) {
		ddi_soft_state_fini(&xencons_soft_state);
		return (rv);
	}
	DEBUGCONT2(XENCONS_DEBUG_INIT, "%s, debug = %x\n",
	    modldrv.drv_linkinfo, debug);
	return (0);
}

int
_fini(void)
{
	int rv;

	if ((rv = mod_remove(&modlinkage)) != 0)
		return (rv);

	ddi_soft_state_fini(&xencons_soft_state);
	return (0);
}

int
_info(struct modinfo *modinfop)
{
	return (mod_info(&modlinkage, modinfop));
}
#
# CDDL HEADER START
#
# The contents of this file are subject to the terms of the
# Common Development and Distribution License (the "License").
# You may not use this file except in compliance with the License.
#
# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
# or http://www.opensolaris.org/os/licensing.
# See the License for the specific language governing permissions
# and limitations under the License.
#
# When distributing Covered Code, include this CDDL HEADER in each
# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
# If applicable, add the following below this CDDL HEADER, with the
# fields enclosed by brackets "[]" replaced with your own identifying
# information: Portions Copyright [yyyy] [name of copyright owner]
#
# CDDL HEADER END
#

#
# Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
# Use is subject to license terms.
#
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*	Copyright (c) 1990, 1991 UNIX System Laboratories, Inc.	*/
/*	Copyright (c) 1984, 1986, 1987, 1988, 1989, 1990 AT&T	*/
/*	  All Rights Reserved 	*/

/*
 * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#ifndef	_SYS_XENCONS_H
#define	_SYS_XENCONS_H

#ifdef __cplusplus
extern "C" {
#endif

#include <sys/tty.h>
#include <sys/ksynch.h>
#include <sys/dditypes.h>
#include <xen/public/io/console.h>


/*
 * Xencons tracing macros.  These are a similar to some macros in sys/vtrace.h.
 *
 * XXX - Needs review:  would it be better to use the macros in sys/vtrace.h ?
 */
#ifdef DEBUG
#define	DEBUGWARN0(fac, format) \
	if (debug & (fac)) \
		cmn_err(CE_WARN, format)
#define	DEBUGNOTE0(fac, format) \
	if (debug & (fac)) \
		cmn_err(CE_NOTE, format)
#define	DEBUGNOTE1(fac, format, arg1) \
	if (debug & (fac)) \
		cmn_err(CE_NOTE, format, arg1)
#define	DEBUGNOTE2(fac, format, arg1, arg2) \
	if (debug & (fac)) \
		cmn_err(CE_NOTE, format, arg1, arg2)
#define	DEBUGNOTE3(fac, format, arg1, arg2, arg3) \
	if (debug & (fac)) \
		cmn_err(CE_NOTE, format, arg1, arg2, arg3)
#define	DEBUGCONT0(fac, format) \
	if (debug & (fac)) \
		cmn_err(CE_CONT, format)
#define	DEBUGCONT1(fac, format, arg1) \
	if (debug & (fac)) \
		cmn_err(CE_CONT, format, arg1)
#define	DEBUGCONT2(fac, format, arg1, arg2) \
	if (debug & (fac)) \
		cmn_err(CE_CONT, format, arg1, arg2)
#define	DEBUGCONT3(fac, format, arg1, arg2, arg3) \
	if (debug & (fac)) \
		cmn_err(CE_CONT, format, arg1, arg2, arg3)
#define	DEBUGCONT4(fac, format, arg1, arg2, arg3, arg4) \
	if (debug & (fac)) \
		cmn_err(CE_CONT, format, arg1, arg2, arg3, arg4)
#define	DEBUGCONT10(fac, format, \
	arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, arg9, arg10) \
	if (debug & (fac)) \
		cmn_err(CE_CONT, format, \
		arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, arg9, arg10)
#else
#define	DEBUGWARN0(fac, format)
#define	DEBUGNOTE0(fac, format)
#define	DEBUGNOTE1(fac, format, arg1)
#define	DEBUGNOTE2(fac, format, arg1, arg2)
#define	DEBUGNOTE3(fac, format, arg1, arg2, arg3)
#define	DEBUGCONT0(fac, format)
#define	DEBUGCONT1(fac, format, arg1)
#define	DEBUGCONT2(fac, format, arg1, arg2)
#define	DEBUGCONT3(fac, format, arg1, arg2, arg3)
#define	DEBUGCONT4(fac, format, arg1, arg2, arg3, arg4)
#define	DEBUGCONT10(fac, format, \
	arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, arg9, arg10)
#endif

/* enum value for sw and hw flow control action */
typedef enum {
	FLOW_CHECK,
	FLOW_STOP,
	FLOW_START
} async_flowc_action;

#define	async_stopc	async_ttycommon.t_stopc
#define	async_startc	async_ttycommon.t_startc

/*
 * Console instance data.
 * Each of the fields in this structure is required to be protected by a
 * mutex lock at the highest priority at which it can be altered.
 */

struct xencons {
	int		flags;	/* random flags  */
	struct asyncline *priv;	/* protocol private data -- asyncline */
	dev_info_t	*dip;	/* dev_info */
	int		unit;	/* which port */
	kmutex_t	excl;	/* adaptive mutex */
	kcondvar_t	excl_cv;	/* condition variable */
	struct cons_polledio polledio;	/* polled I/O functions */
	unsigned char	pollbuf[60];	/* polled I/O data */
	int		polldix;	/* polled data buffer index */
	int		polllen;	/* polled data buffer length */
	volatile struct xencons_interface *ifp;	/* console ring buffers */
	int		console_irq;	/* dom0 console interrupt */
	int		evtchn;		/* console event channel */
};

/*
 * Asychronous protocol private data structure for ASY.
 * Each of the fields in the structure is required to be protected by
 * the lower priority lock except the fields that are set only at
 * base level but cleared (with out lock) at interrupt level.
 */

struct asyncline {
	int		async_flags;	/* random flags */
	kcondvar_t	async_flags_cv; /* condition variable for flags */
	dev_t		async_dev;	/* device major/minor numbers */
	mblk_t		*async_xmitblk;	/* transmit: active msg block */
	struct xencons	*async_common;	/* device common data */
	tty_common_t 	async_ttycommon; /* tty driver common data */
	bufcall_id_t	async_wbufcid;	/* id for pending write-side bufcall */
	timeout_id_t	async_polltid;	/* softint poll timeout id */
	timeout_id_t    async_dtrtid;   /* delaying DTR turn on */
	timeout_id_t    async_utbrktid; /* hold minimum untimed break time id */

	/*
	 * The following fields are protected by the excl_hi lock.
	 * Some, such as async_flowc, are set only at the base level and
	 * cleared (without the lock) only by the interrupt level.
	 */
	uchar_t		*async_optr;	/* output pointer */
	int		async_ocnt;	/* output count */
	ushort_t	async_rput;	/* producing pointer for input */
	ushort_t	async_rget;	/* consuming pointer for input */
	int		async_inflow_source; /* input flow control type */

	union {
		struct {
			uchar_t _hw;	/* overrun (hw) */
			uchar_t _sw;	/* overrun (sw) */
		} _a;
		ushort_t uover_overrun;
	} async_uover;
#define	async_overrun		async_uover._a.uover_overrun
#define	async_hw_overrun	async_uover._a._hw
#define	async_sw_overrun	async_uover._a._sw
	short		async_ext;	/* modem status change count */
	short		async_work;	/* work to do flag */
};

/* definitions for async_flags field */
#define	ASYNC_EXCL_OPEN	 0x10000000	/* exclusive open */
#define	ASYNC_WOPEN	 0x00000001	/* waiting for open to complete */
#define	ASYNC_ISOPEN	 0x00000002	/* open is complete */
#define	ASYNC_STOPPED	 0x00000010	/* output is stopped */
#define	ASYNC_PROGRESS	 0x00001000	/* made progress on output effort */
#define	ASYNC_CLOSING	 0x00002000	/* processing close on stream */
#define	ASYNC_SW_IN_FLOW 0x00020000	/* sw input flow control in effect */
#define	ASYNC_SW_OUT_FLW 0x00040000	/* sw output flow control in effect */
#define	ASYNC_SW_IN_NEEDED 0x00080000	/* sw input flow control char is */
					/* needed to be sent */
#define	ASYNC_OUT_FLW_RESUME 0x00100000 /* output need to be resumed */
					/* because of transition of flow */
					/* control from stop to start */


/* definitions for asy_flags field */
#define	ASY_CONSOLE	0x00000080

/* definitions for async_inflow_source field in struct asyncline */
#define	IN_FLOW_NULL	0x00000000
#define	IN_FLOW_STREAMS	0x00000002
#define	IN_FLOW_USER	0x00000004

#define	XENCONS_BURST	128	/* burst size for console writes */

#ifdef __cplusplus
}
#endif

#endif	/* _SYS_XENCONS_H */
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 * Copyright 2018 Joyent, Inc.
 */

#ifdef DEBUG
#define	XNB_DEBUG 1
#endif /* DEBUG */

#include "xnb.h"

#include <sys/sunddi.h>
#include <sys/sunndi.h>
#include <sys/modctl.h>
#include <sys/conf.h>
#include <sys/mac.h>
#include <sys/mac_impl.h> /* For mac_fix_cksum(). */
#include <sys/dlpi.h>
#include <sys/strsubr.h>
#include <sys/strsun.h>
#include <sys/types.h>
#include <sys/pattr.h>
#include <vm/seg_kmem.h>
#include <vm/hat_i86.h>
#include <xen/sys/xenbus_impl.h>
#include <xen/sys/xendev.h>
#include <sys/balloon_impl.h>
#include <sys/evtchn_impl.h>
#include <sys/gnttab.h>
#include <vm/vm_dep.h>
#include <sys/note.h>
#include <sys/gld.h>
#include <inet/ip.h>
#include <inet/ip_impl.h>

/*
 * The terms "transmit" and "receive" are used in alignment with domU,
 * which means that packets originating from the peer domU are "transmitted"
 * to other parts of the system and packets are "received" from them.
 */

/*
 * Should we allow guests to manipulate multicast group membership?
 */
static boolean_t	xnb_multicast_control = B_TRUE;

static boolean_t	xnb_connect_rings(dev_info_t *);
static void		xnb_disconnect_rings(dev_info_t *);
static void		xnb_oe_state_change(dev_info_t *, ddi_eventcookie_t,
    void *, void *);
static void		xnb_hp_state_change(dev_info_t *, ddi_eventcookie_t,
    void *, void *);

static int	xnb_txbuf_constructor(void *, void *, int);
static void	xnb_txbuf_destructor(void *, void *);
static void	xnb_tx_notify_peer(xnb_t *, boolean_t);
static void	xnb_tx_mark_complete(xnb_t *, RING_IDX, int16_t);

mblk_t		*xnb_to_peer(xnb_t *, mblk_t *);
mblk_t		*xnb_copy_to_peer(xnb_t *, mblk_t *);

static void		setup_gop(xnb_t *, gnttab_copy_t *, uchar_t *,
    size_t, size_t, size_t, grant_ref_t);
static boolean_t	is_foreign(void *);

#define	INVALID_GRANT_HANDLE	((grant_handle_t)-1)
#define	INVALID_GRANT_REF	((grant_ref_t)-1)

static kmutex_t	xnb_alloc_page_lock;

/*
 * On a 32 bit PAE system physical and machine addresses are larger
 * than 32 bits.  ddi_btop() on such systems take an unsigned long
 * argument, and so addresses above 4G are truncated before ddi_btop()
 * gets to see them.  To avoid this, code the shift operation here.
 */
#define	xnb_btop(addr)	((addr) >> PAGESHIFT)

/* DMA attributes for transmit and receive data */
static ddi_dma_attr_t buf_dma_attr = {
	DMA_ATTR_V0,		/* version of this structure */
	0,			/* lowest usable address */
	0xffffffffffffffffULL,	/* highest usable address */
	0x7fffffff,		/* maximum DMAable byte count */
	MMU_PAGESIZE,		/* alignment in bytes */
	0x7ff,			/* bitmap of burst sizes */
	1,			/* minimum transfer */
	0xffffffffU,		/* maximum transfer */
	0xffffffffffffffffULL,	/* maximum segment length */
	1,			/* maximum number of segments */
	1,			/* granularity */
	0,			/* flags (reserved) */
};

/* DMA access attributes for data: NOT to be byte swapped. */
static ddi_device_acc_attr_t data_accattr = {
	DDI_DEVICE_ATTR_V0,
	DDI_NEVERSWAP_ACC,
	DDI_STRICTORDER_ACC
};

/*
 * Statistics.
 */
static const char * const aux_statistics[] = {
	"rx_cksum_deferred",
	"tx_cksum_no_need",
	"rx_rsp_notok",
	"tx_notify_deferred",
	"tx_notify_sent",
	"rx_notify_deferred",
	"rx_notify_sent",
	"tx_too_early",
	"rx_too_early",
	"rx_allocb_failed",
	"tx_allocb_failed",
	"rx_foreign_page",
	"mac_full",
	"spurious_intr",
	"allocation_success",
	"allocation_failure",
	"small_allocation_success",
	"small_allocation_failure",
	"other_allocation_failure",
	"rx_pageboundary_crossed",
	"rx_cpoparea_grown",
	"csum_hardware",
	"csum_software",
	"tx_overflow_page",
	"tx_unexpected_flags",
};

static int
xnb_ks_aux_update(kstat_t *ksp, int flag)
{
	xnb_t *xnbp;
	kstat_named_t *knp;

	if (flag != KSTAT_READ)
		return (EACCES);

	xnbp = ksp->ks_private;
	knp = ksp->ks_data;

	/*
	 * Assignment order should match that of the names in
	 * aux_statistics.
	 */
	(knp++)->value.ui64 = xnbp->xnb_stat_rx_cksum_deferred;
	(knp++)->value.ui64 = xnbp->xnb_stat_tx_cksum_no_need;
	(knp++)->value.ui64 = xnbp->xnb_stat_rx_rsp_notok;
	(knp++)->value.ui64 = xnbp->xnb_stat_tx_notify_deferred;
	(knp++)->value.ui64 = xnbp->xnb_stat_tx_notify_sent;
	(knp++)->value.ui64 = xnbp->xnb_stat_rx_notify_deferred;
	(knp++)->value.ui64 = xnbp->xnb_stat_rx_notify_sent;
	(knp++)->value.ui64 = xnbp->xnb_stat_tx_too_early;
	(knp++)->value.ui64 = xnbp->xnb_stat_rx_too_early;
	(knp++)->value.ui64 = xnbp->xnb_stat_rx_allocb_failed;
	(knp++)->value.ui64 = xnbp->xnb_stat_tx_allocb_failed;
	(knp++)->value.ui64 = xnbp->xnb_stat_rx_foreign_page;
	(knp++)->value.ui64 = xnbp->xnb_stat_mac_full;
	(knp++)->value.ui64 = xnbp->xnb_stat_spurious_intr;
	(knp++)->value.ui64 = xnbp->xnb_stat_allocation_success;
	(knp++)->value.ui64 = xnbp->xnb_stat_allocation_failure;
	(knp++)->value.ui64 = xnbp->xnb_stat_small_allocation_success;
	(knp++)->value.ui64 = xnbp->xnb_stat_small_allocation_failure;
	(knp++)->value.ui64 = xnbp->xnb_stat_other_allocation_failure;
	(knp++)->value.ui64 = xnbp->xnb_stat_rx_pagebndry_crossed;
	(knp++)->value.ui64 = xnbp->xnb_stat_rx_cpoparea_grown;
	(knp++)->value.ui64 = xnbp->xnb_stat_csum_hardware;
	(knp++)->value.ui64 = xnbp->xnb_stat_csum_software;
	(knp++)->value.ui64 = xnbp->xnb_stat_tx_overflow_page;
	(knp++)->value.ui64 = xnbp->xnb_stat_tx_unexpected_flags;

	return (0);
}

static boolean_t
xnb_ks_init(xnb_t *xnbp)
{
	int nstat = sizeof (aux_statistics) /
	    sizeof (aux_statistics[0]);
	const char * const *cp = aux_statistics;
	kstat_named_t *knp;

	/*
	 * Create and initialise kstats.
	 */
	xnbp->xnb_kstat_aux = kstat_create(ddi_driver_name(xnbp->xnb_devinfo),
	    ddi_get_instance(xnbp->xnb_devinfo), "aux_statistics", "net",
	    KSTAT_TYPE_NAMED, nstat, 0);
	if (xnbp->xnb_kstat_aux == NULL)
		return (B_FALSE);

	xnbp->xnb_kstat_aux->ks_private = xnbp;
	xnbp->xnb_kstat_aux->ks_update = xnb_ks_aux_update;

	knp = xnbp->xnb_kstat_aux->ks_data;
	while (nstat > 0) {
		kstat_named_init(knp, *cp, KSTAT_DATA_UINT64);

		knp++;
		cp++;
		nstat--;
	}

	kstat_install(xnbp->xnb_kstat_aux);

	return (B_TRUE);
}

static void
xnb_ks_free(xnb_t *xnbp)
{
	kstat_delete(xnbp->xnb_kstat_aux);
}

/*
 * Calculate and insert the transport checksum for an arbitrary packet.
 */
static mblk_t *
xnb_software_csum(xnb_t *xnbp, mblk_t *mp)
{
	_NOTE(ARGUNUSED(xnbp));

	/*
	 * XXPV dme: shouldn't rely on mac_fix_cksum(), not least
	 * because it doesn't cover all of the interesting cases :-(
	 */
	mac_hcksum_set(mp, 0, 0, 0, 0, HCK_FULLCKSUM);
	mac_hw_emul(&mp, NULL, NULL, MAC_HWCKSUM_EMUL);
	return (mp);
}

mblk_t *
xnb_process_cksum_flags(xnb_t *xnbp, mblk_t *mp, uint32_t capab)
{
	struct ether_header *ehp;
	uint16_t sap;
	uint32_t offset;
	ipha_t *ipha;

	ASSERT(mp->b_next == NULL);

	/*
	 * Check that the packet is contained in a single mblk.  In
	 * the "from peer" path this is true today, but may change
	 * when scatter gather support is added.  In the "to peer"
	 * path we cannot be sure, but in most cases it will be true
	 * (in the xnbo case the packet has come from a MAC device
	 * which is unlikely to split packets).
	 */
	if (mp->b_cont != NULL)
		goto software;

	/*
	 * If the MAC has no hardware capability don't do any further
	 * checking.
	 */
	if (capab == 0)
		goto software;

	ASSERT(MBLKL(mp) >= sizeof (struct ether_header));
	ehp = (struct ether_header *)mp->b_rptr;

	if (ntohs(ehp->ether_type) == VLAN_TPID) {
		struct ether_vlan_header *evhp;

		ASSERT(MBLKL(mp) >= sizeof (struct ether_vlan_header));
		evhp = (struct ether_vlan_header *)mp->b_rptr;
		sap = ntohs(evhp->ether_type);
		offset = sizeof (struct ether_vlan_header);
	} else {
		sap = ntohs(ehp->ether_type);
		offset = sizeof (struct ether_header);
	}

	/*
	 * We only attempt to do IPv4 packets in hardware.
	 */
	if (sap != ETHERTYPE_IP)
		goto software;

	/*
	 * We know that this is an IPv4 packet.
	 */
	ipha = (ipha_t *)(mp->b_rptr + offset);

	switch (ipha->ipha_protocol) {
	case IPPROTO_TCP:
	case IPPROTO_UDP: {
		uint32_t start, length, stuff, cksum;
		uint16_t *stuffp;

		/*
		 * This is a TCP/IPv4 or UDP/IPv4 packet, for which we
		 * can use full IPv4 and partial checksum offload.
		 */
		if ((capab & (HCKSUM_INET_FULL_V4|HCKSUM_INET_PARTIAL)) == 0)
			break;

		start = IP_SIMPLE_HDR_LENGTH;
		length = ntohs(ipha->ipha_length);
		if (ipha->ipha_protocol == IPPROTO_TCP) {
			stuff = start + TCP_CHECKSUM_OFFSET;
			cksum = IP_TCP_CSUM_COMP;
		} else {
			stuff = start + UDP_CHECKSUM_OFFSET;
			cksum = IP_UDP_CSUM_COMP;
		}
		stuffp = (uint16_t *)(mp->b_rptr + offset + stuff);

		if (capab & HCKSUM_INET_FULL_V4) {
			/*
			 * Some devices require that the checksum
			 * field of the packet is zero for full
			 * offload.
			 */
			*stuffp = 0;

			mac_hcksum_set(mp, 0, 0, 0, 0, HCK_FULLCKSUM);

			xnbp->xnb_stat_csum_hardware++;

			return (mp);
		}

		if (capab & HCKSUM_INET_PARTIAL) {
			if (*stuffp == 0) {
				ipaddr_t src, dst;

				/*
				 * Older Solaris guests don't insert
				 * the pseudo-header checksum, so we
				 * calculate it here.
				 */
				src = ipha->ipha_src;
				dst = ipha->ipha_dst;

				cksum += (dst >> 16) + (dst & 0xFFFF);
				cksum += (src >> 16) + (src & 0xFFFF);
				cksum += length - IP_SIMPLE_HDR_LENGTH;

				cksum = (cksum >> 16) + (cksum & 0xFFFF);
				cksum = (cksum >> 16) + (cksum & 0xFFFF);

				ASSERT(cksum <= 0xFFFF);

				*stuffp = (uint16_t)(cksum ? cksum : ~cksum);
			}

			mac_hcksum_set(mp, start, stuff, length, 0,
			    HCK_PARTIALCKSUM);

			xnbp->xnb_stat_csum_hardware++;

			return (mp);
		}

		/* NOTREACHED */
		break;
	}

	default:
		/* Use software. */
		break;
	}

software:
	/*
	 * We are not able to use any offload so do the whole thing in
	 * software.
	 */
	xnbp->xnb_stat_csum_software++;

	return (xnb_software_csum(xnbp, mp));
}

int
xnb_attach(dev_info_t *dip, xnb_flavour_t *flavour, void *flavour_data)
{
	xnb_t *xnbp;
	char *xsname;
	char cachename[32];

	xnbp = kmem_zalloc(sizeof (*xnbp), KM_SLEEP);

	xnbp->xnb_flavour = flavour;
	xnbp->xnb_flavour_data = flavour_data;
	xnbp->xnb_devinfo = dip;
	xnbp->xnb_evtchn = INVALID_EVTCHN;
	xnbp->xnb_irq = B_FALSE;
	xnbp->xnb_tx_ring_handle = INVALID_GRANT_HANDLE;
	xnbp->xnb_rx_ring_handle = INVALID_GRANT_HANDLE;
	xnbp->xnb_connected = B_FALSE;
	xnbp->xnb_hotplugged = B_FALSE;
	xnbp->xnb_detachable = B_FALSE;
	xnbp->xnb_peer = xvdi_get_oeid(dip);
	xnbp->xnb_be_status = XNB_STATE_INIT;
	xnbp->xnb_fe_status = XNB_STATE_INIT;

	xnbp->xnb_tx_buf_count = 0;

	xnbp->xnb_rx_hv_copy = B_FALSE;
	xnbp->xnb_multicast_control = B_FALSE;

	xnbp->xnb_rx_va = vmem_alloc(heap_arena, PAGESIZE, VM_SLEEP);
	ASSERT(xnbp->xnb_rx_va != NULL);

	if (ddi_get_iblock_cookie(dip, 0, &xnbp->xnb_icookie)
	    != DDI_SUCCESS)
		goto failure;

	/* Allocated on demand, when/if we enter xnb_copy_to_peer(). */
	xnbp->xnb_rx_cpop = NULL;
	xnbp->xnb_rx_cpop_count = 0;

	mutex_init(&xnbp->xnb_tx_lock, NULL, MUTEX_DRIVER,
	    xnbp->xnb_icookie);
	mutex_init(&xnbp->xnb_rx_lock, NULL, MUTEX_DRIVER,
	    xnbp->xnb_icookie);
	mutex_init(&xnbp->xnb_state_lock, NULL, MUTEX_DRIVER,
	    xnbp->xnb_icookie);

	/* Set driver private pointer now. */
	ddi_set_driver_private(dip, xnbp);

	(void) sprintf(cachename, "xnb_tx_buf_cache_%d", ddi_get_instance(dip));
	xnbp->xnb_tx_buf_cache = kmem_cache_create(cachename,
	    sizeof (xnb_txbuf_t), 0,
	    xnb_txbuf_constructor, xnb_txbuf_destructor,
	    NULL, xnbp, NULL, 0);
	if (xnbp->xnb_tx_buf_cache == NULL)
		goto failure_0;

	if (!xnb_ks_init(xnbp))
		goto failure_1;

	/*
	 * Receive notification of changes in the state of the
	 * driver in the guest domain.
	 */
	if (xvdi_add_event_handler(dip, XS_OE_STATE, xnb_oe_state_change,
	    NULL) != DDI_SUCCESS)
		goto failure_2;

	/*
	 * Receive notification of hotplug events.
	 */
	if (xvdi_add_event_handler(dip, XS_HP_STATE, xnb_hp_state_change,
	    NULL) != DDI_SUCCESS)
		goto failure_2;

	xsname = xvdi_get_xsname(dip);

	if (xenbus_printf(XBT_NULL, xsname,
	    "feature-multicast-control", "%d",
	    xnb_multicast_control ? 1 : 0) != 0)
		goto failure_3;

	if (xenbus_printf(XBT_NULL, xsname,
	    "feature-rx-copy", "%d",  1) != 0)
		goto failure_3;
	/*
	 * Linux domUs seem to depend on "feature-rx-flip" being 0
	 * in addition to "feature-rx-copy" being 1. It seems strange
	 * to use four possible states to describe a binary decision,
	 * but we might as well play nice.
	 */
	if (xenbus_printf(XBT_NULL, xsname,
	    "feature-rx-flip", "%d", 0) != 0)
		goto failure_3;

	(void) xvdi_switch_state(dip, XBT_NULL, XenbusStateInitWait);
	(void) xvdi_post_event(dip, XEN_HP_ADD);

	return (DDI_SUCCESS);

failure_3:
	xvdi_remove_event_handler(dip, NULL);

failure_2:
	xnb_ks_free(xnbp);

failure_1:
	kmem_cache_destroy(xnbp->xnb_tx_buf_cache);

failure_0:
	mutex_destroy(&xnbp->xnb_state_lock);
	mutex_destroy(&xnbp->xnb_rx_lock);
	mutex_destroy(&xnbp->xnb_tx_lock);

failure:
	vmem_free(heap_arena, xnbp->xnb_rx_va, PAGESIZE);
	kmem_free(xnbp, sizeof (*xnbp));
	return (DDI_FAILURE);
}

void
xnb_detach(dev_info_t *dip)
{
	xnb_t *xnbp = ddi_get_driver_private(dip);

	ASSERT(xnbp != NULL);
	ASSERT(!xnbp->xnb_connected);
	ASSERT(xnbp->xnb_tx_buf_count == 0);

	xnb_disconnect_rings(dip);

	xvdi_remove_event_handler(dip, NULL);

	xnb_ks_free(xnbp);

	kmem_cache_destroy(xnbp->xnb_tx_buf_cache);

	ddi_set_driver_private(dip, NULL);

	mutex_destroy(&xnbp->xnb_state_lock);
	mutex_destroy(&xnbp->xnb_rx_lock);
	mutex_destroy(&xnbp->xnb_tx_lock);

	if (xnbp->xnb_rx_cpop_count > 0)
		kmem_free(xnbp->xnb_rx_cpop, sizeof (xnbp->xnb_rx_cpop[0])
		    * xnbp->xnb_rx_cpop_count);

	ASSERT(xnbp->xnb_rx_va != NULL);
	vmem_free(heap_arena, xnbp->xnb_rx_va, PAGESIZE);

	kmem_free(xnbp, sizeof (*xnbp));
}

/*
 * Allocate a page from the hypervisor to be flipped to the peer.
 *
 * Try to get pages in batches to reduce the overhead of calls into
 * the balloon driver.
 */
static mfn_t
xnb_alloc_page(xnb_t *xnbp)
{
#define	WARNING_RATE_LIMIT 100
#define	BATCH_SIZE 256
	static mfn_t mfns[BATCH_SIZE];	/* common across all instances */
	static int nth = BATCH_SIZE;
	mfn_t mfn;

	mutex_enter(&xnb_alloc_page_lock);
	if (nth == BATCH_SIZE) {
		if (balloon_alloc_pages(BATCH_SIZE, mfns) != BATCH_SIZE) {
			xnbp->xnb_stat_allocation_failure++;
			mutex_exit(&xnb_alloc_page_lock);

			/*
			 * Try for a single page in low memory situations.
			 */
			if (balloon_alloc_pages(1, &mfn) != 1) {
				if ((xnbp->xnb_stat_small_allocation_failure++
				    % WARNING_RATE_LIMIT) == 0)
					cmn_err(CE_WARN, "xnb_alloc_page: "
					    "Cannot allocate memory to "
					    "transfer packets to peer.");
				return (0);
			} else {
				xnbp->xnb_stat_small_allocation_success++;
				return (mfn);
			}
		}

		nth = 0;
		xnbp->xnb_stat_allocation_success++;
	}

	mfn = mfns[nth++];
	mutex_exit(&xnb_alloc_page_lock);

	ASSERT(mfn != 0);

	return (mfn);
#undef BATCH_SIZE
#undef WARNING_RATE_LIMIT
}

/*
 * Free a page back to the hypervisor.
 *
 * This happens only in the error path, so batching is not worth the
 * complication.
 */
static void
xnb_free_page(xnb_t *xnbp, mfn_t mfn)
{
	_NOTE(ARGUNUSED(xnbp));
	int r;
	pfn_t pfn;

	pfn = xen_assign_pfn(mfn);
	pfnzero(pfn, 0, PAGESIZE);
	xen_release_pfn(pfn);

	if ((r = balloon_free_pages(1, &mfn, NULL, NULL)) != 1) {
		cmn_err(CE_WARN, "free_page: cannot decrease memory "
		    "reservation (%d): page kept but unusable (mfn = 0x%lx).",
		    r, mfn);
	}
}

/*
 * Similar to RING_HAS_UNCONSUMED_REQUESTS(&xnbp->rx_ring) but using
 * local variables. Used in both xnb_to_peer() and xnb_copy_to_peer().
 */
#define	XNB_RING_HAS_UNCONSUMED_REQUESTS(_r)		\
	((((_r)->sring->req_prod - loop) <		\
		(RING_SIZE(_r) - (loop - prod))) ?	\
	    ((_r)->sring->req_prod - loop) :		\
	    (RING_SIZE(_r) - (loop - prod)))

/*
 * Pass packets to the peer using page flipping.
 */
mblk_t *
xnb_to_peer(xnb_t *xnbp, mblk_t *mp)
{
	mblk_t *free = mp, *prev = NULL;
	size_t len;
	gnttab_transfer_t *gop;
	boolean_t notify;
	RING_IDX loop, prod, end;

	/*
	 * For each packet the sequence of operations is:
	 *
	 * 1. get a new page from the hypervisor.
	 * 2. get a request slot from the ring.
	 * 3. copy the data into the new page.
	 * 4. transfer the page to the peer.
	 * 5. update the request slot.
	 * 6. kick the peer.
	 * 7. free mp.
	 *
	 * In order to reduce the number of hypercalls, we prepare
	 * several packets for the peer and perform a single hypercall
	 * to transfer them.
	 */

	len = 0;
	mutex_enter(&xnbp->xnb_rx_lock);

	/*
	 * If we are not connected to the peer or have not yet
	 * finished hotplug it is too early to pass packets to the
	 * peer.
	 */
	if (!(xnbp->xnb_connected && xnbp->xnb_hotplugged)) {
		mutex_exit(&xnbp->xnb_rx_lock);
		DTRACE_PROBE(flip_rx_too_early);
		xnbp->xnb_stat_rx_too_early++;
		return (mp);
	}

	loop = xnbp->xnb_rx_ring.req_cons;
	prod = xnbp->xnb_rx_ring.rsp_prod_pvt;
	gop = xnbp->xnb_rx_top;

	while ((mp != NULL) &&
	    XNB_RING_HAS_UNCONSUMED_REQUESTS(&xnbp->xnb_rx_ring)) {

		mfn_t mfn;
		pfn_t pfn;
		netif_rx_request_t *rxreq;
		netif_rx_response_t *rxresp;
		char *valoop;
		mblk_t *ml;
		uint16_t cksum_flags;

		/* 1 */
		if ((mfn = xnb_alloc_page(xnbp)) == 0) {
			xnbp->xnb_stat_rx_defer++;
			break;
		}

		/* 2 */
		rxreq = RING_GET_REQUEST(&xnbp->xnb_rx_ring, loop);

#ifdef XNB_DEBUG
		if (!(rxreq->id < NET_RX_RING_SIZE))
			cmn_err(CE_PANIC, "xnb_to_peer: "
			    "id %d out of range in request 0x%p",
			    rxreq->id, (void *)rxreq);
#endif /* XNB_DEBUG */

		/* Assign a pfn and map the new page at the allocated va. */
		pfn = xen_assign_pfn(mfn);
		hat_devload(kas.a_hat, xnbp->xnb_rx_va, PAGESIZE,
		    pfn, PROT_READ | PROT_WRITE, HAT_LOAD);

		/* 3 */
		len = 0;
		valoop = xnbp->xnb_rx_va;
		for (ml = mp; ml != NULL; ml = ml->b_cont) {
			size_t chunk = ml->b_wptr - ml->b_rptr;

			bcopy(ml->b_rptr, valoop, chunk);
			valoop += chunk;
			len += chunk;
		}

		ASSERT(len < PAGESIZE);

		/* Release the pfn. */
		hat_unload(kas.a_hat, xnbp->xnb_rx_va, PAGESIZE,
		    HAT_UNLOAD_UNMAP);
		xen_release_pfn(pfn);

		/* 4 */
		gop->mfn = mfn;
		gop->domid = xnbp->xnb_peer;
		gop->ref = rxreq->gref;

		/* 5.1 */
		rxresp = RING_GET_RESPONSE(&xnbp->xnb_rx_ring, prod);
		rxresp->offset = 0;
		rxresp->flags = 0;

		cksum_flags = xnbp->xnb_flavour->xf_cksum_to_peer(xnbp, mp);
		if (cksum_flags != 0)
			xnbp->xnb_stat_rx_cksum_deferred++;
		rxresp->flags |= cksum_flags;

		rxresp->id = RING_GET_REQUEST(&xnbp->xnb_rx_ring, prod)->id;
		rxresp->status = len;

		loop++;
		prod++;
		gop++;
		prev = mp;
		mp = mp->b_next;
	}

	/*
	 * Did we actually do anything?
	 */
	if (loop == xnbp->xnb_rx_ring.req_cons) {
		mutex_exit(&xnbp->xnb_rx_lock);
		return (mp);
	}

	end = loop;

	/*
	 * Unlink the end of the 'done' list from the remainder.
	 */
	ASSERT(prev != NULL);
	prev->b_next = NULL;

	if (HYPERVISOR_grant_table_op(GNTTABOP_transfer, xnbp->xnb_rx_top,
	    loop - xnbp->xnb_rx_ring.req_cons) != 0) {
		cmn_err(CE_WARN, "xnb_to_peer: transfer operation failed");
	}

	loop = xnbp->xnb_rx_ring.req_cons;
	prod = xnbp->xnb_rx_ring.rsp_prod_pvt;
	gop = xnbp->xnb_rx_top;

	while (loop < end) {
		int16_t status = NETIF_RSP_OKAY;

		if (gop->status != 0) {
			status = NETIF_RSP_ERROR;

			/*
			 * If the status is anything other than
			 * GNTST_bad_page then we don't own the page
			 * any more, so don't try to give it back.
			 */
			if (gop->status != GNTST_bad_page)
				gop->mfn = 0;
		} else {
			/* The page is no longer ours. */
			gop->mfn = 0;
		}

		if (gop->mfn != 0)
			/*
			 * Give back the page, as we won't be using
			 * it.
			 */
			xnb_free_page(xnbp, gop->mfn);
		else
			/*
			 * We gave away a page, update our accounting
			 * now.
			 */
			balloon_drv_subtracted(1);

		/* 5.2 */
		if (status != NETIF_RSP_OKAY) {
			RING_GET_RESPONSE(&xnbp->xnb_rx_ring, prod)->status =
			    status;
		} else {
			xnbp->xnb_stat_ipackets++;
			xnbp->xnb_stat_rbytes += len;
		}

		loop++;
		prod++;
		gop++;
	}

	xnbp->xnb_rx_ring.req_cons = loop;
	xnbp->xnb_rx_ring.rsp_prod_pvt = prod;

	/* 6 */
	/* LINTED: constant in conditional context */
	RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(&xnbp->xnb_rx_ring, notify);
	if (notify) {
		ec_notify_via_evtchn(xnbp->xnb_evtchn);
		xnbp->xnb_stat_rx_notify_sent++;
	} else {
		xnbp->xnb_stat_rx_notify_deferred++;
	}

	if (mp != NULL)
		xnbp->xnb_stat_rx_defer++;

	mutex_exit(&xnbp->xnb_rx_lock);

	/* Free mblk_t's that we consumed. */
	freemsgchain(free);

	return (mp);
}

/* Helper functions for xnb_copy_to_peer(). */

/*
 * Grow the array of copy operation descriptors.
 */
static boolean_t
grow_cpop_area(xnb_t *xnbp)
{
	size_t count;
	gnttab_copy_t *new;

	ASSERT(MUTEX_HELD(&xnbp->xnb_rx_lock));

	count = xnbp->xnb_rx_cpop_count + CPOP_DEFCNT;

	if ((new = kmem_alloc(sizeof (new[0]) * count, KM_NOSLEEP)) == NULL) {
		xnbp->xnb_stat_other_allocation_failure++;
		return (B_FALSE);
	}

	bcopy(xnbp->xnb_rx_cpop, new,
	    sizeof (xnbp->xnb_rx_cpop[0]) * xnbp->xnb_rx_cpop_count);

	kmem_free(xnbp->xnb_rx_cpop,
	    sizeof (xnbp->xnb_rx_cpop[0]) * xnbp->xnb_rx_cpop_count);

	xnbp->xnb_rx_cpop = new;
	xnbp->xnb_rx_cpop_count = count;

	xnbp->xnb_stat_rx_cpoparea_grown++;

	return (B_TRUE);
}

/*
 * Check whether an address is on a page that's foreign to this domain.
 */
static boolean_t
is_foreign(void *addr)
{
	pfn_t pfn = hat_getpfnum(kas.a_hat, addr);

	return ((pfn & PFN_IS_FOREIGN_MFN) == PFN_IS_FOREIGN_MFN);
}

/*
 * Insert a newly allocated mblk into a chain, replacing the old one.
 */
static mblk_t *
replace_msg(mblk_t *mp, size_t len, mblk_t *mp_prev, mblk_t *ml_prev)
{
	uint32_t	start, stuff, end, value, flags;
	mblk_t		*new_mp;

	new_mp = copyb(mp);
	if (new_mp == NULL) {
		cmn_err(CE_PANIC, "replace_msg: cannot alloc new message"
		    "for %p, len %lu", (void *) mp, len);
	}

	mac_hcksum_get(mp, &start, &stuff, &end, &value, &flags);
	mac_hcksum_set(new_mp, start, stuff, end, value, flags);

	new_mp->b_next = mp->b_next;
	new_mp->b_prev = mp->b_prev;
	new_mp->b_cont = mp->b_cont;

	/* Make sure we only overwrite pointers to the mblk being replaced. */
	if (mp_prev != NULL && mp_prev->b_next == mp)
		mp_prev->b_next = new_mp;

	if (ml_prev != NULL && ml_prev->b_cont == mp)
		ml_prev->b_cont = new_mp;

	mp->b_next = mp->b_prev = mp->b_cont = NULL;
	freemsg(mp);

	return (new_mp);
}

/*
 * Set all the fields in a gnttab_copy_t.
 */
static void
setup_gop(xnb_t *xnbp, gnttab_copy_t *gp, uchar_t *rptr,
    size_t s_off, size_t d_off, size_t len, grant_ref_t d_ref)
{
	ASSERT(xnbp != NULL && gp != NULL);

	gp->source.offset = s_off;
	gp->source.u.gmfn = pfn_to_mfn(hat_getpfnum(kas.a_hat, (caddr_t)rptr));
	gp->source.domid = DOMID_SELF;

	gp->len = (uint16_t)len;
	gp->flags = GNTCOPY_dest_gref;
	gp->status = 0;

	gp->dest.u.ref = d_ref;
	gp->dest.offset = d_off;
	gp->dest.domid = xnbp->xnb_peer;
}

/*
 * Pass packets to the peer using hypervisor copy operations.
 */
mblk_t *
xnb_copy_to_peer(xnb_t *xnbp, mblk_t *mp)
{
	mblk_t		*free = mp, *mp_prev = NULL, *saved_mp = mp;
	mblk_t		*ml, *ml_prev;
	boolean_t	notify;
	RING_IDX	loop, prod;
	int		i;

	/*
	 * If the peer does not pre-post buffers for received packets,
	 * use page flipping to pass packets to it.
	 */
	if (!xnbp->xnb_rx_hv_copy)
		return (xnb_to_peer(xnbp, mp));

	/*
	 * For each packet the sequence of operations is:
	 *
	 *  1. get a request slot from the ring.
	 *  2. set up data for hypercall (see NOTE below)
	 *  3. have the hypervisore copy the data
	 *  4. update the request slot.
	 *  5. kick the peer.
	 *
	 * NOTE ad 2.
	 *  In order to reduce the number of hypercalls, we prepare
	 *  several mblks (mp->b_cont != NULL) for the peer and
	 *  perform a single hypercall to transfer them.  We also have
	 *  to set up a seperate copy operation for every page.
	 *
	 * If we have more than one packet (mp->b_next != NULL), we do
	 * this whole dance repeatedly.
	 */

	mutex_enter(&xnbp->xnb_rx_lock);

	if (!(xnbp->xnb_connected && xnbp->xnb_hotplugged)) {
		mutex_exit(&xnbp->xnb_rx_lock);
		DTRACE_PROBE(copy_rx_too_early);
		xnbp->xnb_stat_rx_too_early++;
		return (mp);
	}

	loop = xnbp->xnb_rx_ring.req_cons;
	prod = xnbp->xnb_rx_ring.rsp_prod_pvt;

	while ((mp != NULL) &&
	    XNB_RING_HAS_UNCONSUMED_REQUESTS(&xnbp->xnb_rx_ring)) {
		netif_rx_request_t	*rxreq;
		size_t			d_offset, len;
		int			item_count;
		gnttab_copy_t		*gop_cp;
		netif_rx_response_t	*rxresp;
		uint16_t		cksum_flags;
		int16_t			status = NETIF_RSP_OKAY;

		/* 1 */
		rxreq = RING_GET_REQUEST(&xnbp->xnb_rx_ring, loop);

#ifdef XNB_DEBUG
		if (!(rxreq->id < NET_RX_RING_SIZE))
			cmn_err(CE_PANIC, "xnb_copy_to_peer: "
			    "id %d out of range in request 0x%p",
			    rxreq->id, (void *)rxreq);
#endif /* XNB_DEBUG */

		/* 2 */
		d_offset = 0;
		len = 0;
		item_count = 0;

		gop_cp = xnbp->xnb_rx_cpop;

		/*
		 * We walk the b_cont pointers and set up a
		 * gnttab_copy_t for each sub-page chunk in each data
		 * block.
		 */
		/* 2a */
		for (ml = mp, ml_prev = NULL; ml != NULL; ml = ml->b_cont) {
			size_t	chunk = ml->b_wptr - ml->b_rptr;
			uchar_t	*r_tmp,	*rpt_align;
			size_t	r_offset;

			/*
			 * The hypervisor will not allow us to
			 * reference a foreign page (e.g. one
			 * belonging to another domain) by mfn in the
			 * copy operation. If the data in this mblk is
			 * on such a page we must copy the data into a
			 * local page before initiating the hypervisor
			 * copy operation.
			 */
			if (is_foreign(ml->b_rptr) || is_foreign(ml->b_wptr)) {
				mblk_t *ml_new = replace_msg(ml, chunk,
				    mp_prev, ml_prev);

				/* We can still use old ml, but not *ml! */
				if (free == ml)
					free = ml_new;
				if (mp == ml)
					mp = ml_new;
				ml = ml_new;

				xnbp->xnb_stat_rx_foreign_page++;
			}

			rpt_align = (uchar_t *)ALIGN2PAGE(ml->b_rptr);
			r_offset = (uint16_t)(ml->b_rptr - rpt_align);
			r_tmp = ml->b_rptr;

			if (d_offset + chunk > PAGESIZE)
				cmn_err(CE_PANIC, "xnb_copy_to_peer: mp %p "
				    "(svd: %p), ml %p,rpt_alg. %p, d_offset "
				    "(%lu) + chunk (%lu) > PAGESIZE %d!",
				    (void *)mp, (void *)saved_mp, (void *)ml,
				    (void *)rpt_align,
				    d_offset, chunk, (int)PAGESIZE);

			while (chunk > 0) {
				size_t part_len;

				if (item_count == xnbp->xnb_rx_cpop_count) {
					if (!grow_cpop_area(xnbp))
						goto failure;
					gop_cp = &xnbp->xnb_rx_cpop[item_count];
				}
				/*
				 * If our mblk crosses a page boundary, we need
				 * to do a seperate copy for each page.
				 */
				if (r_offset + chunk > PAGESIZE) {
					part_len = PAGESIZE - r_offset;

					DTRACE_PROBE3(mblk_page_crossed,
					    (mblk_t *), ml, int, chunk, int,
					    (int)r_offset);

					xnbp->xnb_stat_rx_pagebndry_crossed++;
				} else {
					part_len = chunk;
				}

				setup_gop(xnbp, gop_cp, r_tmp, r_offset,
				    d_offset, part_len, rxreq->gref);

				chunk -= part_len;

				len += part_len;
				d_offset += part_len;
				r_tmp += part_len;
				/*
				 * The 2nd, 3rd ... last copies will always
				 * start at r_tmp, therefore r_offset is 0.
				 */
				r_offset = 0;
				gop_cp++;
				item_count++;
			}
			ml_prev = ml;

			DTRACE_PROBE4(mblk_loop_end, (mblk_t *), ml, int,
			    chunk, int, len, int, item_count);
		}
		/* 3 */
		if (HYPERVISOR_grant_table_op(GNTTABOP_copy, xnbp->xnb_rx_cpop,
		    item_count) != 0) {
			cmn_err(CE_WARN, "xnb_copy_to_peer: copy op. failed");
			DTRACE_PROBE(HV_granttableopfailed);
		}

		/* 4 */
		rxresp = RING_GET_RESPONSE(&xnbp->xnb_rx_ring, prod);
		rxresp->offset = 0;

		rxresp->flags = 0;

		DTRACE_PROBE4(got_RX_rsp, int, (int)rxresp->id, int,
		    (int)rxresp->offset, int, (int)rxresp->flags, int,
		    (int)rxresp->status);

		cksum_flags = xnbp->xnb_flavour->xf_cksum_to_peer(xnbp, mp);
		if (cksum_flags != 0)
			xnbp->xnb_stat_rx_cksum_deferred++;
		rxresp->flags |= cksum_flags;

		rxresp->id = RING_GET_REQUEST(&xnbp->xnb_rx_ring, prod)->id;
		rxresp->status = len;

		DTRACE_PROBE4(RX_rsp_set, int, (int)rxresp->id, int,
		    (int)rxresp->offset, int, (int)rxresp->flags, int,
		    (int)rxresp->status);

		for (i = 0; i < item_count; i++) {
			if (xnbp->xnb_rx_cpop[i].status != 0) {
				DTRACE_PROBE2(cpop_status_nonnull, int,
				    (int)xnbp->xnb_rx_cpop[i].status,
				    int, i);
				status = NETIF_RSP_ERROR;
			}
		}

		/* 5.2 */
		if (status != NETIF_RSP_OKAY) {
			RING_GET_RESPONSE(&xnbp->xnb_rx_ring, prod)->status =
			    status;
			xnbp->xnb_stat_rx_rsp_notok++;
		} else {
			xnbp->xnb_stat_ipackets++;
			xnbp->xnb_stat_rbytes += len;
		}

		loop++;
		prod++;
		mp_prev = mp;
		mp = mp->b_next;
	}
failure:
	/*
	 * Did we actually do anything?
	 */
	if (loop == xnbp->xnb_rx_ring.req_cons) {
		mutex_exit(&xnbp->xnb_rx_lock);
		return (mp);
	}

	/*
	 * Unlink the end of the 'done' list from the remainder.
	 */
	ASSERT(mp_prev != NULL);
	mp_prev->b_next = NULL;

	xnbp->xnb_rx_ring.req_cons = loop;
	xnbp->xnb_rx_ring.rsp_prod_pvt = prod;

	/* 6 */
	/* LINTED: constant in conditional context */
	RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(&xnbp->xnb_rx_ring, notify);
	if (notify) {
		ec_notify_via_evtchn(xnbp->xnb_evtchn);
		xnbp->xnb_stat_rx_notify_sent++;
	} else {
		xnbp->xnb_stat_rx_notify_deferred++;
	}

	if (mp != NULL)
		xnbp->xnb_stat_rx_defer++;

	mutex_exit(&xnbp->xnb_rx_lock);

	/* Free mblk_t structs we have consumed. */
	freemsgchain(free);

	return (mp);
}


static void
xnb_tx_notify_peer(xnb_t *xnbp, boolean_t force)
{
	boolean_t notify;

	ASSERT(MUTEX_HELD(&xnbp->xnb_tx_lock));

	/* LINTED: constant in conditional context */
	RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(&xnbp->xnb_tx_ring, notify);
	if (notify || force) {
		ec_notify_via_evtchn(xnbp->xnb_evtchn);
		xnbp->xnb_stat_tx_notify_sent++;
	} else {
		xnbp->xnb_stat_tx_notify_deferred++;
	}
}

static void
xnb_tx_mark_complete(xnb_t *xnbp, RING_IDX id, int16_t status)
{
	RING_IDX i;
	netif_tx_response_t *txresp;

	ASSERT(MUTEX_HELD(&xnbp->xnb_tx_lock));

	i = xnbp->xnb_tx_ring.rsp_prod_pvt;

	txresp = RING_GET_RESPONSE(&xnbp->xnb_tx_ring, i);
	txresp->id = id;
	txresp->status = status;

	xnbp->xnb_tx_ring.rsp_prod_pvt = i + 1;

	/*
	 * Note that we don't push the change to the peer here - that
	 * is the callers responsibility.
	 */
}

static void
xnb_txbuf_recycle(xnb_txbuf_t *txp)
{
	xnb_t *xnbp = txp->xt_xnbp;

	kmem_cache_free(xnbp->xnb_tx_buf_cache, txp);

	xnbp->xnb_tx_buf_outstanding--;
}

static int
xnb_txbuf_constructor(void *buf, void *arg, int kmflag)
{
	_NOTE(ARGUNUSED(kmflag));
	xnb_txbuf_t *txp = buf;
	xnb_t *xnbp = arg;
	size_t len;
	ddi_dma_cookie_t dma_cookie;
	uint_t ncookies;

	txp->xt_free_rtn.free_func = xnb_txbuf_recycle;
	txp->xt_free_rtn.free_arg = (caddr_t)txp;
	txp->xt_xnbp = xnbp;
	txp->xt_next = NULL;

	if (ddi_dma_alloc_handle(xnbp->xnb_devinfo, &buf_dma_attr,
	    0, 0, &txp->xt_dma_handle) != DDI_SUCCESS)
		goto failure;

	if (ddi_dma_mem_alloc(txp->xt_dma_handle, PAGESIZE, &data_accattr,
	    DDI_DMA_STREAMING, 0, 0, &txp->xt_buf, &len,
	    &txp->xt_acc_handle) != DDI_SUCCESS)
		goto failure_1;

	if (ddi_dma_addr_bind_handle(txp->xt_dma_handle, NULL, txp->xt_buf,
	    len, DDI_DMA_RDWR | DDI_DMA_STREAMING, DDI_DMA_DONTWAIT, 0,
	    &dma_cookie, &ncookies)
	    != DDI_DMA_MAPPED)
		goto failure_2;
	ASSERT(ncookies == 1);

	txp->xt_mfn = xnb_btop(dma_cookie.dmac_laddress);
	txp->xt_buflen = dma_cookie.dmac_size;

	DTRACE_PROBE(txbuf_allocated);

	atomic_inc_32(&xnbp->xnb_tx_buf_count);
	xnbp->xnb_tx_buf_outstanding++;

	return (0);

failure_2:
	ddi_dma_mem_free(&txp->xt_acc_handle);

failure_1:
	ddi_dma_free_handle(&txp->xt_dma_handle);

failure:

	return (-1);
}

static void
xnb_txbuf_destructor(void *buf, void *arg)
{
	xnb_txbuf_t *txp = buf;
	xnb_t *xnbp = arg;

	(void) ddi_dma_unbind_handle(txp->xt_dma_handle);
	ddi_dma_mem_free(&txp->xt_acc_handle);
	ddi_dma_free_handle(&txp->xt_dma_handle);

	atomic_dec_32(&xnbp->xnb_tx_buf_count);
}

/*
 * Take packets from the peer and deliver them onward.
 */
static mblk_t *
xnb_from_peer(xnb_t *xnbp)
{
	RING_IDX start, end, loop;
	gnttab_copy_t *cop;
	xnb_txbuf_t **txpp;
	netif_tx_request_t *txreq;
	boolean_t work_to_do, need_notify = B_FALSE;
	mblk_t *head, *tail;
	int n_data_req, i;

	ASSERT(MUTEX_HELD(&xnbp->xnb_tx_lock));

	head = tail = NULL;
around:

	/* LINTED: constant in conditional context */
	RING_FINAL_CHECK_FOR_REQUESTS(&xnbp->xnb_tx_ring, work_to_do);
	if (!work_to_do) {
finished:
		xnb_tx_notify_peer(xnbp, need_notify);

		return (head);
	}

	start = xnbp->xnb_tx_ring.req_cons;
	end = xnbp->xnb_tx_ring.sring->req_prod;

	if ((end - start) > NET_TX_RING_SIZE) {
		/*
		 * This usually indicates that the frontend driver is
		 * misbehaving, as it's not possible to have more than
		 * NET_TX_RING_SIZE ring elements in play at any one
		 * time.
		 *
		 * We reset the ring pointers to the state declared by
		 * the frontend and try to carry on.
		 */
		cmn_err(CE_WARN, "xnb_from_peer: domain %d tried to give us %u "
		    "items in the ring, resetting and trying to recover.",
		    xnbp->xnb_peer, (end - start));

		/* LINTED: constant in conditional context */
		BACK_RING_ATTACH(&xnbp->xnb_tx_ring,
		    (netif_tx_sring_t *)xnbp->xnb_tx_ring_addr, PAGESIZE);

		goto around;
	}

	loop = start;
	cop = xnbp->xnb_tx_cop;
	txpp = xnbp->xnb_tx_bufp;
	n_data_req = 0;

	while (loop < end) {
		static const uint16_t acceptable_flags =
		    NETTXF_csum_blank |
		    NETTXF_data_validated |
		    NETTXF_extra_info;
		uint16_t unexpected_flags;

		txreq = RING_GET_REQUEST(&xnbp->xnb_tx_ring, loop);

		unexpected_flags = txreq->flags & ~acceptable_flags;
		if (unexpected_flags != 0) {
			/*
			 * The peer used flag bits that we do not
			 * recognize.
			 */
			cmn_err(CE_WARN, "xnb_from_peer: "
			    "unexpected flag bits (0x%x) from peer "
			    "in transmit request",
			    unexpected_flags);
			xnbp->xnb_stat_tx_unexpected_flags++;

			/* Mark this entry as failed. */
			xnb_tx_mark_complete(xnbp, txreq->id, NETIF_RSP_ERROR);
			need_notify = B_TRUE;

		} else if (txreq->flags & NETTXF_extra_info) {
			struct netif_extra_info *erp;
			boolean_t status;

			loop++; /* Consume another slot in the ring. */
			ASSERT(loop <= end);

			erp = (struct netif_extra_info *)
			    RING_GET_REQUEST(&xnbp->xnb_tx_ring, loop);

			switch (erp->type) {
			case XEN_NETIF_EXTRA_TYPE_MCAST_ADD:
				ASSERT(xnbp->xnb_multicast_control);
				status = xnbp->xnb_flavour->xf_mcast_add(xnbp,
				    &erp->u.mcast.addr);
				break;
			case XEN_NETIF_EXTRA_TYPE_MCAST_DEL:
				ASSERT(xnbp->xnb_multicast_control);
				status = xnbp->xnb_flavour->xf_mcast_del(xnbp,
				    &erp->u.mcast.addr);
				break;
			default:
				status = B_FALSE;
				cmn_err(CE_WARN, "xnb_from_peer: "
				    "unknown extra type %d", erp->type);
				break;
			}

			xnb_tx_mark_complete(xnbp, txreq->id,
			    status ? NETIF_RSP_OKAY : NETIF_RSP_ERROR);
			need_notify = B_TRUE;

		} else if ((txreq->offset > PAGESIZE) ||
		    (txreq->offset + txreq->size > PAGESIZE)) {
			/*
			 * Peer attempted to refer to data beyond the
			 * end of the granted page.
			 */
			cmn_err(CE_WARN, "xnb_from_peer: "
			    "attempt to refer beyond the end of granted "
			    "page in txreq (offset %d, size %d).",
			    txreq->offset, txreq->size);
			xnbp->xnb_stat_tx_overflow_page++;

			/* Mark this entry as failed. */
			xnb_tx_mark_complete(xnbp, txreq->id, NETIF_RSP_ERROR);
			need_notify = B_TRUE;

		} else {
			xnb_txbuf_t *txp;

			txp = kmem_cache_alloc(xnbp->xnb_tx_buf_cache,
			    KM_NOSLEEP);
			if (txp == NULL)
				break;

			txp->xt_mblk = desballoc((unsigned char *)txp->xt_buf,
			    txp->xt_buflen, 0, &txp->xt_free_rtn);
			if (txp->xt_mblk == NULL) {
				kmem_cache_free(xnbp->xnb_tx_buf_cache, txp);
				break;
			}

			txp->xt_idx = loop;
			txp->xt_id = txreq->id;

			cop->source.u.ref = txreq->gref;
			cop->source.domid = xnbp->xnb_peer;
			cop->source.offset = txreq->offset;

			cop->dest.u.gmfn = txp->xt_mfn;
			cop->dest.domid = DOMID_SELF;
			cop->dest.offset = 0;

			cop->len = txreq->size;
			cop->flags = GNTCOPY_source_gref;
			cop->status = 0;

			*txpp = txp;

			txpp++;
			cop++;
			n_data_req++;

			ASSERT(n_data_req <= NET_TX_RING_SIZE);
		}

		loop++;
	}

	xnbp->xnb_tx_ring.req_cons = loop;

	if (n_data_req == 0)
		goto around;

	if (HYPERVISOR_grant_table_op(GNTTABOP_copy,
	    xnbp->xnb_tx_cop, n_data_req) != 0) {

		cmn_err(CE_WARN, "xnb_from_peer: copy operation failed");

		txpp = xnbp->xnb_tx_bufp;
		i = n_data_req;
		while (i > 0) {
			kmem_cache_free(xnbp->xnb_tx_buf_cache, *txpp);
			txpp++;
			i--;
		}

		goto finished;
	}

	txpp = xnbp->xnb_tx_bufp;
	cop = xnbp->xnb_tx_cop;
	i = n_data_req;

	while (i > 0) {
		xnb_txbuf_t *txp = *txpp;

		txreq = RING_GET_REQUEST(&xnbp->xnb_tx_ring, txp->xt_idx);

		if (cop->status != 0) {
#ifdef XNB_DEBUG
			cmn_err(CE_WARN, "xnb_from_peer: "
			    "txpp 0x%p failed (%d)",
			    (void *)*txpp, cop->status);
#endif /* XNB_DEBUG */
			xnb_tx_mark_complete(xnbp, txp->xt_id, NETIF_RSP_ERROR);
			freemsg(txp->xt_mblk);
		} else {
			mblk_t *mp;

			mp = txp->xt_mblk;
			mp->b_rptr = mp->b_wptr = (unsigned char *)txp->xt_buf;
			mp->b_wptr += txreq->size;
			mp->b_next = NULL;

			/*
			 * If there are checksum flags, process them
			 * appropriately.
			 */
			if ((txreq->flags &
			    (NETTXF_csum_blank | NETTXF_data_validated))
			    != 0) {
				mp = xnbp->xnb_flavour->xf_cksum_from_peer(xnbp,
				    mp, txreq->flags);
				xnbp->xnb_stat_tx_cksum_no_need++;

				txp->xt_mblk = mp;
			}

			if (head == NULL) {
				ASSERT(tail == NULL);
				head = mp;
			} else {
				ASSERT(tail != NULL);
				tail->b_next = mp;
			}
			tail = mp;

			xnbp->xnb_stat_opackets++;
			xnbp->xnb_stat_obytes += txreq->size;

			xnb_tx_mark_complete(xnbp, txp->xt_id, NETIF_RSP_OKAY);
		}

		txpp++;
		cop++;
		i--;
	}

	goto around;
	/* NOTREACHED */
}

static uint_t
xnb_intr(caddr_t arg)
{
	xnb_t *xnbp = (xnb_t *)arg;
	mblk_t *mp;

	xnbp->xnb_stat_intr++;

	mutex_enter(&xnbp->xnb_tx_lock);

	ASSERT(xnbp->xnb_connected);

	mp = xnb_from_peer(xnbp);

	mutex_exit(&xnbp->xnb_tx_lock);

	if (!xnbp->xnb_hotplugged) {
		xnbp->xnb_stat_tx_too_early++;
		goto fail;
	}
	if (mp == NULL) {
		xnbp->xnb_stat_spurious_intr++;
		goto fail;
	}

	xnbp->xnb_flavour->xf_from_peer(xnbp, mp);

	return (DDI_INTR_CLAIMED);

fail:
	freemsgchain(mp);
	return (DDI_INTR_CLAIMED);
}

/*
 * Read our configuration from xenstore.
 */
boolean_t
xnb_read_xs_config(xnb_t *xnbp)
{
	char *xsname;
	char mac[ETHERADDRL * 3];

	xsname = xvdi_get_xsname(xnbp->xnb_devinfo);

	if (xenbus_scanf(XBT_NULL, xsname,
	    "mac", "%s", mac) != 0) {
		cmn_err(CE_WARN, "xnb_attach: "
		    "cannot read mac address from %s",
		    xsname);
		return (B_FALSE);
	}

	if (ether_aton(mac, xnbp->xnb_mac_addr) != ETHERADDRL) {
		cmn_err(CE_WARN,
		    "xnb_attach: cannot parse mac address %s",
		    mac);
		return (B_FALSE);
	}

	return (B_TRUE);
}

/*
 * Read the configuration of the peer from xenstore.
 */
boolean_t
xnb_read_oe_config(xnb_t *xnbp)
{
	char *oename;
	int i;

	oename = xvdi_get_oename(xnbp->xnb_devinfo);

	if (xenbus_gather(XBT_NULL, oename,
	    "event-channel", "%u", &xnbp->xnb_fe_evtchn,
	    "tx-ring-ref", "%lu", &xnbp->xnb_tx_ring_ref,
	    "rx-ring-ref", "%lu", &xnbp->xnb_rx_ring_ref,
	    NULL) != 0) {
		cmn_err(CE_WARN, "xnb_read_oe_config: "
		    "cannot read other-end details from %s",
		    oename);
		return (B_FALSE);
	}

	/*
	 * Check whether our peer requests receive side hypervisor
	 * copy.
	 */
	if (xenbus_scanf(XBT_NULL, oename,
	    "request-rx-copy", "%d", &i) != 0)
		i = 0;
	if (i != 0)
		xnbp->xnb_rx_hv_copy = B_TRUE;

	/*
	 * Check whether our peer requests multicast_control.
	 */
	if (xenbus_scanf(XBT_NULL, oename,
	    "request-multicast-control", "%d", &i) != 0)
		i = 0;
	if (i != 0)
		xnbp->xnb_multicast_control = B_TRUE;

	/*
	 * The Linux backend driver here checks to see if the peer has
	 * set 'feature-no-csum-offload'. This is used to indicate
	 * that the guest cannot handle receiving packets without a
	 * valid checksum. We don't check here, because packets passed
	 * to the peer _always_ have a valid checksum.
	 *
	 * There are three cases:
	 *
	 * - the NIC is dedicated: packets from the wire should always
	 *   have a valid checksum. If the hardware validates the
	 *   checksum then the relevant bit will be set in the packet
	 *   attributes and we will inform the peer. It can choose to
	 *   ignore the hardware verification.
	 *
	 * - the NIC is shared (VNIC) and a packet originates from the
	 *   wire: this is the same as the case above - the packets
	 *   will have a valid checksum.
	 *
	 * - the NIC is shared (VNIC) and a packet originates from the
	 *   host: the MAC layer ensures that all such packets have a
	 *   valid checksum by calculating one if the stack did not.
	 */

	return (B_TRUE);
}

void
xnb_start_connect(xnb_t *xnbp)
{
	dev_info_t  *dip = xnbp->xnb_devinfo;

	if (!xnb_connect_rings(dip)) {
		cmn_err(CE_WARN, "xnb_start_connect: "
		    "cannot connect rings");
		goto failed;
	}

	if (!xnbp->xnb_flavour->xf_start_connect(xnbp)) {
		cmn_err(CE_WARN, "xnb_start_connect: "
		    "flavour failed to connect");
		goto failed;
	}

	(void) xvdi_switch_state(dip, XBT_NULL, XenbusStateConnected);
	return;

failed:
	xnbp->xnb_flavour->xf_peer_disconnected(xnbp);
	xnb_disconnect_rings(dip);
	(void) xvdi_switch_state(dip, XBT_NULL,
	    XenbusStateClosed);
	(void) xvdi_post_event(dip, XEN_HP_REMOVE);
}

static boolean_t
xnb_connect_rings(dev_info_t *dip)
{
	xnb_t *xnbp = ddi_get_driver_private(dip);
	struct gnttab_map_grant_ref map_op;

	/*
	 * Cannot attempt to connect the rings if already connected.
	 */
	ASSERT(!xnbp->xnb_connected);

	/*
	 * 1. allocate a vaddr for the tx page, one for the rx page.
	 * 2. call GNTTABOP_map_grant_ref to map the relevant pages
	 *    into the allocated vaddr (one for tx, one for rx).
	 * 3. call EVTCHNOP_bind_interdomain to have the event channel
	 *    bound to this domain.
	 * 4. associate the event channel with an interrupt.
	 * 5. enable the interrupt.
	 */

	/* 1.tx */
	xnbp->xnb_tx_ring_addr = vmem_xalloc(heap_arena, PAGESIZE, PAGESIZE,
	    0, 0, 0, 0, VM_SLEEP);
	ASSERT(xnbp->xnb_tx_ring_addr != NULL);

	/* 2.tx */
	map_op.host_addr = (uint64_t)((long)xnbp->xnb_tx_ring_addr);
	map_op.flags = GNTMAP_host_map;
	map_op.ref = xnbp->xnb_tx_ring_ref;
	map_op.dom = xnbp->xnb_peer;
	hat_prepare_mapping(kas.a_hat, xnbp->xnb_tx_ring_addr, NULL);
	if (xen_map_gref(GNTTABOP_map_grant_ref, &map_op, 1, B_FALSE) != 0 ||
	    map_op.status != 0) {
		cmn_err(CE_WARN, "xnb_connect_rings: cannot map tx-ring page.");
		goto fail;
	}
	xnbp->xnb_tx_ring_handle = map_op.handle;

	/* LINTED: constant in conditional context */
	BACK_RING_INIT(&xnbp->xnb_tx_ring,
	    (netif_tx_sring_t *)xnbp->xnb_tx_ring_addr, PAGESIZE);

	/* 1.rx */
	xnbp->xnb_rx_ring_addr = vmem_xalloc(heap_arena, PAGESIZE, PAGESIZE,
	    0, 0, 0, 0, VM_SLEEP);
	ASSERT(xnbp->xnb_rx_ring_addr != NULL);

	/* 2.rx */
	map_op.host_addr = (uint64_t)((long)xnbp->xnb_rx_ring_addr);
	map_op.flags = GNTMAP_host_map;
	map_op.ref = xnbp->xnb_rx_ring_ref;
	map_op.dom = xnbp->xnb_peer;
	hat_prepare_mapping(kas.a_hat, xnbp->xnb_rx_ring_addr, NULL);
	if (xen_map_gref(GNTTABOP_map_grant_ref, &map_op, 1, B_FALSE) != 0 ||
	    map_op.status != 0) {
		cmn_err(CE_WARN, "xnb_connect_rings: cannot map rx-ring page.");
		goto fail;
	}
	xnbp->xnb_rx_ring_handle = map_op.handle;

	/* LINTED: constant in conditional context */
	BACK_RING_INIT(&xnbp->xnb_rx_ring,
	    (netif_rx_sring_t *)xnbp->xnb_rx_ring_addr, PAGESIZE);

	/* 3 */
	if (xvdi_bind_evtchn(dip, xnbp->xnb_fe_evtchn) != DDI_SUCCESS) {
		cmn_err(CE_WARN, "xnb_connect_rings: "
		    "cannot bind event channel %d", xnbp->xnb_evtchn);
		xnbp->xnb_evtchn = INVALID_EVTCHN;
		goto fail;
	}
	xnbp->xnb_evtchn = xvdi_get_evtchn(dip);

	/*
	 * It would be good to set the state to XenbusStateConnected
	 * here as well, but then what if ddi_add_intr() failed?
	 * Changing the state in the store will be noticed by the peer
	 * and cannot be "taken back".
	 */
	mutex_enter(&xnbp->xnb_tx_lock);
	mutex_enter(&xnbp->xnb_rx_lock);

	xnbp->xnb_connected = B_TRUE;

	mutex_exit(&xnbp->xnb_rx_lock);
	mutex_exit(&xnbp->xnb_tx_lock);

	/* 4, 5 */
	if (ddi_add_intr(dip, 0, NULL, NULL, xnb_intr, (caddr_t)xnbp)
	    != DDI_SUCCESS) {
		cmn_err(CE_WARN, "xnb_connect_rings: cannot add interrupt");
		goto fail;
	}
	xnbp->xnb_irq = B_TRUE;

	return (B_TRUE);

fail:
	mutex_enter(&xnbp->xnb_tx_lock);
	mutex_enter(&xnbp->xnb_rx_lock);

	xnbp->xnb_connected = B_FALSE;

	mutex_exit(&xnbp->xnb_rx_lock);
	mutex_exit(&xnbp->xnb_tx_lock);

	return (B_FALSE);
}

static void
xnb_disconnect_rings(dev_info_t *dip)
{
	xnb_t *xnbp = ddi_get_driver_private(dip);

	if (xnbp->xnb_irq) {
		ddi_remove_intr(dip, 0, NULL);
		xnbp->xnb_irq = B_FALSE;
	}

	if (xnbp->xnb_evtchn != INVALID_EVTCHN) {
		xvdi_free_evtchn(dip);
		xnbp->xnb_evtchn = INVALID_EVTCHN;
	}

	if (xnbp->xnb_rx_ring_handle != INVALID_GRANT_HANDLE) {
		struct gnttab_unmap_grant_ref unmap_op;

		unmap_op.host_addr = (uint64_t)(uintptr_t)
		    xnbp->xnb_rx_ring_addr;
		unmap_op.dev_bus_addr = 0;
		unmap_op.handle = xnbp->xnb_rx_ring_handle;
		if (HYPERVISOR_grant_table_op(GNTTABOP_unmap_grant_ref,
		    &unmap_op, 1) != 0)
			cmn_err(CE_WARN, "xnb_disconnect_rings: "
			    "cannot unmap rx-ring page (%d)",
			    unmap_op.status);

		xnbp->xnb_rx_ring_handle = INVALID_GRANT_HANDLE;
	}

	if (xnbp->xnb_rx_ring_addr != NULL) {
		hat_release_mapping(kas.a_hat, xnbp->xnb_rx_ring_addr);
		vmem_free(heap_arena, xnbp->xnb_rx_ring_addr, PAGESIZE);
		xnbp->xnb_rx_ring_addr = NULL;
	}

	if (xnbp->xnb_tx_ring_handle != INVALID_GRANT_HANDLE) {
		struct gnttab_unmap_grant_ref unmap_op;

		unmap_op.host_addr = (uint64_t)(uintptr_t)
		    xnbp->xnb_tx_ring_addr;
		unmap_op.dev_bus_addr = 0;
		unmap_op.handle = xnbp->xnb_tx_ring_handle;
		if (HYPERVISOR_grant_table_op(GNTTABOP_unmap_grant_ref,
		    &unmap_op, 1) != 0)
			cmn_err(CE_WARN, "xnb_disconnect_rings: "
			    "cannot unmap tx-ring page (%d)",
			    unmap_op.status);

		xnbp->xnb_tx_ring_handle = INVALID_GRANT_HANDLE;
	}

	if (xnbp->xnb_tx_ring_addr != NULL) {
		hat_release_mapping(kas.a_hat, xnbp->xnb_tx_ring_addr);
		vmem_free(heap_arena, xnbp->xnb_tx_ring_addr, PAGESIZE);
		xnbp->xnb_tx_ring_addr = NULL;
	}
}

static void
xnb_oe_state_change(dev_info_t *dip, ddi_eventcookie_t id,
    void *arg, void *impl_data)
{
	_NOTE(ARGUNUSED(id, arg));
	xnb_t *xnbp = ddi_get_driver_private(dip);
	XenbusState new_state = *(XenbusState *)impl_data;

	ASSERT(xnbp != NULL);

	switch (new_state) {
	case XenbusStateConnected:
		/* spurious state change */
		if (xnbp->xnb_connected)
			return;

		if (!xnb_read_oe_config(xnbp) ||
		    !xnbp->xnb_flavour->xf_peer_connected(xnbp)) {
			cmn_err(CE_WARN, "xnb_oe_state_change: "
			    "read otherend config error");
			(void) xvdi_switch_state(dip, XBT_NULL,
			    XenbusStateClosed);
			(void) xvdi_post_event(dip, XEN_HP_REMOVE);

			break;
		}


		mutex_enter(&xnbp->xnb_state_lock);
		xnbp->xnb_fe_status = XNB_STATE_READY;
		if (xnbp->xnb_be_status == XNB_STATE_READY)
			xnb_start_connect(xnbp);
		mutex_exit(&xnbp->xnb_state_lock);

		/*
		 * Now that we've attempted to connect it's reasonable
		 * to allow an attempt to detach.
		 */
		xnbp->xnb_detachable = B_TRUE;

		break;

	case XenbusStateClosing:
		(void) xvdi_switch_state(dip, XBT_NULL, XenbusStateClosing);

		break;

	case XenbusStateClosed:
		xnbp->xnb_flavour->xf_peer_disconnected(xnbp);

		mutex_enter(&xnbp->xnb_tx_lock);
		mutex_enter(&xnbp->xnb_rx_lock);

		xnb_disconnect_rings(dip);
		xnbp->xnb_connected = B_FALSE;

		mutex_exit(&xnbp->xnb_rx_lock);
		mutex_exit(&xnbp->xnb_tx_lock);

		(void) xvdi_switch_state(dip, XBT_NULL, XenbusStateClosed);
		(void) xvdi_post_event(dip, XEN_HP_REMOVE);
		/*
		 * In all likelyhood this is already set (in the above
		 * case), but if the peer never attempted to connect
		 * and the domain is destroyed we get here without
		 * having been through the case above, so we set it to
		 * be sure.
		 */
		xnbp->xnb_detachable = B_TRUE;

		break;

	default:
		break;
	}
}

static void
xnb_hp_state_change(dev_info_t *dip, ddi_eventcookie_t id,
    void *arg, void *impl_data)
{
	_NOTE(ARGUNUSED(id, arg));
	xnb_t *xnbp = ddi_get_driver_private(dip);
	xendev_hotplug_state_t state = *(xendev_hotplug_state_t *)impl_data;

	ASSERT(xnbp != NULL);

	switch (state) {
	case Connected:
		/* spurious hotplug event */
		if (xnbp->xnb_hotplugged)
			break;

		if (!xnb_read_xs_config(xnbp))
			break;

		if (!xnbp->xnb_flavour->xf_hotplug_connected(xnbp))
			break;

		mutex_enter(&xnbp->xnb_tx_lock);
		mutex_enter(&xnbp->xnb_rx_lock);

		xnbp->xnb_hotplugged = B_TRUE;

		mutex_exit(&xnbp->xnb_rx_lock);
		mutex_exit(&xnbp->xnb_tx_lock);

		mutex_enter(&xnbp->xnb_state_lock);
		xnbp->xnb_be_status = XNB_STATE_READY;
		if (xnbp->xnb_fe_status == XNB_STATE_READY)
			xnb_start_connect(xnbp);
		mutex_exit(&xnbp->xnb_state_lock);

		break;

	default:
		break;
	}
}

static struct modldrv modldrv = {
	&mod_miscops, "xnb",
};

static struct modlinkage modlinkage = {
	MODREV_1, &modldrv, NULL
};

int
_init(void)
{
	int i;

	mutex_init(&xnb_alloc_page_lock, NULL, MUTEX_DRIVER, NULL);

	i = mod_install(&modlinkage);
	if (i != DDI_SUCCESS)
		mutex_destroy(&xnb_alloc_page_lock);

	return (i);
}

int
_info(struct modinfo *modinfop)
{
	return (mod_info(&modlinkage, modinfop));
}

int
_fini(void)
{
	int i;

	i = mod_remove(&modlinkage);
	if (i == DDI_SUCCESS)
		mutex_destroy(&xnb_alloc_page_lock);

	return (i);
}
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 *
 * xnb.h - definitions for Xen dom0 network driver
 */

#ifndef _SYS_XNB_H
#define	_SYS_XNB_H

#include <sys/types.h>
#include <sys/kstat.h>
#include <sys/stream.h>
#include <sys/ethernet.h>
#include <sys/hypervisor.h>
#include <sys/sysmacros.h>
#include <xen/public/io/netif.h>

#ifdef __cplusplus
extern "C" {
#endif

#define	NET_TX_RING_SIZE  __CONST_RING_SIZE(netif_tx, PAGESIZE)
#define	NET_RX_RING_SIZE  __CONST_RING_SIZE(netif_rx, PAGESIZE)

#define	XNBMAXPKT	1500		/* MTU size */

/* DEBUG flags */
#define	XNBDDI		0x01
#define	XNBTRACE	0x02
#define	XNBSEND		0x04
#define	XNBRECV		0x08
#define	XNBINTR		0x10
#define	XNBRING		0x20
#define	XNBCKSUM	0x40

#define	XNB_STATE_INIT	0x01
#define	XNB_STATE_READY	0x02

typedef struct xnb xnb_t;

/*
 * The xnb module provides core inter-domain network protocol functionality.
 * It is connected to the rest of Solaris in two ways:
 * - as a GLDv3 driver (with xnbu),
 * - as a GLDv3 consumer (with xnbo).
 *
 * The different modes of operation are termed "flavours" and each
 * instance of an xnb based driver operates in one and only one mode.
 * The common xnb driver exports a set of functions to these drivers
 * (declarations at the foot of this file) and calls back into the
 * drivers via the xnb_flavour_t structure.
 */
typedef struct xnb_flavour {
	void		(*xf_from_peer)(xnb_t *, mblk_t *);
	boolean_t	(*xf_peer_connected)(xnb_t *);
	void		(*xf_peer_disconnected)(xnb_t *);
	boolean_t	(*xf_hotplug_connected)(xnb_t *);
	boolean_t	(*xf_start_connect)(xnb_t *);
	mblk_t		*(*xf_cksum_from_peer)(xnb_t *, mblk_t *, uint16_t);
	uint16_t	(*xf_cksum_to_peer)(xnb_t *, mblk_t *);
	boolean_t	(*xf_mcast_add)(xnb_t *, ether_addr_t *);
	boolean_t	(*xf_mcast_del)(xnb_t *, ether_addr_t *);
} xnb_flavour_t;

typedef struct xnb_txbuf {
	frtn_t			xt_free_rtn;
	xnb_t			*xt_xnbp;
	struct xnb_txbuf	*xt_next;
	RING_IDX		xt_id;
	RING_IDX		xt_idx;
	uint16_t		xt_status;

	ddi_dma_handle_t	xt_dma_handle;
	ddi_acc_handle_t	xt_acc_handle;
	caddr_t			xt_buf;
	size_t			xt_buflen;
	mfn_t			xt_mfn;

	mblk_t			*xt_mblk;

	unsigned int		xt_flags;

#define	XNB_TXBUF_INUSE	0x01

} xnb_txbuf_t;

/* Per network-interface-controller driver private structure */
struct xnb {
	/* most interesting stuff first to assist debugging */
	dev_info_t		*xnb_devinfo;	/* System per-device info. */

	xnb_flavour_t		*xnb_flavour;
	void			*xnb_flavour_data;

	boolean_t		xnb_irq;
	unsigned char		xnb_mac_addr[ETHERADDRL];

	uint64_t		xnb_stat_ipackets;
	uint64_t		xnb_stat_opackets;
	uint64_t		xnb_stat_rbytes;
	uint64_t		xnb_stat_obytes;

	uint64_t		xnb_stat_intr;
	uint64_t		xnb_stat_rx_defer;

	uint64_t		xnb_stat_rx_cksum_deferred;
	uint64_t		xnb_stat_tx_cksum_no_need;

	uint64_t		xnb_stat_rx_rsp_notok;

	uint64_t		xnb_stat_tx_notify_sent;
	uint64_t		xnb_stat_tx_notify_deferred;

	uint64_t		xnb_stat_rx_notify_sent;
	uint64_t		xnb_stat_rx_notify_deferred;

	uint64_t		xnb_stat_tx_too_early;
	uint64_t		xnb_stat_rx_too_early;
	uint64_t		xnb_stat_rx_allocb_failed;
	uint64_t		xnb_stat_tx_allocb_failed;
	uint64_t		xnb_stat_rx_foreign_page;
	uint64_t		xnb_stat_tx_overflow_page;
	uint64_t		xnb_stat_tx_unexpected_flags;
	uint64_t		xnb_stat_mac_full;
	uint64_t		xnb_stat_spurious_intr;
	uint64_t		xnb_stat_allocation_success;
	uint64_t		xnb_stat_allocation_failure;
	uint64_t		xnb_stat_small_allocation_success;
	uint64_t		xnb_stat_small_allocation_failure;
	uint64_t		xnb_stat_other_allocation_failure;

	uint64_t		xnb_stat_rx_pagebndry_crossed;
	uint64_t		xnb_stat_rx_cpoparea_grown;

	uint64_t		xnb_stat_csum_hardware;
	uint64_t		xnb_stat_csum_software;

	kstat_t			*xnb_kstat_aux;

	ddi_iblock_cookie_t	xnb_icookie;

	kmutex_t		xnb_rx_lock;
	kmutex_t		xnb_tx_lock;
	kmutex_t		xnb_state_lock;

	int			xnb_be_status;
	int			xnb_fe_status;

	kmem_cache_t		*xnb_tx_buf_cache;
	uint32_t		xnb_tx_buf_count;
	int			xnb_tx_buf_outstanding;

	netif_rx_back_ring_t	xnb_rx_ring;	/* rx interface struct ptr */
	void			*xnb_rx_ring_addr;
	grant_ref_t		xnb_rx_ring_ref;
	grant_handle_t		xnb_rx_ring_handle;

	netif_tx_back_ring_t	xnb_tx_ring;	/* tx interface struct ptr */
	void			*xnb_tx_ring_addr;
	grant_ref_t		xnb_tx_ring_ref;
	grant_handle_t		xnb_tx_ring_handle;

	boolean_t		xnb_connected;
	boolean_t		xnb_hotplugged;
	boolean_t		xnb_detachable;
	int			xnb_evtchn;	/* channel to front end */
	evtchn_port_t		xnb_fe_evtchn;
	domid_t			xnb_peer;

	xnb_txbuf_t		*xnb_tx_bufp[NET_TX_RING_SIZE];
	gnttab_copy_t		xnb_tx_cop[NET_TX_RING_SIZE];

	caddr_t			xnb_rx_va;
	gnttab_transfer_t	xnb_rx_top[NET_RX_RING_SIZE];

	boolean_t		xnb_rx_hv_copy;
	boolean_t		xnb_multicast_control;
	boolean_t		xnb_no_csum_offload;

	gnttab_copy_t		*xnb_rx_cpop;
#define	CPOP_DEFCNT 	8
	size_t			xnb_rx_cpop_count; 	/* in elements */
};

extern int xnb_attach(dev_info_t *, xnb_flavour_t *, void *);
extern void xnb_detach(dev_info_t *);
extern mblk_t *xnb_copy_to_peer(xnb_t *, mblk_t *);
extern mblk_t *xnb_process_cksum_flags(xnb_t *, mblk_t *, uint32_t);

#ifdef __cplusplus
}
#endif

#endif	/* _SYS_XNB_H */
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */


/*
 * Xen network backend - ioemu version.
 *
 * HVM guest domains use an emulated network device (typically the
 * rtl8139) to access the physical network via IO emulation running in
 * a backend domain (generally domain 0).
 *
 * The IO emulation code sends and receives packets using DLPI, usually
 * through a virtual NIC (vnic).
 *
 * The creation of the relevant vnic to correspond to the network interface
 * in the guest domain requires the use of 'hotplug' scripts in the backend
 * domain. This driver ensures that the hotplug scripts are run when
 * such guest domains are created.
 *
 * It is used as a result of the 'compatible' property associated with
 * IO emulated devices. See /etc/driver_aliases and common/xen/os/xvdi.c.
 */

#ifdef DEBUG
#define	XNBE_DEBUG 1
#endif /* DEBUG */

#include <sys/types.h>
#include <sys/conf.h>
#include <sys/sunddi.h>
#include <sys/modctl.h>
#include <xen/sys/xendev.h>
#ifdef XNBE_DEBUG
#include <sys/cmn_err.h>
#endif /* XNBE_DEBUG */

#ifdef XNBE_DEBUG
int xnbe_debug = 0;
#endif /* XNBE_DEBUG */

static int
xnbe_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
{
#ifdef XNBE_DEBUG
	if (xnbe_debug > 0)
		cmn_err(CE_NOTE, "xnbe_attach: dip 0x%p, cmd %d",
		    (void *)dip, cmd);
#endif /* XNBE_DEBUG */

	switch (cmd) {
	case DDI_ATTACH:
		break;
	case DDI_RESUME:
		return (DDI_SUCCESS);
	default:
		return (DDI_FAILURE);
	}

	(void) xvdi_post_event(dip, XEN_HP_ADD);

	return (DDI_SUCCESS);
}

static int
xnbe_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
{
#ifdef XNBE_DEBUG
	if (xnbe_debug > 0)
		cmn_err(CE_NOTE, "detach: dip 0x%p, cmd %d",
		    (void *)dip, cmd);
#endif /* XNBE_DEBUG */

	switch (cmd) {
	case DDI_DETACH:
		return (DDI_SUCCESS);
	case DDI_SUSPEND:
		return (DDI_SUCCESS);
	default:
		return (DDI_FAILURE);
	}
}

static struct cb_ops cb_ops = {
	nulldev,		/* open */
	nulldev,		/* close */
	nodev,			/* strategy */
	nodev,			/* print */
	nodev,			/* dump */
	nodev,			/* read */
	nodev,			/* write */
	nodev,			/* ioctl */
	nodev,			/* devmap */
	nodev,			/* mmap */
	nodev,			/* segmap */
	nochpoll,		/* poll */
	ddi_prop_op,		/* cb_prop_op */
	0,			/* streamtab  */
	D_NEW | D_MP | D_64BIT	/* Driver compatibility flag */
};

static struct dev_ops ops = {
	DEVO_REV,		/* devo_rev */
	0,			/* devo_refcnt  */
	nulldev,		/* devo_getinfo */
	nulldev,		/* devo_identify */
	nulldev,		/* devo_probe */
	xnbe_attach,		/* devo_attach */
	xnbe_detach,		/* devo_detach */
	nodev,			/* devo_reset */
	&cb_ops,		/* devo_cb_ops */
	(struct bus_ops *)0,	/* devo_bus_ops */
	NULL,			/* devo_power */
	ddi_quiesce_not_needed,		/* devo_quiesce */
};

static struct modldrv modldrv = {
	&mod_driverops, "xnbe driver", &ops,
};

static struct modlinkage modlinkage = {
	MODREV_1, &modldrv, NULL
};

int
_init(void)
{
	return (mod_install(&modlinkage));
}

int
_info(struct modinfo *modinfop)
{
	return (mod_info(&modlinkage, modinfop));
}

int
_fini(void)
{
	return (mod_remove(&modlinkage));
}
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

/*
 * Xen network backend - mac client edition.
 *
 * A driver that sits above an existing GLDv3/Nemo MAC driver and
 * relays packets to/from that driver from/to a guest domain.
 */

#ifdef DEBUG
#define	XNBO_DEBUG 1
#endif /* DEBUG */

#include "xnb.h"

#include <sys/sunddi.h>
#include <sys/ddi.h>
#include <sys/modctl.h>
#include <sys/strsubr.h>
#include <sys/mac_client.h>
#include <sys/mac_provider.h>
#include <sys/mac_client_priv.h>
#include <sys/mac.h>
#include <net/if.h>
#include <sys/dlpi.h>
#include <sys/pattr.h>
#include <xen/sys/xenbus_impl.h>
#include <xen/sys/xendev.h>
#include <sys/sdt.h>
#include <sys/note.h>

#ifdef XNBO_DEBUG
boolean_t xnbo_cksum_offload_to_peer = B_TRUE;
boolean_t xnbo_cksum_offload_from_peer = B_TRUE;
#endif /* XNBO_DEBUG */

/* Track multicast addresses. */
typedef struct xmca {
	struct xmca *next;
	ether_addr_t addr;
} xmca_t;

/* State about this device instance. */
typedef struct xnbo {
	mac_handle_t		o_mh;
	mac_client_handle_t	o_mch;
	mac_unicast_handle_t	o_mah;
	mac_promisc_handle_t	o_mphp;
	boolean_t		o_running;
	boolean_t		o_promiscuous;
	uint32_t		o_hcksum_capab;
	xmca_t			*o_mca;
	char			o_link_name[LIFNAMSIZ];
	boolean_t		o_need_rx_filter;
	boolean_t		o_need_setphysaddr;
	boolean_t		o_multicast_control;
} xnbo_t;

static void xnbo_close_mac(xnb_t *);
static void i_xnbo_close_mac(xnb_t *, boolean_t);

/*
 * Packets from the peer come here.  We pass them to the mac device.
 */
static void
xnbo_to_mac(xnb_t *xnbp, mblk_t *mp)
{
	xnbo_t *xnbop = xnbp->xnb_flavour_data;

	ASSERT(mp != NULL);

	if (!xnbop->o_running) {
		xnbp->xnb_stat_tx_too_early++;
		goto fail;
	}

	if (mac_tx(xnbop->o_mch, mp, 0,
	    MAC_DROP_ON_NO_DESC, NULL) != (mac_tx_cookie_t)NULL) {
		xnbp->xnb_stat_mac_full++;
	}

	return;

fail:
	freemsgchain(mp);
}

/*
 * Process the checksum flags `flags' provided by the peer for the
 * packet `mp'.
 */
static mblk_t *
xnbo_cksum_from_peer(xnb_t *xnbp, mblk_t *mp, uint16_t flags)
{
	xnbo_t *xnbop = xnbp->xnb_flavour_data;

	ASSERT(mp->b_next == NULL);

	if ((flags & NETTXF_csum_blank) != 0) {
		uint32_t capab = xnbop->o_hcksum_capab;

#ifdef XNBO_DEBUG
		if (!xnbo_cksum_offload_from_peer)
			capab = 0;
#endif /* XNBO_DEBUG */

		/*
		 * The checksum in the packet is blank.  Determine
		 * whether we can do hardware offload and, if so,
		 * update the flags on the mblk according.  If not,
		 * calculate and insert the checksum using software.
		 */
		mp = xnb_process_cksum_flags(xnbp, mp, capab);
	}

	return (mp);
}

/*
 * Calculate the checksum flags to be relayed to the peer for the
 * packet `mp'.
 */
static uint16_t
xnbo_cksum_to_peer(xnb_t *xnbp, mblk_t *mp)
{
	_NOTE(ARGUNUSED(xnbp));
	uint16_t r = 0;
	uint32_t pflags, csum;

#ifdef XNBO_DEBUG
	if (!xnbo_cksum_offload_to_peer)
		return (0);
#endif /* XNBO_DEBUG */

	/*
	 * We might also check for HCK_PARTIALCKSUM here and,
	 * providing that the partial checksum covers the TCP/UDP
	 * payload, return NETRXF_data_validated.
	 *
	 * It seems that it's probably not worthwhile, as even MAC
	 * devices which advertise HCKSUM_INET_PARTIAL in their
	 * capabilities tend to use HCK_FULLCKSUM on the receive side
	 * - they are actually saying that in the output path the
	 * caller must use HCK_PARTIALCKSUM.
	 *
	 * Then again, if a NIC supports HCK_PARTIALCKSUM in its'
	 * output path, the host IP stack will use it. If such packets
	 * are destined for the peer (i.e. looped around) we would
	 * gain some advantage.
	 */

	mac_hcksum_get(mp, NULL, NULL, NULL, &csum, &pflags);

	/*
	 * If the MAC driver has asserted that the checksum is
	 * good, let the peer know.
	 */
	if (((pflags & HCK_FULLCKSUM) != 0) &&
	    (((pflags & HCK_FULLCKSUM_OK) != 0) ||
	    (csum == 0xffff)))
		r |= NETRXF_data_validated;

	return (r);
}

/*
 * Packets from the mac device come here.  We pass them to the peer.
 */
/*ARGSUSED*/
static void
xnbo_from_mac(void *arg, mac_resource_handle_t mrh, mblk_t *mp,
    boolean_t loopback)
{
	xnb_t *xnbp = arg;

	mp = xnb_copy_to_peer(xnbp, mp);

	if (mp != NULL)
		freemsgchain(mp);
}

/*
 * Packets from the mac device come here. We pass them to the peer if
 * the destination mac address matches or it's a multicast/broadcast
 * address.
 */
static void
xnbo_from_mac_filter(void *arg, mac_resource_handle_t mrh, mblk_t *mp,
    boolean_t loopback)
{
	_NOTE(ARGUNUSED(loopback));
	xnb_t *xnbp = arg;
	xnbo_t *xnbop = xnbp->xnb_flavour_data;
	mblk_t *next, *keep, *keep_head, *free, *free_head;

	keep = keep_head = free = free_head = NULL;

#define	ADD(list, bp)				\
	if (list != NULL)			\
		list->b_next = bp;		\
	else					\
		list##_head = bp;		\
	list = bp;

	for (; mp != NULL; mp = next) {
		mac_header_info_t hdr_info;

		next = mp->b_next;
		mp->b_next = NULL;

		if (mac_header_info(xnbop->o_mh, mp, &hdr_info) != 0) {
			ADD(free, mp);
			continue;
		}

		if ((hdr_info.mhi_dsttype == MAC_ADDRTYPE_BROADCAST) ||
		    (hdr_info.mhi_dsttype == MAC_ADDRTYPE_MULTICAST)) {
			ADD(keep, mp);
			continue;
		}

		if (bcmp(hdr_info.mhi_daddr, xnbp->xnb_mac_addr,
		    sizeof (xnbp->xnb_mac_addr)) == 0) {
			ADD(keep, mp);
			continue;
		}

		ADD(free, mp);
	}
#undef	ADD

	if (keep_head != NULL)
		xnbo_from_mac(xnbp, mrh, keep_head, B_FALSE);

	if (free_head != NULL)
		freemsgchain(free_head);
}

static boolean_t
xnbo_open_mac(xnb_t *xnbp, char *mac)
{
	xnbo_t *xnbop = xnbp->xnb_flavour_data;
	int err;
	const mac_info_t *mi;
	void (*rx_fn)(void *, mac_resource_handle_t, mblk_t *, boolean_t);
	struct ether_addr ea;
	uint_t max_sdu;
	mac_diag_t diag;

	if ((err = mac_open_by_linkname(mac, &xnbop->o_mh)) != 0) {
		cmn_err(CE_WARN, "xnbo_open_mac: "
		    "cannot open mac for link %s (%d)", mac, err);
		return (B_FALSE);
	}
	ASSERT(xnbop->o_mh != NULL);

	mi = mac_info(xnbop->o_mh);
	ASSERT(mi != NULL);

	if (mi->mi_media != DL_ETHER) {
		cmn_err(CE_WARN, "xnbo_open_mac: "
		    "device is not DL_ETHER (%d)", mi->mi_media);
		i_xnbo_close_mac(xnbp, B_TRUE);
		return (B_FALSE);
	}
	if (mi->mi_media != mi->mi_nativemedia) {
		cmn_err(CE_WARN, "xnbo_open_mac: "
		    "device media and native media mismatch (%d != %d)",
		    mi->mi_media, mi->mi_nativemedia);
		i_xnbo_close_mac(xnbp, B_TRUE);
		return (B_FALSE);
	}

	mac_sdu_get(xnbop->o_mh, NULL, &max_sdu);
	if (max_sdu > XNBMAXPKT) {
		cmn_err(CE_WARN, "xnbo_open_mac: mac device SDU too big (%d)",
		    max_sdu);
		i_xnbo_close_mac(xnbp, B_TRUE);
		return (B_FALSE);
	}

	/*
	 * MAC_OPEN_FLAGS_MULTI_PRIMARY is relevant when we are migrating a
	 * guest on the localhost itself. In this case we would have the MAC
	 * client open for the guest being migrated *and* also for the
	 * migrated guest (i.e. the former will be active till the migration
	 * is complete when the latter will be activated). This flag states
	 * that it is OK for mac_unicast_add to add the primary MAC unicast
	 * address multiple times.
	 */
	if (mac_client_open(xnbop->o_mh, &xnbop->o_mch, NULL,
	    MAC_OPEN_FLAGS_USE_DATALINK_NAME |
	    MAC_OPEN_FLAGS_MULTI_PRIMARY) != 0) {
		cmn_err(CE_WARN, "xnbo_open_mac: "
		    "error (%d) opening mac client", err);
		i_xnbo_close_mac(xnbp, B_TRUE);
		return (B_FALSE);
	}

	if (xnbop->o_need_rx_filter)
		rx_fn = xnbo_from_mac_filter;
	else
		rx_fn = xnbo_from_mac;

	err = mac_unicast_add_set_rx(xnbop->o_mch, NULL, MAC_UNICAST_PRIMARY,
	    &xnbop->o_mah, 0, &diag, xnbop->o_multicast_control ? rx_fn : NULL,
	    xnbp);
	if (err != 0) {
		cmn_err(CE_WARN, "xnbo_open_mac: failed to get the primary "
		    "MAC address of %s: %d", mac, err);
		i_xnbo_close_mac(xnbp, B_TRUE);
		return (B_FALSE);
	}
	if (!xnbop->o_multicast_control) {
		err = mac_promisc_add(xnbop->o_mch, MAC_CLIENT_PROMISC_ALL,
		    rx_fn, xnbp, &xnbop->o_mphp, MAC_PROMISC_FLAGS_NO_TX_LOOP |
		    MAC_PROMISC_FLAGS_VLAN_TAG_STRIP);
		if (err != 0) {
			cmn_err(CE_WARN, "xnbo_open_mac: "
			    "cannot enable promiscuous mode of %s: %d",
			    mac, err);
			i_xnbo_close_mac(xnbp, B_TRUE);
			return (B_FALSE);
		}
		xnbop->o_promiscuous = B_TRUE;
	}

	if (xnbop->o_need_setphysaddr) {
		err = mac_unicast_primary_set(xnbop->o_mh, xnbp->xnb_mac_addr);
		/* Warn, but continue on. */
		if (err != 0) {
			bcopy(xnbp->xnb_mac_addr, ea.ether_addr_octet,
			    ETHERADDRL);
			cmn_err(CE_WARN, "xnbo_open_mac: "
			    "cannot set MAC address of %s to "
			    "%s: %d", mac, ether_sprintf(&ea), err);
		}
	}

	if (!mac_capab_get(xnbop->o_mh, MAC_CAPAB_HCKSUM,
	    &xnbop->o_hcksum_capab))
		xnbop->o_hcksum_capab = 0;

	xnbop->o_running = B_TRUE;

	return (B_TRUE);
}

static void
xnbo_close_mac(xnb_t *xnbp)
{
	i_xnbo_close_mac(xnbp, B_FALSE);
}

static void
i_xnbo_close_mac(xnb_t *xnbp, boolean_t locked)
{
	xnbo_t *xnbop = xnbp->xnb_flavour_data;
	xmca_t *loop;

	ASSERT(!locked || MUTEX_HELD(&xnbp->xnb_state_lock));

	if (xnbop->o_mh == NULL)
		return;

	if (xnbop->o_running)
		xnbop->o_running = B_FALSE;

	if (!locked)
		mutex_enter(&xnbp->xnb_state_lock);
	loop = xnbop->o_mca;
	xnbop->o_mca = NULL;
	if (!locked)
		mutex_exit(&xnbp->xnb_state_lock);

	while (loop != NULL) {
		xmca_t *next = loop->next;

		DTRACE_PROBE3(mcast_remove,
		    (char *), "close",
		    (void *), xnbp,
		    (etheraddr_t *), loop->addr);
		(void) mac_multicast_remove(xnbop->o_mch, loop->addr);
		kmem_free(loop, sizeof (*loop));
		loop = next;
	}

	if (xnbop->o_promiscuous) {
		if (xnbop->o_mphp != NULL) {
			mac_promisc_remove(xnbop->o_mphp);
			xnbop->o_mphp = NULL;
		}
		xnbop->o_promiscuous = B_FALSE;
	} else {
		if (xnbop->o_mch != NULL)
			mac_rx_clear(xnbop->o_mch);
	}

	if (xnbop->o_mah != NULL) {
		(void) mac_unicast_remove(xnbop->o_mch, xnbop->o_mah);
		xnbop->o_mah = NULL;
	}

	if (xnbop->o_mch != NULL) {
		mac_client_close(xnbop->o_mch, 0);
		xnbop->o_mch = NULL;
	}

	mac_close(xnbop->o_mh);
	xnbop->o_mh = NULL;
}

/*
 * Hotplug has completed and we are connected to the peer. We have all
 * the information we need to exchange traffic, so open the MAC device
 * and configure it appropriately.
 */
static boolean_t
xnbo_start_connect(xnb_t *xnbp)
{
	xnbo_t *xnbop = xnbp->xnb_flavour_data;

	return (xnbo_open_mac(xnbp, xnbop->o_link_name));
}

/*
 * The guest has successfully synchronize with this instance. We read
 * the configuration of the guest from xenstore to check whether the
 * guest requests multicast control. If not (the default) we make a
 * note that the MAC device needs to be used in promiscious mode.
 */
static boolean_t
xnbo_peer_connected(xnb_t *xnbp)
{
	char *oename;
	int request;
	xnbo_t *xnbop = xnbp->xnb_flavour_data;

	oename = xvdi_get_oename(xnbp->xnb_devinfo);

	if (xenbus_scanf(XBT_NULL, oename,
	    "request-multicast-control", "%d", &request) != 0)
		request = 0;
	xnbop->o_multicast_control = (request > 0);

	return (B_TRUE);
}

/*
 * The guest domain has closed down the inter-domain connection. We
 * close the underlying MAC device.
 */
static void
xnbo_peer_disconnected(xnb_t *xnbp)
{
	xnbo_close_mac(xnbp);
}

/*
 * The hotplug script has completed. We read information from xenstore
 * about our configuration, most notably the name of the MAC device we
 * should use.
 */
static boolean_t
xnbo_hotplug_connected(xnb_t *xnbp)
{
	char *xsname;
	xnbo_t *xnbop = xnbp->xnb_flavour_data;
	int need;

	xsname = xvdi_get_xsname(xnbp->xnb_devinfo);

	if (xenbus_scanf(XBT_NULL, xsname,
	    "nic", "%s", xnbop->o_link_name) != 0) {
		cmn_err(CE_WARN, "xnbo_connect: "
		    "cannot read nic name from %s", xsname);
		return (B_FALSE);
	}

	if (xenbus_scanf(XBT_NULL, xsname,
	    "SUNW-need-rx-filter", "%d", &need) != 0)
		need = 0;
	xnbop->o_need_rx_filter = (need > 0);

	if (xenbus_scanf(XBT_NULL, xsname,
	    "SUNW-need-set-physaddr", "%d", &need) != 0)
		need = 0;
	xnbop->o_need_setphysaddr = (need > 0);

	return (B_TRUE);
}

/*
 * Find the multicast address `addr', return B_TRUE if it is one that
 * we receive. If `remove', remove it from the set received.
 */
static boolean_t
xnbo_mcast_find(xnb_t *xnbp, ether_addr_t *addr, boolean_t remove)
{
	xnbo_t *xnbop = xnbp->xnb_flavour_data;
	xmca_t *prev, *del, *this;

	ASSERT(MUTEX_HELD(&xnbp->xnb_state_lock));
	ASSERT(xnbop->o_promiscuous == B_FALSE);

	prev = del = NULL;

	this = xnbop->o_mca;

	while (this != NULL) {
		if (bcmp(&this->addr, addr, sizeof (this->addr)) == 0) {
			del = this;
			if (remove) {
				if (prev == NULL)
					xnbop->o_mca = this->next;
				else
					prev->next = this->next;
			}
			break;
		}

		prev = this;
		this = this->next;
	}

	if (del == NULL)
		return (B_FALSE);

	if (remove) {
		DTRACE_PROBE3(mcast_remove,
		    (char *), "remove",
		    (void *), xnbp,
		    (etheraddr_t *), del->addr);
		mac_multicast_remove(xnbop->o_mch, del->addr);
		kmem_free(del, sizeof (*del));
	}

	return (B_TRUE);
}

/*
 * Add the multicast address `addr' to the set received.
 */
static boolean_t
xnbo_mcast_add(xnb_t *xnbp, ether_addr_t *addr)
{
	xnbo_t *xnbop = xnbp->xnb_flavour_data;
	boolean_t r = B_FALSE;

	ASSERT(xnbop->o_promiscuous == B_FALSE);

	mutex_enter(&xnbp->xnb_state_lock);

	if (xnbo_mcast_find(xnbp, addr, B_FALSE)) {
		r = B_TRUE;
	} else if (mac_multicast_add(xnbop->o_mch,
	    (const uint8_t *)addr) == 0) {
		xmca_t *mca;

		DTRACE_PROBE3(mcast_add,
		    (char *), "add",
		    (void *), xnbp,
		    (etheraddr_t *), addr);

		mca = kmem_alloc(sizeof (*mca), KM_SLEEP);
		bcopy(addr, &mca->addr, sizeof (mca->addr));

		mca->next = xnbop->o_mca;
		xnbop->o_mca = mca;

		r = B_TRUE;
	}

	mutex_exit(&xnbp->xnb_state_lock);

	return (r);
}

/*
 * Remove the multicast address `addr' from the set received.
 */
static boolean_t
xnbo_mcast_del(xnb_t *xnbp, ether_addr_t *addr)
{
	boolean_t r;

	mutex_enter(&xnbp->xnb_state_lock);
	r = xnbo_mcast_find(xnbp, addr, B_TRUE);
	mutex_exit(&xnbp->xnb_state_lock);

	return (r);
}

static int
xnbo_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
{
	static xnb_flavour_t flavour = {
		xnbo_to_mac, xnbo_peer_connected, xnbo_peer_disconnected,
		xnbo_hotplug_connected, xnbo_start_connect,
		xnbo_cksum_from_peer, xnbo_cksum_to_peer,
		xnbo_mcast_add, xnbo_mcast_del,
	};
	xnbo_t *xnbop;

	switch (cmd) {
	case DDI_ATTACH:
		break;
	case DDI_RESUME:
		return (DDI_SUCCESS);
	default:
		return (DDI_FAILURE);
	}

	xnbop = kmem_zalloc(sizeof (*xnbop), KM_SLEEP);

	if (xnb_attach(dip, &flavour, xnbop) != DDI_SUCCESS) {
		kmem_free(xnbop, sizeof (*xnbop));
		return (DDI_FAILURE);
	}

	return (DDI_SUCCESS);
}

static int
xnbo_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
{
	xnb_t *xnbp = ddi_get_driver_private(dip);
	xnbo_t *xnbop = xnbp->xnb_flavour_data;

	switch (cmd) {
	case DDI_DETACH:
		break;
	case DDI_SUSPEND:
		return (DDI_SUCCESS);
	default:
		return (DDI_FAILURE);
	}

	mutex_enter(&xnbp->xnb_tx_lock);
	mutex_enter(&xnbp->xnb_rx_lock);

	if (!xnbp->xnb_detachable || xnbp->xnb_connected ||
	    (xnbp->xnb_tx_buf_count > 0)) {
		mutex_exit(&xnbp->xnb_rx_lock);
		mutex_exit(&xnbp->xnb_tx_lock);

		return (DDI_FAILURE);
	}

	mutex_exit(&xnbp->xnb_rx_lock);
	mutex_exit(&xnbp->xnb_tx_lock);

	xnbo_close_mac(xnbp);
	kmem_free(xnbop, sizeof (*xnbop));

	xnb_detach(dip);

	return (DDI_SUCCESS);
}

static struct cb_ops cb_ops = {
	nulldev,		/* open */
	nulldev,		/* close */
	nodev,			/* strategy */
	nodev,			/* print */
	nodev,			/* dump */
	nodev,			/* read */
	nodev,			/* write */
	nodev,			/* ioctl */
	nodev,			/* devmap */
	nodev,			/* mmap */
	nodev,			/* segmap */
	nochpoll,		/* poll */
	ddi_prop_op,		/* cb_prop_op */
	0,			/* streamtab  */
	D_NEW | D_MP | D_64BIT	/* Driver compatibility flag */
};

static struct dev_ops ops = {
	DEVO_REV,		/* devo_rev */
	0,			/* devo_refcnt  */
	nulldev,		/* devo_getinfo */
	nulldev,		/* devo_identify */
	nulldev,		/* devo_probe */
	xnbo_attach,		/* devo_attach */
	xnbo_detach,		/* devo_detach */
	nodev,			/* devo_reset */
	&cb_ops,		/* devo_cb_ops */
	(struct bus_ops *)0,	/* devo_bus_ops */
	NULL,			/* devo_power */
	ddi_quiesce_not_needed,		/* devo_quiesce */
};

static struct modldrv modldrv = {
	&mod_driverops, "xnbo driver", &ops,
};

static struct modlinkage modlinkage = {
	MODREV_1, &modldrv, NULL
};

int
_init(void)
{
	return (mod_install(&modlinkage));
}

int
_info(struct modinfo *modinfop)
{
	return (mod_info(&modlinkage, modinfop));
}

int
_fini(void)
{
	return (mod_remove(&modlinkage));
}
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

/*
 * Xen inter-domain backend - GLDv3 driver edition.
 *
 * A traditional GLDv3 driver used to communicate with a guest
 * domain.  This driver is typically plumbed underneath the IP stack
 * or a software ethernet bridge.
 */

#include "xnb.h"

#include <sys/sunddi.h>
#include <sys/conf.h>
#include <sys/modctl.h>
#include <sys/strsubr.h>
#include <sys/dlpi.h>
#include <sys/pattr.h>
#include <sys/mac_provider.h>
#include <sys/mac_ether.h>
#include <xen/sys/xendev.h>
#include <sys/note.h>

/* Required driver entry points for GLDv3 */
static int	xnbu_m_start(void *);
static void	xnbu_m_stop(void *);
static int	xnbu_m_set_mac_addr(void *, const uint8_t *);
static int	xnbu_m_set_multicast(void *, boolean_t, const uint8_t *);
static int	xnbu_m_set_promiscuous(void *, boolean_t);
static int	xnbu_m_stat(void *, uint_t, uint64_t *);
static boolean_t xnbu_m_getcapab(void *, mac_capab_t, void *);
static mblk_t	*xnbu_m_send(void *, mblk_t *);

typedef struct xnbu {
	mac_handle_t		u_mh;
	boolean_t		u_need_sched;
} xnbu_t;

static mac_callbacks_t xnbu_callbacks = {
	MC_GETCAPAB,
	xnbu_m_stat,
	xnbu_m_start,
	xnbu_m_stop,
	xnbu_m_set_promiscuous,
	xnbu_m_set_multicast,
	xnbu_m_set_mac_addr,
	xnbu_m_send,
	NULL,
	NULL,
	xnbu_m_getcapab
};

static void
xnbu_to_host(xnb_t *xnbp, mblk_t *mp)
{
	xnbu_t *xnbup = xnbp->xnb_flavour_data;
	boolean_t sched = B_FALSE;

	ASSERT(mp != NULL);

	mac_rx(xnbup->u_mh, NULL, mp);

	mutex_enter(&xnbp->xnb_rx_lock);

	/*
	 * If a transmit attempt failed because we ran out of ring
	 * space and there is now some space, re-enable the transmit
	 * path.
	 */
	if (xnbup->u_need_sched &&
	    RING_HAS_UNCONSUMED_REQUESTS(&xnbp->xnb_rx_ring)) {
		sched = B_TRUE;
		xnbup->u_need_sched = B_FALSE;
	}

	mutex_exit(&xnbp->xnb_rx_lock);

	if (sched)
		mac_tx_update(xnbup->u_mh);
}

static mblk_t *
xnbu_cksum_from_peer(xnb_t *xnbp, mblk_t *mp, uint16_t flags)
{
	/*
	 * Take a conservative approach - if the checksum is blank
	 * then we fill it in.
	 *
	 * If the consumer of the packet is IP then we might actually
	 * only need fill it in if the data is not validated, but how
	 * do we know who might end up with the packet?
	 */

	if ((flags & NETTXF_csum_blank) != 0) {
		/*
		 * The checksum is blank.  We must fill it in here.
		 */
		mp = xnb_process_cksum_flags(xnbp, mp, 0);

		/*
		 * Because we calculated the checksum ourselves we
		 * know that it must be good, so we assert this.
		 */
		flags |= NETTXF_data_validated;
	}

	if ((flags & NETTXF_data_validated) != 0) {
		/*
		 * The checksum is asserted valid.
		 */
		mac_hcksum_set(mp, 0, 0, 0, 0, HCK_FULLCKSUM_OK);
	}

	return (mp);
}

static uint16_t
xnbu_cksum_to_peer(xnb_t *xnbp, mblk_t *mp)
{
	_NOTE(ARGUNUSED(xnbp));
	uint16_t r = 0;
	uint32_t pflags;

	mac_hcksum_get(mp, NULL, NULL, NULL, NULL, &pflags);

	/*
	 * If the protocol stack has requested checksum
	 * offload, inform the peer that we have not
	 * calculated the checksum.
	 */
	if ((pflags & HCK_FULLCKSUM) != 0)
		r |= NETRXF_csum_blank;

	return (r);
}

static boolean_t
xnbu_start_connect(xnb_t *xnbp)
{
	xnbu_t *xnbup = xnbp->xnb_flavour_data;

	mac_link_update(xnbup->u_mh, LINK_STATE_UP);
	/*
	 * We are able to send packets now - bring them on.
	 */
	mac_tx_update(xnbup->u_mh);

	return (B_TRUE);
}

static boolean_t
xnbu_peer_connected(xnb_t *xnbp)
{
	_NOTE(ARGUNUSED(xnbp));

	return (B_TRUE);
}

static void
xnbu_peer_disconnected(xnb_t *xnbp)
{
	xnbu_t *xnbup = xnbp->xnb_flavour_data;

	mac_link_update(xnbup->u_mh, LINK_STATE_DOWN);
}

/*ARGSUSED*/
static boolean_t
xnbu_hotplug_connected(xnb_t *xnbp)
{
	return (B_TRUE);
}

static mblk_t *
xnbu_m_send(void *arg, mblk_t *mp)
{
	xnb_t *xnbp = arg;
	xnbu_t *xnbup = xnbp->xnb_flavour_data;
	boolean_t sched = B_FALSE;

	mp = xnb_copy_to_peer(arg, mp);

	mutex_enter(&xnbp->xnb_rx_lock);
	/*
	 * If we consumed all of the mblk_t's offered, perhaps we need
	 * to indicate that we can accept more.  Otherwise we are full
	 * and need to wait for space.
	 */
	if (mp == NULL) {
		sched = xnbup->u_need_sched;
		xnbup->u_need_sched = B_FALSE;
	} else {
		xnbup->u_need_sched = B_TRUE;
	}
	mutex_exit(&xnbp->xnb_rx_lock);

	/*
	 * If a previous transmit attempt failed because the ring
	 * was full, try again now.
	 */
	if (sched)
		mac_tx_update(xnbup->u_mh);

	return (mp);
}

/*
 *  xnbu_m_set_mac_addr() -- set the physical network address on the board
 */
/* ARGSUSED */
static int
xnbu_m_set_mac_addr(void *arg, const uint8_t *macaddr)
{
	xnb_t *xnbp = arg;
	xnbu_t *xnbup = xnbp->xnb_flavour_data;

	bcopy(macaddr, xnbp->xnb_mac_addr, ETHERADDRL);
	mac_unicst_update(xnbup->u_mh, xnbp->xnb_mac_addr);

	return (0);
}

/*
 *  xnbu_m_set_multicast() -- set (enable) or disable a multicast address
 */
/*ARGSUSED*/
static int
xnbu_m_set_multicast(void *arg, boolean_t add, const uint8_t *mca)
{
	/*
	 * We always accept all packets from the peer, so nothing to
	 * do for enable or disable.
	 */
	return (0);
}


/*
 * xnbu_m_set_promiscuous() -- set or reset promiscuous mode on the board
 */
/* ARGSUSED */
static int
xnbu_m_set_promiscuous(void *arg, boolean_t on)
{
	/*
	 * We always accept all packets from the peer, so nothing to
	 * do for enable or disable.
	 */
	return (0);
}

/*
 *  xnbu_m_start() -- start the board receiving and enable interrupts.
 */
/*ARGSUSED*/
static int
xnbu_m_start(void *arg)
{
	return (0);
}

/*
 * xnbu_m_stop() - disable hardware
 */
/*ARGSUSED*/
static void
xnbu_m_stop(void *arg)
{
}

static int
xnbu_m_stat(void *arg, uint_t stat, uint64_t *val)
{
	xnb_t *xnbp = arg;

	mutex_enter(&xnbp->xnb_tx_lock);
	mutex_enter(&xnbp->xnb_rx_lock);

#define	map_stat(q, r)				\
	case (MAC_STAT_##q):			\
		*val = xnbp->xnb_stat_##r;	\
		break

	switch (stat) {

	map_stat(IPACKETS, opackets);
	map_stat(OPACKETS, ipackets);
	map_stat(RBYTES, obytes);
	map_stat(OBYTES, rbytes);

	default:
		mutex_exit(&xnbp->xnb_rx_lock);
		mutex_exit(&xnbp->xnb_tx_lock);

		return (ENOTSUP);
	}

#undef map_stat

	mutex_exit(&xnbp->xnb_rx_lock);
	mutex_exit(&xnbp->xnb_tx_lock);

	return (0);
}

static boolean_t
xnbu_m_getcapab(void *arg, mac_capab_t cap, void *cap_data)
{
	_NOTE(ARGUNUSED(arg));

	switch (cap) {
	case MAC_CAPAB_HCKSUM: {
		uint32_t *capab = cap_data;

		*capab = HCKSUM_INET_PARTIAL;
		break;
	}
	default:
		return (B_FALSE);
	}

	return (B_TRUE);
}

/*
 * All packets are passed to the peer, so adding and removing
 * multicast addresses is meaningless.
 */
static boolean_t
xnbu_mcast_add(xnb_t *xnbp, ether_addr_t *addr)
{
	_NOTE(ARGUNUSED(xnbp, addr));

	return (B_TRUE);
}

static boolean_t
xnbu_mcast_del(xnb_t *xnbp, ether_addr_t *addr)
{
	_NOTE(ARGUNUSED(xnbp, addr));

	return (B_TRUE);
}

static int
xnbu_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
{
	static xnb_flavour_t flavour = {
		xnbu_to_host, xnbu_peer_connected, xnbu_peer_disconnected,
		xnbu_hotplug_connected, xnbu_start_connect,
		xnbu_cksum_from_peer, xnbu_cksum_to_peer,
		xnbu_mcast_add, xnbu_mcast_del,
	};
	xnbu_t *xnbup;
	xnb_t *xnbp;
	mac_register_t *mr;
	int err;

	switch (cmd) {
	case DDI_ATTACH:
		break;
	case DDI_RESUME:
		return (DDI_SUCCESS);
	default:
		return (DDI_FAILURE);
	}

	xnbup = kmem_zalloc(sizeof (*xnbup), KM_SLEEP);

	if ((mr = mac_alloc(MAC_VERSION)) == NULL) {
		kmem_free(xnbup, sizeof (*xnbup));
		return (DDI_FAILURE);
	}

	if (xnb_attach(dip, &flavour, xnbup) != DDI_SUCCESS) {
		mac_free(mr);
		kmem_free(xnbup, sizeof (*xnbup));
		return (DDI_FAILURE);
	}

	xnbp = ddi_get_driver_private(dip);
	ASSERT(xnbp != NULL);

	mr->m_dip = dip;
	mr->m_driver = xnbp;

	/*
	 *  Initialize pointers to device specific functions which will be
	 *  used by the generic layer.
	 */
	mr->m_type_ident = MAC_PLUGIN_IDENT_ETHER;
	mr->m_src_addr = xnbp->xnb_mac_addr;
	mr->m_callbacks = &xnbu_callbacks;
	mr->m_min_sdu = 0;
	mr->m_max_sdu = XNBMAXPKT;
	/*
	 * xnbu is a virtual device, and it is not associated with any
	 * physical device. Its margin size is determined by the maximum
	 * packet size it can handle, which is PAGESIZE.
	 */
	mr->m_margin = PAGESIZE - XNBMAXPKT - sizeof (struct ether_header);

	(void) memset(xnbp->xnb_mac_addr, 0xff, ETHERADDRL);
	xnbp->xnb_mac_addr[0] &= 0xfe;
	xnbup->u_need_sched = B_FALSE;

	/*
	 * Register ourselves with the GLDv3 interface.
	 */
	err = mac_register(mr, &xnbup->u_mh);
	mac_free(mr);
	if (err != 0) {
		xnb_detach(dip);
		kmem_free(xnbup, sizeof (*xnbup));
		return (DDI_FAILURE);
	}

	mac_link_update(xnbup->u_mh, LINK_STATE_DOWN);

	return (DDI_SUCCESS);
}

/*ARGSUSED*/
int
xnbu_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
{
	xnb_t *xnbp = ddi_get_driver_private(dip);
	xnbu_t *xnbup = xnbp->xnb_flavour_data;

	switch (cmd) {
	case DDI_DETACH:
		break;
	case DDI_SUSPEND:
		return (DDI_SUCCESS);
	default:
		return (DDI_FAILURE);
	}

	ASSERT(xnbp != NULL);
	ASSERT(xnbup != NULL);

	mutex_enter(&xnbp->xnb_tx_lock);
	mutex_enter(&xnbp->xnb_rx_lock);

	if (!xnbp->xnb_detachable || xnbp->xnb_connected ||
	    (xnbp->xnb_tx_buf_count > 0)) {
		mutex_exit(&xnbp->xnb_rx_lock);
		mutex_exit(&xnbp->xnb_tx_lock);

		return (DDI_FAILURE);
	}

	mutex_exit(&xnbp->xnb_rx_lock);
	mutex_exit(&xnbp->xnb_tx_lock);

	/*
	 * Attempt to unregister the mac.
	 */
	if ((xnbup->u_mh != NULL) && (mac_unregister(xnbup->u_mh) != 0))
		return (DDI_FAILURE);
	kmem_free(xnbup, sizeof (*xnbup));

	xnb_detach(dip);

	return (DDI_SUCCESS);
}

DDI_DEFINE_STREAM_OPS(ops, nulldev, nulldev, xnbu_attach, xnbu_detach,
    nodev, NULL, D_MP, NULL, ddi_quiesce_not_supported);

static struct modldrv modldrv = {
	&mod_driverops, "xnbu driver", &ops
};

static struct modlinkage modlinkage = {
	MODREV_1, &modldrv, NULL
};

int
_init(void)
{
	int i;

	mac_init_ops(&ops, "xnbu");

	i = mod_install(&modlinkage);
	if (i != DDI_SUCCESS)
		mac_fini_ops(&ops);

	return (i);
}

int
_fini(void)
{
	int i;

	i = mod_remove(&modlinkage);
	if (i == DDI_SUCCESS)
		mac_fini_ops(&ops);

	return (i);
}

int
_info(struct modinfo *modinfop)
{
	return (mod_info(&modlinkage, modinfop));
}
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

/*
 * Copyright (c) 2014, 2017 by Delphix. All rights reserved.
 * Copyright 2020 RackTop Systems, Inc.
 */

/*
 *
 * Copyright (c) 2004 Christian Limpach.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. This section intentionally left blank.
 * 4. The name of the author may not be used to endorse or promote products
 *    derived from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */
/*
 * Section 3 of the above license was updated in response to bug 6379571.
 */

/*
 * xnf.c - GLDv3 network driver for domU.
 */

/*
 * This driver uses four per-instance locks:
 *
 * xnf_gref_lock:
 *
 *    Protects access to the grant reference list stored in
 *    xnf_gref_head. Grant references should be acquired and released
 *    using gref_get() and gref_put() respectively.
 *
 * xnf_schedlock:
 *
 *    Protects:
 *    xnf_need_sched - used to record that a previous transmit attempt
 *       failed (and consequently it will be necessary to call
 *       mac_tx_update() when transmit resources are available).
 *    xnf_pending_multicast - the number of multicast requests that
 *       have been submitted to the backend for which we have not
 *       processed responses.
 *
 * xnf_txlock:
 *
 *    Protects the transmit ring (xnf_tx_ring) and associated
 *    structures (notably xnf_tx_pkt_id and xnf_tx_pkt_id_head).
 *
 * xnf_rxlock:
 *
 *    Protects the receive ring (xnf_rx_ring) and associated
 *    structures (notably xnf_rx_pkt_info).
 *
 * If driver-global state that affects both the transmit and receive
 * rings is manipulated, both xnf_txlock and xnf_rxlock should be
 * held, in that order.
 *
 * xnf_schedlock is acquired both whilst holding xnf_txlock and
 * without. It should always be acquired after xnf_txlock if both are
 * held.
 *
 * Notes:
 * - atomic_add_64() is used to manipulate counters where we require
 *   accuracy. For counters intended only for observation by humans,
 *   post increment/decrement are used instead.
 */

#include <sys/types.h>
#include <sys/errno.h>
#include <sys/param.h>
#include <sys/sysmacros.h>
#include <sys/systm.h>
#include <sys/stream.h>
#include <sys/strsubr.h>
#include <sys/strsun.h>
#include <sys/conf.h>
#include <sys/ddi.h>
#include <sys/devops.h>
#include <sys/sunddi.h>
#include <sys/sunndi.h>
#include <sys/dlpi.h>
#include <sys/ethernet.h>
#include <sys/strsun.h>
#include <sys/pattr.h>
#include <inet/ip.h>
#include <inet/ip_impl.h>
#include <inet/tcp.h>
#include <netinet/udp.h>
#include <sys/gld.h>
#include <sys/modctl.h>
#include <sys/mac_provider.h>
#include <sys/mac_ether.h>
#include <sys/bootinfo.h>
#include <sys/mach_mmu.h>
#ifdef	XPV_HVM_DRIVER
#include <sys/xpv_support.h>
#include <sys/hypervisor.h>
#else
#include <sys/hypervisor.h>
#include <sys/evtchn_impl.h>
#include <sys/balloon_impl.h>
#endif
#include <xen/public/io/netif.h>
#include <sys/gnttab.h>
#include <xen/sys/xendev.h>
#include <sys/sdt.h>
#include <sys/note.h>
#include <sys/debug.h>

#include <io/xnf.h>

/*
 * On a 32 bit PAE system physical and machine addresses are larger
 * than 32 bits.  ddi_btop() on such systems take an unsigned long
 * argument, and so addresses above 4G are truncated before ddi_btop()
 * gets to see them.  To avoid this, code the shift operation here.
 */
#define	xnf_btop(addr)	((addr) >> PAGESHIFT)

/*
 * The parameters below should only be changed in /etc/system, never in mdb.
 */

/*
 * Should we use the multicast control feature if the backend provides
 * it?
 */
boolean_t xnf_multicast_control = B_TRUE;

/*
 * Should we allow scatter-gather for tx if backend allows it?
 */
boolean_t xnf_enable_tx_sg = B_TRUE;

/*
 * Should we allow scatter-gather for rx if backend allows it?
 */
boolean_t xnf_enable_rx_sg = B_TRUE;

/*
 * Should we allow lso for tx sends if backend allows it?
 * Requires xnf_enable_tx_sg to be also set to TRUE.
 */
boolean_t xnf_enable_lso = B_TRUE;

/*
 * Should we allow lro on rx if backend supports it?
 * Requires xnf_enable_rx_sg to be also set to TRUE.
 *
 * !! WARNING !!
 * LRO is not yet supported in the OS so this should be left as FALSE.
 * !! WARNING !!
 */
boolean_t xnf_enable_lro = B_FALSE;

/*
 * Received packets below this size are copied to a new streams buffer
 * rather than being desballoc'ed.
 *
 * This value is chosen to accommodate traffic where there are a large
 * number of small packets. For data showing a typical distribution,
 * see:
 *
 * Sinha07a:
 *	Rishi Sinha, Christos Papadopoulos, and John
 *	Heidemann. Internet Packet Size Distributions: Some
 *	Observations. Technical Report ISI-TR-2007-643,
 *	USC/Information Sciences Institute, May, 2007. Orignally
 *	released October 2005 as web page
 *	http://netweb.usc.edu/~sinha/pkt-sizes/.
 *	<http://www.isi.edu/~johnh/PAPERS/Sinha07a.html>.
 */
size_t xnf_rx_copy_limit = 64;

#define	INVALID_GRANT_HANDLE	((grant_handle_t)-1)
#define	INVALID_GRANT_REF	((grant_ref_t)-1)
#define	INVALID_TX_ID		((uint16_t)-1)

#define	TX_ID_TO_TXID(p, id) (&((p)->xnf_tx_pkt_id[(id)]))
#define	TX_ID_VALID(i) \
	(((i) != INVALID_TX_ID) && ((i) < NET_TX_RING_SIZE))

/*
 * calculate how many pages are spanned by an mblk fragment
 */
#define	xnf_mblk_pages(mp)	(MBLKL(mp) == 0 ? 0 : \
    xnf_btop((uintptr_t)mp->b_wptr - 1) - xnf_btop((uintptr_t)mp->b_rptr) + 1)

/* Required system entry points */
static int	xnf_attach(dev_info_t *, ddi_attach_cmd_t);
static int	xnf_detach(dev_info_t *, ddi_detach_cmd_t);

/* Required driver entry points for Nemo */
static int	xnf_start(void *);
static void	xnf_stop(void *);
static int	xnf_set_mac_addr(void *, const uint8_t *);
static int	xnf_set_multicast(void *, boolean_t, const uint8_t *);
static int	xnf_set_promiscuous(void *, boolean_t);
static mblk_t	*xnf_send(void *, mblk_t *);
static uint_t	xnf_intr(caddr_t);
static int	xnf_stat(void *, uint_t, uint64_t *);
static boolean_t xnf_getcapab(void *, mac_capab_t, void *);
static int xnf_getprop(void *, const char *, mac_prop_id_t, uint_t, void *);
static int xnf_setprop(void *, const char *, mac_prop_id_t, uint_t,
    const void *);
static void xnf_propinfo(void *, const char *, mac_prop_id_t,
    mac_prop_info_handle_t);

/* Driver private functions */
static int xnf_alloc_dma_resources(xnf_t *);
static void xnf_release_dma_resources(xnf_t *);
static void xnf_release_mblks(xnf_t *);

static int xnf_buf_constructor(void *, void *, int);
static void xnf_buf_destructor(void *, void *);
static xnf_buf_t *xnf_buf_get(xnf_t *, int, boolean_t);
static void xnf_buf_put(xnf_t *, xnf_buf_t *, boolean_t);
static void xnf_buf_refresh(xnf_buf_t *);
static void xnf_buf_recycle(xnf_buf_t *);

static int xnf_tx_buf_constructor(void *, void *, int);
static void xnf_tx_buf_destructor(void *, void *);

static grant_ref_t xnf_gref_get(xnf_t *);
static void xnf_gref_put(xnf_t *, grant_ref_t);

static xnf_txid_t *xnf_txid_get(xnf_t *);
static void xnf_txid_put(xnf_t *, xnf_txid_t *);

static void xnf_rxbuf_hang(xnf_t *, xnf_buf_t *);
static int xnf_tx_clean_ring(xnf_t  *);
static void oe_state_change(dev_info_t *, ddi_eventcookie_t,
    void *, void *);
static boolean_t xnf_kstat_init(xnf_t *);
static void xnf_rx_collect(xnf_t *);

#define	XNF_CALLBACK_FLAGS	(MC_GETCAPAB | MC_PROPERTIES)

static mac_callbacks_t xnf_callbacks = {
	.mc_callbacks = XNF_CALLBACK_FLAGS,
	.mc_getstat = xnf_stat,
	.mc_start = xnf_start,
	.mc_stop = xnf_stop,
	.mc_setpromisc = xnf_set_promiscuous,
	.mc_multicst = xnf_set_multicast,
	.mc_unicst = xnf_set_mac_addr,
	.mc_tx = xnf_send,
	.mc_getcapab = xnf_getcapab,
	.mc_setprop = xnf_setprop,
	.mc_getprop = xnf_getprop,
	.mc_propinfo = xnf_propinfo,
};

/* DMA attributes for network ring buffer */
static ddi_dma_attr_t ringbuf_dma_attr = {
	.dma_attr_version = DMA_ATTR_V0,
	.dma_attr_addr_lo = 0,
	.dma_attr_addr_hi = 0xffffffffffffffffULL,
	.dma_attr_count_max = 0x7fffffff,
	.dma_attr_align = MMU_PAGESIZE,
	.dma_attr_burstsizes = 0x7ff,
	.dma_attr_minxfer = 1,
	.dma_attr_maxxfer = 0xffffffffU,
	.dma_attr_seg = 0xffffffffffffffffULL,
	.dma_attr_sgllen = 1,
	.dma_attr_granular = 1,
	.dma_attr_flags = 0
};

/* DMA attributes for receive data */
static ddi_dma_attr_t rx_buf_dma_attr = {
	.dma_attr_version = DMA_ATTR_V0,
	.dma_attr_addr_lo = 0,
	.dma_attr_addr_hi = 0xffffffffffffffffULL,
	.dma_attr_count_max = MMU_PAGEOFFSET,
	.dma_attr_align = MMU_PAGESIZE, /* allocation alignment */
	.dma_attr_burstsizes = 0x7ff,
	.dma_attr_minxfer = 1,
	.dma_attr_maxxfer = 0xffffffffU,
	.dma_attr_seg = 0xffffffffffffffffULL,
	.dma_attr_sgllen = 1,
	.dma_attr_granular = 1,
	.dma_attr_flags = 0
};

/* DMA attributes for transmit data */
static ddi_dma_attr_t tx_buf_dma_attr = {
	.dma_attr_version = DMA_ATTR_V0,
	.dma_attr_addr_lo = 0,
	.dma_attr_addr_hi = 0xffffffffffffffffULL,
	.dma_attr_count_max = MMU_PAGEOFFSET,
	.dma_attr_align = 1,
	.dma_attr_burstsizes = 0x7ff,
	.dma_attr_minxfer = 1,
	.dma_attr_maxxfer = 0xffffffffU,
	.dma_attr_seg = XEN_DATA_BOUNDARY - 1, /* segment boundary */
	.dma_attr_sgllen = XEN_MAX_TX_DATA_PAGES, /* max number of segments */
	.dma_attr_granular = 1,
	.dma_attr_flags = 0
};

/* DMA access attributes for registers and descriptors */
static ddi_device_acc_attr_t accattr = {
	DDI_DEVICE_ATTR_V0,
	DDI_STRUCTURE_LE_ACC,	/* This is a little-endian device */
	DDI_STRICTORDER_ACC
};

/* DMA access attributes for data: NOT to be byte swapped. */
static ddi_device_acc_attr_t data_accattr = {
	DDI_DEVICE_ATTR_V0,
	DDI_NEVERSWAP_ACC,
	DDI_STRICTORDER_ACC
};

DDI_DEFINE_STREAM_OPS(xnf_dev_ops, nulldev, nulldev, xnf_attach, xnf_detach,
    nodev, NULL, D_MP, NULL, ddi_quiesce_not_supported);

static struct modldrv xnf_modldrv = {
	&mod_driverops,
	"Virtual Ethernet driver",
	&xnf_dev_ops
};

static struct modlinkage modlinkage = {
	MODREV_1, &xnf_modldrv, NULL
};

int
_init(void)
{
	int r;

	mac_init_ops(&xnf_dev_ops, "xnf");
	r = mod_install(&modlinkage);
	if (r != DDI_SUCCESS)
		mac_fini_ops(&xnf_dev_ops);

	return (r);
}

int
_fini(void)
{
	return (EBUSY); /* XXPV should be removable */
}

int
_info(struct modinfo *modinfop)
{
	return (mod_info(&modlinkage, modinfop));
}

/*
 * Acquire a grant reference.
 */
static grant_ref_t
xnf_gref_get(xnf_t *xnfp)
{
	grant_ref_t gref;

	mutex_enter(&xnfp->xnf_gref_lock);

	do {
		gref = gnttab_claim_grant_reference(&xnfp->xnf_gref_head);

	} while ((gref == INVALID_GRANT_REF) &&
	    (gnttab_alloc_grant_references(16, &xnfp->xnf_gref_head) == 0));

	mutex_exit(&xnfp->xnf_gref_lock);

	if (gref == INVALID_GRANT_REF) {
		xnfp->xnf_stat_gref_failure++;
	} else {
		atomic_inc_64(&xnfp->xnf_stat_gref_outstanding);
		if (xnfp->xnf_stat_gref_outstanding > xnfp->xnf_stat_gref_peak)
			xnfp->xnf_stat_gref_peak =
			    xnfp->xnf_stat_gref_outstanding;
	}

	return (gref);
}

/*
 * Release a grant reference.
 */
static void
xnf_gref_put(xnf_t *xnfp, grant_ref_t gref)
{
	ASSERT(gref != INVALID_GRANT_REF);

	mutex_enter(&xnfp->xnf_gref_lock);
	gnttab_release_grant_reference(&xnfp->xnf_gref_head, gref);
	mutex_exit(&xnfp->xnf_gref_lock);

	atomic_dec_64(&xnfp->xnf_stat_gref_outstanding);
}

/*
 * Acquire a transmit id.
 */
static xnf_txid_t *
xnf_txid_get(xnf_t *xnfp)
{
	xnf_txid_t *tidp;

	ASSERT(MUTEX_HELD(&xnfp->xnf_txlock));

	if (xnfp->xnf_tx_pkt_id_head == INVALID_TX_ID)
		return (NULL);

	ASSERT(TX_ID_VALID(xnfp->xnf_tx_pkt_id_head));

	tidp = TX_ID_TO_TXID(xnfp, xnfp->xnf_tx_pkt_id_head);
	xnfp->xnf_tx_pkt_id_head = tidp->next;
	tidp->next = INVALID_TX_ID;

	ASSERT(tidp->txbuf == NULL);

	return (tidp);
}

/*
 * Release a transmit id.
 */
static void
xnf_txid_put(xnf_t *xnfp, xnf_txid_t *tidp)
{
	ASSERT(MUTEX_HELD(&xnfp->xnf_txlock));
	ASSERT(TX_ID_VALID(tidp->id));
	ASSERT(tidp->next == INVALID_TX_ID);

	tidp->txbuf = NULL;
	tidp->next = xnfp->xnf_tx_pkt_id_head;
	xnfp->xnf_tx_pkt_id_head = tidp->id;
}

static void
xnf_data_txbuf_free(xnf_t *xnfp, xnf_txbuf_t *txp)
{
	ASSERT3U(txp->tx_type, ==, TX_DATA);

	/*
	 * We are either using a lookaside buffer or we are mapping existing
	 * buffers.
	 */
	if (txp->tx_bdesc != NULL) {
		ASSERT(!txp->tx_handle_bound);
		xnf_buf_put(xnfp, txp->tx_bdesc, B_TRUE);
	} else {
		if (txp->tx_txreq.gref != INVALID_GRANT_REF) {
			if (gnttab_query_foreign_access(txp->tx_txreq.gref) !=
			    0) {
				cmn_err(CE_PANIC, "tx grant %d still in use by "
				    "backend domain", txp->tx_txreq.gref);
			}
			(void) gnttab_end_foreign_access_ref(
			    txp->tx_txreq.gref, 1);
			xnf_gref_put(xnfp, txp->tx_txreq.gref);
		}

		if (txp->tx_handle_bound)
			(void) ddi_dma_unbind_handle(txp->tx_dma_handle);
	}

	if (txp->tx_mp != NULL)
		freemsg(txp->tx_mp);

	if (txp->tx_prev != NULL) {
		ASSERT3P(txp->tx_prev->tx_next, ==, txp);
		txp->tx_prev->tx_next = NULL;
	}

	if (txp->tx_txreq.id != INVALID_TX_ID) {
		/*
		 * This should be only possible when resuming from a suspend.
		 */
		ASSERT(!xnfp->xnf_connected);
		xnf_txid_put(xnfp, TX_ID_TO_TXID(xnfp, txp->tx_txreq.id));
		txp->tx_txreq.id = INVALID_TX_ID;
	}

	kmem_cache_free(xnfp->xnf_tx_buf_cache, txp);
}

static void
xnf_data_txbuf_free_chain(xnf_t *xnfp, xnf_txbuf_t *txp)
{
	if (txp == NULL)
		return;

	while (txp->tx_next != NULL)
		txp = txp->tx_next;

	/*
	 * We free the chain in reverse order so that grants can be released
	 * for all dma chunks before unbinding the dma handles. The mblk is
	 * freed last, after all its fragments' dma handles are unbound.
	 */
	xnf_txbuf_t *prev;
	for (; txp != NULL; txp = prev) {
		prev = txp->tx_prev;
		xnf_data_txbuf_free(xnfp, txp);
	}
}

static xnf_txbuf_t *
xnf_data_txbuf_alloc(xnf_t *xnfp, int flag)
{
	xnf_txbuf_t *txp;

	if ((txp = kmem_cache_alloc(xnfp->xnf_tx_buf_cache, flag)) == NULL) {
		return (NULL);
	}

	txp->tx_type = TX_DATA;
	txp->tx_next = NULL;
	txp->tx_prev = NULL;
	txp->tx_head = txp;
	txp->tx_frags_to_ack = 0;
	txp->tx_mp = NULL;
	txp->tx_bdesc = NULL;
	txp->tx_handle_bound = B_FALSE;
	txp->tx_txreq.gref = INVALID_GRANT_REF;
	txp->tx_txreq.id = INVALID_TX_ID;

	return (txp);
}

/*
 * Get `wanted' slots in the transmit ring, waiting for at least that
 * number if `wait' is B_TRUE. Force the ring to be cleaned by setting
 * `wanted' to zero.
 *
 * Return the number of slots available.
 */
static int
xnf_tx_slots_get(xnf_t *xnfp, int wanted, boolean_t wait)
{
	int slotsfree;
	boolean_t forced_clean = (wanted == 0);

	ASSERT(MUTEX_HELD(&xnfp->xnf_txlock));

	/* LINTED: constant in conditional context */
	while (B_TRUE) {
		slotsfree = RING_FREE_REQUESTS(&xnfp->xnf_tx_ring);

		if ((slotsfree < wanted) || forced_clean)
			slotsfree = xnf_tx_clean_ring(xnfp);

		/*
		 * If there are more than we need free, tell other
		 * people to come looking again. We hold txlock, so we
		 * are able to take our slots before anyone else runs.
		 */
		if (slotsfree > wanted)
			cv_broadcast(&xnfp->xnf_cv_tx_slots);

		if (slotsfree >= wanted)
			break;

		if (!wait)
			break;

		cv_wait(&xnfp->xnf_cv_tx_slots, &xnfp->xnf_txlock);
	}

	ASSERT(slotsfree <= RING_SIZE(&(xnfp->xnf_tx_ring)));

	return (slotsfree);
}

static int
xnf_setup_rings(xnf_t *xnfp)
{
	domid_t			oeid;
	struct xenbus_device	*xsd;
	RING_IDX		i;
	int			err;
	xnf_txid_t		*tidp;
	xnf_buf_t **bdescp;

	oeid = xvdi_get_oeid(xnfp->xnf_devinfo);
	xsd = xvdi_get_xsd(xnfp->xnf_devinfo);

	if (xnfp->xnf_tx_ring_ref != INVALID_GRANT_REF)
		gnttab_end_foreign_access(xnfp->xnf_tx_ring_ref, 0, 0);

	err = gnttab_grant_foreign_access(oeid,
	    xnf_btop(pa_to_ma(xnfp->xnf_tx_ring_phys_addr)), 0);
	if (err <= 0) {
		err = -err;
		xenbus_dev_error(xsd, err, "granting access to tx ring page");
		goto out;
	}
	xnfp->xnf_tx_ring_ref = (grant_ref_t)err;

	if (xnfp->xnf_rx_ring_ref != INVALID_GRANT_REF)
		gnttab_end_foreign_access(xnfp->xnf_rx_ring_ref, 0, 0);

	err = gnttab_grant_foreign_access(oeid,
	    xnf_btop(pa_to_ma(xnfp->xnf_rx_ring_phys_addr)), 0);
	if (err <= 0) {
		err = -err;
		xenbus_dev_error(xsd, err, "granting access to rx ring page");
		goto out;
	}
	xnfp->xnf_rx_ring_ref = (grant_ref_t)err;

	mutex_enter(&xnfp->xnf_txlock);

	/*
	 * We first cleanup the TX ring in case we are doing a resume.
	 * Note that this can lose packets, but we expect to stagger on.
	 */
	xnfp->xnf_tx_pkt_id_head = INVALID_TX_ID; /* I.e. emtpy list. */
	for (i = 0, tidp = &xnfp->xnf_tx_pkt_id[0];
	    i < NET_TX_RING_SIZE;
	    i++, tidp++) {
		xnf_txbuf_t *txp = tidp->txbuf;
		if (txp == NULL)
			continue;

		switch (txp->tx_type) {
		case TX_DATA:
			/*
			 * txid_put() will be called for each txbuf's txid in
			 * the chain which will result in clearing tidp->txbuf.
			 */
			xnf_data_txbuf_free_chain(xnfp, txp);

			break;

		case TX_MCAST_REQ:
			txp->tx_type = TX_MCAST_RSP;
			txp->tx_status = NETIF_RSP_DROPPED;
			cv_broadcast(&xnfp->xnf_cv_multicast);

			/*
			 * The request consumed two slots in the ring,
			 * yet only a single xnf_txid_t is used. Step
			 * over the empty slot.
			 */
			i++;
			ASSERT3U(i, <, NET_TX_RING_SIZE);
			break;

		case TX_MCAST_RSP:
			break;
		}
	}

	/*
	 * Now purge old list and add each txid to the new free list.
	 */
	xnfp->xnf_tx_pkt_id_head = INVALID_TX_ID; /* I.e. emtpy list. */
	for (i = 0, tidp = &xnfp->xnf_tx_pkt_id[0];
	    i < NET_TX_RING_SIZE;
	    i++, tidp++) {
		tidp->id = i;
		ASSERT3P(tidp->txbuf, ==, NULL);
		tidp->next = INVALID_TX_ID; /* Appease txid_put(). */
		xnf_txid_put(xnfp, tidp);
	}

	/* LINTED: constant in conditional context */
	SHARED_RING_INIT(xnfp->xnf_tx_ring.sring);
	/* LINTED: constant in conditional context */
	FRONT_RING_INIT(&xnfp->xnf_tx_ring,
	    xnfp->xnf_tx_ring.sring, PAGESIZE);

	mutex_exit(&xnfp->xnf_txlock);

	mutex_enter(&xnfp->xnf_rxlock);

	/*
	 * Clean out any buffers currently posted to the receive ring
	 * before we reset it.
	 */
	for (i = 0, bdescp = &xnfp->xnf_rx_pkt_info[0];
	    i < NET_RX_RING_SIZE;
	    i++, bdescp++) {
		if (*bdescp != NULL) {
			xnf_buf_put(xnfp, *bdescp, B_FALSE);
			*bdescp = NULL;
		}
	}

	/* LINTED: constant in conditional context */
	SHARED_RING_INIT(xnfp->xnf_rx_ring.sring);
	/* LINTED: constant in conditional context */
	FRONT_RING_INIT(&xnfp->xnf_rx_ring,
	    xnfp->xnf_rx_ring.sring, PAGESIZE);

	/*
	 * Fill the ring with buffers.
	 */
	for (i = 0; i < NET_RX_RING_SIZE; i++) {
		xnf_buf_t *bdesc;

		bdesc = xnf_buf_get(xnfp, KM_SLEEP, B_FALSE);
		VERIFY(bdesc != NULL);
		xnf_rxbuf_hang(xnfp, bdesc);
	}

	/* LINTED: constant in conditional context */
	RING_PUSH_REQUESTS(&xnfp->xnf_rx_ring);

	mutex_exit(&xnfp->xnf_rxlock);

	return (0);

out:
	if (xnfp->xnf_tx_ring_ref != INVALID_GRANT_REF)
		gnttab_end_foreign_access(xnfp->xnf_tx_ring_ref, 0, 0);
	xnfp->xnf_tx_ring_ref = INVALID_GRANT_REF;

	if (xnfp->xnf_rx_ring_ref != INVALID_GRANT_REF)
		gnttab_end_foreign_access(xnfp->xnf_rx_ring_ref, 0, 0);
	xnfp->xnf_rx_ring_ref = INVALID_GRANT_REF;

	return (err);
}

/*
 * Connect driver to back end, called to set up communication with
 * back end driver both initially and on resume after restore/migrate.
 */
void
xnf_be_connect(xnf_t *xnfp)
{
	const char	*message;
	xenbus_transaction_t xbt;
	struct		xenbus_device *xsd;
	char		*xsname;
	int		err;

	ASSERT(!xnfp->xnf_connected);

	xsd = xvdi_get_xsd(xnfp->xnf_devinfo);
	xsname = xvdi_get_xsname(xnfp->xnf_devinfo);

	err = xnf_setup_rings(xnfp);
	if (err != 0) {
		cmn_err(CE_WARN, "failed to set up tx/rx rings");
		xenbus_dev_error(xsd, err, "setting up ring");
		return;
	}

again:
	err = xenbus_transaction_start(&xbt);
	if (err != 0) {
		xenbus_dev_error(xsd, EIO, "starting transaction");
		return;
	}

	err = xenbus_printf(xbt, xsname, "tx-ring-ref", "%u",
	    xnfp->xnf_tx_ring_ref);
	if (err != 0) {
		message = "writing tx ring-ref";
		goto abort_transaction;
	}

	err = xenbus_printf(xbt, xsname, "rx-ring-ref", "%u",
	    xnfp->xnf_rx_ring_ref);
	if (err != 0) {
		message = "writing rx ring-ref";
		goto abort_transaction;
	}

	err = xenbus_printf(xbt, xsname, "event-channel", "%u",
	    xnfp->xnf_evtchn);
	if (err != 0) {
		message = "writing event-channel";
		goto abort_transaction;
	}

	err = xenbus_printf(xbt, xsname, "feature-rx-notify", "%d", 1);
	if (err != 0) {
		message = "writing feature-rx-notify";
		goto abort_transaction;
	}

	err = xenbus_printf(xbt, xsname, "request-rx-copy", "%d", 1);
	if (err != 0) {
		message = "writing request-rx-copy";
		goto abort_transaction;
	}

	if (xnfp->xnf_be_mcast_control) {
		err = xenbus_printf(xbt, xsname, "request-multicast-control",
		    "%d", 1);
		if (err != 0) {
			message = "writing request-multicast-control";
			goto abort_transaction;
		}
	}

	/*
	 * Tell backend if we support scatter-gather lists on the rx side.
	 */
	err = xenbus_printf(xbt, xsname, "feature-sg", "%d",
	    xnf_enable_rx_sg ? 1 : 0);
	if (err != 0) {
		message = "writing feature-sg";
		goto abort_transaction;
	}

	/*
	 * Tell backend if we support LRO for IPv4. Scatter-gather on rx is
	 * a prerequisite.
	 */
	err = xenbus_printf(xbt, xsname, "feature-gso-tcpv4", "%d",
	    (xnf_enable_rx_sg && xnf_enable_lro) ? 1 : 0);
	if (err != 0) {
		message = "writing feature-gso-tcpv4";
		goto abort_transaction;
	}

	err = xvdi_switch_state(xnfp->xnf_devinfo, xbt, XenbusStateConnected);
	if (err != 0) {
		message = "switching state to XenbusStateConnected";
		goto abort_transaction;
	}

	err = xenbus_transaction_end(xbt, 0);
	if (err != 0) {
		if (err == EAGAIN)
			goto again;
		xenbus_dev_error(xsd, err, "completing transaction");
	}

	return;

abort_transaction:
	(void) xenbus_transaction_end(xbt, 1);
	xenbus_dev_error(xsd, err, "%s", message);
}

/*
 * Read configuration information from xenstore.
 */
void
xnf_read_config(xnf_t *xnfp)
{
	int err, be_cap;
	char mac[ETHERADDRL * 3];
	char *oename = xvdi_get_oename(xnfp->xnf_devinfo);

	err = xenbus_scanf(XBT_NULL, oename, "mac",
	    "%s", (char *)&mac[0]);
	if (err != 0) {
		/*
		 * bad: we're supposed to be set up with a proper mac
		 * addr. at this point
		 */
		cmn_err(CE_WARN, "%s%d: no mac address",
		    ddi_driver_name(xnfp->xnf_devinfo),
		    ddi_get_instance(xnfp->xnf_devinfo));
			return;
	}
	if (ether_aton(mac, xnfp->xnf_mac_addr) != ETHERADDRL) {
		err = ENOENT;
		xenbus_dev_error(xvdi_get_xsd(xnfp->xnf_devinfo), ENOENT,
		    "parsing %s/mac", xvdi_get_xsname(xnfp->xnf_devinfo));
		return;
	}

	err = xenbus_scanf(XBT_NULL, oename,
	    "feature-rx-copy", "%d", &be_cap);
	/*
	 * If we fail to read the store we assume that the key is
	 * absent, implying an older domain at the far end.  Older
	 * domains cannot do HV copy.
	 */
	if (err != 0)
		be_cap = 0;
	xnfp->xnf_be_rx_copy = (be_cap != 0);

	err = xenbus_scanf(XBT_NULL, oename,
	    "feature-multicast-control", "%d", &be_cap);
	/*
	 * If we fail to read the store we assume that the key is
	 * absent, implying an older domain at the far end.  Older
	 * domains do not support multicast control.
	 */
	if (err != 0)
		be_cap = 0;
	xnfp->xnf_be_mcast_control = (be_cap != 0) && xnf_multicast_control;

	/*
	 * See if back-end supports scatter-gather for transmits. If not,
	 * we will not support LSO and limit the mtu to 1500.
	 */
	err = xenbus_scanf(XBT_NULL, oename, "feature-sg", "%d", &be_cap);
	if (err != 0) {
		be_cap = 0;
		dev_err(xnfp->xnf_devinfo, CE_WARN, "error reading "
		    "'feature-sg' from backend driver");
	}
	if (be_cap == 0) {
		dev_err(xnfp->xnf_devinfo, CE_WARN, "scatter-gather is not "
		    "supported for transmits in the backend driver. LSO is "
		    "disabled and MTU is restricted to 1500 bytes.");
	}
	xnfp->xnf_be_tx_sg = (be_cap != 0) && xnf_enable_tx_sg;

	if (xnfp->xnf_be_tx_sg) {
		/*
		 * Check if LSO is supported. Currently we only check for
		 * IPv4 as Illumos doesn't support LSO for IPv6.
		 */
		err = xenbus_scanf(XBT_NULL, oename, "feature-gso-tcpv4", "%d",
		    &be_cap);
		if (err != 0) {
			be_cap = 0;
			dev_err(xnfp->xnf_devinfo, CE_WARN, "error reading "
			    "'feature-gso-tcpv4' from backend driver");
		}
		if (be_cap == 0) {
			dev_err(xnfp->xnf_devinfo, CE_WARN, "LSO is not "
			    "supported by the backend driver. Performance "
			    "will be affected.");
		}
		xnfp->xnf_be_lso = (be_cap != 0) && xnf_enable_lso;
	}
}

/*
 *  attach(9E) -- Attach a device to the system
 */
static int
xnf_attach(dev_info_t *devinfo, ddi_attach_cmd_t cmd)
{
	mac_register_t *macp;
	xnf_t *xnfp;
	int err;
	char cachename[32];

	switch (cmd) {
	case DDI_RESUME:
		xnfp = ddi_get_driver_private(devinfo);
		xnfp->xnf_gen++;

		(void) xvdi_resume(devinfo);
		(void) xvdi_alloc_evtchn(devinfo);
		xnfp->xnf_evtchn = xvdi_get_evtchn(devinfo);
#ifdef XPV_HVM_DRIVER
		ec_bind_evtchn_to_handler(xnfp->xnf_evtchn, IPL_VIF, xnf_intr,
		    xnfp);
#else
		(void) ddi_add_intr(devinfo, 0, NULL, NULL, xnf_intr,
		    (caddr_t)xnfp);
#endif
		return (DDI_SUCCESS);

	case DDI_ATTACH:
		break;

	default:
		return (DDI_FAILURE);
	}

	/*
	 *  Allocate gld_mac_info_t and xnf_instance structures
	 */
	macp = mac_alloc(MAC_VERSION);
	if (macp == NULL)
		return (DDI_FAILURE);
	xnfp = kmem_zalloc(sizeof (*xnfp), KM_SLEEP);

	xnfp->xnf_tx_pkt_id =
	    kmem_zalloc(sizeof (xnf_txid_t) * NET_TX_RING_SIZE, KM_SLEEP);

	xnfp->xnf_rx_pkt_info =
	    kmem_zalloc(sizeof (xnf_buf_t *) * NET_RX_RING_SIZE, KM_SLEEP);

	macp->m_dip = devinfo;
	macp->m_driver = xnfp;
	xnfp->xnf_devinfo = devinfo;

	macp->m_type_ident = MAC_PLUGIN_IDENT_ETHER;
	macp->m_src_addr = xnfp->xnf_mac_addr;
	macp->m_callbacks = &xnf_callbacks;
	macp->m_min_sdu = 0;
	xnfp->xnf_mtu = ETHERMTU;
	macp->m_max_sdu = xnfp->xnf_mtu;

	xnfp->xnf_running = B_FALSE;
	xnfp->xnf_connected = B_FALSE;
	xnfp->xnf_be_rx_copy = B_FALSE;
	xnfp->xnf_be_mcast_control = B_FALSE;
	xnfp->xnf_need_sched = B_FALSE;

	xnfp->xnf_rx_head = NULL;
	xnfp->xnf_rx_tail = NULL;
	xnfp->xnf_rx_new_buffers_posted = B_FALSE;

#ifdef XPV_HVM_DRIVER
	/* Report our version to dom0 */
	(void) xenbus_printf(XBT_NULL, "guest/xnf", "version", "%d",
	    HVMPV_XNF_VERS);
#endif

	/*
	 * Get the iblock cookie with which to initialize the mutexes.
	 */
	if (ddi_get_iblock_cookie(devinfo, 0, &xnfp->xnf_icookie)
	    != DDI_SUCCESS)
		goto failure;

	mutex_init(&xnfp->xnf_txlock,
	    NULL, MUTEX_DRIVER, xnfp->xnf_icookie);
	mutex_init(&xnfp->xnf_rxlock,
	    NULL, MUTEX_DRIVER, xnfp->xnf_icookie);
	mutex_init(&xnfp->xnf_schedlock,
	    NULL, MUTEX_DRIVER, xnfp->xnf_icookie);
	mutex_init(&xnfp->xnf_gref_lock,
	    NULL, MUTEX_DRIVER, xnfp->xnf_icookie);

	cv_init(&xnfp->xnf_cv_state, NULL, CV_DEFAULT, NULL);
	cv_init(&xnfp->xnf_cv_multicast, NULL, CV_DEFAULT, NULL);
	cv_init(&xnfp->xnf_cv_tx_slots, NULL, CV_DEFAULT, NULL);

	(void) sprintf(cachename, "xnf_buf_cache_%d",
	    ddi_get_instance(devinfo));
	xnfp->xnf_buf_cache = kmem_cache_create(cachename,
	    sizeof (xnf_buf_t), 0,
	    xnf_buf_constructor, xnf_buf_destructor,
	    NULL, xnfp, NULL, 0);
	if (xnfp->xnf_buf_cache == NULL)
		goto failure_0;

	(void) sprintf(cachename, "xnf_tx_buf_cache_%d",
	    ddi_get_instance(devinfo));
	xnfp->xnf_tx_buf_cache = kmem_cache_create(cachename,
	    sizeof (xnf_txbuf_t), 0,
	    xnf_tx_buf_constructor, xnf_tx_buf_destructor,
	    NULL, xnfp, NULL, 0);
	if (xnfp->xnf_tx_buf_cache == NULL)
		goto failure_1;

	xnfp->xnf_gref_head = INVALID_GRANT_REF;

	if (xnf_alloc_dma_resources(xnfp) == DDI_FAILURE) {
		cmn_err(CE_WARN, "xnf%d: failed to allocate and initialize "
		    "driver data structures",
		    ddi_get_instance(xnfp->xnf_devinfo));
		goto failure_2;
	}

	xnfp->xnf_rx_ring.sring->rsp_event =
	    xnfp->xnf_tx_ring.sring->rsp_event = 1;

	xnfp->xnf_tx_ring_ref = INVALID_GRANT_REF;
	xnfp->xnf_rx_ring_ref = INVALID_GRANT_REF;

	/* set driver private pointer now */
	ddi_set_driver_private(devinfo, xnfp);

	if (!xnf_kstat_init(xnfp))
		goto failure_3;

	/*
	 * Allocate an event channel, add the interrupt handler and
	 * bind it to the event channel.
	 */
	(void) xvdi_alloc_evtchn(devinfo);
	xnfp->xnf_evtchn = xvdi_get_evtchn(devinfo);
#ifdef XPV_HVM_DRIVER
	ec_bind_evtchn_to_handler(xnfp->xnf_evtchn, IPL_VIF, xnf_intr, xnfp);
#else
	(void) ddi_add_intr(devinfo, 0, NULL, NULL, xnf_intr, (caddr_t)xnfp);
#endif

	err = mac_register(macp, &xnfp->xnf_mh);
	mac_free(macp);
	macp = NULL;
	if (err != 0)
		goto failure_4;

	if (xvdi_add_event_handler(devinfo, XS_OE_STATE, oe_state_change, NULL)
	    != DDI_SUCCESS)
		goto failure_5;

#ifdef XPV_HVM_DRIVER
	/*
	 * In the HVM case, this driver essentially replaces a driver for
	 * a 'real' PCI NIC. Without the "model" property set to
	 * "Ethernet controller", like the PCI code does, netbooting does
	 * not work correctly, as strplumb_get_netdev_path() will not find
	 * this interface.
	 */
	(void) ndi_prop_update_string(DDI_DEV_T_NONE, devinfo, "model",
	    "Ethernet controller");
#endif

	return (DDI_SUCCESS);

failure_5:
	(void) mac_unregister(xnfp->xnf_mh);

failure_4:
#ifdef XPV_HVM_DRIVER
	ec_unbind_evtchn(xnfp->xnf_evtchn);
	xvdi_free_evtchn(devinfo);
#else
	ddi_remove_intr(devinfo, 0, xnfp->xnf_icookie);
#endif
	xnfp->xnf_evtchn = INVALID_EVTCHN;
	kstat_delete(xnfp->xnf_kstat_aux);

failure_3:
	xnf_release_dma_resources(xnfp);

failure_2:
	kmem_cache_destroy(xnfp->xnf_tx_buf_cache);

failure_1:
	kmem_cache_destroy(xnfp->xnf_buf_cache);

failure_0:
	cv_destroy(&xnfp->xnf_cv_tx_slots);
	cv_destroy(&xnfp->xnf_cv_multicast);
	cv_destroy(&xnfp->xnf_cv_state);

	mutex_destroy(&xnfp->xnf_gref_lock);
	mutex_destroy(&xnfp->xnf_schedlock);
	mutex_destroy(&xnfp->xnf_rxlock);
	mutex_destroy(&xnfp->xnf_txlock);

failure:
	kmem_free(xnfp, sizeof (*xnfp));
	if (macp != NULL)
		mac_free(macp);

	return (DDI_FAILURE);
}

/*  detach(9E) -- Detach a device from the system */
static int
xnf_detach(dev_info_t *devinfo, ddi_detach_cmd_t cmd)
{
	xnf_t *xnfp;		/* Our private device info */

	xnfp = ddi_get_driver_private(devinfo);

	switch (cmd) {
	case DDI_SUSPEND:
#ifdef XPV_HVM_DRIVER
		ec_unbind_evtchn(xnfp->xnf_evtchn);
		xvdi_free_evtchn(devinfo);
#else
		ddi_remove_intr(devinfo, 0, xnfp->xnf_icookie);
#endif

		xvdi_suspend(devinfo);

		mutex_enter(&xnfp->xnf_rxlock);
		mutex_enter(&xnfp->xnf_txlock);

		xnfp->xnf_evtchn = INVALID_EVTCHN;
		xnfp->xnf_connected = B_FALSE;
		mutex_exit(&xnfp->xnf_txlock);
		mutex_exit(&xnfp->xnf_rxlock);

		/* claim link to be down after disconnect */
		mac_link_update(xnfp->xnf_mh, LINK_STATE_DOWN);
		return (DDI_SUCCESS);

	case DDI_DETACH:
		break;

	default:
		return (DDI_FAILURE);
	}

	if (xnfp->xnf_connected)
		return (DDI_FAILURE);

	/*
	 * Cannot detach if we have xnf_buf_t outstanding.
	 */
	if (xnfp->xnf_stat_buf_allocated > 0)
		return (DDI_FAILURE);

	if (mac_unregister(xnfp->xnf_mh) != 0)
		return (DDI_FAILURE);

	kstat_delete(xnfp->xnf_kstat_aux);

	/* Stop the receiver */
	xnf_stop(xnfp);

	xvdi_remove_event_handler(devinfo, XS_OE_STATE);

	/* Remove the interrupt */
#ifdef XPV_HVM_DRIVER
	ec_unbind_evtchn(xnfp->xnf_evtchn);
	xvdi_free_evtchn(devinfo);
#else
	ddi_remove_intr(devinfo, 0, xnfp->xnf_icookie);
#endif

	/* Release any pending xmit mblks */
	xnf_release_mblks(xnfp);

	/* Release all DMA resources */
	xnf_release_dma_resources(xnfp);

	cv_destroy(&xnfp->xnf_cv_tx_slots);
	cv_destroy(&xnfp->xnf_cv_multicast);
	cv_destroy(&xnfp->xnf_cv_state);

	kmem_cache_destroy(xnfp->xnf_tx_buf_cache);
	kmem_cache_destroy(xnfp->xnf_buf_cache);

	mutex_destroy(&xnfp->xnf_gref_lock);
	mutex_destroy(&xnfp->xnf_schedlock);
	mutex_destroy(&xnfp->xnf_rxlock);
	mutex_destroy(&xnfp->xnf_txlock);

	kmem_free(xnfp, sizeof (*xnfp));

	return (DDI_SUCCESS);
}

/*
 *  xnf_set_mac_addr() -- set the physical network address on the board.
 */
static int
xnf_set_mac_addr(void *arg, const uint8_t *macaddr)
{
	_NOTE(ARGUNUSED(arg, macaddr));

	/*
	 * We can't set our macaddr.
	 */
	return (ENOTSUP);
}

/*
 *  xnf_set_multicast() -- set (enable) or disable a multicast address.
 *
 *  Program the hardware to enable/disable the multicast address
 *  in "mca".  Enable if "add" is true, disable if false.
 */
static int
xnf_set_multicast(void *arg, boolean_t add, const uint8_t *mca)
{
	xnf_t *xnfp = arg;
	xnf_txbuf_t *txp;
	int n_slots;
	RING_IDX slot;
	xnf_txid_t *tidp;
	netif_tx_request_t *txrp;
	struct netif_extra_info *erp;
	boolean_t notify, result;

	/*
	 * If the backend does not support multicast control then we
	 * must assume that the right packets will just arrive.
	 */
	if (!xnfp->xnf_be_mcast_control)
		return (0);

	txp = kmem_cache_alloc(xnfp->xnf_tx_buf_cache, KM_SLEEP);

	mutex_enter(&xnfp->xnf_txlock);

	/*
	 * If we're not yet connected then claim success. This is
	 * acceptable because we refresh the entire set of multicast
	 * addresses when we get connected.
	 *
	 * We can't wait around here because the MAC layer expects
	 * this to be a non-blocking operation - waiting ends up
	 * causing a deadlock during resume.
	 */
	if (!xnfp->xnf_connected) {
		mutex_exit(&xnfp->xnf_txlock);
		return (0);
	}

	/*
	 * 1. Acquire two slots in the ring.
	 * 2. Fill in the slots.
	 * 3. Request notification when the operation is done.
	 * 4. Kick the peer.
	 * 5. Wait for the response via xnf_tx_clean_ring().
	 */

	n_slots = xnf_tx_slots_get(xnfp, 2, B_TRUE);
	ASSERT(n_slots >= 2);

	slot = xnfp->xnf_tx_ring.req_prod_pvt;
	tidp = xnf_txid_get(xnfp);
	VERIFY(tidp != NULL);

	txp->tx_type = TX_MCAST_REQ;
	txp->tx_slot = slot;

	txrp = RING_GET_REQUEST(&xnfp->xnf_tx_ring, slot);
	erp = (struct netif_extra_info *)
	    RING_GET_REQUEST(&xnfp->xnf_tx_ring, slot + 1);

	txrp->gref = 0;
	txrp->size = 0;
	txrp->offset = 0;
	/* Set tx_txreq.id to appease xnf_tx_clean_ring(). */
	txrp->id = txp->tx_txreq.id = tidp->id;
	txrp->flags = NETTXF_extra_info;

	erp->type = add ? XEN_NETIF_EXTRA_TYPE_MCAST_ADD :
	    XEN_NETIF_EXTRA_TYPE_MCAST_DEL;
	bcopy((void *)mca, &erp->u.mcast.addr, ETHERADDRL);

	tidp->txbuf = txp;

	xnfp->xnf_tx_ring.req_prod_pvt = slot + 2;

	mutex_enter(&xnfp->xnf_schedlock);
	xnfp->xnf_pending_multicast++;
	mutex_exit(&xnfp->xnf_schedlock);

	/* LINTED: constant in conditional context */
	RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&xnfp->xnf_tx_ring,
	    notify);
	if (notify)
		ec_notify_via_evtchn(xnfp->xnf_evtchn);

	while (txp->tx_type == TX_MCAST_REQ)
		cv_wait(&xnfp->xnf_cv_multicast, &xnfp->xnf_txlock);

	ASSERT3U(txp->tx_type, ==, TX_MCAST_RSP);

	mutex_enter(&xnfp->xnf_schedlock);
	xnfp->xnf_pending_multicast--;
	mutex_exit(&xnfp->xnf_schedlock);

	result = (txp->tx_status == NETIF_RSP_OKAY);

	xnf_txid_put(xnfp, tidp);

	mutex_exit(&xnfp->xnf_txlock);

	kmem_cache_free(xnfp->xnf_tx_buf_cache, txp);

	return (result ? 0 : 1);
}

/*
 * xnf_set_promiscuous() -- set or reset promiscuous mode on the board
 *
 *  Program the hardware to enable/disable promiscuous mode.
 */
static int
xnf_set_promiscuous(void *arg, boolean_t on)
{
	_NOTE(ARGUNUSED(arg, on));

	/*
	 * We can't really do this, but we pretend that we can in
	 * order that snoop will work.
	 */
	return (0);
}

/*
 * Clean buffers that we have responses for from the transmit ring.
 */
static int
xnf_tx_clean_ring(xnf_t *xnfp)
{
	boolean_t work_to_do;

	ASSERT(MUTEX_HELD(&xnfp->xnf_txlock));

loop:
	while (RING_HAS_UNCONSUMED_RESPONSES(&xnfp->xnf_tx_ring)) {
		RING_IDX cons, prod, i;

		cons = xnfp->xnf_tx_ring.rsp_cons;
		prod = xnfp->xnf_tx_ring.sring->rsp_prod;
		membar_consumer();
		/*
		 * Clean tx requests from ring that we have responses
		 * for.
		 */
		DTRACE_PROBE2(xnf_tx_clean_range, int, cons, int, prod);
		for (i = cons; i != prod; i++) {
			netif_tx_response_t *trp;
			xnf_txid_t *tidp;
			xnf_txbuf_t *txp;

			trp = RING_GET_RESPONSE(&xnfp->xnf_tx_ring, i);
			/*
			 * if this slot was occupied by netif_extra_info_t,
			 * then the response will be NETIF_RSP_NULL. In this
			 * case there are no resources to clean up.
			 */
			if (trp->status == NETIF_RSP_NULL)
				continue;

			ASSERT(TX_ID_VALID(trp->id));

			tidp = TX_ID_TO_TXID(xnfp, trp->id);
			ASSERT3U(tidp->id, ==, trp->id);
			ASSERT3U(tidp->next, ==, INVALID_TX_ID);

			txp = tidp->txbuf;
			ASSERT(txp != NULL);
			ASSERT3U(txp->tx_txreq.id, ==, trp->id);

			switch (txp->tx_type) {
			case TX_DATA:
				/*
				 * We must put the txid for each response we
				 * acknowledge to make sure that we never have
				 * more free slots than txids. Because of this
				 * we do it here instead of waiting for it to
				 * be done in xnf_data_txbuf_free_chain().
				 */
				xnf_txid_put(xnfp, tidp);
				txp->tx_txreq.id = INVALID_TX_ID;
				ASSERT3S(txp->tx_head->tx_frags_to_ack, >, 0);
				txp->tx_head->tx_frags_to_ack--;

				/*
				 * We clean the whole chain once we got a
				 * response for each fragment.
				 */
				if (txp->tx_head->tx_frags_to_ack == 0)
					xnf_data_txbuf_free_chain(xnfp, txp);

				break;

			case TX_MCAST_REQ:
				txp->tx_type = TX_MCAST_RSP;
				txp->tx_status = trp->status;
				cv_broadcast(&xnfp->xnf_cv_multicast);

				break;

			default:
				cmn_err(CE_PANIC, "xnf_tx_clean_ring: "
				    "invalid xnf_txbuf_t type: %d",
				    txp->tx_type);
				break;
			}
		}
		/*
		 * Record the last response we dealt with so that we
		 * know where to start next time around.
		 */
		xnfp->xnf_tx_ring.rsp_cons = prod;
		membar_enter();
	}

	/* LINTED: constant in conditional context */
	RING_FINAL_CHECK_FOR_RESPONSES(&xnfp->xnf_tx_ring, work_to_do);
	if (work_to_do)
		goto loop;

	return (RING_FREE_REQUESTS(&xnfp->xnf_tx_ring));
}

/*
 * Allocate and fill in a look-aside buffer for the packet `mp'. Used
 * to ensure that the packet is physically contiguous and contained
 * within a single page.
 */
static xnf_buf_t *
xnf_tx_get_lookaside(xnf_t *xnfp, mblk_t *mp, size_t *plen)
{
	xnf_buf_t *bd;
	caddr_t bp;

	if ((bd = xnf_buf_get(xnfp, KM_NOSLEEP, B_TRUE)) == NULL) {
		return (NULL);
	}

	bp = bd->buf;
	while (mp != NULL) {
		size_t len = MBLKL(mp);

		bcopy(mp->b_rptr, bp, len);
		bp += len;

		mp = mp->b_cont;
	}

	*plen = bp - bd->buf;
	ASSERT3U(*plen, <=, PAGESIZE);

	xnfp->xnf_stat_tx_lookaside++;

	return (bd);
}

/*
 * Insert the pseudo-header checksum into the packet.
 * Assumes packet is IPv4, TCP/UDP since we only advertised support for
 * HCKSUM_INET_FULL_V4.
 */
int
xnf_pseudo_cksum(mblk_t *mp)
{
	struct ether_header *ehp;
	uint16_t sap, iplen, *stuff;
	uint32_t cksum;
	size_t len;
	ipha_t *ipha;
	ipaddr_t src, dst;
	uchar_t *ptr;

	ptr = mp->b_rptr;
	len = MBLKL(mp);

	/* Each header must fit completely in an mblk. */
	ASSERT3U(len, >=, sizeof (*ehp));

	ehp = (struct ether_header *)ptr;

	if (ntohs(ehp->ether_type) == VLAN_TPID) {
		struct ether_vlan_header *evhp;
		ASSERT3U(len, >=, sizeof (*evhp));
		evhp = (struct ether_vlan_header *)ptr;
		sap = ntohs(evhp->ether_type);
		ptr += sizeof (*evhp);
		len -= sizeof (*evhp);
	} else {
		sap = ntohs(ehp->ether_type);
		ptr += sizeof (*ehp);
		len -= sizeof (*ehp);
	}

	ASSERT3U(sap, ==, ETHERTYPE_IP);

	/*
	 * Ethernet and IP headers may be in different mblks.
	 */
	ASSERT3P(ptr, <=, mp->b_wptr);
	if (ptr == mp->b_wptr) {
		mp = mp->b_cont;
		ptr = mp->b_rptr;
		len = MBLKL(mp);
	}

	ASSERT3U(len, >=, sizeof (ipha_t));
	ipha = (ipha_t *)ptr;

	/*
	 * We assume the IP header has no options. (This is enforced in
	 * ire_send_wire_v4() -- search for IXAF_NO_HW_CKSUM).
	 */
	ASSERT3U(IPH_HDR_LENGTH(ipha), ==, IP_SIMPLE_HDR_LENGTH);
	iplen = ntohs(ipha->ipha_length) - IP_SIMPLE_HDR_LENGTH;

	ptr += IP_SIMPLE_HDR_LENGTH;
	len -= IP_SIMPLE_HDR_LENGTH;

	/*
	 * IP and L4 headers may be in different mblks.
	 */
	ASSERT3P(ptr, <=, mp->b_wptr);
	if (ptr == mp->b_wptr) {
		mp = mp->b_cont;
		ptr = mp->b_rptr;
		len = MBLKL(mp);
	}

	switch (ipha->ipha_protocol) {
	case IPPROTO_TCP:
		ASSERT3U(len, >=, sizeof (tcph_t));
		stuff = (uint16_t *)(ptr + TCP_CHECKSUM_OFFSET);
		cksum = IP_TCP_CSUM_COMP;
		break;
	case IPPROTO_UDP:
		ASSERT3U(len, >=, sizeof (struct udphdr));
		stuff = (uint16_t *)(ptr + UDP_CHECKSUM_OFFSET);
		cksum = IP_UDP_CSUM_COMP;
		break;
	default:
		cmn_err(CE_WARN, "xnf_pseudo_cksum: unexpected protocol %d",
		    ipha->ipha_protocol);
		return (EINVAL);
	}

	src = ipha->ipha_src;
	dst = ipha->ipha_dst;

	cksum += (dst >> 16) + (dst & 0xFFFF);
	cksum += (src >> 16) + (src & 0xFFFF);
	cksum += htons(iplen);

	cksum = (cksum >> 16) + (cksum & 0xFFFF);
	cksum = (cksum >> 16) + (cksum & 0xFFFF);

	ASSERT(cksum <= 0xFFFF);

	*stuff = (uint16_t)(cksum ? cksum : ~cksum);

	return (0);
}

/*
 * Push a packet into the transmit ring.
 *
 * Note: the format of a tx packet that spans multiple slots is similar to
 * what is described in xnf_rx_one_packet().
 */
static void
xnf_tx_push_packet(xnf_t *xnfp, xnf_txbuf_t *head)
{
	int nslots = 0;
	int extras = 0;
	RING_IDX slot;
	boolean_t notify;

	ASSERT(MUTEX_HELD(&xnfp->xnf_txlock));
	ASSERT(xnfp->xnf_running);

	slot = xnfp->xnf_tx_ring.req_prod_pvt;

	/*
	 * The caller has already checked that we have enough slots to proceed.
	 */
	for (xnf_txbuf_t *txp = head; txp != NULL; txp = txp->tx_next) {
		xnf_txid_t *tidp;
		netif_tx_request_t *txrp;

		tidp = xnf_txid_get(xnfp);
		VERIFY(tidp != NULL);
		txrp = RING_GET_REQUEST(&xnfp->xnf_tx_ring, slot);

		txp->tx_slot = slot;
		txp->tx_txreq.id = tidp->id;
		*txrp = txp->tx_txreq;

		tidp->txbuf = txp;
		slot++;
		nslots++;

		/*
		 * When present, LSO info is placed in a slot after the first
		 * data segment, and doesn't require a txid.
		 */
		if (txp->tx_txreq.flags & NETTXF_extra_info) {
			netif_extra_info_t *extra;
			ASSERT3U(nslots, ==, 1);

			extra = (netif_extra_info_t *)
			    RING_GET_REQUEST(&xnfp->xnf_tx_ring, slot);
			*extra = txp->tx_extra;
			slot++;
			nslots++;
			extras = 1;
		}
	}

	ASSERT3U(nslots, <=, XEN_MAX_SLOTS_PER_TX);

	/*
	 * Store the number of data fragments.
	 */
	head->tx_frags_to_ack = nslots - extras;

	xnfp->xnf_tx_ring.req_prod_pvt = slot;

	/*
	 * Tell the peer that we sent something, if it cares.
	 */
	/* LINTED: constant in conditional context */
	RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&xnfp->xnf_tx_ring, notify);
	if (notify)
		ec_notify_via_evtchn(xnfp->xnf_evtchn);
}

static xnf_txbuf_t *
xnf_mblk_copy(xnf_t *xnfp, mblk_t *mp)
{
	xnf_txbuf_t *txp;
	size_t length;

	if ((txp = xnf_data_txbuf_alloc(xnfp, KM_NOSLEEP)) == NULL) {
		return (NULL);
	}

	txp->tx_bdesc = xnf_tx_get_lookaside(xnfp, mp, &length);
	if (txp->tx_bdesc == NULL) {
		xnf_data_txbuf_free(xnfp, txp);
		return (NULL);
	}
	txp->tx_mfn = txp->tx_bdesc->buf_mfn;
	txp->tx_txreq.gref = txp->tx_bdesc->grant_ref;
	txp->tx_txreq.size = length;
	txp->tx_txreq.offset = (uintptr_t)txp->tx_bdesc->buf & PAGEOFFSET;
	txp->tx_txreq.flags = 0;

	return (txp);
}

static xnf_txbuf_t *
xnf_mblk_map(xnf_t *xnfp, mblk_t *mp, int *countp)
{
	xnf_txbuf_t *head = NULL;
	xnf_txbuf_t *tail = NULL;
	domid_t oeid;
	int nsegs = 0;

	oeid = xvdi_get_oeid(xnfp->xnf_devinfo);

	for (mblk_t *ml = mp; ml != NULL; ml = ml->b_cont) {
		ddi_dma_handle_t dma_handle;
		const ddi_dma_cookie_t *dma_cookie, *dma_cookie_prev;
		xnf_txbuf_t *txp;

		if (MBLKL(ml) == 0)
			continue;

		if ((txp = xnf_data_txbuf_alloc(xnfp, KM_NOSLEEP)) == NULL) {
			goto error;
		}

		if (head == NULL) {
			head = txp;
		} else {
			ASSERT(tail != NULL);
			TXBUF_SETNEXT(tail, txp);
			txp->tx_head = head;
		}

		/*
		 * The necessary segmentation rules (e.g. not crossing a page
		 * boundary) are enforced by the dma attributes of the handle.
		 */
		dma_handle = txp->tx_dma_handle;
		int ret = ddi_dma_addr_bind_handle(dma_handle,
		    NULL, (char *)ml->b_rptr, MBLKL(ml),
		    DDI_DMA_WRITE | DDI_DMA_STREAMING,
		    DDI_DMA_DONTWAIT, 0, NULL, NULL);
		if (ret != DDI_DMA_MAPPED) {
			if (ret != DDI_DMA_NORESOURCES) {
				dev_err(xnfp->xnf_devinfo, CE_WARN,
				    "ddi_dma_addr_bind_handle() failed "
				    "[dma_error=%d]", ret);
			}
			goto error;
		}
		txp->tx_handle_bound = B_TRUE;

		dma_cookie_prev = NULL;
		while ((dma_cookie = ddi_dma_cookie_iter(dma_handle,
		    dma_cookie_prev)) != NULL) {
			if (nsegs == XEN_MAX_TX_DATA_PAGES) {
				dev_err(xnfp->xnf_devinfo, CE_WARN,
				    "xnf_dmamap_alloc() failed: "
				    "too many segments");
				goto error;
			}
			if (dma_cookie_prev != NULL) {
				if ((txp = xnf_data_txbuf_alloc(xnfp,
				    KM_NOSLEEP)) == NULL) {
					goto error;
				}
				ASSERT(tail != NULL);
				TXBUF_SETNEXT(tail, txp);
				txp->tx_head = head;
			}

			txp->tx_mfn =
			    xnf_btop(pa_to_ma(dma_cookie->dmac_laddress));
			txp->tx_txreq.gref = xnf_gref_get(xnfp);
			if (txp->tx_txreq.gref == INVALID_GRANT_REF) {
				dev_err(xnfp->xnf_devinfo, CE_WARN,
				    "xnf_dmamap_alloc() failed: "
				    "invalid grant ref");
				goto error;
			}
			gnttab_grant_foreign_access_ref(txp->tx_txreq.gref,
			    oeid, txp->tx_mfn, 1);
			txp->tx_txreq.offset =
			    dma_cookie->dmac_laddress & PAGEOFFSET;
			txp->tx_txreq.size = dma_cookie->dmac_size;
			txp->tx_txreq.flags = 0;

			nsegs++;

			if (tail != NULL)
				tail->tx_txreq.flags = NETTXF_more_data;
			tail = txp;

			dma_cookie_prev = dma_cookie;
		}
	}

	*countp = nsegs;
	return (head);

error:
	xnf_data_txbuf_free_chain(xnfp, head);
	return (NULL);
}

static void
xnf_tx_setup_offload(xnf_t *xnfp, xnf_txbuf_t *head,
    uint32_t cksum_flags, uint32_t lso_flags, uint32_t mss)
{
	if (lso_flags != 0) {
		ASSERT3U(lso_flags, ==, HW_LSO);
		ASSERT3P(head->tx_bdesc, ==, NULL);

		head->tx_txreq.flags |= NETTXF_extra_info;
		netif_extra_info_t *extra = &head->tx_extra;
		extra->type = XEN_NETIF_EXTRA_TYPE_GSO;
		extra->flags = 0;
		extra->u.gso.type = XEN_NETIF_GSO_TYPE_TCPV4;
		extra->u.gso.size = mss;
		extra->u.gso.features = 0;
		extra->u.gso.pad = 0;
	} else if (cksum_flags != 0) {
		ASSERT3U(cksum_flags, ==, HCK_FULLCKSUM);
		/*
		 * If the local protocol stack requests checksum
		 * offload we set the 'checksum blank' flag,
		 * indicating to the peer that we need the checksum
		 * calculated for us.
		 *
		 * We _don't_ set the validated flag, because we haven't
		 * validated that the data and the checksum match.
		 *
		 * Note: we already called xnf_pseudo_cksum() in
		 * xnf_send(), so we just set the txreq flag here.
		 */
		head->tx_txreq.flags |= NETTXF_csum_blank;
		xnfp->xnf_stat_tx_cksum_deferred++;
	}
}

/*
 * Send packet mp. Called by the MAC framework.
 */
static mblk_t *
xnf_send(void *arg, mblk_t *mp)
{
	xnf_t *xnfp = arg;
	xnf_txbuf_t *head;
	mblk_t *ml;
	int length;
	int pages, chunks, slots, slots_free;
	uint32_t cksum_flags, lso_flags, mss;
	boolean_t pulledup = B_FALSE;
	boolean_t force_copy = B_FALSE;

	ASSERT3P(mp->b_next, ==, NULL);

	mutex_enter(&xnfp->xnf_txlock);

	/*
	 * Wait until we are connected to the backend.
	 */
	while (!xnfp->xnf_connected)
		cv_wait(&xnfp->xnf_cv_state, &xnfp->xnf_txlock);

	/*
	 * To simplify logic and be in sync with the rescheduling mechanism,
	 * we require the maximum amount of slots that could be used by a
	 * transaction to be free before proceeding. The only downside of doing
	 * this is that it slightly reduces the effective size of the ring.
	 */
	slots_free = xnf_tx_slots_get(xnfp, XEN_MAX_SLOTS_PER_TX, B_FALSE);
	if (slots_free < XEN_MAX_SLOTS_PER_TX) {
		/*
		 * We need to ask for a re-schedule later as the ring is full.
		 */
		mutex_enter(&xnfp->xnf_schedlock);
		xnfp->xnf_need_sched = B_TRUE;
		mutex_exit(&xnfp->xnf_schedlock);

		xnfp->xnf_stat_tx_defer++;
		mutex_exit(&xnfp->xnf_txlock);
		return (mp);
	}

	/*
	 * Get hw offload parameters.
	 * This must be done before pulling up the mp as those parameters
	 * are not copied over.
	 */
	mac_hcksum_get(mp, NULL, NULL, NULL, NULL, &cksum_flags);
	mac_lso_get(mp, &mss, &lso_flags);

	/*
	 * XXX: fix MAC framework so that we can advertise support for
	 * partial checksum for IPv4 only. This way we won't need to calculate
	 * the pseudo header checksum ourselves.
	 */
	if (cksum_flags != 0) {
		ASSERT3U(cksum_flags, ==, HCK_FULLCKSUM);
		(void) xnf_pseudo_cksum(mp);
	}

pulledup:
	for (ml = mp, pages = 0, chunks = 0, length = 0; ml != NULL;
	    ml = ml->b_cont, chunks++) {
		pages += xnf_mblk_pages(ml);
		length += MBLKL(ml);
	}
	DTRACE_PROBE3(packet, int, length, int, chunks, int, pages);
	DTRACE_PROBE3(lso, int, length, uint32_t, lso_flags, uint32_t, mss);

	/*
	 * If the ethernet header crosses a page boundary the packet
	 * will be dropped by the backend. In practice it seems like
	 * this happens fairly rarely so we'll do nothing unless the
	 * packet is small enough to fit in a look-aside buffer.
	 */
	if (((uintptr_t)mp->b_rptr & PAGEOFFSET) +
	    sizeof (struct ether_header) > PAGESIZE) {
		xnfp->xnf_stat_tx_eth_hdr_split++;
		if (length <= PAGESIZE)
			force_copy = B_TRUE;
	}

	if (force_copy || (pages > 1 && !xnfp->xnf_be_tx_sg)) {
		/*
		 * If the packet spans several pages and scatter-gather is not
		 * supported then use a look-aside buffer.
		 */
		ASSERT3U(length, <=, PAGESIZE);
		head = xnf_mblk_copy(xnfp, mp);
		if (head == NULL) {
			dev_err(xnfp->xnf_devinfo, CE_WARN,
			    "xnf_mblk_copy() failed");
			goto drop;
		}
	} else {
		/*
		 * There's a limit for how many pages can be passed to the
		 * backend. If we pass that limit, the packet will be dropped
		 * and some backend implementations (e.g. Linux) could even
		 * offline the interface.
		 */
		if (pages > XEN_MAX_TX_DATA_PAGES) {
			if (pulledup) {
				dev_err(xnfp->xnf_devinfo, CE_WARN,
				    "too many pages, even after pullup: %d.",
				    pages);
				goto drop;
			}

			/*
			 * Defragment packet if it spans too many pages.
			 */
			mblk_t *newmp = msgpullup(mp, -1);
			if (newmp == NULL) {
				dev_err(xnfp->xnf_devinfo, CE_WARN,
				    "msgpullup() failed");
				goto drop;
			}

			freemsg(mp);
			mp = newmp;
			xnfp->xnf_stat_tx_pullup++;
			pulledup = B_TRUE;
			goto pulledup;
		}

		head = xnf_mblk_map(xnfp, mp, &slots);
		if (head == NULL)
			goto drop;

		IMPLY(slots > 1, xnfp->xnf_be_tx_sg);
	}

	/*
	 * Set tx_mp so that mblk is freed when the txbuf chain is freed.
	 */
	head->tx_mp = mp;

	xnf_tx_setup_offload(xnfp, head, cksum_flags, lso_flags, mss);

	/*
	 * The first request must store the total length of the packet.
	 */
	head->tx_txreq.size = length;

	/*
	 * Push the packet we have prepared into the ring.
	 */
	xnf_tx_push_packet(xnfp, head);
	xnfp->xnf_stat_opackets++;
	xnfp->xnf_stat_obytes += length;

	mutex_exit(&xnfp->xnf_txlock);
	return (NULL);

drop:
	freemsg(mp);
	xnfp->xnf_stat_tx_drop++;
	mutex_exit(&xnfp->xnf_txlock);
	return (NULL);
}

/*
 * Notification of RX packets. Currently no TX-complete interrupt is
 * used, as we clean the TX ring lazily.
 */
static uint_t
xnf_intr(caddr_t arg)
{
	xnf_t *xnfp = (xnf_t *)arg;
	mblk_t *mp;
	boolean_t need_sched, clean_ring;

	mutex_enter(&xnfp->xnf_rxlock);

	/*
	 * Interrupts before we are connected are spurious.
	 */
	if (!xnfp->xnf_connected) {
		mutex_exit(&xnfp->xnf_rxlock);
		xnfp->xnf_stat_unclaimed_interrupts++;
		return (DDI_INTR_UNCLAIMED);
	}

	/*
	 * Receive side processing.
	 */
	do {
		/*
		 * Collect buffers from the ring.
		 */
		xnf_rx_collect(xnfp);

		/*
		 * Interrupt me when the next receive buffer is consumed.
		 */
		xnfp->xnf_rx_ring.sring->rsp_event =
		    xnfp->xnf_rx_ring.rsp_cons + 1;
		xen_mb();

	} while (RING_HAS_UNCONSUMED_RESPONSES(&xnfp->xnf_rx_ring));

	if (xnfp->xnf_rx_new_buffers_posted) {
		boolean_t notify;

		/*
		 * Indicate to the peer that we have re-filled the
		 * receive ring, if it cares.
		 */
		/* LINTED: constant in conditional context */
		RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&xnfp->xnf_rx_ring, notify);
		if (notify)
			ec_notify_via_evtchn(xnfp->xnf_evtchn);
		xnfp->xnf_rx_new_buffers_posted = B_FALSE;
	}

	mp = xnfp->xnf_rx_head;
	xnfp->xnf_rx_head = xnfp->xnf_rx_tail = NULL;

	xnfp->xnf_stat_interrupts++;
	mutex_exit(&xnfp->xnf_rxlock);

	if (mp != NULL)
		mac_rx(xnfp->xnf_mh, NULL, mp);

	/*
	 * Transmit side processing.
	 *
	 * If a previous transmit attempt failed or we have pending
	 * multicast requests, clean the ring.
	 *
	 * If we previously stalled transmission and cleaning produces
	 * some free slots, tell upstream to attempt sending again.
	 *
	 * The odd style is to avoid acquiring xnf_txlock unless we
	 * will actually look inside the tx machinery.
	 */
	mutex_enter(&xnfp->xnf_schedlock);
	need_sched = xnfp->xnf_need_sched;
	clean_ring = need_sched || (xnfp->xnf_pending_multicast > 0);
	mutex_exit(&xnfp->xnf_schedlock);

	if (clean_ring) {
		int free_slots;

		mutex_enter(&xnfp->xnf_txlock);
		free_slots = xnf_tx_slots_get(xnfp, 0, B_FALSE);

		if (need_sched && (free_slots >= XEN_MAX_SLOTS_PER_TX)) {
			mutex_enter(&xnfp->xnf_schedlock);
			xnfp->xnf_need_sched = B_FALSE;
			mutex_exit(&xnfp->xnf_schedlock);

			mac_tx_update(xnfp->xnf_mh);
		}
		mutex_exit(&xnfp->xnf_txlock);
	}

	return (DDI_INTR_CLAIMED);
}

/*
 *  xnf_start() -- start the board receiving and enable interrupts.
 */
static int
xnf_start(void *arg)
{
	xnf_t *xnfp = arg;

	mutex_enter(&xnfp->xnf_rxlock);
	mutex_enter(&xnfp->xnf_txlock);

	/* Accept packets from above. */
	xnfp->xnf_running = B_TRUE;

	mutex_exit(&xnfp->xnf_txlock);
	mutex_exit(&xnfp->xnf_rxlock);

	return (0);
}

/* xnf_stop() - disable hardware */
static void
xnf_stop(void *arg)
{
	xnf_t *xnfp = arg;

	mutex_enter(&xnfp->xnf_rxlock);
	mutex_enter(&xnfp->xnf_txlock);

	xnfp->xnf_running = B_FALSE;

	mutex_exit(&xnfp->xnf_txlock);
	mutex_exit(&xnfp->xnf_rxlock);
}

/*
 * Hang buffer `bdesc' on the RX ring.
 */
static void
xnf_rxbuf_hang(xnf_t *xnfp, xnf_buf_t *bdesc)
{
	netif_rx_request_t *reqp;
	RING_IDX hang_ix;

	ASSERT(MUTEX_HELD(&xnfp->xnf_rxlock));

	reqp = RING_GET_REQUEST(&xnfp->xnf_rx_ring,
	    xnfp->xnf_rx_ring.req_prod_pvt);
	hang_ix = (RING_IDX) (reqp - RING_GET_REQUEST(&xnfp->xnf_rx_ring, 0));
	ASSERT(xnfp->xnf_rx_pkt_info[hang_ix] == NULL);

	reqp->id = bdesc->id = hang_ix;
	reqp->gref = bdesc->grant_ref;

	xnfp->xnf_rx_pkt_info[hang_ix] = bdesc;
	xnfp->xnf_rx_ring.req_prod_pvt++;

	xnfp->xnf_rx_new_buffers_posted = B_TRUE;
}

/*
 * Receive an entire packet from the ring, starting from slot *consp.
 * prod indicates the slot of the latest response.
 * On return, *consp will point to the head of the next packet.
 *
 * Note: If slot prod was reached before we could gather a full packet, we will
 * drop the partial packet; this would most likely indicate a bug in either
 * the front-end or the back-end driver.
 *
 * An rx packet can consist of several fragments and thus span multiple slots.
 * Each fragment can contain up to 4k of data.
 *
 * A typical 9000 MTU packet with look like this:
 * +------+---------------------+-------------------+-----------------------+
 * | SLOT | TYPE                | CONTENTS          | FLAGS                 |
 * +------+---------------------+-------------------+-----------------------+
 * | 1    | netif_rx_response_t | 1st data fragment | more_data             |
 * +------+---------------------+-------------------+-----------------------+
 * | 2    | netif_rx_response_t | 2nd data fragment | more_data             |
 * +------+---------------------+-------------------+-----------------------+
 * | 3    | netif_rx_response_t | 3rd data fragment | [none]                |
 * +------+---------------------+-------------------+-----------------------+
 *
 * Fragments are chained by setting NETRXF_more_data in the previous
 * response's flags. If there are additional flags, such as
 * NETRXF_data_validated or NETRXF_extra_info, those should be set on the
 * first fragment.
 *
 * Sometimes extra info can be present. If so, it will follow the first
 * fragment, and NETRXF_extra_info flag will be set on the first response.
 * If LRO is set on a packet, it will be stored in the extra info. Conforming
 * to the spec, extra info can also be chained, but must all be present right
 * after the first fragment.
 *
 * Example of a packet with 2 extra infos:
 * +------+---------------------+-------------------+-----------------------+
 * | SLOT | TYPE                | CONTENTS          | FLAGS                 |
 * +------+---------------------+-------------------+-----------------------+
 * | 1    | netif_rx_response_t | 1st data fragment | extra_info, more_data |
 * +------+---------------------+-------------------+-----------------------+
 * | 2    | netif_extra_info_t  | 1st extra info    | EXTRA_FLAG_MORE       |
 * +------+---------------------+-------------------+-----------------------+
 * | 3    | netif_extra_info_t  | 2nd extra info    | [none]                |
 * +------+---------------------+-------------------+-----------------------+
 * | 4    | netif_rx_response_t | 2nd data fragment | more_data             |
 * +------+---------------------+-------------------+-----------------------+
 * | 5    | netif_rx_response_t | 3rd data fragment | more_data             |
 * +------+---------------------+-------------------+-----------------------+
 * | 6    | netif_rx_response_t | 4th data fragment | [none]                |
 * +------+---------------------+-------------------+-----------------------+
 *
 * In practice, the only extra we expect is for LRO, but only if we advertise
 * that we support it to the backend (xnf_enable_lro == TRUE).
 */
static int
xnf_rx_one_packet(xnf_t *xnfp, RING_IDX prod, RING_IDX *consp, mblk_t **mpp)
{
	mblk_t *head = NULL;
	mblk_t *tail = NULL;
	mblk_t *mp;
	int error = 0;
	RING_IDX cons = *consp;
	netif_extra_info_t lro;
	boolean_t is_lro = B_FALSE;
	boolean_t is_extra = B_FALSE;

	netif_rx_response_t rsp = *RING_GET_RESPONSE(&xnfp->xnf_rx_ring, cons);

	boolean_t hwcsum = (rsp.flags & NETRXF_data_validated) != 0;
	boolean_t more_data = (rsp.flags & NETRXF_more_data) != 0;
	boolean_t more_extra = (rsp.flags & NETRXF_extra_info) != 0;

	IMPLY(more_data, xnf_enable_rx_sg);

	while (cons != prod) {
		xnf_buf_t *bdesc;
		int len, off;
		int rxidx = cons & (NET_RX_RING_SIZE - 1);

		bdesc = xnfp->xnf_rx_pkt_info[rxidx];
		xnfp->xnf_rx_pkt_info[rxidx] = NULL;

		if (is_extra) {
			netif_extra_info_t *extra = (netif_extra_info_t *)&rsp;
			/*
			 * The only extra we expect is for LRO, and it should
			 * only be present once.
			 */
			if (extra->type == XEN_NETIF_EXTRA_TYPE_GSO &&
			    !is_lro) {
				ASSERT(xnf_enable_lro);
				lro = *extra;
				is_lro = B_TRUE;
				DTRACE_PROBE1(lro, netif_extra_info_t *, &lro);
			} else {
				dev_err(xnfp->xnf_devinfo, CE_WARN, "rx packet "
				    "contains unexpected extra info of type %d",
				    extra->type);
				error = EINVAL;
			}
			more_extra =
			    (extra->flags & XEN_NETIF_EXTRA_FLAG_MORE) != 0;

			goto hang_buf;
		}

		ASSERT3U(bdesc->id, ==, rsp.id);

		/*
		 * status stores packet length when >= 0, or errors when < 0.
		 */
		len = rsp.status;
		off = rsp.offset;
		more_data = (rsp.flags & NETRXF_more_data) != 0;

		/*
		 * sanity checks.
		 */
		if (!xnfp->xnf_running) {
			error = EBUSY;
		} else if (len <= 0) {
			xnfp->xnf_stat_errrx++;

			switch (len) {
			case 0:
				xnfp->xnf_stat_runt++;
				break;
			case NETIF_RSP_ERROR:
				xnfp->xnf_stat_mac_rcv_error++;
				break;
			case NETIF_RSP_DROPPED:
				xnfp->xnf_stat_norxbuf++;
				break;
			}
			error = EINVAL;
		} else if (bdesc->grant_ref == INVALID_GRANT_REF) {
			dev_err(xnfp->xnf_devinfo, CE_WARN,
			    "Bad rx grant reference, rsp id %d", rsp.id);
			error = EINVAL;
		} else if ((off + len) > PAGESIZE) {
			dev_err(xnfp->xnf_devinfo, CE_WARN, "Rx packet crosses "
			    "page boundary (offset %d, length %d)", off, len);
			error = EINVAL;
		}

		if (error != 0) {
			/*
			 * If an error has been detected, we do not attempt
			 * to read the data but we still need to replace
			 * the rx bufs.
			 */
			goto hang_buf;
		}

		xnf_buf_t *nbuf = NULL;

		/*
		 * If the packet is below a pre-determined size we will
		 * copy data out of the buf rather than replace it.
		 */
		if (len > xnf_rx_copy_limit)
			nbuf = xnf_buf_get(xnfp, KM_NOSLEEP, B_FALSE);

		if (nbuf != NULL) {
			mp = desballoc((unsigned char *)bdesc->buf,
			    bdesc->len, 0, &bdesc->free_rtn);

			if (mp == NULL) {
				xnfp->xnf_stat_rx_desballoc_fail++;
				xnfp->xnf_stat_norxbuf++;
				error = ENOMEM;
				/*
				 * we free the buf we just allocated as we
				 * will re-hang the old buf.
				 */
				xnf_buf_put(xnfp, nbuf, B_FALSE);
				goto hang_buf;
			}

			mp->b_rptr = mp->b_rptr + off;
			mp->b_wptr = mp->b_rptr + len;

			/*
			 * Release the grant as the backend doesn't need to
			 * access this buffer anymore and grants are scarce.
			 */
			(void) gnttab_end_foreign_access_ref(bdesc->grant_ref,
			    0);
			xnf_gref_put(xnfp, bdesc->grant_ref);
			bdesc->grant_ref = INVALID_GRANT_REF;

			bdesc = nbuf;
		} else {
			/*
			 * We failed to allocate a new buf or decided to reuse
			 * the old one. In either case we copy the data off it
			 * and put it back into the ring.
			 */
			mp = allocb(len, 0);
			if (mp == NULL) {
				xnfp->xnf_stat_rx_allocb_fail++;
				xnfp->xnf_stat_norxbuf++;
				error = ENOMEM;
				goto hang_buf;
			}
			bcopy(bdesc->buf + off, mp->b_wptr, len);
			mp->b_wptr += len;
		}

		if (head == NULL)
			head = mp;
		else
			tail->b_cont = mp;
		tail = mp;

hang_buf:
		/*
		 * No matter what happens, for each response we need to hang
		 * a new buf on the rx ring. Put either the old one, or a new
		 * one if the old one is borrowed by the kernel via desballoc().
		 */
		xnf_rxbuf_hang(xnfp, bdesc);
		cons++;

		/* next response is an extra */
		is_extra = more_extra;

		if (!more_data && !more_extra)
			break;

		/*
		 * Note that since requests and responses are union'd on the
		 * same ring, we copy the response to a local variable instead
		 * of keeping a pointer. Otherwise xnf_rxbuf_hang() would have
		 * overwritten contents of rsp.
		 */
		rsp = *RING_GET_RESPONSE(&xnfp->xnf_rx_ring, cons);
	}

	/*
	 * Check that we do not get stuck in a loop.
	 */
	ASSERT3U(*consp, !=, cons);
	*consp = cons;

	/*
	 * We ran out of responses but the flags indicate there is more data.
	 */
	if (more_data) {
		dev_err(xnfp->xnf_devinfo, CE_WARN, "rx: need more fragments.");
		error = EINVAL;
	}
	if (more_extra) {
		dev_err(xnfp->xnf_devinfo, CE_WARN, "rx: need more fragments "
		    "(extras).");
		error = EINVAL;
	}

	/*
	 * An error means the packet must be dropped. If we have already formed
	 * a partial packet, then discard it.
	 */
	if (error != 0) {
		if (head != NULL)
			freemsg(head);
		xnfp->xnf_stat_rx_drop++;
		return (error);
	}

	ASSERT(head != NULL);

	if (hwcsum) {
		/*
		 * If the peer says that the data has been validated then we
		 * declare that the full checksum has been verified.
		 *
		 * We don't look at the "checksum blank" flag, and hence could
		 * have a packet here that we are asserting is good with
		 * a blank checksum.
		 */
		mac_hcksum_set(head, 0, 0, 0, 0, HCK_FULLCKSUM_OK);
		xnfp->xnf_stat_rx_cksum_no_need++;
	}

	/* XXX: set lro info for packet once LRO is supported in OS. */

	*mpp = head;

	return (0);
}

/*
 * Collect packets from the RX ring, storing them in `xnfp' for later use.
 */
static void
xnf_rx_collect(xnf_t *xnfp)
{
	RING_IDX prod;

	ASSERT(MUTEX_HELD(&xnfp->xnf_rxlock));

	prod = xnfp->xnf_rx_ring.sring->rsp_prod;
	/*
	 * Ensure we see queued responses up to 'prod'.
	 */
	membar_consumer();

	while (xnfp->xnf_rx_ring.rsp_cons != prod) {
		mblk_t *mp;

		/*
		 * Collect a packet.
		 * rsp_cons is updated inside xnf_rx_one_packet().
		 */
		int error = xnf_rx_one_packet(xnfp, prod,
		    &xnfp->xnf_rx_ring.rsp_cons, &mp);
		if (error == 0) {
			xnfp->xnf_stat_ipackets++;
			xnfp->xnf_stat_rbytes += xmsgsize(mp);

			/*
			 * Append the mblk to the rx list.
			 */
			if (xnfp->xnf_rx_head == NULL) {
				ASSERT3P(xnfp->xnf_rx_tail, ==, NULL);
				xnfp->xnf_rx_head = mp;
			} else {
				ASSERT(xnfp->xnf_rx_tail != NULL);
				xnfp->xnf_rx_tail->b_next = mp;
			}
			xnfp->xnf_rx_tail = mp;
		}
	}
}

/*
 *  xnf_alloc_dma_resources() -- initialize the drivers structures
 */
static int
xnf_alloc_dma_resources(xnf_t *xnfp)
{
	dev_info_t		*devinfo = xnfp->xnf_devinfo;
	size_t			len;
	ddi_dma_cookie_t	dma_cookie;
	uint_t			ncookies;
	int			rc;
	caddr_t			rptr;

	/*
	 * The code below allocates all the DMA data structures that
	 * need to be released when the driver is detached.
	 *
	 * Allocate page for the transmit descriptor ring.
	 */
	if (ddi_dma_alloc_handle(devinfo, &ringbuf_dma_attr,
	    DDI_DMA_SLEEP, 0, &xnfp->xnf_tx_ring_dma_handle) != DDI_SUCCESS)
		goto alloc_error;

	if (ddi_dma_mem_alloc(xnfp->xnf_tx_ring_dma_handle,
	    PAGESIZE, &accattr, DDI_DMA_CONSISTENT,
	    DDI_DMA_SLEEP, 0, &rptr, &len,
	    &xnfp->xnf_tx_ring_dma_acchandle) != DDI_SUCCESS) {
		ddi_dma_free_handle(&xnfp->xnf_tx_ring_dma_handle);
		xnfp->xnf_tx_ring_dma_handle = NULL;
		goto alloc_error;
	}

	if ((rc = ddi_dma_addr_bind_handle(xnfp->xnf_tx_ring_dma_handle, NULL,
	    rptr, PAGESIZE, DDI_DMA_RDWR | DDI_DMA_CONSISTENT,
	    DDI_DMA_SLEEP, 0, &dma_cookie, &ncookies)) != DDI_DMA_MAPPED) {
		ddi_dma_mem_free(&xnfp->xnf_tx_ring_dma_acchandle);
		ddi_dma_free_handle(&xnfp->xnf_tx_ring_dma_handle);
		xnfp->xnf_tx_ring_dma_handle = NULL;
		xnfp->xnf_tx_ring_dma_acchandle = NULL;
		if (rc == DDI_DMA_NORESOURCES)
			goto alloc_error;
		else
			goto error;
	}

	ASSERT(ncookies == 1);
	bzero(rptr, PAGESIZE);
	/* LINTED: constant in conditional context */
	SHARED_RING_INIT((netif_tx_sring_t *)rptr);
	/* LINTED: constant in conditional context */
	FRONT_RING_INIT(&xnfp->xnf_tx_ring, (netif_tx_sring_t *)rptr, PAGESIZE);
	xnfp->xnf_tx_ring_phys_addr = dma_cookie.dmac_laddress;

	/*
	 * Allocate page for the receive descriptor ring.
	 */
	if (ddi_dma_alloc_handle(devinfo, &ringbuf_dma_attr,
	    DDI_DMA_SLEEP, 0, &xnfp->xnf_rx_ring_dma_handle) != DDI_SUCCESS)
		goto alloc_error;

	if (ddi_dma_mem_alloc(xnfp->xnf_rx_ring_dma_handle,
	    PAGESIZE, &accattr, DDI_DMA_CONSISTENT,
	    DDI_DMA_SLEEP, 0, &rptr, &len,
	    &xnfp->xnf_rx_ring_dma_acchandle) != DDI_SUCCESS) {
		ddi_dma_free_handle(&xnfp->xnf_rx_ring_dma_handle);
		xnfp->xnf_rx_ring_dma_handle = NULL;
		goto alloc_error;
	}

	if ((rc = ddi_dma_addr_bind_handle(xnfp->xnf_rx_ring_dma_handle, NULL,
	    rptr, PAGESIZE, DDI_DMA_RDWR | DDI_DMA_CONSISTENT,
	    DDI_DMA_SLEEP, 0, &dma_cookie, &ncookies)) != DDI_DMA_MAPPED) {
		ddi_dma_mem_free(&xnfp->xnf_rx_ring_dma_acchandle);
		ddi_dma_free_handle(&xnfp->xnf_rx_ring_dma_handle);
		xnfp->xnf_rx_ring_dma_handle = NULL;
		xnfp->xnf_rx_ring_dma_acchandle = NULL;
		if (rc == DDI_DMA_NORESOURCES)
			goto alloc_error;
		else
			goto error;
	}

	ASSERT(ncookies == 1);
	bzero(rptr, PAGESIZE);
	/* LINTED: constant in conditional context */
	SHARED_RING_INIT((netif_rx_sring_t *)rptr);
	/* LINTED: constant in conditional context */
	FRONT_RING_INIT(&xnfp->xnf_rx_ring, (netif_rx_sring_t *)rptr, PAGESIZE);
	xnfp->xnf_rx_ring_phys_addr = dma_cookie.dmac_laddress;

	return (DDI_SUCCESS);

alloc_error:
	cmn_err(CE_WARN, "xnf%d: could not allocate enough DMA memory",
	    ddi_get_instance(xnfp->xnf_devinfo));
error:
	xnf_release_dma_resources(xnfp);
	return (DDI_FAILURE);
}

/*
 * Release all DMA resources in the opposite order from acquisition
 */
static void
xnf_release_dma_resources(xnf_t *xnfp)
{
	int i;

	/*
	 * Free receive buffers which are currently associated with
	 * descriptors.
	 */
	mutex_enter(&xnfp->xnf_rxlock);
	for (i = 0; i < NET_RX_RING_SIZE; i++) {
		xnf_buf_t *bp;

		if ((bp = xnfp->xnf_rx_pkt_info[i]) == NULL)
			continue;
		xnfp->xnf_rx_pkt_info[i] = NULL;
		xnf_buf_put(xnfp, bp, B_FALSE);
	}
	mutex_exit(&xnfp->xnf_rxlock);

	/* Free the receive ring buffer. */
	if (xnfp->xnf_rx_ring_dma_acchandle != NULL) {
		(void) ddi_dma_unbind_handle(xnfp->xnf_rx_ring_dma_handle);
		ddi_dma_mem_free(&xnfp->xnf_rx_ring_dma_acchandle);
		ddi_dma_free_handle(&xnfp->xnf_rx_ring_dma_handle);
		xnfp->xnf_rx_ring_dma_acchandle = NULL;
	}
	/* Free the transmit ring buffer. */
	if (xnfp->xnf_tx_ring_dma_acchandle != NULL) {
		(void) ddi_dma_unbind_handle(xnfp->xnf_tx_ring_dma_handle);
		ddi_dma_mem_free(&xnfp->xnf_tx_ring_dma_acchandle);
		ddi_dma_free_handle(&xnfp->xnf_tx_ring_dma_handle);
		xnfp->xnf_tx_ring_dma_acchandle = NULL;
	}

}

/*
 * Release any packets and associated structures used by the TX ring.
 */
static void
xnf_release_mblks(xnf_t *xnfp)
{
	RING_IDX i;
	xnf_txid_t *tidp;

	for (i = 0, tidp = &xnfp->xnf_tx_pkt_id[0];
	    i < NET_TX_RING_SIZE;
	    i++, tidp++) {
		xnf_txbuf_t *txp = tidp->txbuf;

		if (txp != NULL) {
			ASSERT(txp->tx_mp != NULL);
			freemsg(txp->tx_mp);

			xnf_txid_put(xnfp, tidp);
			kmem_cache_free(xnfp->xnf_tx_buf_cache, txp);
		}
	}
}

static int
xnf_buf_constructor(void *buf, void *arg, int kmflag)
{
	int (*ddiflags)(caddr_t) = DDI_DMA_SLEEP;
	xnf_buf_t *bdesc = buf;
	xnf_t *xnfp = arg;
	ddi_dma_cookie_t dma_cookie;
	uint_t ncookies;
	size_t len;

	if (kmflag & KM_NOSLEEP)
		ddiflags = DDI_DMA_DONTWAIT;

	/* Allocate a DMA access handle for the buffer. */
	if (ddi_dma_alloc_handle(xnfp->xnf_devinfo, &rx_buf_dma_attr,
	    ddiflags, 0, &bdesc->dma_handle) != DDI_SUCCESS)
		goto failure;

	/* Allocate DMA-able memory for buffer. */
	if (ddi_dma_mem_alloc(bdesc->dma_handle,
	    PAGESIZE, &data_accattr, DDI_DMA_STREAMING, ddiflags, 0,
	    &bdesc->buf, &len, &bdesc->acc_handle) != DDI_SUCCESS)
		goto failure_1;

	/* Bind to virtual address of buffer to get physical address. */
	if (ddi_dma_addr_bind_handle(bdesc->dma_handle, NULL,
	    bdesc->buf, len, DDI_DMA_RDWR | DDI_DMA_STREAMING,
	    ddiflags, 0, &dma_cookie, &ncookies) != DDI_DMA_MAPPED)
		goto failure_2;
	ASSERT(ncookies == 1);

	bdesc->free_rtn.free_func = xnf_buf_recycle;
	bdesc->free_rtn.free_arg = (caddr_t)bdesc;
	bdesc->xnfp = xnfp;
	bdesc->buf_phys = dma_cookie.dmac_laddress;
	bdesc->buf_mfn = pfn_to_mfn(xnf_btop(bdesc->buf_phys));
	bdesc->len = dma_cookie.dmac_size;
	bdesc->grant_ref = INVALID_GRANT_REF;
	bdesc->gen = xnfp->xnf_gen;

	atomic_inc_64(&xnfp->xnf_stat_buf_allocated);

	return (0);

failure_2:
	ddi_dma_mem_free(&bdesc->acc_handle);

failure_1:
	ddi_dma_free_handle(&bdesc->dma_handle);

failure:

	ASSERT(kmflag & KM_NOSLEEP); /* Cannot fail for KM_SLEEP. */
	return (-1);
}

static void
xnf_buf_destructor(void *buf, void *arg)
{
	xnf_buf_t *bdesc = buf;
	xnf_t *xnfp = arg;

	(void) ddi_dma_unbind_handle(bdesc->dma_handle);
	ddi_dma_mem_free(&bdesc->acc_handle);
	ddi_dma_free_handle(&bdesc->dma_handle);

	atomic_dec_64(&xnfp->xnf_stat_buf_allocated);
}

static xnf_buf_t *
xnf_buf_get(xnf_t *xnfp, int flags, boolean_t readonly)
{
	grant_ref_t gref;
	xnf_buf_t *bufp;

	/*
	 * Usually grant references are more scarce than memory, so we
	 * attempt to acquire a grant reference first.
	 */
	gref = xnf_gref_get(xnfp);
	if (gref == INVALID_GRANT_REF)
		return (NULL);

	bufp = kmem_cache_alloc(xnfp->xnf_buf_cache, flags);
	if (bufp == NULL) {
		xnf_gref_put(xnfp, gref);
		return (NULL);
	}

	ASSERT3U(bufp->grant_ref, ==, INVALID_GRANT_REF);

	bufp->grant_ref = gref;

	if (bufp->gen != xnfp->xnf_gen)
		xnf_buf_refresh(bufp);

	gnttab_grant_foreign_access_ref(bufp->grant_ref,
	    xvdi_get_oeid(bufp->xnfp->xnf_devinfo),
	    bufp->buf_mfn, readonly ? 1 : 0);

	atomic_inc_64(&xnfp->xnf_stat_buf_outstanding);

	return (bufp);
}

static void
xnf_buf_put(xnf_t *xnfp, xnf_buf_t *bufp, boolean_t readonly)
{
	if (bufp->grant_ref != INVALID_GRANT_REF) {
		(void) gnttab_end_foreign_access_ref(
		    bufp->grant_ref, readonly ? 1 : 0);
		xnf_gref_put(xnfp, bufp->grant_ref);
		bufp->grant_ref = INVALID_GRANT_REF;
	}

	kmem_cache_free(xnfp->xnf_buf_cache, bufp);

	atomic_dec_64(&xnfp->xnf_stat_buf_outstanding);
}

/*
 * Refresh any cached data about a buffer after resume.
 */
static void
xnf_buf_refresh(xnf_buf_t *bdesc)
{
	bdesc->buf_mfn = pfn_to_mfn(xnf_btop(bdesc->buf_phys));
	bdesc->gen = bdesc->xnfp->xnf_gen;
}

/*
 * Streams `freeb' routine for `xnf_buf_t' when used as transmit
 * look-aside buffers.
 */
static void
xnf_buf_recycle(xnf_buf_t *bdesc)
{
	xnf_t *xnfp = bdesc->xnfp;

	xnf_buf_put(xnfp, bdesc, B_TRUE);
}

static int
xnf_tx_buf_constructor(void *buf, void *arg, int kmflag)
{
	int (*ddiflags)(caddr_t) = DDI_DMA_SLEEP;
	xnf_txbuf_t *txp = buf;
	xnf_t *xnfp = arg;

	if (kmflag & KM_NOSLEEP)
		ddiflags = DDI_DMA_DONTWAIT;

	if (ddi_dma_alloc_handle(xnfp->xnf_devinfo, &tx_buf_dma_attr,
	    ddiflags, 0, &txp->tx_dma_handle) != DDI_SUCCESS) {
		ASSERT(kmflag & KM_NOSLEEP); /* Cannot fail for KM_SLEEP. */
		return (-1);
	}

	return (0);
}

static void
xnf_tx_buf_destructor(void *buf, void *arg)
{
	_NOTE(ARGUNUSED(arg));
	xnf_txbuf_t *txp = buf;

	ddi_dma_free_handle(&txp->tx_dma_handle);
}

/*
 * Statistics.
 */
static char *xnf_aux_statistics[] = {
	"tx_cksum_deferred",
	"rx_cksum_no_need",
	"interrupts",
	"unclaimed_interrupts",
	"tx_pullup",
	"tx_lookaside",
	"tx_drop",
	"tx_eth_hdr_split",
	"buf_allocated",
	"buf_outstanding",
	"gref_outstanding",
	"gref_failure",
	"gref_peak",
	"rx_allocb_fail",
	"rx_desballoc_fail",
};

static int
xnf_kstat_aux_update(kstat_t *ksp, int flag)
{
	xnf_t *xnfp;
	kstat_named_t *knp;

	if (flag != KSTAT_READ)
		return (EACCES);

	xnfp = ksp->ks_private;
	knp = ksp->ks_data;

	/*
	 * Assignment order must match that of the names in
	 * xnf_aux_statistics.
	 */
	(knp++)->value.ui64 = xnfp->xnf_stat_tx_cksum_deferred;
	(knp++)->value.ui64 = xnfp->xnf_stat_rx_cksum_no_need;

	(knp++)->value.ui64 = xnfp->xnf_stat_interrupts;
	(knp++)->value.ui64 = xnfp->xnf_stat_unclaimed_interrupts;
	(knp++)->value.ui64 = xnfp->xnf_stat_tx_pullup;
	(knp++)->value.ui64 = xnfp->xnf_stat_tx_lookaside;
	(knp++)->value.ui64 = xnfp->xnf_stat_tx_drop;
	(knp++)->value.ui64 = xnfp->xnf_stat_tx_eth_hdr_split;

	(knp++)->value.ui64 = xnfp->xnf_stat_buf_allocated;
	(knp++)->value.ui64 = xnfp->xnf_stat_buf_outstanding;
	(knp++)->value.ui64 = xnfp->xnf_stat_gref_outstanding;
	(knp++)->value.ui64 = xnfp->xnf_stat_gref_failure;
	(knp++)->value.ui64 = xnfp->xnf_stat_gref_peak;
	(knp++)->value.ui64 = xnfp->xnf_stat_rx_allocb_fail;
	(knp++)->value.ui64 = xnfp->xnf_stat_rx_desballoc_fail;

	return (0);
}

static boolean_t
xnf_kstat_init(xnf_t *xnfp)
{
	int nstat = sizeof (xnf_aux_statistics) /
	    sizeof (xnf_aux_statistics[0]);
	char **cp = xnf_aux_statistics;
	kstat_named_t *knp;

	/*
	 * Create and initialise kstats.
	 */
	if ((xnfp->xnf_kstat_aux = kstat_create("xnf",
	    ddi_get_instance(xnfp->xnf_devinfo),
	    "aux_statistics", "net", KSTAT_TYPE_NAMED,
	    nstat, 0)) == NULL)
		return (B_FALSE);

	xnfp->xnf_kstat_aux->ks_private = xnfp;
	xnfp->xnf_kstat_aux->ks_update = xnf_kstat_aux_update;

	knp = xnfp->xnf_kstat_aux->ks_data;
	while (nstat > 0) {
		kstat_named_init(knp, *cp, KSTAT_DATA_UINT64);

		knp++;
		cp++;
		nstat--;
	}

	kstat_install(xnfp->xnf_kstat_aux);

	return (B_TRUE);
}

static int
xnf_stat(void *arg, uint_t stat, uint64_t *val)
{
	xnf_t *xnfp = arg;

	mutex_enter(&xnfp->xnf_rxlock);
	mutex_enter(&xnfp->xnf_txlock);

#define	mac_stat(q, r)				\
	case (MAC_STAT_##q):			\
		*val = xnfp->xnf_stat_##r;	\
		break

#define	ether_stat(q, r)			\
	case (ETHER_STAT_##q):			\
		*val = xnfp->xnf_stat_##r;	\
		break

	switch (stat) {

	mac_stat(IPACKETS, ipackets);
	mac_stat(OPACKETS, opackets);
	mac_stat(RBYTES, rbytes);
	mac_stat(OBYTES, obytes);
	mac_stat(NORCVBUF, norxbuf);
	mac_stat(IERRORS, errrx);
	mac_stat(NOXMTBUF, tx_defer);

	ether_stat(MACRCV_ERRORS, mac_rcv_error);
	ether_stat(TOOSHORT_ERRORS, runt);

	/* always claim to be in full duplex mode */
	case ETHER_STAT_LINK_DUPLEX:
		*val = LINK_DUPLEX_FULL;
		break;

	/* always claim to be at 1Gb/s link speed */
	case MAC_STAT_IFSPEED:
		*val = 1000000000ull;
		break;

	default:
		mutex_exit(&xnfp->xnf_txlock);
		mutex_exit(&xnfp->xnf_rxlock);

		return (ENOTSUP);
	}

#undef mac_stat
#undef ether_stat

	mutex_exit(&xnfp->xnf_txlock);
	mutex_exit(&xnfp->xnf_rxlock);

	return (0);
}

static int
xnf_change_mtu(xnf_t *xnfp, uint32_t mtu)
{
	if (mtu > ETHERMTU) {
		if (!xnf_enable_tx_sg) {
			dev_err(xnfp->xnf_devinfo, CE_WARN, "MTU limited to %d "
			    "because scatter-gather is disabled for transmit "
			    "in driver settings", ETHERMTU);
			return (EINVAL);
		} else if (!xnf_enable_rx_sg) {
			dev_err(xnfp->xnf_devinfo, CE_WARN, "MTU limited to %d "
			    "because scatter-gather is disabled for receive "
			    "in driver settings", ETHERMTU);
			return (EINVAL);
		} else if (!xnfp->xnf_be_tx_sg) {
			dev_err(xnfp->xnf_devinfo, CE_WARN, "MTU limited to %d "
			    "because backend doesn't support scatter-gather",
			    ETHERMTU);
			return (EINVAL);
		}
		if (mtu > XNF_MAXPKT)
			return (EINVAL);
	}
	int error = mac_maxsdu_update(xnfp->xnf_mh, mtu);
	if (error == 0)
		xnfp->xnf_mtu = mtu;

	return (error);
}

/*ARGSUSED*/
static int
xnf_getprop(void *data, const char *prop_name, mac_prop_id_t prop_id,
    uint_t prop_val_size, void *prop_val)
{
	xnf_t *xnfp = data;

	switch (prop_id) {
	case MAC_PROP_MTU:
		ASSERT(prop_val_size >= sizeof (uint32_t));
		bcopy(&xnfp->xnf_mtu, prop_val, sizeof (uint32_t));
		break;
	default:
		return (ENOTSUP);
	}
	return (0);
}

/*ARGSUSED*/
static int
xnf_setprop(void *data, const char *prop_name, mac_prop_id_t prop_id,
    uint_t prop_val_size, const void *prop_val)
{
	xnf_t *xnfp = data;
	uint32_t new_mtu;
	int error;

	switch (prop_id) {
	case MAC_PROP_MTU:
		ASSERT(prop_val_size >= sizeof (uint32_t));
		bcopy(prop_val, &new_mtu, sizeof (new_mtu));
		error = xnf_change_mtu(xnfp, new_mtu);
		break;
	default:
		return (ENOTSUP);
	}

	return (error);
}

/*ARGSUSED*/
static void
xnf_propinfo(void *data, const char *prop_name, mac_prop_id_t prop_id,
    mac_prop_info_handle_t prop_handle)
{
	switch (prop_id) {
	case MAC_PROP_MTU:
		mac_prop_info_set_range_uint32(prop_handle, 0, XNF_MAXPKT);
		break;
	default:
		break;
	}
}

static boolean_t
xnf_getcapab(void *arg, mac_capab_t cap, void *cap_data)
{
	xnf_t *xnfp = arg;

	switch (cap) {
	case MAC_CAPAB_HCKSUM: {
		uint32_t *capab = cap_data;

		/*
		 * Whilst the flag used to communicate with the IO
		 * domain is called "NETTXF_csum_blank", the checksum
		 * in the packet must contain the pseudo-header
		 * checksum and not zero.
		 *
		 * To help out the IO domain, we might use
		 * HCKSUM_INET_PARTIAL. Unfortunately our stack will
		 * then use checksum offload for IPv6 packets, which
		 * the IO domain can't handle.
		 *
		 * As a result, we declare outselves capable of
		 * HCKSUM_INET_FULL_V4. This means that we receive
		 * IPv4 packets from the stack with a blank checksum
		 * field and must insert the pseudo-header checksum
		 * before passing the packet to the IO domain.
		 */
		*capab = HCKSUM_INET_FULL_V4;

		/*
		 * TODO: query the "feature-ipv6-csum-offload" capability.
		 * If enabled, that could allow us to use HCKSUM_INET_PARTIAL.
		 */

		break;
	}
	case MAC_CAPAB_LSO: {
		if (!xnfp->xnf_be_lso)
			return (B_FALSE);

		mac_capab_lso_t *lso = cap_data;
		lso->lso_flags = LSO_TX_BASIC_TCP_IPV4;
		lso->lso_basic_tcp_ipv4.lso_max = IP_MAXPACKET;
		break;
	}
	default:
		return (B_FALSE);
	}

	return (B_TRUE);
}

/*
 * The state of the peer has changed - react accordingly.
 */
static void
oe_state_change(dev_info_t *dip, ddi_eventcookie_t id,
    void *arg, void *impl_data)
{
	_NOTE(ARGUNUSED(id, arg));
	xnf_t *xnfp = ddi_get_driver_private(dip);
	XenbusState new_state = *(XenbusState *)impl_data;

	ASSERT(xnfp != NULL);

	switch (new_state) {
	case XenbusStateUnknown:
	case XenbusStateInitialising:
	case XenbusStateInitialised:
	case XenbusStateClosing:
	case XenbusStateClosed:
	case XenbusStateReconfiguring:
	case XenbusStateReconfigured:
		break;

	case XenbusStateInitWait:
		xnf_read_config(xnfp);

		if (!xnfp->xnf_be_rx_copy) {
			cmn_err(CE_WARN,
			    "The xnf driver requires a dom0 that "
			    "supports 'feature-rx-copy'.");
			(void) xvdi_switch_state(xnfp->xnf_devinfo,
			    XBT_NULL, XenbusStateClosed);
			break;
		}

		/*
		 * Connect to the backend.
		 */
		xnf_be_connect(xnfp);

		/*
		 * Our MAC address as discovered by xnf_read_config().
		 */
		mac_unicst_update(xnfp->xnf_mh, xnfp->xnf_mac_addr);

		/*
		 * We do not know if some features such as LSO are supported
		 * until we connect to the backend. We request the MAC layer
		 * to poll our capabilities again.
		 */
		mac_capab_update(xnfp->xnf_mh);

		break;

	case XenbusStateConnected:
		mutex_enter(&xnfp->xnf_rxlock);
		mutex_enter(&xnfp->xnf_txlock);

		xnfp->xnf_connected = B_TRUE;
		/*
		 * Wake up any threads waiting to send data to
		 * backend.
		 */
		cv_broadcast(&xnfp->xnf_cv_state);

		mutex_exit(&xnfp->xnf_txlock);
		mutex_exit(&xnfp->xnf_rxlock);

		/*
		 * Kick the peer in case it missed any transmits
		 * request in the TX ring.
		 */
		ec_notify_via_evtchn(xnfp->xnf_evtchn);

		/*
		 * There may already be completed receive requests in
		 * the ring sent by backend after it gets connected
		 * but before we see its state change here, so we call
		 * xnf_intr() to handle them, if any.
		 */
		(void) xnf_intr((caddr_t)xnfp);

		/*
		 * Mark the link up now that we are connected.
		 */
		mac_link_update(xnfp->xnf_mh, LINK_STATE_UP);

		/*
		 * Tell the backend about the multicast addresses in
		 * which we are interested.
		 */
		mac_multicast_refresh(xnfp->xnf_mh, NULL, xnfp, B_TRUE);

		break;

	default:
		break;
	}
}
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

/*
 * Copyright (c) 2014, 2017 by Delphix. All rights reserved.
 */

#ifndef _SYS_XNF_H
#define	_SYS_XNF_H

#ifdef __cplusplus
extern "C" {
#endif

/*
 * As of April 2017, TX and RX ring sizes are fixed to 1 page in
 * size and Xen doesn't support changing it.
 * This represents 256 entries.
 */
#define	NET_TX_RING_SIZE  __CONST_RING_SIZE(netif_tx, PAGESIZE)
#define	NET_RX_RING_SIZE  __CONST_RING_SIZE(netif_rx, PAGESIZE)

/*
 * There is no MTU limit, however for all practical purposes hardware won't
 * support anything much larger than 9k. We put an arbitrary 16k limit.
 */
#define	XNF_MAXPKT	16384
#define	XNF_FRAMESIZE	1514		/* frame size including MAC header */

/*
 * Based on XEN_NETIF_NR_SLOTS_MIN in Linux. Packets that span more pages
 * than this must be defragmented or dropped.
 */
#define	XEN_MAX_TX_DATA_PAGES	18
/*
 * We keep one extra slot for LSO
 */
#define	XEN_MAX_SLOTS_PER_TX	(XEN_MAX_TX_DATA_PAGES + 1)

#define	XEN_DATA_BOUNDARY	0x1000

/*
 * Information about each receive buffer and any transmit look-aside
 * buffers.
 */
typedef struct xnf_buf {
	frtn_t			free_rtn;
	struct xnf		*xnfp;
	ddi_dma_handle_t	dma_handle;
	caddr_t			buf;		/* DMA-able data buffer */
	paddr_t			buf_phys;
	mfn_t			buf_mfn;
	size_t			len;
	struct xnf_buf		*next;	/* For linking into free list */
	ddi_acc_handle_t	acc_handle;
	grant_ref_t		grant_ref;	/* grant table reference */
	uint16_t		id;		/* buffer id */
	unsigned int		gen;
} xnf_buf_t;

/*
 * Information about each transmit buffer.
 */
typedef enum xnf_txbuf_type {
	TX_DATA = 1,
	TX_MCAST_REQ,
	TX_MCAST_RSP
} xnf_txbuf_type_t;

/*
 * A xnf_txbuf is used to store ancillary data for a netif_tx_request_t.
 * A tx packet can span multiple xnf_txbuf's, linked together through tx_next
 * and tx_prev; tx_head points to the head of the chain.
 */
typedef struct xnf_txbuf {
	struct xnf_txbuf	*tx_next;
	struct xnf_txbuf	*tx_prev;
	struct xnf_txbuf	*tx_head;
	xnf_txbuf_type_t	tx_type;
	netif_tx_request_t	tx_txreq;
	netif_extra_info_t	tx_extra;
	/* Used for TX_DATA types */
	ddi_dma_handle_t	tx_dma_handle;
	boolean_t		tx_handle_bound;
	mblk_t			*tx_mp;
	xnf_buf_t		*tx_bdesc; /* Look-aside buffer, if used. */
	int			tx_frags_to_ack;
	/* Used for TX_MCAST types */
	int16_t			tx_status;
	/* Used for debugging */
	mfn_t			tx_mfn;
	RING_IDX		tx_slot;
} xnf_txbuf_t;

#define	TXBUF_SETNEXT(head, next)	\
	head->tx_next = next;		\
	next->tx_prev = head;

/*
 * Information about each outstanding transmit operation.
 */
typedef struct xnf_txid {
	uint16_t	id;	/* Id of this transmit buffer. */
	uint16_t	next;	/* Freelist of ids. */
	xnf_txbuf_t	*txbuf;	/* Buffer details. */
} xnf_txid_t;

/*
 * Per-instance data.
 */
typedef struct xnf {
	/* most interesting stuff first to assist debugging */
	dev_info_t		*xnf_devinfo;
	mac_handle_t		xnf_mh;
	unsigned char		xnf_mac_addr[ETHERADDRL];
	uint32_t		xnf_mtu;

	unsigned int		xnf_gen;	/* Increments on resume. */

	boolean_t		xnf_connected;
	boolean_t		xnf_running;

	boolean_t		xnf_be_rx_copy;
	boolean_t		xnf_be_mcast_control;
	boolean_t		xnf_be_tx_sg;
	boolean_t		xnf_be_lso;

	uint64_t		xnf_stat_interrupts;
	uint64_t		xnf_stat_unclaimed_interrupts;
	uint64_t		xnf_stat_norxbuf;
	uint64_t		xnf_stat_rx_drop;
	uint64_t		xnf_stat_errrx;

	uint64_t		xnf_stat_tx_pullup;
	uint64_t		xnf_stat_tx_lookaside;
	uint64_t		xnf_stat_tx_defer;
	uint64_t		xnf_stat_tx_drop;
	uint64_t		xnf_stat_tx_eth_hdr_split;
	uint64_t		xnf_stat_mac_rcv_error;
	uint64_t		xnf_stat_runt;

	uint64_t		xnf_stat_ipackets;
	uint64_t		xnf_stat_opackets;
	uint64_t		xnf_stat_rbytes;
	uint64_t		xnf_stat_obytes;

	uint64_t		xnf_stat_tx_cksum_deferred;
	uint64_t		xnf_stat_rx_cksum_no_need;

	uint64_t		xnf_stat_buf_allocated;
	uint64_t		xnf_stat_buf_outstanding;
	uint64_t		xnf_stat_gref_outstanding;
	uint64_t		xnf_stat_gref_failure;
	uint64_t		xnf_stat_gref_peak;
	uint64_t		xnf_stat_rx_allocb_fail;
	uint64_t		xnf_stat_rx_desballoc_fail;

	kstat_t			*xnf_kstat_aux;

	ddi_iblock_cookie_t	xnf_icookie;

	netif_tx_front_ring_t	xnf_tx_ring;
	ddi_dma_handle_t	xnf_tx_ring_dma_handle;
	ddi_acc_handle_t	xnf_tx_ring_dma_acchandle;
	paddr_t			xnf_tx_ring_phys_addr;
	grant_ref_t		xnf_tx_ring_ref;

	xnf_txid_t		*xnf_tx_pkt_id;
	uint16_t		xnf_tx_pkt_id_head;
	kmutex_t		xnf_txlock;
	kmutex_t		xnf_schedlock;
	boolean_t		xnf_need_sched;
	kcondvar_t		xnf_cv_tx_slots;
	kmem_cache_t		*xnf_tx_buf_cache;

	netif_rx_front_ring_t	xnf_rx_ring;
	ddi_dma_handle_t	xnf_rx_ring_dma_handle;
	ddi_acc_handle_t	xnf_rx_ring_dma_acchandle;
	paddr_t			xnf_rx_ring_phys_addr;
	grant_ref_t		xnf_rx_ring_ref;

	xnf_buf_t		**xnf_rx_pkt_info;
	kmutex_t		xnf_rxlock;
	mblk_t			*xnf_rx_head;
	mblk_t			*xnf_rx_tail;
	boolean_t		xnf_rx_new_buffers_posted;
	kmem_cache_t		*xnf_buf_cache;

	uint16_t		xnf_evtchn;

	kmutex_t		xnf_gref_lock;
	grant_ref_t		xnf_gref_head;

	kcondvar_t		xnf_cv_state;
	kcondvar_t		xnf_cv_multicast;
	uint_t			xnf_pending_multicast;
} xnf_t;

#ifdef __cplusplus
}
#endif

#endif	/* _SYS_XNF_H */
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#include <sys/types.h>
#include <sys/hypervisor.h>
#include <sys/sunndi.h>
#include <sys/sunddi.h>
#include <sys/ddi_subrdefs.h>
#include <sys/bootconf.h>
#include <sys/psw.h>
#include <sys/modctl.h>
#include <sys/errno.h>
#include <sys/reboot.h>
#include <sys/hypervisor.h>
#include <xen/sys/xenbus_comms.h>
#include <xen/sys/xenbus_impl.h>
#include <xen/sys/xendev.h>

extern int xen_boot_debug;

/*
 * Internal structures and functions
 */
int xendev_nounload = 0;
void xendev_enumerate(int);

/*
 * Interface routines
 */

static struct modlmisc modlmisc = {
	&mod_miscops, "virtual device probe"
};

static struct modlinkage modlinkage = {
	MODREV_1, (void *)&modlmisc, NULL
};

int
_init(void)
{
	int	err;

	if ((err = mod_install(&modlinkage)) != 0)
		return (err);

	impl_bus_add_probe(xendev_enumerate);
	return (0);
}

int
_fini(void)
{
	int	err;

	if (xendev_nounload)
		return (EBUSY);

	if ((err = mod_remove(&modlinkage)) != 0)
		return (err);

	impl_bus_delete_probe(xendev_enumerate);
	return (0);
}

int
_info(struct modinfo *modinfop)
{
	return (mod_info(&modlinkage, modinfop));
}


/*
 * This functions is invoked twice, first time with reprogram=0 to
 * set up the xpvd portion of the device tree. The second time is
 * ignored.
 */
void
xendev_enumerate(int reprogram)
{
	dev_info_t *dip;

	if (reprogram != 0)
		return;

	ndi_devi_alloc_sleep(ddi_root_node(), "xpvd",
	    (pnode_t)DEVI_SID_NODEID, &dip);

	(void) ndi_devi_bind_driver(dip, 0);

	/*
	 * Too early to enumerate split device drivers in domU
	 * since we need to create taskq thread during enumeration.
	 * So, we only enumerate softdevs and console here.
	 */
	xendev_enum_all(dip, B_TRUE);
}
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

/*
 * Copyright 2012 Garrett D'Amore <garrett@damore.org>.  All rights reserved.
 * Copyright (c) 2014 by Delphix. All rights reserved.
 * Copyright 2017 Nexenta Systems, Inc.
 * Copyright 2023 Oxide Computer Company
 */

/*
 *	Host to hypervisor virtual devices nexus driver
 *
 * TODO:
 * - Add watchpoints on vbd/vif and enumerate/offline on watch callback
 * - Add DR IOCTLs
 * - Filter/restrict property lookups into xenstore
 */

#include <sys/conf.h>
#include <sys/kmem.h>
#include <sys/debug.h>
#include <sys/modctl.h>
#include <sys/autoconf.h>
#include <sys/ddi_impldefs.h>
#include <sys/ddi_subrdefs.h>
#include <sys/ddi.h>
#include <sys/sunddi.h>
#include <sys/sunndi.h>
#include <sys/avintr.h>
#include <sys/psm.h>
#include <sys/spl.h>
#include <sys/promif.h>
#include <sys/list.h>
#include <sys/bootconf.h>
#include <sys/bootsvcs.h>
#include <util/sscanf.h>
#include <sys/mach_intr.h>
#include <sys/bootinfo.h>
#ifdef XPV_HVM_DRIVER
#include <sys/xpv_support.h>
#include <sys/hypervisor.h>
#include <sys/archsystm.h>
#include <sys/cpu.h>
#include <public/xen.h>
#include <public/event_channel.h>
#include <public/io/xenbus.h>
#else
#include <sys/hypervisor.h>
#include <sys/evtchn_impl.h>
#include <sys/xen_mmu.h>
#endif
#include <xen/sys/xenbus_impl.h>
#include <xen/sys/xendev.h>

/*
 * DDI dev_ops entrypoints
 */
static int xpvd_info(dev_info_t *, ddi_info_cmd_t, void *, void **);
static int xpvd_attach(dev_info_t *devi, ddi_attach_cmd_t cmd);
static int xpvd_detach(dev_info_t *devi, ddi_detach_cmd_t cmd);


/*
 * NDI bus_ops entrypoints
 */
static int xpvd_ctlops(dev_info_t *, dev_info_t *, ddi_ctl_enum_t, void *,
	void *);
static int xpvd_intr_ops(dev_info_t *, dev_info_t *, ddi_intr_op_t,
	ddi_intr_handle_impl_t *, void *);
static int xpvd_prop_op(dev_t, dev_info_t *, dev_info_t *, ddi_prop_op_t,
	int, char *, caddr_t, int *);
static int xpvd_bus_config(dev_info_t *, uint_t, ddi_bus_config_op_t,
	void *, dev_info_t **);
static int xpvd_bus_unconfig(dev_info_t *, uint_t, ddi_bus_config_op_t,
    void *);
static int xpvd_get_eventcookie(dev_info_t *, dev_info_t *,
    char *, ddi_eventcookie_t *);
static int xpvd_add_eventcall(dev_info_t *, dev_info_t *,
    ddi_eventcookie_t, void (*)(dev_info_t *,
    ddi_eventcookie_t, void *, void *),
    void *, ddi_callback_id_t *);
static int xpvd_remove_eventcall(dev_info_t *, ddi_callback_id_t);
static int xpvd_post_event(dev_info_t *, dev_info_t *,
    ddi_eventcookie_t, void *);

/*
 * misc functions
 */
static int xpvd_enable_intr(dev_info_t *, ddi_intr_handle_impl_t *, int);
static void xpvd_disable_intr(dev_info_t *, ddi_intr_handle_impl_t *, int);
static int xpvd_removechild(dev_info_t *);
static int xpvd_initchild(dev_info_t *);
static int xpvd_name_child(dev_info_t *, char *, int);
static boolean_t i_xpvd_parse_devname(char *, xendev_devclass_t *,
    domid_t *, int *);


/* Extern declarations */
extern int (*psm_intr_ops)(dev_info_t *, ddi_intr_handle_impl_t *,
    psm_intr_op_t, int *);

struct bus_ops xpvd_bus_ops = {
	BUSO_REV,
	i_ddi_bus_map,
	NULL,
	NULL,
	NULL,
	i_ddi_map_fault,
	NULL,
	ddi_dma_allochdl,
	ddi_dma_freehdl,
	ddi_dma_bindhdl,
	ddi_dma_unbindhdl,
	ddi_dma_flush,
	ddi_dma_win,
	ddi_dma_mctl,
	xpvd_ctlops,
	xpvd_prop_op,
	xpvd_get_eventcookie,
	xpvd_add_eventcall,
	xpvd_remove_eventcall,
	xpvd_post_event,
	0,		/* (*bus_intr_ctl)(); */
	xpvd_bus_config,
	xpvd_bus_unconfig,
	NULL,		/* (*bus_fm_init)(); */
	NULL,		/* (*bus_fm_fini)(); */
	NULL,		/* (*bus_fm_access_enter)(); */
	NULL,		/* (*bus_fm_access_exit)(); */
	NULL,		/* (*bus_power)(); */
	xpvd_intr_ops	/* (*bus_intr_op)(); */
};

struct dev_ops xpvd_ops = {
	DEVO_REV,		/* devo_rev */
	0,			/* refcnt  */
	xpvd_info,		/* info */
	nulldev,		/* identify */
	nulldev,		/* probe */
	xpvd_attach,		/* attach */
	xpvd_detach,		/* detach */
	nulldev,		/* reset */
	(struct cb_ops *)0,	/* driver operations */
	&xpvd_bus_ops,		/* bus operations */
	NULL,			/* power */
	ddi_quiesce_not_needed,		/* quiesce */
};


dev_info_t *xpvd_dip;

#define	CF_DBG		0x1
#define	ALL_DBG		0xff

static ndi_event_definition_t xpvd_ndi_event_defs[] = {
	{ 0, XS_OE_STATE, EPL_KERNEL, NDI_EVENT_POST_TO_TGT },
	{ 1, XS_HP_STATE, EPL_KERNEL, NDI_EVENT_POST_TO_TGT },
};

#define	XENDEV_N_NDI_EVENTS \
	(sizeof (xpvd_ndi_event_defs) / sizeof (xpvd_ndi_event_defs[0]))

static ndi_event_set_t xpvd_ndi_events = {
	NDI_EVENTS_REV1, XENDEV_N_NDI_EVENTS, xpvd_ndi_event_defs
};

static ndi_event_hdl_t xpvd_ndi_event_handle;

/*
 * Hypervisor interrupt capabilities
 */
#define	XENDEV_INTR_CAPABILITIES \
	(DDI_INTR_FLAG_EDGE | DDI_INTR_FLAG_MASKABLE | DDI_INTR_FLAG_PENDING)

/*
 * Module linkage information for the kernel.
 */

static struct modldrv modldrv = {
	&mod_driverops, /* Type of module */
	"virtual device nexus driver",
	&xpvd_ops,	/* driver ops */
};

static struct modlinkage modlinkage = {
	MODREV_1,
	(void *)&modldrv,
	NULL
};

int
_init(void)
{
	return (mod_install(&modlinkage));
}

int
_fini(void)
{
	return (mod_remove(&modlinkage));
}

int
_info(struct modinfo *modinfop)
{
	return (mod_info(&modlinkage, modinfop));
}

/* ARGSUSED */
static int
xpvd_info(dev_info_t *dip, ddi_info_cmd_t cmd, void *arg, void **result)
{
	switch (cmd) {
	default:
		return (DDI_FAILURE);

	case DDI_INFO_DEVT2INSTANCE:
		*result = (void *)0;
		return (DDI_SUCCESS);

	case DDI_INFO_DEVT2DEVINFO:
		*result = (void *)xpvd_dip;
		return (DDI_SUCCESS);
	}
}

/*ARGSUSED*/
static int
xpvd_attach(dev_info_t *devi, ddi_attach_cmd_t cmd)
{
	extern void xvdi_watch_devices(int);
#ifdef XPV_HVM_DRIVER
	extern dev_info_t *xpv_dip;

	if (xpv_dip == NULL) {
		if (ddi_hold_installed_driver(ddi_name_to_major("xpv")) ==
		    NULL) {
			cmn_err(CE_WARN, "Couldn't initialize xpv framework");
			return (DDI_FAILURE);
		}
	}
#endif /* XPV_HVM_DRIVER */

	if (ndi_event_alloc_hdl(devi, 0, &xpvd_ndi_event_handle,
	    NDI_SLEEP) != NDI_SUCCESS) {
		xpvd_dip = NULL;
		return (DDI_FAILURE);
	}
	if (ndi_event_bind_set(xpvd_ndi_event_handle, &xpvd_ndi_events,
	    NDI_SLEEP) != NDI_SUCCESS) {
		(void) ndi_event_free_hdl(xpvd_ndi_event_handle);
		xpvd_dip = NULL;
		return (DDI_FAILURE);
	}
	if (ddi_create_minor_node(devi, "devctl", S_IFCHR,
	    ddi_get_instance(devi), DDI_PSEUDO, 0) != DDI_SUCCESS) {
		(void) ndi_event_unbind_set(xpvd_ndi_event_handle,
		    &xpvd_ndi_events, NDI_SLEEP);
		(void) ndi_event_free_hdl(xpvd_ndi_event_handle);
		xpvd_dip = NULL;
		return (DDI_FAILURE);
	}

#ifdef XPV_HVM_DRIVER
	(void) ddi_prop_update_int(DDI_DEV_T_NONE, devi, DDI_NO_AUTODETACH, 1);

	/* Report our version to dom0 */
	(void) xenbus_printf(XBT_NULL, "guest/xpvd", "version", "%d",
	    HVMPV_XPVD_VERS);
#endif /* XPV_HVM_DRIVER */

	/* watch both frontend and backend for new devices */
	if (DOMAIN_IS_INITDOMAIN(xen_info))
		(void) xs_register_xenbus_callback(xvdi_watch_devices);
	else
		xvdi_watch_devices(XENSTORE_UP);

	xpvd_dip = devi;
	ddi_report_dev(devi);

	return (DDI_SUCCESS);
}

/*ARGSUSED*/
static int
xpvd_detach(dev_info_t *devi, ddi_detach_cmd_t cmd)
{
	return (DDI_FAILURE);
}

/*
 * xpvd_prop_op()
 *
 * Query xenstore for the value of properties if DDI_PROP_NOTPROM
 * is not set.  Xenstore property values are represented as ascii strings.
 */
static int
xpvd_prop_op(dev_t dev, dev_info_t *dip, dev_info_t *ch_dip,
    ddi_prop_op_t prop_op, int mod_flags, char *name, caddr_t valuep,
    int *lengthp)
{
	caddr_t buff;
	struct xendev_ppd *pdp;
	void *prop_str;
	size_t prop_len;
	unsigned int len;
	int rv;

	pdp = (struct xendev_ppd *)ddi_get_parent_data(ch_dip);

	if ((pdp == NULL) || !(mod_flags & (DDI_PROP_CANSLEEP)) ||
	    (mod_flags & DDI_PROP_NOTPROM) || (pdp->xd_xsdev.nodename == NULL))
		goto toss_off;
	/*
	 * First try reading the property off the the frontend. if that
	 * fails, try and read it from the backend node.  If that
	 * also fails, pass the request on the DDI framework
	 */
	prop_str = NULL;
	if ((xenbus_read(XBT_NULL, pdp->xd_xsdev.nodename, name, &prop_str,
	    &len) == 0) && (prop_str != NULL) && (strlen(prop_str) != 0))
		goto got_xs_prop;

	prop_str = NULL;
	if ((pdp->xd_xsdev.otherend != NULL) &&
	    (xenbus_read(XBT_NULL, pdp->xd_xsdev.otherend, name, &prop_str,
	    &len) == 0) && (prop_str != NULL) && (strlen(prop_str) != 0))
		goto got_xs_prop;

toss_off:
	return (ddi_bus_prop_op(dev, dip, ch_dip, prop_op,
	    mod_flags | DDI_PROP_NOTPROM, name, valuep, lengthp));

got_xs_prop:
	prop_len = strlen(prop_str) + 1;
	rv = DDI_PROP_SUCCESS;

	switch (prop_op) {
	case PROP_LEN:
		*lengthp = prop_len;
		break;

	case PROP_LEN_AND_VAL_ALLOC:
		buff = kmem_alloc((size_t)prop_len, KM_SLEEP);
		*(caddr_t *)valuep = (caddr_t)buff;
		break;
	case PROP_LEN_AND_VAL_BUF:
		buff = (caddr_t)valuep;
		if (*lengthp < prop_len)
			rv = DDI_PROP_BUF_TOO_SMALL;
		break;
	default:
		rv = DDI_PROP_INVAL_ARG;
		break;
	}

	if (rv == DDI_PROP_SUCCESS) {
		if (prop_op != PROP_LEN) {
			bcopy(prop_str, buff, prop_len);
			*lengthp = prop_len;
		}
	}
	kmem_free(prop_str, len);
	return (rv);
}


/*
 * return address of the device's interrupt spec structure.
 */
/*ARGSUSED*/
struct intrspec *
xpvd_get_ispec(dev_info_t *rdip, uint_t inumber)
{
	struct xendev_ppd *pdp;

	ASSERT(inumber == 0);

	if ((pdp = ddi_get_parent_data(rdip)) == NULL)
		return (NULL);

	return (&pdp->xd_ispec);
}

/*
 * return (and determine) the interrupt priority of the device.
 */
/*ARGSUSED*/
static int
xpvd_get_priority(dev_info_t *dip, int inum, int *pri)
{
	struct xendev_ppd *pdp;
	struct intrspec *ispec;
	int	*intpriorities;
	uint_t	num_intpriorities;

	DDI_INTR_NEXDBG((CE_CONT, "xpvd_get_priority: dip = 0x%p\n",
	    (void *)dip));

	ASSERT(inum == 0);

	if ((pdp = ddi_get_parent_data(dip)) == NULL)
		return (DDI_FAILURE);

	ispec = &pdp->xd_ispec;

	/*
	 * Set the default priority based on the device class.  The
	 * "interrupt-priorities" property can be used to override
	 * the default.
	 */
	if (ispec->intrspec_pri == 0) {
		ispec->intrspec_pri = xendev_devclass_ipl(pdp->xd_devclass);
		if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, dip,
		    DDI_PROP_NOTPROM | DDI_PROP_DONTPASS,
		    "interrupt-priorities", &intpriorities,
		    &num_intpriorities) == DDI_PROP_SUCCESS) {
			ispec->intrspec_pri = intpriorities[0];
			ddi_prop_free(intpriorities);
		}
	}
	*pri = ispec->intrspec_pri;
	return (DDI_SUCCESS);
}


/*
 * xpvd_intr_ops: bus_intr_op() function for interrupt support
 */
/* ARGSUSED */
static int
xpvd_intr_ops(dev_info_t *pdip, dev_info_t *rdip, ddi_intr_op_t intr_op,
    ddi_intr_handle_impl_t *hdlp, void *result)
{
	int priority = 0;
	struct intrspec *ispec;
	struct xendev_ppd *pdp;

	DDI_INTR_NEXDBG((CE_CONT,
	    "xpvd_intr_ops: pdip 0x%p, rdip 0x%p, op %x handle 0x%p\n",
	    (void *)pdip, (void *)rdip, intr_op, (void *)hdlp));

	/* Process the request */
	switch (intr_op) {
	case DDI_INTROP_SUPPORTED_TYPES:
		/* Fixed supported by default */
		*(int *)result = DDI_INTR_TYPE_FIXED;
		break;

	case DDI_INTROP_NINTRS:
		*(int *)result = 1;
		break;

	case DDI_INTROP_ALLOC:
		/*
		 * FIXED interrupts: just return available interrupts
		 */
		if (hdlp->ih_type == DDI_INTR_TYPE_FIXED) {
			/*
			 * event channels are edge-triggered, maskable,
			 * and support int pending.
			 */
			hdlp->ih_cap |= XENDEV_INTR_CAPABILITIES;
			*(int *)result = 1;	/* DDI_INTR_TYPE_FIXED */
		} else {
			return (DDI_FAILURE);
		}
		break;

	case DDI_INTROP_FREE:
		ispec = xpvd_get_ispec(rdip, (int)hdlp->ih_inum);
		if (ispec == NULL)
			return (DDI_FAILURE);
		ispec->intrspec_pri = 0; /* mark as un-initialized */
		break;

	case DDI_INTROP_GETPRI:
		if (xpvd_get_priority(rdip, hdlp->ih_inum, &priority) !=
		    DDI_SUCCESS)
			return (DDI_FAILURE);
		DDI_INTR_NEXDBG((CE_CONT, "xpvd_intr_ops: priority = 0x%x\n",
		    priority));
		*(int *)result = priority;
		break;

	case DDI_INTROP_SETPRI:
		/* Validate the interrupt priority passed */
		if (*(int *)result > LOCK_LEVEL)
			return (DDI_FAILURE);

		/* Ensure that PSM is all initialized */
		if (psm_intr_ops == NULL)
			return (DDI_FAILURE);

		/* Change the priority */
		if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_PRI, result) ==
		    PSM_FAILURE)
			return (DDI_FAILURE);

		ispec = xpvd_get_ispec(rdip, (int)hdlp->ih_inum);
		if (ispec == NULL)
			return (DDI_FAILURE);
		ispec->intrspec_pri = *(int *)result;
		break;

	case DDI_INTROP_ADDISR:
		/* update ispec */
		ispec = xpvd_get_ispec(rdip, (int)hdlp->ih_inum);
		if (ispec == NULL)
			return (DDI_FAILURE);
		ispec->intrspec_func = hdlp->ih_cb_func;

		break;

	case DDI_INTROP_REMISR:
		ispec = xpvd_get_ispec(rdip, (int)hdlp->ih_inum);
		pdp = (struct xendev_ppd *)ddi_get_parent_data(rdip);

		ASSERT(pdp != NULL);
		ASSERT(pdp->xd_evtchn != INVALID_EVTCHN);

		if (ispec) {
			ispec->intrspec_vec = 0;
			ispec->intrspec_func = (uint_t (*)()) 0;
		}
		pdp->xd_evtchn = INVALID_EVTCHN;
		break;

	case DDI_INTROP_GETCAP:
		if (hdlp->ih_type ==  DDI_INTR_TYPE_FIXED) {
			/*
			 * event channels are edge-triggered, maskable,
			 * and support int pending.
			 */
			*(int *)result = XENDEV_INTR_CAPABILITIES;
		} else {
			*(int *)result = 0;
			return (DDI_FAILURE);
		}
		DDI_INTR_NEXDBG((CE_CONT, "xpvd: GETCAP returned = %x\n",
		    *(int *)result));
		break;
	case DDI_INTROP_SETCAP:
		DDI_INTR_NEXDBG((CE_CONT, "xpvd_intr_ops: SETCAP cap=0x%x\n",
		    *(int *)result));
		if (psm_intr_ops == NULL)
			return (DDI_FAILURE);

		if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_CAP, result)) {
			DDI_INTR_NEXDBG((CE_CONT, "GETCAP: psm_intr_ops"
			    " returned failure\n"));
			return (DDI_FAILURE);
		}
		break;

	case DDI_INTROP_ENABLE:
		if (psm_intr_ops == NULL)
			return (DDI_FAILURE);

		if (xpvd_enable_intr(rdip, hdlp, (int)hdlp->ih_inum) !=
		    DDI_SUCCESS)
			return (DDI_FAILURE);

		DDI_INTR_NEXDBG((CE_CONT, "xpvd_intr_ops: ENABLE vec=0x%x\n",
		    hdlp->ih_vector));
		break;

	case DDI_INTROP_DISABLE:
		if (psm_intr_ops == NULL)
			return (DDI_FAILURE);
		xpvd_disable_intr(rdip, hdlp, hdlp->ih_inum);
		DDI_INTR_NEXDBG((CE_CONT, "xpvd_intr_ops: DISABLE vec = %x\n",
		    hdlp->ih_vector));
		break;

	case DDI_INTROP_BLOCKENABLE:
	case DDI_INTROP_BLOCKDISABLE:
		return (DDI_FAILURE);

	case DDI_INTROP_SETMASK:
	case DDI_INTROP_CLRMASK:
#ifdef XPV_HVM_DRIVER
		return (DDI_ENOTSUP);
#else
		/*
		 * Handle this here
		 */
		if (hdlp->ih_type != DDI_INTR_TYPE_FIXED)
			return (DDI_FAILURE);
		if (intr_op == DDI_INTROP_SETMASK) {
			ec_disable_irq(hdlp->ih_vector);
		} else {
			ec_enable_irq(hdlp->ih_vector);
		}
		break;
#endif
	case DDI_INTROP_GETPENDING:
#ifdef XPV_HVM_DRIVER
		return (DDI_ENOTSUP);
#else
		if (hdlp->ih_type != DDI_INTR_TYPE_FIXED)
			return (DDI_FAILURE);
		*(int *)result = ec_pending_irq(hdlp->ih_vector);
		DDI_INTR_NEXDBG((CE_CONT, "xpvd: GETPENDING returned = %x\n",
		    *(int *)result));
		break;
#endif

	case DDI_INTROP_NAVAIL:
		*(int *)result = 1;
		DDI_INTR_NEXDBG((CE_CONT, "xpvd: NAVAIL returned = %x\n",
		    *(int *)result));
		break;

	default:
		return (i_ddi_intr_ops(pdip, rdip, intr_op, hdlp, result));
	}

	return (DDI_SUCCESS);
}


static int
xpvd_enable_intr(dev_info_t *rdip, ddi_intr_handle_impl_t *hdlp, int inum)
{
	int		vector;
	ihdl_plat_t	*ihdl_plat_datap = (ihdl_plat_t *)hdlp->ih_private;

	DDI_INTR_NEXDBG((CE_CONT, "xpvd_enable_intr: hdlp %p inum %x\n",
	    (void *)hdlp, inum));

	ihdl_plat_datap->ip_ispecp = xpvd_get_ispec(rdip, inum);
	if (ihdl_plat_datap->ip_ispecp == NULL)
		return (DDI_FAILURE);

	/* translate the interrupt if needed */
	(void) (*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_XLATE_VECTOR, &vector);
	DDI_INTR_NEXDBG((CE_CONT, "xpvd_enable_intr: priority=%x vector=%x\n",
	    hdlp->ih_pri, vector));

	/* Add the interrupt handler */
	if (!add_avintr((void *)hdlp, hdlp->ih_pri, hdlp->ih_cb_func,
	    DEVI(rdip)->devi_name, vector, hdlp->ih_cb_arg1,
	    hdlp->ih_cb_arg2, NULL, rdip))
		return (DDI_FAILURE);

	/* Note this really is an irq. */
	hdlp->ih_vector = (ushort_t)vector;

	return (DDI_SUCCESS);
}


static void
xpvd_disable_intr(dev_info_t *rdip, ddi_intr_handle_impl_t *hdlp, int inum)
{
	int		vector;
	ihdl_plat_t	*ihdl_plat_datap = (ihdl_plat_t *)hdlp->ih_private;

	DDI_INTR_NEXDBG((CE_CONT, "xpvd_disable_intr: \n"));
	ihdl_plat_datap->ip_ispecp = xpvd_get_ispec(rdip, inum);
	if (ihdl_plat_datap->ip_ispecp == NULL)
		return;

	/* translate the interrupt if needed */
	(void) (*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_XLATE_VECTOR, &vector);

	/* Disable the interrupt handler */
	rem_avintr((void *)hdlp, hdlp->ih_pri, hdlp->ih_cb_func, vector);
	ihdl_plat_datap->ip_ispecp = NULL;
}

/*ARGSUSED*/
static int
xpvd_ctlops(dev_info_t *dip, dev_info_t *rdip,
    ddi_ctl_enum_t ctlop, void *arg, void *result)
{
	switch (ctlop) {
	case DDI_CTLOPS_REPORTDEV:
		if (rdip == (dev_info_t *)0)
			return (DDI_FAILURE);
		cmn_err(CE_CONT, "?%s@%s, %s%d\n", ddi_node_name(rdip),
		    ddi_get_name_addr(rdip), ddi_driver_name(rdip),
		    ddi_get_instance(rdip));
		return (DDI_SUCCESS);

	case DDI_CTLOPS_INITCHILD:
		return (xpvd_initchild((dev_info_t *)arg));

	case DDI_CTLOPS_UNINITCHILD:
		return (xpvd_removechild((dev_info_t *)arg));

	case DDI_CTLOPS_SIDDEV:
		return (DDI_SUCCESS);

	case DDI_CTLOPS_REGSIZE:
	case DDI_CTLOPS_NREGS:
		return (DDI_FAILURE);

	case DDI_CTLOPS_POWER: {
		return (ddi_ctlops(dip, rdip, ctlop, arg, result));
	}

	default:
		return (ddi_ctlops(dip, rdip, ctlop, arg, result));
	}

	/* NOTREACHED */

}

/*
 * Assign the address portion of the node name
 */
static int
xpvd_name_child(dev_info_t *child, char *addr, int addrlen)
{
	int *domain, *vdev;
	uint_t ndomain, nvdev;
	char *prop_str;

	/*
	 * i_xpvd_parse_devname() knows the formats used by this
	 * routine.  If this code changes, so must that.
	 */

	if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, child, DDI_PROP_DONTPASS,
	    "domain", &domain, &ndomain) != DDI_PROP_SUCCESS)
		return (DDI_FAILURE);
	ASSERT(ndomain == 1);

	/*
	 * Use "domain" and "vdev" properties (backend drivers).
	 */
	if (*domain != DOMID_SELF) {
		if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, child,
		    DDI_PROP_DONTPASS, "vdev", &vdev, &nvdev)
		    != DDI_PROP_SUCCESS) {
			ddi_prop_free(domain);
			return (DDI_FAILURE);
		}
		ASSERT(nvdev == 1);

		(void) snprintf(addr, addrlen, "%d,%d", domain[0], vdev[0]);
		ddi_prop_free(vdev);
		ddi_prop_free(domain);
		return (DDI_SUCCESS);
	}
	ddi_prop_free(domain);

	/*
	 * Use "vdev" and "unit-address" properties (frontend/softdev drivers).
	 * At boot time, only the vdev property is available on xdf disks.
	 */
	if (ddi_prop_lookup_string(DDI_DEV_T_ANY, child, DDI_PROP_DONTPASS,
	    "unit-address", &prop_str) != DDI_PROP_SUCCESS) {
		if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, child,
		    DDI_PROP_DONTPASS, "vdev", &vdev,
		    &nvdev) != DDI_PROP_SUCCESS)
			return (DDI_FAILURE);
		ASSERT(nvdev == 1);
		(void) snprintf(addr, addrlen, "%d", vdev[0]);
		ddi_prop_free(vdev);
		return (DDI_SUCCESS);
	}
	(void) strlcpy(addr, prop_str, addrlen);
	ddi_prop_free(prop_str);
	return (DDI_SUCCESS);
}

static int
xpvd_initchild(dev_info_t *child)
{
	char addr[80];

	/*
	 * Pseudo nodes indicate a prototype node with per-instance
	 * properties to be merged into the real h/w device node.
	 */
	if (ndi_dev_is_persistent_node(child) == 0) {
		ddi_set_parent_data(child, NULL);
		if (xpvd_name_child(child, addr, sizeof (addr)) != DDI_SUCCESS)
			return (DDI_FAILURE);
		ddi_set_name_addr(child, addr);

		/*
		 * Try to merge the properties from this prototype
		 * node into real h/w nodes.
		 */
		if (ndi_merge_node(child, xpvd_name_child) == DDI_SUCCESS) {
			/*
			 * Merged ok - return failure to remove the node.
			 */
			ddi_set_name_addr(child, NULL);
			return (DDI_FAILURE);
		}

		/*
		 * The child was not merged into a h/w node,
		 * but there's not much we can do with it other
		 * than return failure to cause the node to be removed.
		 */
		cmn_err(CE_WARN, "!%s@%s: %s.conf properties not merged",
		    ddi_get_name(child), ddi_get_name_addr(child),
		    ddi_get_name(child));
		ddi_set_name_addr(child, NULL);
		return (DDI_NOT_WELL_FORMED);
	}

	if (xvdi_init_dev(child) != DDI_SUCCESS)
		return (DDI_FAILURE);

	if (xpvd_name_child(child, addr, sizeof (addr)) != DDI_SUCCESS) {
		xvdi_uninit_dev(child);
		return (DDI_FAILURE);
	}
	ddi_set_name_addr(child, addr);

	return (DDI_SUCCESS);
}

static int
xpvd_removechild(dev_info_t *dip)
{
	xvdi_uninit_dev(dip);

	ddi_set_name_addr(dip, NULL);

	/*
	 * Strip the node to properly convert it back to prototype
	 * form.
	 */
	ddi_remove_minor_node(dip, NULL);

	return (DDI_SUCCESS);
}

static int
xpvd_bus_unconfig(dev_info_t *parent, uint_t flag, ddi_bus_config_op_t op,
    void *device_name)
{
	return (ndi_busop_bus_unconfig(parent, flag, op, device_name));
}

/*
 * Given the name of a child of xpvd, determine the device class,
 * domain and vdevnum to which it refers.
 */
static boolean_t
i_xpvd_parse_devname(char *name, xendev_devclass_t *devclassp,
    domid_t *domp, int *vdevp)
{
	int len = strlen(name) + 1;
	char *device_name = i_ddi_strdup(name, KM_SLEEP);
	char *cname = NULL, *caddr = NULL;
	boolean_t ret = B_FALSE;

	i_ddi_parse_name(device_name, &cname, &caddr, NULL);

	if ((cname == NULL) || (strlen(cname) == 0) ||
	    (caddr == NULL) || (strlen(caddr) == 0)) {
		ret = B_FALSE;
		goto done;
	}

	*devclassp = xendev_nodename_to_devclass(cname);
	if (*devclassp < 0) {
		ret = B_FALSE;
		goto done;
	}

	/*
	 * Parsing the address component requires knowledge of how
	 * xpvd_name_child() works.  If that code changes, so must
	 * this.
	 */

	/* Backend format is "<domain>,<vdev>". */
	if (sscanf(caddr, "%hu,%d", domp, vdevp) == 2) {
		ret = B_TRUE;
		goto done;
	}

	/* Frontend format is "<vdev>". */
	*domp = DOMID_SELF;
	if (sscanf(caddr, "%d", vdevp) == 1)
		ret = B_TRUE;
done:
	kmem_free(device_name, len);
	return (ret);
}

/*
 * xpvd_bus_config()
 *
 * BUS_CONFIG_ONE:
 *	Enumerate the exact instance of a driver.
 *
 * BUS_CONFIG_ALL:
 *	Enumerate all the instances of all the possible children (seen before
 *	and never seen before).
 *
 * BUS_CONFIG_DRIVER:
 *	Enumerate all the instances of a particular driver.
 */
static int
xpvd_bus_config(dev_info_t *parent, uint_t flag, ddi_bus_config_op_t op,
    void *arg, dev_info_t **childp)
{
	char *cname = NULL;

	ndi_devi_enter(parent);

	switch (op) {
	case BUS_CONFIG_ONE: {
		xendev_devclass_t devclass;
		domid_t dom;
		int vdev;

		if (!i_xpvd_parse_devname(arg, &devclass, &dom, &vdev)) {
			ndi_devi_exit(parent);
			return (NDI_FAILURE);
		}

		*childp = xvdi_find_dev(parent, devclass, dom, vdev);
		if (*childp == NULL)
			*childp = xvdi_create_dev(parent, devclass, dom, vdev);

		ndi_devi_exit(parent);

		if (*childp == NULL)
			return (NDI_FAILURE);
		else
			return (ndi_busop_bus_config(parent, flag,
			    op, arg, childp, 0));
	}

	case BUS_CONFIG_DRIVER: {
		xendev_devclass_t devclass = XEN_INVAL;

		cname = ddi_major_to_name((major_t)(uintptr_t)arg);
		if (cname != NULL)
			devclass = xendev_nodename_to_devclass(cname);

		if (devclass == XEN_INVAL) {
			ndi_devi_exit(parent);
			return (NDI_FAILURE);
		} else {
			xendev_enum_class(parent, devclass);
			ndi_devi_exit(parent);
			return (ndi_busop_bus_config(parent, flag, op,
			    arg, childp, 0));
		}
		/* NOTREACHED */
	}

	case BUS_CONFIG_ALL:
		xendev_enum_all(parent, B_FALSE);
		ndi_devi_exit(parent);

		return (ndi_busop_bus_config(parent, flag, op,
		    arg, childp, 0));

	default:
		ndi_devi_exit(parent);
		return (NDI_FAILURE);
	}
}

/*ARGSUSED*/
static int
xpvd_get_eventcookie(dev_info_t *dip, dev_info_t *rdip,
    char *eventname, ddi_eventcookie_t *cookie)
{
	return (ndi_event_retrieve_cookie(xpvd_ndi_event_handle,
	    rdip, eventname, cookie, NDI_EVENT_NOPASS));
}

/*ARGSUSED*/
static int
xpvd_add_eventcall(dev_info_t *dip, dev_info_t *rdip,
    ddi_eventcookie_t cookie, void (*callback)(dev_info_t *dip,
    ddi_eventcookie_t cookie, void *arg, void *bus_impldata),
    void *arg, ddi_callback_id_t *cb_id)
{
	return (ndi_event_add_callback(xpvd_ndi_event_handle,
	    rdip, cookie, callback, arg, NDI_SLEEP, cb_id));
}

/*ARGSUSED*/
static int
xpvd_remove_eventcall(dev_info_t *dip, ddi_callback_id_t cb_id)
{
	return (ndi_event_remove_callback(xpvd_ndi_event_handle,
	    cb_id));
}

/*ARGSUSED*/
static int
xpvd_post_event(dev_info_t *dip, dev_info_t *rdip,
    ddi_eventcookie_t cookie, void *bus_impldata)
{
	return (ndi_event_run_callbacks(xpvd_ndi_event_handle, rdip,
	    cookie, bus_impldata));
}
#
# CDDL HEADER START
#
# The contents of this file are subject to the terms of the
# Common Development and Distribution License (the "License").
# You may not use this file except in compliance with the License.
#
# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
# or http://www.opensolaris.org/os/licensing.
# See the License for the specific language governing permissions
# and limitations under the License.
#
# When distributing Covered Code, include this CDDL HEADER in each
# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
# If applicable, add the following below this CDDL HEADER, with the
# fields enclosed by brackets "[]" replaced with your own identifying
# information: Portions Copyright [yyyy] [name of copyright owner]
#
# CDDL HEADER END
#
#
# Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
# Use is subject to license terms.
#

# ident	"%Z%%M%	%I%	%E% SMI"

name="xpvd" class="root";
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 * Copyright 2017 Joyent, Inc.
 */


#include <sys/errno.h>
#include <sys/types.h>
#include <sys/conf.h>
#include <sys/kmem.h>
#include <sys/ddi.h>
#include <sys/stat.h>
#include <sys/sunddi.h>
#include <sys/file.h>
#include <sys/open.h>
#include <sys/modctl.h>
#include <sys/ddi_impldefs.h>
#include <sys/sysmacros.h>
#include <sys/ddidevmap.h>
#include <sys/policy.h>

#include <sys/vmsystm.h>
#include <vm/hat_i86.h>
#include <vm/hat_pte.h>
#include <vm/seg_kmem.h>
#include <vm/seg_mf.h>

#include <xen/io/blkif_impl.h>
#include <xen/io/blk_common.h>
#include <xen/io/xpvtap.h>


static int xpvtap_open(dev_t *devp, int flag, int otyp, cred_t *cred);
static int xpvtap_close(dev_t devp, int flag, int otyp, cred_t *cred);
static int xpvtap_ioctl(dev_t dev, int cmd, intptr_t arg, int mode,
    cred_t *cred, int *rval);
static int xpvtap_devmap(dev_t dev, devmap_cookie_t dhp, offset_t off,
    size_t len, size_t *maplen, uint_t model);
static int xpvtap_segmap(dev_t dev, off_t off, struct as *asp, caddr_t *addrp,
    off_t len, unsigned int prot, unsigned int maxprot, unsigned int flags,
    cred_t *cred_p);
static int xpvtap_chpoll(dev_t dev, short events, int anyyet, short *reventsp,
    struct pollhead **phpp);

static 	struct cb_ops xpvtap_cb_ops = {
	xpvtap_open,		/* cb_open */
	xpvtap_close,		/* cb_close */
	nodev,			/* cb_strategy */
	nodev,			/* cb_print */
	nodev,			/* cb_dump */
	nodev,			/* cb_read */
	nodev,			/* cb_write */
	xpvtap_ioctl,		/* cb_ioctl */
	xpvtap_devmap,		/* cb_devmap */
	nodev,			/* cb_mmap */
	xpvtap_segmap,		/* cb_segmap */
	xpvtap_chpoll,		/* cb_chpoll */
	ddi_prop_op,		/* cb_prop_op */
	NULL,			/* cb_stream */
	D_NEW | D_MP | D_64BIT | D_DEVMAP,	/* cb_flag */
	CB_REV
};

static int xpvtap_getinfo(dev_info_t *dip, ddi_info_cmd_t cmd, void *arg,
    void **result);
static int xpvtap_attach(dev_info_t *devi, ddi_attach_cmd_t cmd);
static int xpvtap_detach(dev_info_t *devi, ddi_detach_cmd_t cmd);

static struct dev_ops xpvtap_dev_ops = {
	DEVO_REV,		/* devo_rev */
	0,			/* devo_refcnt */
	xpvtap_getinfo,		/* devo_getinfo */
	nulldev,		/* devo_identify */
	nulldev,		/* devo_probe */
	xpvtap_attach,		/* devo_attach */
	xpvtap_detach,		/* devo_detach */
	nodev,			/* devo_reset */
	&xpvtap_cb_ops,		/* devo_cb_ops */
	NULL,			/* devo_bus_ops */
	NULL			/* power */
};


static struct modldrv xpvtap_modldrv = {
	&mod_driverops,		/* Type of module.  This one is a driver */
	"xpvtap driver",	/* Name of the module. */
	&xpvtap_dev_ops,	/* driver ops */
};

static struct modlinkage xpvtap_modlinkage = {
	MODREV_1,
	(void *) &xpvtap_modldrv,
	NULL
};


void *xpvtap_statep;


static xpvtap_state_t *xpvtap_drv_init(int instance);
static void xpvtap_drv_fini(xpvtap_state_t *state);
static uint_t xpvtap_intr(caddr_t arg);

typedef void (*xpvtap_rs_cleanup_t)(xpvtap_state_t *state, uint_t rs);
static void xpvtap_rs_init(uint_t min_val, uint_t max_val,
    xpvtap_rs_hdl_t *handle);
static void xpvtap_rs_fini(xpvtap_rs_hdl_t *handle);
static int xpvtap_rs_alloc(xpvtap_rs_hdl_t handle, uint_t *rs);
static void xpvtap_rs_free(xpvtap_rs_hdl_t handle, uint_t rs);
static void xpvtap_rs_flush(xpvtap_rs_hdl_t handle,
    xpvtap_rs_cleanup_t callback, void *arg);

static int xpvtap_segmf_register(xpvtap_state_t *state);
static void xpvtap_segmf_unregister(struct as *as, void *arg, uint_t event);

static int xpvtap_user_init(xpvtap_state_t *state);
static void xpvtap_user_fini(xpvtap_state_t *state);
static int xpvtap_user_ring_init(xpvtap_state_t *state);
static void xpvtap_user_ring_fini(xpvtap_state_t *state);
static int xpvtap_user_thread_init(xpvtap_state_t *state);
static void xpvtap_user_thread_fini(xpvtap_state_t *state);
static void xpvtap_user_thread_start(caddr_t arg);
static void xpvtap_user_thread_stop(xpvtap_state_t *state);
static void xpvtap_user_thread(void *arg);

static void xpvtap_user_app_stop(caddr_t arg);

static int xpvtap_user_request_map(xpvtap_state_t *state, blkif_request_t *req,
    uint_t *uid);
static int xpvtap_user_request_push(xpvtap_state_t *state,
    blkif_request_t *req, uint_t uid);
static int xpvtap_user_response_get(xpvtap_state_t *state,
    blkif_response_t *resp, uint_t *uid);
static void xpvtap_user_request_unmap(xpvtap_state_t *state, uint_t uid);


/*
 * _init()
 */
int
_init(void)
{
	int e;

	e = ddi_soft_state_init(&xpvtap_statep, sizeof (xpvtap_state_t), 1);
	if (e != 0) {
		return (e);
	}

	e = mod_install(&xpvtap_modlinkage);
	if (e != 0) {
		ddi_soft_state_fini(&xpvtap_statep);
		return (e);
	}

	return (0);
}


/*
 * _info()
 */
int
_info(struct modinfo *modinfop)
{
	return (mod_info(&xpvtap_modlinkage, modinfop));
}


/*
 * _fini()
 */
int
_fini(void)
{
	int e;

	e = mod_remove(&xpvtap_modlinkage);
	if (e != 0) {
		return (e);
	}

	ddi_soft_state_fini(&xpvtap_statep);

	return (0);
}


/*
 * xpvtap_attach()
 */
static int
xpvtap_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
{
	blk_ringinit_args_t args;
	xpvtap_state_t *state;
	int instance;
	int e;


	switch (cmd) {
	case DDI_ATTACH:
		break;

	case DDI_RESUME:
		return (DDI_SUCCESS);

	default:
		return (DDI_FAILURE);
	}

	/* initialize our state info */
	instance = ddi_get_instance(dip);
	state = xpvtap_drv_init(instance);
	if (state == NULL) {
		return (DDI_FAILURE);
	}
	state->bt_dip = dip;

	/* Initialize the guest ring */
	args.ar_dip = state->bt_dip;
	args.ar_intr = xpvtap_intr;
	args.ar_intr_arg = (caddr_t)state;
	args.ar_ringup = xpvtap_user_thread_start;
	args.ar_ringup_arg = (caddr_t)state;
	args.ar_ringdown = xpvtap_user_app_stop;
	args.ar_ringdown_arg = (caddr_t)state;
	e = blk_ring_init(&args, &state->bt_guest_ring);
	if (e != DDI_SUCCESS) {
		goto attachfail_ringinit;
	}

	/* create the minor node (for ioctl/mmap) */
	e = ddi_create_minor_node(dip, "xpvtap", S_IFCHR, instance,
	    DDI_PSEUDO, 0);
	if (e != DDI_SUCCESS) {
		goto attachfail_minor_node;
	}

	/* Report that driver was loaded */
	ddi_report_dev(dip);

	return (DDI_SUCCESS);

attachfail_minor_node:
	blk_ring_fini(&state->bt_guest_ring);
attachfail_ringinit:
	xpvtap_drv_fini(state);
	return (DDI_FAILURE);
}


/*
 * xpvtap_detach()
 */
static int
xpvtap_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
{
	xpvtap_state_t *state;
	int instance;


	instance = ddi_get_instance(dip);
	state = ddi_get_soft_state(xpvtap_statep, instance);
	if (state == NULL) {
		return (DDI_FAILURE);
	}

	switch (cmd) {
	case DDI_DETACH:
		break;

	case DDI_SUSPEND:
	default:
		return (DDI_FAILURE);
	}

	xpvtap_user_thread_stop(state);
	blk_ring_fini(&state->bt_guest_ring);
	xpvtap_drv_fini(state);
	ddi_remove_minor_node(dip, NULL);

	return (DDI_SUCCESS);
}


/*
 * xpvtap_getinfo()
 */
/*ARGSUSED*/
static int
xpvtap_getinfo(dev_info_t *dip, ddi_info_cmd_t cmd, void *arg, void **result)
{
	xpvtap_state_t *state;
	int instance;
	dev_t dev;
	int e;


	dev = (dev_t)arg;
	instance = getminor(dev);

	switch (cmd) {
	case DDI_INFO_DEVT2DEVINFO:
		state = ddi_get_soft_state(xpvtap_statep, instance);
		if (state == NULL) {
			return (DDI_FAILURE);
		}
		*result = (void *)state->bt_dip;
		e = DDI_SUCCESS;
		break;

	case DDI_INFO_DEVT2INSTANCE:
		*result = (void *)(uintptr_t)instance;
		e = DDI_SUCCESS;
		break;

	default:
		e = DDI_FAILURE;
		break;
	}

	return (e);
}


/*
 * xpvtap_open()
 */
/*ARGSUSED*/
static int
xpvtap_open(dev_t *devp, int flag, int otyp, cred_t *cred)
{
	xpvtap_state_t *state;
	int instance;


	if (secpolicy_xvm_control(cred)) {
		return (EPERM);
	}

	instance = getminor(*devp);
	state = ddi_get_soft_state(xpvtap_statep, instance);
	if (state == NULL) {
		return (ENXIO);
	}

	/* we should only be opened once */
	mutex_enter(&state->bt_open.bo_mutex);
	if (state->bt_open.bo_opened) {
		mutex_exit(&state->bt_open.bo_mutex);
		return (EBUSY);
	}
	state->bt_open.bo_opened = B_TRUE;
	mutex_exit(&state->bt_open.bo_mutex);

	/*
	 * save the apps address space. need it for mapping/unmapping grefs
	 * since will be doing it in a separate kernel thread.
	 */
	state->bt_map.um_as = curproc->p_as;

	return (0);
}


/*
 * xpvtap_close()
 */
/*ARGSUSED*/
static int
xpvtap_close(dev_t devp, int flag, int otyp, cred_t *cred)
{
	xpvtap_state_t *state;
	int instance;


	instance = getminor(devp);
	state = ddi_get_soft_state(xpvtap_statep, instance);
	if (state == NULL) {
		return (ENXIO);
	}

	/*
	 * wake thread so it can cleanup and wait for it to exit so we can
	 * be sure it's not in the middle of processing a request/response.
	 */
	mutex_enter(&state->bt_thread.ut_mutex);
	state->bt_thread.ut_wake = B_TRUE;
	state->bt_thread.ut_exit = B_TRUE;
	cv_signal(&state->bt_thread.ut_wake_cv);
	if (!state->bt_thread.ut_exit_done) {
		cv_wait(&state->bt_thread.ut_exit_done_cv,
		    &state->bt_thread.ut_mutex);
	}
	ASSERT(state->bt_thread.ut_exit_done);
	mutex_exit(&state->bt_thread.ut_mutex);

	state->bt_map.um_as = NULL;
	state->bt_map.um_guest_pages = NULL;

	/*
	 * when the ring is brought down, a userland hotplug script is run
	 * which tries to bring the userland app down. We'll wait for a bit
	 * for the user app to exit. Notify the thread waiting that the app
	 * has closed the driver.
	 */
	mutex_enter(&state->bt_open.bo_mutex);
	ASSERT(state->bt_open.bo_opened);
	state->bt_open.bo_opened = B_FALSE;
	cv_signal(&state->bt_open.bo_exit_cv);
	mutex_exit(&state->bt_open.bo_mutex);

	return (0);
}


/*
 * xpvtap_ioctl()
 */
/*ARGSUSED*/
static int
xpvtap_ioctl(dev_t dev, int cmd, intptr_t arg, int mode, cred_t *cred,
    int *rval)
{
	xpvtap_state_t *state;
	int instance;


	if (secpolicy_xvm_control(cred)) {
		return (EPERM);
	}

	instance = getminor(dev);
	if (instance == -1) {
		return (EBADF);
	}

	state = ddi_get_soft_state(xpvtap_statep, instance);
	if (state == NULL) {
		return (EBADF);
	}

	switch (cmd) {
	case XPVTAP_IOCTL_RESP_PUSH:
		/*
		 * wake thread, thread handles guest requests and user app
		 * responses.
		 */
		mutex_enter(&state->bt_thread.ut_mutex);
		state->bt_thread.ut_wake = B_TRUE;
		cv_signal(&state->bt_thread.ut_wake_cv);
		mutex_exit(&state->bt_thread.ut_mutex);
		break;

	default:
		cmn_err(CE_WARN, "ioctl(%d) not supported\n", cmd);
		return (ENXIO);
	}

	return (0);
}


/*
 * xpvtap_segmap()
 */
/*ARGSUSED*/
static int
xpvtap_segmap(dev_t dev, off_t off, struct as *asp, caddr_t *addrp,
    off_t len, unsigned int prot, unsigned int maxprot, unsigned int flags,
    cred_t *cred_p)
{
	struct segmf_crargs a;
	xpvtap_state_t *state;
	int instance;
	int e;


	if (secpolicy_xvm_control(cred_p)) {
		return (EPERM);
	}

	instance = getminor(dev);
	state = ddi_get_soft_state(xpvtap_statep, instance);
	if (state == NULL) {
		return (EBADF);
	}

	/* the user app should be doing a MAP_SHARED mapping */
	if ((flags & MAP_TYPE) != MAP_SHARED) {
		return (EINVAL);
	}

	/*
	 * if this is the user ring (offset = 0), devmap it (which ends up in
	 * xpvtap_devmap). devmap will alloc and map the ring into the
	 * app's VA space.
	 */
	if (off == 0) {
		e = devmap_setup(dev, (offset_t)off, asp, addrp, (size_t)len,
		    prot, maxprot, flags, cred_p);
		return (e);
	}

	/* this should be the mmap for the gref pages (offset = PAGESIZE) */
	if (off != PAGESIZE) {
		return (EINVAL);
	}

	/* make sure we get the size we're expecting */
	if (len != XPVTAP_GREF_BUFSIZE) {
		return (EINVAL);
	}

	/*
	 * reserve user app VA space for the gref pages and use segmf to
	 * manage the backing store for the physical memory. segmf will
	 * map in/out the grefs and fault them in/out.
	 */
	ASSERT(asp == state->bt_map.um_as);
	as_rangelock(asp);
	if ((flags & MAP_FIXED) == 0) {
		map_addr(addrp, len, 0, 0, flags);
		if (*addrp == NULL) {
			as_rangeunlock(asp);
			return (ENOMEM);
		}
	} else {
		/* User specified address */
		(void) as_unmap(asp, *addrp, len);
	}
	a.dev = dev;
	a.prot = (uchar_t)prot;
	a.maxprot = (uchar_t)maxprot;
	e = as_map(asp, *addrp, len, segmf_create, &a);
	if (e != 0) {
		as_rangeunlock(asp);
		return (e);
	}
	as_rangeunlock(asp);

	/*
	 * Stash user base address, and compute address where the request
	 * array will end up.
	 */
	state->bt_map.um_guest_pages = (caddr_t)*addrp;
	state->bt_map.um_guest_size = (size_t)len;

	/* register an as callback so we can cleanup when the app goes away */
	e = as_add_callback(asp, xpvtap_segmf_unregister, state,
	    AS_UNMAP_EVENT, *addrp, len, KM_SLEEP);
	if (e != 0) {
		(void) as_unmap(asp, *addrp, len);
		return (EINVAL);
	}

	/* wake thread to see if there are requests already queued up */
	mutex_enter(&state->bt_thread.ut_mutex);
	state->bt_thread.ut_wake = B_TRUE;
	cv_signal(&state->bt_thread.ut_wake_cv);
	mutex_exit(&state->bt_thread.ut_mutex);

	return (0);
}


/*
 * xpvtap_devmap()
 */
/*ARGSUSED*/
static int
xpvtap_devmap(dev_t dev, devmap_cookie_t dhp, offset_t off, size_t len,
    size_t *maplen, uint_t model)
{
	xpvtap_user_ring_t *usring;
	xpvtap_state_t *state;
	int instance;
	int e;


	instance = getminor(dev);
	state = ddi_get_soft_state(xpvtap_statep, instance);
	if (state == NULL) {
		return (EBADF);
	}

	/* we should only get here if the offset was == 0 */
	if (off != 0) {
		return (EINVAL);
	}

	/* we should only be mapping in one page */
	if (len != PAGESIZE) {
		return (EINVAL);
	}

	/*
	 * we already allocated the user ring during driver attach, all we
	 * need to do is map it into the user app's VA.
	 */
	usring = &state->bt_user_ring;
	e = devmap_umem_setup(dhp, state->bt_dip, NULL, usring->ur_cookie, 0,
	    PAGESIZE, PROT_ALL, DEVMAP_DEFAULTS, NULL);
	if (e < 0) {
		return (e);
	}

	/* return the size to compete the devmap */
	*maplen = PAGESIZE;

	return (0);
}


/*
 * xpvtap_chpoll()
 */
static int
xpvtap_chpoll(dev_t dev, short events, int anyyet, short *reventsp,
    struct pollhead **phpp)
{
	xpvtap_user_ring_t *usring;
	xpvtap_state_t *state;
	int instance;


	instance = getminor(dev);
	if (instance == -1) {
		return (EBADF);
	}
	state = ddi_get_soft_state(xpvtap_statep, instance);
	if (state == NULL) {
		return (EBADF);
	}

	if (((events & (POLLIN | POLLRDNORM)) == 0) && !anyyet) {
		return (EINVAL);
	}

	/*
	 * if we pushed requests on the user ring since the last poll, wakeup
	 * the user app
	 */
	*reventsp = 0;
	usring = &state->bt_user_ring;
	if (usring->ur_prod_polled != usring->ur_ring.req_prod_pvt) {

		/*
		 * XXX - is this faster here or xpvtap_user_request_push??
		 * prelim data says here.  Because less membars or because
		 * user thread will spin in poll requests before getting to
		 * responses?
		 */
		RING_PUSH_REQUESTS(&usring->ur_ring);

		usring->ur_prod_polled = usring->ur_ring.sring->req_prod;
		*reventsp =  POLLIN | POLLRDNORM;
	}

	if ((*reventsp == 0 && !anyyet) || (events & POLLET)) {
		*phpp = &state->bt_pollhead;
	}

	return (0);
}


/*
 * xpvtap_drv_init()
 */
static xpvtap_state_t *
xpvtap_drv_init(int instance)
{
	xpvtap_state_t *state;
	int e;


	e = ddi_soft_state_zalloc(xpvtap_statep, instance);
	if (e != DDI_SUCCESS) {
		return (NULL);
	}
	state = ddi_get_soft_state(xpvtap_statep, instance);
	if (state == NULL) {
		goto drvinitfail_get_soft_state;
	}

	state->bt_instance = instance;
	mutex_init(&state->bt_open.bo_mutex, NULL, MUTEX_DRIVER, NULL);
	cv_init(&state->bt_open.bo_exit_cv, NULL, CV_DRIVER, NULL);
	state->bt_open.bo_opened = B_FALSE;
	state->bt_map.um_registered = B_FALSE;

	/* initialize user ring, thread, mapping state */
	e = xpvtap_user_init(state);
	if (e != DDI_SUCCESS) {
		goto drvinitfail_userinit;
	}

	return (state);

drvinitfail_userinit:
	cv_destroy(&state->bt_open.bo_exit_cv);
	mutex_destroy(&state->bt_open.bo_mutex);
drvinitfail_get_soft_state:
	(void) ddi_soft_state_free(xpvtap_statep, instance);
	return (NULL);
}


/*
 * xpvtap_drv_fini()
 */
static void
xpvtap_drv_fini(xpvtap_state_t *state)
{
	xpvtap_user_fini(state);
	cv_destroy(&state->bt_open.bo_exit_cv);
	mutex_destroy(&state->bt_open.bo_mutex);
	(void) ddi_soft_state_free(xpvtap_statep, state->bt_instance);
}


/*
 * xpvtap_intr()
 *    this routine will be called when we have a request on the guest ring.
 */
static uint_t
xpvtap_intr(caddr_t arg)
{
	xpvtap_state_t *state;


	state = (xpvtap_state_t *)arg;

	/* wake thread, thread handles guest requests and user app responses */
	mutex_enter(&state->bt_thread.ut_mutex);
	state->bt_thread.ut_wake = B_TRUE;
	cv_signal(&state->bt_thread.ut_wake_cv);
	mutex_exit(&state->bt_thread.ut_mutex);

	return (DDI_INTR_CLAIMED);
}


/*
 * xpvtap_segmf_register()
 */
static int
xpvtap_segmf_register(xpvtap_state_t *state)
{
	struct seg *seg;
	uint64_t pte_ma;
	struct as *as;
	caddr_t uaddr;
	uint_t pgcnt;
	int i;


	as = state->bt_map.um_as;
	pgcnt = btopr(state->bt_map.um_guest_size);
	uaddr = state->bt_map.um_guest_pages;

	if (pgcnt == 0) {
		return (DDI_FAILURE);
	}

	AS_LOCK_ENTER(as, RW_READER);

	seg = as_findseg(as, state->bt_map.um_guest_pages, 0);
	if ((seg == NULL) || ((uaddr + state->bt_map.um_guest_size) >
	    (seg->s_base + seg->s_size))) {
		AS_LOCK_EXIT(as);
		return (DDI_FAILURE);
	}

	/*
	 * lock down the htables so the HAT can't steal them. Register the
	 * PTE MA's for each gref page with seg_mf so we can do user space
	 * gref mappings.
	 */
	for (i = 0; i < pgcnt; i++) {
		hat_prepare_mapping(as->a_hat, uaddr, &pte_ma);
		hat_devload(as->a_hat, uaddr, PAGESIZE, (pfn_t)0,
		    PROT_READ | PROT_WRITE | PROT_USER | HAT_UNORDERED_OK,
		    HAT_LOAD_NOCONSIST | HAT_LOAD_LOCK);
		hat_release_mapping(as->a_hat, uaddr);
		segmf_add_gref_pte(seg, uaddr, pte_ma);
		uaddr += PAGESIZE;
	}

	state->bt_map.um_registered = B_TRUE;

	AS_LOCK_EXIT(as);

	return (DDI_SUCCESS);
}


/*
 * xpvtap_segmf_unregister()
 *    as_callback routine
 */
/*ARGSUSED*/
static void
xpvtap_segmf_unregister(struct as *as, void *arg, uint_t event)
{
	xpvtap_state_t *state;
	caddr_t uaddr;
	uint_t pgcnt;
	int i;


	state = (xpvtap_state_t *)arg;
	if (!state->bt_map.um_registered) {
		/* remove the callback (which is this routine) */
		(void) as_delete_callback(as, arg);
		return;
	}

	pgcnt = btopr(state->bt_map.um_guest_size);
	uaddr = state->bt_map.um_guest_pages;

	/* unmap any outstanding req's grefs */
	xpvtap_rs_flush(state->bt_map.um_rs, xpvtap_user_request_unmap, state);

	/* Unlock the gref pages */
	for (i = 0; i < pgcnt; i++) {
		AS_LOCK_ENTER(as, RW_WRITER);
		hat_prepare_mapping(as->a_hat, uaddr, NULL);
		hat_unload(as->a_hat, uaddr, PAGESIZE, HAT_UNLOAD_UNLOCK);
		hat_release_mapping(as->a_hat, uaddr);
		AS_LOCK_EXIT(as);
		uaddr += PAGESIZE;
	}

	/* remove the callback (which is this routine) */
	(void) as_delete_callback(as, arg);

	state->bt_map.um_registered = B_FALSE;
}


/*
 * xpvtap_user_init()
 */
static int
xpvtap_user_init(xpvtap_state_t *state)
{
	xpvtap_user_map_t *map;
	int e;


	map = &state->bt_map;

	/* Setup the ring between the driver and user app */
	e = xpvtap_user_ring_init(state);
	if (e != DDI_SUCCESS) {
		return (DDI_FAILURE);
	}

	/*
	 * the user ring can handle BLKIF_RING_SIZE outstanding requests. This
	 * is the same number of requests as the guest ring. Initialize the
	 * state we use to track request IDs to the user app. These IDs will
	 * also identify which group of gref pages correspond with the
	 * request.
	 */
	xpvtap_rs_init(0, (BLKIF_RING_SIZE - 1), &map->um_rs);

	/*
	 * allocate the space to store a copy of each outstanding requests. We
	 * will need to reference the ID and the number of segments when we
	 * get the response from the user app.
	 */
	map->um_outstanding_reqs = kmem_zalloc(
	    sizeof (*map->um_outstanding_reqs) * BLKIF_RING_SIZE,
	    KM_SLEEP);

	/*
	 * initialize the thread we use to process guest requests and user
	 * responses.
	 */
	e = xpvtap_user_thread_init(state);
	if (e != DDI_SUCCESS) {
		goto userinitfail_user_thread_init;
	}

	return (DDI_SUCCESS);

userinitfail_user_thread_init:
	xpvtap_rs_fini(&map->um_rs);
	kmem_free(map->um_outstanding_reqs,
	    sizeof (*map->um_outstanding_reqs) * BLKIF_RING_SIZE);
	xpvtap_user_ring_fini(state);
	return (DDI_FAILURE);
}


/*
 * xpvtap_user_ring_init()
 */
static int
xpvtap_user_ring_init(xpvtap_state_t *state)
{
	xpvtap_user_ring_t *usring;


	usring = &state->bt_user_ring;

	/* alocate and initialize the page for the shared user ring */
	usring->ur_sring = (blkif_sring_t *)ddi_umem_alloc(PAGESIZE,
	    DDI_UMEM_SLEEP, &usring->ur_cookie);
	SHARED_RING_INIT(usring->ur_sring);
	FRONT_RING_INIT(&usring->ur_ring, usring->ur_sring, PAGESIZE);
	usring->ur_prod_polled = 0;

	return (DDI_SUCCESS);
}


/*
 * xpvtap_user_thread_init()
 */
static int
xpvtap_user_thread_init(xpvtap_state_t *state)
{
	xpvtap_user_thread_t *thread;
	char taskqname[32];


	thread = &state->bt_thread;

	mutex_init(&thread->ut_mutex, NULL, MUTEX_DRIVER, NULL);
	cv_init(&thread->ut_wake_cv, NULL, CV_DRIVER, NULL);
	cv_init(&thread->ut_exit_done_cv, NULL, CV_DRIVER, NULL);
	thread->ut_wake = B_FALSE;
	thread->ut_exit = B_FALSE;
	thread->ut_exit_done = B_TRUE;

	/* create but don't start the user thread */
	(void) sprintf(taskqname, "xvptap_%d", state->bt_instance);
	thread->ut_taskq = ddi_taskq_create(state->bt_dip, taskqname, 1,
	    TASKQ_DEFAULTPRI, 0);
	if (thread->ut_taskq == NULL) {
		goto userinitthrfail_taskq_create;
	}

	return (DDI_SUCCESS);

userinitthrfail_taskq_create:
	cv_destroy(&thread->ut_exit_done_cv);
	cv_destroy(&thread->ut_wake_cv);
	mutex_destroy(&thread->ut_mutex);

	return (DDI_FAILURE);
}


/*
 * xpvtap_user_thread_start()
 */
static void
xpvtap_user_thread_start(caddr_t arg)
{
	xpvtap_user_thread_t *thread;
	xpvtap_state_t *state;
	int e;


	state = (xpvtap_state_t *)arg;
	thread = &state->bt_thread;

	/* start the user thread */
	thread->ut_exit_done = B_FALSE;
	e = ddi_taskq_dispatch(thread->ut_taskq, xpvtap_user_thread, state,
	    DDI_SLEEP);
	if (e != DDI_SUCCESS) {
		thread->ut_exit_done = B_TRUE;
		cmn_err(CE_WARN, "Unable to start user thread\n");
	}
}


/*
 * xpvtap_user_thread_stop()
 */
static void
xpvtap_user_thread_stop(xpvtap_state_t *state)
{
	/* wake thread so it can exit */
	mutex_enter(&state->bt_thread.ut_mutex);
	state->bt_thread.ut_wake = B_TRUE;
	state->bt_thread.ut_exit = B_TRUE;
	cv_signal(&state->bt_thread.ut_wake_cv);
	if (!state->bt_thread.ut_exit_done) {
		cv_wait(&state->bt_thread.ut_exit_done_cv,
		    &state->bt_thread.ut_mutex);
	}
	mutex_exit(&state->bt_thread.ut_mutex);
	ASSERT(state->bt_thread.ut_exit_done);
}


/*
 * xpvtap_user_fini()
 */
static void
xpvtap_user_fini(xpvtap_state_t *state)
{
	xpvtap_user_map_t *map;


	map = &state->bt_map;

	xpvtap_user_thread_fini(state);
	xpvtap_rs_fini(&map->um_rs);
	kmem_free(map->um_outstanding_reqs,
	    sizeof (*map->um_outstanding_reqs) * BLKIF_RING_SIZE);
	xpvtap_user_ring_fini(state);
}


/*
 * xpvtap_user_ring_fini()
 */
static void
xpvtap_user_ring_fini(xpvtap_state_t *state)
{
	ddi_umem_free(state->bt_user_ring.ur_cookie);
}


/*
 * xpvtap_user_thread_fini()
 */
static void
xpvtap_user_thread_fini(xpvtap_state_t *state)
{
	ddi_taskq_destroy(state->bt_thread.ut_taskq);
	cv_destroy(&state->bt_thread.ut_exit_done_cv);
	cv_destroy(&state->bt_thread.ut_wake_cv);
	mutex_destroy(&state->bt_thread.ut_mutex);
}


/*
 * xpvtap_user_thread()
 */
static void
xpvtap_user_thread(void *arg)
{
	xpvtap_user_thread_t *thread;
	blkif_response_t resp;
	xpvtap_state_t *state;
	blkif_request_t req;
	boolean_t b;
	uint_t uid;
	int e;


	state = (xpvtap_state_t *)arg;
	thread = &state->bt_thread;

xpvtap_thread_start:
	/* See if we are supposed to exit */
	mutex_enter(&thread->ut_mutex);
	if (thread->ut_exit) {
		thread->ut_exit_done = B_TRUE;
		cv_signal(&state->bt_thread.ut_exit_done_cv);
		mutex_exit(&thread->ut_mutex);
		return;
	}

	/*
	 * if we aren't supposed to be awake, wait until someone wakes us.
	 * when we wake up, check for a kill or someone telling us to exit.
	 */
	if (!thread->ut_wake) {
		e = cv_wait_sig(&thread->ut_wake_cv, &thread->ut_mutex);
		if ((e == 0) || (thread->ut_exit)) {
			thread->ut_exit = B_TRUE;
			mutex_exit(&thread->ut_mutex);
			goto xpvtap_thread_start;
		}
	}

	/* if someone didn't wake us, go back to the start of the thread */
	if (!thread->ut_wake) {
		mutex_exit(&thread->ut_mutex);
		goto xpvtap_thread_start;
	}

	/* we are awake */
	thread->ut_wake = B_FALSE;
	mutex_exit(&thread->ut_mutex);

	/* process requests from the guest */
	do {
		/*
		 * check for requests from the guest. if we don't have any,
		 * break out of the loop.
		 */
		e = blk_ring_request_get(state->bt_guest_ring, &req);
		if (e == B_FALSE) {
			break;
		}

		/* we got a request, map the grefs into the user app's VA */
		e = xpvtap_user_request_map(state, &req, &uid);
		if (e != DDI_SUCCESS) {
			/*
			 * If we couldn't map the request (e.g. user app hasn't
			 * opened the device yet), requeue it and try again
			 * later
			 */
			blk_ring_request_requeue(state->bt_guest_ring);
			break;
		}

		/* push the request to the user app */
		e = xpvtap_user_request_push(state, &req, uid);
		if (e != DDI_SUCCESS) {
			resp.id = req.id;
			resp.operation = req.operation;
			resp.status = BLKIF_RSP_ERROR;
			blk_ring_response_put(state->bt_guest_ring, &resp);
		}
	} while (!thread->ut_exit);

	/* process reponses from the user app */
	do {
		/*
		 * check for responses from the user app. if we don't have any,
		 * break out of the loop.
		 */
		b = xpvtap_user_response_get(state, &resp, &uid);
		if (b != B_TRUE) {
			break;
		}

		/*
		 * if we got a response, unmap the grefs from the matching
		 * request.
		 */
		xpvtap_user_request_unmap(state, uid);

		/* push the response to the guest */
		blk_ring_response_put(state->bt_guest_ring, &resp);
	} while (!thread->ut_exit);

	goto xpvtap_thread_start;
}


/*
 * xpvtap_user_request_map()
 */
static int
xpvtap_user_request_map(xpvtap_state_t *state, blkif_request_t *req,
    uint_t *uid)
{
	grant_ref_t gref[BLKIF_MAX_SEGMENTS_PER_REQUEST];
	struct seg *seg;
	struct as *as;
	domid_t domid;
	caddr_t uaddr;
	uint_t flags;
	int i;
	int e;


	domid = xvdi_get_oeid(state->bt_dip);

	as = state->bt_map.um_as;
	if ((as == NULL) || (state->bt_map.um_guest_pages == NULL)) {
		return (DDI_FAILURE);
	}

	/* has to happen after segmap returns */
	if (!state->bt_map.um_registered) {
		/* register the pte's with segmf */
		e = xpvtap_segmf_register(state);
		if (e != DDI_SUCCESS) {
			return (DDI_FAILURE);
		}
	}

	/* alloc an ID for the user ring */
	e = xpvtap_rs_alloc(state->bt_map.um_rs, uid);
	if (e != DDI_SUCCESS) {
		return (DDI_FAILURE);
	}

	/* if we don't have any segments to map, we're done */
	if ((req->operation == BLKIF_OP_WRITE_BARRIER) ||
	    (req->operation == BLKIF_OP_FLUSH_DISKCACHE) ||
	    (req->nr_segments == 0)) {
		return (DDI_SUCCESS);
	}

	/* get the apps gref address */
	uaddr = XPVTAP_GREF_REQADDR(state->bt_map.um_guest_pages, *uid);

	AS_LOCK_ENTER(as, RW_READER);
	seg = as_findseg(as, state->bt_map.um_guest_pages, 0);
	if ((seg == NULL) || ((uaddr + mmu_ptob(req->nr_segments)) >
	    (seg->s_base + seg->s_size))) {
		AS_LOCK_EXIT(as);
		return (DDI_FAILURE);
	}

	/* if we are reading from disk, we are writing into memory */
	flags = 0;
	if (req->operation == BLKIF_OP_READ) {
		flags |= SEGMF_GREF_WR;
	}

	/* Load the grefs into seg_mf */
	for (i = 0; i < req->nr_segments; i++) {
		gref[i] = req->seg[i].gref;
	}
	(void) segmf_add_grefs(seg, uaddr, flags, gref, req->nr_segments,
	    domid);

	AS_LOCK_EXIT(as);

	return (DDI_SUCCESS);
}


/*
 * xpvtap_user_request_push()
 */
static int
xpvtap_user_request_push(xpvtap_state_t *state, blkif_request_t *req,
    uint_t uid)
{
	blkif_request_t *outstanding_req;
	blkif_front_ring_t *uring;
	blkif_request_t *target;
	xpvtap_user_map_t *map;


	uring = &state->bt_user_ring.ur_ring;
	map = &state->bt_map;

	target = RING_GET_REQUEST(uring, uring->req_prod_pvt);

	/*
	 * Save request from the frontend. used for ID mapping and unmap
	 * on response/cleanup
	 */
	outstanding_req = &map->um_outstanding_reqs[uid];
	bcopy(req, outstanding_req, sizeof (*outstanding_req));

	/* put the request on the user ring */
	bcopy(req, target, sizeof (*req));
	target->id = (uint64_t)uid;
	uring->req_prod_pvt++;

	pollwakeup(&state->bt_pollhead, POLLIN | POLLRDNORM);

	return (DDI_SUCCESS);
}


static void
xpvtap_user_request_unmap(xpvtap_state_t *state, uint_t uid)
{
	blkif_request_t *req;
	struct seg *seg;
	struct as *as;
	caddr_t uaddr;
	int e;


	as = state->bt_map.um_as;
	if (as == NULL) {
		return;
	}

	/* get a copy of the original request */
	req = &state->bt_map.um_outstanding_reqs[uid];

	/* unmap the grefs for this request */
	if ((req->operation != BLKIF_OP_WRITE_BARRIER) &&
	    (req->operation != BLKIF_OP_FLUSH_DISKCACHE) &&
	    (req->nr_segments != 0)) {
		uaddr = XPVTAP_GREF_REQADDR(state->bt_map.um_guest_pages, uid);
		AS_LOCK_ENTER(as, RW_READER);
		seg = as_findseg(as, state->bt_map.um_guest_pages, 0);
		if ((seg == NULL) || ((uaddr + mmu_ptob(req->nr_segments)) >
		    (seg->s_base + seg->s_size))) {
			AS_LOCK_EXIT(as);
			xpvtap_rs_free(state->bt_map.um_rs, uid);
			return;
		}

		e = segmf_release_grefs(seg, uaddr, req->nr_segments);
		if (e != 0) {
			cmn_err(CE_WARN, "unable to release grefs");
		}

		AS_LOCK_EXIT(as);
	}

	/* free up the user ring id */
	xpvtap_rs_free(state->bt_map.um_rs, uid);
}


static int
xpvtap_user_response_get(xpvtap_state_t *state, blkif_response_t *resp,
    uint_t *uid)
{
	blkif_front_ring_t *uring;
	blkif_response_t *target;


	uring = &state->bt_user_ring.ur_ring;

	if (!RING_HAS_UNCONSUMED_RESPONSES(uring)) {
		return (B_FALSE);
	}

	target = NULL;
	target = RING_GET_RESPONSE(uring, uring->rsp_cons);
	if (target == NULL) {
		return (B_FALSE);
	}

	/* copy out the user app response */
	bcopy(target, resp, sizeof (*resp));
	uring->rsp_cons++;

	/* restore the quests id from the original request */
	*uid = (uint_t)resp->id;
	resp->id = state->bt_map.um_outstanding_reqs[*uid].id;

	return (B_TRUE);
}


/*
 * xpvtap_user_app_stop()
 */
static void xpvtap_user_app_stop(caddr_t arg)
{
	xpvtap_state_t *state;
	clock_t rc;

	state = (xpvtap_state_t *)arg;

	/*
	 * Give the app 10 secs to exit. If it doesn't exit, it's not a serious
	 * problem, we just won't auto-detach the driver.
	 */
	mutex_enter(&state->bt_open.bo_mutex);
	if (state->bt_open.bo_opened) {
		rc = cv_reltimedwait(&state->bt_open.bo_exit_cv,
		    &state->bt_open.bo_mutex, drv_usectohz(10000000),
		    TR_CLOCK_TICK);
		if (rc <= 0) {
			cmn_err(CE_NOTE, "!user process still has driver open, "
			    "deferring detach\n");
		}
	}
	mutex_exit(&state->bt_open.bo_mutex);
}


/*
 * xpvtap_rs_init()
 *    Initialize the resource structure. init() returns a handle to be used
 *    for the rest of the resource functions. This code is written assuming
 *    that min_val will be close to 0. Therefore, we will allocate the free
 *    buffer only taking max_val into account.
 */
static void
xpvtap_rs_init(uint_t min_val, uint_t max_val, xpvtap_rs_hdl_t *handle)
{
	xpvtap_rs_t *rstruct;
	uint_t array_size;
	uint_t index;


	ASSERT(handle != NULL);
	ASSERT(min_val < max_val);

	/* alloc space for resource structure */
	rstruct = kmem_alloc(sizeof (xpvtap_rs_t), KM_SLEEP);

	/*
	 * Test to see if the max value is 64-bit aligned. If so, we don't need
	 * to allocate an extra 64-bit word. alloc space for free buffer
	 * (8 bytes per uint64_t).
	 */
	if ((max_val & 0x3F) == 0) {
		rstruct->rs_free_size = (max_val >> 6) * 8;
	} else {
		rstruct->rs_free_size = ((max_val >> 6) + 1) * 8;
	}
	rstruct->rs_free = kmem_alloc(rstruct->rs_free_size, KM_SLEEP);

	/* Initialize resource structure */
	rstruct->rs_min = min_val;
	rstruct->rs_last = min_val;
	rstruct->rs_max = max_val;
	mutex_init(&rstruct->rs_mutex, NULL, MUTEX_DRIVER, NULL);
	rstruct->rs_flushing = B_FALSE;

	/* Mark all resources as free */
	array_size = rstruct->rs_free_size >> 3;
	for (index = 0; index < array_size; index++) {
		rstruct->rs_free[index] = (uint64_t)0xFFFFFFFFFFFFFFFF;
	}

	/* setup handle which is returned from this function */
	*handle = rstruct;
}


/*
 * xpvtap_rs_fini()
 *    Frees up the space allocated in init().  Notice that a pointer to the
 *    handle is used for the parameter.  fini() will set the handle to NULL
 *    before returning.
 */
static void
xpvtap_rs_fini(xpvtap_rs_hdl_t *handle)
{
	xpvtap_rs_t *rstruct;


	ASSERT(handle != NULL);

	rstruct = (xpvtap_rs_t *)*handle;

	mutex_destroy(&rstruct->rs_mutex);
	kmem_free(rstruct->rs_free, rstruct->rs_free_size);
	kmem_free(rstruct, sizeof (xpvtap_rs_t));

	/* set handle to null.  This helps catch bugs. */
	*handle = NULL;
}


/*
 * xpvtap_rs_alloc()
 *    alloc a resource. If alloc fails, we are out of resources.
 */
static int
xpvtap_rs_alloc(xpvtap_rs_hdl_t handle, uint_t *resource)
{
	xpvtap_rs_t *rstruct;
	uint_t array_idx;
	uint64_t free;
	uint_t index;
	uint_t last;
	uint_t min;
	uint_t max;


	ASSERT(handle != NULL);
	ASSERT(resource != NULL);

	rstruct = (xpvtap_rs_t *)handle;

	mutex_enter(&rstruct->rs_mutex);
	min = rstruct->rs_min;
	max = rstruct->rs_max;

	/*
	 * Find a free resource. This will return out of the loop once it finds
	 * a free resource. There are a total of 'max'-'min'+1 resources.
	 * Performs a round robin allocation.
	 */
	for (index = min; index <= max; index++) {

		array_idx = rstruct->rs_last >> 6;
		free = rstruct->rs_free[array_idx];
		last = rstruct->rs_last & 0x3F;

		/* if the next resource to check is free */
		if ((free & ((uint64_t)1 << last)) != 0) {
			/* we are using this resource */
			*resource = rstruct->rs_last;

			/* take it out of the free list */
			rstruct->rs_free[array_idx] &= ~((uint64_t)1 << last);

			/*
			 * increment the last count so we start checking the
			 * next resource on the next alloc().  Note the rollover
			 * at 'max'+1.
			 */
			rstruct->rs_last++;
			if (rstruct->rs_last > max) {
				rstruct->rs_last = rstruct->rs_min;
			}

			/* unlock the resource structure */
			mutex_exit(&rstruct->rs_mutex);

			return (DDI_SUCCESS);
		}

		/*
		 * This resource is not free, lets go to the next one. Note the
		 * rollover at 'max'.
		 */
		rstruct->rs_last++;
		if (rstruct->rs_last > max) {
			rstruct->rs_last = rstruct->rs_min;
		}
	}

	mutex_exit(&rstruct->rs_mutex);

	return (DDI_FAILURE);
}


/*
 * xpvtap_rs_free()
 *    Free the previously alloc'd resource.  Once a resource has been free'd,
 *    it can be used again when alloc is called.
 */
static void
xpvtap_rs_free(xpvtap_rs_hdl_t handle, uint_t resource)
{
	xpvtap_rs_t *rstruct;
	uint_t array_idx;
	uint_t offset;


	ASSERT(handle != NULL);

	rstruct = (xpvtap_rs_t *)handle;
	ASSERT(resource >= rstruct->rs_min);
	ASSERT(resource <= rstruct->rs_max);

	if (!rstruct->rs_flushing) {
		mutex_enter(&rstruct->rs_mutex);
	}

	/* Put the resource back in the free list */
	array_idx = resource >> 6;
	offset = resource & 0x3F;
	rstruct->rs_free[array_idx] |= ((uint64_t)1 << offset);

	if (!rstruct->rs_flushing) {
		mutex_exit(&rstruct->rs_mutex);
	}
}


/*
 * xpvtap_rs_flush()
 */
static void
xpvtap_rs_flush(xpvtap_rs_hdl_t handle, xpvtap_rs_cleanup_t callback,
    void *arg)
{
	xpvtap_rs_t *rstruct;
	uint_t array_idx;
	uint64_t free;
	uint_t index;
	uint_t last;
	uint_t min;
	uint_t max;


	ASSERT(handle != NULL);

	rstruct = (xpvtap_rs_t *)handle;

	mutex_enter(&rstruct->rs_mutex);
	min = rstruct->rs_min;
	max = rstruct->rs_max;

	rstruct->rs_flushing = B_TRUE;

	/*
	 * for all resources not free, call the callback routine to clean it
	 * up.
	 */
	for (index = min; index <= max; index++) {

		array_idx = rstruct->rs_last >> 6;
		free = rstruct->rs_free[array_idx];
		last = rstruct->rs_last & 0x3F;

		/* if the next resource to check is not free */
		if ((free & ((uint64_t)1 << last)) == 0) {
			/* call the callback to cleanup */
			(*callback)(arg, rstruct->rs_last);

			/* put it back in the free list */
			rstruct->rs_free[array_idx] |= ((uint64_t)1 << last);
		}

		/* go to the next one. Note the rollover at 'max' */
		rstruct->rs_last++;
		if (rstruct->rs_last > max) {
			rstruct->rs_last = rstruct->rs_min;
		}
	}

	mutex_exit(&rstruct->rs_mutex);
}
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#ifndef _SYS_XPVTAP_H
#define	_SYS_XPVTAP_H


#ifdef __cplusplus
extern "C" {
#endif

#include <sys/types.h>

/* Notification from user app that it has pushed responses */
#define	XPVTAP_IOCTL_RESP_PUSH		1

/* Number of bytes the user app should mmap for the gref pages */
#define	XPVTAP_GREF_BUFSIZE	\
	(BLKIF_RING_SIZE * BLKIF_MAX_SEGMENTS_PER_REQUEST * PAGESIZE)


#ifdef	_KERNEL

#include <xen/io/blk_common.h>


#define	XPVTAP_GREF_REQADDR(base, id) (caddr_t) \
	((uintptr_t)base + (id * BLKIF_MAX_SEGMENTS_PER_REQUEST * PAGESIZE))

/* structure used to keep track of resources */
typedef struct xpvtap_rs_s {
	/*
	 * Bounds of resource allocation. We will start allocating at rs_min
	 * and rollover at rs_max+1 (rs_max is included). e.g. for rs_min=0
	 * and rs_max=7, we will have 8 total resources which can be alloced.
	 */
	uint_t rs_min;
	uint_t rs_max;

	/*
	 * rs_free points to an array of 64-bit values used to track resource
	 * allocation. rs_free_size is the free buffer size in bytes.
	 */
	uint64_t *rs_free;
	uint_t rs_free_size;

	/*
	 * last tracks the last alloc'd resource. This allows us to do a round
	 * robin allocation.
	 */
	uint_t rs_last;

	/*
	 * set when flushing all allocated resources. We'll know the lock
	 * is held.
	 */
	boolean_t rs_flushing;

	kmutex_t rs_mutex;
} xpvtap_rs_t;
typedef struct xpvtap_rs_s *xpvtap_rs_hdl_t;

/* track if user app has the device open, and sleep waiting for close */
typedef struct xpvtap_open_s {
	kmutex_t	bo_mutex;
	boolean_t	bo_opened;
	kcondvar_t	bo_exit_cv;
} xpvtap_open_t;

/*
 * ring between driver and user app. requests are forwared from the
 * guest to the user app on this ring. reponses from the user app come in
 * on this ring are then are forwarded to the guest.
 */
typedef struct xpvtap_user_ring_s {
	/* ring state */
	blkif_front_ring_t	ur_ring;

	/*
	 * pointer to allocated memory for the ring which is shared between
	 * the driver and the app.
	 */
	blkif_sring_t		*ur_sring;

	/* umem cookie for free'ing up the umem */
	ddi_umem_cookie_t	ur_cookie;

	RING_IDX		ur_prod_polled;
} xpvtap_user_ring_t;

/*
 * track the requests that come in from the guest. we need to track the
 * requests for two reasons. first, we need to know how many grefs we need
 * to unmap when the app sends the response. second, since we use the ID in
 * the request to index into um_guest_pages (tells the app where the segments
 * are mapped), we need to have a mapping between the the ID we sent in the
 * request to the app and the ID we got from the guest request. The response
 * to the guest needs to have the later.
 */
typedef struct xpvtap_user_map_s {
	/* address space of the user app. grab this in open */
	struct as		*um_as;

	/* state to track request IDs we can send to the user app */
	xpvtap_rs_hdl_t		um_rs;

	/*
	 * base user app VA of the mapped grefs. this VA space is large enough
	 * to map the max pages per request * max outstanding requests.
	 */
	caddr_t			um_guest_pages;
	size_t			um_guest_size;

	/*
	 * have we locked down the gref buffer's ptes and registered
	 * them with segmf. This needs to happen after the user app
	 * has mmaped the gref buf.
	 */
	boolean_t		um_registered;

	/*
	 * array of outstanding requests to the user app. Index into this
	 * array using the ID in the user app request.
	 */
	blkif_request_t		*um_outstanding_reqs;
} xpvtap_user_map_t;

/* thread start, wake, exit state */
typedef struct xpvtap_user_thread_s {
	kmutex_t		ut_mutex;
	kcondvar_t		ut_wake_cv;
	volatile boolean_t	ut_wake;
	volatile boolean_t	ut_exit;
	kcondvar_t		ut_exit_done_cv;
	volatile boolean_t	ut_exit_done;
	ddi_taskq_t		*ut_taskq;
} xpvtap_user_thread_t;

/* driver state */
typedef struct xpvtap_state_s {
	dev_info_t		*bt_dip;
	int			bt_instance;

	/* ring between the guest and xpvtap */
	blk_ring_t		bt_guest_ring;

	/* ring between xpvtap and the user app */
	xpvtap_user_ring_t	bt_user_ring;

	xpvtap_user_map_t	bt_map;
	xpvtap_user_thread_t	bt_thread;
	struct pollhead		bt_pollhead;
	xpvtap_open_t		bt_open;
} xpvtap_state_t;

#endif /* _KERNEL */

#ifdef __cplusplus
}
#endif

#endif /* _SYS_XPVTAP_H */
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

/*
 * gnttab.c
 *
 * Granting foreign access to our memory reservation.
 *
 * Copyright (c) 2005-2006, Christopher Clark
 * Copyright (c) 2004-2005, K A Fraser
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License version 2
 * as published by the Free Software Foundation; or, when distributed
 * separately from the Linux kernel or incorporated into other
 * software packages, subject to the following license:
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this source file (the "Software"), to deal in the Software without
 * restriction, including without limitation the rights to use, copy, modify,
 * merge, publish, distribute, sublicense, and/or sell copies of the Software,
 * and to permit persons to whom the Software is furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 */

#include <sys/types.h>
#include <sys/archsystm.h>
#ifdef XPV_HVM_DRIVER
#include <sys/xpv_support.h>
#include <sys/mman.h>
#include <vm/hat.h>
#endif
#include <sys/hypervisor.h>
#include <sys/gnttab.h>
#include <sys/sysmacros.h>
#include <sys/machsystm.h>
#include <sys/systm.h>
#include <sys/mutex.h>
#include <sys/atomic.h>
#include <sys/spl.h>
#include <sys/condvar.h>
#include <sys/cpuvar.h>
#include <sys/taskq.h>
#include <sys/panic.h>
#include <sys/cmn_err.h>
#include <sys/promif.h>
#include <sys/cpu.h>
#include <sys/vmem.h>
#include <vm/hat_i86.h>
#include <sys/bootconf.h>
#include <sys/bootsvcs.h>
#ifndef XPV_HVM_DRIVER
#include <sys/bootinfo.h>
#include <sys/multiboot.h>
#include <vm/kboot_mmu.h>
#endif
#include <sys/bootvfs.h>
#include <sys/bootprops.h>
#include <vm/seg_kmem.h>
#include <sys/mman.h>

/* Globals */

static grant_ref_t **gnttab_list;
static uint_t nr_grant_frames;
static int gnttab_free_count;
static grant_ref_t gnttab_free_head;
static kmutex_t gnttab_list_lock;
static grant_entry_t *shared;
static struct gnttab_free_callback *gnttab_free_callback_list;

/* Macros */

#define	GT_PGADDR(i) ((uintptr_t)shared + ((i) << MMU_PAGESHIFT))
#define	VALID_GRANT_REF(r) ((r) < (nr_grant_frames * GREFS_PER_GRANT_FRAME))
#define	RPP (PAGESIZE / sizeof (grant_ref_t))
#define	GNTTAB_ENTRY(entry) (gnttab_list[(entry) / RPP][(entry) % RPP])
#define	CMPXCHG(t, c, n) atomic_cas_16((t), (c), (n))
/* External tools reserve first few grant table entries. */
#define	NR_RESERVED_ENTRIES 8
#define	GNTTAB_LIST_END 0xffffffff
#define	GREFS_PER_GRANT_FRAME (PAGESIZE / sizeof (grant_entry_t))

/* Implementation */

static uint_t
max_nr_grant_frames(void)
{
	struct gnttab_query_size query;
	int rc;

	query.dom = DOMID_SELF;

	rc = HYPERVISOR_grant_table_op(GNTTABOP_query_size, &query, 1);
	if ((rc < 0) || (query.status != GNTST_okay))
		return (4); /* Legacy max supported number of frames */

	ASSERT(query.max_nr_frames);
	return (query.max_nr_frames);
}

static void
do_free_callbacks(void)
{
	struct gnttab_free_callback *callback, *next;

	callback = gnttab_free_callback_list;
	gnttab_free_callback_list = NULL;

	while (callback != NULL) {
		next = callback->next;
		if (gnttab_free_count >= callback->count) {
			callback->next = NULL;
			callback->fn(callback->arg);
		} else {
			callback->next = gnttab_free_callback_list;
			gnttab_free_callback_list = callback;
		}
		callback = next;
	}
}

static void
check_free_callbacks(void)
{
	if (gnttab_free_callback_list)
		do_free_callbacks();
}

static int
grow_gnttab_list(uint_t more_frames)
{
	uint_t new_nr_grant_frames, extra_entries, i;
	uint_t nr_glist_frames, new_nr_glist_frames;

	ASSERT(MUTEX_HELD(&gnttab_list_lock));

	new_nr_grant_frames = nr_grant_frames + more_frames;
	extra_entries = more_frames * GREFS_PER_GRANT_FRAME;

	nr_glist_frames = (nr_grant_frames * GREFS_PER_GRANT_FRAME + RPP - 1)
	    / RPP;
	new_nr_glist_frames = (new_nr_grant_frames * GREFS_PER_GRANT_FRAME
	    + RPP - 1) / RPP;
	for (i = nr_glist_frames; i < new_nr_glist_frames; i++)
		gnttab_list[i] = kmem_alloc(PAGESIZE, KM_SLEEP);

	for (i = GREFS_PER_GRANT_FRAME * nr_grant_frames;
	    i < GREFS_PER_GRANT_FRAME * new_nr_grant_frames - 1; i++)
		GNTTAB_ENTRY(i) = i + 1;

	GNTTAB_ENTRY(i) = gnttab_free_head;
	gnttab_free_head = GREFS_PER_GRANT_FRAME * nr_grant_frames;
	gnttab_free_count += extra_entries;

	nr_grant_frames = new_nr_grant_frames;

	check_free_callbacks();

	return (0);
}

static int
gnttab_expand(uint_t req_entries)
{
	uint_t cur, extra;

	ASSERT(MUTEX_HELD(&gnttab_list_lock));

	cur = nr_grant_frames;
	extra = ((req_entries + (GREFS_PER_GRANT_FRAME - 1)) /
	    GREFS_PER_GRANT_FRAME);
	if (cur + extra > max_nr_grant_frames())
		return (-1);

	return (grow_gnttab_list(extra));
}

static int
get_free_entries(int count)
{
	int ref, rc;
	grant_ref_t head;

	mutex_enter(&gnttab_list_lock);
	if (gnttab_free_count < count &&
	    ((rc = gnttab_expand(count - gnttab_free_count)) < 0)) {
		mutex_exit(&gnttab_list_lock);
		return (rc);
	}
	ref = head = gnttab_free_head;
	gnttab_free_count -= count;
	while (count-- > 1)
		head = GNTTAB_ENTRY(head);
	gnttab_free_head = GNTTAB_ENTRY(head);
	GNTTAB_ENTRY(head) = GNTTAB_LIST_END;
	mutex_exit(&gnttab_list_lock);
	return (ref);
}

static void
put_free_entry(grant_ref_t ref)
{
	ASSERT(VALID_GRANT_REF(ref));

	mutex_enter(&gnttab_list_lock);
	GNTTAB_ENTRY(ref) = gnttab_free_head;
	gnttab_free_head = ref;
	gnttab_free_count++;
	check_free_callbacks();
	mutex_exit(&gnttab_list_lock);
}

/*
 * Public grant-issuing interface functions
 */

int
gnttab_grant_foreign_access(domid_t domid, gnttab_frame_t frame, int readonly)
{
	int ref;

	if ((ref = get_free_entries(1)) == -1)
		return (-1);

	ASSERT(VALID_GRANT_REF(ref));

	shared[ref].frame = frame;
	shared[ref].domid = domid;
	membar_producer();
	shared[ref].flags = GTF_permit_access | (readonly ? GTF_readonly : 0);

	return (ref);
}

void
gnttab_grant_foreign_access_ref(grant_ref_t ref, domid_t domid,
				gnttab_frame_t frame, int readonly)
{
	ASSERT(VALID_GRANT_REF(ref));

	shared[ref].frame = frame;
	shared[ref].domid = domid;
	membar_producer();
	shared[ref].flags = GTF_permit_access | (readonly ? GTF_readonly : 0);
}


int
gnttab_query_foreign_access(grant_ref_t ref)
{
	uint16_t nflags;

	ASSERT(VALID_GRANT_REF(ref));

	nflags = shared[ref].flags;

	return (nflags & (GTF_reading|GTF_writing));
}

/* ARGSUSED */
int
gnttab_end_foreign_access_ref(grant_ref_t ref, int readonly)
{
	uint16_t flags, nflags;

	ASSERT(VALID_GRANT_REF(ref));

	nflags = shared[ref].flags;
	do {
		if ((flags = nflags) & (GTF_reading|GTF_writing)) {
			cmn_err(CE_WARN, "g.e. still in use!");
			return (0);
		}
	} while ((nflags = CMPXCHG(&shared[ref].flags, flags, 0)) != flags);

	return (1);
}

void
gnttab_end_foreign_access(grant_ref_t ref, int readonly, gnttab_frame_t page)
{
	ASSERT(VALID_GRANT_REF(ref));

	if (gnttab_end_foreign_access_ref(ref, readonly)) {
		put_free_entry(ref);
		/*
		 * XXPV - we don't support freeing a page here
		 */
		if (page != 0) {
			cmn_err(CE_WARN,
	"gnttab_end_foreign_access_ref: using unsupported free_page interface");
			/* free_page(page); */
		}
	} else {
		/*
		 * XXX This needs to be fixed so that the ref and page are
		 * placed on a list to be freed up later.
		 */
		cmn_err(CE_WARN, "leaking g.e. and page still in use!");
	}
}

int
gnttab_grant_foreign_transfer(domid_t domid, pfn_t pfn)
{
	int ref;

	if ((ref = get_free_entries(1)) == -1)
		return (-1);

	ASSERT(VALID_GRANT_REF(ref));

	gnttab_grant_foreign_transfer_ref(ref, domid, pfn);

	return (ref);
}

void
gnttab_grant_foreign_transfer_ref(grant_ref_t ref, domid_t domid, pfn_t pfn)
{
	ASSERT(VALID_GRANT_REF(ref));

	shared[ref].frame = pfn;
	shared[ref].domid = domid;
	membar_producer();
	shared[ref].flags = GTF_accept_transfer;
}

gnttab_frame_t
gnttab_end_foreign_transfer_ref(grant_ref_t ref)
{
	gnttab_frame_t frame;
	uint16_t flags;

	ASSERT(VALID_GRANT_REF(ref));

	/*
	 * If a transfer is not even yet started, try to reclaim the grant
	 * reference and return failure (== 0).
	 */
	while (!((flags = shared[ref].flags) & GTF_transfer_committed)) {
		if (CMPXCHG(&shared[ref].flags, flags, 0) == flags)
			return (0);
		(void) HYPERVISOR_yield();
	}

	/* If a transfer is in progress then wait until it is completed. */
	while (!(flags & GTF_transfer_completed)) {
		flags = shared[ref].flags;
		(void) HYPERVISOR_yield();
	}

	/* Read the frame number /after/ reading completion status. */
	membar_consumer();
	frame = shared[ref].frame;
	ASSERT(frame != 0);

	return (frame);
}

gnttab_frame_t
gnttab_end_foreign_transfer(grant_ref_t ref)
{
	gnttab_frame_t frame;

	ASSERT(VALID_GRANT_REF(ref));

	frame = gnttab_end_foreign_transfer_ref(ref);
	put_free_entry(ref);
	return (frame);
}

void
gnttab_free_grant_reference(grant_ref_t ref)
{
	ASSERT(VALID_GRANT_REF(ref));

	put_free_entry(ref);
}

void
gnttab_free_grant_references(grant_ref_t head)
{
	grant_ref_t ref;
	int count = 1;

	if (head == GNTTAB_LIST_END)
		return;
	mutex_enter(&gnttab_list_lock);
	ref = head;
	while (GNTTAB_ENTRY(ref) != GNTTAB_LIST_END) {
		ref = GNTTAB_ENTRY(ref);
		count++;
	}
	GNTTAB_ENTRY(ref) = gnttab_free_head;
	gnttab_free_head = head;
	gnttab_free_count += count;
	check_free_callbacks();
	mutex_exit(&gnttab_list_lock);
}

int
gnttab_alloc_grant_references(uint16_t count, grant_ref_t *head)
{
	int h = get_free_entries(count);

	if (h == -1)
		return (-1);

	*head = h;

	return (0);
}

int
gnttab_empty_grant_references(const grant_ref_t *private_head)
{
	return (*private_head == GNTTAB_LIST_END);
}

int
gnttab_claim_grant_reference(grant_ref_t *private_head)
{
	grant_ref_t g = *private_head;

	if (g == GNTTAB_LIST_END)
		return (-1);
	*private_head = GNTTAB_ENTRY(g);
	return (g);
}

void
gnttab_release_grant_reference(grant_ref_t *private_head, grant_ref_t release)
{
	ASSERT(VALID_GRANT_REF(release));

	GNTTAB_ENTRY(release) = *private_head;
	*private_head = release;
}

void
gnttab_request_free_callback(struct gnttab_free_callback *callback,
	void (*fn)(void *), void *arg, uint16_t count)
{
	mutex_enter(&gnttab_list_lock);
	if (callback->next)
		goto out;
	callback->fn = fn;
	callback->arg = arg;
	callback->count = count;
	callback->next = gnttab_free_callback_list;
	gnttab_free_callback_list = callback;
	check_free_callbacks();
out:
	mutex_exit(&gnttab_list_lock);
}

void
gnttab_cancel_free_callback(struct gnttab_free_callback *callback)
{
	struct gnttab_free_callback **pcb;

	mutex_enter(&gnttab_list_lock);
	for (pcb = &gnttab_free_callback_list; *pcb; pcb = &(*pcb)->next) {
		if (*pcb == callback) {
			*pcb = callback->next;
			break;
		}
	}
	mutex_exit(&gnttab_list_lock);
}

static gnttab_frame_t *
gnttab_setup(gnttab_setup_table_t *pset)
{
	gnttab_frame_t *frames;

	frames = kmem_alloc(pset->nr_frames * sizeof (gnttab_frame_t),
	    KM_SLEEP);

	/*LINTED: constant in conditional context*/
	set_xen_guest_handle(pset->frame_list, frames);

#ifndef XPV_HVM_DRIVER
	/*
	 * Take pset->nr_frames pages of grant table space from
	 * the hypervisor and map it
	 */
	if ((HYPERVISOR_grant_table_op(GNTTABOP_setup_table, pset, 1) != 0) ||
	    (pset->status != 0)) {
		cmn_err(CE_PANIC, "Grant Table setup failed");
	}
#endif

	return (frames);
}

#ifdef XPV_HVM_DRIVER
static void
gnttab_map(void)
{
	struct xen_add_to_physmap xatp;
	caddr_t va;
	pfn_t pfn;
	int i;

	va = (caddr_t)shared;
	for (i = 0; i < max_nr_grant_frames(); i++) {
		if ((pfn = hat_getpfnum(kas.a_hat, va)) == PFN_INVALID)
			cmn_err(CE_PANIC, "gnttab_map: Invalid pfn");

		xatp.domid = DOMID_SELF;
		xatp.idx = i;
		xatp.space = XENMAPSPACE_grant_table;
		xatp.gpfn = pfn;
		hat_unload(kas.a_hat, va, MMU_PAGESIZE, HAT_UNLOAD);
		/*
		 * This call replaces the existing machine page backing
		 * the given gpfn with the page from the allocated grant
		 * table at index idx. The existing machine page is
		 * returned to the free list.
		 */
		if (HYPERVISOR_memory_op(XENMEM_add_to_physmap, &xatp) != 0)
			panic("Couldn't map grant table");
		hat_devload(kas.a_hat, va, MMU_PAGESIZE, pfn,
		    PROT_READ | PROT_WRITE | HAT_STORECACHING_OK,
		    HAT_LOAD | HAT_LOAD_LOCK | HAT_LOAD_NOCONSIST);
		va += MMU_PAGESIZE;
	}
}
#endif /* XPV_HVM_DRIVER */

void
gnttab_init(void)
{
	gnttab_setup_table_t set;
	int i;
	uint_t nr_init_grefs, max_nr_glist_frames, nr_glist_frames;
	gnttab_frame_t *frames;

	/*
	 * gnttab_init() should only be invoked once.
	 */
	mutex_enter(&gnttab_list_lock);
	ASSERT(nr_grant_frames == 0);
	nr_grant_frames = 1;
	mutex_exit(&gnttab_list_lock);

	max_nr_glist_frames = (max_nr_grant_frames() *
	    GREFS_PER_GRANT_FRAME / RPP);

	set.dom = DOMID_SELF;
	set.nr_frames = max_nr_grant_frames();
	frames = gnttab_setup(&set);

#ifdef XPV_HVM_DRIVER
	shared = (grant_entry_t *)xen_alloc_pages(set.nr_frames);

	gnttab_map();
#else /* XPV_HVM_DRIVER */
	shared = vmem_xalloc(heap_arena, set.nr_frames * MMU_PAGESIZE,
	    MMU_PAGESIZE, 0, 0, 0, 0, VM_SLEEP);
	for (i = 0; i < set.nr_frames; i++) {
		hat_devload(kas.a_hat, (caddr_t)GT_PGADDR(i), PAGESIZE,
		    xen_assign_pfn(frames[i]),
		    PROT_READ | PROT_WRITE | HAT_STORECACHING_OK,
		    HAT_LOAD_LOCK);
	}
#endif

	gnttab_list = kmem_alloc(max_nr_glist_frames * sizeof (grant_ref_t *),
	    KM_SLEEP);

	nr_glist_frames = (nr_grant_frames * GREFS_PER_GRANT_FRAME + RPP - 1)
	    / RPP;
	for (i = 0; i < nr_glist_frames; i++) {
		gnttab_list[i] = kmem_alloc(PAGESIZE, KM_SLEEP);
	}

	kmem_free(frames, set.nr_frames * sizeof (gnttab_frame_t));

	nr_init_grefs = nr_grant_frames * GREFS_PER_GRANT_FRAME;

	for (i = NR_RESERVED_ENTRIES; i < nr_init_grefs - 1; i++)
		GNTTAB_ENTRY(i) = i + 1;

	GNTTAB_ENTRY(nr_init_grefs - 1) = GNTTAB_LIST_END;
	gnttab_free_count = nr_init_grefs - NR_RESERVED_ENTRIES;
	gnttab_free_head  = NR_RESERVED_ENTRIES;
}

void
gnttab_resume(void)
{
	gnttab_setup_table_t set;
	int i;
	gnttab_frame_t *frames;
	uint_t available_frames = max_nr_grant_frames();

	if (available_frames < nr_grant_frames) {
		cmn_err(CE_PANIC, "Hypervisor does not have enough grant "
		    "frames: required(%u), available(%u)", nr_grant_frames,
		    available_frames);
	}

#ifdef XPV_HVM_DRIVER
	gnttab_map();
#endif /* XPV_HVM_DRIVER */

	set.dom = DOMID_SELF;
	set.nr_frames = available_frames;
	frames = gnttab_setup(&set);

	for (i = 0; i < available_frames; i++) {
		(void) HYPERVISOR_update_va_mapping(GT_PGADDR(i),
		    FRAME_TO_MA(frames[i]) | PT_VALID | PT_WRITABLE,
		    UVMF_INVLPG | UVMF_ALL);
	}
	kmem_free(frames, set.nr_frames * sizeof (gnttab_frame_t));
}

void
gnttab_suspend(void)
{
	int i;

	/*
	 * clear grant table mappings before suspending
	 */
	for (i = 0; i < max_nr_grant_frames(); i++) {
		(void) HYPERVISOR_update_va_mapping(GT_PGADDR(i),
		    0, UVMF_INVLPG);
	}
}

/*
 * Local variables:
 *  c-file-style: "solaris"
 *  indent-tabs-mode: t
 *  c-indent-level: 8
 *  c-basic-offset: 8
 *  tab-width: 8
 * End:
 */
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

/*
 * Provides basic C wrappers around hypervisor invocation.
 *
 * i386: eax = vector: ebx, ecx, edx, esi, edi = args 1-5
 *	 eax = return value
 *	 (argument registers may be clobbered on return)
 *
 * amd64:rax = vector: rdi, rsi, rdx, r10, r8, r9 = args 1-6
 *	 rax = return value
 *	 (arguments registers not clobbered on return; rcx, r11 are)
 */

#include <sys/types.h>
#include <sys/errno.h>
#ifndef __xpv
#include <sys/xpv_support.h>
#else
#include <sys/xpv_user.h>
#endif

#include <sys/hypervisor.h>
#include <xen/public/sched.h>
#include <sys/debug.h>
#include <sys/archsystm.h>

long
HYPERVISOR_set_trap_table(trap_info_t *table)
{
	return (__hypercall1(__HYPERVISOR_set_trap_table, (ulong_t)table));
}

int
HYPERVISOR_mmu_update(mmu_update_t *req, int count, int *success_count,
    domid_t domain_id)
{
	return (__hypercall4_int(__HYPERVISOR_mmu_update,
	    (ulong_t)req, (long)count, (ulong_t)success_count,
	    (ulong_t)domain_id));
}

long
HYPERVISOR_set_gdt(ulong_t *frame_list, int entries)
{
	return (__hypercall2(
	    __HYPERVISOR_set_gdt, (ulong_t)frame_list, (long)entries));
}

/*
 * XXPV Seems like "sp" would be a better name for both amd64 and i386?
 * For now stay consistent with xen project source.
 */
long
HYPERVISOR_stack_switch(ulong_t ss, ulong_t esp)
{
	return (__hypercall2(__HYPERVISOR_stack_switch, ss, esp));
}

#if defined(__amd64)

long
HYPERVISOR_set_callbacks(ulong_t event_address, ulong_t failsafe_address,
    ulong_t syscall_address)
{
	return (__hypercall3(__HYPERVISOR_set_callbacks,
	    event_address, failsafe_address, syscall_address));
}

#endif	/* __amd64 */

long
HYPERVISOR_fpu_taskswitch(int set)
{
	return (__hypercall1(__HYPERVISOR_fpu_taskswitch, (long)set));
}

/* *** __HYPERVISOR_sched_op_compat *** OBSOLETED */

long
HYPERVISOR_platform_op(xen_platform_op_t *platform_op)
{
	return (__hypercall1(__HYPERVISOR_platform_op, (ulong_t)platform_op));
}

/* *** __HYPERVISOR_set_debugreg *** NOT IMPLEMENTED */

/* *** __HYPERVISOR_get_debugreg *** NOT IMPLEMENTED */

long
HYPERVISOR_update_descriptor(maddr_t ma, uint64_t desc)
{
#if defined(__amd64)

	return (__hypercall2(__HYPERVISOR_update_descriptor, ma, desc));

#endif
}

long
HYPERVISOR_memory_op(int cmd, void *arg)
{
	return (__hypercall2(__HYPERVISOR_memory_op, (long)cmd,
	    (ulong_t)arg));
}

long
HYPERVISOR_multicall(void *call_list, uint_t nr_calls)
{
	return (__hypercall2(__HYPERVISOR_multicall,
	    (ulong_t)call_list, (ulong_t)nr_calls));
}

int
HYPERVISOR_update_va_mapping(ulong_t va, uint64_t new_pte, ulong_t flags)
{
#if !defined(_BOOT)
	if (IN_XPV_PANIC())
		return (0);
#endif
#if defined(__amd64)

	return (__hypercall3_int(__HYPERVISOR_update_va_mapping, va,
	    new_pte, flags));

#endif	/* __amd64 */
}

/*
 * Note: this timeout must be the Xen system time not hrtime (see
 * xpv_timestamp.c).
 */
long
HYPERVISOR_set_timer_op(uint64_t timeout)
{
#if defined(__amd64)

	return (__hypercall1(__HYPERVISOR_set_timer_op, timeout));

#endif	/* __amd64 */
}

/* *** __HYPERVISOR_event_channel_op_compat *** OBSOLETED */

long
HYPERVISOR_xen_version(int cmd, void *arg)
{
	return (__hypercall2(__HYPERVISOR_xen_version, (long)cmd,
	    (ulong_t)arg));
}

long
HYPERVISOR_console_io(int cmd, int count, char *str)
{
	return (__hypercall3(__HYPERVISOR_console_io, (long)cmd, (long)count,
	    (ulong_t)str));
}

/* *** __HYPERVISOR_physdev_op_compat *** OBSOLETED */

/*
 * ****
 * NOTE: this hypercall should not be called directly for a
 * GNTTABOP_map_grant_ref. Instead xen_map_gref() should be called.
 * ****
 */
long
HYPERVISOR_grant_table_op(uint_t cmd, void *uop, uint_t count)
{
	int ret_val;
	ret_val = __hypercall3(__HYPERVISOR_grant_table_op,
	    (long)cmd, (ulong_t)uop, (ulong_t)count);
	return (ret_val);
}

long
HYPERVISOR_vm_assist(uint_t cmd, uint_t type)
{
	return (__hypercall2(__HYPERVISOR_vm_assist,
	    (ulong_t)cmd, (ulong_t)type));
}

int
HYPERVISOR_update_va_mapping_otherdomain(ulong_t va,
    uint64_t new_pte, ulong_t flags, domid_t domain_id)
{
#if defined(__amd64)

	return (__hypercall4_int(__HYPERVISOR_update_va_mapping_otherdomain,
	    va, new_pte, flags, (ulong_t)domain_id));

#endif	/* __amd64 */
}

/*
 * *** __HYPERVISOR_iret ***
 *   see HYPERVISOR_IRET() macro in i86xpv/sys/machprivregs.h
 */

long
HYPERVISOR_vcpu_op(int cmd, int vcpuid, void *extra_args)
{
	return (__hypercall3(__HYPERVISOR_vcpu_op, (long)cmd, (long)vcpuid,
	    (ulong_t)extra_args));
}

#if defined(__amd64)

long
HYPERVISOR_set_segment_base(int reg, ulong_t value)
{
	return (__hypercall2(__HYPERVISOR_set_segment_base, (long)reg, value));
}

#endif	/* __amd64 */

int
HYPERVISOR_mmuext_op(struct mmuext_op *req, int count, uint_t *success_count,
    domid_t domain_id)
{
	return (__hypercall4_int(__HYPERVISOR_mmuext_op,
	    (ulong_t)req, (long)count, (ulong_t)success_count,
	    (ulong_t)domain_id));
}

long
HYPERVISOR_nmi_op(int cmd, void *arg)
{
	return (__hypercall2(__HYPERVISOR_nmi_op, (long)cmd, (ulong_t)arg));
}

long
HYPERVISOR_sched_op(int cmd, void *arg)
{
	return (__hypercall2(__HYPERVISOR_sched_op,
	    (ulong_t)cmd, (ulong_t)arg));
}

long
HYPERVISOR_callback_op(int cmd, void *arg)
{
	return (__hypercall2(__HYPERVISOR_callback_op,
	    (ulong_t)cmd, (ulong_t)arg));
}

/* *** __HYPERVISOR_xenoprof_op *** NOT IMPLEMENTED */

long
HYPERVISOR_event_channel_op(int cmd, void *arg)
{
	return (__hypercall2(__HYPERVISOR_event_channel_op, (long)cmd,
	    (ulong_t)arg));
}

long
HYPERVISOR_physdev_op(int cmd, void *arg)
{
	return (__hypercall2(__HYPERVISOR_physdev_op, (long)cmd,
	    (ulong_t)arg));
}

long
HYPERVISOR_hvm_op(int cmd, void *arg)
{
	return (__hypercall2(__HYPERVISOR_hvm_op, (long)cmd, (ulong_t)arg));
}

#if defined(__xpv)
long
HYPERVISOR_xsm_op(struct xen_acmctl *arg)
{
	return (__hypercall1(__HYPERVISOR_xsm_op, (ulong_t)arg));
}

long
HYPERVISOR_sysctl(xen_sysctl_t *sysctl)
{
	return (__hypercall1(__HYPERVISOR_sysctl, (ulong_t)sysctl));
}

long
HYPERVISOR_domctl(xen_domctl_t *domctl)
{
	return (__hypercall1(__HYPERVISOR_domctl, (ulong_t)domctl));
}
#endif /* __xpv */

/* *** __HYPERVISOR_kexec_op *** NOT IMPLEMENTED */

/*
 *
 * HYPERCALL HELPER ROUTINES
 *    These don't have there own unique hypercalls.
 *
 */

long
HYPERVISOR_yield(void)
{
	return (HYPERVISOR_sched_op(SCHEDOP_yield, NULL));
}

long
HYPERVISOR_block(void)
{
	return (HYPERVISOR_sched_op(SCHEDOP_block, NULL));
}

long
HYPERVISOR_shutdown(uint_t reason)
{
	struct sched_shutdown sched_shutdown;

	sched_shutdown.reason = reason;

	return (HYPERVISOR_sched_op(SCHEDOP_shutdown, &sched_shutdown));
}

/*
 * Poll one or more event-channel ports, and return when pending.
 * An optional timeout (in nanoseconds, absolute time since boot) may be
 * specified. Note: this timeout must be the Xen system time not hrtime (see
 * xpv_timestamp.c).
 */
long
HYPERVISOR_poll(evtchn_port_t *ports, uint_t nr_ports, uint64_t timeout)
{
	struct sched_poll sched_poll;

	/*LINTED: constant in conditional context*/
	set_xen_guest_handle(sched_poll.ports, ports);
	sched_poll.nr_ports = nr_ports;
	sched_poll.timeout = timeout;

	return (HYPERVISOR_sched_op(SCHEDOP_poll, &sched_poll));
}

long
HYPERVISOR_suspend(ulong_t start_info_mfn)
{
	struct sched_shutdown sched_shutdown;

	sched_shutdown.reason = SHUTDOWN_suspend;

	return (__hypercall3(__HYPERVISOR_sched_op, SCHEDOP_shutdown,
	    (ulong_t)&sched_shutdown, start_info_mfn));
}

long
HYPERVISOR_mca(uint32_t cmd, xen_mc_t *xmcp)
{
	long rv;

	switch (cmd) {
	case XEN_MC_fetch:
	case XEN_MC_physcpuinfo:
	case XEN_MC_msrinject:
	case XEN_MC_mceinject:
		break;

	case XEN_MC_notifydomain:
		return (ENOTSUP);

	default:
		return (EINVAL);
	}

	xmcp->interface_version = XEN_MCA_INTERFACE_VERSION;
	xmcp->cmd = cmd;

	rv = __hypercall1(__HYPERVISOR_mca, (ulong_t)xmcp);

	return (rv);
}
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

/*
 * Copyright (c) 2014 by Delphix. All rights reserved.
 * Copyright 2018 Nexenta Systems, Inc.
 * Copyright 2023 Oxide Computer Company
 */

/*
 * Xen virtual device driver interfaces
 */

/*
 * todo:
 * + name space clean up:
 *	xvdi_* - public xen interfaces, for use by all leaf drivers
 *	xd_* - public xen data structures
 *	i_xvdi_* - implementation private functions
 *	xendev_* - xendev driver interfaces, both internal and in cb_ops/bus_ops
 * + add mdb dcmds to dump ring status
 * + implement xvdi_xxx to wrap xenbus_xxx read/write function
 * + convert (xendev_ring_t *) into xvdi_ring_handle_t
 */
#include <sys/conf.h>
#include <sys/param.h>
#include <sys/kmem.h>
#include <vm/seg_kmem.h>
#include <sys/debug.h>
#include <sys/modctl.h>
#include <sys/autoconf.h>
#include <sys/ddi_impldefs.h>
#include <sys/ddi_subrdefs.h>
#include <sys/ddi.h>
#include <sys/sunddi.h>
#include <sys/sunndi.h>
#include <sys/sunldi.h>
#include <sys/fs/dv_node.h>
#include <sys/avintr.h>
#include <sys/psm.h>
#include <sys/spl.h>
#include <sys/promif.h>
#include <sys/list.h>
#include <sys/bootconf.h>
#include <sys/bootsvcs.h>
#include <sys/bootinfo.h>
#include <sys/note.h>
#include <sys/sysmacros.h>
#ifdef XPV_HVM_DRIVER
#include <sys/xpv_support.h>
#include <sys/hypervisor.h>
#include <public/grant_table.h>
#include <public/xen.h>
#include <public/io/xenbus.h>
#include <public/io/xs_wire.h>
#include <public/event_channel.h>
#include <public/io/xenbus.h>
#else /* XPV_HVM_DRIVER */
#include <sys/hypervisor.h>
#include <sys/xen_mmu.h>
#include <xen/sys/xenbus_impl.h>
#include <sys/evtchn_impl.h>
#endif /* XPV_HVM_DRIVER */
#include <sys/gnttab.h>
#include <xen/sys/xendev.h>
#include <vm/hat_i86.h>
#include <sys/scsi/generic/inquiry.h>
#include <util/sscanf.h>
#include <xen/public/io/xs_wire.h>


#define	isdigit(ch)	((ch) >= '0' && (ch) <= '9')
#define	isxdigit(ch)	(isdigit(ch) || ((ch) >= 'a' && (ch) <= 'f') || \
			((ch) >= 'A' && (ch) <= 'F'))

static void xvdi_ring_init_sring(xendev_ring_t *);
static void xvdi_ring_init_front_ring(xendev_ring_t *, size_t, size_t);
#ifndef XPV_HVM_DRIVER
static void xvdi_ring_init_back_ring(xendev_ring_t *, size_t, size_t);
#endif
static void xvdi_reinit_ring(dev_info_t *, grant_ref_t *, xendev_ring_t *);

static int i_xvdi_add_watches(dev_info_t *);
static void i_xvdi_rem_watches(dev_info_t *);

static int i_xvdi_add_watch_oestate(dev_info_t *);
static void i_xvdi_rem_watch_oestate(dev_info_t *);
static void i_xvdi_oestate_cb(struct xenbus_device *, XenbusState);
static void i_xvdi_oestate_handler(void *);

static int i_xvdi_add_watch_hpstate(dev_info_t *);
static void i_xvdi_rem_watch_hpstate(dev_info_t *);
static void i_xvdi_hpstate_cb(struct xenbus_watch *, const char **,
    unsigned int);
static void i_xvdi_hpstate_handler(void *);

static int i_xvdi_add_watch_bepath(dev_info_t *);
static void i_xvdi_rem_watch_bepath(dev_info_t *);
static void i_xvdi_bepath_cb(struct xenbus_watch *, const char **,
    unsigned in);

static void xendev_offline_device(void *);

static void i_xvdi_probe_path_cb(struct xenbus_watch *, const char **,
    unsigned int);
static void i_xvdi_probe_path_handler(void *);

typedef struct oestate_evt {
	dev_info_t *dip;
	XenbusState state;
} i_oestate_evt_t;

typedef struct xd_cfg {
	xendev_devclass_t devclass;
	char *xsdev;
	char *xs_path_fe;
	char *xs_path_be;
	char *node_fe;
	char *node_be;
	char *device_type;
	int xd_ipl;
	int flags;
} i_xd_cfg_t;

#define	XD_DOM_ZERO	0x01	/* dom0 only. */
#define	XD_DOM_GUEST	0x02	/* Guest domains (i.e. non-dom0). */
#define	XD_DOM_IO	0x04	/* IO domains. */

#define	XD_DOM_ALL	(XD_DOM_ZERO | XD_DOM_GUEST)

static i_xd_cfg_t xdci[] = {
#ifndef XPV_HVM_DRIVER
	{ XEN_CONSOLE, NULL, NULL, NULL, "xencons", NULL,
	    "console", IPL_CONS, XD_DOM_ALL, },
#endif

	{ XEN_VNET, "vif", "device/vif", "backend/vif", "xnf", "xnb",
	    "network", IPL_VIF, XD_DOM_ALL, },

	{ XEN_VBLK, "vbd", "device/vbd", "backend/vbd", "xdf", "xdb",
	    "block", IPL_VBD, XD_DOM_ALL, },

	{ XEN_BLKTAP, "tap", NULL, "backend/tap", NULL, "xpvtap",
	    "block", IPL_VBD, XD_DOM_ALL, },

#ifndef XPV_HVM_DRIVER
	{ XEN_XENBUS, NULL, NULL, NULL, "xenbus", NULL,
	    NULL, 0, XD_DOM_ALL, },

	{ XEN_DOMCAPS, NULL, NULL, NULL, "domcaps", NULL,
	    NULL, 0, XD_DOM_ALL, },

	{ XEN_BALLOON, NULL, NULL, NULL, "balloon", NULL,
	    NULL, 0, XD_DOM_ALL, },
#endif

	{ XEN_EVTCHN, NULL, NULL, NULL, "evtchn", NULL,
	    NULL, 0, XD_DOM_ZERO, },

	{ XEN_PRIVCMD, NULL, NULL, NULL, "privcmd", NULL,
	    NULL, 0, XD_DOM_ZERO, },
};
#define	NXDC	(sizeof (xdci) / sizeof (xdci[0]))

static void i_xvdi_enum_fe(dev_info_t *, i_xd_cfg_t *);
static void i_xvdi_enum_be(dev_info_t *, i_xd_cfg_t *);
static void i_xvdi_enum_worker(dev_info_t *, i_xd_cfg_t *, char *);

/*
 * Xen device channel device access and DMA attributes
 */
static ddi_device_acc_attr_t xendev_dc_accattr = {
	DDI_DEVICE_ATTR_V0, DDI_NEVERSWAP_ACC, DDI_STRICTORDER_ACC
};

static ddi_dma_attr_t xendev_dc_dmaattr = {
	DMA_ATTR_V0,		/* version of this structure */
	0,			/* lowest usable address */
	0xffffffffffffffffULL,	/* highest usable address */
	0x7fffffff,		/* maximum DMAable byte count */
	MMU_PAGESIZE,		/* alignment in bytes */
	0x7ff,			/* bitmap of burst sizes */
	1,			/* minimum transfer */
	0xffffffffU,		/* maximum transfer */
	0xffffffffffffffffULL,	/* maximum segment length */
	1,			/* maximum number of segments */
	1,			/* granularity */
	0,			/* flags (reserved) */
};

static dev_info_t *xendev_dip = NULL;

#define	XVDI_DBG_STATE	0x01
#define	XVDI_DBG_PROBE	0x02

#ifdef DEBUG
int i_xvdi_debug = 0;

#define	XVDI_DPRINTF(flag, format, ...)			\
{							\
	if (i_xvdi_debug & (flag))			\
		prom_printf((format), __VA_ARGS__);	\
}
#else
#define	XVDI_DPRINTF(flag, format, ...)
#endif /* DEBUG */

static i_xd_cfg_t *
i_xvdi_devclass2cfg(xendev_devclass_t devclass)
{
	i_xd_cfg_t *xdcp;
	int i;

	for (i = 0, xdcp = xdci; i < NXDC; i++, xdcp++)
		if (xdcp->devclass == devclass)
			return (xdcp);

	return (NULL);
}

int
xvdi_init_dev(dev_info_t *dip)
{
	xendev_devclass_t devcls;
	int vdevnum;
	domid_t domid;
	struct xendev_ppd *pdp;
	i_xd_cfg_t *xdcp;
	boolean_t backend;
	char xsnamebuf[TYPICALMAXPATHLEN];
	char *xsname;
	void *prop_str;
	unsigned int prop_len;
	char unitaddr[16];

	devcls = ddi_prop_get_int(DDI_DEV_T_ANY, dip,
	    DDI_PROP_DONTPASS, "devclass", XEN_INVAL);
	vdevnum = ddi_prop_get_int(DDI_DEV_T_ANY, dip,
	    DDI_PROP_DONTPASS, "vdev", VDEV_NOXS);
	domid = (domid_t)ddi_prop_get_int(DDI_DEV_T_ANY, dip,
	    DDI_PROP_DONTPASS, "domain", DOMID_SELF);

	backend = (domid != DOMID_SELF);
	xdcp = i_xvdi_devclass2cfg(devcls);
	if (xdcp->device_type != NULL)
		(void) ndi_prop_update_string(DDI_DEV_T_NONE, dip,
		    "device_type", xdcp->device_type);

	pdp = kmem_zalloc(sizeof (*pdp), KM_SLEEP);
	pdp->xd_domain = domid;
	pdp->xd_vdevnum = vdevnum;
	pdp->xd_devclass = devcls;
	pdp->xd_evtchn = INVALID_EVTCHN;
	list_create(&pdp->xd_xb_watches, sizeof (xd_xb_watches_t),
	    offsetof(xd_xb_watches_t, xxw_list));
	mutex_init(&pdp->xd_evt_lk, NULL, MUTEX_DRIVER, NULL);
	mutex_init(&pdp->xd_ndi_lk, NULL, MUTEX_DRIVER, NULL);
	ddi_set_parent_data(dip, pdp);

	/*
	 * devices that do not need to interact with xenstore
	 */
	if (vdevnum == VDEV_NOXS) {
		(void) ndi_prop_update_string(DDI_DEV_T_NONE, dip,
		    "unit-address", "0");
		if (devcls == XEN_CONSOLE)
			(void) ndi_prop_update_string(DDI_DEV_T_NONE, dip,
			    "pm-hardware-state", "needs-suspend-resume");
		return (DDI_SUCCESS);
	}

	/*
	 * PV devices that need to probe xenstore
	 */

	(void) ndi_prop_update_string(DDI_DEV_T_NONE, dip,
	    "pm-hardware-state", "needs-suspend-resume");

	xsname = xsnamebuf;
	if (!backend)
		(void) snprintf(xsnamebuf, sizeof (xsnamebuf),
		    "%s/%d", xdcp->xs_path_fe, vdevnum);
	else
		(void) snprintf(xsnamebuf, sizeof (xsnamebuf),
		    "%s/%d/%d", xdcp->xs_path_be, domid, vdevnum);
	if ((xenbus_read_driver_state(xsname) >= XenbusStateClosing)) {
		/* Don't try to init a dev that may be closing */
		mutex_destroy(&pdp->xd_ndi_lk);
		mutex_destroy(&pdp->xd_evt_lk);
		kmem_free(pdp, sizeof (*pdp));
		ddi_set_parent_data(dip, NULL);
		return (DDI_FAILURE);
	}

	pdp->xd_xsdev.nodename = i_ddi_strdup(xsname, KM_SLEEP);
	pdp->xd_xsdev.devicetype = xdcp->xsdev;
	pdp->xd_xsdev.frontend = (backend ? 0 : 1);
	pdp->xd_xsdev.data = dip;
	pdp->xd_xsdev.otherend_id = (backend ? domid : -1);
	if (i_xvdi_add_watches(dip) != DDI_SUCCESS) {
		cmn_err(CE_WARN, "xvdi_init_dev: "
		    "cannot add watches for %s", xsname);
		xvdi_uninit_dev(dip);
		return (DDI_FAILURE);
	}

	if (backend)
		return (DDI_SUCCESS);

	/*
	 * The unit-address for frontend devices is the name of the
	 * of the xenstore node containing the device configuration
	 * and is contained in the 'vdev' property.
	 * VIF devices are named using an incrementing integer.
	 * VBD devices are either named using the 32-bit dev_t value
	 * for linux 'hd' and 'xvd' devices, or a simple integer value
	 * in the range 0..767.  768 is the base value of the linux
	 * dev_t namespace, the dev_t value for 'hda'.
	 */
	(void) snprintf(unitaddr, sizeof (unitaddr), "%d", vdevnum);
	(void) ndi_prop_update_string(DDI_DEV_T_NONE, dip, "unit-address",
	    unitaddr);

	switch (devcls) {
	case XEN_VNET:
		if (xenbus_read(XBT_NULL, xsname, "mac", (void *)&prop_str,
		    &prop_len) != 0)
			break;
		(void) ndi_prop_update_string(DDI_DEV_T_NONE, dip, "mac",
		    prop_str);
		kmem_free(prop_str, prop_len);
		break;
	case XEN_VBLK:
		/*
		 * cache a copy of the otherend name
		 * for ease of observeability
		 */
		if (xenbus_read(XBT_NULL, pdp->xd_xsdev.otherend, "dev",
		    &prop_str, &prop_len) != 0)
			break;
		(void) ndi_prop_update_string(DDI_DEV_T_NONE, dip,
		    "dev-address", prop_str);
		kmem_free(prop_str, prop_len);
		break;
	default:
		break;
	}

	return (DDI_SUCCESS);
}

void
xvdi_uninit_dev(dev_info_t *dip)
{
	struct xendev_ppd *pdp = ddi_get_parent_data(dip);

	if (pdp != NULL) {
		/* Remove any registered callbacks. */
		xvdi_remove_event_handler(dip, NULL);

		/* Remove any registered watches. */
		i_xvdi_rem_watches(dip);

		/* tell other end to close */
		if (pdp->xd_xsdev.otherend_id != (domid_t)-1)
			(void) xvdi_switch_state(dip, XBT_NULL,
			    XenbusStateClosed);

		if (pdp->xd_xsdev.nodename != NULL)
			kmem_free((char *)(pdp->xd_xsdev.nodename),
			    strlen(pdp->xd_xsdev.nodename) + 1);

		ddi_set_parent_data(dip, NULL);

		mutex_destroy(&pdp->xd_ndi_lk);
		mutex_destroy(&pdp->xd_evt_lk);
		kmem_free(pdp, sizeof (*pdp));
	}
}

/*
 * Bind the event channel for this device instance.
 * Currently we only support one evtchn per device instance.
 */
int
xvdi_bind_evtchn(dev_info_t *dip, evtchn_port_t evtchn)
{
	struct xendev_ppd *pdp;
	domid_t oeid;
	int r;

	pdp = ddi_get_parent_data(dip);
	ASSERT(pdp != NULL);
	ASSERT(pdp->xd_evtchn == INVALID_EVTCHN);

	mutex_enter(&pdp->xd_evt_lk);
	if (pdp->xd_devclass == XEN_CONSOLE) {
		if (!DOMAIN_IS_INITDOMAIN(xen_info)) {
			pdp->xd_evtchn = xen_info->console.domU.evtchn;
		} else {
			pdp->xd_evtchn = INVALID_EVTCHN;
			mutex_exit(&pdp->xd_evt_lk);
			return (DDI_SUCCESS);
		}
	} else {
		oeid = pdp->xd_xsdev.otherend_id;
		if (oeid == (domid_t)-1) {
			mutex_exit(&pdp->xd_evt_lk);
			return (DDI_FAILURE);
		}

		if ((r = xen_bind_interdomain(oeid, evtchn, &pdp->xd_evtchn))) {
			xvdi_dev_error(dip, r, "bind event channel");
			mutex_exit(&pdp->xd_evt_lk);
			return (DDI_FAILURE);
		}
	}
#ifndef XPV_HVM_DRIVER
	pdp->xd_ispec.intrspec_vec = ec_bind_evtchn_to_irq(pdp->xd_evtchn);
#endif
	mutex_exit(&pdp->xd_evt_lk);

	return (DDI_SUCCESS);
}

/*
 * Allocate an event channel for this device instance.
 * Currently we only support one evtchn per device instance.
 */
int
xvdi_alloc_evtchn(dev_info_t *dip)
{
	struct xendev_ppd *pdp;
	domid_t oeid;
	int rv;

	pdp = ddi_get_parent_data(dip);
	ASSERT(pdp != NULL);
	ASSERT(pdp->xd_evtchn == INVALID_EVTCHN);

	mutex_enter(&pdp->xd_evt_lk);
	if (pdp->xd_devclass == XEN_CONSOLE) {
		if (!DOMAIN_IS_INITDOMAIN(xen_info)) {
			pdp->xd_evtchn = xen_info->console.domU.evtchn;
		} else {
			pdp->xd_evtchn = INVALID_EVTCHN;
			mutex_exit(&pdp->xd_evt_lk);
			return (DDI_SUCCESS);
		}
	} else {
		oeid = pdp->xd_xsdev.otherend_id;
		if (oeid == (domid_t)-1) {
			mutex_exit(&pdp->xd_evt_lk);
			return (DDI_FAILURE);
		}

		if ((rv = xen_alloc_unbound_evtchn(oeid, &pdp->xd_evtchn))) {
			xvdi_dev_error(dip, rv, "bind event channel");
			mutex_exit(&pdp->xd_evt_lk);
			return (DDI_FAILURE);
		}
	}
#ifndef XPV_HVM_DRIVER
	pdp->xd_ispec.intrspec_vec = ec_bind_evtchn_to_irq(pdp->xd_evtchn);
#endif
	mutex_exit(&pdp->xd_evt_lk);

	return (DDI_SUCCESS);
}

/*
 * Unbind the event channel for this device instance.
 * Currently we only support one evtchn per device instance.
 */
void
xvdi_free_evtchn(dev_info_t *dip)
{
	struct xendev_ppd *pdp;

	pdp = ddi_get_parent_data(dip);
	ASSERT(pdp != NULL);

	mutex_enter(&pdp->xd_evt_lk);
	if (pdp->xd_evtchn != INVALID_EVTCHN) {
#ifndef XPV_HVM_DRIVER
		ec_unbind_irq(pdp->xd_ispec.intrspec_vec);
		pdp->xd_ispec.intrspec_vec = 0;
#endif
		pdp->xd_evtchn = INVALID_EVTCHN;
	}
	mutex_exit(&pdp->xd_evt_lk);
}

#ifndef XPV_HVM_DRIVER
/*
 * Map an inter-domain communication ring for a virtual device.
 * This is used by backend drivers.
 */
int
xvdi_map_ring(dev_info_t *dip, size_t nentry, size_t entrysize,
    grant_ref_t gref, xendev_ring_t **ringpp)
{
	domid_t oeid;
	gnttab_map_grant_ref_t mapop;
	gnttab_unmap_grant_ref_t unmapop;
	caddr_t ringva;
	ddi_acc_hdl_t *ap;
	ddi_acc_impl_t *iap;
	xendev_ring_t *ring;
	int err;
	char errstr[] = "mapping in ring buffer";

	ring = kmem_zalloc(sizeof (xendev_ring_t), KM_SLEEP);
	oeid = xvdi_get_oeid(dip);

	/* alloc va in backend dom for ring buffer */
	ringva = vmem_xalloc(heap_arena, PAGESIZE, PAGESIZE,
	    0, 0, 0, 0, VM_SLEEP);

	/* map in ring page */
	hat_prepare_mapping(kas.a_hat, ringva, NULL);
	mapop.host_addr = (uint64_t)(uintptr_t)ringva;
	mapop.flags = GNTMAP_host_map;
	mapop.ref = gref;
	mapop.dom = oeid;
	err = xen_map_gref(GNTTABOP_map_grant_ref, &mapop, 1, B_FALSE);
	if (err) {
		xvdi_fatal_error(dip, err, errstr);
		goto errout1;
	}

	if (mapop.status != 0) {
		xvdi_fatal_error(dip, err, errstr);
		goto errout2;
	}
	ring->xr_vaddr = ringva;
	ring->xr_grant_hdl = mapop.handle;
	ring->xr_gref = gref;

	/*
	 * init an acc handle and associate it w/ this ring
	 * this is only for backend drivers. we get the memory by calling
	 * vmem_xalloc(), instead of calling any ddi function, so we have
	 * to init an acc handle by ourselves
	 */
	ring->xr_acc_hdl = impl_acc_hdl_alloc(KM_SLEEP, NULL);
	ap = impl_acc_hdl_get(ring->xr_acc_hdl);
	ap->ah_vers = VERS_ACCHDL;
	ap->ah_dip = dip;
	ap->ah_xfermodes = DDI_DMA_CONSISTENT;
	ap->ah_acc = xendev_dc_accattr;
	iap = (ddi_acc_impl_t *)ap->ah_platform_private;
	iap->ahi_acc_attr |= DDI_ACCATTR_CPU_VADDR;
	impl_acc_hdl_init(ap);
	ap->ah_offset = 0;
	ap->ah_len = (off_t)PAGESIZE;
	ap->ah_addr = ring->xr_vaddr;

	/* init backend ring */
	xvdi_ring_init_back_ring(ring, nentry, entrysize);

	*ringpp = ring;

	return (DDI_SUCCESS);

errout2:
	/* unmap ring page */
	unmapop.host_addr = (uint64_t)(uintptr_t)ringva;
	unmapop.handle = ring->xr_grant_hdl;
	unmapop.dev_bus_addr = 0;
	(void) HYPERVISOR_grant_table_op(GNTTABOP_unmap_grant_ref, &unmapop, 1);
	hat_release_mapping(kas.a_hat, ringva);
errout1:
	vmem_xfree(heap_arena, ringva, PAGESIZE);
	kmem_free(ring, sizeof (xendev_ring_t));
	return (DDI_FAILURE);
}

/*
 * Unmap a ring for a virtual device.
 * This is used by backend drivers.
 */
void
xvdi_unmap_ring(xendev_ring_t *ring)
{
	gnttab_unmap_grant_ref_t unmapop;

	ASSERT((ring != NULL) && (ring->xr_vaddr != NULL));

	impl_acc_hdl_free(ring->xr_acc_hdl);
	unmapop.host_addr = (uint64_t)(uintptr_t)ring->xr_vaddr;
	unmapop.handle = ring->xr_grant_hdl;
	unmapop.dev_bus_addr = 0;
	(void) HYPERVISOR_grant_table_op(GNTTABOP_unmap_grant_ref, &unmapop, 1);
	hat_release_mapping(kas.a_hat, ring->xr_vaddr);
	vmem_xfree(heap_arena, ring->xr_vaddr, PAGESIZE);
	kmem_free(ring, sizeof (xendev_ring_t));
}
#endif /* XPV_HVM_DRIVER */

/*
 * Re-initialise an inter-domain communications ring for the backend domain.
 * ring will be re-initialized after re-grant succeed
 * ring will be freed if fails to re-grant access to backend domain
 * so, don't keep useful data in the ring
 * used only in frontend driver
 */
static void
xvdi_reinit_ring(dev_info_t *dip, grant_ref_t *gref, xendev_ring_t *ringp)
{
	paddr_t rpaddr;
	maddr_t rmaddr;

	ASSERT((ringp != NULL) && (ringp->xr_paddr != 0));
	rpaddr = ringp->xr_paddr;

	rmaddr = DOMAIN_IS_INITDOMAIN(xen_info) ? rpaddr : pa_to_ma(rpaddr);
	gnttab_grant_foreign_access_ref(ringp->xr_gref, xvdi_get_oeid(dip),
	    rmaddr >> PAGESHIFT, 0);
	*gref = ringp->xr_gref;

	/* init frontend ring */
	xvdi_ring_init_sring(ringp);
	xvdi_ring_init_front_ring(ringp, ringp->xr_sring.fr.nr_ents,
	    ringp->xr_entry_size);
}

/*
 * allocate Xen inter-domain communications ring for Xen virtual devices
 * used only in frontend driver
 * if *ringpp is not NULL, we'll simply re-init it
 */
int
xvdi_alloc_ring(dev_info_t *dip, size_t nentry, size_t entrysize,
    grant_ref_t *gref, xendev_ring_t **ringpp)
{
	size_t len;
	xendev_ring_t *ring;
	ddi_dma_cookie_t dma_cookie;
	uint_t ncookies;
	grant_ref_t ring_gref;
	domid_t oeid;
	maddr_t rmaddr;

	if (*ringpp) {
		xvdi_reinit_ring(dip, gref, *ringpp);
		return (DDI_SUCCESS);
	}

	*ringpp = ring = kmem_zalloc(sizeof (xendev_ring_t), KM_SLEEP);
	oeid = xvdi_get_oeid(dip);

	/*
	 * Allocate page for this ring buffer
	 */
	if (ddi_dma_alloc_handle(dip, &xendev_dc_dmaattr, DDI_DMA_SLEEP,
	    0, &ring->xr_dma_hdl) != DDI_SUCCESS)
		goto err;

	if (ddi_dma_mem_alloc(ring->xr_dma_hdl, PAGESIZE,
	    &xendev_dc_accattr, DDI_DMA_CONSISTENT, DDI_DMA_SLEEP, 0,
	    &ring->xr_vaddr, &len, &ring->xr_acc_hdl) != DDI_SUCCESS) {
		ddi_dma_free_handle(&ring->xr_dma_hdl);
		goto err;
	}

	if (ddi_dma_addr_bind_handle(ring->xr_dma_hdl, NULL,
	    ring->xr_vaddr, len, DDI_DMA_RDWR | DDI_DMA_CONSISTENT,
	    DDI_DMA_SLEEP, 0, &dma_cookie, &ncookies) != DDI_DMA_MAPPED) {
		ddi_dma_mem_free(&ring->xr_acc_hdl);
		ring->xr_vaddr = NULL;
		ddi_dma_free_handle(&ring->xr_dma_hdl);
		goto err;
	}
	ASSERT(ncookies == 1);
	ring->xr_paddr = dma_cookie.dmac_laddress;
	rmaddr = DOMAIN_IS_INITDOMAIN(xen_info) ? ring->xr_paddr :
	    pa_to_ma(ring->xr_paddr);

	if ((ring_gref = gnttab_grant_foreign_access(oeid,
	    rmaddr >> PAGESHIFT, 0)) == (grant_ref_t)-1) {
		(void) ddi_dma_unbind_handle(ring->xr_dma_hdl);
		ddi_dma_mem_free(&ring->xr_acc_hdl);
		ring->xr_vaddr = NULL;
		ddi_dma_free_handle(&ring->xr_dma_hdl);
		goto err;
	}
	*gref = ring->xr_gref = ring_gref;

	/* init frontend ring */
	xvdi_ring_init_sring(ring);
	xvdi_ring_init_front_ring(ring, nentry, entrysize);

	return (DDI_SUCCESS);

err:
	kmem_free(ring, sizeof (xendev_ring_t));
	return (DDI_FAILURE);
}

/*
 * Release ring buffers allocated for Xen devices
 * used for frontend driver
 */
void
xvdi_free_ring(xendev_ring_t *ring)
{
	ASSERT((ring != NULL) && (ring->xr_vaddr != NULL));

	(void) gnttab_end_foreign_access_ref(ring->xr_gref, 0);
	(void) ddi_dma_unbind_handle(ring->xr_dma_hdl);
	ddi_dma_mem_free(&ring->xr_acc_hdl);
	ddi_dma_free_handle(&ring->xr_dma_hdl);
	kmem_free(ring, sizeof (xendev_ring_t));
}

dev_info_t *
xvdi_create_dev(dev_info_t *parent, xendev_devclass_t devclass,
    domid_t dom, int vdev)
{
	dev_info_t *dip;
	boolean_t backend;
	i_xd_cfg_t *xdcp;
	char xsnamebuf[TYPICALMAXPATHLEN];
	char *type, *node = NULL, *xsname = NULL;
	unsigned int tlen;
	int ret;

	ASSERT(DEVI_BUSY_OWNED(parent));

	backend = (dom != DOMID_SELF);
	xdcp = i_xvdi_devclass2cfg(devclass);
	ASSERT(xdcp != NULL);

	if (vdev != VDEV_NOXS) {
		if (!backend) {
			(void) snprintf(xsnamebuf, sizeof (xsnamebuf),
			    "%s/%d", xdcp->xs_path_fe, vdev);
			xsname = xsnamebuf;
			node = xdcp->node_fe;
		} else {
			(void) snprintf(xsnamebuf, sizeof (xsnamebuf),
			    "%s/%d/%d", xdcp->xs_path_be, dom, vdev);
			xsname = xsnamebuf;
			node = xdcp->node_be;
		}
	} else {
		node = xdcp->node_fe;
	}

	/* Must have a driver to use. */
	if (node == NULL)
		return (NULL);

	/*
	 * We need to check the state of this device before we go
	 * further, otherwise we'll end up with a dead loop if
	 * anything goes wrong.
	 */
	if ((xsname != NULL) &&
	    (xenbus_read_driver_state(xsname) >= XenbusStateClosing))
		return (NULL);

	ndi_devi_alloc_sleep(parent, node, DEVI_SID_NODEID, &dip);

	/*
	 * Driver binding uses the compatible property _before_ the
	 * node name, so we set the node name to the 'model' of the
	 * device (i.e. 'xnb' or 'xdb') and, if 'type' is present,
	 * encode both the model and the type in a compatible property
	 * (i.e. 'xnb,netfront' or 'xnb,SUNW_mac').  This allows a
	 * driver binding based on the <model,type> pair _before_ a
	 * binding based on the node name.
	 */
	if ((xsname != NULL) &&
	    (xenbus_read(XBT_NULL, xsname, "type", (void *)&type, &tlen)
	    == 0)) {
		size_t clen;
		char *c[1];

		clen = strlen(node) + strlen(type) + 2;
		c[0] = kmem_alloc(clen, KM_SLEEP);
		(void) snprintf(c[0], clen, "%s,%s", node, type);

		(void) ndi_prop_update_string_array(DDI_DEV_T_NONE,
		    dip, "compatible", (char **)c, 1);

		kmem_free(c[0], clen);
		kmem_free(type, tlen);
	}

	(void) ndi_prop_update_int(DDI_DEV_T_NONE, dip, "devclass", devclass);
	(void) ndi_prop_update_int(DDI_DEV_T_NONE, dip, "domain", dom);
	(void) ndi_prop_update_int(DDI_DEV_T_NONE, dip, "vdev", vdev);

	if (i_ddi_devi_attached(parent))
		ret = ndi_devi_online(dip, 0);
	else
		ret = ndi_devi_bind_driver(dip, 0);
	if (ret != NDI_SUCCESS)
		(void) ndi_devi_offline(dip, NDI_DEVI_REMOVE);

	return (dip);
}

/*
 * xendev_enum_class()
 */
void
xendev_enum_class(dev_info_t *parent, xendev_devclass_t devclass)
{
	boolean_t dom0 = DOMAIN_IS_INITDOMAIN(xen_info);
	boolean_t domU = !dom0;
	i_xd_cfg_t *xdcp;

	xdcp = i_xvdi_devclass2cfg(devclass);
	ASSERT(xdcp != NULL);

	if (dom0 && !(xdcp->flags & XD_DOM_ZERO))
		return;

	if (domU && !(xdcp->flags & XD_DOM_GUEST))
		return;

	if (xdcp->xsdev == NULL) {
		/*
		 * Don't need to probe this kind of device from the
		 * store, just create one if it doesn't exist.
		 */

		ndi_devi_enter(parent);
		if (xvdi_find_dev(parent, devclass, DOMID_SELF, VDEV_NOXS)
		    == NULL)
			(void) xvdi_create_dev(parent, devclass,
			    DOMID_SELF, VDEV_NOXS);
		ndi_devi_exit(parent);
	} else {
		/*
		 * Probe this kind of device from the store, both
		 * frontend and backend.
		 */
		if (xdcp->node_fe != NULL) {
			i_xvdi_enum_fe(parent, xdcp);
		}
		if (xdcp->node_be != NULL) {
			i_xvdi_enum_be(parent, xdcp);
		}
	}
}

/*
 * xendev_enum_all()
 */
void
xendev_enum_all(dev_info_t *parent, boolean_t store_unavailable)
{
	int i;
	i_xd_cfg_t *xdcp;
	boolean_t dom0 = DOMAIN_IS_INITDOMAIN(xen_info);

	for (i = 0, xdcp = xdci; i < NXDC; i++, xdcp++) {
		/*
		 * Dom0 relies on watchpoints to create non-soft
		 * devices - don't attempt to iterate over the store.
		 */
		if (dom0 && (xdcp->xsdev != NULL))
			continue;

		/*
		 * If the store is not yet available, don't attempt to
		 * iterate.
		 */
		if (store_unavailable && (xdcp->xsdev != NULL))
			continue;

		xendev_enum_class(parent, xdcp->devclass);
	}
}

xendev_devclass_t
xendev_nodename_to_devclass(char *nodename)
{
	int i;
	i_xd_cfg_t *xdcp;

	/*
	 * This relies on the convention that variants of a base
	 * driver share the same prefix and that there are no drivers
	 * which share a common prefix with the name of any other base
	 * drivers.
	 *
	 * So for a base driver 'xnb' (which is the name listed in
	 * xdci) the variants all begin with the string 'xnb' (in fact
	 * they are 'xnbe', 'xnbo' and 'xnbu') and there are no other
	 * base drivers which have the prefix 'xnb'.
	 */
	ASSERT(nodename != NULL);
	for (i = 0, xdcp = xdci; i < NXDC; i++, xdcp++) {
		if (((xdcp->node_fe != NULL) &&
		    (strncmp(nodename, xdcp->node_fe,
		    strlen(xdcp->node_fe)) == 0)) ||
		    ((xdcp->node_be != NULL) &&
		    (strncmp(nodename, xdcp->node_be,
		    strlen(xdcp->node_be)) == 0)))

			return (xdcp->devclass);
	}
	return (XEN_INVAL);
}

int
xendev_devclass_ipl(xendev_devclass_t devclass)
{
	i_xd_cfg_t *xdcp;

	xdcp = i_xvdi_devclass2cfg(devclass);
	ASSERT(xdcp != NULL);

	return (xdcp->xd_ipl);
}

/*
 * Determine if a devinfo instance exists of a particular device
 * class, domain and xenstore virtual device number.
 */
dev_info_t *
xvdi_find_dev(dev_info_t *parent, xendev_devclass_t devclass,
    domid_t dom, int vdev)
{
	dev_info_t *dip;

	ASSERT(DEVI_BUSY_OWNED(parent));

	switch (devclass) {
	case XEN_CONSOLE:
	case XEN_XENBUS:
	case XEN_DOMCAPS:
	case XEN_BALLOON:
	case XEN_EVTCHN:
	case XEN_PRIVCMD:
		/* Console and soft devices have no vdev. */
		vdev = VDEV_NOXS;
		break;
	default:
		break;
	}

	for (dip = ddi_get_child(parent); dip != NULL;
	    dip = ddi_get_next_sibling(dip)) {
		int *vdevnump, *domidp, *devclsp, vdevnum;
		uint_t ndomid, nvdevnum, ndevcls;
		xendev_devclass_t devcls;
		domid_t domid;
		struct xendev_ppd *pdp = ddi_get_parent_data(dip);

		if (pdp == NULL) {
			if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, dip,
			    DDI_PROP_DONTPASS, "domain", &domidp, &ndomid) !=
			    DDI_PROP_SUCCESS)
				continue;
			ASSERT(ndomid == 1);
			domid = (domid_t)*domidp;
			ddi_prop_free(domidp);

			if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, dip,
			    DDI_PROP_DONTPASS, "vdev", &vdevnump, &nvdevnum) !=
			    DDI_PROP_SUCCESS)
				continue;
			ASSERT(nvdevnum == 1);
			vdevnum = *vdevnump;
			ddi_prop_free(vdevnump);

			if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, dip,
			    DDI_PROP_DONTPASS, "devclass", &devclsp,
			    &ndevcls) != DDI_PROP_SUCCESS)
				continue;
			ASSERT(ndevcls == 1);
			devcls = (xendev_devclass_t)*devclsp;
			ddi_prop_free(devclsp);
		} else {
			domid = pdp->xd_domain;
			vdevnum = pdp->xd_vdevnum;
			devcls = pdp->xd_devclass;
		}

		if ((domid == dom) && (vdevnum == vdev) && (devcls == devclass))
			return (dip);
	}
	return (NULL);
}

int
xvdi_get_evtchn(dev_info_t *xdip)
{
	struct xendev_ppd *pdp = ddi_get_parent_data(xdip);

	ASSERT(pdp != NULL);
	return (pdp->xd_evtchn);
}

int
xvdi_get_vdevnum(dev_info_t *xdip)
{
	struct xendev_ppd *pdp = ddi_get_parent_data(xdip);

	ASSERT(pdp != NULL);
	return (pdp->xd_vdevnum);
}

char *
xvdi_get_xsname(dev_info_t *xdip)
{
	struct xendev_ppd *pdp = ddi_get_parent_data(xdip);

	ASSERT(pdp != NULL);
	return ((char *)(pdp->xd_xsdev.nodename));
}

char *
xvdi_get_oename(dev_info_t *xdip)
{
	struct xendev_ppd *pdp = ddi_get_parent_data(xdip);

	ASSERT(pdp != NULL);
	if (pdp->xd_devclass == XEN_CONSOLE)
		return (NULL);
	return ((char *)(pdp->xd_xsdev.otherend));
}

struct xenbus_device *
xvdi_get_xsd(dev_info_t *xdip)
{
	struct xendev_ppd *pdp = ddi_get_parent_data(xdip);

	ASSERT(pdp != NULL);
	return (&pdp->xd_xsdev);
}

domid_t
xvdi_get_oeid(dev_info_t *xdip)
{
	struct xendev_ppd *pdp = ddi_get_parent_data(xdip);

	ASSERT(pdp != NULL);
	if (pdp->xd_devclass == XEN_CONSOLE)
		return ((domid_t)-1);
	return ((domid_t)(pdp->xd_xsdev.otherend_id));
}

void
xvdi_dev_error(dev_info_t *dip, int errno, char *errstr)
{
	struct xendev_ppd *pdp = ddi_get_parent_data(dip);

	ASSERT(pdp != NULL);
	xenbus_dev_error(&pdp->xd_xsdev, errno, errstr);
}

void
xvdi_fatal_error(dev_info_t *dip, int errno, char *errstr)
{
	struct xendev_ppd *pdp = ddi_get_parent_data(dip);

	ASSERT(pdp != NULL);
	xenbus_dev_fatal(&pdp->xd_xsdev, errno, errstr);
}

static void
i_xvdi_oestate_handler(void *arg)
{
	i_oestate_evt_t *evt = (i_oestate_evt_t *)arg;
	dev_info_t *dip = evt->dip;
	struct xendev_ppd *pdp = ddi_get_parent_data(dip);
	XenbusState oestate = pdp->xd_xsdev.otherend_state;
	XenbusState curr_oestate = evt->state;
	ddi_eventcookie_t evc;

	/* evt is alloc'ed in i_xvdi_oestate_cb */
	kmem_free(evt, sizeof (i_oestate_evt_t));

	/*
	 * If the oestate we're handling is not the latest one,
	 * it does not make any sense to continue handling it.
	 */
	if (curr_oestate != oestate)
		return;

	mutex_enter(&pdp->xd_ndi_lk);

	if (pdp->xd_oe_ehid != NULL) {
		/* send notification to driver */
		if (ddi_get_eventcookie(dip, XS_OE_STATE,
		    &evc) == DDI_SUCCESS) {
			mutex_exit(&pdp->xd_ndi_lk);
			(void) ndi_post_event(dip, dip, evc, &oestate);
			mutex_enter(&pdp->xd_ndi_lk);
		}
	} else {
		/*
		 * take default action, if driver hasn't registered its
		 * event handler yet
		 */
		if (oestate == XenbusStateClosing) {
			(void) xvdi_switch_state(dip, XBT_NULL,
			    XenbusStateClosed);
		} else if (oestate == XenbusStateClosed) {
			(void) xvdi_switch_state(dip, XBT_NULL,
			    XenbusStateClosed);
			(void) xvdi_post_event(dip, XEN_HP_REMOVE);
		}
	}

	mutex_exit(&pdp->xd_ndi_lk);

	/*
	 * We'll try to remove the devinfo node of this device if the
	 * other end has closed.
	 */
	if (oestate == XenbusStateClosed)
		(void) ddi_taskq_dispatch(DEVI(ddi_get_parent(dip))->devi_taskq,
		    xendev_offline_device, dip, DDI_SLEEP);
}

static void
i_xvdi_hpstate_handler(void *arg)
{
	dev_info_t *dip = (dev_info_t *)arg;
	struct xendev_ppd *pdp = ddi_get_parent_data(dip);
	ddi_eventcookie_t evc;
	char *hp_status;
	unsigned int hpl;

	mutex_enter(&pdp->xd_ndi_lk);
	if ((ddi_get_eventcookie(dip, XS_HP_STATE, &evc) == DDI_SUCCESS) &&
	    (xenbus_read(XBT_NULL, pdp->xd_hp_watch.node, "",
	    (void *)&hp_status, &hpl) == 0)) {

		xendev_hotplug_state_t new_state = Unrecognized;

		if (strcmp(hp_status, "connected") == 0)
			new_state = Connected;

		mutex_exit(&pdp->xd_ndi_lk);

		(void) ndi_post_event(dip, dip, evc, &new_state);
		kmem_free(hp_status, hpl);
		return;
	}
	mutex_exit(&pdp->xd_ndi_lk);
}

void
xvdi_notify_oe(dev_info_t *dip)
{
	struct xendev_ppd *pdp;

	pdp = ddi_get_parent_data(dip);
	ASSERT(pdp->xd_evtchn != INVALID_EVTCHN);
	ec_notify_via_evtchn(pdp->xd_evtchn);
}

static void
i_xvdi_bepath_cb(struct xenbus_watch *w, const char **vec, unsigned int len)
{
	dev_info_t *dip = (dev_info_t *)w->dev;
	struct xendev_ppd *pdp = ddi_get_parent_data(dip);
	char *be = NULL;
	unsigned int bel;

	ASSERT(len > XS_WATCH_PATH);
	ASSERT(vec[XS_WATCH_PATH] != NULL);

	/*
	 * If the backend is not the same as that we already stored,
	 * re-set our watch for its' state.
	 */
	if ((xenbus_read(XBT_NULL, "", vec[XS_WATCH_PATH], (void *)be, &bel)
	    == 0) && (strcmp(be, pdp->xd_xsdev.otherend) != 0))
		(void) i_xvdi_add_watch_oestate(dip);

	if (be != NULL) {
		ASSERT(bel > 0);
		kmem_free(be, bel);
	}
}

static void
i_xvdi_xb_watch_free(xd_xb_watches_t *xxwp)
{
	ASSERT(xxwp->xxw_ref == 0);
	strfree((char *)xxwp->xxw_watch.node);
	kmem_free(xxwp, sizeof (*xxwp));
}

static void
i_xvdi_xb_watch_release(xd_xb_watches_t *xxwp)
{
	ASSERT(MUTEX_HELD(&xxwp->xxw_xppd->xd_ndi_lk));
	ASSERT(xxwp->xxw_ref > 0);
	if (--xxwp->xxw_ref == 0)
		i_xvdi_xb_watch_free(xxwp);
}

static void
i_xvdi_xb_watch_hold(xd_xb_watches_t *xxwp)
{
	ASSERT(MUTEX_HELD(&xxwp->xxw_xppd->xd_ndi_lk));
	ASSERT(xxwp->xxw_ref > 0);
	xxwp->xxw_ref++;
}

static void
i_xvdi_xb_watch_cb_tq(void *arg)
{
	xd_xb_watches_t		*xxwp = (xd_xb_watches_t *)arg;
	dev_info_t		*dip = (dev_info_t *)xxwp->xxw_watch.dev;
	struct xendev_ppd	*pdp = xxwp->xxw_xppd;

	xxwp->xxw_cb(dip, xxwp->xxw_watch.node, xxwp->xxw_arg);

	mutex_enter(&pdp->xd_ndi_lk);
	i_xvdi_xb_watch_release(xxwp);
	mutex_exit(&pdp->xd_ndi_lk);
}

static void
i_xvdi_xb_watch_cb(struct xenbus_watch *w, const char **vec, unsigned int len)
{
	dev_info_t		*dip = (dev_info_t *)w->dev;
	struct xendev_ppd	*pdp = ddi_get_parent_data(dip);
	xd_xb_watches_t		*xxwp;

	ASSERT(len > XS_WATCH_PATH);
	ASSERT(vec[XS_WATCH_PATH] != NULL);

	mutex_enter(&pdp->xd_ndi_lk);
	for (xxwp = list_head(&pdp->xd_xb_watches); xxwp != NULL;
	    xxwp = list_next(&pdp->xd_xb_watches, xxwp)) {
		if (w == &xxwp->xxw_watch)
			break;
	}

	if (xxwp == NULL) {
		mutex_exit(&pdp->xd_ndi_lk);
		return;
	}

	i_xvdi_xb_watch_hold(xxwp);
	(void) ddi_taskq_dispatch(pdp->xd_xb_watch_taskq,
	    i_xvdi_xb_watch_cb_tq, xxwp, DDI_SLEEP);
	mutex_exit(&pdp->xd_ndi_lk);
}

/*
 * Any watches registered with xvdi_add_xb_watch_handler() get torn down during
 * a suspend operation.  So if a frontend driver want's to use these interfaces,
 * that driver is responsible for re-registering any watches it had before
 * the suspend operation.
 */
int
xvdi_add_xb_watch_handler(dev_info_t *dip, const char *dir, const char *node,
    xvdi_xb_watch_cb_t cb, void *arg)
{
	struct xendev_ppd	*pdp = ddi_get_parent_data(dip);
	xd_xb_watches_t		*xxw_new, *xxwp;
	char			*path;
	int			n;

	ASSERT((dip != NULL) && (dir != NULL) && (node != NULL));
	ASSERT(cb != NULL);

	n = strlen(dir) + 1 + strlen(node) + 1;
	path = kmem_zalloc(n, KM_SLEEP);
	(void) strlcat(path, dir, n);
	(void) strlcat(path, "/", n);
	(void) strlcat(path, node, n);
	ASSERT((strlen(path) + 1) == n);

	xxw_new = kmem_zalloc(sizeof (*xxw_new), KM_SLEEP);
	xxw_new->xxw_ref = 1;
	xxw_new->xxw_watch.node = path;
	xxw_new->xxw_watch.callback = i_xvdi_xb_watch_cb;
	xxw_new->xxw_watch.dev = (struct xenbus_device *)dip;
	xxw_new->xxw_xppd = pdp;
	xxw_new->xxw_cb = cb;
	xxw_new->xxw_arg = arg;

	mutex_enter(&pdp->xd_ndi_lk);

	/*
	 * If this is the first watch we're setting up, create a taskq
	 * to dispatch watch events and initialize the watch list.
	 */
	if (pdp->xd_xb_watch_taskq == NULL) {
		char tq_name[TASKQ_NAMELEN];

		ASSERT(list_is_empty(&pdp->xd_xb_watches));

		(void) snprintf(tq_name, sizeof (tq_name),
		    "%s_xb_watch_tq", ddi_get_name(dip));

		if ((pdp->xd_xb_watch_taskq = ddi_taskq_create(dip, tq_name,
		    1, TASKQ_DEFAULTPRI, 0)) == NULL) {
			i_xvdi_xb_watch_release(xxw_new);
			mutex_exit(&pdp->xd_ndi_lk);
			return (DDI_FAILURE);
		}
	}

	/* Don't allow duplicate watches to be registered */
	for (xxwp = list_head(&pdp->xd_xb_watches); xxwp != NULL;
	    xxwp = list_next(&pdp->xd_xb_watches, xxwp)) {

		ASSERT(strcmp(xxwp->xxw_watch.node, path) != 0);
		if (strcmp(xxwp->xxw_watch.node, path) != 0)
			continue;
		i_xvdi_xb_watch_release(xxw_new);
		mutex_exit(&pdp->xd_ndi_lk);
		return (DDI_FAILURE);
	}

	if (register_xenbus_watch(&xxw_new->xxw_watch) != 0) {
		if (list_is_empty(&pdp->xd_xb_watches)) {
			ddi_taskq_destroy(pdp->xd_xb_watch_taskq);
			pdp->xd_xb_watch_taskq = NULL;
		}
		i_xvdi_xb_watch_release(xxw_new);
		mutex_exit(&pdp->xd_ndi_lk);
		return (DDI_FAILURE);
	}

	list_insert_head(&pdp->xd_xb_watches, xxw_new);
	mutex_exit(&pdp->xd_ndi_lk);
	return (DDI_SUCCESS);
}

/*
 * Tear down all xenbus watches registered by the specified dip.
 */
void
xvdi_remove_xb_watch_handlers(dev_info_t *dip)
{
	struct xendev_ppd	*pdp = ddi_get_parent_data(dip);
	xd_xb_watches_t		*xxwp;
	ddi_taskq_t		*tq;

	mutex_enter(&pdp->xd_ndi_lk);

	while ((xxwp = list_remove_head(&pdp->xd_xb_watches)) != NULL) {
		mutex_exit(&pdp->xd_ndi_lk);
		unregister_xenbus_watch(&xxwp->xxw_watch);
		mutex_enter(&pdp->xd_ndi_lk);
		i_xvdi_xb_watch_release(xxwp);
	}
	ASSERT(list_is_empty(&pdp->xd_xb_watches));

	/*
	 * We can't hold xd_ndi_lk while we destroy the xd_xb_watch_taskq.
	 * This is because if there are currently any executing taskq threads,
	 * we will block until they are finished, and to finish they need
	 * to aquire xd_ndi_lk in i_xvdi_xb_watch_cb_tq() so they can release
	 * their reference on their corresponding xxwp structure.
	 */
	tq = pdp->xd_xb_watch_taskq;
	pdp->xd_xb_watch_taskq = NULL;
	mutex_exit(&pdp->xd_ndi_lk);
	if (tq != NULL)
		ddi_taskq_destroy(tq);
}

static int
i_xvdi_add_watch_oestate(dev_info_t *dip)
{
	struct xendev_ppd *pdp = ddi_get_parent_data(dip);

	ASSERT(pdp != NULL);
	ASSERT(pdp->xd_xsdev.nodename != NULL);
	ASSERT(mutex_owned(&pdp->xd_ndi_lk));

	/*
	 * Create taskq for delivering other end state change event to
	 * this device later.
	 *
	 * Set nthreads to 1 to make sure that events can be delivered
	 * in order.
	 *
	 * Note: It is _not_ guaranteed that driver can see every
	 * xenstore change under the path that it is watching. If two
	 * changes happen consecutively in a very short amount of
	 * time, it is likely that the driver will see only the last
	 * one.
	 */
	if (pdp->xd_oe_taskq == NULL)
		if ((pdp->xd_oe_taskq = ddi_taskq_create(dip,
		    "xendev_oe_taskq", 1, TASKQ_DEFAULTPRI, 0)) == NULL)
			return (DDI_FAILURE);

	/*
	 * Watch for changes to the XenbusState of otherend.
	 */
	pdp->xd_xsdev.otherend_state = XenbusStateUnknown;
	pdp->xd_xsdev.otherend_changed = i_xvdi_oestate_cb;

	if (talk_to_otherend(&pdp->xd_xsdev) != 0) {
		i_xvdi_rem_watch_oestate(dip);
		return (DDI_FAILURE);
	}

	return (DDI_SUCCESS);
}

static void
i_xvdi_rem_watch_oestate(dev_info_t *dip)
{
	struct xendev_ppd *pdp;
	struct xenbus_device *dev;

	pdp = ddi_get_parent_data(dip);
	ASSERT(pdp != NULL);
	ASSERT(mutex_owned(&pdp->xd_ndi_lk));

	dev = &pdp->xd_xsdev;

	/* Unwatch for changes to XenbusState of otherend */
	if (dev->otherend_watch.node != NULL) {
		mutex_exit(&pdp->xd_ndi_lk);
		unregister_xenbus_watch(&dev->otherend_watch);
		mutex_enter(&pdp->xd_ndi_lk);
	}

	/* make sure no event handler is running */
	if (pdp->xd_oe_taskq != NULL) {
		mutex_exit(&pdp->xd_ndi_lk);
		ddi_taskq_destroy(pdp->xd_oe_taskq);
		mutex_enter(&pdp->xd_ndi_lk);
		pdp->xd_oe_taskq = NULL;
	}

	/* clean up */
	dev->otherend_state = XenbusStateUnknown;
	dev->otherend_id = (domid_t)-1;
	if (dev->otherend_watch.node != NULL)
		kmem_free((void *)dev->otherend_watch.node,
		    strlen(dev->otherend_watch.node) + 1);
	dev->otherend_watch.node = NULL;
	if (dev->otherend != NULL)
		kmem_free((void *)dev->otherend, strlen(dev->otherend) + 1);
	dev->otherend = NULL;
}

static int
i_xvdi_add_watch_hpstate(dev_info_t *dip)
{
	struct xendev_ppd *pdp = ddi_get_parent_data(dip);

	ASSERT(pdp != NULL);
	ASSERT(pdp->xd_xsdev.frontend == 0);
	ASSERT(mutex_owned(&pdp->xd_ndi_lk));

	/*
	 * Create taskq for delivering hotplug status change event to
	 * this device later.
	 *
	 * Set nthreads to 1 to make sure that events can be delivered
	 * in order.
	 *
	 * Note: It is _not_ guaranteed that driver can see every
	 * hotplug status change under the path that it is
	 * watching. If two changes happen consecutively in a very
	 * short amount of time, it is likely that the driver only
	 * sees the last one.
	 */
	if (pdp->xd_hp_taskq == NULL)
		if ((pdp->xd_hp_taskq = ddi_taskq_create(dip,
		    "xendev_hp_taskq", 1, TASKQ_DEFAULTPRI, 0)) == NULL)
			return (DDI_FAILURE);

	if (pdp->xd_hp_watch.node == NULL) {
		size_t len;
		char *path;

		ASSERT(pdp->xd_xsdev.nodename != NULL);

		len = strlen(pdp->xd_xsdev.nodename) +
		    strlen("/hotplug-status") + 1;
		path = kmem_alloc(len, KM_SLEEP);
		(void) snprintf(path, len, "%s/hotplug-status",
		    pdp->xd_xsdev.nodename);

		pdp->xd_hp_watch.node = path;
		pdp->xd_hp_watch.callback = i_xvdi_hpstate_cb;
		pdp->xd_hp_watch.dev = (struct xenbus_device *)dip; /* yuck! */
		if (register_xenbus_watch(&pdp->xd_hp_watch) != 0) {
			i_xvdi_rem_watch_hpstate(dip);
			return (DDI_FAILURE);
		}
	}

	return (DDI_SUCCESS);
}

static void
i_xvdi_rem_watch_hpstate(dev_info_t *dip)
{
	struct xendev_ppd *pdp;
	pdp = ddi_get_parent_data(dip);

	ASSERT(pdp != NULL);
	ASSERT(pdp->xd_xsdev.frontend == 0);
	ASSERT(mutex_owned(&pdp->xd_ndi_lk));

	/* Unwatch for changes to "hotplug-status" node for backend device. */
	if (pdp->xd_hp_watch.node != NULL) {
		mutex_exit(&pdp->xd_ndi_lk);
		unregister_xenbus_watch(&pdp->xd_hp_watch);
		mutex_enter(&pdp->xd_ndi_lk);
	}

	/* Make sure no event handler is running. */
	if (pdp->xd_hp_taskq != NULL) {
		mutex_exit(&pdp->xd_ndi_lk);
		ddi_taskq_destroy(pdp->xd_hp_taskq);
		mutex_enter(&pdp->xd_ndi_lk);
		pdp->xd_hp_taskq = NULL;
	}

	/* Clean up. */
	if (pdp->xd_hp_watch.node != NULL) {
		kmem_free((void *)pdp->xd_hp_watch.node,
		    strlen(pdp->xd_hp_watch.node) + 1);
		pdp->xd_hp_watch.node = NULL;
	}
}

static int
i_xvdi_add_watches(dev_info_t *dip)
{
	struct xendev_ppd *pdp = ddi_get_parent_data(dip);

	ASSERT(pdp != NULL);

	mutex_enter(&pdp->xd_ndi_lk);

	if (i_xvdi_add_watch_oestate(dip) != DDI_SUCCESS) {
		mutex_exit(&pdp->xd_ndi_lk);
		return (DDI_FAILURE);
	}

	if (pdp->xd_xsdev.frontend == 1) {
		/*
		 * Frontend devices must watch for the backend path
		 * changing.
		 */
		if (i_xvdi_add_watch_bepath(dip) != DDI_SUCCESS)
			goto unwatch_and_fail;
	} else {
		/*
		 * Backend devices must watch for hotplug events.
		 */
		if (i_xvdi_add_watch_hpstate(dip) != DDI_SUCCESS)
			goto unwatch_and_fail;
	}

	mutex_exit(&pdp->xd_ndi_lk);

	return (DDI_SUCCESS);

unwatch_and_fail:
	i_xvdi_rem_watch_oestate(dip);
	mutex_exit(&pdp->xd_ndi_lk);

	return (DDI_FAILURE);
}

static void
i_xvdi_rem_watches(dev_info_t *dip)
{
	struct xendev_ppd *pdp = ddi_get_parent_data(dip);

	ASSERT(pdp != NULL);

	mutex_enter(&pdp->xd_ndi_lk);

	i_xvdi_rem_watch_oestate(dip);

	if (pdp->xd_xsdev.frontend == 1)
		i_xvdi_rem_watch_bepath(dip);
	else
		i_xvdi_rem_watch_hpstate(dip);

	mutex_exit(&pdp->xd_ndi_lk);

	xvdi_remove_xb_watch_handlers(dip);
}

static int
i_xvdi_add_watch_bepath(dev_info_t *dip)
{
	struct xendev_ppd *pdp = ddi_get_parent_data(dip);

	ASSERT(pdp != NULL);
	ASSERT(pdp->xd_xsdev.frontend == 1);

	/*
	 * Frontend devices need to watch for the backend path changing.
	 */
	if (pdp->xd_bepath_watch.node == NULL) {
		size_t len;
		char *path;

		ASSERT(pdp->xd_xsdev.nodename != NULL);

		len = strlen(pdp->xd_xsdev.nodename) + strlen("/backend") + 1;
		path = kmem_alloc(len, KM_SLEEP);
		(void) snprintf(path, len, "%s/backend",
		    pdp->xd_xsdev.nodename);

		pdp->xd_bepath_watch.node = path;
		pdp->xd_bepath_watch.callback = i_xvdi_bepath_cb;
		pdp->xd_bepath_watch.dev = (struct xenbus_device *)dip;
		if (register_xenbus_watch(&pdp->xd_bepath_watch) != 0) {
			kmem_free(path, len);
			pdp->xd_bepath_watch.node = NULL;
			return (DDI_FAILURE);
		}
	}

	return (DDI_SUCCESS);
}

static void
i_xvdi_rem_watch_bepath(dev_info_t *dip)
{
	struct xendev_ppd *pdp = ddi_get_parent_data(dip);

	ASSERT(pdp != NULL);
	ASSERT(pdp->xd_xsdev.frontend == 1);
	ASSERT(mutex_owned(&pdp->xd_ndi_lk));

	if (pdp->xd_bepath_watch.node != NULL) {
		mutex_exit(&pdp->xd_ndi_lk);
		unregister_xenbus_watch(&pdp->xd_bepath_watch);
		mutex_enter(&pdp->xd_ndi_lk);

		kmem_free((void *)(pdp->xd_bepath_watch.node),
		    strlen(pdp->xd_bepath_watch.node) + 1);
		pdp->xd_bepath_watch.node = NULL;
	}
}

int
xvdi_switch_state(dev_info_t *dip, xenbus_transaction_t xbt,
    XenbusState newState)
{
	int rv;
	struct xendev_ppd *pdp;

	pdp = ddi_get_parent_data(dip);
	ASSERT(pdp != NULL);

	XVDI_DPRINTF(XVDI_DBG_STATE,
	    "xvdi_switch_state: %s@%s's xenbus state moves to %d\n",
	    ddi_binding_name(dip) == NULL ? "null" : ddi_binding_name(dip),
	    ddi_get_name_addr(dip) == NULL ? "null" : ddi_get_name_addr(dip),
	    newState);

	rv = xenbus_switch_state(&pdp->xd_xsdev, xbt, newState);
	if (rv > 0)
		cmn_err(CE_WARN, "xvdi_switch_state: change state failed");

	return (rv);
}

/*
 * Notify hotplug script running in userland
 */
int
xvdi_post_event(dev_info_t *dip, xendev_hotplug_cmd_t hpc)
{
	struct xendev_ppd *pdp;
	nvlist_t *attr_list = NULL;
	i_xd_cfg_t *xdcp;
	sysevent_id_t eid;
	int err;
	char devname[256]; /* XXPV dme: ? */

	pdp = ddi_get_parent_data(dip);
	ASSERT(pdp != NULL);

	xdcp = i_xvdi_devclass2cfg(pdp->xd_devclass);
	ASSERT(xdcp != NULL);

	(void) snprintf(devname, sizeof (devname) - 1, "%s%d",
	    ddi_driver_name(dip),  ddi_get_instance(dip));

	err = nvlist_alloc(&attr_list, NV_UNIQUE_NAME, KM_NOSLEEP);
	if (err != DDI_SUCCESS)
		goto failure;

	err = nvlist_add_int32(attr_list, "domain", pdp->xd_domain);
	if (err != DDI_SUCCESS)
		goto failure;
	err = nvlist_add_int32(attr_list, "vdev", pdp->xd_vdevnum);
	if (err != DDI_SUCCESS)
		goto failure;
	err = nvlist_add_string(attr_list, "devclass", xdcp->xsdev);
	if (err != DDI_SUCCESS)
		goto failure;
	err = nvlist_add_string(attr_list, "device", devname);
	if (err != DDI_SUCCESS)
		goto failure;
	err = nvlist_add_string(attr_list, "fob",
	    ((pdp->xd_xsdev.frontend == 1) ? "frontend" : "backend"));
	if (err != DDI_SUCCESS)
		goto failure;

	switch (hpc) {
	case XEN_HP_ADD:
		err = ddi_log_sysevent(dip, DDI_VENDOR_SUNW, "EC_xendev",
		    "add", attr_list, &eid, DDI_NOSLEEP);
		break;
	case XEN_HP_REMOVE:
		err = ddi_log_sysevent(dip, DDI_VENDOR_SUNW, "EC_xendev",
		    "remove", attr_list, &eid, DDI_NOSLEEP);
		break;
	default:
		err = DDI_FAILURE;
		goto failure;
	}

failure:
	nvlist_free(attr_list);

	return (err);
}

/* ARGSUSED */
static void
i_xvdi_probe_path_cb(struct xenbus_watch *w, const char **vec,
    unsigned int len)
{
	char *path;

	if (xendev_dip == NULL)
		xendev_dip = ddi_find_devinfo("xpvd", -1, 0);

	path = i_ddi_strdup((char *)vec[XS_WATCH_PATH], KM_SLEEP);

	(void) ddi_taskq_dispatch(DEVI(xendev_dip)->devi_taskq,
	    i_xvdi_probe_path_handler, (void *)path, DDI_SLEEP);
}

static void
i_xvdi_watch_device(char *path)
{
	struct xenbus_watch *w;

	ASSERT(path != NULL);

	w = kmem_zalloc(sizeof (*w), KM_SLEEP);
	w->node = path;
	w->callback = &i_xvdi_probe_path_cb;
	w->dev = NULL;

	if (register_xenbus_watch(w) != 0) {
		cmn_err(CE_WARN, "i_xvdi_watch_device: "
		    "cannot set watch on %s", path);
		kmem_free(w, sizeof (*w));
		return;
	}
}

void
xvdi_watch_devices(int newstate)
{
	int devclass;

	/*
	 * Watch for devices being created in the store.
	 */
	if (newstate == XENSTORE_DOWN)
		return;
	for (devclass = 0; devclass < NXDC; devclass++) {
		if (xdci[devclass].xs_path_fe != NULL)
			i_xvdi_watch_device(xdci[devclass].xs_path_fe);
		if (xdci[devclass].xs_path_be != NULL)
			i_xvdi_watch_device(xdci[devclass].xs_path_be);
	}
}

/*
 * Iterate over the store looking for backend devices to create.
 */
static void
i_xvdi_enum_be(dev_info_t *parent, i_xd_cfg_t *xdcp)
{
	char **domains;
	unsigned int ndomains;
	int ldomains, i;

	if ((domains = xenbus_directory(XBT_NULL, xdcp->xs_path_be, "",
	    &ndomains)) == NULL)
		return;

	for (i = 0, ldomains = 0; i < ndomains; i++) {
		ldomains += strlen(domains[i]) + 1 + sizeof (char *);

		i_xvdi_enum_worker(parent, xdcp, domains[i]);
	}
	kmem_free(domains, ldomains);
}

/*
 * Iterate over the store looking for frontend devices to create.
 */
static void
i_xvdi_enum_fe(dev_info_t *parent, i_xd_cfg_t *xdcp)
{
	i_xvdi_enum_worker(parent, xdcp, NULL);
}

static void
i_xvdi_enum_worker(dev_info_t *parent, i_xd_cfg_t *xdcp,
    char *domain)
{
	char *path, *domain_path, *ep;
	char **devices;
	unsigned int ndevices;
	int ldevices, j;
	domid_t dom;
	long tmplong;

	if (domain == NULL) {
		dom = DOMID_SELF;
		path = xdcp->xs_path_fe;
		domain_path = "";
	} else {
		(void) ddi_strtol(domain, &ep, 0, &tmplong);
		dom = tmplong;
		path = xdcp->xs_path_be;
		domain_path = domain;
	}

	if ((devices = xenbus_directory(XBT_NULL, path, domain_path,
	    &ndevices)) == NULL)
		return;

	for (j = 0, ldevices = 0; j < ndevices; j++) {
		int vdev;

		ldevices += strlen(devices[j]) + 1 + sizeof (char *);
		(void) ddi_strtol(devices[j], &ep, 0, &tmplong);
		vdev = tmplong;

		ndi_devi_enter(parent);

		if (xvdi_find_dev(parent, xdcp->devclass, dom, vdev) == NULL)
			(void) xvdi_create_dev(parent, xdcp->devclass,
			    dom, vdev);

		ndi_devi_exit(parent);
	}
	kmem_free(devices, ldevices);
}

/*
 * Leaf drivers should call this in their detach() routine during suspend.
 */
void
xvdi_suspend(dev_info_t *dip)
{
	i_xvdi_rem_watches(dip);
}

/*
 * Leaf drivers should call this in their attach() routine during resume.
 */
int
xvdi_resume(dev_info_t *dip)
{
	return (i_xvdi_add_watches(dip));
}

/*
 * Add event handler for the leaf driver
 * to handle event triggered by the change in xenstore
 */
int
xvdi_add_event_handler(dev_info_t *dip, char *name,
    void (*evthandler)(dev_info_t *, ddi_eventcookie_t, void *, void *),
    void *arg)
{
	ddi_eventcookie_t ecv;
	struct xendev_ppd *pdp = ddi_get_parent_data(dip);
	ddi_callback_id_t *cbid;
	boolean_t call_handler;
	i_oestate_evt_t *evt = NULL;
	XenbusState oestate;

	ASSERT(pdp != NULL);

	mutex_enter(&pdp->xd_ndi_lk);

	if (strcmp(name, XS_OE_STATE) == 0) {
		ASSERT(pdp->xd_xsdev.otherend != NULL);

		cbid = &pdp->xd_oe_ehid;
	} else if (strcmp(name, XS_HP_STATE) == 0) {
		if (pdp->xd_xsdev.frontend == 1) {
			mutex_exit(&pdp->xd_ndi_lk);
			return (DDI_FAILURE);
		}

		ASSERT(pdp->xd_hp_watch.node != NULL);

		cbid = &pdp->xd_hp_ehid;
	} else {
		/* Unsupported watch. */
		mutex_exit(&pdp->xd_ndi_lk);
		return (DDI_FAILURE);
	}

	/*
	 * No event handler provided, take default action to handle
	 * event.
	 */
	if (evthandler == NULL) {
		mutex_exit(&pdp->xd_ndi_lk);
		return (DDI_SUCCESS);
	}

	ASSERT(*cbid == NULL);

	if (ddi_get_eventcookie(dip, name, &ecv) != DDI_SUCCESS) {
		cmn_err(CE_WARN, "failed to find %s cookie for %s@%s",
		    name, ddi_get_name(dip), ddi_get_name_addr(dip));
		mutex_exit(&pdp->xd_ndi_lk);
		return (DDI_FAILURE);
	}
	if (ddi_add_event_handler(dip, ecv, evthandler, arg, cbid)
	    != DDI_SUCCESS) {
		cmn_err(CE_WARN, "failed to add %s event handler for %s@%s",
		    name, ddi_get_name(dip), ddi_get_name_addr(dip));
		*cbid = NULL;
		mutex_exit(&pdp->xd_ndi_lk);
		return (DDI_FAILURE);
	}

	/*
	 * if we're adding an oe state callback, and the ring has already
	 * transitioned out of Unknown, call the handler after we release
	 * the mutex.
	 */
	call_handler = B_FALSE;
	if ((strcmp(name, XS_OE_STATE) == 0) &&
	    (pdp->xd_xsdev.otherend_state != XenbusStateUnknown)) {
		oestate = pdp->xd_xsdev.otherend_state;
		call_handler = B_TRUE;
	}

	mutex_exit(&pdp->xd_ndi_lk);

	if (call_handler) {
		evt = kmem_alloc(sizeof (i_oestate_evt_t), KM_SLEEP);
		evt->dip = dip;
		evt->state = oestate;
		(void) ddi_taskq_dispatch(pdp->xd_oe_taskq,
		    i_xvdi_oestate_handler, (void *)evt, DDI_SLEEP);
	}

	return (DDI_SUCCESS);
}

/*
 * Remove event handler for the leaf driver and unwatch xenstore
 * so, driver will not be notified when xenstore entry changed later
 */
void
xvdi_remove_event_handler(dev_info_t *dip, char *name)
{
	struct xendev_ppd *pdp;
	boolean_t rem_oe = B_FALSE, rem_hp = B_FALSE;
	ddi_callback_id_t oeid = NULL, hpid = NULL;

	pdp = ddi_get_parent_data(dip);
	ASSERT(pdp != NULL);

	if (name == NULL) {
		rem_oe = B_TRUE;
		rem_hp = B_TRUE;
	} else if (strcmp(name, XS_OE_STATE) == 0) {
		rem_oe = B_TRUE;
	} else if (strcmp(name, XS_HP_STATE) == 0) {
		rem_hp = B_TRUE;
	} else {
		cmn_err(CE_WARN, "event %s not supported, cannot remove", name);
		return;
	}

	mutex_enter(&pdp->xd_ndi_lk);

	if (rem_oe && (pdp->xd_oe_ehid != NULL)) {
		oeid = pdp->xd_oe_ehid;
		pdp->xd_oe_ehid = NULL;
	}

	if (rem_hp && (pdp->xd_hp_ehid != NULL)) {
		hpid = pdp->xd_hp_ehid;
		pdp->xd_hp_ehid = NULL;
	}

	mutex_exit(&pdp->xd_ndi_lk);

	if (oeid != NULL)
		(void) ddi_remove_event_handler(oeid);
	if (hpid != NULL)
		(void) ddi_remove_event_handler(hpid);
}


/*
 * common ring interfaces
 */

#define	FRONT_RING(_ringp)	(&(_ringp)->xr_sring.fr)
#define	BACK_RING(_ringp)	(&(_ringp)->xr_sring.br)
#define	GET_RING_SIZE(_ringp)	RING_SIZE(FRONT_RING(ringp))
#define	GET_RING_ENTRY_FE(_ringp, _idx)		\
	(FRONT_RING(_ringp)->sring->ring +	\
	(_ringp)->xr_entry_size * ((_idx) & (GET_RING_SIZE(_ringp) - 1)))
#define	GET_RING_ENTRY_BE(_ringp, _idx)		\
	(BACK_RING(_ringp)->sring->ring +	\
	(_ringp)->xr_entry_size * ((_idx) & (GET_RING_SIZE(_ringp) - 1)))

unsigned int
xvdi_ring_avail_slots(xendev_ring_t *ringp)
{
	comif_ring_fe_t *frp;
	comif_ring_be_t *brp;

	if (ringp->xr_frontend) {
		frp = FRONT_RING(ringp);
		return (GET_RING_SIZE(ringp) -
		    (frp->req_prod_pvt - frp->rsp_cons));
	} else {
		brp = BACK_RING(ringp);
		return (GET_RING_SIZE(ringp) -
		    (brp->rsp_prod_pvt - brp->req_cons));
	}
}

int
xvdi_ring_has_unconsumed_requests(xendev_ring_t *ringp)
{
	comif_ring_be_t *brp;

	ASSERT(!ringp->xr_frontend);
	brp = BACK_RING(ringp);
	return ((brp->req_cons !=
	    ddi_get32(ringp->xr_acc_hdl, &brp->sring->req_prod)) &&
	    ((brp->req_cons - brp->rsp_prod_pvt) != RING_SIZE(brp)));
}

int
xvdi_ring_has_incomp_request(xendev_ring_t *ringp)
{
	comif_ring_fe_t *frp;

	ASSERT(ringp->xr_frontend);
	frp = FRONT_RING(ringp);
	return (frp->req_prod_pvt !=
	    ddi_get32(ringp->xr_acc_hdl, &frp->sring->rsp_prod));
}

int
xvdi_ring_has_unconsumed_responses(xendev_ring_t *ringp)
{
	comif_ring_fe_t *frp;

	ASSERT(ringp->xr_frontend);
	frp = FRONT_RING(ringp);
	return (frp->rsp_cons !=
	    ddi_get32(ringp->xr_acc_hdl, &frp->sring->rsp_prod));
}

/* NOTE: req_event will be increased as needed */
void *
xvdi_ring_get_request(xendev_ring_t *ringp)
{
	comif_ring_fe_t *frp;
	comif_ring_be_t *brp;

	if (ringp->xr_frontend) {
		/* for frontend ring */
		frp = FRONT_RING(ringp);
		if (!RING_FULL(frp))
			return (GET_RING_ENTRY_FE(ringp, frp->req_prod_pvt++));
		else
			return (NULL);
	} else {
		/* for backend ring */
		brp = BACK_RING(ringp);
		/* RING_FINAL_CHECK_FOR_REQUESTS() */
		if (xvdi_ring_has_unconsumed_requests(ringp))
			return (GET_RING_ENTRY_BE(ringp, brp->req_cons++));
		else {
			ddi_put32(ringp->xr_acc_hdl, &brp->sring->req_event,
			    brp->req_cons + 1);
			membar_enter();
			if (xvdi_ring_has_unconsumed_requests(ringp))
				return (GET_RING_ENTRY_BE(ringp,
				    brp->req_cons++));
			else
				return (NULL);
		}
	}
}

int
xvdi_ring_push_request(xendev_ring_t *ringp)
{
	RING_IDX old, new, reqevt;
	comif_ring_fe_t *frp;

	/* only frontend should be able to push request */
	ASSERT(ringp->xr_frontend);

	/* RING_PUSH_REQUEST_AND_CHECK_NOTIFY() */
	frp = FRONT_RING(ringp);
	old = ddi_get32(ringp->xr_acc_hdl, &frp->sring->req_prod);
	new = frp->req_prod_pvt;
	ddi_put32(ringp->xr_acc_hdl, &frp->sring->req_prod, new);
	membar_enter();
	reqevt = ddi_get32(ringp->xr_acc_hdl, &frp->sring->req_event);
	return ((RING_IDX)(new - reqevt) < (RING_IDX)(new - old));
}

/* NOTE: rsp_event will be increased as needed */
void *
xvdi_ring_get_response(xendev_ring_t *ringp)
{
	comif_ring_fe_t *frp;
	comif_ring_be_t *brp;

	if (!ringp->xr_frontend) {
		/* for backend ring */
		brp = BACK_RING(ringp);
		return (GET_RING_ENTRY_BE(ringp, brp->rsp_prod_pvt++));
	} else {
		/* for frontend ring */
		frp = FRONT_RING(ringp);
		/* RING_FINAL_CHECK_FOR_RESPONSES() */
		if (xvdi_ring_has_unconsumed_responses(ringp))
			return (GET_RING_ENTRY_FE(ringp, frp->rsp_cons++));
		else {
			ddi_put32(ringp->xr_acc_hdl, &frp->sring->rsp_event,
			    frp->rsp_cons + 1);
			membar_enter();
			if (xvdi_ring_has_unconsumed_responses(ringp))
				return (GET_RING_ENTRY_FE(ringp,
				    frp->rsp_cons++));
			else
				return (NULL);
		}
	}
}

int
xvdi_ring_push_response(xendev_ring_t *ringp)
{
	RING_IDX old, new, rspevt;
	comif_ring_be_t *brp;

	/* only backend should be able to push response */
	ASSERT(!ringp->xr_frontend);

	/* RING_PUSH_RESPONSE_AND_CHECK_NOTIFY() */
	brp = BACK_RING(ringp);
	old = ddi_get32(ringp->xr_acc_hdl, &brp->sring->rsp_prod);
	new = brp->rsp_prod_pvt;
	ddi_put32(ringp->xr_acc_hdl, &brp->sring->rsp_prod, new);
	membar_enter();
	rspevt = ddi_get32(ringp->xr_acc_hdl, &brp->sring->rsp_event);
	return ((RING_IDX)(new - rspevt) < (RING_IDX)(new - old));
}

static void
xvdi_ring_init_sring(xendev_ring_t *ringp)
{
	ddi_acc_handle_t acchdl;
	comif_sring_t *xsrp;
	int i;

	xsrp = (comif_sring_t *)ringp->xr_vaddr;
	acchdl = ringp->xr_acc_hdl;

	/* shared ring initialization */
	ddi_put32(acchdl, &xsrp->req_prod, 0);
	ddi_put32(acchdl, &xsrp->rsp_prod, 0);
	ddi_put32(acchdl, &xsrp->req_event, 1);
	ddi_put32(acchdl, &xsrp->rsp_event, 1);
	for (i = 0; i < sizeof (xsrp->pad); i++)
		ddi_put8(acchdl, xsrp->pad + i, 0);
}

static void
xvdi_ring_init_front_ring(xendev_ring_t *ringp, size_t nentry, size_t entrysize)
{
	comif_ring_fe_t *xfrp;

	xfrp = &ringp->xr_sring.fr;
	xfrp->req_prod_pvt = 0;
	xfrp->rsp_cons = 0;
	xfrp->nr_ents = nentry;
	xfrp->sring = (comif_sring_t *)ringp->xr_vaddr;

	ringp->xr_frontend = 1;
	ringp->xr_entry_size = entrysize;
}

#ifndef XPV_HVM_DRIVER
static void
xvdi_ring_init_back_ring(xendev_ring_t *ringp, size_t nentry, size_t entrysize)
{
	comif_ring_be_t *xbrp;

	xbrp = &ringp->xr_sring.br;
	xbrp->rsp_prod_pvt = 0;
	xbrp->req_cons = 0;
	xbrp->nr_ents = nentry;
	xbrp->sring = (comif_sring_t *)ringp->xr_vaddr;

	ringp->xr_frontend = 0;
	ringp->xr_entry_size = entrysize;
}
#endif /* XPV_HVM_DRIVER */

static void
xendev_offline_device(void *arg)
{
	dev_info_t *dip = (dev_info_t *)arg;
	char devname[MAXNAMELEN] = {0};

	/*
	 * This is currently the only chance to delete a devinfo node, which
	 * is _not_ always successful.
	 */
	(void) ddi_deviname(dip, devname);
	(void) devfs_clean(ddi_get_parent(dip), devname + 1, DV_CLEAN_FORCE);
	(void) ndi_devi_offline(dip, NDI_DEVI_REMOVE);
}

static void
i_xvdi_oestate_cb(struct xenbus_device *dev, XenbusState oestate)
{
	dev_info_t *dip = (dev_info_t *)dev->data;
	struct xendev_ppd *pdp = ddi_get_parent_data(dip);
	i_oestate_evt_t *evt = NULL;
	boolean_t call_handler;

	XVDI_DPRINTF(XVDI_DBG_STATE,
	    "i_xvdi_oestate_cb: %s@%s sees oestate change to %d\n",
	    ddi_binding_name(dip) == NULL ? "null" : ddi_binding_name(dip),
	    ddi_get_name_addr(dip) == NULL ? "null" : ddi_get_name_addr(dip),
	    oestate);

	/* only call the handler if our state has changed */
	call_handler = B_FALSE;
	mutex_enter(&pdp->xd_ndi_lk);
	if (dev->otherend_state != oestate) {
		dev->otherend_state = oestate;
		call_handler = B_TRUE;
	}
	mutex_exit(&pdp->xd_ndi_lk);

	if (call_handler) {
		/*
		 * Try to deliver the oestate change event to the dip
		 */
		evt = kmem_alloc(sizeof (i_oestate_evt_t), KM_SLEEP);
		evt->dip = dip;
		evt->state = oestate;
		(void) ddi_taskq_dispatch(pdp->xd_oe_taskq,
		    i_xvdi_oestate_handler, (void *)evt, DDI_SLEEP);
	}
}

/*ARGSUSED*/
static void
i_xvdi_hpstate_cb(struct xenbus_watch *w, const char **vec,
    unsigned int len)
{
	dev_info_t *dip = (dev_info_t *)w->dev;
	struct xendev_ppd *pdp = ddi_get_parent_data(dip);

#ifdef DEBUG
	char *hp_status = NULL;
	unsigned int hpl = 0;

	(void) xenbus_read(XBT_NULL, pdp->xd_hp_watch.node, "",
	    (void *)&hp_status, &hpl);
	XVDI_DPRINTF(XVDI_DBG_STATE,
	    "i_xvdi_hpstate_cb: %s@%s sees hpstate change to %s\n",
	    ddi_binding_name(dip) == NULL ?  "null" : ddi_binding_name(dip),
	    ddi_get_name_addr(dip) == NULL ?  "null" : ddi_get_name_addr(dip),
	    hp_status == NULL ? "null" : hp_status);
	if (hp_status != NULL)
		kmem_free(hp_status, hpl);
#endif /* DEBUG */

	(void) ddi_taskq_dispatch(pdp->xd_hp_taskq,
	    i_xvdi_hpstate_handler, (void *)dip, DDI_SLEEP);
}

static void
i_xvdi_probe_path_handler(void *arg)
{
	dev_info_t *parent;
	char *path = arg, *p = NULL;
	int i, vdev;
	i_xd_cfg_t *xdcp;
	boolean_t frontend;
	domid_t dom;

	for (i = 0, xdcp = &xdci[0]; i < NXDC; i++, xdcp++) {

		if ((xdcp->xs_path_fe != NULL) &&
		    (strncmp(path, xdcp->xs_path_fe, strlen(xdcp->xs_path_fe))
		    == 0)) {

			frontend = B_TRUE;
			p = path + strlen(xdcp->xs_path_fe);
			break;
		}

		if ((xdcp->xs_path_be != NULL) &&
		    (strncmp(path, xdcp->xs_path_be, strlen(xdcp->xs_path_be))
		    == 0)) {

			frontend = B_FALSE;
			p = path + strlen(xdcp->xs_path_be);
			break;
		}

	}

	if (p == NULL) {
		cmn_err(CE_WARN, "i_xvdi_probe_path_handler: "
		    "unexpected path prefix in %s", path);
		goto done;
	}

	if (frontend) {
		dom = DOMID_SELF;
		if (sscanf(p, "/%d/", &vdev) != 1) {
			XVDI_DPRINTF(XVDI_DBG_PROBE,
			    "i_xvdi_probe_path_handler: "
			    "cannot parse frontend path %s",
			    path);
			goto done;
		}
	} else {
		if (sscanf(p, "/%hu/%d/", &dom, &vdev) != 2) {
			XVDI_DPRINTF(XVDI_DBG_PROBE,
			    "i_xvdi_probe_path_handler: "
			    "cannot parse backend path %s",
			    path);
			goto done;
		}
	}

	/*
	 * This is an oxymoron, so indicates a bogus configuration we
	 * must check for.
	 */
	if (vdev == VDEV_NOXS) {
		cmn_err(CE_WARN, "i_xvdi_probe_path_handler: "
		    "invalid path %s", path);
		goto done;
	}

	parent = xendev_dip;
	ASSERT(parent != NULL);

	ndi_devi_enter(parent);

	if (xvdi_find_dev(parent, xdcp->devclass, dom, vdev) == NULL) {
		XVDI_DPRINTF(XVDI_DBG_PROBE,
		    "i_xvdi_probe_path_handler: create for %s", path);
		(void) xvdi_create_dev(parent, xdcp->devclass, dom, vdev);
	} else {
		XVDI_DPRINTF(XVDI_DBG_PROBE,
		    "i_xvdi_probe_path_handler: %s already exists", path);
	}

	ndi_devi_exit(parent);

done:
	kmem_free(path, strlen(path) + 1);
}
XEN NOTICE
==========

This copyright applies to all files within this subdirectory and its
subdirectories:
  include/public/*.h
  include/public/hvm/*.h
  include/public/io/*.h

The intention is that these files can be freely copied into the source
tree of an operating system when porting that OS to run on Xen. Doing
so does *not* cause the OS to become subject to the terms of the GPL.

All other files in the Xen source distribution are covered by version
2 of the GNU General Public License except where explicitly stated
otherwise within individual source files.

 -- Keir Fraser (on behalf of the Xen team)

=====================================================================

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to
deal in the Software without restriction, including without limitation the
rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
sell copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
DEALINGS IN THE SOFTWARE.
#
# CDDL HEADER START
#
# The contents of this file are subject to the terms of the
# Common Development and Distribution License (the "License").
# You may not use this file except in compliance with the License.
#
# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
# or http://www.opensolaris.org/os/licensing.
# See the License for the specific language governing permissions
# and limitations under the License.
#
# When distributing Covered Code, include this CDDL HEADER in each
# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
# If applicable, add the following below this CDDL HEADER, with the
# fields enclosed by brackets "[]" replaced with your own identifying
# information: Portions Copyright [yyyy] [name of copyright owner]
#
# CDDL HEADER END
#

#
# Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
# Use is subject to license terms.
#
# ident "@(#)prototype.Makefile 1.15    06/02/08 SMI"
#

These files should not be edited in ON. They are copies from a specific
build of the xen consolidation which can be found in:
  xen.hg/xen/include/public

Any changes to these files should be done in the xen consolidation.

/******************************************************************************
 * arch-x86/mca.h
 *
 * Contributed by Advanced Micro Devices, Inc.
 * Author: Christoph Egger <Christoph.Egger@amd.com>
 *
 * Guest OS machine check interface to x86 Xen.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to
 * deal in the Software without restriction, including without limitation the
 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
 * sell copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 * DEALINGS IN THE SOFTWARE.
 */

/* Full MCA functionality has the following Usecases from the guest side:
 *
 * Must have's:
 * 1. Dom0 and DomU register machine check trap callback handlers
 *    (already done via "set_trap_table" hypercall)
 * 2. Dom0 registers machine check event callback handler
 *    (doable via EVTCHNOP_bind_virq)
 * 3. Dom0 and DomU fetches machine check data
 * 4. Dom0 wants Xen to notify a DomU
 * 5. Dom0 gets DomU ID from physical address
 * 6. Dom0 wants Xen to kill DomU (already done for "xm destroy")
 *
 * Nice to have's:
 * 7. Dom0 wants Xen to deactivate a physical CPU
 *    This is better done as separate task, physical CPU hotplugging,
 *    and hypercall(s) should be sysctl's
 * 8. Page migration proposed from Xen NUMA work, where Dom0 can tell Xen to
 *    move a DomU (or Dom0 itself) away from a malicious page
 *    producing correctable errors.
 * 9. offlining physical page:
 *    Xen free's and never re-uses a certain physical page.
 * 10. Testfacility: Allow Dom0 to write values into machine check MSR's
 *     and tell Xen to trigger a machine check
 */

#ifndef __XEN_PUBLIC_ARCH_X86_MCA_H__
#define __XEN_PUBLIC_ARCH_X86_MCA_H__

/* Hypercall */
#define __HYPERVISOR_mca __HYPERVISOR_arch_0

/*
 * The xen-unstable repo has interface version 0x03000001; out interface
 * is incompatible with that and any future minor revisions, so we
 * choose a different version number range that is numerically less
 * than that used in xen-unstable.
 */
#define XEN_MCA_INTERFACE_VERSION 0x01ecc003

/* IN: Dom0 calls hypercall to retrieve nonurgent telemetry */
#define XEN_MC_NONURGENT  0x0001
/* IN: Dom0/DomU calls hypercall to retrieve urgent telemetry */
#define XEN_MC_URGENT     0x0002
/* IN: Dom0 acknowledges previosly-fetched telemetry */
#define XEN_MC_ACK        0x0004

/* OUT: All is ok */
#define XEN_MC_OK           0x0
/* OUT: Domain could not fetch data. */
#define XEN_MC_FETCHFAILED  0x1
/* OUT: There was no machine check data to fetch. */
#define XEN_MC_NODATA       0x2
/* OUT: Between notification time and this hypercall an other
 *  (most likely) correctable error happened. The fetched data,
 *  does not match the original machine check data. */
#define XEN_MC_NOMATCH      0x4

/* OUT: DomU did not register MC NMI handler. Try something else. */
#define XEN_MC_CANNOTHANDLE 0x8
/* OUT: Notifying DomU failed. Retry later or try something else. */
#define XEN_MC_NOTDELIVERED 0x10
/* Note, XEN_MC_CANNOTHANDLE and XEN_MC_NOTDELIVERED are mutually exclusive. */


#ifndef __ASSEMBLY__

#define VIRQ_MCA VIRQ_ARCH_0 /* G. (DOM0) Machine Check Architecture */

/*
 * Machine Check Architecure:
 * structs are read-only and used to report all kinds of
 * correctable and uncorrectable errors detected by the HW.
 * Dom0 and DomU: register a handler to get notified.
 * Dom0 only: Correctable errors are reported via VIRQ_MCA
 * Dom0 and DomU: Uncorrectable errors are reported via nmi handlers
 */
#define MC_TYPE_GLOBAL          0
#define MC_TYPE_BANK            1
#define MC_TYPE_EXTENDED        2
#define MC_TYPE_RECOVERY        3

struct mcinfo_common {
    uint16_t type;      /* structure type */
    uint16_t size;      /* size of this struct in bytes */
};


#define MC_FLAG_CORRECTABLE     (1 << 0)
#define MC_FLAG_UNCORRECTABLE   (1 << 1)
#define MC_FLAG_RECOVERABLE	(1 << 2)
#define MC_FLAG_POLLED		(1 << 3)
#define MC_FLAG_RESET		(1 << 4)
#define MC_FLAG_CMCI		(1 << 5)
#define MC_FLAG_MCE		(1 << 6)
/* contains global x86 mc information */
struct mcinfo_global {
    struct mcinfo_common common;

    /* running domain at the time in error (most likely the impacted one) */
    uint16_t mc_domid;
    uint16_t mc_vcpuid; /* virtual cpu scheduled for mc_domid */
    uint32_t mc_socketid; /* physical socket of the physical core */
    uint16_t mc_coreid; /* physical impacted core */
    uint16_t mc_core_threadid; /* core thread of physical core */
    uint32_t mc_apicid;
    uint32_t mc_flags;
    uint64_t mc_gstatus; /* global status */
};

/* contains bank local x86 mc information */
struct mcinfo_bank {
    struct mcinfo_common common;

    uint16_t mc_bank; /* bank nr */
    uint16_t mc_domid; /* Usecase 5: domain referenced by mc_addr on dom0
                        * and if mc_addr is valid. Never valid on DomU. */
    uint64_t mc_status; /* bank status */
    uint64_t mc_addr;   /* bank address, only valid
                         * if addr bit is set in mc_status */
    uint64_t mc_misc;
    uint64_t mc_ctrl2;
    uint64_t mc_tsc;
};


struct mcinfo_msr {
    uint64_t reg;   /* MSR */
    uint64_t value; /* MSR value */
};

/* contains mc information from other
 * or additional mc MSRs */
struct mcinfo_extended {
    struct mcinfo_common common;

    /* You can fill up to five registers.
     * If you need more, then use this structure
     * multiple times. */

    uint32_t mc_msrs; /* Number of msr with valid values. */
    /*
     * Currently Intel extended MSR (32/64) include all gp registers
     * and E(R)FLAGS, E(R)IP, E(R)MISC, up to 11/19 of them might be
     * useful at present. So expand this array to 16/32 to leave room.
     */
    struct mcinfo_msr mc_msr[sizeof(void *) * 4];
};

/* Recovery Action flags. Giving recovery result information to DOM0 */

/* Xen takes successful recovery action, the error is recovered */
#define REC_ACTION_RECOVERED (0x1 << 0)
/* No action is performed by XEN */
#define REC_ACTION_NONE (0x1 << 1)
/* It's possible DOM0 might take action ownership in some case */
#define REC_ACTION_NEED_RESET (0x1 << 2)

/* Different Recovery Action types, if the action is performed successfully,
 * REC_ACTION_RECOVERED flag will be returned.
 */

/* Page Offline Action */
#define MC_ACTION_PAGE_OFFLINE (0x1 << 0)
/* CPU offline Action */
#define MC_ACTION_CPU_OFFLINE (0x1 << 1)
/* L3 cache disable Action */
#define MC_ACTION_CACHE_SHRINK (0x1 << 2)

/* Below interface used between XEN/DOM0 for passing XEN's recovery action
 * information to DOM0.
 * usage Senario: After offlining broken page, XEN might pass its page offline
 * recovery action result to DOM0. DOM0 will save the information in
 * non-volatile memory for further proactive actions, such as offlining the
 * easy broken page earlier when doing next reboot.
*/
struct page_offline_action
{
    /* Params for passing the offlined page number to DOM0 */
    uint64_t mfn;
    uint64_t status;
};

struct cpu_offline_action
{
    /* Params for passing the identity of the offlined CPU to DOM0 */
    uint32_t mc_socketid;
    uint16_t mc_coreid;
    uint16_t mc_core_threadid;
};

#define MAX_UNION_SIZE 16
struct mcinfo_recovery
{
    struct mcinfo_common common;
    uint16_t mc_bank; /* bank nr */
    uint8_t action_flags;
    uint8_t action_types;
    union {
        struct page_offline_action page_retire;
        struct cpu_offline_action cpu_offline;
        uint8_t pad[MAX_UNION_SIZE];
    } action_info;
};


#define MCINFO_HYPERCALLSIZE	1024
#define MCINFO_MAXSIZE		768

struct mc_info {
    /* Number of mcinfo_* entries in mi_data */
    uint32_t mi_nentries;
    uint32_t _pad0;
    uint64_t mi_data[(MCINFO_MAXSIZE - 1) / 8];
};
typedef struct mc_info mc_info_t;
DEFINE_XEN_GUEST_HANDLE(mc_info_t);

#define __MC_MSR_ARRAYSIZE 8
#define __MC_NMSRS 1
#define MC_NCAPS	7	/* 7 CPU feature flag words */
#define MC_CAPS_STD_EDX	0	/* cpuid level 0x00000001 (%edx) */
#define MC_CAPS_AMD_EDX	1	/* cpuid level 0x80000001 (%edx) */
#define MC_CAPS_TM	2	/* cpuid level 0x80860001 (TransMeta) */
#define MC_CAPS_LINUX	3	/* Linux-defined */
#define MC_CAPS_STD_ECX	4	/* cpuid level 0x00000001 (%ecx) */
#define MC_CAPS_VIA	5	/* cpuid level 0xc0000001 */
#define MC_CAPS_AMD_ECX	6	/* cpuid level 0x80000001 (%ecx) */

struct mcinfo_logical_cpu {
    uint32_t mc_cpunr;
    uint32_t mc_chipid;
    uint16_t mc_coreid;
    uint16_t mc_threadid;
    uint32_t mc_apicid;
    uint32_t mc_clusterid;
    uint32_t mc_ncores;
    uint32_t mc_ncores_active;
    uint32_t mc_nthreads;
    int32_t mc_cpuid_level;
    uint32_t mc_family;
    uint32_t mc_vendor;
    uint32_t mc_model;
    uint32_t mc_step;
    char mc_vendorid[16];
    char mc_brandid[64];
    uint32_t mc_cpu_caps[MC_NCAPS];
    uint32_t mc_cache_size;
    uint32_t mc_cache_alignment;
    int32_t mc_nmsrvals;
    struct mcinfo_msr mc_msrvalues[__MC_MSR_ARRAYSIZE];
};
typedef struct mcinfo_logical_cpu xen_mc_logical_cpu_t;
DEFINE_XEN_GUEST_HANDLE(xen_mc_logical_cpu_t);


/*
 * OS's should use these instead of writing their own lookup function
 * each with its own bugs and drawbacks.
 * We use macros instead of static inline functions to allow guests
 * to include this header in assembly files (*.S).
 */
/* Prototype:
 *    uint32_t x86_mcinfo_nentries(struct mc_info *mi);
 */
#define x86_mcinfo_nentries(_mi)    \
    (_mi)->mi_nentries
/* Prototype:
 *    struct mcinfo_common *x86_mcinfo_first(struct mc_info *mi);
 */
#define x86_mcinfo_first(_mi)       \
    ((struct mcinfo_common *)(_mi)->mi_data)
/* Prototype:
 *    struct mcinfo_common *x86_mcinfo_next(struct mcinfo_common *mic);
 */
#define x86_mcinfo_next(_mic)       \
    ((struct mcinfo_common *)((uint8_t *)(_mic) + (_mic)->size))

/* Prototype:
 *    void x86_mcinfo_lookup(void *ret, struct mc_info *mi, uint16_t type);
 */
#define x86_mcinfo_lookup(_ret, _mi, _type)    \
    do {                                                        \
        uint32_t found, i;                                      \
        struct mcinfo_common *_mic;                             \
                                                                \
        found = 0;                                              \
	(_ret) = NULL;						\
	if (_mi == NULL) break;					\
        _mic = x86_mcinfo_first(_mi);                           \
        for (i = 0; i < x86_mcinfo_nentries(_mi); i++) {        \
            if (_mic->type == (_type)) {                        \
                found = 1;                                      \
                break;                                          \
            }                                                   \
            _mic = x86_mcinfo_next(_mic);                       \
        }                                                       \
        (_ret) = found ? _mic : NULL;                           \
    } while (0)


/* Usecase 1
 * Register machine check trap callback handler
 *    (already done via "set_trap_table" hypercall)
 */

/* Usecase 2
 * Dom0 registers machine check event callback handler
 * done by EVTCHNOP_bind_virq
 */

/* Usecase 3
 * Fetch machine check data from hypervisor.
 * Note, this hypercall is special, because both Dom0 and DomU must use this.
 */
#define XEN_MC_fetch            1
struct xen_mc_fetch {
    /* IN/OUT variables. */
    uint32_t flags;	/* IN: XEN_MC_NONURGENT, XEN_MC_URGENT,
                           XEN_MC_ACK if ack'ing an earlier fetch */
			/* OUT: XEN_MC_OK, XEN_MC_FETCHFAILED,
			   XEN_MC_NODATA, XEN_MC_NOMATCH */
    uint32_t _pad0;
    uint64_t fetch_id;	/* OUT: id for ack, IN: id we are ack'ing */

    /* OUT variables. */
    XEN_GUEST_HANDLE(mc_info_t) data;
};
typedef struct xen_mc_fetch xen_mc_fetch_t;
DEFINE_XEN_GUEST_HANDLE(xen_mc_fetch_t);


/* Usecase 4
 * This tells the hypervisor to notify a DomU about the machine check error
 */
#define XEN_MC_notifydomain     2
struct xen_mc_notifydomain {
    /* IN variables. */
    uint16_t mc_domid;    /* The unprivileged domain to notify. */
    uint16_t mc_vcpuid;   /* The vcpu in mc_domid to notify.
                           * Usually echo'd value from the fetch hypercall. */

    /* IN/OUT variables. */
    uint32_t flags;

/* IN: XEN_MC_CORRECTABLE, XEN_MC_TRAP */
/* OUT: XEN_MC_OK, XEN_MC_CANNOTHANDLE, XEN_MC_NOTDELIVERED, XEN_MC_NOMATCH */
};
typedef struct xen_mc_notifydomain xen_mc_notifydomain_t;
DEFINE_XEN_GUEST_HANDLE(xen_mc_notifydomain_t);

#define XEN_MC_physcpuinfo 3
struct xen_mc_physcpuinfo {
	/* IN/OUT */
	uint32_t ncpus;
	uint32_t _pad0;
	/* OUT */
	XEN_GUEST_HANDLE(xen_mc_logical_cpu_t) info;
};

#define XEN_MC_msrinject    4
#define MC_MSRINJ_MAXMSRS       8
struct xen_mc_msrinject {
       /* IN */
	uint32_t mcinj_cpunr;           /* target processor id */
	uint32_t mcinj_flags;           /* see MC_MSRINJ_F_* below */
	uint32_t mcinj_count;           /* 0 .. count-1 in array are valid */
	uint32_t _pad0;
	struct mcinfo_msr mcinj_msr[MC_MSRINJ_MAXMSRS];
};

/* Flags for mcinj_flags above; bits 16-31 are reserved */
#define MC_MSRINJ_F_INTERPOSE   0x1

#define XEN_MC_mceinject    5
struct xen_mc_mceinject {
	unsigned int mceinj_cpunr;      /* target processor id */
};

struct xen_mc {
    uint32_t cmd;
    uint32_t interface_version; /* XEN_MCA_INTERFACE_VERSION */
    union {
        struct xen_mc_fetch        mc_fetch;
        struct xen_mc_notifydomain mc_notifydomain;
        struct xen_mc_physcpuinfo  mc_physcpuinfo;
        struct xen_mc_msrinject    mc_msrinject;
        struct xen_mc_mceinject    mc_mceinject;
    } u;
};
typedef struct xen_mc xen_mc_t;
DEFINE_XEN_GUEST_HANDLE(xen_mc_t);

#endif /* __ASSEMBLY__ */

#endif /* __XEN_PUBLIC_ARCH_X86_MCA_H__ */
/******************************************************************************
 * xen-x86_32.h
 *
 * Guest OS interface to x86 32-bit Xen.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to
 * deal in the Software without restriction, including without limitation the
 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
 * sell copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 * DEALINGS IN THE SOFTWARE.
 *
 * Copyright (c) 2004-2007, K A Fraser
 */

#ifndef __XEN_PUBLIC_ARCH_X86_XEN_X86_32_H__
#define __XEN_PUBLIC_ARCH_X86_XEN_X86_32_H__

/*
 * Hypercall interface:
 *  Input:  %ebx, %ecx, %edx, %esi, %edi (arguments 1-5)
 *  Output: %eax
 * Access is via hypercall page (set up by guest loader or via a Xen MSR):
 *  call hypercall_page + hypercall-number * 32
 * Clobbered: Argument registers (e.g., 2-arg hypercall clobbers %ebx,%ecx)
 */

/*
 * Direct hypercall interface:
 * As above, except the entry sequence to the hypervisor is:
 *  mov $hypercall-number*32,%eax ; int $0x82
 */
#if !defined(_ASM)
#define TRAP_INSTR "int $0x82"
#else
#define TRAP_INSTR int $0x82
#endif

/*
 * These flat segments are in the Xen-private section of every GDT. Since these
 * are also present in the initial GDT, many OSes will be able to avoid
 * installing their own GDT.
 */
#define FLAT_RING1_CS 0xe019    /* GDT index 259 */
#define FLAT_RING1_DS 0xe021    /* GDT index 260 */
#define FLAT_RING1_SS 0xe021    /* GDT index 260 */
#define FLAT_RING3_CS 0xe02b    /* GDT index 261 */
#define FLAT_RING3_DS 0xe033    /* GDT index 262 */
#define FLAT_RING3_SS 0xe033    /* GDT index 262 */

#define FLAT_KERNEL_CS FLAT_RING1_CS
#define FLAT_KERNEL_DS FLAT_RING1_DS
#define FLAT_KERNEL_SS FLAT_RING1_SS
#define FLAT_USER_CS    FLAT_RING3_CS
#define FLAT_USER_DS    FLAT_RING3_DS
#define FLAT_USER_SS    FLAT_RING3_SS

#define __HYPERVISOR_VIRT_START_PAE    0xF5800000
#define __MACH2PHYS_VIRT_START_PAE     0xF5800000
#define __MACH2PHYS_VIRT_END_PAE       0xF6800000
#define HYPERVISOR_VIRT_START_PAE      \
    mk_unsigned_long(__HYPERVISOR_VIRT_START_PAE)
#define MACH2PHYS_VIRT_START_PAE       \
    mk_unsigned_long(__MACH2PHYS_VIRT_START_PAE)
#define MACH2PHYS_VIRT_END_PAE         \
    mk_unsigned_long(__MACH2PHYS_VIRT_END_PAE)

/* Non-PAE bounds are obsolete. */
#define __HYPERVISOR_VIRT_START_NONPAE 0xFC000000
#define __MACH2PHYS_VIRT_START_NONPAE  0xFC000000
#define __MACH2PHYS_VIRT_END_NONPAE    0xFC400000
#define HYPERVISOR_VIRT_START_NONPAE   \
    mk_unsigned_long(__HYPERVISOR_VIRT_START_NONPAE)
#define MACH2PHYS_VIRT_START_NONPAE    \
    mk_unsigned_long(__MACH2PHYS_VIRT_START_NONPAE)
#define MACH2PHYS_VIRT_END_NONPAE      \
    mk_unsigned_long(__MACH2PHYS_VIRT_END_NONPAE)

#define __HYPERVISOR_VIRT_START __HYPERVISOR_VIRT_START_PAE
#define __MACH2PHYS_VIRT_START  __MACH2PHYS_VIRT_START_PAE
#define __MACH2PHYS_VIRT_END    __MACH2PHYS_VIRT_END_PAE

#ifndef HYPERVISOR_VIRT_START
#define HYPERVISOR_VIRT_START mk_unsigned_long(__HYPERVISOR_VIRT_START)
#endif

#define MACH2PHYS_VIRT_START  mk_unsigned_long(__MACH2PHYS_VIRT_START)
#define MACH2PHYS_VIRT_END    mk_unsigned_long(__MACH2PHYS_VIRT_END)
#define MACH2PHYS_NR_ENTRIES  ((MACH2PHYS_VIRT_END-MACH2PHYS_VIRT_START)>>2)
#ifndef machine_to_phys_mapping
#define machine_to_phys_mapping ((unsigned long *)MACH2PHYS_VIRT_START)
#endif

/* 32-/64-bit invariability for control interfaces (domctl/sysctl). */
#if defined(__XEN__) || defined(__XEN_TOOLS__)
#undef ___DEFINE_XEN_GUEST_HANDLE

#ifdef __GNUC__

#define ___DEFINE_XEN_GUEST_HANDLE(name, type)                  \
    typedef struct { type *p; }                                 \
        __guest_handle_ ## name;                                \
    typedef struct { union { type *p; uint64_aligned_t q; }; }  \
        __guest_handle_64_ ## name

#else /* __GNUC__ */

/*
 * Workaround for 6671857.
 */
#define ___DEFINE_XEN_GUEST_HANDLE(name, type)                  \
    typedef struct { type *p; }                                 \
        __guest_handle_ ## name;                                \
    typedef struct { union { type *p; uint64_aligned_t q; } u; }\
        __guest_handle_64_ ## name

#endif /* __GNUC__ */

#undef set_xen_guest_handle
#define set_xen_guest_handle(hnd, val)                      \
    do { if ( sizeof(hnd) == 8 ) *(uint64_t *)&(hnd) = 0;   \
         (hnd).p = val;                                     \
    } while ( 0 )
#define uint64_aligned_t uint64_t __attribute__((aligned(8)))
#define __XEN_GUEST_HANDLE_64(name) __guest_handle_64_ ## name
#define XEN_GUEST_HANDLE_64(name) __XEN_GUEST_HANDLE_64(name)
#endif

#ifndef __ASSEMBLY__

struct cpu_user_regs {
    uint32_t ebx;
    uint32_t ecx;
    uint32_t edx;
    uint32_t esi;
    uint32_t edi;
    uint32_t ebp;
    uint32_t eax;
    uint16_t error_code;    /* private */
    uint16_t entry_vector;  /* private */
    uint32_t eip;
    uint16_t cs;
    uint8_t  saved_upcall_mask;
    uint8_t  _pad0;
    uint32_t eflags;        /* eflags.IF == !saved_upcall_mask */
    uint32_t esp;
    uint16_t ss, _pad1;
    uint16_t es, _pad2;
    uint16_t ds, _pad3;
    uint16_t fs, _pad4;
    uint16_t gs, _pad5;
};
typedef struct cpu_user_regs cpu_user_regs_t;
DEFINE_XEN_GUEST_HANDLE(cpu_user_regs_t);

/*
 * Page-directory addresses above 4GB do not fit into architectural %cr3.
 * When accessing %cr3, or equivalent field in vcpu_guest_context, guests
 * must use the following accessor macros to pack/unpack valid MFNs.
 */
#define xen_pfn_to_cr3(pfn) (((unsigned)(pfn) << 12) | ((unsigned)(pfn) >> 20))
#define xen_cr3_to_pfn(cr3) (((unsigned)(cr3) >> 12) | ((unsigned)(cr3) << 20))

struct arch_vcpu_info {
    unsigned long cr2;
    unsigned long pad[5]; /* sizeof(vcpu_info_t) == 64 */
};
typedef struct arch_vcpu_info arch_vcpu_info_t;

struct xen_callback {
    unsigned long cs;
    unsigned long eip;
};
typedef struct xen_callback xen_callback_t;

/*
 * Structure used to capture the register state at panic time.  This struct
 * is built to mimic a similar structure in Solaris.  If there is interest
 * in making this panic implementation an official part of Xen, this should
 * be made more platform-neutral.
 */
struct panic_regs {
	unsigned long pad1;
	unsigned long pad2;

	unsigned long gs;
	unsigned long fs;
	unsigned long es;
	unsigned long ds;
	unsigned long edi;
	unsigned long esi;
	unsigned long ebp;
	unsigned long esp;
	unsigned long ebx;
	unsigned long edx;
	unsigned long ecx;
	unsigned long eax;
	unsigned long pad3;
	unsigned long pad4;
	unsigned long eip;
	unsigned long cs;
	unsigned long efl;
	unsigned long pad5;
	unsigned long ss;
};

#endif /* !__ASSEMBLY__ */

/* Offsets of each field in the xen_panic_regs structure.  */
#define PANIC_REG_PAD1		0
#define PANIC_REG_PAD2		4
#define PANIC_REG_GS		8
#define PANIC_REG_FS		12
#define PANIC_REG_ES		16
#define PANIC_REG_DS		20
#define PANIC_REG_EDI		24
#define PANIC_REG_ESI		28
#define PANIC_REG_EBP		32
#define PANIC_REG_ESP		36
#define PANIC_REG_EBX		40
#define PANIC_REG_EDX		44
#define PANIC_REG_ECX		48
#define PANIC_REG_EAX		52
#define PANIC_REG_PAD3		56
#define PANIC_REG_PAD4		60
#define PANIC_REG_EIP		64
#define PANIC_REG_CS		68
#define PANIC_REG_EFL		72
#define PANIC_REG_PAD5		76
#define PANIC_REG_SS		80
#define PANIC_REG_STRUCT_SIZE	84

#endif /* __XEN_PUBLIC_ARCH_X86_XEN_X86_32_H__ */

/*
 * Local variables:
 * mode: C
 * c-set-style: "BSD"
 * c-basic-offset: 4
 * tab-width: 4
 * indent-tabs-mode: nil
 * End:
 */
/******************************************************************************
 * xen-x86_64.h
 *
 * Guest OS interface to x86 64-bit Xen.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to
 * deal in the Software without restriction, including without limitation the
 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
 * sell copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 * DEALINGS IN THE SOFTWARE.
 *
 * Copyright (c) 2004-2006, K A Fraser
 */

#ifndef __XEN_PUBLIC_ARCH_X86_XEN_X86_64_H__
#define __XEN_PUBLIC_ARCH_X86_XEN_X86_64_H__

/*
 * Hypercall interface:
 *  Input:  %rdi, %rsi, %rdx, %r10, %r8 (arguments 1-5)
 *  Output: %rax
 * Access is via hypercall page (set up by guest loader or via a Xen MSR):
 *  call hypercall_page + hypercall-number * 32
 * Clobbered: argument registers (e.g., 2-arg hypercall clobbers %rdi,%rsi)
 */

/*
 * Direct hypercall interface:
 * As above, except the entry sequence to the hypervisor is:
 *  mov $hypercall-number*32,%eax ; syscall
 * Clobbered: %rcx, %r11, argument registers (as above)
 */
#if !defined(_ASM)
#define TRAP_INSTR "syscall"
#else
#define TRAP_INSTR syscall
#endif

/*
 * 64-bit segment selectors
 * These flat segments are in the Xen-private section of every GDT. Since these
 * are also present in the initial GDT, many OSes will be able to avoid
 * installing their own GDT.
 */

#define FLAT_RING3_CS32 0xe023  /* GDT index 260 */
#define FLAT_RING3_CS64 0xe033  /* GDT index 261 */
#define FLAT_RING3_DS32 0xe02b  /* GDT index 262 */
#define FLAT_RING3_DS64 0x0000  /* NULL selector */
#define FLAT_RING3_SS32 0xe02b  /* GDT index 262 */
#define FLAT_RING3_SS64 0xe02b  /* GDT index 262 */

#define FLAT_KERNEL_DS64 FLAT_RING3_DS64
#define FLAT_KERNEL_DS32 FLAT_RING3_DS32
#define FLAT_KERNEL_DS   FLAT_KERNEL_DS64
#define FLAT_KERNEL_CS64 FLAT_RING3_CS64
#define FLAT_KERNEL_CS32 FLAT_RING3_CS32
#define FLAT_KERNEL_CS   FLAT_KERNEL_CS64
#define FLAT_KERNEL_SS64 FLAT_RING3_SS64
#define FLAT_KERNEL_SS32 FLAT_RING3_SS32
#define FLAT_KERNEL_SS   FLAT_KERNEL_SS64

#define FLAT_USER_DS64 FLAT_RING3_DS64
#define FLAT_USER_DS32 FLAT_RING3_DS32
#define FLAT_USER_DS   FLAT_USER_DS64
#define FLAT_USER_CS64 FLAT_RING3_CS64
#define FLAT_USER_CS32 FLAT_RING3_CS32
#define FLAT_USER_CS   FLAT_USER_CS64
#define FLAT_USER_SS64 FLAT_RING3_SS64
#define FLAT_USER_SS32 FLAT_RING3_SS32
#define FLAT_USER_SS   FLAT_USER_SS64

#define __HYPERVISOR_VIRT_START 0xFFFF800000000000
#define __HYPERVISOR_VIRT_END   0xFFFF880000000000
#define __MACH2PHYS_VIRT_START  0xFFFF800000000000
#define __MACH2PHYS_VIRT_END    0xFFFF804000000000

#ifndef HYPERVISOR_VIRT_START
#define HYPERVISOR_VIRT_START mk_unsigned_long(__HYPERVISOR_VIRT_START)
#define HYPERVISOR_VIRT_END   mk_unsigned_long(__HYPERVISOR_VIRT_END)
#endif

#define MACH2PHYS_VIRT_START  mk_unsigned_long(__MACH2PHYS_VIRT_START)
#define MACH2PHYS_VIRT_END    mk_unsigned_long(__MACH2PHYS_VIRT_END)
#define MACH2PHYS_NR_ENTRIES  ((MACH2PHYS_VIRT_END-MACH2PHYS_VIRT_START)>>3)
#ifndef machine_to_phys_mapping
#define machine_to_phys_mapping ((unsigned long *)HYPERVISOR_VIRT_START)
#endif

/*
 * int HYPERVISOR_set_segment_base(unsigned int which, unsigned long base)
 *  @which == SEGBASE_*  ;  @base == 64-bit base address
 * Returns 0 on success.
 */
#define SEGBASE_FS          0
#define SEGBASE_GS_USER     1
#define SEGBASE_GS_KERNEL   2
#define SEGBASE_GS_USER_SEL 3 /* Set user %gs specified in base[15:0] */

/*
 * int HYPERVISOR_iret(void)
 * All arguments are on the kernel stack, in the following format.
 * Never returns if successful. Current kernel context is lost.
 * The saved CS is mapped as follows:
 *   RING0 -> RING3 kernel mode.
 *   RING1 -> RING3 kernel mode.
 *   RING2 -> RING3 kernel mode.
 *   RING3 -> RING3 user mode.
 * However RING0 indicates that the guest kernel should return to iteself
 * directly with
 *      orb   $3,1*8(%rsp)
 *      iretq
 * If flags contains VGCF_in_syscall:
 *   Restore RAX, RIP, RFLAGS, RSP.
 *   Discard R11, RCX, CS, SS.
 * Otherwise:
 *   Restore RAX, R11, RCX, CS:RIP, RFLAGS, SS:RSP.
 * All other registers are saved on hypercall entry and restored to user.
 */
/* Guest exited in SYSCALL context? Return to guest with SYSRET? */
#define _VGCF_in_syscall 8
#define VGCF_in_syscall  (1<<_VGCF_in_syscall)
#define VGCF_IN_SYSCALL  VGCF_in_syscall

#ifndef __ASSEMBLY__

struct iret_context {
    /* Top of stack (%rsp at point of hypercall). */
    uint64_t rax, r11, rcx, flags, rip, cs, rflags, rsp, ss;
    /* Bottom of iret stack frame. */
};

#if defined(__GNUC__) && !defined(__STRICT_ANSI__)
/* Anonymous union includes both 32- and 64-bit names (e.g., eax/rax). */
#define __DECL_REG(name) union { \
    uint64_t r ## name, e ## name; \
    uint32_t _e ## name; \
}
#else
/* Non-gcc sources must always use the proper 64-bit name (e.g., rax). */
#define __DECL_REG(name) uint64_t r ## name
#endif

struct cpu_user_regs {
    uint64_t r15;
    uint64_t r14;
    uint64_t r13;
    uint64_t r12;
    __DECL_REG(bp);
    __DECL_REG(bx);
    uint64_t r11;
    uint64_t r10;
    uint64_t r9;
    uint64_t r8;
    __DECL_REG(ax);
    __DECL_REG(cx);
    __DECL_REG(dx);
    __DECL_REG(si);
    __DECL_REG(di);
    uint32_t error_code;    /* private */
    uint32_t entry_vector;  /* private */
    __DECL_REG(ip);
    uint16_t cs, _pad0[1];
    uint8_t  saved_upcall_mask;
    uint8_t  _pad1[3];
    __DECL_REG(flags);      /* rflags.IF == !saved_upcall_mask */
    __DECL_REG(sp);
    uint16_t ss, _pad2[3];
    uint16_t es, _pad3[3];
    uint16_t ds, _pad4[3];
    uint16_t fs, _pad5[3]; /* Non-zero => takes precedence over fs_base.     */
    uint16_t gs, _pad6[3]; /* Non-zero => takes precedence over gs_base_usr. */
};
typedef struct cpu_user_regs cpu_user_regs_t;
DEFINE_XEN_GUEST_HANDLE(cpu_user_regs_t);

#undef __DECL_REG

#define xen_pfn_to_cr3(pfn) ((unsigned long)(pfn) << 12)
#define xen_cr3_to_pfn(cr3) ((unsigned long)(cr3) >> 12)

struct arch_vcpu_info {
    unsigned long cr2;
    unsigned long pad; /* sizeof(vcpu_info_t) == 64 */
};
typedef struct arch_vcpu_info arch_vcpu_info_t;

typedef unsigned long xen_callback_t;

/*
 * Structure used to capture the register state at panic time.  This struct
 * is built to mimic a similar structure in Solaris.  If there is interest
 * in making this panic implementation an official part of Xen, this should
 * be made more platform-neutral.
 */
struct panic_regs {
	unsigned long pad1;
	unsigned long pad2;
	unsigned long rdi;
	unsigned long rsi;
	unsigned long rdx;
	unsigned long rcx;
	unsigned long r8;
	unsigned long r9;
	unsigned long rax;
	unsigned long rbx;
	unsigned long rbp;
	unsigned long r10;
	unsigned long r11;
	unsigned long r12;
	unsigned long r13;
	unsigned long r14;
	unsigned long r15;
	unsigned long pad3;
	unsigned long pad4;
	unsigned long ds;
	unsigned long es;
	unsigned long fs;
	unsigned long gs;
	unsigned long pad5;
	unsigned long pad6;
	unsigned long rip;
	unsigned long cs;
	unsigned long rfl;
	unsigned long rsp;
	unsigned long ss;
};

#endif /* !__ASSEMBLY__ */

/* Offsets of each field in the xen_panic_regs structure.  */
#define PANIC_REG_PAD1		0
#define PANIC_REG_PAD2		8
#define PANIC_REG_RDI		16
#define PANIC_REG_RSI		24
#define PANIC_REG_RDX		32
#define PANIC_REG_RCX		40
#define PANIC_REG_R8		48
#define PANIC_REG_R9		56
#define PANIC_REG_RAX		64
#define PANIC_REG_RBX		72
#define PANIC_REG_RBP		80
#define PANIC_REG_R10		88
#define PANIC_REG_R11		96
#define PANIC_REG_R12		104
#define PANIC_REG_R13		112
#define PANIC_REG_R14		120
#define PANIC_REG_R15		128
#define PANIC_REG_PAD3		136
#define PANIC_REG_PAD4		144
#define PANIC_REG_DS		152
#define PANIC_REG_ES		160
#define PANIC_REG_FS		168
#define PANIC_REG_GS		176
#define PANIC_REG_PAD5		184
#define PANIC_REG_PAD6		192
#define PANIC_REG_RIP		200
#define PANIC_REG_CS		208
#define PANIC_REG_RFL		216
#define PANIC_REG_RSP		224
#define PANIC_REG_SS		232
#define PANIC_REG_STRUCT_SIZE	240

#endif /* __XEN_PUBLIC_ARCH_X86_XEN_X86_64_H__ */

/*
 * Local variables:
 * mode: C
 * c-set-style: "BSD"
 * c-basic-offset: 4
 * tab-width: 4
 * indent-tabs-mode: nil
 * End:
 */
/******************************************************************************
 * arch-x86/xen.h
 *
 * Guest OS interface to x86 Xen.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to
 * deal in the Software without restriction, including without limitation the
 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
 * sell copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 * DEALINGS IN THE SOFTWARE.
 *
 * Copyright (c) 2004-2006, K A Fraser
 */

#include "../xen.h"

#ifndef __XEN_PUBLIC_ARCH_X86_XEN_H__
#define __XEN_PUBLIC_ARCH_X86_XEN_H__

/* Structural guest handles introduced in 0x00030201. */
#if __XEN_INTERFACE_VERSION__ >= 0x00030201
#define ___DEFINE_XEN_GUEST_HANDLE(name, type) \
    typedef struct { type *p; } __guest_handle_ ## name
#else
#define ___DEFINE_XEN_GUEST_HANDLE(name, type) \
    typedef type * __guest_handle_ ## name
#endif

#define __DEFINE_XEN_GUEST_HANDLE(name, type) \
    ___DEFINE_XEN_GUEST_HANDLE(name, type);   \
    ___DEFINE_XEN_GUEST_HANDLE(const_##name, const type)
#define DEFINE_XEN_GUEST_HANDLE(name)   __DEFINE_XEN_GUEST_HANDLE(name, name)
#define __XEN_GUEST_HANDLE(name)        __guest_handle_ ## name
#define XEN_GUEST_HANDLE(name)          __XEN_GUEST_HANDLE(name)
#define set_xen_guest_handle(hnd, val)  do { (hnd).p = val; } while (0)
#ifdef __XEN_TOOLS__
#define get_xen_guest_handle(val, hnd)  do { val = (hnd).p; } while (0)
#endif

#if defined(__i386__)
#include "xen-x86_32.h"
#elif defined(__x86_64__)
#include "xen-x86_64.h"
#endif

#ifndef __ASSEMBLY__
typedef unsigned long xen_pfn_t;
#define PRI_xen_pfn "lx"
#endif

/*
 * SEGMENT DESCRIPTOR TABLES
 */
/*
 * A number of GDT entries are reserved by Xen. These are not situated at the
 * start of the GDT because some stupid OSes export hard-coded selector values
 * in their ABI. These hard-coded values are always near the start of the GDT,
 * so Xen places itself out of the way, at the far end of the GDT.
 */
#define FIRST_RESERVED_GDT_PAGE  14
#define FIRST_RESERVED_GDT_BYTE  (FIRST_RESERVED_GDT_PAGE * 4096)
#define FIRST_RESERVED_GDT_ENTRY (FIRST_RESERVED_GDT_BYTE / 8)

/* Maximum number of virtual CPUs in multi-processor guests. */
#define MAX_VIRT_CPUS 32

#ifndef __ASSEMBLY__

typedef unsigned long xen_ulong_t;

/*
 * Send an array of these to HYPERVISOR_set_trap_table().
 * The privilege level specifies which modes may enter a trap via a software
 * interrupt. On x86/64, since rings 1 and 2 are unavailable, we allocate
 * privilege levels as follows:
 *  Level == 0: Noone may enter
 *  Level == 1: Kernel may enter
 *  Level == 2: Kernel may enter
 *  Level == 3: Everyone may enter
 */
#define TI_GET_DPL(_ti)      ((_ti)->flags & 3)
#define TI_GET_IF(_ti)       ((_ti)->flags & 4)
#define TI_SET_DPL(_ti,_dpl) ((_ti)->flags |= (_dpl))
#define TI_SET_IF(_ti,_if)   ((_ti)->flags |= ((!!(_if))<<2))
struct trap_info {
    uint8_t       vector;  /* exception vector                              */
    uint8_t       flags;   /* 0-3: privilege level; 4: clear event enable?  */
    uint16_t      cs;      /* code selector                                 */
    unsigned long address; /* code offset                                   */
};
typedef struct trap_info trap_info_t;
DEFINE_XEN_GUEST_HANDLE(trap_info_t);

typedef uint64_t tsc_timestamp_t; /* RDTSC timestamp */

/*
 * The following is all CPU context. Note that the fpu_ctxt block is filled
 * in by FXSAVE if the CPU has feature FXSR; otherwise FSAVE is used.
 */
struct vcpu_guest_context {
    /* FPU registers come first so they can be aligned for FXSAVE/FXRSTOR. */
    struct { char x[512]; } fpu_ctxt;       /* User-level FPU registers     */
#define VGCF_I387_VALID                (1<<0)
#define VGCF_IN_KERNEL                 (1<<2)
#define _VGCF_i387_valid               0
#define VGCF_i387_valid                (1<<_VGCF_i387_valid)
#define _VGCF_in_kernel                2
#define VGCF_in_kernel                 (1<<_VGCF_in_kernel)
#define _VGCF_failsafe_disables_events 3
#define VGCF_failsafe_disables_events  (1<<_VGCF_failsafe_disables_events)
#define _VGCF_syscall_disables_events  4
#define VGCF_syscall_disables_events   (1<<_VGCF_syscall_disables_events)
#define _VGCF_online                   5
#define VGCF_online                    (1<<_VGCF_online)
    unsigned long flags;                    /* VGCF_* flags                 */
    struct cpu_user_regs user_regs;         /* User-level CPU registers     */
    struct trap_info trap_ctxt[256];        /* Virtual IDT                  */
    unsigned long ldt_base, ldt_ents;       /* LDT (linear address, # ents) */
    unsigned long gdt_frames[16], gdt_ents; /* GDT (machine frames, # ents) */
    unsigned long kernel_ss, kernel_sp;     /* Virtual TSS (only SS1/SP1)   */
    /* NB. User pagetable on x86/64 is placed in ctrlreg[1]. */
    unsigned long ctrlreg[8];               /* CR0-CR7 (control registers)  */
    unsigned long debugreg[8];              /* DB0-DB7 (debug registers)    */
#ifdef __i386__
    unsigned long event_callback_cs;        /* CS:EIP of event callback     */
    unsigned long event_callback_eip;
    unsigned long failsafe_callback_cs;     /* CS:EIP of failsafe callback  */
    unsigned long failsafe_callback_eip;
#else
    unsigned long event_callback_eip;
    unsigned long failsafe_callback_eip;
#ifdef __XEN__
    union {
        unsigned long syscall_callback_eip;
        struct {
            unsigned int event_callback_cs;    /* compat CS of event cb     */
            unsigned int failsafe_callback_cs; /* compat CS of failsafe cb  */
        };
    };
#else
    unsigned long syscall_callback_eip;
#endif
#endif
    unsigned long vm_assist;                /* VMASST_TYPE_* bitmap */
#ifdef __x86_64__
    /* Segment base addresses. */
    uint64_t      fs_base;
    uint64_t      gs_base_kernel;
    uint64_t      gs_base_user;
#endif
};
typedef struct vcpu_guest_context vcpu_guest_context_t;
DEFINE_XEN_GUEST_HANDLE(vcpu_guest_context_t);

struct arch_shared_info {
    unsigned long max_pfn;                  /* max pfn that appears in table */
    /* Frame containing list of mfns containing list of mfns containing p2m. */
    xen_pfn_t     pfn_to_mfn_frame_list_list;
    unsigned long nmi_reason;
    uint64_t pad[32];
};
typedef struct arch_shared_info arch_shared_info_t;

#define	MCA_PANICDATA_MAGIC	0x5044  /* "PD" */
#define MCA_PANICDATA_VERS	1

typedef struct xpv_mca_panic_data {
	uint16_t mpd_magic;
	uint16_t mpd_version;
	int mpd_fwdptr_offset;
	int mpd_revptr_offset;
	int mpd_dataptr_offset;
	void *mpd_urgent_processing;
	void *mpd_urgent_dangling;
	void *mpd_urgent_committed;
	void *mpd_nonurgent_processing;
	void *mpd_nonurgent_dangling;
	void *mpd_nonurgent_committed;
} xpv_mca_panic_data_t;

typedef struct panic_regs panic_regs_t;
struct panic_info {
	int		pi_version;	/* panic_info format version */
	panic_regs_t	*pi_regs;	/* register state */
	void		*pi_apic;	/* local APIC address */
	char		*pi_panicstr;	/* panic message */
	void		*pi_ram_start;	/* Start of all-RAM mapping region */
	void		*pi_ram_end;	/* End of all-RAM mapping region */
	void		*pi_xen_start;	/* Start of Xen's text/heap */
	void		*pi_xen_end;	/* End of Xen's text/heap */
	void		*pi_stktop;	/* Top of current Xen stack */
	struct domain	*pi_domain;	/* Panicking domain */
	struct vcpu	*pi_vcpu;	/* Panicking vcpu */
	int		pi_dom0cpu;	/* cpu number - if a dom0 panic */
	xpv_mca_panic_data_t pi_mca;	/* Machine check error telemetry */
};

struct panic_frame {
	unsigned long	pf_fp;
	unsigned long	pf_pc;
};

#define PANIC_INFO_VERSION	3

#endif /* !__ASSEMBLY__ */

/*
 * Prefix forces emulation of some non-trapping instructions.
 * Currently only CPUID.
 */
#ifdef __ASSEMBLY__
#define XEN_EMULATE_PREFIX .byte 0x0f,0x0b,0x78,0x65,0x6e ;
#define XEN_CPUID          XEN_EMULATE_PREFIX cpuid
#else
#define XEN_EMULATE_PREFIX ".byte 0x0f,0x0b,0x78,0x65,0x6e ; "
#define XEN_CPUID          XEN_EMULATE_PREFIX "cpuid"
#endif

#endif /* __XEN_PUBLIC_ARCH_X86_XEN_H__ */

/*
 * Local variables:
 * mode: C
 * c-set-style: "BSD"
 * c-basic-offset: 4
 * tab-width: 4
 * indent-tabs-mode: nil
 * End:
 */
/******************************************************************************
 * callback.h
 *
 * Register guest OS callbacks with Xen.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to
 * deal in the Software without restriction, including without limitation the
 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
 * sell copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 * DEALINGS IN THE SOFTWARE.
 *
 * Copyright (c) 2006, Ian Campbell
 */

#ifndef __XEN_PUBLIC_CALLBACK_H__
#define __XEN_PUBLIC_CALLBACK_H__

#include "xen.h"

/*
 * Prototype for this hypercall is:
 *   long callback_op(int cmd, void *extra_args)
 * @cmd        == CALLBACKOP_??? (callback operation).
 * @extra_args == Operation-specific extra arguments (NULL if none).
 */

/* ia64, x86: Callback for event delivery. */
#define CALLBACKTYPE_event                 0

/* x86: Failsafe callback when guest state cannot be restored by Xen. */
#define CALLBACKTYPE_failsafe              1

/* x86/64 hypervisor: Syscall by 64-bit guest app ('64-on-64-on-64'). */
#define CALLBACKTYPE_syscall               2

/*
 * x86/32 hypervisor: Only available on x86/32 when supervisor_mode_kernel
 *     feature is enabled. Do not use this callback type in new code.
 */
#define CALLBACKTYPE_sysenter_deprecated   3

/* x86: Callback for NMI delivery. */
#define CALLBACKTYPE_nmi                   4

/*
 * x86: sysenter is only available as follows:
 * - 32-bit hypervisor: with the supervisor_mode_kernel feature enabled
 * - 64-bit hypervisor: 32-bit guest applications on Intel CPUs
 *                      ('32-on-32-on-64', '32-on-64-on-64')
 *                      [nb. also 64-bit guest applications on Intel CPUs
 *                           ('64-on-64-on-64'), but syscall is preferred]
 */
#define CALLBACKTYPE_sysenter              5

/*
 * x86/64 hypervisor: Syscall by 32-bit guest app on AMD CPUs
 *                    ('32-on-32-on-64', '32-on-64-on-64')
 */
#define CALLBACKTYPE_syscall32             7

/*
 * Disable event deliver during callback? This flag is ignored for event and
 * NMI callbacks: event delivery is unconditionally disabled.
 */
#define _CALLBACKF_mask_events             0
#define CALLBACKF_mask_events              (1U << _CALLBACKF_mask_events)

/*
 * Register a callback.
 */
#define CALLBACKOP_register                0
struct callback_register {
    uint16_t type;
    uint16_t flags;
    xen_callback_t address;
};
typedef struct callback_register callback_register_t;
DEFINE_XEN_GUEST_HANDLE(callback_register_t);

/*
 * Unregister a callback.
 *
 * Not all callbacks can be unregistered. -EINVAL will be returned if
 * you attempt to unregister such a callback.
 */
#define CALLBACKOP_unregister              1
struct callback_unregister {
    uint16_t type;
    uint16_t _unused;
};
typedef struct callback_unregister callback_unregister_t;
DEFINE_XEN_GUEST_HANDLE(callback_unregister_t);

#if __XEN_INTERFACE_VERSION__ < 0x00030207
#undef CALLBACKTYPE_sysenter
#define CALLBACKTYPE_sysenter CALLBACKTYPE_sysenter_deprecated
#endif

#endif /* __XEN_PUBLIC_CALLBACK_H__ */

/*
 * Local variables:
 * mode: C
 * c-set-style: "BSD"
 * c-basic-offset: 4
 * tab-width: 4
 * indent-tabs-mode: nil
 * End:
 */
/******************************************************************************
 * domctl.h
 *
 * Domain management operations. For use by node control stack.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to
 * deal in the Software without restriction, including without limitation the
 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
 * sell copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 * DEALINGS IN THE SOFTWARE.
 *
 * Copyright (c) 2002-2003, B Dragovic
 * Copyright (c) 2002-2006, K Fraser
 */

#ifndef __XEN_PUBLIC_DOMCTL_H__
#define __XEN_PUBLIC_DOMCTL_H__

#if !defined(__XEN__) && !defined(__XEN_TOOLS__)
#error "domctl operations are intended for use by node control tools only"
#endif

#include "xen.h"

#define XEN_DOMCTL_INTERFACE_VERSION 0x00000005

struct xenctl_cpumap {
    XEN_GUEST_HANDLE_64(uint8) bitmap;
    uint32_t nr_cpus;
};

/*
 * NB. xen_domctl.domain is an IN/OUT parameter for this operation.
 * If it is specified as zero, an id is auto-allocated and returned.
 */
#define XEN_DOMCTL_createdomain       1
struct xen_domctl_createdomain {
    /* IN parameters */
    uint32_t ssidref;
    xen_domain_handle_t handle;
 /* Is this an HVM guest (as opposed to a PV guest)? */
#define _XEN_DOMCTL_CDF_hvm_guest     0
#define XEN_DOMCTL_CDF_hvm_guest      (1U<<_XEN_DOMCTL_CDF_hvm_guest)
 /* Use hardware-assisted paging if available? */
#define _XEN_DOMCTL_CDF_hap           1
#define XEN_DOMCTL_CDF_hap            (1U<<_XEN_DOMCTL_CDF_hap)
 /* Should domain memory integrity be verifed by tboot during Sx? */
#define _XEN_DOMCTL_CDF_s3_integrity  2
#define XEN_DOMCTL_CDF_s3_integrity   (1U<<_XEN_DOMCTL_CDF_s3_integrity)
    uint32_t flags;
};
typedef struct xen_domctl_createdomain xen_domctl_createdomain_t;
DEFINE_XEN_GUEST_HANDLE(xen_domctl_createdomain_t);

#define XEN_DOMCTL_destroydomain      2
#define XEN_DOMCTL_pausedomain        3
#define XEN_DOMCTL_unpausedomain      4
#define XEN_DOMCTL_resumedomain      27

#define XEN_DOMCTL_getdomaininfo      5
struct xen_domctl_getdomaininfo {
    /* OUT variables. */
    domid_t  domain;              /* Also echoed in domctl.domain */
 /* Domain is scheduled to die. */
#define _XEN_DOMINF_dying     0
#define XEN_DOMINF_dying      (1U<<_XEN_DOMINF_dying)
 /* Domain is an HVM guest (as opposed to a PV guest). */
#define _XEN_DOMINF_hvm_guest 1
#define XEN_DOMINF_hvm_guest  (1U<<_XEN_DOMINF_hvm_guest)
 /* The guest OS has shut down. */
#define _XEN_DOMINF_shutdown  2
#define XEN_DOMINF_shutdown   (1U<<_XEN_DOMINF_shutdown)
 /* Currently paused by control software. */
#define _XEN_DOMINF_paused    3
#define XEN_DOMINF_paused     (1U<<_XEN_DOMINF_paused)
 /* Currently blocked pending an event.     */
#define _XEN_DOMINF_blocked   4
#define XEN_DOMINF_blocked    (1U<<_XEN_DOMINF_blocked)
 /* Domain is currently running.            */
#define _XEN_DOMINF_running   5
#define XEN_DOMINF_running    (1U<<_XEN_DOMINF_running)
 /* Being debugged.  */
#define _XEN_DOMINF_debugged  6
#define XEN_DOMINF_debugged   (1U<<_XEN_DOMINF_debugged)
 /* XEN_DOMINF_shutdown guest-supplied code.  */
#define XEN_DOMINF_shutdownmask 255
#define XEN_DOMINF_shutdownshift 16
    uint32_t flags;              /* XEN_DOMINF_* */
    uint64_aligned_t tot_pages;
    uint64_aligned_t max_pages;
    uint64_aligned_t shared_info_frame; /* GMFN of shared_info struct */
    uint64_aligned_t cpu_time;
    uint32_t nr_online_vcpus;    /* Number of VCPUs currently online. */
    uint32_t max_vcpu_id;        /* Maximum VCPUID in use by this domain. */
    uint32_t ssidref;
    xen_domain_handle_t handle;
};
typedef struct xen_domctl_getdomaininfo xen_domctl_getdomaininfo_t;
DEFINE_XEN_GUEST_HANDLE(xen_domctl_getdomaininfo_t);


#define XEN_DOMCTL_getmemlist         6
struct xen_domctl_getmemlist {
    /* IN variables. */
    /* Max entries to write to output buffer. */
    uint64_aligned_t max_pfns;
    /* Start index in guest's page list. */
    uint64_aligned_t start_pfn;
    XEN_GUEST_HANDLE_64(uint64) buffer;
    /* OUT variables. */
    uint64_aligned_t num_pfns;
};
typedef struct xen_domctl_getmemlist xen_domctl_getmemlist_t;
DEFINE_XEN_GUEST_HANDLE(xen_domctl_getmemlist_t);


#define XEN_DOMCTL_getpageframeinfo   7

#define XEN_DOMCTL_PFINFO_LTAB_SHIFT 28
#define XEN_DOMCTL_PFINFO_NOTAB   (0x0U<<28)
#define XEN_DOMCTL_PFINFO_L1TAB   (0x1U<<28)
#define XEN_DOMCTL_PFINFO_L2TAB   (0x2U<<28)
#define XEN_DOMCTL_PFINFO_L3TAB   (0x3U<<28)
#define XEN_DOMCTL_PFINFO_L4TAB   (0x4U<<28)
#define XEN_DOMCTL_PFINFO_LTABTYPE_MASK (0x7U<<28)
#define XEN_DOMCTL_PFINFO_LPINTAB (0x1U<<31)
#define XEN_DOMCTL_PFINFO_XTAB    (0xfU<<28) /* invalid page */
#define XEN_DOMCTL_PFINFO_LTAB_MASK (0xfU<<28)

struct xen_domctl_getpageframeinfo {
    /* IN variables. */
    uint64_aligned_t gmfn; /* GMFN to query */
    /* OUT variables. */
    /* Is the page PINNED to a type? */
    uint32_t type;         /* see above type defs */
};
typedef struct xen_domctl_getpageframeinfo xen_domctl_getpageframeinfo_t;
DEFINE_XEN_GUEST_HANDLE(xen_domctl_getpageframeinfo_t);


#define XEN_DOMCTL_getpageframeinfo2  8
struct xen_domctl_getpageframeinfo2 {
    /* IN variables. */
    uint64_aligned_t num;
    /* IN/OUT variables. */
    XEN_GUEST_HANDLE_64(uint32) array;
};
typedef struct xen_domctl_getpageframeinfo2 xen_domctl_getpageframeinfo2_t;
DEFINE_XEN_GUEST_HANDLE(xen_domctl_getpageframeinfo2_t);


/*
 * Control shadow pagetables operation
 */
#define XEN_DOMCTL_shadow_op         10

/* Disable shadow mode. */
#define XEN_DOMCTL_SHADOW_OP_OFF         0

/* Enable shadow mode (mode contains ORed XEN_DOMCTL_SHADOW_ENABLE_* flags). */
#define XEN_DOMCTL_SHADOW_OP_ENABLE      32

/* Log-dirty bitmap operations. */
 /* Return the bitmap and clean internal copy for next round. */
#define XEN_DOMCTL_SHADOW_OP_CLEAN       11
 /* Return the bitmap but do not modify internal copy. */
#define XEN_DOMCTL_SHADOW_OP_PEEK        12

/* Memory allocation accessors. */
#define XEN_DOMCTL_SHADOW_OP_GET_ALLOCATION   30
#define XEN_DOMCTL_SHADOW_OP_SET_ALLOCATION   31

/* Legacy enable operations. */
 /* Equiv. to ENABLE with no mode flags. */
#define XEN_DOMCTL_SHADOW_OP_ENABLE_TEST       1
 /* Equiv. to ENABLE with mode flag ENABLE_LOG_DIRTY. */
#define XEN_DOMCTL_SHADOW_OP_ENABLE_LOGDIRTY   2
 /* Equiv. to ENABLE with mode flags ENABLE_REFCOUNT and ENABLE_TRANSLATE. */
#define XEN_DOMCTL_SHADOW_OP_ENABLE_TRANSLATE  3

/* Mode flags for XEN_DOMCTL_SHADOW_OP_ENABLE. */
 /*
  * Shadow pagetables are refcounted: guest does not use explicit mmu
  * operations nor write-protect its pagetables.
  */
#define XEN_DOMCTL_SHADOW_ENABLE_REFCOUNT  (1 << 1)
 /*
  * Log pages in a bitmap as they are dirtied.
  * Used for live relocation to determine which pages must be re-sent.
  */
#define XEN_DOMCTL_SHADOW_ENABLE_LOG_DIRTY (1 << 2)
 /*
  * Automatically translate GPFNs into MFNs.
  */
#define XEN_DOMCTL_SHADOW_ENABLE_TRANSLATE (1 << 3)
 /*
  * Xen does not steal virtual address space from the guest.
  * Requires HVM support.
  */
#define XEN_DOMCTL_SHADOW_ENABLE_EXTERNAL  (1 << 4)

struct xen_domctl_shadow_op_stats {
    uint32_t fault_count;
    uint32_t dirty_count;
};
typedef struct xen_domctl_shadow_op_stats xen_domctl_shadow_op_stats_t;
DEFINE_XEN_GUEST_HANDLE(xen_domctl_shadow_op_stats_t);

struct xen_domctl_shadow_op {
    /* IN variables. */
    uint32_t       op;       /* XEN_DOMCTL_SHADOW_OP_* */

    /* OP_ENABLE */
    uint32_t       mode;     /* XEN_DOMCTL_SHADOW_ENABLE_* */

    /* OP_GET_ALLOCATION / OP_SET_ALLOCATION */
    uint32_t       mb;       /* Shadow memory allocation in MB */

    /* OP_PEEK / OP_CLEAN */
    XEN_GUEST_HANDLE_64(uint8) dirty_bitmap;
    uint64_aligned_t pages; /* Size of buffer. Updated with actual size. */
    struct xen_domctl_shadow_op_stats stats;
};
typedef struct xen_domctl_shadow_op xen_domctl_shadow_op_t;
DEFINE_XEN_GUEST_HANDLE(xen_domctl_shadow_op_t);


#define XEN_DOMCTL_max_mem           11
struct xen_domctl_max_mem {
    /* IN variables. */
    uint64_aligned_t max_memkb;
};
typedef struct xen_domctl_max_mem xen_domctl_max_mem_t;
DEFINE_XEN_GUEST_HANDLE(xen_domctl_max_mem_t);


#define XEN_DOMCTL_setvcpucontext    12
#define XEN_DOMCTL_getvcpucontext    13
struct xen_domctl_vcpucontext {
    uint32_t              vcpu;                  /* IN */
    XEN_GUEST_HANDLE_64(vcpu_guest_context_t) ctxt; /* IN/OUT */
};
typedef struct xen_domctl_vcpucontext xen_domctl_vcpucontext_t;
DEFINE_XEN_GUEST_HANDLE(xen_domctl_vcpucontext_t);


#define XEN_DOMCTL_getvcpuinfo       14
struct xen_domctl_getvcpuinfo {
    /* IN variables. */
    uint32_t vcpu;
    /* OUT variables. */
    uint8_t  online;                  /* currently online (not hotplugged)? */
    uint8_t  blocked;                 /* blocked waiting for an event? */
    uint8_t  running;                 /* currently scheduled on its CPU? */
    uint64_aligned_t cpu_time;        /* total cpu time consumed (ns) */
    uint32_t cpu;                     /* current mapping   */
};
typedef struct xen_domctl_getvcpuinfo xen_domctl_getvcpuinfo_t;
DEFINE_XEN_GUEST_HANDLE(xen_domctl_getvcpuinfo_t);


/* Get/set which physical cpus a vcpu can execute on. */
#define XEN_DOMCTL_setvcpuaffinity    9
#define XEN_DOMCTL_getvcpuaffinity   25
struct xen_domctl_vcpuaffinity {
    uint32_t  vcpu;              /* IN */
    struct xenctl_cpumap cpumap; /* IN/OUT */
};
typedef struct xen_domctl_vcpuaffinity xen_domctl_vcpuaffinity_t;
DEFINE_XEN_GUEST_HANDLE(xen_domctl_vcpuaffinity_t);


#define XEN_DOMCTL_max_vcpus         15
struct xen_domctl_max_vcpus {
    uint32_t max;           /* maximum number of vcpus */
};
typedef struct xen_domctl_max_vcpus xen_domctl_max_vcpus_t;
DEFINE_XEN_GUEST_HANDLE(xen_domctl_max_vcpus_t);


#define XEN_DOMCTL_scheduler_op      16
/* Scheduler types. */
#define XEN_SCHEDULER_SEDF     4
#define XEN_SCHEDULER_CREDIT   5
/* Set or get info? */
#define XEN_DOMCTL_SCHEDOP_putinfo 0
#define XEN_DOMCTL_SCHEDOP_getinfo 1
struct xen_domctl_scheduler_op {
    uint32_t sched_id;  /* XEN_SCHEDULER_* */
    uint32_t cmd;       /* XEN_DOMCTL_SCHEDOP_* */
    union {
        struct xen_domctl_sched_sedf {
            uint64_aligned_t period;
            uint64_aligned_t slice;
            uint64_aligned_t latency;
            uint32_t extratime;
            uint32_t weight;
        } sedf;
        struct xen_domctl_sched_credit {
            uint16_t weight;
            uint16_t cap;
        } credit;
    } u;
};
typedef struct xen_domctl_scheduler_op xen_domctl_scheduler_op_t;
DEFINE_XEN_GUEST_HANDLE(xen_domctl_scheduler_op_t);


#define XEN_DOMCTL_setdomainhandle   17
struct xen_domctl_setdomainhandle {
    xen_domain_handle_t handle;
};
typedef struct xen_domctl_setdomainhandle xen_domctl_setdomainhandle_t;
DEFINE_XEN_GUEST_HANDLE(xen_domctl_setdomainhandle_t);


#define XEN_DOMCTL_setdebugging      18
struct xen_domctl_setdebugging {
    uint8_t enable;
};
typedef struct xen_domctl_setdebugging xen_domctl_setdebugging_t;
DEFINE_XEN_GUEST_HANDLE(xen_domctl_setdebugging_t);


#define XEN_DOMCTL_irq_permission    19
struct xen_domctl_irq_permission {
    uint8_t pirq;
    uint8_t allow_access;    /* flag to specify enable/disable of IRQ access */
};
typedef struct xen_domctl_irq_permission xen_domctl_irq_permission_t;
DEFINE_XEN_GUEST_HANDLE(xen_domctl_irq_permission_t);


#define XEN_DOMCTL_iomem_permission  20
struct xen_domctl_iomem_permission {
    uint64_aligned_t first_mfn;/* first page (physical page number) in range */
    uint64_aligned_t nr_mfns;  /* number of pages in range (>0) */
    uint8_t  allow_access;     /* allow (!0) or deny (0) access to range? */
};
typedef struct xen_domctl_iomem_permission xen_domctl_iomem_permission_t;
DEFINE_XEN_GUEST_HANDLE(xen_domctl_iomem_permission_t);


#define XEN_DOMCTL_ioport_permission 21
struct xen_domctl_ioport_permission {
    uint32_t first_port;              /* first port int range */
    uint32_t nr_ports;                /* size of port range */
    uint8_t  allow_access;            /* allow or deny access to range? */
};
typedef struct xen_domctl_ioport_permission xen_domctl_ioport_permission_t;
DEFINE_XEN_GUEST_HANDLE(xen_domctl_ioport_permission_t);


#define XEN_DOMCTL_hypercall_init    22
struct xen_domctl_hypercall_init {
    uint64_aligned_t  gmfn;           /* GMFN to be initialised */
};
typedef struct xen_domctl_hypercall_init xen_domctl_hypercall_init_t;
DEFINE_XEN_GUEST_HANDLE(xen_domctl_hypercall_init_t);


#define XEN_DOMCTL_arch_setup        23
#define _XEN_DOMAINSETUP_hvm_guest 0
#define XEN_DOMAINSETUP_hvm_guest  (1UL<<_XEN_DOMAINSETUP_hvm_guest)
#define _XEN_DOMAINSETUP_query 1 /* Get parameters (for save)  */
#define XEN_DOMAINSETUP_query  (1UL<<_XEN_DOMAINSETUP_query)
#define _XEN_DOMAINSETUP_sioemu_guest 2
#define XEN_DOMAINSETUP_sioemu_guest  (1UL<<_XEN_DOMAINSETUP_sioemu_guest)
typedef struct xen_domctl_arch_setup {
    uint64_aligned_t flags;  /* XEN_DOMAINSETUP_* */
#ifdef __ia64__
    uint64_aligned_t bp;     /* mpaddr of boot param area */
    uint64_aligned_t maxmem; /* Highest memory address for MDT.  */
    uint64_aligned_t xsi_va; /* Xen shared_info area virtual address.  */
    uint32_t hypercall_imm;  /* Break imm for Xen hypercalls.  */
    int8_t vhpt_size_log2;   /* Log2 of VHPT size. */
#endif
} xen_domctl_arch_setup_t;
DEFINE_XEN_GUEST_HANDLE(xen_domctl_arch_setup_t);


#define XEN_DOMCTL_settimeoffset     24
struct xen_domctl_settimeoffset {
    int32_t  time_offset_seconds; /* applied to domain wallclock time */
};
typedef struct xen_domctl_settimeoffset xen_domctl_settimeoffset_t;
DEFINE_XEN_GUEST_HANDLE(xen_domctl_settimeoffset_t);


#define XEN_DOMCTL_gethvmcontext     33
#define XEN_DOMCTL_sethvmcontext     34
typedef struct xen_domctl_hvmcontext {
    uint32_t size; /* IN/OUT: size of buffer / bytes filled */
    XEN_GUEST_HANDLE_64(uint8) buffer; /* IN/OUT: data, or call
                                        * gethvmcontext with NULL
                                        * buffer to get size req'd */
} xen_domctl_hvmcontext_t;
DEFINE_XEN_GUEST_HANDLE(xen_domctl_hvmcontext_t);


#define XEN_DOMCTL_set_address_size  35
#define XEN_DOMCTL_get_address_size  36
typedef struct xen_domctl_address_size {
    uint32_t size;
} xen_domctl_address_size_t;
DEFINE_XEN_GUEST_HANDLE(xen_domctl_address_size_t);


#define XEN_DOMCTL_real_mode_area    26
struct xen_domctl_real_mode_area {
    uint32_t log; /* log2 of Real Mode Area size */
};
typedef struct xen_domctl_real_mode_area xen_domctl_real_mode_area_t;
DEFINE_XEN_GUEST_HANDLE(xen_domctl_real_mode_area_t);


#define XEN_DOMCTL_sendtrigger       28
#define XEN_DOMCTL_SENDTRIGGER_NMI    0
#define XEN_DOMCTL_SENDTRIGGER_RESET  1
#define XEN_DOMCTL_SENDTRIGGER_INIT   2
#define XEN_DOMCTL_SENDTRIGGER_POWER  3
struct xen_domctl_sendtrigger {
    uint32_t  trigger;  /* IN */
    uint32_t  vcpu;     /* IN */
};
typedef struct xen_domctl_sendtrigger xen_domctl_sendtrigger_t;
DEFINE_XEN_GUEST_HANDLE(xen_domctl_sendtrigger_t);


/* Assign PCI device to HVM guest. Sets up IOMMU structures. */
#define XEN_DOMCTL_assign_device      37
#define XEN_DOMCTL_test_assign_device 45
#define XEN_DOMCTL_deassign_device 47
struct xen_domctl_assign_device {
    uint32_t  machine_bdf;   /* machine PCI ID of assigned device */
};
typedef struct xen_domctl_assign_device xen_domctl_assign_device_t;
DEFINE_XEN_GUEST_HANDLE(xen_domctl_assign_device_t);

/* Retrieve sibling devices infomation of machine_bdf */
#define XEN_DOMCTL_get_device_group 50
struct xen_domctl_get_device_group {
    uint32_t  machine_bdf;      /* IN */
    uint32_t  max_sdevs;        /* IN */
    uint32_t  num_sdevs;        /* OUT */
    XEN_GUEST_HANDLE_64(uint32)  sdev_array;   /* OUT */
};
typedef struct xen_domctl_get_device_group xen_domctl_get_device_group_t;
DEFINE_XEN_GUEST_HANDLE(xen_domctl_get_device_group_t);

/* Pass-through interrupts: bind real irq -> hvm devfn. */
#define XEN_DOMCTL_bind_pt_irq       38
#define XEN_DOMCTL_unbind_pt_irq     48
typedef enum pt_irq_type_e {
    PT_IRQ_TYPE_PCI,
    PT_IRQ_TYPE_ISA,
    PT_IRQ_TYPE_MSI,
    PT_IRQ_TYPE_MSI_TRANSLATE,
} pt_irq_type_t;
struct xen_domctl_bind_pt_irq {
    uint32_t machine_irq;
    pt_irq_type_t irq_type;
    uint32_t hvm_domid;

    union {
        struct {
            uint8_t isa_irq;
        } isa;
        struct {
            uint8_t bus;
            uint8_t device;
            uint8_t intx;
        } pci;
        struct {
            uint8_t gvec;
            uint32_t gflags;
            uint64_aligned_t gtable;
        } msi;
    } u;
};
typedef struct xen_domctl_bind_pt_irq xen_domctl_bind_pt_irq_t;
DEFINE_XEN_GUEST_HANDLE(xen_domctl_bind_pt_irq_t);


/* Bind machine I/O address range -> HVM address range. */
#define XEN_DOMCTL_memory_mapping    39
#define DPCI_ADD_MAPPING         1
#define DPCI_REMOVE_MAPPING      0
struct xen_domctl_memory_mapping {
    uint64_aligned_t first_gfn; /* first page (hvm guest phys page) in range */
    uint64_aligned_t first_mfn; /* first page (machine page) in range */
    uint64_aligned_t nr_mfns;   /* number of pages in range (>0) */
    uint32_t add_mapping;       /* add or remove mapping */
    uint32_t padding;           /* padding for 64-bit aligned structure */
};
typedef struct xen_domctl_memory_mapping xen_domctl_memory_mapping_t;
DEFINE_XEN_GUEST_HANDLE(xen_domctl_memory_mapping_t);


/* Bind machine I/O port range -> HVM I/O port range. */
#define XEN_DOMCTL_ioport_mapping    40
struct xen_domctl_ioport_mapping {
    uint32_t first_gport;     /* first guest IO port*/
    uint32_t first_mport;     /* first machine IO port */
    uint32_t nr_ports;        /* size of port range */
    uint32_t add_mapping;     /* add or remove mapping */
};
typedef struct xen_domctl_ioport_mapping xen_domctl_ioport_mapping_t;
DEFINE_XEN_GUEST_HANDLE(xen_domctl_ioport_mapping_t);


/*
 * Pin caching type of RAM space for x86 HVM domU.
 */
#define XEN_DOMCTL_pin_mem_cacheattr 41
/* Caching types: these happen to be the same as x86 MTRR/PAT type codes. */
#define XEN_DOMCTL_MEM_CACHEATTR_UC  0
#define XEN_DOMCTL_MEM_CACHEATTR_WC  1
#define XEN_DOMCTL_MEM_CACHEATTR_WT  4
#define XEN_DOMCTL_MEM_CACHEATTR_WP  5
#define XEN_DOMCTL_MEM_CACHEATTR_WB  6
#define XEN_DOMCTL_MEM_CACHEATTR_UCM 7
struct xen_domctl_pin_mem_cacheattr {
    uint64_aligned_t start, end;
    unsigned int type; /* XEN_DOMCTL_MEM_CACHEATTR_* */
};
typedef struct xen_domctl_pin_mem_cacheattr xen_domctl_pin_mem_cacheattr_t;
DEFINE_XEN_GUEST_HANDLE(xen_domctl_pin_mem_cacheattr_t);


#define XEN_DOMCTL_set_ext_vcpucontext 42
#define XEN_DOMCTL_get_ext_vcpucontext 43
struct xen_domctl_ext_vcpucontext {
    /* IN: VCPU that this call applies to. */
    uint32_t         vcpu;
    /*
     * SET: Size of struct (IN)
     * GET: Size of struct (OUT)
     */
    uint32_t         size;
#if defined(__i386__) || defined(__x86_64__)
    /* SYSCALL from 32-bit mode and SYSENTER callback information. */
    /* NB. SYSCALL from 64-bit mode is contained in vcpu_guest_context_t */
    uint64_aligned_t syscall32_callback_eip;
    uint64_aligned_t sysenter_callback_eip;
    uint16_t         syscall32_callback_cs;
    uint16_t         sysenter_callback_cs;
    uint8_t          syscall32_disables_events;
    uint8_t          sysenter_disables_events;
#endif
};
typedef struct xen_domctl_ext_vcpucontext xen_domctl_ext_vcpucontext_t;
DEFINE_XEN_GUEST_HANDLE(xen_domctl_ext_vcpucontext_t);

/*
 * Set optimizaton features for a domain
 */
#define XEN_DOMCTL_set_opt_feature    44
struct xen_domctl_set_opt_feature {
#if defined(__ia64__)
    struct xen_ia64_opt_feature optf;
#else
    /* Make struct non-empty: do not depend on this field name! */
    uint64_t dummy;
#endif
};
typedef struct xen_domctl_set_opt_feature xen_domctl_set_opt_feature_t;
DEFINE_XEN_GUEST_HANDLE(xen_domctl_set_opt_feature_t);

/*
 * Set the target domain for a domain
 */
#define XEN_DOMCTL_set_target    46
struct xen_domctl_set_target {
    domid_t target;
};
typedef struct xen_domctl_set_target xen_domctl_set_target_t;
DEFINE_XEN_GUEST_HANDLE(xen_domctl_set_target_t);

#if defined(__i386__) || defined(__x86_64__)
# define XEN_CPUID_INPUT_UNUSED  0xFFFFFFFF
# define XEN_DOMCTL_set_cpuid 49
struct xen_domctl_cpuid {
  unsigned int  input[2];
  unsigned int  eax;
  unsigned int  ebx;
  unsigned int  ecx;
  unsigned int  edx;
};
typedef struct xen_domctl_cpuid xen_domctl_cpuid_t;
DEFINE_XEN_GUEST_HANDLE(xen_domctl_cpuid_t);
#endif

#define XEN_DOMCTL_subscribe          29
struct xen_domctl_subscribe {
    uint32_t port; /* IN */
};
typedef struct xen_domctl_subscribe xen_domctl_subscribe_t;
DEFINE_XEN_GUEST_HANDLE(xen_domctl_subscribe_t);

/*
 * Define the maximum machine address size which should be allocated
 * to a guest.
 */
#define XEN_DOMCTL_set_machine_address_size  51
#define XEN_DOMCTL_get_machine_address_size  52

/*
 * Do not inject spurious page faults into this domain.
 */
#define XEN_DOMCTL_suppress_spurious_page_faults 53

#define XEN_DOMCTL_debug_op    54
#define XEN_DOMCTL_DEBUG_OP_SINGLE_STEP_OFF         0
#define XEN_DOMCTL_DEBUG_OP_SINGLE_STEP_ON          1
struct xen_domctl_debug_op {
    uint32_t op;   /* IN */
    uint32_t vcpu; /* IN */
};
typedef struct xen_domctl_debug_op xen_domctl_debug_op_t;
DEFINE_XEN_GUEST_HANDLE(xen_domctl_debug_op_t);

/*
 * Request a particular record from the HVM context
 */
#define XEN_DOMCTL_gethvmcontext_partial   55
typedef struct xen_domctl_hvmcontext_partial {
    uint32_t type;                      /* IN: Type of record required */
    uint32_t instance;                  /* IN: Instance of that type */
    XEN_GUEST_HANDLE_64(uint8) buffer;  /* OUT: buffer to write record into */
} xen_domctl_hvmcontext_partial_t;
DEFINE_XEN_GUEST_HANDLE(xen_domctl_hvmcontext_partial_t);


struct xen_domctl {
    uint32_t cmd;
    uint32_t interface_version; /* XEN_DOMCTL_INTERFACE_VERSION */
    domid_t  domain;
    union {
        struct xen_domctl_createdomain      createdomain;
        struct xen_domctl_getdomaininfo     getdomaininfo;
        struct xen_domctl_getmemlist        getmemlist;
        struct xen_domctl_getpageframeinfo  getpageframeinfo;
        struct xen_domctl_getpageframeinfo2 getpageframeinfo2;
        struct xen_domctl_vcpuaffinity      vcpuaffinity;
        struct xen_domctl_shadow_op         shadow_op;
        struct xen_domctl_max_mem           max_mem;
        struct xen_domctl_vcpucontext       vcpucontext;
        struct xen_domctl_getvcpuinfo       getvcpuinfo;
        struct xen_domctl_max_vcpus         max_vcpus;
        struct xen_domctl_scheduler_op      scheduler_op;
        struct xen_domctl_setdomainhandle   setdomainhandle;
        struct xen_domctl_setdebugging      setdebugging;
        struct xen_domctl_irq_permission    irq_permission;
        struct xen_domctl_iomem_permission  iomem_permission;
        struct xen_domctl_ioport_permission ioport_permission;
        struct xen_domctl_hypercall_init    hypercall_init;
        struct xen_domctl_arch_setup        arch_setup;
        struct xen_domctl_settimeoffset     settimeoffset;
        struct xen_domctl_real_mode_area    real_mode_area;
        struct xen_domctl_hvmcontext        hvmcontext;
        struct xen_domctl_hvmcontext_partial hvmcontext_partial;
        struct xen_domctl_address_size      address_size;
        struct xen_domctl_sendtrigger       sendtrigger;
        struct xen_domctl_get_device_group  get_device_group;
        struct xen_domctl_assign_device     assign_device;
        struct xen_domctl_bind_pt_irq       bind_pt_irq;
        struct xen_domctl_memory_mapping    memory_mapping;
        struct xen_domctl_ioport_mapping    ioport_mapping;
        struct xen_domctl_pin_mem_cacheattr pin_mem_cacheattr;
        struct xen_domctl_ext_vcpucontext   ext_vcpucontext;
        struct xen_domctl_set_opt_feature   set_opt_feature;
        struct xen_domctl_set_target        set_target;
        struct xen_domctl_subscribe         subscribe;
        struct xen_domctl_debug_op          debug_op;
#if defined(__i386__) || defined(__x86_64__)
        struct xen_domctl_cpuid             cpuid;
#endif
        uint8_t                             pad[128];
    } u;
};
typedef struct xen_domctl xen_domctl_t;
DEFINE_XEN_GUEST_HANDLE(xen_domctl_t);

#endif /* __XEN_PUBLIC_DOMCTL_H__ */

/*
 * Local variables:
 * mode: C
 * c-set-style: "BSD"
 * c-basic-offset: 4
 * tab-width: 4
 * indent-tabs-mode: nil
 * End:
 */
/******************************************************************************
 * elfnote.h
 *
 * Definitions used for the Xen ELF notes.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to
 * deal in the Software without restriction, including without limitation the
 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
 * sell copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 * DEALINGS IN THE SOFTWARE.
 *
 * Copyright (c) 2006, Ian Campbell, XenSource Ltd.
 */

#ifndef __XEN_PUBLIC_ELFNOTE_H__
#define __XEN_PUBLIC_ELFNOTE_H__

/*
 * The notes should live in a PT_NOTE segment and have "Xen" in the
 * name field.
 *
 * Numeric types are either 4 or 8 bytes depending on the content of
 * the desc field.
 *
 * LEGACY indicated the fields in the legacy __xen_guest string which
 * this a note type replaces.
 */

/*
 * NAME=VALUE pair (string).
 */
#define XEN_ELFNOTE_INFO           0

/*
 * The virtual address of the entry point (numeric).
 *
 * LEGACY: VIRT_ENTRY
 */
#define XEN_ELFNOTE_ENTRY          1

/* The virtual address of the hypercall transfer page (numeric).
 *
 * LEGACY: HYPERCALL_PAGE. (n.b. legacy value is a physical page
 * number not a virtual address)
 */
#define XEN_ELFNOTE_HYPERCALL_PAGE 2

/* The virtual address where the kernel image should be mapped (numeric).
 *
 * Defaults to 0.
 *
 * LEGACY: VIRT_BASE
 */
#define XEN_ELFNOTE_VIRT_BASE      3

/*
 * The offset of the ELF paddr field from the acutal required
 * psuedo-physical address (numeric).
 *
 * This is used to maintain backwards compatibility with older kernels
 * which wrote __PAGE_OFFSET into that field. This field defaults to 0
 * if not present.
 *
 * LEGACY: ELF_PADDR_OFFSET. (n.b. legacy default is VIRT_BASE)
 */
#define XEN_ELFNOTE_PADDR_OFFSET   4

/*
 * The version of Xen that we work with (string).
 *
 * LEGACY: XEN_VER
 */
#define XEN_ELFNOTE_XEN_VERSION    5

/*
 * The name of the guest operating system (string).
 *
 * LEGACY: GUEST_OS
 */
#define XEN_ELFNOTE_GUEST_OS       6

/*
 * The version of the guest operating system (string).
 *
 * LEGACY: GUEST_VER
 */
#define XEN_ELFNOTE_GUEST_VERSION  7

/*
 * The loader type (string).
 *
 * LEGACY: LOADER
 */
#define XEN_ELFNOTE_LOADER         8

/*
 * The kernel supports PAE (x86/32 only, string = "yes", "no" or
 * "bimodal").
 *
 * For compatibility with Xen 3.0.3 and earlier the "bimodal" setting
 * may be given as "yes,bimodal" which will cause older Xen to treat
 * this kernel as PAE.
 *
 * LEGACY: PAE (n.b. The legacy interface included a provision to
 * indicate 'extended-cr3' support allowing L3 page tables to be
 * placed above 4G. It is assumed that any kernel new enough to use
 * these ELF notes will include this and therefore "yes" here is
 * equivalent to "yes[entended-cr3]" in the __xen_guest interface.
 */
#define XEN_ELFNOTE_PAE_MODE       9

/*
 * The features supported/required by this kernel (string).
 *
 * The string must consist of a list of feature names (as given in
 * features.h, without the "XENFEAT_" prefix) separated by '|'
 * characters. If a feature is required for the kernel to function
 * then the feature name must be preceded by a '!' character.
 *
 * LEGACY: FEATURES
 */
#define XEN_ELFNOTE_FEATURES      10

/*
 * The kernel requires the symbol table to be loaded (string = "yes" or "no")
 * LEGACY: BSD_SYMTAB (n.b. The legacy treated the presence or absence
 * of this string as a boolean flag rather than requiring "yes" or
 * "no".
 */
#define XEN_ELFNOTE_BSD_SYMTAB    11

/*
 * The lowest address the hypervisor hole can begin at (numeric).
 *
 * This must not be set higher than HYPERVISOR_VIRT_START. Its presence
 * also indicates to the hypervisor that the kernel can deal with the
 * hole starting at a higher address.
 */
#define XEN_ELFNOTE_HV_START_LOW  12

/*
 * List of maddr_t-sized mask/value pairs describing how to recognize
 * (non-present) L1 page table entries carrying valid MFNs (numeric).
 */
#define XEN_ELFNOTE_L1_MFN_VALID  13

/*
 * Whether or not the guest supports cooperative suspend cancellation.
 */
#define XEN_ELFNOTE_SUSPEND_CANCEL 14

/*
 * The (non-default) location the initial phys-to-machine map should be
 * placed at by the hypervisor (Dom0) or the tools (DomU).
 * The kernel must be prepared for this mapping to be established using
 * large pages, despite such otherwise not being available to guests.
 * The kernel must also be able to handle the page table pages used for
 * this mapping not being accessible through the initial mapping.
 * (Only x86-64 supports this at present.)
 */
#define XEN_ELFNOTE_INIT_P2M      15

/*
 * The number of the highest elfnote defined.
 */
#define XEN_ELFNOTE_MAX XEN_ELFNOTE_INIT_P2M

/*
 * System information exported through crash notes.
 *
 * The kexec / kdump code will create one XEN_ELFNOTE_CRASH_INFO
 * note in case of a system crash. This note will contain various
 * information about the system, see xen/include/xen/elfcore.h.
 */
#define XEN_ELFNOTE_CRASH_INFO 0x1000001

/*
 * System registers exported through crash notes.
 *
 * The kexec / kdump code will create one XEN_ELFNOTE_CRASH_REGS
 * note per cpu in case of a system crash. This note is architecture
 * specific and will contain registers not saved in the "CORE" note.
 * See xen/include/xen/elfcore.h for more information.
 */
#define XEN_ELFNOTE_CRASH_REGS 0x1000002


/*
 * xen dump-core none note.
 * xm dump-core code will create one XEN_ELFNOTE_DUMPCORE_NONE
 * in its dump file to indicate that the file is xen dump-core
 * file. This note doesn't have any other information.
 * See tools/libxc/xc_core.h for more information.
 */
#define XEN_ELFNOTE_DUMPCORE_NONE               0x2000000

/*
 * xen dump-core header note.
 * xm dump-core code will create one XEN_ELFNOTE_DUMPCORE_HEADER
 * in its dump file.
 * See tools/libxc/xc_core.h for more information.
 */
#define XEN_ELFNOTE_DUMPCORE_HEADER             0x2000001

/*
 * xen dump-core xen version note.
 * xm dump-core code will create one XEN_ELFNOTE_DUMPCORE_XEN_VERSION
 * in its dump file. It contains the xen version obtained via the
 * XENVER hypercall.
 * See tools/libxc/xc_core.h for more information.
 */
#define XEN_ELFNOTE_DUMPCORE_XEN_VERSION        0x2000002

/*
 * xen dump-core format version note.
 * xm dump-core code will create one XEN_ELFNOTE_DUMPCORE_FORMAT_VERSION
 * in its dump file. It contains a format version identifier.
 * See tools/libxc/xc_core.h for more information.
 */
#define XEN_ELFNOTE_DUMPCORE_FORMAT_VERSION     0x2000003

#endif /* __XEN_PUBLIC_ELFNOTE_H__ */

/*
 * Local variables:
 * mode: C
 * c-set-style: "BSD"
 * c-basic-offset: 4
 * tab-width: 4
 * indent-tabs-mode: nil
 * End:
 */
/******************************************************************************
 * event_channel.h
 *
 * Event channels between domains.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to
 * deal in the Software without restriction, including without limitation the
 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
 * sell copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 * DEALINGS IN THE SOFTWARE.
 *
 * Copyright (c) 2003-2004, K A Fraser.
 */

#ifndef __XEN_PUBLIC_EVENT_CHANNEL_H__
#define __XEN_PUBLIC_EVENT_CHANNEL_H__

/*
 * Prototype for this hypercall is:
 *  int event_channel_op(int cmd, void *args)
 * @cmd  == EVTCHNOP_??? (event-channel operation).
 * @args == Operation-specific extra arguments (NULL if none).
 */

typedef uint32_t evtchn_port_t;
DEFINE_XEN_GUEST_HANDLE(evtchn_port_t);

/*
 * EVTCHNOP_alloc_unbound: Allocate a port in domain <dom> and mark as
 * accepting interdomain bindings from domain <remote_dom>. A fresh port
 * is allocated in <dom> and returned as <port>.
 * NOTES:
 *  1. If the caller is unprivileged then <dom> must be DOMID_SELF.
 *  2. <rdom> may be DOMID_SELF, allowing loopback connections.
 */
#define EVTCHNOP_alloc_unbound    6
struct evtchn_alloc_unbound {
    /* IN parameters */
    domid_t dom, remote_dom;
    /* OUT parameters */
    evtchn_port_t port;
};
typedef struct evtchn_alloc_unbound evtchn_alloc_unbound_t;

/*
 * EVTCHNOP_bind_interdomain: Construct an interdomain event channel between
 * the calling domain and <remote_dom>. <remote_dom,remote_port> must identify
 * a port that is unbound and marked as accepting bindings from the calling
 * domain. A fresh port is allocated in the calling domain and returned as
 * <local_port>.
 * NOTES:
 *  2. <remote_dom> may be DOMID_SELF, allowing loopback connections.
 */
#define EVTCHNOP_bind_interdomain 0
struct evtchn_bind_interdomain {
    /* IN parameters. */
    domid_t remote_dom;
    evtchn_port_t remote_port;
    /* OUT parameters. */
    evtchn_port_t local_port;
};
typedef struct evtchn_bind_interdomain evtchn_bind_interdomain_t;

/*
 * EVTCHNOP_bind_virq: Bind a local event channel to VIRQ <irq> on specified
 * vcpu.
 * NOTES:
 *  1. Virtual IRQs are classified as per-vcpu or global. See the VIRQ list
 *     in xen.h for the classification of each VIRQ.
 *  2. Global VIRQs must be allocated on VCPU0 but can subsequently be
 *     re-bound via EVTCHNOP_bind_vcpu.
 *  3. Per-vcpu VIRQs may be bound to at most one event channel per vcpu.
 *     The allocated event channel is bound to the specified vcpu and the
 *     binding cannot be changed.
 */
#define EVTCHNOP_bind_virq        1
struct evtchn_bind_virq {
    /* IN parameters. */
    uint32_t virq;
    uint32_t vcpu;
    /* OUT parameters. */
    evtchn_port_t port;
};
typedef struct evtchn_bind_virq evtchn_bind_virq_t;

/*
 * EVTCHNOP_bind_pirq: Bind a local event channel to PIRQ <irq>.
 * NOTES:
 *  1. A physical IRQ may be bound to at most one event channel per domain.
 *  2. Only a sufficiently-privileged domain may bind to a physical IRQ.
 */
#define EVTCHNOP_bind_pirq        2
struct evtchn_bind_pirq {
    /* IN parameters. */
    uint32_t pirq;
#define BIND_PIRQ__WILL_SHARE 1
    uint32_t flags; /* BIND_PIRQ__* */
    /* OUT parameters. */
    evtchn_port_t port;
};
typedef struct evtchn_bind_pirq evtchn_bind_pirq_t;

/*
 * EVTCHNOP_bind_ipi: Bind a local event channel to receive events.
 * NOTES:
 *  1. The allocated event channel is bound to the specified vcpu. The binding
 *     may not be changed.
 */
#define EVTCHNOP_bind_ipi         7
struct evtchn_bind_ipi {
    uint32_t vcpu;
    /* OUT parameters. */
    evtchn_port_t port;
};
typedef struct evtchn_bind_ipi evtchn_bind_ipi_t;

/*
 * EVTCHNOP_close: Close a local event channel <port>. If the channel is
 * interdomain then the remote end is placed in the unbound state
 * (EVTCHNSTAT_unbound), awaiting a new connection.
 */
#define EVTCHNOP_close            3
struct evtchn_close {
    /* IN parameters. */
    evtchn_port_t port;
};
typedef struct evtchn_close evtchn_close_t;

/*
 * EVTCHNOP_send: Send an event to the remote end of the channel whose local
 * endpoint is <port>.
 */
#define EVTCHNOP_send             4
struct evtchn_send {
    /* IN parameters. */
    evtchn_port_t port;
};
typedef struct evtchn_send evtchn_send_t;

/*
 * EVTCHNOP_status: Get the current status of the communication channel which
 * has an endpoint at <dom, port>.
 * NOTES:
 *  1. <dom> may be specified as DOMID_SELF.
 *  2. Only a sufficiently-privileged domain may obtain the status of an event
 *     channel for which <dom> is not DOMID_SELF.
 */
#define EVTCHNOP_status           5
struct evtchn_status {
    /* IN parameters */
    domid_t  dom;
    evtchn_port_t port;
    /* OUT parameters */
#define EVTCHNSTAT_closed       0  /* Channel is not in use.                 */
#define EVTCHNSTAT_unbound      1  /* Channel is waiting interdom connection.*/
#define EVTCHNSTAT_interdomain  2  /* Channel is connected to remote domain. */
#define EVTCHNSTAT_pirq         3  /* Channel is bound to a phys IRQ line.   */
#define EVTCHNSTAT_virq         4  /* Channel is bound to a virtual IRQ line */
#define EVTCHNSTAT_ipi          5  /* Channel is bound to a virtual IPI line */
    uint32_t status;
    uint32_t vcpu;                 /* VCPU to which this channel is bound.   */
    union {
        struct {
            domid_t dom;
        } unbound; /* EVTCHNSTAT_unbound */
        struct {
            domid_t dom;
            evtchn_port_t port;
        } interdomain; /* EVTCHNSTAT_interdomain */
        uint32_t pirq;      /* EVTCHNSTAT_pirq        */
        uint32_t virq;      /* EVTCHNSTAT_virq        */
    } u;
};
typedef struct evtchn_status evtchn_status_t;

/*
 * EVTCHNOP_bind_vcpu: Specify which vcpu a channel should notify when an
 * event is pending.
 * NOTES:
 *  1. IPI-bound channels always notify the vcpu specified at bind time.
 *     This binding cannot be changed.
 *  2. Per-VCPU VIRQ channels always notify the vcpu specified at bind time.
 *     This binding cannot be changed.
 *  3. All other channels notify vcpu0 by default. This default is set when
 *     the channel is allocated (a port that is freed and subsequently reused
 *     has its binding reset to vcpu0).
 */
#define EVTCHNOP_bind_vcpu        8
struct evtchn_bind_vcpu {
    /* IN parameters. */
    evtchn_port_t port;
    uint32_t vcpu;
};
typedef struct evtchn_bind_vcpu evtchn_bind_vcpu_t;

/*
 * EVTCHNOP_unmask: Unmask the specified local event-channel port and deliver
 * a notification to the appropriate VCPU if an event is pending.
 */
#define EVTCHNOP_unmask           9
struct evtchn_unmask {
    /* IN parameters. */
    evtchn_port_t port;
};
typedef struct evtchn_unmask evtchn_unmask_t;

/*
 * EVTCHNOP_reset: Close all event channels associated with specified domain.
 * NOTES:
 *  1. <dom> may be specified as DOMID_SELF.
 *  2. Only a sufficiently-privileged domain may specify other than DOMID_SELF.
 */
#define EVTCHNOP_reset           10
struct evtchn_reset {
    /* IN parameters. */
    domid_t dom;
};
typedef struct evtchn_reset evtchn_reset_t;

/*
 * Argument to event_channel_op_compat() hypercall. Superceded by new
 * event_channel_op() hypercall since 0x00030202.
 */
struct evtchn_op {
    uint32_t cmd; /* EVTCHNOP_* */
    union {
        struct evtchn_alloc_unbound    alloc_unbound;
        struct evtchn_bind_interdomain bind_interdomain;
        struct evtchn_bind_virq        bind_virq;
        struct evtchn_bind_pirq        bind_pirq;
        struct evtchn_bind_ipi         bind_ipi;
        struct evtchn_close            close;
        struct evtchn_send             send;
        struct evtchn_status           status;
        struct evtchn_bind_vcpu        bind_vcpu;
        struct evtchn_unmask           unmask;
    } u;
};
typedef struct evtchn_op evtchn_op_t;
DEFINE_XEN_GUEST_HANDLE(evtchn_op_t);

#endif /* __XEN_PUBLIC_EVENT_CHANNEL_H__ */

/*
 * Local variables:
 * mode: C
 * c-set-style: "BSD"
 * c-basic-offset: 4
 * tab-width: 4
 * indent-tabs-mode: nil
 * End:
 */
/******************************************************************************
 * features.h
 *
 * Feature flags, reported by XENVER_get_features.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to
 * deal in the Software without restriction, including without limitation the
 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
 * sell copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 * DEALINGS IN THE SOFTWARE.
 *
 * Copyright (c) 2006, Keir Fraser <keir@xensource.com>
 */

#ifndef __XEN_PUBLIC_FEATURES_H__
#define __XEN_PUBLIC_FEATURES_H__

/*
 * If set, the guest does not need to write-protect its pagetables, and can
 * update them via direct writes.
 */
#define XENFEAT_writable_page_tables       0

/*
 * If set, the guest does not need to write-protect its segment descriptor
 * tables, and can update them via direct writes.
 */
#define XENFEAT_writable_descriptor_tables 1

/*
 * If set, translation between the guest's 'pseudo-physical' address space
 * and the host's machine address space are handled by the hypervisor. In this
 * mode the guest does not need to perform phys-to/from-machine translations
 * when performing page table operations.
 */
#define XENFEAT_auto_translated_physmap    2

/* If set, the guest is running in supervisor mode (e.g., x86 ring 0). */
#define XENFEAT_supervisor_mode_kernel     3

/*
 * If set, the guest does not need to allocate x86 PAE page directories
 * below 4GB. This flag is usually implied by auto_translated_physmap.
 */
#define XENFEAT_pae_pgdir_above_4gb        4

/* x86: Does this Xen host support the MMU_PT_UPDATE_PRESERVE_AD hypercall? */
#define XENFEAT_mmu_pt_update_preserve_ad  5

/* x86: Does this Xen host support the MMU_{CLEAR,COPY}_PAGE hypercall? */
#define XENFEAT_highmem_assist             6

/*
 * If set, GNTTABOP_map_grant_ref honors flags to be placed into guest kernel
 * available pte bits.
 */
#define XENFEAT_gnttab_map_avail_bits      7

#define XENFEAT_NR_SUBMAPS 1

#endif /* __XEN_PUBLIC_FEATURES_H__ */

/*
 * Local variables:
 * mode: C
 * c-set-style: "BSD"
 * c-basic-offset: 4
 * tab-width: 4
 * indent-tabs-mode: nil
 * End:
 */
/******************************************************************************
 * grant_table.h
 *
 * Interface for granting foreign access to page frames, and receiving
 * page-ownership transfers.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to
 * deal in the Software without restriction, including without limitation the
 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
 * sell copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 * DEALINGS IN THE SOFTWARE.
 *
 * Copyright (c) 2004, K A Fraser
 */

#ifndef __XEN_PUBLIC_GRANT_TABLE_H__
#define __XEN_PUBLIC_GRANT_TABLE_H__


/***********************************
 * GRANT TABLE REPRESENTATION
 */

/* Some rough guidelines on accessing and updating grant-table entries
 * in a concurrency-safe manner. For more information, Linux contains a
 * reference implementation for guest OSes (arch/xen/kernel/grant_table.c).
 *
 * NB. WMB is a no-op on current-generation x86 processors. However, a
 *     compiler barrier will still be required.
 *
 * Introducing a valid entry into the grant table:
 *  1. Write ent->domid.
 *  2. Write ent->frame:
 *      GTF_permit_access:   Frame to which access is permitted.
 *      GTF_accept_transfer: Pseudo-phys frame slot being filled by new
 *                           frame, or zero if none.
 *  3. Write memory barrier (WMB).
 *  4. Write ent->flags, inc. valid type.
 *
 * Invalidating an unused GTF_permit_access entry:
 *  1. flags = ent->flags.
 *  2. Observe that !(flags & (GTF_reading|GTF_writing)).
 *  3. Check result of SMP-safe CMPXCHG(&ent->flags, flags, 0).
 *  NB. No need for WMB as reuse of entry is control-dependent on success of
 *      step 3, and all architectures guarantee ordering of ctrl-dep writes.
 *
 * Invalidating an in-use GTF_permit_access entry:
 *  This cannot be done directly. Request assistance from the domain controller
 *  which can set a timeout on the use of a grant entry and take necessary
 *  action. (NB. This is not yet implemented!).
 *
 * Invalidating an unused GTF_accept_transfer entry:
 *  1. flags = ent->flags.
 *  2. Observe that !(flags & GTF_transfer_committed). [*]
 *  3. Check result of SMP-safe CMPXCHG(&ent->flags, flags, 0).
 *  NB. No need for WMB as reuse of entry is control-dependent on success of
 *      step 3, and all architectures guarantee ordering of ctrl-dep writes.
 *  [*] If GTF_transfer_committed is set then the grant entry is 'committed'.
 *      The guest must /not/ modify the grant entry until the address of the
 *      transferred frame is written. It is safe for the guest to spin waiting
 *      for this to occur (detect by observing GTF_transfer_completed in
 *      ent->flags).
 *
 * Invalidating a committed GTF_accept_transfer entry:
 *  1. Wait for (ent->flags & GTF_transfer_completed).
 *
 * Changing a GTF_permit_access from writable to read-only:
 *  Use SMP-safe CMPXCHG to set GTF_readonly, while checking !GTF_writing.
 *
 * Changing a GTF_permit_access from read-only to writable:
 *  Use SMP-safe bit-setting instruction.
 */

/*
 * A grant table comprises a packed array of grant entries in one or more
 * page frames shared between Xen and a guest.
 * [XEN]: This field is written by Xen and read by the sharing guest.
 * [GST]: This field is written by the guest and read by Xen.
 */
struct grant_entry {
    /* GTF_xxx: various type and flag information.  [XEN,GST] */
    uint16_t flags;
    /* The domain being granted foreign privileges. [GST] */
    domid_t  domid;
    /*
     * GTF_permit_access: Frame that @domid is allowed to map and access. [GST]
     * GTF_accept_transfer: Frame whose ownership transferred by @domid. [XEN]
     */
    uint32_t frame;
};
typedef struct grant_entry grant_entry_t;

/*
 * Type of grant entry.
 *  GTF_invalid: This grant entry grants no privileges.
 *  GTF_permit_access: Allow @domid to map/access @frame.
 *  GTF_accept_transfer: Allow @domid to transfer ownership of one page frame
 *                       to this guest. Xen writes the page number to @frame.
 */
#define GTF_invalid         (0U<<0)
#define GTF_permit_access   (1U<<0)
#define GTF_accept_transfer (2U<<0)
#define GTF_type_mask       (3U<<0)

/*
 * Subflags for GTF_permit_access.
 *  GTF_readonly: Restrict @domid to read-only mappings and accesses. [GST]
 *  GTF_reading: Grant entry is currently mapped for reading by @domid. [XEN]
 *  GTF_writing: Grant entry is currently mapped for writing by @domid. [XEN]
 *  GTF_PAT, GTF_PWT, GTF_PCD: (x86) cache attribute flags for the grant [GST]
 */
#define _GTF_readonly       (2)
#define GTF_readonly        (1U<<_GTF_readonly)
#define _GTF_reading        (3)
#define GTF_reading         (1U<<_GTF_reading)
#define _GTF_writing        (4)
#define GTF_writing         (1U<<_GTF_writing)
#define _GTF_PWT            (5)
#define GTF_PWT             (1U<<_GTF_PWT)
#define _GTF_PCD            (6)
#define GTF_PCD             (1U<<_GTF_PCD)
#define _GTF_PAT            (7)
#define GTF_PAT             (1U<<_GTF_PAT)

/*
 * Subflags for GTF_accept_transfer:
 *  GTF_transfer_committed: Xen sets this flag to indicate that it is committed
 *      to transferring ownership of a page frame. When a guest sees this flag
 *      it must /not/ modify the grant entry until GTF_transfer_completed is
 *      set by Xen.
 *  GTF_transfer_completed: It is safe for the guest to spin-wait on this flag
 *      after reading GTF_transfer_committed. Xen will always write the frame
 *      address, followed by ORing this flag, in a timely manner.
 */
#define _GTF_transfer_committed (2)
#define GTF_transfer_committed  (1U<<_GTF_transfer_committed)
#define _GTF_transfer_completed (3)
#define GTF_transfer_completed  (1U<<_GTF_transfer_completed)


/***********************************
 * GRANT TABLE QUERIES AND USES
 */

/*
 * Reference to a grant entry in a specified domain's grant table.
 */
typedef uint32_t grant_ref_t;

/*
 * Handle to track a mapping created via a grant reference.
 */
typedef uint32_t grant_handle_t;

/*
 * GNTTABOP_map_grant_ref: Map the grant entry (<dom>,<ref>) for access
 * by devices and/or host CPUs. If successful, <handle> is a tracking number
 * that must be presented later to destroy the mapping(s). On error, <handle>
 * is a negative status code.
 * NOTES:
 *  1. If GNTMAP_device_map is specified then <dev_bus_addr> is the address
 *     via which I/O devices may access the granted frame.
 *  2. If GNTMAP_host_map is specified then a mapping will be added at
 *     either a host virtual address in the current address space, or at
 *     a PTE at the specified machine address.  The type of mapping to
 *     perform is selected through the GNTMAP_contains_pte flag, and the
 *     address is specified in <host_addr>.
 *  3. Mappings should only be destroyed via GNTTABOP_unmap_grant_ref. If a
 *     host mapping is destroyed by other means then it is *NOT* guaranteed
 *     to be accounted to the correct grant reference!
 */
#define GNTTABOP_map_grant_ref        0
struct gnttab_map_grant_ref {
    /* IN parameters. */
    uint64_t host_addr;
    uint32_t flags;               /* GNTMAP_* */
    grant_ref_t ref;
    domid_t  dom;
    /* OUT parameters. */
    int16_t  status;              /* GNTST_* */
    grant_handle_t handle;
    uint64_t dev_bus_addr;
};
typedef struct gnttab_map_grant_ref gnttab_map_grant_ref_t;
DEFINE_XEN_GUEST_HANDLE(gnttab_map_grant_ref_t);

/*
 * GNTTABOP_unmap_grant_ref: Destroy one or more grant-reference mappings
 * tracked by <handle>. If <host_addr> or <dev_bus_addr> is zero, that
 * field is ignored. If non-zero, they must refer to a device/host mapping
 * that is tracked by <handle>
 * NOTES:
 *  1. The call may fail in an undefined manner if either mapping is not
 *     tracked by <handle>.
 *  3. After executing a batch of unmaps, it is guaranteed that no stale
 *     mappings will remain in the device or host TLBs.
 */
#define GNTTABOP_unmap_grant_ref      1
struct gnttab_unmap_grant_ref {
    /* IN parameters. */
    uint64_t host_addr;
    uint64_t dev_bus_addr;
    grant_handle_t handle;
    /* OUT parameters. */
    int16_t  status;              /* GNTST_* */
};
typedef struct gnttab_unmap_grant_ref gnttab_unmap_grant_ref_t;
DEFINE_XEN_GUEST_HANDLE(gnttab_unmap_grant_ref_t);

/*
 * GNTTABOP_setup_table: Set up a grant table for <dom> comprising at least
 * <nr_frames> pages. The frame addresses are written to the <frame_list>.
 * Only <nr_frames> addresses are written, even if the table is larger.
 * NOTES:
 *  1. <dom> may be specified as DOMID_SELF.
 *  2. Only a sufficiently-privileged domain may specify <dom> != DOMID_SELF.
 *  3. Xen may not support more than a single grant-table page per domain.
 */
#define GNTTABOP_setup_table          2
struct gnttab_setup_table {
    /* IN parameters. */
    domid_t  dom;
    uint32_t nr_frames;
    /* OUT parameters. */
    int16_t  status;              /* GNTST_* */
    XEN_GUEST_HANDLE(ulong) frame_list;
};
typedef struct gnttab_setup_table gnttab_setup_table_t;
DEFINE_XEN_GUEST_HANDLE(gnttab_setup_table_t);

/*
 * GNTTABOP_dump_table: Dump the contents of the grant table to the
 * xen console. Debugging use only.
 */
#define GNTTABOP_dump_table           3
struct gnttab_dump_table {
    /* IN parameters. */
    domid_t dom;
    /* OUT parameters. */
    int16_t status;               /* GNTST_* */
};
typedef struct gnttab_dump_table gnttab_dump_table_t;
DEFINE_XEN_GUEST_HANDLE(gnttab_dump_table_t);

/*
 * GNTTABOP_transfer_grant_ref: Transfer <frame> to a foreign domain. The
 * foreign domain has previously registered its interest in the transfer via
 * <domid, ref>.
 *
 * Note that, even if the transfer fails, the specified page no longer belongs
 * to the calling domain *unless* the error is GNTST_bad_page.
 */
#define GNTTABOP_transfer                4
struct gnttab_transfer {
    /* IN parameters. */
    xen_pfn_t     mfn;
    domid_t       domid;
    grant_ref_t   ref;
    /* OUT parameters. */
    int16_t       status;
};
typedef struct gnttab_transfer gnttab_transfer_t;
DEFINE_XEN_GUEST_HANDLE(gnttab_transfer_t);


/*
 * GNTTABOP_copy: Hypervisor based copy
 * source and destinations can be eithers MFNs or, for foreign domains,
 * grant references. the foreign domain has to grant read/write access
 * in its grant table.
 *
 * The flags specify what type source and destinations are (either MFN
 * or grant reference).
 *
 * Note that this can also be used to copy data between two domains
 * via a third party if the source and destination domains had previously
 * grant appropriate access to their pages to the third party.
 *
 * source_offset specifies an offset in the source frame, dest_offset
 * the offset in the target frame and  len specifies the number of
 * bytes to be copied.
 */

#define _GNTCOPY_source_gref      (0)
#define GNTCOPY_source_gref       (1<<_GNTCOPY_source_gref)
#define _GNTCOPY_dest_gref        (1)
#define GNTCOPY_dest_gref         (1<<_GNTCOPY_dest_gref)

#define GNTTABOP_copy                 5
typedef struct gnttab_copy {
    /* IN parameters. */
    struct {
        union {
            grant_ref_t ref;
            xen_pfn_t   gmfn;
        } u;
        domid_t  domid;
        uint16_t offset;
    } source, dest;
    uint16_t      len;
    uint16_t      flags;          /* GNTCOPY_* */
    /* OUT parameters. */
    int16_t       status;
} gnttab_copy_t;
DEFINE_XEN_GUEST_HANDLE(gnttab_copy_t);

/*
 * GNTTABOP_query_size: Query the current and maximum sizes of the shared
 * grant table.
 * NOTES:
 *  1. <dom> may be specified as DOMID_SELF.
 *  2. Only a sufficiently-privileged domain may specify <dom> != DOMID_SELF.
 */
#define GNTTABOP_query_size           6
struct gnttab_query_size {
    /* IN parameters. */
    domid_t  dom;
    /* OUT parameters. */
    uint32_t nr_frames;
    uint32_t max_nr_frames;
    int16_t  status;              /* GNTST_* */
};
typedef struct gnttab_query_size gnttab_query_size_t;
DEFINE_XEN_GUEST_HANDLE(gnttab_query_size_t);

/*
 * GNTTABOP_unmap_and_replace: Destroy one or more grant-reference mappings
 * tracked by <handle> but atomically replace the page table entry with one
 * pointing to the machine address under <new_addr>.  <new_addr> will be
 * redirected to the null entry.
 * NOTES:
 *  1. The call may fail in an undefined manner if either mapping is not
 *     tracked by <handle>.
 *  2. After executing a batch of unmaps, it is guaranteed that no stale
 *     mappings will remain in the device or host TLBs.
 */
#define GNTTABOP_unmap_and_replace    7
struct gnttab_unmap_and_replace {
    /* IN parameters. */
    uint64_t host_addr;
    uint64_t new_addr;
    grant_handle_t handle;
    /* OUT parameters. */
    int16_t  status;              /* GNTST_* */
};
typedef struct gnttab_unmap_and_replace gnttab_unmap_and_replace_t;
DEFINE_XEN_GUEST_HANDLE(gnttab_unmap_and_replace_t);


/*
 * Bitfield values for gnttab_map_grant_ref.flags.
 */
 /* Map the grant entry for access by I/O devices. */
#define _GNTMAP_device_map      (0)
#define GNTMAP_device_map       (1<<_GNTMAP_device_map)
 /* Map the grant entry for access by host CPUs. */
#define _GNTMAP_host_map        (1)
#define GNTMAP_host_map         (1<<_GNTMAP_host_map)
 /* Accesses to the granted frame will be restricted to read-only access. */
#define _GNTMAP_readonly        (2)
#define GNTMAP_readonly         (1<<_GNTMAP_readonly)
 /*
  * GNTMAP_host_map subflag:
  *  0 => The host mapping is usable only by the guest OS.
  *  1 => The host mapping is usable by guest OS + current application.
  */
#define _GNTMAP_application_map (3)
#define GNTMAP_application_map  (1<<_GNTMAP_application_map)

 /*
  * GNTMAP_contains_pte subflag:
  *  0 => This map request contains a host virtual address.
  *  1 => This map request contains the machine addess of the PTE to update.
  */
#define _GNTMAP_contains_pte    (4)
#define GNTMAP_contains_pte     (1<<_GNTMAP_contains_pte)

/*
 * Bits to be placed in guest kernel available PTE bits (architecture
 * dependent; only supported when XENFEAT_gnttab_map_avail_bits is set).
 */
#define _GNTMAP_guest_avail0    (16)
#define GNTMAP_guest_avail_mask ((uint32_t)~0 << _GNTMAP_guest_avail0)

/*
 * Values for error status returns. All errors are -ve.
 */
#define GNTST_okay             (0)  /* Normal return.                        */
#define GNTST_general_error    (-1) /* General undefined error.              */
#define GNTST_bad_domain       (-2) /* Unrecognsed domain id.                */
#define GNTST_bad_gntref       (-3) /* Unrecognised or inappropriate gntref. */
#define GNTST_bad_handle       (-4) /* Unrecognised or inappropriate handle. */
#define GNTST_bad_virt_addr    (-5) /* Inappropriate virtual address to map. */
#define GNTST_bad_dev_addr     (-6) /* Inappropriate device address to unmap.*/
#define GNTST_no_device_space  (-7) /* Out of space in I/O MMU.              */
#define GNTST_permission_denied (-8) /* Not enough privilege for operation.  */
#define GNTST_bad_page         (-9) /* Specified page was invalid for op.    */
#define GNTST_bad_copy_arg    (-10) /* copy arguments cross page boundary.   */
#define GNTST_address_too_big (-11) /* transfer page address too large.      */

#define GNTTABOP_error_msgs {                   \
    "okay",                                     \
    "undefined error",                          \
    "unrecognised domain id",                   \
    "invalid grant reference",                  \
    "invalid mapping handle",                   \
    "invalid virtual address",                  \
    "invalid device address",                   \
    "no spare translation slot in the I/O MMU", \
    "permission denied",                        \
    "bad page",                                 \
    "copy arguments cross page boundary",       \
    "page address size too large"               \
}

#endif /* __XEN_PUBLIC_GRANT_TABLE_H__ */

/*
 * Local variables:
 * mode: C
 * c-set-style: "BSD"
 * c-basic-offset: 4
 * tab-width: 4
 * indent-tabs-mode: nil
 * End:
 */
/*
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to
 * deal in the Software without restriction, including without limitation the
 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
 * sell copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 * DEALINGS IN THE SOFTWARE.
 */

#ifndef __XEN_PUBLIC_HVM_HVM_OP_H__
#define __XEN_PUBLIC_HVM_HVM_OP_H__

/* Get/set subcommands: extra argument == pointer to xen_hvm_param struct. */
#define HVMOP_set_param           0
#define HVMOP_get_param           1
struct xen_hvm_param {
    domid_t  domid;    /* IN */
    uint32_t index;    /* IN */
    uint64_t value;    /* IN/OUT */
};
typedef struct xen_hvm_param xen_hvm_param_t;
DEFINE_XEN_GUEST_HANDLE(xen_hvm_param_t);

/* Set the logical level of one of a domain's PCI INTx wires. */
#define HVMOP_set_pci_intx_level  2
struct xen_hvm_set_pci_intx_level {
    /* Domain to be updated. */
    domid_t  domid;
    /* PCI INTx identification in PCI topology (domain:bus:device:intx). */
    uint8_t  domain, bus, device, intx;
    /* Assertion level (0 = unasserted, 1 = asserted). */
    uint8_t  level;
};
typedef struct xen_hvm_set_pci_intx_level xen_hvm_set_pci_intx_level_t;
DEFINE_XEN_GUEST_HANDLE(xen_hvm_set_pci_intx_level_t);

/* Set the logical level of one of a domain's ISA IRQ wires. */
#define HVMOP_set_isa_irq_level   3
struct xen_hvm_set_isa_irq_level {
    /* Domain to be updated. */
    domid_t  domid;
    /* ISA device identification, by ISA IRQ (0-15). */
    uint8_t  isa_irq;
    /* Assertion level (0 = unasserted, 1 = asserted). */
    uint8_t  level;
};
typedef struct xen_hvm_set_isa_irq_level xen_hvm_set_isa_irq_level_t;
DEFINE_XEN_GUEST_HANDLE(xen_hvm_set_isa_irq_level_t);

#define HVMOP_set_pci_link_route  4
struct xen_hvm_set_pci_link_route {
    /* Domain to be updated. */
    domid_t  domid;
    /* PCI link identifier (0-3). */
    uint8_t  link;
    /* ISA IRQ (1-15), or 0 (disable link). */
    uint8_t  isa_irq;
};
typedef struct xen_hvm_set_pci_link_route xen_hvm_set_pci_link_route_t;
DEFINE_XEN_GUEST_HANDLE(xen_hvm_set_pci_link_route_t);

/* Flushes all VCPU TLBs: @arg must be NULL. */
#define HVMOP_flush_tlbs          5

/* Following tools-only interfaces may change in future. */
#if defined(__XEN__) || defined(__XEN_TOOLS__)

/* Track dirty VRAM. */
#define HVMOP_track_dirty_vram    6
struct xen_hvm_track_dirty_vram {
    /* Domain to be tracked. */
    domid_t  domid;
    /* First pfn to track. */
    uint64_aligned_t first_pfn;
    /* Number of pages to track. */
    uint64_aligned_t nr;
    /* OUT variable. */
    /* Dirty bitmap buffer. */
    XEN_GUEST_HANDLE_64(uint8) dirty_bitmap;
};
typedef struct xen_hvm_track_dirty_vram xen_hvm_track_dirty_vram_t;
DEFINE_XEN_GUEST_HANDLE(xen_hvm_track_dirty_vram_t);

/* Notify that some pages got modified by the Device Model. */
#define HVMOP_modified_memory    7
struct xen_hvm_modified_memory {
    /* Domain to be updated. */
    domid_t  domid;
    /* First pfn. */
    uint64_aligned_t first_pfn;
    /* Number of pages. */
    uint64_aligned_t nr;
};
typedef struct xen_hvm_modified_memory xen_hvm_modified_memory_t;
DEFINE_XEN_GUEST_HANDLE(xen_hvm_modified_memory_t);

#define HVMOP_set_mem_type    8
typedef enum {
    HVMMEM_ram_rw,             /* Normal read/write guest RAM */
    HVMMEM_ram_ro,             /* Read-only; writes are discarded */
    HVMMEM_mmio_dm,            /* Reads and write go to the device model */
} hvmmem_type_t;
/* Notify that a region of memory is to be treated in a specific way. */
struct xen_hvm_set_mem_type {
    /* Domain to be updated. */
    domid_t domid;
    /* Memory type */
    hvmmem_type_t hvmmem_type;
    /* First pfn. */
    uint64_aligned_t first_pfn;
    /* Number of pages. */
    uint64_aligned_t nr;
};
typedef struct xen_hvm_set_mem_type xen_hvm_set_mem_type_t;
DEFINE_XEN_GUEST_HANDLE(xen_hvm_set_mem_type_t);


#endif /* defined(__XEN__) || defined(__XEN_TOOLS__) */

#endif /* __XEN_PUBLIC_HVM_HVM_OP_H__ */
/*
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to
 * deal in the Software without restriction, including without limitation the
 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
 * sell copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 * DEALINGS IN THE SOFTWARE.
 */

#ifndef __XEN_PUBLIC_HVM_PARAMS_H__
#define __XEN_PUBLIC_HVM_PARAMS_H__

#include "hvm_op.h"

/*
 * Parameter space for HVMOP_{set,get}_param.
 */

/*
 * How should CPU0 event-channel notifications be delivered?
 * val[63:56] == 0: val[55:0] is a delivery GSI (Global System Interrupt).
 * val[63:56] == 1: val[55:0] is a delivery PCI INTx line, as follows:
 *                  Domain = val[47:32], Bus  = val[31:16],
 *                  DevFn  = val[15: 8], IntX = val[ 1: 0]
 * If val == 0 then CPU0 event-channel notifications are not delivered.
 */
#define HVM_PARAM_CALLBACK_IRQ 0

/*
 * These are not used by Xen. They are here for convenience of HVM-guest
 * xenbus implementations.
 */
#define HVM_PARAM_STORE_PFN    1
#define HVM_PARAM_STORE_EVTCHN 2

#define HVM_PARAM_PAE_ENABLED  4

#define HVM_PARAM_IOREQ_PFN    5

#define HVM_PARAM_BUFIOREQ_PFN 6

#ifdef __ia64__

#define HVM_PARAM_NVRAM_FD     7
#define HVM_PARAM_VHPT_SIZE    8
#define HVM_PARAM_BUFPIOREQ_PFN	9

#elif defined(__i386__) || defined(__x86_64__)

/* Expose Viridian interfaces to this HVM guest? */
#define HVM_PARAM_VIRIDIAN     9

#endif

/*
 * Set mode for virtual timers (currently x86 only):
 *  delay_for_missed_ticks (default):
 *   Do not advance a vcpu's time beyond the correct delivery time for
 *   interrupts that have been missed due to preemption. Deliver missed
 *   interrupts when the vcpu is rescheduled and advance the vcpu's virtual
 *   time stepwise for each one.
 *  no_delay_for_missed_ticks:
 *   As above, missed interrupts are delivered, but guest time always tracks
 *   wallclock (i.e., real) time while doing so.
 *  no_missed_ticks_pending:
 *   No missed interrupts are held pending. Instead, to ensure ticks are
 *   delivered at some non-zero rate, if we detect missed ticks then the
 *   internal tick alarm is not disabled if the VCPU is preempted during the
 *   next tick period.
 *  one_missed_tick_pending:
 *   Missed interrupts are collapsed together and delivered as one 'late tick'.
 *   Guest time always tracks wallclock (i.e., real) time.
 */
#define HVM_PARAM_TIMER_MODE   10
#define HVMPTM_delay_for_missed_ticks    0
#define HVMPTM_no_delay_for_missed_ticks 1
#define HVMPTM_no_missed_ticks_pending   2
#define HVMPTM_one_missed_tick_pending   3

/* Boolean: Enable virtual HPET (high-precision event timer)? (x86-only) */
#define HVM_PARAM_HPET_ENABLED 11

/* Identity-map page directory used by Intel EPT when CR0.PG=0. */
#define HVM_PARAM_IDENT_PT     12

/* Device Model domain, defaults to 0. */
#define HVM_PARAM_DM_DOMAIN    13

/* ACPI S state: currently support S0 and S3 on x86. */
#define HVM_PARAM_ACPI_S_STATE 14

/* TSS used on Intel when CR0.PE=0. */
#define HVM_PARAM_VM86_TSS     15

/* Boolean: Enable aligning all periodic vpts to reduce interrupts */
#define HVM_PARAM_VPT_ALIGN    16

#define HVM_NR_PARAMS          17

#endif /* __XEN_PUBLIC_HVM_PARAMS_H__ */
/******************************************************************************
 * blkif.h
 *
 * Unified block-device I/O interface for Xen guest OSes.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to
 * deal in the Software without restriction, including without limitation the
 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
 * sell copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 * DEALINGS IN THE SOFTWARE.
 *
 * Copyright (c) 2003-2004, Keir Fraser
 */

#ifndef __XEN_PUBLIC_IO_BLKIF_H__
#define __XEN_PUBLIC_IO_BLKIF_H__

#include "ring.h"
#include "../grant_table.h"

/*
 * Front->back notifications: When enqueuing a new request, sending a
 * notification can be made conditional on req_event (i.e., the generic
 * hold-off mechanism provided by the ring macros). Backends must set
 * req_event appropriately (e.g., using RING_FINAL_CHECK_FOR_REQUESTS()).
 *
 * Back->front notifications: When enqueuing a new response, sending a
 * notification can be made conditional on rsp_event (i.e., the generic
 * hold-off mechanism provided by the ring macros). Frontends must set
 * rsp_event appropriately (e.g., using RING_FINAL_CHECK_FOR_RESPONSES()).
 */

#ifndef blkif_vdev_t
#define blkif_vdev_t   uint16_t
#endif
#define blkif_sector_t uint64_t

/*
 * REQUEST CODES.
 */
#define BLKIF_OP_READ              0
#define BLKIF_OP_WRITE             1
/*
 * Recognised only if "feature-barrier" is present in backend xenbus info.
 * The "feature-barrier" node contains a boolean indicating whether barrier
 * requests are likely to succeed or fail. Either way, a barrier request
 * may fail at any time with BLKIF_RSP_EOPNOTSUPP if it is unsupported by
 * the underlying block-device hardware. The boolean simply indicates whether
 * or not it is worthwhile for the frontend to attempt barrier requests.
 * If a backend does not recognise BLKIF_OP_WRITE_BARRIER, it should *not*
 * create the "feature-barrier" node!
 */
#define BLKIF_OP_WRITE_BARRIER     2
/*
 * Recognised if "feature-flush-cache" is present in backend xenbus
 * info.  A flush will ask the underlying storage hardware to flush its
 * non-volatile caches as appropriate.  The "feature-flush-cache" node
 * contains a boolean indicating whether flush requests are likely to
 * succeed or fail. Either way, a flush request may fail at any time
 * with BLKIF_RSP_EOPNOTSUPP if it is unsupported by the underlying
 * block-device hardware. The boolean simply indicates whether or not it
 * is worthwhile for the frontend to attempt flushes.  If a backend does
 * not recognise BLKIF_OP_WRITE_FLUSH_CACHE, it should *not* create the
 * "feature-flush-cache" node!
 */
#define BLKIF_OP_FLUSH_DISKCACHE   3

/*
 * Maximum scatter/gather segments per request.
 * This is carefully chosen so that sizeof(blkif_ring_t) <= PAGE_SIZE.
 * NB. This could be 12 if the ring indexes weren't stored in the same page.
 */
#define BLKIF_MAX_SEGMENTS_PER_REQUEST 11

/*
 * NB. first_sect and last_sect in blkif_request_segment, as well as
 * sector_number in blkif_request, are always expressed in 512-byte units.
 * However they must be properly aligned to the real sector size of the
 * physical disk, which is reported in the "sector-size" node in the backend
 * xenbus info. Also the xenbus "sectors" node is expressed in 512-byte units.
 */
struct blkif_request_segment {
    grant_ref_t gref;        /* reference to I/O buffer frame        */
    /* @first_sect: first sector in frame to transfer (inclusive).   */
    /* @last_sect: last sector in frame to transfer (inclusive).     */
    uint8_t     first_sect, last_sect;
};

struct blkif_request {
    uint8_t        operation;    /* BLKIF_OP_???                         */
    uint8_t        nr_segments;  /* number of segments                   */
    blkif_vdev_t   handle;       /* only for read/write requests         */
    uint64_t       id;           /* private guest value, echoed in resp  */
    blkif_sector_t sector_number;/* start sector idx on disk (r/w only)  */
    struct blkif_request_segment seg[BLKIF_MAX_SEGMENTS_PER_REQUEST];
};
typedef struct blkif_request blkif_request_t;

struct blkif_response {
    uint64_t        id;              /* copied from request */
    uint8_t         operation;       /* copied from request */
    int16_t         status;          /* BLKIF_RSP_???       */
};
typedef struct blkif_response blkif_response_t;

/*
 * STATUS RETURN CODES.
 */
 /* Operation not supported (only happens on barrier writes). */
#define BLKIF_RSP_EOPNOTSUPP  -2
 /* Operation failed for some unspecified reason (-EIO). */
#define BLKIF_RSP_ERROR       -1
 /* Operation completed successfully. */
#define BLKIF_RSP_OKAY         0

/*
 * Generate blkif ring structures and types.
 */

DEFINE_RING_TYPES(blkif, struct blkif_request, struct blkif_response);

#define VDISK_CDROM        0x1
#define VDISK_REMOVABLE    0x2
#define VDISK_READONLY     0x4

#endif /* __XEN_PUBLIC_IO_BLKIF_H__ */

/*
 * Local variables:
 * mode: C
 * c-set-style: "BSD"
 * c-basic-offset: 4
 * tab-width: 4
 * indent-tabs-mode: nil
 * End:
 */
/******************************************************************************
 * console.h
 *
 * Console I/O interface for Xen guest OSes.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to
 * deal in the Software without restriction, including without limitation the
 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
 * sell copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 * DEALINGS IN THE SOFTWARE.
 *
 * Copyright (c) 2005, Keir Fraser
 */

#ifndef __XEN_PUBLIC_IO_CONSOLE_H__
#define __XEN_PUBLIC_IO_CONSOLE_H__

typedef uint32_t XENCONS_RING_IDX;

#define MASK_XENCONS_IDX(idx, ring) ((idx) & (sizeof(ring)-1))

struct xencons_interface {
    char in[1024];
    char out[2048];
    XENCONS_RING_IDX in_cons, in_prod;
    XENCONS_RING_IDX out_cons, out_prod;
};

#endif /* __XEN_PUBLIC_IO_CONSOLE_H__ */

/*
 * Local variables:
 * mode: C
 * c-set-style: "BSD"
 * c-basic-offset: 4
 * tab-width: 4
 * indent-tabs-mode: nil
 * End:
 */
/******************************************************************************
 * netif.h
 *
 * Unified network-device I/O interface for Xen guest OSes.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to
 * deal in the Software without restriction, including without limitation the
 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
 * sell copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 * DEALINGS IN THE SOFTWARE.
 *
 * Copyright (c) 2003-2004, Keir Fraser
 */

#ifndef __XEN_PUBLIC_IO_NETIF_H__
#define __XEN_PUBLIC_IO_NETIF_H__

#include "ring.h"
#include "../grant_table.h"

/*
 * Notifications after enqueuing any type of message should be conditional on
 * the appropriate req_event or rsp_event field in the shared ring.
 * If the client sends notification for rx requests then it should specify
 * feature 'feature-rx-notify' via xenbus. Otherwise the backend will assume
 * that it cannot safely queue packets (as it may not be kicked to send them).
 */

/*
 * This is the 'wire' format for packets:
 *  Request 1: netif_tx_request -- NETTXF_* (any flags)
 * [Request 2: netif_tx_extra]  (only if request 1 has NETTXF_extra_info)
 * [Request 3: netif_tx_extra]  (only if request 2 has XEN_NETIF_EXTRA_MORE)
 *  Request 4: netif_tx_request -- NETTXF_more_data
 *  Request 5: netif_tx_request -- NETTXF_more_data
 *  ...
 *  Request N: netif_tx_request -- 0
 */

/* Protocol checksum field is blank in the packet (hardware offload)? */
#define _NETTXF_csum_blank     (0)
#define  NETTXF_csum_blank     (1U<<_NETTXF_csum_blank)

/* Packet data has been validated against protocol checksum. */
#define _NETTXF_data_validated (1)
#define  NETTXF_data_validated (1U<<_NETTXF_data_validated)

/* Packet continues in the next request descriptor. */
#define _NETTXF_more_data      (2)
#define  NETTXF_more_data      (1U<<_NETTXF_more_data)

/* Packet to be followed by extra descriptor(s). */
#define _NETTXF_extra_info     (3)
#define  NETTXF_extra_info     (1U<<_NETTXF_extra_info)

struct netif_tx_request {
    grant_ref_t gref;      /* Reference to buffer page */
    uint16_t offset;       /* Offset within buffer page */
    uint16_t flags;        /* NETTXF_* */
    uint16_t id;           /* Echoed in response message. */
    uint16_t size;         /* Packet size in bytes.       */
};
typedef struct netif_tx_request netif_tx_request_t;

/* Types of netif_extra_info descriptors. */
#define XEN_NETIF_EXTRA_TYPE_NONE      (0)  /* Never used - invalid */
#define XEN_NETIF_EXTRA_TYPE_GSO       (1)  /* u.gso */
#define XEN_NETIF_EXTRA_TYPE_MCAST_ADD (2)  /* u.mcast */
#define XEN_NETIF_EXTRA_TYPE_MCAST_DEL (3)  /* u.mcast */
#define XEN_NETIF_EXTRA_TYPE_MAX       (4)

/* netif_extra_info flags. */
#define _XEN_NETIF_EXTRA_FLAG_MORE (0)
#define XEN_NETIF_EXTRA_FLAG_MORE  (1U<<_XEN_NETIF_EXTRA_FLAG_MORE)

/* GSO types - only TCPv4 currently supported. */
#define XEN_NETIF_GSO_TYPE_TCPV4        (1)

/*
 * This structure needs to fit within both netif_tx_request and
 * netif_rx_response for compatibility.
 */
struct netif_extra_info {
    uint8_t type;  /* XEN_NETIF_EXTRA_TYPE_* */
    uint8_t flags; /* XEN_NETIF_EXTRA_FLAG_* */

    union {
        /*
         * XEN_NETIF_EXTRA_TYPE_GSO:
         */
        struct {
            /*
             * Maximum payload size of each segment. For example, for TCP this
             * is just the path MSS.
             */
            uint16_t size;

            /*
             * GSO type. This determines the protocol of the packet and any
             * extra features required to segment the packet properly.
             */
            uint8_t type; /* XEN_NETIF_GSO_TYPE_* */

            /* Future expansion. */
            uint8_t pad;

            /*
             * GSO features. This specifies any extra GSO features required
             * to process this packet, such as ECN support for TCPv4.
             */
            uint16_t features; /* XEN_NETIF_GSO_FEAT_* */
        } gso;

        /*
         * XEN_NETIF_EXTRA_TYPE_MCAST_{ADD,DEL}:
         * Backend advertises availability via 'feature-multicast-control'
         * xenbus node containing value '1'.
         * Frontend requests this feature by advertising
         * 'request-multicast-control' xenbus node containing value '1'.
         * If multicast control is requested then multicast flooding is
         * disabled and the frontend must explicitly register its interest
         * in multicast groups using dummy transmit requests containing
         * MCAST_{ADD,DEL} extra-info fragments.
         */
        struct {
            uint8_t addr[6]; /* Address to add/remove. */
        } mcast;

        uint16_t pad[3];
    } u;
};
typedef struct netif_extra_info netif_extra_info_t;

struct netif_tx_response {
    uint16_t id;
    int16_t  status;       /* NETIF_RSP_* */
};
typedef struct netif_tx_response netif_tx_response_t;

struct netif_rx_request {
    uint16_t    id;        /* Echoed in response message.        */
    grant_ref_t gref;      /* Reference to incoming granted frame */
};
typedef struct netif_rx_request netif_rx_request_t;

/* Packet data has been validated against protocol checksum. */
#define _NETRXF_data_validated (0)
#define  NETRXF_data_validated (1U<<_NETRXF_data_validated)

/* Protocol checksum field is blank in the packet (hardware offload)? */
#define _NETRXF_csum_blank     (1)
#define  NETRXF_csum_blank     (1U<<_NETRXF_csum_blank)

/* Packet continues in the next request descriptor. */
#define _NETRXF_more_data      (2)
#define  NETRXF_more_data      (1U<<_NETRXF_more_data)

/* Packet to be followed by extra descriptor(s). */
#define _NETRXF_extra_info     (3)
#define  NETRXF_extra_info     (1U<<_NETRXF_extra_info)

struct netif_rx_response {
    uint16_t id;
    uint16_t offset;       /* Offset in page of start of received packet  */
    uint16_t flags;        /* NETRXF_* */
    int16_t  status;       /* -ve: BLKIF_RSP_* ; +ve: Rx'ed pkt size. */
};
typedef struct netif_rx_response netif_rx_response_t;

/*
 * Generate netif ring structures and types.
 */

DEFINE_RING_TYPES(netif_tx, struct netif_tx_request, struct netif_tx_response);
DEFINE_RING_TYPES(netif_rx, struct netif_rx_request, struct netif_rx_response);

#define NETIF_RSP_DROPPED         -2
#define NETIF_RSP_ERROR           -1
#define NETIF_RSP_OKAY             0
/* No response: used for auxiliary requests (e.g., netif_tx_extra). */
#define NETIF_RSP_NULL             1

#endif

/*
 * Local variables:
 * mode: C
 * c-set-style: "BSD"
 * c-basic-offset: 4
 * tab-width: 4
 * indent-tabs-mode: nil
 * End:
 */
/******************************************************************************
 * protocols.h
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to
 * deal in the Software without restriction, including without limitation the
 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
 * sell copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 * DEALINGS IN THE SOFTWARE.
 */

#ifndef __XEN_PROTOCOLS_H__
#define __XEN_PROTOCOLS_H__

#define XEN_IO_PROTO_ABI_X86_32     "x86_32-abi"
#define XEN_IO_PROTO_ABI_X86_64     "x86_64-abi"
#define XEN_IO_PROTO_ABI_IA64       "ia64-abi"

#if defined(__i386__)
# define XEN_IO_PROTO_ABI_NATIVE XEN_IO_PROTO_ABI_X86_32
#elif defined(__x86_64__)
# define XEN_IO_PROTO_ABI_NATIVE XEN_IO_PROTO_ABI_X86_64
#elif defined(__ia64__)
# define XEN_IO_PROTO_ABI_NATIVE XEN_IO_PROTO_ABI_IA64
#else
# error arch fixup needed here
#endif

#endif
/******************************************************************************
 * ring.h
 *
 * Shared producer-consumer ring macros.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to
 * deal in the Software without restriction, including without limitation the
 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
 * sell copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 * DEALINGS IN THE SOFTWARE.
 *
 * Tim Deegan and Andrew Warfield November 2004.
 */

#ifndef __XEN_PUBLIC_IO_RING_H__
#define __XEN_PUBLIC_IO_RING_H__

#include "../xen-compat.h"

#if __XEN_INTERFACE_VERSION__ < 0x00030208
#define xen_mb()  mb()
#define xen_rmb() rmb()
#define xen_wmb() wmb()
#endif

typedef unsigned int RING_IDX;

/* Round a 32-bit unsigned constant down to the nearest power of two. */
#define __RD2(_x)  (((_x) & 0x00000002) ? 0x2                  : ((_x) & 0x1))
#define __RD4(_x)  (((_x) & 0x0000000c) ? __RD2((_x)>>2)<<2    : __RD2(_x))
#define __RD8(_x)  (((_x) & 0x000000f0) ? __RD4((_x)>>4)<<4    : __RD4(_x))
#define __RD16(_x) (((_x) & 0x0000ff00) ? __RD8((_x)>>8)<<8    : __RD8(_x))
#define __RD32(_x) (((_x) & 0xffff0000) ? __RD16((_x)>>16)<<16 : __RD16(_x))

/*
 * Calculate size of a shared ring, given the total available space for the
 * ring and indexes (_sz), and the name tag of the request/response structure.
 * A ring contains as many entries as will fit, rounded down to the nearest
 * power of two (so we can mask with (size-1) to loop around).
 */
#define __CONST_RING_SIZE(_s, _sz) \
    (__RD32(((_sz) - offsetof(struct _s##_sring, ring)) / \
       sizeof(((struct _s##_sring *)0)->ring[0])))
/*
 * The same for passing in an actual pointer instead of a name tag.
 */
#define __RING_SIZE(_s, _sz) \
    (__RD32(((_sz) - (long)(_s)->ring + (long)(_s)) / sizeof((_s)->ring[0])))

/*
 * Macros to make the correct C datatypes for a new kind of ring.
 *
 * To make a new ring datatype, you need to have two message structures,
 * let's say request_t, and response_t already defined.
 *
 * In a header where you want the ring datatype declared, you then do:
 *
 *     DEFINE_RING_TYPES(mytag, request_t, response_t);
 *
 * These expand out to give you a set of types, as you can see below.
 * The most important of these are:
 *
 *     mytag_sring_t      - The shared ring.
 *     mytag_front_ring_t - The 'front' half of the ring.
 *     mytag_back_ring_t  - The 'back' half of the ring.
 *
 * To initialize a ring in your code you need to know the location and size
 * of the shared memory area (PAGE_SIZE, for instance). To initialise
 * the front half:
 *
 *     mytag_front_ring_t front_ring;
 *     SHARED_RING_INIT((mytag_sring_t *)shared_page);
 *     FRONT_RING_INIT(&front_ring, (mytag_sring_t *)shared_page, PAGE_SIZE);
 *
 * Initializing the back follows similarly (note that only the front
 * initializes the shared ring):
 *
 *     mytag_back_ring_t back_ring;
 *     BACK_RING_INIT(&back_ring, (mytag_sring_t *)shared_page, PAGE_SIZE);
 */

#define DEFINE_RING_TYPES(__name, __req_t, __rsp_t)                     \
                                                                        \
/* Shared ring entry */                                                 \
union __name##_sring_entry {                                            \
    __req_t req;                                                        \
    __rsp_t rsp;                                                        \
};                                                                      \
                                                                        \
/* Shared ring page */                                                  \
struct __name##_sring {                                                 \
    RING_IDX req_prod, req_event;                                       \
    RING_IDX rsp_prod, rsp_event;                                       \
    uint8_t  pad[48];                                                   \
    union __name##_sring_entry ring[1]; /* variable-length */           \
};                                                                      \
                                                                        \
/* "Front" end's private variables */                                   \
struct __name##_front_ring {                                            \
    RING_IDX req_prod_pvt;                                              \
    RING_IDX rsp_cons;                                                  \
    unsigned int nr_ents;                                               \
    struct __name##_sring *sring;                                       \
};                                                                      \
                                                                        \
/* "Back" end's private variables */                                    \
struct __name##_back_ring {                                             \
    RING_IDX rsp_prod_pvt;                                              \
    RING_IDX req_cons;                                                  \
    unsigned int nr_ents;                                               \
    struct __name##_sring *sring;                                       \
};                                                                      \
                                                                        \
/* Syntactic sugar */                                                   \
typedef struct __name##_sring __name##_sring_t;                         \
typedef struct __name##_front_ring __name##_front_ring_t;               \
typedef struct __name##_back_ring __name##_back_ring_t

/*
 * Macros for manipulating rings.
 *
 * FRONT_RING_whatever works on the "front end" of a ring: here
 * requests are pushed on to the ring and responses taken off it.
 *
 * BACK_RING_whatever works on the "back end" of a ring: here
 * requests are taken off the ring and responses put on.
 *
 * N.B. these macros do NO INTERLOCKS OR FLOW CONTROL.
 * This is OK in 1-for-1 request-response situations where the
 * requestor (front end) never has more than RING_SIZE()-1
 * outstanding requests.
 */

/* Initialising empty rings */
#define SHARED_RING_INIT(_s) do {                                       \
    (_s)->req_prod  = (_s)->rsp_prod  = 0;                              \
    (_s)->req_event = (_s)->rsp_event = 1;                              \
    (void)memset((_s)->pad, 0, sizeof((_s)->pad));                      \
} while(0)

#define FRONT_RING_INIT(_r, _s, __size) do {                            \
    (_r)->req_prod_pvt = 0;                                             \
    (_r)->rsp_cons = 0;                                                 \
    (_r)->nr_ents = __RING_SIZE(_s, __size);                            \
    (_r)->sring = (_s);                                                 \
} while (0)

#define BACK_RING_INIT(_r, _s, __size) do {                             \
    (_r)->rsp_prod_pvt = 0;                                             \
    (_r)->req_cons = 0;                                                 \
    (_r)->nr_ents = __RING_SIZE(_s, __size);                            \
    (_r)->sring = (_s);                                                 \
} while (0)

/* Initialize to existing shared indexes -- for recovery */
#define FRONT_RING_ATTACH(_r, _s, __size) do {                          \
    (_r)->sring = (_s);                                                 \
    (_r)->req_prod_pvt = (_s)->req_prod;                                \
    (_r)->rsp_cons = (_s)->rsp_prod;                                    \
    (_r)->nr_ents = __RING_SIZE(_s, __size);                            \
} while (0)

#define BACK_RING_ATTACH(_r, _s, __size) do {                           \
    (_r)->sring = (_s);                                                 \
    (_r)->rsp_prod_pvt = (_s)->rsp_prod;                                \
    (_r)->req_cons = (_s)->req_prod;                                    \
    (_r)->nr_ents = __RING_SIZE(_s, __size);                            \
} while (0)

/* How big is this ring? */
#define RING_SIZE(_r)                                                   \
    ((_r)->nr_ents)

/* Number of free requests (for use on front side only). */
#define RING_FREE_REQUESTS(_r)                                          \
    (RING_SIZE(_r) - ((_r)->req_prod_pvt - (_r)->rsp_cons))

/* Test if there is an empty slot available on the front ring.
 * (This is only meaningful from the front. )
 */
#define RING_FULL(_r)                                                   \
    (RING_FREE_REQUESTS(_r) == 0)

/* Test if there are outstanding messages to be processed on a ring. */
#define RING_HAS_UNCONSUMED_RESPONSES(_r)                               \
    ((_r)->sring->rsp_prod - (_r)->rsp_cons)

#ifdef __GNUC__
#define RING_HAS_UNCONSUMED_REQUESTS(_r) ({                             \
    unsigned int req = (_r)->sring->req_prod - (_r)->req_cons;          \
    unsigned int rsp = RING_SIZE(_r) -                                  \
        ((_r)->req_cons - (_r)->rsp_prod_pvt);                          \
    req < rsp ? req : rsp;                                              \
})
#else
/* Same as above, but without the nice GCC ({ ... }) syntax. */
#define RING_HAS_UNCONSUMED_REQUESTS(_r)                                \
    ((((_r)->sring->req_prod - (_r)->req_cons) <                        \
      (RING_SIZE(_r) - ((_r)->req_cons - (_r)->rsp_prod_pvt))) ?        \
     ((_r)->sring->req_prod - (_r)->req_cons) :                         \
     (RING_SIZE(_r) - ((_r)->req_cons - (_r)->rsp_prod_pvt)))
#endif

/* Direct access to individual ring elements, by index. */
#define RING_GET_REQUEST(_r, _idx)                                      \
    (&((_r)->sring->ring[((_idx) & (RING_SIZE(_r) - 1))].req))

#define RING_GET_RESPONSE(_r, _idx)                                     \
    (&((_r)->sring->ring[((_idx) & (RING_SIZE(_r) - 1))].rsp))

/* Loop termination condition: Would the specified index overflow the ring? */
#define RING_REQUEST_CONS_OVERFLOW(_r, _cons)                           \
    (((_cons) - (_r)->rsp_prod_pvt) >= RING_SIZE(_r))

#define RING_PUSH_REQUESTS(_r) do {                                     \
    xen_wmb(); /* back sees requests /before/ updated producer index */ \
    (_r)->sring->req_prod = (_r)->req_prod_pvt;                         \
} while (0)

#define RING_PUSH_RESPONSES(_r) do {                                    \
    xen_wmb(); /* front sees resps /before/ updated producer index */   \
    (_r)->sring->rsp_prod = (_r)->rsp_prod_pvt;                         \
} while (0)

/*
 * Notification hold-off (req_event and rsp_event):
 *
 * When queueing requests or responses on a shared ring, it may not always be
 * necessary to notify the remote end. For example, if requests are in flight
 * in a backend, the front may be able to queue further requests without
 * notifying the back (if the back checks for new requests when it queues
 * responses).
 *
 * When enqueuing requests or responses:
 *
 *  Use RING_PUSH_{REQUESTS,RESPONSES}_AND_CHECK_NOTIFY(). The second argument
 *  is a boolean return value. True indicates that the receiver requires an
 *  asynchronous notification.
 *
 * After dequeuing requests or responses (before sleeping the connection):
 *
 *  Use RING_FINAL_CHECK_FOR_REQUESTS() or RING_FINAL_CHECK_FOR_RESPONSES().
 *  The second argument is a boolean return value. True indicates that there
 *  are pending messages on the ring (i.e., the connection should not be put
 *  to sleep).
 *
 *  These macros will set the req_event/rsp_event field to trigger a
 *  notification on the very next message that is enqueued. If you want to
 *  create batches of work (i.e., only receive a notification after several
 *  messages have been enqueued) then you will need to create a customised
 *  version of the FINAL_CHECK macro in your own code, which sets the event
 *  field appropriately.
 */

#define RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(_r, _notify) do {           \
    RING_IDX __old = (_r)->sring->req_prod;                             \
    RING_IDX __new = (_r)->req_prod_pvt;                                \
    xen_wmb(); /* back sees requests /before/ updated producer index */ \
    (_r)->sring->req_prod = __new;                                      \
    xen_mb(); /* back sees new requests /before/ we check req_event */  \
    (_notify) = ((RING_IDX)(__new - (_r)->sring->req_event) <           \
                 (RING_IDX)(__new - __old));                            \
} while (0)

#define RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(_r, _notify) do {          \
    RING_IDX __old = (_r)->sring->rsp_prod;                             \
    RING_IDX __new = (_r)->rsp_prod_pvt;                                \
    xen_wmb(); /* front sees resps /before/ updated producer index */   \
    (_r)->sring->rsp_prod = __new;                                      \
    xen_mb(); /* front sees new resps /before/ we check rsp_event */    \
    (_notify) = ((RING_IDX)(__new - (_r)->sring->rsp_event) <           \
                 (RING_IDX)(__new - __old));                            \
} while (0)

#define RING_FINAL_CHECK_FOR_REQUESTS(_r, _work_to_do) do {             \
    (_work_to_do) = RING_HAS_UNCONSUMED_REQUESTS(_r);                   \
    if (_work_to_do) break;                                             \
    (_r)->sring->req_event = (_r)->req_cons + 1;                        \
    xen_mb();                                                           \
    (_work_to_do) = RING_HAS_UNCONSUMED_REQUESTS(_r);                   \
} while (0)

#define RING_FINAL_CHECK_FOR_RESPONSES(_r, _work_to_do) do {            \
    (_work_to_do) = RING_HAS_UNCONSUMED_RESPONSES(_r);                  \
    if (_work_to_do) break;                                             \
    (_r)->sring->rsp_event = (_r)->rsp_cons + 1;                        \
    xen_mb();                                                           \
    (_work_to_do) = RING_HAS_UNCONSUMED_RESPONSES(_r);                  \
} while (0)

#endif /* __XEN_PUBLIC_IO_RING_H__ */

/*
 * Local variables:
 * mode: C
 * c-set-style: "BSD"
 * c-basic-offset: 4
 * tab-width: 4
 * indent-tabs-mode: nil
 * End:
 */
/*****************************************************************************
 * xenbus.h
 *
 * Xenbus protocol details.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to
 * deal in the Software without restriction, including without limitation the
 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
 * sell copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 * DEALINGS IN THE SOFTWARE.
 *
 * Copyright (C) 2005 XenSource Ltd.
 */

#ifndef _XEN_PUBLIC_IO_XENBUS_H
#define _XEN_PUBLIC_IO_XENBUS_H

/*
 * The state of either end of the Xenbus, i.e. the current communication
 * status of initialisation across the bus.  States here imply nothing about
 * the state of the connection between the driver and the kernel's device
 * layers.
 */
enum xenbus_state {
    XenbusStateUnknown       = 0,

    XenbusStateInitialising  = 1,

    /*
     * InitWait: Finished early initialisation but waiting for information
     * from the peer or hotplug scripts.
     */
    XenbusStateInitWait      = 2,

    /*
     * Initialised: Waiting for a connection from the peer.
     */
    XenbusStateInitialised   = 3,

    XenbusStateConnected     = 4,

    /*
     * Closing: The device is being closed due to an error or an unplug event.
     */
    XenbusStateClosing       = 5,

    XenbusStateClosed        = 6,

    /*
     * Reconfiguring: The device is being reconfigured.
     */
    XenbusStateReconfiguring = 7,

    XenbusStateReconfigured  = 8
};
typedef enum xenbus_state XenbusState;

#endif /* _XEN_PUBLIC_IO_XENBUS_H */

/*
 * Local variables:
 * mode: C
 * c-set-style: "BSD"
 * c-basic-offset: 4
 * tab-width: 4
 * indent-tabs-mode: nil
 * End:
 */
/*
 * Details of the "wire" protocol between Xen Store Daemon and client
 * library or guest kernel.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to
 * deal in the Software without restriction, including without limitation the
 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
 * sell copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 * DEALINGS IN THE SOFTWARE.
 *
 * Copyright (C) 2005 Rusty Russell IBM Corporation
 */

#ifndef _XS_WIRE_H
#define _XS_WIRE_H

enum xsd_sockmsg_type
{
    XS_DEBUG,
    XS_DIRECTORY,
    XS_READ,
    XS_GET_PERMS,
    XS_WATCH,
    XS_UNWATCH,
    XS_TRANSACTION_START,
    XS_TRANSACTION_END,
    XS_INTRODUCE,
    XS_RELEASE,
    XS_GET_DOMAIN_PATH,
    XS_WRITE,
    XS_MKDIR,
    XS_RM,
    XS_SET_PERMS,
    XS_WATCH_EVENT,
    XS_ERROR,
    XS_IS_DOMAIN_INTRODUCED,
    XS_RESUME,
    XS_SET_TARGET
};

#define XS_WRITE_NONE "NONE"
#define XS_WRITE_CREATE "CREATE"
#define XS_WRITE_CREATE_EXCL "CREATE|EXCL"

/* We hand errors as strings, for portability. */
struct xsd_errors
{
    int errnum;
    const char *errstring;
};
#define XSD_ERROR(x) { x, #x }
/* LINTED: static unused */
static struct xsd_errors xsd_errors[]
#if defined(__GNUC__)
__attribute__((unused))
#endif
    = {
    XSD_ERROR(EINVAL),
    XSD_ERROR(EACCES),
    XSD_ERROR(EEXIST),
    XSD_ERROR(EISDIR),
    XSD_ERROR(ENOENT),
    XSD_ERROR(ENOMEM),
    XSD_ERROR(ENOSPC),
    XSD_ERROR(EIO),
    XSD_ERROR(ENOTEMPTY),
    XSD_ERROR(ENOSYS),
    XSD_ERROR(EROFS),
    XSD_ERROR(EBUSY),
    XSD_ERROR(EAGAIN),
    XSD_ERROR(EISCONN)
};

struct xsd_sockmsg
{
    uint32_t type;  /* XS_??? */
    uint32_t req_id;/* Request identifier, echoed in daemon's response.  */
    uint32_t tx_id; /* Transaction id (0 if not related to a transaction). */
    uint32_t len;   /* Length of data following this. */

    /* Generally followed by nul-terminated string(s). */
};

enum xs_watch_type
{
    XS_WATCH_PATH = 0,
    XS_WATCH_TOKEN
};

/* Inter-domain shared memory communications. */
#define XENSTORE_RING_SIZE 1024
typedef uint32_t XENSTORE_RING_IDX;
#define MASK_XENSTORE_IDX(idx) ((idx) & (XENSTORE_RING_SIZE-1))
struct xenstore_domain_interface {
    char req[XENSTORE_RING_SIZE]; /* Requests to xenstore daemon. */
    char rsp[XENSTORE_RING_SIZE]; /* Replies and async watch events. */
    XENSTORE_RING_IDX req_cons, req_prod;
    XENSTORE_RING_IDX rsp_cons, rsp_prod;
};

/* Violating this is very bad.  See docs/misc/xenstore.txt. */
#define XENSTORE_PAYLOAD_MAX 4096

/* Violating these just gets you an error back */
#define XENSTORE_ABS_PATH_MAX 3072
#define XENSTORE_REL_PATH_MAX 2048

#endif /* _XS_WIRE_H */

/*
 * Local variables:
 * mode: C
 * c-set-style: "BSD"
 * c-basic-offset: 4
 * tab-width: 4
 * indent-tabs-mode: nil
 * End:
 */
/******************************************************************************
 * memory.h
 *
 * Memory reservation and information.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to
 * deal in the Software without restriction, including without limitation the
 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
 * sell copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 * DEALINGS IN THE SOFTWARE.
 *
 * Copyright (c) 2005, Keir Fraser <keir@xensource.com>
 */

#ifndef __XEN_PUBLIC_MEMORY_H__
#define __XEN_PUBLIC_MEMORY_H__

/*
 * Increase or decrease the specified domain's memory reservation. Returns the
 * number of extents successfully allocated or freed.
 * arg == addr of struct xen_memory_reservation.
 */
#define XENMEM_increase_reservation 0
#define XENMEM_decrease_reservation 1
#define XENMEM_populate_physmap     6

#if __XEN_INTERFACE_VERSION__ >= 0x00030209
/*
 * Maximum # bits addressable by the user of the allocated region (e.g., I/O
 * devices often have a 32-bit limitation even in 64-bit systems). If zero
 * then the user has no addressing restriction. This field is not used by
 * XENMEM_decrease_reservation.
 */
#define XENMEMF_address_bits(x)     (x)
#define XENMEMF_get_address_bits(x) ((x) & 0xffu)
/* NUMA node to allocate from. */
#define XENMEMF_node(x)     (((x) + 1) << 8)
#define XENMEMF_get_node(x) ((((x) >> 8) - 1) & 0xffu)
/* Flag to populate physmap with populate-on-demand entries */
#define XENMEMF_populate_on_demand (1<<16)
#endif

struct xen_memory_reservation {

    /*
     * XENMEM_increase_reservation:
     *   OUT: MFN (*not* GMFN) bases of extents that were allocated
     * XENMEM_decrease_reservation:
     *   IN:  GMFN bases of extents to free
     * XENMEM_populate_physmap:
     *   IN:  GPFN bases of extents to populate with memory
     *   OUT: GMFN bases of extents that were allocated
     *   (NB. This command also updates the mach_to_phys translation table)
     */
    XEN_GUEST_HANDLE(xen_pfn_t) extent_start;

    /* Number of extents, and size/alignment of each (2^extent_order pages). */
    xen_ulong_t    nr_extents;
    unsigned int   extent_order;

#if __XEN_INTERFACE_VERSION__ >= 0x00030209
    /* XENMEMF flags. */
    unsigned int   mem_flags;
#else
    unsigned int   address_bits;
#endif

    /*
     * Domain whose reservation is being changed.
     * Unprivileged domains can specify only DOMID_SELF.
     */
    domid_t        domid;
};
typedef struct xen_memory_reservation xen_memory_reservation_t;
DEFINE_XEN_GUEST_HANDLE(xen_memory_reservation_t);

/*
 * An atomic exchange of memory pages. If return code is zero then
 * @out.extent_list provides GMFNs of the newly-allocated memory.
 * Returns zero on complete success, otherwise a negative error code.
 * On complete success then always @nr_exchanged == @in.nr_extents.
 * On partial success @nr_exchanged indicates how much work was done.
 */
#define XENMEM_exchange             11
struct xen_memory_exchange {
    /*
     * [IN] Details of memory extents to be exchanged (GMFN bases).
     * Note that @in.address_bits is ignored and unused.
     */
    struct xen_memory_reservation in;

    /*
     * [IN/OUT] Details of new memory extents.
     * We require that:
     *  1. @in.domid == @out.domid
     *  2. @in.nr_extents  << @in.extent_order ==
     *     @out.nr_extents << @out.extent_order
     *  3. @in.extent_start and @out.extent_start lists must not overlap
     *  4. @out.extent_start lists GPFN bases to be populated
     *  5. @out.extent_start is overwritten with allocated GMFN bases
     */
    struct xen_memory_reservation out;

    /*
     * [OUT] Number of input extents that were successfully exchanged:
     *  1. The first @nr_exchanged input extents were successfully
     *     deallocated.
     *  2. The corresponding first entries in the output extent list correctly
     *     indicate the GMFNs that were successfully exchanged.
     *  3. All other input and output extents are untouched.
     *  4. If not all input exents are exchanged then the return code of this
     *     command will be non-zero.
     *  5. THIS FIELD MUST BE INITIALISED TO ZERO BY THE CALLER!
     */
    xen_ulong_t nr_exchanged;
};
typedef struct xen_memory_exchange xen_memory_exchange_t;
DEFINE_XEN_GUEST_HANDLE(xen_memory_exchange_t);

/*
 * Returns the maximum machine frame number of mapped RAM in this system.
 * This command always succeeds (it never returns an error code).
 * arg == NULL.
 */
#define XENMEM_maximum_ram_page     2

/*
 * Returns the current or maximum memory reservation, in pages, of the
 * specified domain (may be DOMID_SELF). Returns -ve errcode on failure.
 * arg == addr of domid_t.
 */
#define XENMEM_current_reservation  3
#define XENMEM_maximum_reservation  4

/*
 * Returns the maximum GPFN in use by the guest, or -ve errcode on failure.
 */
#define XENMEM_maximum_gpfn         14

/*
 * Returns a list of MFN bases of 2MB extents comprising the machine_to_phys
 * mapping table. Architectures which do not have a m2p table do not implement
 * this command.
 * arg == addr of xen_machphys_mfn_list_t.
 */
#define XENMEM_machphys_mfn_list    5
struct xen_machphys_mfn_list {
    /*
     * Size of the 'extent_start' array. Fewer entries will be filled if the
     * machphys table is smaller than max_extents * 2MB.
     */
    unsigned int max_extents;

    /*
     * Pointer to buffer to fill with list of extent starts. If there are
     * any large discontiguities in the machine address space, 2MB gaps in
     * the machphys table will be represented by an MFN base of zero.
     */
    XEN_GUEST_HANDLE(xen_pfn_t) extent_start;

    /*
     * Number of extents written to the above array. This will be smaller
     * than 'max_extents' if the machphys table is smaller than max_e * 2MB.
     */
    unsigned int nr_extents;
};
typedef struct xen_machphys_mfn_list xen_machphys_mfn_list_t;
DEFINE_XEN_GUEST_HANDLE(xen_machphys_mfn_list_t);

/*
 * Returns the location in virtual address space of the machine_to_phys
 * mapping table. Architectures which do not have a m2p table, or which do not
 * map it by default into guest address space, do not implement this command.
 * arg == addr of xen_machphys_mapping_t.
 */
#define XENMEM_machphys_mapping     12
struct xen_machphys_mapping {
    xen_ulong_t v_start, v_end; /* Start and end virtual addresses.   */
    xen_ulong_t max_mfn;        /* Maximum MFN that can be looked up. */
};
typedef struct xen_machphys_mapping xen_machphys_mapping_t;
DEFINE_XEN_GUEST_HANDLE(xen_machphys_mapping_t);

/*
 * Sets the GPFN at which a particular page appears in the specified guest's
 * pseudophysical address space.
 * arg == addr of xen_add_to_physmap_t.
 */
#define XENMEM_add_to_physmap      7
struct xen_add_to_physmap {
    /* Which domain to change the mapping for. */
    domid_t domid;

    /* Source mapping space. */
#define XENMAPSPACE_shared_info 0 /* shared info page */
#define XENMAPSPACE_grant_table 1 /* grant table page */
#define XENMAPSPACE_gmfn        2 /* GMFN */
    unsigned int space;

    /* Index into source mapping space. */
    xen_ulong_t idx;

    /* GPFN where the source mapping page should appear. */
    xen_pfn_t     gpfn;
};
typedef struct xen_add_to_physmap xen_add_to_physmap_t;
DEFINE_XEN_GUEST_HANDLE(xen_add_to_physmap_t);

/*** REMOVED ***/
/*#define XENMEM_translate_gpfn_list  8*/

/*
 * Returns the pseudo-physical memory map as it was when the domain
 * was started (specified by XENMEM_set_memory_map).
 * arg == addr of xen_memory_map_t.
 */
#define XENMEM_memory_map           9
struct xen_memory_map {
    /*
     * On call the number of entries which can be stored in buffer. On
     * return the number of entries which have been stored in
     * buffer.
     */
    unsigned int nr_entries;

    /*
     * Entries in the buffer are in the same format as returned by the
     * BIOS INT 0x15 EAX=0xE820 call.
     */
    XEN_GUEST_HANDLE(void) buffer;
};
typedef struct xen_memory_map xen_memory_map_t;
DEFINE_XEN_GUEST_HANDLE(xen_memory_map_t);

/*
 * Returns the real physical memory map. Passes the same structure as
 * XENMEM_memory_map.
 * arg == addr of xen_memory_map_t.
 */
#define XENMEM_machine_memory_map   10

/*
 * Set the pseudo-physical memory map of a domain, as returned by
 * XENMEM_memory_map.
 * arg == addr of xen_foreign_memory_map_t.
 */
#define XENMEM_set_memory_map       13
struct xen_foreign_memory_map {
    domid_t domid;
    struct xen_memory_map map;
};
typedef struct xen_foreign_memory_map xen_foreign_memory_map_t;
DEFINE_XEN_GUEST_HANDLE(xen_foreign_memory_map_t);

#define XENMEM_set_pod_target       16
#define XENMEM_get_pod_target       17
struct xen_pod_target {
    /* IN */
    uint64_t target_pages;
    /* OUT */
    uint64_t tot_pages;
    uint64_t pod_cache_pages;
    uint64_t pod_entries;
    /* IN */
    domid_t domid;
};
typedef struct xen_pod_target xen_pod_target_t;
#endif /* __XEN_PUBLIC_MEMORY_H__ */

/*
 * Local variables:
 * mode: C
 * c-set-style: "BSD"
 * c-basic-offset: 4
 * tab-width: 4
 * indent-tabs-mode: nil
 * End:
 */
/******************************************************************************
 * nmi.h
 *
 * NMI callback registration and reason codes.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to
 * deal in the Software without restriction, including without limitation the
 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
 * sell copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 * DEALINGS IN THE SOFTWARE.
 *
 * Copyright (c) 2005, Keir Fraser <keir@xensource.com>
 */

#ifndef __XEN_PUBLIC_NMI_H__
#define __XEN_PUBLIC_NMI_H__

/*
 * NMI reason codes:
 * Currently these are x86-specific, stored in arch_shared_info.nmi_reason.
 */
 /* I/O-check error reported via ISA port 0x61, bit 6. */
#define _XEN_NMIREASON_io_error     0
#define XEN_NMIREASON_io_error      (1UL << _XEN_NMIREASON_io_error)
 /* Parity error reported via ISA port 0x61, bit 7. */
#define _XEN_NMIREASON_parity_error 1
#define XEN_NMIREASON_parity_error  (1UL << _XEN_NMIREASON_parity_error)
 /* Unknown hardware-generated NMI. */
#define _XEN_NMIREASON_unknown      2
#define XEN_NMIREASON_unknown       (1UL << _XEN_NMIREASON_unknown)

/*
 * long nmi_op(unsigned int cmd, void *arg)
 * NB. All ops return zero on success, else a negative error code.
 */

/*
 * Register NMI callback for this (calling) VCPU. Currently this only makes
 * sense for domain 0, vcpu 0. All other callers will be returned EINVAL.
 * arg == pointer to xennmi_callback structure.
 */
#define XENNMI_register_callback   0
struct xennmi_callback {
    unsigned long handler_address;
    unsigned long pad;
};
typedef struct xennmi_callback xennmi_callback_t;
DEFINE_XEN_GUEST_HANDLE(xennmi_callback_t);

/*
 * Deregister NMI callback for this (calling) VCPU.
 * arg == NULL.
 */
#define XENNMI_unregister_callback 1

#endif /* __XEN_PUBLIC_NMI_H__ */

/*
 * Local variables:
 * mode: C
 * c-set-style: "BSD"
 * c-basic-offset: 4
 * tab-width: 4
 * indent-tabs-mode: nil
 * End:
 */
/*
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to
 * deal in the Software without restriction, including without limitation the
 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
 * sell copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 * DEALINGS IN THE SOFTWARE.
 */

#ifndef __XEN_PUBLIC_PHYSDEV_H__
#define __XEN_PUBLIC_PHYSDEV_H__

/*
 * Prototype for this hypercall is:
 *  int physdev_op(int cmd, void *args)
 * @cmd  == PHYSDEVOP_??? (physdev operation).
 * @args == Operation-specific extra arguments (NULL if none).
 */

/*
 * Notify end-of-interrupt (EOI) for the specified IRQ.
 * @arg == pointer to physdev_eoi structure.
 */
#define PHYSDEVOP_eoi                   12
struct physdev_eoi {
    /* IN */
    uint32_t irq;
};
typedef struct physdev_eoi physdev_eoi_t;
DEFINE_XEN_GUEST_HANDLE(physdev_eoi_t);

/*
 * Register a shared page for the hypervisor to indicate whether the guest
 * must issue PHYSDEVOP_eoi. The semantics of PHYSDEVOP_eoi change slightly
 * once the guest used this function in that the associated event channel
 * will automatically get unmasked. The page registered is used as a bit
 * array indexed by Xen's PIRQ value.
 */
#define PHYSDEVOP_pirq_eoi_gmfn         17
struct physdev_pirq_eoi_gmfn {
    /* IN */
    xen_pfn_t gmfn;
};
typedef struct physdev_pirq_eoi_gmfn physdev_pirq_eoi_gmfn_t;
DEFINE_XEN_GUEST_HANDLE(physdev_pirq_eoi_gmfn_t);

/*
 * Query the status of an IRQ line.
 * @arg == pointer to physdev_irq_status_query structure.
 */
#define PHYSDEVOP_irq_status_query       5
struct physdev_irq_status_query {
    /* IN */
    uint32_t irq;
    /* OUT */
    uint32_t flags; /* XENIRQSTAT_* */
};
typedef struct physdev_irq_status_query physdev_irq_status_query_t;
DEFINE_XEN_GUEST_HANDLE(physdev_irq_status_query_t);

/* Need to call PHYSDEVOP_eoi when the IRQ has been serviced? */
#define _XENIRQSTAT_needs_eoi   (0)
#define  XENIRQSTAT_needs_eoi   (1U<<_XENIRQSTAT_needs_eoi)

/* IRQ shared by multiple guests? */
#define _XENIRQSTAT_shared      (1)
#define  XENIRQSTAT_shared      (1U<<_XENIRQSTAT_shared)

/*
 * Set the current VCPU's I/O privilege level.
 * @arg == pointer to physdev_set_iopl structure.
 */
#define PHYSDEVOP_set_iopl               6
struct physdev_set_iopl {
    /* IN */
    uint32_t iopl;
};
typedef struct physdev_set_iopl physdev_set_iopl_t;
DEFINE_XEN_GUEST_HANDLE(physdev_set_iopl_t);

/*
 * Set the current VCPU's I/O-port permissions bitmap.
 * @arg == pointer to physdev_set_iobitmap structure.
 */
#define PHYSDEVOP_set_iobitmap           7
struct physdev_set_iobitmap {
    /* IN */
#if __XEN_INTERFACE_VERSION__ >= 0x00030205
    XEN_GUEST_HANDLE(uint8) bitmap;
#else
    uint8_t *bitmap;
#endif
    uint32_t nr_ports;
};
typedef struct physdev_set_iobitmap physdev_set_iobitmap_t;
DEFINE_XEN_GUEST_HANDLE(physdev_set_iobitmap_t);

/*
 * Read or write an IO-APIC register.
 * @arg == pointer to physdev_apic structure.
 */
#define PHYSDEVOP_apic_read              8
#define PHYSDEVOP_apic_write             9
struct physdev_apic {
    /* IN */
    unsigned long apic_physbase;
    uint32_t reg;
    /* IN or OUT */
    uint32_t value;
};
typedef struct physdev_apic physdev_apic_t;
DEFINE_XEN_GUEST_HANDLE(physdev_apic_t);

/*
 * Allocate or free a physical upcall vector for the specified IRQ line.
 * @arg == pointer to physdev_irq structure.
 */
#define PHYSDEVOP_alloc_irq_vector      10
#define PHYSDEVOP_free_irq_vector       11
struct physdev_irq {
    /* IN */
    uint32_t irq;
    /* IN or OUT */
    uint32_t vector;
};
typedef struct physdev_irq physdev_irq_t;
DEFINE_XEN_GUEST_HANDLE(physdev_irq_t);

#define MAP_PIRQ_TYPE_MSI               0x0
#define MAP_PIRQ_TYPE_GSI               0x1
#define MAP_PIRQ_TYPE_UNKNOWN           0x2

#define PHYSDEVOP_map_pirq               13
struct physdev_map_pirq {
    domid_t domid;
    /* IN */
    int type;
    /* IN */
    int index;
    /* IN or OUT */
    int pirq;
    /* IN */
    int bus;
    /* IN */
    int devfn;
    /* IN */
    int entry_nr;
    /* IN */
    uint64_t table_base;
};
typedef struct physdev_map_pirq physdev_map_pirq_t;
DEFINE_XEN_GUEST_HANDLE(physdev_map_pirq_t);

#define PHYSDEVOP_unmap_pirq             14
struct physdev_unmap_pirq {
    domid_t domid;
    /* IN */
    int pirq;
};

typedef struct physdev_unmap_pirq physdev_unmap_pirq_t;
DEFINE_XEN_GUEST_HANDLE(physdev_unmap_pirq_t);

#define PHYSDEVOP_manage_pci_add         15
#define PHYSDEVOP_manage_pci_remove      16
struct physdev_manage_pci {
    /* IN */
    uint8_t bus;
    uint8_t devfn;
};

typedef struct physdev_manage_pci physdev_manage_pci_t;
DEFINE_XEN_GUEST_HANDLE(physdev_manage_pci_t);

#define PHYSDEVOP_restore_msi            19
struct physdev_restore_msi {
    /* IN */
    uint8_t bus;
    uint8_t devfn;
};
typedef struct physdev_restore_msi physdev_restore_msi_t;
DEFINE_XEN_GUEST_HANDLE(physdev_restore_msi_t);

#define PHYSDEVOP_manage_pci_add_ext     20
struct physdev_manage_pci_ext {
    /* IN */
    uint8_t bus;
    uint8_t devfn;
    unsigned is_extfn;
    unsigned is_virtfn;
    struct {
        uint8_t bus;
        uint8_t devfn;
    } physfn;
};

typedef struct physdev_manage_pci_ext physdev_manage_pci_ext_t;
DEFINE_XEN_GUEST_HANDLE(physdev_manage_pci_ext_t);

/*
 * Argument to physdev_op_compat() hypercall. Superceded by new physdev_op()
 * hypercall since 0x00030202.
 */
struct physdev_op {
    uint32_t cmd;
    union {
        struct physdev_irq_status_query      irq_status_query;
        struct physdev_set_iopl              set_iopl;
        struct physdev_set_iobitmap          set_iobitmap;
        struct physdev_apic                  apic_op;
        struct physdev_irq                   irq_op;
    } u;
};
typedef struct physdev_op physdev_op_t;
DEFINE_XEN_GUEST_HANDLE(physdev_op_t);

/*
 * Notify that some PIRQ-bound event channels have been unmasked.
 * ** This command is obsolete since interface version 0x00030202 and is **
 * ** unsupported by newer versions of Xen.                              **
 */
#define PHYSDEVOP_IRQ_UNMASK_NOTIFY      4

/*
 * These all-capitals physdev operation names are superceded by the new names
 * (defined above) since interface version 0x00030202.
 */
#define PHYSDEVOP_IRQ_STATUS_QUERY       PHYSDEVOP_irq_status_query
#define PHYSDEVOP_SET_IOPL               PHYSDEVOP_set_iopl
#define PHYSDEVOP_SET_IOBITMAP           PHYSDEVOP_set_iobitmap
#define PHYSDEVOP_APIC_READ              PHYSDEVOP_apic_read
#define PHYSDEVOP_APIC_WRITE             PHYSDEVOP_apic_write
#define PHYSDEVOP_ASSIGN_VECTOR          PHYSDEVOP_alloc_irq_vector
#define PHYSDEVOP_FREE_VECTOR            PHYSDEVOP_free_irq_vector
#define PHYSDEVOP_IRQ_NEEDS_UNMASK_NOTIFY XENIRQSTAT_needs_eoi
#define PHYSDEVOP_IRQ_SHARED             XENIRQSTAT_shared

#endif /* __XEN_PUBLIC_PHYSDEV_H__ */

/*
 * Local variables:
 * mode: C
 * c-set-style: "BSD"
 * c-basic-offset: 4
 * tab-width: 4
 * indent-tabs-mode: nil
 * End:
 */
/******************************************************************************
 * platform.h
 *
 * Hardware platform operations. Intended for use by domain-0 kernel.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to
 * deal in the Software without restriction, including without limitation the
 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
 * sell copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 * DEALINGS IN THE SOFTWARE.
 *
 * Copyright (c) 2002-2006, K Fraser
 */

#ifndef __XEN_PUBLIC_PLATFORM_H__
#define __XEN_PUBLIC_PLATFORM_H__

#include "xen.h"

#define XENPF_INTERFACE_VERSION 0x03000001

/*
 * Set clock such that it would read <secs,nsecs> after 00:00:00 UTC,
 * 1 January, 1970 if the current system time was <system_time>.
 */
#define XENPF_settime             17
struct xenpf_settime {
    /* IN variables. */
    uint32_t secs;
    uint32_t nsecs;
    uint64_t system_time;
};
typedef struct xenpf_settime xenpf_settime_t;
DEFINE_XEN_GUEST_HANDLE(xenpf_settime_t);

/*
 * Request memory range (@mfn, @mfn+@nr_mfns-1) to have type @type.
 * On x86, @type is an architecture-defined MTRR memory type.
 * On success, returns the MTRR that was used (@reg) and a handle that can
 * be passed to XENPF_DEL_MEMTYPE to accurately tear down the new setting.
 * (x86-specific).
 */
#define XENPF_add_memtype         31
struct xenpf_add_memtype {
    /* IN variables. */
    xen_pfn_t mfn;
    uint64_t nr_mfns;
    uint32_t type;
    /* OUT variables. */
    uint32_t handle;
    uint32_t reg;
};
typedef struct xenpf_add_memtype xenpf_add_memtype_t;
DEFINE_XEN_GUEST_HANDLE(xenpf_add_memtype_t);

/*
 * Tear down an existing memory-range type. If @handle is remembered then it
 * should be passed in to accurately tear down the correct setting (in case
 * of overlapping memory regions with differing types). If it is not known
 * then @handle should be set to zero. In all cases @reg must be set.
 * (x86-specific).
 */
#define XENPF_del_memtype         32
struct xenpf_del_memtype {
    /* IN variables. */
    uint32_t handle;
    uint32_t reg;
};
typedef struct xenpf_del_memtype xenpf_del_memtype_t;
DEFINE_XEN_GUEST_HANDLE(xenpf_del_memtype_t);

/* Read current type of an MTRR (x86-specific). */
#define XENPF_read_memtype        33
struct xenpf_read_memtype {
    /* IN variables. */
    uint32_t reg;
    /* OUT variables. */
    xen_pfn_t mfn;
    uint64_t nr_mfns;
    uint32_t type;
};
typedef struct xenpf_read_memtype xenpf_read_memtype_t;
DEFINE_XEN_GUEST_HANDLE(xenpf_read_memtype_t);

#define XENPF_microcode_update    35
struct xenpf_microcode_update {
    /* IN variables. */
    XEN_GUEST_HANDLE(const_void) data;/* Pointer to microcode data */
    uint32_t length;                  /* Length of microcode data. */
};
typedef struct xenpf_microcode_update xenpf_microcode_update_t;
DEFINE_XEN_GUEST_HANDLE(xenpf_microcode_update_t);

#define XENPF_platform_quirk      39
#define QUIRK_NOIRQBALANCING      1 /* Do not restrict IO-APIC RTE targets */
#define QUIRK_IOAPIC_BAD_REGSEL   2 /* IO-APIC REGSEL forgets its value    */
#define QUIRK_IOAPIC_GOOD_REGSEL  3 /* IO-APIC REGSEL behaves properly     */
struct xenpf_platform_quirk {
    /* IN variables. */
    uint32_t quirk_id;
};
typedef struct xenpf_platform_quirk xenpf_platform_quirk_t;
DEFINE_XEN_GUEST_HANDLE(xenpf_platform_quirk_t);

#define XENPF_firmware_info       50
#define XEN_FW_DISK_INFO          1 /* from int 13 AH=08/41/48 */
#define XEN_FW_DISK_MBR_SIGNATURE 2 /* from MBR offset 0x1b8 */
#define XEN_FW_VBEDDC_INFO        3 /* from int 10 AX=4f15 */
struct xenpf_firmware_info {
    /* IN variables. */
    uint32_t type;
    uint32_t index;
    /* OUT variables. */
    union {
        struct {
            /* Int13, Fn48: Check Extensions Present. */
            uint8_t device;                   /* %dl: bios device number */
            uint8_t version;                  /* %ah: major version      */
            uint16_t interface_support;       /* %cx: support bitmap     */
            /* Int13, Fn08: Legacy Get Device Parameters. */
            uint16_t legacy_max_cylinder;     /* %cl[7:6]:%ch: max cyl # */
            uint8_t legacy_max_head;          /* %dh: max head #         */
            uint8_t legacy_sectors_per_track; /* %cl[5:0]: max sector #  */
            /* Int13, Fn41: Get Device Parameters (as filled into %ds:%esi). */
            /* NB. First uint16_t of buffer must be set to buffer size.      */
            XEN_GUEST_HANDLE(void) edd_params;
        } disk_info; /* XEN_FW_DISK_INFO */
        struct {
            uint8_t device;                   /* bios device number  */
            uint32_t mbr_signature;           /* offset 0x1b8 in mbr */
        } disk_mbr_signature; /* XEN_FW_DISK_MBR_SIGNATURE */
        struct {
            /* Int10, AX=4F15: Get EDID info. */
            uint8_t capabilities;
            uint8_t edid_transfer_time;
            /* must refer to 128-byte buffer */
            XEN_GUEST_HANDLE(uint8) edid;
        } vbeddc_info; /* XEN_FW_VBEDDC_INFO */
    } u;
};
typedef struct xenpf_firmware_info xenpf_firmware_info_t;
DEFINE_XEN_GUEST_HANDLE(xenpf_firmware_info_t);

#define XENPF_enter_acpi_sleep    51
struct xenpf_enter_acpi_sleep {
    /* IN variables */
    uint16_t pm1a_cnt_val;      /* PM1a control value. */
    uint16_t pm1b_cnt_val;      /* PM1b control value. */
    uint32_t sleep_state;       /* Which state to enter (Sn). */
    uint32_t flags;             /* Must be zero. */
};
typedef struct xenpf_enter_acpi_sleep xenpf_enter_acpi_sleep_t;
DEFINE_XEN_GUEST_HANDLE(xenpf_enter_acpi_sleep_t);

#define XENPF_change_freq         52
struct xenpf_change_freq {
    /* IN variables */
    uint32_t flags; /* Must be zero. */
    uint32_t cpu;   /* Physical cpu. */
    uint64_t freq;  /* New frequency (Hz). */
};
typedef struct xenpf_change_freq xenpf_change_freq_t;
DEFINE_XEN_GUEST_HANDLE(xenpf_change_freq_t);

/*
 * Get idle times (nanoseconds since boot) for physical CPUs specified in the
 * @cpumap_bitmap with range [0..@cpumap_nr_cpus-1]. The @idletime array is
 * indexed by CPU number; only entries with the corresponding @cpumap_bitmap
 * bit set are written to. On return, @cpumap_bitmap is modified so that any
 * non-existent CPUs are cleared. Such CPUs have their @idletime array entry
 * cleared.
 */
#define XENPF_getidletime         53
struct xenpf_getidletime {
    /* IN/OUT variables */
    /* IN: CPUs to interrogate; OUT: subset of IN which are present */
    XEN_GUEST_HANDLE(uint8) cpumap_bitmap;
    /* IN variables */
    /* Size of cpumap bitmap. */
    uint32_t cpumap_nr_cpus;
    /* Must be indexable for every cpu in cpumap_bitmap. */
    XEN_GUEST_HANDLE(uint64) idletime;
    /* OUT variables */
    /* System time when the idletime snapshots were taken. */
    uint64_t now;
};
typedef struct xenpf_getidletime xenpf_getidletime_t;
DEFINE_XEN_GUEST_HANDLE(xenpf_getidletime_t);

#define XENPF_set_processor_pminfo      54

/* ability bits */
#define XEN_PROCESSOR_PM_CX	1
#define XEN_PROCESSOR_PM_PX	2
#define XEN_PROCESSOR_PM_TX	4

/* cmd type */
#define XEN_PM_CX   0
#define XEN_PM_PX   1
#define XEN_PM_TX   2

/* Px sub info type */
#define XEN_PX_PCT   1
#define XEN_PX_PSS   2
#define XEN_PX_PPC   4
#define XEN_PX_PSD   8

struct xen_power_register {
    uint32_t     space_id;
    uint32_t     bit_width;
    uint32_t     bit_offset;
    uint32_t     access_size;
    uint64_t     address;
};

struct xen_processor_csd {
    uint32_t    domain;      /* domain number of one dependent group */
    uint32_t    coord_type;  /* coordination type */
    uint32_t    num;         /* number of processors in same domain */
};
typedef struct xen_processor_csd xen_processor_csd_t;
DEFINE_XEN_GUEST_HANDLE(xen_processor_csd_t);

struct xen_processor_cx {
    struct xen_power_register  reg; /* GAS for Cx trigger register */
    uint8_t     type;     /* cstate value, c0: 0, c1: 1, ... */
    uint32_t    latency;  /* worst latency (ms) to enter/exit this cstate */
    uint32_t    power;    /* average power consumption(mW) */
    uint32_t    dpcnt;    /* number of dependency entries */
    XEN_GUEST_HANDLE(xen_processor_csd_t) dp; /* NULL if no dependency */
};
typedef struct xen_processor_cx xen_processor_cx_t;
DEFINE_XEN_GUEST_HANDLE(xen_processor_cx_t);

struct xen_processor_flags {
    uint32_t bm_control:1;
    uint32_t bm_check:1;
    uint32_t has_cst:1;
    uint32_t power_setup_done:1;
    uint32_t bm_rld_set:1;
};

struct xen_processor_power {
    uint32_t count;  /* number of C state entries in array below */
    struct xen_processor_flags flags;  /* global flags of this processor */
    XEN_GUEST_HANDLE(xen_processor_cx_t) states; /* supported c states */
};

struct xen_pct_register {
    uint8_t  descriptor;
    uint16_t length;
    uint8_t  space_id;
    uint8_t  bit_width;
    uint8_t  bit_offset;
    uint8_t  reserved;
    uint64_t address;
};

struct xen_processor_px {
    uint64_t core_frequency; /* megahertz */
    uint64_t power;      /* milliWatts */
    uint64_t transition_latency; /* microseconds */
    uint64_t bus_master_latency; /* microseconds */
    uint64_t control;        /* control value */
    uint64_t status;     /* success indicator */
};
typedef struct xen_processor_px xen_processor_px_t;
DEFINE_XEN_GUEST_HANDLE(xen_processor_px_t);

struct xen_psd_package {
    uint64_t num_entries;
    uint64_t revision;
    uint64_t domain;
    uint64_t coord_type;
    uint64_t num_processors;
};

struct xen_processor_performance {
    uint32_t flags;     /* flag for Px sub info type */
    uint32_t platform_limit;  /* Platform limitation on freq usage */
    struct xen_pct_register control_register;
    struct xen_pct_register status_register;
    uint32_t state_count;     /* total available performance states */
    XEN_GUEST_HANDLE(xen_processor_px_t) states;
    struct xen_psd_package domain_info;
    uint32_t shared_type;     /* coordination type of this processor */
};
typedef struct xen_processor_performance xen_processor_performance_t;
DEFINE_XEN_GUEST_HANDLE(xen_processor_performance_t);

struct xenpf_set_processor_pminfo {
    /* IN variables */
    uint32_t id;    /* ACPI CPU ID */
    uint32_t type;  /* {XEN_PM_CX, XEN_PM_PX} */
    union {
        struct xen_processor_power          power;/* Cx: _CST/_CSD */
        struct xen_processor_performance    perf; /* Px: _PPC/_PCT/_PSS/_PSD */
    } u;
};
typedef struct xenpf_set_processor_pminfo xenpf_set_processor_pminfo_t;
DEFINE_XEN_GUEST_HANDLE(xenpf_set_processor_pminfo_t);

#define XENPF_panic_init          40
struct xenpf_panic_init {
    unsigned long panic_addr;
};
typedef struct xenpf_panic_init xenpf_panic_init_t;
DEFINE_XEN_GUEST_HANDLE(xenpf_panic_init_t);

struct xen_platform_op {
    uint32_t cmd;
    uint32_t interface_version; /* XENPF_INTERFACE_VERSION */
    union {
        struct xenpf_settime           settime;
        struct xenpf_add_memtype       add_memtype;
        struct xenpf_del_memtype       del_memtype;
        struct xenpf_read_memtype      read_memtype;
        struct xenpf_microcode_update  microcode;
        struct xenpf_platform_quirk    platform_quirk;
        struct xenpf_firmware_info     firmware_info;
        struct xenpf_enter_acpi_sleep  enter_acpi_sleep;
        struct xenpf_change_freq       change_freq;
        struct xenpf_getidletime       getidletime;
        struct xenpf_set_processor_pminfo set_pminfo;
        struct xenpf_panic_init        panic_init;
        uint8_t                        pad[128];
    } u;
};
typedef struct xen_platform_op xen_platform_op_t;
DEFINE_XEN_GUEST_HANDLE(xen_platform_op_t);

#endif /* __XEN_PUBLIC_PLATFORM_H__ */

/*
 * Local variables:
 * mode: C
 * c-set-style: "BSD"
 * c-basic-offset: 4
 * tab-width: 4
 * indent-tabs-mode: nil
 * End:
 */
/******************************************************************************
 * sched.h
 *
 * Scheduler state interactions
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to
 * deal in the Software without restriction, including without limitation the
 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
 * sell copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 * DEALINGS IN THE SOFTWARE.
 *
 * Copyright (c) 2005, Keir Fraser <keir@xensource.com>
 */

#ifndef __XEN_PUBLIC_SCHED_H__
#define __XEN_PUBLIC_SCHED_H__

#include "event_channel.h"

/*
 * The prototype for this hypercall is:
 *  long sched_op(int cmd, void *arg)
 * @cmd == SCHEDOP_??? (scheduler operation).
 * @arg == Operation-specific extra argument(s), as described below.
 *
 * Versions of Xen prior to 3.0.2 provided only the following legacy version
 * of this hypercall, supporting only the commands yield, block and shutdown:
 *  long sched_op(int cmd, unsigned long arg)
 * @cmd == SCHEDOP_??? (scheduler operation).
 * @arg == 0               (SCHEDOP_yield and SCHEDOP_block)
 *      == SHUTDOWN_* code (SCHEDOP_shutdown)
 * This legacy version is available to new guests as sched_op_compat().
 */

/*
 * Voluntarily yield the CPU.
 * @arg == NULL.
 */
#define SCHEDOP_yield       0

/*
 * Block execution of this VCPU until an event is received for processing.
 * If called with event upcalls masked, this operation will atomically
 * reenable event delivery and check for pending events before blocking the
 * VCPU. This avoids a "wakeup waiting" race.
 * @arg == NULL.
 */
#define SCHEDOP_block       1

/*
 * Halt execution of this domain (all VCPUs) and notify the system controller.
 * @arg == pointer to sched_shutdown structure.
 */
#define SCHEDOP_shutdown    2
struct sched_shutdown {
    unsigned int reason; /* SHUTDOWN_* */
};
typedef struct sched_shutdown sched_shutdown_t;
DEFINE_XEN_GUEST_HANDLE(sched_shutdown_t);

/*
 * Poll a set of event-channel ports. Return when one or more are pending. An
 * optional timeout may be specified.
 * @arg == pointer to sched_poll structure.
 */
#define SCHEDOP_poll        3
struct sched_poll {
    XEN_GUEST_HANDLE(evtchn_port_t) ports;
    unsigned int nr_ports;
    uint64_t timeout;
};
typedef struct sched_poll sched_poll_t;
DEFINE_XEN_GUEST_HANDLE(sched_poll_t);

/*
 * Declare a shutdown for another domain. The main use of this function is
 * in interpreting shutdown requests and reasons for fully-virtualized
 * domains.  A para-virtualized domain may use SCHEDOP_shutdown directly.
 * @arg == pointer to sched_remote_shutdown structure.
 */
#define SCHEDOP_remote_shutdown        4
struct sched_remote_shutdown {
    domid_t domain_id;         /* Remote domain ID */
    unsigned int reason;       /* SHUTDOWN_xxx reason */
};
typedef struct sched_remote_shutdown sched_remote_shutdown_t;
DEFINE_XEN_GUEST_HANDLE(sched_remote_shutdown_t);

/*
 * Reason codes for SCHEDOP_shutdown. These may be interpreted by control
 * software to determine the appropriate action. For the most part, Xen does
 * not care about the shutdown code.
 */
#define SHUTDOWN_poweroff   0  /* Domain exited normally. Clean up and kill. */
#define SHUTDOWN_reboot     1  /* Clean up, kill, and then restart.          */
#define SHUTDOWN_suspend    2  /* Clean up, save suspend info, kill.         */
#define SHUTDOWN_crash      3  /* Tell controller we've crashed.             */

#endif /* __XEN_PUBLIC_SCHED_H__ */

/*
 * Local variables:
 * mode: C
 * c-set-style: "BSD"
 * c-basic-offset: 4
 * tab-width: 4
 * indent-tabs-mode: nil
 * End:
 */
/******************************************************************************
 * sysctl.h
 *
 * System management operations. For use by node control stack.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to
 * deal in the Software without restriction, including without limitation the
 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
 * sell copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 * DEALINGS IN THE SOFTWARE.
 *
 * Copyright (c) 2002-2006, K Fraser
 */

#ifndef __XEN_PUBLIC_SYSCTL_H__
#define __XEN_PUBLIC_SYSCTL_H__

#if !defined(__XEN__) && !defined(__XEN_TOOLS__)
#error "sysctl operations are intended for use by node control tools only"
#endif

#include "xen.h"
#include "domctl.h"

#define XEN_SYSCTL_INTERFACE_VERSION 0x00000006

/*
 * Read console content from Xen buffer ring.
 */
#define XEN_SYSCTL_readconsole       1
struct xen_sysctl_readconsole {
    /* IN: Non-zero -> clear after reading. */
    uint8_t clear;
    /* IN: Non-zero -> start index specified by @index field. */
    uint8_t incremental;
    uint8_t pad0, pad1;
    /*
     * IN:  Start index for consuming from ring buffer (if @incremental);
     * OUT: End index after consuming from ring buffer.
     */
    uint32_t index;
    /* IN: Virtual address to write console data. */
    XEN_GUEST_HANDLE_64(char) buffer;
    /* IN: Size of buffer; OUT: Bytes written to buffer. */
    uint32_t count;
};
typedef struct xen_sysctl_readconsole xen_sysctl_readconsole_t;
DEFINE_XEN_GUEST_HANDLE(xen_sysctl_readconsole_t);

/* Get trace buffers machine base address */
#define XEN_SYSCTL_tbuf_op           2
struct xen_sysctl_tbuf_op {
    /* IN variables */
#define XEN_SYSCTL_TBUFOP_get_info     0
#define XEN_SYSCTL_TBUFOP_set_cpu_mask 1
#define XEN_SYSCTL_TBUFOP_set_evt_mask 2
#define XEN_SYSCTL_TBUFOP_set_size     3
#define XEN_SYSCTL_TBUFOP_enable       4
#define XEN_SYSCTL_TBUFOP_disable      5
    uint32_t cmd;
    /* IN/OUT variables */
    struct xenctl_cpumap cpu_mask;
    uint32_t             evt_mask;
    /* OUT variables */
    uint64_aligned_t buffer_mfn;
    uint32_t size;
};
typedef struct xen_sysctl_tbuf_op xen_sysctl_tbuf_op_t;
DEFINE_XEN_GUEST_HANDLE(xen_sysctl_tbuf_op_t);

/*
 * Get physical information about the host machine
 */
#define XEN_SYSCTL_physinfo          3
 /* (x86) The platform supports HVM guests. */
#define _XEN_SYSCTL_PHYSCAP_hvm          0
#define XEN_SYSCTL_PHYSCAP_hvm           (1u<<_XEN_SYSCTL_PHYSCAP_hvm)
 /* (x86) The platform supports HVM-guest direct access to I/O devices. */
#define _XEN_SYSCTL_PHYSCAP_hvm_directio 1
#define XEN_SYSCTL_PHYSCAP_hvm_directio  (1u<<_XEN_SYSCTL_PHYSCAP_hvm_directio)
struct xen_sysctl_physinfo {
    uint32_t threads_per_core;
    uint32_t cores_per_socket;
    uint32_t nr_cpus;
    uint32_t nr_nodes;
    uint32_t cpu_khz;
    uint64_aligned_t total_pages;
    uint64_aligned_t free_pages;
    uint64_aligned_t scrub_pages;
    uint32_t hw_cap[8];

    /*
     * IN: maximum addressable entry in the caller-provided cpu_to_node array.
     * OUT: largest cpu identifier in the system.
     * If OUT is greater than IN then the cpu_to_node array is truncated!
     */
    uint32_t max_cpu_id;
    /*
     * If not NULL, this array is filled with node identifier for each cpu.
     * If a cpu has no node information (e.g., cpu not present) then the
     * sentinel value ~0u is written.
     * The size of this array is specified by the caller in @max_cpu_id.
     * If the actual @max_cpu_id is smaller than the array then the trailing
     * elements of the array will not be written by the sysctl.
     */
    XEN_GUEST_HANDLE_64(uint32) cpu_to_node;

    /* XEN_SYSCTL_PHYSCAP_??? */
    uint32_t capabilities;
};
typedef struct xen_sysctl_physinfo xen_sysctl_physinfo_t;
DEFINE_XEN_GUEST_HANDLE(xen_sysctl_physinfo_t);

/*
 * Get the ID of the current scheduler.
 */
#define XEN_SYSCTL_sched_id          4
struct xen_sysctl_sched_id {
    /* OUT variable */
    uint32_t sched_id;
};
typedef struct xen_sysctl_sched_id xen_sysctl_sched_id_t;
DEFINE_XEN_GUEST_HANDLE(xen_sysctl_sched_id_t);

/* Interface for controlling Xen software performance counters. */
#define XEN_SYSCTL_perfc_op          5
/* Sub-operations: */
#define XEN_SYSCTL_PERFCOP_reset 1   /* Reset all counters to zero. */
#define XEN_SYSCTL_PERFCOP_query 2   /* Get perfctr information. */
struct xen_sysctl_perfc_desc {
    char         name[80];             /* name of perf counter */
    uint32_t     nr_vals;              /* number of values for this counter */
};
typedef struct xen_sysctl_perfc_desc xen_sysctl_perfc_desc_t;
DEFINE_XEN_GUEST_HANDLE(xen_sysctl_perfc_desc_t);
typedef uint32_t xen_sysctl_perfc_val_t;
DEFINE_XEN_GUEST_HANDLE(xen_sysctl_perfc_val_t);

struct xen_sysctl_perfc_op {
    /* IN variables. */
    uint32_t       cmd;                /*  XEN_SYSCTL_PERFCOP_??? */
    /* OUT variables. */
    uint32_t       nr_counters;       /*  number of counters description  */
    uint32_t       nr_vals;           /*  number of values  */
    /* counter information (or NULL) */
    XEN_GUEST_HANDLE_64(xen_sysctl_perfc_desc_t) desc;
    /* counter values (or NULL) */
    XEN_GUEST_HANDLE_64(xen_sysctl_perfc_val_t) val;
};
typedef struct xen_sysctl_perfc_op xen_sysctl_perfc_op_t;
DEFINE_XEN_GUEST_HANDLE(xen_sysctl_perfc_op_t);

#define XEN_SYSCTL_getdomaininfolist 6
struct xen_sysctl_getdomaininfolist {
    /* IN variables. */
    domid_t               first_domain;
    uint32_t              max_domains;
    XEN_GUEST_HANDLE_64(xen_domctl_getdomaininfo_t) buffer;
    /* OUT variables. */
    uint32_t              num_domains;
};
typedef struct xen_sysctl_getdomaininfolist xen_sysctl_getdomaininfolist_t;
DEFINE_XEN_GUEST_HANDLE(xen_sysctl_getdomaininfolist_t);

/* Inject debug keys into Xen. */
#define XEN_SYSCTL_debug_keys        7
struct xen_sysctl_debug_keys {
    /* IN variables. */
    XEN_GUEST_HANDLE_64(char) keys;
    uint32_t nr_keys;
};
typedef struct xen_sysctl_debug_keys xen_sysctl_debug_keys_t;
DEFINE_XEN_GUEST_HANDLE(xen_sysctl_debug_keys_t);

/* Get physical CPU information. */
#define XEN_SYSCTL_getcpuinfo        8
struct xen_sysctl_cpuinfo {
    uint64_aligned_t idletime;
};
typedef struct xen_sysctl_cpuinfo xen_sysctl_cpuinfo_t;
DEFINE_XEN_GUEST_HANDLE(xen_sysctl_cpuinfo_t);
struct xen_sysctl_getcpuinfo {
    /* IN variables. */
    uint32_t max_cpus;
    XEN_GUEST_HANDLE_64(xen_sysctl_cpuinfo_t) info;
    /* OUT variables. */
    uint32_t nr_cpus;
};
typedef struct xen_sysctl_getcpuinfo xen_sysctl_getcpuinfo_t;
DEFINE_XEN_GUEST_HANDLE(xen_sysctl_getcpuinfo_t);

#define XEN_SYSCTL_availheap         9
struct xen_sysctl_availheap {
    /* IN variables. */
    uint32_t min_bitwidth;  /* Smallest address width (zero if don't care). */
    uint32_t max_bitwidth;  /* Largest address width (zero if don't care). */
    int32_t  node;          /* NUMA node of interest (-1 for all nodes). */
    /* OUT variables. */
    uint64_aligned_t avail_bytes;/* Bytes available in the specified region. */
};
typedef struct xen_sysctl_availheap xen_sysctl_availheap_t;
DEFINE_XEN_GUEST_HANDLE(xen_sysctl_availheap_t);

#define XEN_SYSCTL_get_pmstat        10
struct pm_px_val {
    uint64_aligned_t freq;        /* Px core frequency */
    uint64_aligned_t residency;   /* Px residency time */
    uint64_aligned_t count;       /* Px transition count */
};
typedef struct pm_px_val pm_px_val_t;
DEFINE_XEN_GUEST_HANDLE(pm_px_val_t);

struct pm_px_stat {
    uint8_t total;        /* total Px states */
    uint8_t usable;       /* usable Px states */
    uint8_t last;         /* last Px state */
    uint8_t cur;          /* current Px state */
    XEN_GUEST_HANDLE_64(uint64) trans_pt;   /* Px transition table */
    XEN_GUEST_HANDLE_64(pm_px_val_t) pt;
};
typedef struct pm_px_stat pm_px_stat_t;
DEFINE_XEN_GUEST_HANDLE(pm_px_stat_t);

struct pm_cx_stat {
    uint32_t nr;    /* entry nr in triggers & residencies, including C0 */
    uint32_t last;  /* last Cx state */
    uint64_aligned_t idle_time;                 /* idle time from boot */
    XEN_GUEST_HANDLE_64(uint64) triggers;    /* Cx trigger counts */
    XEN_GUEST_HANDLE_64(uint64) residencies; /* Cx residencies */
};

struct xen_sysctl_get_pmstat {
#define PMSTAT_CATEGORY_MASK 0xf0
#define PMSTAT_PX            0x10
#define PMSTAT_CX            0x20
#define PMSTAT_get_max_px    (PMSTAT_PX | 0x1)
#define PMSTAT_get_pxstat    (PMSTAT_PX | 0x2)
#define PMSTAT_reset_pxstat  (PMSTAT_PX | 0x3)
#define PMSTAT_get_max_cx    (PMSTAT_CX | 0x1)
#define PMSTAT_get_cxstat    (PMSTAT_CX | 0x2)
#define PMSTAT_reset_cxstat  (PMSTAT_CX | 0x3)
    uint32_t type;
    uint32_t cpuid;
    union {
        struct pm_px_stat getpx;
        struct pm_cx_stat getcx;
        /* other struct for tx, etc */
    } u;
};
typedef struct xen_sysctl_get_pmstat xen_sysctl_get_pmstat_t;
DEFINE_XEN_GUEST_HANDLE(xen_sysctl_get_pmstat_t);

/*
 * Status codes. Must be greater than 0 to avoid confusing
 * sysctl callers that see 0 as a plain successful return.
 */
#define XEN_CPU_HOTPLUG_STATUS_OFFLINE 1
#define XEN_CPU_HOTPLUG_STATUS_ONLINE  2
#define XEN_CPU_HOTPLUG_STATUS_NEW     3

#define XEN_SYSCTL_cpu_hotplug       11
struct xen_sysctl_cpu_hotplug {
    /* IN variables */
    uint32_t cpu;   /* Physical cpu. */
#define XEN_SYSCTL_CPU_HOTPLUG_ONLINE  0
#define XEN_SYSCTL_CPU_HOTPLUG_OFFLINE 1
#define XEN_SYSCTL_CPU_HOTPLUG_STATUS 2
    uint32_t op;    /* hotplug opcode */
};
typedef struct xen_sysctl_cpu_hotplug xen_sysctl_cpu_hotplug_t;
DEFINE_XEN_GUEST_HANDLE(xen_sysctl_cpu_hotplug_t);

/*
 * Get/set xen power management, include
 * 1. cpufreq governors and related parameters
 */
#define XEN_SYSCTL_pm_op        12
struct xen_userspace {
    uint32_t scaling_setspeed;
};
typedef struct xen_userspace xen_userspace_t;

struct xen_ondemand {
    uint32_t sampling_rate_max;
    uint32_t sampling_rate_min;

    uint32_t sampling_rate;
    uint32_t up_threshold;
};
typedef struct xen_ondemand xen_ondemand_t;

/*
 * cpufreq para name of this structure named
 * same as sysfs file name of native linux
 */
#define CPUFREQ_NAME_LEN 16
struct xen_get_cpufreq_para {
    /* IN/OUT variable */
    uint32_t cpu_num;
    uint32_t freq_num;
    uint32_t gov_num;

    /* for all governors */
    /* OUT variable */
    XEN_GUEST_HANDLE_64(uint32) affected_cpus;
    XEN_GUEST_HANDLE_64(uint32) scaling_available_frequencies;
    XEN_GUEST_HANDLE_64(char)   scaling_available_governors;
    char scaling_driver[CPUFREQ_NAME_LEN];

    uint32_t cpuinfo_cur_freq;
    uint32_t cpuinfo_max_freq;
    uint32_t cpuinfo_min_freq;
    uint32_t scaling_cur_freq;

    char scaling_governor[CPUFREQ_NAME_LEN];
    uint32_t scaling_max_freq;
    uint32_t scaling_min_freq;

    /* for specific governor */
    union {
        struct  xen_userspace userspace;
        struct  xen_ondemand ondemand;
    } u;
};

struct xen_set_cpufreq_gov {
    char scaling_governor[CPUFREQ_NAME_LEN];
};

struct xen_set_cpufreq_para {
    #define SCALING_MAX_FREQ           1
    #define SCALING_MIN_FREQ           2
    #define SCALING_SETSPEED           3
    #define SAMPLING_RATE              4
    #define UP_THRESHOLD               5

    uint32_t ctrl_type;
    uint32_t ctrl_value;
};

/* Get physical CPU topology information. */
#define INVALID_TOPOLOGY_ID  (~0U)
struct xen_get_cputopo {
     /* IN: maximum addressable entry in
      * the caller-provided cpu_to_core/socket.
      */
    uint32_t max_cpus;
    XEN_GUEST_HANDLE_64(uint32) cpu_to_core;
    XEN_GUEST_HANDLE_64(uint32) cpu_to_socket;

    /* OUT: number of cpus returned
     * If OUT is greater than IN then the cpu_to_core/socket is truncated!
     */
    uint32_t nr_cpus;
};

struct xen_sysctl_pm_op {
    #define PM_PARA_CATEGORY_MASK      0xf0
    #define CPUFREQ_PARA               0x10

    /* cpufreq command type */
    #define GET_CPUFREQ_PARA           (CPUFREQ_PARA | 0x01)
    #define SET_CPUFREQ_GOV            (CPUFREQ_PARA | 0x02)
    #define SET_CPUFREQ_PARA           (CPUFREQ_PARA | 0x03)
    #define GET_CPUFREQ_AVGFREQ        (CPUFREQ_PARA | 0x04)

    /* get CPU topology */
    #define XEN_SYSCTL_pm_op_get_cputopo  0x20

    /* set/reset scheduler power saving option */
    #define XEN_SYSCTL_pm_op_set_sched_opt_smt    0x21

    /* cpuidle max_cstate access command */
    #define XEN_SYSCTL_pm_op_get_max_cstate       0x22
    #define XEN_SYSCTL_pm_op_set_max_cstate       0x23

    /* set scheduler migration cost value */
    #define XEN_SYSCTL_pm_op_set_vcpu_migration_delay   0x24
    #define XEN_SYSCTL_pm_op_get_vcpu_migration_delay   0x25

    uint32_t cmd;
    uint32_t cpuid;
    union {
        struct xen_get_cpufreq_para get_para;
        struct xen_set_cpufreq_gov  set_gov;
        struct xen_set_cpufreq_para set_para;
        uint64_t get_avgfreq;
        struct xen_get_cputopo      get_topo;
        uint32_t                    set_sched_opt_smt;
        uint32_t                    get_max_cstate;
        uint32_t                    set_max_cstate;
        uint32_t                    get_vcpu_migration_delay;
        uint32_t                    set_vcpu_migration_delay;
    } u;
};

#define XEN_SYSCTL_page_offline_op        14
struct xen_sysctl_page_offline_op {
    /* IN: range of page to be offlined */
#define sysctl_page_offline     1
#define sysctl_page_online      2
#define sysctl_query_page_offline  3
    uint32_t cmd;
    uint32_t start;
    uint32_t end;
    /* OUT: result of page offline request */
    /*
     * bit 0~15: result flags
     * bit 16~31: owner
     */
    XEN_GUEST_HANDLE(uint32) status;
};

#define PG_OFFLINE_STATUS_MASK    (0xFFUL)

/* The result is invalid, i.e. HV does not handle it */
#define PG_OFFLINE_INVALID   (0x1UL << 0)

#define PG_OFFLINE_OFFLINED  (0x1UL << 1)
#define PG_OFFLINE_PENDING   (0x1UL << 2)
#define PG_OFFLINE_FAILED    (0x1UL << 3)

#define PG_ONLINE_FAILED     PG_OFFLINE_FAILED
#define PG_ONLINE_ONLINED    PG_OFFLINE_OFFLINED

#define PG_OFFLINE_STATUS_OFFLINED              (0x1UL << 1)
#define PG_OFFLINE_STATUS_ONLINE                (0x1UL << 2)
#define PG_OFFLINE_STATUS_OFFLINE_PENDING       (0x1UL << 3)
#define PG_OFFLINE_STATUS_BROKEN                (0x1UL << 4)

#define PG_OFFLINE_MISC_MASK    (0xFFUL << 4)

/* only valid when PG_OFFLINE_FAILED */
#define PG_OFFLINE_XENPAGE   (0x1UL << 8)
#define PG_OFFLINE_DOM0PAGE  (0x1UL << 9)
#define PG_OFFLINE_ANONYMOUS (0x1UL << 10)
#define PG_OFFLINE_NOT_CONV_RAM   (0x1UL << 11)
#define PG_OFFLINE_OWNED     (0x1UL << 12)

#define PG_OFFLINE_BROKEN    (0x1UL << 13)
#define PG_ONLINE_BROKEN     PG_OFFLINE_BROKEN

#define PG_OFFLINE_OWNER_SHIFT 16

struct xen_sysctl {
    uint32_t cmd;
    uint32_t interface_version; /* XEN_SYSCTL_INTERFACE_VERSION */
    union {
        struct xen_sysctl_readconsole       readconsole;
        struct xen_sysctl_tbuf_op           tbuf_op;
        struct xen_sysctl_physinfo          physinfo;
        struct xen_sysctl_sched_id          sched_id;
        struct xen_sysctl_perfc_op          perfc_op;
        struct xen_sysctl_getdomaininfolist getdomaininfolist;
        struct xen_sysctl_debug_keys        debug_keys;
        struct xen_sysctl_getcpuinfo        getcpuinfo;
        struct xen_sysctl_availheap         availheap;
        struct xen_sysctl_get_pmstat        get_pmstat;
        struct xen_sysctl_cpu_hotplug       cpu_hotplug;
        struct xen_sysctl_pm_op             pm_op;
        struct xen_sysctl_page_offline_op   page_offline;
        uint8_t                             pad[128];
    } u;
};
typedef struct xen_sysctl xen_sysctl_t;
DEFINE_XEN_GUEST_HANDLE(xen_sysctl_t);

#endif /* __XEN_PUBLIC_SYSCTL_H__ */

/*
 * Local variables:
 * mode: C
 * c-set-style: "BSD"
 * c-basic-offset: 4
 * tab-width: 4
 * indent-tabs-mode: nil
 * End:
 */
/******************************************************************************
 * include/public/trace.h
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to
 * deal in the Software without restriction, including without limitation the
 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
 * sell copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 * DEALINGS IN THE SOFTWARE.
 *
 * Mark Williamson, (C) 2004 Intel Research Cambridge
 * Copyright (C) 2005 Bin Ren
 */

#ifndef __XEN_PUBLIC_TRACE_H__
#define __XEN_PUBLIC_TRACE_H__

#define TRACE_EXTRA_MAX    7
#define TRACE_EXTRA_SHIFT 28

/* Trace classes */
#define TRC_CLS_SHIFT 16
#define TRC_GEN      0x0001f000    /* General trace            */
#define TRC_SCHED    0x0002f000    /* Xen Scheduler trace      */
#define TRC_DOM0OP   0x0004f000    /* Xen DOM0 operation trace */
#define TRC_HVM      0x0008f000    /* Xen HVM trace            */
#define TRC_MEM      0x0010f000    /* Xen memory trace         */
#define TRC_PV       0x0020f000    /* Xen PV traces            */
#define TRC_SHADOW   0x0040f000    /* Xen shadow tracing       */
#define TRC_PM       0x0080f000    /* Xen power management trace */
#define TRC_ALL      0x0ffff000
#define TRC_HD_TO_EVENT(x) ((x)&0x0fffffff)
#define TRC_HD_CYCLE_FLAG (1UL<<31)
#define TRC_HD_INCLUDES_CYCLE_COUNT(x) ( !!( (x) & TRC_HD_CYCLE_FLAG ) )
#define TRC_HD_EXTRA(x)    (((x)>>TRACE_EXTRA_SHIFT)&TRACE_EXTRA_MAX)

/* Trace subclasses */
#define TRC_SUBCLS_SHIFT 12

/* trace subclasses for SVM */
#define TRC_HVM_ENTRYEXIT 0x00081000   /* VMENTRY and #VMEXIT       */
#define TRC_HVM_HANDLER   0x00082000   /* various HVM handlers      */

#define TRC_SCHED_MIN       0x00021000   /* Just runstate changes */
#define TRC_SCHED_VERBOSE   0x00028000   /* More inclusive scheduling */

/* Trace events per class */
#define TRC_LOST_RECORDS        (TRC_GEN + 1)
#define TRC_TRACE_WRAP_BUFFER  (TRC_GEN + 2)
#define TRC_TRACE_CPU_CHANGE    (TRC_GEN + 3)

#define TRC_SCHED_RUNSTATE_CHANGE   (TRC_SCHED_MIN + 1)
#define TRC_SCHED_CONTINUE_RUNNING  (TRC_SCHED_MIN + 2)
#define TRC_SCHED_DOM_ADD        (TRC_SCHED_VERBOSE +  1)
#define TRC_SCHED_DOM_REM        (TRC_SCHED_VERBOSE +  2)
#define TRC_SCHED_SLEEP          (TRC_SCHED_VERBOSE +  3)
#define TRC_SCHED_WAKE           (TRC_SCHED_VERBOSE +  4)
#define TRC_SCHED_YIELD          (TRC_SCHED_VERBOSE +  5)
#define TRC_SCHED_BLOCK          (TRC_SCHED_VERBOSE +  6)
#define TRC_SCHED_SHUTDOWN       (TRC_SCHED_VERBOSE +  7)
#define TRC_SCHED_CTL            (TRC_SCHED_VERBOSE +  8)
#define TRC_SCHED_ADJDOM         (TRC_SCHED_VERBOSE +  9)
#define TRC_SCHED_SWITCH         (TRC_SCHED_VERBOSE + 10)
#define TRC_SCHED_S_TIMER_FN     (TRC_SCHED_VERBOSE + 11)
#define TRC_SCHED_T_TIMER_FN     (TRC_SCHED_VERBOSE + 12)
#define TRC_SCHED_DOM_TIMER_FN   (TRC_SCHED_VERBOSE + 13)
#define TRC_SCHED_SWITCH_INFPREV (TRC_SCHED_VERBOSE + 14)
#define TRC_SCHED_SWITCH_INFNEXT (TRC_SCHED_VERBOSE + 15)

#define TRC_MEM_PAGE_GRANT_MAP      (TRC_MEM + 1)
#define TRC_MEM_PAGE_GRANT_UNMAP    (TRC_MEM + 2)
#define TRC_MEM_PAGE_GRANT_TRANSFER (TRC_MEM + 3)

#define TRC_PV_HYPERCALL             (TRC_PV +  1)
#define TRC_PV_TRAP                  (TRC_PV +  3)
#define TRC_PV_PAGE_FAULT            (TRC_PV +  4)
#define TRC_PV_FORCED_INVALID_OP     (TRC_PV +  5)
#define TRC_PV_EMULATE_PRIVOP        (TRC_PV +  6)
#define TRC_PV_EMULATE_4GB           (TRC_PV +  7)
#define TRC_PV_MATH_STATE_RESTORE    (TRC_PV +  8)
#define TRC_PV_PAGING_FIXUP          (TRC_PV +  9)
#define TRC_PV_GDT_LDT_MAPPING_FAULT (TRC_PV + 10)
#define TRC_PV_PTWR_EMULATION        (TRC_PV + 11)
#define TRC_PV_PTWR_EMULATION_PAE    (TRC_PV + 12)
  /* Indicates that addresses in trace record are 64 bits */
#define TRC_64_FLAG               (0x100)

#define TRC_SHADOW_NOT_SHADOW                 (TRC_SHADOW +  1)
#define TRC_SHADOW_FAST_PROPAGATE             (TRC_SHADOW +  2)
#define TRC_SHADOW_FAST_MMIO                  (TRC_SHADOW +  3)
#define TRC_SHADOW_FALSE_FAST_PATH            (TRC_SHADOW +  4)
#define TRC_SHADOW_MMIO                       (TRC_SHADOW +  5)
#define TRC_SHADOW_FIXUP                      (TRC_SHADOW +  6)
#define TRC_SHADOW_DOMF_DYING                 (TRC_SHADOW +  7)
#define TRC_SHADOW_EMULATE                    (TRC_SHADOW +  8)
#define TRC_SHADOW_EMULATE_UNSHADOW_USER      (TRC_SHADOW +  9)
#define TRC_SHADOW_EMULATE_UNSHADOW_EVTINJ    (TRC_SHADOW + 10)
#define TRC_SHADOW_EMULATE_UNSHADOW_UNHANDLED (TRC_SHADOW + 11)
#define TRC_SHADOW_WRMAP_BF                   (TRC_SHADOW + 12)
#define TRC_SHADOW_PREALLOC_UNPIN             (TRC_SHADOW + 13)
#define TRC_SHADOW_RESYNC_FULL                (TRC_SHADOW + 14)
#define TRC_SHADOW_RESYNC_ONLY                (TRC_SHADOW + 15)

/* trace events per subclass */
#define TRC_HVM_VMENTRY         (TRC_HVM_ENTRYEXIT + 0x01)
#define TRC_HVM_VMEXIT          (TRC_HVM_ENTRYEXIT + 0x02)
#define TRC_HVM_VMEXIT64        (TRC_HVM_ENTRYEXIT + TRC_64_FLAG + 0x02)
#define TRC_HVM_PF_XEN          (TRC_HVM_HANDLER + 0x01)
#define TRC_HVM_PF_XEN64        (TRC_HVM_HANDLER + TRC_64_FLAG + 0x01)
#define TRC_HVM_PF_INJECT       (TRC_HVM_HANDLER + 0x02)
#define TRC_HVM_PF_INJECT64     (TRC_HVM_HANDLER + TRC_64_FLAG + 0x02)
#define TRC_HVM_INJ_EXC         (TRC_HVM_HANDLER + 0x03)
#define TRC_HVM_INJ_VIRQ        (TRC_HVM_HANDLER + 0x04)
#define TRC_HVM_REINJ_VIRQ      (TRC_HVM_HANDLER + 0x05)
#define TRC_HVM_IO_READ         (TRC_HVM_HANDLER + 0x06)
#define TRC_HVM_IO_WRITE        (TRC_HVM_HANDLER + 0x07)
#define TRC_HVM_CR_READ         (TRC_HVM_HANDLER + 0x08)
#define TRC_HVM_CR_READ64       (TRC_HVM_HANDLER + TRC_64_FLAG + 0x08)
#define TRC_HVM_CR_WRITE        (TRC_HVM_HANDLER + 0x09)
#define TRC_HVM_CR_WRITE64      (TRC_HVM_HANDLER + TRC_64_FLAG + 0x09)
#define TRC_HVM_DR_READ         (TRC_HVM_HANDLER + 0x0A)
#define TRC_HVM_DR_WRITE        (TRC_HVM_HANDLER + 0x0B)
#define TRC_HVM_MSR_READ        (TRC_HVM_HANDLER + 0x0C)
#define TRC_HVM_MSR_WRITE       (TRC_HVM_HANDLER + 0x0D)
#define TRC_HVM_CPUID           (TRC_HVM_HANDLER + 0x0E)
#define TRC_HVM_INTR            (TRC_HVM_HANDLER + 0x0F)
#define TRC_HVM_NMI             (TRC_HVM_HANDLER + 0x10)
#define TRC_HVM_SMI             (TRC_HVM_HANDLER + 0x11)
#define TRC_HVM_VMMCALL         (TRC_HVM_HANDLER + 0x12)
#define TRC_HVM_HLT             (TRC_HVM_HANDLER + 0x13)
#define TRC_HVM_INVLPG          (TRC_HVM_HANDLER + 0x14)
#define TRC_HVM_INVLPG64        (TRC_HVM_HANDLER + TRC_64_FLAG + 0x14)
#define TRC_HVM_MCE             (TRC_HVM_HANDLER + 0x15)
#define TRC_HVM_IOPORT_READ     (TRC_HVM_HANDLER + 0x16)
#define TRC_HVM_IOMEM_READ      (TRC_HVM_HANDLER + 0x17)
#define TRC_HVM_CLTS            (TRC_HVM_HANDLER + 0x18)
#define TRC_HVM_LMSW            (TRC_HVM_HANDLER + 0x19)
#define TRC_HVM_LMSW64          (TRC_HVM_HANDLER + TRC_64_FLAG + 0x19)
#define TRC_HVM_INTR_WINDOW     (TRC_HVM_HANDLER + 0x20)
#define TRC_HVM_IOPORT_WRITE    (TRC_HVM_HANDLER + 0x216)
#define TRC_HVM_IOMEM_WRITE     (TRC_HVM_HANDLER + 0x217)

/* trace subclasses for power management */
#define TRC_PM_FREQ     0x00801000      /* xen cpu freq events */
#define TRC_PM_IDLE     0x00802000      /* xen cpu idle events */

/* trace events for per class */
#define TRC_PM_FREQ_CHANGE      (TRC_PM_FREQ + 0x01)
#define TRC_PM_IDLE_ENTRY       (TRC_PM_IDLE + 0x01)
#define TRC_PM_IDLE_EXIT        (TRC_PM_IDLE + 0x02)

/* This structure represents a single trace buffer record. */
struct t_rec {
    uint32_t event:28;
    uint32_t extra_u32:3;         /* # entries in trailing extra_u32[] array */
    uint32_t cycles_included:1;   /* u.cycles or u.no_cycles? */
    union {
        struct {
            uint32_t cycles_lo, cycles_hi; /* cycle counter timestamp */
            uint32_t extra_u32[7];         /* event data items */
        } cycles;
        struct {
            uint32_t extra_u32[7];         /* event data items */
        } nocycles;
    } u;
};

/*
 * This structure contains the metadata for a single trace buffer.  The head
 * field, indexes into an array of struct t_rec's.
 */
struct t_buf {
    /* Assume the data buffer size is X.  X is generally not a power of 2.
     * CONS and PROD are incremented modulo (2*X):
     *     0 <= cons < 2*X
     *     0 <= prod < 2*X
     * This is done because addition modulo X breaks at 2^32 when X is not a
     * power of 2:
     *     (((2^32 - 1) % X) + 1) % X != (2^32) % X
     */
    uint32_t cons;   /* Offset of next item to be consumed by control tools. */
    uint32_t prod;   /* Offset of next item to be produced by Xen.           */
    /*  Records follow immediately after the meta-data header.    */
};

#endif /* __XEN_PUBLIC_TRACE_H__ */

/*
 * Local variables:
 * mode: C
 * c-set-style: "BSD"
 * c-basic-offset: 4
 * tab-width: 4
 * indent-tabs-mode: nil
 * End:
 */
/******************************************************************************
 * vcpu.h
 *
 * VCPU initialisation, query, and hotplug.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to
 * deal in the Software without restriction, including without limitation the
 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
 * sell copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 * DEALINGS IN THE SOFTWARE.
 *
 * Copyright (c) 2005, Keir Fraser <keir@xensource.com>
 */

#ifndef __XEN_PUBLIC_VCPU_H__
#define __XEN_PUBLIC_VCPU_H__

/*
 * Prototype for this hypercall is:
 *  int vcpu_op(int cmd, int vcpuid, void *extra_args)
 * @cmd        == VCPUOP_??? (VCPU operation).
 * @vcpuid     == VCPU to operate on.
 * @extra_args == Operation-specific extra arguments (NULL if none).
 */

/*
 * Initialise a VCPU. Each VCPU can be initialised only once. A
 * newly-initialised VCPU will not run until it is brought up by VCPUOP_up.
 *
 * @extra_arg == pointer to vcpu_guest_context structure containing initial
 *               state for the VCPU.
 */
#define VCPUOP_initialise            0

/*
 * Bring up a VCPU. This makes the VCPU runnable. This operation will fail
 * if the VCPU has not been initialised (VCPUOP_initialise).
 */
#define VCPUOP_up                    1

/*
 * Bring down a VCPU (i.e., make it non-runnable).
 * There are a few caveats that callers should observe:
 *  1. This operation may return, and VCPU_is_up may return false, before the
 *     VCPU stops running (i.e., the command is asynchronous). It is a good
 *     idea to ensure that the VCPU has entered a non-critical loop before
 *     bringing it down. Alternatively, this operation is guaranteed
 *     synchronous if invoked by the VCPU itself.
 *  2. After a VCPU is initialised, there is currently no way to drop all its
 *     references to domain memory. Even a VCPU that is down still holds
 *     memory references via its pagetable base pointer and GDT. It is good
 *     practise to move a VCPU onto an 'idle' or default page table, LDT and
 *     GDT before bringing it down.
 */
#define VCPUOP_down                  2

/* Returns 1 if the given VCPU is up. */
#define VCPUOP_is_up                 3

/*
 * Return information about the state and running time of a VCPU.
 * @extra_arg == pointer to vcpu_runstate_info structure.
 */
#define VCPUOP_get_runstate_info     4
struct vcpu_runstate_info {
    /* VCPU's current state (RUNSTATE_*). */
    int      state;
    /* When was current state entered (system time, ns)? */
    uint64_t state_entry_time;
    /*
     * Time spent in each RUNSTATE_* (ns). The sum of these times is
     * guaranteed not to drift from system time.
     */
    uint64_t time[4];
};
typedef struct vcpu_runstate_info vcpu_runstate_info_t;
DEFINE_XEN_GUEST_HANDLE(vcpu_runstate_info_t);

/* VCPU is currently running on a physical CPU. */
#define RUNSTATE_running  0

/* VCPU is runnable, but not currently scheduled on any physical CPU. */
#define RUNSTATE_runnable 1

/* VCPU is blocked (a.k.a. idle). It is therefore not runnable. */
#define RUNSTATE_blocked  2

/*
 * VCPU is not runnable, but it is not blocked.
 * This is a 'catch all' state for things like hotplug and pauses by the
 * system administrator (or for critical sections in the hypervisor).
 * RUNSTATE_blocked dominates this state (it is the preferred state).
 */
#define RUNSTATE_offline  3

/*
 * Register a shared memory area from which the guest may obtain its own
 * runstate information without needing to execute a hypercall.
 * Notes:
 *  1. The registered address may be virtual or physical or guest handle,
 *     depending on the platform. Virtual address or guest handle should be
 *     registered on x86 systems.
 *  2. Only one shared area may be registered per VCPU. The shared area is
 *     updated by the hypervisor each time the VCPU is scheduled. Thus
 *     runstate.state will always be RUNSTATE_running and
 *     runstate.state_entry_time will indicate the system time at which the
 *     VCPU was last scheduled to run.
 * @extra_arg == pointer to vcpu_register_runstate_memory_area structure.
 */
#define VCPUOP_register_runstate_memory_area 5
struct vcpu_register_runstate_memory_area {
    union {
        XEN_GUEST_HANDLE(vcpu_runstate_info_t) h;
        struct vcpu_runstate_info *v;
        uint64_t p;
    } addr;
};
typedef struct vcpu_register_runstate_memory_area vcpu_register_runstate_memory_area_t;
DEFINE_XEN_GUEST_HANDLE(vcpu_register_runstate_memory_area_t);

/*
 * Set or stop a VCPU's periodic timer. Every VCPU has one periodic timer
 * which can be set via these commands. Periods smaller than one millisecond
 * may not be supported.
 */
#define VCPUOP_set_periodic_timer    6 /* arg == vcpu_set_periodic_timer_t */
#define VCPUOP_stop_periodic_timer   7 /* arg == NULL */
struct vcpu_set_periodic_timer {
    uint64_t period_ns;
};
typedef struct vcpu_set_periodic_timer vcpu_set_periodic_timer_t;
DEFINE_XEN_GUEST_HANDLE(vcpu_set_periodic_timer_t);

/*
 * Set or stop a VCPU's single-shot timer. Every VCPU has one single-shot
 * timer which can be set via these commands.
 */
#define VCPUOP_set_singleshot_timer  8 /* arg == vcpu_set_singleshot_timer_t */
#define VCPUOP_stop_singleshot_timer 9 /* arg == NULL */
struct vcpu_set_singleshot_timer {
    uint64_t timeout_abs_ns;   /* Absolute system time value in nanoseconds. */
    uint32_t flags;            /* VCPU_SSHOTTMR_??? */
};
typedef struct vcpu_set_singleshot_timer vcpu_set_singleshot_timer_t;
DEFINE_XEN_GUEST_HANDLE(vcpu_set_singleshot_timer_t);

/* Flags to VCPUOP_set_singleshot_timer. */
 /* Require the timeout to be in the future (return -ETIME if it's passed). */
#define _VCPU_SSHOTTMR_future (0)
#define VCPU_SSHOTTMR_future  (1U << _VCPU_SSHOTTMR_future)

/*
 * Register a memory location in the guest address space for the
 * vcpu_info structure.  This allows the guest to place the vcpu_info
 * structure in a convenient place, such as in a per-cpu data area.
 * The pointer need not be page aligned, but the structure must not
 * cross a page boundary.
 *
 * This may be called only once per vcpu.
 */
#define VCPUOP_register_vcpu_info   10  /* arg == vcpu_register_vcpu_info_t */
struct vcpu_register_vcpu_info {
    uint64_t mfn;    /* mfn of page to place vcpu_info */
    uint32_t offset; /* offset within page */
    uint32_t rsvd;   /* unused */
};
typedef struct vcpu_register_vcpu_info vcpu_register_vcpu_info_t;
DEFINE_XEN_GUEST_HANDLE(vcpu_register_vcpu_info_t);

/* Send an NMI to the specified VCPU. @extra_arg == NULL. */
#define VCPUOP_send_nmi             11

/*
 * Get the physical ID information for a pinned vcpu's underlying physical
 * processor.  The physical ID informmation is architecture-specific.
 * On x86: id[31:0]=apic_id, id[63:32]=acpi_id, and all values 0xff and
 *         greater are reserved.
 * This command returns -EINVAL if it is not a valid operation for this VCPU.
 */
#define VCPUOP_get_physid           12 /* arg == vcpu_get_physid_t */
struct vcpu_get_physid {
    uint64_t phys_id;
};
typedef struct vcpu_get_physid vcpu_get_physid_t;
DEFINE_XEN_GUEST_HANDLE(vcpu_get_physid_t);
#define xen_vcpu_physid_to_x86_apicid(physid) \
    ((((uint32_t)(physid)) >= 0xff) ? 0xff : ((uint8_t)(physid)))
#define xen_vcpu_physid_to_x86_acpiid(physid) \
    ((((uint32_t)((physid)>>32)) >= 0xff) ? 0xff : ((uint8_t)((physid)>>32)))

#endif /* __XEN_PUBLIC_VCPU_H__ */

/*
 * Local variables:
 * mode: C
 * c-set-style: "BSD"
 * c-basic-offset: 4
 * tab-width: 4
 * indent-tabs-mode: nil
 * End:
 */
/******************************************************************************
 * version.h
 *
 * Xen version, type, and compile information.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to
 * deal in the Software without restriction, including without limitation the
 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
 * sell copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 * DEALINGS IN THE SOFTWARE.
 *
 * Copyright (c) 2005, Nguyen Anh Quynh <aquynh@gmail.com>
 * Copyright (c) 2005, Keir Fraser <keir@xensource.com>
 */

#ifndef __XEN_PUBLIC_VERSION_H__
#define __XEN_PUBLIC_VERSION_H__

/* NB. All ops return zero on success, except XENVER_{version,pagesize} */

/* arg == NULL; returns major:minor (16:16). */
#define XENVER_version      0

/* arg == xen_extraversion_t. */
#define XENVER_extraversion 1
typedef char xen_extraversion_t[16];
#define XEN_EXTRAVERSION_LEN (sizeof(xen_extraversion_t))

/* arg == xen_compile_info_t. */
#define XENVER_compile_info 2
struct xen_compile_info {
    char compiler[64];
    char compile_by[16];
    char compile_domain[32];
    char compile_date[32];
};
typedef struct xen_compile_info xen_compile_info_t;

#define XENVER_capabilities 3
typedef char xen_capabilities_info_t[1024];
#define XEN_CAPABILITIES_INFO_LEN (sizeof(xen_capabilities_info_t))

#define XENVER_changeset 4
typedef char xen_changeset_info_t[64];
#define XEN_CHANGESET_INFO_LEN (sizeof(xen_changeset_info_t))

#define XENVER_platform_parameters 5
struct xen_platform_parameters {
    unsigned long virt_start;
};
typedef struct xen_platform_parameters xen_platform_parameters_t;

#define XENVER_get_features 6
struct xen_feature_info {
    unsigned int submap_idx;    /* IN: which 32-bit submap to return */
    uint32_t     submap;        /* OUT: 32-bit submap */
};
typedef struct xen_feature_info xen_feature_info_t;

/* Declares the features reported by XENVER_get_features. */
#include "features.h"

/* arg == NULL; returns host memory page size. */
#define XENVER_pagesize 7

/* arg == xen_domain_handle_t. */
#define XENVER_guest_handle 8

#endif /* __XEN_PUBLIC_VERSION_H__ */

/*
 * Local variables:
 * mode: C
 * c-set-style: "BSD"
 * c-basic-offset: 4
 * tab-width: 4
 * indent-tabs-mode: nil
 * End:
 */
/******************************************************************************
 * xen-compat.h
 *
 * Guest OS interface to Xen.  Compatibility layer.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to
 * deal in the Software without restriction, including without limitation the
 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
 * sell copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 * DEALINGS IN THE SOFTWARE.
 *
 * Copyright (c) 2006, Christian Limpach
 */

#ifndef __XEN_PUBLIC_XEN_COMPAT_H__
#define __XEN_PUBLIC_XEN_COMPAT_H__

#define __XEN_LATEST_INTERFACE_VERSION__ 0x00030209

#if defined(__XEN__) || defined(__XEN_TOOLS__)
/* Xen is built with matching headers and implements the latest interface. */
#define __XEN_INTERFACE_VERSION__ __XEN_LATEST_INTERFACE_VERSION__
#elif !defined(__XEN_INTERFACE_VERSION__)
/* Guests which do not specify a version get the legacy interface. */
#define __XEN_INTERFACE_VERSION__ 0x00000000
#endif

#if __XEN_INTERFACE_VERSION__ > __XEN_LATEST_INTERFACE_VERSION__
#error "These header files do not support the requested interface version."
#endif

#endif /* __XEN_PUBLIC_XEN_COMPAT_H__ */
/******************************************************************************
 * xen.h
 *
 * Guest OS interface to Xen.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to
 * deal in the Software without restriction, including without limitation the
 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
 * sell copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 * DEALINGS IN THE SOFTWARE.
 *
 * Copyright (c) 2004, K A Fraser
 */

#ifndef __XEN_PUBLIC_XEN_H__
#define __XEN_PUBLIC_XEN_H__

#include "xen-compat.h"

#if defined(__i386) && !defined(__i386__)
#define __i386__ /* foo */
#endif

#if defined(__amd64) && !defined(__x86_64__)
#define __x86_64__
#endif

#if defined(_ASM) && !defined(__ASSEMBLY__)
#define __ASSEMBLY__
#endif

#if defined(__i386__) || defined(__x86_64__)
#include "arch-x86/xen.h"
#elif defined(__ia64__)
#include "arch-ia64.h"
#else
#error "Unsupported architecture"
#endif

#ifndef __ASSEMBLY__
/* Guest handles for primitive C types. */
DEFINE_XEN_GUEST_HANDLE(char);
__DEFINE_XEN_GUEST_HANDLE(uchar, unsigned char);
DEFINE_XEN_GUEST_HANDLE(int);
__DEFINE_XEN_GUEST_HANDLE(uint,  unsigned int);
DEFINE_XEN_GUEST_HANDLE(long);
__DEFINE_XEN_GUEST_HANDLE(ulong, unsigned long);
DEFINE_XEN_GUEST_HANDLE(void);

DEFINE_XEN_GUEST_HANDLE(xen_pfn_t);
#endif

/*
 * HYPERCALLS
 */

#define __HYPERVISOR_set_trap_table        0
#define __HYPERVISOR_mmu_update            1
#define __HYPERVISOR_set_gdt               2
#define __HYPERVISOR_stack_switch          3
#define __HYPERVISOR_set_callbacks         4
#define __HYPERVISOR_fpu_taskswitch        5
#define __HYPERVISOR_sched_op_compat       6 /* compat since 0x00030101 */
#define __HYPERVISOR_platform_op           7
#define __HYPERVISOR_set_debugreg          8
#define __HYPERVISOR_get_debugreg          9
#define __HYPERVISOR_update_descriptor    10
#define __HYPERVISOR_memory_op            12
#define __HYPERVISOR_multicall            13
#define __HYPERVISOR_update_va_mapping    14
#define __HYPERVISOR_set_timer_op         15
#define __HYPERVISOR_event_channel_op_compat 16 /* compat since 0x00030202 */
#define __HYPERVISOR_xen_version          17
#define __HYPERVISOR_console_io           18
#define __HYPERVISOR_physdev_op_compat    19 /* compat since 0x00030202 */
#define __HYPERVISOR_grant_table_op       20
#define __HYPERVISOR_vm_assist            21
#define __HYPERVISOR_update_va_mapping_otherdomain 22
#define __HYPERVISOR_iret                 23 /* x86 only */
#define __HYPERVISOR_vcpu_op              24
#define __HYPERVISOR_set_segment_base     25 /* x86/64 only */
#define __HYPERVISOR_mmuext_op            26
#define __HYPERVISOR_xsm_op               27
#define __HYPERVISOR_nmi_op               28
#define __HYPERVISOR_sched_op             29
#define __HYPERVISOR_callback_op          30
#define __HYPERVISOR_xenoprof_op          31
#define __HYPERVISOR_event_channel_op     32
#define __HYPERVISOR_physdev_op           33
#define __HYPERVISOR_hvm_op               34
#define __HYPERVISOR_sysctl               35
#define __HYPERVISOR_domctl               36
#define __HYPERVISOR_kexec_op             37

/* Architecture-specific hypercall definitions. */
#define __HYPERVISOR_arch_0               48
#define __HYPERVISOR_arch_1               49
#define __HYPERVISOR_arch_2               50
#define __HYPERVISOR_arch_3               51
#define __HYPERVISOR_arch_4               52
#define __HYPERVISOR_arch_5               53
#define __HYPERVISOR_arch_6               54
#define __HYPERVISOR_arch_7               55

/*
 * HYPERCALL COMPATIBILITY.
 */

/* New sched_op hypercall introduced in 0x00030101. */
#if __XEN_INTERFACE_VERSION__ < 0x00030101
#undef __HYPERVISOR_sched_op
#define __HYPERVISOR_sched_op __HYPERVISOR_sched_op_compat
#endif

/* New event-channel and physdev hypercalls introduced in 0x00030202. */
#if __XEN_INTERFACE_VERSION__ < 0x00030202
#undef __HYPERVISOR_event_channel_op
#define __HYPERVISOR_event_channel_op __HYPERVISOR_event_channel_op_compat
#undef __HYPERVISOR_physdev_op
#define __HYPERVISOR_physdev_op __HYPERVISOR_physdev_op_compat
#endif

/* New platform_op hypercall introduced in 0x00030204. */
#if __XEN_INTERFACE_VERSION__ < 0x00030204
#define __HYPERVISOR_dom0_op __HYPERVISOR_platform_op
#endif

/*
 * VIRTUAL INTERRUPTS
 *
 * Virtual interrupts that a guest OS may receive from Xen.
 *
 * In the side comments, 'V.' denotes a per-VCPU VIRQ while 'G.' denotes a
 * global VIRQ. The former can be bound once per VCPU and cannot be re-bound.
 * The latter can be allocated only once per guest: they must initially be
 * allocated to VCPU0 but can subsequently be re-bound.
 */
#define VIRQ_TIMER      0  /* V. Timebase update, and/or requested timeout.  */
#define VIRQ_DEBUG      1  /* V. Request guest to dump debug info.           */
#define VIRQ_CONSOLE    2  /* G. (DOM0) Bytes received on emergency console. */
#define VIRQ_DOM_EXC    3  /* G. (DOM0) Exceptional event for some domain.   */
#define VIRQ_TBUF       4  /* G. (DOM0) Trace buffer has records available.  */
#define VIRQ_DEBUGGER   6  /* G. (DOM0) A domain has paused for debugging.   */
#define VIRQ_XENOPROF   7  /* V. XenOprofile interrupt: new sample available */
#define VIRQ_CON_RING   8  /* G. (DOM0) Bytes received on console            */

/* Architecture-specific VIRQ definitions. */
#define VIRQ_ARCH_0    16
#define VIRQ_ARCH_1    17
#define VIRQ_ARCH_2    18
#define VIRQ_ARCH_3    19
#define VIRQ_ARCH_4    20
#define VIRQ_ARCH_5    21
#define VIRQ_ARCH_6    22
#define VIRQ_ARCH_7    23

#define NR_VIRQS       24

/*
 * MMU-UPDATE REQUESTS
 *
 * HYPERVISOR_mmu_update() accepts a list of (ptr, val) pairs.
 * A foreigndom (FD) can be specified (or DOMID_SELF for none).
 * Where the FD has some effect, it is described below.
 * ptr[1:0] specifies the appropriate MMU_* command.
 *
 * ptr[1:0] == MMU_NORMAL_PT_UPDATE:
 * Updates an entry in a page table. If updating an L1 table, and the new
 * table entry is valid/present, the mapped frame must belong to the FD, if
 * an FD has been specified. If attempting to map an I/O page then the
 * caller assumes the privilege of the FD.
 * FD == DOMID_IO: Permit /only/ I/O mappings, at the priv level of the caller.
 * FD == DOMID_XEN: Map restricted areas of Xen's heap space.
 * ptr[:2]  -- Machine address of the page-table entry to modify.
 * val      -- Value to write.
 *
 * ptr[1:0] == MMU_MACHPHYS_UPDATE:
 * Updates an entry in the machine->pseudo-physical mapping table.
 * ptr[:2]  -- Machine address within the frame whose mapping to modify.
 *             The frame must belong to the FD, if one is specified.
 * val      -- Value to write into the mapping entry.
 *
 * ptr[1:0] == MMU_PT_UPDATE_PRESERVE_AD:
 * As MMU_NORMAL_PT_UPDATE above, but A/D bits currently in the PTE are ORed
 * with those in @val.
 */
#define MMU_NORMAL_PT_UPDATE      0 /* checked '*ptr = val'. ptr is MA.      */
#define MMU_MACHPHYS_UPDATE       1 /* ptr = MA of frame to modify entry for */
#define MMU_PT_UPDATE_PRESERVE_AD 2 /* atomically: *ptr = val | (*ptr&(A|D)) */

/*
 * MMU EXTENDED OPERATIONS
 *
 * HYPERVISOR_mmuext_op() accepts a list of mmuext_op structures.
 * A foreigndom (FD) can be specified (or DOMID_SELF for none).
 * Where the FD has some effect, it is described below.
 *
 * cmd: MMUEXT_(UN)PIN_*_TABLE
 * mfn: Machine frame number to be (un)pinned as a p.t. page.
 *      The frame must belong to the FD, if one is specified.
 *
 * cmd: MMUEXT_NEW_BASEPTR
 * mfn: Machine frame number of new page-table base to install in MMU.
 *
 * cmd: MMUEXT_NEW_USER_BASEPTR [x86/64 only]
 * mfn: Machine frame number of new page-table base to install in MMU
 *      when in user space.
 *
 * cmd: MMUEXT_TLB_FLUSH_LOCAL
 * No additional arguments. Flushes local TLB.
 *
 * cmd: MMUEXT_INVLPG_LOCAL
 * linear_addr: Linear address to be flushed from the local TLB.
 *
 * cmd: MMUEXT_TLB_FLUSH_MULTI
 * vcpumask: Pointer to bitmap of VCPUs to be flushed.
 *
 * cmd: MMUEXT_INVLPG_MULTI
 * linear_addr: Linear address to be flushed.
 * vcpumask: Pointer to bitmap of VCPUs to be flushed.
 *
 * cmd: MMUEXT_TLB_FLUSH_ALL
 * No additional arguments. Flushes all VCPUs' TLBs.
 *
 * cmd: MMUEXT_INVLPG_ALL
 * linear_addr: Linear address to be flushed from all VCPUs' TLBs.
 *
 * cmd: MMUEXT_FLUSH_CACHE
 * No additional arguments. Writes back and flushes cache contents.
 *
 * cmd: MMUEXT_SET_LDT
 * linear_addr: Linear address of LDT base (NB. must be page-aligned).
 * nr_ents: Number of entries in LDT.
 *
 * cmd: MMUEXT_CLEAR_PAGE
 * mfn: Machine frame number to be cleared.
 *
 * cmd: MMUEXT_COPY_PAGE
 * mfn: Machine frame number of the destination page.
 * src_mfn: Machine frame number of the source page.
 */
#define MMUEXT_PIN_L1_TABLE      0
#define MMUEXT_PIN_L2_TABLE      1
#define MMUEXT_PIN_L3_TABLE      2
#define MMUEXT_PIN_L4_TABLE      3
#define MMUEXT_UNPIN_TABLE       4
#define MMUEXT_NEW_BASEPTR       5
#define MMUEXT_TLB_FLUSH_LOCAL   6
#define MMUEXT_INVLPG_LOCAL      7
#define MMUEXT_TLB_FLUSH_MULTI   8
#define MMUEXT_INVLPG_MULTI      9
#define MMUEXT_TLB_FLUSH_ALL    10
#define MMUEXT_INVLPG_ALL       11
#define MMUEXT_FLUSH_CACHE      12
#define MMUEXT_SET_LDT          13
#define MMUEXT_NEW_USER_BASEPTR 15
#define MMUEXT_CLEAR_PAGE       16
#define MMUEXT_COPY_PAGE        17

#ifndef __ASSEMBLY__
struct mmuext_op {
    unsigned int cmd;
    union {
        /* [UN]PIN_TABLE, NEW_BASEPTR, NEW_USER_BASEPTR
         * CLEAR_PAGE, COPY_PAGE */
        xen_pfn_t     mfn;
        /* INVLPG_LOCAL, INVLPG_ALL, SET_LDT */
        unsigned long linear_addr;
    } arg1;
    union {
        /* SET_LDT */
        unsigned int nr_ents;
        /* TLB_FLUSH_MULTI, INVLPG_MULTI */
#if __XEN_INTERFACE_VERSION__ >= 0x00030205
        XEN_GUEST_HANDLE(const_void) vcpumask;
#else
        const void *vcpumask;
#endif
        /* COPY_PAGE */
        xen_pfn_t src_mfn;
    } arg2;
};
typedef struct mmuext_op mmuext_op_t;
DEFINE_XEN_GUEST_HANDLE(mmuext_op_t);
#endif

/* These are passed as 'flags' to update_va_mapping. They can be ORed. */
/* When specifying UVMF_MULTI, also OR in a pointer to a CPU bitmap.   */
/* UVMF_LOCAL is merely UVMF_MULTI with a NULL bitmap pointer.         */
#define UVMF_NONE               (0UL<<0) /* No flushing at all.   */
#define UVMF_TLB_FLUSH          (1UL<<0) /* Flush entire TLB(s).  */
#define UVMF_INVLPG             (2UL<<0) /* Flush only one entry. */
#define UVMF_FLUSHTYPE_MASK     (3UL<<0)
#define UVMF_MULTI              (0UL<<2) /* Flush subset of TLBs. */
#define UVMF_LOCAL              (0UL<<2) /* Flush local TLB.      */
#define UVMF_ALL                (1UL<<2) /* Flush all TLBs.       */

/*
 * Commands to HYPERVISOR_console_io().
 */
#define CONSOLEIO_write         0
#define CONSOLEIO_read          1
#define CONSOLEIO_get_device   32

/*
 * Commands to HYPERVISOR_vm_assist().
 */
#define VMASST_CMD_enable                0
#define VMASST_CMD_disable               1

/* x86/32 guests: simulate full 4GB segment limits. */
#define VMASST_TYPE_4gb_segments         0

/* x86/32 guests: trap (vector 15) whenever above vmassist is used. */
#define VMASST_TYPE_4gb_segments_notify  1

/*
 * x86 guests: support writes to bottom-level PTEs.
 * NB1. Page-directory entries cannot be written.
 * NB2. Guest must continue to remove all writable mappings of PTEs.
 */
#define VMASST_TYPE_writable_pagetables  2

/* x86/PAE guests: support PDPTs above 4GB. */
#define VMASST_TYPE_pae_extended_cr3     3

#define MAX_VMASST_TYPE                  3

#ifndef __ASSEMBLY__

typedef uint16_t domid_t;

/* Domain ids >= DOMID_FIRST_RESERVED cannot be used for ordinary domains. */
#define DOMID_FIRST_RESERVED (0x7FF0U)

/* DOMID_SELF is used in certain contexts to refer to oneself. */
#define DOMID_SELF (0x7FF0U)

/*
 * DOMID_IO is used to restrict page-table updates to mapping I/O memory.
 * Although no Foreign Domain need be specified to map I/O pages, DOMID_IO
 * is useful to ensure that no mappings to the OS's own heap are accidentally
 * installed. (e.g., in Linux this could cause havoc as reference counts
 * aren't adjusted on the I/O-mapping code path).
 * This only makes sense in MMUEXT_SET_FOREIGNDOM, but in that context can
 * be specified by any calling domain.
 */
#define DOMID_IO   (0x7FF1U)

/*
 * DOMID_XEN is used to allow privileged domains to map restricted parts of
 * Xen's heap space (e.g., the machine_to_phys table).
 * This only makes sense in MMUEXT_SET_FOREIGNDOM, and is only permitted if
 * the caller is privileged.
 */
#define DOMID_XEN  (0x7FF2U)

/* DOMID_INVALID is used to identity invalid domid */
#define DOMID_INVALID (0x7FFFU)

/*
 * Send an array of these to HYPERVISOR_mmu_update().
 * NB. The fields are natural pointer/address size for this architecture.
 */
struct mmu_update {
    uint64_t ptr;       /* Machine address of PTE. */
    uint64_t val;       /* New contents of PTE.    */
};
typedef struct mmu_update mmu_update_t;
DEFINE_XEN_GUEST_HANDLE(mmu_update_t);

/*
 * Send an array of these to HYPERVISOR_multicall().
 * NB. The fields are natural register size for this architecture.
 */
struct multicall_entry {
    unsigned long op, result;
    unsigned long args[6];
};
typedef struct multicall_entry multicall_entry_t;
DEFINE_XEN_GUEST_HANDLE(multicall_entry_t);

/*
 * Event channel endpoints per domain:
 *  1024 if a long is 32 bits; 4096 if a long is 64 bits.
 */
#define NR_EVENT_CHANNELS (sizeof(unsigned long) * sizeof(unsigned long) * 64)

struct vcpu_time_info {
    /*
     * Updates to the following values are preceded and followed by an
     * increment of 'version'. The guest can therefore detect updates by
     * looking for changes to 'version'. If the least-significant bit of
     * the version number is set then an update is in progress and the guest
     * must wait to read a consistent set of values.
     * The correct way to interact with the version number is similar to
     * Linux's seqlock: see the implementations of read_seqbegin/read_seqretry.
     */
    uint32_t version;
    uint32_t pad0;
    uint64_t tsc_timestamp;   /* TSC at last update of time vals.  */
    uint64_t system_time;     /* Time, in nanosecs, since boot.    */
    /*
     * Current system time:
     *   system_time +
     *   ((((tsc - tsc_timestamp) << tsc_shift) * tsc_to_system_mul) >> 32)
     * CPU frequency (Hz):
     *   ((10^9 << 32) / tsc_to_system_mul) >> tsc_shift
     */
    uint32_t tsc_to_system_mul;
    int8_t   tsc_shift;
    int8_t   pad1[3];
}; /* 32 bytes */
typedef struct vcpu_time_info vcpu_time_info_t;

struct vcpu_info {
    /*
     * 'evtchn_upcall_pending' is written non-zero by Xen to indicate
     * a pending notification for a particular VCPU. It is then cleared
     * by the guest OS /before/ checking for pending work, thus avoiding
     * a set-and-check race. Note that the mask is only accessed by Xen
     * on the CPU that is currently hosting the VCPU. This means that the
     * pending and mask flags can be updated by the guest without special
     * synchronisation (i.e., no need for the x86 LOCK prefix).
     * This may seem suboptimal because if the pending flag is set by
     * a different CPU then an IPI may be scheduled even when the mask
     * is set. However, note:
     *  1. The task of 'interrupt holdoff' is covered by the per-event-
     *     channel mask bits. A 'noisy' event that is continually being
     *     triggered can be masked at source at this very precise
     *     granularity.
     *  2. The main purpose of the per-VCPU mask is therefore to restrict
     *     reentrant execution: whether for concurrency control, or to
     *     prevent unbounded stack usage. Whatever the purpose, we expect
     *     that the mask will be asserted only for short periods at a time,
     *     and so the likelihood of a 'spurious' IPI is suitably small.
     * The mask is read before making an event upcall to the guest: a
     * non-zero mask therefore guarantees that the VCPU will not receive
     * an upcall activation. The mask is cleared when the VCPU requests
     * to block: this avoids wakeup-waiting races.
     */
    uint8_t evtchn_upcall_pending;
    uint8_t evtchn_upcall_mask;
    unsigned long evtchn_pending_sel;
    struct arch_vcpu_info arch;
    struct vcpu_time_info time;
}; /* 64 bytes (x86) */
#ifndef __XEN__
typedef struct vcpu_info vcpu_info_t;
#endif

/*
 * Xen/kernel shared data -- pointer provided in start_info.
 *
 * This structure is defined to be both smaller than a page, and the
 * only data on the shared page, but may vary in actual size even within
 * compatible Xen versions; guests should not rely on the size
 * of this structure remaining constant.
 */
struct shared_info {
    struct vcpu_info vcpu_info[MAX_VIRT_CPUS];

    /*
     * A domain can create "event channels" on which it can send and receive
     * asynchronous event notifications. There are three classes of event that
     * are delivered by this mechanism:
     *  1. Bi-directional inter- and intra-domain connections. Domains must
     *     arrange out-of-band to set up a connection (usually by allocating
     *     an unbound 'listener' port and avertising that via a storage service
     *     such as xenstore).
     *  2. Physical interrupts. A domain with suitable hardware-access
     *     privileges can bind an event-channel port to a physical interrupt
     *     source.
     *  3. Virtual interrupts ('events'). A domain can bind an event-channel
     *     port to a virtual interrupt source, such as the virtual-timer
     *     device or the emergency console.
     *
     * Event channels are addressed by a "port index". Each channel is
     * associated with two bits of information:
     *  1. PENDING -- notifies the domain that there is a pending notification
     *     to be processed. This bit is cleared by the guest.
     *  2. MASK -- if this bit is clear then a 0->1 transition of PENDING
     *     will cause an asynchronous upcall to be scheduled. This bit is only
     *     updated by the guest. It is read-only within Xen. If a channel
     *     becomes pending while the channel is masked then the 'edge' is lost
     *     (i.e., when the channel is unmasked, the guest must manually handle
     *     pending notifications as no upcall will be scheduled by Xen).
     *
     * To expedite scanning of pending notifications, any 0->1 pending
     * transition on an unmasked channel causes a corresponding bit in a
     * per-vcpu selector word to be set. Each bit in the selector covers a
     * 'C long' in the PENDING bitfield array.
     */
    unsigned long evtchn_pending[sizeof(unsigned long) * 8];
    unsigned long evtchn_mask[sizeof(unsigned long) * 8];

    /*
     * Wallclock time: updated only by control software. Guests should base
     * their gettimeofday() syscall on this wallclock-base value.
     */
    uint32_t wc_version;      /* Version counter: see vcpu_time_info_t. */
    uint32_t wc_sec;          /* Secs  00:00:00 UTC, Jan 1, 1970.  */
    uint32_t wc_nsec;         /* Nsecs 00:00:00 UTC, Jan 1, 1970.  */

    struct arch_shared_info arch;

};
#ifndef __XEN__
typedef struct shared_info shared_info_t;
#endif

/*
 * Start-of-day memory layout:
 *  1. The domain is started within contiguous virtual-memory region.
 *  2. The contiguous region ends on an aligned 4MB boundary.
 *  3. This the order of bootstrap elements in the initial virtual region:
 *      a. relocated kernel image
 *      b. initial ram disk              [mod_start, mod_len]
 *      c. list of allocated page frames [mfn_list, nr_pages]
 *         (unless relocated due to XEN_ELFNOTE_INIT_P2M)
 *      d. start_info_t structure        [register ESI (x86)]
 *      e. bootstrap page tables         [pt_base, CR3 (x86)]
 *      f. bootstrap stack               [register ESP (x86)]
 *  4. Bootstrap elements are packed together, but each is 4kB-aligned.
 *  5. The initial ram disk may be omitted.
 *  6. The list of page frames forms a contiguous 'pseudo-physical' memory
 *     layout for the domain. In particular, the bootstrap virtual-memory
 *     region is a 1:1 mapping to the first section of the pseudo-physical map.
 *  7. All bootstrap elements are mapped read-writable for the guest OS. The
 *     only exception is the bootstrap page table, which is mapped read-only.
 *  8. There is guaranteed to be at least 512kB padding after the final
 *     bootstrap element. If necessary, the bootstrap virtual region is
 *     extended by an extra 4MB to ensure this.
 */

#define MAX_GUEST_CMDLINE 1024
struct start_info {
    /* THE FOLLOWING ARE FILLED IN BOTH ON INITIAL BOOT AND ON RESUME.    */
    char magic[32];             /* "xen-<version>-<platform>".            */
    unsigned long nr_pages;     /* Total pages allocated to this domain.  */
    unsigned long shared_info;  /* MACHINE address of shared info struct. */
    uint32_t flags;             /* SIF_xxx flags.                         */
    xen_pfn_t store_mfn;        /* MACHINE page number of shared page.    */
    uint32_t store_evtchn;      /* Event channel for store communication. */
    union {
        struct {
            xen_pfn_t mfn;      /* MACHINE page number of console page.   */
            uint32_t  evtchn;   /* Event channel for console page.        */
        } domU;
        struct {
            uint32_t info_off;  /* Offset of console_info struct.         */
            uint32_t info_size; /* Size of console_info struct from start.*/
        } dom0;
    } console;
    /* THE FOLLOWING ARE ONLY FILLED IN ON INITIAL BOOT (NOT RESUME).     */
    unsigned long pt_base;      /* VIRTUAL address of page directory.     */
    unsigned long nr_pt_frames; /* Number of bootstrap p.t. frames.       */
    unsigned long mfn_list;     /* VIRTUAL address of page-frame list.    */
    unsigned long mod_start;    /* VIRTUAL address of pre-loaded module.  */
    unsigned long mod_len;      /* Size (bytes) of pre-loaded module.     */
    int8_t cmd_line[MAX_GUEST_CMDLINE];
    /* The pfn range here covers both page table and p->m table frames.   */
    unsigned long first_p2m_pfn;/* 1st pfn forming initial P->M table.    */
    unsigned long nr_p2m_frames;/* # of pfns forming initial P->M table.  */
};
typedef struct start_info start_info_t;

/* New console union for dom0 introduced in 0x00030203. */
#if __XEN_INTERFACE_VERSION__ < 0x00030203
#define console_mfn    console.domU.mfn
#define console_evtchn console.domU.evtchn
#endif

/* These flags are passed in the 'flags' field of start_info_t. */
#define SIF_PRIVILEGED    (1<<0)  /* Is the domain privileged? */
#define SIF_INITDOMAIN    (1<<1)  /* Is this the initial control domain? */
#define SIF_PM_MASK       (0xFF<<8) /* reserve 1 byte for xen-pm options */

#define XEN_CONSOLE_INVALID   -1
#define XEN_CONSOLE_COM1       0
#define XEN_CONSOLE_COM2       1
#define XEN_CONSOLE_VGA        2

typedef struct dom0_vga_console_info {
    uint8_t video_type; /* DOM0_VGA_CONSOLE_??? */
#define XEN_VGATYPE_TEXT_MODE_3 0x03
#define XEN_VGATYPE_VESA_LFB    0x23

    union {
        struct {
            /* Font height, in pixels. */
            uint16_t font_height;
            /* Cursor location (column, row). */
            uint16_t cursor_x, cursor_y;
            /* Number of rows and columns (dimensions in characters). */
            uint16_t rows, columns;
        } text_mode_3;

        struct {
            /* Width and height, in pixels. */
            uint16_t width, height;
            /* Bytes per scan line. */
            uint16_t bytes_per_line;
            /* Bits per pixel. */
            uint16_t bits_per_pixel;
            /* LFB physical address, and size (in units of 64kB). */
            uint32_t lfb_base;
            uint32_t lfb_size;
            /* RGB mask offsets and sizes, as defined by VBE 1.2+ */
            uint8_t  red_pos, red_size;
            uint8_t  green_pos, green_size;
            uint8_t  blue_pos, blue_size;
            uint8_t  rsvd_pos, rsvd_size;
#if __XEN_INTERFACE_VERSION__ >= 0x00030206
            /* VESA capabilities (offset 0xa, VESA command 0x4f00). */
            uint32_t gbl_caps;
            /* Mode attributes (offset 0x0, VESA command 0x4f01). */
            uint16_t mode_attrs;
#endif
        } vesa_lfb;
    } u;
} dom0_vga_console_info_t;
#define xen_vga_console_info dom0_vga_console_info
#define xen_vga_console_info_t dom0_vga_console_info_t

typedef uint8_t xen_domain_handle_t[16];

/* Turn a plain number into a C unsigned long constant. */
#define __mk_unsigned_long(x) x ## UL
#define mk_unsigned_long(x) __mk_unsigned_long(x)

__DEFINE_XEN_GUEST_HANDLE(uint8,  uint8_t);
__DEFINE_XEN_GUEST_HANDLE(uint16, uint16_t);
__DEFINE_XEN_GUEST_HANDLE(uint32, uint32_t);
__DEFINE_XEN_GUEST_HANDLE(uint64, uint64_t);

#else /* __ASSEMBLY__ */

/* In assembly code we cannot use C numeric constant suffixes. */
#define mk_unsigned_long(x) x

#endif /* !__ASSEMBLY__ */

/* Default definitions for macros used by domctl/sysctl. */
#if defined(__XEN__) || defined(__XEN_TOOLS__)
#ifndef uint64_aligned_t
#define uint64_aligned_t uint64_t
#endif
#ifndef XEN_GUEST_HANDLE_64
#define XEN_GUEST_HANDLE_64(name) XEN_GUEST_HANDLE(name)
#endif
#endif

#endif /* __XEN_PUBLIC_XEN_H__ */

/*
 * Local variables:
 * mode: C
 * c-set-style: "BSD"
 * c-basic-offset: 4
 * tab-width: 4
 * indent-tabs-mode: nil
 * End:
 */
/*
 * acm.h: Xen access control module interface defintions
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to
 * deal in the Software without restriction, including without limitation the
 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
 * sell copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 * DEALINGS IN THE SOFTWARE.
 *
 * Reiner Sailer <sailer@watson.ibm.com>
 * Copyright (c) 2005, International Business Machines Corporation.
 */

#ifndef _XEN_PUBLIC_ACM_H
#define _XEN_PUBLIC_ACM_H

#include "../xen.h"

/* default ssid reference value if not supplied */
#define ACM_DEFAULT_SSID  0x0
#define ACM_DEFAULT_LOCAL_SSID  0x0

/* Internal ACM ERROR types */
#define ACM_OK     0
#define ACM_UNDEF   -1
#define ACM_INIT_SSID_ERROR  -2
#define ACM_INIT_SOID_ERROR  -3
#define ACM_ERROR          -4

/* External ACCESS DECISIONS */
#define ACM_ACCESS_PERMITTED        0
#define ACM_ACCESS_DENIED           -111
#define ACM_NULL_POINTER_ERROR      -200

/*
   Error codes reported in when trying to test for a new policy
   These error codes are reported in an array of tuples where
   each error code is followed by a parameter describing the error
   more closely, such as a domain id.
*/
#define ACM_EVTCHN_SHARING_VIOLATION       0x100
#define ACM_GNTTAB_SHARING_VIOLATION       0x101
#define ACM_DOMAIN_LOOKUP                  0x102
#define ACM_CHWALL_CONFLICT                0x103
#define ACM_SSIDREF_IN_USE                 0x104


/* primary policy in lower 4 bits */
#define ACM_NULL_POLICY 0
#define ACM_CHINESE_WALL_POLICY 1
#define ACM_SIMPLE_TYPE_ENFORCEMENT_POLICY 2
#define ACM_POLICY_UNDEFINED 15

/* combinations have secondary policy component in higher 4bit */
#define ACM_CHINESE_WALL_AND_SIMPLE_TYPE_ENFORCEMENT_POLICY \
    ((ACM_SIMPLE_TYPE_ENFORCEMENT_POLICY << 4) | ACM_CHINESE_WALL_POLICY)

/* policy: */
#define ACM_POLICY_NAME(X) \
 ((X) == (ACM_NULL_POLICY)) ? "NULL" :                        \
    ((X) == (ACM_CHINESE_WALL_POLICY)) ? "CHINESE WALL" :        \
    ((X) == (ACM_SIMPLE_TYPE_ENFORCEMENT_POLICY)) ? "SIMPLE TYPE ENFORCEMENT" : \
    ((X) == (ACM_CHINESE_WALL_AND_SIMPLE_TYPE_ENFORCEMENT_POLICY)) ? "CHINESE WALL AND SIMPLE TYPE ENFORCEMENT" : \
     "UNDEFINED"

/* the following policy versions must be increased
 * whenever the interpretation of the related
 * policy's data structure changes
 */
#define ACM_POLICY_VERSION 4
#define ACM_CHWALL_VERSION 1
#define ACM_STE_VERSION  1

/* defines a ssid reference used by xen */
typedef uint32_t ssidref_t;

/* hooks that are known to domains */
#define ACMHOOK_none          0
#define ACMHOOK_sharing       1
#define ACMHOOK_authorization 2
#define ACMHOOK_conflictset   3

/* -------security policy relevant type definitions-------- */

/* type identifier; compares to "equal" or "not equal" */
typedef uint16_t domaintype_t;

/* CHINESE WALL POLICY DATA STRUCTURES
 *
 * current accumulated conflict type set:
 * When a domain is started and has a type that is in
 * a conflict set, the conflicting types are incremented in
 * the aggregate set. When a domain is destroyed, the
 * conflicting types to its type are decremented.
 * If a domain has multiple types, this procedure works over
 * all those types.
 *
 * conflict_aggregate_set[i] holds the number of
 *   running domains that have a conflict with type i.
 *
 * running_types[i] holds the number of running domains
 *        that include type i in their ssidref-referenced type set
 *
 * conflict_sets[i][j] is "0" if type j has no conflict
 *    with type i and is "1" otherwise.
 */
/* high-16 = version, low-16 = check magic */
#define ACM_MAGIC  0x0001debc

/* size of the SHA1 hash identifying the XML policy from which the
   binary policy was created */
#define ACM_SHA1_HASH_SIZE    20

/* each offset in bytes from start of the struct they
 * are part of */

/* V3 of the policy buffer aded a version structure */
struct acm_policy_version
{
    uint32_t major;
    uint32_t minor;
};


/* each buffer consists of all policy information for
 * the respective policy given in the policy code
 *
 * acm_policy_buffer, acm_chwall_policy_buffer,
 * and acm_ste_policy_buffer need to stay 32-bit aligned
 * because we create binary policies also with external
 * tools that assume packed representations (e.g. the java tool)
 */
struct acm_policy_buffer {
    uint32_t magic;
    uint32_t policy_version; /* ACM_POLICY_VERSION */
    uint32_t len;
    uint32_t policy_reference_offset;
    uint32_t primary_policy_code;
    uint32_t primary_buffer_offset;
    uint32_t secondary_policy_code;
    uint32_t secondary_buffer_offset;
    struct acm_policy_version xml_pol_version; /* add in V3 */
    uint8_t xml_policy_hash[ACM_SHA1_HASH_SIZE]; /* added in V4 */
};


struct acm_policy_reference_buffer {
    uint32_t len;
};

struct acm_chwall_policy_buffer {
    uint32_t policy_version; /* ACM_CHWALL_VERSION */
    uint32_t policy_code;
    uint32_t chwall_max_types;
    uint32_t chwall_max_ssidrefs;
    uint32_t chwall_max_conflictsets;
    uint32_t chwall_ssid_offset;
    uint32_t chwall_conflict_sets_offset;
    uint32_t chwall_running_types_offset;
    uint32_t chwall_conflict_aggregate_offset;
};

struct acm_ste_policy_buffer {
    uint32_t policy_version; /* ACM_STE_VERSION */
    uint32_t policy_code;
    uint32_t ste_max_types;
    uint32_t ste_max_ssidrefs;
    uint32_t ste_ssid_offset;
};

struct acm_stats_buffer {
    uint32_t magic;
    uint32_t len;
    uint32_t primary_policy_code;
    uint32_t primary_stats_offset;
    uint32_t secondary_policy_code;
    uint32_t secondary_stats_offset;
};

struct acm_ste_stats_buffer {
    uint32_t ec_eval_count;
    uint32_t gt_eval_count;
    uint32_t ec_denied_count;
    uint32_t gt_denied_count;
    uint32_t ec_cachehit_count;
    uint32_t gt_cachehit_count;
};

struct acm_ssid_buffer {
    uint32_t len;
    ssidref_t ssidref;
    uint32_t policy_reference_offset;
    uint32_t primary_policy_code;
    uint32_t primary_max_types;
    uint32_t primary_types_offset;
    uint32_t secondary_policy_code;
    uint32_t secondary_max_types;
    uint32_t secondary_types_offset;
};

#endif

/*
 * Local variables:
 * mode: C
 * c-set-style: "BSD"
 * c-basic-offset: 4
 * tab-width: 4
 * indent-tabs-mode: nil
 * End:
 */
/*
 * acm_ops.h: Xen access control module hypervisor commands
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to
 * deal in the Software without restriction, including without limitation the
 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
 * sell copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 * DEALINGS IN THE SOFTWARE.
 *
 * Reiner Sailer <sailer@watson.ibm.com>
 * Copyright (c) 2005,2006 International Business Machines Corporation.
 */

#ifndef __XEN_PUBLIC_ACM_OPS_H__
#define __XEN_PUBLIC_ACM_OPS_H__

#include "../xen.h"
#include "acm.h"

/*
 * Make sure you increment the interface version whenever you modify this file!
 * This makes sure that old versions of acm tools will stop working in a
 * well-defined way (rather than crashing the machine, for instance).
 */
#define ACM_INTERFACE_VERSION   0xAAAA000A

/************************************************************************/

/*
 * Prototype for this hypercall is:
 *  int acm_op(int cmd, void *args)
 * @cmd  == ACMOP_??? (access control module operation).
 * @args == Operation-specific extra arguments (NULL if none).
 */


#define ACMOP_setpolicy         1
struct acm_setpolicy {
    /* IN */
    XEN_GUEST_HANDLE_64(void) pushcache;
    uint32_t pushcache_size;
};


#define ACMOP_getpolicy         2
struct acm_getpolicy {
    /* IN */
    XEN_GUEST_HANDLE_64(void) pullcache;
    uint32_t pullcache_size;
};


#define ACMOP_dumpstats         3
struct acm_dumpstats {
    /* IN */
    XEN_GUEST_HANDLE_64(void) pullcache;
    uint32_t pullcache_size;
};


#define ACMOP_getssid           4
#define ACM_GETBY_ssidref  1
#define ACM_GETBY_domainid 2
struct acm_getssid {
    /* IN */
    uint32_t get_ssid_by; /* ACM_GETBY_* */
    union {
        domaintype_t domainid;
        ssidref_t    ssidref;
    } id;
    XEN_GUEST_HANDLE_64(void) ssidbuf;
    uint32_t ssidbuf_size;
};

#define ACMOP_getdecision      5
struct acm_getdecision {
    /* IN */
    uint32_t get_decision_by1; /* ACM_GETBY_* */
    uint32_t get_decision_by2; /* ACM_GETBY_* */
    union {
        domaintype_t domainid;
        ssidref_t    ssidref;
    } id1;
    union {
        domaintype_t domainid;
        ssidref_t    ssidref;
    } id2;
    uint32_t hook;
    /* OUT */
    uint32_t acm_decision;
};


#define ACMOP_chgpolicy        6
struct acm_change_policy {
    /* IN */
    XEN_GUEST_HANDLE_64(void) policy_pushcache;
    uint32_t policy_pushcache_size;
    XEN_GUEST_HANDLE_64(void) del_array;
    uint32_t delarray_size;
    XEN_GUEST_HANDLE_64(void) chg_array;
    uint32_t chgarray_size;
    /* OUT */
    /* array with error code */
    XEN_GUEST_HANDLE_64(void) err_array;
    uint32_t errarray_size;
};

#define ACMOP_relabeldoms       7
struct acm_relabel_doms {
    /* IN */
    XEN_GUEST_HANDLE_64(void) relabel_map;
    uint32_t relabel_map_size;
    /* OUT */
    XEN_GUEST_HANDLE_64(void) err_array;
    uint32_t errarray_size;
};

/* future interface to Xen */
struct xen_acmctl {
    uint32_t cmd;
    uint32_t interface_version;
    union {
        struct acm_setpolicy     setpolicy;
        struct acm_getpolicy     getpolicy;
        struct acm_dumpstats     dumpstats;
        struct acm_getssid       getssid;
        struct acm_getdecision   getdecision;
        struct acm_change_policy change_policy;
        struct acm_relabel_doms  relabel_doms;
    } u;
};

typedef struct xen_acmctl xen_acmctl_t;
DEFINE_XEN_GUEST_HANDLE(xen_acmctl_t);

#endif /* __XEN_PUBLIC_ACM_OPS_H__ */

/*
 * Local variables:
 * mode: C
 * c-set-style: "BSD"
 * c-basic-offset: 4
 * tab-width: 4
 * indent-tabs-mode: nil
 * End:
 */
#
# CDDL HEADER START
#
# The contents of this file are subject to the terms of the
# Common Development and Distribution License (the "License").
# You may not use this file except in compliance with the License.
#
# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
# or http://www.opensolaris.org/os/licensing.
# See the License for the specific language governing permissions
# and limitations under the License.
#
# When distributing Covered Code, include this CDDL HEADER in each
# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
# If applicable, add the following below this CDDL HEADER, with the
# fields enclosed by brackets "[]" replaced with your own identifying
# information: Portions Copyright [yyyy] [name of copyright owner]
#
# CDDL HEADER END
#

#
# Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
# Use is subject to license terms.
#
# ident "@(#)prototype.Makefile 1.15    06/02/08 SMI"
#

evtchn.h, privcmd.h, and xenbus.h should not be edited in ON. They are
copies from a specific build of the xen consolidation which can be found in:
  xen.hg/tools/libxc/xen/solaris

Any changes to these files should be done in the xen consolidation.

/******************************************************************************
 * evtchn.h
 *
 * Interface to /dev/xen/evtchn.
 *
 * Copyright (c) 2003-2005, K A Fraser
 *
 * This file may be distributed separately from the Linux kernel, or
 * incorporated into other software packages, subject to the following license:
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this source file (the "Software"), to deal in the Software without
 * restriction, including without limitation the rights to use, copy, modify,
 * merge, publish, distribute, sublicense, and/or sell copies of the Software,
 * and to permit persons to whom the Software is furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 */

/*
 * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#ifndef _XEN_SYS_EVTCHN_H
#define _XEN_SYS_EVTCHN_H

#define	_IOC_NONE	0
#define	_IOC(flag, letter, inum, size)	((letter) << 8 | (inum))

/*
 * Bind a fresh port to VIRQ @virq.
 * Return allocated port.
 */
#define IOCTL_EVTCHN_BIND_VIRQ				\
	_IOC(_IOC_NONE, 'E', 0, sizeof(struct ioctl_evtchn_bind_virq))
struct ioctl_evtchn_bind_virq {
	unsigned int virq;
};

/*
 * Bind a fresh port to remote <@remote_domain, @remote_port>.
 * Return allocated port.
 */
#define IOCTL_EVTCHN_BIND_INTERDOMAIN			\
	_IOC(_IOC_NONE, 'E', 1, sizeof(struct ioctl_evtchn_bind_interdomain))
struct ioctl_evtchn_bind_interdomain {
	unsigned int remote_domain, remote_port;
};

/*
 * Allocate a fresh port for binding to @remote_domain.
 * Return allocated port.
 */
#define IOCTL_EVTCHN_BIND_UNBOUND_PORT                  \
        _IOC(_IOC_NONE, 'E', 2, sizeof(struct ioctl_evtchn_bind_unbound_port))
struct ioctl_evtchn_bind_unbound_port {
        unsigned int remote_domain;
};

/*
 * Unbind previously allocated @port.
 */
#define IOCTL_EVTCHN_UNBIND				\
	_IOC(_IOC_NONE, 'E', 3, sizeof(struct ioctl_evtchn_unbind))
struct ioctl_evtchn_unbind {
	unsigned int port;
};

/*
 * Notify the given @port.
 */
#define IOCTL_EVTCHN_NOTIFY				\
	_IOC(_IOC_NONE, 'E', 4, sizeof(struct ioctl_evtchn_notify))
struct ioctl_evtchn_notify {
	unsigned int port;
};

#endif /* _XEN_SYS_EVTCHN_H */

/*
 * Local variables:
 *  c-file-style: "solaris"
 *  indent-tabs-mode: t
 *  c-indent-level: 8
 *  c-basic-offset: 8
 *  tab-width: 8
 * End:
 */
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#ifndef	_SYS_GNTTAB_H
#define	_SYS_GNTTAB_H

/*
 * gnttab.h
 *
 * Two sets of functionality:
 * 1. Granting foreign access to our memory reservation.
 * 2. Accessing others' memory reservations via grant references.
 * (i.e., mechanisms for both sender and recipient of grant references)
 *
 * Copyright (c) 2004-2005, K A Fraser
 * Copyright (c) 2005, Christopher Clark
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License version 2
 * as published by the Free Software Foundation; or, when distributed
 * separately from the Linux kernel or incorporated into other
 * software packages, subject to the following license:
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this source file (the "Software"), to deal in the Software without
 * restriction, including without limitation the rights to use, copy, modify,
 * merge, publish, distribute, sublicense, and/or sell copies of the Software,
 * and to permit persons to whom the Software is furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 */

#include <sys/hypervisor.h>
#include <xen/public/grant_table.h>
#include <xen/public/features.h>

#ifdef __cplusplus
extern "C" {
#endif

struct gnttab_free_callback {
	struct gnttab_free_callback *next;
	void (*fn)(void *);
	void *arg;
	uint16_t count;
};

/*
 * For i86xpv the "frames" in grant table terminology are really MFNs.
 */
typedef mfn_t gnttab_frame_t;
#define	FRAME_TO_MA(f)	((maddr_t)(f) << PAGESHIFT)

int gnttab_grant_foreign_access(domid_t, gnttab_frame_t, int readonly);

/*
 * End access through the given grant reference, iff the grant entry is no
 * longer in use.  Return 1 if the grant entry was freed, 0 if it is still in
 * use.
 */
int gnttab_end_foreign_access_ref(grant_ref_t ref, int readonly);

/*
 * Eventually end access through the given grant reference, and once that
 * access has been ended, free the given page too.  Access will be ended
 * immediately iff the grant entry is not in use, otherwise it will happen
 * some time later.  page may be 0, in which case no freeing will occur.
 */
void gnttab_end_foreign_access(grant_ref_t ref, int readonly,
	gnttab_frame_t page);

int gnttab_grant_foreign_transfer(domid_t domid, pfn_t pfn);

gnttab_frame_t gnttab_end_foreign_transfer_ref(grant_ref_t ref);
gnttab_frame_t gnttab_end_foreign_transfer(grant_ref_t ref);

int gnttab_query_foreign_access(grant_ref_t ref);

/*
 * operations on reserved batches of grant references
 */
int gnttab_alloc_grant_references(uint16_t count, grant_ref_t *pprivate_head);

void gnttab_free_grant_reference(grant_ref_t ref);

void gnttab_free_grant_references(grant_ref_t head);

int gnttab_empty_grant_references(const grant_ref_t *pprivate_head);

int gnttab_claim_grant_reference(grant_ref_t *pprivate_head);

void gnttab_release_grant_reference(grant_ref_t *private_head,
	grant_ref_t release);

void gnttab_request_free_callback(struct gnttab_free_callback *callback,
	void (*fn)(void *), void *arg, uint16_t count);

void gnttab_cancel_free_callback(struct gnttab_free_callback *callback);

void gnttab_grant_foreign_access_ref(grant_ref_t ref, domid_t domid,
	gnttab_frame_t frame, int readonly);

void gnttab_grant_foreign_transfer_ref(grant_ref_t, domid_t domid,
	pfn_t pfn);

#define	gnttab_map_vaddr(map) ((void *)(map.host_virt_addr))

/*
 * framework
 */
void gnttab_init(void);
void gnttab_suspend(void);
void gnttab_resume(void);

#ifdef	__cplusplus
}
#endif

#endif /* _SYS_GNTTAB_H */
/*
 * Copyright (c) 2003-2005, K A Fraser
 *
 * This file may be distributed separately from the Linux kernel, or
 * incorporated into other software packages, subject to the following license:
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this source file (the "Software"), to deal in the Software without
 * restriction, including without limitation the rights to use, copy, modify,
 * merge, publish, distribute, sublicense, and/or sell copies of the Software,
 * and to permit persons to whom the Software is furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 */

/*
 * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#ifndef	_XEN_SYS_PRIVCMD_H
#define	_XEN_SYS_PRIVCMD_H

/*
 * WARNING:
 *	These numbers and structure are built into the ON privcmd
 *	driver, as well as the low-level tools and libraries in
 *	the Xen consolidation.
 */

#include <sys/types.h>

#ifdef __cplusplus
extern "C" {
#endif

/*
 * ioctl numbers and corresponding data structures
 */

#define	__PRIVCMD_IOC			(('p'<<24)|('r'<<16)|('v'<<8))

#define	IOCTL_PRIVCMD_HYPERCALL		(__PRIVCMD_IOC|0)
#define	IOCTL_PRIVCMD_MMAP		(__PRIVCMD_IOC|1)
#define	IOCTL_PRIVCMD_MMAPBATCH		(__PRIVCMD_IOC|2)

typedef struct __privcmd_hypercall {
	unsigned long op;
	unsigned long arg[5];
} privcmd_hypercall_t;

typedef struct __privcmd_mmap_entry {
	unsigned long va;
	unsigned long mfn;
	unsigned long npages;
} privcmd_mmap_entry_t;

typedef struct __privcmd_mmap {
	int num;
	domid_t dom;	/* target domain */
	privcmd_mmap_entry_t *entry;
} privcmd_mmap_t;

typedef struct __privcmd_mmapbatch {
	int num;	/* number of pages to populate */
	domid_t dom;	/* target domain */
	unsigned long addr;	  /* virtual address */
	unsigned long *arr;	  /* array of mfns - top nibble set on err */
} privcmd_mmapbatch_t;

#ifdef __cplusplus
}
#endif

#endif /* _XEN_SYS_PRIVCMD_H */
/*
 * This file may be distributed separately from the Linux kernel, or
 * incorporated into other software packages, subject to the following license:
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this source file (the "Software"), to deal in the Software without
 * restriction, including without limitation the rights to use, copy, modify,
 * merge, publish, distribute, sublicense, and/or sell copies of the Software,
 * and to permit persons to whom the Software is furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 */

/*
 * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#ifndef _XEN_SYS_XENBUS_H
#define	_XEN_SYS_XENBUS_H

/*
 * Return the xenstore event channel.
 */
#define	IOCTL_XENBUS_XENSTORE_EVTCHN ('X' << 8)

/*
 * Notify the kernel that the xenstore is up and running
 */
#define	IOCTL_XENBUS_NOTIFY_UP ('U' << 8)

#endif /* _XEN_SYS_XENBUS_H */
/*
 * Private include for xenbus communications.
 *
 * Copyright (C) 2005 Rusty Russell, IBM Corporation
 *
 * This file may be distributed separately from the Linux kernel, or
 * incorporated into other software packages, subject to the following license:
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this source file (the "Software"), to deal in the Software without
 * restriction, including without limitation the rights to use, copy, modify,
 * merge, publish, distribute, sublicense, and/or sell copies of the Software,
 * and to permit persons to whom the Software is furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 */

/*
 * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#ifndef	_SYS_XENBUS_COMMS_H
#define	_SYS_XENBUS_COMMS_H

#include <sys/sunddi.h>

#ifdef __cplusplus
extern "C" {
#endif

/* xenbus interface interrupt */
#define	IPL_XENBUS	0x01

void xs_early_init(void);
void xs_domu_init(void);
void xs_dom0_init(void);
void xb_suspend(void);
void xb_init(void);
void xb_setup_intr(void);

/* Low level routines. */
int xb_write(const void *data, unsigned len);
int xb_read(void *data, unsigned len);

ddi_umem_cookie_t xb_xenstore_cookie(void);

#ifdef __cplusplus
}
#endif

#endif /* _SYS_XENBUS_COMMS_H */
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

/*
 *
 * xenbus.h (renamed to xenbus_impl.h)
 *
 * Talks to Xen Store to figure out what devices we have.
 *
 * Copyright (C) 2005 Rusty Russell, IBM Corporation
 *
 * This file may be distributed separately from the Linux kernel, or
 * incorporated into other software packages, subject to the following license:
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this source file (the "Software"), to deal in the Software without
 * restriction, including without limitation the rights to use, copy, modify,
 * merge, publish, distribute, sublicense, and/or sell copies of the Software,
 * and to permit persons to whom the Software is furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 */

#ifndef _SYS_XENBUS_H
#define	_SYS_XENBUS_H

#include <sys/mutex.h>
#include <sys/list.h>

#ifdef	__cplusplus
extern "C" {
#endif

#define	XBT_NULL 0

typedef uint32_t xenbus_transaction_t;

/* Register callback to watch this node. */
struct xenbus_watch;
typedef void (*xenbus_watch_cb_t)(struct xenbus_watch *,
    const char **vec, unsigned int len);
struct xenbus_watch {
	list_node_t 		list;
	const char		*node;	/* path being watched */
	xenbus_watch_cb_t	callback;
	struct xenbus_device	*dev;
};

/*
 * Call this function when xenstore is available, i.e. the daemon is
 * connected to the xenbus device.
 */
struct xenbus_notify {
	list_node_t list;
	void (*notify_func) (int);
};

/* A xenbus device. */
struct xenbus_device {
	const char *devicetype;
	const char *nodename;
	const char *otherend;
	int otherend_id;
	int otherend_state;
	struct xenbus_watch otherend_watch;
	int has_error;
	int frontend;
	void (*otherend_changed)(struct xenbus_device *, XenbusState);
	void *data;
};

typedef void (*xvdi_xb_watch_cb_t)(dev_info_t *dip, const char *path,
    void *arg);

typedef struct xd_xb_watches {
	list_node_t		xxw_list;
	int			xxw_ref;
	struct xenbus_watch	xxw_watch;
	struct xendev_ppd	*xxw_xppd;
	xvdi_xb_watch_cb_t	xxw_cb;
	void			*xxw_arg;
} xd_xb_watches_t;

extern char **xenbus_directory(xenbus_transaction_t t, const char *dir,
	    const char *node, unsigned int *num);
extern int xenbus_read(xenbus_transaction_t t, const char *dir,
	    const char *node, void **rstr, unsigned int *len);
extern int xenbus_read_str(const char *dir, const char *node, char **rstr);
extern int xenbus_write(xenbus_transaction_t t, const char *dir,
	    const char *node, const char *string);
extern int xenbus_mkdir(xenbus_transaction_t t, const char *dir,
	    const char *node);
extern boolean_t xenbus_exists(const char *dir, const char *node);
extern boolean_t xenbus_exists_dir(const char *dir, const char *node);
extern int xenbus_rm(xenbus_transaction_t t, const char *dir,
	    const char *node);
extern int xenbus_transaction_start(xenbus_transaction_t *t);
extern int xenbus_transaction_end(xenbus_transaction_t t, int abort);

/* Single read and scanf: returns errno or num scanned if > 0. */
extern int xenbus_scanf(xenbus_transaction_t t, const char *dir,
	    const char *node, const char *fmt, ...);

/* Single printf and write: returns errno or 0. */
extern int xenbus_printf(xenbus_transaction_t t, const char *dir,
	    const char *node, const char *fmt, ...);

/*
 * Generic read function: NULL-terminated triples of name,
 * sprintf-style type string, and pointer. Returns 0 or errno.
 */
extern int xenbus_gather(xenbus_transaction_t t, const char *dir, ...);

extern int register_xenbus_watch(struct xenbus_watch *watch);
extern void unregister_xenbus_watch(struct xenbus_watch *watch);
extern void reregister_xenbus_watches(void);

/* Called from xen core code. */
extern void xenbus_suspend(void);
extern void xenbus_resume(void);

#define	XENBUS_EXIST_ERR(err) ((err) == ENOENT || (err) == ERANGE)

/*
 * Register a watch on the given path, using the given xenbus_watch structure
 * for storage, and the given callback function as the callback.  Return 0 on
 * success, or errno on error.  On success, the given path will be saved as
 * watch->node, and remains the caller's to free.  On error, watch->node will
 * be NULL, the device will switch to XenbusStateClosing, and the error will
 * be saved in the store.
 */
extern int xenbus_watch_path(struct xenbus_device *dev, const char *path,
			struct xenbus_watch *watch,
			void (*callback)(struct xenbus_watch *,
			const char **, unsigned int));


/*
 * Register a watch on the given path/path2, using the given xenbus_watch
 * structure for storage, and the given callback function as the callback.
 * Return 0 on success, or errno on error.  On success, the watched path
 * (path/path2) will be saved as watch->node, and becomes the caller's to
 * kfree().  On error, watch->node will be NULL, so the caller has nothing to
 * free, the device will switch to XenbusStateClosing, and the error will be
 * saved in the store.
 */
extern int xenbus_watch_path2(struct xenbus_device *dev, const char *path,
			const char *path2, struct xenbus_watch *watch,
			void (*callback)(struct xenbus_watch *,
			const char **, unsigned int));


/*
 * Advertise in the store a change of the given driver to the given new_state.
 * Perform the change inside the given transaction xbt.  xbt may be NULL, in
 * which case this is performed inside its own transaction.  Return 0 on
 * success, or errno on error.  On error, the device will switch to
 * XenbusStateClosing, and the error will be saved in the store.
 */
extern int xenbus_switch_state(struct xenbus_device *dev,
			xenbus_transaction_t xbt,
			XenbusState new_state);


/*
 * Grant access to the given ring_mfn to the peer of the given device.  Return
 * 0 on success, or errno on error.  On error, the device will switch to
 * XenbusStateClosing, and the error will be saved in the store.
 */
extern int xenbus_grant_ring(struct xenbus_device *dev, unsigned long ring_mfn);


/*
 * Allocate an event channel for the given xenbus_device, assigning the newly
 * created local port to *port.  Return 0 on success, or errno on error.  On
 * error, the device will switch to XenbusStateClosing, and the error will be
 * saved in the store.
 */
extern int xenbus_alloc_evtchn(struct xenbus_device *dev, int *port);


/*
 * Return the state of the driver rooted at the given store path, or
 * XenbusStateClosed if no state can be read.
 */
extern XenbusState xenbus_read_driver_state(const char *path);


/*
 * Report the given negative errno into the store, along with the given
 * formatted message.
 */
extern void xenbus_dev_error(struct xenbus_device *dev, int err,
	const char *fmt, ...);


/*
 * Equivalent to xenbus_dev_error(dev, err, fmt, args), followed by
 * xenbus_switch_state(dev, NULL, XenbusStateClosing) to schedule an orderly
 * closedown of this driver and its peer.
 */
extern void xenbus_dev_fatal(struct xenbus_device *dev,
	int err, const char *fmt, ...);

/* Clear any error. */
extern void xenbus_dev_ok(struct xenbus_device *dev);

/*
 * Set up watches on other end of split device.
 */
extern int talk_to_otherend(struct xenbus_device *dev);

#define	XENSTORE_DOWN	0	/* xenstore is down */
#define	XENSTORE_UP	1	/* xenstore is up */

/*
 * Register a notify callback function.
 */
extern int xs_register_xenbus_callback(void (*callback)(int));

/*
 * Notify clients that xenstore is up
 */
extern void xs_notify_xenstore_up(void);

/*
 * Notify clients that xenstore is down
 */
extern void xs_notify_xenstore_down(void);

struct xsd_sockmsg;

extern int xenbus_dev_request_and_reply(struct xsd_sockmsg *, void **);

#ifdef	__cplusplus
}
#endif

#endif /* _SYS_XENBUS_H */
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#ifndef	_SYS_XENDEV_H
#define	_SYS_XENDEV_H


#include <sys/hypervisor.h>
#include <sys/taskq.h>
#ifndef	__xpv
#include <public/io/ring.h>
#include <public/event_channel.h>
#include <public/grant_table.h>
#endif
#include <xen/sys/xenbus_impl.h>

#ifdef	__cplusplus
extern "C" {
#endif

/*
 * Xenbus property interfaces, initialized by framework
 */
#define	XBP_HP_STATUS		"hotplug-status"	/* backend prop: str */
#define	XBV_HP_STATUS_CONN	"connected"		/* backend prop val */
#define	XBP_DEV_TYPE		"device-type"		/* backend prop: str */
#define	XBV_DEV_TYPE_CD		"cdrom"			/* backend prop val */

/*
 * Xenbus property interfaces, initialized by backend disk driver
 */
#define	XBP_SECTOR_SIZE	"sector-size"		/* backend prop: uint */
#define	XBP_SECTORS	"sectors"		/* backend prop: uint64 */
#define	XBP_INFO	"info"			/* backend prop: uint */
#define	XBP_FB		"feature-barrier"	/* backend prop: boolean int */

/*
 * Xenbus property interfaces, initialized by frontend disk driver
 */
#define	XBP_RING_REF	"ring-ref"		/* frontend prop: long */
#define	XBP_EVENT_CHAN	"event-channel"		/* frontend prop: long */
#define	XBP_PROTOCOL	"protocol"		/* frontend prop: string */

/*
 * Xenbus CDROM property interfaces, used by backend and frontend
 *
 * XBP_MEDIA_REQ_SUP
 *	- Backend xenbus property located at:
 *		backend/vbd/<domU_id>/<domU_dev>/media-req-sup
 *	- Set by the backend, consumed by the frontend.
 *	- Cosumed by the frontend.
 *	- A boolean integer property indicating backend support
 *	  for the XBP_MEDIA_REQ property.
 *
 * XBP_MEDIA_REQ
 *	- Frontend xenbus property located at:
 *		/local/domain/<domU_id>/device/vbd/<domU_dev>/media-req
 *	- Set and consumed by both the frontend and backend.
 *	- Possible values:
 *		XBV_MEDIA_REQ_NONE, XBV_MEDIA_REQ_LOCK, and XBV_MEDIA_REQ_EJECT
 *	- Only applies to CDROM devices.
 *
 * XBV_MEDIA_REQ_NONE
 * 	- XBP_MEDIA_REQ property valud
 *	- Set and consumed by both the frontend and backend.
 *	- Indicates that there are no currently outstanding media requet
 *	  operations.
 *
 * XBV_MEDIA_REQ_LOCK
 * 	- XBP_MEDIA_REQ property valud
 *	- Set by the frontend, consumed by the backend.
 *	- Indicates to the backend that the currenct media is locked
 *	  and changes to the media (via xm block-configure for example)
 *	  should not be allowed.
 *
 * XBV_MEDIA_REQ_EJECT
 * 	- XBP_MEDIA_REQ property valud
 *	- Set by the frontend, consumed by the backend.
 *	- Indicates to the backend that the currenct media should be ejected.
 *	  This means that the backend should close it's connection to
 *	  the frontend device, close it's current backing store device/file,
 *	  and then set the media-req property to XBV_MEDIA_REQ_NONE.  (to
 *	  indicate that the eject operation is complete.)
 */
#define	XBP_MEDIA_REQ_SUP	"media-req-sup"	/* backend prop: boolean int */
#define	XBP_MEDIA_REQ		"media-req"	/* frontend prop: str */
#define	XBV_MEDIA_REQ_NONE	"none"		/* frontend prop val */
#define	XBV_MEDIA_REQ_LOCK	"lock"		/* frontend prop val */
#define	XBV_MEDIA_REQ_EJECT	"eject"		/* frontend prop val */

/*
 * Xen device class codes
 */
typedef enum {
	XEN_INVAL = -1,
	XEN_CONSOLE = 0,
	XEN_VNET,
	XEN_VBLK,
	XEN_XENBUS,
	XEN_DOMCAPS,
	XEN_BALLOON,
	XEN_EVTCHN,
	XEN_PRIVCMD,
	XEN_BLKTAP,
	XEN_LASTCLASS
} xendev_devclass_t;

/*
 * Hotplug request sent to userland event handler.
 */
typedef enum {
	XEN_HP_ADD,
	XEN_HP_REMOVE
} xendev_hotplug_cmd_t;

/*
 * Hotplug status.
 *
 * In fact, the Xen tools can write any arbitrary string into the
 * hotplug-status node. We represent the known values here - anything
 * else will be 'Unrecognized'.
 */
typedef enum {
	Unrecognized,
	Connected
} xendev_hotplug_state_t;

struct xendev_ppd {
	kmutex_t		xd_evt_lk;
	int			xd_evtchn;
	struct intrspec		xd_ispec;

	xendev_devclass_t	xd_devclass;
	domid_t			xd_domain;
	int			xd_vdevnum;

	kmutex_t		xd_ndi_lk;
	struct xenbus_device	xd_xsdev;
	struct xenbus_watch	xd_hp_watch;
	struct xenbus_watch	xd_bepath_watch;
	ddi_callback_id_t	xd_oe_ehid;
	ddi_callback_id_t	xd_hp_ehid;
	ddi_taskq_t		*xd_oe_taskq;
	ddi_taskq_t		*xd_hp_taskq;
	ddi_taskq_t		*xd_xb_watch_taskq;
	list_t			xd_xb_watches;
};

#define	XS_OE_STATE	"SUNW,xendev:otherend_state"
#define	XS_HP_STATE	"SUNW,xendev:hotplug_state"

/*
 * A device with xd_vdevnum == VDEV_NOXS does not participate in
 * xenstore.
 */
#define	VDEV_NOXS	(-1)

void	xendev_enum_class(dev_info_t *, xendev_devclass_t);
void	xendev_enum_all(dev_info_t *, boolean_t);
xendev_devclass_t	xendev_nodename_to_devclass(char *);
int	xendev_devclass_ipl(xendev_devclass_t);
struct intrspec *xendev_get_ispec(dev_info_t *, uint_t);
void	xvdi_suspend(dev_info_t *);
int	xvdi_resume(dev_info_t *);
int	xvdi_alloc_evtchn(dev_info_t *);
int	xvdi_bind_evtchn(dev_info_t *, evtchn_port_t);
void	xvdi_free_evtchn(dev_info_t *);
int	xvdi_add_event_handler(dev_info_t *, char *,
	void (*)(dev_info_t *, ddi_eventcookie_t, void *, void *),
	void *arg);
void	xvdi_remove_event_handler(dev_info_t *, char *);
int	xvdi_get_evtchn(dev_info_t *);
int	xvdi_get_vdevnum(dev_info_t *);
char	*xvdi_get_xsname(dev_info_t *);
char	*xvdi_get_oename(dev_info_t *);
domid_t	xvdi_get_oeid(dev_info_t *);
void	xvdi_dev_error(dev_info_t *, int, char *);
void	xvdi_fatal_error(dev_info_t *, int, char *);
void	xvdi_notify_oe(dev_info_t *);
int	xvdi_post_event(dev_info_t *, xendev_hotplug_cmd_t);
struct  xenbus_device *xvdi_get_xsd(dev_info_t *);
int	xvdi_switch_state(dev_info_t *, xenbus_transaction_t, XenbusState);
dev_info_t	*xvdi_create_dev(dev_info_t *, xendev_devclass_t,
    domid_t, int);
int	xvdi_init_dev(dev_info_t *);
void	xvdi_uninit_dev(dev_info_t *);
dev_info_t	*xvdi_find_dev(dev_info_t *, xendev_devclass_t, domid_t, int);

extern int xvdi_add_xb_watch_handler(dev_info_t *, const char *,
    const char *, xvdi_xb_watch_cb_t cb, void *);
extern void xvdi_remove_xb_watch_handlers(dev_info_t *);

/*
 * common ring interfaces
 */

/*
 * we need the pad between ring index
 * and the real ring containing requests/responses,
 * so that we can map comif_sring_t structure to
 * any xxxif_sring_t structure defined via macros in ring.h
 */
#define	SRINGPAD		48

typedef struct comif_sring {
	RING_IDX req_prod, req_event;
	RING_IDX rsp_prod, rsp_event;
	uint8_t  pad[SRINGPAD];
	/*
	 * variable length
	 * stores real request/response entries
	 * entry size is fixed per ring
	 */
	char ring[1];
} comif_sring_t;

typedef struct comif_ring_fe {
	/*
	 * keep the member names as defined in ring.h
	 * in order to make use of the pre-defined macros
	 */
	RING_IDX req_prod_pvt;
	RING_IDX rsp_cons;
	unsigned int nr_ents;
	comif_sring_t *sring;
} comif_ring_fe_t;

typedef struct comif_ring_be {
	/*
	 * keep the member names as defined in ring.h
	 * in order to make use of the pre-defined macros
	 */
	RING_IDX rsp_prod_pvt;
	RING_IDX req_cons;
	unsigned int nr_ents;
	comif_sring_t  *sring;
} comif_ring_be_t;

typedef union comif_ring {
	comif_ring_fe_t fr;
	comif_ring_be_t br;
} comif_ring_t;

typedef struct xendev_req {
	unsigned long next;
	void *req;
} xendev_req_t;

typedef struct xendev_ring {
	ddi_dma_handle_t xr_dma_hdl;
	ddi_acc_handle_t xr_acc_hdl;
	grant_handle_t xr_grant_hdl;
	caddr_t xr_vaddr;
	paddr_t xr_paddr;
	grant_ref_t xr_gref;
	int xr_entry_size;
	int xr_frontend;
	comif_ring_t xr_sring;
} xendev_ring_t;

int	xvdi_alloc_ring(dev_info_t *, size_t, size_t, grant_ref_t *,
	xendev_ring_t **);
void	xvdi_free_ring(xendev_ring_t *);
int	xvdi_map_ring(dev_info_t *, size_t, size_t, grant_ref_t,
	xendev_ring_t **);
void	xvdi_unmap_ring(xendev_ring_t *);
uint_t	xvdi_ring_avail_slots(xendev_ring_t *);
int	xvdi_ring_has_unconsumed_requests(xendev_ring_t *);
int	xvdi_ring_has_incomp_request(xendev_ring_t *);
int	xvdi_ring_has_unconsumed_responses(xendev_ring_t *);
void*	xvdi_ring_get_request(xendev_ring_t *);
int	xvdi_ring_push_request(xendev_ring_t *);
void*	xvdi_ring_get_response(xendev_ring_t *);
int	xvdi_ring_push_response(xendev_ring_t *);

#ifdef	__cplusplus
}
#endif

#endif	/* _SYS_XENDEV_H */