/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#include <sys/errno.h>
#include <sys/types.h>
#include <sys/conf.h>
#include <sys/kmem.h>
#include <sys/ddi.h>
#include <sys/stat.h>
#include <sys/sunddi.h>
#include <sys/file.h>
#include <sys/open.h>
#include <sys/modctl.h>
#include <sys/ddi_impldefs.h>
#include <sys/sysmacros.h>

#include <sys/ioat.h>

static int ioat_open(dev_t *devp, int flag, int otyp, cred_t *cred);
static int ioat_close(dev_t devp, int flag, int otyp, cred_t *cred);
static int ioat_attach(dev_info_t *devi, ddi_attach_cmd_t cmd);
static int ioat_detach(dev_info_t *devi, ddi_detach_cmd_t cmd);
static int ioat_getinfo(dev_info_t *dip, ddi_info_cmd_t cmd, void *arg,
    void **result);
static int ioat_quiesce(dev_info_t *dip);

static 	struct cb_ops ioat_cb_ops = {
	ioat_open,		/* cb_open */
	ioat_close,		/* cb_close */
	nodev,			/* cb_strategy */
	nodev,			/* cb_print */
	nodev,			/* cb_dump */
	nodev,			/* cb_read */
	nodev,			/* cb_write */
	ioat_ioctl,		/* cb_ioctl */
	nodev,			/* cb_devmap */
	nodev,			/* cb_mmap */
	nodev,			/* cb_segmap */
	nochpoll,		/* cb_chpoll */
	ddi_prop_op,		/* cb_prop_op */
	NULL,			/* cb_stream */
	D_NEW | D_MP | D_64BIT | D_DEVMAP,	/* cb_flag */
	CB_REV
};

static struct dev_ops ioat_dev_ops = {
	DEVO_REV,		/* devo_rev */
	0,			/* devo_refcnt */
	ioat_getinfo,		/* devo_getinfo */
	nulldev,		/* devo_identify */
	nulldev,		/* devo_probe */
	ioat_attach,		/* devo_attach */
	ioat_detach,		/* devo_detach */
	nodev,			/* devo_reset */
	&ioat_cb_ops,		/* devo_cb_ops */
	NULL,			/* devo_bus_ops */
	NULL,			/* devo_power */
	ioat_quiesce,		/* devo_quiesce */
};

static struct modldrv ioat_modldrv = {
	&mod_driverops,		/* Type of module.  This one is a driver */
	"ioat driver",		/* Name of the module. */
	&ioat_dev_ops,		/* driver ops */
};

static struct modlinkage ioat_modlinkage = {
	MODREV_1,
	(void *) &ioat_modldrv,
	NULL
};


void *ioat_statep;

static int ioat_chip_init(ioat_state_t *state);
static void ioat_chip_fini(ioat_state_t *state);
static int ioat_drv_init(ioat_state_t *state);
static void ioat_drv_fini(ioat_state_t *state);
static uint_t ioat_isr(caddr_t parm);
static void ioat_intr_enable(ioat_state_t *state);
static void ioat_intr_disable(ioat_state_t *state);
void ioat_detach_finish(ioat_state_t *state);


ddi_device_acc_attr_t ioat_acc_attr = {
	DDI_DEVICE_ATTR_V0,		/* devacc_attr_version */
	DDI_NEVERSWAP_ACC,		/* devacc_attr_endian_flags */
	DDI_STORECACHING_OK_ACC,	/* devacc_attr_dataorder */
	DDI_DEFAULT_ACC			/* devacc_attr_access */
};

/* dcopy callback interface */
dcopy_device_cb_t ioat_cb = {
	DCOPY_DEVICECB_V0,
	0,		/* reserved */
	ioat_channel_alloc,
	ioat_channel_free,
	ioat_cmd_alloc,
	ioat_cmd_free,
	ioat_cmd_post,
	ioat_cmd_poll,
	ioat_unregister_complete
};

/*
 * _init()
 */
int
_init(void)
{
	int e;

	e = ddi_soft_state_init(&ioat_statep, sizeof (ioat_state_t), 1);
	if (e != 0) {
		return (e);
	}

	e = mod_install(&ioat_modlinkage);
	if (e != 0) {
		ddi_soft_state_fini(&ioat_statep);
		return (e);
	}

	return (0);
}

/*
 * _info()
 */
int
_info(struct modinfo *modinfop)
{
	return (mod_info(&ioat_modlinkage, modinfop));
}

/*
 * _fini()
 */
int
_fini(void)
{
	int e;

	e = mod_remove(&ioat_modlinkage);
	if (e != 0) {
		return (e);
	}

	ddi_soft_state_fini(&ioat_statep);

	return (0);
}

/*
 * ioat_attach()
 */
static int
ioat_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
{
	ioat_state_t *state;
	int instance;
	int e;


	switch (cmd) {
	case DDI_ATTACH:
		break;

	case DDI_RESUME:
		instance = ddi_get_instance(dip);
		state = ddi_get_soft_state(ioat_statep, instance);
		if (state == NULL) {
			return (DDI_FAILURE);
		}
		e = ioat_channel_resume(state);
		if (e != DDI_SUCCESS) {
			return (DDI_FAILURE);
		}
		ioat_intr_enable(state);
		return (DDI_SUCCESS);

	default:
		return (DDI_FAILURE);
	}

	instance = ddi_get_instance(dip);
	e = ddi_soft_state_zalloc(ioat_statep, instance);
	if (e != DDI_SUCCESS) {
		return (DDI_FAILURE);
	}
	state = ddi_get_soft_state(ioat_statep, instance);
	if (state == NULL) {
		goto attachfail_get_soft_state;
	}

	state->is_dip = dip;
	state->is_instance = instance;

	/* setup the registers, save away some device info */
	e = ioat_chip_init(state);
	if (e != DDI_SUCCESS) {
		goto attachfail_chip_init;
	}

	/* initialize driver state, must be after chip init */
	e = ioat_drv_init(state);
	if (e != DDI_SUCCESS) {
		goto attachfail_drv_init;
	}

	/* create the minor node (for the ioctl) */
	e = ddi_create_minor_node(dip, "ioat", S_IFCHR, instance, DDI_PSEUDO,
	    0);
	if (e != DDI_SUCCESS) {
		goto attachfail_minor_node;
	}

	/* Enable device interrupts */
	ioat_intr_enable(state);

	/* Report that driver was loaded */
	ddi_report_dev(dip);

	/* register with dcopy */
	e = dcopy_device_register(state, &state->is_deviceinfo,
	    &state->is_device_handle);
	if (e != DCOPY_SUCCESS) {
		goto attachfail_register;
	}

	return (DDI_SUCCESS);

attachfail_register:
	ioat_intr_disable(state);
	ddi_remove_minor_node(dip, NULL);
attachfail_minor_node:
	ioat_drv_fini(state);
attachfail_drv_init:
	ioat_chip_fini(state);
attachfail_chip_init:
attachfail_get_soft_state:
	(void) ddi_soft_state_free(ioat_statep, instance);

	return (DDI_FAILURE);
}

/*
 * ioat_detach()
 */
static int
ioat_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
{
	ioat_state_t *state;
	int instance;
	int e;


	instance = ddi_get_instance(dip);
	state = ddi_get_soft_state(ioat_statep, instance);
	if (state == NULL) {
		return (DDI_FAILURE);
	}

	switch (cmd) {
	case DDI_DETACH:
		break;

	case DDI_SUSPEND:
		ioat_channel_suspend(state);
		return (DDI_SUCCESS);

	default:
		return (DDI_FAILURE);
	}

	/*
	 * try to unregister from dcopy.  Since this driver doesn't follow the
	 * traditional parent/child model, we may still be in use so we can't
	 * detach yet.
	 */
	e = dcopy_device_unregister(&state->is_device_handle);
	if (e != DCOPY_SUCCESS) {
		if (e == DCOPY_PENDING) {
			cmn_err(CE_NOTE, "device busy, performing asynchronous"
			    " detach\n");
		}
		return (DDI_FAILURE);
	}

	ioat_detach_finish(state);

	return (DDI_SUCCESS);
}

/*
 * ioat_getinfo()
 */
/*ARGSUSED*/
static int
ioat_getinfo(dev_info_t *dip, ddi_info_cmd_t cmd, void *arg, void **result)
{
	ioat_state_t *state;
	int instance;
	dev_t dev;
	int e;


	dev = (dev_t)arg;
	instance = getminor(dev);

	switch (cmd) {
	case DDI_INFO_DEVT2DEVINFO:
		state = ddi_get_soft_state(ioat_statep, instance);
		if (state == NULL) {
			return (DDI_FAILURE);
		}
		*result = (void *)state->is_dip;
		e = DDI_SUCCESS;
		break;

	case DDI_INFO_DEVT2INSTANCE:
		*result = (void *)(uintptr_t)instance;
		e = DDI_SUCCESS;
		break;

	default:
		e = DDI_FAILURE;
		break;
	}

	return (e);
}


/*
 * ioat_open()
 */
/*ARGSUSED*/
static int
ioat_open(dev_t *devp, int flag, int otyp, cred_t *cred)
{
	ioat_state_t *state;
	int instance;

	instance = getminor(*devp);
	state = ddi_get_soft_state(ioat_statep, instance);
	if (state == NULL) {
		return (ENXIO);
	}

	return (0);
}


/*
 * ioat_close()
 */
/*ARGSUSED*/
static int
ioat_close(dev_t devp, int flag, int otyp, cred_t *cred)
{
	return (0);
}


/*
 * ioat_chip_init()
 */
static int
ioat_chip_init(ioat_state_t *state)
{
	ddi_device_acc_attr_t attr;
	int e;


	attr.devacc_attr_version = DDI_DEVICE_ATTR_V0;
	attr.devacc_attr_endian_flags = DDI_NEVERSWAP_ACC;
	attr.devacc_attr_dataorder = DDI_STRICTORDER_ACC;

	e =  ddi_regs_map_setup(state->is_dip, 1, (caddr_t *)&state->is_genregs,
	    0, 0, &attr, &state->is_reg_handle);
	if (e != DDI_SUCCESS) {
		goto chipinitfail_regsmap;
	}

	/* save away ioat chip info */
	state->is_num_channels = (uint_t)ddi_get8(state->is_reg_handle,
	    &state->is_genregs[IOAT_CHANCNT]);

	/*
	 * If we get a bogus value, something is wrong with the H/W, fail to
	 * attach.
	 */
	if (state->is_num_channels == 0) {
		goto chipinitfail_numchan;
	}

	state->is_maxxfer = (uint_t)ddi_get8(state->is_reg_handle,
	    &state->is_genregs[IOAT_XFERCAP]);
	state->is_chanoff = (uintptr_t)ddi_get16(state->is_reg_handle,
	    (uint16_t *)&state->is_genregs[IOAT_PERPORT_OFF]);
	state->is_cbver = (uint_t)ddi_get8(state->is_reg_handle,
	    &state->is_genregs[IOAT_CBVER]);
	state->is_intrdelay = (uint_t)ddi_get16(state->is_reg_handle,
	    (uint16_t *)&state->is_genregs[IOAT_INTRDELAY]);
	state->is_status = (uint_t)ddi_get16(state->is_reg_handle,
	    (uint16_t *)&state->is_genregs[IOAT_CSSTATUS]);
	state->is_capabilities = (uint_t)ddi_get32(state->is_reg_handle,
	    (uint32_t *)&state->is_genregs[IOAT_DMACAPABILITY]);

	if (state->is_cbver & 0x10) {
		state->is_ver = IOAT_CBv1;
	} else if (state->is_cbver & 0x20) {
		state->is_ver = IOAT_CBv2;
	} else {
		goto chipinitfail_version;
	}

	return (DDI_SUCCESS);

chipinitfail_version:
chipinitfail_numchan:
	ddi_regs_map_free(&state->is_reg_handle);
chipinitfail_regsmap:
	return (DDI_FAILURE);
}


/*
 * ioat_chip_fini()
 */
static void
ioat_chip_fini(ioat_state_t *state)
{
	ddi_regs_map_free(&state->is_reg_handle);
}


/*
 * ioat_drv_init()
 */
static int
ioat_drv_init(ioat_state_t *state)
{
	ddi_acc_handle_t handle;
	int e;


	mutex_init(&state->is_mutex, NULL, MUTEX_DRIVER, NULL);

	state->is_deviceinfo.di_dip = state->is_dip;
	state->is_deviceinfo.di_num_dma = state->is_num_channels;
	state->is_deviceinfo.di_maxxfer = state->is_maxxfer;
	state->is_deviceinfo.di_capabilities = state->is_capabilities;
	state->is_deviceinfo.di_cb = &ioat_cb;

	e = pci_config_setup(state->is_dip, &handle);
	if (e != DDI_SUCCESS) {
		goto drvinitfail_config_setup;
	}

	/* read in Vendor ID */
	state->is_deviceinfo.di_id = (uint64_t)pci_config_get16(handle, 0);
	state->is_deviceinfo.di_id = state->is_deviceinfo.di_id << 16;

	/* read in Device ID */
	state->is_deviceinfo.di_id |= (uint64_t)pci_config_get16(handle, 2);
	state->is_deviceinfo.di_id = state->is_deviceinfo.di_id << 32;

	/* Add in chipset version */
	state->is_deviceinfo.di_id |= (uint64_t)state->is_cbver;
	pci_config_teardown(&handle);

	e = ddi_intr_hilevel(state->is_dip, 0);
	if (e != 0) {
		cmn_err(CE_WARN, "hilevel interrupt not supported\n");
		goto drvinitfail_hilevel;
	}

	/* we don't support MSIs for v2 yet */
	e = ddi_add_intr(state->is_dip, 0, NULL, NULL, ioat_isr,
	    (caddr_t)state);
	if (e != DDI_SUCCESS) {
		goto drvinitfail_add_intr;
	}

	e = ddi_get_iblock_cookie(state->is_dip, 0, &state->is_iblock_cookie);
	if (e != DDI_SUCCESS) {
		goto drvinitfail_iblock_cookie;
	}

	e = ioat_channel_init(state);
	if (e != DDI_SUCCESS) {
		goto drvinitfail_channel_init;
	}

	return (DDI_SUCCESS);

drvinitfail_channel_init:
drvinitfail_iblock_cookie:
	ddi_remove_intr(state->is_dip, 0, state->is_iblock_cookie);
drvinitfail_add_intr:
drvinitfail_hilevel:
drvinitfail_config_setup:
	mutex_destroy(&state->is_mutex);

	return (DDI_FAILURE);
}


/*
 * ioat_drv_fini()
 */
static void
ioat_drv_fini(ioat_state_t *state)
{
	ioat_channel_fini(state);
	ddi_remove_intr(state->is_dip, 0, state->is_iblock_cookie);
	mutex_destroy(&state->is_mutex);
}


/*
 * ioat_unregister_complete()
 */
void
ioat_unregister_complete(void *device_private, int status)
{
	ioat_state_t *state;


	state = device_private;

	if (status != DCOPY_SUCCESS) {
		cmn_err(CE_WARN, "asynchronous detach aborted\n");
		return;
	}

	cmn_err(CE_CONT, "detach completing\n");
	ioat_detach_finish(state);
}


/*
 * ioat_detach_finish()
 */
void
ioat_detach_finish(ioat_state_t *state)
{
	ioat_intr_disable(state);
	ddi_remove_minor_node(state->is_dip, NULL);
	ioat_drv_fini(state);
	ioat_chip_fini(state);
	(void) ddi_soft_state_free(ioat_statep, state->is_instance);
}


/*
 * ioat_intr_enable()
 */
static void
ioat_intr_enable(ioat_state_t *state)
{
	uint32_t intr_status;


	/* Clear any pending interrupts */
	intr_status = ddi_get32(state->is_reg_handle,
	    (uint32_t *)&state->is_genregs[IOAT_ATTNSTATUS]);
	if (intr_status != 0) {
		ddi_put32(state->is_reg_handle,
		    (uint32_t *)&state->is_genregs[IOAT_ATTNSTATUS],
		    intr_status);
	}

	/* Enable interrupts on the device */
	ddi_put8(state->is_reg_handle, &state->is_genregs[IOAT_INTRCTL],
	    IOAT_INTRCTL_MASTER_EN);
}


/*
 * ioat_intr_disable()
 */
static void
ioat_intr_disable(ioat_state_t *state)
{
	/*
	 * disable interrupts on the device. A read of the interrupt control
	 * register clears the enable bit.
	 */
	(void) ddi_get8(state->is_reg_handle,
	    &state->is_genregs[IOAT_INTRCTL]);
}


/*
 * ioat_isr()
 */
static uint_t
ioat_isr(caddr_t parm)
{
	uint32_t intr_status;
	ioat_state_t *state;
	uint8_t intrctrl;
	uint32_t chan;
	uint_t r;
	int i;

	state = (ioat_state_t *)parm;

	intrctrl = ddi_get8(state->is_reg_handle,
	    &state->is_genregs[IOAT_INTRCTL]);
	/* master interrupt enable should always be set */
	ASSERT(intrctrl & IOAT_INTRCTL_MASTER_EN);

	/* If the interrupt status bit isn't set, it's not ours */
	if (!(intrctrl & IOAT_INTRCTL_INTR_STAT)) {
		/* re-set master interrupt enable (since it clears on read) */
		ddi_put8(state->is_reg_handle,
		    &state->is_genregs[IOAT_INTRCTL], intrctrl);
		return (DDI_INTR_UNCLAIMED);
	}

	/* see which channels generated the interrupt */
	intr_status = ddi_get32(state->is_reg_handle,
	    (uint32_t *)&state->is_genregs[IOAT_ATTNSTATUS]);

	/* call the intr handler for the channels */
	r = DDI_INTR_UNCLAIMED;
	chan = 1;
	for (i = 0; i < state->is_num_channels; i++) {
		if (intr_status & chan) {
			ioat_channel_intr(&state->is_channel[i]);
			r = DDI_INTR_CLAIMED;
		}
		chan = chan << 1;
	}

	/*
	 * if interrupt status bit was set, there should have been an
	 * attention status bit set too.
	 */
	ASSERT(r == DDI_INTR_CLAIMED);

	/* re-set master interrupt enable (since it clears on read) */
	ddi_put8(state->is_reg_handle, &state->is_genregs[IOAT_INTRCTL],
	    intrctrl);

	return (r);
}

static int
ioat_quiesce(dev_info_t *dip)
{
	ioat_state_t *state;
	int instance;

	instance = ddi_get_instance(dip);
	state = ddi_get_soft_state(ioat_statep, instance);
	if (state == NULL) {
		return (DDI_FAILURE);
	}

	ioat_intr_disable(state);
	ioat_channel_quiesce(state);

	return (DDI_SUCCESS);
}
#
# CDDL HEADER START
#
# The contents of this file are subject to the terms of the
# Common Development and Distribution License (the "License").
# You may not use this file except in compliance with the License.
#
# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
# or http://www.opensolaris.org/os/licensing.
# See the License for the specific language governing permissions
# and limitations under the License.
#
# When distributing Covered Code, include this CDDL HEADER in each
# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
# If applicable, add the following below this CDDL HEADER, with the
# fields enclosed by brackets "[]" replaced with your own identifying
# information: Portions Copyright [yyyy] [name of copyright owner]
#
# CDDL HEADER END
#
#
# Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
# Use is subject to license terms.
#
#ident	"%Z%%M%	%I%	%E% SMI"

#
# force attach this driver to support misc/driver
ddi-forceattach=1;

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

/*
 * Copyright (c) 2009, Intel Corporation.
 * All rights reserved.
 */

#include <sys/errno.h>
#include <sys/types.h>
#include <sys/conf.h>
#include <sys/kmem.h>
#include <sys/ddi.h>
#include <sys/stat.h>
#include <sys/sunddi.h>
#include <sys/file.h>
#include <sys/open.h>
#include <sys/modctl.h>
#include <sys/ddi_impldefs.h>
#include <sys/sysmacros.h>
#include <vm/hat.h>
#include <vm/as.h>
#include <sys/mach_mmu.h>
#ifdef __xpv
#include <sys/hypervisor.h>
#endif

#include <sys/ioat.h>


extern ddi_device_acc_attr_t ioat_acc_attr;

/* dma attr for the descriptor rings */
ddi_dma_attr_t ioat_desc_dma_attr = {
	DMA_ATTR_V0,		/* dma_attr_version */
	0x0,			/* dma_attr_addr_lo */
	0xffffffffffffffff,	/* dma_attr_addr_hi */
	0xffffffff,		/* dma_attr_count_max */
	0x1000,			/* dma_attr_align */
	0x1,			/* dma_attr_burstsizes */
	0x1,			/* dma_attr_minxfer */
	0xffffffff,		/* dma_attr_maxxfer */
	0xffffffff,		/* dma_attr_seg */
	0x1,			/* dma_attr_sgllen */
	0x1,			/* dma_attr_granular */
	0x0,			/* dma_attr_flags */
};

/* dma attr for the completion buffers */
ddi_dma_attr_t ioat_cmpl_dma_attr = {
	DMA_ATTR_V0,		/* dma_attr_version */
	0x0,			/* dma_attr_addr_lo */
	0xffffffffffffffff,	/* dma_attr_addr_hi */
	0xffffffff,		/* dma_attr_count_max */
	0x40,			/* dma_attr_align */
	0x1,			/* dma_attr_burstsizes */
	0x1,			/* dma_attr_minxfer */
	0xffffffff,		/* dma_attr_maxxfer */
	0xffffffff,		/* dma_attr_seg */
	0x1,			/* dma_attr_sgllen */
	0x1,			/* dma_attr_granular */
	0x0,			/* dma_attr_flags */
};

static int ioat_completion_alloc(ioat_channel_t channel);
static void ioat_completion_free(ioat_channel_t channel);
static void ioat_channel_start(ioat_channel_t channel);
static void ioat_channel_reset(ioat_channel_t channel);

int ioat_ring_alloc(ioat_channel_t channel, uint_t desc_cnt);
void ioat_ring_free(ioat_channel_t channel);
void ioat_ring_seed(ioat_channel_t channel, ioat_chan_dma_desc_t *desc);
int ioat_ring_reserve(ioat_channel_t channel, ioat_channel_ring_t *ring,
    dcopy_cmd_t cmd);

static void ioat_cmd_post_copy(ioat_channel_ring_t *ring, uint64_t src_addr,
    uint64_t dest_addr, uint32_t size, uint32_t ctrl);
static void ioat_cmd_post_dca(ioat_channel_ring_t *ring, uint32_t dca_id);


/*
 * ioat_channel_init()
 */
int
ioat_channel_init(ioat_state_t *state)
{
	int i;

	/*
	 * initialize each dma channel's state which doesn't change across
	 * channel alloc/free.
	 */
	state->is_chansize = sizeof (struct ioat_channel_s) *
	    state->is_num_channels;
	state->is_channel = kmem_zalloc(state->is_chansize, KM_SLEEP);
	for (i = 0; i < state->is_num_channels; i++) {
		state->is_channel[i].ic_state = state;
		state->is_channel[i].ic_regs = (uint8_t *)
		    ((uintptr_t)state->is_genregs +
		    (uintptr_t)(IOAT_CHANNELREG_OFFSET * (i + 1)));
	}

	/* initial the allocator (from 0 to state->is_num_channels) */
	ioat_rs_init(state, 0, state->is_num_channels, &state->is_channel_rs);

	return (DDI_SUCCESS);
}


/*
 * ioat_channel_fini()
 */
void
ioat_channel_fini(ioat_state_t *state)
{
	ioat_rs_fini(&state->is_channel_rs);
	kmem_free(state->is_channel, state->is_chansize);
}


/*
 * ioat_channel_alloc()
 *   NOTE: We intentionaly don't handle DCOPY_SLEEP (if no channels are
 *	available)
 */
/*ARGSUSED*/
int
ioat_channel_alloc(void *device_private, dcopy_handle_t handle, int flags,
    uint_t size, dcopy_query_channel_t *info, void *channel_private)
{
#define	CHANSTRSIZE	20
	struct ioat_channel_s *channel;
	char chanstr[CHANSTRSIZE];
	ioat_channel_t *chan;
	ioat_state_t *state;
	size_t cmd_size;
	uint_t chan_num;
	uint32_t estat;
	int e;


	state = (ioat_state_t *)device_private;
	chan = (ioat_channel_t *)channel_private;

	/* allocate a H/W channel */
	e = ioat_rs_alloc(state->is_channel_rs, &chan_num);
	if (e != DDI_SUCCESS) {
		return (DCOPY_NORESOURCES);
	}

	channel = &state->is_channel[chan_num];
	channel->ic_inuse = B_TRUE;
	channel->ic_chan_num = chan_num;
	channel->ic_ver = state->is_ver;
	channel->ic_dca_active = B_FALSE;
	channel->ic_channel_state = IOAT_CHANNEL_OK;
	channel->ic_dcopy_handle = handle;

#ifdef	DEBUG
	{
		/* if we're cbv2, verify that the V2 compatibility bit is set */
		uint16_t reg;
		if (channel->ic_ver == IOAT_CBv2) {
			reg = ddi_get16(state->is_reg_handle,
			    (uint16_t *)&channel->ic_regs[IOAT_CHAN_COMP]);
			ASSERT(reg & 0x2);
		}
	}
#endif

	/*
	 * Configure DMA channel
	 *   Channel In Use
	 *   Error Interrupt Enable
	 *   Any Error Abort Enable
	 *   Error Completion Enable
	 */
	ddi_put16(state->is_reg_handle,
	    (uint16_t *)&channel->ic_regs[IOAT_CHAN_CTL], 0x011C);

	/* check channel error register, clear any errors */
	estat = ddi_get32(state->is_reg_handle,
	    (uint32_t *)&channel->ic_regs[IOAT_CHAN_ERR]);
	if (estat != 0) {
#ifdef	DEBUG
		cmn_err(CE_CONT, "cleared errors (0x%x) before channel (%d) "
		    "enable\n", estat, channel->ic_chan_num);
#endif
		ddi_put32(state->is_reg_handle,
		    (uint32_t *)&channel->ic_regs[IOAT_CHAN_ERR], estat);
	}

	/* allocate and initialize the descriptor buf */
	e = ioat_ring_alloc(channel, size);
	if (e != DDI_SUCCESS) {
		goto chinitfail_desc_alloc;
	}

	/* allocate and initialize the completion space */
	e = ioat_completion_alloc(channel);
	if (e != DDI_SUCCESS) {
		goto chinitfail_completion_alloc;
	}

	/* setup kmem_cache for commands */
	cmd_size = sizeof (struct dcopy_cmd_s) +
	    sizeof (struct dcopy_cmd_priv_s) +
	    sizeof (struct ioat_cmd_private_s);
	(void) snprintf(chanstr, CHANSTRSIZE, "ioat%dchan%dcmd",
	    state->is_instance, channel->ic_chan_num);
	channel->ic_cmd_cache = kmem_cache_create(chanstr, cmd_size, 64,
	    NULL, NULL, NULL, NULL, NULL, 0);
	if (channel->ic_cmd_cache == NULL) {
		goto chinitfail_kmem_cache;
	}

	/* start-up the channel */
	ioat_channel_start(channel);

	/* fill in the channel info returned to dcopy */
	info->qc_version = DCOPY_QUERY_CHANNEL_V0;
	info->qc_id = state->is_deviceinfo.di_id;
	info->qc_capabilities = (uint64_t)state->is_capabilities;
	info->qc_channel_size = (uint64_t)size;
	info->qc_chan_num = (uint64_t)channel->ic_chan_num;
	if (channel->ic_ver == IOAT_CBv1) {
		info->qc_dca_supported = B_FALSE;
	} else {
		if (info->qc_capabilities & IOAT_DMACAP_DCA) {
			info->qc_dca_supported = B_TRUE;
		} else {
			info->qc_dca_supported = B_FALSE;
		}
	}

	*chan = channel;

	return (DCOPY_SUCCESS);

chinitfail_kmem_cache:
	ioat_completion_free(channel);
chinitfail_completion_alloc:
	ioat_ring_free(channel);
chinitfail_desc_alloc:
	return (DCOPY_FAILURE);
}


/*
 * ioat_channel_suspend()
 */
/*ARGSUSED*/
void
ioat_channel_suspend(ioat_state_t *state)
{
	/*
	 * normally you would disable interrupts and reset the H/W here. But
	 * since the suspend framework doesn't know who is using us, it may
	 * not suspend their I/O before us.  Since we won't actively be doing
	 * any DMA or interrupts unless someone asks us to, it's safe to not
	 * do anything here.
	 */
}


/*
 * ioat_channel_resume()
 */
int
ioat_channel_resume(ioat_state_t *state)
{
	ioat_channel_ring_t *ring;
	ioat_channel_t channel;
	uint32_t estat;
	int i;


	for (i = 0; i < state->is_num_channels; i++) {
		channel = &state->is_channel[i];
		ring = channel->ic_ring;

		if (!channel->ic_inuse) {
			continue;
		}

		/*
		 * Configure DMA channel
		 *   Channel In Use
		 *   Error Interrupt Enable
		 *   Any Error Abort Enable
		 *   Error Completion Enable
		 */
		ddi_put16(state->is_reg_handle,
		    (uint16_t *)&channel->ic_regs[IOAT_CHAN_CTL], 0x011C);

		/* check channel error register, clear any errors */
		estat = ddi_get32(state->is_reg_handle,
		    (uint32_t *)&channel->ic_regs[IOAT_CHAN_ERR]);
		if (estat != 0) {
#ifdef	DEBUG
			cmn_err(CE_CONT, "cleared errors (0x%x) before channel"
			    " (%d) enable\n", estat, channel->ic_chan_num);
#endif
			ddi_put32(state->is_reg_handle,
			    (uint32_t *)&channel->ic_regs[IOAT_CHAN_ERR],
			    estat);
		}

		/* Re-initialize the ring */
		bzero(ring->cr_desc, channel->ic_desc_alloc_size);
		/* write the physical address into the chain address register */
		if (channel->ic_ver == IOAT_CBv1) {
			ddi_put32(state->is_reg_handle,
			    (uint32_t *)&channel->ic_regs[IOAT_V1_CHAN_ADDR_LO],
			    (uint32_t)(ring->cr_phys_desc & 0xffffffff));
			ddi_put32(state->is_reg_handle,
			    (uint32_t *)&channel->ic_regs[IOAT_V1_CHAN_ADDR_HI],
			    (uint32_t)(ring->cr_phys_desc >> 32));
		} else {
			ASSERT(channel->ic_ver == IOAT_CBv2);
			ddi_put32(state->is_reg_handle,
			    (uint32_t *)&channel->ic_regs[IOAT_V2_CHAN_ADDR_LO],
			    (uint32_t)(ring->cr_phys_desc & 0xffffffff));
			ddi_put32(state->is_reg_handle,
			    (uint32_t *)&channel->ic_regs[IOAT_V2_CHAN_ADDR_HI],
			    (uint32_t)(ring->cr_phys_desc >> 32));
		}

		/* re-initialize the completion buffer */
		bzero((void *)channel->ic_cmpl, channel->ic_cmpl_alloc_size);
		/* write the phys addr into the completion address register */
		ddi_put32(state->is_reg_handle,
		    (uint32_t *)&channel->ic_regs[IOAT_CHAN_CMPL_LO],
		    (uint32_t)(channel->ic_phys_cmpl & 0xffffffff));
		ddi_put32(state->is_reg_handle,
		    (uint32_t *)&channel->ic_regs[IOAT_CHAN_CMPL_HI],
		    (uint32_t)(channel->ic_phys_cmpl >> 32));

		/* start-up the channel */
		ioat_channel_start(channel);

	}

	return (DDI_SUCCESS);
}

/*
 * quiesce(9E) entry point.
 *
 * This function is called when the system is single-threaded at high
 * PIL with preemption disabled. Therefore, this function must not be
 * blocked.
 *
 * This function returns DDI_SUCCESS on success, or DDI_FAILURE on failure.
 * DDI_FAILURE indicates an error condition and should almost never happen.
 */
void
ioat_channel_quiesce(ioat_state_t *state)
{
	int i;

	/*
	 * Walk through all channels and quiesce
	 */
	for (i = 0; i < state->is_num_channels; i++) {

		ioat_channel_t	channel = state->is_channel + i;

		if (!channel->ic_inuse)
			continue;

		/* disable the interrupts */
		ddi_put16(state->is_reg_handle,
		    (uint16_t *)&channel->ic_regs[IOAT_CHAN_CTL],
		    0x0);

		ioat_channel_reset(channel);
	}
}


/*
 * ioat_channel_free()
 */
void
ioat_channel_free(void *channel_private)
{
	struct ioat_channel_s *channel;
	ioat_channel_t *chan;
	ioat_state_t *state;
	uint_t chan_num;


	chan = (ioat_channel_t *)channel_private;
	channel = *chan;

	state = channel->ic_state;
	chan_num = channel->ic_chan_num;

	/* disable the interrupts */
	ddi_put16(state->is_reg_handle,
	    (uint16_t *)&channel->ic_regs[IOAT_CHAN_CTL], 0x0);

	ioat_channel_reset(channel);

	/* cleanup command cache */
	kmem_cache_destroy(channel->ic_cmd_cache);

	/* clean-up/free-up the completion space and descriptors */
	ioat_completion_free(channel);
	ioat_ring_free(channel);

	channel->ic_inuse = B_FALSE;

	/* free the H/W DMA engine */
	ioat_rs_free(state->is_channel_rs, chan_num);

	*chan = NULL;
}


/*
 * ioat_channel_intr()
 */
void
ioat_channel_intr(ioat_channel_t channel)
{
	ioat_state_t *state;
	uint16_t chanctrl;
	uint32_t chanerr;
	uint32_t status;


	state = channel->ic_state;

	if (channel->ic_ver == IOAT_CBv1) {
		status = ddi_get32(state->is_reg_handle,
		    (uint32_t *)&channel->ic_regs[IOAT_V1_CHAN_STS_LO]);
	} else {
		ASSERT(channel->ic_ver == IOAT_CBv2);
		status = ddi_get32(state->is_reg_handle,
		    (uint32_t *)&channel->ic_regs[IOAT_V2_CHAN_STS_LO]);
	}

	/* if that status isn't ACTIVE or IDLE, the channel has failed */
	if (status & IOAT_CHAN_STS_FAIL_MASK) {
		chanerr = ddi_get32(state->is_reg_handle,
		    (uint32_t *)&channel->ic_regs[IOAT_CHAN_ERR]);
		cmn_err(CE_WARN, "channel(%d) fatal failure! "
		    "chanstat_lo=0x%X; chanerr=0x%X\n",
		    channel->ic_chan_num, status, chanerr);
		channel->ic_channel_state = IOAT_CHANNEL_IN_FAILURE;
		ioat_channel_reset(channel);

		return;
	}

	/*
	 * clear interrupt disable bit if set (it's a RW1C). Read it back to
	 * ensure the write completes.
	 */
	chanctrl = ddi_get16(state->is_reg_handle,
	    (uint16_t *)&channel->ic_regs[IOAT_CHAN_CTL]);
	ddi_put16(state->is_reg_handle,
	    (uint16_t *)&channel->ic_regs[IOAT_CHAN_CTL], chanctrl);
	(void) ddi_get16(state->is_reg_handle,
	    (uint16_t *)&channel->ic_regs[IOAT_CHAN_CTL]);

	/* tell dcopy we have seen a completion on this channel */
	dcopy_device_channel_notify(channel->ic_dcopy_handle, DCOPY_COMPLETION);
}


/*
 * ioat_channel_start()
 */
void
ioat_channel_start(ioat_channel_t channel)
{
	ioat_chan_dma_desc_t desc;

	/* set the first descriptor up as a NULL descriptor */
	bzero(&desc, sizeof (desc));
	desc.dd_size = 0;
	desc.dd_ctrl = IOAT_DESC_CTRL_OP_DMA | IOAT_DESC_DMACTRL_NULL |
	    IOAT_DESC_CTRL_CMPL;
	desc.dd_next_desc = 0x0;

	/* setup the very first descriptor */
	ioat_ring_seed(channel, &desc);
}


/*
 * ioat_channel_reset()
 */
void
ioat_channel_reset(ioat_channel_t channel)
{
	ioat_state_t *state;

	state = channel->ic_state;

	/* hit the reset bit */
	if (channel->ic_ver == IOAT_CBv1) {
		ddi_put8(state->is_reg_handle,
		    &channel->ic_regs[IOAT_V1_CHAN_CMD], 0x20);
	} else {
		ASSERT(channel->ic_ver == IOAT_CBv2);
		ddi_put8(state->is_reg_handle,
		    &channel->ic_regs[IOAT_V2_CHAN_CMD], 0x20);
	}
}


/*
 * ioat_completion_alloc()
 */
int
ioat_completion_alloc(ioat_channel_t channel)
{
	ioat_state_t *state;
	size_t real_length;
	uint_t cookie_cnt;
	int e;


	state = channel->ic_state;

	/*
	 * allocate memory for the completion status, zero it out, and get
	 * the paddr. We'll allocate a physically contiguous cache line.
	 */
	e = ddi_dma_alloc_handle(state->is_dip, &ioat_cmpl_dma_attr,
	    DDI_DMA_SLEEP, NULL, &channel->ic_cmpl_dma_handle);
	if (e != DDI_SUCCESS) {
		goto cmplallocfail_alloc_handle;
	}
	channel->ic_cmpl_alloc_size = 64;
	e = ddi_dma_mem_alloc(channel->ic_cmpl_dma_handle,
	    channel->ic_cmpl_alloc_size, &ioat_acc_attr,
	    DDI_DMA_CONSISTENT, DDI_DMA_SLEEP, NULL,
	    (caddr_t *)&channel->ic_cmpl, &real_length,
	    &channel->ic_cmpl_handle);
	if (e != DDI_SUCCESS) {
		goto cmplallocfail_mem_alloc;
	}
	bzero((void *)channel->ic_cmpl, channel->ic_cmpl_alloc_size);
	e = ddi_dma_addr_bind_handle(channel->ic_cmpl_dma_handle, NULL,
	    (caddr_t)channel->ic_cmpl, channel->ic_cmpl_alloc_size,
	    DDI_DMA_RDWR | DDI_DMA_CONSISTENT, DDI_DMA_SLEEP, NULL,
	    &channel->ic_cmpl_cookie, &cookie_cnt);
	if (e != DDI_SUCCESS) {
		goto cmplallocfail_addr_bind;
	}
	ASSERT(cookie_cnt == 1);
	ASSERT(channel->ic_cmpl_cookie.dmac_size ==
	    channel->ic_cmpl_alloc_size);
	channel->ic_phys_cmpl = channel->ic_cmpl_cookie.dmac_laddress;

	/* write the physical address into the completion address register */
	ddi_put32(state->is_reg_handle,
	    (uint32_t *)&channel->ic_regs[IOAT_CHAN_CMPL_LO],
	    (uint32_t)(channel->ic_phys_cmpl & 0xffffffff));
	ddi_put32(state->is_reg_handle,
	    (uint32_t *)&channel->ic_regs[IOAT_CHAN_CMPL_HI],
	    (uint32_t)(channel->ic_phys_cmpl >> 32));

	return (DDI_SUCCESS);

cmplallocfail_addr_bind:
	ddi_dma_mem_free(&channel->ic_desc_handle);
cmplallocfail_mem_alloc:
	ddi_dma_free_handle(&channel->ic_desc_dma_handle);
cmplallocfail_alloc_handle:
	return (DDI_FAILURE);
}


/*
 * ioat_completion_free()
 */
void
ioat_completion_free(ioat_channel_t channel)
{
	ioat_state_t *state;

	state = channel->ic_state;

	/* reset the completion address register */
	ddi_put32(state->is_reg_handle,
	    (uint32_t *)&channel->ic_regs[IOAT_CHAN_CMPL_LO], 0x0);
	ddi_put32(state->is_reg_handle,
	    (uint32_t *)&channel->ic_regs[IOAT_CHAN_CMPL_HI], 0x0);

	/* unbind, then free up the memory, dma handle */
	(void) ddi_dma_unbind_handle(channel->ic_cmpl_dma_handle);
	ddi_dma_mem_free(&channel->ic_cmpl_handle);
	ddi_dma_free_handle(&channel->ic_cmpl_dma_handle);
}

/*
 * ioat_ring_alloc()
 */
int
ioat_ring_alloc(ioat_channel_t channel, uint_t desc_cnt)
{
	ioat_channel_ring_t *ring;
	ioat_state_t *state;
	size_t real_length;
	uint_t cookie_cnt;
	int e;


	state = channel->ic_state;

	ring = kmem_zalloc(sizeof (ioat_channel_ring_t), KM_SLEEP);
	channel->ic_ring = ring;
	ring->cr_chan = channel;
	ring->cr_post_cnt = 0;

	mutex_init(&ring->cr_cmpl_mutex, NULL, MUTEX_DRIVER,
	    channel->ic_state->is_iblock_cookie);
	mutex_init(&ring->cr_desc_mutex, NULL, MUTEX_DRIVER,
	    channel->ic_state->is_iblock_cookie);

	/*
	 * allocate memory for the ring, zero it out, and get the paddr.
	 * We'll allocate a physically contiguous chunck of memory  which
	 * simplifies the completion logic.
	 */
	e = ddi_dma_alloc_handle(state->is_dip, &ioat_desc_dma_attr,
	    DDI_DMA_SLEEP, NULL, &channel->ic_desc_dma_handle);
	if (e != DDI_SUCCESS) {
		goto ringallocfail_alloc_handle;
	}
	/*
	 * allocate one extra descriptor so we can simplify the empty/full
	 * logic. Then round that number up to a whole multiple of 4.
	 */
	channel->ic_chan_desc_cnt = ((desc_cnt + 1) + 3) & ~0x3;
	ring->cr_desc_last = channel->ic_chan_desc_cnt - 1;
	channel->ic_desc_alloc_size = channel->ic_chan_desc_cnt *
	    sizeof (ioat_chan_desc_t);
	e = ddi_dma_mem_alloc(channel->ic_desc_dma_handle,
	    channel->ic_desc_alloc_size, &ioat_acc_attr,
	    DDI_DMA_CONSISTENT, DDI_DMA_SLEEP, NULL,
	    (caddr_t *)&ring->cr_desc, &real_length, &channel->ic_desc_handle);
	if (e != DDI_SUCCESS) {
		goto ringallocfail_mem_alloc;
	}
	bzero(ring->cr_desc, channel->ic_desc_alloc_size);
	e = ddi_dma_addr_bind_handle(channel->ic_desc_dma_handle, NULL,
	    (caddr_t)ring->cr_desc, channel->ic_desc_alloc_size,
	    DDI_DMA_RDWR | DDI_DMA_CONSISTENT, DDI_DMA_SLEEP, NULL,
	    &channel->ic_desc_cookies, &cookie_cnt);
	if (e != DDI_SUCCESS) {
		goto ringallocfail_addr_bind;
	}
	ASSERT(cookie_cnt == 1);
	ASSERT(channel->ic_desc_cookies.dmac_size ==
	    channel->ic_desc_alloc_size);
	ring->cr_phys_desc = channel->ic_desc_cookies.dmac_laddress;

	/* write the physical address into the chain address register */
	if (channel->ic_ver == IOAT_CBv1) {
		ddi_put32(state->is_reg_handle,
		    (uint32_t *)&channel->ic_regs[IOAT_V1_CHAN_ADDR_LO],
		    (uint32_t)(ring->cr_phys_desc & 0xffffffff));
		ddi_put32(state->is_reg_handle,
		    (uint32_t *)&channel->ic_regs[IOAT_V1_CHAN_ADDR_HI],
		    (uint32_t)(ring->cr_phys_desc >> 32));
	} else {
		ASSERT(channel->ic_ver == IOAT_CBv2);
		ddi_put32(state->is_reg_handle,
		    (uint32_t *)&channel->ic_regs[IOAT_V2_CHAN_ADDR_LO],
		    (uint32_t)(ring->cr_phys_desc & 0xffffffff));
		ddi_put32(state->is_reg_handle,
		    (uint32_t *)&channel->ic_regs[IOAT_V2_CHAN_ADDR_HI],
		    (uint32_t)(ring->cr_phys_desc >> 32));
	}

	return (DCOPY_SUCCESS);

ringallocfail_addr_bind:
	ddi_dma_mem_free(&channel->ic_desc_handle);
ringallocfail_mem_alloc:
	ddi_dma_free_handle(&channel->ic_desc_dma_handle);
ringallocfail_alloc_handle:
	mutex_destroy(&ring->cr_desc_mutex);
	mutex_destroy(&ring->cr_cmpl_mutex);
	kmem_free(channel->ic_ring, sizeof (ioat_channel_ring_t));

	return (DCOPY_FAILURE);
}


/*
 * ioat_ring_free()
 */
void
ioat_ring_free(ioat_channel_t channel)
{
	ioat_state_t *state;


	state = channel->ic_state;

	/* reset the chain address register */
	if (channel->ic_ver == IOAT_CBv1) {
		ddi_put32(state->is_reg_handle,
		    (uint32_t *)&channel->ic_regs[IOAT_V1_CHAN_ADDR_LO], 0x0);
		ddi_put32(state->is_reg_handle,
		    (uint32_t *)&channel->ic_regs[IOAT_V1_CHAN_ADDR_HI], 0x0);
	} else {
		ASSERT(channel->ic_ver == IOAT_CBv2);
		ddi_put32(state->is_reg_handle,
		    (uint32_t *)&channel->ic_regs[IOAT_V2_CHAN_ADDR_LO], 0x0);
		ddi_put32(state->is_reg_handle,
		    (uint32_t *)&channel->ic_regs[IOAT_V2_CHAN_ADDR_HI], 0x0);
	}

	/* unbind, then free up the memory, dma handle */
	(void) ddi_dma_unbind_handle(channel->ic_desc_dma_handle);
	ddi_dma_mem_free(&channel->ic_desc_handle);
	ddi_dma_free_handle(&channel->ic_desc_dma_handle);

	mutex_destroy(&channel->ic_ring->cr_desc_mutex);
	mutex_destroy(&channel->ic_ring->cr_cmpl_mutex);
	kmem_free(channel->ic_ring, sizeof (ioat_channel_ring_t));

}


/*
 * ioat_ring_seed()
 *    write the first descriptor in the ring.
 */
void
ioat_ring_seed(ioat_channel_t channel, ioat_chan_dma_desc_t *in_desc)
{
	ioat_channel_ring_t *ring;
	ioat_chan_dma_desc_t *desc;
	ioat_chan_dma_desc_t *prev;
	ioat_state_t *state;


	state = channel->ic_state;
	ring = channel->ic_ring;

	/* init the completion state */
	ring->cr_cmpl_gen = 0x0;
	ring->cr_cmpl_last = 0x0;

	/* write in the descriptor and init the descriptor state */
	ring->cr_post_cnt++;
	channel->ic_ring->cr_desc[0] = *(ioat_chan_desc_t *)in_desc;
	ring->cr_desc_gen = 0;
	ring->cr_desc_prev = 0;
	ring->cr_desc_next = 1;

	if (channel->ic_ver == IOAT_CBv1) {
		/* hit the start bit */
		ddi_put8(state->is_reg_handle,
		    &channel->ic_regs[IOAT_V1_CHAN_CMD], 0x1);
	} else {
		/*
		 * if this is CBv2, link the descriptor to an empty
		 * descriptor
		 */
		ASSERT(ring->cr_chan->ic_ver == IOAT_CBv2);
		desc = (ioat_chan_dma_desc_t *)
		    &ring->cr_desc[ring->cr_desc_next];
		prev = (ioat_chan_dma_desc_t *)
		    &ring->cr_desc[ring->cr_desc_prev];

		desc->dd_ctrl = 0;
		desc->dd_next_desc = 0x0;

		prev->dd_next_desc = ring->cr_phys_desc +
		    (ring->cr_desc_next << 6);

		ddi_put16(state->is_reg_handle,
		    (uint16_t *)&channel->ic_regs[IOAT_V2_CHAN_CNT],
		    (uint16_t)1);
	}

}

/*
 * ioat_ring_loop()
 * Make the ring loop for CB v1
 * This function assume we are in the ring->cr_desc_mutex mutex context
 */
int
ioat_ring_loop(ioat_channel_ring_t *ring, dcopy_cmd_t cmd)
{
	uint64_t count;
	ioat_channel_t channel;
	ioat_chan_dma_desc_t *curr;
	ioat_cmd_private_t *prevpriv;
	ioat_cmd_private_t *currpriv;

	currpriv = NULL;
	channel = ring->cr_chan;
	ASSERT(channel->ic_ver == IOAT_CBv1);

	/*
	 * For each cmd in the command queue, check whether they are continuous
	 * in descriptor ring. Return error if not continuous.
	 */
	for (count = 0, prevpriv = NULL;
	    cmd != NULL && count <= channel->ic_chan_desc_cnt;
	    prevpriv = currpriv) {
		currpriv = cmd->dp_private->pr_device_cmd_private;
		if (prevpriv != NULL &&
		    currpriv->ip_index + 1 != prevpriv->ip_start &&
		    currpriv->ip_index + 1 != prevpriv->ip_start +
		    channel->ic_chan_desc_cnt) {
			/* Non-continuous, other commands get interleaved */
			return (DCOPY_FAILURE);
		}
		if (currpriv->ip_index < currpriv->ip_start) {
			count += channel->ic_chan_desc_cnt
			    + currpriv->ip_index - currpriv->ip_start + 1;
		} else {
			count += currpriv->ip_index - currpriv->ip_start + 1;
		}
		cmd = currpriv->ip_next;
	}
	/*
	 * Check for too many descriptors which would cause wrap around in
	 * descriptor ring. And make sure there is space for cancel operation.
	 */
	if (count >= channel->ic_chan_desc_cnt) {
		return (DCOPY_FAILURE);
	}

	/* Point next descriptor to header of chain. */
	curr = (ioat_chan_dma_desc_t *)&ring->cr_desc[ring->cr_desc_prev];
	curr->dd_next_desc = ring->cr_phys_desc + (currpriv->ip_start << 6);

	/* sync the last desc */
	(void) ddi_dma_sync(channel->ic_desc_dma_handle,
	    ring->cr_desc_prev << 6, 64, DDI_DMA_SYNC_FORDEV);

	return (DCOPY_SUCCESS);
}


/*
 * ioat_cmd_alloc()
 */
int
ioat_cmd_alloc(void *private, int flags, dcopy_cmd_t *cmd)
{
	ioat_cmd_private_t *priv;
	ioat_channel_t channel;
	dcopy_cmd_t oldcmd;
	int kmflag;


	channel = (ioat_channel_t)private;

	if (flags & DCOPY_NOSLEEP) {
		kmflag = KM_NOSLEEP;
	} else {
		kmflag = KM_SLEEP;
	}

	/* save the command passed incase DCOPY_ALLOC_LINK is set */
	oldcmd = *cmd;

	*cmd = kmem_cache_alloc(channel->ic_cmd_cache, kmflag);
	if (*cmd == NULL) {
		return (DCOPY_NORESOURCES);
	}

	/* setup the dcopy and ioat private state pointers */
	(*cmd)->dp_version = DCOPY_CMD_V0;
	(*cmd)->dp_cmd = 0;
	(*cmd)->dp_private = (struct dcopy_cmd_priv_s *)
	    ((uintptr_t)(*cmd) + sizeof (struct dcopy_cmd_s));
	(*cmd)->dp_private->pr_device_cmd_private =
	    (struct ioat_cmd_private_s *)((uintptr_t)(*cmd)->dp_private +
	    sizeof (struct dcopy_cmd_priv_s));

	/*
	 * if DCOPY_ALLOC_LINK is set, link the old command to the new one
	 * just allocated.
	 */
	priv = (*cmd)->dp_private->pr_device_cmd_private;
	if (flags & DCOPY_ALLOC_LINK) {
		priv->ip_next = oldcmd;
	} else {
		priv->ip_next = NULL;
	}

	return (DCOPY_SUCCESS);
}


/*
 * ioat_cmd_free()
 */
void
ioat_cmd_free(void *private, dcopy_cmd_t *cmdp)
{
	ioat_cmd_private_t *priv;
	ioat_channel_t channel;
	dcopy_cmd_t next;
	dcopy_cmd_t cmd;


	channel = (ioat_channel_t)private;
	cmd = *(cmdp);

	/*
	 * free all the commands in the chain (see DCOPY_ALLOC_LINK in
	 * ioat_cmd_alloc() for more info).
	 */
	while (cmd != NULL) {
		priv = cmd->dp_private->pr_device_cmd_private;
		next = priv->ip_next;
		kmem_cache_free(channel->ic_cmd_cache, cmd);
		cmd = next;
	}
	*cmdp = NULL;
}


/*
 * ioat_cmd_post()
 */
int
ioat_cmd_post(void *private, dcopy_cmd_t cmd)
{
	ioat_channel_ring_t *ring;
	ioat_cmd_private_t *priv;
	ioat_channel_t channel;
	ioat_state_t *state;
	uint64_t dest_paddr;
	uint64_t src_paddr;
	uint64_t dest_addr;
	uint32_t dest_size;
	uint64_t src_addr;
	uint32_t src_size;
	size_t xfer_size;
	uint32_t ctrl;
	size_t size;
	int e;


	channel = (ioat_channel_t)private;
	priv = cmd->dp_private->pr_device_cmd_private;

	state = channel->ic_state;
	ring = channel->ic_ring;

	/*
	 * Special support for DCOPY_CMD_LOOP option, only supported on CBv1.
	 * DCOPY_CMD_QUEUE should also be set if DCOPY_CMD_LOOP is set.
	 */
	if ((cmd->dp_flags & DCOPY_CMD_LOOP) &&
	    (channel->ic_ver != IOAT_CBv1 ||
	    (cmd->dp_flags & DCOPY_CMD_QUEUE))) {
		return (DCOPY_FAILURE);
	}

	if ((cmd->dp_flags & DCOPY_CMD_NOWAIT) == 0) {
		mutex_enter(&ring->cr_desc_mutex);

	/*
	 * Try to acquire mutex if NOWAIT flag is set.
	 * Return failure if failed to acquire mutex.
	 */
	} else if (mutex_tryenter(&ring->cr_desc_mutex) == 0) {
		return (DCOPY_FAILURE);
	}

	/* if the channel has had a fatal failure, return failure */
	if (channel->ic_channel_state == IOAT_CHANNEL_IN_FAILURE) {
		mutex_exit(&ring->cr_desc_mutex);
		return (DCOPY_FAILURE);
	}

	/* make sure we have space for the descriptors */
	e = ioat_ring_reserve(channel, ring, cmd);
	if (e != DCOPY_SUCCESS) {
		mutex_exit(&ring->cr_desc_mutex);
		return (DCOPY_NORESOURCES);
	}

	/* if we support DCA, and the DCA flag is set, post a DCA desc */
	if ((channel->ic_ver == IOAT_CBv2) &&
	    (cmd->dp_flags & DCOPY_CMD_DCA)) {
		ioat_cmd_post_dca(ring, cmd->dp_dca_id);
	}

	/*
	 * the dma copy may have to be broken up into multiple descriptors
	 * since we can't cross a page boundary.
	 */
	ASSERT(cmd->dp_version == DCOPY_CMD_V0);
	ASSERT(cmd->dp_cmd == DCOPY_CMD_COPY);
	src_addr = cmd->dp.copy.cc_source;
	dest_addr = cmd->dp.copy.cc_dest;
	size = cmd->dp.copy.cc_size;
	priv->ip_start = ring->cr_desc_next;
	while (size > 0) {
		src_paddr = pa_to_ma(src_addr);
		dest_paddr = pa_to_ma(dest_addr);

		/* adjust for any offset into the page */
		if ((src_addr & PAGEOFFSET) == 0) {
			src_size = PAGESIZE;
		} else {
			src_size = PAGESIZE - (src_addr & PAGEOFFSET);
		}
		if ((dest_addr & PAGEOFFSET) == 0) {
			dest_size = PAGESIZE;
		} else {
			dest_size = PAGESIZE - (dest_addr & PAGEOFFSET);
		}

		/* take the smallest of the three */
		xfer_size = MIN(src_size, dest_size);
		xfer_size = MIN(xfer_size, size);

		/*
		 * if this is the last descriptor, and we are supposed to
		 * generate a completion, generate a completion. same logic
		 * for interrupt.
		 */
		ctrl = 0;
		if (cmd->dp_flags & DCOPY_CMD_NOSRCSNP) {
			ctrl |= IOAT_DESC_CTRL_NOSRCSNP;
		}
		if (cmd->dp_flags & DCOPY_CMD_NODSTSNP) {
			ctrl |= IOAT_DESC_CTRL_NODSTSNP;
		}
		if (xfer_size == size) {
			if (!(cmd->dp_flags & DCOPY_CMD_NOSTAT)) {
				ctrl |= IOAT_DESC_CTRL_CMPL;
			}
			if ((cmd->dp_flags & DCOPY_CMD_INTR)) {
				ctrl |= IOAT_DESC_CTRL_INTR;
			}
		}

		ioat_cmd_post_copy(ring, src_paddr, dest_paddr, xfer_size,
		    ctrl);

		/* go to the next page */
		src_addr += xfer_size;
		dest_addr += xfer_size;
		size -= xfer_size;
	}

	/* save away the state so we can poll on it. */
	priv->ip_generation = ring->cr_desc_gen_prev;
	priv->ip_index = ring->cr_desc_prev;

	/* if queue not defined, tell the DMA engine about it */
	if (!(cmd->dp_flags & DCOPY_CMD_QUEUE)) {
		/*
		 * Link the ring to a loop (currently only for FIPE).
		 */
		if (cmd->dp_flags & DCOPY_CMD_LOOP) {
			e = ioat_ring_loop(ring, cmd);
			if (e != DCOPY_SUCCESS) {
				mutex_exit(&ring->cr_desc_mutex);
				return (DCOPY_FAILURE);
			}
		}

		if (channel->ic_ver == IOAT_CBv1) {
			ddi_put8(state->is_reg_handle,
			    (uint8_t *)&channel->ic_regs[IOAT_V1_CHAN_CMD],
			    0x2);
		} else {
			ASSERT(channel->ic_ver == IOAT_CBv2);
			ddi_put16(state->is_reg_handle,
			    (uint16_t *)&channel->ic_regs[IOAT_V2_CHAN_CNT],
			    (uint16_t)(ring->cr_post_cnt & 0xFFFF));
		}
	}

	mutex_exit(&ring->cr_desc_mutex);

	return (DCOPY_SUCCESS);
}


/*
 * ioat_cmd_post_dca()
 */
static void
ioat_cmd_post_dca(ioat_channel_ring_t *ring, uint32_t dca_id)
{
	ioat_chan_dca_desc_t *saved_prev;
	ioat_chan_dca_desc_t *desc;
	ioat_chan_dca_desc_t *prev;
	ioat_channel_t channel;
	uint64_t next_desc_phys;
	off_t prev_offset;
	off_t next_offset;


	channel = ring->cr_chan;
	desc = (ioat_chan_dca_desc_t *)&ring->cr_desc[ring->cr_desc_next];
	prev = (ioat_chan_dca_desc_t *)&ring->cr_desc[ring->cr_desc_prev];

	/* keep track of the number of descs posted for cbv2 */
	ring->cr_post_cnt++;

	/*
	 * post a context change desriptor. If dca has never been used on
	 * this channel, or if the id doesn't match the last id used on this
	 * channel, set CONTEXT_CHANGE bit and dca id, set dca state to active,
	 * and save away the id we're using.
	 */
	desc->dd_ctrl = IOAT_DESC_CTRL_OP_CNTX;
	desc->dd_next_desc = 0x0;
	if (!channel->ic_dca_active || (channel->ic_dca_current != dca_id)) {
		channel->ic_dca_active = B_TRUE;
		channel->ic_dca_current = dca_id;
		desc->dd_ctrl |= IOAT_DESC_CTRL_CNTX_CHNG;
		desc->dd_cntx = dca_id;
	}

	/*
	 * save next desc and prev offset for when we link the two
	 * descriptors together.
	 */
	saved_prev = prev;
	prev_offset = ring->cr_desc_prev << 6;
	next_offset = ring->cr_desc_next << 6;
	next_desc_phys = ring->cr_phys_desc + next_offset;

	/* save the current desc_next and desc_last for the completion */
	ring->cr_desc_prev = ring->cr_desc_next;
	ring->cr_desc_gen_prev = ring->cr_desc_gen;

	/* increment next/gen so it points to the next free desc */
	ring->cr_desc_next++;
	if (ring->cr_desc_next > ring->cr_desc_last) {
		ring->cr_desc_next = 0;
		ring->cr_desc_gen++;
	}

	/*
	 * if this is CBv2, link the descriptor to an empty descriptor. Since
	 * we always leave on desc empty to detect full, this works out.
	 */
	if (ring->cr_chan->ic_ver == IOAT_CBv2) {
		desc = (ioat_chan_dca_desc_t *)
		    &ring->cr_desc[ring->cr_desc_next];
		prev = (ioat_chan_dca_desc_t *)
		    &ring->cr_desc[ring->cr_desc_prev];
		desc->dd_ctrl = 0;
		desc->dd_next_desc = 0x0;
		(void) ddi_dma_sync(channel->ic_desc_dma_handle,
		    ring->cr_desc_next << 6, 64, DDI_DMA_SYNC_FORDEV);
		prev->dd_next_desc = ring->cr_phys_desc +
		    (ring->cr_desc_next << 6);
	}

	/* Put the descriptors physical address in the previous descriptor */
	/*LINTED:E_TRUE_LOGICAL_EXPR*/
	ASSERT(sizeof (ioat_chan_dca_desc_t) == 64);

	/* sync the current desc */
	(void) ddi_dma_sync(channel->ic_desc_dma_handle, next_offset, 64,
	    DDI_DMA_SYNC_FORDEV);

	/* update the previous desc and sync it too */
	saved_prev->dd_next_desc = next_desc_phys;
	(void) ddi_dma_sync(channel->ic_desc_dma_handle, prev_offset, 64,
	    DDI_DMA_SYNC_FORDEV);
}


/*
 * ioat_cmd_post_copy()
 *
 */
static void
ioat_cmd_post_copy(ioat_channel_ring_t *ring, uint64_t src_addr,
    uint64_t dest_addr, uint32_t size, uint32_t ctrl)
{
	ioat_chan_dma_desc_t *saved_prev;
	ioat_chan_dma_desc_t *desc;
	ioat_chan_dma_desc_t *prev;
	ioat_channel_t channel;
	uint64_t next_desc_phy;
	off_t prev_offset;
	off_t next_offset;


	channel = ring->cr_chan;
	desc = (ioat_chan_dma_desc_t *)&ring->cr_desc[ring->cr_desc_next];
	prev = (ioat_chan_dma_desc_t *)&ring->cr_desc[ring->cr_desc_prev];

	/* keep track of the number of descs posted for cbv2 */
	ring->cr_post_cnt++;

	/* write in the DMA desc */
	desc->dd_ctrl = IOAT_DESC_CTRL_OP_DMA | ctrl;
	desc->dd_size = size;
	desc->dd_src_paddr = src_addr;
	desc->dd_dest_paddr = dest_addr;
	desc->dd_next_desc = 0x0;

	/*
	 * save next desc and prev offset for when we link the two
	 * descriptors together.
	 */
	saved_prev = prev;
	prev_offset = ring->cr_desc_prev << 6;
	next_offset = ring->cr_desc_next << 6;
	next_desc_phy = ring->cr_phys_desc + next_offset;

	/* increment next/gen so it points to the next free desc */
	ring->cr_desc_prev = ring->cr_desc_next;
	ring->cr_desc_gen_prev = ring->cr_desc_gen;

	/* increment next/gen so it points to the next free desc */
	ring->cr_desc_next++;
	if (ring->cr_desc_next > ring->cr_desc_last) {
		ring->cr_desc_next = 0;
		ring->cr_desc_gen++;
	}

	/*
	 * if this is CBv2, link the descriptor to an empty descriptor. Since
	 * we always leave on desc empty to detect full, this works out.
	 */
	if (ring->cr_chan->ic_ver == IOAT_CBv2) {
		desc = (ioat_chan_dma_desc_t *)
		    &ring->cr_desc[ring->cr_desc_next];
		prev = (ioat_chan_dma_desc_t *)
		    &ring->cr_desc[ring->cr_desc_prev];
		desc->dd_size = 0;
		desc->dd_ctrl = 0;
		desc->dd_next_desc = 0x0;
		(void) ddi_dma_sync(channel->ic_desc_dma_handle,
		    ring->cr_desc_next << 6, 64, DDI_DMA_SYNC_FORDEV);
		prev->dd_next_desc = ring->cr_phys_desc +
		    (ring->cr_desc_next << 6);
	}

	/* Put the descriptors physical address in the previous descriptor */
	/*LINTED:E_TRUE_LOGICAL_EXPR*/
	ASSERT(sizeof (ioat_chan_dma_desc_t) == 64);

	/* sync the current desc */
	(void) ddi_dma_sync(channel->ic_desc_dma_handle, next_offset, 64,
	    DDI_DMA_SYNC_FORDEV);

	/* update the previous desc and sync it too */
	saved_prev->dd_next_desc = next_desc_phy;
	(void) ddi_dma_sync(channel->ic_desc_dma_handle, prev_offset, 64,
	    DDI_DMA_SYNC_FORDEV);
}


/*
 * ioat_cmd_poll()
 */
int
ioat_cmd_poll(void *private, dcopy_cmd_t cmd)
{
	ioat_channel_ring_t *ring;
	ioat_cmd_private_t *priv;
	ioat_channel_t channel;
	uint64_t generation;
	uint64_t last_cmpl;

	ASSERT(cmd != NULL);
	channel = (ioat_channel_t)private;
	priv = cmd->dp_private->pr_device_cmd_private;

	ring = channel->ic_ring;
	ASSERT(ring != NULL);

	if ((cmd->dp_flags & DCOPY_CMD_NOWAIT) == 0) {
		mutex_enter(&ring->cr_cmpl_mutex);

	/*
	 * Try to acquire mutex if NOWAIT flag is set.
	 * Return failure if failed to acquire mutex.
	 */
	} else if (mutex_tryenter(&ring->cr_cmpl_mutex) == 0) {
		return (DCOPY_FAILURE);
	}

	/* if the channel had a fatal failure, fail all polls */
	if ((channel->ic_channel_state == IOAT_CHANNEL_IN_FAILURE) ||
	    IOAT_CMPL_FAILED(channel)) {
		mutex_exit(&ring->cr_cmpl_mutex);
		return (DCOPY_FAILURE);
	}

	/*
	 * if the current completion is the same as the last time we read one,
	 * post is still pending, nothing further to do. We track completions
	 * as indexes into the ring since post uses VAs and the H/W returns
	 * PAs. We grab a snapshot of generation and last_cmpl in the mutex.
	 */
	(void) ddi_dma_sync(channel->ic_cmpl_dma_handle, 0, 0,
	    DDI_DMA_SYNC_FORCPU);
	last_cmpl = IOAT_CMPL_INDEX(channel);
	if (last_cmpl != ring->cr_cmpl_last) {
		/*
		 * if we wrapped the ring, increment the generation. Store
		 * the last cmpl. This logic assumes a physically contiguous
		 * ring.
		 */
		if (last_cmpl < ring->cr_cmpl_last) {
			ring->cr_cmpl_gen++;
		}
		ring->cr_cmpl_last = last_cmpl;
		generation = ring->cr_cmpl_gen;

	} else {
		generation = ring->cr_cmpl_gen;
	}

	mutex_exit(&ring->cr_cmpl_mutex);

	/*
	 * if cmd isn't passed in, well return.  Useful for updating the
	 * consumer pointer (ring->cr_cmpl_last).
	 */
	if (cmd->dp_flags & DCOPY_CMD_SYNC) {
		return (DCOPY_PENDING);
	}

	/*
	 * if the post's generation is old, this post has completed. No reason
	 * to go check the last completion. if the generation is the same
	 * and if the post is before or = to the last completion processed,
	 * the post has completed.
	 */
	if (priv->ip_generation < generation) {
		return (DCOPY_COMPLETED);
	} else if ((priv->ip_generation == generation) &&
	    (priv->ip_index <= last_cmpl)) {
		return (DCOPY_COMPLETED);
	}

	return (DCOPY_PENDING);
}


/*
 * ioat_ring_reserve()
 */
int
ioat_ring_reserve(ioat_channel_t channel, ioat_channel_ring_t *ring,
    dcopy_cmd_t cmd)
{
	uint64_t dest_addr;
	uint32_t dest_size;
	uint64_t src_addr;
	uint32_t src_size;
	size_t xfer_size;
	uint64_t desc;
	int num_desc;
	size_t size;
	int i;


	/*
	 * figure out how many descriptors we need. This can include a dca
	 * desc and multiple desc for a dma copy.
	 */
	num_desc = 0;
	if ((channel->ic_ver == IOAT_CBv2) &&
	    (cmd->dp_flags & DCOPY_CMD_DCA)) {
		num_desc++;
	}
	src_addr = cmd->dp.copy.cc_source;
	dest_addr = cmd->dp.copy.cc_dest;
	size = cmd->dp.copy.cc_size;
	while (size > 0) {
		num_desc++;

		/* adjust for any offset into the page */
		if ((src_addr & PAGEOFFSET) == 0) {
			src_size = PAGESIZE;
		} else {
			src_size = PAGESIZE - (src_addr & PAGEOFFSET);
		}
		if ((dest_addr & PAGEOFFSET) == 0) {
			dest_size = PAGESIZE;
		} else {
			dest_size = PAGESIZE - (dest_addr & PAGEOFFSET);
		}

		/* take the smallest of the three */
		xfer_size = MIN(src_size, dest_size);
		xfer_size = MIN(xfer_size, size);

		/* go to the next page */
		src_addr += xfer_size;
		dest_addr += xfer_size;
		size -= xfer_size;
	}

	/* Make sure we have space for these descriptors */
	desc = ring->cr_desc_next;
	for (i = 0; i < num_desc; i++) {

		/*
		 * if this is the last descriptor in the ring, see if the
		 * last completed descriptor is #0.
		 */
		if (desc == ring->cr_desc_last) {
			if (ring->cr_cmpl_last == 0) {
				/*
				 * if we think the ring is full, update where
				 * the H/W really is and check for full again.
				 */
				cmd->dp_flags |= DCOPY_CMD_SYNC;
				(void) ioat_cmd_poll(channel, cmd);
				cmd->dp_flags &= ~DCOPY_CMD_SYNC;
				if (ring->cr_cmpl_last == 0) {
					return (DCOPY_NORESOURCES);
				}
			}

			/*
			 * go to the next descriptor which is zero in this
			 * case.
			 */
			desc = 0;

		/*
		 * if this is not the last descriptor in the ring, see if
		 * the last completion we saw was the next descriptor.
		 */
		} else {
			if ((desc + 1) == ring->cr_cmpl_last) {
				/*
				 * if we think the ring is full, update where
				 * the H/W really is and check for full again.
				 */
				cmd->dp_flags |= DCOPY_CMD_SYNC;
				(void) ioat_cmd_poll(channel, cmd);
				cmd->dp_flags &= ~DCOPY_CMD_SYNC;
				if ((desc + 1) == ring->cr_cmpl_last) {
					return (DCOPY_NORESOURCES);
				}
			}

			/* go to the next descriptor */
			desc++;
		}
	}

	return (DCOPY_SUCCESS);
}
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#include <sys/errno.h>
#include <sys/types.h>
#include <sys/conf.h>
#include <sys/kmem.h>
#include <sys/ddi.h>
#include <sys/stat.h>
#include <sys/sunddi.h>
#include <sys/file.h>
#include <sys/open.h>
#include <sys/modctl.h>
#include <sys/ddi_impldefs.h>
#include <sys/sysmacros.h>

#include <vm/hat.h>
#include <vm/as.h>

#include <sys/ioat.h>


extern void *ioat_statep;
#define	ptob64(x)	(((uint64_t)(x)) << PAGESHIFT)

static int ioat_ioctl_rdreg(ioat_state_t *state, void *arg, int mode);
#ifdef	DEBUG
static int ioat_ioctl_wrreg(ioat_state_t *state, void *arg, int mode);
static int ioat_ioctl_test(ioat_state_t *state, void *arg, int mode);
#endif

/*
 * ioat_ioctl()
 */
/*ARGSUSED*/
int
ioat_ioctl(dev_t dev, int cmd, intptr_t arg, int mode, cred_t *cred, int *rval)
{
	ioat_state_t *state;
	int instance;
	int e;


	e = drv_priv(cred);
	if (e != 0) {
		return (EPERM);
	}
	instance = getminor(dev);
	if (instance == -1) {
		return (EBADF);
	}
	state = ddi_get_soft_state(ioat_statep, instance);
	if (state == NULL) {
		return (EBADF);
	}

	switch (cmd) {
	case IOAT_IOCTL_READ_REG:
		e = ioat_ioctl_rdreg(state, (void *)arg, mode);
		break;
#ifdef	DEBUG
	case IOAT_IOCTL_WRITE_REG:
		e = ioat_ioctl_wrreg(state, (void *)arg, mode);
		break;
	case IOAT_IOCTL_TEST:
		e = ioat_ioctl_test(state, (void *)arg, mode);
		break;
#endif

	default:
		e = ENXIO;
	}

	return (e);
}


/*
 * ioat_ioctl_rdreg()
 */
static int
ioat_ioctl_rdreg(ioat_state_t *state, void *arg, int mode)
{
	ioat_ioctl_rdreg_t rdreg;
	int e;


	e = ddi_copyin(arg, &rdreg, sizeof (ioat_ioctl_rdreg_t), mode);
	if (e != 0) {
		return (EFAULT);
	}

	/*
	 * read a device register, where size is read size in bits, addr is
	 * the offset into MMIO registers.
	 */
	switch (rdreg.size) {
	case 8:
		rdreg.data = (uint64_t)ddi_get8(state->is_reg_handle,
		    (uint8_t *)&state->is_genregs[rdreg.addr]);
		break;
	case 16:
		rdreg.data = (uint64_t)ddi_get16(state->is_reg_handle,
		    (uint16_t *)&state->is_genregs[rdreg.addr]);
		break;
	case 32:
		rdreg.data = (uint64_t)ddi_get32(state->is_reg_handle,
		    (uint32_t *)&state->is_genregs[rdreg.addr]);
		break;
	case 64:
		rdreg.data = (uint64_t)ddi_get64(state->is_reg_handle,
		    (uint64_t *)&state->is_genregs[rdreg.addr]);
		break;
	default:
		return (EFAULT);
	}

	e = ddi_copyout(&rdreg, arg, sizeof (ioat_ioctl_rdreg_t), mode);
	if (e != 0) {
		return (EFAULT);
	}

	return (0);
}


#ifdef	DEBUG
/*
 * ioat_ioctl_wrreg()
 */
static int
ioat_ioctl_wrreg(ioat_state_t *state, void *arg, int mode)
{
	ioat_ioctl_wrreg_t wrreg;
	int e;


	e = ddi_copyin(arg, &wrreg, sizeof (ioat_ioctl_wrreg_t), mode);
	if (e != 0) {
		return (EFAULT);
	}

	/*
	 * write a device register, where size is write size in bits, addr is
	 * the offset into MMIO registers.
	 */
	switch (wrreg.size) {
	case 8:
		ddi_put8(state->is_reg_handle,
		    (uint8_t *)&state->is_genregs[wrreg.addr],
		    (uint8_t)wrreg.data);
		break;
	case 16:
		ddi_put16(state->is_reg_handle,
		    (uint16_t *)&state->is_genregs[wrreg.addr],
		    (uint16_t)wrreg.data);
		break;
	case 32:
		ddi_put32(state->is_reg_handle,
		    (uint32_t *)&state->is_genregs[wrreg.addr],
		    (uint32_t)wrreg.data);
		break;
	case 64:
		ddi_put64(state->is_reg_handle,
		    (uint64_t *)&state->is_genregs[wrreg.addr],
		    (uint64_t)wrreg.data);
		break;
	default:
		return (EFAULT);
	}

	return (0);
}


/*
 * ioat_ioctl_test()
 */
/*ARGSUSED*/
static int
ioat_ioctl_test(ioat_state_t *state, void *arg, int mode)
{
	dcopy_handle_t channel;
	dcopy_cmd_t cmd;
	uint8_t *source;
	uint_t buf_size;
	uint_t poll_cnt;
	uint8_t *dest;
	uint8_t *buf;
	int flags;
	int i;
	int e;


	/* allocate 2 paged aligned 4k pages */
	buf_size = 0x1000;
	buf = kmem_zalloc((buf_size * 2) + 0x1000, KM_SLEEP);
	source = (uint8_t *)(((uintptr_t)buf + PAGEOFFSET) & PAGEMASK);
	dest = source + buf_size;

	/* Init source buffer */
	for (i = 0; i < buf_size; i++) {
		source[i] = (uint8_t)(i & 0xFF);
	}

	/* allocate a DMA channel */
	e = dcopy_alloc(DCOPY_SLEEP, &channel);
	if (e != DCOPY_SUCCESS) {
		cmn_err(CE_CONT, "dcopy_alloc() failed\n");
		goto testfail_alloc;
	}

	/*
	 * post 32 DMA copy's from dest to dest.  These will complete in order
	 * so they won't stomp on each other. We don't care about the data
	 * right now which is why we go dest to dest.
	 */
	flags = DCOPY_SLEEP;
	for (i = 0; i < 32; i++) {
		/*
		 * if this is the second command, link the commands from here
		 * on out. We only want to keep track of the last command. We
		 * will poll on the last command completing (which infers that
		 * the other commands completed). If any of the previous
		 * commands fail, so will the last one. Linking the commands
		 * also allows us to only call free for the last command. free
		 * will free up the entire chain of commands.
		 */
		if (i == 1) {
			flags |= DCOPY_ALLOC_LINK;
		}
		e = dcopy_cmd_alloc(channel, flags, &cmd);
		if (e != DCOPY_SUCCESS) {
			cmn_err(CE_CONT, "dcopy_cmd_alloc() failed\n");
			goto testfail_alloc;
		}

		ASSERT(cmd->dp_version == DCOPY_CMD_V0);
		cmd->dp_cmd = DCOPY_CMD_COPY;
		cmd->dp_flags = DCOPY_CMD_NOFLAGS;

		/* do a bunch of dest to dest DMA's */
		cmd->dp.copy.cc_source = ptob64(hat_getpfnum(kas.a_hat,
		    (caddr_t)source)) + ((uintptr_t)dest & PAGEOFFSET);
		cmd->dp.copy.cc_dest = ptob64(hat_getpfnum(kas.a_hat,
		    (caddr_t)dest)) + ((uintptr_t)dest & PAGEOFFSET);
		cmd->dp.copy.cc_size = PAGESIZE;

		e = dcopy_cmd_post(cmd);
		if (e != DCOPY_SUCCESS) {
			cmn_err(CE_CONT, "dcopy_post() failed\n");
			goto testfail_post;
		}
	}

	e = dcopy_cmd_alloc(channel, flags, &cmd);
	if (e != DCOPY_SUCCESS) {
		cmn_err(CE_CONT, "dcopy_cmd_alloc() failed\n");
		goto testfail_alloc;
	}

	/* now queue up the DMA we are going to check status and data for  */
	cmd->dp_cmd = DCOPY_CMD_COPY;
	cmd->dp_flags = DCOPY_CMD_INTR;
	cmd->dp.copy.cc_source = ptob64(hat_getpfnum(kas.a_hat,
	    (caddr_t)source)) + ((uintptr_t)source & PAGEOFFSET);
	cmd->dp.copy.cc_dest = ptob64(hat_getpfnum(kas.a_hat,
	    (caddr_t)dest)) + ((uintptr_t)dest & PAGEOFFSET);
	cmd->dp.copy.cc_size = PAGESIZE;
	e = dcopy_cmd_post(cmd);
	if (e != DCOPY_SUCCESS) {
		cmn_err(CE_CONT, "dcopy_post() failed\n");
		goto testfail_post;
	}

	/* check the status of the last command */
	poll_cnt = 0;
	flags = DCOPY_POLL_NOFLAGS;
	while ((e = dcopy_cmd_poll(cmd, flags)) == DCOPY_PENDING) {
		poll_cnt++;
		if (poll_cnt >= 16) {
			flags |= DCOPY_POLL_BLOCK;
		}
	}
	if (e != DCOPY_COMPLETED) {
		cmn_err(CE_CONT, "dcopy_poll() failed\n");
		goto testfail_poll;
	}

	/* since the cmd's are linked we only need to pass in the last cmd */
	dcopy_cmd_free(&cmd);
	dcopy_free(&channel);

	/* verify the data */
	for (i = 0; i < PAGESIZE; i++) {
		if (dest[i] != (uint8_t)(i & 0xFF)) {
			cmn_err(CE_CONT,
			    "dcopy_data_compare() failed, %p[%d]: %x, %x\n",
			    (void *)dest, i, dest[i], i & 0xFF);
			return (-1);
		}
	}

	kmem_free(buf, (buf_size * 2) + 0x1000);

	return (0);

testfail_poll:
testfail_post:
	dcopy_cmd_free(&cmd);
	dcopy_free(&channel);
testfail_alloc:
	kmem_free(buf, (buf_size * 2) + 0x1000);

	return (-1);
}
#endif
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#include <sys/kmem.h>
#include <sys/types.h>
#include <sys/conf.h>
#include <sys/ddi.h>
#include <sys/sunddi.h>

#include <sys/ioat.h>


/* structure used to keep track of resources */
typedef struct ioat_rs_s {
	/*
	 * Bounds of resource allocation. We will start allocating at rs_min
	 * and rollover at rs_max+1 (rs_max is included). e.g. for rs_min=0
	 * and rs_max=7, we will have 8 total resources which can be alloced.
	 */
	uint_t rs_min;
	uint_t rs_max;

	/*
	 * rs_free points to an array of 64-bit values used to track resource
	 * allocation. rs_free_size is the free buffer size in bytes.
	 */
	uint64_t *rs_free;
	uint_t rs_free_size;

	/*
	 * last tracks the last alloc'd resource. This allows us to do a round
	 * robin allocation.
	 */
	uint_t rs_last;

	kmutex_t rs_mutex;
} ioat_rs_t;


/*
 * ioat_rs_init()
 *    Initialize the resource structure. This structure will be protected
 *    by a mutex at the iblock_cookie passed in. init() returns a handle to be
 *    used for the rest of the resource functions. This code is written assuming
 *    that min_val will be close to 0. Therefore, we will allocate the free
 *    buffer only taking max_val into account.
 */
void
ioat_rs_init(ioat_state_t *state, uint_t min_val, uint_t max_val,
    ioat_rs_hdl_t *handle)
{
	ioat_rs_t *rstruct;
	uint_t array_size;
	uint_t index;


	ASSERT(handle != NULL);
	ASSERT(min_val < max_val);

	/* alloc space for resource structure */
	rstruct = kmem_alloc(sizeof (ioat_rs_t), KM_SLEEP);

	/*
	 * Test to see if the max value is 64-bit aligned. If so, we don't need
	 * to allocate an extra 64-bit word. alloc space for free buffer
	 * (8 bytes per uint64_t).
	 */
	if ((max_val & 0x3F) == 0) {
		rstruct->rs_free_size = (max_val >> 6) * 8;
	} else {
		rstruct->rs_free_size = ((max_val >> 6) + 1) * 8;
	}
	rstruct->rs_free = kmem_alloc(rstruct->rs_free_size, KM_SLEEP);

	/* Initialize resource structure */
	rstruct->rs_min = min_val;
	rstruct->rs_last = min_val;
	rstruct->rs_max = max_val;
	mutex_init(&rstruct->rs_mutex, NULL, MUTEX_DRIVER,
	    state->is_iblock_cookie);

	/* Mark all resources as free */
	array_size = rstruct->rs_free_size >> 3;
	for (index = 0; index < array_size; index++) {
		rstruct->rs_free[index] = (uint64_t)0xFFFFFFFFFFFFFFFF;
	}

	/* setup handle which is returned from this function */
	*handle = rstruct;
}


/*
 * ioat_rs_fini()
 *    Frees up the space allocated in init().  Notice that a pointer to the
 *    handle is used for the parameter.  fini() will set the handle to NULL
 *    before returning.
 */
void
ioat_rs_fini(ioat_rs_hdl_t *handle)
{
	ioat_rs_t *rstruct;


	ASSERT(handle != NULL);

	rstruct = (ioat_rs_t *)*handle;

	mutex_destroy(&rstruct->rs_mutex);
	kmem_free(rstruct->rs_free, rstruct->rs_free_size);
	kmem_free(rstruct, sizeof (ioat_rs_t));

	/* set handle to null.  This helps catch bugs. */
	*handle = NULL;
}


/*
 * ioat_rs_alloc()
 *    alloc a resource. If alloc fails, we are out of resources.
 */
int
ioat_rs_alloc(ioat_rs_hdl_t handle, uint_t *resource)
{
	ioat_rs_t *rstruct;
	uint_t array_idx;
	uint64_t free;
	uint_t index;
	uint_t last;
	uint_t min;
	uint_t max;


	ASSERT(handle != NULL);
	ASSERT(resource != NULL);

	rstruct = (ioat_rs_t *)handle;

	mutex_enter(&rstruct->rs_mutex);
	min = rstruct->rs_min;
	max = rstruct->rs_max;

	/*
	 * Find a free resource. This will return out of the loop once it finds
	 * a free resource. There are a total of 'max'-'min'+1 resources.
	 * Performs a round robin allocation.
	 */
	for (index = min; index <= max; index++) {

		array_idx = rstruct->rs_last >> 6;
		free = rstruct->rs_free[array_idx];
		last = rstruct->rs_last & 0x3F;

		/* if the next resource to check is free */
		if ((free & ((uint64_t)1 << last)) != 0) {
			/* we are using this resource */
			*resource = rstruct->rs_last;

			/* take it out of the free list */
			rstruct->rs_free[array_idx] &= ~((uint64_t)1 << last);

			/*
			 * increment the last count so we start checking the
			 * next resource on the next alloc().  Note the rollover
			 * at 'max'+1.
			 */
			rstruct->rs_last++;
			if (rstruct->rs_last > max) {
				rstruct->rs_last = rstruct->rs_min;
			}

			/* unlock the resource structure */
			mutex_exit(&rstruct->rs_mutex);

			return (DDI_SUCCESS);
		}

		/*
		 * This resource is not free, lets go to the next one. Note the
		 * rollover at 'max'.
		 */
		rstruct->rs_last++;
		if (rstruct->rs_last > max) {
			rstruct->rs_last = rstruct->rs_min;
		}
	}

	mutex_exit(&rstruct->rs_mutex);

	return (DDI_FAILURE);
}


/*
 * ioat_rs_free()
 *    Free the previously alloc'd resource.  Once a resource has been free'd,
 *    it can be used again when alloc is called.
 */
void
ioat_rs_free(ioat_rs_hdl_t handle, uint_t resource)
{
	ioat_rs_t *rstruct;
	uint_t array_idx;
	uint_t offset;


	ASSERT(handle != NULL);

	rstruct = (ioat_rs_t *)handle;
	ASSERT(resource >= rstruct->rs_min);
	ASSERT(resource <= rstruct->rs_max);

	mutex_enter(&rstruct->rs_mutex);

	/* Put the resource back in the free list */
	array_idx = resource >> 6;
	offset = resource & 0x3F;
	rstruct->rs_free[array_idx] |= ((uint64_t)1 << offset);

	mutex_exit(&rstruct->rs_mutex);
}