mc - zygaena/hammerhead

root / base / usr / src / common / mc
mc Plain Text 7559 lines 218.2 KB
/*
 * This file and its contents are supplied under the terms of the
 * Common Development and Distribution License ("CDDL"), version 1.0.
 * You may only use this file in accordance with the terms of version
 * 1.0 of the CDDL.
 *
 * A full copy of the text of the CDDL should have accompanied this
 * source.  A copy of the CDDL is also available via the Internet at
 * http://www.illumos.org/license/CDDL.
 */

/*
 * Copyright 2019 Joyent, Inc.
 */

/*
 * Memory decoding logic.
 *
 * This file is part of the 'imc' driver on x86. It supports taking a physical
 * address and determining what the corresponding DIMM is. This is shared
 * between the kernel and userland for easier testing.
 *
 * For more information about the different parts of the decoding process,
 * please see the file 'uts/i86pc/io/imc/imc.c'.
 */

#include <sys/sysmacros.h>

#ifndef _KERNEL
#include <stdint.h>
#include <strings.h>
#define	BITX(u, h, l)	(((u) >> (l)) & ((1LU << ((h) - (l) + 1LU)) - 1LU))
#endif	/* !_KERNEL */

#include "imc.h"

/*
 * Address ranges for decoding system addresses. There are three ranges that
 * exist on x86, traditional DOS memory (hi 640 KiB), low memory, and high
 * memory. Low memory always starts at 1 MiB and high memory always starts at 4
 * GiB. The upper bounds of these ranges is based on registers on the system.
 */
#define	IMC_DECODE_CONV_BASE	0UL
#define	IMC_DECODE_CONV_MAX	0x00009ffffULL	/* 640 KiB - 1 */
#define	IMC_DECODE_LOW_BASE	0x000100000ULL	/* 1 M */
#define	IMC_DECODE_HIGH_BASE	0x100000000ULL /* 4 GiB */

typedef struct imc_legacy_range {
	uint64_t	ilr_base;
	size_t		ilr_len;
	const char	*ilr_desc;
} imc_legacy_range_t;

/*
 * These represent regions of memory that are reserved for use and will not be
 * decoded by DRAM.
 */
static imc_legacy_range_t imc_legacy_ranges[] = {
	{ 0x00000A0000ULL,	128 * 1024,	"VGA" },
	{ 0x00000C0000ULL,	256 * 1024,	"PAM" },
	{ 0x0000F00000ULL,	1024 * 1024,	"Reserved" },
	{ 0x00FE000000ULL,	32 * 1024 * 1024, "Unknown" },
	{ 0x00FF000000ULL,	16 * 1024 * 1024, "Firmware" },
	{ 0x00FED20000ULL,	384 * 1024,	"TXT" },
	{ 0x00FED00000ULL,	1024 * 1024,	"PCH" },
	{ 0x00FEC00000ULL,	1024 * 1024,	"IOAPIC" },
	{ 0x00FEB80000ULL,	512 * 1024,	"Reserved" },
	{ 0x00FEB00000ULL,	64 * 1024,	"Reserved" }
};

/*
 * Determine whether or not this address is in one of the reserved regions or if
 * it falls outside of the explicit DRAM ranges.
 */
static boolean_t
imc_decode_addr_resvd(const imc_t *imc, imc_decode_state_t *dec)
{
	uint_t i;
	const imc_sad_t *sad;

	for (i = 0; i < ARRAY_SIZE(imc_legacy_ranges); i++) {
		uint64_t end = imc_legacy_ranges[i].ilr_base +
		    imc_legacy_ranges[i].ilr_len;

		if (dec->ids_pa >= imc_legacy_ranges[i].ilr_base &&
		    dec->ids_pa < end) {
			dec->ids_fail = IMC_DECODE_F_LEGACY_RANGE;
			dec->ids_fail_data = i;
			return (B_TRUE);
		}
	}

	/*
	 * For checking and determining whether or not we fit in DRAM, we need
	 * to check against the top of low memory and the top of high memory.
	 * While we technically have this information on a per-socket basis, we
	 * have to rely on the fact that both processors have the same
	 * information. A requirement which if not true, would lead to chaos
	 * depending on what socket we're running on.
	 */
	sad = &imc->imc_sockets[0].isock_sad;
	if (sad->isad_valid != IMC_SAD_V_VALID) {
		dec->ids_fail = IMC_DECODE_F_BAD_SAD;
		return (B_TRUE);
	}

	/*
	 * An address may fall into three ranges. It may fall into conventional
	 * memory. It may fall into low memory. It may fall into high memory.
	 * The conventional memory range is inclusive at the top. The others
	 * have been translated such that they are uniformly exclusive at the
	 * top. Because the bottom of conventional memory is at zero, the
	 * compiler will be angry if we compare against IMC_DECODE_CONV_BASE as
	 * it is always true.
	 */
	if (dec->ids_pa <= IMC_DECODE_CONV_MAX) {
		return (B_FALSE);
	}

	if (dec->ids_pa >= IMC_DECODE_LOW_BASE &&
	    dec->ids_pa < sad->isad_tolm) {
		return (B_FALSE);
	}

	if (dec->ids_pa >= IMC_DECODE_HIGH_BASE &&
	    dec->ids_pa < sad->isad_tohm) {
		return (B_FALSE);
	}

	/*
	 * Memory fell outside of the valid range. It's not for us.
	 */
	dec->ids_fail = IMC_DECODE_F_OUTSIDE_DRAM;
	return (B_TRUE);
}

static uint_t
imc_decode_sad_interleave(const imc_sad_rule_t *rule, uint64_t pa)
{
	uint_t itgt = 0;

	switch (rule->isr_imode) {
	case IMC_SAD_IMODE_8t6:
		if (rule->isr_a7mode) {
			itgt = BITX(pa, 9, 9);
			itgt |= (BITX(pa, 8, 7) << 1);
		} else {
			itgt = BITX(pa, 8, 6);
		}
		break;
	case IMC_SAD_IMODE_8t6XOR:
		if (rule->isr_a7mode) {
			itgt = BITX(pa, 9, 9);
			itgt |= (BITX(pa, 8, 7) << 1);
		} else {
			itgt = BITX(pa, 8, 6);
		}
		itgt ^= BITX(pa, 18, 16);
		break;
	case IMC_SAD_IMODE_10t8:
		itgt = BITX(pa, 10, 8);
		break;
	case IMC_SAD_IMODE_14t12:
		itgt = BITX(pa, 14, 12);
		break;
	case IMC_SAD_IMODE_32t30:
		itgt = BITX(pa, 32, 30);
		break;
	}

	return (itgt);
}

/*
 * Use the system address decoder to try and find a valid SAD entry for this
 * address. We always use socket zero's SAD as the SAD rules should be the same
 * between the different sockets.
 */
static boolean_t
imc_decode_sad(const imc_t *imc, imc_decode_state_t *dec)
{
	uint_t i, ileaveidx;
	uint8_t ileavetgt;
	uint32_t nodeid, tadid, channelid;
	uint64_t base;
	const imc_socket_t *socket = &imc->imc_sockets[0];
	const imc_sad_t *sad = &socket->isock_sad;
	const imc_sad_rule_t *rule;
	boolean_t loop = B_FALSE;

	/*
	 * Note, all SAD rules have been adjusted so that they are uniformly
	 * exclusive.
	 */
start:
	for (rule = NULL, i = 0, base = 0; i < sad->isad_nrules; i++) {
		rule = &sad->isad_rules[i];

		if (rule->isr_enable && dec->ids_pa >= base &&
		    dec->ids_pa < rule->isr_limit) {
			break;
		}

		base = rule->isr_limit;
	}

	if (rule == NULL || i == sad->isad_nrules) {
		dec->ids_fail = IMC_DECODE_F_NO_SAD_RULE;
		return (B_FALSE);
	}

	/*
	 * Store the SAD rule in the decode information for debugging's sake.
	 */
	dec->ids_sad = sad;
	dec->ids_sad_rule = rule;

	/*
	 * We have found a SAD rule. We now need to transform that into the
	 * corresponding target based on its mode, etc. The way we do this
	 * varies based on the generation.
	 *
	 * The first thing we need to do is to figure out the target in the
	 * interleave list.
	 */
	ileaveidx = imc_decode_sad_interleave(rule, dec->ids_pa);
	if (ileaveidx >= rule->isr_ntargets) {
		dec->ids_fail = IMC_DECODE_F_BAD_SAD_INTERLEAVE;
		dec->ids_fail_data = ileaveidx;
		return (B_FALSE);
	}
	ileavetgt = rule->isr_targets[ileaveidx];
	if (imc->imc_gen >= IMC_GEN_SKYLAKE &&
	    IMC_SAD_ILEAVE_SKX_LOCAL(ileavetgt) == 0) {
		/*
		 * If we're in this case, the interleave rule said we had a
		 * remote target. That means we need to find the correct SAD
		 * based on the Node ID and then do all of this over again.
		 */
		nodeid = IMC_SAD_ILEAVE_SKX_TARGET(ileavetgt);

		if (loop) {
			dec->ids_fail = IMC_DECODE_F_SAD_SEARCH_LOOP;
			return (B_FALSE);
		}

		for (i = 0; i < imc->imc_nsockets; i++) {
			if (imc->imc_sockets[i].isock_valid ==
			    IMC_SOCKET_V_VALID &&
			    imc->imc_sockets[i].isock_nodeid == nodeid) {
				socket = &imc->imc_sockets[i];
				sad = &imc->imc_sockets[i].isock_sad;
				loop = B_TRUE;
				goto start;
			}
		}

		dec->ids_fail = IMC_DECODE_F_BAD_REMOTE_MC_ROUTE;
		dec->ids_fail_data = nodeid;
		return (B_FALSE);
	}

	/*
	 * On some platforms we need to derive the target channel based on the
	 * physical address and additional rules in the SAD. If we do, do that
	 * here. The idea is that this may overrule the memory channel route
	 * table target that was determined from the SAD rule.
	 */
	if (rule->isr_need_mod3) {
		uint64_t addr;
		uint8_t channel;

		switch (rule->isr_mod_mode) {
		case IMC_SAD_MOD_MODE_45t6:
			addr = dec->ids_pa >> 6;
			break;
		case IMC_SAD_MOD_MODE_45t8:
			addr = dec->ids_pa >> 8;
			break;
		case IMC_SAD_MOD_MODE_45t12:
			addr = dec->ids_pa >> 12;
			break;
		default:
			dec->ids_fail = IMC_DECODE_F_SAD_BAD_MOD;
			return (B_FALSE);
		}

		switch (rule->isr_mod_type) {
		case IMC_SAD_MOD_TYPE_MOD3:
			channel = (addr % 3) << 1;
			channel |= ileavetgt & 1;
			break;
		case IMC_SAD_MOD_TYPE_MOD2_01:
			channel = (addr % 2) << 1;
			channel |= ileavetgt & 1;
			break;
		case IMC_SAD_MOD_TYPE_MOD2_12:
			channel = (addr % 2) << 2;
			channel |= (~addr % 2) << 1;
			channel |= ileavetgt & 1;
			break;
		case IMC_SAD_MOD_TYPE_MOD2_02:
			channel = (addr % 2) << 2;
			channel |= ileavetgt & 1;
			break;
		default:
			dec->ids_fail = IMC_DECODE_F_SAD_BAD_MOD;
			return (B_FALSE);
		}

		ileavetgt = channel;
	}

	switch (imc->imc_gen) {
	case IMC_GEN_SANDY:
		/*
		 * Sandy Bridge systems only have a single home agent, so the
		 * interleave target is always the node id.
		 */
		nodeid = ileavetgt;
		tadid = 0;
		channelid = UINT32_MAX;
		break;
	case IMC_GEN_IVY:
	case IMC_GEN_HASWELL:
	case IMC_GEN_BROADWELL:
		/*
		 * On these generations, the interleave NodeID in the SAD
		 * encodes both the nodeid and the home agent ID that we care
		 * about.
		 */
		nodeid = IMC_NODEID_IVY_BRD_UPPER(ileavetgt) |
		    IMC_NODEID_IVY_BRD_LOWER(ileavetgt);
		tadid = IMC_NODEID_IVY_BRD_HA(ileavetgt);
		channelid = UINT32_MAX;
		break;
	case IMC_GEN_SKYLAKE:
		/*
		 * On Skylake generation systems we take the interleave target
		 * and use that to look up both the memory controller and the
		 * physical channel in the route table. The nodeid is already
		 * known because its SAD rules redirect us.
		 */
		nodeid = socket->isock_nodeid;
		if (ileavetgt > IMC_SAD_ILEAVE_SKX_MAX) {
			dec->ids_fail = IMC_DECODE_F_BAD_SAD_INTERLEAVE;
			dec->ids_fail_data = ileavetgt;
			return (B_FALSE);
		}
		ileavetgt = IMC_SAD_ILEAVE_SKX_TARGET(ileavetgt);
		if (ileavetgt > sad->isad_mcroute.ismc_nroutes) {
			dec->ids_fail = IMC_DECODE_F_BAD_SAD_INTERLEAVE;
			dec->ids_fail_data = ileavetgt;
			return (B_FALSE);
		}
		tadid = sad->isad_mcroute.ismc_mcroutes[ileavetgt].ismce_imc;
		channelid =
		    sad->isad_mcroute.ismc_mcroutes[ileavetgt].ismce_pchannel;
		break;
	default:
		nodeid = tadid = channelid = UINT32_MAX;
		break;
	}

	/*
	 * Map to the correct socket based on the nodeid. Make sure that we have
	 * a valid TAD.
	 */
	dec->ids_socket = NULL;
	for (i = 0; i < imc->imc_nsockets; i++) {
		if (imc->imc_sockets[i].isock_nodeid == nodeid) {
			dec->ids_socket = &imc->imc_sockets[i];
			break;
		}
	}
	if (dec->ids_socket == NULL) {
		dec->ids_fail = IMC_DECODE_F_SAD_BAD_SOCKET;
		dec->ids_fail_data = nodeid;
		return (B_FALSE);
	}

	if (tadid >= dec->ids_socket->isock_ntad) {
		dec->ids_fail = IMC_DECODE_F_SAD_BAD_TAD;
		dec->ids_fail_data = tadid;
		return (B_FALSE);
	}

	dec->ids_nodeid = nodeid;
	dec->ids_tadid = tadid;
	dec->ids_channelid = channelid;
	dec->ids_tad = &dec->ids_socket->isock_tad[tadid];
	dec->ids_mc = &dec->ids_socket->isock_imcs[tadid];

	return (B_TRUE);
}

/*
 * For Sandy Bridge through Broadwell we need to decode the memory channel that
 * we're targeting. This is determined based on the number of ways that the
 * socket and channel are supposed to be interleaved. The TAD has a target
 * channel list sitting with the TAD rule. To figure out the appropriate index,
 * the algorithm is roughly:
 *
 *    idx = [(dec->ids_pa >> 6) / socket-ways] % channel-ways
 *
 * The shift by six, comes from taking the number of bits that are in theory in
 * the cache line size. Of course, if things were this simple, that'd be great.
 * The first complication is a7mode / MCChanShiftUpEnable. When this is enabled,
 * more cache lines are used for this. The next complication comes when the
 * feature MCChanHashEn is enabled. This means that we have to hash the
 * resulting address before we do the modulus based on the number of channel
 * ways.
 *
 * The last, and most complicated problem is when the number of channel ways is
 * set to three. When this is the case, the base address of the range may not
 * actually start at index zero. The nominal solution is to use the offset
 * that's programmed on a per-channel basis to offset the system address.
 * However, to get that information we would have to know what channel we're on,
 * which is what we're trying to figure out. Regretfully, proclaim that we can't
 * in this case.
 */
static boolean_t
imc_decode_tad_channel(const imc_t *imc, imc_decode_state_t *dec)
{
	uint64_t index;
	const imc_tad_rule_t *rule = dec->ids_tad_rule;

	index = dec->ids_pa >> 6;
	if ((dec->ids_tad->itad_flags & IMC_TAD_FLAG_CHANSHIFT) != 0) {
		index = index >> 1;
	}

	/*
	 * When performing a socket way equals three comparison, this would not
	 * work.
	 */
	index = index / rule->itr_sock_way;

	if ((dec->ids_tad->itad_flags & IMC_TAD_FLAG_CHANHASH) != 0) {
		uint_t i;
		for (i = 12; i < 28; i += 2) {
			uint64_t shift = (dec->ids_pa >> i) & 0x3;
			index ^= shift;
		}
	}

	index %= rule->itr_chan_way;
	if (index >= rule->itr_ntargets) {
		dec->ids_fail = IMC_DECODE_F_TAD_BAD_TARGET_INDEX;
		dec->ids_fail_data = index;
		return (B_FALSE);
	}

	dec->ids_channelid = rule->itr_targets[index];
	return (B_TRUE);
}

static uint_t
imc_tad_gran_to_shift(const imc_tad_t *tad, imc_tad_gran_t gran)
{
	uint_t shift = 0;

	switch (gran) {
	case IMC_TAD_GRAN_64B:
		shift = 6;
		if ((tad->itad_flags & IMC_TAD_FLAG_CHANSHIFT) != 0) {
			shift++;
		}
		break;
	case IMC_TAD_GRAN_256B:
		shift = 8;
		break;
	case IMC_TAD_GRAN_4KB:
		shift = 12;
		break;
	case IMC_TAD_GRAN_1GB:
		shift = 30;
		break;
	}

	return (shift);
}

static boolean_t
imc_decode_tad(const imc_t *imc, imc_decode_state_t *dec)
{
	uint_t i, tadruleno;
	uint_t sockshift, chanshift, sockmask, chanmask;
	uint64_t off, chanaddr;
	const imc_tad_t *tad = dec->ids_tad;
	const imc_mc_t *mc = dec->ids_mc;
	const imc_tad_rule_t *rule = NULL;
	const imc_channel_t *chan;

	/*
	 * The first step in all of this is to determine which TAD rule applies
	 * for this address.
	 */
	for (i = 0; i < tad->itad_nrules; i++) {
		rule = &tad->itad_rules[i];

		if (dec->ids_pa >= rule->itr_base &&
		    dec->ids_pa < rule->itr_limit) {
			break;
		}
	}

	if (rule == NULL || i == tad->itad_nrules) {
		dec->ids_fail = IMC_DECODE_F_NO_TAD_RULE;
		return (B_FALSE);
	}
	tadruleno = i;
	dec->ids_tad_rule = rule;

	/*
	 * Check if our TAD rule requires 3-way interleaving on the channel. We
	 * basically can't do that right now. For more information, see the
	 * comment above imc_decode_tad_channel().
	 */
	if (rule->itr_chan_way == 3) {
		dec->ids_fail = IMC_DECODE_F_TAD_3_ILEAVE;
		return (B_FALSE);
	}

	/*
	 * On some platforms, we need to now calculate the channel index from
	 * this. The way that we calculate this is nominally straightforward,
	 * but complicated by a number of different issues.
	 */
	switch (imc->imc_gen) {
	case IMC_GEN_SANDY:
	case IMC_GEN_IVY:
	case IMC_GEN_HASWELL:
	case IMC_GEN_BROADWELL:
		if (!imc_decode_tad_channel(imc, dec)) {
			return (B_FALSE);
		}
		break;
	default:
		/*
		 * On Skylake and newer platforms we should have already decoded
		 * the target channel based on using the memory controller route
		 * table above.
		 */
		break;
	}

	/*
	 * We initialize ids_channelid to UINT32_MAX, so this should make sure
	 * that we catch an incorrect channel as well.
	 */
	if (dec->ids_channelid >= mc->icn_nchannels) {
		dec->ids_fail = IMC_DECODE_F_BAD_CHANNEL_ID;
		dec->ids_fail_data = dec->ids_channelid;
		return (B_FALSE);
	}
	chan = &mc->icn_channels[dec->ids_channelid];
	dec->ids_chan = chan;

	if (tadruleno >= chan->ich_ntad_offsets) {
		dec->ids_fail = IMC_DECODE_F_BAD_CHANNEL_TAD_OFFSET;
		dec->ids_fail_data = tadruleno;
		return (B_FALSE);
	}

	/*
	 * Now we can go ahead and calculate the channel address, which is
	 * roughly equal to:
	 *
	 * chan_addr = (sys_addr - off) / (chan way * sock way).
	 *
	 * The catch is that we want to preserve the low bits where possible.
	 * The number of bits is based on the interleaving granularities, the
	 * way that's calculated is based on information in the TAD rule.
	 * However, if a7mode is enabled on Ivy Bridge through Broadwell, then
	 * we need to add one to that. So we will save the smallest number of
	 * bits that are left after interleaving.
	 *
	 * Because the interleaving occurs at different granularities, we need
	 * to break this into two discrete steps, one where we apply the socket
	 * interleaving and one where we apply the channel interleaving,
	 * shifting and dividing at each step.
	 */
	off = chan->ich_tad_offsets[tadruleno];
	if (off > dec->ids_pa) {
		dec->ids_fail = IMC_DECODE_F_CHANOFF_UNDERFLOW;
		return (B_FALSE);
	}
	chanshift = imc_tad_gran_to_shift(tad, rule->itr_chan_gran);
	sockshift = imc_tad_gran_to_shift(tad, rule->itr_sock_gran);
	chanmask = (1 << chanshift) - 1;
	sockmask = (1 << sockshift) - 1;

	chanaddr = dec->ids_pa - off;
	chanaddr >>= sockshift;
	chanaddr /= rule->itr_sock_way;
	chanaddr <<= sockshift;
	chanaddr |= dec->ids_pa & sockmask;
	chanaddr >>= chanshift;
	chanaddr /= rule->itr_chan_way;
	chanaddr <<= chanshift;
	chanaddr |= dec->ids_pa & chanmask;

	dec->ids_chanaddr = chanaddr;

	return (B_TRUE);
}

static boolean_t
imc_decode_rir(const imc_t *imc, imc_decode_state_t *dec)
{
	const imc_mc_t *mc = dec->ids_mc;
	const imc_channel_t *chan = dec->ids_chan;
	const imc_rank_ileave_t *rir = NULL;
	const imc_rank_ileave_entry_t *rirtarg;
	const imc_dimm_t *dimm;
	uint32_t shift, index;
	uint_t i, dimmid, rankid;
	uint64_t mask, base, rankaddr;

	if (mc->icn_closed) {
		shift = IMC_PAGE_BITS_CLOSED;
	} else {
		shift = IMC_PAGE_BITS_OPEN;
	}
	mask = (1UL << shift) - 1;

	for (i = 0, base = 0; i < chan->ich_nrankileaves; i++) {
		rir = &chan->ich_rankileaves[i];
		if (rir->irle_enabled && dec->ids_chanaddr >= base &&
		    dec->ids_chanaddr < rir->irle_limit) {
			break;
		}

		base = rir->irle_limit;
	}

	if (rir == NULL || i == chan->ich_nrankileaves) {
		dec->ids_fail = IMC_DECODE_F_NO_RIR_RULE;
		return (B_FALSE);
	}
	dec->ids_rir = rir;

	/*
	 * Determine the index of the rule that we care about. This is done by
	 * shifting the address based on the open and closed page bits and then
	 * just modding it by the number of ways in question.
	 */
	index = (dec->ids_chanaddr >> shift) % rir->irle_nways;
	if (index >= rir->irle_nentries) {
		dec->ids_fail = IMC_DECODE_F_BAD_RIR_ILEAVE_TARGET;
		dec->ids_fail_data = index;
		return (B_FALSE);
	}
	rirtarg = &rir->irle_entries[index];

	/*
	 * The rank interleaving register has information about a physical rank
	 * target. This is within the notion of the physical chip selects that
	 * exist. While the memory controller only has eight actual chip
	 * selects, the physical values that are programmed depend a bit on the
	 * underlying hardware. Effectively, in this ID space, each DIMM has
	 * four ranks associated with it. Even when we only have two ranks with
	 * each physical channel, they'll be programmed so we can simply do the
	 * following match:
	 *
	 * DIMM = rank id / 4
	 * RANK = rank id % 4
	 */
	dec->ids_physrankid = rirtarg->irle_target;
	dimmid = dec->ids_physrankid / 4;
	rankid = dec->ids_physrankid % 4;

	if (dimmid >= chan->ich_ndimms) {
		dec->ids_fail = IMC_DECODE_F_BAD_DIMM_INDEX;
		dec->ids_fail_data = dimmid;
		return (B_FALSE);
	}

	dimm = &chan->ich_dimms[dimmid];
	if (!dimm->idimm_present) {
		dec->ids_fail = IMC_DECODE_F_DIMM_NOT_PRESENT;
		return (B_FALSE);
	}
	dec->ids_dimmid = dimmid;
	dec->ids_dimm = dimm;

	if (rankid >= dimm->idimm_nranks) {
		dec->ids_fail = IMC_DECODE_F_BAD_DIMM_RANK;
		dec->ids_fail_data = rankid;
		return (B_FALSE);
	}
	dec->ids_rankid = rankid;

	/*
	 * Calculate the rank address. We need to divide the address by the
	 * number of rank ways and then or in the lower bits.
	 */
	rankaddr = dec->ids_chanaddr;
	rankaddr >>= shift;
	rankaddr /= rir->irle_nways;
	rankaddr <<= shift;
	rankaddr |= dec->ids_chanaddr & mask;

	if (rirtarg->irle_offset > rankaddr) {
		dec->ids_fail = IMC_DECODE_F_RANKOFF_UNDERFLOW;
		return (B_FALSE);
	}
	rankaddr -= rirtarg->irle_offset;
	dec->ids_rankaddr = rankaddr;

	return (B_TRUE);
}

boolean_t
imc_decode_pa(const imc_t *imc, uint64_t pa, imc_decode_state_t *dec)
{
	bzero(dec, sizeof (*dec));
	dec->ids_pa = pa;
	dec->ids_nodeid = dec->ids_tadid = dec->ids_channelid = UINT32_MAX;

	/*
	 * We need to rely on socket zero's information. Make sure that it both
	 * exists and is considered valid.
	 */
	if (imc->imc_nsockets < 1 ||
	    imc->imc_sockets[0].isock_valid != IMC_SOCKET_V_VALID) {
		dec->ids_fail = IMC_DECODE_F_BAD_SOCKET;
		dec->ids_fail_data = 0;
		return (B_FALSE);
	}

	/*
	 * First, we need to make sure that the PA we've been given actually is
	 * meant to target a DRAM address. This address may fall to MMIO, MMCFG,
	 * be an address that's outside of DRAM, or belong to a legacy address
	 * range that is interposed.
	 */
	if (imc_decode_addr_resvd(imc, dec)) {
		return (B_FALSE);
	}

	/*
	 * Now that we have this data, we want to go through and look at the
	 * SAD. The SAD will point us to a specific socket and an IMC / home
	 * agent on that socket which will tell us which TAD we need to use.
	 */
	if (!imc_decode_sad(imc, dec)) {
		return (B_FALSE);
	}

	/*
	 * The decoded SAD information has pointed us a TAD. We need to use this
	 * to point us to the corresponding memory channel and the corresponding
	 * address on the channel.
	 */
	if (!imc_decode_tad(imc, dec)) {
		return (B_FALSE);
	}

	/*
	 * Use the rank interleaving data to determine which DIMM this is, the
	 * relevant rank, and the rank address.
	 */
	if (!imc_decode_rir(imc, dec)) {
		return (B_FALSE);
	}

	return (B_TRUE);
}
/*
 * This file and its contents are supplied under the terms of the
 * Common Development and Distribution License ("CDDL"), version 1.0.
 * You may only use this file in accordance with the terms of version
 * 1.0 of the CDDL.
 *
 * A full copy of the text of the CDDL should have accompanied this
 * source.  A copy of the CDDL is also available via the Internet at
 * http://www.illumos.org/license/CDDL.
 */

/*
 * Copyright 2019 Joyent, Inc.
 */

/*
 * This implements logic to allow us to dump IMC data for decoding purposes,
 * such that we can later encode it elsewhere. In general, dumping is done by
 * the kernel and reconstituting this data is done by user land.
 */

#include "imc.h"

#ifndef _KERNEL
#include <stdint.h>
#include <strings.h>
#endif	/* !_KERNEL */


static nvlist_t *
imc_dump_sad(imc_sad_t *sad)
{
	uint_t i;
	nvlist_t *nvl;
	nvlist_t *rules[IMC_MAX_SAD_RULES];
	nvlist_t *routes[IMC_MAX_SAD_MCROUTES];

	nvl = fnvlist_alloc();
	fnvlist_add_uint32(nvl, "isad_flags", sad->isad_flags);
	fnvlist_add_uint32(nvl, "isad_valid", sad->isad_valid);
	fnvlist_add_uint64(nvl, "isad_tolm", sad->isad_tolm);
	fnvlist_add_uint64(nvl, "isad_tohm", sad->isad_tohm);

	for (i = 0; i < sad->isad_nrules; i++) {
		nvlist_t *n = fnvlist_alloc();
		imc_sad_rule_t *r = &sad->isad_rules[i];

		fnvlist_add_boolean_value(n, "isr_enable", r->isr_enable);
		fnvlist_add_boolean_value(n, "isr_a7mode", r->isr_a7mode);
		fnvlist_add_boolean_value(n, "isr_need_mod3", r->isr_need_mod3);
		fnvlist_add_uint64(n, "isr_limit", r->isr_limit);
		fnvlist_add_uint32(n, "isr_type", r->isr_type);
		fnvlist_add_uint32(n, "isr_imode", r->isr_imode);
		fnvlist_add_uint32(n, "isr_mod_mode", r->isr_mod_mode);
		fnvlist_add_uint32(n, "isr_mod_type", r->isr_mod_type);
		fnvlist_add_uint8_array(n, "isr_targets", r->isr_targets,
		    r->isr_ntargets);

		rules[i] = n;
	}
	fnvlist_add_nvlist_array(nvl, "isad_rules", rules, sad->isad_nrules);
	for (i = 0; i < sad->isad_nrules; i++) {
		nvlist_free(rules[i]);
	}

	if (sad->isad_mcroute.ismc_nroutes == 0) {
		return (nvl);
	}

	for (i = 0; i <  sad->isad_mcroute.ismc_nroutes; i++) {
		nvlist_t *r = fnvlist_alloc();
		imc_sad_mcroute_entry_t *e =
		    &sad->isad_mcroute.ismc_mcroutes[i];

		fnvlist_add_uint8(r, "ismce_imc", e->ismce_imc);
		fnvlist_add_uint8(r, "ismce_pchannel", e->ismce_pchannel);
		routes[i] = r;
	}
	fnvlist_add_nvlist_array(nvl, "isad_mcroute", routes, i);
	for (i = 0; i <  sad->isad_mcroute.ismc_nroutes; i++) {
		nvlist_free(routes[i]);
	}

	return (nvl);
}

static nvlist_t *
imc_dump_tad(imc_tad_t *tad)
{
	uint_t i;
	nvlist_t *nvl;
	nvlist_t *rules[IMC_MAX_TAD_RULES];

	nvl = fnvlist_alloc();
	fnvlist_add_uint32(nvl, "itad_valid", tad->itad_valid);
	fnvlist_add_uint32(nvl, "itad_flags", tad->itad_flags);
	for (i = 0; i < tad->itad_nrules; i++) {
		nvlist_t *t = fnvlist_alloc();
		imc_tad_rule_t *r = &tad->itad_rules[i];

		fnvlist_add_uint64(t, "itr_base", r->itr_base);
		fnvlist_add_uint64(t, "itr_limit", r->itr_limit);
		fnvlist_add_uint8(t, "itr_sock_way", r->itr_sock_way);
		fnvlist_add_uint8(t, "itr_chan_way", r->itr_chan_way);
		fnvlist_add_uint32(t, "itr_sock_gran", r->itr_sock_gran);
		fnvlist_add_uint32(t, "itr_chan_gran", r->itr_chan_gran);
		fnvlist_add_uint8_array(t, "itr_targets", r->itr_targets,
		    r->itr_ntargets);

		rules[i] = t;
	}
	fnvlist_add_nvlist_array(nvl, "itad_rules", rules, tad->itad_nrules);
	for (i = 0; i < tad->itad_nrules; i++) {
		nvlist_free(rules[i]);
	}

	return (nvl);
}

static nvlist_t *
imc_dump_channel(imc_channel_t *chan)
{
	uint_t i;
	nvlist_t *nvl;
	nvlist_t *dimms[IMC_MAX_DIMMPERCHAN];
	nvlist_t *ranks[IMC_MAX_RANK_WAYS];

	nvl = fnvlist_alloc();
	fnvlist_add_uint32(nvl, "ich_valid", chan->ich_valid);
	for (i = 0; i < chan->ich_ndimms; i++) {
		nvlist_t *d = fnvlist_alloc();
		imc_dimm_t *dimm = &chan->ich_dimms[i];

		fnvlist_add_uint32(d, "idimm_valid", dimm->idimm_valid);
		fnvlist_add_boolean_value(d, "idimm_present",
		    dimm->idimm_present);
		if (!dimm->idimm_present)
			goto add;

		fnvlist_add_uint8(d, "idimm_nbanks", dimm->idimm_nbanks);
		fnvlist_add_uint8(d, "idimm_nranks", dimm->idimm_nranks);
		fnvlist_add_uint8(d, "idimm_width", dimm->idimm_width);
		fnvlist_add_uint8(d, "idimm_density", dimm->idimm_density);
		fnvlist_add_uint8(d, "idimm_nrows", dimm->idimm_nrows);
		fnvlist_add_uint8(d, "idimm_ncolumns", dimm->idimm_ncolumns);
		fnvlist_add_uint64(d, "idimm_size", dimm->idimm_size);
add:
		dimms[i] = d;
	}
	fnvlist_add_nvlist_array(nvl, "ich_dimms", dimms, i);
	for (i = 0; i < chan->ich_ndimms; i++) {
		nvlist_free(dimms[i]);
	}

	fnvlist_add_uint64_array(nvl, "ich_tad_offsets", chan->ich_tad_offsets,
	    chan->ich_ntad_offsets);

	for (i = 0; i < chan->ich_nrankileaves; i++) {
		uint_t j;
		nvlist_t *r = fnvlist_alloc();
		nvlist_t *ileaves[IMC_MAX_RANK_INTERLEAVES];
		imc_rank_ileave_t *rank = &chan->ich_rankileaves[i];

		fnvlist_add_boolean_value(r, "irle_enabled",
		    rank->irle_enabled);
		fnvlist_add_uint8(r, "irle_nways", rank->irle_nways);
		fnvlist_add_uint8(r, "irle_nwaysbits", rank->irle_nwaysbits);
		fnvlist_add_uint64(r, "irle_limit", rank->irle_limit);

		for (j = 0; j < rank->irle_nentries; j++) {
			nvlist_t *e = fnvlist_alloc();

			fnvlist_add_uint8(e, "irle_target",
			    rank->irle_entries[j].irle_target);
			fnvlist_add_uint64(e, "irle_offset",
			    rank->irle_entries[j].irle_offset);
			ileaves[j] = e;
		}
		fnvlist_add_nvlist_array(r, "irle_entries", ileaves, j);
		for (j = 0; j < rank->irle_nentries; j++) {
			nvlist_free(ileaves[j]);
		}

		ranks[i] = r;
	}
	fnvlist_add_nvlist_array(nvl, "ich_rankileaves", ranks, i);
	for (i = 0; i < chan->ich_nrankileaves; i++) {
		nvlist_free(ranks[i]);
	}

	return (nvl);
}

static nvlist_t *
imc_dump_mc(imc_mc_t *mc)
{
	uint_t i;
	nvlist_t *nvl;
	nvlist_t *channels[IMC_MAX_CHANPERMC];

	nvl = fnvlist_alloc();
	fnvlist_add_boolean_value(nvl, "icn_ecc", mc->icn_ecc);
	fnvlist_add_boolean_value(nvl, "icn_lockstep", mc->icn_lockstep);
	fnvlist_add_boolean_value(nvl, "icn_closed", mc->icn_closed);
	fnvlist_add_uint32(nvl, "icn_dimm_type", mc->icn_dimm_type);

	for (i = 0; i < mc->icn_nchannels; i++) {
		channels[i] = imc_dump_channel(&mc->icn_channels[i]);
	}
	fnvlist_add_nvlist_array(nvl, "icn_channels", channels, i);
	for (i = 0; i < mc->icn_nchannels; i++) {
		nvlist_free(channels[i]);
	}

	return (nvl);
}

static nvlist_t *
imc_dump_socket(imc_socket_t *sock)
{
	uint_t i;
	nvlist_t *nvl, *sad;
	nvlist_t *tad[IMC_MAX_TAD];
	nvlist_t *mc[IMC_MAX_IMCPERSOCK];

	nvl = fnvlist_alloc();

	sad = imc_dump_sad(&sock->isock_sad);
	fnvlist_add_nvlist(nvl, "isock_sad", sad);
	nvlist_free(sad);

	for (i = 0; i < sock->isock_ntad; i++) {
		tad[i] = imc_dump_tad(&sock->isock_tad[i]);
	}
	fnvlist_add_nvlist_array(nvl, "isock_tad", tad, i);
	for (i = 0; i < sock->isock_ntad; i++) {
		fnvlist_free(tad[i]);
	}

	fnvlist_add_uint32(nvl, "isock_nodeid", sock->isock_nodeid);

	for (i = 0; i  < sock->isock_nimc; i++) {
		mc[i] = imc_dump_mc(&sock->isock_imcs[i]);
	}
	fnvlist_add_nvlist_array(nvl, "isock_imcs", mc, i);
	for (i = 0; i < sock->isock_nimc; i++) {
		fnvlist_free(mc[i]);
	}
	return (nvl);
}

nvlist_t *
imc_dump_decoder(imc_t *imc)
{
	uint_t i;
	nvlist_t *nvl, *invl;
	nvlist_t *sockets[IMC_MAX_SOCKETS];

	nvl = fnvlist_alloc();
	fnvlist_add_uint32(nvl, "mc_dump_version", 0);
	fnvlist_add_string(nvl, "mc_dump_driver", "imc");

	invl = fnvlist_alloc();
	fnvlist_add_uint32(invl, "imc_gen", imc->imc_gen);

	for (i = 0; i < imc->imc_nsockets; i++) {
		sockets[i] = imc_dump_socket(&imc->imc_sockets[i]);
	}
	fnvlist_add_nvlist_array(invl, "imc_sockets", sockets, i);
	fnvlist_add_nvlist(nvl, "imc", invl);

	for (i = 0; i < imc->imc_nsockets; i++) {
		nvlist_free(sockets[i]);
	}
	nvlist_free(invl);

	return (nvl);
}

static boolean_t
imc_restore_sad(nvlist_t *nvl, imc_sad_t *sad)
{
	nvlist_t **rules, **routes;
	uint_t i, nroutes;

	if (nvlist_lookup_uint32(nvl, "isad_flags", &sad->isad_flags) != 0 ||
	    nvlist_lookup_uint32(nvl, "isad_valid", &sad->isad_valid) != 0 ||
	    nvlist_lookup_uint64(nvl, "isad_tolm", &sad->isad_tolm) != 0 ||
	    nvlist_lookup_uint64(nvl, "isad_tohm", &sad->isad_tohm) != 0 ||
	    nvlist_lookup_nvlist_array(nvl, "isad_rules",
	    &rules, &sad->isad_nrules) != 0) {
		return (B_FALSE);
	}

	for (i = 0; i < sad->isad_nrules; i++) {
		imc_sad_rule_t *r = &sad->isad_rules[i];
		uint8_t *targs;

		if (nvlist_lookup_boolean_value(rules[i], "isr_enable",
		    &r->isr_enable) != 0 ||
		    nvlist_lookup_boolean_value(rules[i], "isr_a7mode",
		    &r->isr_a7mode) != 0 ||
		    nvlist_lookup_boolean_value(rules[i], "isr_need_mod3",
		    &r->isr_need_mod3) != 0 ||
		    nvlist_lookup_uint64(rules[i], "isr_limit",
		    &r->isr_limit) != 0 ||
		    nvlist_lookup_uint32(rules[i], "isr_type",
		    &r->isr_type) != 0 ||
		    nvlist_lookup_uint32(rules[i], "isr_imode",
		    &r->isr_imode) != 0 ||
		    nvlist_lookup_uint32(rules[i], "isr_mod_mode",
		    &r->isr_mod_mode) != 0 ||
		    nvlist_lookup_uint32(rules[i], "isr_mod_type",
		    &r->isr_mod_type) != 0 ||
		    nvlist_lookup_uint8_array(rules[i], "isr_targets", &targs,
		    &r->isr_ntargets) != 0 ||
		    r->isr_ntargets > IMC_MAX_SAD_RULES) {
			return (B_FALSE);
		}

		bcopy(targs, r->isr_targets, r->isr_ntargets *
		    sizeof (uint8_t));
	}

	/*
	 * The mcroutes entry right now is only included conditionally.
	 */
	if (nvlist_lookup_nvlist_array(nvl, "isad_mcroute", &routes,
	    &nroutes) == 0) {
		if (nroutes > IMC_MAX_SAD_MCROUTES)
			return (B_FALSE);
		sad->isad_mcroute.ismc_nroutes = nroutes;
		for (i = 0; i < nroutes; i++) {
			imc_sad_mcroute_entry_t *r =
			    &sad->isad_mcroute.ismc_mcroutes[i];
			if (nvlist_lookup_uint8(routes[i], "ismce_imc",
			    &r->ismce_imc) != 0 ||
			    nvlist_lookup_uint8(routes[i], "ismce_pchannel",
			    &r->ismce_pchannel) != 0) {
				return (B_FALSE);
			}
		}
	}

	return (B_TRUE);
}

static boolean_t
imc_restore_tad(nvlist_t *nvl, imc_tad_t *tad)
{
	nvlist_t **rules;

	if (nvlist_lookup_uint32(nvl, "itad_valid", &tad->itad_valid) != 0 ||
	    nvlist_lookup_uint32(nvl, "itad_flags", &tad->itad_flags) != 0 ||
	    nvlist_lookup_nvlist_array(nvl, "itad_rules", &rules,
	    &tad->itad_nrules) != 0 || tad->itad_nrules > IMC_MAX_TAD_RULES) {
		return (B_FALSE);
	}

	for (uint_t i = 0; i < tad->itad_nrules; i++) {
		imc_tad_rule_t *r = &tad->itad_rules[i];
		uint8_t *targs;

		if (nvlist_lookup_uint64(rules[i], "itr_base",
		    &r->itr_base) != 0 ||
		    nvlist_lookup_uint64(rules[i], "itr_limit",
		    &r->itr_limit) != 0 ||
		    nvlist_lookup_uint8(rules[i], "itr_sock_way",
		    &r->itr_sock_way) != 0 ||
		    nvlist_lookup_uint8(rules[i], "itr_chan_way",
		    &r->itr_chan_way) != 0 ||
		    nvlist_lookup_uint32(rules[i], "itr_sock_gran",
		    &r->itr_sock_gran) != 0 ||
		    nvlist_lookup_uint32(rules[i], "itr_chan_gran",
		    &r->itr_chan_gran) != 0 ||
		    nvlist_lookup_uint8_array(rules[i], "itr_targets",
		    &targs, &r->itr_ntargets) != 0 ||
		    r->itr_ntargets > IMC_MAX_TAD_TARGETS) {
			return (B_FALSE);
		}

		bcopy(targs, r->itr_targets, r->itr_ntargets *
		    sizeof (uint8_t));
	}

	return (B_TRUE);
}

static boolean_t
imc_restore_channel(nvlist_t *nvl, imc_channel_t *chan)
{
	nvlist_t **dimms, **rir;
	uint64_t *tadoff;

	if (nvlist_lookup_uint32(nvl, "ich_valid", &chan->ich_valid) != 0 ||
	    nvlist_lookup_nvlist_array(nvl, "ich_dimms", &dimms,
	    &chan->ich_ndimms) != 0 ||
	    chan->ich_ndimms > IMC_MAX_DIMMPERCHAN ||
	    nvlist_lookup_uint64_array(nvl, "ich_tad_offsets", &tadoff,
	    &chan->ich_ntad_offsets) != 0 ||
	    chan->ich_ntad_offsets > IMC_MAX_TAD_RULES ||
	    nvlist_lookup_nvlist_array(nvl, "ich_rankileaves", &rir,
	    &chan->ich_nrankileaves) != 0 ||
	    chan->ich_nrankileaves > IMC_MAX_RANK_WAYS) {
		return (B_FALSE);
	}

	for (uint_t i = 0; i < chan->ich_ndimms; i++) {
		imc_dimm_t *d = &chan->ich_dimms[i];

		if (nvlist_lookup_uint32(dimms[i], "idimm_valid",
		    &d->idimm_valid) != 0 ||
		    nvlist_lookup_boolean_value(dimms[i], "idimm_present",
		    &d->idimm_present) != 0) {
			return (B_FALSE);
		}

		if (!d->idimm_present)
			continue;

		if (nvlist_lookup_uint8(dimms[i], "idimm_nbanks",
		    &d->idimm_nbanks) != 0 ||
		    nvlist_lookup_uint8(dimms[i], "idimm_nranks",
		    &d->idimm_nranks) != 0 ||
		    nvlist_lookup_uint8(dimms[i], "idimm_width",
		    &d->idimm_width) != 0 ||
		    nvlist_lookup_uint8(dimms[i], "idimm_density",
		    &d->idimm_density) != 0 ||
		    nvlist_lookup_uint8(dimms[i], "idimm_nrows",
		    &d->idimm_nrows) != 0 ||
		    nvlist_lookup_uint8(dimms[i], "idimm_ncolumns",
		    &d->idimm_ncolumns) != 0 ||
		    nvlist_lookup_uint64(dimms[i], "idimm_size",
		    &d->idimm_size) != 0) {
			return (B_FALSE);
		}
	}

	bcopy(tadoff, chan->ich_tad_offsets, chan->ich_ntad_offsets *
	    sizeof (uint64_t));

	for (uint_t i = 0; i < chan->ich_nrankileaves; i++) {
		nvlist_t **ileaves;
		imc_rank_ileave_t *r = &chan->ich_rankileaves[i];

		if (nvlist_lookup_boolean_value(rir[i], "irle_enabled",
		    &r->irle_enabled) != 0 ||
		    nvlist_lookup_uint8(rir[i], "irle_nways",
		    &r->irle_nways) != 0 ||
		    nvlist_lookup_uint8(rir[i], "irle_nwaysbits",
		    &r->irle_nwaysbits) != 0 ||
		    nvlist_lookup_uint64(rir[i], "irle_limit",
		    &r->irle_limit) != 0 ||
		    nvlist_lookup_nvlist_array(rir[i], "irle_entries",
		    &ileaves, &r->irle_nentries) != 0 ||
		    r->irle_nentries > IMC_MAX_RANK_INTERLEAVES) {
			return (B_FALSE);
		}

		for (uint_t j = 0; j < r->irle_nentries; j++) {
			imc_rank_ileave_entry_t *ril = &r->irle_entries[j];

			if (nvlist_lookup_uint8(ileaves[j], "irle_target",
			    &ril->irle_target) != 0 ||
			    nvlist_lookup_uint64(ileaves[j], "irle_offset",
			    &ril->irle_offset) != 0) {
				return (B_FALSE);
			}
		}
	}

	return (B_TRUE);
}

static boolean_t
imc_restore_mc(nvlist_t *nvl, imc_mc_t *mc)
{
	nvlist_t **channels;

	if (nvlist_lookup_boolean_value(nvl, "icn_ecc", &mc->icn_ecc) != 0 ||
	    nvlist_lookup_boolean_value(nvl, "icn_lockstep",
	    &mc->icn_lockstep) != 0 ||
	    nvlist_lookup_boolean_value(nvl, "icn_closed",
	    &mc->icn_closed) != 0 ||
	    nvlist_lookup_uint32(nvl, "icn_dimm_type",
	    &mc->icn_dimm_type) != 0 ||
	    nvlist_lookup_nvlist_array(nvl, "icn_channels", &channels,
	    &mc->icn_nchannels) != 0 || mc->icn_nchannels > IMC_MAX_CHANPERMC) {
		return (B_FALSE);
	}

	for (uint_t i = 0; i < mc->icn_nchannels; i++) {
		if (!imc_restore_channel(channels[i], &mc->icn_channels[i])) {
			return (B_FALSE);
		}
	}

	return (B_TRUE);
}

static boolean_t
imc_restore_socket(nvlist_t *nvl, imc_socket_t *sock)
{
	uint_t i;
	nvlist_t *sad, **tads, **imcs;

	if (nvlist_lookup_nvlist(nvl, "isock_sad", &sad) != 0 ||
	    nvlist_lookup_nvlist_array(nvl, "isock_tad", &tads,
	    &sock->isock_ntad) != 0 ||
	    nvlist_lookup_uint32(nvl, "isock_nodeid",
	    &sock->isock_nodeid) != 0 ||
	    nvlist_lookup_nvlist_array(nvl, "isock_imcs", &imcs,
	    &sock->isock_nimc) != 0 ||
	    sock->isock_ntad > IMC_MAX_TAD ||
	    sock->isock_nimc > IMC_MAX_IMCPERSOCK) {
		return (B_FALSE);
	}

	if (!imc_restore_sad(sad, &sock->isock_sad)) {
		return (B_FALSE);
	}

	for (i = 0; i < sock->isock_ntad; i++) {
		if (!imc_restore_tad(tads[i], &sock->isock_tad[i])) {
			return (B_FALSE);
		}
	}

	for (i = 0; i < sock->isock_nimc; i++) {
		if (!imc_restore_mc(imcs[i], &sock->isock_imcs[i])) {
			return (B_FALSE);
		}
	}

	return (B_TRUE);
}

boolean_t
imc_restore_decoder(nvlist_t *nvl, imc_t *imc)
{
	uint_t i;
	uint32_t vers;
	nvlist_t *invl, **socks;
	char *driver;

	bzero(imc, sizeof (imc_t));

	if (nvlist_lookup_uint32(nvl, "mc_dump_version", &vers) != 0 ||
	    vers != 0 ||
	    nvlist_lookup_string(nvl, "mc_dump_driver", &driver) != 0 ||
	    strcmp(driver, "imc") != 0 ||
	    nvlist_lookup_nvlist(nvl, "imc", &invl) != 0) {
		return (B_FALSE);
	}

	if (nvlist_lookup_uint32(invl, "imc_gen", &imc->imc_gen) != 0 ||
	    nvlist_lookup_nvlist_array(invl, "imc_sockets", &socks,
	    &imc->imc_nsockets) != 0 ||
	    imc->imc_nsockets > IMC_MAX_SOCKETS) {
		return (B_FALSE);
	}

	for (i = 0; i < imc->imc_nsockets; i++) {
		if (!imc_restore_socket(socks[i], &imc->imc_sockets[i]))
			return (B_FALSE);
	}

	return (B_TRUE);
}
#
# CDDL HEADER START
#
# The contents of this file are subject to the terms of the
# Common Development and Distribution License, Version 1.0 only
# (the "License").  You may not use this file except in compliance
# with the License.
#
# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
# or http://www.opensolaris.org/os/licensing.
# See the License for the specific language governing permissions
# and limitations under the License.
#
# When distributing Covered Code, include this CDDL HEADER in each
# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
# If applicable, add the following below this CDDL HEADER, with the
# fields enclosed by brackets "[]" replaced with your own identifying
# information: Portions Copyright [yyyy] [name of copyright owner]
#
# CDDL HEADER END
#
#
# Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
# Use is subject to license terms.
#
#ident	"%Z%%M%	%I%	%E% SMI"

MCAMD_CMN_SRCS = \
	mcamd_err.c \
	mcamd_misc.c \
	mcamd_patounum.c \
	mcamd_unumtopa.c \
	mcamd_synd.c \
	mcamd_rowcol_tbl.c \
	mcamd_rowcol.c

MCAMD_CMN_OBJS = $(MCAMD_CMN_SRCS:%.c=%.o)
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#ifndef _MCAMD_API_H
#define	_MCAMD_API_H

/*
 * Primary header file for mcamd_* routines in $SRC/common/mc.  The
 * routines not implemented there are required to be implemented in the
 * kernel or userland consumer of this interface (such as the mc-amd driver).
 * The common code must use the wrapper functions provided by the consumer
 * to navigate the MC tree, get properties etc.
 */

#if defined(_KERNEL)
#include <sys/systm.h>
#include <sys/sunddi.h>
#else
#include <string.h>
#include <assert.h>
#endif

#include <sys/types.h>
#include <sys/mc.h>
#include <sys/mca_amd.h>
#include <sys/mc_amd.h>

#ifdef __cplusplus
extern "C" {
#endif

/*
 * Consumers of this common code must implement the following types.
 */
typedef struct mcamd_node mcamd_node_t;
struct mcamd_hdl;

/*
 * Properties and raw register values for an mcamd_node_t are retrieved via
 * mcamd_get_numprop(s) and mcamd_get_cfgreg(s) specifying a property or
 * register code below.
 */

typedef uint64_t mcamd_prop_t;
typedef uint32_t mcamd_cfgreg_t;

typedef enum mcamd_propcode {
	/*
	 * Common properties
	 */
	MCAMD_PROP_NUM = 0x4000,
#define	MCAMD_PROPSTR_NUM		"num"
	MCAMD_PROP_SIZE,
#define	MCAMD_PROPSTR_SIZE		"size"
	MCAMD_PROP_BASE_ADDR,
#define	MCAMD_PROPSTR_BASE_ADDR		"base-addr"
	/*
	 * Memory controller properties
	 */
	MCAMD_PROP_REV = 0x5000,
#define	MCAMD_PROPSTR_REV		"revision"
	MCAMD_PROP_LIM_ADDR,
#define	MCAMD_PROPSTR_LIM_ADDR		"lim-addr"
	MCAMD_PROP_ILEN,
#define	MCAMD_PROPSTR_ILEN		"node-ilen"
	MCAMD_PROP_ILSEL,
#define	MCAMD_PROPSTR_ILSEL		"node-ilsel"
	MCAMD_PROP_CSINTLVFCTR,
#define	MCAMD_PROPSTR_CSINTLVFCTR	"cs-intlv-factor"
	MCAMD_PROP_DRAMHOLE_SIZE,
#define	MCAMD_PROPSTR_DRAMHOLE_SIZE	"dram-hole-size"
	MCAMD_PROP_ACCESS_WIDTH,
#define	MCAMD_PROPSTR_ACCESS_WIDTH	"access-width"
	MCAMD_PROP_CSBANKMAPREG,
#define	MCAMD_PROPSTR_CSBANKMAPREG	"bank-mapping"
	MCAMD_PROP_BANKSWZL,
#define	MCAMD_PROPSTR_BANKSWZL		"bankswizzle"
	MCAMD_PROP_MOD64MUX,
#define	MCAMD_PROPSTR_MOD64MUX		"mismatched-dimm-support"
	MCAMD_PROP_SPARECS,
#define	MCAMD_PROPSTR_SPARECS		"spare-csnum"
	MCAMD_PROP_BADCS,
#define	MCAMD_PROPSTR_BADCS		"bad-csnum"
	/*
	 * Chip-select properties
	 */
	MCAMD_PROP_MASK = 0x6000,
#define	MCAMD_PROPSTR_MASK		"mask"
	MCAMD_PROP_CSBE,
#define	MCAMD_PROPSTR_CSBE		"cs-bank-enable"
	MCAMD_PROP_SPARE,
#define	MCAMD_PROPSTR_SPARE		"online-spare"
	MCAMD_PROP_TESTFAIL,
#define	MCAMD_PROPSTR_TESTFAIL		"failed-test"
	MCAMD_PROP_CSDIMM1,
#define	MCAMD_PROPSTR_CSDIMM1		"dimm1-num"
	MCAMD_PROP_CSDIMM2,
#define	MCAMD_PROPSTR_CSDIMM2		"dimm2-num"
	MCAMD_PROP_DIMMRANK
#define	MCAMD_PROPSTR_DIMMRANK		"dimm-rank"
} mcamd_propcode_t;

typedef enum mcamd_regcode {
	MCAMD_REG_DRAMBASE = 0x7000,
	MCAMD_REG_DRAMLIMIT,
	MCAMD_REG_DRAMHOLE,
	MCAMD_REG_DRAMCFGLO,
	MCAMD_REG_DRAMCFGHI,
	MCAMD_REG_CSBASE,
	MCAMD_REG_CSMASK
} mcamd_regcode_t;
/*
 * Flags for mcamd_dprintf
 */
#define	MCAMD_DBG_ERR		0x1
#define	MCAMD_DBG_FLOW		0x2

typedef union mcamd_dimm_offset_un mcamd_dimm_offset_un_t;

/*
 * Offset definition.  Encode everything in a single uint64_t, allowing some
 * room for growth in numbers of rows/columns/banks in future MC revisions.
 * Some consumers will handle this as an opaque uint64 to be passed around,
 * while others will want to look inside via the union defined below.  Since
 * we must support a 32-bit kernel we structure this as two uint32_t.
 */

#define	MCAMD_OFFSET_VERSION_0			0x0
#define	MCAMD_OFFSET_VERSION			MCAMD_OFFSET_VERSION_0

union mcamd_dimm_offset_un {
	uint64_t _dou_offset;
	struct {
		struct {
			uint32_t dou_col:20;	/* column address */
			uint32_t dou_bank:4;	/* internal sdram bank number */
			uint32_t unused:8;
		} lo;
		struct {
			uint32_t dou_row:20;	/* row address */
			uint32_t dou_rank:3;	/* cs rank on dimm */
			uint32_t unused:4;
			uint32_t dou_version:4;	/* offset encoding version */
			uint32_t dou_valid:1;	/* set if valid */
		} hi;
	} _dou_hilo;
};

#define	do_offset  _dou_offset

#define	do_valid _dou_hilo.hi.dou_valid
#define	do_version _dou_hilo.hi.dou_version
#define	do_rank _dou_hilo.hi.dou_rank
#define	do_row _dou_hilo.hi.dou_row
#define	do_bank _dou_hilo.lo.dou_bank
#define	do_col _dou_hilo.lo.dou_col

/*
 * The following work on an offset treated as a uint64_t.
 */
#define	MCAMD_RC_OFFSET_VALID(offset) (((uint64_t)(offset) & (1ULL << 63)) != 0)
#define	MCAMD_RC_OFFSET_VERSION(offset) (((uint64_t)offset >> 59) & 0xf)

/*
 * Value to be used to indicate an invalid offset.
 */
#define	MCAMD_RC_INVALID_OFFSET 0x0

/*
 * Routines provided by the common mcamd code.
 */
extern const char *mcamd_get_propname(mcamd_propcode_t);

extern int mcamd_patounum(struct mcamd_hdl *, mcamd_node_t *, uint64_t,
    uint8_t, uint8_t, uint32_t, int, mc_unum_t *);
extern int mcamd_unumtopa(struct mcamd_hdl *, mcamd_node_t *, mc_unum_t *,
    uint64_t *);
extern int mc_pa_to_offset(struct mcamd_hdl *, mcamd_node_t *, mcamd_node_t *,
    uint64_t, uint64_t *);
extern int mc_offset_to_pa(struct mcamd_hdl *, mcamd_node_t *, mcamd_node_t *,
    uint64_t, uint64_t *);

extern int mcamd_cs_size(struct mcamd_hdl *, mcamd_node_t *, int, size_t *);

extern int mcamd_synd_validate(struct mcamd_hdl *, uint32_t, int);
extern int mcamd_eccsynd_decode(struct mcamd_hdl *, uint32_t, uint_t *);
extern int mcamd_cksynd_decode(struct mcamd_hdl *, uint32_t, uint_t *,
    uint_t *);
extern int mcamd_cksym_decode(struct mcamd_hdl *, uint_t, int *, int *,
    int *, int *);

extern void *mcamd_set_errno_ptr(struct mcamd_hdl *, int);
extern const char *mcamd_strerror(int);
extern const char *mcamd_errmsg(struct mcamd_hdl *);

/*
 * Routines to be provided by wrapper code.
 */
extern mcamd_node_t *mcamd_mc_next(struct mcamd_hdl *, mcamd_node_t *,
    mcamd_node_t *);
extern mcamd_node_t *mcamd_cs_next(struct mcamd_hdl *, mcamd_node_t *,
    mcamd_node_t *);
extern mcamd_node_t *mcamd_dimm_next(struct mcamd_hdl *, mcamd_node_t *,
    mcamd_node_t *);

extern mcamd_node_t *mcamd_cs_mc(struct mcamd_hdl *, mcamd_node_t *);
extern mcamd_node_t *mcamd_dimm_mc(struct mcamd_hdl *, mcamd_node_t *);

extern int mcamd_get_numprop(struct mcamd_hdl *, mcamd_node_t *,
    mcamd_propcode_t, mcamd_prop_t *);
extern int mcamd_get_numprops(struct mcamd_hdl *, ...);
extern int mcamd_get_cfgreg(struct mcamd_hdl *, mcamd_node_t *,
    mcamd_regcode_t, mcamd_cfgreg_t *);
extern int mcamd_get_cfgregs(struct mcamd_hdl *, ...);

extern int mcamd_errno(struct mcamd_hdl *);
extern int mcamd_set_errno(struct mcamd_hdl *, int);
extern void mcamd_dprintf(struct mcamd_hdl *, int, const char *, ...);

#ifdef __cplusplus
}
#endif

#endif /* _MCAMD_API_H */
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 *
 * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#include <mcamd_api.h>
#include <mcamd_err.h>

static const char *const _mcamd_errlist[] = {
	"Invalid syndrome",			/* EMCAMD_SYNDINVALID */
	"Invalid configuration tree",		/* EMCAMD_TREEINVALID */
	"Address not found",			/* EMCAMD_NOADDR */
	"Operation not supported",		/* EMCAMD_NOTSUP */
	"Too few valid address bits",		/* EMCAMD_INSUFF_RES */
};

static const int _mcamd_nerr = sizeof (_mcamd_errlist) /
    sizeof (_mcamd_errlist[0]);

void *
mcamd_set_errno_ptr(struct mcamd_hdl *mcamd, int err)
{
	(void) mcamd_set_errno(mcamd, err);
	return (NULL);
}

const char *
mcamd_strerror(int err)
{
	const char *str = NULL;

	if (err >= EMCAMD_BASE && (err - EMCAMD_BASE) < _mcamd_nerr)
		str = _mcamd_errlist[err - EMCAMD_BASE];

	return (str == NULL ? "Unknown error" : str);
}

const char *
mcamd_errmsg(struct mcamd_hdl *mcamd)
{
	return (mcamd_strerror(mcamd_errno(mcamd)));
}
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 *
 * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#ifndef _MCAMD_ERR_H
#define	_MCAMD_ERR_H

#ifdef __cplusplus
extern "C" {
#endif

#define	EMCAMD_BASE	2000	/* out of system's and consumer's way */

enum {
	EMCAMD_SYNDINVALID = EMCAMD_BASE,	/* invalid syndrome */
	EMCAMD_TREEINVALID,			/* invalid configuration tree */
	EMCAMD_NOADDR,				/* address not found */
	EMCAMD_NOTSUP,				/* operation not supported */
	EMCAMD_INSUFF_RES			/* insufficient resolution */
};

extern const char *mcamd_errmsg(struct mcamd_hdl *);
extern const char *mcamd_strerror(int);
extern int mcamd_errno(struct mcamd_hdl *);
extern int mcamd_set_errno(struct mcamd_hdl *, int);
extern void *mcamd_set_errno_ptr(struct mcamd_hdl *, int);

#ifdef __cplusplus
}
#endif

#endif /* _MCAMD_ERR_H */
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 *
 * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#include <sys/types.h>

#include <mcamd_api.h>

static struct mcproptostr {
	mcamd_propcode_t code;
	const char *name;
} _propstrings[] = {
	/*
	 * Common codes
	 */
	{ MCAMD_PROP_NUM, MCAMD_PROPSTR_NUM },
	{ MCAMD_PROP_SIZE, MCAMD_PROPSTR_SIZE },
	{ MCAMD_PROP_BASE_ADDR, MCAMD_PROPSTR_BASE_ADDR },
	/*
	 * Memory controller properties
	 */
	{ MCAMD_PROP_REV, MCAMD_PROPSTR_REV },
	{ MCAMD_PROP_LIM_ADDR, MCAMD_PROPSTR_LIM_ADDR },
	{ MCAMD_PROP_ILEN, MCAMD_PROPSTR_ILEN },
	{ MCAMD_PROP_ILSEL, MCAMD_PROPSTR_ILSEL },
	{ MCAMD_PROP_CSINTLVFCTR, MCAMD_PROPSTR_CSINTLVFCTR },
	{ MCAMD_PROP_ACCESS_WIDTH, MCAMD_PROPSTR_ACCESS_WIDTH },
	{ MCAMD_PROP_CSBANKMAPREG, MCAMD_PROPSTR_CSBANKMAPREG },
	{ MCAMD_PROP_BANKSWZL, MCAMD_PROPSTR_BANKSWZL },
	{ MCAMD_PROP_DRAMHOLE_SIZE, MCAMD_PROPSTR_DRAMHOLE_SIZE },
	{ MCAMD_PROP_MOD64MUX, MCAMD_PROPSTR_MOD64MUX },
	{ MCAMD_PROP_SPARECS, MCAMD_PROPSTR_SPARECS },
	{ MCAMD_PROP_BADCS, MCAMD_PROPSTR_BADCS },
	/*
	 * Chip-select properties
	 */
	{ MCAMD_PROP_MASK, MCAMD_PROPSTR_MASK },
	{ MCAMD_PROP_CSBE, MCAMD_PROPSTR_CSBE },
	{ MCAMD_PROP_SPARE, MCAMD_PROPSTR_SPARE },
	{ MCAMD_PROP_TESTFAIL, MCAMD_PROPSTR_TESTFAIL },
	{ MCAMD_PROP_CSDIMM1, MCAMD_PROPSTR_CSDIMM1 },
	{ MCAMD_PROP_CSDIMM2, MCAMD_PROPSTR_CSDIMM2 },
	{ MCAMD_PROP_DIMMRANK, MCAMD_PROPSTR_DIMMRANK },
};

static const int _nprop = sizeof (_propstrings) /
    sizeof (struct mcproptostr);

const char *
mcamd_get_propname(mcamd_propcode_t code)
{
	int i;

	for (i = 0; i < _nprop; i++) {
		if (_propstrings[i].code == code)
			return (_propstrings[i].name);
	}

	return (NULL);
}
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 *
 * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

/*
 * Given a physical address and an optional syndrome, determine the
 * name of the memory module that contains it.
 */

#include <sys/errno.h>
#include <sys/types.h>
#include <sys/mc.h>

#include <mcamd_api.h>
#include <mcamd_err.h>

#define	MC_SYSADDR_MSB	39
#define	MC_SYSADDR_LSB	3

#define	CSDIMM1	0x1
#define	CSDIMM2	0x2

#define	BITS(val, high, low) \
	((val) & (((2ULL << (high)) - 1) & ~((1ULL << (low)) - 1)))

/*
 * iaddr_gen generates a "normalized" DRAM controller input address
 * from a system address (physical address) if it falls within the
 * mapped range for this memory controller.  Normalisation is
 * performed by subtracting the node base address from the system address,
 * allowing from hoisting, and excising any bits being used in node
 * interleaving.
 */
static int
iaddr_gen(struct mcamd_hdl *hdl, mcamd_node_t *mc, uint64_t pa,
    uint64_t *iaddrp)
{
	uint64_t orig = pa;
	uint64_t mcnum, base, lim, dramaddr, ilen, ilsel, top, holesz;

	if (!mcamd_get_numprops(hdl,
	    mc, MCAMD_PROP_NUM, &mcnum,
	    mc, MCAMD_PROP_BASE_ADDR, &base,
	    mc, MCAMD_PROP_LIM_ADDR, &lim,
	    mc, MCAMD_PROP_ILEN, &ilen,
	    mc, MCAMD_PROP_ILSEL, &ilsel,
	    mc, MCAMD_PROP_DRAMHOLE_SIZE, &holesz,
	    NULL)) {
		mcamd_dprintf(hdl, MCAMD_DBG_ERR, "iaddr_gen: failed to "
		    "lookup required properties");
		return (mcamd_set_errno(hdl, EMCAMD_TREEINVALID));
	}

	/*
	 * A node with no mapped memory (no active chip-selects is usually
	 * mapped with base and lim both zero.  We'll cover that case and
	 * any other where the range is 0.
	 */
	if (base == lim)
		return (mcamd_set_errno(hdl, EMCAMD_NOADDR));

	if (pa < base || pa > lim) {
		mcamd_dprintf(hdl, MCAMD_DBG_FLOW, "iaddr_gen: PA 0x%llx not "
		    "in range [0x%llx, 0x%llx] of MC %d\n", pa, base, lim,
		    (int)mcnum);
		return (mcamd_set_errno(hdl, EMCAMD_NOADDR));
	}

	/*
	 * Rev E and later added the DRAM Hole Address Register for
	 * memory hoisting.  In earlier revisions memory hoisting is
	 * achieved by following some algorithm to modify the CS bases etc,
	 * and this pa to unum algorithm will simply see those modified
	 * values.  But if the Hole Address Register is being used then
	 * we need to reduce any address at or above 4GB by the size of
	 * the hole.
	 */
	if (holesz != 0 && pa >= 0x100000000) {
		pa -= holesz;
		mcamd_dprintf(hdl, MCAMD_DBG_FLOW, "iaddr_gen: dram hole "
		    "valid; pa decremented from 0x%llx to 0x%llx for "
		    "a dramhole size of 0x%llx\n", orig, pa, holesz);
	}

	dramaddr = BITS(pa, 39, 0) - BITS(base, 39, 24);

	if (ilen != 0) {
		int pailsel;

		if (ilen != 1 && ilen != 3 && ilen != 7) {
			mcamd_dprintf(hdl, MCAMD_DBG_ERR, "Invalid intlven "
			    "of %d for MC %d\n", (int)ilen, (int)mcnum);
			return (mcamd_set_errno(hdl, EMCAMD_TREEINVALID));
		}

		if ((pailsel = BITS(pa, 14, 12) >> 12 & ilen) != ilsel) {
			mcamd_dprintf(hdl, MCAMD_DBG_FLOW, "iaddr_gen: "
			    "PA 0x%llx in a %d-way node interleave indicates "
			    "selection %d, MC %d has ilsel of %d\n",
			    pa, (int)ilen + 1, pailsel, (int)mcnum, (int)ilsel);
			return (mcamd_set_errno(hdl, EMCAMD_NOADDR));
		}

		if (ilen == 1)
			top = BITS(dramaddr, 36, 13) >> 1;
		else if (ilen == 3)
			top = BITS(dramaddr, 37, 14) >> 2;
		else if (ilen == 7)
			top = BITS(dramaddr, 38, 15) >> 3;
	} else {
		top = BITS(dramaddr, 35, 12);
	}

	*iaddrp = top | BITS(dramaddr, 11, 0);

	mcamd_dprintf(hdl, MCAMD_DBG_FLOW, "iaddr_gen: PA 0x%llx in range "
	    "[0x%llx, 0x%llx] of MC %d; normalized address for cs compare "
	    "is 0x%llx\n", pa, base, lim, (int)mcnum, *iaddrp);

	return (0);
}

/*
 * cs_match determines whether the given DRAM controller input address
 * would be responded to by the given chip-select (which may or may not
 * be interleaved with other chip-selects).  Since we include nodes
 * for spare chip-selects (if any) and those marked TestFail (if any)
 * we must check chip-select-bank-enable.
 */
static int
cs_match(struct mcamd_hdl *hdl, uint64_t iaddr, mcamd_node_t *cs)
{
	uint64_t csnum, csbase, csmask, csbe;
	int match = 0;

	if (!mcamd_get_numprops(hdl,
	    cs, MCAMD_PROP_NUM, &csnum,
	    cs, MCAMD_PROP_BASE_ADDR, &csbase,
	    cs, MCAMD_PROP_MASK, &csmask,
	    cs, MCAMD_PROP_CSBE, &csbe,
	    NULL)) {
		mcamd_dprintf(hdl, MCAMD_DBG_ERR, "cs_match: failed to lookup "
		    "required properties\n");
		return (0);
	}

	if (csbe) {
		match = ((iaddr & ~csmask) == (csbase & ~csmask));

		mcamd_dprintf(hdl, MCAMD_DBG_FLOW, "cs_match: iaddr 0x%llx "
		    "does %smatch CS %d (base 0x%llx, mask 0x%llx)\n", iaddr,
		    match ? "" : "not ", (int)csnum, csbase, csmask);
	} else {
		mcamd_dprintf(hdl, MCAMD_DBG_FLOW, "cs_match: iaddr 0x%llx "
		    "does not match disabled CS %d\n", iaddr, (int)csnum);
	}

	return (match);
}

/*
 * Given a chip-select node determine whether it has been substituted
 * by the online spare chip-select.
 */
static mcamd_node_t *
cs_sparedto(struct mcamd_hdl *hdl, mcamd_node_t *cs, mcamd_node_t *mc)
{
	uint64_t csnum, badcsnum, sparecsnum, tmpcsnum;

	if (!mcamd_get_numprops(hdl,
	    cs, MCAMD_PROP_NUM, &csnum,
	    mc, MCAMD_PROP_BADCS, &badcsnum,
	    mc, MCAMD_PROP_SPARECS, &sparecsnum,
	    NULL)) {
		mcamd_dprintf(hdl, MCAMD_DBG_ERR, "cs_sparedto: failed to "
		    "lookup required properties\n");
		return (NULL);
	}

	if ((badcsnum == MC_INVALNUM && sparecsnum == MC_INVALNUM) ||
	    csnum != badcsnum)
		return (NULL);

	for (cs = mcamd_cs_next(hdl, mc, NULL); cs != NULL;
	    cs = mcamd_cs_next(hdl, mc, cs)) {
		if (!mcamd_get_numprop(hdl, cs, MCAMD_PROP_NUM, &tmpcsnum)) {
			mcamd_dprintf(hdl, MCAMD_DBG_ERR, "cs_sparedto: "
			    "fail to lookup csnum - cannot reroute to spare\n");
			return (NULL);
		}
		if (tmpcsnum == sparecsnum)
			break;
	}

	if (cs != NULL) {
		mcamd_dprintf(hdl, MCAMD_DBG_FLOW, "cs_sparedto: cs#%d is "
		    "redirected to active online spare of cs#%d\n", csnum,
		    sparecsnum);
	} else {
		mcamd_dprintf(hdl, MCAMD_DBG_ERR, "cs_sparedto: cs#%d is "
		    "redirected but cannot find spare cs# - cannout reroute to "
		    "cs#%d\n", csnum, sparecsnum);
	}

	return (cs);
}

/*
 * Having determined which node and chip-select an address maps to,
 * as well as whether it is a dimm1, dimm2 or dimm1/dimm2 pair
 * involved, fill the unum structure including an optional dimm offset
 * member.
 */
static int
unum_fill(struct mcamd_hdl *hdl, mcamd_node_t *cs, int which,
    uint64_t iaddr, mc_unum_t *unump, int incloff)
{
	uint64_t chipnum, csnum, dimm1, dimm2, ranknum;
	mcamd_node_t *mc, *dimm;
	int offsetdimm;
	int i;

	if ((mc = mcamd_cs_mc(hdl, cs)) == NULL ||
	    !mcamd_get_numprops(hdl,
	    mc, MCAMD_PROP_NUM, &chipnum,
	    cs, MCAMD_PROP_NUM, &csnum,
	    cs, MCAMD_PROP_DIMMRANK, &ranknum,
	    NULL)) {
		mcamd_dprintf(hdl, MCAMD_DBG_ERR, "unum_fill: failed to "
		    "lookup required properties\n");
		return (mcamd_set_errno(hdl, EMCAMD_TREEINVALID));
	}

	if ((which & CSDIMM1) &&
	    !mcamd_get_numprop(hdl, cs, MCAMD_PROP_CSDIMM1, &dimm1) ||
	    (which & CSDIMM2) &&
	    !mcamd_get_numprop(hdl, cs, MCAMD_PROP_CSDIMM2, &dimm2)) {
		mcamd_dprintf(hdl, MCAMD_DBG_ERR, "unum_fill: failed to "
		    "lookup dimm1/dimm2 properties\n");
		return (mcamd_set_errno(hdl, EMCAMD_TREEINVALID));
	}

	unump->unum_board = 0;
	unump->unum_chip = (int)chipnum;
	unump->unum_mc = 0;
	unump->unum_chan = MC_INVALNUM;
	unump->unum_cs = (int)csnum;
	unump->unum_rank = (int)ranknum;

	for (i = 0; i < MC_UNUM_NDIMM; i++) {
		unump->unum_dimms[i] = MC_INVALNUM;
	}
	switch (which) {
	case CSDIMM1:
		unump->unum_dimms[0] = (int)dimm1;
		offsetdimm = (int)dimm1;
		break;
	case CSDIMM2:
		unump->unum_dimms[0] = (int)dimm2;
		offsetdimm = (int)dimm2;
		break;
	case CSDIMM1 | CSDIMM2:
		unump->unum_dimms[0] = (int)dimm1;
		unump->unum_dimms[1] = (int)dimm2;
		offsetdimm = (int)dimm1;
		break;
	}

	if (!incloff) {
		unump->unum_offset = MCAMD_RC_INVALID_OFFSET;
		return (0);
	}

	/*
	 * We wish to calculate a dimm offset.  In the paired case we will
	 * lookup dimm1 (see offsetdimm above).
	 */
	for (dimm = mcamd_dimm_next(hdl, mc, NULL); dimm != NULL;
	    dimm = mcamd_dimm_next(hdl, mc, dimm)) {
		uint64_t dnum;
		if (!mcamd_get_numprop(hdl, dimm, MCAMD_PROP_NUM, &dnum)) {
			mcamd_dprintf(hdl, MCAMD_DBG_ERR, "unum_fill: failed "
			    "to lookup dimm number property\n");
			continue;
		}
		if (dnum == offsetdimm)
			break;
	}

	if (dimm == NULL) {
		mcamd_dprintf(hdl, MCAMD_DBG_ERR, "unum_fill: failed to "
		    "find dimm with number %d for offset calculation\n",
		    offsetdimm);
		unump->unum_offset = MCAMD_RC_INVALID_OFFSET;
		return (mcamd_set_errno(hdl, EMCAMD_TREEINVALID));
	}

	/*
	 * mc_pa_to_offset sets the offset to an invalid value if
	 * it hits an error.
	 */
	(void) mc_pa_to_offset(hdl, mc, cs, iaddr, &unump->unum_offset);

	return (0);
}

/*
 * We have translated a system address to a (node, chip-select), and wish
 * to determine the associated dimm or dimms.
 *
 * A (node, chip-select) pair identifies one (in 64-bit MC mode) or two (in
 * 128-bit MC mode) DIMMs.  In the case of a single dimm it is usually in a
 * lodimm (channel A) slot, but if mismatched dimm support is present it may
 * be an updimm (channel B).
 *
 * Where just one dimm is associated with the chip-select we are done.
 * Where there are two dimms associated with the chip-select we can
 * use the ECC type and/or syndrome to determine which of the pair we
 * resolve to, if the error is correctable.  If the error is uncorrectable
 * then in 64/8 ECC mode we can still resolve to a single dimm (since ECC
 * is calculated and checked on each half of the data separately), but
 * in ChipKill mode we cannot resolve down to a single dimm.
 */
static int
mc_whichdimm(struct mcamd_hdl *hdl, mcamd_node_t *cs, uint64_t pa,
    uint8_t valid_lo, uint32_t synd, int syndtype)
{
	int lobit, hibit, data, check;
	uint64_t dimm1, dimm2;
	uint_t sym, pat;
	int ndimm;

	/*
	 * Read the associated dimm instance numbers.  The provider must
	 * assure that if there is just one dimm then it is in the first
	 * property, and if there are two then the first must be on
	 * channel A.
	 */
	if (!mcamd_get_numprops(hdl,
	    cs, MCAMD_PROP_CSDIMM1, &dimm1,
	    cs, MCAMD_PROP_CSDIMM2, &dimm2,
	    NULL)) {
		mcamd_dprintf(hdl, MCAMD_DBG_ERR, "mc_whichdimm: failed to "
		    "lookup required properties");
		return (mcamd_set_errno(hdl, EMCAMD_TREEINVALID));
	}
	ndimm = (dimm1 != MC_INVALNUM) + (dimm2 != MC_INVALNUM);
	if (ndimm == 0) {
		mcamd_dprintf(hdl, MCAMD_DBG_ERR, "mc_whichdimm: found no "
		    "dimms associated with chip-select");
		return (mcamd_set_errno(hdl, EMCAMD_TREEINVALID));
	}

	if (ndimm == 1) {
		mcamd_dprintf(hdl, MCAMD_DBG_FLOW, "mc_whichdimm: just one "
		    "dimm associated with this chip-select");
		return (CSDIMM1);
	}

	/*
	 * 64/8 ECC is checked separately for the upper and lower
	 * halves, so even an uncorrectable error is contained within
	 * one of the two halves.  If we have sufficient address resolution
	 * then we can determine which DIMM.
	 */
	if (syndtype == AMD_SYNDTYPE_ECC) {
		if (valid_lo <= MC_SYSADDR_LSB) {
			mcamd_dprintf(hdl, MCAMD_DBG_FLOW, "mc_whichdimm: 64/8 "
			    "ECC in 128-bit mode, PA 0x%llx is in %s half\n",
			    pa, pa & 0x8 ? "upper" : "lower");
			return (pa & 0x8 ? CSDIMM2 : CSDIMM1);
		} else {
			mcamd_dprintf(hdl, MCAMD_DBG_FLOW, "mc_whichdimm: "
			    "64/8 ECC in 128-bit mode, PA 0x%llx with least "
			    "significant valid bit %d cannot be resolved to "
			    "a single DIMM\n", pa, valid_lo);
			return (mcamd_set_errno(hdl, EMCAMD_INSUFF_RES));
		}
	}

	/*
	 * ChipKill ECC
	 */
	if (mcamd_cksynd_decode(hdl, synd, &sym, &pat)) {
		/*
		 * A correctable ChipKill syndrome and we can tell
		 * which half the error was in from the symbol number.
		 */
		if (mcamd_cksym_decode(hdl, sym, &lobit, &hibit, &data,
		    &check) == 0)
			return (mcamd_set_errno(hdl, EMCAMD_SYNDINVALID));

		if (data && hibit <= 63 || check && hibit <= 7) {
			mcamd_dprintf(hdl, MCAMD_DBG_FLOW, "mc_whichdimm: "
			    "ChipKill symbol %d (%s %d..%d), so LODIMM\n", sym,
			    data ? "data" : "check", lobit, hibit);
			return (CSDIMM1);
		} else {
			mcamd_dprintf(hdl, MCAMD_DBG_FLOW, "mc_whichdimm: "
			    "ChipKill symbol %d (%s %d..%d), so UPDIMM\n", sym,
			    data ? "data" : "check", lobit, hibit);
			return (CSDIMM2);
		}
	} else {
		/*
		 * An uncorrectable error while in ChipKill ECC mode - can't
		 * tell which dimm or dimms the errors lie within.
		 */
		mcamd_dprintf(hdl, MCAMD_DBG_FLOW, "mc_whichhdimm: "
		    "uncorrectable ChipKill, could be either LODIMM "
		    "or UPDIMM\n");
		return (CSDIMM1 | CSDIMM2);
	}
}

#ifdef DEBUG
/*
 * Brute-force BKDG pa to cs translation, coded to look as much like the
 * BKDG code as possible.
 */
static int
mc_bkdg_patounum(struct mcamd_hdl *hdl, mcamd_node_t *mc, uint64_t pa,
    uint8_t valid_lo, uint32_t synd, int syndtype,
    mc_unum_t *unump)
{
	int which;
	uint64_t mcnum, rev;
	mcamd_node_t *cs;
	/*
	 * Raw registers as per BKDG
	 */
	uint32_t HoleEn;
	uint32_t DramBase, DramLimit;
	uint32_t CSBase,  CSMask;
	/*
	 * Variables as per BKDG
	 */
	int Ilog;
	uint32_t SystemAddr = (uint32_t)(pa >> 8);
	uint64_t IntlvEn, IntlvSel;
	uint32_t HoleOffset;
	uint32_t InputAddr, Temp;

	if (!mcamd_get_numprops(hdl,
	    mc, MCAMD_PROP_NUM, &mcnum,
	    mc, MCAMD_PROP_REV, &rev, NULL) || !mcamd_get_cfgregs(hdl,
	    mc, MCAMD_REG_DRAMBASE, &DramBase,
	    mc, MCAMD_REG_DRAMLIMIT, &DramLimit,
	    mc, MCAMD_REG_DRAMHOLE, &HoleEn, NULL)) {
		mcamd_dprintf(hdl, MCAMD_DBG_ERR, "mc_bkdg_patounm: failed "
		    "to lookup required properties and registers\n");
		return (mcamd_set_errno(hdl, EMCAMD_TREEINVALID));
	}

	/*
	 * BKDG line to skip		Why
	 *
	 * F1Offset = ...		Register already read,
	 * DramBase = Get_PCI()		and retrieved above.
	 * DramEn = ...			Function only called for enabled nodes.
	 */
	IntlvEn = (DramBase & 0x00000700) >> 8;
	DramBase &= 0xffff0000;
	/* DramLimit = Get_PCI()	Retrieved above */
	IntlvSel = (DramLimit & 0x00000700) >> 8;
	DramLimit |= 0x0000ffff;
	/* HoleEn = ...			Retrieved above */
	HoleOffset = (HoleEn & 0x0000ff00) << 8;
	HoleEn &= 0x00000001;

	if (!(DramBase <= SystemAddr && SystemAddr <= DramLimit)) {
		mcamd_dprintf(hdl, MCAMD_DBG_FLOW, "mc_bkdg_patounum: "
		    "SystemAddr 0x%x derived from PA 0x%llx is not in the "
		    "address range [0x%x, 0x%x] of MC %d\n",
		    SystemAddr, pa, DramBase, DramLimit, (int)mcnum);
		return (mcamd_set_errno(hdl, EMCAMD_NOADDR));
	}

	if (HoleEn && SystemAddr > 0x00ffffff)
		InputAddr = SystemAddr - HoleOffset;

	InputAddr = SystemAddr - DramBase;

	if (IntlvEn) {
		if (IntlvSel == ((SystemAddr >> 4) & IntlvEn)) {
			switch (IntlvEn) {
			case 1:
				Ilog = 1;
				break;
			case 3:
				Ilog = 2;
				break;
			case 7:
				Ilog = 3;
				break;
			default:
				return (mcamd_set_errno(hdl,
				    EMCAMD_TREEINVALID));
			}
			Temp = (InputAddr >> (4 + Ilog)) << 4;
			InputAddr = (Temp | (SystemAddr & 0x0000000f));
		} else {
			/* not this node */
			mcamd_dprintf(hdl, MCAMD_DBG_FLOW, "mc_bkdg_patounum: "
			    "Node interleaving, MC node %d not selected\n",
			    (int)mcnum);
			return (mcamd_set_errno(hdl, EMCAMD_NOADDR));
		}
	}

	if (!MC_REV_MATCH(rev, MC_F_REVS_FG))
		InputAddr <<= 4;

	for (cs = mcamd_cs_next(hdl, mc, NULL); cs != NULL;
	    cs = mcamd_cs_next(hdl, mc, cs)) {
		uint64_t csnum, CSEn;

		if (!mcamd_get_cfgregs(hdl,
		    cs, MCAMD_REG_CSBASE, &CSBase,
		    cs, MCAMD_REG_CSMASK, &CSMask,
		    NULL) ||
		    !mcamd_get_numprops(hdl,
		    cs, MCAMD_PROP_NUM, &csnum,
		    cs, MCAMD_PROP_CSBE, &CSEn,
		    NULL)) {
			mcamd_dprintf(hdl, MCAMD_DBG_ERR, "mc_bkdg_patounm: "
			    "failed to read cs registers\n");
			return (mcamd_set_errno(hdl, EMCAMD_TREEINVALID));
		}

		/*
		 * BKDG line to skip		Why
		 *
		 * F2Offset =			Register already read,
		 * F2MaskOffset (rev F)		Register already read
		 * CSBase =			Register already read
		 * CSEn =			We only keep enabled cs.
		 */
		if (MC_REV_MATCH(rev, MC_F_REVS_FG)) {
			CSBase &= 0x1ff83fe0;
			/* CSMask = Get_PCI()		Retrieved above */
			CSMask = (CSMask | 0x0007c01f) & 0x1fffffff;
		} else {
			CSBase &= 0xffe0fe00;
			/* CSMask = Get_PCI()		Retrieved above */
			CSMask = (CSMask | 0x001f01ff) & 0x3fffffff;
		}

		if (CSEn && (InputAddr & ~CSMask) == (CSBase & ~CSMask)) {
			mcamd_node_t *sparecs;

			mcamd_dprintf(hdl, MCAMD_DBG_FLOW, "mc_bkdg_patounum: "
			    "match for chip select %d of MC %d\n", (int)csnum,
			    (int)mcnum);

			if ((sparecs = cs_sparedto(hdl, cs, mc)) != NULL)
				cs = sparecs;

			if ((which = mc_whichdimm(hdl, cs, pa, valid_lo,
			    synd, syndtype)) < 0)
				return (-1); /* errno is set for us */

			/*
			 * The BKDG algorithm drops low-order bits that
			 * are unimportant in deriving chip-select but are
			 * included in row/col/bank mapping, so do not
			 * perform offset calculation in this case.
			 */
			if (unum_fill(hdl, cs, which, InputAddr, unump, 0) < 0)
				return (-1); /* errno is set for us */

			return (0);
		}
	}

	mcamd_dprintf(hdl, MCAMD_DBG_ERR, "mc_bkdg_patounum: in range "
	    "for MC %d but no cs responds\n", (int)mcnum);

	return (mcamd_set_errno(hdl, EMCAMD_NOADDR));
}
#endif /* DEBUG */

/*
 * Called for each memory controller to see if the given address is
 * mapped to this node (as determined in iaddr_gen) and, if so, which
 * chip-select on this node responds.
 */

/*ARGSUSED*/
static int
mc_patounum(struct mcamd_hdl *hdl, mcamd_node_t *mc, uint64_t pa,
    uint8_t valid_lo, uint32_t synd, int syndtype, mc_unum_t *unump)
{
	uint64_t iaddr;
	mcamd_node_t *cs, *sparecs;
	int which;
#ifdef DEBUG
	mc_unum_t bkdg_unum;
	int bkdgres;

	/*
	 * We perform the translation twice, once using the brute-force
	 * approach of the BKDG and again using a more elegant but more
	 * difficult to review against the BKDG approach.
	 */
	mcamd_dprintf(hdl, MCAMD_DBG_FLOW, "BKDG brute-force method begins\n");
	bkdgres = mc_bkdg_patounum(hdl, mc, pa, valid_lo, synd,
	    syndtype, &bkdg_unum);
	mcamd_dprintf(hdl, MCAMD_DBG_FLOW, "BKDG brute-force method ends\n");
#endif

	if (iaddr_gen(hdl, mc, pa, &iaddr) < 0)
		return (-1); /* errno is set for us */

	for (cs = mcamd_cs_next(hdl, mc, NULL); cs != NULL;
	    cs = mcamd_cs_next(hdl, mc, cs)) {
		if (cs_match(hdl, iaddr, cs))
			break;
	}

	if (cs == NULL)
		return (mcamd_set_errno(hdl, EMCAMD_NOADDR));

	/*
	 * If the spare chip-select has been swapped in for the one just
	 * matched then it is really the spare that we are after.  Note that
	 * when the swap is done the csbase, csmask and CSBE of the spare
	 * rank do not change - accesses to the bad rank (as nominated in
	 * the Online Spare Control Register) are redirect to the spare.
	 */
	if ((sparecs = cs_sparedto(hdl, cs, mc)) != NULL) {
		cs = sparecs;
	}

	if ((which = mc_whichdimm(hdl, cs, pa, valid_lo, synd,
	    syndtype)) < 0)
		return (-1); /* errno is set for us */

	if (unum_fill(hdl, cs, which, iaddr, unump, 1) < 0)
		return (-1); /* errno is set for us */

#ifdef DEBUG
	mcamd_dprintf(hdl, MCAMD_DBG_FLOW, "bkdgres=%d res=0\n", bkdgres);
	/* offset is not checked - see note in BKDG algorithm */
	if (bkdgres != 0) {
		mcamd_dprintf(hdl, MCAMD_DBG_ERR, "BKDG alg failed while "
		    "ours succeeded\n");
	} else if (!(unump->unum_board == bkdg_unum.unum_board &&
	    unump->unum_chip == bkdg_unum.unum_chip &&
	    unump->unum_mc == bkdg_unum.unum_mc &&
	    unump->unum_chan == bkdg_unum.unum_chan &&
	    unump->unum_cs == bkdg_unum.unum_cs &&
	    unump->unum_dimms[0] == bkdg_unum.unum_dimms[0] &&
	    unump->unum_dimms[1] == bkdg_unum.unum_dimms[1])) {
		mcamd_dprintf(hdl, MCAMD_DBG_ERR,
		    "BKDG: node %d mc %d cs %d dimm(s) %d/%d\n"
		    "Ours: node 5d mc %d cs %d dimm(s) %d/%d\n",
		    bkdg_unum.unum_chip, bkdg_unum.unum_mc, bkdg_unum.unum_cs,
		    bkdg_unum.unum_dimms[0], bkdg_unum.unum_dimms[1],
		    unump->unum_chip, unump->unum_mc, unump->unum_cs,
		    unump->unum_dimms[0], unump->unum_dimms[1]);
	}
#endif /* DEBUG */

	mcamd_dprintf(hdl, MCAMD_DBG_FLOW, "Result: chip %d mc %d cs %d "
	    "offset 0x%llx\n", unump->unum_chip, unump->unum_mc,
	    unump->unum_cs, unump->unum_offset);

	return (0);
}

int
mcamd_patounum(struct mcamd_hdl *hdl, mcamd_node_t *root, uint64_t pa,
    uint8_t valid_hi, uint8_t valid_lo, uint32_t synd, int syndtype,
    mc_unum_t *unump)
{
	mcamd_node_t *mc;

	mcamd_dprintf(hdl, MCAMD_DBG_FLOW, "mcamd_patounum: pa=0x%llx, "
	    "synd=0x%x, syndtype=%d\n", pa, synd, syndtype);

	if (valid_hi < MC_SYSADDR_MSB) {
		mcamd_dprintf(hdl, MCAMD_DBG_FLOW, "mcamd_patounum: require "
		    "pa<%d> to be valid\n", MC_SYSADDR_MSB);
		return (mcamd_set_errno(hdl, EMCAMD_INSUFF_RES));
	}

	if (!mcamd_synd_validate(hdl, synd, syndtype))
		return (mcamd_set_errno(hdl, EMCAMD_SYNDINVALID));

	for (mc = mcamd_mc_next(hdl, root, NULL); mc != NULL;
	    mc = mcamd_mc_next(hdl, root, mc)) {
		if (mc_patounum(hdl, mc, pa, valid_lo, synd,
		    syndtype, unump) == 0)
			return (0);

		if (mcamd_errno(hdl) != EMCAMD_NOADDR)
			break;
	}

	return (-1); /* errno is set for us */
}
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 *
 * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

/*
 * Copyright (c) 2018, Joyent, Inc.
 */

#include <mcamd_api.h>
#include <mcamd_err.h>
#include <mcamd_rowcol_impl.h>

/*
 * Convenience structures to stash MC and CS properties in.
 */
struct mcprops {
	mcamd_prop_t num;		/* corresponding chip number */
	mcamd_prop_t rev;		/* revision */
	mcamd_prop_t width;		/* access width */
	mcamd_prop_t base;		/* MC base address */
	mcamd_prop_t lim;		/* MC limit address */
	mcamd_prop_t csbnkmap_reg;	/* chip-select bank map */
	mcamd_prop_t intlven;		/* Node-intlv mask */
	mcamd_prop_t intlvsel;		/* Node-intlv selection for this node */
	mcamd_prop_t csintlvfctr;	/* cs intlv factor on this node */
	mcamd_prop_t bnkswzl;		/* bank-swizzle mode */
	mcamd_prop_t sparecs;		/* spare cs#, if any */
	mcamd_prop_t badcs;		/* substituted cs#, if any */
};

struct csprops {
	mcamd_prop_t num;		/* chip-select number */
	mcamd_prop_t base;		/* chip-select base address */
	mcamd_prop_t mask;		/* chip-select mask */
	mcamd_prop_t testfail;		/* marked testFail */
	mcamd_prop_t dimmrank;		/* rank number on dimm(s) */
};

static int
getmcprops(struct mcamd_hdl *hdl, mcamd_node_t *mc, const char *caller,
    struct mcprops *pp)
{
	if (!mcamd_get_numprops(hdl,
	    mc, MCAMD_PROP_NUM, &pp->num,
	    mc, MCAMD_PROP_REV, &pp->rev,
	    mc, MCAMD_PROP_ACCESS_WIDTH, &pp->width,
	    mc, MCAMD_PROP_BASE_ADDR, &pp->base,
	    mc, MCAMD_PROP_LIM_ADDR, &pp->lim,
	    mc, MCAMD_PROP_CSBANKMAPREG, &pp->csbnkmap_reg,
	    mc, MCAMD_PROP_ILEN, &pp->intlven,
	    mc, MCAMD_PROP_ILSEL, &pp->intlvsel,
	    mc, MCAMD_PROP_CSINTLVFCTR, &pp->csintlvfctr,
	    mc, MCAMD_PROP_BANKSWZL, &pp->bnkswzl,
	    mc, MCAMD_PROP_SPARECS, &pp->sparecs,
	    mc, MCAMD_PROP_BADCS, &pp->badcs,
	    NULL)) {
		mcamd_dprintf(hdl, MCAMD_DBG_ERR, "%s: failed to read mc "
		    "props for mc 0x%p\n", caller, mc);
		return (mcamd_set_errno(hdl, EMCAMD_TREEINVALID));
	}

	return (0);
}

static int
getcsprops(struct mcamd_hdl *hdl, mcamd_node_t *cs, const char *caller,
    struct csprops *csp)
{
	if (!mcamd_get_numprops(hdl,
	    cs, MCAMD_PROP_NUM, &csp->num,
	    cs, MCAMD_PROP_BASE_ADDR, &csp->base,
	    cs, MCAMD_PROP_MASK, &csp->mask,
	    cs, MCAMD_PROP_TESTFAIL, &csp->testfail,
	    cs, MCAMD_PROP_DIMMRANK, &csp->dimmrank,
	    NULL))  {
		mcamd_dprintf(hdl, MCAMD_DBG_ERR, "%s: failed to read cs "
		    "props for cs 0x%p\n", caller, cs);
		return (mcamd_set_errno(hdl, EMCAMD_TREEINVALID));
	}

	return (0);
}

static int
gettbls(struct mcamd_hdl *hdl, uint_t csmode, struct mcprops *mcpp,
    const struct rct_bnkaddrmode **bamp, const struct rct_rcbmap **rcbmp,
    const struct rct_bnkswzlinfo **swzlp, struct rct_csintlv *csid,
    const char *caller)
{
	uint_t rev = (uint_t)mcpp->rev;
	int width = (int)mcpp->width;

	if (bamp && (*bamp = rct_bnkaddrmode(rev, csmode)) == NULL) {
		mcamd_dprintf(hdl, MCAMD_DBG_FLOW, "%s: no bank address mode "
		    "table for MC rev %d csmode %d\n", caller, rev, csmode);
		return (mcamd_set_errno(hdl, EMCAMD_NOTSUP));
	}

	if (rcbmp && (*rcbmp = rct_rcbmap(rev, width, csmode)) == NULL) {
		mcamd_dprintf(hdl, MCAMD_DBG_FLOW, "%s: no dram address map "
		    "table for MC rev %d csmode %d\n", caller,
		    rev, csmode);
		return (mcamd_set_errno(hdl, EMCAMD_NOTSUP));
	}

	if (swzlp && (*swzlp = rct_bnkswzlinfo(rev, width)) == NULL) {
		mcamd_dprintf(hdl, MCAMD_DBG_FLOW, "%s: no bank swizzling "
		    "table for MC rev %d width %d\n", caller, rev, width);
		return (mcamd_set_errno(hdl, EMCAMD_NOTSUP));
	}

	if (csid) {
		if (mcpp->csintlvfctr > 1) {
			rct_csintlv_bits(rev, width, csmode,
			    mcpp->csintlvfctr, csid);
			if (csid->csi_factor == 0) {
				mcamd_dprintf(hdl, MCAMD_DBG_FLOW, "%s: "
				    "could not work out cs interleave "
				    "paramters for MC rev %d, width %d, "
				    "csmode %d, factor %d\n", caller,
				    rev, width, csmode,
				    (int)mcpp->csintlvfctr);
				return (mcamd_set_errno(hdl, EMCAMD_NOTSUP));
			}
		} else {
			csid->csi_factor = 0;
		}
	}

	return (0);
}

static uint64_t
iaddr_add(struct mcamd_hdl *hdl, uint64_t in, uint64_t add, const char *what)
{
	uint64_t new = in | add;

	mcamd_dprintf(hdl, MCAMD_DBG_FLOW, "%s: 0x%llx | 0x%llx --> 0x%llx",
	    what, in, add, new);

	return (add);
}

/*
 * Where the number of row/col address bits is ambiguous (affects CG and
 * earlier only) we will assign the "floating" bit to row address.  If
 * we adopt the same convention in address reconstruction then all should work.
 */
static uint32_t
iaddr_to_row(struct mcamd_hdl *hdl, const struct rct_bnkaddrmode *bamp,
    const struct rct_rcbmap *rcbm, struct rct_csintlv *csid, uint64_t iaddr)
{
	uint32_t addr = 0;
	int abitno, ibitno;
	int nbits = bamp->bam_nrows;
	int swapped = 0;

	for (abitno = 0; abitno < nbits; abitno++) {
		ibitno = rcbm->rcb_rowbit[abitno];
		if (MC_RC_CSI_SWAPPED_BIT(csid, ibitno)) {
			ibitno = MC_RC_CSI_BITSWAP(csid, ibitno);
			swapped++;
		}
		if (BITVAL(iaddr, ibitno) != 0)
			SETBIT(addr, abitno);
	}

	mcamd_dprintf(hdl, MCAMD_DBG_FLOW, "iaddr_to_row: iaddr 0x%llx --> "
	    "row 0x%x (%d bits swapped for cs intlv)\n", iaddr, addr, swapped);

	return (addr);
}

/*ARGSUSED*/
static uint64_t
row_to_iaddr(struct mcamd_hdl *hdl, const struct rct_bnkaddrmode *bamp,
    const struct rct_rcbmap *rcbm, struct rct_csintlv *csid, uint32_t rowaddr)
{
	uint64_t iaddr = 0;
	int abitno, ibitno;
	int nbits = bamp->bam_nrows;

	for (abitno = 0; abitno < nbits; abitno++) {
		if (BIT(rowaddr, abitno) == 0)
			continue;
		ibitno = rcbm->rcb_rowbit[abitno];
		if (MC_RC_CSI_SWAPPED_BIT(csid, ibitno)) {
			ibitno = MC_RC_CSI_BITSWAP(csid, ibitno);
		}
		SETBIT(iaddr, ibitno);
	}

	return (iaddr);
}


static uint32_t
iaddr_to_col(struct mcamd_hdl *hdl, const struct rct_bnkaddrmode *bamp,
    const struct rct_rcbmap *rcbm, uint64_t iaddr)
{
	uint32_t addr = 0;
	int abitno, ibitno, bias = 0;
	int nbits = bamp->bam_ncols;

	/*
	 * Knock off a column bit if the numbers are ambiguous
	 */
	if (bamp->bam_ambig)
		nbits--;

	for (abitno = 0; abitno < nbits; abitno++) {
		if (abitno == MC_PC_COLADDRBIT)
			bias = 1;

		ibitno = rcbm->rcb_colbit[abitno + bias];

		if (BITVAL(iaddr, ibitno) != 0)
			SETBIT(addr, abitno);
	}

	mcamd_dprintf(hdl, MCAMD_DBG_FLOW, "iaddr_to_col: iaddr 0x%llx --> "
	    "col 0x%x\n", iaddr, addr);

	return (addr);
}

/*ARGSUSED*/
static uint64_t
col_to_iaddr(struct mcamd_hdl *hdl, const struct rct_bnkaddrmode *bamp,
    const struct rct_rcbmap *rcbm, uint32_t coladdr)
{
	uint64_t iaddr = 0;
	int abitno, ibitno, bias = 0;
	int nbits = bamp->bam_ncols;

	/*
	 * Knock off a column bit if the numbers are ambiguous
	 */
	if (bamp->bam_ambig)
		nbits--;

	for (abitno = 0; abitno < nbits; abitno++) {
		if (BIT(coladdr, abitno) == 0)
			continue;

		if (abitno == MC_PC_COLADDRBIT)
			bias = 1;

		ibitno = rcbm->rcb_colbit[abitno + bias];
		SETBIT(iaddr, ibitno);
	}

	return (iaddr);
}

/*
 * Extract bank bit arguments and swizzle if requested.
 */
static uint32_t
iaddr_to_bank(struct mcamd_hdl *hdl, const struct rct_rcbmap *rcbm,
    const struct rct_bnkswzlinfo *swzlp, uint64_t iaddr)
{
	uint32_t addr = 0;
	int abitno, ibitno, i;

	for (abitno = 0; abitno < rcbm->rcb_nbankbits; abitno++) {
		uint32_t val;

		/*
		 * rcb_bankbit[abitno] tells us which iaddr bit number
		 * will form bit abitno of the bank address
		 */
		ibitno = rcbm->rcb_bankbit[abitno];
		val = BITVAL(iaddr, ibitno);

		/*
		 * If bank swizzling is in operation then xor the bit value
		 * obtained above with other iaddr bits.
		 */
		if (swzlp) {
			for (i = 0; i < MC_RC_SWZLBITS; i++) {
				ibitno = swzlp->bswz_rowbits[abitno][i];
				val ^= BITVAL(iaddr, ibitno);
			}
		}

		if (val)
			SETBIT(addr, abitno);
	}

	mcamd_dprintf(hdl, MCAMD_DBG_FLOW, "iaddr_to_bank: iaddr 0x%llx --> "
	    "bank 0x%x\n", iaddr, addr);

	return (addr);
}

/*
 * bank_to_iaddr requires the iaddr reconstructed thus far with at least the
 * row bits repopulated.  That's because in bank swizzle mode
 * the bank bits are the result of xor'ing three original iaddr bits
 * together - two of which come from the row address and the third we
 * can reconstruct here.  Note that a zero bankaddr bit *can* result
 * in a nonzero iaddr bit (unlike in row and col reconstruction).
 */
/*ARGSUSED*/
static uint64_t
bank_to_iaddr(struct mcamd_hdl *hdl, const struct rct_rcbmap *rcbm,
    const struct rct_bnkswzlinfo *swzlp, uint64_t partiaddr, uint32_t bankaddr)
{
	uint64_t iaddr = 0;
	int abitno, pibitno, i;

	for (abitno = 0; abitno < rcbm->rcb_nbankbits; abitno++) {
		uint32_t val = BITVAL(bankaddr, abitno);
		if (swzlp) {
			for (i = 0; i < MC_RC_SWZLBITS; i++) {
				pibitno = swzlp->bswz_rowbits[abitno][i];
				val ^= BITVAL(partiaddr, pibitno);
			}
		}
		if (val)
			SETBIT(iaddr, rcbm->rcb_bankbit[abitno]);
	}

	return (iaddr);
}

static int
iaddr_to_rcb(struct mcamd_hdl *hdl, uint_t csmode, struct mcprops *mcpp,
    uint64_t iaddr, uint32_t *rowp, uint32_t *colp, uint32_t *bankp)
{
	const struct rct_bnkaddrmode *bamp;
	const struct rct_rcbmap *rcbmp;
	const struct rct_bnkswzlinfo *swzlp = NULL;
	struct rct_csintlv csi;

	if (gettbls(hdl, csmode, mcpp, &bamp, &rcbmp,
	    mcpp->bnkswzl ? &swzlp : NULL, &csi,
	    "iaddr_to_rcb") < 0)
		return (-1);	/* errno already set */

	*rowp = iaddr_to_row(hdl, bamp, rcbmp, &csi, iaddr);
	*colp = iaddr_to_col(hdl, bamp, rcbmp, iaddr);
	*bankp = iaddr_to_bank(hdl, rcbmp, swzlp, iaddr);

	return (0);
}

/*
 * Take a reconstructed InputAddr and undo the normalization described in
 * BKDG 3.29 3.4.4 to include the base address of the MC if no node
 * interleave or to insert the node interleave selection bits.
 */
static int
iaddr_unnormalize(struct mcamd_hdl *hdl, struct mcprops *mcpp, uint64_t iaddr,
    uint64_t *rsltp)
{
	uint64_t dramaddr;
	int intlvbits;

	switch (mcpp->intlven) {
	case 0x0:
		intlvbits = 0;
		break;
	case 0x1:
		intlvbits = 1;
		break;
	case 0x3:
		intlvbits = 2;
		break;
	case 0x7:
		intlvbits = 3;
		break;
	default:
		mcamd_dprintf(hdl, MCAMD_DBG_ERR, "iaddr_unnormalize: "
		    "illegal IntlvEn of %d for MC 0x%p\n",
		    (int)mcpp->intlven, (int)mcpp->num);
		return (mcamd_set_errno(hdl, EMCAMD_TREEINVALID));
	}

	if (intlvbits != 0) {
		/*
		 * For a 2/4/8 way interleave iaddr was formed by excising
		 * 1, 2, or 3 bits 12:12, 13:12, or 14:12 from dramaddr,
		 * the removed bits having done their job by selecting the
		 * responding node.  So we must move bits 35:12 of the
		 * reconstructed iaddr up to make a 1, 2 or 3 bit hole and
		 * then fill those bits with the current IntlvSel value for
		 * this node.  The node base address must be zero if nodes
		 * are interleaved.
		 *
		 * Note that the DRAM controller InputAddr is still 36 bits
		 * 35:0 on rev F.
		 */
		dramaddr = (BITS(iaddr, 35, 12) << intlvbits) |
		    (mcpp->intlvsel << 12) | BITS(iaddr, 11, 0);
	} else {
		dramaddr = iaddr + mcpp->base;
	}

	*rsltp = dramaddr;

	mcamd_dprintf(hdl, MCAMD_DBG_FLOW, "iaddr_unnormalize: iaddr 0x%llx "
	    "intlven 0x%x intlvsel 0x%x MC base 0x%llx --> 0x%llx\n",
	    iaddr, (int)mcpp->intlven, (int)mcpp->intlvsel, (int)mcpp->base,
	    dramaddr);

	return (0);
}

int
mc_pa_to_offset(struct mcamd_hdl *hdl, mcamd_node_t *mc, mcamd_node_t *cs,
    uint64_t iaddr, uint64_t *offsetp)
{
	mcamd_dimm_offset_un_t offset_un;
	uint_t csmode;
	uint32_t bankaddr, rowaddr, coladdr;
	struct mcprops mcp;
	struct csprops csp;

	*offsetp = MCAMD_RC_INVALID_OFFSET;

	if (getmcprops(hdl, mc, "mc_dimm_offset", &mcp) < 0 ||
	    getcsprops(hdl, cs, "mc_dimm_offset", &csp) < 0)
		return (-1);	/* errno already set */

	csmode = MC_CS_MODE(mcp.csbnkmap_reg, csp.num);

	if (iaddr_to_rcb(hdl, csmode, &mcp, iaddr, &rowaddr,
	    &coladdr, &bankaddr) < 0)
		return (-1);	/* errno already set */

	offset_un.do_offset = 0;

	offset_un.do_valid = 1;
	offset_un.do_version = MCAMD_OFFSET_VERSION;
	offset_un.do_rank = (uint32_t)csp.dimmrank;
	offset_un.do_row = rowaddr;
	offset_un.do_bank = bankaddr;
	offset_un.do_col = coladdr;

	*offsetp = offset_un.do_offset;

	return (0);
}

/*
 * Given an MC, DIMM and offset (dimm rank, row, col, internal bank) we
 * find the corresponding chip-select for the rank and then reconstruct
 * a system address.  In the absence of serial number support it is possible
 * that we may be asked to perform this operation on a dimm which has been
 * swapped, perhaps even for a dimm of different size and number of ranks.
 * This may happen if fmadm repair has not been used.  There are some
 * unused bits in the offset and we could guard against this a little
 * by recording in those bit some of the physical characteristic of the
 * original DIMM such as size, number of ranks etc.
 */
int
mc_offset_to_pa(struct mcamd_hdl *hdl, mcamd_node_t *mc, mcamd_node_t *dimm,
    uint64_t offset, uint64_t *pap)
{
	mcamd_node_t *cs;
	mcamd_dimm_offset_un_t off_un;
	uint32_t rank, rowaddr, bankaddr, coladdr;
	uint64_t iaddr = 0;
	const struct rct_bnkaddrmode *bamp;
	const struct rct_rcbmap *rcbmp;
	const struct rct_bnkswzlinfo *swzlp = NULL;
	struct rct_csintlv csi;
	struct mcprops mcp;
	struct csprops csp;
	uint64_t csmode;
	int maskhi_hi, maskhi_lo, masklo_hi, masklo_lo;

	off_un.do_offset = offset;
	rank = off_un.do_rank;
	bankaddr = off_un.do_bank;
	rowaddr = off_un.do_row;
	coladdr = off_un.do_col;

	mcamd_dprintf(hdl, MCAMD_DBG_FLOW, "mc_offset_to_pa: offset 0x%llx "
	    "-> rank %d bank %d row 0x%x col 0x%x\n", offset,
	    rank, bankaddr, rowaddr, coladdr);

	if (getmcprops(hdl, mc, "mc_offset_to_pa", &mcp) < 0)
		return (-1);	/* errno already set */

	maskhi_hi = MC_CSMASKHI_HIBIT(mcp.rev);
	maskhi_lo = MC_CSMASKHI_LOBIT(mcp.rev);
	masklo_hi = MC_CSMASKLO_HIBIT(mcp.rev);
	masklo_lo = MC_CSMASKLO_LOBIT(mcp.rev);

	/*
	 * Find the chip-select on this dimm using the given rank.
	 */
	for (cs = mcamd_cs_next(hdl, dimm, NULL); cs != NULL;
	    cs = mcamd_cs_next(hdl, dimm, cs)) {
		if (getcsprops(hdl, cs, "mc_offset_to_pa", &csp) < 0)
			return (-1);	/* errno already set */

		if (csp.dimmrank == rank)
			break;
	}

	if (cs == NULL) {
		mcamd_dprintf(hdl, MCAMD_DBG_FLOW, "mc_offset_to_pa: Current "
		    "dimm in this slot does not have a cs using rank %d\n",
		    rank);
		return (mcamd_set_errno(hdl, EMCAMD_NOADDR));
	}

	/*
	 * If the cs# has been substituted by the online spare then the
	 * given unum is not actually contributing to the system address
	 * map since all accesses to it are redirected.
	 *
	 * If the cs# failed BIOS test it is not in the address map.
	 *
	 * If the cs# is the online spare cs# then it is contributing to
	 * the system address map only if swapped in, and the csbase etc
	 * parameters to use must be those of the bad cs#.
	 */
	if (mcp.badcs != MC_INVALNUM && csp.num == mcp.badcs) {
		return (mcamd_set_errno(hdl, EMCAMD_NOADDR));
	} else if (csp.testfail) {
		return (mcamd_set_errno(hdl, EMCAMD_NOADDR));
	} else if (mcp.sparecs != MC_INVALNUM && csp.num == mcp.sparecs &&
	    mcp.badcs != MC_INVALNUM) {
		/*
		 * Iterate over all cs# of this memory controller to find
		 * the bad one - the bad cs# need not be on the same dimm
		 * as the spare.
		 */
		for (cs = mcamd_cs_next(hdl, mc, NULL); cs != NULL;
		    cs = mcamd_cs_next(hdl, mc, cs)) {
			mcamd_prop_t csnum;

			if (!mcamd_get_numprop(hdl, cs, MCAMD_PROP_NUM,
			    &csnum)) {
				mcamd_dprintf(hdl, MCAMD_DBG_ERR,
				    "mcamd_offset_to_pa: csnum lookup failed "
				    "while looking for bad cs#");
				return (mcamd_set_errno(hdl,
				    EMCAMD_TREEINVALID));
			}
			if (csnum == mcp.badcs)
				break;
		}

		if (cs == NULL) {
			mcamd_dprintf(hdl, MCAMD_DBG_ERR, "mcamd_offset_to_pa: "
			    "failed to find cs for bad cs#%d\n", mcp.badcs);
			return (mcamd_set_errno(hdl, EMCAMD_TREEINVALID));
		}

		/* found bad cs - reread properties from it instead of spare */
		if (getcsprops(hdl, cs, "mc_offset_to_pa", &csp) < 0)
			return (-1);	/* errno already set */
	}

	csmode = MC_CS_MODE(mcp.csbnkmap_reg, csp.num);

	if (gettbls(hdl, csmode, &mcp, &bamp, &rcbmp,
	    mcp.bnkswzl ? &swzlp : NULL, &csi,
	    "mc_offset_to_pa") < 0)
		return (-1);	/* errno already set */

	/*
	 * If there are umaskable DRAM InputAddr bits the add those bits
	 * to iaddr from the cs base address.
	 */
	if (MC_CSMASK_UNMASKABLE(mcp.rev) != 0) {
		iaddr |= iaddr_add(hdl, iaddr,
		    BITS(csp.base, maskhi_hi + MC_CSMASK_UNMASKABLE(mcp.rev),
		    maskhi_hi + 1), "unmaskable cs basehi bits");
	}

	/*
	 * basehi bits not meing masked pass straight through to the
	 * iaddr.
	 */
	iaddr |= iaddr_add(hdl, iaddr,
	    BITS(csp.base, maskhi_hi, maskhi_lo) &
	    ~BITS(csp.mask, maskhi_hi, maskhi_lo),
	    "cs basehi bits not being masked");

	/*
	 * if cs interleaving is active then baselo address bit are being
	 * masked - pass the rest through.
	 */
	if (mcp.csintlvfctr > 1) {
		iaddr |= iaddr_add(hdl, iaddr,
		    BITS(csp.base, masklo_hi, masklo_lo) &
		    ~BITS(csp.mask, masklo_hi, masklo_lo),
		    "cs baselo bits not being masked");
	}

	/*
	 * Reconstruct iaddr bits from known row address
	 */
	iaddr |= iaddr_add(hdl, iaddr,
	    row_to_iaddr(hdl, bamp, rcbmp, &csi, rowaddr),
	    "add iaddr bits from row");

	/*
	 * Reconstruct iaddr bits from known column address
	 */
	iaddr |= iaddr_add(hdl, iaddr,
	    col_to_iaddr(hdl, bamp, rcbmp, coladdr),
	    "add iaddr bits from col");

	/*
	 * Reconstruct iaddr bits from known internal banksel address
	 */
	iaddr |= iaddr_add(hdl, iaddr,
	    bank_to_iaddr(hdl, rcbmp, swzlp, iaddr, bankaddr),
	    "add iaddr bits from bank");

	/*
	 * Move iaddr up into the range for this MC and insert any
	 * node interleave selection bits.
	 */
	if (iaddr_unnormalize(hdl, &mcp, iaddr, pap) < 0)
		return (-1);	/* errno already set */

	return (0);
}

int
mcamd_cs_size(struct mcamd_hdl *hdl, mcamd_node_t *mc, int csnum, size_t *szp)
{
	uint_t csmode;
	struct mcprops mcp;
	const struct rct_bnkaddrmode *bamp;

	if (getmcprops(hdl, mc, "mcamd_cs_size", &mcp) < 0)
		return (-1);	/* errno already set */

	csmode = MC_CS_MODE(mcp.csbnkmap_reg, csnum);

	if (gettbls(hdl, csmode, &mcp, &bamp, NULL, NULL, NULL,
	    "mcamd_cs_size") < 0)
		return (-1);	/* errno already set */

	*szp = MC_CS_SIZE(bamp, mcp.width);

	return (0);
}
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 *
 * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#ifndef _MCAMD_ROWCOL_IMPL_H
#define	_MCAMD_ROWCOL_IMPL_H

#include <mcamd_api.h>
#include <sys/mc_amd.h>

#ifdef __cplusplus
extern "C" {
#endif

#define	MC_PC_COLADDRBIT	10	/* col address bit used for precharge */
#define	MC_PC_ALL		-1	/* marker used in tables */

#define	MC_CS_SCALE		(1024 * 1024)
#define	MC_CS_SIZE(bam, width) \
	((size_t)bam->bam_sizemb * MC_CS_SCALE * ((width) == 128 ? 2 : 1))

#define	MC_CS_MODE(csbmap, csnum) \
	(csbmap >> MC_CHIP_DIMMPAIR(csnum) * MC_DC_BAM_CSBANK_SHIFT & \
	MC_DC_BAM_CSBANK_MASK)

#define	BIT(val, num) ((val) & 1ULL << num)

#define	BITS(val, high, low) \
	((val) & (((2ULL << (high)) - 1) & ~((1ULL << (low)) - 1)))

#define	SETBIT(var, num) (var |= (1ULL << (num)))

#define	BITVAL(var, num) ((BIT(var, num) >> (num)) & 1ULL)

#define	MC_RC_ROW_MAX		16	/* maximum number of row address bits */
#define	MC_RC_COL_MAX		12	/* maximum number of col address bits */
#define	MC_RC_BANKBITS_MAX	3	/* number of internal banksel bits */
#define	MC_RC_CSMODES		16	/* max number of cs bankaddr modes */
#define	MC_RC_SWZLBITS		2	/* number of row bits in swizzle */

struct rct_bnkaddrmode {
	int bam_sizemb;			/* DIMM size in MB */
	int bam_nrows;			/* number of row address bits */
	int bam_ncols;			/* number of column address bits */
	int bam_ambig;			/* numbers are maximums; keep last */
};

struct rct_rcbmap {
	int rcb_nbankbits;			/* # of bank address bits */
	int rcb_bankbit[MC_RC_BANKBITS_MAX];	/* bank address bits */
	int rcb_rowbit[MC_RC_ROW_MAX];
	int rcb_colbit[MC_RC_COL_MAX + 1];	/* one for MC_PC_ALL */
};

struct rct_bnkswzlinfo {
	int bswz_rowbits[MC_RC_BANKBITS_MAX][MC_RC_SWZLBITS];
};

struct rct_csintlv {
	int csi_factor;			/* cs interleave factor */
	int csi_hibit;			/* first non-offset bit in addr */
	int csi_lobit;			/* first row bit in addr */
	int csi_nbits;			/* number of bits to swap in mask */
};

#define	MC_RC_CSI_SWAPPED_BIT(csidp, n)				\
	(csidp->csi_factor && n >= csidp->csi_lobit &&		\
	n <= csidp->csi_lobit + csidp->csi_nbits - 1)

#define	MC_RC_CSI_BITSWAP(csidp, n)				\
	(csidp->csi_hibit + n - csidp->csi_lobit)

extern const struct rct_bnkaddrmode *rct_bnkaddrmode(uint_t, uint_t);
extern const struct rct_rcbmap *rct_rcbmap(uint_t, int, uint_t);
extern const struct rct_bnkswzlinfo *rct_bnkswzlinfo(uint_t, int);
extern void rct_csintlv_bits(uint_t, int, uint_t, int, struct rct_csintlv *);

#ifdef __cplusplus
}
#endif

#endif /* _MCAMD_ROWCOL_IMPL_H */
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 *
 * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#include <mcamd_api.h>
#include <mcamd_err.h>
#include <mcamd_rowcol_impl.h>

/*
 * =========== Chip-Select Bank Address Mode Encodings =======================
 */

/* Individual table declarations */
static const struct rct_bnkaddrmode bnkaddr_tbls_pre_d[];
static const struct rct_bnkaddrmode bnkaddr_tbls_d_e[];
static const struct rct_bnkaddrmode bnkaddr_tbls_f[];

/* Managing bank address mode tables */
static const struct _bnkaddrmode_tbldesc {
	uint_t	revmask;
	int	nmodes;
	const struct rct_bnkaddrmode *modetbl;
} bnkaddr_tbls[] = {
	{ MC_F_REVS_BC, 7, bnkaddr_tbls_pre_d },
	{ MC_F_REVS_DE, 11, bnkaddr_tbls_d_e },
	{ MC_F_REVS_FG, 12, bnkaddr_tbls_f },
};

/*
 * =========== DRAM Address Mappings for bank/row/column =====================
 */


/* Individual table declarations */
struct _rcbmap_tbl {
	uint_t mt_revmask;		/* revision to which this applies */
	int mt_width;			/* MC mode (64 or 128) */
	const struct rct_rcbmap mt_csmap[MC_RC_CSMODES];
};

static const struct _rcbmap_tbl dram_addrmap_pre_d_64;
static const struct _rcbmap_tbl dram_addrmap_pre_d_128;
static const struct _rcbmap_tbl dram_addrmap_d_e_64;
static const struct _rcbmap_tbl dram_addrmap_d_e_128;
static const struct _rcbmap_tbl dram_addrmap_f_64;
static const struct _rcbmap_tbl dram_addrmap_f_128;

/* Managing row/column/bank tables */
static const struct _rcbmap_tbldesc {
	int nmodes;
	const struct _rcbmap_tbl *rcbmap;
} rcbmap_tbls[] = {
	{ 7, &dram_addrmap_pre_d_64 },
	{ 7, &dram_addrmap_pre_d_128 },
	{ 11, &dram_addrmap_d_e_64 },
	{ 11, &dram_addrmap_d_e_128 },
	{ 12, &dram_addrmap_f_64 },
	{ 12, &dram_addrmap_f_128 },
};

/*
 * =========== Bank swizzling information ====================================
 */

/* Individual table declarations */
struct _bnkswzl_tbl {
	uint_t swzt_revmask;		/* revision to which this applies */
	int swzt_width;			/* MC mode (64 or 128) */
	const struct rct_bnkswzlinfo swzt_bits;
};

static const struct _bnkswzl_tbl bnswzl_info_e_64;
static const struct _bnkswzl_tbl bnswzl_info_e_128;
static const struct _bnkswzl_tbl bnswzl_info_f_64;
static const struct _bnkswzl_tbl bnswzl_info_f_128;

/* Managing bank swizzle tables */
static const struct _bnkswzl_tbl *bnkswzl_tbls[] = {
	&bnswzl_info_e_64,
	&bnswzl_info_e_128,
	&bnswzl_info_f_64,
	&bnswzl_info_f_128,
};

/*
 * ======================================================================
 * | Tables reflecting those in the BKDG				|
 * ======================================================================
 */

/*
 * DRAM Address Mapping in Interleaving Mode
 *
 * Chip-select interleave is performed by addressing across the columns
 * of the first row of internal bank-select 0 on a chip-select, then the
 * next row on internal bank-select 1, then 2 then 3;  instead of then
 * moving on to the next row of this chip-select we then rotate across
 * other chip-selects in the interleave.  The row/column/bank mappings
 * described elsewhere in this file show that a DRAM InputAddr breaks down
 * as follows, using an example for CS Mode 0000 revision CG and earlier 64-bit
 * mode; the cs size is 32MB, requiring 25 bits to address all of it.
 *
 * chip-selection bits |    offset within chip-select bits      |
 *		       | row bits | bank bits | column bits | - |
 *                      24      13 12       11 10          3 2 0
 *
 * The high-order chip-selection bits select the chip-select and the
 * offset bits offset within the chosen chip-select.
 *
 * To establish say a 2-way interleave in which we consume all of one
 * row number and all internal bank numbers on one cs before moving on
 * to the next to do the same we will target the first row bit - bit 13;
 * a 4-way interleave would use bits 14 and 13, and an 8-way interleave
 * bits 15, 14 and 13.  We swap the chosen bits with the least significant
 * high order chip-selection bits.
 *
 * The BKDG interleave tables really just describe the above.  Working
 * out the high-order bits to swap is easy since that is derived directly
 * from the chip-select size.  The low-order bits depend on the device
 * parameters since we need to target the least significant row address bits -
 * but we have that information from the rcbmap_tbls since the first row bit
 * simply follows the last bank address bit.
 */

/*
 * General notes for CS Bank Address Mode Encoding tables.
 *
 * These are indexed by chip-select mode.  Where the numbers of rows and
 * columns is ambiguous (as it is for a number of rev CG and earlier cases)
 * the bam_config should be initialized to 1 and the numbers of rows
 * and columns should be the maximums.
 */

/*
 * Chip Select Bank Address Mode Encoding for rev CG and earlier.
 */
static const struct rct_bnkaddrmode bnkaddr_tbls_pre_d[] = {
	{	/* 000 */
		32, 12, 8
	},
	{	/* 001 */
		64, 12, 9
	},
	{	/* 010 */
		128, 13, 10, 1	/* AMBIG */
	},
	{	/* 011 */
		256, 13, 11, 1	/* AMBIG */
	},
	{	/* 100 */
		512, 14, 11, 1	/* AMBIG */
	},
	{	/* 101 */
		1024, 14, 12, 1	/* AMBIG */
	},
	{	/* 110 */
		2048, 14, 12
	}
};

/*
 * Chip Select Bank Address Mode Encoding for revs D and E.
 */
static const struct rct_bnkaddrmode bnkaddr_tbls_d_e[] = {
	{	/* 0000 */
		32, 12, 8
	},
	{	/* 0001 */
		64, 12, 9
	},
	{	/* 0010 */
		128, 13, 9
	},
	{	/* 0011 */
		128, 12, 10
	},
	{	/* 0100 */
		256, 13, 10
	},
	{	/* 0101 */
		512, 14, 10
	},
	{	/* 0110 */
		256, 12, 11
	},
	{	/* 0111 */
		512, 13, 11
	},
	{	/* 1000 */
		1024, 14, 11
	},
	{	/* 1001 */
		1024, 13, 12
	},
	{	/* 1010 */
		2048, 14, 12
	}
};

/*
 * Chip Select Bank Address Mode Encoding for rev F
 */
static const struct rct_bnkaddrmode bnkaddr_tbls_f[] = {
	{	/* 0000 */
		128, 13, 9
	},
	{	/* 0001 */
		256, 13, 10
	},
	{	/* 0010 */
		512, 14, 10
	},
	{	/* 0011 */
		512, 13, 11
	},
	{	/* 0100 */
		512, 13, 10
	},
	{	/* 0101 */
		1024, 14, 10
	},
	{	/* 0110 */
		1024, 14, 11
	},
	{	/* 0111 */
		2048, 15, 10
	},
	{	/* 1000 */
		2048, 14, 11
	},
	{	/* 1001 */
		4096, 15, 11
	},
	{	/* 1010 */
		4096, 16, 10
	},
	{	/* 1011 */
		8192, 16, 11
	}

};

/*
 * General notes on Row/Column/Bank table initialisation.
 *
 * These are the tables 7, 8, 9, 10, 11 and 12 of BKDG 3.29 section 3.5.6.1.
 * They apply in non-interleave (node or cs) mode and describe how for
 * a given revision, access width, bank-swizzle mode, and current chip-select
 * mode the row, column and internal sdram bank are derived from the
 * normalizied InputAddr presented to the DRAM controller.
 *
 * The mt_csmap array is indexed by chip-select mode.  Within it the
 * bankargs, rowbits and colbits arrays are indexed by bit number, so
 * match the BKDG tables if the latter are read right-to-left.
 *
 * The bankargs list up to three bit numbers per bank bit.  For revisions
 * CG and earlier there is no bank swizzling, so just a single number
 * should be listed.  Revisions D and E have the same row/column/bank mapping,
 * but rev E has the additional feature of being able to xor two row bits
 * into each bank bit.  The consumer will know whether they are using bank
 * swizzling - if so then they should xor the bankargs bits together.
 * The first argument must be the bit number not already used in forming
 * part of the row address - eg in table 12 for csmode 0000b bank address
 * bit 0 is bit 12 xor bit 18 xor bit 21, and 18 and 21 are also mentioned in
 * the row address (bits 10 and 1) so we must list bit 12 first.  We will
 * use this information in chip-select interleave decoding in which we need
 * to know which is the first bit after column and bank address bits.
 *
 * Column address A10 is always used for the Precharge All signal.  Where
 * "PC" appears in the BKDG tables we will include MC_PC_ALL in the
 * corresponding bit position.
 *
 * For some rev CG and earlier chipselect modes the number of rows and columns
 * is ambiguous.  This is reflected in these tables by some bit being
 * duplicated between row and column address.  In practice we will follow
 * the convention of always assigning the floating bit to the row address.
 */

/*
 * Row/Column/Bank address mappings for rev CG in 64-bit mode, no interleave.
 * See BKDG 3.29 3.5.6 Table 7.
 */
static const struct _rcbmap_tbl dram_addrmap_pre_d_64 = {
	MC_F_REVS_BC,
	64,
	{
	{   /* 000 */
	    2, { 11, 12 },
	    { 19, 20, 21, 22, 23, 24, 13, 14, 15, 16, 17, 18 },
	    { 3, 4, 5, 6, 7, 8, 9, 10 }
	},
	{   /* 001 */
	    2, { 13, 12 },
	    { 19, 20, 21, 22, 23, 24, 25, 14, 15, 16, 17, 18 },
	    { 3, 4, 5, 6, 7, 8, 9, 10, 11 }
	},
	{   /* 010 */
	    2, { 13, 12 },
	    { 19, 20, 21, 22, 23, 24, 25, 14, 15, 16, 17, 18, 26 },
	    { 3, 4, 5, 6, 7, 8, 9, 10, 11, 26 }
	},
	{   /* 011 */
	    2, { 13, 14 },
	    { 19, 20, 21, 22, 23, 24, 25, 26, 15, 16, 17, 18, 27 },
	    { 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, MC_PC_ALL, 27 }
	},
	{   /* 100 */
	    2, { 13, 14 },
	    { 19, 20, 21, 22, 23, 24, 25, 26, 15, 16, 17, 18, 27, 28 },
	    { 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, MC_PC_ALL, 28 }
	},
	{   /* 101 */
	    2, { 15, 14 },
	    { 19, 20, 21, 22, 23, 24, 25, 26, 29, 16, 17, 18, 27, 28 },
	    { 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, MC_PC_ALL, 13, 28 }
	},
	{   /* 110 */
	    2, { 15, 14 },
	    { 19, 20, 21, 22, 23, 24, 25, 26, 29, 16, 17, 18, 27, 28 },
	    { 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, MC_PC_ALL, 13, 30 }
	},
	/*
	 * remainder unused
	 */
	}

};

/*
 * Row/Column/Bank address mappings for rev CG in 128-bit mode, no interleave.
 * See BKDG 3.29 3.5.6 Table 8.
 */
static const struct _rcbmap_tbl dram_addrmap_pre_d_128 = {
	MC_F_REVS_BC,
	128,
	{
	{   /* 000 */
	    2, { 12, 13 },
	    { 20, 21, 22, 23, 24, 25, 14, 15, 16, 17, 18, 19 },
	    { 4, 5, 6, 7, 8, 9, 10, 11 }
	},
	{   /* 001 */
	    2, { 14, 13 },
	    { 20, 21, 22, 23, 24, 25, 26, 15, 16, 17, 18, 19 },
	    { 4, 5, 6, 7, 8, 9, 10, 11, 12 }
	},
	{   /* 010 */
	    2, { 14, 13 },
	    { 20, 21, 22, 23, 24, 25, 26, 15, 16, 17, 18, 19, 27 },
	    { 4, 5, 6, 7, 8, 9, 10, 11, 12, 27 }
	},
	{   /* 011 */
	    2, { 14, 15 },
	    { 20, 21, 22, 23, 24, 25, 26, 27, 16, 17, 18, 19, 28 },
	    { 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, MC_PC_ALL, 28 }
	},
	{   /* 100 */
	    2, { 14, 15 },
	    { 20, 21, 22, 23, 24, 25, 26, 27, 16, 17, 18, 19, 28, 29 },
	    { 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, MC_PC_ALL, 29 }
	},
	{   /* 101 */
	    2, { 16, 15 },
	    { 20, 21, 22, 23, 24, 25, 26, 27, 30, 17, 18, 19, 28, 29 },
	    { 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, MC_PC_ALL, 14, 29 }
	},
	{   /* 110 */
	    2, { 16, 15 },
	    { 20, 21, 22, 23, 24, 25, 26, 27, 30, 17, 18, 19, 28, 29 },
	    { 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, MC_PC_ALL, 14, 31 }
	},
	/*
	 * remainder unused
	 */
	}
};

/*
 * Row/Column/Bank address mappings for rev D/E in 64-bit mode, no interleave.
 * See BKDG 3.29 3.5.6 Table 9.
 */
static const struct _rcbmap_tbl dram_addrmap_d_e_64 = {
	MC_F_REVS_DE,
	64,
	{
	{   /* 0000 */
	    2, { 11, 12 },
	    { 19, 20, 21, 22, 23, 24, 13, 14, 15, 16, 17, 18 },
	    { 3, 4, 5, 6, 7, 8, 9, 10 }
	},
	{   /* 0001 */
	    2, { 12, 13 },
	    { 19, 20, 21, 22, 23, 24, 25, 14, 15, 16, 17, 18, 26 },
	    { 3, 4, 5, 6, 7, 8, 9, 10, 11 }
	},
	{   /* 0010 */
	    2, { 12, 13 },
	    { 19, 20, 21, 22, 23, 24, 25, 14, 15, 16, 17, 18, 26 },
	    { 3, 4, 5, 6, 7, 8, 9, 10, 11 }
	},
	{   /* 0011 */
	    2, { 13, 14 },
	    { 19, 20, 21, 22, 23, 24, 25, 26, 15, 16, 17, 18, 27, 28 },
	    { 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 }
	},
	{   /* 0100 */
	    2, { 13, 14 },
	    { 19, 20, 21, 22, 23, 24, 25, 26, 15, 16, 17, 18, 27, 28 },
	    { 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 }
	},
	{   /* 0101 */
	    2, { 13, 14 },
	    { 19, 20, 21, 22, 23, 24, 25, 26, 15, 16, 17, 18, 27, 28 },
	    { 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 }
	},
	{   /* 0110 */
	    2, { 14, 15 },
	    { 19, 20, 21, 22, 23, 24, 25, 26, 27, 16, 17, 18, 28, 29 },
	    { 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, MC_PC_ALL, 13 }
	},
	{   /* 0111 */
	    2, { 14, 15 },
	    { 19, 20, 21, 22, 23, 24, 25, 26, 27, 16, 17, 18, 28, 29 },
	    { 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, MC_PC_ALL, 13 }
	},
	{   /* 1000 */
	    2, { 14, 15 },
	    { 19, 20, 21, 22, 23, 24, 25, 26, 27, 16, 17, 18, 28, 29 },
	    { 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, MC_PC_ALL, 13 }
	},
	{   /* 1001 */
	    2, { 15, 16 },
	    { 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 17, 18, 29, 30 },
	    { 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, MC_PC_ALL, 13, 14 }
	},
	{   /* 1010 */
	    2, { 15, 16 },
	    { 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 17, 18, 29, 30 },
	    { 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, MC_PC_ALL, 13, 14 }
	},
	/*
	 * remainder unused
	 */
	}
};

/*
 * Row/Column/Bank address mappings for rev D/E in 128-bit mode, no interleave.
 * See BKDG 3.29 3.5.6 Table 9.
 */
static const struct _rcbmap_tbl dram_addrmap_d_e_128 = {
	MC_F_REVS_DE,
	128,
	{
	{   /* 0000 */
	    2, { 12, 13 },
	    { 20, 21, 22, 23, 24, 25, 14, 15, 16, 17, 18, 19 },
	    { 4, 5, 6, 7, 8, 9, 10, 11 }
	},
	{   /* 0001 */
	    2, { 13, 14 },
	    { 20, 21, 22, 23, 24, 25, 26, 15, 16, 17, 18, 19, 27 },
	    { 4, 5, 6, 7, 8, 9, 10, 11, 12 }
	},
	{   /* 0010 */
	    2, { 13, 14 },
	    { 20, 21, 22, 23, 24, 25, 26, 15, 16, 17, 18, 19, 27 },
	    { 4, 5, 6, 7, 8, 9, 10, 11, 12 }
	},
	{   /* 0011 */
	    2, { 14, 15 },
	    { 20, 21, 22, 23, 24, 25, 26, 27, 16, 17, 18, 19, 28, 29 },
	    { 4, 5, 6, 7, 8, 9, 10, 11, 12, 13 }
	},
	{   /* 0100 */
	    2, { 14, 15 },
	    { 20, 21, 22, 23, 24, 25, 26, 27, 16, 17, 18, 19, 28, 29 },
	    { 4, 5, 6, 7, 8, 9, 10, 11, 12, 13 }
	},
	{   /* 0101 */
	    2, { 14, 15 },
	    { 20, 21, 22, 23, 24, 25, 26, 27, 16, 17, 18, 19, 28, 29 },
	    { 4, 5, 6, 7, 8, 9, 10, 11, 12, 13 }
	},
	{   /* 0110 */
	    2, { 15, 16 },
	    { 20, 21, 22, 23, 24, 25, 26, 27, 28, 17, 18, 19, 29, 30 },
	    { 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, MC_PC_ALL, 14 }
	},
	{   /* 0111 */
	    2, { 15, 16 },
	    { 20, 21, 22, 23, 24, 25, 26, 27, 28, 17, 18, 19, 29, 30 },
	    { 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, MC_PC_ALL, 14 }
	},
	{   /* 1000 */
	    2, { 15, 16 },
	    { 20, 21, 22, 23, 24, 25, 26, 27, 28, 17, 18, 19, 29, 30 },
	    { 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, MC_PC_ALL, 14 }
	},
	{   /* 1001 */
	    2, { 16, 17 },
	    { 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 18, 19, 30, 31 },
	    { 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, MC_PC_ALL, 14, 15 }
	},
	{   /* 1010 */
	    2, { 16, 17 },
	    { 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 18, 19, 30, 31 },
	    { 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, MC_PC_ALL, 14, 15 }
	},
	/*
	 * remainder unused
	 */
	}
};

/*
 * Row/Column/Bank address mappings for revs F/G in 64-bit mode, no interleave.
 */
static const struct _rcbmap_tbl dram_addrmap_f_64 = {
	MC_F_REVS_FG,
	64,
	{
	{	/* 0000 */
		2, { 12, 13 },
		{ 18, 19, 20, 21, 22, 23, 24, 25, 26, 14, 15, 16, 17 },
		{ 3, 4, 5, 6, 7, 8, 9, 10, 11 },
	},
	{	/* 0001 */
		2, { 13, 14 },
		{ 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 15, 16, 17 },
		{ 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 },
	},
	{	/* 0010 */
		2, { 13, 14 },
		{ 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 15, 16, 17 },
		{ 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 },
	},
	{	/* 0011 */
		2, { 14, 15 },
		{ 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 16, 17 },
		{ 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, MC_PC_ALL, 13 },
	},
	{	/* 0100 */
		3, { 13, 14, 15 },
		{ 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 16, 17 },
		{ 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, MC_PC_ALL, 13 },
	},
	{	/* 0101 */
		3, { 13, 14, 15 },
		{ 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 16, 17 },
		{ 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 }
	},
	{	/* 0110 */
		2, { 14, 15 },
		{ 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 16, 17 },
		{ 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, MC_PC_ALL, 13 },
	},
	{	/* 0111 */
		3, { 13, 14, 15 },
		{ 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 16, 17 },
		{ 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 }
	},
	{	/* 1000 */
		3, { 14, 15, 16 },
		{ 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 17 },
		{ 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, MC_PC_ALL, 13 },
	},
	{	/* 1001 */
		3, { 14, 15, 16 },
		{ 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 17 },
		{ 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, MC_PC_ALL, 13 },
	},
	{	/* 1010 */
		3, { 13, 14, 15 },
		{ 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31,
		    16, 17 },
		{ 3, 4, 5, 6, 7, 8, 9, 10, 11, 12 }
	},
	{	/* 1011 */
		3, { 14, 15, 16 },
		{ 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32,
		    17 },
		{ 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, MC_PC_ALL, 13 },
	},
	/*
	 * remainder unused
	 */
	}
};

/*
 * Row/Column/Bank address mappings for revs F/G in 128-bit mode, no interleave.
 */
static const struct _rcbmap_tbl dram_addrmap_f_128 = {
	MC_F_REVS_FG,
	128,
	{
	{	/* 0000 */
		2, { 13, 14 },
		{ 19, 20, 21, 22, 23, 24, 25, 26, 27, 15, 16, 17, 18 },
		{ 4, 5, 6, 7, 8, 9, 10, 11, 12 },
	},
	{	/* 0001 */
		2, { 14, 15 },
		{ 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 16, 17, 18 },
		{ 4, 5, 6, 7, 8, 9, 10, 11, 12, 13 },
	},
	{	/* 0010 */
		2, { 14, 15 },
		{ 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 16, 17, 18 },
		{ 4, 5, 6, 7, 8, 9, 10, 11, 12, 13 },
	},
	{	/* 0011 */
		2, { 15, 16 },
		{ 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 17, 18 },
		{ 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, MC_PC_ALL, 14 },
	},
	{	/* 0100 */
		3, { 14, 15, 16 },
		{ 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 17, 18 },
		{ 4, 5, 6, 7, 8, 9, 10, 11, 12, 13 },
	},
	{	/* 0101 */
		3, { 14, 15, 16 },
		{ 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 17, 18 },
		{ 4, 5, 6, 7, 8, 9, 10, 11, 12, 13 },
	},
	{	/* 0110 */
		2, { 15, 16 },
		{ 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 17, 18 },
		{ 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, MC_PC_ALL, 14 },
	},
	{	/* 0111 */
		3, { 14, 15, 16 },
		{ 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31,
		    17, 18 },
		{ 4, 5, 6, 7, 8, 9, 10, 11, 12, 13 },
	},
	{	/* 1000 */
		3, { 15, 16, 17 },
		{ 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31,
		    18 },
		{ 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, MC_PC_ALL, 14 },
	},
	{	/* 1001 */
		3, { 15, 16, 17 },
		{ 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32,
		    18 },
		{ 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, MC_PC_ALL, 14 },
	},
	{	/* 1010 */
		3, { 14, 15, 16 },
		{ 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32,
		    17, 18 },
		{ 4, 5, 6, 7, 8, 9, 10, 11, 12, 13 },
	},
	{	/* 1011 */
		3, { 15, 16, 17 },
		{ 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33,
		    18 },
		{ 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, MC_PC_ALL, 14 },
	},
	/*
	 * remainder unused
	 */
	}
};

/*
 * Bank swizzling is an option in revisions E and later.  Each internal-bank-
 * select address bit is xor'd with two row address bits.  Which row
 * address bits to use is not dependent on bank address mode but on
 * revision and dram controller width alone.
 *
 * While rev E only supports 2 bank address bits, rev F supports 3 but not
 * all chip-select bank address modes use all 3.  These tables will list
 * the row bits to use in swizzling for the maximum number of supported
 * bank address bits - the consumer musr determine how many should be
 * applied (listed in the above row/col/bank tables).
 */

static const struct _bnkswzl_tbl bnswzl_info_e_64 = {
	MC_F_REV_E,
	64,
	{
	    {
		{ 17, 20 },		/* rows bits to swizzle with BA0 */
		{ 18, 21 },		/* rows bits to swizzle with BA1 */
		/* only 2 bankaddr bits on rev E */
	    }
	}
};

static const struct _bnkswzl_tbl bnswzl_info_e_128 = {
	MC_F_REV_E,
	128,
	{
	    {
		{ 18, 21 },		/* rows bits to swizzle with BA0 */
		{ 19, 22 },		/* rows bits to swizzle with BA1 */
		/* only 2 bankaddr bits on rev E */
	    }
	}
};

static const struct _bnkswzl_tbl bnswzl_info_f_64 = {
	MC_F_REVS_FG,
	64,
	{
	    {
		{ 17, 22 },		/* rows bits to swizzle with BA0 */
		{ 18, 23 },		/* rows bits to swizzle with BA1 */
		{ 19, 24 },		/* rows bits to swizzle with BA2 */
	    }
	}
};

static const struct _bnkswzl_tbl bnswzl_info_f_128 = {
	MC_F_REVS_FG,
	128,
	{
	    {
		{ 18, 23 },		/* rows bits to swizzle with BA0 */
		{ 19, 24 },		/* rows bits to swizzle with BA1 */
		{ 20, 25 },		/* rows bits to swizzle with BA2 */
	    }
	}
};

/*
 * Yet another highbit function.  This really needs to go to common source.
 * Returns range 0 to 64 inclusive;
 */
static int
topbit(uint64_t i)
{
	int h = 1;

	if (i == 0)
		return (0);

	if (i & 0xffffffff00000000ULL) {
		h += 32;
		i >>= 32;
	}

	if (i & 0xffff0000) {
		h += 16;
		i >>= 16;
	}

	if (i & 0xff00) {
		h += 8;
		i >>= 8;
	}

	if (i & 0xf0) {
		h += 4;
		i >>= 4;
	}

	if (i & 0xc) {
		h += 2;
		i >>= 2;
	}

	if (i & 0x2)
		h += 1;

	return (h);
}

/*
 * Lookup the Chip-Select Bank Address Mode Encoding table for a given
 * chip revision and chip-select mode.
 */
const struct rct_bnkaddrmode *
rct_bnkaddrmode(uint_t mcrev, uint_t csmode)
{
	int i;
	const struct _bnkaddrmode_tbldesc *bdp = bnkaddr_tbls;

	for (i = 0; i < sizeof (bnkaddr_tbls) /
	    sizeof (struct _bnkaddrmode_tbldesc);
	    i++, bdp++) {
		if (MC_REV_MATCH(mcrev, bdp->revmask) && csmode < bdp->nmodes)
			return (&bdp->modetbl[csmode]);

	}

	return (NULL);
}

/*
 * Lookup the DRAM Address Mapping table for a given chip revision, access
 * width, bank-swizzle and chip-select mode.
 */
const struct rct_rcbmap *
rct_rcbmap(uint_t mcrev, int width, uint_t csmode)
{
	const struct _rcbmap_tbl *rcbm;
	int i;

	for (i = 0; i < sizeof (rcbmap_tbls) /
	    sizeof (struct _rcbmap_tbldesc); i++) {
		rcbm = rcbmap_tbls[i].rcbmap;
		if (MC_REV_MATCH(mcrev, rcbm->mt_revmask) &&
		    rcbm->mt_width == width && csmode < rcbmap_tbls[i].nmodes)
			return (&rcbm->mt_csmap[csmode]);
	}

	return (NULL);
}

/*
 * Lookup the bank swizzling information for a given chip revision and
 * access width.
 */
const struct rct_bnkswzlinfo *
rct_bnkswzlinfo(uint_t mcrev, int width)
{
	int i;
	const struct _bnkswzl_tbl *swztp;

	for (i = 0; i < sizeof (bnkswzl_tbls) /
	    sizeof (struct rcb_bnkswzl_tbl *); i++) {
		swztp = bnkswzl_tbls[i];
		if (MC_REV_MATCH(mcrev, swztp->swzt_revmask) &&
		    swztp->swzt_width == width)
			return (&swztp->swzt_bits);
	}

	return (NULL);
}

void
rct_csintlv_bits(uint_t mcrev, int width, uint_t csmode, int factor,
    struct rct_csintlv *csid)
{
	int i, lstbnkbit;
	size_t csz;
	const struct rct_bnkaddrmode *bam;
	const struct rct_rcbmap *rcm;

	/*
	 * 8-way cs interleave for some large cs sizes in 128-bit mode is
	 * not implemented prior to rev F.
	 */
	if (factor == 8 && width == 128 &&
	    ((MC_REV_MATCH(mcrev, MC_F_REVS_BC) && csmode == 0x6) ||
	    (MC_REV_MATCH(mcrev, MC_F_REVS_DE) &&
	    (csmode == 0x9 || csmode == 0xa)))) {
		csid->csi_factor = 0;
		return;
	}

	if ((bam = rct_bnkaddrmode(mcrev, csmode)) == NULL ||
	    (rcm = rct_rcbmap(mcrev, width, csmode)) == NULL) {
		csid->csi_factor = 0;
		return;
	}

	csz = MC_CS_SIZE(bam, width);

	switch (factor) {
		case 2:
			csid->csi_nbits = 1;
			break;
		case 4:
			csid->csi_nbits = 2;
			break;
		case 8:
			csid->csi_nbits = 3;
			break;
		default:
			csid->csi_factor = 0;
			return;
	}

	csid->csi_hibit = topbit(csz) - 1;

	/*
	 * The first row bit is immediately after the last bank bit.
	 */
	lstbnkbit = 0;
	for (i = 0; i < rcm->rcb_nbankbits; i++)
		if (rcm->rcb_bankbit[i] > lstbnkbit)
			lstbnkbit = rcm->rcb_bankbit[i];

	csid->csi_lobit = lstbnkbit + 1;

	csid->csi_factor = factor;
}
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License, Version 1.0 only
 * (the "License").  You may not use this file except in compliance
 * with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 *
 * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#include <mcamd_api.h>

/*
 * Indexed by syndrome, value is bit number.  If value is -1, a multi-bit
 * error has been detected.  A special case is the zero'th entry - a
 * syndrome of 0x0 indicates no bits in error.
 */
static char eccsynd[] = {
	-1, 64, 65, -1, 66, -1, -1, -1, 67, -1, -1, 17, -1, -1, 16, -1,
	68, -1, -1, 18, -1, 19, 20, -1, -1, 21, 22, -1, 23, -1, -1, -1,
	69, -1, -1,  8, -1,  9, 10, -1, -1, 11, 12, -1, 13, -1, -1, -1,
	-1, 14, -1, -1, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
	70, -1, -1, -1, -1, -1, -1, -1, -1, -1, 33, -1, -1, -1, -1, 32,
	-1, -1, 34, -1, 35, -1, -1, 36, 37, -1, -1, 38, -1, 39, -1, -1,
	-1, -1, 56, -1, 57, -1, -1, 58, 59, -1, -1, 60, -1, 61, -1, -1,
	62, -1, -1, -1, -1, 63, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
	71, -1, -1, -1, -1, -1, -1, -1, -1, -1, 49, -1, -1, -1, -1, 48,
	-1, -1, 50, -1, 51, -1, -1, 52, 53, -1, -1, 54, -1, 55, -1, -1,
	-1, -1, 40, -1, 41, -1, -1, 42, 43, -1, -1, 44, -1, 45, -1, -1,
	46, -1, -1, -1, -1, 47, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
	-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,  1, -1, -1,  0, -1,
	-1, -1, -1,  2, -1,  3,  4, -1, -1,  5,  6, -1,  7, -1, -1, -1,
	-1, -1, -1, 24, -1, 25, 26, -1, -1, 27, 28, -1, 29, -1, -1, -1,
	-1, 30, -1, -1, 31, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1
};

/*
 * The first dimension of this table is the errored bit pattern, which is the
 * column dimension of the table in the BKDG.  Conveniently, the bit pattern
 * is also the lowest-order nibble in the syndrome, thus allowing the first
 * dimension value to be calculated.  The second dimension is the symbol
 * number, which can be found by searching for a matching syndrome.
 *
 * Note that the first dimension is actually (errored_bit_pattern - 1) since
 * 0 is not a valid errored bit pattern.
 */
#define	MCAMD_CKSYND_NPATS	15
#define	MCAMD_CKSYND_NSYMS	36

static uint16_t cksynd[MCAMD_CKSYND_NPATS][MCAMD_CKSYND_NSYMS] = {
	/* table column 0x1 */
	{ 0xe821, 0x5d31, 0x0001, 0x2021, 0x5041, 0xbe21, 0x4951, 0x74e1,
	0x15c1, 0x3d01, 0x9801, 0xd131, 0xe1d1, 0x6051, 0xa4c1, 0x11c1,
	0x45d1, 0x63e1, 0xb741, 0xdd41, 0x2bd1, 0x83c1, 0x8fd1, 0x4791,
	0x5781, 0xbf41, 0x9391, 0xcce1, 0xa761, 0xff61, 0x5451, 0x6fc1,
	0xbe01, 0x4101, 0xc441, 0x7621 },

	/* table column 0x2 */
	{ 0x7c32, 0xa612, 0x0002, 0x3032, 0xa082, 0xd732, 0x8ea2, 0x9872,
	0x2a42, 0x1602, 0xec02, 0x6212, 0x7262, 0xb0a2, 0xf842, 0x2242,
	0x8a62, 0xb172, 0xd982, 0x6682, 0x3d62, 0xc142, 0xc562, 0x89e2,
	0xa9c2, 0xd582, 0xe1e2, 0x4472, 0xf9b2, 0x55b2, 0xa8a2, 0xb542,
	0xd702, 0x8202, 0x4882, 0x9b32 },

	/* table column 0x3 */
	{ 0x9413, 0xfb23, 0x0003, 0x1013, 0xf0c3, 0x6913, 0xc7f3, 0xec93,
	0x3f83, 0x2b03, 0x7403, 0xb323, 0x93b3, 0xd0f3, 0x5c83, 0x3383,
	0xcfb3, 0xd293, 0x6ec3, 0xbbc3, 0x16b3, 0x4283, 0x4ab3, 0xce73,
	0xfe43, 0x6ac3, 0x7273, 0x8893, 0x5ed3, 0xaad3, 0xfcf3, 0xda83,
	0x6903, 0xc303, 0x8cc3, 0xed13 },

	/* table column 0x4 */
	{ 0xbb44, 0x9584, 0x0004, 0x4044, 0x9054, 0x2144, 0x5394, 0xd6b4,
	0xcef4, 0x8504, 0x6b04, 0x3884, 0xb834, 0x1094, 0xe6f4, 0xc8f4,
	0x5e34, 0x14b4, 0x2254, 0x3554, 0x4f34, 0xa4f4, 0xa934, 0x5264,
	0x92a4, 0x2954, 0x6464, 0xfdb4, 0xe214, 0x7914, 0x9694, 0x19f4,
	0x2104, 0x5804, 0xf654, 0xda44 },

	/* table column 0x5 */
	{ 0x5365, 0xc8b5, 0x0005, 0x6065, 0xc015, 0x9f65, 0x1ac5, 0xa255,
	0xdb35, 0xb805, 0xf305, 0xe9b5, 0x59e5, 0x70c5, 0x4235, 0xd935,
	0x1be5, 0x7755, 0x9515, 0xe815, 0x64e5, 0x2735, 0x26e5, 0x15f5,
	0xc525, 0x9615, 0xf7f5, 0x3155, 0x4575, 0x8675, 0xc2c5, 0x7635,
	0x9f05, 0x1905, 0x3215, 0xac65 },

	/* table column 0x6 */
	{ 0xc776, 0x3396, 0x0006, 0x7076, 0x30d6, 0xf676, 0xdd36, 0x4ec6,
	0xe4b6, 0x9306, 0x8706, 0x5a96, 0xca56, 0xa036, 0x1eb6, 0xeab6,
	0xd456, 0xa5c6, 0xfbd6, 0x53d6, 0x7256, 0x65b6, 0x6c56, 0xdb86,
	0x3b66, 0xfcd6, 0x8586, 0xb9c6, 0x1ba6, 0x2ca6, 0x3e36, 0xacb6,
	0xf606, 0xda06, 0xbed6, 0x4176 },

	/* table column 0x7 */
	{ 0x2f57, 0x6ea7, 0x0007, 0x5057, 0x6097, 0x4857, 0x9467, 0x3a27,
	0xf177, 0xae07, 0x1f07, 0x8ba7, 0x2b87, 0xc067, 0xba77, 0xfb77,
	0x9187, 0xc627, 0x4c97, 0x8e97, 0x5987, 0xe677, 0xe387, 0x9c17,
	0x6ce7, 0x4397, 0x1617, 0x7527, 0xbcc7, 0xd3c7, 0x6a67, 0xc377,
	0x4807, 0x9b07, 0x7a97, 0x3757 },

	/* table column 0x8 */
	{ 0xdd88, 0xeac8, 0x0008, 0x8088, 0xe0a8, 0x3288, 0xa1e8, 0x6bd8,
	0x4758, 0xca08, 0xbd08, 0x1cc8, 0xdc18, 0x20e8, 0x7b58, 0x4c58,
	0xa718, 0x28d8, 0x33a8, 0x1aa8, 0x8518, 0xf858, 0xfe18, 0xa3b8,
	0xe3f8, 0x3ea8, 0xb8b8, 0x56d8, 0x7328, 0x9e28, 0xebe8, 0x2e58,
	0x3208, 0xac08, 0x5ba8, 0x6f88 },

	/* table column 0x9 */
	{ 0x35a9, 0xb7f9, 0x0009, 0xa0a9, 0xb0e9, 0x8ca9, 0xe8b9, 0x1f39,
	0x5299, 0xf709, 0x2509, 0xcdf9, 0x3dc9, 0x40b9, 0xdf99, 0x5d99,
	0xe2c9, 0x4b39, 0x84e9, 0xc7e9, 0xaec9, 0x7b99, 0x71c9, 0xe429,
	0xb479, 0x81e9, 0x2b29, 0x9a39, 0xd449, 0x6149, 0xbfb9, 0x4199,
	0x8c09, 0xed09, 0x9fe9, 0x19a9 },

	/* table column 0xa */
	{ 0xa1ba, 0x4cda, 0x000a, 0xb0ba, 0x402a, 0xe5ba, 0x2f4a, 0xf3aa,
	0x6d1a, 0xdc0a, 0x510a, 0x7eda, 0xae7a, 0x904a, 0x831a, 0x6e1a,
	0x2d7a, 0x99aa, 0xea2a, 0x7c2a, 0xb87a, 0x391a, 0x3b7a, 0x2a5a,
	0x4a3a, 0xeb2a, 0x595a, 0x12aa, 0x8a9a, 0xcb9a, 0x434a, 0x9b1a,
	0xe50a, 0x2e0a, 0x132a, 0xf4ba },

	/* table column 0xb */
	{ 0x499b, 0x11eb, 0x000b, 0x909b, 0x106b, 0x5b9b, 0x661b, 0x874b,
	0x78db, 0xe10b, 0xc90b, 0xafeb, 0x4fab, 0xf01b, 0x27db, 0x7fdb,
	0x68ab, 0xfa4b, 0x5d6b, 0xa16b, 0x93ab, 0xbadb, 0xb4ab, 0x6dcb,
	0x1dbb, 0x546b, 0xcacb, 0xde4b, 0x2dfb, 0x34fb, 0x171b, 0xf4db,
	0x5b0b, 0x6f0b, 0xd76b, 0x829b },

	/* table column 0xc */
	{ 0x66cc, 0x7f4c, 0x000c, 0xc0cc, 0x70fc, 0x13cc, 0xf27c, 0xbd6c,
	0x89ac, 0x4f0c, 0xd60c, 0x244c, 0x642c, 0x307c, 0x9dac, 0x84ac,
	0xf92c, 0x3c6c, 0x11fc, 0x2ffc, 0xca2c, 0x5cac, 0x572c, 0xf1dc,
	0x715c, 0x17fc, 0xdcdc, 0xab6c, 0x913c, 0xe73c, 0x7d7c, 0x37ac,
	0x130c, 0xf40c, 0xadfc, 0xb5cc },

	/* table column 0xd */
	{ 0x8eed, 0x227d, 0x000d, 0xe0ed, 0x20bd, 0xaded, 0xbb2d, 0xc98d,
	0x9c6d, 0x720d, 0x4e0d, 0xf57d, 0x85fd, 0x502d, 0x396d, 0x956d,
	0xbcfd, 0x5f8d, 0xa6bd, 0xf2bd, 0xe1fd, 0xdf6d, 0xd8fd, 0xb64d,
	0x26dd, 0xa8bd, 0x4f4d, 0x678d, 0x365d, 0x185d, 0x292d, 0x586d,
	0xad0d, 0xb50d, 0x69bd, 0xc3ed },

	/* table column 0xe */
	{ 0x1afe, 0xd95e, 0x000e, 0xf0fe, 0xd07e, 0xc4fe, 0x7cde, 0x251e,
	0xa3ee, 0x590e, 0x3a0e, 0x465e, 0x164e, 0x80de, 0x65ee, 0xa6ee,
	0x734e, 0x8d1e, 0xc87e, 0x497e, 0xf74e, 0x9dee, 0x924e, 0x783e,
	0xd89e, 0xc27e, 0x3d3e, 0xef1e, 0x688e, 0xb28e, 0xd5de, 0x82ee,
	0xc40e, 0x760e, 0xe57e, 0x2efe },

	/* table column 0xf */
	{ 0xf2df, 0x846f, 0x000f, 0xd0df, 0x803f, 0x7adf, 0x358f, 0x51ff,
	0xb62f, 0x640f, 0xa20f, 0x976f, 0xf79f, 0xe08f, 0xc12f, 0xb72f,
	0x369f, 0xeeff, 0x7f3f, 0x943f, 0xdc9f, 0x1e2f, 0x1d9f, 0x3faf,
	0x8f1f, 0x7d3f, 0xaeaf, 0x23ff, 0xcfef, 0x4def, 0x818f, 0xed2f,
	0x7a0f, 0x370f, 0x213f, 0x58df }
};

int
mcamd_synd_validate(struct mcamd_hdl *hdl, uint32_t synd, int syndtype)
{
	int result;

	switch (syndtype) {
	case AMD_SYNDTYPE_ECC:
		result = (synd > 0 && synd <= 0xff);
		break;
	case AMD_SYNDTYPE_CHIPKILL:
		result = (synd > 0 && synd <= 0xffff);
		break;
	default:
		mcamd_dprintf(hdl, MCAMD_DBG_FLOW,
		    "mcamd_synd_validate: invalid syndtype %d\n", syndtype);
		return (0);
	}

	if (result == 0)
		mcamd_dprintf(hdl, MCAMD_DBG_FLOW, "mcamd_synd_validate: "
		    "invalid %s syndrome 0x%x\n",
		    syndtype == AMD_SYNDTYPE_ECC ? "64/8" : "ChipKill",
		    synd);

	return (result);
}

int
mcamd_eccsynd_decode(struct mcamd_hdl *hdl, uint32_t synd, uint_t *bitp)
{
	char bit;

	if (synd > 0xff) {
		mcamd_dprintf(hdl, MCAMD_DBG_FLOW, "mcamd_eccsynd_decode: "
		    "invalid synd 0x%x\n", synd);
		return (0);
	}
	if ((bit = eccsynd[synd]) == -1) {
		mcamd_dprintf(hdl, MCAMD_DBG_FLOW, "mcamd_eccsynd_decode: "
		    "synd 0x%x is a multi-bit syndrome\n", synd);
		return (0);
	}

	mcamd_dprintf(hdl, MCAMD_DBG_FLOW, "mcamd_eccsynd_decode: "
	    "synd 0x%x is single-bit and indicates %s bit %d\n", synd,
	    bit >= 64 ? "check" : "data",
	    bit >= 64 ? bit - 64 : bit);

	*bitp = bit;
	return (1);
}

int
mcamd_cksynd_decode(struct mcamd_hdl *hdl, uint32_t synd, uint_t *symp,
    uint_t *patp)
{
	int pat = synd & 0xf;
	int i;

	if (pat == 0) {
		mcamd_dprintf(hdl, MCAMD_DBG_FLOW, "mcamd_cksynd_decode: "
		    "synd 0x%x is not a correctable syndrome\n", synd);
		return (0);
	}

	for (i = 0; i < MCAMD_CKSYND_NSYMS; i++) {
		if (cksynd[pat - 1][i] == synd) {
			*symp = i;
			*patp = pat;
			mcamd_dprintf(hdl, MCAMD_DBG_FLOW,
			    "mcamd_cksynd_decode: synd 0x%x is correctable "
			    "and indicates symbol %d\n", synd, i);
			return (1);
		}
	}

	mcamd_dprintf(hdl, MCAMD_DBG_FLOW, "mcamd_cksynd_decode: "
	    "synd 0x%x is not a correctable syndrome\n", synd);
	return (0);
}

/*
 * symbols 0 to 0xf:		data[63:0]
 * symbols 0x10 to 0x1f:	data[127:64]
 * symbols 0x20, 0x21:		checkbits for [63:0]
 * symbols 0x22, 0x23:		checkbits for [127:64]
 */
/*ARGSUSED*/
int
mcamd_cksym_decode(struct mcamd_hdl *hdl, uint_t sym, int *lowbitp,
    int *hibitp, int *data, int *check)
{
	if (sym <= 0xf || sym >= 0x10 && sym <= 0x1f) {
		*data = 1;
		*check = 0;
		*lowbitp = sym * 4;
		*hibitp = (sym + 1) * 4 - 1;
	} else if (sym >= 0x20 && sym <= 0x23) {
		*data = 0;
		*check = 1;
		*lowbitp = (sym - 0x20) * 4;
		*hibitp = (sym + 1 - 0x20) * 4 - 1;
	} else {
		return (0);
	}

	return (1);
}
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 *
 * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

/*
 * Given a unum including an offset calculate the associated system
 * address.  This may be different to when the original PA to unum
 * calculation took place if interleave etc has changed.
 */

#include <sys/errno.h>
#include <sys/types.h>
#include <sys/mc.h>

#include <mcamd_api.h>
#include <mcamd_err.h>

/*
 * The submitted unum must have the MC and DIMM numbers and an offset.
 * Any cs info it has will not be used - we will reconstruct cs info.
 * This is because cs is not in the topology used for diagnosis.
 */
int
mcamd_unumtopa(struct mcamd_hdl *hdl, mcamd_node_t *root, mc_unum_t *unump,
    uint64_t *pa)
{
	mcamd_node_t *mc, *dimm;
	uint64_t num, holesz;

	mcamd_dprintf(hdl, MCAMD_DBG_FLOW, "mcamd_unumtopa: chip %d "
	    "mc %d dimm %d offset 0x%llx\n", unump->unum_chip, unump->unum_mc,
	    unump->unum_dimms[0], unump->unum_offset);

	if (!MCAMD_RC_OFFSET_VALID(unump->unum_offset)) {
		mcamd_dprintf(hdl, MCAMD_DBG_FLOW, "mcamd_unumtopa: offset "
		    "invalid\n");
		return (mcamd_set_errno(hdl, EMCAMD_NOADDR));
	}

	/*
	 * Search current config for a MC number matching the chip in the
	 * unum.
	 */
	for (mc = mcamd_mc_next(hdl, root, NULL); mc != NULL;
	    mc = mcamd_mc_next(hdl, root, mc)) {
		if (!mcamd_get_numprops(hdl,
		    mc, MCAMD_PROP_NUM, &num,
		    mc, MCAMD_PROP_DRAMHOLE_SIZE, &holesz,
		    NULL)) {
			mcamd_dprintf(hdl, MCAMD_DBG_ERR, "mcamd_unumtopa: "
			    "failed to lookup num, dramhole for MC 0x%p\n", mc);
			return (mcamd_set_errno(hdl, EMCAMD_TREEINVALID));
		}
		if (num == unump->unum_chip)
			break;
	}
	if (mc == NULL) {
		mcamd_dprintf(hdl, MCAMD_DBG_FLOW, "mcamd_unumtopa; "
		    "no match for MC %d\n", unump->unum_chip);
		return (mcamd_set_errno(hdl, EMCAMD_NOADDR));
	}

	/*
	 * Search DIMMs of this MC.  We can match against the
	 * first dimm in the unum - if there is more than one they all
	 * share the same chip-selects anyway and the pa we will resolve
	 * to is not finer grained than the 128-bits of a dimm pair.
	 */
	for (dimm = mcamd_dimm_next(hdl, mc, NULL); dimm != NULL;
	    dimm = mcamd_dimm_next(hdl, mc, dimm)) {
		if (!mcamd_get_numprop(hdl, dimm, MCAMD_PROP_NUM, &num)) {
			mcamd_dprintf(hdl, MCAMD_DBG_ERR, "mcamd_unumtopa: "
			    "failed to lookup num for dimm 0xx%p\n",
			    dimm);
			return (mcamd_set_errno(hdl, EMCAMD_TREEINVALID));
		}
		if (num == unump->unum_dimms[0])
			break;
	}
	if (dimm == NULL) {
		mcamd_dprintf(hdl, MCAMD_DBG_FLOW, "mcamd_unumtopa; "
		    "no match for dimm %d cs %d on MC %d\n",
		    unump->unum_dimms[0], unump->unum_cs, unump->unum_chip);
		return (mcamd_set_errno(hdl, EMCAMD_NOADDR));
	}

	mcamd_dprintf(hdl, MCAMD_DBG_FLOW, "mcamd_unumtopa: matched "
	    "mc 0x%p dimm 0x%p; resolving offset 0x%llx\n",
	    mc, dimm, unump->unum_offset);

	if (mc_offset_to_pa(hdl, mc, dimm, unump->unum_offset, pa) < 0) {
		mcamd_dprintf(hdl, MCAMD_DBG_FLOW, "mcamd_unumtopa: "
		    "mc_offset_to_pa failed: %s\n", mcamd_errmsg(hdl));
		return (-1);	/* errno already set */
	}

	mcamd_dprintf(hdl, MCAMD_DBG_FLOW, "mcamd_unumtopa: "
	    "mc_offset_to_pa succeeded and returned pa=0x%llx: %\n",
	    *pa);

	/*
	 * If this MC has a dram address hole just below 4GB then we must
	 * hoist all address from the hole start upwards by the hole size
	 */
	if (holesz != 0) {
		if (*pa >= 0x100000000 - holesz)
			*pa += holesz;
		mcamd_dprintf(hdl, MCAMD_DBG_FLOW, "mcamd_untopa: hoist "
		    "above dram hole of size 0x%llx to get pa=0x%llx",
		    holesz, *pa);
	}

	return (0);
}
/*
 * This file and its contents are supplied under the terms of the
 * Common Development and Distribution License ("CDDL"), version 1.0.
 * You may only use this file in accordance with the terms of version
 * 1.0 of the CDDL.
 *
 * A full copy of the text of the CDDL should have accompanied this
 * source.  A copy of the CDDL is also available via the Internet at
 * http://www.illumos.org/license/CDDL.
 */

/*
 * Copyright 2025 Oxide Computer Company
 */

/*
 * Zen UMC Decoding logic. See zen_umc.c for an overview of everything. This
 * implements shared userland/kernel decoding.
 */

#include "zen_umc.h"

#ifndef _KERNEL
#include <strings.h>
#endif

/*
 * Address constants.
 */
#define	ZEN_UMC_TOM2_START	0x100000000ULL
#define	ZEN_UMC_TOM2_RSVD_BEGIN	0xfd00000000ULL
#define	ZEN_UMC_TOM2_RSVD_END	0x10000000000ULL

/*
 * COD based hashing constants.
 */
#define	ZEN_UMC_COD_NBITS	3
#define	ZEN_UMC_NPS_MOD_NBITS	3

/*
 * Enumeration that represents which parts of the NPS 1K/2K non-power of 2 hash
 * we should use. These are ordered such their indexes correspond with the
 * 'hashes' array indexes used in zen_umc_decode_ileave_nps_k_mod().
 */
typedef enum {
	ZEN_UMC_NP2_K_HASH_8 = 0,
	ZEN_UMC_NP2_K_HASH_9,
	ZEN_UMC_NP2_K_HASH_12,
	ZEN_UMC_NP2_K_HASH_13
} zen_umc_np2_k_hash_t;

typedef struct {
	/*
	 * Indicates what the type of this rule is.
	 */
	df_chan_ileave_t zukr_type;
	/*
	 * This is the modulus that this rule uses.
	 */
	uint32_t zukr_mod;
	/*
	 * Indicates that this rule requires socket interleaving. Otherwise we
	 * expect no socket interleaving to be enabled.
	 */
	boolean_t zukr_sock;
	/*
	 * This is the 'high' portion of the original address that is used as
	 * part of the division and modulus logic when we take it. This bit is
	 * inclusive, e.g. a value of 12 indicates we want addr[64:12].
	 */
	uint32_t zukr_high;
	/*
	 * This indicates at what point in the modulus address the high bits
	 * should arrive at.
	 */
	uint32_t zukr_mod_shift;
	/*
	 * This indicates how we should fill the remaining bits in the modulus
	 * address. This is either zero filled or an original address bit. Only
	 * address bits 8 or 9 are ever used so we cheat and treat a zero here
	 * as zero filled. Only the first zukr_mod_shift bits will be
	 * considered. This and zukr_mod_shit are used prior to the modulus
	 * calculation.
	 */
	uint32_t zukr_mod_fill[5];
	/*
	 * The next series of values defines how to construct the channel. The
	 * channel is always made up of some number of bits from the modulus
	 * value and then optionally some of the hash bits. The first value
	 * indicates how many bits to shift the resulting modulus value by. Any
	 * bit that it is shifted over by must be filled by a hashed value. The
	 * indication of which hash bit is indicated by its starting address
	 * number.
	 */
	uint32_t zukr_chan_mod_shift;
	zen_umc_np2_k_hash_t zukr_chan_fill[2];
	/*
	 * Next, it's time to describe how to construct the normalized address.
	 * There is a portion of it which is divided by the modulus. This is
	 * always going to be the high bits, but sometimes includes additional
	 * lower parts of the physical address ORed in. The first value
	 * indicates how many consecutive address bits should be included. The
	 * second indicates the starting address.
	 */
	uint32_t zukr_div_addr;
	uint32_t zukr_div_naddr;
	/*
	 * Finally the middle portion of the normalized address.
	 */
	uint32_t zukr_norm_addr;
	uint32_t zukr_norm_naddr;
} zen_umc_np2_k_rule_t;

const zen_umc_np2_k_rule_t zen_umc_np2_k_rules[] = { {
	.zukr_type = DF_CHAN_ILEAVE_NPS4_3CH_1K,
	.zukr_mod = 3,
	.zukr_high = 12,
	.zukr_mod_shift = 2,
	.zukr_mod_fill = { 8, 9 },
	.zukr_chan_mod_shift = 0,
	.zukr_div_addr = 8,
	.zukr_div_naddr = 2,
	.zukr_norm_addr = 10,
	.zukr_norm_naddr = 2
}, {
	.zukr_type = DF_CHAN_ILEAVE_NPS4_3CH_2K,
	.zukr_mod = 3,
	.zukr_high = 12,
	.zukr_mod_shift = 2,
	.zukr_mod_fill = { 0, 8 },
	.zukr_chan_mod_shift = 0,
	.zukr_div_addr = 8,
	.zukr_div_naddr = 1,
	.zukr_norm_addr = 9,
	.zukr_norm_naddr = 3
}, {
	.zukr_type = DF_CHAN_ILEAVE_NPS2_6CH_1K,
	.zukr_mod = 3,
	.zukr_high = 12,
	.zukr_mod_shift = 2,
	.zukr_mod_fill = { 0, 9 },
	.zukr_chan_mod_shift = 1,
	.zukr_chan_fill = { ZEN_UMC_NP2_K_HASH_8 },
	.zukr_div_addr = 9,
	.zukr_div_naddr = 1,
	.zukr_norm_addr = 10,
	.zukr_norm_naddr = 2
}, {
	.zukr_type = DF_CHAN_ILEAVE_NPS2_6CH_2K,
	.zukr_mod = 3,
	.zukr_high = 12,
	.zukr_mod_shift = 2,
	.zukr_mod_fill = { 0, 0 },
	.zukr_chan_mod_shift = 1,
	.zukr_chan_fill = { ZEN_UMC_NP2_K_HASH_8 },
	.zukr_div_naddr = 0,
	.zukr_norm_addr = 9,
	.zukr_norm_naddr = 3
}, {
	.zukr_type = DF_CHAN_ILEAVE_NPS1_12CH_1K,
	.zukr_mod = 3,
	.zukr_high = 12,
	.zukr_mod_shift = 2,
	.zukr_mod_fill = { 0, 0 },
	.zukr_chan_mod_shift = 2,
	.zukr_chan_fill = { ZEN_UMC_NP2_K_HASH_8, ZEN_UMC_NP2_K_HASH_9 },
	.zukr_div_naddr = 0,
	.zukr_norm_addr = 10,
	.zukr_norm_naddr = 2
}, {
	.zukr_type = DF_CHAN_ILEAVE_NPS1_12CH_1K,
	.zukr_mod = 3,
	.zukr_high = 12,
	.zukr_mod_shift = 2,
	.zukr_mod_fill = { 0, 0 },
	.zukr_chan_mod_shift = 2,
	.zukr_chan_fill = { ZEN_UMC_NP2_K_HASH_8, ZEN_UMC_NP2_K_HASH_9 },
	.zukr_div_naddr = 0,
	.zukr_norm_addr = 10,
	.zukr_norm_naddr = 2
}, {
	.zukr_type = DF_CHAN_ILEAVE_NPS1_12CH_2K,
	.zukr_mod = 3,
	.zukr_high = 13,
	.zukr_mod_shift = 3,
	.zukr_mod_fill = { 0, 0, 0 },
	.zukr_chan_mod_shift = 2,
	.zukr_chan_fill = { ZEN_UMC_NP2_K_HASH_8, ZEN_UMC_NP2_K_HASH_12 },
	.zukr_div_naddr = 0,
	.zukr_norm_addr = 9,
	.zukr_norm_naddr = 3
}, {
	.zukr_type = DF_CHAN_ILEAVE_NPS0_24CH_1K,
	.zukr_mod = 3,
	.zukr_sock = B_TRUE,
	.zukr_high = 13,
	.zukr_mod_shift = 3,
	.zukr_mod_fill = { 0, 0, 0 },
	.zukr_chan_mod_shift = 2,
	.zukr_chan_fill = { ZEN_UMC_NP2_K_HASH_9, ZEN_UMC_NP2_K_HASH_12 },
	.zukr_div_naddr = 0,
	.zukr_norm_addr = 10,
	.zukr_norm_naddr = 2
}, {
	.zukr_type = DF_CHAN_ILEAVE_NPS0_24CH_2K,
	.zukr_mod = 3,
	.zukr_sock = B_TRUE,
	.zukr_high = 14,
	.zukr_mod_shift = 4,
	.zukr_mod_fill = { 0, 0, 0, 0 },
	.zukr_chan_mod_shift = 2,
	.zukr_chan_fill = { ZEN_UMC_NP2_K_HASH_12, ZEN_UMC_NP2_K_HASH_13 },
	.zukr_div_naddr = 0,
	.zukr_norm_addr = 9,
	.zukr_norm_naddr = 3
}, {
	.zukr_type = DF_CHAN_ILEAVE_NPS2_5CH_1K,
	.zukr_mod = 5,
	.zukr_high = 12,
	.zukr_mod_shift = 2,
	.zukr_mod_fill = { 8, 9 },
	.zukr_chan_mod_shift = 0,
	.zukr_div_addr = 8,
	.zukr_div_naddr = 2,
	.zukr_norm_addr = 10,
	.zukr_norm_naddr = 2
}, {
	.zukr_type = DF_CHAN_ILEAVE_NPS2_5CH_2K,
	.zukr_mod = 5,
	.zukr_high = 12,
	.zukr_mod_shift = 2,
	.zukr_mod_fill = { 0, 8 },
	.zukr_chan_mod_shift = 0,
	.zukr_div_addr = 8,
	.zukr_div_naddr = 1,
	.zukr_norm_addr = 9,
	.zukr_norm_naddr = 3
}, {
	.zukr_type = DF_CHAN_ILEAVE_NPS1_10CH_1K,
	.zukr_mod = 5,
	.zukr_high = 12,
	.zukr_mod_shift = 2,
	.zukr_mod_fill = { 0, 9 },
	.zukr_chan_mod_shift = 1,
	.zukr_chan_fill = { ZEN_UMC_NP2_K_HASH_8 },
	.zukr_div_addr = 9,
	.zukr_div_naddr = 1,
	.zukr_norm_addr = 10,
	.zukr_norm_naddr = 2
}, {
	.zukr_type = DF_CHAN_ILEAVE_NPS1_10CH_2K,
	.zukr_mod = 5,
	.zukr_high = 12,
	.zukr_mod_shift = 2,
	.zukr_mod_fill = { 0, 0 },
	.zukr_chan_mod_shift = 1,
	.zukr_chan_fill = { ZEN_UMC_NP2_K_HASH_8 },
	.zukr_div_naddr = 0,
	.zukr_norm_addr = 9,
	.zukr_norm_naddr = 3
} };

/*
 * We want to apply some initial heuristics to determine if a physical address
 * is DRAM before we proceed because of the MMIO hole and related. The DRAM
 * ranges can overlap with these system reserved ranges so we have to manually
 * check these.  Effectively this means that we have a few valid ranges:
 *
 *  o [ 0, TOM )
 *  o [ 4 GiB, TOM2 )
 *
 * However, the above 4 GiB runs into trouble depending on size. There is a 12
 * GiB system reserved address region right below 1 TiB. So it really turns
 * into the following when we have more than 1 TiB of DRAM:
 *
 *  o [ 0, TOM )
 *  o [ 4 GiB, 1 TiB - 12 GiB )
 *  o [ 1 TiB, TOM2 )
 *
 * Note, this does not currently scan MTRRs or MMIO rules for what might be
 * redirected to MMIO.
 */
static boolean_t
zen_umc_decode_is_dram(const zen_umc_t *umc, zen_umc_decoder_t *dec)
{
	if (dec->dec_pa < umc->umc_tom) {
		return (B_TRUE);
	}

	if (dec->dec_pa >= umc->umc_tom2) {
		dec->dec_fail = ZEN_UMC_DECODE_F_OUTSIDE_DRAM;
		return (B_FALSE);
	}

	/*
	 * If the address is in the reserved hole around 1 TiB, do not proceed.
	 */
	if (dec->dec_pa >= ZEN_UMC_TOM2_RSVD_BEGIN &&
	    dec->dec_pa < ZEN_UMC_TOM2_RSVD_END) {
		dec->dec_fail = ZEN_UMC_DECODE_F_OUTSIDE_DRAM;
		return (B_FALSE);
	}

	/*
	 * Now that we've validated we're not in the hole, check to see if we're
	 * actually in a valid region for TOM2.
	 */
	if (dec->dec_pa >= ZEN_UMC_TOM2_START &&
	    dec->dec_pa < umc->umc_tom2) {
		return (B_TRUE);
	}

	/*
	 * At this point we have eliminated all known DRAM regions described by
	 * TOM and TOM2, so we have to conclude that whatever we're looking at
	 * is now not part of DRAM.
	 */
	dec->dec_fail = ZEN_UMC_DECODE_F_OUTSIDE_DRAM;
	return (B_FALSE);
}

/*
 * In our first stop on decoding, we need to go through and take a physical
 * address and figure out what the corresponding initial DF rule that applies
 * is. This rule will then be used to figure out which target on the data fabric
 * we should be going to and what interleaving rules apply.
 *
 * Our DRAM rule may reflect that the DRAM hole is active. In this case the
 * specified range in the rule will be larger than the actual amount of DRAM
 * present. MMIO accesses take priority over DRAM accesses in the core and
 * therefore the MMIO portion of the rule is not actually decoded. When trying
 * to match a rule we do not need to worry about that and can just look whether
 * our physical address matches a rule. We will take into account whether
 * hoisting should adjust the address when we translate from a system address to
 * a normal address (e.g. an address in the channel) which will be done in a
 * subsequent step. If an address is in the hole, that has already been
 * accounted for.
 *
 * While gathering information, we have all the DRAM rules for a given CCM that
 * corresponds to a CPU core. This allows us to review all DRAM rules in one
 * place rather than walking through what's been assigned to each UMC instance,
 * which only has the rules that are directed towards that particular channel
 * and matter for determining channel offsets.
 */
static boolean_t
zen_umc_decode_find_df_rule(const zen_umc_t *umc, zen_umc_decoder_t *dec)
{
	const zen_umc_df_t *df = &umc->umc_dfs[0];

	for (uint_t i = 0; i < df->zud_dram_nrules; i++) {
		const df_dram_rule_t *rule = &df->zud_rules[i];

		/*
		 * If this rule is not enabled, skip it.
		 */
		if ((rule->ddr_flags & DF_DRAM_F_VALID) == 0)
			continue;

		if (dec->dec_pa >= rule->ddr_base &&
		    dec->dec_pa < rule->ddr_limit) {
			dec->dec_df_ruleno = i;
			dec->dec_df_rule = rule;
			dec->dec_df_rulesrc = df;
			return (B_TRUE);
		}
	}

	dec->dec_fail = ZEN_UMC_DECODE_F_NO_DF_RULE;
	return (B_FALSE);
}

/*
 * This function takes care of the common logic of adjusting an address by the
 * base value in the rule and determining if we need to apply the DRAM hole or
 * not. This function is used in two different places:
 *
 *   o As part of adjusting the system address to construct the interleave
 *     address for DFv4 and Zen 3 based 6-channel hashing (see
 *     zen_umc_determine_ileave_addr() below).
 *   o As part of adjusting the system address at the beginning of normalization
 *     to a channel address.
 *
 * One thing to highlight is that the same adjustment we make in the first case
 * applies to a subset of things for interleaving; however, it applies to
 * everything when normalizing.
 */
static boolean_t
zen_umc_adjust_dram_addr(const zen_umc_t *umc, zen_umc_decoder_t *dec,
    uint64_t *addrp, zen_umc_decode_failure_t errno)
{
	const uint64_t init_addr = *addrp;
	const df_dram_rule_t *rule = dec->dec_df_rule;
	const zen_umc_df_t *df = dec->dec_df_rulesrc;
	uint64_t mod_addr = init_addr;

	ASSERT3U(init_addr, >=, rule->ddr_base);
	ASSERT3U(init_addr, <, rule->ddr_limit);
	mod_addr -= rule->ddr_base;

	/*
	 * Determine if the hole applies to this rule.
	 */
	if ((rule->ddr_flags & DF_DRAM_F_HOLE) != 0 &&
	    (df->zud_flags & ZEN_UMC_DF_F_HOLE_VALID) != 0 &&
	    init_addr >= ZEN_UMC_TOM2_START) {
		uint64_t hole_size;
		hole_size = ZEN_UMC_TOM2_START -
		    umc->umc_dfs[0].zud_hole_base;
		if (mod_addr < hole_size) {
			dec->dec_fail = errno;
			dec->dec_fail_data = dec->dec_df_ruleno;
			return (B_FALSE);
		}

		mod_addr -= hole_size;
	}

	*addrp = mod_addr;
	return (B_TRUE);
}

/*
 * Take care of constructing the address we need to use for determining the
 * interleaving target fabric id. See the big theory statement in zen_umc.c for
 * more on this.
 */
static boolean_t
zen_umc_determine_ileave_addr(const zen_umc_t *umc, zen_umc_decoder_t *dec)
{
	const df_dram_rule_t *rule = dec->dec_df_rule;

	if ((umc->umc_df_rev <= DF_REV_3 &&
	    rule->ddr_chan_ileave != DF_CHAN_ILEAVE_6CH) ||
	    umc->umc_df_rev >= DF_REV_4D2) {
		dec->dec_ilv_pa = dec->dec_pa;
		return (B_TRUE);
	}

	dec->dec_ilv_pa = dec->dec_pa;
	if (!zen_umc_adjust_dram_addr(umc, dec, &dec->dec_ilv_pa,
	    ZEN_UMC_DECODE_F_ILEAVE_UNDERFLOW)) {
		return (B_FALSE);
	}

	return (B_TRUE);
}

/*
 * This is a simple interleaving case where we simply extract bits. No hashing
 * required! Per zen_umc.c, from lowest to highest, we have channel, die, and
 * then socket bits.
 */
static boolean_t
zen_umc_decode_ileave_nohash(const zen_umc_t *umc, zen_umc_decoder_t *dec)
{
	uint32_t nchan_bit, ndie_bit, nsock_bit, addr_bit;
	const df_dram_rule_t *rule = dec->dec_df_rule;

	nsock_bit = rule->ddr_sock_ileave_bits;
	ndie_bit = rule->ddr_die_ileave_bits;
	switch (rule->ddr_chan_ileave) {
	case DF_CHAN_ILEAVE_1CH:
		nchan_bit = 0;
		break;
	case DF_CHAN_ILEAVE_2CH:
		nchan_bit = 1;
		break;
	case DF_CHAN_ILEAVE_4CH:
		nchan_bit = 2;
		break;
	case DF_CHAN_ILEAVE_8CH:
		nchan_bit = 3;
		break;
	case DF_CHAN_ILEAVE_16CH:
		nchan_bit = 4;
		break;
	case DF_CHAN_ILEAVE_32CH:
		nchan_bit = 5;
		break;
	default:
		dec->dec_fail = ZEN_UMC_DECODE_F_CHAN_ILEAVE_NOTSUP;
		dec->dec_fail_data = rule->ddr_chan_ileave;
		return (B_FALSE);
	}

	/*
	 * Zero all of these out in case no bits are dedicated to this purpose.
	 * In those cases, then the value for this is always zero.
	 */
	dec->dec_ilv_sock = dec->dec_ilv_die = dec->dec_ilv_chan = 0;
	addr_bit = rule->ddr_addr_start;
	if (nchan_bit > 0) {
		dec->dec_ilv_chan = bitx64(dec->dec_ilv_pa,
		    addr_bit + nchan_bit - 1, addr_bit);
		addr_bit += nchan_bit;
	}

	if (ndie_bit > 0) {
		dec->dec_ilv_die = bitx64(dec->dec_ilv_pa,
		    addr_bit + ndie_bit - 1, addr_bit);
		addr_bit += ndie_bit;
	}

	if (nsock_bit > 0) {
		dec->dec_ilv_sock = bitx64(dec->dec_ilv_pa,
		    addr_bit + nsock_bit - 1, addr_bit);
		addr_bit += nsock_bit;
	}

	return (B_TRUE);
}

/*
 * Perform the Zen 2/Zen 3 "COD" based hashing. See the zen_umc.c interleaving
 * section of the big theory statement for an overview of how this works.
 */
static boolean_t
zen_umc_decode_ileave_cod(const zen_umc_t *umc, zen_umc_decoder_t *dec)
{
	uint32_t nchan_bit;
	const df_dram_rule_t *rule = dec->dec_df_rule;
	/*
	 * The order of bits here is defined by AMD. Yes, we do use the rule's
	 * address bit first and then skip to bit 12 for the second hash bit.
	 */
	const uint32_t addr_bits[3] = { rule->ddr_addr_start, 12, 13 };

	if (rule->ddr_sock_ileave_bits != 0 || rule->ddr_die_ileave_bits != 0) {
		dec->dec_fail = ZEN_UMC_DECODE_F_COD_BAD_ILEAVE;
		dec->dec_fail_data = dec->dec_df_ruleno;
		return (B_FALSE);
	}

	switch (rule->ddr_chan_ileave) {
	case DF_CHAN_ILEAVE_COD4_2CH:
		nchan_bit = 1;
		break;
	case DF_CHAN_ILEAVE_COD2_4CH:
		nchan_bit = 2;
		break;
	case DF_CHAN_ILEAVE_COD1_8CH:
		nchan_bit = 3;
		break;
	default:
		dec->dec_fail = ZEN_UMC_DECODE_F_CHAN_ILEAVE_NOTSUP;
		dec->dec_fail_data = rule->ddr_chan_ileave;
		return (B_FALSE);
	}

	dec->dec_ilv_sock = dec->dec_ilv_die = dec->dec_ilv_chan = 0;

	/*
	 * Proceed to calculate the address hash based on the number of bits
	 * that we have been told to use based on the DF rule. Use the flags in
	 * the rule to determine which additional address ranges to hash in.
	 */
	for (uint_t i = 0; i < nchan_bit; i++) {
		uint8_t hash = 0;

		hash = bitx64(dec->dec_ilv_pa, addr_bits[i], addr_bits[i]);
		if ((rule->ddr_flags & DF_DRAM_F_HASH_16_18) != 0) {
			uint8_t val = bitx64(dec->dec_ilv_pa, 16 + i, 16 + i);
			hash ^= val;
		}

		if ((rule->ddr_flags & DF_DRAM_F_HASH_21_23) != 0) {
			uint8_t val = bitx64(dec->dec_ilv_pa, 21 + i, 21 + i);
			hash ^= val;
		}

		if ((rule->ddr_flags & DF_DRAM_F_HASH_30_32) != 0) {
			uint8_t val = bitx64(dec->dec_ilv_pa, 30 + i, 30 + i);
			hash ^= val;
		}

		dec->dec_ilv_chan |= hash << i;
	}

	return (B_TRUE);
}

/*
 * Common logic to perform hashing across the NPS, NPS 1K, and NPS 2K variants.
 */
static void
zen_umc_decode_ileave_nps_common(zen_umc_decoder_t *dec,
    const uint32_t *addr_bits, const uint32_t *adj, uint32_t nsock_bits,
    uint32_t nchan_bits, boolean_t df4p0)
{
	const df_dram_rule_t *rule = dec->dec_df_rule;

	for (uint32_t i = 0; i < nchan_bits + nsock_bits; i++) {
		uint8_t hash = 0;

		hash = bitx64(dec->dec_ilv_pa, addr_bits[i], addr_bits[i]);
		if ((rule->ddr_flags & DF_DRAM_F_HASH_16_18) != 0) {
			uint8_t val = bitx64(dec->dec_ilv_pa, 16 + adj[i],
			    16 + adj[i]);
			hash ^= val;
		}

		if ((rule->ddr_flags & DF_DRAM_F_HASH_21_23) != 0) {
			uint8_t val = bitx64(dec->dec_ilv_pa, 21 + adj[i],
			    21 + adj[i]);
			hash ^= val;
		}

		if ((rule->ddr_flags & DF_DRAM_F_HASH_30_32) != 0) {
			uint8_t val = bitx64(dec->dec_ilv_pa, 30 + adj[i], 30 +
			    adj[i]);
			hash ^= val;
		}

		/*
		 * While 1T is only supported in the NPS 1K/2K variant, rule
		 * normalization means this won't be set in the plain NPS case.
		 */
		if ((rule->ddr_flags & DF_DRAM_F_HASH_40_42) != 0) {
			uint8_t val = bitx64(dec->dec_ilv_pa, 40 + adj[i],
			    40 + adj[i]);
			hash ^= val;
		}

		/*
		 * If this is the first bit and we're not doing socket
		 * interleaving, then we need to add bit 14 to the running hash.
		 * This is only true for a strict DF v4.0 NPS style hash. We
		 * don't perform this for the 1K/2K variant.
		 */
		if (i == 0 && nsock_bits == 0 && df4p0) {
			uint8_t val = bitx64(dec->dec_ilv_pa, 14, 14);
			hash ^= val;
		}

		/*
		 * If socket interleaving is going on we need to store the first
		 * bit as the socket hash and then redirect the remaining bits
		 * to the channel, taking into account that the shift will be
		 * adjusted as a result.
		 */
		if (nsock_bits > 0) {
			if (i == 0) {
				dec->dec_ilv_sock = hash;
			} else {
				dec->dec_ilv_chan |= hash << (i - 1);
			}
		} else {
			dec->dec_ilv_chan |= hash << i;
		}
	}
}


/*
 * This implements the standard NPS hash for power of 2 based channel
 * configurations that is found in DFv4. For more information, please see the
 * interleaving portion of the zen_umc.c big theory statement.
 */
static boolean_t
zen_umc_decode_ileave_nps(const zen_umc_t *umc, zen_umc_decoder_t *dec)
{
	uint32_t nchan_bit, nsock_bit;
	const df_dram_rule_t *rule = dec->dec_df_rule;
	/*
	 * The order of bits here is defined by AMD. Yes, this is start with the
	 * defined address bit and then skip to bit 12.
	 */
	const uint32_t addr_bits[4] = { rule->ddr_addr_start, 12, 13, 14 };
	const uint32_t adj[4] = { 0, 1, 2, 3 };

	if (rule->ddr_die_ileave_bits != 0) {
		dec->dec_fail = ZEN_UMC_DECODE_F_NPS_BAD_ILEAVE;
		dec->dec_fail_data = dec->dec_df_ruleno;
		return (B_FALSE);
	}

	nsock_bit = rule->ddr_sock_ileave_bits;
	switch (rule->ddr_chan_ileave) {
	case DF_CHAN_ILEAVE_NPS4_2CH:
		nchan_bit = 1;
		break;
	case DF_CHAN_ILEAVE_NPS2_4CH:
		nchan_bit = 2;
		break;
	case DF_CHAN_ILEAVE_NPS1_8CH:
		nchan_bit = 3;
		break;
	default:
		dec->dec_fail = ZEN_UMC_DECODE_F_CHAN_ILEAVE_NOTSUP;
		dec->dec_fail_data = rule->ddr_chan_ileave;
		return (B_FALSE);
	}

	ASSERT3U(nchan_bit + nsock_bit, <=, 4);
	dec->dec_ilv_sock = dec->dec_ilv_die = dec->dec_ilv_chan = 0;

	zen_umc_decode_ileave_nps_common(dec, addr_bits, adj, nsock_bit,
	    nchan_bit, B_TRUE);
	return (B_TRUE);
}

/*
 * This implements the Zen 5 (really DF 4D2) NPS variants that work on both 1K
 * and 2K hashing.
 */
static boolean_t
zen_umc_decode_ileave_nps_k(const zen_umc_t *umc, zen_umc_decoder_t *dec)
{
	uint32_t nchan_bit, nsock_bit;
	const df_dram_rule_t *rule = dec->dec_df_rule;
	const uint32_t addr_bits_1k[5] = { rule->ddr_addr_start, 9, 12, 13,
	    14 };
	const uint32_t addr_bits_2k[4] = { rule->ddr_addr_start, 12, 13, 14 };
	const uint32_t adj_1k[5] = { 0, 1, 2, 3, 4 };
	const uint32_t adj_2k[4] = { 0, 2, 3, 4 };
	const uint32_t *addr_bits;
	const uint32_t *adj;

	if (rule->ddr_die_ileave_bits != 0 || rule->ddr_addr_start != 8) {
		dec->dec_fail = ZEN_UMC_DECODE_F_NPS_BAD_ILEAVE;
		dec->dec_fail_data = dec->dec_df_ruleno;
		return (B_FALSE);
	}

	nsock_bit = rule->ddr_sock_ileave_bits;
	switch (rule->ddr_chan_ileave) {
	case DF_CHAN_ILEAVE_NPS4_2CH_1K:
	case DF_CHAN_ILEAVE_NPS4_2CH_2K:
		nchan_bit = 1;
		break;
	case DF_CHAN_ILEAVE_NPS2_4CH_1K:
	case DF_CHAN_ILEAVE_NPS2_4CH_2K:
		nchan_bit = 2;
		break;
	case DF_CHAN_ILEAVE_NPS1_8CH_1K:
	case DF_CHAN_ILEAVE_NPS1_8CH_2K:
		nchan_bit = 3;
		break;
	case DF_CHAN_ILEAVE_NPS1_16CH_1K:
	case DF_CHAN_ILEAVE_NPS1_16CH_2K:
		nchan_bit = 4;
		break;
	default:
		dec->dec_fail = ZEN_UMC_DECODE_F_CHAN_ILEAVE_NOTSUP;
		dec->dec_fail_data = rule->ddr_chan_ileave;
		return (B_FALSE);
	}

	switch (rule->ddr_chan_ileave) {
	case DF_CHAN_ILEAVE_NPS4_2CH_1K:
	case DF_CHAN_ILEAVE_NPS2_4CH_1K:
	case DF_CHAN_ILEAVE_NPS1_8CH_1K:
	case DF_CHAN_ILEAVE_NPS1_16CH_1K:
		ASSERT3U(nchan_bit + nsock_bit, <=, 5);
		addr_bits = addr_bits_1k;
		adj = adj_1k;
		break;
	case DF_CHAN_ILEAVE_NPS4_2CH_2K:
	case DF_CHAN_ILEAVE_NPS2_4CH_2K:
	case DF_CHAN_ILEAVE_NPS1_8CH_2K:
	case DF_CHAN_ILEAVE_NPS1_16CH_2K:
		ASSERT3U(nchan_bit + nsock_bit, <=, 4);
		addr_bits = addr_bits_2k;
		adj = adj_2k;
		break;
	default:
		dec->dec_fail = ZEN_UMC_DECODE_F_CHAN_ILEAVE_NOTSUP;
		dec->dec_fail_data = rule->ddr_chan_ileave;
		return (B_FALSE);
	}

	dec->dec_ilv_sock = dec->dec_ilv_die = dec->dec_ilv_chan = 0;
	zen_umc_decode_ileave_nps_common(dec, addr_bits, adj, nsock_bit,
	    nchan_bit, B_FALSE);
	return (B_TRUE);
}

/*
 * This implements the logic to perform the Zen 3 6ch special hash. It's worth
 * calling out that unlike all other hash functions, this does not support the
 * use of the DF_DRAM_F_HASH_16_18 flag.
 */
static void
zen_umc_decode_hash_zen3_6ch(const df_dram_rule_t *rule, uint64_t pa,
    uint8_t hashes[3])
{
	uint32_t addr_bit = rule->ddr_addr_start;
	/*
	 * Yes, we use these in a weird order. No, there is no 64K.
	 */
	const uint32_t bits_2M[3] = { 23, 21, 22 };
	const uint32_t bits_1G[3] = { 32, 30, 31 };

	hashes[0] = hashes[1] = hashes[2] = 0;
	for (uint_t i = 0; i < ZEN_UMC_COD_NBITS; i++) {
		hashes[i] = bitx64(pa, addr_bit + i, addr_bit + i);
		if (i == 0) {
			uint8_t val = bitx64(pa, addr_bit + 3, addr_bit + 3);
			hashes[i] ^= val;
		}

		if ((rule->ddr_flags & DF_DRAM_F_HASH_21_23) != 0) {
			uint8_t val = bitx64(pa, bits_2M[i], bits_2M[i]);
			hashes[i] ^= val;
		}

		if ((rule->ddr_flags & DF_DRAM_F_HASH_30_32) != 0) {
			uint8_t val = bitx64(pa, bits_1G[i], bits_1G[i]);
			hashes[i] ^= val;
		}
	}
}

/*
 * Perform Zen 3 6-channel hashing. This is pretty weird compared to others. See
 * the zen_umc.c big theory statement for the thorny details.
 */
static boolean_t
zen_umc_decode_ileave_zen3_6ch(const zen_umc_t *umc, zen_umc_decoder_t *dec)
{
	uint8_t hashes[3] = { 0 };
	const df_dram_rule_t *rule = dec->dec_df_rule;
	uint32_t addr_bit = rule->ddr_addr_start;

	if (rule->ddr_sock_ileave_bits != 0 || rule->ddr_die_ileave_bits != 0) {
		dec->dec_fail = ZEN_UMC_DECODE_F_COD_BAD_ILEAVE;
		dec->dec_fail_data = dec->dec_df_ruleno;
		return (B_FALSE);
	}

	zen_umc_decode_hash_zen3_6ch(rule, dec->dec_ilv_pa, hashes);
	dec->dec_ilv_sock = dec->dec_ilv_die = dec->dec_ilv_chan = 0;
	dec->dec_ilv_chan = hashes[0];
	if (hashes[1] == 1 && hashes[2] == 1) {
		uint64_t mod_addr = dec->dec_ilv_pa >> (addr_bit + 3);
		dec->dec_ilv_chan |= (mod_addr % 3) << 1;
	} else {
		dec->dec_ilv_chan |= hashes[1] << 1;
		dec->dec_ilv_chan |= hashes[2] << 2;
	}

	return (B_TRUE);
}

/*
 * This is the standard hash function for the non-power of two based NPS hashes.
 * See the big theory statement for more information. Unlike the normal NPS hash
 * which uses bit 14 conditionally based on socket interleaving, here it is
 * always used.
 */
static void
zen_umc_decode_hash_nps_mod(const df_dram_rule_t *rule, uint64_t pa,
    uint8_t hashes[3])
{
	const uint32_t addr_bits[3] = { rule->ddr_addr_start, 12, 13 };

	for (uint_t i = 0; i < ZEN_UMC_NPS_MOD_NBITS; i++) {
		hashes[i] = bitx64(pa, addr_bits[i], addr_bits[i]);
		if (i == 0) {
			uint8_t val = bitx64(pa, 14, 14);
			hashes[i] ^= val;
		}

		if ((rule->ddr_flags & DF_DRAM_F_HASH_16_18) != 0) {
			uint8_t val = bitx64(pa, 16 + i, 16 + i);
			hashes[i] ^= val;
		}

		if ((rule->ddr_flags & DF_DRAM_F_HASH_21_23) != 0) {
			uint8_t val = bitx64(pa, 21 + i, 21 + i);
			hashes[i] ^= val;
		}

		if ((rule->ddr_flags & DF_DRAM_F_HASH_30_32) != 0) {
			uint8_t val = bitx64(pa, 30 + i, 30 + i);
			hashes[i] ^= val;
		}
	}
}

static void
zen_umc_decode_hash_nps_k_mod(const df_dram_rule_t *rule, uint64_t pa,
    uint8_t hashes[4])
{
	const uint32_t addr_bits[4] = { rule->ddr_addr_start, 9, 12, 13 };

	for (size_t i = 0; i < ARRAY_SIZE(addr_bits); i++) {
		hashes[i] = bitx64(pa, addr_bits[i], addr_bits[i]);
		if (i == 0) {
			uint8_t val = bitx64(pa, 14, 14);
			hashes[i] ^= val;
		}

		if ((rule->ddr_flags & DF_DRAM_F_HASH_16_18) != 0) {
			uint8_t val = bitx64(pa, 16 + i, 16 + i);
			hashes[i] ^= val;
		}

		if ((rule->ddr_flags & DF_DRAM_F_HASH_21_23) != 0) {
			uint8_t val = bitx64(pa, 21 + i, 21 + i);
			hashes[i] ^= val;
		}

		if ((rule->ddr_flags & DF_DRAM_F_HASH_30_32) != 0) {
			uint8_t val = bitx64(pa, 30 + i, 30 + i);
			hashes[i] ^= val;
		}

		if ((rule->ddr_flags & DF_DRAM_F_HASH_40_42) != 0) {
			uint8_t val = bitx64(pa, 40 + i, 40 + i);
			hashes[i] ^= val;
		}
	}
}

/*
 * See the big theory statement in zen_umc.c which describes the rules for this
 * computation. This is a little less weird than the Zen 3 one, but still,
 * unique.
 */
static boolean_t
zen_umc_decode_ileave_nps_mod(const zen_umc_t *umc, zen_umc_decoder_t *dec)
{
	uint8_t hashes[3] = { 0 };
	uint32_t nsock_bit, chan_mod;
	const df_dram_rule_t *rule = dec->dec_df_rule;

	if (rule->ddr_die_ileave_bits != 0) {
		dec->dec_fail = ZEN_UMC_DECODE_F_NPS_BAD_ILEAVE;
		dec->dec_fail_data = dec->dec_df_ruleno;
		return (B_FALSE);
	}

	nsock_bit = rule->ddr_sock_ileave_bits;
	switch (rule->ddr_chan_ileave) {
	case DF_CHAN_ILEAVE_NPS4_3CH:
	case DF_CHAN_ILEAVE_NPS2_6CH:
	case DF_CHAN_ILEAVE_NPS1_12CH:
		chan_mod = 3;
		break;
	case DF_CHAN_ILEAVE_NPS2_5CH:
	case DF_CHAN_ILEAVE_NPS1_10CH:
		chan_mod = 5;
		break;
	default:
		dec->dec_fail = ZEN_UMC_DECODE_F_CHAN_ILEAVE_NOTSUP;
		dec->dec_fail_data = rule->ddr_chan_ileave;
		return (B_FALSE);
	}

	dec->dec_ilv_sock = dec->dec_ilv_die = dec->dec_ilv_chan = 0;
	zen_umc_decode_hash_nps_mod(rule, dec->dec_ilv_pa, hashes);

	if (nsock_bit > 0) {
		ASSERT3U(nsock_bit, ==, 1);
		dec->dec_ilv_sock = hashes[0];
	}

	dec->dec_ilv_chan = bitx64(dec->dec_ilv_pa, 63, 14) % chan_mod;
	if (hashes[0] == 1) {
		dec->dec_ilv_chan = (dec->dec_ilv_chan + 1) % chan_mod;
	}

	/*
	 * Use the remaining hash bits based on the number of channels. There is
	 * nothing else to do for 3/5 channel configs.
	 */
	switch (rule->ddr_chan_ileave) {
	case DF_CHAN_ILEAVE_NPS4_3CH:
	case DF_CHAN_ILEAVE_NPS2_5CH:
		break;
	case DF_CHAN_ILEAVE_NPS2_6CH:
	case DF_CHAN_ILEAVE_NPS1_10CH:
		dec->dec_ilv_chan += hashes[2] * chan_mod;
		break;
	case DF_CHAN_ILEAVE_NPS1_12CH:
		dec->dec_ilv_chan += ((hashes[2] << 1) | hashes[1]) * chan_mod;
		break;
	default:
		dec->dec_fail = ZEN_UMC_DECODE_F_CHAN_ILEAVE_NOTSUP;
		dec->dec_fail_data = rule->ddr_chan_ileave;
		return (B_FALSE);
	}

	return (B_TRUE);
}

/*
 * Determine the interleave address for the NPS 1K/2K non-power of 2 based
 * values. Each of these uses a similar style of calculation with rather
 * different values and as such we use a data table for each of these that maps
 * to a given rule.
 */
static boolean_t
zen_umc_decode_ileave_nps_k_mod(const zen_umc_t *umc, zen_umc_decoder_t *dec)
{
	uint8_t hashes[4] = { 0 };
	uint32_t chan, mod_val;
	uint64_t mod_addr;
	const df_dram_rule_t *rule = dec->dec_df_rule;
	const zen_umc_np2_k_rule_t *np2 = NULL;

	for (size_t i = 0; i < ARRAY_SIZE(zen_umc_np2_k_rules); i++) {
		if (rule->ddr_chan_ileave == zen_umc_np2_k_rules[i].zukr_type) {
			np2 = &zen_umc_np2_k_rules[i];
			break;
		}
	}

	if (np2 == NULL) {
		dec->dec_fail = ZEN_UMC_DECODE_F_CHAN_ILEAVE_NOTSUP;
		dec->dec_fail_data = rule->ddr_chan_ileave;
		return (B_FALSE);
	}

	if (rule->ddr_die_ileave_bits != 0 || rule->ddr_addr_start != 8) {
		dec->dec_fail = ZEN_UMC_DECODE_F_NPS_BAD_ILEAVE;
		dec->dec_fail_data = dec->dec_df_ruleno;
		return (B_FALSE);
	}

	/*
	 * These rules either require that socket interleaving is enabled or
	 * not. Make sure that this matches before we proceed.
	 */
	if (np2->zukr_sock != (rule->ddr_sock_ileave_bits == 1)) {
		dec->dec_fail = ZEN_UMC_DECODE_F_NPS_BAD_ILEAVE;
		dec->dec_fail_data = dec->dec_df_ruleno;
		return (B_FALSE);
	}

	dec->dec_ilv_sock = dec->dec_ilv_die = dec->dec_ilv_chan = 0;
	zen_umc_decode_hash_nps_k_mod(rule, dec->dec_ilv_pa, hashes);
	if (rule->ddr_sock_ileave_bits > 0) {
		ASSERT3U(rule->ddr_sock_ileave_bits, ==, 1);
		dec->dec_ilv_sock = hashes[0];
	}

	mod_addr = bitx64(dec->dec_ilv_pa, 63, np2->zukr_high);
	mod_addr = mod_addr << np2->zukr_mod_shift;
	for (uint32_t i = 0; i < np2->zukr_mod_shift; i++) {
		uint32_t bit = np2->zukr_mod_fill[i];
		if (bit != 0) {
			uint64_t val = bitx64(dec->dec_ilv_pa, bit, bit);
			mod_addr = bitset64(mod_addr, i, i, val);
		}
	}

	mod_val = (uint32_t)(mod_addr % np2->zukr_mod);
	chan = mod_val << np2->zukr_chan_mod_shift;
	for (uint32_t i = 0; i < np2->zukr_chan_mod_shift; i++) {
		VERIFY3U(np2->zukr_chan_fill[i], <, ARRAY_SIZE(hashes));
		uint32_t bit = np2->zukr_chan_fill[i];
		uint32_t val = hashes[np2->zukr_chan_fill[i]];
		chan = bitset32(chan, bit, bit, val);
	}

	dec->dec_ilv_chan = chan;
	return (B_TRUE);
}

/*
 * Our next task is to attempt to translate the PA and the DF rule from a system
 * address into a normalized address and a particular DRAM channel that it's
 * targeting. There are several things that we need to take into account here
 * when performing interleaving and translation:
 *
 *  o The DRAM Hole modifying our base address
 *  o The various interleave bits
 *  o Potentially hashing based on channel and global settings
 *  o Potential CS re-targeting registers (only on some systems)
 *  o Finally, the question of how to adjust for the DRAM hole and the base
 *    address changes based on the DF generation and channel configuration. This
 *    influences what address we start interleaving with.
 *
 * Note, this phase does not actually construct the normalized (e.g. channel)
 * address. That's done in a subsequent step. For more background, please see
 * the 'Data Fabric Interleaving' section of the zen_umc.c big theory statement.
 */
static boolean_t
zen_umc_decode_sysaddr_to_csid(const zen_umc_t *umc, zen_umc_decoder_t *dec)
{
	uint32_t sock, die, chan, remap_ruleset;
	const df_dram_rule_t *rule = dec->dec_df_rule;
	const zen_umc_cs_remap_t *remap;

	/*
	 * First, we must determine what the actual address used for
	 * interleaving is. This varies based on the interleaving and DF
	 * generation.
	 */
	if (!zen_umc_determine_ileave_addr(umc, dec)) {
		return (B_FALSE);
	}

	switch (rule->ddr_chan_ileave) {
	case DF_CHAN_ILEAVE_1CH:
	case DF_CHAN_ILEAVE_2CH:
	case DF_CHAN_ILEAVE_4CH:
	case DF_CHAN_ILEAVE_8CH:
	case DF_CHAN_ILEAVE_16CH:
	case DF_CHAN_ILEAVE_32CH:
		if (!zen_umc_decode_ileave_nohash(umc, dec)) {
			return (B_FALSE);
		}
		break;
	case DF_CHAN_ILEAVE_COD4_2CH:
	case DF_CHAN_ILEAVE_COD2_4CH:
	case DF_CHAN_ILEAVE_COD1_8CH:
		if (!zen_umc_decode_ileave_cod(umc, dec)) {
			return (B_FALSE);
		}
		break;
	case DF_CHAN_ILEAVE_NPS4_2CH:
	case DF_CHAN_ILEAVE_NPS2_4CH:
	case DF_CHAN_ILEAVE_NPS1_8CH:
		if (!zen_umc_decode_ileave_nps(umc, dec)) {
			return (B_FALSE);
		}
		break;
	case DF_CHAN_ILEAVE_6CH:
		if (!zen_umc_decode_ileave_zen3_6ch(umc, dec)) {
			return (B_FALSE);
		}
		break;
	case DF_CHAN_ILEAVE_NPS4_3CH:
	case DF_CHAN_ILEAVE_NPS2_6CH:
	case DF_CHAN_ILEAVE_NPS1_12CH:
	case DF_CHAN_ILEAVE_NPS2_5CH:
	case DF_CHAN_ILEAVE_NPS1_10CH:
		if (!zen_umc_decode_ileave_nps_mod(umc, dec)) {
			return (B_FALSE);
		}
		break;
	case DF_CHAN_ILEAVE_NPS4_2CH_1K:
	case DF_CHAN_ILEAVE_NPS2_4CH_1K:
	case DF_CHAN_ILEAVE_NPS1_8CH_1K:
	case DF_CHAN_ILEAVE_NPS1_16CH_1K:
	case DF_CHAN_ILEAVE_NPS4_2CH_2K:
	case DF_CHAN_ILEAVE_NPS2_4CH_2K:
	case DF_CHAN_ILEAVE_NPS1_8CH_2K:
	case DF_CHAN_ILEAVE_NPS1_16CH_2K:
		if (!zen_umc_decode_ileave_nps_k(umc, dec)) {
			return (B_FALSE);
		}
		break;
	case DF_CHAN_ILEAVE_NPS4_3CH_1K:
	case DF_CHAN_ILEAVE_NPS2_6CH_1K:
	case DF_CHAN_ILEAVE_NPS1_12CH_1K:
	case DF_CHAN_ILEAVE_NPS0_24CH_1K:
	case DF_CHAN_ILEAVE_NPS2_5CH_1K:
	case DF_CHAN_ILEAVE_NPS1_10CH_1K:
	case DF_CHAN_ILEAVE_NPS4_3CH_2K:
	case DF_CHAN_ILEAVE_NPS2_6CH_2K:
	case DF_CHAN_ILEAVE_NPS1_12CH_2K:
	case DF_CHAN_ILEAVE_NPS0_24CH_2K:
	case DF_CHAN_ILEAVE_NPS2_5CH_2K:
	case DF_CHAN_ILEAVE_NPS1_10CH_2K:
		if (!zen_umc_decode_ileave_nps_k_mod(umc, dec)) {
			return (B_FALSE);
		}
		break;
	case DF_CHAN_ILEAVE_MI3H_8CH:
	case DF_CHAN_ILEAVE_MI3H_16CH:
	case DF_CHAN_ILEAVE_MI3H_32CH:
	default:
		dec->dec_fail = ZEN_UMC_DECODE_F_CHAN_ILEAVE_NOTSUP;
		dec->dec_fail_data = rule->ddr_chan_ileave;
		return (B_FALSE);
	}

	/*
	 * At this point we have dealt with decoding the interleave into the
	 * logical elements that it contains. We need to transform that back
	 * into a fabric ID, so we can add it to the base fabric ID in our rule.
	 * After that, we need to see if there is any CS remapping going on. If
	 * there is, we will replace the component part of the decomposed fabric
	 * ID. With that done, we can then transform the components back into
	 * our target fabric ID, which indicates which UMC we're after.
	 */
	zen_fabric_id_compose(&umc->umc_decomp, dec->dec_ilv_sock,
	    dec->dec_ilv_die, dec->dec_ilv_chan, &dec->dec_ilv_fabid);
	dec->dec_log_fabid = dec->dec_ilv_fabid + rule->ddr_dest_fabid;

	/*
	 * If there's no remapping to do, then we're done. Simply assign the
	 * logical ID as our target.
	 */
	zen_fabric_id_decompose(&umc->umc_decomp, dec->dec_log_fabid, &sock,
	    &die, &chan);
	if ((rule->ddr_flags & DF_DRAM_F_REMAP_EN) == 0) {
		dec->dec_targ_fabid = dec->dec_log_fabid;
		return (B_TRUE);
	}

	/*
	 * The DF contains multiple remapping tables. We must figure out which
	 * of these to actually use. There are two different ways that this can
	 * work. The first way is the one added in DFv4 and is used since then.
	 * In that case, the DRAM rule includes both that remapping was enabled
	 * and which of the multiple mapping tables to use.
	 *
	 * This feature also exists prior to DFv4, but only in Milan. In that
	 * world, indicated by the DF_DRAM_F_REMAP_SOCK flag, there is one table
	 * in each DF per-socket. Based on the destination socket from the data
	 * fabric ID, you pick the actual table to use.
	 *
	 * Once the table has been selected, we maintain the socket and die
	 * portions of the fabric ID as constants and replace the component with
	 * the one the remapping table indicates.
	 *
	 * Technically each DF has its own copy of the remapping tables. To make
	 * this work we rely on the following assumption: a given DF node has to
	 * be able to fully route all DRAM rules to a target. That is, a given
	 * DF node doesn't really forward a system address to the remote die for
	 * further interleave processing and therefore we must have enough
	 * information here to map it totally from the same DF that we got the
	 * CCM rules from in the first place, DF 0.
	 */
	if ((rule->ddr_flags & DF_DRAM_F_REMAP_SOCK) != 0) {
		remap_ruleset = sock;
	} else {
		remap_ruleset = rule->ddr_remap_ent;
	}

	if (remap_ruleset >= dec->dec_df_rulesrc->zud_cs_nremap) {
		dec->dec_fail = ZEN_UMC_DECODE_F_BAD_REMAP_SET;
		dec->dec_fail_data = remap_ruleset;
		return (B_FALSE);
	}

	remap = &dec->dec_df_rulesrc->zud_remap[remap_ruleset];
	if (chan >= remap->csr_nremaps) {
		dec->dec_fail = ZEN_UMC_DECODE_F_BAD_REMAP_ENTRY;
		dec->dec_fail_data = chan;
		return (B_FALSE);
	}

	dec->dec_remap_comp = remap->csr_remaps[chan];
	if ((dec->dec_remap_comp & ~umc->umc_decomp.dfd_comp_mask) != 0) {
		dec->dec_fail = ZEN_UMC_DECODE_F_REMAP_HAS_BAD_COMP;
		dec->dec_fail_data = dec->dec_remap_comp;
		return (B_FALSE);
	}

	zen_fabric_id_compose(&umc->umc_decomp, sock, die, dec->dec_remap_comp,
	    &dec->dec_targ_fabid);

	return (B_TRUE);
}

/*
 * Our next step here is to actually take our target ID and find the
 * corresponding DF, UMC, and actual rule that was used. Note, we don't
 * decompose the ID and look things up that way for a few reasons. While each
 * UMC should map linearly to its instance/component ID, there are suggestions
 * that they can be renumbered. This makes it simplest to just walk over
 * everything (and there aren't that many things to walk over either).
 */
static boolean_t
zen_umc_decode_find_umc_rule(const zen_umc_t *umc, zen_umc_decoder_t *dec)
{
	for (uint_t dfno = 0; dfno < umc->umc_ndfs; dfno++) {
		const zen_umc_df_t *df = &umc->umc_dfs[dfno];
		for (uint_t umcno = 0; umcno < df->zud_nchan; umcno++) {
			const zen_umc_chan_t *chan = &df->zud_chan[umcno];

			if (chan->chan_fabid != dec->dec_targ_fabid) {
				continue;
			}

			/*
			 * At this point we have found the UMC that we were
			 * looking for. Snapshot that and then figure out which
			 * rule index of it corresponds to our mapping so we can
			 * properly determine an offset. We will still use the
			 * primary CCM rule for all other calculations.
			 */
			dec->dec_umc_chan = chan;
			for (uint32_t ruleno = 0; ruleno < chan->chan_nrules;
			    ruleno++) {
				const df_dram_rule_t *rule =
				    &chan->chan_rules[ruleno];
				if ((rule->ddr_flags & DF_DRAM_F_VALID) == 0) {
					continue;
				}

				if (dec->dec_pa >= rule->ddr_base &&
				    dec->dec_pa < rule->ddr_limit) {
					dec->dec_umc_ruleno = ruleno;
					return (B_TRUE);
				}
			}

			dec->dec_fail = ZEN_UMC_DECODE_F_UMC_DOESNT_HAVE_PA;
			return (B_FALSE);
		}
	}

	dec->dec_fail = ZEN_UMC_DECODE_F_CANNOT_MAP_FABID;
	return (B_FALSE);
}

/*
 * Non-hashing interleave modes system address normalization logic. See the
 * zen_umc.c big theory statement for more information.
 */
static boolean_t
zen_umc_decode_normalize_nohash(const zen_umc_t *umc, zen_umc_decoder_t *dec)
{
	uint_t nbits = 0;
	const df_dram_rule_t *rule = dec->dec_df_rule;

	nbits += rule->ddr_sock_ileave_bits;
	nbits += rule->ddr_die_ileave_bits;
	switch (rule->ddr_chan_ileave) {
	case DF_CHAN_ILEAVE_1CH:
		break;
	case DF_CHAN_ILEAVE_2CH:
		nbits += 1;
		break;
	case DF_CHAN_ILEAVE_4CH:
		nbits += 2;
		break;
	case DF_CHAN_ILEAVE_8CH:
		nbits += 3;
		break;
	case DF_CHAN_ILEAVE_16CH:
		nbits += 4;
		break;
	case DF_CHAN_ILEAVE_32CH:
		nbits += 5;
		break;
	default:
		dec->dec_fail = ZEN_UMC_DECODE_F_CHAN_ILEAVE_NOTSUP;
		dec->dec_fail_data = rule->ddr_chan_ileave;
		return (B_FALSE);
	}

	/*
	 * If we have a really simple configuration (e.g. no interleaving at
	 * all), then make sure that we do not actually do anything here.
	 */
	if (nbits > 0) {
		dec->dec_norm_addr = bitdel64(dec->dec_norm_addr,
		    rule->ddr_addr_start + nbits - 1, rule->ddr_addr_start);
	}

	return (B_TRUE);
}

/*
 * COD/NPS system address normalization logic. See the zen_umc.c big theory
 * statement for more information.
 */
static boolean_t
zen_umc_decode_normalize_hash(const zen_umc_t *umc, zen_umc_decoder_t *dec)
{
	uint_t nbits = 0, nstart = 0;
	const df_dram_rule_t *rule = dec->dec_df_rule;

	/*
	 * NPS 1K hashes remove bits 8 and 9 first. Determine how many bits to
	 * remove from the starting location. This will later be reduced based
	 * upon how many address bits there actually are.
	 */
	switch (rule->ddr_chan_ileave) {
	case DF_CHAN_ILEAVE_NPS4_2CH_1K:
	case DF_CHAN_ILEAVE_NPS2_4CH_1K:
	case DF_CHAN_ILEAVE_NPS1_8CH_1K:
	case DF_CHAN_ILEAVE_NPS1_16CH_1K:
		nstart = 2;
		break;
	default:
		nstart = 1;
		break;
	}

	/*
	 * NPS hashes allow for socket interleaving, COD hashes do not. Add
	 * socket interleaving, skip die.
	 */
	nbits += rule->ddr_sock_ileave_bits;
	switch (rule->ddr_chan_ileave) {
	case DF_CHAN_ILEAVE_COD4_2CH:
	case DF_CHAN_ILEAVE_NPS4_2CH:
	case DF_CHAN_ILEAVE_NPS4_2CH_1K:
	case DF_CHAN_ILEAVE_NPS4_2CH_2K:
		nbits += 1;
		break;
	case DF_CHAN_ILEAVE_COD2_4CH:
	case DF_CHAN_ILEAVE_NPS2_4CH:
	case DF_CHAN_ILEAVE_NPS2_4CH_1K:
	case DF_CHAN_ILEAVE_NPS2_4CH_2K:
		nbits += 2;
		break;
	case DF_CHAN_ILEAVE_COD1_8CH:
	case DF_CHAN_ILEAVE_NPS1_8CH:
	case DF_CHAN_ILEAVE_NPS1_8CH_1K:
	case DF_CHAN_ILEAVE_NPS1_8CH_2K:
		nbits += 3;
		break;
	case DF_CHAN_ILEAVE_NPS1_16CH_1K:
	case DF_CHAN_ILEAVE_NPS1_16CH_2K:
		nbits += 4;
		break;
	default:
		dec->dec_fail = ZEN_UMC_DECODE_F_CHAN_ILEAVE_NOTSUP;
		dec->dec_fail_data = rule->ddr_chan_ileave;
	}

	/*
	 * Don't remove more bits from the start than exist.
	 */
	if (nstart > nbits) {
		nstart = nbits;
	}

	/*
	 * Always remove high order bits before low order bits so we don't have
	 * to adjust the bits we need to remove.
	 */
	if (nbits > nstart) {
		uint_t start = 12;
		uint_t end = start + (nbits - nstart - 1);
		dec->dec_norm_addr = bitdel64(dec->dec_norm_addr, end, start);
	}

	dec->dec_norm_addr = bitdel64(dec->dec_norm_addr,
	    rule->ddr_addr_start + nstart - 1, rule->ddr_addr_start);
	return (B_TRUE);
}

/*
 * Now it's time to perform normalization of our favorite interleaving type.
 * Please see the comments in zen_umc.c on this to understand what we're doing
 * here and why.
 */
static boolean_t
zen_umc_decode_normalize_zen3_6ch(const zen_umc_t *umc, zen_umc_decoder_t *dec)
{
	uint8_t hashes[3] = { 0 };
	uint_t start, end;
	const df_dram_rule_t *rule = dec->dec_df_rule;

	/*
	 * As per the theory statement, we always remove the hash bits here from
	 * the starting address. Because this is a 6-channel config, that turns
	 * into 3. Perform the hash again first.
	 */
	zen_umc_decode_hash_zen3_6ch(rule, dec->dec_norm_addr, hashes);
	start = rule->ddr_addr_start;
	end = rule->ddr_addr_start + ZEN_UMC_COD_NBITS - 1;
	dec->dec_norm_addr = bitdel64(dec->dec_norm_addr, end, start);

	/*
	 * This is the case the theory statement warned about. This gets
	 * normalized to the top of the DIMM's range (its two upper most bits
	 * are set).
	 */
	if (hashes[1] == 1 && hashes[2] == 1) {
		uint_t start = 14 - ZEN_UMC_COD_NBITS +
		    dec->dec_umc_chan->chan_np2_space0;
		dec->dec_norm_addr = bitset64(dec->dec_norm_addr, start + 1,
		    start, 0x3);
	}

	return (B_TRUE);
}

/*
 * Based on the algorithm of sorts described in zen_umc.c, we have a few
 * different phases of extraction and combination. This isn't quite like the
 * others where we simply delete bits.
 */
static boolean_t
zen_umc_decode_normalize_nps_mod(const zen_umc_t *umc, zen_umc_decoder_t *dec)
{
	uint64_t low, high, mid;
	uint_t nbits, chan_mod, sock_bits, nmid_bits;
	uint_t mid_start, mid_end;
	uint8_t hashes[3] = { 0 };
	const df_dram_rule_t *rule = dec->dec_df_rule;

	sock_bits = rule->ddr_sock_ileave_bits;
	switch (rule->ddr_chan_ileave) {
	case DF_CHAN_ILEAVE_NPS4_3CH:
		chan_mod = 3;
		nbits = 1;
		break;
	case DF_CHAN_ILEAVE_NPS2_5CH:
		chan_mod = 5;
		nbits = 1;
		break;
	case DF_CHAN_ILEAVE_NPS2_6CH:
		chan_mod = 3;
		nbits = 2;
		break;
	case DF_CHAN_ILEAVE_NPS1_10CH:
		chan_mod = 5;
		nbits = 2;
		break;
	case DF_CHAN_ILEAVE_NPS1_12CH:
		chan_mod = 3;
		nbits = 3;
		break;
	default:
		dec->dec_fail = ZEN_UMC_DECODE_F_CHAN_ILEAVE_NOTSUP;
		dec->dec_fail_data = rule->ddr_chan_ileave;
		return (B_FALSE);
	}

	/*
	 * First extract the low bit range that we're using which is everything
	 * below the starting interleave address. We also always extract the
	 * high bits, which are always [63:14] and divide it by the modulus.
	 * Note, we apply the hash after any such division if needed. It becomes
	 * the new least significant bit.
	 */
	low = bitx64(dec->dec_norm_addr, rule->ddr_addr_start - 1, 0);
	high = bitx64(dec->dec_norm_addr, 63, 14) / chan_mod;
	zen_umc_decode_hash_nps_mod(rule, dec->dec_norm_addr, hashes);
	if (sock_bits == 0) {
		high = (high << 1) | hashes[0];
	}

	/*
	 * Now for the weirdest bit here, extracting the middle bits. Recall
	 * this hash uses bit 8, then 13, then 12 (the hash order is still 8,
	 * 12, 13, but it uses the hashes[2] before hashes[1] in
	 * zen_umc_decode_ileave_nps_mod()). So if we're only using 1 interleave
	 * bit, we just remove bit 8 (assuming that is our starting address) and
	 * our range is [13:9]. If we're using two, our range becomes [12:9],
	 * and if three, [11:9]. The 6 - nbits below comes from the fact that in
	 * a 1 bit interleave we have 5 bits. Because our mid_start/mid_end
	 * range is inclusive, we subtract one at the end from mid_end.
	 */
	nmid_bits = 6 - nbits;
	mid_start = rule->ddr_addr_start + 1;
	mid_end = mid_start + nmid_bits - 1;
	mid = bitx64(dec->dec_norm_addr, mid_end, mid_start);

	/*
	 * Because we've been removing bits, we don't use any of the start and
	 * ending ranges we calculated above for shifts, as that was what we
	 * needed from the original address.
	 */
	dec->dec_norm_addr = low | (mid << rule->ddr_addr_start) | (high <<
	    (rule->ddr_addr_start + nmid_bits));

	return (B_TRUE);
}

/*
 * Construct the normalized address for the NPS 1K/2K non-power of 2 instances.
 * See the theory statement for the rough formula used here. While each variant
 * uses slightly different values, that has been abstracted based on our data
 * table.
 */
static boolean_t
zen_umc_decode_normalize_nps_k_mod(const zen_umc_t *umc, zen_umc_decoder_t *dec)
{
	uint64_t high, mid, low;
	uint_t mid_end;
	const df_dram_rule_t *rule = dec->dec_df_rule;
	const zen_umc_np2_k_rule_t *np2 = NULL;

	for (size_t i = 0; i < ARRAY_SIZE(zen_umc_np2_k_rules); i++) {
		if (rule->ddr_chan_ileave == zen_umc_np2_k_rules[i].zukr_type) {
			np2 = &zen_umc_np2_k_rules[i];
			break;
		}
	}

	if (np2 == NULL) {
		dec->dec_fail = ZEN_UMC_DECODE_F_CHAN_ILEAVE_NOTSUP;
		dec->dec_fail_data = rule->ddr_chan_ileave;
		return (B_FALSE);
	}

	low = bitx64(dec->dec_norm_addr, rule->ddr_addr_start - 1, 0);
	mid_end = np2->zukr_norm_addr + np2->zukr_norm_naddr - 1;
	VERIFY3U(mid_end, >=, rule->ddr_addr_start);
	mid = bitx64(dec->dec_norm_addr, mid_end, np2->zukr_norm_addr);

	high = bitx64(dec->dec_norm_addr, 63, np2->zukr_high);
	if (np2->zukr_div_naddr > 0) {
		uint_t ins_end = np2->zukr_div_addr + np2->zukr_div_naddr - 1;
		uint64_t insert = bitx64(dec->dec_norm_addr, ins_end,
		    np2->zukr_div_addr);

		high = high << np2->zukr_div_naddr;
		high = bitset64(high, np2->zukr_div_naddr - 1, 0, insert);
	}
	high = high / np2->zukr_mod;

	dec->dec_norm_addr = low | (mid << rule->ddr_addr_start) | (high <<
	    (rule->ddr_addr_start + np2->zukr_norm_naddr));
	return (B_TRUE);
}

/*
 * Now we need to go through and try to construct a normalized address using all
 * the information that we've gathered to date. To do this we need to take into
 * account all of the following transformations on the address that need to
 * occur. We apply modifications to the address in the following order:
 *
 *   o The base address of the rule
 *   o DRAM hole changes
 *   o Normalization of the address due to interleaving (more fun)
 *   o The DRAM offset register of the rule
 */
static boolean_t
zen_umc_decode_sysaddr_to_norm(const zen_umc_t *umc, zen_umc_decoder_t *dec)
{
	const zen_umc_chan_t *chan = dec->dec_umc_chan;
	const df_dram_rule_t *rule = dec->dec_df_rule;

	dec->dec_norm_addr = dec->dec_pa;
	if (!zen_umc_adjust_dram_addr(umc, dec, &dec->dec_norm_addr,
	    ZEN_UMC_DECODE_F_CALC_NORM_UNDERFLOW)) {
		return (B_FALSE);
	}

	/*
	 * Now for the most annoying part of this whole thing, normalizing based
	 * on our actual interleave format. The reason for this is that when
	 * interleaving is going on, it actually is removing bits that are just
	 * being used to direct it somewhere; however, it's actually generally
	 * speaking the same value in each location. See the big theory
	 * statement in zen_umc.c for more information.
	 */
	switch (rule->ddr_chan_ileave) {
	case DF_CHAN_ILEAVE_1CH:
	case DF_CHAN_ILEAVE_2CH:
	case DF_CHAN_ILEAVE_4CH:
	case DF_CHAN_ILEAVE_8CH:
	case DF_CHAN_ILEAVE_16CH:
	case DF_CHAN_ILEAVE_32CH:
		if (!zen_umc_decode_normalize_nohash(umc, dec)) {
			return (B_FALSE);
		}
		break;
	case DF_CHAN_ILEAVE_COD4_2CH:
	case DF_CHAN_ILEAVE_COD2_4CH:
	case DF_CHAN_ILEAVE_COD1_8CH:
	case DF_CHAN_ILEAVE_NPS4_2CH:
	case DF_CHAN_ILEAVE_NPS2_4CH:
	case DF_CHAN_ILEAVE_NPS1_8CH:
	case DF_CHAN_ILEAVE_NPS4_2CH_1K:
	case DF_CHAN_ILEAVE_NPS2_4CH_1K:
	case DF_CHAN_ILEAVE_NPS1_8CH_1K:
	case DF_CHAN_ILEAVE_NPS1_16CH_1K:
	case DF_CHAN_ILEAVE_NPS4_2CH_2K:
	case DF_CHAN_ILEAVE_NPS2_4CH_2K:
	case DF_CHAN_ILEAVE_NPS1_8CH_2K:
	case DF_CHAN_ILEAVE_NPS1_16CH_2K:
		if (!zen_umc_decode_normalize_hash(umc, dec)) {
			return (B_FALSE);
		}
		break;
	case DF_CHAN_ILEAVE_6CH:
		if (!zen_umc_decode_normalize_zen3_6ch(umc, dec)) {
			return (B_FALSE);
		}
		break;
	case DF_CHAN_ILEAVE_NPS4_3CH:
	case DF_CHAN_ILEAVE_NPS2_6CH:
	case DF_CHAN_ILEAVE_NPS1_12CH:
	case DF_CHAN_ILEAVE_NPS2_5CH:
	case DF_CHAN_ILEAVE_NPS1_10CH:
		if (!zen_umc_decode_normalize_nps_mod(umc, dec)) {
			return (B_FALSE);
		}
		break;
	case DF_CHAN_ILEAVE_NPS4_3CH_1K:
	case DF_CHAN_ILEAVE_NPS2_6CH_1K:
	case DF_CHAN_ILEAVE_NPS1_12CH_1K:
	case DF_CHAN_ILEAVE_NPS0_24CH_1K:
	case DF_CHAN_ILEAVE_NPS2_5CH_1K:
	case DF_CHAN_ILEAVE_NPS1_10CH_1K:
	case DF_CHAN_ILEAVE_NPS4_3CH_2K:
	case DF_CHAN_ILEAVE_NPS2_6CH_2K:
	case DF_CHAN_ILEAVE_NPS1_12CH_2K:
	case DF_CHAN_ILEAVE_NPS0_24CH_2K:
	case DF_CHAN_ILEAVE_NPS2_5CH_2K:
	case DF_CHAN_ILEAVE_NPS1_10CH_2K:
		if (!zen_umc_decode_normalize_nps_k_mod(umc, dec)) {
			return (B_FALSE);
		}
		break;
	case DF_CHAN_ILEAVE_MI3H_8CH:
	case DF_CHAN_ILEAVE_MI3H_16CH:
	case DF_CHAN_ILEAVE_MI3H_32CH:
	default:
		dec->dec_fail = ZEN_UMC_DECODE_F_CHAN_ILEAVE_NOTSUP;
		dec->dec_fail_data = rule->ddr_chan_ileave;
		return (B_FALSE);
	}

	/*
	 * Determine if this rule has an offset to apply. Note, there is never
	 * an offset for rule 0, hence the index into this is one less than the
	 * actual rule number. Unlike other transformations these offsets
	 * describe the start of a normalized range. Therefore we need to
	 * actually add this value instead of subtract.
	 */
	if (dec->dec_umc_ruleno > 0) {
		uint32_t offno = dec->dec_umc_ruleno - 1;
		const chan_offset_t *offset = &chan->chan_offsets[offno];

		if (offset->cho_valid) {
			dec->dec_norm_addr += offset->cho_offset;
		}
	}

	return (B_TRUE);
}

/*
 * This applies the formula that determines a chip-select actually matches which
 * is defined as (address & ~mask) == (base & ~mask) in the PPR. There is both a
 * primary and secondary mask here. We need to pay attention to which is used
 * (if any) for later on.
 */
static boolean_t
zen_umc_decoder_cs_matches(const umc_cs_t *cs, const uint64_t norm,
    boolean_t *matched_sec)
{
	if ((cs->ucs_flags & UMC_CS_F_DECODE_EN) == 0) {
		return (B_FALSE);
	}

	if (cs->ucs_base.udb_valid != 0) {
		uint64_t imask = ~cs->ucs_base_mask;
		if ((norm & imask) == (cs->ucs_base.udb_base & imask)) {
			*matched_sec = B_FALSE;
			return (B_TRUE);
		}
	}

	if (cs->ucs_sec.udb_valid != 0) {
		uint64_t imask = ~cs->ucs_sec_mask;
		if ((norm & imask) == (cs->ucs_sec.udb_base & imask)) {
			*matched_sec = B_TRUE;
			return (B_TRUE);
		}
	}

	return (B_FALSE);
}

/*
 * Go through with our normalized address and map it to a given chip-select.
 * This as a side effect indicates which DIMM we're going out on as well. Note,
 * the final DIMM can change due to chip-select hashing; however, we use this
 * DIMM for determining all of the actual address translations.
 */
static boolean_t
zen_umc_decode_find_cs(const zen_umc_t *umc, zen_umc_decoder_t *dec)
{
	const zen_umc_chan_t *chan = dec->dec_umc_chan;

	for (uint_t dimmno = 0; dimmno < ZEN_UMC_MAX_DIMMS; dimmno++) {
		const umc_dimm_t *dimm = &chan->chan_dimms[dimmno];

		if ((dimm->ud_flags & UMC_DIMM_F_VALID) == 0)
			continue;

		for (uint_t csno = 0; csno < ZEN_UMC_MAX_CS_PER_DIMM; csno++) {
			const umc_cs_t *cs = &dimm->ud_cs[csno];
			boolean_t is_sec = B_FALSE;

			if (zen_umc_decoder_cs_matches(cs, dec->dec_norm_addr,
			    &is_sec)) {
				dec->dec_dimm = dimm;
				dec->dec_cs = cs;
				dec->dec_log_csno = dimmno * ZEN_UMC_MAX_DIMMS +
				    csno;
				dec->dec_cs_sec = is_sec;
				return (B_TRUE);
			}
		}
	}

	dec->dec_fail = ZEN_UMC_DECODE_F_NO_CS_BASE_MATCH;
	return (B_FALSE);
}

/*
 * Extract the column from the address. For once, something that is almost
 * straightforward.
 */
static boolean_t
zen_umc_decode_cols(const zen_umc_t *umc, zen_umc_decoder_t *dec)
{
	uint32_t cols = 0;
	const umc_cs_t *cs = dec->dec_cs;

	for (uint_t i = 0; i < cs->ucs_ncol; i++) {
		uint32_t index;

		index = cs->ucs_col_bits[i];
		cols |= bitx64(dec->dec_norm_addr, index, index) << i;
	}

	dec->dec_dimm_col = cols;
	return (B_TRUE);
}

/*
 * The row is split into two different regions. There's a low and high value,
 * though the high value is only present in DDR4. Unlike the column, where each
 * bit is spelled out, each set of row bits are contiguous (low and high are
 * independent).
 */
static boolean_t
zen_umc_decode_rows(const zen_umc_t *umc, zen_umc_decoder_t *dec)
{
	uint32_t row = 0;
	uint8_t inv;
	const umc_cs_t *cs = dec->dec_cs;
	const uint_t total_bits = cs->ucs_nrow_lo + cs->ucs_nrow_hi;
	const uint_t lo_end = cs->ucs_nrow_lo + cs->ucs_row_low_bit - 1;

	row = bitx64(dec->dec_norm_addr, lo_end, cs->ucs_row_low_bit);
	if (cs->ucs_nrow_hi > 0) {
		const uint_t hi_end = cs->ucs_nrow_hi + cs->ucs_row_hi_bit - 1;
		const uint32_t hi = bitx64(dec->dec_norm_addr, hi_end,
		    cs->ucs_row_hi_bit);

		row |= hi << cs->ucs_nrow_lo;
	}

	if (dec->dec_cs_sec) {
		inv = cs->ucs_inv_msbs_sec;
	} else {
		inv = cs->ucs_inv_msbs;
	}

	/*
	 * We need to potentially invert the top two bits of the row address
	 * based on the low two bits of the inverted register below. Note, inv
	 * only has two valid bits below. So we shift them into place to perform
	 * the XOR. See the big theory statement in zen_umc.c for more on why
	 * this works.
	 */
	inv = inv << (total_bits - 2);
	row = row ^ inv;

	dec->dec_dimm_row = row;
	return (B_TRUE);
}

/*
 * Several of the hash schemes ask us to go through and xor all the bits that
 * are in an address to transform it into a single bit. This implements that for
 * a uint32_t. This is basically a bitwise XOR reduce.
 */
static uint8_t
zen_umc_running_xor32(const uint32_t in)
{
	uint8_t run = 0;

	for (uint_t i = 0; i < sizeof (in) * NBBY; i++) {
		run ^= bitx32(in, i, i);
	}

	return (run);
}

static uint8_t
zen_umc_running_xor64(const uint64_t in)
{
	uint8_t run = 0;

	for (uint_t i = 0; i < sizeof (in) * NBBY; i++) {
		run ^= bitx64(in, i, i);
	}

	return (run);
}

/*
 * Our goal here is to extract the number of banks and bank groups that are
 * used, if any.
 */
static boolean_t
zen_umc_decode_banks(const zen_umc_t *umc, zen_umc_decoder_t *dec)
{
	uint8_t bank = 0;
	const umc_cs_t *cs = dec->dec_cs;
	const umc_chan_hash_t *hash = &dec->dec_umc_chan->chan_hash;

	/*
	 * Get an initial bank address bit and then perform any hashing if
	 * bank hashing is enabled. Note, the memory controller's nbanks is the
	 * total number of bank and bank group bits, hence why it's used for
	 * the loop counter.
	 */
	for (uint_t i = 0; i < cs->ucs_nbanks; i++) {
		uint32_t row_hash, col_hash;
		uint8_t row_xor, col_xor;
		uint_t targ = cs->ucs_bank_bits[i];
		uint8_t val = bitx64(dec->dec_norm_addr, targ, targ);
		const umc_bank_hash_t *bank_hash = &hash->uch_bank_hashes[i];

		if ((hash->uch_flags & UMC_CHAN_HASH_F_BANK) == 0 ||
		    !hash->uch_bank_hashes[i].ubh_en) {
			bank |= val << i;
			continue;
		}

		/*
		 * See the big theory statement for more on this. Short form,
		 * bit-wise AND the row and column, then XOR shenanigans.
		 */
		row_hash = dec->dec_dimm_row & bank_hash->ubh_row_xor;
		col_hash = dec->dec_dimm_col & bank_hash->ubh_col_xor;
		row_xor = zen_umc_running_xor32(row_hash);
		col_xor = zen_umc_running_xor32(col_hash);
		bank |= (row_xor ^ col_xor ^ val) << i;
	}

	/*
	 * The bank and bank group are conjoined in the register and bit
	 * definitions. Once we've calculated that, extract it.
	 */
	dec->dec_dimm_bank_group = bitx8(bank, cs->ucs_nbank_groups - 1, 0);
	dec->dec_dimm_bank = bitx8(bank, cs->ucs_nbanks, cs->ucs_nbank_groups);
	return (B_TRUE);
}

/*
 * Extract the sub-channel. If not a DDR5 based device, simply set it to zero
 * and return. We can't forget to hash this if required.
 */
static boolean_t
zen_umc_decode_subchan(const zen_umc_t *umc, zen_umc_decoder_t *dec)
{
	uint8_t subchan;
	uint32_t row_hash, col_hash, bank_hash;
	uint8_t row_xor, col_xor, bank_xor;
	const umc_cs_t *cs = dec->dec_cs;
	const umc_chan_hash_t *hash = &dec->dec_umc_chan->chan_hash;

	switch (dec->dec_umc_chan->chan_type) {
	case UMC_DIMM_T_DDR5:
	case UMC_DIMM_T_LPDDR5:
		break;
	default:
		dec->dec_dimm_subchan = 0;
		return (B_TRUE);
	}

	subchan = bitx64(dec->dec_norm_addr, cs->ucs_subchan, cs->ucs_subchan);
	if ((hash->uch_flags & UMC_CHAN_HASH_F_PC) == 0 ||
	    !hash->uch_pc_hash.uph_en) {
		dec->dec_dimm_subchan = subchan;
		return (B_TRUE);
	}

	row_hash = dec->dec_dimm_row & hash->uch_pc_hash.uph_row_xor;
	col_hash = dec->dec_dimm_col & hash->uch_pc_hash.uph_col_xor;
	bank_hash = dec->dec_dimm_bank & hash->uch_pc_hash.uph_bank_xor;
	row_xor = zen_umc_running_xor32(row_hash);
	col_xor = zen_umc_running_xor32(col_hash);
	bank_xor = zen_umc_running_xor32(bank_hash);

	dec->dec_dimm_subchan = subchan ^ row_xor ^ col_xor ^ bank_xor;
	return (B_TRUE);
}

/*
 * Note that we have normalized the RM bits between the primary and secondary
 * base/mask registers so that way even though the DDR5 controller always uses
 * the same RM selection bits, it works in a uniform way for both DDR4 and DDR5.
 */
static boolean_t
zen_umc_decode_rank_mul(const zen_umc_t *umc, zen_umc_decoder_t *dec)
{
	uint8_t rm = 0;
	const umc_cs_t *cs = dec->dec_cs;
	const umc_chan_hash_t *hash = &dec->dec_umc_chan->chan_hash;

	for (uint_t i = 0; i < cs->ucs_nrm; i++) {
		uint8_t index = cs->ucs_rm_bits[i];
		uint8_t bit = bitx64(dec->dec_norm_addr, index, index);

		if ((hash->uch_flags & UMC_CHAN_HASH_F_RM) != 0 &&
		    hash->uch_rm_hashes[i].uah_en) {
			uint64_t norm_mask = dec->dec_norm_addr &
			    hash->uch_rm_hashes[i].uah_addr_xor;
			uint8_t norm_hash = zen_umc_running_xor64(norm_mask);
			bit = bit ^ norm_hash;
		}

		rm |= bit << i;
	}

	dec->dec_dimm_rm = rm;
	return (B_TRUE);
}

/*
 * Go through and determine the actual chip-select activated. This is subject to
 * hashing. Note, we first constructed a logical chip-select value based on
 * which of the four base/mask registers in the UMC we activated for the
 * channel. That basically seeded the two bit value we start with.
 */
static boolean_t
zen_umc_decode_chipsel(const zen_umc_t *umc, zen_umc_decoder_t *dec)
{
	uint8_t csno = 0;
	const umc_cs_t *cs = dec->dec_cs;
	const umc_chan_hash_t *hash = &dec->dec_umc_chan->chan_hash;

	for (uint_t i = 0; i < ZEN_UMC_MAX_CS_BITS; i++) {
		uint8_t bit = bitx8(dec->dec_log_csno, i, i);
		if ((hash->uch_flags & UMC_CHAN_HASH_F_CS) != 0 &&
		    hash->uch_cs_hashes[i].uah_en) {
			uint64_t mask = dec->dec_norm_addr &
			    hash->uch_cs_hashes[i].uah_addr_xor;
			uint8_t rxor = zen_umc_running_xor64(mask);
			bit = bit ^ rxor;
		}
		csno |= bit << i;
	}

	/*
	 * It is not entirely clear what the circumstances are that we need to
	 * apply the chip-select xor. Right now we always apply it. This only
	 * exists on a few DDR5 SoCs, it seems, and we zero out other cases to
	 * try and have a uniform and reasonable path. This tells us what the
	 * absolute chip-select is in the channel. We record this for debugging
	 * purposes and to derive the DIMM and CS.
	 */
	dec->dec_chan_csno = (csno ^ cs->ucs_cs_xor) & 0x3;

	/*
	 * Now that we actually know which chip-select we're targeting, go back
	 * and actual indicate which DIMM we'll go out to and what chip-select
	 * it is relative to the DIMM. This may have changed out due to CS
	 * hashing. As such we have to now snapshot our final DIMM and
	 * chip-select.
	 */
	dec->dec_dimm_no = dec->dec_chan_csno >> 1;
	dec->dec_dimm_csno = dec->dec_chan_csno % 2;
	return (B_TRUE);
}

/*
 * Initialize the decoder state. We do this by first zeroing it all and then
 * setting various result addresses to the UINTXX_MAX that is appropriate. These
 * work as better sentinel values than zero; however, we always zero the
 * structure to be defensive, cover pointers, etc.
 */
static void
zen_umc_decoder_init(zen_umc_decoder_t *dec)
{
	bzero(dec, sizeof (*dec));

	dec->dec_pa = dec->dec_ilv_pa = UINT64_MAX;
	dec->dec_df_ruleno = UINT32_MAX;
	dec->dec_ilv_sock = dec->dec_ilv_die = dec->dec_ilv_chan =
	    dec->dec_ilv_fabid = dec->dec_log_fabid = dec->dec_remap_comp =
	    dec->dec_targ_fabid = UINT32_MAX;
	dec->dec_umc_ruleno = UINT32_MAX;
	dec->dec_norm_addr = UINT64_MAX;
	dec->dec_dimm_col = dec->dec_dimm_row = UINT32_MAX;
	dec->dec_log_csno = dec->dec_dimm_bank = dec->dec_dimm_bank_group =
	    dec->dec_dimm_subchan = dec->dec_dimm_rm = dec->dec_chan_csno =
	    dec->dec_dimm_no = dec->dec_dimm_csno = UINT8_MAX;
}

boolean_t
zen_umc_decode_pa(const zen_umc_t *umc, const uint64_t pa,
    zen_umc_decoder_t *dec)
{
	zen_umc_decoder_init(dec);
	dec->dec_pa = pa;

	/*
	 * Before we proceed through decoding, the first thing we should try to
	 * do is verify that this is even something that could be DRAM.
	 */
	if (!zen_umc_decode_is_dram(umc, dec)) {
		ASSERT3U(dec->dec_fail, !=, ZEN_UMC_DECODE_F_NONE);
		return (B_FALSE);
	}

	/*
	 * The very first thing that we need to do is find a data fabric rule
	 * that corresponds to this memory address. This will be used to
	 * determine which set of rules for interleave and related we actually
	 * should then use.
	 */
	if (!zen_umc_decode_find_df_rule(umc, dec)) {
		ASSERT3U(dec->dec_fail, !=, ZEN_UMC_DECODE_F_NONE);
		return (B_FALSE);
	}

	/*
	 * Now that we have a DF rule, we must take a more involved step of
	 * mapping to a given CS, e.g. a specific UMC channel. This will tell us
	 * the socket and die as well. This takes care of all the interleaving
	 * and remapping and produces a target fabric ID.
	 */
	if (!zen_umc_decode_sysaddr_to_csid(umc, dec)) {
		ASSERT3U(dec->dec_fail, !=, ZEN_UMC_DECODE_F_NONE);
		return (B_FALSE);
	}

	/*
	 * With that target ID known, now actually map this to a corresponding
	 * UMC.
	 */
	if (!zen_umc_decode_find_umc_rule(umc, dec)) {
		ASSERT3U(dec->dec_fail, !=, ZEN_UMC_DECODE_F_NONE);
		return (B_FALSE);
	}

	/*
	 * With the target and corresponding rules and offset information,
	 * actually perform normalization.
	 */
	if (!zen_umc_decode_sysaddr_to_norm(umc, dec)) {
		ASSERT3U(dec->dec_fail, !=, ZEN_UMC_DECODE_F_NONE);
		return (B_FALSE);
	}

	/*
	 * Finally, we somehow managed to actually construct a normalized
	 * address. Now we must begin the act of transforming this channel
	 * address into something that makes sense to address a DIMM. To start
	 * with determine which logical chip-select, which determines where we
	 * source all our data to use.
	 */
	if (!zen_umc_decode_find_cs(umc, dec)) {
		ASSERT3U(dec->dec_fail, !=, ZEN_UMC_DECODE_F_NONE);
		return (B_FALSE);
	}

	/*
	 * Now that we have the logical chip-select matched that we're sourcing
	 * our data from, the next this is a bit more involved: we need to
	 * extract the row, column, rank/rank multiplication, bank, and bank
	 * group out of all this, while taking into account all of our hashes.
	 *
	 * To do this, we begin by first calculating the row and column as those
	 * will be needed to determine some of our other values here.
	 */
	if (!zen_umc_decode_rows(umc, dec)) {
		ASSERT3U(dec->dec_fail, !=, ZEN_UMC_DECODE_F_NONE);
		return (B_FALSE);
	}

	if (!zen_umc_decode_cols(umc, dec)) {
		ASSERT3U(dec->dec_fail, !=, ZEN_UMC_DECODE_F_NONE);
		return (B_FALSE);
	}

	/*
	 * Now that we have the rows and columns we can go through and determine
	 * the bank and bank group. This depends on the above.
	 */
	if (!zen_umc_decode_banks(umc, dec)) {
		ASSERT3U(dec->dec_fail, !=, ZEN_UMC_DECODE_F_NONE);
		return (B_FALSE);
	}

	/*
	 * If we have a DDR5 generation DIMM then we need to consider the
	 * subchannel. This doesn't exist in DDR4 systems (the function handles
	 * this reality). Because of potential hashing, this needs to come after
	 * the row, column, and bank have all been determined.
	 */
	if (!zen_umc_decode_subchan(umc, dec)) {
		ASSERT3U(dec->dec_fail, !=, ZEN_UMC_DECODE_F_NONE);
		return (B_FALSE);
	}

	/*
	 * Time for the last two pieces here: the actual chip select used and
	 * then figuring out which rank, taking into account rank
	 * multiplication. Don't worry, these both have hashing opportunities.
	 */
	if (!zen_umc_decode_rank_mul(umc, dec)) {
		ASSERT3U(dec->dec_fail, !=, ZEN_UMC_DECODE_F_NONE);
		return (B_FALSE);
	}

	if (!zen_umc_decode_chipsel(umc, dec)) {
		ASSERT3U(dec->dec_fail, !=, ZEN_UMC_DECODE_F_NONE);
		return (B_FALSE);
	}

	/*
	 * Somehow, that's it.
	 */
	return (B_TRUE);
}
/*
 * This file and its contents are supplied under the terms of the
 * Common Development and Distribution License ("CDDL"), version 1.0.
 * You may only use this file in accordance with the terms of version
 * 1.0 of the CDDL.
 *
 * A full copy of the text of the CDDL should have accompanied this
 * source.  A copy of the CDDL is also available via the Internet at
 * http://www.illumos.org/license/CDDL.
 */

/*
 * Copyright 2025 Oxide Computer Company
 */

/*
 * Dump and restore logic for external processing. Dump generally runs in kernel
 * context from a well formed structure created by the driver. Restore is used
 * in userland as part of testing and related.
 *
 * Note, there are a lot of fields in these structures that are not serialized
 * because they are not used as part of the decoder (e.g. the various raw values
 * which are captured to aid future debugging).
 */

#include "zen_umc.h"
#ifndef _KERNEL
#include <string.h>
#include <strings.h>
#include <libnvpair.h>
#endif

static nvlist_t *
zen_umc_dump_dram_rule(df_dram_rule_t *rule)
{
	nvlist_t *nvl;

	nvl = fnvlist_alloc();
	fnvlist_add_uint32(nvl, "ddr_flags", rule->ddr_flags);
	fnvlist_add_uint64(nvl, "ddr_base", rule->ddr_base);
	fnvlist_add_uint64(nvl, "ddr_limit", rule->ddr_limit);
	fnvlist_add_uint16(nvl, "ddr_dest_fabid", rule->ddr_dest_fabid);
	fnvlist_add_uint8(nvl, "ddr_sock_ileave_bits",
	    rule->ddr_sock_ileave_bits);
	fnvlist_add_uint8(nvl, "ddr_die_ileave_bits",
	    rule->ddr_die_ileave_bits);
	fnvlist_add_uint8(nvl, "ddr_addr_start", rule->ddr_addr_start);
	fnvlist_add_uint8(nvl, "ddr_remap_ent", rule->ddr_remap_ent);
	fnvlist_add_uint32(nvl, "ddr_chan_ileave", rule->ddr_chan_ileave);

	return (nvl);
}

static nvlist_t *
zen_umc_dump_cs(umc_cs_t *cs)
{
	nvlist_t *nvl = fnvlist_alloc();
	nvlist_t *base = fnvlist_alloc();
	nvlist_t *sec = fnvlist_alloc();

	fnvlist_add_uint32(nvl, "ucs_flags", cs->ucs_flags);
	fnvlist_add_uint64(base, "udb_base", cs->ucs_base.udb_base);
	fnvlist_add_uint8(base, "udb_valid", cs->ucs_base.udb_valid);
	fnvlist_add_nvlist(nvl, "ucs_base", base);
	nvlist_free(base);
	fnvlist_add_uint64(sec, "udb_base", cs->ucs_sec.udb_base);
	fnvlist_add_uint8(sec, "udb_valid", cs->ucs_sec.udb_valid);
	fnvlist_add_nvlist(nvl, "ucs_sec", sec);
	nvlist_free(sec);
	fnvlist_add_uint64(nvl, "ucs_base_mask", cs->ucs_base_mask);
	fnvlist_add_uint64(nvl, "ucs_sec_mask", cs->ucs_sec_mask);
	fnvlist_add_uint8(nvl, "ucs_nrow_lo", cs->ucs_nrow_lo);
	fnvlist_add_uint8(nvl, "ucs_nrow_hi", cs->ucs_nrow_hi);
	fnvlist_add_uint8(nvl, "ucs_nbank_groups", cs->ucs_nbank_groups);
	fnvlist_add_uint8(nvl, "ucs_cs_xor", cs->ucs_cs_xor);
	fnvlist_add_uint8(nvl, "ucs_row_hi_bit", cs->ucs_row_hi_bit);
	fnvlist_add_uint8(nvl, "ucs_row_low_bit", cs->ucs_row_low_bit);
	fnvlist_add_uint8_array(nvl, "ucs_bank_bits", cs->ucs_bank_bits,
	    cs->ucs_nbanks);
	fnvlist_add_uint8_array(nvl, "ucs_col_bits", cs->ucs_col_bits,
	    cs->ucs_ncol);
	fnvlist_add_uint8(nvl, "ucs_inv_msbs", cs->ucs_inv_msbs);
	fnvlist_add_uint8_array(nvl, "ucs_rm_bits", cs->ucs_rm_bits,
	    cs->ucs_nrm);
	fnvlist_add_uint8(nvl, "ucs_inv_msbs_sec", cs->ucs_inv_msbs_sec);
	fnvlist_add_uint8_array(nvl, "ucs_rm_bits_sec", cs->ucs_rm_bits_sec,
	    cs->ucs_nrm);
	fnvlist_add_uint8(nvl, "ucs_subchan", cs->ucs_subchan);

	return (nvl);
}

static nvlist_t *
zen_umc_dump_dimm(umc_dimm_t *dimm)
{
	nvlist_t *nvl = fnvlist_alloc();
	nvlist_t *cs[ZEN_UMC_MAX_CS_PER_DIMM];

	fnvlist_add_uint32(nvl, "ud_flags", dimm->ud_flags);
	fnvlist_add_uint32(nvl, "ud_width", dimm->ud_width);
	fnvlist_add_uint32(nvl, "ud_kind", dimm->ud_kind);
	fnvlist_add_uint32(nvl, "ud_dimmno", dimm->ud_dimmno);

	for (uint_t i = 0; i < ZEN_UMC_MAX_CS_PER_DIMM; i++) {
		cs[i] = zen_umc_dump_cs(&dimm->ud_cs[i]);
	}
	fnvlist_add_nvlist_array(nvl, "ud_cs", cs, ZEN_UMC_MAX_CS_PER_DIMM);
	for (uint_t i = 0; i < ZEN_UMC_MAX_CS_PER_DIMM; i++) {
		nvlist_free(cs[i]);
	}

	return (nvl);
}

static nvlist_t *
zen_umc_dump_chan_hash(umc_chan_hash_t *hash)
{
	nvlist_t *nvl = fnvlist_alloc();

	fnvlist_add_uint32(nvl, "uch_flags", hash->uch_flags);

	if (hash->uch_flags & UMC_CHAN_HASH_F_BANK) {
		nvlist_t *banks[ZEN_UMC_MAX_CHAN_BANK_HASH];
		for (uint_t i = 0; i < ZEN_UMC_MAX_CHAN_BANK_HASH; i++) {
			banks[i] = fnvlist_alloc();

			fnvlist_add_uint32(banks[i], "ubh_row_xor",
			    hash->uch_bank_hashes[i].ubh_row_xor);
			fnvlist_add_uint32(banks[i], "ubh_col_xor",
			    hash->uch_bank_hashes[i].ubh_col_xor);
			fnvlist_add_boolean_value(banks[i], "ubh_en",
			    hash->uch_bank_hashes[i].ubh_en);
		}
		fnvlist_add_nvlist_array(nvl, "uch_bank_hashes", banks,
		    ZEN_UMC_MAX_CHAN_BANK_HASH);

		for (uint_t i = 0; i < ZEN_UMC_MAX_CHAN_BANK_HASH; i++) {
			nvlist_free(banks[i]);
		}
	}

	if (hash->uch_flags & UMC_CHAN_HASH_F_RM) {
		nvlist_t *rm[ZEN_UMC_MAX_CHAN_RM_HASH];
		for (uint_t i = 0; i < ZEN_UMC_MAX_CHAN_RM_HASH; i++) {
			rm[i] = fnvlist_alloc();

			fnvlist_add_uint64(rm[i], "uah_addr_xor",
			    hash->uch_rm_hashes[i].uah_addr_xor);
			fnvlist_add_boolean_value(rm[i], "uah_en",
			    hash->uch_rm_hashes[i].uah_en);
		}
		fnvlist_add_nvlist_array(nvl, "uch_rm_hashes", rm,
		    ZEN_UMC_MAX_CHAN_RM_HASH);

		for (uint_t i = 0; i < ZEN_UMC_MAX_CHAN_RM_HASH; i++) {
			nvlist_free(rm[i]);
		}
	}

	if (hash->uch_flags & UMC_CHAN_HASH_F_CS) {
		nvlist_t *cs[ZEN_UMC_MAX_CHAN_CS_HASH];
		for (uint_t i = 0; i < ZEN_UMC_MAX_CHAN_CS_HASH; i++) {
			cs[i] = fnvlist_alloc();

			fnvlist_add_uint64(cs[i], "uah_addr_xor",
			    hash->uch_rm_hashes[i].uah_addr_xor);
			fnvlist_add_boolean_value(cs[i], "uah_en",
			    hash->uch_rm_hashes[i].uah_en);
		}
		fnvlist_add_nvlist_array(nvl, "uch_cs_hashes", cs,
		    ZEN_UMC_MAX_CHAN_CS_HASH);

		for (uint_t i = 0; i < ZEN_UMC_MAX_CHAN_CS_HASH; i++) {
			nvlist_free(cs[i]);
		}
	}

	if (hash->uch_flags & UMC_CHAN_HASH_F_PC) {
		nvlist_t *pc = fnvlist_alloc();

		fnvlist_add_uint32(pc, "uph_row_xor",
		    hash->uch_pc_hash.uph_row_xor);
		fnvlist_add_uint32(pc, "uph_col_xor",
		    hash->uch_pc_hash.uph_col_xor);
		fnvlist_add_uint8(pc, "uph_bank_xor",
		    hash->uch_pc_hash.uph_bank_xor);
		fnvlist_add_boolean_value(pc, "uph_en",
		    hash->uch_pc_hash.uph_en);

		fnvlist_add_nvlist(nvl, "uch_pch_hash", pc);
		fnvlist_free(pc);

	}

	return (nvl);
}

static nvlist_t *
zen_umc_dump_chan(zen_umc_chan_t *chan)
{
	nvlist_t *nvl, *hash;
	nvlist_t *rules[ZEN_UMC_MAX_CS_RULES];
	nvlist_t *offsets[ZEN_UMC_MAX_DRAM_OFFSET];
	nvlist_t *dimms[ZEN_UMC_MAX_DIMMS];

	nvl = fnvlist_alloc();
	fnvlist_add_uint32(nvl, "chan_flags", chan->chan_flags);
	fnvlist_add_uint32(nvl, "chan_fabid", chan->chan_fabid);
	fnvlist_add_uint32(nvl, "chan_instid", chan->chan_instid);
	fnvlist_add_uint32(nvl, "chan_logid", chan->chan_logid);
	fnvlist_add_uint32(nvl, "chan_np2_space0", chan->chan_np2_space0);
	fnvlist_add_uint32(nvl, "chan_type", chan->chan_type);

	for (uint_t i = 0; i < chan->chan_nrules; i++) {
		rules[i] = zen_umc_dump_dram_rule(&chan->chan_rules[i]);
	}

	for (uint_t i = 0; i < chan->chan_nrules - 1; i++) {
		offsets[i] = fnvlist_alloc();
		fnvlist_add_boolean_value(offsets[i], "cho_valid",
		    chan->chan_offsets[i].cho_valid);
		fnvlist_add_uint64(offsets[i], "cho_offset",
		    chan->chan_offsets[i].cho_offset);
	}

	for (uint_t i = 0; i < ZEN_UMC_MAX_DIMMS; i++) {
		dimms[i] = zen_umc_dump_dimm(&chan->chan_dimms[i]);
	}

	fnvlist_add_nvlist_array(nvl, "chan_rules", rules, chan->chan_nrules);
	fnvlist_add_nvlist_array(nvl, "chan_offsets", offsets,
	    chan->chan_nrules - 1);
	fnvlist_add_nvlist_array(nvl, "chan_dimms", dimms, ZEN_UMC_MAX_DIMMS);
	hash = zen_umc_dump_chan_hash(&chan->chan_hash);
	fnvlist_add_nvlist(nvl, "chan_hash", hash);

	for (uint_t i = 0; i < chan->chan_nrules; i++) {
		nvlist_free(rules[i]);
	}

	for (uint_t i = 0; i < chan->chan_nrules - 1; i++) {
		nvlist_free(offsets[i]);
	}

	for (uint_t i = 0; i < ZEN_UMC_MAX_DIMMS; i++) {
		nvlist_free(dimms[i]);
	}

	nvlist_free(hash);

	return (nvl);
}

static nvlist_t *
zen_umc_dump_df(zen_umc_df_t *df)
{
	nvlist_t *nvl;
	nvlist_t *rules[ZEN_UMC_MAX_DRAM_RULES];
	nvlist_t *remap[ZEN_UMC_MAX_CS_REMAPS];
	nvlist_t *chan[ZEN_UMC_MAX_UMCS];

	nvl = fnvlist_alloc();
	fnvlist_add_uint32(nvl, "zud_flags", df->zud_flags);
	fnvlist_add_uint32(nvl, "zud_dfno", df->zud_dfno);
	fnvlist_add_uint32(nvl, "zud_ccm_inst", df->zud_ccm_inst);
	fnvlist_add_uint64(nvl, "zud_hole_base", df->zud_hole_base);

	for (uint_t i = 0; i < df->zud_dram_nrules; i++) {
		rules[i] = zen_umc_dump_dram_rule(&df->zud_rules[i]);
	}

	for (uint_t i = 0; i < df->zud_cs_nremap; i++) {
		remap[i] = fnvlist_alloc();
		fnvlist_add_uint16_array(remap[i], "csr_remaps",
		    df->zud_remap[i].csr_remaps, df->zud_remap[i].csr_nremaps);
	}

	for (uint_t i = 0; i < df->zud_nchan; i++) {
		chan[i] = zen_umc_dump_chan(&df->zud_chan[i]);
	}

	fnvlist_add_nvlist_array(nvl, "zud_rules", rules, df->zud_dram_nrules);
	fnvlist_add_nvlist_array(nvl, "zud_remap", remap, df->zud_cs_nremap);
	fnvlist_add_nvlist_array(nvl, "zud_chan", chan, df->zud_nchan);

	for (uint_t i = 0; i < df->zud_dram_nrules; i++) {
		nvlist_free(rules[i]);
	}

	for (uint_t i = 0; i < df->zud_cs_nremap; i++) {
		nvlist_free(remap[i]);
	}

	for (uint_t i = 0; i < df->zud_nchan; i++) {
		nvlist_free(chan[i]);
	}

	return (nvl);
}

nvlist_t *
zen_umc_dump_decoder(zen_umc_t *umc)
{
	nvlist_t *nvl, *umc_nvl, *decomp;
	nvlist_t *dfs[ZEN_UMC_MAX_DFS];

	nvl = fnvlist_alloc();
	fnvlist_add_uint32(nvl, "mc_dump_version", 0);
	fnvlist_add_string(nvl, "mc_dump_driver", "zen_umc");

	umc_nvl = fnvlist_alloc();
	fnvlist_add_uint64(umc_nvl, "umc_tom", umc->umc_tom);
	fnvlist_add_uint64(umc_nvl, "umc_tom2", umc->umc_tom2);
	fnvlist_add_uint32(umc_nvl, "umc_family", umc->umc_family);
	fnvlist_add_uint32(umc_nvl, "umc_df_rev", umc->umc_df_rev);

	decomp = fnvlist_alloc();
	fnvlist_add_uint32(decomp, "dfd_sock_mask",
	    umc->umc_decomp.dfd_sock_mask);
	fnvlist_add_uint32(decomp, "dfd_die_mask",
	    umc->umc_decomp.dfd_die_mask);
	fnvlist_add_uint32(decomp, "dfd_node_mask",
	    umc->umc_decomp.dfd_node_mask);
	fnvlist_add_uint32(decomp, "dfd_comp_mask",
	    umc->umc_decomp.dfd_comp_mask);
	fnvlist_add_uint8(decomp, "dfd_sock_shift",
	    umc->umc_decomp.dfd_sock_shift);
	fnvlist_add_uint8(decomp, "dfd_die_shift",
	    umc->umc_decomp.dfd_die_shift);
	fnvlist_add_uint8(decomp, "dfd_node_shift",
	    umc->umc_decomp.dfd_node_shift);
	fnvlist_add_uint8(decomp, "dfd_comp_shift",
	    umc->umc_decomp.dfd_comp_shift);
	fnvlist_add_nvlist(umc_nvl, "umc_decomp", decomp);
	nvlist_free(decomp);

	for (uint_t i = 0; i < umc->umc_ndfs; i++) {
		dfs[i] = zen_umc_dump_df(&umc->umc_dfs[i]);
	}

	fnvlist_add_nvlist_array(umc_nvl, "umc_dfs", dfs, umc->umc_ndfs);
	fnvlist_add_nvlist(nvl, "zen_umc", umc_nvl);
	for (uint_t i = 0; i < umc->umc_ndfs; i++) {
		nvlist_free(dfs[i]);
	}

	return (nvl);
}

static boolean_t
zen_umc_restore_dram_rule(nvlist_t *nvl, df_dram_rule_t *rule)
{
	return (nvlist_lookup_pairs(nvl, 0,
	    "ddr_flags", DATA_TYPE_UINT32, &rule->ddr_flags,
	    "ddr_base", DATA_TYPE_UINT64, &rule->ddr_base,
	    "ddr_limit", DATA_TYPE_UINT64, &rule->ddr_limit,
	    "ddr_dest_fabid", DATA_TYPE_UINT16, &rule->ddr_dest_fabid,
	    "ddr_sock_ileave_bits", DATA_TYPE_UINT8,
	    &rule->ddr_sock_ileave_bits,
	    "ddr_die_ileave_bits", DATA_TYPE_UINT8, &rule->ddr_die_ileave_bits,
	    "ddr_addr_start", DATA_TYPE_UINT8, &rule->ddr_addr_start,
	    "ddr_remap_ent", DATA_TYPE_UINT8, &rule->ddr_remap_ent,
	    "ddr_chan_ileave", DATA_TYPE_UINT32, &rule->ddr_chan_ileave,
	    NULL) == 0);
}

static boolean_t
zen_umc_restore_cs(nvlist_t *nvl, umc_cs_t *cs)
{
	nvlist_t *base, *sec;
	uint8_t *bank_bits, *col_bits, *rm_bits, *rm_bits_sec;
	uint_t nbanks, ncols, nrm, nrm_sec;

	if (nvlist_lookup_pairs(nvl, 0,
	    "ucs_flags", DATA_TYPE_UINT32, &cs->ucs_flags,
	    "ucs_base", DATA_TYPE_NVLIST, &base,
	    "ucs_sec", DATA_TYPE_NVLIST, &sec,
	    "ucs_base_mask", DATA_TYPE_UINT64, &cs->ucs_base_mask,
	    "ucs_sec_mask", DATA_TYPE_UINT64, &cs->ucs_sec_mask,
	    "ucs_nrow_lo", DATA_TYPE_UINT8, &cs->ucs_nrow_lo,
	    "ucs_nrow_hi", DATA_TYPE_UINT8, &cs->ucs_nrow_hi,
	    "ucs_nbank_groups", DATA_TYPE_UINT8, &cs->ucs_nbank_groups,
	    "ucs_cs_xor", DATA_TYPE_UINT8, &cs->ucs_cs_xor,
	    "ucs_row_hi_bit", DATA_TYPE_UINT8, &cs->ucs_row_hi_bit,
	    "ucs_row_low_bit", DATA_TYPE_UINT8, &cs->ucs_row_low_bit,
	    "ucs_bank_bits", DATA_TYPE_UINT8_ARRAY, &bank_bits, &nbanks,
	    "ucs_col_bits", DATA_TYPE_UINT8_ARRAY, &col_bits, &ncols,
	    "ucs_inv_msbs", DATA_TYPE_UINT8, &cs->ucs_inv_msbs,
	    "ucs_rm_bits", DATA_TYPE_UINT8_ARRAY, &rm_bits, &nrm,
	    "ucs_inv_msbs_sec", DATA_TYPE_UINT8, &cs->ucs_inv_msbs_sec,
	    "ucs_rm_bits_sec", DATA_TYPE_UINT8_ARRAY, &rm_bits_sec, &nrm_sec,
	    "ucs_subchan", DATA_TYPE_UINT8, &cs->ucs_subchan,
	    NULL) != 0) {
		return (B_FALSE);
	}

	if (nbanks > ZEN_UMC_MAX_BANK_BITS ||
	    ncols > ZEN_UMC_MAX_COL_BITS ||
	    nrm > ZEN_UMC_MAX_RM_BITS ||
	    nrm != nrm_sec) {
		return (B_FALSE);
	}

	cs->ucs_nbanks = nbanks;
	cs->ucs_ncol = ncols;
	cs->ucs_nrm = nrm;

	bcopy(bank_bits, cs->ucs_bank_bits, cs->ucs_nbanks *
	    sizeof (uint8_t));
	bcopy(col_bits, cs->ucs_col_bits, cs->ucs_ncol * sizeof (uint8_t));
	bcopy(rm_bits, cs->ucs_rm_bits, cs->ucs_nrm * sizeof (uint8_t));
	bcopy(rm_bits_sec, cs->ucs_rm_bits_sec, cs->ucs_nrm *
	    sizeof (uint8_t));

	if (nvlist_lookup_pairs(base, 0,
	    "udb_base", DATA_TYPE_UINT64, &cs->ucs_base.udb_base,
	    "udb_valid", DATA_TYPE_UINT8, &cs->ucs_base.udb_valid,
	    NULL) != 0) {
		return (B_FALSE);
	}

	if (nvlist_lookup_pairs(sec, 0,
	    "udb_base", DATA_TYPE_UINT64, &cs->ucs_sec.udb_base,
	    "udb_valid", DATA_TYPE_UINT8, &cs->ucs_sec.udb_valid,
	    NULL) != 0) {
		return (B_FALSE);
	}

	return (B_TRUE);
}

static boolean_t
zen_umc_restore_dimm(nvlist_t *nvl, umc_dimm_t *dimm)
{
	nvlist_t **cs;
	uint_t ncs;

	if (nvlist_lookup_pairs(nvl, 0,
	    "ud_flags", DATA_TYPE_UINT32, &dimm->ud_flags,
	    "ud_width", DATA_TYPE_UINT32, &dimm->ud_width,
	    "ud_kind", DATA_TYPE_UINT32, &dimm->ud_kind,
	    "ud_dimmno", DATA_TYPE_UINT32, &dimm->ud_dimmno,
	    "ud_cs", DATA_TYPE_NVLIST_ARRAY, &cs, &ncs,
	    NULL) != 0) {
		return (B_FALSE);
	}

	if (ncs != ZEN_UMC_MAX_CS_PER_DIMM) {
		return (B_FALSE);
	}

	for (uint_t i = 0; i < ZEN_UMC_MAX_CS_PER_DIMM; i++) {
		if (!zen_umc_restore_cs(cs[i], &dimm->ud_cs[i])) {
			return (B_FALSE);
		}
	}

	return (B_TRUE);
}

static boolean_t
zen_umc_restore_hash(nvlist_t *nvl, umc_chan_hash_t *hash)
{
	if (nvlist_lookup_uint32(nvl, "uch_flags", &hash->uch_flags) != 0) {
		return (B_FALSE);
	}

	if (hash->uch_flags & UMC_CHAN_HASH_F_BANK) {
		nvlist_t **banks;
		uint_t nbanks;

		if (nvlist_lookup_nvlist_array(nvl, "uch_bank_hashes", &banks,
		    &nbanks) != 0) {
			return (B_FALSE);
		}

		if (nbanks != ZEN_UMC_MAX_CHAN_BANK_HASH) {
			return (B_FALSE);
		}

		for (uint_t i = 0; i < nbanks; i++) {
			if (nvlist_lookup_pairs(banks[i], 0,
			    "ubh_row_xor", DATA_TYPE_UINT32,
			    &hash->uch_bank_hashes[i].ubh_row_xor,
			    "ubh_col_xor", DATA_TYPE_UINT32,
			    &hash->uch_bank_hashes[i].ubh_col_xor,
			    "ubh_en", DATA_TYPE_BOOLEAN_VALUE,
			    &hash->uch_bank_hashes[i].ubh_en,
			    NULL) != 0) {
				return (B_FALSE);
			}
		}
	}

	if (hash->uch_flags & UMC_CHAN_HASH_F_RM) {
		nvlist_t **rm;
		uint_t nrm;

		if (nvlist_lookup_nvlist_array(nvl, "uch_rm_hashes", &rm,
		    &nrm) != 0) {
			return (B_FALSE);
		}

		if (nrm != ZEN_UMC_MAX_CHAN_RM_HASH) {
			return (B_FALSE);
		}

		for (uint_t i = 0; i < nrm; i++) {
			if (nvlist_lookup_pairs(rm[i], 0,
			    "uah_addr_xor", DATA_TYPE_UINT64,
			    &hash->uch_rm_hashes[i].uah_addr_xor,
			    "uah_en", DATA_TYPE_BOOLEAN_VALUE,
			    &hash->uch_rm_hashes[i].uah_en,
			    NULL) != 0) {
				return (B_FALSE);
			}
		}
	}

	if (hash->uch_flags & UMC_CHAN_HASH_F_CS) {
		nvlist_t **cs;
		uint_t ncs;

		if (nvlist_lookup_nvlist_array(nvl, "uch_cs_hashes", &cs,
		    &ncs) != 0) {
			return (B_FALSE);
		}

		if (ncs != ZEN_UMC_MAX_CHAN_CS_HASH) {
			return (B_FALSE);
		}

		for (uint_t i = 0; i < ncs; i++) {
			if (nvlist_lookup_pairs(cs[i], 0,
			    "uah_addr_xor", DATA_TYPE_UINT64,
			    &hash->uch_cs_hashes[i].uah_addr_xor,
			    "uah_en", DATA_TYPE_BOOLEAN_VALUE,
			    &hash->uch_cs_hashes[i].uah_en,
			    NULL) != 0) {
				return (B_FALSE);
			}
		}
	}

	if (hash->uch_flags & UMC_CHAN_HASH_F_PC) {
		nvlist_t *pc;

		if (nvlist_lookup_nvlist(nvl, "uch_pch_hash", &pc) != 0) {
			return (B_FALSE);
		}

		if (nvlist_lookup_pairs(pc, 0,
		    "uph_row_xor", DATA_TYPE_UINT32,
		    &hash->uch_pc_hash.uph_row_xor,
		    "uph_col_xor", DATA_TYPE_UINT32,
		    &hash->uch_pc_hash.uph_col_xor,
		    "uph_bank_xor", DATA_TYPE_UINT32,
		    &hash->uch_pc_hash.uph_bank_xor,
		    "uph_en", DATA_TYPE_BOOLEAN_VALUE,
		    &hash->uch_pc_hash.uph_en,
		    NULL) != 0) {
			return (B_FALSE);
		}
	}
	return (B_TRUE);
}

static boolean_t
zen_umc_restore_chan(nvlist_t *nvl, zen_umc_chan_t *chan)
{
	uint_t noffsets, ndimms;
	nvlist_t **rules, **offsets, **dimms, *hash;

	if (nvlist_lookup_pairs(nvl, 0,
	    "chan_flags", DATA_TYPE_UINT32, &chan->chan_flags,
	    "chan_fabid", DATA_TYPE_UINT32, &chan->chan_fabid,
	    "chan_instid", DATA_TYPE_UINT32, &chan->chan_instid,
	    "chan_logid", DATA_TYPE_UINT32, &chan->chan_logid,
	    "chan_rules", DATA_TYPE_NVLIST_ARRAY, &rules, &chan->chan_nrules,
	    "chan_np2_space0", DATA_TYPE_UINT32, &chan->chan_np2_space0,
	    "chan_type", DATA_TYPE_UINT32, &chan->chan_np2_space0,
	    "chan_offsets", DATA_TYPE_NVLIST_ARRAY, &offsets, &noffsets,
	    "chan_dimms", DATA_TYPE_NVLIST_ARRAY, &dimms, &ndimms,
	    "chan_hash", DATA_TYPE_NVLIST, &hash,
	    NULL) != 0) {
		return (B_FALSE);
	}

	if (chan->chan_nrules > ZEN_UMC_MAX_CS_RULES ||
	    noffsets != chan->chan_nrules - 1 || ndimms != ZEN_UMC_MAX_DIMMS) {
		return (B_FALSE);
	}

	for (uint_t i = 0; i < chan->chan_nrules; i++) {
		if (!zen_umc_restore_dram_rule(rules[i],
		    &chan->chan_rules[i])) {
			return (B_FALSE);
		}
	}

	for (uint_t i = 0; i < chan->chan_nrules - 1; i++) {
		chan_offset_t *coff = &chan->chan_offsets[i];

		if (nvlist_lookup_pairs(offsets[i], 0,
		    "cho_valid", DATA_TYPE_BOOLEAN_VALUE, &coff->cho_valid,
		    "cho_offset", DATA_TYPE_UINT64, &coff->cho_offset,
		    NULL) != 0) {
			return (B_FALSE);
		}
	}

	for (uint_t i = 0; i < ZEN_UMC_MAX_DIMMS; i++) {
		if (!zen_umc_restore_dimm(dimms[i], &chan->chan_dimms[i])) {
			return (B_FALSE);
		}
	}

	if (!zen_umc_restore_hash(hash, &chan->chan_hash)) {
		return (B_FALSE);
	}

	return (B_TRUE);
}

static boolean_t
zen_umc_restore_df(nvlist_t *nvl, zen_umc_df_t *df)
{
	nvlist_t **rules, **chan, **remap;

	if (nvlist_lookup_pairs(nvl, 0,
	    "zud_flags", DATA_TYPE_UINT32, &df->zud_flags,
	    "zud_dfno", DATA_TYPE_UINT32, &df->zud_dfno,
	    "zud_ccm_inst", DATA_TYPE_UINT32, &df->zud_ccm_inst,
	    "zud_hole_base", DATA_TYPE_UINT64, &df->zud_hole_base,
	    "zud_rules", DATA_TYPE_NVLIST_ARRAY, &rules, &df->zud_dram_nrules,
	    "zud_remap", DATA_TYPE_NVLIST_ARRAY, &remap, &df->zud_cs_nremap,
	    "zud_chan", DATA_TYPE_NVLIST_ARRAY, &chan, &df->zud_nchan,
	    NULL != 0) ||
	    df->zud_dram_nrules > ZEN_UMC_MAX_DRAM_RULES ||
	    df->zud_cs_nremap > ZEN_UMC_MAX_CS_REMAPS ||
	    df->zud_nchan > ZEN_UMC_MAX_UMCS) {
		return (B_FALSE);
	}

	for (uint_t i = 0; i < df->zud_dram_nrules; i++) {
		if (!zen_umc_restore_dram_rule(rules[i], &df->zud_rules[i])) {
			return (B_FALSE);
		}
	}

	for (uint_t i = 0; i < df->zud_cs_nremap; i++) {
		uint16_t *u16p;
		if (nvlist_lookup_uint16_array(remap[i], "csr_remaps", &u16p,
		    &df->zud_remap[i].csr_nremaps) != 0 ||
		    df->zud_remap[i].csr_nremaps > ZEN_UMC_MAX_REMAP_ENTS) {
			return (B_FALSE);
		}
		bcopy(u16p, df->zud_remap[i].csr_remaps,
		    df->zud_remap[i].csr_nremaps);
	}

	for (uint_t i = 0; i < df->zud_nchan; i++) {
		if (!zen_umc_restore_chan(chan[i], &df->zud_chan[i])) {
			return (B_FALSE);
		}
	}

	return (B_TRUE);
}

boolean_t
zen_umc_restore_decoder(nvlist_t *nvl, zen_umc_t *umc)
{
	uint32_t vers;
	char *driver;
	nvlist_t *umc_nvl, *decomp, **dfs;
	bzero(umc, sizeof (zen_umc_t));

	if (nvlist_lookup_pairs(nvl, 0,
	    "mc_dump_version", DATA_TYPE_UINT32, &vers,
	    "mc_dump_driver", DATA_TYPE_STRING, &driver,
	    NULL) != 0 || vers != 0 || strcmp(driver, "zen_umc") != 0 ||
	    nvlist_lookup_nvlist(nvl, "zen_umc", &umc_nvl) != 0) {
		return (B_FALSE);
	}

	if (nvlist_lookup_pairs(umc_nvl, 0,
	    "umc_tom", DATA_TYPE_UINT64, &umc->umc_tom,
	    "umc_tom2", DATA_TYPE_UINT64, &umc->umc_tom2,
	    "umc_family", DATA_TYPE_UINT32, &umc->umc_family,
	    "umc_df_rev", DATA_TYPE_UINT32, &umc->umc_df_rev,
	    "umc_decomp", DATA_TYPE_NVLIST, &decomp,
	    "umc_dfs", DATA_TYPE_NVLIST_ARRAY, &dfs, &umc->umc_ndfs,
	    NULL) != 0 || umc->umc_ndfs > ZEN_UMC_MAX_DFS) {
		return (B_FALSE);
	}


	if (nvlist_lookup_pairs(decomp, 0,
	    "dfd_sock_mask", DATA_TYPE_UINT32, &umc->umc_decomp.dfd_sock_mask,
	    "dfd_die_mask", DATA_TYPE_UINT32, &umc->umc_decomp.dfd_die_mask,
	    "dfd_node_mask", DATA_TYPE_UINT32, &umc->umc_decomp.dfd_node_mask,
	    "dfd_comp_mask", DATA_TYPE_UINT32, &umc->umc_decomp.dfd_comp_mask,
	    "dfd_sock_shift", DATA_TYPE_UINT8, &umc->umc_decomp.dfd_sock_shift,
	    "dfd_die_shift", DATA_TYPE_UINT8, &umc->umc_decomp.dfd_die_shift,
	    "dfd_node_shift", DATA_TYPE_UINT8, &umc->umc_decomp.dfd_node_shift,
	    "dfd_comp_shift", DATA_TYPE_UINT8, &umc->umc_decomp.dfd_comp_shift,
	    NULL) != 0) {
		return (B_FALSE);
	}

	for (uint_t i = 0; i < umc->umc_ndfs; i++) {
		if (!zen_umc_restore_df(dfs[i], &umc->umc_dfs[i])) {
			return (B_FALSE);
		}
	}

	return (B_TRUE);
}