cpu - zygaena/hammerhead

root / base / usr / src / uts / i86pc / cpu
cpu Plain Text 8189 lines 218.3 KB
# CDDL HEADER START
#
# The contents of this file are subject to the terms of the
# Common Development and Distribution License, Version 1.0 only
# (the "License").  You may not use this file except in compliance
# with the License.
#
# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
# or http://www.opensolaris.org/os/licensing.
# See the License for the specific language governing permissions
# and limitations under the License.
#
# When distributing Covered Code, include this CDDL HEADER in each
# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
# If applicable, add the following below this CDDL HEADER, with the
# fields enclosed by brackets "[]" replaced with your own identifying
# information: Portions Copyright [yyyy] [name of copyright owner]
#
# CDDL HEADER END
#
# Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
# Use is subject to license terms.
#
# ident	"%Z%%M%	%I%	%E% SMI"
#

include $(UTSBASE)/i86pc/Makefile.i86pc
include $(UTSBASE)/i86pc/cpu/Makefile.files
# CDDL HEADER START
#
# The contents of this file are subject to the terms of the
# Common Development and Distribution License (the "License").
# You may not use this file except in compliance with the License.
#
# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
# or http://www.opensolaris.org/os/licensing.
# See the License for the specific language governing permissions
# and limitations under the License.
#
# When distributing Covered Code, include this CDDL HEADER in each
# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
# If applicable, add the following below this CDDL HEADER, with the
# fields enclosed by brackets "[]" replaced with your own identifying
# information: Portions Copyright [yyyy] [name of copyright owner]
#
# CDDL HEADER END
#
# Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
# Use is subject to license terms.
#

CPU_AO_OBJS = \
	ao_cpu.o \
	ao_main.o \
	ao_mca.o \
	ao_mca_disp.o \
	ao_poll.o

CPU_GCPU_OBJS = \
	gcpu_main.o \
	gcpu_mca.o \
	gcpu_poll_subr.o

CPU_GCPU_NTV_OBJS = \
	gcpu_poll_ntv.o

CPU_GCPU_XPV_OBJS = \
	gcpu_mca_xpv.o \
	gcpu_poll_xpv.o

CPU_AUTHAMD_OBJS = \
	authamd_main.o

CPU_GENINTEL_OBJS = \
	gintel_main.o
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright (c) 2006, 2010, Oracle and/or its affiliates. All rights reserved.
 */

#ifndef _AO_H
#define	_AO_H

#include <sys/types.h>
#include <sys/mc.h>
#include <sys/mca_amd.h>
#include <sys/mc_amd.h>
#include <sys/cpu_module_ms_impl.h>
#include <sys/nvpair.h>
#include <sys/cyclic.h>
#include <sys/errorq.h>
#include <sys/kobj.h>
#include <sys/fm/util.h>

#ifdef __cplusplus
extern "C" {
#endif

#define	AO_MAX_CHIPS		8

#define	AO_MCA_MAX_ERRORS	10

typedef struct ao_ms_data ao_ms_data_t;

/*
 * Rather than using torturous conditionals, we match errors using a table of
 * ao_error_disp_t's.  The members in the ao_error_disp_t are matched against
 * the value of MCi_STATUS, with a successful match indicating that the given
 * error occurred.
 *
 * While aed_stat_code will match most of the status code bits, a few of the
 * status code fields are either/or, and are treated separately so as to
 * minimize the number of ao_error_disp_t structures that must be created.
 * For example, the dc.tag_par error can have r4 values drd or dwr.  Rather
 * than creating two ao_error_disp_t's, we use the separate aed_stat_r4_bits
 * field to indicate both AO_MCA_R4_BIT_DRD and AO_MCA_R4_BIT_DWD.  As the
 * matching r4 values are drawn from aed_stat_r4_bits, we don't use the r4
 * bits in aed_stat_code for matching.  Similar reasoning lies behind the
 * creation of the pp and ii fields.
 */
#define	AO_AED_PANIC_NEVER	0x00
#define	AO_AED_PANIC_IFMCE	0x01
#define	AO_AED_PANIC_ALWAYS	0x80

/*
 * The AO_AED_F_* flags tell us how to interpret aspects of the error
 * telemetry, such as which bits of the captured address are valid for
 * this error.
 */
					/* MCi_ADDR ... */
#define	AO_AED_F_LINEAR		0x01	/* is a linear address */
#define	AO_AED_F_PHYSICAL	0x02	/* is a physical address */
#define	AO_AED_F_PAGEALIGNED	0x04	/* aligns to page size */
#define	AO_AED_F_L2SETWAY	0x08	/* 3:0 = way, 15/14/13/12:6 = set */

#define	AO_AED_FLAGS_ADDRTYPE	(AO_AED_F_LINEAR | AO_AED_F_PHYSICAL | \
    AO_AED_F_PAGEALIGNED | AO_AED_F_L2SETWAY)

/*
 * The AO_AED_ET_* flags group individual error dispositions into
 * error types.  This is used to nominate additional telemetry beyond the
 * architectural bank registers to capture for this error type.
 */
#define	AO_AED_ET_MEMECC		0x0001	/* Main memory ECC error */

typedef struct ao_error_disp {
	const char *aed_class;		/* ereport class for use if match */
	uint64_t aed_ereport_members;	/* ereport contents flags if match */
	uint64_t aed_stat_mask;		/* status msr bits for match */
	uint64_t aed_stat_mask_res;	/* status mask result for match */
	uint16_t aed_stat_code;		/* status code for match */
	uint8_t aed_stat_extcode;	/* extended status code for match */
	uint8_t aed_stat_pp_bits:4;	/* AO_MCA_PP_BIT_* for pp matching */
	uint8_t aed_stat_ii_bits:4;	/* AO_MCA_II_BIT_* for ii matching */
	uint16_t aed_stat_r4_bits;	/* AO_MCA_R4_BIT_* for r4 matching */
	uint8_t aed_addrvalid_hi;	/* most significant valid addr bit */
	uint8_t aed_addrvalid_lo;	/* least significant valid addr bit */
	uint8_t aed_panic_when;		/* extra conditions for panic */
	uint16_t aed_flags;		/* AO_AED_F_* */
	uint16_t aed_errtype;		/* AO_AED_ET_* */
} ao_error_disp_t;

/*
 * We store non-architectutal config as inherited from the BIOS to assist
 * in troubleshooting.
 */
struct ao_bios_cfg {
	uint64_t *bcfg_bank_mask;
};

/*
 * The master data structure used to hold MCA-related state.
 */
typedef struct ao_ms_mca {
	struct ao_bios_cfg ao_mca_bios_cfg;
	kmutex_t ao_mca_poll_lock;	/* keep pollers from colliding */
	uint_t ao_mca_flags;		/* AO_MCA_F_* */
} ao_ms_mca_t;

/*
 * Per-chip shared state
 */
struct ao_chipshared {
	x86_chiprev_t aos_chiprev;
	volatile ulong_t aos_cfgonce;	/* Config performed once per chip */
	hrtime_t aos_nb_poll_timestamp;
	cmi_hdl_t aos_nb_poll_owner;
	uint64_t aos_bcfg_nb_misc;	/* BIOS value of MC4_MISC MSR */
	uint32_t aos_bcfg_nb_cfg;	/* BIOS value of NB MCA Config */
	uint32_t aos_bcfg_nb_sparectl;	/* BIOS value of Online Spare Control */
	uint32_t aos_bcfg_dcfg_lo;	/* BIOS value of DRAM Config Low */
	uint32_t aos_bcfg_dcfg_hi;	/* BIOS value of DRAM Config High */
	uint32_t aos_bcfg_scrubctl;	/* BIOS value of scrub control */
};

/* Bit numbers for once-per-chip operations policed by cms_once */
enum ao_cfgonce_bitnum {
	AO_CFGONCE_NBMCA,
	AO_CFGONCE_NBCFG,
	AO_CFGONCE_DRAMCFG
};

/*
 * Per-CPU model-specific state
 */
struct ao_ms_data {
	cmi_hdl_t ao_ms_hdl;
	ao_ms_mca_t ao_ms_mca;
	struct ao_chipshared *ao_ms_shared;
	uint64_t ao_ms_hwcr_val;
};

#ifdef _KERNEL

struct regs;

/*
 * Our cms_ops operations and function prototypes for all non-NULL members.
 */
extern const cms_ops_t _cms_ops;

extern int ao_ms_init(cmi_hdl_t, void **);
extern void ao_ms_post_startup(cmi_hdl_t);
extern void ao_ms_post_mpstartup(cmi_hdl_t);
extern uint64_t ao_ms_mcgctl_val(cmi_hdl_t, int, uint64_t);
extern boolean_t ao_ms_bankctl_skipinit(cmi_hdl_t, int);
extern uint64_t ao_ms_bankctl_val(cmi_hdl_t, int, uint64_t);
extern void ao_ms_mca_init(cmi_hdl_t, int);
extern uint64_t ao_ms_poll_ownermask(cmi_hdl_t, hrtime_t);
extern uint32_t ao_ms_error_action(cmi_hdl_t, int, int, uint64_t,
    uint64_t, uint64_t, void *);
extern cms_cookie_t ao_ms_disp_match(cmi_hdl_t, int, int, uint64_t, uint64_t,
    uint64_t, void *);
extern void ao_ms_ereport_class(cmi_hdl_t, cms_cookie_t, const char **,
    const char **);
extern boolean_t ao_ms_ereport_includestack(cmi_hdl_t, cms_cookie_t);
extern void ao_ms_ereport_add_logout(cmi_hdl_t, nvlist_t *,
    nv_alloc_t *, int, uint64_t, uint64_t, uint64_t, void *, void *);
extern cms_errno_t ao_ms_msrinject(cmi_hdl_t, uint_t, uint64_t);

/*
 * Local functions
 */
extern void ao_procnode_scrubber_enable(cmi_hdl_t, ao_ms_data_t *);
extern void ao_pcicfg_write(uint_t, uint_t, uint_t, uint32_t);
extern uint32_t ao_pcicfg_read(uint_t, uint_t, uint_t);
extern void ao_bankstatus_prewrite(cmi_hdl_t, ao_ms_data_t *);
extern void ao_bankstatus_postwrite(cmi_hdl_t, ao_ms_data_t *);

#endif /* _KERNEL */

#ifdef __cplusplus
}
#endif

#endif /* _AO_H */
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#include <sys/types.h>
#include <sys/pghw.h>
#include <sys/cmn_err.h>
#include <sys/sysmacros.h>
#include <sys/fm/protocol.h>
#include <sys/x86_archext.h>
#include <sys/pci_cfgspace.h>

#include "ao.h"

/*
 * AMD Opteron CPU Subroutines
 *
 * The following two tunables are used to determine the scrubbing rates for
 * the D$ and L2$.  The values range from 0x00-0x16 as described in BKDG
 * Scrub Control Register.  A value of zero disables the scrubber.  Values
 * above zero indicate rates in descending order.
 *
 * The current default values are used on several Sun systems.  In the future
 * this code should assign values dynamically based on cache sizing.  If you
 * tune these values manually be aware of the following architectural issue:
 * At present, Opteron can only survive certain kinds of multi-bit errors if
 * they are detected by the scrubbers.  Therefore in general we want these
 * values tuned as high as possible without impacting workload performance.
 */
uint32_t ao_scrub_rate_dcache = 8;	/* 64B every 5.12 us */
uint32_t ao_scrub_rate_l2cache = 9;	/* 64B every 10.2 us */

enum {
	AO_SCRUB_BIOSDEFAULT,		/* retain system default values */
	AO_SCRUB_FIXED,			/* assign ao_scrub_rate_* values */
	AO_SCRUB_MAX			/* assign max of system and tunables */
} ao_scrub_policy = AO_SCRUB_MAX;

void
ao_pcicfg_write(uint_t procnodeid, uint_t func, uint_t reg, uint32_t val)
{
	ASSERT(procnodeid + 24 <= 31);
	ASSERT((func & 7) == func);
	ASSERT((reg & 3) == 0 && reg < 256);

	cmi_pci_putl(0, procnodeid + 24, func, reg, 0, val);
}

uint32_t
ao_pcicfg_read(uint_t procnodeid, uint_t func, uint_t reg)
{
	ASSERT(procnodeid + 24 <= 31);
	ASSERT((func & 7) == func);
	ASSERT((reg & 3) == 0 && reg < 256);

	return (cmi_pci_getl(0, procnodeid + 24, func, reg, 0, 0));
}


/*
 * Return the maximum scrubbing rate between r1 and r2, where r2 is extracted
 * from the specified 'cfg' register value using 'mask' and 'shift'.  If a
 * value is zero, scrubbing is off so return the opposite value.  Otherwise
 * the maximum rate is the smallest non-zero value of the two values.
 */
static uint32_t
ao_scrubber_max(uint32_t r1, uint32_t r2)
{
	if (r1 != 0 && r2 != 0)
		return (MIN(r1, r2));

	return (r1 ? r1 : r2);
}

/*
 * Enable the node-specific hardware scrubbers for the D$ and L2$.  We set
 * the scrubber rate based on a set of tunables defined at the top of the file.
 */
void
ao_procnode_scrubber_enable(cmi_hdl_t hdl, ao_ms_data_t *ao)
{
	uint_t procnodeid = cmi_hdl_procnodeid(hdl);
	union mcreg_scrubctl scrubctl;

	ao->ao_ms_shared->aos_bcfg_scrubctl = MCREG_VAL32(&scrubctl) =
	    ao_pcicfg_read(procnodeid, MC_FUNC_MISCCTL, MC_CTL_REG_SCRUBCTL);

	if (ao_scrub_policy == AO_SCRUB_BIOSDEFAULT)
		return;

	if (ao_scrub_rate_dcache > AMD_NB_SCRUBCTL_RATE_MAX) {
		cmn_err(CE_WARN, "ao_scrub_rate_dcache is too large; "
		    "resetting to 0x%x\n", AMD_NB_SCRUBCTL_RATE_MAX);
		ao_scrub_rate_dcache = AMD_NB_SCRUBCTL_RATE_MAX;
	}

	if (ao_scrub_rate_l2cache > AMD_NB_SCRUBCTL_RATE_MAX) {
		cmn_err(CE_WARN, "ao_scrub_rate_l2cache is too large; "
		    "resetting to 0x%x\n", AMD_NB_SCRUBCTL_RATE_MAX);
		ao_scrub_rate_l2cache = AMD_NB_SCRUBCTL_RATE_MAX;
	}

	switch (ao_scrub_policy) {
	case AO_SCRUB_FIXED:
		/* Use the system values checked above */
		break;

	default:
		cmn_err(CE_WARN, "Unknown ao_scrub_policy value %d - "
		    "using default policy of AO_SCRUB_MAX", ao_scrub_policy);
		/*FALLTHRU*/

	case AO_SCRUB_MAX:
		ao_scrub_rate_dcache =
		    ao_scrubber_max(ao_scrub_rate_dcache,
		    MCREG_FIELD_CMN(&scrubctl, DcacheScrub));

		ao_scrub_rate_l2cache =
		    ao_scrubber_max(ao_scrub_rate_l2cache,
		    MCREG_FIELD_CMN(&scrubctl, L2Scrub));
		break;
	}

	MCREG_FIELD_CMN(&scrubctl, DcacheScrub) = ao_scrub_rate_dcache;
	MCREG_FIELD_CMN(&scrubctl, L2Scrub) = ao_scrub_rate_l2cache;

	ao_pcicfg_write(procnodeid, MC_FUNC_MISCCTL, MC_CTL_REG_SCRUBCTL,
	    MCREG_VAL32(&scrubctl));
}
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright (c) 2006, 2010, Oracle and/or its affiliates. All rights reserved.
 */

/*
 * The CPU module for the AMD Athlon64 and Opteron processors
 */

#include <sys/types.h>
#include <sys/cmn_err.h>
#include <sys/sunddi.h>
#include <sys/cpu_module.h>
#include <sys/cpu_module_ms_impl.h>
#include <sys/cpuvar.h>
#include <sys/x86_archext.h>
#include <sys/kmem.h>
#include <sys/pghw.h>
#include <sys/modctl.h>
#include <sys/mc.h>
#include <sys/mca_x86.h>

#include "ao.h"

int ao_ms_support_disable = 0;

static struct ao_chipshared *ao_shared[AO_MAX_CHIPS];

/*
 * This cpu module supports AMD family 0xf revisions B/C/D/E/F/G.  If
 * a family 0xf cpu beyond the rev G model limit is detected then
 * return ENOTSUP and let the generic x86 CPU module load instead.
 */
uint_t ao_model_limit = 0x6f;

int
ao_ms_init(cmi_hdl_t hdl, void **datap)
{
	uint_t chipid = cmi_hdl_chipid(hdl);
	struct ao_chipshared *sp, *osp;
	ao_ms_data_t *ao;
	uint64_t cap;

	if (ao_ms_support_disable || cmi_hdl_model(hdl) >= ao_model_limit)
		return (ENOTSUP);

	if (!is_x86_feature(x86_featureset, X86FSET_MCA))
		return (ENOTSUP);

	if (cmi_hdl_rdmsr(hdl, IA32_MSR_MCG_CAP, &cap) != CMI_SUCCESS)
		return (ENOTSUP);

	if (!(cap & MCG_CAP_CTL_P))
		return (ENOTSUP);

	if ((cap & MCG_CAP_COUNT_MASK) != AMD_MCA_BANK_COUNT) {
		cmn_err(CE_WARN, "Chip %d core %d has %llu MCA banks, "
		    "expected %u: disabling AMD-specific MCA support on "
		    "this CPU", chipid, cmi_hdl_coreid(hdl),
		    (u_longlong_t)cap & MCG_CAP_COUNT_MASK,
		    AMD_MCA_BANK_COUNT);
		return (ENOTSUP);
	}

	ao = *datap = kmem_zalloc(sizeof (ao_ms_data_t), KM_SLEEP);
	cmi_hdl_hold(hdl);	/* release in fini */
	ao->ao_ms_hdl = hdl;

	/*
	 * Allocate the chipshared structure if it appears not to have been
	 * allocated already (by a sibling core).  Install the newly
	 * allocated pointer atomically in case a sibling core beats
	 * us to it.
	 */
	if ((sp = ao_shared[chipid]) == NULL) {
		sp = kmem_zalloc(sizeof (struct ao_chipshared), KM_SLEEP);
		sp->aos_chiprev = cmi_hdl_chiprev(hdl);
		membar_producer();

		osp = atomic_cas_ptr(&ao_shared[chipid], NULL, sp);
		if (osp != NULL) {
			kmem_free(sp, sizeof (struct ao_chipshared));
			sp = osp;
		}
	}
	ao->ao_ms_shared = sp;

	return (0);
}

/*ARGSUSED*/
void
ao_ms_post_mpstartup(cmi_hdl_t hdl)
{
	(void) ddi_install_driver("mc-amd");
}

cms_api_ver_t _cms_api_version = CMS_API_VERSION_2;

const cms_ops_t _cms_ops = {
	ao_ms_init,			/* cms_init */
	ao_ms_post_startup,		/* cms_post_startup */
	ao_ms_post_mpstartup,		/* cms_post_mpstartup */
	NULL,				/* cms_logout_size */
	ao_ms_mcgctl_val,		/* cms_mcgctl_val */
	ao_ms_bankctl_skipinit,		/* cms_bankctl_skipinit */
	ao_ms_bankctl_val,		/* cms_bankctl_val */
	NULL,				/* cms_bankstatus_skipinit */
	NULL,				/* cms_bankstatus_val */
	ao_ms_mca_init,			/* cms_mca_init */
	ao_ms_poll_ownermask,		/* cms_poll_ownermask */
	NULL,				/* cms_bank_logout */
	ao_ms_error_action,		/* cms_error_action */
	ao_ms_disp_match,		/* cms_disp_match */
	ao_ms_ereport_class,		/* cms_ereport_class */
	NULL,				/* cms_ereport_detector */
	ao_ms_ereport_includestack,	/* cms_ereport_includestack */
	ao_ms_ereport_add_logout,	/* cms_ereport_add_logout */
	ao_ms_msrinject,		/* cms_msrinject */
	NULL,				/* cms_fini */
};

static struct modlcpu modlcpu = {
	&mod_cpuops,
	"AMD Athlon64/Opteron Model-Specific Support"
};

static struct modlinkage modlinkage = {
	MODREV_1,
	(void *)&modlcpu,
	NULL
};

int
_init(void)
{
	return (mod_install(&modlinkage));
}

int
_info(struct modinfo *modinfop)
{
	return (mod_info(&modlinkage, modinfop));
}

int
_fini(void)
{
	return (mod_remove(&modlinkage));
}
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright (c) 2006, 2010, Oracle and/or its affiliates. All rights reserved.
 * Copyright 2022 Oxide Computer Co.
 */

#include <sys/types.h>
#include <sys/regset.h>
#include <sys/privregs.h>
#include <sys/pci_impl.h>
#include <sys/cpuvar.h>
#include <sys/x86_archext.h>
#include <sys/cmn_err.h>
#include <sys/systm.h>
#include <sys/sysmacros.h>
#include <sys/pghw.h>
#include <sys/cyclic.h>
#include <sys/sysevent.h>
#include <sys/smbios.h>
#include <sys/mca_x86.h>
#include <sys/mca_amd.h>
#include <sys/mc.h>
#include <sys/mc_amd.h>
#include <sys/psw.h>
#include <sys/ddi.h>
#include <sys/sunddi.h>
#include <sys/sdt.h>
#include <sys/fm/util.h>
#include <sys/fm/protocol.h>
#include <sys/fm/cpu/AMD.h>
#include <sys/fm/smb/fmsmb.h>
#include <sys/acpi/acpi.h>
#include <sys/acpi/acpi_pci.h>
#include <sys/acpica.h>
#include <sys/cpu_module.h>

#include "ao.h"
#include "ao_mca_disp.h"

#define	AO_F_REVS_FG (X86_CHIPREV_AMD_LEGACY_F_REV_F | \
	X86_CHIPREV_AMD_LEGACY_F_REV_G)

int ao_mca_smi_disable = 1;		/* attempt to disable SMI polling */

extern int x86gentopo_legacy;	/* x86 generic topology support */

struct ao_ctl_init {
	uint32_t ctl_revmask;	/* rev(s) to which this applies */
	uint64_t ctl_bits;	/* mca ctl reg bitmask to set */
};

/*
 * Additional NB MCA ctl initialization for revs F and G
 */
static const struct ao_ctl_init ao_nb_ctl_init[] = {
	{ AO_F_REVS_FG, AMD_NB_CTL_INIT_REV_FG },
	{ X86_CHIPREV_UNKNOWN, 0 }
};

typedef struct ao_bank_cfg {
	uint64_t bank_ctl_init_cmn;			/* Common init value */
	const struct ao_ctl_init *bank_ctl_init_extra;	/* Extra for each rev */
	void (*bank_misc_initfunc)(cmi_hdl_t, ao_ms_data_t *, uint32_t);
	uint_t bank_ctl_mask;
} ao_bank_cfg_t;

static void nb_mcamisc_init(cmi_hdl_t, ao_ms_data_t *, uint32_t);

static const ao_bank_cfg_t ao_bank_cfgs[] = {
	{ AMD_DC_CTL_INIT_CMN, NULL, NULL, AMD_MSR_DC_MASK },
	{ AMD_IC_CTL_INIT_CMN, NULL, NULL, AMD_MSR_IC_MASK },
	{ AMD_BU_CTL_INIT_CMN, NULL, NULL, AMD_MSR_BU_MASK },
	{ AMD_LS_CTL_INIT_CMN, NULL, NULL, AMD_MSR_LS_MASK },
	{ AMD_NB_CTL_INIT_CMN, &ao_nb_ctl_init[0], nb_mcamisc_init,
		AMD_MSR_NB_MASK },
};

static int ao_nbanks = sizeof (ao_bank_cfgs) / sizeof (ao_bank_cfgs[0]);

/*
 * This is quite awful but necessary to work around x86 system vendor's view of
 * the world.  Other operating systems (you know who you are) don't understand
 * Opteron-specific error handling, so BIOS and system vendors often hide these
 * conditions from them by using SMI polling to copy out any errors from the
 * machine-check registers.  When Solaris runs on a system with this feature,
 * we want to disable the SMI polling so we can use FMA instead.  Sadly, there
 * isn't even a standard self-describing way to express the whole situation,
 * so we have to resort to hard-coded values.  This should all be changed to
 * be a self-describing vendor-specific SMBIOS structure in the future.
 */
static const struct ao_smi_disable {
	const char *asd_sys_vendor;	/* SMB_TYPE_SYSTEM vendor prefix */
	const char *asd_sys_product;	/* SMB_TYPE_SYSTEM product prefix */
	const char *asd_bios_vendor;	/* SMB_TYPE_BIOS vendor prefix */
	uint8_t asd_code;		/* output code for SMI disable */
} ao_smi_disable[] = {
	{ "Sun Microsystems", "Galaxy12",
	    "American Megatrends", 0x59 },
	{ "Sun Microsystems", "Sun Fire X4100 Server",
	    "American Megatrends", 0x59 },
	{ "Sun Microsystems", "Sun Fire X4200 Server",
	    "American Megatrends", 0x59 },
	{ NULL, NULL, NULL, 0 }
};

static int
ao_disp_match_r4(uint16_t ref, uint8_t r4)
{
	static const uint16_t ao_r4_map[] = {
		AO_MCA_R4_BIT_ERR,	/* MCAX86_ERRCODE_RRRR_ERR */
		AO_MCA_R4_BIT_RD,	/* MCAX86_ERRCODE_RRRR_RD */
		AO_MCA_R4_BIT_WR,	/* MCAX86_ERRCODE_RRRR_WR */
		AO_MCA_R4_BIT_DRD,	/* MCAX86_ERRCODE_RRRR_DRD */
		AO_MCA_R4_BIT_DWR,	/* MCAX86_ERRCODE_RRRR_DWR */
		AO_MCA_R4_BIT_IRD,	/* MCAX86_ERRCODE_RRRR_IRD */
		AO_MCA_R4_BIT_PREFETCH,	/* MCAX86_ERRCODE_RRRR_PREFETCH */
		AO_MCA_R4_BIT_EVICT,	/* MCAX86_ERRCODE_RRRR_EVICT */
		AO_MCA_R4_BIT_SNOOP	/* MCAX86_ERRCODE_RRRR_SNOOP */
	};

	ASSERT(r4 < sizeof (ao_r4_map) / sizeof (uint16_t));

	return ((ref & ao_r4_map[r4]) != 0);
}

static int
ao_disp_match_pp(uint8_t ref, uint8_t pp)
{
	static const uint8_t ao_pp_map[] = {
		AO_MCA_PP_BIT_SRC,	/* MCAX86_ERRCODE_PP_SRC */
		AO_MCA_PP_BIT_RES,	/* MCAX86_ERRCODE_PP_RES */
		AO_MCA_PP_BIT_OBS,	/* MCAX86_ERRCODE_PP_OBS */
		AO_MCA_PP_BIT_GEN	/* MCAX86_ERRCODE_PP_GEN */
	};

	ASSERT(pp < sizeof (ao_pp_map) / sizeof (uint8_t));

	return ((ref & ao_pp_map[pp]) != 0);
}

static int
ao_disp_match_ii(uint8_t ref, uint8_t ii)
{
	static const uint8_t ao_ii_map[] = {
		AO_MCA_II_BIT_MEM,	/* MCAX86_ERRCODE_II_MEM */
		0,
		AO_MCA_II_BIT_IO,	/* MCAX86_ERRCODE_II_IO */
		AO_MCA_II_BIT_GEN	/* MCAX86_ERRCODE_II_GEN */
	};

	ASSERT(ii < sizeof (ao_ii_map) / sizeof (uint8_t));

	return ((ref & ao_ii_map[ii]) != 0);
}

static uint8_t
bit_strip(uint16_t *codep, uint16_t mask, uint16_t shift)
{
	uint8_t val = (*codep & mask) >> shift;
	*codep &= ~mask;
	return (val);
}

#define	BIT_STRIP(codep, name) \
	bit_strip(codep, MCAX86_ERRCODE_##name##_MASK, \
	MCAX86_ERRCODE_##name##_SHIFT)

/*ARGSUSED*/
static int
ao_disp_match_one(const ao_error_disp_t *aed, uint64_t status, uint32_t rev,
    int bankno)
{
	uint16_t code = MCAX86_ERRCODE(status);
	uint8_t extcode = AMD_EXT_ERRCODE(status);
	uint64_t stat_mask = aed->aed_stat_mask;
	uint64_t stat_mask_res = aed->aed_stat_mask_res;

	/*
	 * If the bank's status register indicates overflow, then we can no
	 * longer rely on the value of CECC: our experience with actual fault
	 * injection has shown that multiple CE's overwriting each other shows
	 * AMD_BANK_STAT_CECC and AMD_BANK_STAT_UECC both set to zero.  This
	 * should be clarified in a future BKDG or by the Revision Guide.
	 * This behaviour is fixed in revision F.
	 */
	if (bankno == AMD_MCA_BANK_NB &&
	    !chiprev_at_least(rev, X86_CHIPREV_AMD_LEGACY_F_REV_F) &&
	    status & MSR_MC_STATUS_OVER) {
		stat_mask &= ~AMD_BANK_STAT_CECC;
		stat_mask_res &= ~AMD_BANK_STAT_CECC;
	}

	if ((status & stat_mask) != stat_mask_res)
		return (0);

	/*
	 * r4 and pp bits are stored separately, so we mask off and compare them
	 * for the code types that use them.  Once we've taken the r4 and pp
	 * bits out of the equation, we can directly compare the resulting code
	 * with the one stored in the ao_error_disp_t.
	 */
	if (AMD_ERRCODE_ISMEM(code)) {
		uint8_t r4 = BIT_STRIP(&code, RRRR);

		if (!ao_disp_match_r4(aed->aed_stat_r4_bits, r4))
			return (0);

	} else if (AMD_ERRCODE_ISBUS(code)) {
		uint8_t r4 = BIT_STRIP(&code, RRRR);
		uint8_t pp = BIT_STRIP(&code, PP);
		uint8_t ii = BIT_STRIP(&code, II);

		if (!ao_disp_match_r4(aed->aed_stat_r4_bits, r4) ||
		    !ao_disp_match_pp(aed->aed_stat_pp_bits, pp) ||
		    !ao_disp_match_ii(aed->aed_stat_ii_bits, ii))
			return (0);
	}

	return (code == aed->aed_stat_code && extcode == aed->aed_stat_extcode);
}

/*ARGSUSED*/
cms_cookie_t
ao_ms_disp_match(cmi_hdl_t hdl, int ismc, int banknum, uint64_t status,
    uint64_t addr, uint64_t misc, void *mslogout)
{
	ao_ms_data_t *ao = cms_hdl_getcmsdata(hdl);
	x86_chiprev_t rev = ao->ao_ms_shared->aos_chiprev;
	const ao_error_disp_t *aed;

	for (aed = ao_error_disp[banknum]; aed->aed_stat_mask != 0; aed++) {
		if (ao_disp_match_one(aed, status, rev, banknum))
			return ((cms_cookie_t)aed);
	}

	return (NULL);
}

/*ARGSUSED*/
void
ao_ms_ereport_class(cmi_hdl_t hdl, cms_cookie_t mscookie,
    const char **cpuclsp, const char **leafclsp)
{
	const ao_error_disp_t *aed = mscookie;

	if (aed != NULL) {
		*cpuclsp = FM_EREPORT_CPU_AMD;
		*leafclsp = aed->aed_class;
	}
}

static int
ao_chip_once(ao_ms_data_t *ao, enum ao_cfgonce_bitnum what)
{
	return (atomic_set_long_excl(&ao->ao_ms_shared->aos_cfgonce,
	    what) == 0 ?  B_TRUE : B_FALSE);
}

/*
 * This knob exists in case any platform has a problem with our default
 * policy of disabling any interrupt registered in the NB MC4_MISC
 * register.  Setting this may cause Solaris and external entities
 * who also have an interest in this register to argue over available
 * telemetry (so setting it is generally not recommended).
 */
int ao_nb_cfg_mc4misc_noseize = 0;

/*
 * The BIOS may have setup to receive SMI on counter overflow.  It may also
 * have locked various fields or made them read-only.  We will clear any
 * SMI request and leave the register locked.  We will also clear the
 * counter and enable counting - while we don't use the counter it is nice
 * to have it enabled for verification and debug work.
 */
static void
nb_mcamisc_init(cmi_hdl_t hdl, ao_ms_data_t *ao, uint32_t rev)
{
	uint64_t val, nval;

	if (!chiprev_matches(rev, AO_F_REVS_FG))
		return;

	if (cmi_hdl_rdmsr(hdl, AMD_MSR_NB_MISC, &val) != CMI_SUCCESS)
		return;

	ao->ao_ms_shared->aos_bcfg_nb_misc = val;

	if (ao_nb_cfg_mc4misc_noseize)
		return;		/* stash BIOS value, but no changes */


	/*
	 * The Valid bit tells us whether the CtrP bit is defined; if it
	 * is the CtrP bit tells us whether an ErrCount field is present.
	 * If not then there is nothing for us to do.
	 */
	if (!(val & AMD_NB_MISC_VALID) || !(val & AMD_NB_MISC_CTRP))
		return;


	nval = val;
	nval |= AMD_NB_MISC_CNTEN;		/* enable ECC error counting */
	nval &= ~AMD_NB_MISC_ERRCOUNT_MASK;	/* clear ErrCount */
	nval &= ~AMD_NB_MISC_OVRFLW;		/* clear Ovrflw */
	nval &= ~AMD_NB_MISC_INTTYPE_MASK;	/* no interrupt on overflow */
	nval |= AMD_NB_MISC_LOCKED;

	if (nval != val) {
		uint64_t locked = val & AMD_NB_MISC_LOCKED;

		if (locked)
			ao_bankstatus_prewrite(hdl, ao);

		(void) cmi_hdl_wrmsr(hdl, AMD_MSR_NB_MISC, nval);

		if (locked)
			ao_bankstatus_postwrite(hdl, ao);
	}
}

/*
 * NorthBridge (NB) MCA Configuration.
 *
 * We add and remove bits from the BIOS-configured value, rather than
 * writing an absolute value.  The variables ao_nb_cfg_{add,remove}_cmn and
 * ap_nb_cfg_{add,remove}_revFG are available for modification via kmdb
 * and /etc/system.  The revision-specific adds and removes are applied
 * after the common changes, and one write is made to the config register.
 * These are not intended for watchdog configuration via these variables -
 * use the watchdog policy below.
 */

/*
 * Bits to be added to the NB configuration register - all revs.
 */
uint32_t ao_nb_cfg_add_cmn = AMD_NB_CFG_ADD_CMN;

/*
 * Bits to be cleared from the NB configuration register - all revs.
 */
uint32_t ao_nb_cfg_remove_cmn = AMD_NB_CFG_REMOVE_CMN;

/*
 * Bits to be added to the NB configuration register - revs F and G.
 */
uint32_t ao_nb_cfg_add_revFG = AMD_NB_CFG_ADD_REV_FG;

/*
 * Bits to be cleared from the NB configuration register - revs F and G.
 */
uint32_t ao_nb_cfg_remove_revFG = AMD_NB_CFG_REMOVE_REV_FG;

struct ao_nb_cfg {
	uint32_t cfg_revmask;
	uint32_t *cfg_add_p;
	uint32_t *cfg_remove_p;
};

static const struct ao_nb_cfg ao_cfg_extra[] = {
	{ AO_F_REVS_FG, &ao_nb_cfg_add_revFG, &ao_nb_cfg_remove_revFG },
	{ X86_CHIPREV_UNKNOWN, NULL, NULL }
};

/*
 * Bits to be used if we configure the NorthBridge (NB) Watchdog.  The watchdog
 * triggers a machine check exception when no response to an NB system access
 * occurs within a specified time interval.
 */
uint32_t ao_nb_cfg_wdog =
    AMD_NB_CFG_WDOGTMRCNTSEL_4095 |
    AMD_NB_CFG_WDOGTMRBASESEL_1MS;

/*
 * The default watchdog policy is to enable it (at the above rate) if it
 * is disabled;  if it is enabled then we leave it enabled at the rate
 * chosen by the BIOS.
 */
enum {
	AO_NB_WDOG_LEAVEALONE,		/* Don't touch watchdog config */
	AO_NB_WDOG_DISABLE,		/* Always disable watchdog */
	AO_NB_WDOG_ENABLE_IF_DISABLED,	/* If disabled, enable at our rate */
	AO_NB_WDOG_ENABLE_FORCE_RATE	/* Enable and set our rate */
} ao_nb_watchdog_policy = AO_NB_WDOG_ENABLE_IF_DISABLED;

static void
ao_nb_cfg(ao_ms_data_t *ao, uint32_t rev)
{
	const struct ao_nb_cfg *nbcp = &ao_cfg_extra[0];
	uint_t procnodeid = pg_plat_hw_instance_id(CPU, PGHW_PROCNODE);
	uint32_t val;

	/*
	 * Read the NorthBridge (NB) configuration register in PCI space,
	 * modify the settings accordingly, and store the new value back.
	 * Note that the stashed BIOS config value aos_bcfg_nb_cfg is used
	 * in ereport payload population to determine ECC syndrome type for
	 * memory errors.
	 */
	ao->ao_ms_shared->aos_bcfg_nb_cfg = val =
	    ao_pcicfg_read(procnodeid, MC_FUNC_MISCCTL, MC_CTL_REG_NBCFG);

	switch (ao_nb_watchdog_policy) {
	case AO_NB_WDOG_LEAVEALONE:
		break;

	case AO_NB_WDOG_DISABLE:
		val &= ~AMD_NB_CFG_WDOGTMRBASESEL_MASK;
		val &= ~AMD_NB_CFG_WDOGTMRCNTSEL_MASK;
		val |= AMD_NB_CFG_WDOGTMRDIS;
		break;

	default:
		cmn_err(CE_NOTE, "ao_nb_watchdog_policy=%d unrecognised, "
		    "using default policy", ao_nb_watchdog_policy);
		/*FALLTHRU*/

	case AO_NB_WDOG_ENABLE_IF_DISABLED:
		if (!(val & AMD_NB_CFG_WDOGTMRDIS))
			break;	/* if enabled leave rate intact */
		/*FALLTHRU*/

	case AO_NB_WDOG_ENABLE_FORCE_RATE:
		val &= ~AMD_NB_CFG_WDOGTMRBASESEL_MASK;
		val &= ~AMD_NB_CFG_WDOGTMRCNTSEL_MASK;
		val &= ~AMD_NB_CFG_WDOGTMRDIS;
		val |= ao_nb_cfg_wdog;
		break;
	}

	/*
	 * Now apply bit adds and removes, first those common to all revs
	 * and then the revision-specific ones.
	 */
	val &= ~ao_nb_cfg_remove_cmn;
	val |= ao_nb_cfg_add_cmn;

	while (nbcp->cfg_revmask != X86_CHIPREV_UNKNOWN) {
		if (chiprev_matches(rev, nbcp->cfg_revmask)) {
			val &= ~(*nbcp->cfg_remove_p);
			val |= *nbcp->cfg_add_p;
		}
		nbcp++;
	}

	ao_pcicfg_write(procnodeid, MC_FUNC_MISCCTL, MC_CTL_REG_NBCFG, val);
}

static void
ao_dram_cfg(ao_ms_data_t *ao, uint32_t rev)
{
	uint_t procnodeid = pg_plat_hw_instance_id(CPU, PGHW_PROCNODE);
	union mcreg_dramcfg_lo dcfglo;

	ao->ao_ms_shared->aos_bcfg_dcfg_lo = MCREG_VAL32(&dcfglo) =
	    ao_pcicfg_read(procnodeid, MC_FUNC_DRAMCTL, MC_DC_REG_DRAMCFGLO);
	ao->ao_ms_shared->aos_bcfg_dcfg_hi =
	    ao_pcicfg_read(procnodeid, MC_FUNC_DRAMCTL, MC_DC_REG_DRAMCFGHI);
#ifdef OPTERON_ERRATUM_172
	if (chiprev_matches(rev, AO_F_REVS_FG) &&
	    MCREG_FIELD_F_revFG(&dcfglo, ParEn)) {
		MCREG_FIELD_F_revFG(&dcfglo, ParEn) = 0;
		ao_pcicfg_write(procnodeid, MC_FUNC_DRAMCTL,
		    MC_DC_REG_DRAMCFGLO, MCREG_VAL32(&dcfglo));
	}
#endif
}

/*
 * This knob exists in case any platform has a problem with our default
 * policy of disabling any interrupt registered in the online spare
 * control register.  Setting this may cause Solaris and external entities
 * who also have an interest in this register to argue over available
 * telemetry (so setting it is generally not recommended).
 */
int ao_nb_cfg_sparectl_noseize = 0;

/*
 * Setup the online spare control register (revs F and G).  We disable
 * any interrupt registered by the BIOS and zero all error counts.
 */
static void
ao_sparectl_cfg(ao_ms_data_t *ao)
{
	uint_t procnodeid = pg_plat_hw_instance_id(CPU, PGHW_PROCNODE);
	union mcreg_sparectl sparectl;
	int chan, cs;

	ao->ao_ms_shared->aos_bcfg_nb_sparectl = MCREG_VAL32(&sparectl) =
	    ao_pcicfg_read(procnodeid, MC_FUNC_MISCCTL, MC_CTL_REG_SPARECTL);

	if (ao_nb_cfg_sparectl_noseize)
		return;	/* stash BIOS value, but no changes */

	/*
	 * If the BIOS has requested SMI interrupt type for ECC count
	 * overflow for a chip-select or channel force those off.
	 */
	MCREG_FIELD_F_revFG(&sparectl, EccErrInt) = 0;
	MCREG_FIELD_F_revFG(&sparectl, SwapDoneInt) = 0;

	/*
	 * Zero EccErrCnt and write this back to all chan/cs combinations.
	 */
	MCREG_FIELD_F_revFG(&sparectl, EccErrCntWrEn) = 1;
	MCREG_FIELD_F_revFG(&sparectl, EccErrCnt) = 0;
	for (chan = 0; chan < MC_CHIP_NDRAMCHAN; chan++) {
		MCREG_FIELD_F_revFG(&sparectl, EccErrCntDramChan) = chan;

		for (cs = 0; cs < MC_CHIP_NCS; cs++) {
			MCREG_FIELD_F_revFG(&sparectl, EccErrCntDramCs) = cs;
			ao_pcicfg_write(procnodeid, MC_FUNC_MISCCTL,
			    MC_CTL_REG_SPARECTL, MCREG_VAL32(&sparectl));
		}
	}
}

int ao_forgive_uc = 0;		/* For test/debug only */
int ao_forgive_pcc = 0;		/* For test/debug only */
int ao_fake_poison = 0;		/* For test/debug only */

uint32_t
ao_ms_error_action(cmi_hdl_t hdl, int ismc, int banknum,
    uint64_t status, uint64_t addr, uint64_t misc, void *mslogout)
{
	const ao_error_disp_t *aed;
	uint32_t retval = 0;
	uint8_t when;
	int en;

	if (ao_forgive_uc)
		retval |= CMS_ERRSCOPE_CLEARED_UC;

	if (ao_forgive_pcc)
		retval |= CMS_ERRSCOPE_CURCONTEXT_OK;

	if (ao_fake_poison && status & MSR_MC_STATUS_UC)
		retval |= CMS_ERRSCOPE_POISONED;

	if (retval)
		return (retval);

	aed = ao_ms_disp_match(hdl, ismc, banknum, status, addr, misc,
	    mslogout);

	/*
	 * If we do not recognise the error let the cpu module apply
	 * the generic criteria to decide how to react.
	 */
	if (aed == NULL)
		return (0);

	en = (status & MSR_MC_STATUS_EN) != 0;

	if ((when = aed->aed_panic_when) == AO_AED_PANIC_NEVER)
		retval |= CMS_ERRSCOPE_IGNORE_ERR;

	if ((when & AO_AED_PANIC_ALWAYS) ||
	    ((when & AO_AED_PANIC_IFMCE) && (en || ismc)))
		retval |= CMS_ERRSCOPE_FORCE_FATAL;

	/*
	 * The original AMD implementation would panic on a machine check
	 * (not a poll) if the status overflow bit was set, with an
	 * exception for the case of rev F or later with an NB error
	 * indicating CECC.  This came from the perception that the
	 * overflow bit was not correctly managed on rev E and earlier, for
	 * example that repeated correctable memeory errors did not set
	 * OVER but somehow clear CECC.
	 *
	 * We will leave the generic support to evaluate overflow errors
	 * and decide to panic on their individual merits, e.g., if PCC
	 * is set and so on.  The AMD docs do say (as Intel does) that
	 * the status information is *all* from the higher-priority
	 * error in the case of an overflow, so it is at least as serious
	 * as the original and we can decide panic etc based on it.
	 */

	return (retval);
}

/*
 * Will need to change for family 0x10
 */
static uint_t
ao_ereport_synd(ao_ms_data_t *ao, uint64_t status, uint_t *typep,
    int is_nb)
{
	if (is_nb) {
		if (ao->ao_ms_shared->aos_bcfg_nb_cfg &
		    AMD_NB_CFG_CHIPKILLECCEN) {
			*typep = AMD_SYNDTYPE_CHIPKILL;
			return (AMD_NB_STAT_CKSYND(status));
		} else {
			*typep = AMD_SYNDTYPE_ECC;
			return (AMD_BANK_SYND(status));
		}
	} else {
		*typep = AMD_SYNDTYPE_ECC;
		return (AMD_BANK_SYND(status));
	}
}

static nvlist_t *
ao_ereport_create_resource_elem(cmi_hdl_t hdl, nv_alloc_t *nva,
    mc_unum_t *unump, int dimmnum)
{
	nvlist_t *nvl, *snvl;
	nvlist_t *board_list = NULL;

	if ((nvl = fm_nvlist_create(nva)) == NULL)	/* freed by caller */
		return (NULL);

	if ((snvl = fm_nvlist_create(nva)) == NULL) {
		fm_nvlist_destroy(nvl, nva ? FM_NVA_RETAIN : FM_NVA_FREE);
		return (NULL);
	}

	(void) nvlist_add_uint64(snvl, FM_FMRI_HC_SPECIFIC_OFFSET,
	    unump->unum_offset);

	if (!x86gentopo_legacy) {
		board_list = cmi_hdl_smb_bboard(hdl);

		if (board_list == NULL) {
			fm_nvlist_destroy(nvl,
			    nva ? FM_NVA_RETAIN : FM_NVA_FREE);
			fm_nvlist_destroy(snvl,
			    nva ? FM_NVA_RETAIN : FM_NVA_FREE);
			return (NULL);
		}

		fm_fmri_hc_create(nvl, FM_HC_SCHEME_VERSION, NULL, snvl,
		    board_list, 4,
		    "chip", cmi_hdl_smb_chipid(hdl),
		    "memory-controller", unump->unum_mc,
		    "dimm", unump->unum_dimms[dimmnum],
		    "rank", unump->unum_rank);
	} else {
		fm_fmri_hc_set(nvl, FM_HC_SCHEME_VERSION, NULL, snvl, 5,
		    "motherboard", unump->unum_board,
		    "chip", unump->unum_chip,
		    "memory-controller", unump->unum_mc,
		    "dimm", unump->unum_dimms[dimmnum],
		    "rank", unump->unum_rank);
	}

	fm_nvlist_destroy(snvl, nva ? FM_NVA_RETAIN : FM_NVA_FREE);

	return (nvl);
}

static void
ao_ereport_add_resource(cmi_hdl_t hdl, nvlist_t *payload, nv_alloc_t *nva,
    mc_unum_t *unump)
{

	nvlist_t *elems[MC_UNUM_NDIMM];
	int nelems = 0;
	int i;

	for (i = 0; i < MC_UNUM_NDIMM; i++) {
		if (unump->unum_dimms[i] == MC_INVALNUM)
			break;

		if ((elems[nelems] = ao_ereport_create_resource_elem(hdl, nva,
		    unump, i)) == NULL)
			break;

		nelems++;
	}

	if (nelems == 0)
		return;

	fm_payload_set(payload, FM_EREPORT_PAYLOAD_NAME_RESOURCE,
	    DATA_TYPE_NVLIST_ARRAY, nelems, elems, NULL);

	for (i = 0; i < nelems; i++)
		fm_nvlist_destroy(elems[i], nva ? FM_NVA_RETAIN : FM_NVA_FREE);
}

/*ARGSUSED*/
void
ao_ms_ereport_add_logout(cmi_hdl_t hdl, nvlist_t *ereport,
    nv_alloc_t *nva, int banknum, uint64_t status, uint64_t addr,
    uint64_t misc, void *mslogout, cms_cookie_t mscookie)
{
	ao_ms_data_t *ao = cms_hdl_getcmsdata(hdl);
	const ao_error_disp_t *aed = mscookie;
	uint_t synd, syndtype;
	uint64_t members;

	if (aed == NULL)
		return;

	members = aed->aed_ereport_members;

	synd = ao_ereport_synd(ao, status, &syndtype,
	    banknum == AMD_MCA_BANK_NB);

	if (members & FM_EREPORT_PAYLOAD_FLAG_SYND) {
		fm_payload_set(ereport, FM_EREPORT_PAYLOAD_NAME_SYND,
		    DATA_TYPE_UINT16, synd, NULL);
	}

	if (members & FM_EREPORT_PAYLOAD_FLAG_SYND_TYPE) {
		fm_payload_set(ereport, FM_EREPORT_PAYLOAD_NAME_SYND_TYPE,
		    DATA_TYPE_STRING, (syndtype == AMD_SYNDTYPE_CHIPKILL ?
		    "C4" : "E"), NULL);
	}

	if (members & FM_EREPORT_PAYLOAD_FLAG_RESOURCE) {
		mc_unum_t unum;

		if (((aed->aed_flags & AO_AED_FLAGS_ADDRTYPE) ==
		    AO_AED_F_PHYSICAL) && (status & MSR_MC_STATUS_ADDRV) &&
		    cmi_mc_patounum(addr, aed->aed_addrvalid_hi,
		    aed->aed_addrvalid_lo, synd, syndtype, &unum) ==
		    CMI_SUCCESS)
			ao_ereport_add_resource(hdl, ereport, nva, &unum);
	}
}

/*ARGSUSED*/
boolean_t
ao_ms_ereport_includestack(cmi_hdl_t hdl, cms_cookie_t mscookie)
{
	const ao_error_disp_t *aed = mscookie;

	if (aed == NULL)
		return (0);

	return ((aed->aed_ereport_members &
	    FM_EREPORT_PAYLOAD_FLAG_STACK) != 0);
}

cms_errno_t
ao_ms_msrinject(cmi_hdl_t hdl, uint_t msr, uint64_t val)
{
	ao_ms_data_t *ao = cms_hdl_getcmsdata(hdl);
	cms_errno_t rv = CMSERR_BADMSRWRITE;

	ao_bankstatus_prewrite(hdl, ao);
	if (cmi_hdl_wrmsr(hdl, msr, val) == CMI_SUCCESS)
		rv = CMS_SUCCESS;
	ao_bankstatus_postwrite(hdl, ao);

	return (rv);
}

/*ARGSUSED*/
uint64_t
ao_ms_mcgctl_val(cmi_hdl_t hdl, int nbanks, uint64_t def)
{
	return ((1ULL << nbanks) - 1);
}

boolean_t
ao_ms_bankctl_skipinit(cmi_hdl_t hdl, int banknum)
{
	ao_ms_data_t *ao = cms_hdl_getcmsdata(hdl);

	if (banknum != AMD_MCA_BANK_NB)
		return (B_FALSE);

	/*
	 * If we are the first to atomically set the "I'll do it" bit
	 * then return B_FALSE (do not skip), otherwise skip with B_TRUE.
	 */
	return (ao_chip_once(ao, AO_CFGONCE_NBMCA) == B_TRUE ?
	    B_FALSE : B_TRUE);
}

uint64_t
ao_ms_bankctl_val(cmi_hdl_t hdl, int banknum, uint64_t def)
{
	ao_ms_data_t *ao = cms_hdl_getcmsdata(hdl);
	const struct ao_ctl_init *extrap;
	const ao_bank_cfg_t *bankcfg;
	uint64_t mcictl;
	x86_chiprev_t rev = ao->ao_ms_shared->aos_chiprev;

	if (banknum >= sizeof (ao_bank_cfgs) / sizeof (ao_bank_cfgs[0]))
		return (def);

	bankcfg = &ao_bank_cfgs[banknum];
	extrap = bankcfg->bank_ctl_init_extra;

	mcictl = bankcfg->bank_ctl_init_cmn;

	while (extrap != NULL && extrap->ctl_revmask != X86_CHIPREV_UNKNOWN) {
		if (chiprev_matches(rev, extrap->ctl_revmask))
			mcictl |= extrap->ctl_bits;
		extrap++;
	}

	return (mcictl);
}

/*ARGSUSED*/
void
ao_bankstatus_prewrite(cmi_hdl_t hdl, ao_ms_data_t *ao)
{
#ifndef __xpv
	uint64_t hwcr;

	if (cmi_hdl_rdmsr(hdl, MSR_AMD_HWCR, &hwcr) != CMI_SUCCESS)
		return;

	ao->ao_ms_hwcr_val = hwcr;

	if (!(hwcr & AMD_HWCR_MCI_STATUS_WREN)) {
		hwcr |= AMD_HWCR_MCI_STATUS_WREN;
		(void) cmi_hdl_wrmsr(hdl, MSR_AMD_HWCR, hwcr);
	}
#endif
}

/*ARGSUSED*/
void
ao_bankstatus_postwrite(cmi_hdl_t hdl, ao_ms_data_t *ao)
{
#ifndef __xpv
	uint64_t hwcr = ao->ao_ms_hwcr_val;

	if (!(hwcr & AMD_HWCR_MCI_STATUS_WREN)) {
		hwcr &= ~AMD_HWCR_MCI_STATUS_WREN;
		(void) cmi_hdl_wrmsr(hdl, MSR_AMD_HWCR, hwcr);
	}
#endif
}

void
ao_ms_mca_init(cmi_hdl_t hdl, int nbanks)
{
	ao_ms_data_t *ao = cms_hdl_getcmsdata(hdl);
	x86_chiprev_t rev = ao->ao_ms_shared->aos_chiprev;
	ao_ms_mca_t *mca = &ao->ao_ms_mca;
	uint64_t *maskp;
	int i;

	maskp = mca->ao_mca_bios_cfg.bcfg_bank_mask = kmem_zalloc(nbanks *
	    sizeof (uint64_t), KM_SLEEP);

	/*
	 * Read the bank ctl mask MSRs, but only as many as we know
	 * certainly exist - don't calculate the register address.
	 * Also initialize the MCi_MISC register where required.
	 */
	for (i = 0; i < MIN(nbanks, ao_nbanks); i++) {
		(void) cmi_hdl_rdmsr(hdl, ao_bank_cfgs[i].bank_ctl_mask,
		    maskp++);
		if (ao_bank_cfgs[i].bank_misc_initfunc != NULL)
			ao_bank_cfgs[i].bank_misc_initfunc(hdl, ao, rev);

	}

	if (ao_chip_once(ao, AO_CFGONCE_NBCFG) == B_TRUE) {
		ao_nb_cfg(ao, rev);

		if (chiprev_matches(rev, AO_F_REVS_FG))
			ao_sparectl_cfg(ao);
	}

	if (ao_chip_once(ao, AO_CFGONCE_DRAMCFG) == B_TRUE)
		ao_dram_cfg(ao, rev);

	ao_procnode_scrubber_enable(hdl, ao);
}

/*
 * Note that although this cpu module is loaded before the PSMs are
 * loaded (and hence before acpica is loaded), this function is
 * called from post_startup(), after PSMs are initialized and acpica
 * is loaded.
 */
static int
ao_acpi_find_smicmd(int *asd_port)
{
	ACPI_TABLE_FADT *fadt = NULL;

	/*
	 * AcpiGetTable works even if ACPI is disabled, so a failure
	 * here means we weren't able to retreive a pointer to the FADT.
	 */
	if (AcpiGetTable(ACPI_SIG_FADT, 1, (ACPI_TABLE_HEADER **)&fadt) !=
	    AE_OK)
		return (-1);

	ASSERT(fadt != NULL);

	*asd_port = fadt->SmiCommand;
	return (0);
}

/*ARGSUSED*/
void
ao_ms_post_startup(cmi_hdl_t hdl)
{
	const struct ao_smi_disable *asd;
	id_t id;
	int rv = -1, asd_port;

	smbios_system_t sy;
	smbios_bios_t sb;
	smbios_info_t si;

	/*
	 * Fetch the System and BIOS vendor strings from SMBIOS and see if they
	 * match a value in our table.  If so, disable SMI error polling.  This
	 * is grotesque and should be replaced by self-describing vendor-
	 * specific SMBIOS data or a specification enhancement instead.
	 */
	if (ao_mca_smi_disable && ksmbios != NULL &&
	    smbios_info_bios(ksmbios, &sb) != SMB_ERR &&
	    (id = smbios_info_system(ksmbios, &sy)) != SMB_ERR &&
	    smbios_info_common(ksmbios, id, &si) != SMB_ERR) {

		for (asd = ao_smi_disable; asd->asd_sys_vendor != NULL; asd++) {
			if (strncmp(asd->asd_sys_vendor, si.smbi_manufacturer,
			    strlen(asd->asd_sys_vendor)) != 0 ||
			    strncmp(asd->asd_sys_product, si.smbi_product,
			    strlen(asd->asd_sys_product)) != 0 ||
			    strncmp(asd->asd_bios_vendor, sb.smbb_vendor,
			    strlen(asd->asd_bios_vendor)) != 0)
				continue;

			/*
			 * Look for the SMI_CMD port in the ACPI FADT,
			 * if the port is 0, this platform doesn't support
			 * SMM, so there is no SMI error polling to disable.
			 */
			if ((rv = ao_acpi_find_smicmd(&asd_port)) == 0 &&
			    asd_port != 0) {
				cmn_err(CE_CONT, "?SMI polling disabled in "
				    "favor of Solaris Fault Management for "
				    "AMD Processors\n");

				outb(asd_port, asd->asd_code);

			} else if (rv < 0) {
				cmn_err(CE_CONT, "?Solaris Fault Management "
				    "for AMD Processors could not disable SMI "
				    "polling because an error occurred while "
				    "trying to determine the SMI command port "
				    "from the ACPI FADT table\n");
			}
			break;
		}
	}
}
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 *
 * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#ifndef _AO_MCA_DISP_H
#define	_AO_MCA_DISP_H

#include <sys/types.h>
#include <sys/mca_amd.h>
#include <sys/fm/cpu/AMD.h>

#include <ao.h>

#ifdef __cplusplus
extern "C" {
#endif

#define	AO_MCA_PP_BIT_SRC	0x1
#define	AO_MCA_PP_BIT_RES	0x2
#define	AO_MCA_PP_BIT_OBS	0x4
#define	AO_MCA_PP_BIT_GEN	0x8

#define	AO_MCA_II_BIT_MEM	0x1
#define	AO_MCA_II_BIT_IO	0x2
#define	AO_MCA_II_BIT_GEN	0x4

#define	AO_MCA_R4_BIT_ERR	0x001
#define	AO_MCA_R4_BIT_RD	0x002
#define	AO_MCA_R4_BIT_WR	0x004
#define	AO_MCA_R4_BIT_DRD	0x008
#define	AO_MCA_R4_BIT_DWD	0x010
#define	AO_MCA_R4_BIT_DWR	0x020
#define	AO_MCA_R4_BIT_IRD	0x040
#define	AO_MCA_R4_BIT_PREFETCH	0x080
#define	AO_MCA_R4_BIT_EVICT	0x100
#define	AO_MCA_R4_BIT_SNOOP	0x200

extern const ao_error_disp_t *ao_error_disp[];

#ifdef __cplusplus
}
#endif

#endif /* _AO_MCA_DISP_H */
# CDDL HEADER START
#
# The contents of this file are subject to the terms of the
# Common Development and Distribution License (the "License").
# You may not use this file except in compliance with the License.
#
# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
# or http://www.opensolaris.org/os/licensing.
# See the License for the specific language governing permissions
# and limitations under the License.
#
# When distributing Covered Code, include this CDDL HEADER in each
# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
# If applicable, add the following below this CDDL HEADER, with the
# fields enclosed by brackets "[]" replaced with your own identifying
# information: Portions Copyright [yyyy] [name of copyright owner]
#
# CDDL HEADER END
#
# Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
# Use is subject to license terms.
#
# ident	"%Z%%M%	%I%	%E% SMI"
#

funcunit = dc

desc = Correctable D$ data infill from system memory
error = ereport.cpu.amd.dc.inf_sys_ecc1

mask on  = AMD_BANK_STAT_CECC
mask off = MSR_MC_STATUS_UC, AMD_BANK_STAT_UECC

#	ext	type	pp	t	rrrr	addr	ii	ll	tt
#       ------- -------	-------	-------	-------	-------	------- ------- -----
code =	0000	bus	src	0	drd	<39:6>	mem/io	lg	-

panic = never
flags = physical
errtype =

# ---

desc = Correctable D$ data infill from L2$
error = ereport.cpu.amd.dc.inf_l2_ecc1

mask on  = AMD_BANK_STAT_CECC
mask off = MSR_MC_STATUS_UC, AMD_BANK_STAT_UECC

#	ext	type	pp	t	rrrr	addr	ii	ll	tt
#       ------- -------	-------	-------	-------	-------	------- ------- -----
code =	0000	mem	-	-	drd	<39:6>	-	l2	data

panic = never
flags = physical
errtype =

# ---

desc = Uncorrectable D$ data infill from system memory
error =  ereport.cpu.amd.dc.inf_sys_eccm

mask on  = MSR_MC_STATUS_UC, AMD_BANK_STAT_UECC
mask off = AMD_BANK_STAT_CECC

#	ext	type	pp	t	rrrr	addr	ii	ll	tt
#       ------- -------	-------	-------	-------	-------	------- ------- -----
code =	0000	bus	src	0	drd	<39:6>	mem/io	lg	-

panic = always
flags = physical
errtype =

# ---

desc = Uncorrectable D$ data infill from L2$
error = ereport.cpu.amd.dc.inf_l2_eccm

mask on  = MSR_MC_STATUS_UC, AMD_BANK_STAT_UECC
mask off = AMD_BANK_STAT_CECC

#	ext	type	pp	t	rrrr	addr	ii	ll	tt
#       ------- -------	-------	-------	-------	-------	------- ------- -----
code =	0000	mem	-	-	drd	<39:6>	-	l2	data

panic = always
flags = physical
errtype =

# ---

desc = Correctable single-bit error in Data Array from scrub
error = ereport.cpu.amd.dc.data_ecc1

mask on  = AMD_BANK_STAT_CECC, AMD_BANK_STAT_SCRUB
mask off = MSR_MC_STATUS_UC, AMD_BANK_STAT_UECC

#	ext	type	pp	t	rrrr	addr	ii	ll	tt
#       ------- -------	-------	-------	-------	-------	------- ------- -----
code =	0000	mem	-	-	err	<11:3>	-	l1	data

panic = never
flags = physical
errtype =

# ---

desc = Uncorrectable single-bit error in Data Array
error = ereport.cpu.amd.dc.data_ecc1_uc

mask on  = MSR_MC_STATUS_UC, AMD_BANK_STAT_CECC
mask off = AMD_BANK_STAT_SCRUB

#	ext	type	pp	t	rrrr	addr	ii	ll	tt
#       ------- -------	-------	-------	-------	-------	------- ------- -----
code =	0000	mem	-	-	drd/dwr	<39:3>	-	l1	data
code =	0000	mem	-	-	ev/snp	<11:6>	-	l1	data

panic = always
flags = physical
errtype =

# ---

desc = Uncorrectable multi-bit error in Data Array
error = ereport.cpu.amd.dc.data_eccm

mask on  = MSR_MC_STATUS_UC, AMD_BANK_STAT_UECC
mask off = AMD_BANK_STAT_CECC, AMD_BANK_STAT_SCRUB

#	ext	type	pp	t	rrrr	addr	ii	ll	tt
#       ------- -------	-------	-------	-------	-------	------- ------- -----
code =	0000	mem	-	-	drd/dwr	<39:3>	-	l1	data
code =	0000	mem	-	-	ev/snp	<11:6>	-	l1	data

panic = always
flags = physical
errtype =

# ---

desc = Uncorrectable multi-bit error in Data Array from scrub
error = ereport.cpu.amd.dc.data_eccm

mask on  = MSR_MC_STATUS_UC, AMD_BANK_STAT_UECC, AMD_BANK_STAT_SCRUB
mask off = AMD_BANK_STAT_CECC

#	ext	type	pp	t	rrrr	addr	ii	ll	tt
#       ------- -------	-------	-------	-------	-------	------- ------- -----
code =	0000	mem	-	-	err	<11:3>	-	l1	data

panic = always
flags = physical
errtype =

# ---

desc = Main Tag Array Parity Error
error = ereport.cpu.amd.dc.tag_par

mask on  = MSR_MC_STATUS_UC
mask off = AMD_BANK_STAT_CECC, AMD_BANK_STAT_UECC

#	ext	type	pp	t	rrrr	addr	ii	ll	tt
#       ------- -------	-------	-------	-------	-------	------- ------- -----
code =	0000	mem	-	-	drd/dwr	<39:3>	-	l1	data

panic = always
flags = physical
errtype =

# ---

desc = Snoop Tag Array Parity Error
error = ereport.cpu.amd.dc.stag_par

mask on  = MSR_MC_STATUS_UC
mask off = AMD_BANK_STAT_CECC, AMD_BANK_STAT_UECC

#	ext	type	pp	t	rrrr	addr	ii	ll	tt
#       ------- -------	-------	-------	-------	-------	------- ------- -----
code =	0000	mem	-	-	snp/ev	<11:6>	-	l1	data

panic = always
flags = physical
errtype =

# ---

desc = L1 DTLB Parity Error
error = ereport.cpu.amd.dc.l1tlb_par

mask on  = MSR_MC_STATUS_UC
mask off = AMD_BANK_STAT_CECC, AMD_BANK_STAT_UECC

#	ext	type	pp	t	rrrr	addr	ii	ll	tt
#       ------- -------	-------	-------	-------	-------	------- ------- -----
code =	0000	tlb	-	-	-	<47:12>	-	l1	data

panic = always
flags = linear
errtype =

# ---

desc = L1 DTLB Parity Error (multimatch)
error = ereport.cpu.amd.dc.l1tlb_par

mask on  = MSR_MC_STATUS_UC
mask off = AMD_BANK_STAT_CECC, AMD_BANK_STAT_UECC

#	ext	type	pp	t	rrrr	addr	ii	ll	tt
#       ------- -------	-------	-------	-------	-------	------- ------- -----
code =	0001	tlb	-	-	-	<47:12>	-	l1	data

panic = always
flags =  linear
errtype =

# ---

desc = L2 DTLB Parity Error
error = ereport.cpu.amd.dc.l2tlb_par

mask on  = MSR_MC_STATUS_UC
mask off = AMD_BANK_STAT_CECC, AMD_BANK_STAT_UECC

#	ext	type	pp	t	rrrr	addr	ii	ll	tt
#       ------- -------	-------	-------	-------	-------	------- ------- -----
code =	0000	tlb	-	-	-	<47:12>	-	l2	data

panic = always
flags = linear
errtype =

# ---

desc = L2 DTLB Parity Error (multimatch)
error = ereport.cpu.amd.dc.l2tlb_par

mask on  = MSR_MC_STATUS_UC
mask off = AMD_BANK_STAT_CECC, AMD_BANK_STAT_UECC

#	ext	type	pp	t	rrrr	addr	ii	ll	tt
#       ------- -------	-------	-------	-------	-------	------- ------- -----
code =	0001	tlb	-	-	-	<47:12>	-	l2	data

panic = always
flags = linear
errtype =

#
# Instruction Cache Functional Unit
#

funcunit = ic

desc = Correctable I$ data infill from system memory
error = ereport.cpu.amd.ic.inf_sys_ecc1

mask on  = AMD_BANK_STAT_CECC
mask off = MSR_MC_STATUS_UC, AMD_BANK_STAT_UECC

#	ext	type	pp	t	rrrr	addr	ii	ll	tt
#       ------- -------	-------	-------	-------	-------	------- ------- -----
code =	0000	bus	src	0	ird	<39:6>	mem	lg	-

panic = never
flags = physical
errtype =

# ----

desc = Correctable I$ data infill from L2$
error = ereport.cpu.amd.ic.inf_l2_ecc1

mask on  = AMD_BANK_STAT_CECC
mask off = MSR_MC_STATUS_UC, AMD_BANK_STAT_UECC

#	ext	type	pp	t	rrrr	addr	ii	ll	tt
#       ------- -------	-------	-------	-------	-------	------- ------- -----
code =	0000	mem	-	-	ird	<39:6>	-	l2	instr

panic = never
flags = physical
errtype =

# ----

desc = Uncorrectable I$ data infill from system memory
error = ereport.cpu.amd.ic.inf_sys_eccm

mask on  = AMD_BANK_STAT_UECC, MSR_MC_STATUS_UC
mask off = AMD_BANK_STAT_CECC

#	ext	type	pp	t	rrrr	addr	ii	ll	tt
#       ------- -------	-------	-------	-------	-------	------- ------- -----
code =	0000	bus	src	0	ird	<39:6>	mem	lg	-

panic = always
flags = physical
errtype =

# ---

desc = Uncorrectable I$ data infill from L2$
error = ereport.cpu.amd.ic.inf_l2_eccm

mask on  = AMD_BANK_STAT_UECC, MSR_MC_STATUS_UC
mask off = AMD_BANK_STAT_CECC

#	ext	type	pp	t	rrrr	addr	ii	ll	tt
#       ------- -------	-------	-------	-------	-------	------- ------- -----
code =	0000	mem	-	-	ird	<39:6>	-	l2	instr

panic = always
flags = physical
errtype =

# ---

desc = Data Array Parity Error
error = ereport.cpu.amd.ic.data_par

mask on  = 
mask off = AMD_BANK_STAT_CECC, AMD_BANK_STAT_UECC, MSR_MC_STATUS_UC

#	ext	type	pp	t	rrrr	addr	ii	ll	tt
#       ------- -------	-------	-------	-------	-------	------- ------- -----
code =	0000	mem	-	-	ird	<47:4>	-	l1	instr

panic = never
flags = linear
errtype =

# ---

desc = Main Tag Array Parity Error
error = ereport.cpu.amd.ic.tag_par

mask on  = 
mask off = AMD_BANK_STAT_CECC, AMD_BANK_STAT_UECC, MSR_MC_STATUS_UC

#	ext	type	pp	t	rrrr	addr	ii	ll	tt
#       ------- -------	-------	-------	-------	-------	------- ------- -----
code =	0000	mem	-	-	ird	<47:6>	-	l1	instr
code =	0000	mem	-	-	ev	none	-	l1	instr

panic = never
flags = linear
errtype =

# ---

desc = Snoop Tag Array Parity Error
error = ereport.cpu.amd.ic.stag_par

mask on  = MSR_MC_STATUS_UC
mask off = AMD_BANK_STAT_CECC, AMD_BANK_STAT_UECC

#	ext	type	pp	t	rrrr	addr	ii	ll	tt
#       ------- -------	-------	-------	-------	-------	------- ------- -----
code =	0000	mem	-	-	snp	<39:6>	-	l1	instr
code =	0000	mem	-	-	ev	none	-	l1	instr

panic = always
flags = physical
errtype =

# ---

desc = L1 ITLB Parity Error
error = ereport.cpu.amd.ic.l1tlb_par

mask on  =
mask off = AMD_BANK_STAT_CECC, AMD_BANK_STAT_UECC, MSR_MC_STATUS_UC

#	ext	type	pp	t	rrrr	addr	ii	ll	tt
#       ------- -------	-------	-------	-------	-------	------- ------- -----
code =	0000	tlb	-	-	-	<47:12>	-	l1	instr

panic = never
flags = linear,pagealigned
errtype =

# ---

desc = L1 ITLB Parity Error (multimatch)
error = ereport.cpu.amd.ic.l1tlb_par

mask on  =
mask off = AMD_BANK_STAT_CECC, AMD_BANK_STAT_UECC, MSR_MC_STATUS_UC

#	ext	type	pp	t	rrrr	addr	ii	ll	tt
#       ------- -------	-------	-------	-------	-------	------- ------- -----
code =	0001	tlb	-	-	-	<47:12>	-	l1	instr

panic = never
flags = linear,pagealigned
errtype =

# ---

desc = L2 ITLB Parity Error
error = ereport.cpu.amd.ic.l2tlb_par

mask on  =
mask off = AMD_BANK_STAT_CECC, AMD_BANK_STAT_UECC, MSR_MC_STATUS_UC

#	ext	type	pp	t	rrrr	addr	ii	ll	tt
#       ------- -------	-------	-------	-------	-------	------- ------- -----
code =	0000	tlb	-	-	-	<47:12>	-	l2	instr

panic = never
flags = linear
errtype =

# ---

desc = L2 ITLB Parity Error (multimatch)
error = ereport.cpu.amd.ic.l2tlb_par

mask on  =
mask off = AMD_BANK_STAT_CECC, AMD_BANK_STAT_UECC, MSR_MC_STATUS_UC

#	ext	type	pp	t	rrrr	addr	ii	ll	tt
#       ------- -------	-------	-------	-------	-------	------- ------- -----
code =	0001	tlb	-	-	-	<47:12>	-	l2	instr

panic = never
flags = linear
errtype =

# ---

desc = System Data Read Error
error = ereport.cpu.amd.ic.rdde

mask on  = MSR_MC_STATUS_UC
mask off = AMD_BANK_STAT_CECC, AMD_BANK_STAT_UECC

#	ext	type	pp	t	rrrr	addr	ii	ll	tt
#       ------- -------	-------	-------	-------	-------	------- ------- -----
code =	0000	bus	src	0	ird	none	mem	lg	-

panic = ifmce
flags =
errtype =

#
# ---
#

funcunit = bu

# ---

desc = L2 data array single-bit ECC during TLB reload, snoop, or copyback
error = ereport.cpu.amd.bu.l2d_ecc1

mask on  = AMD_BANK_STAT_CECC
mask off = AMD_BANK_STAT_UECC, MSR_MC_STATUS_UC

#	ext	type	pp	t	rrrr	addr	ii	ll	tt
#       ------- -------	-------	-------	-------	-------	------- ------- -----
code =	0000	mem	-	-	rd/snp/ev <39:6> -	l2	gen

panic = never
flags = physical
errtype =

# ---

desc = L2 data array multi-bit ECC during TLB reload, snoop, or copyback
error = ereport.cpu.amd.bu.l2d_eccm

mask on  = AMD_BANK_STAT_UECC, MSR_MC_STATUS_UC
mask off = AMD_BANK_STAT_CECC

#	ext	type	pp	t	rrrr	addr	ii	ll	tt
#       ------- -------	-------	-------	-------	-------	------- ------- -----
code =	0000	mem	-	-	rd/snp/ev <39:6> -	l2	gen

panic = always
flags = physical
errtype =

# ---

desc = L2 main tag array single-bit ECC error on scrubber access
error = ereport.cpu.amd.bu.l2t_ecc1

mask on  = AMD_BANK_STAT_CECC, AMD_BANK_STAT_SCRUB
mask off = AMD_BANK_STAT_UECC, MSR_MC_STATUS_UC

#	ext	type	pp	t	rrrr	addr	ii	ll	tt
#       ------- -------	-------	-------	-------	-------	------- ------- -----
code =	0010	mem	-	-	err	<15:0>	-	l2	instr

panic = never
flags = physical,l2setway
errtype =

# ---

desc = L2 main tag array multi-bit ECC error on scrubber access
error = ereport.cpu.amd.bu.l2t_eccm

mask on  = AMD_BANK_STAT_UECC, MSR_MC_STATUS_UC, AMD_BANK_STAT_SCRUB
mask off = AMD_BANK_STAT_CECC

#	ext	type	pp	t	rrrr	addr	ii	ll	tt
#       ------- -------	-------	-------	-------	-------	------- ------- -----
code =	0010	mem	-	-	err	<15:0>	-	l2	instr

panic = always
flags = physical,l2setway
errtype =

# ---

desc = L2 main tag array parity error on I$ fetch
error = ereport.cpu.amd.bu.l2t_par

mask on  = MSR_MC_STATUS_UC
mask off = AMD_BANK_STAT_CECC, AMD_BANK_STAT_UECC

#	ext	type	pp	t	rrrr	addr	ii	ll	tt
#       ------- -------	-------	-------	-------	-------	------- ------- -----
code =	0010	mem	-	-	ird	<15:0>	-	l2	instr

panic = always
flags = physical,l2setway
errtype =

# ---

desc = L2 main tag array parity error on D$ fetch
error = ereport.cpu.amd.bu.l2t_par

mask on  = MSR_MC_STATUS_UC
mask off = AMD_BANK_STAT_CECC, AMD_BANK_STAT_UECC

#	ext	type	pp	t	rrrr	addr	ii	ll	tt
#       ------- -------	-------	-------	-------	-------	------- ------- -----
code =	0010	mem	-	-	drd	<15:0>	-	l2	data

panic = always
flags = physical,l2setway
errtype =

# ---

desc = L2 main tag array parity error on TLB reload, snoop, or copyback
error = ereport.cpu.amd.bu.l2t_par

mask on  = MSR_MC_STATUS_UC
mask off = AMD_BANK_STAT_CECC, AMD_BANK_STAT_UECC

#	ext	type	pp	t	rrrr	addr	ii	ll	tt
#       ------- -------	-------	-------	-------	-------	------- ------- -----
code =	0010	mem	-	-	rd/snp/ev <15:0> -	l2	gen

panic = always
flags = physical,l2setway
errtype =

# ---

desc = L2 main tag array parity error on scrubber access
error = ereport.cpu.amd.bu.l2t_par

mask on  = MSR_MC_STATUS_UC, AMD_BANK_STAT_SCRUB
mask off = AMD_BANK_STAT_CECC, AMD_BANK_STAT_UECC

#	ext	type	pp	t	rrrr	addr	ii	ll	tt
#       ------- -------	-------	-------	-------	-------	------- ------- -----
code =	0010	mem	-	-	err	<15:0>	-	l2	instr

panic = always
flags = physical,l2setway
errtype =

# ---

desc = System data single-bit ECC for hardware prefetch or TLB reload
error = ereport.cpu.amd.bu.s_ecc1

mask on  = AMD_BANK_STAT_CECC
mask off = MSR_MC_STATUS_UC, AMD_BANK_STAT_UECC

#	ext	type	pp	t	rrrr	addr	ii	ll	tt
#       ------- -------	-------	-------	-------	-------	------- ------- -----
code =	0000	bus	src	0	rd	<39:6>	mem/io	lg	-
code =	0000	bus	src	0	pf	none	mem/io	lg	-

panic = never
flags = physical
errtype =

# ---

desc = System data multi-bit ECC for hardware prefetch or TLB reload
error = ereport.cpu.amd.bu.s_eccm

mask on  = MSR_MC_STATUS_UC, AMD_BANK_STAT_UECC
mask off = AMD_BANK_STAT_CECC

#	ext	type	pp	t	rrrr	addr	ii	ll	tt
#       ------- -------	-------	-------	-------	-------	------- ------- -----
code =	0000	bus	src	0	rd	<39:6>	mem/io	lg	-
code =	0000	bus	src	0	pf	none	mem/io	lg	-

panic = always
flags = physical
errtype =

# ---

desc = System data read error for TLB reload or hardware prefetch
error = ereport.cpu.amd.bu.s_rde

mask on  = MSR_MC_STATUS_UC
mask off = AMD_BANK_STAT_CECC, AMD_BANK_STAT_UECC

#	ext	type	pp	t	rrrr	addr	ii	ll	tt
#       ------- -------	-------	-------	-------	-------	------- ------- -----
code =	0000	bus	src	0	rd/pf	<39:6>	mem/io	lg	-

panic = ifmce
flags = physical
errtype =

#
# ---
#

funcunit = ls

desc = System data read error
error = ereport.cpu.amd.ls.s_rde

mask on  = MSR_MC_STATUS_UC
mask off = AMD_BANK_STAT_CECC, AMD_BANK_STAT_UECC

#	ext	type	pp	t	rrrr	addr	ii	ll	tt
#       ------- -------	-------	-------	-------	-------	------- ------- -----
code =	0000	bus	src	0	rd/wr	<39:6>	mem/io	lg	-

panic = ifmce
flags = physical
errtype =

#
# ---
#

funcunit = nb

desc = Correctable ECC error from Normal ECC
error = ereport.cpu.amd.nb.mem_ce

mask on  = AMD_BANK_STAT_CECC
mask off = MSR_MC_STATUS_UC, AMD_BANK_STAT_UECC

#	ext	type	pp	t	rrrr	addr	ii	ll	tt
#       ------- -------	-------	-------	-------	-------	------- ------- -----
code  = 0000	bus	src/res	0	rd/wr	<39:3>	mem	lg	-

panic = never
flags = physical
errtype =

# ---

desc = Uncorrectable ECC error from Normal ECC
error = ereport.cpu.amd.nb.mem_ue

mask on  = MSR_MC_STATUS_UC, AMD_BANK_STAT_UECC
mask off = AMD_BANK_STAT_CECC

#	ext	type	pp	t	rrrr	addr	ii	ll	tt
#       ------- -------	-------	-------	-------	-------	------- ------- -----
code  = 0000	bus	src/res	0	rd/wr	<39:3>	mem	lg	-

panic = always
flags = physical
errtype =

# ---

desc = Correctable ECC error from ChipKill ECC
error = ereport.cpu.amd.nb.mem_ce

mask on  = AMD_BANK_STAT_CECC
mask off = MSR_MC_STATUS_UC, AMD_BANK_STAT_UECC

#	ext	type	pp	t	rrrr	addr	ii	ll	tt
#       ------- -------	-------	-------	-------	-------	------- ------- -----
code  = 1000	bus	src/res	0	rd/wr	<39:3>	mem	lg	-

panic = never
flags = physical
errtype =

# ---

desc = Uncorrectable ECC error from ChipKill ECC
error = ereport.cpu.amd.nb.mem_ue

mask on  = MSR_MC_STATUS_UC, AMD_BANK_STAT_UECC
mask off = AMD_BANK_STAT_CECC

#	ext	type	pp	t	rrrr	addr	ii	ll	tt
#       ------- -------	-------	-------	-------	-------	------- ------- -----
code  = 1000	bus	src/res	0	rd/wr	<39:3>	mem	lg	-

panic = always
flags = physical
errtype =

# ---

desc = Hypertransport CRC error
error = ereport.cpu.amd.nb.ht_crc

mask on  = MSR_MC_STATUS_UC
mask off = AMD_BANK_STAT_CECC, AMD_BANK_STAT_UECC

#	ext	type	pp	t	rrrr	addr	ii	ll	tt
#       ------- -------	-------	-------	-------	-------	------- ------- -----
code  = 0001	bus	obs	0	err	none	gen	lg	-

panic = always
flags = 
errtype =

# ---

desc = Hypertransport Sync packet error
error = ereport.cpu.amd.nb.ht_sync

mask on  = MSR_MC_STATUS_UC
mask off = AMD_BANK_STAT_CECC, AMD_BANK_STAT_UECC

#	ext	type	pp	t	rrrr	addr	ii	ll	tt
#       ------- -------	-------	-------	-------	-------	------- ------- -----
code  = 0010	bus	obs	0	err	none	gen	lg	-

panic = always
flags = 
errtype =

# ---

desc = Master Abort
error = ereport.cpu.amd.nb.ma

mask on  = MSR_MC_STATUS_UC
mask off = AMD_BANK_STAT_CECC, AMD_BANK_STAT_UECC

#	ext	type	pp	t	rrrr	addr	ii	ll	tt
#       ------- -------	-------	-------	-------	-------	------- ------- -----
code  = 0011	bus	src/obs	0	rd/wr	<39:3>	mem/io	lg	-

panic = never
flags = physical
errtype =

# ---

desc = Target Abort
error = ereport.cpu.amd.nb.ta

mask on  = MSR_MC_STATUS_UC
mask off = AMD_BANK_STAT_CECC, AMD_BANK_STAT_UECC

#	ext	type	pp	t	rrrr	addr	ii	ll	tt
#       ------- -------	-------	-------	-------	-------	------- ------- -----
code  = 0100	bus	src/obs	0	rd/wr	<39:3>	mem/io	lg	-

panic = never
flags = physical
errtype =

# ---

desc = GART Table Walk Error
error = ereport.cpu.amd.nb.gart_walk

mask on  = MSR_MC_STATUS_UC
mask off = AMD_BANK_STAT_CECC, AMD_BANK_STAT_UECC

#	ext	type	pp	t	rrrr	addr	ii	ll	tt
#       ------- -------	-------	-------	-------	-------	------- ------- -----
code  = 0101	tlb	-	-	-	<39:3>	-	lg	gen

panic = never
flags = physical
errtype =

# ---

desc = Atomic Read/Modify/Write error
error = ereport.cpu.amd.nb.rmw

mask on  = MSR_MC_STATUS_UC
mask off = AMD_BANK_STAT_CECC, AMD_BANK_STAT_UECC

#	ext	type	pp	t	rrrr	addr	ii	ll	tt
#       ------- -------	-------	-------	-------	-------	------- ------- -----
code  = 0110	bus	obs	0	err	<39:3>	io	lg	-

panic = always
flags = physical
errtype =

# ---

desc = Watchdog error (timeout)
error = ereport.cpu.amd.nb.wdog

mask on  = MSR_MC_STATUS_UC
mask off = AMD_BANK_STAT_CECC, AMD_BANK_STAT_UECC

#	ext	type	pp	t	rrrr	addr	ii	ll	tt
#       ------- -------	-------	-------	-------	-------	------- ------- -----
code  = 0111	bus	gen	1	err	<39:3>	gen	lg	-

panic = always
flags = 
errtype =

# ---

desc = DRAM Address Parity Error
error = ereport.cpu.amd.nb.dramaddr_par

mask on  = MSR_MC_STATUS_UC
mask off = AMD_BANK_STAT_CECC, AMD_BANK_STAT_UECC

#	ext	type	pp	t	rrrr	addr	ii	ll	tt
#       ------- -------	-------	-------	-------	-------	------- ------- -----
code  = 1101	bus	obs	0	err	none	mem	lg	-

panic = always
flags = 
errtype =
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

/*
 * AMD Athlon64/Opteron Model-Specific Poller Implementation
 */

#include <sys/types.h>

#include "ao.h"

/*
 * Decide whether the caller should poll the NB.  The decision is made
 * and any poll is performed under protection of the chip-wide mutex
 * enforced at the caller's level.  That mutex already ensures that all
 * pollers on a chip are serialized - the following is simply to
 * avoid the NB poll ping-ponging between different detectors.
 */
uint64_t
ao_ms_poll_ownermask(cmi_hdl_t hdl, hrtime_t pintvl)
{
	ao_ms_data_t *ao = cms_hdl_getcmsdata(hdl);
	hrtime_t now = gethrtime_waitfree();
	hrtime_t last = ao->ao_ms_shared->aos_nb_poll_timestamp;
	int dopoll = 0;

	if (now - last > 2 * pintvl || last == 0) {
		/*
		 * If no last value has been recorded assume ownership.
		 * Otherwise only take over if the current "owner" seems
		 * to be making little progress.
		 */
		ao->ao_ms_shared->aos_nb_poll_owner = hdl;
		dopoll = 1;
	} else if (ao->ao_ms_shared->aos_nb_poll_owner == hdl) {
		/*
		 * This is the current owner and it is making progress.
		 */
		dopoll = 1;
	}

	if (dopoll)
		ao->ao_ms_shared->aos_nb_poll_timestamp = now;

	return (dopoll ? -1ULL : ~(1 << AMD_MCA_BANK_NB));
}
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
 */

#ifndef _AUTHAMD_H
#define	_AUTHAMD_H

#include <sys/types.h>
#include <sys/mca_amd.h>
#include <sys/cpu_module_ms_impl.h>

#ifdef __cplusplus
extern "C" {
#endif

#define	AUTHAMD_MAX_NODES		8	/* max number of nodes */
#define	AUTHAMD_DRAM_NCHANNEL		2	/* dram channels per node */
#define	AUTHAMD_DRAM_NCS		8	/* chip-selects per channel */

#define	AUTHAMD_FAMILY_6		0x6
#define	AUTHAMD_FAMILY_F		0xf
#define	AUTHAMD_FAMILY_10		0x10

#define	AUTHAMD_SYNDTYPE_64_8		0x0
#define	AUTHAMD_SYNDTYPE_128_16		0x1

typedef struct authamd_data authamd_data_t;

typedef struct authamd_error_disp {
	const char *aad_subclass;
	const char *aad_leafclass;
	uint64_t aad_ereport_members;
} authamd_error_disp_t;

/*
 * Model-specific logout structure.
 */
#pragma pack(1)
typedef struct authamd_logout {
	uint8_t aal_eccerrcnt[AUTHAMD_DRAM_NCHANNEL][AUTHAMD_DRAM_NCS];
} authamd_logout_t;
#pragma pack()

/*
 * Per node shared state
 */
struct authamd_nodeshared {
	uint_t ans_chipid;
	uint_t ans_procnodeid;
	uint_t ans_family;		/* family number */
	x86_chiprev_t ans_rev;		/* revision per cpuid_getchiprev */
	volatile ulong_t ans_cfgonce;	/* Config performed once per chip */
	hrtime_t ans_poll_timestamp;	/* Checks poll owner is alive */
	cmi_hdl_t ans_pollowner;	/* poller of shared resources */
	char *ans_eccsymsz;		/* DRAM ChipKill ECC Symbol Size */
};

enum authamd_cfgonce_bitnum {
	AUTHAMD_CFGONCE_ONLNSPRCFG,
	AUTHAMD_CFGONCE_NBTHRESH,
	AUTHAMD_CFGONCE_NBMCACFG,
	AUTHAMD_CFGONCE_CACHESCRUB,
	AUTHAMD_CFGONCE_NBMCA,
	AUTHAMD_CFGONCE_ECCSYMSZ
};

/*
 * Per-CPU model-specific state
 */
struct authamd_data {
	cmi_hdl_t amd_hdl;			/* cpu we're associated with */
	uint64_t amd_hwcr;
	struct authamd_nodeshared *amd_shared;
};

#ifdef _KERNEL

/*
 * Our cms_ops operations and function prototypes for all non-NULL members.
 */
extern const cms_ops_t _cms_ops;

extern int authamd_init(cmi_hdl_t, void **);
extern size_t authamd_logout_size(cmi_hdl_t);
extern uint64_t authamd_mcgctl_val(cmi_hdl_t, int, uint64_t);
extern boolean_t authamd_bankctl_skipinit(cmi_hdl_t, int);
extern uint64_t authamd_bankctl_val(cmi_hdl_t, int, uint64_t);
extern void authamd_mca_init(cmi_hdl_t, int);
extern void authamd_bank_logout(cmi_hdl_t, int, uint64_t, uint64_t,
    uint64_t, void *);
extern uint32_t authamd_error_action(cmi_hdl_t, int, int, uint64_t,
    uint64_t, uint64_t, void *);
extern cms_cookie_t authamd_disp_match(cmi_hdl_t, int, int, uint64_t, uint64_t,
    uint64_t, void *);
extern void authamd_ereport_class(cmi_hdl_t, cms_cookie_t, const char **,
    const char **);
extern void authamd_ereport_add_logout(cmi_hdl_t, nvlist_t *,
    nv_alloc_t *, int, uint64_t, uint64_t, uint64_t, void *, cms_cookie_t);
extern cms_errno_t authamd_msrinject(cmi_hdl_t, uint_t, uint64_t);

#endif /* _KERNEL */

#ifdef __cplusplus
}
#endif

#endif /* _AUTHAMD_H */
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
 * Copyright 2022 Oxide Computer Co.
 */

/*
 * "Generic AMD" model-specific support.  If no more-specific support can
 * be found, or such modules declines to initialize, then for AuthenticAMD
 * cpus this module can have a crack at providing some AMD model-specific
 * support that at least goes beyond common MCA architectural features
 * if not down to the nitty-gritty level for a particular model.  We
 * are layered on top of a cpu module, likely cpu.generic, so there is no
 * need for us to perform common architecturally-accessible functions.
 */

#include <sys/types.h>
#include <sys/cmn_err.h>
#include <sys/modctl.h>
#include <sys/cpu_module.h>
#include <sys/mca_x86.h>
#include <sys/pci_cfgspace.h>
#include <sys/x86_archext.h>
#include <sys/mc_amd.h>
#include <sys/fm/protocol.h>
#include <sys/fm/cpu/GENAMD.h>
#include <sys/fm/smb/fmsmb.h>
#include <sys/fm/util.h>
#include <sys/nvpair.h>
#include <sys/controlregs.h>
#include <sys/pghw.h>
#include <sys/sunddi.h>
#include <sys/sysmacros.h>
#include <sys/cpu_module_ms_impl.h>

#include "authamd.h"

extern int x86gentopo_legacy; /* x86 generic topo support */

int authamd_ms_support_disable = 0;

#define	AUTHAMD_F_REVS_BCDE \
	(X86_CHIPREV_AMD_LEGACY_F_REV_B | X86_CHIPREV_AMD_LEGACY_F_REV_C0 | \
	X86_CHIPREV_AMD_LEGACY_F_REV_CG | X86_CHIPREV_AMD_LEGACY_F_REV_D | \
	X86_CHIPREV_AMD_LEGACY_F_REV_E)

#define	AUTHAMD_F_REVS_FG \
	(X86_CHIPREV_AMD_LEGACY_F_REV_F | X86_CHIPREV_AMD_LEGACY_F_REV_G)

#define	AUTHAMD_10_REVS_AB \
	(X86_CHIPREV_AMD_LEGACY_10_REV_A | X86_CHIPREV_AMD_LEGACY_10_REV_B)

/*
 * Bitmasks of support for various features.  Try to enable features
 * via inclusion in one of these bitmasks and check that at the
 * feature imlementation - that way new family support may often simply
 * simply need to update these bitmasks.
 */

/*
 * Models that include an on-chip NorthBridge.
 */
#define	AUTHAMD_NBONCHIP(rev) \
	(chiprev_at_least(rev, X86_CHIPREV_AMD_LEGACY_F_REV_B) || \
	chiprev_at_least(rev, X86_CHIPREV_AMD_LEGACY_10_REV_A))

/*
 * Families/revisions for which we can recognise main memory ECC errors.
 */
#define	AUTHAMD_MEMECC_RECOGNISED(rev) \
	(chiprev_at_least(rev, X86_CHIPREV_AMD_LEGACY_F_REV_B) || \
	chiprev_at_least(rev, X86_CHIPREV_AMD_LEGACY_10_REV_A))

/*
 * Families/revisions that have an Online Spare Control Register
 */
#define	AUTHAMD_HAS_ONLINESPARECTL(rev) \
	(chiprev_at_least(rev, X86_CHIPREV_AMD_LEGACY_F_REV_F) || \
	chiprev_at_least(rev, X86_CHIPREV_AMD_LEGACY_10_REV_A))

/*
 * Families/revisions for which we will perform NB MCA Config changes
 */
#define	AUTHAMD_DO_NBMCACFG(rev) \
	(chiprev_at_least(rev, X86_CHIPREV_AMD_LEGACY_F_REV_B) || \
	chiprev_at_least(rev, X86_CHIPREV_AMD_LEGACY_10_REV_A))

/*
 * Families/revisions that have chip cache scrubbers.
 */
#define	AUTHAMD_HAS_CHIPSCRUB(rev) \
	(chiprev_at_least(rev, X86_CHIPREV_AMD_LEGACY_F_REV_B) || \
	chiprev_at_least(rev, X86_CHIPREV_AMD_LEGACY_10_REV_A))

/*
 * Families/revisions that have a NB misc register or registers -
 * evaluates to 0 if no support, otherwise the number of MC4_MISCj.
 */
#define	AUTHAMD_NBMISC_NUM(rev) \
	(chiprev_at_least(rev, X86_CHIPREV_AMD_LEGACY_F_REV_F)? 1 : \
	(chiprev_at_least(rev, X86_CHIPREV_AMD_LEGACY_10_REV_A) ? 3 : 0))

/*
 * Families/revision for which we wish not to machine check for GART
 * table walk errors - bit 10 of NB CTL.
 */
#define	AUTHAMD_NOGARTTBLWLK_MC(rev) \
	(chiprev_at_least(rev, X86_CHIPREV_AMD_LEGACY_F_REV_B) || \
	chiprev_at_least(rev, X86_CHIPREV_AMD_LEGACY_10_REV_A))

/*
 * Families/revisions that are potentially L3 capable
 */
#define	AUTHAMD_L3CAPABLE(rev) \
	(chiprev_at_least(rev, X86_CHIPREV_AMD_LEGACY_10_REV_A))

/*
 * Families/revisions that support x8 ChipKill ECC
 */
#define	AUTHAMD_SUPPORTS_X8ECC(rev) \
	(chiprev_at_least(rev, X86_CHIPREV_AMD_LEGACY_10_REV_D0))

/*
 * We recognise main memory ECC errors for AUTHAMD_MEMECC_RECOGNISED
 * revisions as:
 *
 *	- being reported by the NB
 *	- being a compound bus/interconnect error (external to chip)
 *	- having LL of LG
 *	- having II of MEM (but could still be a master/target abort)
 *	- having CECC or UECC set
 *
 * We do not check the extended error code (first nibble of the
 * model-specific error code on AMD) since this has changed from
 * family 0xf to family 0x10 (ext code 0 now reserved on family 0x10).
 * Instead we use CECC/UECC to separate off the master/target
 * abort cases.
 *
 * We insist that the detector be the NorthBridge bank;  although
 * IC/DC can report some main memory errors, they do not capture
 * an address at sufficient resolution to be useful and the NB will
 * report most errors.
 */
#define	AUTHAMD_IS_MEMECCERR(bank, status) \
	((bank) == AMD_MCA_BANK_NB && \
	MCAX86_ERRCODE_ISBUS_INTERCONNECT(MCAX86_ERRCODE(status)) && \
	MCAX86_ERRCODE_LL(MCAX86_ERRCODE(status)) == MCAX86_ERRCODE_LL_LG && \
	MCAX86_ERRCODE_II(MCAX86_ERRCODE(status)) == MCAX86_ERRCODE_II_MEM && \
	((status) & (AMD_BANK_STAT_CECC | AMD_BANK_STAT_UECC)))

static authamd_error_disp_t authamd_memce_disp = {
	FM_EREPORT_CPU_GENAMD,
	FM_EREPORT_CPU_GENAMD_MEM_CE,
	FM_EREPORT_GENAMD_PAYLOAD_FLAGS_MEM_CE
};

static authamd_error_disp_t authamd_memue_disp = {
	FM_EREPORT_CPU_GENAMD,
	FM_EREPORT_CPU_GENAMD_MEM_UE,
	FM_EREPORT_GENAMD_PAYLOAD_FLAGS_MEM_UE
};

static authamd_error_disp_t authamd_ckmemce_disp = {
	FM_EREPORT_CPU_GENAMD,
	FM_EREPORT_CPU_GENAMD_CKMEM_CE,
	FM_EREPORT_GENAMD_PAYLOAD_FLAGS_CKMEM_CE
};

static authamd_error_disp_t authamd_ckmemue_disp = {
	FM_EREPORT_CPU_GENAMD,
	FM_EREPORT_CPU_GENAMD_CKMEM_UE,
	FM_EREPORT_GENAMD_PAYLOAD_FLAGS_CKMEM_UE
};

/*
 * We recognise GART walk errors as:
 *
 *	- being reported by the NB
 *	- being a compound TLB error
 *	- having LL of LG and TT of GEN
 *	- having UC set
 *	- possibly having PCC set (if source CPU)
 */
#define	AUTHAMD_IS_GARTERR(bank, status) \
	((bank) == AMD_MCA_BANK_NB && \
	MCAX86_ERRCODE_ISTLB(MCAX86_ERRCODE(status)) && \
	MCAX86_ERRCODE_LL(MCAX86_ERRCODE(status)) == MCAX86_ERRCODE_LL_LG && \
	MCAX86_ERRCODE_TT(MCAX86_ERRCODE(status)) == MCAX86_ERRCODE_TT_GEN && \
	(status) & MSR_MC_STATUS_UC)

static authamd_error_disp_t authamd_gart_disp = {
	FM_EREPORT_CPU_GENAMD,			/* use generic subclass */
	FM_EREPORT_CPU_GENADM_GARTTBLWLK,	/* use generic leafclass */
	0					/* no additional payload */
};


static struct authamd_nodeshared *authamd_shared[AUTHAMD_MAX_NODES];

static int
authamd_chip_once(authamd_data_t *authamd, enum authamd_cfgonce_bitnum what)
{
	return (atomic_set_long_excl(&authamd->amd_shared->ans_cfgonce,
	    what) == 0 ?  B_TRUE : B_FALSE);
}

static void
authamd_pcicfg_write(uint_t procnodeid, uint_t func, uint_t reg, uint32_t val)
{
	ASSERT(procnodeid + 24 <= 31);
	ASSERT((func & 7) == func);
	ASSERT((reg & 3) == 0 && reg < 4096);

	cmi_pci_putl(0, procnodeid + 24, func, reg, 0, val);
}

static uint32_t
authamd_pcicfg_read(uint_t procnodeid, uint_t func, uint_t reg)
{
	ASSERT(procnodeid + 24 <= 31);
	ASSERT((func & 7) == func);
	ASSERT((reg & 3) == 0 && reg < 4096);

	return (cmi_pci_getl(0, procnodeid + 24, func, reg, 0, 0));
}

void
authamd_bankstatus_prewrite(cmi_hdl_t hdl, authamd_data_t *authamd)
{
	uint64_t hwcr;

	if (cmi_hdl_rdmsr(hdl, MSR_AMD_HWCR, &hwcr) != CMI_SUCCESS)
		return;

	authamd->amd_hwcr = hwcr;

	if (!(hwcr & AMD_HWCR_MCI_STATUS_WREN)) {
		hwcr |= AMD_HWCR_MCI_STATUS_WREN;
		(void) cmi_hdl_wrmsr(hdl, MSR_AMD_HWCR, hwcr);
	}
}

void
authamd_bankstatus_postwrite(cmi_hdl_t hdl, authamd_data_t *authamd)
{
	uint64_t hwcr = authamd->amd_hwcr;

	if (!(hwcr & AMD_HWCR_MCI_STATUS_WREN)) {
		hwcr &= ~AMD_HWCR_MCI_STATUS_WREN;
		(void) cmi_hdl_wrmsr(hdl, MSR_AMD_HWCR, hwcr);
	}
}

/*
 * Read EccCnt repeatedly for all possible channel/chip-select combos:
 *
 *	- read sparectl register
 *	- if EccErrCntWrEn is set, clear that bit in the just-read value
 *	  and write it back to sparectl;  this *may* clobber the EccCnt
 *	  for the channel/chip-select combination currently selected, so
 *	  we leave this bit clear if we had to clear it
 *	- cycle through all channel/chip-select combinations writing each
 *	  combination to sparectl before reading the register back for
 *	  EccCnt for that combination;  since EccErrCntWrEn is clear
 *	  the writes to select what count to read will not themselves
 *	  zero any counts
 */
static int
authamd_read_ecccnt(authamd_data_t *authamd, struct authamd_logout *msl)
{
	union mcreg_sparectl sparectl;
	uint_t procnodeid = authamd->amd_shared->ans_procnodeid;
	uint_t family = authamd->amd_shared->ans_family;
	x86_chiprev_t rev = authamd->amd_shared->ans_rev;
	int chan, cs;

	/*
	 * Check for feature support;  this macro will test down to the
	 * family revision number, whereafter we'll switch on family
	 * assuming that future revisions will use the same register
	 * format.
	 */
	if (!AUTHAMD_HAS_ONLINESPARECTL(rev)) {
		bzero(&msl->aal_eccerrcnt, sizeof (msl->aal_eccerrcnt));
		return (0);
	}

	MCREG_VAL32(&sparectl) =
	    authamd_pcicfg_read(procnodeid, MC_FUNC_MISCCTL,
	    MC_CTL_REG_SPARECTL);

	switch (family) {
	case AUTHAMD_FAMILY_F:
		MCREG_FIELD_F_revFG(&sparectl, EccErrCntWrEn) = 0;
		break;

	case AUTHAMD_FAMILY_10:
		MCREG_FIELD_10_revAB(&sparectl, EccErrCntWrEn) = 0;
		break;
	}

	for (chan = 0; chan < AUTHAMD_DRAM_NCHANNEL; chan++) {
		switch (family) {
		case AUTHAMD_FAMILY_F:
			MCREG_FIELD_F_revFG(&sparectl, EccErrCntDramChan) =
			    chan;
			break;

		case AUTHAMD_FAMILY_10:
			MCREG_FIELD_10_revAB(&sparectl, EccErrCntDramChan) =
			    chan;
			break;
		}

		for (cs = 0; cs < AUTHAMD_DRAM_NCS; cs++) {
			switch (family) {
			case AUTHAMD_FAMILY_F:
				MCREG_FIELD_F_revFG(&sparectl,
				    EccErrCntDramCs) = cs;
				break;

			case AUTHAMD_FAMILY_10:
				MCREG_FIELD_10_revAB(&sparectl,
				    EccErrCntDramCs) = cs;
				break;
			}

			authamd_pcicfg_write(procnodeid, MC_FUNC_MISCCTL,
			    MC_CTL_REG_SPARECTL, MCREG_VAL32(&sparectl));

			MCREG_VAL32(&sparectl) = authamd_pcicfg_read(procnodeid,
			    MC_FUNC_MISCCTL, MC_CTL_REG_SPARECTL);

			switch (family) {
			case AUTHAMD_FAMILY_F:
				msl->aal_eccerrcnt[chan][cs] =
				    MCREG_FIELD_F_revFG(&sparectl, EccErrCnt);
				break;
			case AUTHAMD_FAMILY_10:
				msl->aal_eccerrcnt[chan][cs] =
				    MCREG_FIELD_10_revAB(&sparectl, EccErrCnt);
				break;
			}
		}
	}

	return (1);
}

/*
 * Clear EccCnt for all possible channel/chip-select combos:
 *
 *	- set EccErrCntWrEn in sparectl, if necessary
 *	- write 0 to EccCnt for all channel/chip-select combinations
 *	- clear EccErrCntWrEn
 *
 * If requested also disable the interrupts taken on counter overflow
 * and on swap done.
 */
static void
authamd_clear_ecccnt(authamd_data_t *authamd, boolean_t clrint)
{
	union mcreg_sparectl sparectl;
	uint_t procnodeid = authamd->amd_shared->ans_procnodeid;
	uint_t family = authamd->amd_shared->ans_family;
	x86_chiprev_t rev = authamd->amd_shared->ans_rev;
	int chan, cs;

	if (!AUTHAMD_HAS_ONLINESPARECTL(rev))
		return;

	MCREG_VAL32(&sparectl) =
	    authamd_pcicfg_read(procnodeid, MC_FUNC_MISCCTL,
	    MC_CTL_REG_SPARECTL);

	switch (family) {
	case AUTHAMD_FAMILY_F:
		MCREG_FIELD_F_revFG(&sparectl, EccErrCntWrEn) = 1;
		if (clrint) {
			MCREG_FIELD_F_revFG(&sparectl, EccErrInt) = 0;
			MCREG_FIELD_F_revFG(&sparectl, SwapDoneInt) = 0;
		}
		break;

	case AUTHAMD_FAMILY_10:
		MCREG_FIELD_10_revAB(&sparectl, EccErrCntWrEn) = 1;
		if (clrint) {
			MCREG_FIELD_10_revAB(&sparectl, EccErrInt) = 0;
			MCREG_FIELD_10_revAB(&sparectl, SwapDoneInt) = 0;
		}
		break;
	}

	authamd_pcicfg_write(procnodeid, MC_FUNC_MISCCTL,
	    MC_CTL_REG_SPARECTL, MCREG_VAL32(&sparectl));

	for (chan = 0; chan < AUTHAMD_DRAM_NCHANNEL; chan++) {
		switch (family) {
		case AUTHAMD_FAMILY_F:
			MCREG_FIELD_F_revFG(&sparectl, EccErrCntDramChan) =
			    chan;
			break;

		case AUTHAMD_FAMILY_10:
			MCREG_FIELD_10_revAB(&sparectl, EccErrCntDramChan) =
			    chan;
			break;
		}

		for (cs = 0; cs < AUTHAMD_DRAM_NCS; cs++) {
			switch (family) {
			case AUTHAMD_FAMILY_F:
				MCREG_FIELD_F_revFG(&sparectl,
				    EccErrCntDramCs) = cs;
				MCREG_FIELD_F_revFG(&sparectl,
				    EccErrCnt) = 0;
				break;

			case AUTHAMD_FAMILY_10:
				MCREG_FIELD_10_revAB(&sparectl,
				    EccErrCntDramCs) = cs;
				MCREG_FIELD_10_revAB(&sparectl,
				    EccErrCnt) = 0;
				break;
			}

			authamd_pcicfg_write(procnodeid, MC_FUNC_MISCCTL,
			    MC_CTL_REG_SPARECTL, MCREG_VAL32(&sparectl));
		}
	}
}


/*
 * Return
 *	1: supported
 *	0: unsupported
 */
static int
authamd_supported(cmi_hdl_t hdl)
{
	uint_t family = cmi_hdl_family(hdl);

	switch (family) {
	case AUTHAMD_FAMILY_6:
	case AUTHAMD_FAMILY_F:
	case AUTHAMD_FAMILY_10:
		return (1);
	default:
		return (0);
	}
}

/*
 * cms_init entry point.
 *
 * This module provides broad model-specific support for AMD families
 * 0x6, 0xf and 0x10.  Future families will have to be evaluated once their
 * documentation is available.
 */
int
authamd_init(cmi_hdl_t hdl, void **datap)
{
	uint_t chipid = cmi_hdl_chipid(hdl);
	uint_t procnodeid = cmi_hdl_procnodeid(hdl);
	struct authamd_nodeshared *sp, *osp;
	uint_t family = cmi_hdl_family(hdl);
	x86_chiprev_t rev = cmi_hdl_chiprev(hdl);
	authamd_data_t *authamd;
	uint64_t cap;

	if (authamd_ms_support_disable ||
	    !authamd_supported(hdl))
		return (ENOTSUP);

	if (!is_x86_feature(x86_featureset, X86FSET_MCA))
		return (ENOTSUP);

	if (cmi_hdl_rdmsr(hdl, IA32_MSR_MCG_CAP, &cap) != CMI_SUCCESS)
		return (ENOTSUP);

	if (!(cap & MCG_CAP_CTL_P))
		return (ENOTSUP);

	authamd = *datap = kmem_zalloc(sizeof (authamd_data_t), KM_SLEEP);
	cmi_hdl_hold(hdl);	/* release in fini */
	authamd->amd_hdl = hdl;

	if ((sp = authamd_shared[procnodeid]) == NULL) {
		sp = kmem_zalloc(sizeof (struct authamd_nodeshared), KM_SLEEP);
		sp->ans_chipid = chipid;
		sp->ans_procnodeid = procnodeid;
		sp->ans_family = family;
		sp->ans_rev = rev;
		membar_producer();

		osp = atomic_cas_ptr(&authamd_shared[procnodeid], NULL, sp);
		if (osp != NULL) {
			kmem_free(sp, sizeof (struct authamd_nodeshared));
			sp = osp;
		}
	}
	authamd->amd_shared = sp;

	return (0);
}

/*
 * cms_logout_size entry point.
 */
/*ARGSUSED*/
size_t
authamd_logout_size(cmi_hdl_t hdl)
{
	return (sizeof (struct authamd_logout));
}

/*
 * cms_mcgctl_val entry point
 *
 * Instead of setting all bits to 1 we can set just those for the
 * error detector banks known to exist.
 */
/*ARGSUSED*/
uint64_t
authamd_mcgctl_val(cmi_hdl_t hdl, int nbanks, uint64_t proposed)
{
	return (nbanks < 64 ? (1ULL << nbanks) - 1 : proposed);
}

/*
 * cms_bankctl_skipinit entry point
 *
 * On K6 we do not initialize MC0_CTL since, reportedly, this bank (for DC)
 * may produce spurious machine checks.
 *
 * Only allow a single core to setup the NorthBridge MCi_CTL register.
 */
/*ARGSUSED*/
boolean_t
authamd_bankctl_skipinit(cmi_hdl_t hdl, int bank)
{
	authamd_data_t *authamd = cms_hdl_getcmsdata(hdl);
	x86_chiprev_t rev = authamd->amd_shared->ans_rev;

	if (authamd->amd_shared->ans_family == AUTHAMD_FAMILY_6)
		return (bank == 0 ?  B_TRUE : B_FALSE);

	if (AUTHAMD_NBONCHIP(rev) && bank == AMD_MCA_BANK_NB) {
		return (authamd_chip_once(authamd, AUTHAMD_CFGONCE_NBMCA) ==
		    B_TRUE ? B_FALSE : B_TRUE);
	}

	return (B_FALSE);
}

/*
 * cms_bankctl_val entry point
 */
uint64_t
authamd_bankctl_val(cmi_hdl_t hdl, int bank, uint64_t proposed)
{
	authamd_data_t *authamd = cms_hdl_getcmsdata(hdl);
	x86_chiprev_t rev = authamd->amd_shared->ans_rev;
	uint64_t val = proposed;

	/*
	 * The Intel MCA says we can write all 1's to enable #MC for
	 * all errors, and AMD docs say much the same.  But, depending
	 * perhaps on other config registers, taking machine checks
	 * for some errors such as GART TLB errors and master/target
	 * aborts may be bad - they set UC and sometime also PCC, but
	 * we should not always panic for these error types.
	 *
	 * Our cms_error_action entry point can suppress such panics,
	 * however we can also use the cms_bankctl_val entry point to
	 * veto enabling of some of the known villains in the first place.
	 */
	if (bank == AMD_MCA_BANK_NB && AUTHAMD_NOGARTTBLWLK_MC(rev))
		val &= ~AMD_NB_EN_GARTTBLWK;

	return (val);
}

/*
 * Bits to add to NB MCA config (after watchdog config).
 */
uint32_t authamd_nb_mcacfg_add = AMD_NB_CFG_ADD_CMN;

/*
 * Bits to remove from NB MCA config (after watchdog config)
 */
uint32_t authamd_nb_mcacfg_remove = AMD_NB_CFG_REMOVE_CMN;

/*
 * NB Watchdog policy, and rate we use if enabling.
 */
enum {
	AUTHAMD_NB_WDOG_LEAVEALONE,
	AUTHAMD_NB_WDOG_DISABLE,
	AUTHAMD_NB_WDOG_ENABLE_IF_DISABLED,
	AUTHAMD_NB_WDOG_ENABLE_FORCE_RATE
} authamd_nb_watchdog_policy = AUTHAMD_NB_WDOG_ENABLE_IF_DISABLED;

uint32_t authamd_nb_mcacfg_wdog = AMD_NB_CFG_WDOGTMRCNTSEL_4095 |
    AMD_NB_CFG_WDOGTMRBASESEL_1MS;

/*
 * Per-core cache scrubbing policy and rates.
 */
enum {
	AUTHAMD_SCRUB_BIOSDEFAULT,	/* leave as BIOS configured */
	AUTHAMD_SCRUB_FIXED,		/* assign our chosen rate */
	AUTHAMD_SCRUB_MAX		/* use higher of ours and BIOS rate */
} authamd_scrub_policy = AUTHAMD_SCRUB_MAX;

uint32_t authamd_scrub_rate_dcache = 0xf;	/* 64K per 0.67 seconds */
uint32_t authamd_scrub_rate_l2cache = 0xe;	/* 1MB per 5.3 seconds */
uint32_t authamd_scrub_rate_l3cache = 0xd;	/* 1MB per 2.7 seconds */

static uint32_t
authamd_scrubrate(uint32_t osrate, uint32_t biosrate, const char *varnm)
{
	uint32_t rate;

	if (osrate > AMD_NB_SCRUBCTL_RATE_MAX) {
		cmn_err(CE_WARN, "%s is too large, resetting to 0x%x\n",
		    varnm, AMD_NB_SCRUBCTL_RATE_MAX);
		osrate = AMD_NB_SCRUBCTL_RATE_MAX;
	}

	switch (authamd_scrub_policy) {
	case AUTHAMD_SCRUB_FIXED:
		rate = osrate;
		break;

	default:
		cmn_err(CE_WARN, "Unknown authamd_scrub_policy %d - "
		    "using default policy of AUTHAMD_SCRUB_MAX",
		    authamd_scrub_policy);
		/*FALLTHRU*/

	case AUTHAMD_SCRUB_MAX:
		if (osrate != 0 && biosrate != 0)
			rate = MIN(osrate, biosrate);	/* small is fast */
		else
			rate = osrate ? osrate : biosrate;
	}

	return (rate);
}

/*
 * cms_mca_init entry point.
 */
/*ARGSUSED*/
void
authamd_mca_init(cmi_hdl_t hdl, int nbanks)
{
	authamd_data_t *authamd = cms_hdl_getcmsdata(hdl);
	x86_chiprev_t rev = authamd->amd_shared->ans_rev;
	uint_t procnodeid = authamd->amd_shared->ans_procnodeid;

	/*
	 * On chips with a NB online spare control register take control
	 * and clear ECC counts.
	 */
	if (AUTHAMD_HAS_ONLINESPARECTL(rev) &&
	    authamd_chip_once(authamd, AUTHAMD_CFGONCE_ONLNSPRCFG)) {
		authamd_clear_ecccnt(authamd, B_TRUE);
	}

	/*
	 * And since we are claiming the telemetry stop the BIOS receiving
	 * an SMI on NB threshold overflow.
	 */
	if (AUTHAMD_NBMISC_NUM(rev) &&
	    authamd_chip_once(authamd, AUTHAMD_CFGONCE_NBTHRESH)) {
		union mcmsr_nbmisc nbm;
		int i;

		authamd_bankstatus_prewrite(hdl, authamd);

		for (i = 0; i < AUTHAMD_NBMISC_NUM(rev); i++) {
			if (cmi_hdl_rdmsr(hdl, MC_MSR_NB_MISC(i),
			    (uint64_t *)&nbm) != CMI_SUCCESS)
				continue;

			if (chiprev_at_least(rev,
			    X86_CHIPREV_AMD_LEGACY_F_REV_F) &&
			    MCMSR_FIELD_F_revFG(&nbm, mcmisc_Valid) &&
			    MCMSR_FIELD_F_revFG(&nbm, mcmisc_CntP)) {
				MCMSR_FIELD_F_revFG(&nbm, mcmisc_IntType) = 0;
			} else if (chiprev_at_least(rev,
			    X86_CHIPREV_AMD_LEGACY_10_REV_A) &&
			    MCMSR_FIELD_10_revAB(&nbm, mcmisc_Valid) &&
			    MCMSR_FIELD_10_revAB(&nbm, mcmisc_CntP)) {
				MCMSR_FIELD_10_revAB(&nbm, mcmisc_IntType) = 0;
			}

			(void) cmi_hdl_wrmsr(hdl, MC_MSR_NB_MISC(i),
			    MCMSR_VAL(&nbm));
		}

		authamd_bankstatus_postwrite(hdl, authamd);
	}

	/*
	 * NB MCA Configuration Register.
	 */
	if (AUTHAMD_DO_NBMCACFG(rev) &&
	    authamd_chip_once(authamd, AUTHAMD_CFGONCE_NBMCACFG)) {
		uint32_t val = authamd_pcicfg_read(procnodeid, MC_FUNC_MISCCTL,
		    MC_CTL_REG_NBCFG);

		switch (authamd_nb_watchdog_policy) {
		case AUTHAMD_NB_WDOG_LEAVEALONE:
			break;

		case AUTHAMD_NB_WDOG_DISABLE:
			val &= ~(AMD_NB_CFG_WDOGTMRBASESEL_MASK |
			    AMD_NB_CFG_WDOGTMRCNTSEL_MASK);
			val |= AMD_NB_CFG_WDOGTMRDIS;
			break;

		default:
			cmn_err(CE_NOTE, "authamd_nb_watchdog_policy=%d "
			    "unrecognised, using default policy",
			    authamd_nb_watchdog_policy);
			/*FALLTHRU*/

		case AUTHAMD_NB_WDOG_ENABLE_IF_DISABLED:
			if (!(val & AMD_NB_CFG_WDOGTMRDIS))
				break;	/* if enabled leave rate intact */
			/*FALLTHRU*/

		case AUTHAMD_NB_WDOG_ENABLE_FORCE_RATE:
			val &= ~(AMD_NB_CFG_WDOGTMRBASESEL_MASK |
			    AMD_NB_CFG_WDOGTMRCNTSEL_MASK |
			    AMD_NB_CFG_WDOGTMRDIS);
			val |= authamd_nb_mcacfg_wdog;
			break;
		}

		/*
		 * Bit 0 of the NB MCA Config register is reserved on family
		 * 0x10.
		 */
		if (chiprev_at_least(rev, X86_CHIPREV_AMD_LEGACY_10_REV_A))
			authamd_nb_mcacfg_add &= ~AMD_NB_CFG_CPUECCERREN;

		val &= ~authamd_nb_mcacfg_remove;
		val |= authamd_nb_mcacfg_add;

		authamd_pcicfg_write(procnodeid, MC_FUNC_MISCCTL,
		    MC_CTL_REG_NBCFG, val);
	}

	/*
	 * Cache scrubbing.  We can't enable DRAM scrubbing since
	 * we don't know the DRAM base for this node.
	 */
	if (AUTHAMD_HAS_CHIPSCRUB(rev) &&
	    authamd_scrub_policy != AUTHAMD_SCRUB_BIOSDEFAULT &&
	    authamd_chip_once(authamd, AUTHAMD_CFGONCE_CACHESCRUB)) {
		uint32_t val = authamd_pcicfg_read(procnodeid, MC_FUNC_MISCCTL,
		    MC_CTL_REG_SCRUBCTL);
		int l3cap = 0;

		if (AUTHAMD_L3CAPABLE(rev)) {
			l3cap = (authamd_pcicfg_read(procnodeid,
			    MC_FUNC_MISCCTL, MC_CTL_REG_NBCAP) &
			    MC_NBCAP_L3CAPABLE) != 0;
		}

		authamd_scrub_rate_dcache =
		    authamd_scrubrate(authamd_scrub_rate_dcache,
		    (val & AMD_NB_SCRUBCTL_DC_MASK) >> AMD_NB_SCRUBCTL_DC_SHIFT,
		    "authamd_scrub_rate_dcache");

		authamd_scrub_rate_l2cache =
		    authamd_scrubrate(authamd_scrub_rate_l2cache,
		    (val & AMD_NB_SCRUBCTL_L2_MASK) >> AMD_NB_SCRUBCTL_L2_SHIFT,
		    "authamd_scrub_rate_l2cache");

		authamd_scrub_rate_l3cache = l3cap ?
		    authamd_scrubrate(authamd_scrub_rate_l3cache,
		    (val & AMD_NB_SCRUBCTL_L3_MASK) >> AMD_NB_SCRUBCTL_L3_SHIFT,
		    "authamd_scrub_rate_l3cache") : 0;

		val = AMD_NB_MKSCRUBCTL(authamd_scrub_rate_l3cache,
		    authamd_scrub_rate_dcache, authamd_scrub_rate_l2cache,
		    val & AMD_NB_SCRUBCTL_DRAM_MASK);

		authamd_pcicfg_write(procnodeid, MC_FUNC_MISCCTL,
		    MC_CTL_REG_SCRUBCTL, val);
	}

	/*
	 * ECC symbol size. Defaults to 4.
	 * Set to 8 on systems that support x8 ECC and have it enabled.
	 */
	if (authamd_chip_once(authamd, AUTHAMD_CFGONCE_ECCSYMSZ)) {
		authamd->amd_shared->ans_eccsymsz = "C4";
		if (AUTHAMD_SUPPORTS_X8ECC(rev) &&
		    (authamd_pcicfg_read(procnodeid, MC_FUNC_MISCCTL,
		    MC_CTL_REG_EXTNBCFG) & MC_EXTNBCFG_ECCSYMSZ))
			authamd->amd_shared->ans_eccsymsz = "C8";
	}
}

/*
 * cms_poll_ownermask entry point.
 */
uint64_t
authamd_poll_ownermask(cmi_hdl_t hdl, hrtime_t pintvl)
{
	authamd_data_t *authamd = cms_hdl_getcmsdata(hdl);
	struct authamd_nodeshared *ansp = authamd->amd_shared;
	hrtime_t now = gethrtime_waitfree();
	hrtime_t last = ansp->ans_poll_timestamp;
	int dopoll = 0;

	if (now - last > 2 * pintvl || last == 0) {
		ansp->ans_pollowner = hdl;
		dopoll = 1;
	} else if (ansp->ans_pollowner == hdl) {
		dopoll = 1;
	}

	if (dopoll)
		ansp->ans_poll_timestamp = now;

	return (dopoll ? -1ULL : ~(1 << AMD_MCA_BANK_NB));

}

/*
 * cms_bank_logout entry point.
 */
/*ARGSUSED*/
void
authamd_bank_logout(cmi_hdl_t hdl, int bank, uint64_t status,
    uint64_t addr, uint64_t misc, void *mslogout)
{
	authamd_data_t *authamd = cms_hdl_getcmsdata(hdl);
	struct authamd_logout *msl = mslogout;
	x86_chiprev_t rev = authamd->amd_shared->ans_rev;

	if (msl == NULL)
		return;

	/*
	 * For main memory ECC errors on revisions with an Online Spare
	 * Control Register grab the ECC counts by channel and chip-select
	 * and reset them to 0.
	 */
	if (AUTHAMD_MEMECC_RECOGNISED(rev) &&
	    AUTHAMD_IS_MEMECCERR(bank, status) &&
	    AUTHAMD_HAS_ONLINESPARECTL(rev)) {
		if (authamd_read_ecccnt(authamd, msl))
			authamd_clear_ecccnt(authamd, B_FALSE);
	}
}

/*
 * cms_error_action entry point
 */

int authamd_forgive_uc = 0;	/* For test/debug only */
int authamd_forgive_pcc = 0;	/* For test/debug only */
int authamd_fake_poison = 0;	/* For test/debug only */

/*ARGSUSED*/
uint32_t
authamd_error_action(cmi_hdl_t hdl, int ismc, int bank,
    uint64_t status, uint64_t addr, uint64_t misc, void *mslogout)
{
	authamd_error_disp_t *disp;
	uint32_t rv = 0;

	if (authamd_forgive_uc)
		rv |= CMS_ERRSCOPE_CLEARED_UC;

	if (authamd_forgive_pcc)
		rv |= CMS_ERRSCOPE_CURCONTEXT_OK;

	if (authamd_fake_poison && status & MSR_MC_STATUS_UC)
		rv |= CMS_ERRSCOPE_POISONED;

	if (rv)
		return (rv);

	disp = authamd_disp_match(hdl, ismc, bank, status, addr, misc,
	    mslogout);

	if (disp == &authamd_gart_disp) {
		/*
		 * GART walk errors set UC and possibly PCC (if source CPU)
		 * but should not be regarded as terminal.
		 */
		return (CMS_ERRSCOPE_IGNORE_ERR);
	}

	/*
	 * May also want to consider master abort and target abort.  These
	 * also set UC and PCC (if src CPU) but the requester gets -1
	 * and I believe the IO stuff in Solaris will handle that.
	 */

	return (rv);
}

/*
 * cms_disp_match entry point
 */
/*ARGSUSED*/
cms_cookie_t
authamd_disp_match(cmi_hdl_t hdl, int ismc, int bank, uint64_t status,
    uint64_t addr, uint64_t misc, void *mslogout)
{
	authamd_data_t *authamd = cms_hdl_getcmsdata(hdl);
	/* uint16_t errcode = MCAX86_ERRCODE(status); */
	uint16_t exterrcode = AMD_EXT_ERRCODE(status);
	x86_chiprev_t rev = authamd->amd_shared->ans_rev;

	/*
	 * Recognise main memory ECC errors
	 */
	if (AUTHAMD_MEMECC_RECOGNISED(rev) &&
	    AUTHAMD_IS_MEMECCERR(bank, status)) {
		if (status & AMD_BANK_STAT_CECC) {
			return (exterrcode == 0 ? &authamd_memce_disp :
			    &authamd_ckmemce_disp);
		} else if (status & AMD_BANK_STAT_UECC) {
			return (exterrcode == 0 ? &authamd_memue_disp :
			    &authamd_ckmemue_disp);
		}
	}

	/*
	 * Recognise GART walk errors
	 */
	if (AUTHAMD_NOGARTTBLWLK_MC(rev) && AUTHAMD_IS_GARTERR(bank, status))
		return (&authamd_gart_disp);

	return (NULL);
}

/*
 * cms_ereport_class entry point
 */
/*ARGSUSED*/
void
authamd_ereport_class(cmi_hdl_t hdl, cms_cookie_t mscookie,
    const char **cpuclsp, const char **leafclsp)
{
	const authamd_error_disp_t *aed = mscookie;

	if (aed == NULL)
		return;

	if (aed->aad_subclass != NULL)
		*cpuclsp = aed->aad_subclass;
	if (aed->aad_leafclass != NULL)
		*leafclsp = aed->aad_leafclass;
}

/*ARGSUSED*/
static void
authamd_ereport_add_resource(cmi_hdl_t hdl, authamd_data_t *authamd,
    nvlist_t *ereport, nv_alloc_t *nva, void *mslogout)
{
	nvlist_t *elems[AUTHAMD_DRAM_NCHANNEL * AUTHAMD_DRAM_NCS];
	uint8_t counts[AUTHAMD_DRAM_NCHANNEL * AUTHAMD_DRAM_NCS];
	authamd_logout_t *msl;
	nvlist_t *nvl;
	int nelems = 0;
	int i, chan, cs, mc;
	nvlist_t *board_list = NULL;

	if ((msl = mslogout) == NULL)
		return;

	/* Assume all processors have the same number of nodes */
	mc = authamd->amd_shared->ans_procnodeid %
	    cpuid_get_procnodes_per_pkg(CPU);

	for (chan = 0; chan < AUTHAMD_DRAM_NCHANNEL; chan++) {
		for (cs = 0; cs < AUTHAMD_DRAM_NCS; cs++) {
			if (msl->aal_eccerrcnt[chan][cs] == 0)
				continue;

			if ((nvl = fm_nvlist_create(nva)) == NULL)
				continue;

			elems[nelems] = nvl;
			counts[nelems++] = msl->aal_eccerrcnt[chan][cs];

			if (!x86gentopo_legacy) {
				board_list = cmi_hdl_smb_bboard(hdl);
				if (board_list == NULL)
					continue;
				fm_fmri_hc_create(nvl, FM_HC_SCHEME_VERSION,
				    NULL, NULL, board_list, 4,
				    "chip", cmi_hdl_smb_chipid(hdl),
				    "memory-controller", 0,
				    "dram-channel", chan,
				    "chip-select", cs);
			} else {
				fm_fmri_hc_set(nvl, FM_HC_SCHEME_VERSION,
				    NULL, NULL, 5,
				    "motherboard", 0,
				    "chip", authamd->amd_shared->ans_chipid,
				    "memory-controller", mc,
				    "dram-channel", chan,
				    "chip-select", cs);
			}
		}
	}

	if (nelems == 0)
		return;

	fm_payload_set(ereport, FM_EREPORT_GENAMD_PAYLOAD_NAME_RESOURCE,
	    DATA_TYPE_NVLIST_ARRAY, nelems, elems,
	    NULL);

	fm_payload_set(ereport, FM_EREPORT_GENAMD_PAYLOAD_NAME_RESOURCECNT,
	    DATA_TYPE_UINT8_ARRAY, nelems, &counts[0],
	    NULL);

	for (i = 0; i < nelems; i++)
		fm_nvlist_destroy(elems[i], nva ? FM_NVA_RETAIN : FM_NVA_FREE);
}

/*
 * cms_ereport_add_logout entry point
 */
/*ARGSUSED*/
void
authamd_ereport_add_logout(cmi_hdl_t hdl, nvlist_t *ereport, nv_alloc_t *nva,
    int bank, uint64_t status, uint64_t addr, uint64_t misc,
    void *mslogout, cms_cookie_t mscookie)
{
	authamd_data_t *authamd = cms_hdl_getcmsdata(hdl);
	const authamd_error_disp_t *aed = mscookie;
	uint64_t members;

	if (aed == NULL)
		return;

	members = aed->aad_ereport_members;

	if (members & FM_EREPORT_GENAMD_PAYLOAD_FLAG_SYND) {
		fm_payload_set(ereport, FM_EREPORT_GENAMD_PAYLOAD_NAME_SYND,
		    DATA_TYPE_UINT16, (uint16_t)AMD_BANK_SYND(status),
		    NULL);

		if (members & FM_EREPORT_GENAMD_PAYLOAD_FLAG_SYNDTYPE) {
			fm_payload_set(ereport,
			    FM_EREPORT_GENAMD_PAYLOAD_NAME_SYNDTYPE,
			    DATA_TYPE_STRING, "E",
			    NULL);
		}
	}

	if (members & FM_EREPORT_GENAMD_PAYLOAD_FLAG_CKSYND) {
		fm_payload_set(ereport, FM_EREPORT_GENAMD_PAYLOAD_NAME_CKSYND,
		    DATA_TYPE_UINT16, (uint16_t)AMD_NB_STAT_CKSYND(status),
		    NULL);

		if (members & FM_EREPORT_GENAMD_PAYLOAD_FLAG_SYNDTYPE) {
			fm_payload_set(ereport,
			    FM_EREPORT_GENAMD_PAYLOAD_NAME_SYNDTYPE,
			    DATA_TYPE_STRING, authamd->amd_shared->ans_eccsymsz,
			    NULL);
		}
	}

	if (members & FM_EREPORT_GENAMD_PAYLOAD_FLAG_RESOURCE &&
	    status & MSR_MC_STATUS_ADDRV) {
		authamd_ereport_add_resource(hdl, authamd, ereport, nva,
		    mslogout);
	}
}

/*
 * cms_msrinject entry point
 */
cms_errno_t
authamd_msrinject(cmi_hdl_t hdl, uint_t msr, uint64_t val)
{
	authamd_data_t *authamd = cms_hdl_getcmsdata(hdl);
	cms_errno_t rv = CMSERR_BADMSRWRITE;

	authamd_bankstatus_prewrite(hdl, authamd);
	if (cmi_hdl_wrmsr(hdl, msr, val) == CMI_SUCCESS)
		rv = CMS_SUCCESS;
	authamd_bankstatus_postwrite(hdl, authamd);

	return (rv);
}

cms_api_ver_t _cms_api_version = CMS_API_VERSION_2;

const cms_ops_t _cms_ops = {
	authamd_init,			/* cms_init */
	NULL,				/* cms_post_startup */
	NULL,				/* cms_post_mpstartup */
	authamd_logout_size,		/* cms_logout_size */
	authamd_mcgctl_val,		/* cms_mcgctl_val */
	authamd_bankctl_skipinit,	/* cms_bankctl_skipinit */
	authamd_bankctl_val,		/* cms_bankctl_val */
	NULL,				/* cms_bankstatus_skipinit */
	NULL,				/* cms_bankstatus_val */
	authamd_mca_init,		/* cms_mca_init */
	authamd_poll_ownermask,		/* cms_poll_ownermask */
	authamd_bank_logout,		/* cms_bank_logout */
	authamd_error_action,		/* cms_error_action */
	authamd_disp_match,		/* cms_disp_match */
	authamd_ereport_class,		/* cms_ereport_class */
	NULL,				/* cms_ereport_detector */
	NULL,				/* cms_ereport_includestack */
	authamd_ereport_add_logout,	/* cms_ereport_add_logout */
	authamd_msrinject,		/* cms_msrinject */
	NULL,				/* cms_fini */
};

static struct modlcpu modlcpu = {
	&mod_cpuops,
	"Generic AMD model-specific MCA"
};

static struct modlinkage modlinkage = {
	MODREV_1,
	(void *)&modlcpu,
	NULL
};

int
_init(void)
{
	return (mod_install(&modlinkage));
}

int
_info(struct modinfo *modinfop)
{
	return (mod_info(&modlinkage, modinfop));
}

int
_fini(void)
{
	return (mod_remove(&modlinkage));
}
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright (c) 2006, 2010, Oracle and/or its affiliates. All rights reserved.
 * Copyright (c) 2018, Joyent, Inc.
 */

#ifndef _GCPU_H
#define	_GCPU_H

#include <sys/types.h>
#include <sys/cpu_module_impl.h>
#include <sys/cpu_module_ms.h>
#include <sys/ksynch.h>
#include <sys/systm.h>
#include <sys/fm/util.h>

#ifdef __cplusplus
extern "C" {
#endif

#define	GCPU_MCA_ERRS_PERCPU	10	/* errorq slots per cpu */
#define	GCPU_MCA_MIN_ERRORS	30	/* minimum total errorq slots */
#define	GCPU_MCA_MAX_ERRORS	100	/* maximum total errorq slots */

typedef struct gcpu_data gcpu_data_t;

#define	GCPU_ERRCODE_MASK_ALL		0xffff

typedef struct gcpu_error_disp {
	const char *ged_class_fmt;	/* ereport class formatter (last bit) */
	const char *ged_compound_fmt;	/* compound error formatter */
	uint64_t ged_ereport_members;	/* ereport payload members */
	uint16_t ged_errcode_mask_on;	/* errcode bits that must be set ... */
	uint16_t ged_errcode_mask_off;	/* ... and must be clear for a match */
} gcpu_error_disp_t;

/*
 * For errorq_dispatch we need to have a single contiguous structure
 * capturing all our logout data.  We do not know in advance how many
 * error detector banks there are in this cpu model, so we'll manually
 * allocate additional space for the gcl_banks array below.
 */
typedef struct gcpu_bank_logout {
	uint64_t gbl_status;		/* MCi_STATUS value */
	uint64_t gbl_addr;		/* MCi_ADDR value */
	uint64_t gbl_misc;		/* MCi_MISC value */
	uint64_t gbl_disp;		/* Error disposition for this bank */
	uint32_t gbl_clrdefcnt;		/* Count of deferred status clears */
} gcpu_bank_logout_t;

/*
 * The data structure we "logout" all error telemetry from all banks of
 * a cpu to.  The gcl_data array declared with 1 member below will actually
 * have gcl_nbanks members - variable with the actual cpu model present.
 * After the gcl_data array there is a further model-specific array that
 * may be allocated, and gcl_ms_logout will point to that if present.
 * This cpu logout data must form one contiguous chunk of memory for
 * dispatch with errorq_dispatch.
 */
typedef struct gcpu_logout {
	gcpu_data_t *gcl_gcpu;		/* pointer to per-cpu gcpu_data_t */
	uintptr_t gcl_ip;		/* instruction pointer from #mc trap */
	uint64_t gcl_timestamp;		/* gethrtime() at logout */
	uint64_t gcl_mcg_status;	/* MCG_STATUS register value */
	uint64_t gcl_flags;		/* Flags */
	pc_t gcl_stack[FM_STK_DEPTH];	/* saved stack trace, if any */
	int gcl_stackdepth;		/* saved stack trace depth */
	int ismc;			/* is a machine check flag */
	int gcl_nbanks;			/* number of banks in array below */
	void *gcl_ms_logout;		/* Model-specific area after gcl_data */
	gcpu_bank_logout_t gcl_data[1];	/* Bank logout areas - must be last */
} gcpu_logout_t;

/*
 * gcl_flag values
 */
#define	GCPU_GCL_F_PRIV		0x1	/* #MC during privileged code */
#define	GCPU_GCL_F_TES_P	0x2	/* MCG_CAP indicates TES_P */

struct gcpu_bios_bankcfg {
	uint64_t bios_bank_ctl;
	uint64_t bios_bank_status;
	uint64_t bios_bank_addr;
	uint64_t bios_bank_misc;
};

struct gcpu_bios_cfg {
	uint64_t bios_mcg_cap;
	uint64_t bios_mcg_ctl;
	struct gcpu_bios_bankcfg *bios_bankcfg;
};

/*
 * Events types in poll trace records.  Keep these in sync with
 * the generic cpu mdb module names for each (see gcpu_mpt_dump in mdb).
 */
#define	GCPU_MPT_WHAT_CYC_ERR		0	/* cyclic-induced poll */
#define	GCPU_MPT_WHAT_POKE_ERR		1	/* manually-induced poll */
#define	GCPU_MPT_WHAT_UNFAULTING	2	/* discarded error state */
#define	GCPU_MPT_WHAT_MC_ERR		3	/* MC# */
#define	GCPU_MPT_WHAT_CMCI_ERR		4	/* CMCI interrupt */
#define	GCPU_MPT_WHAT_XPV_VIRQ		5	/* MCA_VIRQ in dom0 */
#define	GCPU_MPT_WHAT_XPV_VIRQ_LOGOUT	6	/* MCA_VIRQ logout complete */

typedef struct gcpu_poll_trace {
	hrtime_t mpt_when;		/* timestamp of event */
	uint8_t mpt_what;		/* GCPU_MPT_WHAT_* (which event?) */
	uint8_t mpt_nerr;		/* number of errors discovered */
	uint16_t mpt_pad1;
	uint32_t mpt_pad2;
} gcpu_poll_trace_t;

typedef struct gcpu_poll_trace_ctl {
	gcpu_poll_trace_t *mptc_tbufs;	/* trace buffers */
	uint_t mptc_curtrace;			/* last buffer filled */
} gcpu_poll_trace_ctl_t;


/*
 * For counting some of the important number or time for runtime
 * cmci enable/disable
 */
typedef struct gcpu_mca_cmci {
	uint32_t cmci_cap;	/* cmci capability for this bank */
	uint32_t ncmci;		/* number of correctable errors between polls */
	uint32_t drtcmci;	/* duration of no cmci when cmci is disabled */
	uint32_t cmci_enabled;	/* cmci enable/disable status for this bank */
} gcpu_mca_cmci_t;

/* Index for gcpu_mca_logout array below */
#define	GCPU_MCA_LOGOUT_EXCEPTION	0	/* area for #MC */
#define	GCPU_MCA_LOGOUT_POLLER_1	1	/* next/prev poll area */
#define	GCPU_MCA_LOGOUT_POLLER_2	2	/* prev/next poll area */
#define	GCPU_MCA_LOGOUT_NUM		3

typedef struct gcpu_mca {
	gcpu_logout_t *gcpu_mca_logout[GCPU_MCA_LOGOUT_NUM];
	uint32_t gcpu_mca_nextpoll_idx;	/* logout area for next poll */
	struct gcpu_bios_cfg gcpu_mca_bioscfg;
	uint_t gcpu_mca_nbanks;
	size_t gcpu_mca_lgsz;		/* size of gcpu_mca_logout structs */
	uint_t gcpu_mca_flags;		/* GCPU_MCA_F_* */
	hrtime_t gcpu_mca_lastpoll;
	gcpu_poll_trace_ctl_t gcpu_polltrace;
	uint32_t gcpu_mca_first_poll_cmci_enabled; /* cmci on in first poll */
	gcpu_mca_cmci_t *gcpu_bank_cmci;
} gcpu_mca_t;

typedef struct gcpu_mce_status {
	uint_t mce_nerr;	/* total errors found in logout of all banks */
	uint64_t mce_disp;	/* Disposition information */
	uint_t mce_npcc;	/* number of errors with PCC */
	uint_t mce_npcc_ok;	/* PCC with CMS_ERRSCOPE_CURCONTEXT_OK */
	uint_t mce_nuc;		/* number of errors with UC */
	uint_t mce_nuc_ok;	/* UC with CMS_ERRSCOPE_CLEARED_UC */
	uint_t mce_nuc_poisoned; /* UC with CMS_ERRSCOPE_POISONED */
	uint_t mce_forcefatal;	/* CMS_ERRSCOPE_FORCE_FATAL */
	uint_t mce_ignored;	/* CMS_ERRSCOPE_IGNORE_ERR */
} gcpu_mce_status_t;

/*
 * Flags for gcpu_mca_flags
 */
#define	GCPU_MCA_F_UNFAULTING		0x1	/* CPU exiting faulted state */
#define	GCPU_MCA_F_CMCI_CAPABLE		0x2	/* CPU supports CMCI */
#define	GCPU_MCA_F_CMCI_ENABLE		0x4	/* CPU CMCI enabled */

/*
 * State shared by all cpus on a chip
 */
struct gcpu_chipshared {
	kmutex_t gcpus_cfglock;		/* serial MCA config from chip cores */
	kmutex_t gcpus_poll_lock;	/* serialize pollers on the same chip */
	uint32_t gcpus_actv_banks;	/* MCA bank numbers active on chip */
	volatile uint32_t gcpus_actv_cnt; /* active cpu count in this chip */
	char *gcpus_ident;		/* ident string, if available */
};

struct gcpu_data {
	gcpu_mca_t gcpu_mca;			/* MCA state for this CPU */
	cmi_hdl_t gcpu_hdl;			/* associated handle */
	struct gcpu_chipshared *gcpu_shared;	/* Shared state for the chip */
};

#ifdef _KERNEL

struct regs;

/*
 * CMI implementation
 */
extern int gcpu_init(cmi_hdl_t, void **);
extern void gcpu_fini(cmi_hdl_t);
extern void gcpu_post_startup(cmi_hdl_t);
extern void gcpu_post_mpstartup(cmi_hdl_t);
extern void gcpu_faulted_enter(cmi_hdl_t);
extern void gcpu_faulted_exit(cmi_hdl_t);
extern void gcpu_mca_init(cmi_hdl_t);
extern void gcpu_mca_fini(cmi_hdl_t);
extern void gcpu_mca_cmci_enable(cmi_hdl_t);
extern cmi_errno_t gcpu_msrinject(cmi_hdl_t, cmi_mca_regs_t *, uint_t, int);
#ifndef __xpv
extern uint64_t gcpu_mca_trap(cmi_hdl_t, struct regs *);
extern void gcpu_cmci_trap(cmi_hdl_t);
extern void gcpu_hdl_poke(cmi_hdl_t);
#else
extern void gcpu_xpv_panic_callback(void);
#endif

/*
 * Local functions
 */
extern void gcpu_mca_poll_init(cmi_hdl_t);
extern void gcpu_mca_poll_fini(cmi_hdl_t);
extern void gcpu_mca_poll_start(cmi_hdl_t);
extern void gcpu_poll_trace_init(gcpu_poll_trace_ctl_t *);
extern void gcpu_poll_trace(gcpu_poll_trace_ctl_t *, uint8_t, uint8_t);
extern void gcpu_mca_logout(cmi_hdl_t, struct regs *, uint64_t,
    gcpu_mce_status_t *, boolean_t, int);
#ifdef __xpv
extern void gcpu_xpv_mca_init(int);
#endif /* __xpv */

#endif /* _KERNEL */

#ifdef __cplusplus
}
#endif

#endif /* _GCPU_H */
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 * Copyright (c) 2018, Joyent, Inc.
 */
/*
 * Copyright (c) 2010, Intel Corporation.
 * All rights reserved.
 */

/*
 * Copyright (c) 2018, Joyent, Inc.
 * Copyright 2020 RackTop Systems, Inc.
 */

/*
 * Generic x86 CPU Module
 *
 * This CPU module is used for generic x86 CPUs when Solaris has no other
 * CPU-specific support module available.  Code in this module should be the
 * absolute bare-bones support and must be cognizant of both Intel and AMD etc.
 */

#include <sys/types.h>
#include <sys/cpu_module_impl.h>
#include <sys/cpuvar.h>
#include <sys/kmem.h>
#include <sys/modctl.h>
#include <sys/pghw.h>
#include <sys/x86_archext.h>

#include "gcpu.h"

/*
 * Prevent generic cpu support from loading.
 */
int gcpu_disable = 0;

#define	GCPU_MAX_CHIPID		32
static struct gcpu_chipshared *gcpu_shared[GCPU_MAX_CHIPID];
#ifdef	DEBUG
int gcpu_id_disable = 0;
static const char *gcpu_id_override[GCPU_MAX_CHIPID] = { NULL };
#endif

#ifndef	__xpv

/*
 * The purpose of this is to construct a unique identifier for a given processor
 * that can be used by things like FMA to determine when a FRU has been
 * replaced. It is supported on Intel Xeon Platforms since Ivy Bridge and AMD
 * 17h processors since Rome. See cpuid_pass1_ppin() for how we determine if a
 * CPU is supported.
 *
 * The protected processor inventory number (PPIN) can be used to create a
 * unique identifier when combined with the processor's cpuid signature. We
 * create a versioned, synthetic ID using the following scheme for the
 * identifier: iv0-<vendor>-<signature>-<PPIN>. The iv0 is the illumos version
 * zero of the ID. If we have a new scheme for a new generation of processors,
 * then that should rev the version field, otherwise for a given processor, this
 * synthetic ID should not change.
 *
 * We use the string "INTC" for Intel and "AMD" for AMD. None of these or the
 * formatting of the values can change without changing the version string.
 */
static char *
gcpu_init_ident_ppin(cmi_hdl_t hdl)
{
	uint_t ppin_ctl_msr, ppin_msr;
	uint64_t value;
	const char *vendor;

	/*
	 * This list should be extended as new Intel Xeon family processors come
	 * out.
	 */
	switch (cmi_hdl_vendor(hdl)) {
	case X86_VENDOR_Intel:
		ppin_ctl_msr = MSR_PPIN_CTL_INTC;
		ppin_msr = MSR_PPIN_INTC;
		vendor = "INTC";
		break;
	case X86_VENDOR_AMD:
		ppin_ctl_msr = MSR_PPIN_CTL_AMD;
		ppin_msr = MSR_PPIN_AMD;
		vendor = "AMD";
		break;
	default:
		return (NULL);
	}

	if (cmi_hdl_rdmsr(hdl, ppin_ctl_msr, &value) != CMI_SUCCESS) {
		return (NULL);
	}

	/*
	 * If the PPIN is not enabled and not locked, attempt to enable it.
	 * Note: in some environments such as Amazon EC2 the PPIN appears
	 * to be disabled and unlocked but our attempts to enable it don't
	 * stick, and when we attempt to read the PPIN we get an uncaught
	 * #GP. To avoid that happening we read the MSR back and verify it
	 * has taken the new value.
	 */
	if ((value & MSR_PPIN_CTL_ENABLED) == 0) {
		if ((value & MSR_PPIN_CTL_LOCKED) != 0) {
			return (NULL);
		}

		if (cmi_hdl_wrmsr(hdl, ppin_ctl_msr, MSR_PPIN_CTL_ENABLED) !=
		    CMI_SUCCESS) {
			return (NULL);
		}

		if (cmi_hdl_rdmsr(hdl, ppin_ctl_msr, &value) != CMI_SUCCESS) {
			return (NULL);
		}

		if ((value & MSR_PPIN_CTL_ENABLED) == 0) {
			return (NULL);
		}
	}

	if (cmi_hdl_rdmsr(hdl, ppin_msr, &value) != CMI_SUCCESS) {
		return (NULL);
	}

	/*
	 * Now that we've read data, lock the PPIN. Don't worry about success or
	 * failure of this part, as we will have gotten everything that we need.
	 * It is possible that it locked open, for example.
	 */
	if (cmi_hdl_wrmsr(hdl, ppin_ctl_msr, MSR_PPIN_CTL_DISABLED) ==
	    CMI_SUCCESS) {
		(void) cmi_hdl_wrmsr(hdl, ppin_ctl_msr, MSR_PPIN_CTL_LOCKED);
	}

	return (kmem_asprintf("iv0-%s-%x-%llx", vendor, cmi_hdl_chipsig(hdl),
	    value));
}
#endif	/* __xpv */

static void
gcpu_init_ident(cmi_hdl_t hdl, struct gcpu_chipshared *sp)
{
#ifdef	DEBUG
	uint_t chipid;

	/*
	 * On debug, allow a developer to override the string to more
	 * easily test CPU autoreplace without needing to physically
	 * replace a CPU.
	 */
	if (gcpu_id_disable != 0) {
		return;
	}

	chipid = cmi_hdl_chipid(hdl);
	if (gcpu_id_override[chipid] != NULL) {
		sp->gcpus_ident = strdup(gcpu_id_override[chipid]);
		return;
	}
#endif

#ifndef __xpv
	if (is_x86_feature(x86_featureset, X86FSET_PPIN)) {
		sp->gcpus_ident = gcpu_init_ident_ppin(hdl);
	}
#endif	/* __xpv */
}

/*
 * Our cmi_init entry point, called during startup of each cpu instance.
 */
int
gcpu_init(cmi_hdl_t hdl, void **datap)
{
	uint_t chipid = cmi_hdl_chipid(hdl);
	struct gcpu_chipshared *sp, *osp;
	gcpu_data_t *gcpu;

	if (gcpu_disable || chipid >= GCPU_MAX_CHIPID)
		return (ENOTSUP);

	/*
	 * Allocate the state structure for this cpu.  We will only
	 * allocate the bank logout areas in gcpu_mca_init once we
	 * know how many banks there are.
	 */
	gcpu = *datap = kmem_zalloc(sizeof (gcpu_data_t), KM_SLEEP);
	cmi_hdl_hold(hdl);	/* release in gcpu_fini */
	gcpu->gcpu_hdl = hdl;

	/*
	 * Allocate a chipshared structure if no sibling cpu has already
	 * allocated it, but allow for the fact that a sibling core may
	 * be starting up in parallel.
	 */
	if ((sp = gcpu_shared[chipid]) == NULL) {
		sp = kmem_zalloc(sizeof (struct gcpu_chipshared), KM_SLEEP);
		mutex_init(&sp->gcpus_poll_lock, NULL, MUTEX_DRIVER, NULL);
		mutex_init(&sp->gcpus_cfglock, NULL, MUTEX_DRIVER, NULL);
		osp = atomic_cas_ptr(&gcpu_shared[chipid], NULL, sp);
		if (osp != NULL) {
			mutex_destroy(&sp->gcpus_cfglock);
			mutex_destroy(&sp->gcpus_poll_lock);
			kmem_free(sp, sizeof (struct gcpu_chipshared));
			sp = osp;
		} else {
			gcpu_init_ident(hdl, sp);
		}
	}

	atomic_inc_32(&sp->gcpus_actv_cnt);
	gcpu->gcpu_shared = sp;

	return (0);
}

/*
 * deconfigure gcpu_init()
 */
void
gcpu_fini(cmi_hdl_t hdl)
{
	uint_t chipid = cmi_hdl_chipid(hdl);
	gcpu_data_t *gcpu = cmi_hdl_getcmidata(hdl);
	struct gcpu_chipshared *sp;

	if (gcpu_disable || chipid >= GCPU_MAX_CHIPID)
		return;

	gcpu_mca_fini(hdl);

	/*
	 * Keep shared data in cache for reuse.
	 */
	sp = gcpu_shared[chipid];
	ASSERT(sp != NULL);
	atomic_dec_32(&sp->gcpus_actv_cnt);

	if (gcpu != NULL)
		kmem_free(gcpu, sizeof (gcpu_data_t));

	/* Release reference count held in gcpu_init(). */
	cmi_hdl_rele(hdl);
}

void
gcpu_post_startup(cmi_hdl_t hdl)
{
	gcpu_data_t *gcpu = cmi_hdl_getcmidata(hdl);

	if (gcpu_disable)
		return;

	if (gcpu != NULL)
		cms_post_startup(hdl);
#ifdef __xpv
	/*
	 * All cpu handles are initialized so we can begin polling now.
	 * Furthermore, our virq mechanism requires that everything
	 * be run on cpu 0 so we can assure that by starting from here.
	 */
	gcpu_mca_poll_start(hdl);
#else
	/*
	 * The boot CPU has a bit of a chicken and egg problem for CMCI. Its MCA
	 * initialization is run before we have initialized the PSM module that
	 * we would use for enabling CMCI. Therefore, we use this as a chance to
	 * enable CMCI for the boot CPU. For all other CPUs, this chicken and
	 * egg problem will have already been solved.
	 */
	gcpu_mca_cmci_enable(hdl);
#endif
}

void
gcpu_post_mpstartup(cmi_hdl_t hdl)
{
	if (gcpu_disable)
		return;

	cms_post_mpstartup(hdl);

#ifndef __xpv
	/*
	 * All cpu handles are initialized only once all cpus are started, so we
	 * can begin polling post mp startup.
	 */
	gcpu_mca_poll_start(hdl);
#endif
}

const char *
gcpu_ident(cmi_hdl_t hdl)
{
	uint_t chipid;
	struct gcpu_chipshared *sp;

	if (gcpu_disable)
		return (NULL);

	chipid = cmi_hdl_chipid(hdl);
	if (chipid >= GCPU_MAX_CHIPID)
		return (NULL);

	if (cmi_hdl_getcmidata(hdl) == NULL)
		return (NULL);

	sp = gcpu_shared[cmi_hdl_chipid(hdl)];
	return (sp->gcpus_ident);
}

#ifdef __xpv
#define	GCPU_OP(ntvop, xpvop)	xpvop
#else
#define	GCPU_OP(ntvop, xpvop)	ntvop
#endif

cmi_api_ver_t _cmi_api_version = CMI_API_VERSION_3;

const cmi_ops_t _cmi_ops = {
	gcpu_init,				/* cmi_init */
	gcpu_post_startup,			/* cmi_post_startup */
	gcpu_post_mpstartup,			/* cmi_post_mpstartup */
	gcpu_faulted_enter,			/* cmi_faulted_enter */
	gcpu_faulted_exit,			/* cmi_faulted_exit */
	gcpu_mca_init,				/* cmi_mca_init */
	GCPU_OP(gcpu_mca_trap, NULL),		/* cmi_mca_trap */
	GCPU_OP(gcpu_cmci_trap, NULL),		/* cmi_cmci_trap */
	gcpu_msrinject,				/* cmi_msrinject */
	GCPU_OP(gcpu_hdl_poke, NULL),		/* cmi_hdl_poke */
	gcpu_fini,				/* cmi_fini */
	GCPU_OP(NULL, gcpu_xpv_panic_callback),	/* cmi_panic_callback */
	gcpu_ident				/* cmi_ident */
};

static struct modlcpu modlcpu = {
	&mod_cpuops,
	"Generic x86 CPU Module"
};

static struct modlinkage modlinkage = {
	MODREV_1,
	(void *)&modlcpu,
	NULL
};

int
_init(void)
{
	return (mod_install(&modlinkage));
}

int
_info(struct modinfo *modinfop)
{
	return (mod_info(&modlinkage, modinfop));
}

int
_fini(void)
{
	return (mod_remove(&modlinkage));
}
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright (c) 2006, 2010, Oracle and/or its affiliates. All rights reserved.
 * Copyright (c) 2018, Joyent, Inc.
 * Copyright 2022 Oxide Computer Co.
 */
/*
 * Copyright (c) 2010, Intel Corporation.
 * All rights reserved.
 */

#include <sys/mca_x86.h>
#include <sys/cpu_module_impl.h>
#include <sys/cpu_module_ms.h>
#include <sys/cmn_err.h>
#include <sys/cpuvar.h>
#include <sys/pghw.h>
#include <sys/x86_archext.h>
#include <sys/sysmacros.h>
#include <sys/regset.h>
#include <sys/privregs.h>
#include <sys/systm.h>
#include <sys/types.h>
#include <sys/log.h>
#include <sys/psw.h>
#include <sys/fm/protocol.h>
#include <sys/fm/util.h>
#include <sys/errorq.h>
#include <sys/mca_x86.h>
#include <sys/fm/cpu/GMCA.h>
#include <sys/fm/smb/fmsmb.h>
#include <sys/sysevent.h>
#include <sys/ontrap.h>
#include <sys/smp_impldefs.h>

#include "gcpu.h"

extern int x86gentopo_legacy;	/* x86 generic topology support */

static uint_t gcpu_force_addr_in_payload = 0;

/*
 * Clear to log telemetry found at initialization.  While processor docs
 * say you should process this telemetry on all but Intel family 0x6
 * there are way too many exceptions and we want to avoid bogus
 * diagnoses.
 */
int gcpu_suppress_log_on_init = 1;

/*
 * gcpu_mca_stack_flag is a debug assist option to capture a stack trace at
 * error logout time.  The stack will be included in the ereport if the
 * error type selects stack inclusion, or in all cases if
 * gcpu_mca_stack_ereport_include is nonzero.
 */
int gcpu_mca_stack_flag = 0;
int gcpu_mca_stack_ereport_include = 0;

/*
 * The number of times to re-read MCA telemetry to try to obtain a
 * consistent snapshot if we find it to be changing under our feet.
 */
int gcpu_mca_telemetry_retries = 5;

#ifndef __xpv
int gcpu_mca_cmci_throttling_threshold = 10;
int gcpu_mca_cmci_reenable_threshold = 1000;

/*
 * This is used to determine whether or not we have registered the CMCI CPU
 * setup function. This is protected by cpu_lock.
 */
static boolean_t gcpu_mca_cpu_registered = B_FALSE;
#endif

static gcpu_error_disp_t gcpu_errtypes[] = {

	/*
	 * Unclassified
	 */
	{
		FM_EREPORT_CPU_GENERIC_UNCLASSIFIED,
		NULL,
		FM_EREPORT_PAYLOAD_FLAGS_COMMON,
		MCAX86_SIMPLE_UNCLASSIFIED_MASKON,
		MCAX86_SIMPLE_UNCLASSIFIED_MASKOFF
	},

	/*
	 * Microcode ROM Parity Error
	 */
	{
		FM_EREPORT_CPU_GENERIC_MC_CODE_PARITY,
		NULL,
		FM_EREPORT_PAYLOAD_FLAGS_COMMON,
		MCAX86_SIMPLE_MC_CODE_PARITY_MASKON,
		MCAX86_SIMPLE_MC_CODE_PARITY_MASKOFF
	},

	/*
	 * External - BINIT# from another processor during power-on config
	 */
	{
		FM_EREPORT_CPU_GENERIC_EXTERNAL,
		NULL,
		FM_EREPORT_PAYLOAD_FLAGS_COMMON,
		MCAX86_SIMPLE_EXTERNAL_MASKON,
		MCAX86_SIMPLE_EXTERNAL_MASKOFF
	},

	/*
	 * Functional redundancy check master/slave error
	 */
	{
		FM_EREPORT_CPU_GENERIC_FRC,
		NULL,
		FM_EREPORT_PAYLOAD_FLAGS_COMMON,
		MCAX86_SIMPLE_FRC_MASKON,
		MCAX86_SIMPLE_FRC_MASKOFF
	},

	/*
	 * Internal parity error
	 */
	{
		FM_EREPORT_CPU_GENERIC_INTERNAL_PARITY,
		NULL,
		FM_EREPORT_PAYLOAD_FLAGS_COMMON,
		MCAX86_SIMPLE_INTERNAL_PARITY_MASKON,
		MCAX86_SIMPLE_INTERNAL_PARITY_MASKOFF
	},


	/*
	 * Internal timer error
	 */
	{
		FM_EREPORT_CPU_GENERIC_INTERNAL_TIMER,
		NULL,
		FM_EREPORT_PAYLOAD_FLAGS_COMMON,
		MCAX86_SIMPLE_INTERNAL_TIMER_MASKON,
		MCAX86_SIMPLE_INTERNAL_TIMER_MASKOFF
	},

	/*
	 * Internal unclassified
	 */
	{
		FM_EREPORT_CPU_GENERIC_INTERNAL_UNCLASS,
		NULL,
		FM_EREPORT_PAYLOAD_FLAGS_COMMON,
		MCAX86_SIMPLE_INTERNAL_UNCLASS_MASK_MASKON,
		MCAX86_SIMPLE_INTERNAL_UNCLASS_MASK_MASKOFF
	},

	/*
	 * Compound error codes - generic memory hierarchy
	 */
	{
		FM_EREPORT_CPU_GENERIC_GENMEMHIER,
		NULL,
		FM_EREPORT_PAYLOAD_FLAGS_COMMON, /* yes, no compound name */
		MCAX86_COMPOUND_GENERIC_MEMHIER_MASKON,
		MCAX86_COMPOUND_GENERIC_MEMHIER_MASKOFF
	},

	/*
	 * Compound error codes - TLB errors
	 */
	{
		FM_EREPORT_CPU_GENERIC_TLB,
		"%1$s" "TLB" "%2$s" "_ERR",
		FM_EREPORT_PAYLOAD_FLAGS_COMPOUND_ERR,
		MCAX86_COMPOUND_TLB_MASKON,
		MCAX86_COMPOUND_TLB_MASKOFF
	},

	/*
	 * Compound error codes - memory hierarchy
	 */
	{
		FM_EREPORT_CPU_GENERIC_MEMHIER,
		"%1$s" "CACHE" "%2$s" "_" "%3$s" "_ERR",
		FM_EREPORT_PAYLOAD_FLAGS_COMPOUND_ERR,
		MCAX86_COMPOUND_MEMHIER_MASKON,
		MCAX86_COMPOUND_MEMHIER_MASKOFF
	},

	/*
	 * Compound error codes - bus and interconnect errors
	 */
	{
		FM_EREPORT_CPU_GENERIC_BUS_INTERCONNECT,
		"BUS" "%2$s" "_" "%4$s" "_" "%3$s" "_" "%5$s" "_" "%6$s" "_ERR",
		FM_EREPORT_PAYLOAD_FLAGS_COMPOUND_ERR,
		MCAX86_COMPOUND_BUS_INTERCONNECT_MASKON,
		MCAX86_COMPOUND_BUS_INTERCONNECT_MASKOFF
	},
	/*
	 * Compound error codes - memory controller errors
	 */
	{
		FM_EREPORT_CPU_GENERIC_MEMORY_CONTROLLER,
		"MC" "_" "%8$s" "_" "%9$s" "_ERR",
		FM_EREPORT_PAYLOAD_FLAGS_COMPOUND_ERR,
		MCAX86_COMPOUND_MEMORY_CONTROLLER_MASKON,
		MCAX86_COMPOUND_MEMORY_CONTROLLER_MASKOFF
	},
};

static gcpu_error_disp_t gcpu_unknown = {
	FM_EREPORT_CPU_GENERIC_UNKNOWN,
	"UNKNOWN",
	FM_EREPORT_PAYLOAD_FLAGS_COMMON,
	0,
	0
};

static errorq_t *gcpu_mca_queue;
static kmutex_t gcpu_mca_queue_lock;

#ifdef __xpv
static int isxpv = 1;
#else
static int isxpv = 0;
#endif

static const gcpu_error_disp_t *
gcpu_disp_match(uint16_t code)
{
	const gcpu_error_disp_t *ged = gcpu_errtypes;
	int i;

	for (i = 0; i < sizeof (gcpu_errtypes) / sizeof (gcpu_error_disp_t);
	    i++, ged++) {
		uint16_t on = ged->ged_errcode_mask_on;
		uint16_t off = ged->ged_errcode_mask_off;

		if ((code & on) == on && (code & off) == 0)
			return (ged);
	}

	return (NULL);
}

static uint16_t
bit_strip(uint16_t code, uint16_t mask, uint16_t shift)
{
	return ((code & mask) >> shift);
}

#define	BIT_STRIP(code, name) \
	bit_strip(code, MCAX86_ERRCODE_##name##_MASK, \
	MCAX86_ERRCODE_##name##_SHIFT)

#define	GCPU_MNEMONIC_UNDEF	"undefined"
#define	GCPU_MNEMONIC_RESVD	"reserved"

/*
 * Mappings of TT, LL, RRRR, PP, II and T values to compound error name
 * mnemonics and to ereport class name components.
 */

struct gcpu_mnexp {
	const char *mne_compound;	/* used in expanding compound errname */
	const char *mne_ereport;	/* used in expanding ereport class */
};

static struct gcpu_mnexp gcpu_TT_mnemonics[] = { /* MCAX86_ERRCODE_TT_* */
	{ "I", FM_EREPORT_CPU_GENERIC_TT_INSTR },		/* INSTR */
	{ "D", FM_EREPORT_CPU_GENERIC_TT_DATA },		/* DATA */
	{ "G", FM_EREPORT_CPU_GENERIC_TT_GEN },			/* GEN */
	{ GCPU_MNEMONIC_UNDEF, "" }
};

static struct gcpu_mnexp gcpu_LL_mnemonics[] = { /* MCAX86_ERRCODE_LL_* */
	{ "LO", FM_EREPORT_CPU_GENERIC_LL_L0 },			/* L0 */
	{ "L1",	FM_EREPORT_CPU_GENERIC_LL_L1 },			/* L1 */
	{ "L2",	FM_EREPORT_CPU_GENERIC_LL_L2 },			/* L2 */
	{ "LG", FM_EREPORT_CPU_GENERIC_LL_LG }			/* LG */
};

static struct gcpu_mnexp gcpu_RRRR_mnemonics[] = { /* MCAX86_ERRCODE_RRRR_* */
	{ "ERR", FM_EREPORT_CPU_GENERIC_RRRR_ERR },		/* ERR */
	{ "RD",	FM_EREPORT_CPU_GENERIC_RRRR_RD },		/* RD */
	{ "WR", FM_EREPORT_CPU_GENERIC_RRRR_WR },		/* WR */
	{ "DRD", FM_EREPORT_CPU_GENERIC_RRRR_DRD },		/* DRD */
	{ "DWR", FM_EREPORT_CPU_GENERIC_RRRR_DWR },		/* DWR */
	{ "IRD", FM_EREPORT_CPU_GENERIC_RRRR_IRD },		/* IRD */
	{ "PREFETCH", FM_EREPORT_CPU_GENERIC_RRRR_PREFETCH },	/* PREFETCH */
	{ "EVICT", FM_EREPORT_CPU_GENERIC_RRRR_EVICT },		/* EVICT */
	{ "SNOOP", FM_EREPORT_CPU_GENERIC_RRRR_SNOOP },		/* SNOOP */
};

static struct gcpu_mnexp gcpu_PP_mnemonics[] = { /* MCAX86_ERRCODE_PP_* */
	{ "SRC", FM_EREPORT_CPU_GENERIC_PP_SRC },		/* SRC */
	{ "RES", FM_EREPORT_CPU_GENERIC_PP_RES },		/* RES */
	{ "OBS", FM_EREPORT_CPU_GENERIC_PP_OBS },		/* OBS */
	{ "", FM_EREPORT_CPU_GENERIC_PP_GEN }			/* GEN */
};

static struct gcpu_mnexp gcpu_II_mnemonics[] = { /* MCAX86_ERRCODE_II_* */
	{ "M", FM_EREPORT_CPU_GENERIC_II_MEM },			/* MEM */
	{ GCPU_MNEMONIC_RESVD, "" },
	{ "IO", FM_EREPORT_CPU_GENERIC_II_IO },			/* IO */
	{ "", FM_EREPORT_CPU_GENERIC_II_GEN }			/* GEN */
};

static struct gcpu_mnexp gcpu_T_mnemonics[] = {	 /* MCAX86_ERRCODE_T_* */
	{ "NOTIMEOUT", FM_EREPORT_CPU_GENERIC_T_NOTIMEOUT },	/* NONE */
	{ "TIMEOUT", FM_EREPORT_CPU_GENERIC_T_TIMEOUT }		/* TIMEOUT */
};

static struct gcpu_mnexp gcpu_CCCC_mnemonics[] = { /* MCAX86_ERRCODE_CCCC_* */
	{ "CH0", FM_EREPORT_CPU_GENERIC_CCCC },		/* CH0 */
	{ "CH1", FM_EREPORT_CPU_GENERIC_CCCC },		/* CH1 */
	{ "CH2", FM_EREPORT_CPU_GENERIC_CCCC },		/* CH2 */
	{ "CH3", FM_EREPORT_CPU_GENERIC_CCCC },		/* CH3 */
	{ "CH4", FM_EREPORT_CPU_GENERIC_CCCC },		/* CH4 */
	{ "CH5", FM_EREPORT_CPU_GENERIC_CCCC },		/* CH5 */
	{ "CH6", FM_EREPORT_CPU_GENERIC_CCCC },		/* CH6 */
	{ "CH7", FM_EREPORT_CPU_GENERIC_CCCC },		/* CH7 */
	{ "CH8", FM_EREPORT_CPU_GENERIC_CCCC },		/* CH8 */
	{ "CH9", FM_EREPORT_CPU_GENERIC_CCCC },		/* CH9 */
	{ "CH10", FM_EREPORT_CPU_GENERIC_CCCC },	/* CH10 */
	{ "CH11", FM_EREPORT_CPU_GENERIC_CCCC },	/* CH11 */
	{ "CH12", FM_EREPORT_CPU_GENERIC_CCCC },	/* CH12 */
	{ "CH13", FM_EREPORT_CPU_GENERIC_CCCC },	/* CH13 */
	{ "CH14", FM_EREPORT_CPU_GENERIC_CCCC },	/* CH14 */
	{ "CH", FM_EREPORT_CPU_GENERIC_CCCC }		/* GEN */
};

static struct gcpu_mnexp gcpu_MMM_mnemonics[] = { /* MCAX86_ERRCODE_MMM_* */
	{ "GEN", FM_EREPORT_CPU_GENERIC_MMM_ERR },	/* GEN ERR */
	{ "RD", FM_EREPORT_CPU_GENERIC_MMM_RD },	/* READ  */
	{ "WR", FM_EREPORT_CPU_GENERIC_MMM_WR },	/* WRITE  */
	{ "ADDR_CMD", FM_EREPORT_CPU_GENERIC_MMM_ADRCMD },	/* ADDR, CMD  */
	{ "SCRUB", FM_EREPORT_CPU_GENERIC_MMM_SCRUB },
	{ GCPU_MNEMONIC_RESVD, ""},			/* RESERVED  */
	{ GCPU_MNEMONIC_RESVD, ""},			/* RESERVED  */
	{ GCPU_MNEMONIC_RESVD, ""}			/* RESERVED  */
};

enum gcpu_mn_namespace {
	GCPU_MN_NAMESPACE_COMPOUND,
	GCPU_MN_NAMESPACE_EREPORT
};

static const char *
gcpu_mnemonic(const struct gcpu_mnexp *tbl, size_t tbl_sz, uint16_t val,
    enum gcpu_mn_namespace nspace)
{
	if (val >= tbl_sz || val > 0xff)
		return (GCPU_MNEMONIC_UNDEF);	/* for all namespaces */

	switch (nspace) {
	case GCPU_MN_NAMESPACE_COMPOUND:
		return (tbl[val].mne_compound);
		/*NOTREACHED*/

	case GCPU_MN_NAMESPACE_EREPORT:
		return (tbl[val].mne_ereport);
		/*NOTREACHED*/

	default:
		return (GCPU_MNEMONIC_UNDEF);
		/*NOTREACHED*/
	}
}

/*
 * The ereport class leaf component is either a simple string with no
 * format specifiers, or a string with one or more embedded %n$s specifiers -
 * positional selection for string arguments.  The kernel snprintf does
 * not support %n$ (and teaching it to do so is too big a headache) so
 * we will expand this restricted format string ourselves.
 */

#define	GCPU_CLASS_VARCOMPS	9

#define	GCPU_MNEMONIC(code, name, nspace) \
	gcpu_mnemonic(gcpu_##name##_mnemonics, \
	sizeof (gcpu_##name##_mnemonics) / sizeof (struct gcpu_mnexp), \
	BIT_STRIP(code, name), nspace)

static void
gcpu_mn_fmt(const char *fmt, char *buf, size_t buflen, uint64_t status,
    enum gcpu_mn_namespace nspace)
{
	uint16_t code = MCAX86_ERRCODE(status);
	const char *mn[GCPU_CLASS_VARCOMPS];
	char *p = buf;			/* current position in buf */
	char *q = buf + buflen;		/* pointer past last char in buf */
	int which, expfmtchar, error;
	char c;

	mn[0] = GCPU_MNEMONIC(code, TT, nspace);
	mn[1] = GCPU_MNEMONIC(code, LL, nspace);
	mn[2] = GCPU_MNEMONIC(code, RRRR, nspace);
	mn[3] = GCPU_MNEMONIC(code, PP, nspace);
	mn[4] = GCPU_MNEMONIC(code, II, nspace);
	mn[5] = GCPU_MNEMONIC(code, T, nspace);
	mn[6] = (status & MSR_MC_STATUS_UC) ? "_uc" : "";
	mn[7] = GCPU_MNEMONIC(code, CCCC, nspace);
	mn[8] = GCPU_MNEMONIC(code, MMM, nspace);

	while (p < q - 1 && (c = *fmt++) != '\0') {
		if (c != '%') {
			/* not the beginning of a format specifier - copy */
			*p++ = c;
			continue;
		}

		error = 0;
		which = -1;
		expfmtchar = -1;

nextfmt:
		if ((c = *fmt++) == '\0')
			break;	/* early termination of fmt specifier */

		switch (c) {
		case '1':
		case '2':
		case '3':
		case '4':
		case '5':
		case '6':
		case '7':
		case '8':
		case '9':
			if (which != -1) { /* allow only one positional digit */
				error++;
				break;
			}
			which = c - '1';
			goto nextfmt;
			/*NOTREACHED*/

		case '$':
			if (which == -1) { /* no position specified */
				error++;
				break;
			}
			expfmtchar = 's';
			goto nextfmt;
			/*NOTREACHED*/

		case 's':
			if (expfmtchar != 's') {
				error++;
				break;
			}
			(void) snprintf(p, (uintptr_t)q - (uintptr_t)p, "%s",
			    mn[which]);
			p += strlen(p);
			break;

		default:
			error++;
			break;
		}

		if (error)
			break;
	}

	*p = '\0';	/* NUL termination */
}

static void
gcpu_erpt_clsfmt(const char *fmt, char *buf, size_t buflen, uint64_t status,
    const char *cpuclass, const char *leafclass)
{
	char *p = buf;			/* current position in buf */
	char *q = buf + buflen;		/* pointer past last char in buf */

	(void) snprintf(buf, (uintptr_t)q - (uintptr_t)p, "%s.%s.",
	    FM_ERROR_CPU, cpuclass ? cpuclass : FM_EREPORT_CPU_GENERIC);

	p += strlen(p);
	if (p >= q)
		return;

	if (leafclass == NULL) {
		gcpu_mn_fmt(fmt, p, (uintptr_t)q - (uintptr_t)p, status,
		    GCPU_MN_NAMESPACE_EREPORT);
	} else {
		(void) snprintf(p, (uintptr_t)q - (uintptr_t)p, "%s",
		    leafclass);
	}
}

/*
 * Create an "hc" scheme FMRI identifying the given cpu with
 * motherboard/chip/core/strand instance numbers.
 */
static nvlist_t *
gcpu_fmri_create(cmi_hdl_t hdl, nv_alloc_t *nva)
{
	nvlist_t *nvl, *fmri;

	if ((nvl = fm_nvlist_create(nva)) == NULL)
		return (NULL);

	if (!x86gentopo_legacy) {
		fmri = cmi_hdl_smb_bboard(hdl);
		if (fmri == NULL)
			return (NULL);

		fm_fmri_hc_create(nvl, FM_HC_SCHEME_VERSION,
		    NULL, NULL, fmri, 3,
		    "chip", cmi_hdl_smb_chipid(hdl),
		    "core", cmi_hdl_coreid(hdl),
		    "strand", cmi_hdl_strandid(hdl));
	} else {
		fm_fmri_hc_set(nvl, FM_HC_SCHEME_VERSION, NULL, NULL, 4,
		    "motherboard", 0,
		    "chip", cmi_hdl_chipid(hdl),
		    "core", cmi_hdl_coreid(hdl),
		    "strand", cmi_hdl_strandid(hdl));
	}

	return (nvl);
}

int gcpu_bleat_count_thresh = 5;
hrtime_t gcpu_bleat_min_interval = 10 * 1000000000ULL;

/*
 * Called when we are unable to propogate a logout structure onto an
 * errorq for subsequent ereport preparation and logging etc.  The caller
 * should usually only decide to call this for severe errors - those we
 * suspect we may need to panic for.
 */
static void
gcpu_bleat(cmi_hdl_t hdl, gcpu_logout_t *gcl)
{
	hrtime_t now  = gethrtime_waitfree();
	static hrtime_t gcpu_last_bleat;
	gcpu_bank_logout_t *gbl;
	static int bleatcount;
	int i;

	/*
	 * Throttle spamming of the console.  The first gcpu_bleat_count_thresh
	 * can come as fast as we like, but once we've spammed that many
	 * to the console we require a minimum interval to pass before
	 * any more complaints.
	 */
	if (++bleatcount > gcpu_bleat_count_thresh) {
		if (now - gcpu_last_bleat < gcpu_bleat_min_interval)
			return;
		else
			bleatcount = 0;
	}
	gcpu_last_bleat = now;

	cmn_err(CE_WARN,
	    "Machine-Check Errors unlogged on chip %d core %d strand %d, "
	    "raw dump follows", cmi_hdl_chipid(hdl), cmi_hdl_coreid(hdl),
	    cmi_hdl_strandid(hdl));
	cmn_err(CE_WARN, "MCG_STATUS 0x%016llx",
	    (u_longlong_t)gcl->gcl_mcg_status);
	for (i = 0, gbl = &gcl->gcl_data[0]; i < gcl->gcl_nbanks; i++, gbl++) {
		uint64_t status = gbl->gbl_status;

		if (!(status & MSR_MC_STATUS_VAL))
			continue;

		/* Force ADDRV for AMD Family 0xf and above */
		if (gcpu_force_addr_in_payload)
			status = status | MSR_MC_STATUS_ADDRV;

		switch (status & (MSR_MC_STATUS_ADDRV | MSR_MC_STATUS_MISCV)) {
		case MSR_MC_STATUS_ADDRV | MSR_MC_STATUS_MISCV:
			cmn_err(CE_WARN, "Bank %d (offset 0x%llx) "
			    "STAT 0x%016llx ADDR 0x%016llx MISC 0x%016llx",
			    i, IA32_MSR_MC(i, STATUS),
			    (u_longlong_t)gbl->gbl_status,
			    (u_longlong_t)gbl->gbl_addr,
			    (u_longlong_t)gbl->gbl_misc);
			break;

		case MSR_MC_STATUS_ADDRV:
			cmn_err(CE_WARN, "Bank %d (offset 0x%llx) "
			    "STAT 0x%016llx ADDR 0x%016llx",
			    i, IA32_MSR_MC(i, STATUS),
			    (u_longlong_t)gbl->gbl_status,
			    (u_longlong_t)gbl->gbl_addr);
			break;

		case MSR_MC_STATUS_MISCV:
			cmn_err(CE_WARN, "Bank %d (offset 0x%llx) "
			    "STAT 0x%016llx MISC 0x%016llx",
			    i, IA32_MSR_MC(i, STATUS),
			    (u_longlong_t)gbl->gbl_status,
			    (u_longlong_t)gbl->gbl_misc);
			break;

		default:
			cmn_err(CE_WARN, "Bank %d (offset 0x%llx) "
			    "STAT 0x%016llx",
			    i, IA32_MSR_MC(i, STATUS),
			    (u_longlong_t)gbl->gbl_status);
			break;

		}
	}
}

#define	_GCPU_BSTATUS(status, what) \
	FM_EREPORT_PAYLOAD_NAME_MC_STATUS_##what, DATA_TYPE_BOOLEAN_VALUE, \
	(status) & MSR_MC_STATUS_##what ? B_TRUE : B_FALSE

static void
gcpu_ereport_add_logout(nvlist_t *ereport, const gcpu_logout_t *gcl,
    uint_t bankno, const gcpu_error_disp_t *ged, uint16_t code)
{
	uint64_t members = ged ? ged->ged_ereport_members :
	    FM_EREPORT_PAYLOAD_FLAGS_COMMON;
	uint64_t mcg = gcl->gcl_mcg_status;
	int mcip = mcg & MCG_STATUS_MCIP;
	const gcpu_bank_logout_t *gbl = &gcl->gcl_data[bankno];
	uint64_t bstat = gbl->gbl_status;

	/*
	 * Include the compound error name if requested and if this
	 * is a compound error type.
	 */
	if (members & FM_EREPORT_PAYLOAD_FLAG_COMPOUND_ERR && ged &&
	    ged->ged_compound_fmt != NULL) {
		char buf[FM_MAX_CLASS];

		gcpu_mn_fmt(ged->ged_compound_fmt, buf, sizeof (buf), code,
		    GCPU_MN_NAMESPACE_COMPOUND);
		fm_payload_set(ereport, FM_EREPORT_PAYLOAD_NAME_COMPOUND_ERR,
		    DATA_TYPE_STRING, buf, NULL);
	}

	/*
	 * Include disposition information for this error
	 */
	if (members & FM_EREPORT_PAYLOAD_FLAG_DISP &&
	    gbl->gbl_disp != 0) {
		int i, empty = 1;
		char buf[128];
		char *p = buf, *q = buf + 128;
		static struct _gcpu_disp_name {
			uint64_t dv;
			const char *dn;
		} disp_names[] = {
			{ CMI_ERRDISP_CURCTXBAD,
			    "processor_context_corrupt" },
			{ CMI_ERRDISP_RIPV_INVALID,
			    "return_ip_invalid" },
			{ CMI_ERRDISP_UC_UNCONSTRAINED,
			    "unconstrained" },
			{ CMI_ERRDISP_FORCEFATAL,
			    "forcefatal" },
			{ CMI_ERRDISP_IGNORED,
			    "ignored" },
			{ CMI_ERRDISP_PCC_CLEARED,
			    "corrupt_context_cleared" },
			{ CMI_ERRDISP_UC_CLEARED,
			    "uncorrected_data_cleared" },
			{ CMI_ERRDISP_POISONED,
			    "poisoned" },
			{ CMI_ERRDISP_INCONSISTENT,
			    "telemetry_unstable" },
		};

		for (i = 0; i < sizeof (disp_names) /
		    sizeof (struct _gcpu_disp_name); i++) {
			if ((gbl->gbl_disp & disp_names[i].dv) == 0)
				continue;

			(void) snprintf(p, (uintptr_t)q - (uintptr_t)p,
			    "%s%s", empty ? "" : ",", disp_names[i].dn);
			p += strlen(p);
			empty = 0;
		}

		if (p != buf)
			fm_payload_set(ereport, FM_EREPORT_PAYLOAD_NAME_DISP,
			    DATA_TYPE_STRING, buf, NULL);
	}

	/*
	 * If MCG_STATUS is included add that and an indication of whether
	 * this ereport was the result of a machine check or poll.
	 */
	if (members & FM_EREPORT_PAYLOAD_FLAG_MCG_STATUS) {
		fm_payload_set(ereport, FM_EREPORT_PAYLOAD_NAME_MCG_STATUS,
		    DATA_TYPE_UINT64, mcg, NULL);

		fm_payload_set(ereport, FM_EREPORT_PAYLOAD_NAME_MCG_STATUS_MCIP,
		    DATA_TYPE_BOOLEAN_VALUE, mcip ? B_TRUE : B_FALSE, NULL);
	}

	/*
	 * If an instruction pointer is to be included add one provided
	 * MCG_STATUS indicated it is valid; meaningless for polled events.
	 */
	if (mcip && members & FM_EREPORT_PAYLOAD_FLAG_IP &&
	    mcg & MCG_STATUS_EIPV) {
		fm_payload_set(ereport, FM_EREPORT_PAYLOAD_NAME_IP,
		    DATA_TYPE_UINT64, gcl->gcl_ip, NULL);
	}

	/*
	 * Add an indication of whether the trap occured during privileged code.
	 */
	if (mcip && members & FM_EREPORT_PAYLOAD_FLAG_PRIV) {
		fm_payload_set(ereport, FM_EREPORT_PAYLOAD_NAME_PRIV,
		    DATA_TYPE_BOOLEAN_VALUE,
		    gcl->gcl_flags & GCPU_GCL_F_PRIV ? B_TRUE : B_FALSE, NULL);
	}

	/*
	 * If requested, add the index of the MCA bank.  This indicates the
	 * n'th bank of 4 MCA registers, and does not necessarily correspond
	 * to MCi_* - use the bank offset to correlate
	 */
	if (members & FM_EREPORT_PAYLOAD_FLAG_BANK_NUM) {
		fm_payload_set(ereport,
		    /* Bank number */
		    FM_EREPORT_PAYLOAD_NAME_BANK_NUM, DATA_TYPE_UINT8, bankno,
		    /* Offset of MCi_CTL */
		    FM_EREPORT_PAYLOAD_NAME_BANK_MSR_OFFSET, DATA_TYPE_UINT64,
		    IA32_MSR_MC(bankno, CTL),
		    NULL);
	}

	/*
	 * Add MCi_STATUS if requested, and decode it.
	 */
	if (members & FM_EREPORT_PAYLOAD_FLAG_MC_STATUS) {
		const char *tbes[] = {
			"No tracking",			/* 00 */
			"Green - below threshold",	/* 01 */
			"Yellow - above threshold",	/* 10 */
			"Reserved"			/* 11 */
		};

		fm_payload_set(ereport,
		    /* Bank MCi_STATUS */
		    FM_EREPORT_PAYLOAD_NAME_MC_STATUS, DATA_TYPE_UINT64, bstat,
		    /* Overflow? */
		    _GCPU_BSTATUS(bstat, OVER),
		    /* Uncorrected? */
		    _GCPU_BSTATUS(bstat, UC),
		    /* Enabled? */
		    _GCPU_BSTATUS(bstat, EN),
		    /* Processor context corrupt? */
		    _GCPU_BSTATUS(bstat, PCC),
		    /* Error code */
		    FM_EREPORT_PAYLOAD_NAME_MC_STATUS_ERRCODE,
		    DATA_TYPE_UINT16, MCAX86_ERRCODE(bstat),
		    /* Model-specific error code */
		    FM_EREPORT_PAYLOAD_NAME_MC_STATUS_EXTERRCODE,
		    DATA_TYPE_UINT16, MCAX86_MSERRCODE(bstat),
		    NULL);

		/*
		 * If MCG_CAP.TES_P indicates that that thresholding info
		 * is present in the architural component of the bank status
		 * then include threshold information for this bank.
		 */
		if (gcl->gcl_flags & GCPU_GCL_F_TES_P) {
			fm_payload_set(ereport,
			    FM_EREPORT_PAYLOAD_NAME_MC_STATUS_TES,
			    DATA_TYPE_STRING, tbes[MCAX86_TBES_VALUE(bstat)],
			    NULL);
		}
	}

	/*
	 * Add MCi_ADDR info if requested and valid. We force addition of
	 * MCi_ADDR, even if its not valid on AMD family 0xf and above,
	 * to aid in analysis of ereports, for WatchDog errors.
	 */
	if (members & FM_EREPORT_PAYLOAD_FLAG_MC_ADDR &&
	    ((bstat & MSR_MC_STATUS_ADDRV) ||
	    gcpu_force_addr_in_payload)) {
		fm_payload_set(ereport, FM_EREPORT_PAYLOAD_NAME_MC_ADDR,
		    DATA_TYPE_UINT64, gbl->gbl_addr, NULL);
	}

	/*
	 * MCi_MISC if requested and MCi_STATUS.MISCV).
	 */
	if (members & FM_EREPORT_PAYLOAD_FLAG_MC_MISC &&
	    bstat & MSR_MC_STATUS_MISCV) {
		fm_payload_set(ereport, FM_EREPORT_PAYLOAD_NAME_MC_MISC,
		    DATA_TYPE_UINT64, gbl->gbl_misc, NULL);
	}

}

/*
 * Construct and post an ereport based on the logout information from a
 * single MCA bank.  We are not necessarily running on the cpu that
 * detected the error.
 */
static void
gcpu_ereport_post(const gcpu_logout_t *gcl, int bankidx,
    const gcpu_error_disp_t *ged, cms_cookie_t mscookie, uint64_t status)
{
	gcpu_data_t *gcpu = gcl->gcl_gcpu;
	cmi_hdl_t hdl = gcpu->gcpu_hdl;
	const gcpu_bank_logout_t *gbl = &gcl->gcl_data[bankidx];
	const char *cpuclass = NULL, *leafclass = NULL;
	uint16_t code = MCAX86_ERRCODE(status);
	errorq_elem_t *eqep, *scr_eqep;
	nvlist_t *ereport, *detector;
	char buf[FM_MAX_CLASS];
	const char *classfmt;
	nv_alloc_t *nva;

	if (panicstr) {
		if ((eqep = errorq_reserve(ereport_errorq)) == NULL)
			return;
		ereport = errorq_elem_nvl(ereport_errorq, eqep);

		/*
		 * Allocate another element for scratch space, but fallback
		 * to the one we have if that fails.  We'd like to use the
		 * additional scratch space for nvlist construction.
		 */
		if ((scr_eqep = errorq_reserve(ereport_errorq)) != NULL)
			nva = errorq_elem_nva(ereport_errorq, scr_eqep);
		else
			nva = errorq_elem_nva(ereport_errorq, eqep);
	} else {
		ereport = fm_nvlist_create(NULL);
		nva = NULL;
		eqep = NULL;
		scr_eqep = NULL;
	}

	if (ereport == NULL)
		return;

	/*
	 * Common payload data required by the protocol:
	 *	- ereport class
	 *	- detector
	 *	- ENA
	 */

	/*
	 * Ereport class - call into model-specific support to allow it to
	 * provide a cpu class or leaf class, otherwise calculate our own.
	 */
	cms_ereport_class(hdl, mscookie, &cpuclass, &leafclass);
	classfmt = ged ?  ged->ged_class_fmt : FM_EREPORT_CPU_GENERIC_UNKNOWN;
	gcpu_erpt_clsfmt(classfmt, buf, sizeof (buf), status, cpuclass,
	    leafclass);

	/*
	 * The detector FMRI.
	 */
	if ((detector = cms_ereport_detector(hdl, bankidx, mscookie,
	    nva)) == NULL)
		detector = gcpu_fmri_create(hdl, nva);

	/*
	 * Should we define a new ENA format 3?? for chip/core/strand?
	 * It will be better when virtualized.
	 */
	fm_ereport_set(ereport, FM_EREPORT_VERSION, buf,
	    fm_ena_generate_cpu(gcl->gcl_timestamp,
	    cmi_hdl_chipid(hdl) << 6 | cmi_hdl_coreid(hdl) << 3 |
	    cmi_hdl_strandid(hdl), FM_ENA_FMT1), detector, NULL);

	if (panicstr) {
		fm_nvlist_destroy(detector, FM_NVA_RETAIN);
		nv_alloc_reset(nva);
	} else {
		fm_nvlist_destroy(detector, FM_NVA_FREE);
	}

	/*
	 * Add the architectural ereport class-specific payload data.
	 */
	gcpu_ereport_add_logout(ereport, gcl, bankidx, ged, code);

	/*
	 * Allow model-specific code to add ereport members.
	 */
	cms_ereport_add_logout(hdl, ereport, nva, bankidx, gbl->gbl_status,
	    gbl->gbl_addr, gbl->gbl_misc, gcl->gcl_ms_logout, mscookie);

	/*
	 * Include stack if options is turned on and either selected in
	 * the payload member bitmask or inclusion is forced.
	 */
	if (gcpu_mca_stack_flag &&
	    (cms_ereport_includestack(hdl, mscookie) ==
	    B_TRUE || gcpu_mca_stack_ereport_include)) {
		fm_payload_stack_add(ereport, gcl->gcl_stack,
		    gcl->gcl_stackdepth);
	}

	/*
	 * If injection has taken place anytime in the past then note this
	 * on the ereport.
	 */
	if (cmi_inj_tainted() == B_TRUE) {
		fm_payload_set(ereport, "__injected", DATA_TYPE_BOOLEAN_VALUE,
		    B_TRUE, NULL);
	}

	/*
	 * Post ereport.
	 */
	if (panicstr) {
		errorq_commit(ereport_errorq, eqep, ERRORQ_SYNC);
		if (scr_eqep)
			errorq_cancel(ereport_errorq, scr_eqep);
	} else {
		(void) fm_ereport_post(ereport, EVCH_TRYHARD);
		fm_nvlist_destroy(ereport, FM_NVA_FREE);
	}

}

/*ARGSUSED*/
void
gcpu_mca_drain(void *ignored, const void *data, const errorq_elem_t *eqe)
{
	const gcpu_logout_t *gcl = data;
	const gcpu_bank_logout_t *gbl;
	int ismc;
	int i;

	ismc = gcl->ismc;
	for (i = 0, gbl = &gcl->gcl_data[0]; i < gcl->gcl_nbanks; i++, gbl++) {
		const gcpu_error_disp_t *gened;
		cms_cookie_t mscookie;

		if (gbl->gbl_status & MSR_MC_STATUS_VAL &&
		    !(gbl->gbl_disp & CMI_ERRDISP_INCONSISTENT)) {
			uint16_t code = MCAX86_ERRCODE(gbl->gbl_status);

			/*
			 * Perform a match based on IA32 MCA architectural
			 * components alone.
			 */
			gened = gcpu_disp_match(code); /* may be NULL */

			/*
			 * Now see if an model-specific match can be made.
			 */
			mscookie = cms_disp_match(gcl->gcl_gcpu->gcpu_hdl, ismc,
			    i, gbl->gbl_status, gbl->gbl_addr, gbl->gbl_misc,
			    gcl->gcl_ms_logout);

			/*
			 * Prepare and dispatch an ereport for logging and
			 * diagnosis.
			 */
			gcpu_ereport_post(gcl, i, gened, mscookie,
			    gbl->gbl_status);
		} else if (gbl->gbl_status & MSR_MC_STATUS_VAL &&
		    (gbl->gbl_disp & CMI_ERRDISP_INCONSISTENT)) {
			/*
			 * Telemetry kept changing as we tried to read
			 * it.  Force an unknown ereport leafclass but
			 * keep the telemetry unchanged for logging.
			 */
			gcpu_ereport_post(gcl, i, &gcpu_unknown, NULL,
			    gbl->gbl_status);
		}
	}
}

static size_t gcpu_mca_queue_datasz = 0;

/*
 * The following code is ready to make a weak attempt at growing the
 * errorq structure size.  Since it is not foolproof (we don't know
 * who may already be producing to the outgoing errorq) our caller
 * instead assures that we'll always be called with no greater data
 * size than on our first call.
 */
static void
gcpu_errorq_init(size_t datasz)
{
	int slots;

	mutex_enter(&gcpu_mca_queue_lock);

	if (gcpu_mca_queue_datasz >= datasz) {
		mutex_exit(&gcpu_mca_queue_lock);
		return;
	}

	membar_producer();
	if (gcpu_mca_queue) {
		gcpu_mca_queue_datasz = 0;
		errorq_destroy(gcpu_mca_queue);
	}

	slots = MAX(GCPU_MCA_ERRS_PERCPU * max_ncpus, GCPU_MCA_MIN_ERRORS);
	slots = MIN(slots, GCPU_MCA_MAX_ERRORS);

	gcpu_mca_queue = errorq_create("gcpu_mca_queue", gcpu_mca_drain,
	    NULL, slots, datasz, 1, ERRORQ_VITAL);

	if (gcpu_mca_queue != NULL)
		gcpu_mca_queue_datasz = datasz;

	mutex_exit(&gcpu_mca_queue_lock);
}

/*
 * Perform MCA initialization as described in section 14.6 of Intel 64
 * and IA-32 Architectures Software Developer's Manual Volume 3A.
 */

static uint_t global_nbanks;

#ifndef __xpv
/*ARGSUSED*/
int
gcpu_cmci_cpu_setup(cpu_setup_t what, int cpuid, void *arg)
{
	/*
	 * In general, we'd expect that in a multi-socket configuration, either
	 * all CPUs would support CMCI or none of them would.  Unfortunately,
	 * that may not be the case in the wild.  While we'd rather check the
	 * handle's enablement state here, that itself is a bit complicated. We
	 * don't have a guarantee in a heterogenous situation that the CPU in
	 * question is using the generic CPU module or not, even though we've
	 * been registered. As such, we allow the interrupt to be registered and
	 * written to the local apic anyways. We won't have a CMCI interrupt
	 * generated anyways because the MCA banks will not be programmed as
	 * such for that CPU by the polling thread.
	 */
	switch (what) {
	case CPU_ON:
		psm_cmci_setup(cpuid, B_TRUE);
		break;
	case CPU_OFF:
		psm_cmci_setup(cpuid, B_FALSE);
		break;
	default:
		break;
	}

	return (0);
}

void
gcpu_mca_cmci_enable(cmi_hdl_t hdl)
{
	gcpu_data_t *gcpu = cmi_hdl_getcmidata(hdl);
	gcpu_mca_t *mca = &gcpu->gcpu_mca;

	/*
	 * If this CPU doesn't support CMCI, don't do anything.
	 */
	if ((mca->gcpu_mca_flags & GCPU_MCA_F_CMCI_CAPABLE) == 0)
		return;

	/*
	 * If we don't have support from the PSM module, then there's nothing we
	 * can do. Note that this changes as we start up the system. The only
	 * case where it may be mistakenly NULL is for the boot CPU. The boot
	 * CPU will have this taken care of for it in gcpu_post_startup(), once
	 * we know for certain whether or not the PSM module supports CMCI.
	 */
	if (psm_cmci_setup == NULL) {
		return;
	}

	mca->gcpu_mca_flags |= GCPU_MCA_F_CMCI_ENABLE;
	if (MUTEX_HELD(&cpu_lock)) {
		if (!gcpu_mca_cpu_registered) {
			register_cpu_setup_func(gcpu_cmci_cpu_setup, NULL);
			gcpu_mca_cpu_registered = B_TRUE;
		}
	} else {
		mutex_enter(&cpu_lock);
		if (!gcpu_mca_cpu_registered) {
			register_cpu_setup_func(gcpu_cmci_cpu_setup, NULL);
			gcpu_mca_cpu_registered = B_TRUE;
		}
		mutex_exit(&cpu_lock);
	}

	/*
	 * Call the PSM op to make sure that we initialize things on
	 * this CPU.
	 */
	psm_cmci_setup(cmi_hdl_logical_id(hdl), B_TRUE);
}
#endif	/* !__xpv */

void
gcpu_mca_init(cmi_hdl_t hdl)
{
	gcpu_data_t *gcpu = cmi_hdl_getcmidata(hdl);
	uint64_t cap;
	uint_t vendor = cmi_hdl_vendor(hdl);
	uint_t family = cmi_hdl_family(hdl);
	gcpu_mca_t *mca = &gcpu->gcpu_mca;
	int mcg_ctl_present;
	uint_t nbanks;
	uint32_t ctl_skip_mask = 0;
	uint32_t status_skip_mask = 0;
	size_t mslsz;
	int i;
#ifndef __xpv
	int mcg_ctl2_present;
	uint32_t cmci_capable = 0;
#endif
	if (gcpu == NULL)
		return;

	/* We add MCi_ADDR always for AMD Family 0xf and above */
	if (family >= 0xf)
		gcpu_force_addr_in_payload = 1;

	/*
	 * Protect from some silly /etc/system settings.
	 */
	if (gcpu_mca_telemetry_retries < 0 || gcpu_mca_telemetry_retries > 100)
		gcpu_mca_telemetry_retries = 5;

	if (cmi_hdl_rdmsr(hdl, IA32_MSR_MCG_CAP, &cap) != CMI_SUCCESS)
		return;

	/*
	 * CPU startup code only calls cmi_mca_init if x86_featureset indicates
	 * both MCA and MCE support (i.e., X86FSET_MCA).  P5, K6, and earlier
	 * processors, which have their own more primitive way of doing
	 * machine checks, will not have cmi_mca_init called since their
	 * CPUID information will not indicate both MCA and MCE features.
	 */
	ASSERT(is_x86_feature(x86_featureset, X86FSET_MCA));

	/*
	 * Determine whether the IA32_MCG_CTL register is present.  If it
	 * is we will enable all features by writing -1 to it towards
	 * the end of this initialization;  if it is absent then volume 3A
	 * says we must nonetheless continue to initialize the individual
	 * banks.
	 */
	mcg_ctl_present = cap & MCG_CAP_CTL_P;
#ifndef __xpv
	mcg_ctl2_present = cap & MCG_CAP_CTL2_P;
#endif

	/*
	 * We squirell values away for inspection/debugging.
	 */
	mca->gcpu_mca_bioscfg.bios_mcg_cap = cap;
	if (mcg_ctl_present)
		(void) cmi_hdl_rdmsr(hdl, IA32_MSR_MCG_CTL,
		    &mca->gcpu_mca_bioscfg.bios_mcg_ctl);

	/*
	 * Determine the number of error-reporting banks implemented.
	 */
	mca->gcpu_mca_nbanks = nbanks = cap & MCG_CAP_COUNT_MASK;

	if (nbanks != 0 && global_nbanks == 0)
		global_nbanks = nbanks;	/* no race - BSP will get here first */

	/*
	 * If someone is hiding the number of banks (perhaps we are fully
	 * virtualized?) or if this processor has more banks than the
	 * first to set global_nbanks then bail.  The latter requirement
	 * is because we need to size our errorq data structure and we
	 * don't want to have to grow the errorq (destroy and recreate)
	 * which may just lose some telemetry.
	 */
	if (nbanks == 0 || nbanks > global_nbanks)
		return;

	mca->gcpu_mca_bioscfg.bios_bankcfg = kmem_zalloc(nbanks *
	    sizeof (struct gcpu_bios_bankcfg), KM_SLEEP);

	/*
	 * Calculate the size we need to allocate for a gcpu_logout_t
	 * with a gcl_data array big enough for all banks of this cpu.
	 * Add any space requested by the model-specific logout support.
	 */
	mslsz = cms_logout_size(hdl);
	mca->gcpu_mca_lgsz = sizeof (gcpu_logout_t) +
	    (nbanks - 1) * sizeof (gcpu_bank_logout_t) + mslsz;

	for (i = 0; i < GCPU_MCA_LOGOUT_NUM; i++) {
		gcpu_logout_t *gcl;

		mca->gcpu_mca_logout[i] = gcl =
		    kmem_zalloc(mca->gcpu_mca_lgsz, KM_SLEEP);
		gcl->gcl_gcpu = gcpu;
		gcl->gcl_nbanks = nbanks;
		gcl->gcl_ms_logout = (mslsz == 0) ? NULL :
		    (char *)(&gcl->gcl_data[0]) + nbanks *
		    sizeof (gcpu_bank_logout_t);

	}

#ifdef __xpv
	gcpu_xpv_mca_init(nbanks);
#endif

	mca->gcpu_mca_nextpoll_idx = GCPU_MCA_LOGOUT_POLLER_1;

#ifndef __xpv
	mca->gcpu_bank_cmci = kmem_zalloc(sizeof (gcpu_mca_cmci_t) * nbanks,
	    KM_SLEEP);
#endif

	/*
	 * Create our errorq to transport the logout structures.  This
	 * can fail so users of gcpu_mca_queue must be prepared for NULL.
	 */
	gcpu_errorq_init(mca->gcpu_mca_lgsz);

	/*
	 * Not knowing which, if any, banks are shared between cores we
	 * assure serialization of MCA bank initialization by each cpu
	 * on the chip.  On chip architectures in which some banks are
	 * shared this will mean the shared resource is initialized more
	 * than once - we're simply aiming to avoid simultaneous MSR writes
	 * to the shared resource.
	 *
	 * Even with these precautions, some platforms may yield a GP fault
	 * if a core other than a designated master tries to write anything
	 * but all 0's to MCi_{STATUS,ADDR,CTL}.  So we will perform
	 * those writes under on_trap protection.
	 */
	mutex_enter(&gcpu->gcpu_shared->gcpus_cfglock);

	/*
	 * Initialize poller data, but don't start polling yet.
	 */
	gcpu_mca_poll_init(hdl);

	/*
	 * Work out which MCA banks we will initialize.  In MCA logout
	 * code we will only read those banks which we initialize here.
	 */
	for (i = 0; i < nbanks; i++) {
		boolean_t skipctl = cms_bankctl_skipinit(hdl, i);
		boolean_t skipstatus = cms_bankstatus_skipinit(hdl, i);

		if (!cms_present(hdl)) {
			/*
			 * Model-specific support is not present, try to use
			 * sane defaults.
			 *
			 * On AMD family 6 processors, reports about spurious
			 * machine checks indicate that bank 0 should be
			 * skipped.
			 *
			 * On Intel family 6 processors, the documentation tells
			 * us not to write to MC0_CTL.
			 *
			 */
			if (i == 0 && family == 6) {
				switch (vendor) {
				case X86_VENDOR_AMD:
					skipstatus = B_TRUE;
					/*FALLTHRU*/
				case X86_VENDOR_Intel:
					skipctl = B_TRUE;
					break;
				}
			}
		}

		ctl_skip_mask |= skipctl << i;
		status_skip_mask |= skipstatus << i;

		if (skipctl && skipstatus)
			continue;

		/*
		 * Record which MCA banks were enabled, from the point of view
		 * of the whole chip (if some cores share a bank we must be
		 * sure either can logout from it).
		 */
		atomic_or_32(&gcpu->gcpu_shared->gcpus_actv_banks, 1 << i);

#ifndef __xpv
		/*
		 * check CMCI capability
		 */
		if (mcg_ctl2_present) {
			uint64_t ctl2;
			uint32_t cap = 0;
			(void) cmi_hdl_rdmsr(hdl, IA32_MSR_MC_CTL2(i), &ctl2);
			if (ctl2 & MSR_MC_CTL2_EN)
				continue;
			ctl2 |= MSR_MC_CTL2_EN;
			(void) cmi_hdl_wrmsr(hdl, IA32_MSR_MC_CTL2(i), ctl2);
			(void) cmi_hdl_rdmsr(hdl, IA32_MSR_MC_CTL2(i), &ctl2);
			mca->gcpu_bank_cmci[i].cmci_cap = cap =
			    (ctl2 & MSR_MC_CTL2_EN) ? 1 : 0;
			if (cap)
				cmci_capable ++;
			/*
			 * Set threshold to 1 while unset the en field, to avoid
			 * CMCI trigged before APIC LVT entry init.
			 */
			ctl2 = (ctl2 & (~MSR_MC_CTL2_EN)) | 1;
			(void) cmi_hdl_wrmsr(hdl, IA32_MSR_MC_CTL2(i), ctl2);

			/*
			 * init cmci related count
			 */
			mca->gcpu_bank_cmci[i].cmci_enabled = 0;
			mca->gcpu_bank_cmci[i].drtcmci = 0;
			mca->gcpu_bank_cmci[i].ncmci = 0;
		}
#endif
	}

#ifndef __xpv
	if (cmci_capable) {
		mca->gcpu_mca_flags |= GCPU_MCA_F_CMCI_CAPABLE;
		gcpu_mca_cmci_enable(hdl);
	}
#endif

#ifndef __xpv
	/*
	 * Log any valid telemetry lurking in the MCA banks, but do not
	 * clear the status registers.  Ignore the disposition returned -
	 * we have already paniced or reset for any nasty errors found here.
	 *
	 * Intel vol 3A says that we should not do this on family 0x6,
	 * and that for any extended family the BIOS clears things
	 * on power-on reset so you'll only potentially find valid telemetry
	 * on warm reset (we do it for both - on power-on reset we should
	 * just see zeroes).
	 *
	 * AMD docs since K7 say we should process anything we find here.
	 */
	if (!gcpu_suppress_log_on_init &&
	    ((vendor == X86_VENDOR_Intel && family >= 0xf) ||
	    vendor == X86_VENDOR_AMD ||
	    vendor == X86_VENDOR_HYGON))
		gcpu_mca_logout(hdl, NULL, -1ULL, NULL, B_FALSE,
		    GCPU_MPT_WHAT_POKE_ERR);

	/*
	 * Initialize all MCi_CTL and clear all MCi_STATUS, allowing the
	 * model-specific module the power of veto.
	 */
	for (i = 0; i < nbanks; i++) {
		struct gcpu_bios_bankcfg *bcfgp =
		    mca->gcpu_mca_bioscfg.bios_bankcfg + i;

		/*
		 * Stash inherited bank MCA state, even for banks we will
		 * not initialize ourselves.  Do not read the MISC register
		 * unconditionally - on some processors that will #GP on
		 * banks that do not implement the MISC register (would be
		 * caught by on_trap, anyway).
		 */
		(void) cmi_hdl_rdmsr(hdl, IA32_MSR_MC(i, CTL),
		    &bcfgp->bios_bank_ctl);

		(void) cmi_hdl_rdmsr(hdl, IA32_MSR_MC(i, STATUS),
		    &bcfgp->bios_bank_status);

		if ((bcfgp->bios_bank_status & MSR_MC_STATUS_ADDRV) ||
		    gcpu_force_addr_in_payload) {
			(void) cmi_hdl_rdmsr(hdl, IA32_MSR_MC(i, ADDR),
			    &bcfgp->bios_bank_addr);
		}

		/*
		 * In some old BIOS the status value after boot can indicate
		 * MISCV when there is actually no MISC register for
		 * that bank.  The following read could therefore
		 * aggravate a general protection fault.  This should be
		 * caught by on_trap, but the #GP fault handler is busted
		 * and can suffer a double fault even before we get to
		 * trap() to check for on_trap protection.  Until that
		 * issue is fixed we remove the one access that we know
		 * can cause a #GP.
		 *
		 * if (bcfgp->bios_bank_status & MSR_MC_STATUS_MISCV)
		 *	(void) cmi_hdl_rdmsr(hdl, IA32_MSR_MC(i, MISC),
		 *	    &bcfgp->bios_bank_misc);
		 */
		bcfgp->bios_bank_misc = 0;

		if (!(ctl_skip_mask & (1 << i))) {
			(void) cmi_hdl_wrmsr(hdl, IA32_MSR_MC(i, CTL),
			    cms_bankctl_val(hdl, i, -1ULL));
		}

		if (!(status_skip_mask & (1 << i))) {
			(void) cmi_hdl_wrmsr(hdl, IA32_MSR_MC(i, STATUS),
			    cms_bankstatus_val(hdl, i, 0ULL));
		}
	}
#endif
	/*
	 * Now let the model-specific support perform further initialization
	 * of non-architectural features.
	 */
	cms_mca_init(hdl, nbanks);

#ifndef __xpv
	(void) cmi_hdl_wrmsr(hdl, IA32_MSR_MCG_STATUS, 0ULL);
	membar_producer();

	/* enable all machine-check features */
	if (mcg_ctl_present)
		(void) cmi_hdl_wrmsr(hdl, IA32_MSR_MCG_CTL,
		    cms_mcgctl_val(hdl, nbanks, -1ULL));
#endif

	mutex_exit(&gcpu->gcpu_shared->gcpus_cfglock);

#ifndef __xpv
	/* enable machine-check exception in CR4 */
	cmi_hdl_enable_mce(hdl);
#endif
}

static uint64_t
gcpu_mca_process(cmi_hdl_t hdl, struct regs *rp, int nerr, gcpu_data_t *gcpu,
    gcpu_logout_t *gcl, int ismc, gcpu_mce_status_t *mcesp)
{
	int curctxbad = 0, unconstrained = 0, forcefatal = 0;
	gcpu_mca_t *mca = &gcpu->gcpu_mca;
	int nbanks = mca->gcpu_mca_nbanks;
	gcpu_mce_status_t mce;
	gcpu_bank_logout_t *gbl;
	uint64_t disp = 0;
	int i;

	if (mcesp == NULL)
		mcesp = &mce;

	mcesp->mce_nerr = nerr;

	mcesp->mce_npcc = mcesp->mce_npcc_ok = mcesp->mce_nuc =
	    mcesp->mce_nuc_ok = mcesp->mce_nuc_poisoned =
	    mcesp->mce_forcefatal = mcesp->mce_ignored = 0;

	/*
	 * If this a machine check then if the return instruction pointer
	 * is not valid the current context is lost.
	 */
	if (ismc && !(gcl->gcl_mcg_status & MCG_STATUS_RIPV))
		disp |= CMI_ERRDISP_RIPV_INVALID;
	gcl->ismc = ismc;

	for (i = 0, gbl = &gcl->gcl_data[0]; i < nbanks; i++, gbl++) {
		uint64_t mcistatus = gbl->gbl_status;
		uint32_t ms_scope;
		int pcc, uc;
		int poisoned;

		if (!(mcistatus & MSR_MC_STATUS_VAL))
			continue;

		if (gbl->gbl_disp & CMI_ERRDISP_INCONSISTENT)
			continue;

		pcc = (mcistatus & MSR_MC_STATUS_PCC) != 0;
		uc = (mcistatus & MSR_MC_STATUS_UC) != 0;
		mcesp->mce_npcc += pcc;
		mcesp->mce_nuc += uc;

		ms_scope = cms_error_action(hdl, ismc, i, mcistatus,
		    gbl->gbl_addr, gbl->gbl_misc, gcl->gcl_ms_logout);

		if (pcc && ms_scope & CMS_ERRSCOPE_CURCONTEXT_OK) {
			pcc = 0;
			mcesp->mce_npcc_ok++;
			gbl->gbl_disp |= CMI_ERRDISP_PCC_CLEARED;
		}

		if (uc && ms_scope & CMS_ERRSCOPE_CLEARED_UC) {
			uc = 0;
			mcesp->mce_nuc_ok++;
			gbl->gbl_disp |= CMI_ERRDISP_UC_CLEARED;
		}

		if (uc) {
			poisoned = (ms_scope & CMS_ERRSCOPE_POISONED) != 0;
			if (poisoned) {
				mcesp->mce_nuc_poisoned++;
				gbl->gbl_disp |= CMI_ERRDISP_POISONED;
			}
		}

		if ((ms_scope & CMS_ERRSCOPE_IGNORE_ERR) == 0) {
			/*
			 * We're not being instructed to ignore the error,
			 * so apply our standard disposition logic to it.
			 */
			if (uc && !poisoned) {
				unconstrained++;
				gbl->gbl_disp |= disp |
				    CMI_ERRDISP_UC_UNCONSTRAINED;
			}

			if (pcc && ismc) {
				curctxbad++;
				gbl->gbl_disp |= disp |
				    CMI_ERRDISP_CURCTXBAD;
			}

			/*
			 * Even if the above may not indicate that the error
			 * is terminal, model-specific support may insist
			 * that we treat it as such.  Such errors wil be
			 * fatal even if discovered via poll.
			 */
			if (ms_scope & CMS_ERRSCOPE_FORCE_FATAL) {
				forcefatal++;
				mcesp->mce_forcefatal++;
				gbl->gbl_disp |= disp |
				    CMI_ERRDISP_FORCEFATAL;
			}
		} else {
			mcesp->mce_ignored++;
			gbl->gbl_disp |= disp | CMI_ERRDISP_IGNORED;
		}
	}

	if (unconstrained > 0)
		disp |= CMI_ERRDISP_UC_UNCONSTRAINED;

	if (curctxbad > 0)
		disp |= CMI_ERRDISP_CURCTXBAD;

	if (forcefatal > 0)
		disp |= CMI_ERRDISP_FORCEFATAL;

	if (gcpu_mca_queue != NULL) {
		int how;

		if (ismc) {
			how = cmi_mce_response(rp, disp) ?
			    ERRORQ_ASYNC :	/* no panic, so arrange drain */
			    ERRORQ_SYNC;	/* panic flow will drain */
		} else {
			how = (disp & CMI_ERRDISP_FORCEFATAL &&
			    cmi_panic_on_ue()) ?
			    ERRORQ_SYNC :	/* poller will panic */
			    ERRORQ_ASYNC;	/* no panic */
		}

		errorq_dispatch(gcpu_mca_queue, gcl, mca->gcpu_mca_lgsz, how);
	} else if (disp != 0) {
		gcpu_bleat(hdl, gcl);
	}

	mcesp->mce_disp = disp;

	return (disp);
}

/*
 * Gather error telemetry from our source, and then submit it for
 * processing.
 */

#define	IS_MCE_CANDIDATE(status) (((status) & MSR_MC_STATUS_EN) != 0 && \
	((status) & (MSR_MC_STATUS_UC | MSR_MC_STATUS_PCC)) != 0)

#define	STATUS_EQV(s1, s2) \
	(((s1) & ~MSR_MC_STATUS_OVER) == ((s2) & ~MSR_MC_STATUS_OVER))

static uint32_t gcpu_deferrred_polled_clears;

#ifndef __xpv
static void
gcpu_cmci_logout(cmi_hdl_t hdl, int bank, gcpu_mca_cmci_t *bank_cmci_p,
    uint64_t status, int what)
{
	uint64_t ctl2;

	if (bank_cmci_p->cmci_cap && (what == GCPU_MPT_WHAT_CYC_ERR) &&
	    (!(status & MSR_MC_STATUS_VAL) || ((status & MSR_MC_STATUS_VAL) &&
	    !(status & MSR_MC_STATUS_CEC_MASK)))) {

		if (!(bank_cmci_p->cmci_enabled)) {
			/*
			 * when cmci is disabled, and the bank has no error or
			 * no corrected error for
			 * gcpu_mca_cmci_reenable_threshold consecutive polls,
			 * turn on this bank's cmci.
			 */

			bank_cmci_p->drtcmci ++;

			if (bank_cmci_p->drtcmci >=
			    gcpu_mca_cmci_reenable_threshold) {

				/* turn on cmci */

				(void) cmi_hdl_rdmsr(hdl,
				    IA32_MSR_MC_CTL2(bank), &ctl2);
				ctl2 |= MSR_MC_CTL2_EN;
				(void) cmi_hdl_wrmsr(hdl,
				    IA32_MSR_MC_CTL2(bank), ctl2);

				/* reset counter and set flag */
				bank_cmci_p->drtcmci = 0;
				bank_cmci_p->cmci_enabled = 1;
			}
		} else {
			/*
			 * when cmci is enabled,if is in cyclic poll and the
			 * bank has no error or no corrected error, reset ncmci
			 * counter
			 */
			bank_cmci_p->ncmci = 0;
		}
	}
}

static void
gcpu_cmci_throttle(cmi_hdl_t hdl, int bank, gcpu_mca_cmci_t *bank_cmci_p,
    int what)
{
	uint64_t ctl2 = 0;

	/*
	 * if cmci of this bank occurred beyond
	 * gcpu_mca_cmci_throttling_threshold between 2 polls,
	 * turn off this bank's CMCI;
	 */
	if (bank_cmci_p->cmci_enabled && what == GCPU_MPT_WHAT_CMCI_ERR) {

		/* if it is cmci trap, increase the count */
		bank_cmci_p->ncmci++;

		if (bank_cmci_p->ncmci >= gcpu_mca_cmci_throttling_threshold) {

			/* turn off cmci */

			(void) cmi_hdl_rdmsr(hdl, IA32_MSR_MC_CTL2(bank),
			    &ctl2);
			ctl2 &= ~MSR_MC_CTL2_EN;
			(void) cmi_hdl_wrmsr(hdl, IA32_MSR_MC_CTL2(bank),
			    ctl2);

			/* clear the flag and count */

			bank_cmci_p->cmci_enabled = 0;
			bank_cmci_p->ncmci = 0;
		}
	}
}
#endif

static void
clear_mc(int first, int last, int ismc, boolean_t clrstatus,
    cmi_hdl_t hdl, gcpu_logout_t *gcl, gcpu_logout_t *pgcl)
{
	int i;
	gcpu_bank_logout_t *gbl, *pgbl;
	uint64_t status;

	if (first < 0 || last < 0)
		return;

	for (i = first, gbl = &gcl->gcl_data[first]; i <= last; i++, gbl++) {
		status = gbl->gbl_status;
		if (status == 0)
			continue;
		if (clrstatus == B_FALSE)
			goto serialize;

		/*
		 * For i86xpv we always clear status in order to invalidate
		 * the interposed telemetry.
		 *
		 * For native machine checks we always clear status here.  For
		 * native polls we must be a little more cautious since there
		 * is an outside chance that we may clear telemetry from a
		 * shared MCA bank on which a sibling core is machine checking.
		 *
		 * For polled observations of errors that look like they may
		 * produce a machine check (UC/PCC and ENabled, although these
		 * do not guarantee a machine check on error occurence)
		 * we will not clear the status at this wakeup unless
		 * we saw the same status at the previous poll.	 We will
		 * always process and log the current observations - it
		 * is only the clearing of MCi_STATUS which may be
		 * deferred until the next wakeup.
		 */
		if (isxpv || ismc || !IS_MCE_CANDIDATE(status)) {
			(void) cmi_hdl_wrmsr(hdl, IA32_MSR_MC(i, STATUS), 0ULL);
			goto serialize;
		}

		/*
		 * We have a polled observation of a machine check
		 * candidate.  If we saw essentially the same status at the
		 * last poll then clear the status now since this appears
		 * not to be a #MC candidate after all.	 If we see quite
		 * different status now then do not clear, but reconsider at
		 * the next poll.  In no actual machine check clears
		 * the status in the interim then the status should not
		 * keep changing forever (meaning we'd never clear it)
		 * since before long we'll simply have latched the highest-
		 * priority error and set the OVerflow bit.  Nonetheless
		 * we count how many times we defer clearing and after
		 * a while insist on clearing the status.
		 */
		pgbl = &pgcl->gcl_data[i];
		if (pgbl->gbl_clrdefcnt != 0) {
			/* We deferred clear on this bank at last wakeup */
			if (STATUS_EQV(status, pgcl->gcl_data[i].gbl_status) ||
			    pgbl->gbl_clrdefcnt > 5) {
				/*
				 * Status is unchanged so clear it now and,
				 * since we have already logged this info,
				 * avoid logging it again.
				 */
				gbl->gbl_status = 0;
				(void) cmi_hdl_wrmsr(hdl,
				    IA32_MSR_MC(i, STATUS), 0ULL);
			} else {
				/* Record deferral for next wakeup */
				gbl->gbl_clrdefcnt = pgbl->gbl_clrdefcnt + 1;
			}
		} else {
			/* Record initial deferral for next wakeup */
			gbl->gbl_clrdefcnt = 1;
			gcpu_deferrred_polled_clears++;
		}

serialize:
		{
#ifdef __xpv
			;
#else
			/*
			 * Intel Vol 3A says to execute a serializing
			 * instruction here, ie CPUID.	Well WRMSR is also
			 * defined to be serializing, so the status clear above
			 * should suffice.  To be a good citizen, and since
			 * some clears are deferred, we'll execute a CPUID
			 * instruction here.
			 */
			struct cpuid_regs tmp;
			(void) __cpuid_insn(&tmp);
#endif
		}
	}
}

/*ARGSUSED5*/
void
gcpu_mca_logout(cmi_hdl_t hdl, struct regs *rp, uint64_t bankmask,
    gcpu_mce_status_t *mcesp, boolean_t clrstatus, int what)
{
	gcpu_data_t *gcpu = cmi_hdl_getcmidata(hdl);
	gcpu_mca_t *mca = &gcpu->gcpu_mca;
	int nbanks = mca->gcpu_mca_nbanks;
	gcpu_bank_logout_t *gbl, *pgbl;
	gcpu_logout_t *gcl, *pgcl;
	int ismc = (rp != NULL);
	int ispoll = !ismc;
	int i, nerr = 0;
	cmi_errno_t err;
	uint64_t mcg_status;
	uint64_t disp;
	uint64_t cap;
	int first = -1;
	int last = -1;
	int willpanic = 0;

	if (cmi_hdl_rdmsr(hdl, IA32_MSR_MCG_STATUS, &mcg_status) !=
	    CMI_SUCCESS || cmi_hdl_rdmsr(hdl, IA32_MSR_MCG_CAP, &cap) !=
	    CMI_SUCCESS) {
		if (mcesp != NULL)
			mcesp->mce_nerr = mcesp->mce_disp = 0;
		return;
	}

	if (ismc) {
		gcl = mca->gcpu_mca_logout[GCPU_MCA_LOGOUT_EXCEPTION];
		pgcl = NULL;
	} else {
		int pidx = mca->gcpu_mca_nextpoll_idx;
		int ppidx = (pidx == GCPU_MCA_LOGOUT_POLLER_1) ?
		    GCPU_MCA_LOGOUT_POLLER_2 : GCPU_MCA_LOGOUT_POLLER_1;

		gcl = mca->gcpu_mca_logout[pidx];	/* current logout */
		pgcl = mca->gcpu_mca_logout[ppidx];	/* previous logout */
		mca->gcpu_mca_nextpoll_idx = ppidx;	/* switch next time */
	}

	gcl->gcl_timestamp = gethrtime_waitfree();
	gcl->gcl_mcg_status = mcg_status;
	gcl->gcl_ip = rp ? rp->r_pc : 0;

	gcl->gcl_flags = (rp && USERMODE(rp->r_cs)) ? GCPU_GCL_F_PRIV : 0;
	if (cap & MCG_CAP_TES_P)
		gcl->gcl_flags |= GCPU_GCL_F_TES_P;

	for (i = 0, gbl = &gcl->gcl_data[0]; i < nbanks; i++, gbl++) {
		uint64_t status, status2, addr, misc;
		int retries = gcpu_mca_telemetry_retries;

		gbl->gbl_status = 0;
		gbl->gbl_disp = 0;
		gbl->gbl_clrdefcnt = 0;

		/*
		 * Only logout from MCA banks we have initialized from at
		 * least one core.  If a core shares an MCA bank with another
		 * but perhaps lost the race to initialize it, then it must
		 * still be allowed to logout from the shared bank.
		 */
		if (!(gcpu->gcpu_shared->gcpus_actv_banks & 1 << i))
			continue;

		/*
		 * On a poll look only at the banks we've been asked to check.
		 */
		if (rp == NULL && !(bankmask & 1 << i))
			continue;


		if (cmi_hdl_rdmsr(hdl, IA32_MSR_MC(i, STATUS), &status) !=
		    CMI_SUCCESS)
			continue;

#ifndef __xpv
		gcpu_cmci_logout(hdl, i, &mca->gcpu_bank_cmci[i], status, what);
#endif

retry:
		if (!(status & MSR_MC_STATUS_VAL))
			continue;

		/* First and last bank that have valid status */
		if (first < 0)
			first = i;
		last = i;

		addr = -1;
		misc = 0;

		if ((status & MSR_MC_STATUS_ADDRV) ||
		    gcpu_force_addr_in_payload)
			(void) cmi_hdl_rdmsr(hdl, IA32_MSR_MC(i, ADDR), &addr);

		if (status & MSR_MC_STATUS_MISCV)
			(void) cmi_hdl_rdmsr(hdl, IA32_MSR_MC(i, MISC), &misc);

#ifndef __xpv
		gcpu_cmci_throttle(hdl, i, &mca->gcpu_bank_cmci[i], what);
#endif

		/*
		 * Allow the model-specific code to extract bank telemetry.
		 */
		cms_bank_logout(hdl, i, status, addr, misc, gcl->gcl_ms_logout);

		/*
		 * Not all cpu models assure us that the status/address/misc
		 * data will not change during the above sequence of MSR reads,
		 * or that it can only change by the addition of the OVerflow
		 * bit to the status register.  If the status has changed
		 * other than in the overflow bit then we attempt to reread
		 * for a consistent snapshot, but eventually give up and
		 * go with what we've got.  We only perform this check
		 * for a poll - a further #MC during a #MC will reset, and
		 * polled errors should not overwrite higher-priority
		 * trapping errors (but could set the overflow bit).
		 */
		if (ispoll && (err = cmi_hdl_rdmsr(hdl, IA32_MSR_MC(i, STATUS),
		    &status2)) == CMI_SUCCESS) {
			if (!STATUS_EQV(status, status2)) {
				if (retries-- > 0) {
					status = status2;
					goto retry;
				} else {
					gbl->gbl_disp |=
					    CMI_ERRDISP_INCONSISTENT;
				}
			}
		} else if (ispoll && err != CMI_SUCCESS) {
			gbl->gbl_disp |= CMI_ERRDISP_INCONSISTENT;
		}

		nerr++;
		gbl->gbl_status = status;
		gbl->gbl_addr = addr;
		gbl->gbl_misc = misc;

		/*
		 * For polled observation, if the count of deferred status
		 * clears updated in the clear_mc() is nonzero and the
		 * MCi_STATUS has not changed, the last wakeup has produced
		 * the ereport of the error. Therefore, clear the status in
		 * this wakeup to avoid duplicate ereport.
		 */
		pgbl = &pgcl->gcl_data[i];
		if (!isxpv && ispoll && IS_MCE_CANDIDATE(status) &&
		    pgbl->gbl_clrdefcnt != 0) {
			if (STATUS_EQV(status, pgcl->gcl_data[i].gbl_status)) {
				gbl->gbl_status = 0;
				(void) cmi_hdl_wrmsr(hdl,
				    IA32_MSR_MC(i, STATUS), 0ULL);
			}
		}
	}

	if (gcpu_mca_stack_flag)
		gcl->gcl_stackdepth = getpcstack(gcl->gcl_stack, FM_STK_DEPTH);
	else
		gcl->gcl_stackdepth = 0;

	/*
	 * Decide our disposition for this error or errors, and submit for
	 * logging and subsequent diagnosis.
	 */
	if (nerr != 0) {
		disp = gcpu_mca_process(hdl, rp, nerr, gcpu, gcl, ismc, mcesp);

		willpanic = (ismc && cmi_mce_response(rp, disp) == 0);

		if (!willpanic)
			clear_mc(first, last, ismc, clrstatus, hdl, gcl, pgcl);
	} else {
		disp = 0;
		if (mcesp) {
			mcesp->mce_nerr = mcesp->mce_disp = 0;
		}
	}

	/*
	 * Clear MCG_STATUS if MCIP is set (machine check in progress).
	 * If a second #MC had occured before now the system would have
	 * reset.  We can only do thise once gcpu_mca_process has copied
	 * the logout structure.
	 */
	if (ismc && mcg_status & MCG_STATUS_MCIP)
		(void) cmi_hdl_wrmsr(hdl, IA32_MSR_MCG_STATUS, 0);

	/*
	 * At this point we have read and logged all telemetry that is visible
	 * under the MCA.  On architectures for which the NorthBridge is
	 * on-chip this may include NB-observed errors, but where the NB
	 * is off chip it may have been the source of the #MC request and
	 * so we must call into the memory-controller driver to give it
	 * a chance to log errors.
	 */
	if (ismc) {
		cmi_mc_logout(hdl, 1, willpanic);
	}
}

#ifndef __xpv
int gcpu_mca_trap_vomit_summary = 0;

/*
 * On a native machine check exception we come here from mcetrap via
 * cmi_mca_trap.  A machine check on one cpu of a chip does not trap others
 * cpus of the chip, so it is possible that another cpu on this chip could
 * initiate a poll while we're in the #mc handler;  it is also possible that
 * this trap has occured during a poll on this cpu.  So we must acquire
 * the chip-wide poll lock, but be careful to avoid deadlock.
 *
 * The 'data' pointer cannot be NULL due to init order.
 */
uint64_t
gcpu_mca_trap(cmi_hdl_t hdl, struct regs *rp)
{
	gcpu_data_t *gcpu = cmi_hdl_getcmidata(hdl);
	kmutex_t *poll_lock = NULL;
	gcpu_mce_status_t mce;
	uint64_t mcg_status;
	int tooklock = 0;

	if (cmi_hdl_rdmsr(hdl, IA32_MSR_MCG_STATUS, &mcg_status) !=
	    CMI_SUCCESS || !(mcg_status & MCG_STATUS_MCIP))
		return (0);

	/*
	 * Synchronize with any poller from another core that may happen
	 * to share access to one or more of the MCA banks.
	 */
	if (gcpu->gcpu_shared != NULL)
		poll_lock = &gcpu->gcpu_shared->gcpus_poll_lock;

	if (poll_lock != NULL && !mutex_owned(poll_lock)) {
		/*
		 * The lock is not owned by the thread we have
		 * interrupted.  Spin for this adaptive lock.
		 */
		while (!mutex_tryenter(poll_lock)) {
			while (mutex_owner(poll_lock) != NULL)
				;
		}
		tooklock = 1;
	}

	gcpu_mca_logout(hdl, rp, 0, &mce, B_TRUE, GCPU_MPT_WHAT_MC_ERR);

	if (tooklock)
		mutex_exit(poll_lock);

	/*
	 * gcpu_mca_trap_vomit_summary may be set for debug assistance.
	 */
	if (mce.mce_nerr != 0 && gcpu_mca_trap_vomit_summary) {
		cmn_err(CE_WARN, "MCE: %u errors, disp=0x%llx, "
		    "%u PCC (%u ok), "
		    "%u UC (%d ok, %u poisoned), "
		    "%u forcefatal, %u ignored",
		    mce.mce_nerr, (u_longlong_t)mce.mce_disp,
		    mce.mce_npcc, mce.mce_npcc_ok,
		    mce.mce_nuc, mce.mce_nuc_ok, mce.mce_nuc_poisoned,
		    mce.mce_forcefatal, mce.mce_ignored);
	}

	return (mce.mce_disp);
}
#endif

/*ARGSUSED*/
void
gcpu_faulted_enter(cmi_hdl_t hdl)
{
	/* Nothing to do here */
}

/*ARGSUSED*/
void
gcpu_faulted_exit(cmi_hdl_t hdl)
{
	gcpu_data_t *gcpu = cmi_hdl_getcmidata(hdl);

	gcpu->gcpu_mca.gcpu_mca_flags |= GCPU_MCA_F_UNFAULTING;
}

/*
 * Write the requested values to the indicated MSRs.  Having no knowledge
 * of the model-specific requirements for writing to these model-specific
 * registers, we will only blindly write to those MSRs if the 'force'
 * argument is nonzero.  That option should only be used in prototyping
 * and debugging.
 */
/*ARGSUSED*/
cmi_errno_t
gcpu_msrinject(cmi_hdl_t hdl, cmi_mca_regs_t *regs, uint_t nregs,
    int force)
{
	int i, errs = 0;

	for (i = 0; i < nregs; i++) {
		uint_t msr = regs[i].cmr_msrnum;
		uint64_t val = regs[i].cmr_msrval;

		if (cms_present(hdl)) {
			if (cms_msrinject(hdl, msr, val) != CMS_SUCCESS)
				errs++;
		} else if (force) {
			errs += (cmi_hdl_wrmsr(hdl, msr, val) != CMI_SUCCESS);
		} else {
			errs++;
		}
	}

	return (errs == 0 ? CMI_SUCCESS : CMIERR_UNKNOWN);
}

/* deconfigure gcpu_mca_init() */
void
gcpu_mca_fini(cmi_hdl_t hdl)
{
	gcpu_data_t *gcpu = cmi_hdl_getcmidata(hdl);
	gcpu_mca_t *mca = &gcpu->gcpu_mca;
	int i;

	/*
	 * CPU startup code only calls cmi_mca_init if x86_featureset indicates
	 * both MCA and MCE support (i.e., X86FSET_MCA).  P5, K6, and earlier
	 * processors, which have their own more primitive way of doing
	 * machine checks, will not have cmi_mca_init called since their
	 * CPUID information will not indicate both MCA and MCE features.
	 */
	if (!is_x86_feature(x86_featureset, X86FSET_MCA))
		return;
#ifndef __xpv
	/*
	 * disable machine check in CR4
	 */
	cmi_ntv_hwdisable_mce(hdl);
#endif
	mutex_enter(&gcpu->gcpu_shared->gcpus_cfglock);
	gcpu_mca_poll_fini(hdl);
	mutex_exit(&gcpu->gcpu_shared->gcpus_cfglock);

	/*
	 * free resources allocated during init
	 */
	if (mca->gcpu_bank_cmci != NULL) {
		kmem_free(mca->gcpu_bank_cmci, sizeof (gcpu_mca_cmci_t) *
		    mca->gcpu_mca_nbanks);
	}

	for (i = 0; i < GCPU_MCA_LOGOUT_NUM; i++) {
		if (mca->gcpu_mca_logout[i] != NULL) {
			kmem_free(mca->gcpu_mca_logout[i], mca->gcpu_mca_lgsz);
		}
	}

	if (mca->gcpu_mca_bioscfg.bios_bankcfg != NULL) {
		kmem_free(mca->gcpu_mca_bioscfg.bios_bankcfg,
		    sizeof (struct gcpu_bios_bankcfg) * mca->gcpu_mca_nbanks);
	}
}
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 * Copyright (c) 2018, Joyent, Inc.
 */
/*
 * Copyright (c) 2010, Intel Corporation.
 * All rights reserved.
 */

/*
 * Native MCA polling.  We establish an ommipresent cyclic to fire on all
 * online cpus to check their MCA state and log any valid errors for
 * diagnosis.
 */

#include <sys/types.h>
#include <sys/atomic.h>
#include <sys/cyclic.h>
#include <sys/x86_archext.h>
#include <sys/mca_x86.h>

#include "gcpu.h"

hrtime_t gcpu_mca_poll_interval = NANOSEC * 10ULL;	/* tuneable */
static cyclic_id_t gcpu_mca_poll_cycid;
static volatile uint_t gcpu_mca_poll_inits;
extern int gcpu_poll_trace_always;
extern uint_t gcpu_poll_trace_nent;

/*
 * Return nonzero of the given handle should poll the MCH.  We stick with
 * the same handle as before unless the timestamp has not been updated
 * for a while.  There is no need to keep a hold on the mch_poll_owner
 * handle.
 */

static kmutex_t mch_poll_lock;
static hrtime_t mch_poll_timestamp;
static cmi_hdl_t mch_poll_owner;

static int
mch_pollowner(cmi_hdl_t hdl)
{
	hrtime_t now = gethrtime_waitfree();
	int dopoll = 0;

	mutex_enter(&mch_poll_lock);
	if (now - mch_poll_timestamp > 2 * gcpu_mca_poll_interval ||
	    mch_poll_timestamp == 0) {
		mch_poll_owner = hdl;
		dopoll = 1;
	} else if (mch_poll_owner == hdl) {
		dopoll = 1;
	}

	if (dopoll)
		mch_poll_timestamp = now;

	mutex_exit(&mch_poll_lock);
	return (dopoll);
}


static void
gcpu_ntv_mca_poll(cmi_hdl_t hdl, int what)
{
	gcpu_data_t *gcpu = cmi_hdl_getcmidata(hdl);
	gcpu_mca_t *mca = &gcpu->gcpu_mca;
	gcpu_mce_status_t mce;
	int willpanic;
	uint64_t bankmask;

	ASSERT(MUTEX_HELD(&gcpu->gcpu_shared->gcpus_poll_lock));

	/* Enable CMCI in first poll if is supported */
	if ((mca->gcpu_mca_flags & GCPU_MCA_F_CMCI_ENABLE) != 0 &&
	    (!mca->gcpu_mca_first_poll_cmci_enabled)) {
		int i;
		uint64_t ctl2;

		for (i = 0; i < mca->gcpu_mca_nbanks; i++) {
			if (mca->gcpu_bank_cmci[i].cmci_cap) {
				(void) cmi_hdl_rdmsr(hdl, IA32_MSR_MC_CTL2(i),
				    &ctl2);
				ctl2 |= MSR_MC_CTL2_EN;
				(void) cmi_hdl_wrmsr(hdl, IA32_MSR_MC_CTL2(i),
				    ctl2);
				mca->gcpu_bank_cmci[i].cmci_enabled = 1;
			}
		}
		mca->gcpu_mca_first_poll_cmci_enabled = 1;
	}

	if (mca->gcpu_mca_flags & GCPU_MCA_F_UNFAULTING) {
		int i;

		mca->gcpu_mca_flags &= ~GCPU_MCA_F_UNFAULTING;
		gcpu_poll_trace(&gcpu->gcpu_mca.gcpu_polltrace,
		    GCPU_MPT_WHAT_UNFAULTING, 0);

		/*
		 * On the first cyclic poll after unfaulting a CPU we
		 * clear the status registers; see gcpu_faulted_exit
		 * for details.  We don't do this if the poll was
		 * initiated manually (presumably from some injection
		 * activity).
		 */
		if (what == GCPU_MPT_WHAT_CYC_ERR) {
			for (i = 0; i < mca->gcpu_mca_nbanks; i++) {
				(void) cmi_hdl_wrmsr(hdl,
				    IA32_MSR_MC(i, STATUS), 0ULL);
			}
			return;
		}
	}

	/*
	 * Logout errors of the MCA banks of this cpu.
	 */
	if (what == GCPU_MPT_WHAT_CMCI_ERR) {
		/*
		 * for CMCI, all banks should be scanned for log out
		 */
		bankmask = -1ULL;
	} else {
		bankmask = cms_poll_ownermask(hdl, gcpu_mca_poll_interval);
	}
	gcpu_mca_logout(hdl, NULL, bankmask, &mce, B_TRUE, what);

	if (mce.mce_nerr != 0)
		gcpu_poll_trace(&gcpu->gcpu_mca.gcpu_polltrace, what,
		    mce.mce_nerr);

	mca->gcpu_mca_lastpoll = gethrtime_waitfree();

	willpanic = mce.mce_disp & CMI_ERRDISP_FORCEFATAL && cmi_panic_on_ue();

	if (what != GCPU_MPT_WHAT_CMCI_ERR) {
		/*
		 * Call to the memory-controller driver which may report some
		 * errors not visible under the MCA (for off-chip NB).
		 * Since there is typically a single MCH we arrange that
		 * just one cpu perform this task at each cyclic fire.
		 */
		if (mch_pollowner(hdl))
			cmi_mc_logout(hdl, 0, willpanic);
	}

	/*
	 * In the common case any polled error is considered non-fatal,
	 * even if it indicates PCC or UC etc.  The only condition on which
	 * we will panic for a polled error is if model-specific support
	 * forces the error to be terminal regardless of how it is
	 * encountered.
	 */
	if (willpanic) {
#ifdef DEBUG
		cmn_err(CE_WARN, "MCA Poll: %u errors, disp=0x%llx, "
		    "%u PCC (%u ok), "
		    "%u UC (%u ok, %u poisoned), "
		    "%u forcefatal, %u ignored",
		    mce.mce_nerr, (u_longlong_t)mce.mce_disp,
		    mce.mce_npcc, mce.mce_npcc_ok,
		    mce.mce_nuc, mce.mce_nuc_ok, mce.mce_nuc_poisoned,
		    mce.mce_forcefatal, mce.mce_ignored);

#endif
		fm_panic("Unrecoverable Machine-Check Exception (Polled)");
	}
}

/*
 * See gcpu_mca_trap for an explanation of why preemption is disabled here.
 * Note that we disable preemption and then contend for an adaptive mutex -
 * we could block during the mutex operation, but once we return with the
 * mutex held we nust perform no operation that can block and we cannot
 * be preempted so we will stay on cpu for the duration.  The disabling
 * of preemption also means we cannot migrate cpus once we have returned
 * with the mutex held - cyclic invocations can't migrate, anyway, but
 * others could if they have failed to bind before this point.
 */
static void
gcpu_ntv_mca_poll_wrapper(cmi_hdl_t hdl, int what)
{
	gcpu_data_t *gcpu;

	if (hdl == NULL || (gcpu = cmi_hdl_getcmidata(hdl)) == NULL ||
	    gcpu->gcpu_mca.gcpu_mca_lgsz == 0)
		return;

	kpreempt_disable();
	mutex_enter(&gcpu->gcpu_shared->gcpus_poll_lock);
	gcpu_ntv_mca_poll(hdl, what);
	mutex_exit(&gcpu->gcpu_shared->gcpus_poll_lock);
	kpreempt_enable();
}

static void
gcpu_ntv_mca_poll_cyclic(void *arg)
{
	gcpu_ntv_mca_poll_wrapper((cmi_hdl_t)arg, GCPU_MPT_WHAT_CYC_ERR);
}

/*ARGSUSED*/
static void
gcpu_ntv_mca_poll_online(void *arg, cpu_t *cp, cyc_handler_t *cyh,
    cyc_time_t *cyt)
{
	cmi_hdl_t hdl;

	/*
	 * Lookup and hold a handle for this cpu (any hold released in
	 * our offline function).  If we chose not to initialize a handle
	 * for this cpu back at cmi_init time then this lookup will return
	 * NULL, so the cyh_func we appoint must be prepared for that.
	 */
	hdl = cmi_hdl_lookup(CMI_HDL_NATIVE, cmi_ntv_hwchipid(cp),
	    cmi_ntv_hwcoreid(cp), cmi_ntv_hwstrandid(cp));

	cyt->cyt_when = 0;
	cyt->cyt_interval = gcpu_mca_poll_interval;
	cyh->cyh_func = gcpu_ntv_mca_poll_cyclic;
	cyh->cyh_arg = (void *)hdl;
	cyh->cyh_level = CY_LOW_LEVEL;
}

/*ARGSUSED*/
static void
gcpu_ntv_mca_poll_offline(void *arg, cpu_t *cpu, void *cyh_arg)
{
	cmi_hdl_t hdl = (cmi_hdl_t)cyh_arg;

	if (hdl != NULL)
		cmi_hdl_rele(hdl);
}

static void
gcpu_ntv_mca_poll_start(void)
{
	cyc_omni_handler_t cyo;

	if (gcpu_mca_poll_interval == 0 || gcpu_mca_poll_inits == 0)
		return;

	cyo.cyo_online = gcpu_ntv_mca_poll_online;
	cyo.cyo_offline = gcpu_ntv_mca_poll_offline;
	cyo.cyo_arg = NULL;

	mutex_enter(&cpu_lock);
	gcpu_mca_poll_cycid = cyclic_add_omni(&cyo);
	mutex_exit(&cpu_lock);
}

/*
 * gcpu_mca_poll_init is called from gcpu_mca_init for each cpu handle
 * that we initialize for.  It should prepare for polling by allocating
 * control structures and the like, but must not kick polling off yet.
 */

void
gcpu_mca_poll_init(cmi_hdl_t hdl)
{
	gcpu_data_t *gcpu = cmi_hdl_getcmidata(hdl);
	gcpu_poll_trace_ctl_t *ptc = &gcpu->gcpu_mca.gcpu_polltrace;

	ASSERT(cmi_hdl_class(hdl) == CMI_HDL_NATIVE);

	gcpu_poll_trace_init(ptc);

	atomic_inc_uint(&gcpu_mca_poll_inits);
}

/* deconfigure gcpu_mca_poll_init() */
void
gcpu_mca_poll_fini(cmi_hdl_t hdl)
{
	gcpu_data_t *gcpu = cmi_hdl_getcmidata(hdl);
	gcpu_poll_trace_ctl_t *ptc = &gcpu->gcpu_mca.gcpu_polltrace;

	ASSERT(cmi_hdl_class(hdl) == CMI_HDL_NATIVE);

	if (gcpu_poll_trace_always && (ptc->mptc_tbufs != NULL)) {
		kmem_free(ptc->mptc_tbufs, sizeof (gcpu_poll_trace_t) *
		    gcpu_poll_trace_nent);
	}

	atomic_dec_uint(&gcpu_mca_poll_inits);
}

void
gcpu_mca_poll_start(cmi_hdl_t hdl)
{
	ASSERT(cmi_hdl_class(hdl) == CMI_HDL_NATIVE);
	gcpu_ntv_mca_poll_start();
}

void
gcpu_hdl_poke(cmi_hdl_t hdl)
{
	ASSERT(cmi_hdl_class(hdl) == CMI_HDL_NATIVE);
	gcpu_ntv_mca_poll_wrapper(hdl, GCPU_MPT_WHAT_POKE_ERR);
}

void
gcpu_cmci_trap(cmi_hdl_t hdl)
{
	gcpu_ntv_mca_poll_wrapper(hdl, GCPU_MPT_WHAT_CMCI_ERR);
}
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

/*
 * Generic x86 CPU MCA poller - support functions for native and xpv pollers.
 */

#include <sys/types.h>
#include <sys/sysmacros.h>
#include <sys/sdt.h>
#include <sys/cmn_err.h>

#include "gcpu.h"

uint_t gcpu_poll_trace_nent = 100;

#ifdef DEBUG
int gcpu_poll_trace_always = 1;
#else
int gcpu_poll_trace_always = 0;
#endif

void
gcpu_poll_trace(gcpu_poll_trace_ctl_t *ptc, uint8_t what, uint8_t nerr)
{
	gcpu_poll_trace_t *pt;
	uint_t next;

	DTRACE_PROBE2(gcpu__mca__poll__trace, uint32_t, what, uint32_t, nerr);

	if (ptc->mptc_tbufs == NULL)
		return; /* poll trace buffer is disabled */

	next = (ptc->mptc_curtrace + 1) % gcpu_poll_trace_nent;
	pt = &ptc->mptc_tbufs[next];

	pt->mpt_when = 0;
	pt->mpt_what = what;

	pt->mpt_nerr = MIN(nerr, UINT8_MAX);

	pt->mpt_when = gethrtime_waitfree();
	ptc->mptc_curtrace = next;
}

void
gcpu_poll_trace_init(gcpu_poll_trace_ctl_t *ptc)
{
	gcpu_poll_trace_t *tbufs = NULL;

	if (gcpu_poll_trace_always) {
		tbufs = kmem_zalloc(sizeof (gcpu_poll_trace_t) *
		    gcpu_poll_trace_nent, KM_SLEEP);
	}

	ptc->mptc_tbufs = tbufs;
	ptc->mptc_curtrace = 0;
}
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
 */

/*
 * Intel model-specific support.  Right now all this conists of is
 * to modify the ereport subclass to produce different ereport classes
 * so that we can have different diagnosis rules and corresponding faults.
 */

#include <sys/types.h>
#include <sys/cmn_err.h>
#include <sys/modctl.h>
#include <sys/mca_x86.h>
#include <sys/cpu_module_ms_impl.h>
#include <sys/mc_intel.h>
#include <sys/pci_cfgspace.h>
#include <sys/fm/protocol.h>
#include <sys/fm/util.h>
#include <sys/fm/smb/fmsmb.h>

extern int x86gentopo_legacy;

int gintel_ms_support_disable = 0;
int gintel_error_action_return = 0;
int gintel_ms_unconstrained = 0;

int quickpath;
int max_bus_number = 0xff;

#define	ERR_COUNTER_INDEX	2
#define	MAX_CPU_NODES		2
#define	N_MC_COR_ECC_CNT	6
uint32_t err_counter_array[MAX_CPU_NODES][ERR_COUNTER_INDEX][N_MC_COR_ECC_CNT];
uint8_t	err_counter_index[MAX_CPU_NODES];

#define	MAX_BUS_NUMBER  max_bus_number
#define	SOCKET_BUS(cpu) (MAX_BUS_NUMBER - (cpu))

#define	MC_COR_ECC_CNT(chipid, reg)	(*pci_getl_func)(SOCKET_BUS(chipid), \
    NEHALEM_EP_MEMORY_CONTROLLER_DEV, NEHALEM_EP_MEMORY_CONTROLLER_FUNC, \
    0x80 + (reg) * 4)

#define	MSCOD_MEM_ECC_READ	0x1
#define	MSCOD_MEM_ECC_SCRUB	0x2
#define	MSCOD_MEM_WR_PARITY	0x4
#define	MSCOD_MEM_REDUNDANT_MEM	0x8
#define	MSCOD_MEM_SPARE_MEM	0x10
#define	MSCOD_MEM_ILLEGAL_ADDR	0x20
#define	MSCOD_MEM_BAD_ID	0x40
#define	MSCOD_MEM_ADDR_PARITY	0x80
#define	MSCOD_MEM_BYTE_PARITY	0x100

#define	GINTEL_ERROR_MEM	0x1000
#define	GINTEL_ERROR_QUICKPATH	0x2000
#define	GINTEL_ERROR_UNKNOWN	0x4000

#define	GINTEL_ERR_SPARE_MEM	(GINTEL_ERROR_MEM | 1)
#define	GINTEL_ERR_MEM_UE	(GINTEL_ERROR_MEM | 2)
#define	GINTEL_ERR_MEM_CE	(GINTEL_ERROR_MEM | 3)
#define	GINTEL_ERR_MEM_PARITY	(GINTEL_ERROR_MEM | 4)
#define	GINTEL_ERR_MEM_ADDR_PARITY	(GINTEL_ERROR_MEM | 5)
#define	GINTEL_ERR_MEM_REDUNDANT (GINTEL_ERROR_MEM | 6)
#define	GINTEL_ERR_MEM_BAD_ADDR	(GINTEL_ERROR_MEM | 7)
#define	GINTEL_ERR_MEM_BAD_ID	(GINTEL_ERROR_MEM | 8)
#define	GINTEL_ERR_MEM_UNKNOWN	(GINTEL_ERROR_MEM | 0xfff)

#define	MSR_MC_MISC_MEM_CHANNEL_MASK	0x00000000000c0000ULL
#define	MSR_MC_MISC_MEM_CHANNEL_SHIFT	18
#define	MSR_MC_MISC_MEM_DIMM_MASK	0x0000000000030000ULL
#define	MSR_MC_MISC_MEM_DIMM_SHIFT	16
#define	MSR_MC_MISC_MEM_SYNDROME_MASK	0xffffffff00000000ULL
#define	MSR_MC_MISC_MEM_SYNDROME_SHIFT	32

#define	CPU_GENERATION_DONT_CARE	0
#define	CPU_GENERATION_NEHALEM_EP	1

#define	INTEL_CPU_6_ID			0x6
#define	INTEL_NEHALEM_CPU_FAMILY_ID	0x6
#define	INTEL_NEHALEM_CPU_MODEL_ID	0x1A

#define	NEHALEM_EP_MEMORY_CONTROLLER_DEV	0x3
#define	NEHALEM_EP_MEMORY_CONTROLLER_FUNC	0x2

/*ARGSUSED*/
int
gintel_init(cmi_hdl_t hdl, void **datap)
{
	uint32_t nb_chipset;

	if (gintel_ms_support_disable)
		return (ENOTSUP);

	if (!is_x86_feature(x86_featureset, X86FSET_MCA))
		return (ENOTSUP);

	nb_chipset = (*pci_getl_func)(0, 0, 0, 0x0);
	switch (nb_chipset) {
	case INTEL_NB_7300:
	case INTEL_NB_5000P:
	case INTEL_NB_5000X:
	case INTEL_NB_5000V:
	case INTEL_NB_5000Z:
	case INTEL_NB_5400:
	case INTEL_NB_5400A:
	case INTEL_NB_5400B:
		if (!gintel_ms_unconstrained)
			gintel_error_action_return |= CMS_ERRSCOPE_POISONED;
		break;
	case INTEL_QP_IO:
	case INTEL_QP_WP:
	case INTEL_QP_36D:
	case INTEL_QP_24D:
	case INTEL_QP_U1:
	case INTEL_QP_U2:
	case INTEL_QP_U3:
	case INTEL_QP_U4:
	case INTEL_QP_JF:
	case INTEL_QP_JF0:
	case INTEL_QP_JF1:
	case INTEL_QP_JF2:
	case INTEL_QP_JF3:
	case INTEL_QP_JF4:
	case INTEL_QP_JF5:
	case INTEL_QP_JF6:
	case INTEL_QP_JF7:
	case INTEL_QP_JF8:
	case INTEL_QP_JF9:
	case INTEL_QP_JFa:
	case INTEL_QP_JFb:
	case INTEL_QP_JFc:
	case INTEL_QP_JFd:
	case INTEL_QP_JFe:
	case INTEL_QP_JFf:
		quickpath = 1;
		break;
	default:
		break;
	}
	return (0);
}

/*ARGSUSED*/
uint32_t
gintel_error_action(cmi_hdl_t hdl, int ismc, int bank,
    uint64_t status, uint64_t addr, uint64_t misc, void *mslogout)
{
	uint32_t rc;

	if (ismc == 0 && bank == 0 &&
	    cmi_hdl_family(hdl) == INTEL_CPU_6_ID &&
	    cmi_hdl_model(hdl) < INTEL_NEHALEM_CPU_MODEL_ID &&
	    MCAX86_ERRCODE_ISBUS_INTERCONNECT(MCAX86_ERRCODE(status)) &&
	    MCAX86_MSERRCODE(status) == 0) {
		rc = CMS_ERRSCOPE_CURCONTEXT_OK | CMS_ERRSCOPE_CLEARED_UC;
	} else if ((status & MSR_MC_STATUS_PCC) == 0) {
		rc = gintel_error_action_return;
	} else {
		rc = gintel_error_action_return & ~CMS_ERRSCOPE_POISONED;
	}
	return (rc);
}

/*ARGSUSED*/
cms_cookie_t
gintel_disp_match(cmi_hdl_t hdl, int ismc, int bank, uint64_t status,
    uint64_t addr, uint64_t misc, void *mslogout)
{
	cms_cookie_t rt = (cms_cookie_t)NULL;
	uint16_t mcacode = MCAX86_ERRCODE(status);
	uint16_t mscode = MCAX86_MSERRCODE(status);

	if (MCAX86_ERRCODE_ISMEMORY_CONTROLLER(mcacode)) {
		/*
		 * memory controller errors
		 */
		if (mscode & MSCOD_MEM_SPARE_MEM) {
			rt = (cms_cookie_t)GINTEL_ERR_SPARE_MEM;
		} else if (mscode & (MSCOD_MEM_ECC_READ |
		    MSCOD_MEM_ECC_SCRUB)) {
			if (status & MSR_MC_STATUS_UC)
				rt = (cms_cookie_t)GINTEL_ERR_MEM_UE;
			else
				rt = (cms_cookie_t)GINTEL_ERR_MEM_CE;
		} else if (mscode & (MSCOD_MEM_WR_PARITY |
		    MSCOD_MEM_BYTE_PARITY)) {
			rt = (cms_cookie_t)GINTEL_ERR_MEM_PARITY;
		} else if (mscode & MSCOD_MEM_ADDR_PARITY) {
			rt = (cms_cookie_t)GINTEL_ERR_MEM_ADDR_PARITY;
		} else if (mscode & MSCOD_MEM_REDUNDANT_MEM) {
			rt = (cms_cookie_t)GINTEL_ERR_MEM_REDUNDANT;
		} else if (mscode & MSCOD_MEM_ILLEGAL_ADDR) {
			rt = (cms_cookie_t)GINTEL_ERR_MEM_BAD_ADDR;
		} else if (mscode & MSCOD_MEM_BAD_ID) {
			rt = (cms_cookie_t)GINTEL_ERR_MEM_BAD_ID;
		} else {
			rt = (cms_cookie_t)GINTEL_ERR_MEM_UNKNOWN;
		}
	} else if (quickpath &&
	    MCAX86_ERRCODE_ISBUS_INTERCONNECT(MCAX86_ERRCODE(status))) {
		rt = (cms_cookie_t)GINTEL_ERROR_QUICKPATH;
	} else if (ismc == 0 && bank == 0 &&
	    cmi_hdl_family(hdl) == INTEL_CPU_6_ID &&
	    cmi_hdl_model(hdl) < INTEL_NEHALEM_CPU_MODEL_ID &&
	    MCAX86_ERRCODE_ISBUS_INTERCONNECT(MCAX86_ERRCODE(status)) &&
	    MCAX86_MSERRCODE(status) == 0) {
		rt = (cms_cookie_t)GINTEL_ERROR_UNKNOWN;
	}
	return (rt);
}

/*ARGSUSED*/
void
gintel_ereport_class(cmi_hdl_t hdl, cms_cookie_t mscookie,
    const char **cpuclsp, const char **leafclsp)
{
	*cpuclsp = FM_EREPORT_CPU_INTEL;
	switch ((uintptr_t)mscookie) {
	case GINTEL_ERROR_QUICKPATH:
		*leafclsp = "quickpath.interconnect";
		break;
	case GINTEL_ERR_SPARE_MEM:
		*leafclsp = "quickpath.mem_spare";
		break;
	case GINTEL_ERR_MEM_UE:
		*leafclsp = "quickpath.mem_ue";
		break;
	case GINTEL_ERR_MEM_CE:
		*leafclsp = "quickpath.mem_ce";
		break;
	case GINTEL_ERR_MEM_PARITY:
		*leafclsp = "quickpath.mem_parity";
		break;
	case GINTEL_ERR_MEM_ADDR_PARITY:
		*leafclsp = "quickpath.mem_addr_parity";
		break;
	case GINTEL_ERR_MEM_REDUNDANT:
		*leafclsp = "quickpath.mem_redundant";
		break;
	case GINTEL_ERR_MEM_BAD_ADDR:
		*leafclsp = "quickpath.mem_bad_addr";
		break;
	case GINTEL_ERR_MEM_BAD_ID:
		*leafclsp = "quickpath.mem_bad_id";
		break;
	case GINTEL_ERR_MEM_UNKNOWN:
		*leafclsp = "quickpath.mem_unknown";
		break;
	case GINTEL_ERROR_UNKNOWN:
		*leafclsp = "unknown";
		break;
	}
}

static nvlist_t *
gintel_gentopo_ereport_detector(cmi_hdl_t hdl, cms_cookie_t mscookie,
    nv_alloc_t *nva)
{
	nvlist_t *nvl = (nvlist_t *)NULL;
	nvlist_t *board_list = (nvlist_t *)NULL;

	if (mscookie) {
		board_list = cmi_hdl_smb_bboard(hdl);

		if (board_list == NULL)
			return (NULL);

		if ((nvl = fm_nvlist_create(nva)) == NULL)
			return (NULL);

		if ((uintptr_t)mscookie & GINTEL_ERROR_QUICKPATH) {
			fm_fmri_hc_create(nvl, FM_HC_SCHEME_VERSION,
			    NULL, NULL, board_list, 1,
			    "chip", cmi_hdl_smb_chipid(hdl));
		} else {
			fm_fmri_hc_create(nvl, FM_HC_SCHEME_VERSION,
			    NULL, NULL, board_list, 2,
			    "chip", cmi_hdl_smb_chipid(hdl),
			    "memory-controller", 0);
		}
	}
	return (nvl);
}

/*ARGSUSED*/
nvlist_t *
gintel_ereport_detector(cmi_hdl_t hdl, int bankno, cms_cookie_t mscookie,
    nv_alloc_t *nva)
{
	nvlist_t *nvl = (nvlist_t *)NULL;

	if (!x86gentopo_legacy) {
		nvl = gintel_gentopo_ereport_detector(hdl, mscookie, nva);
		return (nvl);
	}

	if (mscookie) {
		if ((nvl = fm_nvlist_create(nva)) == NULL)
			return (NULL);
		if (((uintptr_t)mscookie & GINTEL_ERROR_QUICKPATH) ||
		    ((uintptr_t)mscookie & GINTEL_ERROR_UNKNOWN)) {
			fm_fmri_hc_set(nvl, FM_HC_SCHEME_VERSION, NULL, NULL, 2,
			    "motherboard", 0,
			    "chip", cmi_hdl_chipid(hdl));
		} else {
			fm_fmri_hc_set(nvl, FM_HC_SCHEME_VERSION, NULL, NULL, 3,
			    "motherboard", 0,
			    "chip", cmi_hdl_chipid(hdl),
			    "memory-controller", 0);
		}
	}
	return (nvl);
}

static nvlist_t *
gintel_gentopo_ereport_create_resource_elem(cmi_hdl_t hdl, nv_alloc_t *nva,
    mc_unum_t *unump)
{
	nvlist_t *nvl, *snvl;
	nvlist_t *board_list = NULL;

	board_list = cmi_hdl_smb_bboard(hdl);
	if (board_list == NULL) {
		return (NULL);
	}

	if ((nvl = fm_nvlist_create(nva)) == NULL)	/* freed by caller */
		return (NULL);

	if ((snvl = fm_nvlist_create(nva)) == NULL) {
		fm_nvlist_destroy(nvl, nva ? FM_NVA_RETAIN : FM_NVA_FREE);
		return (NULL);
	}

	(void) nvlist_add_uint64(snvl, FM_FMRI_HC_SPECIFIC_OFFSET,
	    unump->unum_offset);

	if (unump->unum_chan == -1) {
		fm_fmri_hc_create(nvl, FM_HC_SCHEME_VERSION, NULL, snvl,
		    board_list, 2,
		    "chip", cmi_hdl_smb_chipid(hdl),
		    "memory-controller", unump->unum_mc);
	} else if (unump->unum_cs == -1) {
		fm_fmri_hc_create(nvl, FM_HC_SCHEME_VERSION, NULL, snvl,
		    board_list, 3,
		    "chip", cmi_hdl_smb_chipid(hdl),
		    "memory-controller", unump->unum_mc,
		    "dram-channel", unump->unum_chan);
	} else if (unump->unum_rank == -1) {
		fm_fmri_hc_create(nvl, FM_HC_SCHEME_VERSION, NULL, snvl,
		    board_list, 4,
		    "chip", cmi_hdl_smb_chipid(hdl),
		    "memory-controller", unump->unum_mc,
		    "dram-channel", unump->unum_chan,
		    "dimm", unump->unum_cs);
	} else {
		fm_fmri_hc_create(nvl, FM_HC_SCHEME_VERSION, NULL, snvl,
		    board_list, 5,
		    "chip", cmi_hdl_smb_chipid(hdl),
		    "memory-controller", unump->unum_mc,
		    "dram-channel", unump->unum_chan,
		    "dimm", unump->unum_cs,
		    "rank", unump->unum_rank);
	}

	fm_nvlist_destroy(snvl, nva ? FM_NVA_RETAIN : FM_NVA_FREE);

	return (nvl);
}

static nvlist_t *
gintel_ereport_create_resource_elem(nv_alloc_t *nva, mc_unum_t *unump)
{
	nvlist_t *nvl, *snvl;

	if ((nvl = fm_nvlist_create(nva)) == NULL)	/* freed by caller */
		return (NULL);

	if ((snvl = fm_nvlist_create(nva)) == NULL) {
		fm_nvlist_destroy(nvl, nva ? FM_NVA_RETAIN : FM_NVA_FREE);
		return (NULL);
	}

	(void) nvlist_add_uint64(snvl, FM_FMRI_HC_SPECIFIC_OFFSET,
	    unump->unum_offset);

	if (unump->unum_chan == -1) {
		fm_fmri_hc_set(nvl, FM_HC_SCHEME_VERSION, NULL, snvl, 3,
		    "motherboard", unump->unum_board,
		    "chip", unump->unum_chip,
		    "memory-controller", unump->unum_mc);
	} else if (unump->unum_cs == -1) {
		fm_fmri_hc_set(nvl, FM_HC_SCHEME_VERSION, NULL, snvl, 4,
		    "motherboard", unump->unum_board,
		    "chip", unump->unum_chip,
		    "memory-controller", unump->unum_mc,
		    "dram-channel", unump->unum_chan);
	} else if (unump->unum_rank == -1) {
		fm_fmri_hc_set(nvl, FM_HC_SCHEME_VERSION, NULL, snvl, 5,
		    "motherboard", unump->unum_board,
		    "chip", unump->unum_chip,
		    "memory-controller", unump->unum_mc,
		    "dram-channel", unump->unum_chan,
		    "dimm", unump->unum_cs);
	} else {
		fm_fmri_hc_set(nvl, FM_HC_SCHEME_VERSION, NULL, snvl, 6,
		    "motherboard", unump->unum_board,
		    "chip", unump->unum_chip,
		    "memory-controller", unump->unum_mc,
		    "dram-channel", unump->unum_chan,
		    "dimm", unump->unum_cs,
		    "rank", unump->unum_rank);
	}

	fm_nvlist_destroy(snvl, nva ? FM_NVA_RETAIN : FM_NVA_FREE);

	return (nvl);
}

static void
nehalem_ep_ereport_add_memory_error_counter(uint_t  chipid,
    uint32_t *this_err_counter_array)
{
	int	index;

	for (index = 0; index < N_MC_COR_ECC_CNT; index ++)
		this_err_counter_array[index] = MC_COR_ECC_CNT(chipid, index);
}

static int
gintel_cpu_generation(cmi_hdl_t hdl)
{
	int	cpu_generation = CPU_GENERATION_DONT_CARE;

	if ((cmi_hdl_family(hdl) == INTEL_NEHALEM_CPU_FAMILY_ID) &&
	    (cmi_hdl_model(hdl) == INTEL_NEHALEM_CPU_MODEL_ID))
		cpu_generation = CPU_GENERATION_NEHALEM_EP;

	return (cpu_generation);
}

/*ARGSUSED*/
void
gintel_ereport_add_logout(cmi_hdl_t hdl, nvlist_t *ereport,
    nv_alloc_t *nva, int banknum, uint64_t status, uint64_t addr,
    uint64_t misc, void *mslogout, cms_cookie_t mscookie)
{
	mc_unum_t unum;
	nvlist_t *resource;
	uint32_t synd = 0;
	int  chan = MCAX86_ERRCODE_CCCC(status);
	uint8_t last_index, this_index;
	int chipid;

	if (chan == 0xf)
		chan = -1;

	if ((uintptr_t)mscookie & GINTEL_ERROR_MEM) {
		unum.unum_board = 0;
		unum.unum_chip = cmi_hdl_chipid(hdl);
		unum.unum_mc = 0;
		unum.unum_chan = chan;
		unum.unum_cs = -1;
		unum.unum_rank = -1;
		unum.unum_offset = -1ULL;
		if (status & MSR_MC_STATUS_MISCV) {
			unum.unum_chan =
			    (misc & MSR_MC_MISC_MEM_CHANNEL_MASK) >>
			    MSR_MC_MISC_MEM_CHANNEL_SHIFT;
			unum.unum_cs =
			    (misc & MSR_MC_MISC_MEM_DIMM_MASK) >>
			    MSR_MC_MISC_MEM_DIMM_SHIFT;
			synd = (misc & MSR_MC_MISC_MEM_SYNDROME_MASK) >>
			    MSR_MC_MISC_MEM_SYNDROME_SHIFT;
			fm_payload_set(ereport, FM_EREPORT_PAYLOAD_ECC_SYND,
			    DATA_TYPE_UINT32, synd, 0);
		}
		if (status & MSR_MC_STATUS_ADDRV) {
			fm_payload_set(ereport, FM_FMRI_MEM_PHYSADDR,
			    DATA_TYPE_UINT64, addr, NULL);
			(void) cmi_mc_patounum(addr, 0, 0, synd, 0, &unum);
			if (unum.unum_offset != -1ULL &&
			    (unum.unum_offset & OFFSET_ROW_BANK_COL) != 0) {
				fm_payload_set(ereport,
				    FM_EREPORT_PAYLOAD_NAME_BANK,
				    DATA_TYPE_INT32,
				    TCODE_OFFSET_BANK(unum.unum_offset), NULL);
				fm_payload_set(ereport,
				    FM_EREPORT_PAYLOAD_NAME_CAS,
				    DATA_TYPE_INT32,
				    TCODE_OFFSET_CAS(unum.unum_offset), NULL);
				fm_payload_set(ereport,
				    FM_EREPORT_PAYLOAD_NAME_RAS,
				    DATA_TYPE_INT32,
				    TCODE_OFFSET_RAS(unum.unum_offset), NULL);
			}
		}

		if (!x86gentopo_legacy) {
			resource = gintel_gentopo_ereport_create_resource_elem(
			    hdl, nva, &unum);
		} else {
			resource = gintel_ereport_create_resource_elem(nva,
			    &unum);
		}

		fm_payload_set(ereport, FM_EREPORT_PAYLOAD_NAME_RESOURCE,
		    DATA_TYPE_NVLIST_ARRAY, 1, &resource, NULL);
		fm_nvlist_destroy(resource, nva ? FM_NVA_RETAIN:FM_NVA_FREE);

		if (gintel_cpu_generation(hdl) == CPU_GENERATION_NEHALEM_EP) {

			chipid = unum.unum_chip;
			if (chipid < MAX_CPU_NODES) {
				last_index = err_counter_index[chipid];
				this_index =
				    (last_index + 1) % ERR_COUNTER_INDEX;
				err_counter_index[chipid] = this_index;
				nehalem_ep_ereport_add_memory_error_counter(
				    chipid,
				    err_counter_array[chipid][this_index]);
				fm_payload_set(ereport,
				    FM_EREPORT_PAYLOAD_MEM_ECC_COUNTER_THIS,
				    DATA_TYPE_UINT32_ARRAY, N_MC_COR_ECC_CNT,
				    err_counter_array[chipid][this_index],
				    NULL);
				fm_payload_set(ereport,
				    FM_EREPORT_PAYLOAD_MEM_ECC_COUNTER_LAST,
				    DATA_TYPE_UINT32_ARRAY, N_MC_COR_ECC_CNT,
				    err_counter_array[chipid][last_index],
				    NULL);
			}
		}
	}
}

boolean_t
gintel_bankctl_skipinit(cmi_hdl_t hdl, int banknum)
{
	/*
	 * On Intel family 6 before QuickPath we must not enable machine check
	 * from bank 0 detectors. bank 0 is reserved for the platform
	 */

	if (banknum == 0 &&
	    cmi_hdl_family(hdl) == INTEL_NEHALEM_CPU_FAMILY_ID &&
	    cmi_hdl_model(hdl) < INTEL_NEHALEM_CPU_MODEL_ID)
		return (1);
	else
		return (0);
}

cms_api_ver_t _cms_api_version = CMS_API_VERSION_2;

const cms_ops_t _cms_ops = {
	gintel_init,		/* cms_init */
	NULL,			/* cms_post_startup */
	NULL,			/* cms_post_mpstartup */
	NULL,			/* cms_logout_size */
	NULL,			/* cms_mcgctl_val */
	gintel_bankctl_skipinit, /* cms_bankctl_skipinit */
	NULL,			/* cms_bankctl_val */
	NULL,			/* cms_bankstatus_skipinit */
	NULL,			/* cms_bankstatus_val */
	NULL,			/* cms_mca_init */
	NULL,			/* cms_poll_ownermask */
	NULL,			/* cms_bank_logout */
	gintel_error_action,	/* cms_error_action */
	gintel_disp_match,	/* cms_disp_match */
	gintel_ereport_class,	/* cms_ereport_class */
	gintel_ereport_detector,	/* cms_ereport_detector */
	NULL,			/* cms_ereport_includestack */
	gintel_ereport_add_logout,	/* cms_ereport_add_logout */
	NULL,			/* cms_msrinject */
	NULL,			/* cms_fini */
};

static struct modlcpu modlcpu = {
	&mod_cpuops,
	"Generic Intel model-specific MCA"
};

static struct modlinkage modlinkage = {
	MODREV_1,
	(void *)&modlcpu,
	NULL
};

int
_init(void)
{
	return (mod_install(&modlinkage));
}

int
_info(struct modinfo *modinfop)
{
	return (mod_info(&modlinkage, modinfop));
}

int
_fini(void)
{
	return (mod_remove(&modlinkage));
}
#
# CDDL HEADER START
#
# The contents of this file are subject to the terms of the
# Common Development and Distribution License (the "License").
# You may not use this file except in compliance with the License.
#
# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
# or http://www.opensolaris.org/os/licensing.
# See the License for the specific language governing permissions
# and limitations under the License.
#
# When distributing Covered Code, include this CDDL HEADER in each
# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
# If applicable, add the following below this CDDL HEADER, with the
# fields enclosed by brackets "[]" replaced with your own identifying
# information: Portions Copyright [yyyy] [name of copyright owner]
#
# CDDL HEADER END
#

#
# Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
# Use is subject to license terms.
#
# ident	"%Z%%M%	%I%	%E% SMI"
#

PERLFILES= \
	ao_gendisp

include ../../../Makefile.uts

.KEEP_STATE:

def all install setup: $(PERLFILES)

clean clobber:
	$(RM) $(PERLFILES)

include ../../../Makefile.targ
#!/bin/perl
#
# CDDL HEADER START
#
# The contents of this file are subject to the terms of the
# Common Development and Distribution License (the "License").
# You may not use this file except in compliance with the License.
#
# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
# or http://www.opensolaris.org/os/licensing.
# See the License for the specific language governing permissions
# and limitations under the License.
#
# When distributing Covered Code, include this CDDL HEADER in each
# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
# If applicable, add the following below this CDDL HEADER, with the
# fields enclosed by brackets "[]" replaced with your own identifying
# information: Portions Copyright [yyyy] [name of copyright owner]
#
# CDDL HEADER END
#

#
# Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
# Use is subject to license terms.
#
# ident	"%Z%%M%	%I%	%E% SMI"
#

use strict;
use File::Basename;

my $PROGNAME = basename($0);

my ($funcunit, $error);
my @funcunits = ();
my @errorrefs = ();

my $codelinesin = 0;	# number of input 'code' lines for an ereport type
my $codeoutlen = 0;	# number of output lines from sub state_code

my $state = "initial";

sub usage() {
	print STDERR "Usage: $PROGNAME inputfile\n";
	exit(2);
}

sub bail() {
	print STDERR "$PROGNAME: ", join(" ", @_), "\n";
	exit(1);
}

sub parsebail() {
	print STDERR "$PROGNAME: $::infile: $.: ", join(" ", @_), "\n";
	exit(1);
}

sub error_alloc() {
	my @a = ();

	push(@::errorrefs, \@a);
	return (\@a);
}

sub error_dup() {
	my ($drop) = @_;
	my $newref = &error_alloc();

	my $zeroref = $::errorrefs[0];

	my $n = $#$zeroref - $drop;

	@$newref = @$zeroref[0 .. $n];
}

sub code_lines() {
	return ($::codelinesin++);
}

sub error_init() {
	&error_alloc();
	$::codelinesin = 0;
}

sub error_reset() {
	@::errorrefs = ();
	$::codelinesin = 0;
	$::codeoutlen = 0;
}

sub errout() {
	my ($line) = @_;

	foreach my $ref (@::errorrefs) {
		push(@$ref, $line);
	}
}

sub errout_N() {
	my ($instance, $line) = @_;
	my $ref = @::errorrefs[$instance];
	push(@$ref, $line);
	return 1;
}

sub print_errout() {
	foreach my $ref (@::errorrefs) {
		print @$ref;
	}
}

sub print_header() {
	print "#include <sys/mca_x86.h>\n";
	print "#include \"ao_mca_disp.h\"\n\n";
}

sub print_footer() {
	print 'const ao_error_disp_t *ao_error_disp[] = {' . "\n";

	foreach my $name (@funcunits) {
		print "\t$name,\n";
	}

	print "};\n";
}

sub funcunit_begin() {
	my $arrnm = "ao_error_disp_" . $_[0];
	print "static const ao_error_disp_t " . $arrnm . "[] = {\n";

	@funcunits = (@funcunits, $arrnm);
}

sub funcunit_end() {
	print "\t{ NULL }\n};\n\n";
}

sub error_begin() {
	my ($ereport_name) = @_;

	$ereport_name =~ tr/[a-z]./[A-Z]_/;
	my $flags_name = $ereport_name;
	$flags_name =~ s/EREPORT_/EREPORT_PAYLOAD_FLAGS_/;

	&errout("\tFM_$ereport_name,\n\tFM_$flags_name,\n");
}

sub error_end() {
	&errout("\t},\n\n");

	&print_errout();

	&error_reset();
}

sub print_bits() {
	my $name = $_[0];
	my @bits = @_[1..$#_];
	my $out = "";

	if (@bits == 0) {
		$out = "\t0,";
	} elsif (@bits == 1) {
		$out = "\t$bits[0],";
	} else {
		$out = "\t( " . join(" | ", @bits) . " ),";
	}

	$out .= " /* $name */" if (defined $name);
	$out .= "\n";

	return ($out);
}

sub field_burst() {
	my ($field, $valuesref, $name, $prefix) = @_;

	if ($field eq "-") {
		return ();
	}

	map {
		if (!defined ${$valuesref}{$_}) {
			&parsebail("unknown $name value `$_'");
		}
		$_ = ${$valuesref}{$_};
		tr/[a-z]/[A-Z]/;
		$prefix . "_" . $_;
	} split(/\//, $field);
}

sub bin2dec() {
	my $bin = $_[0];
	my $dec = 0;

	foreach my $bit (split(//, $bin)) {
		$dec = $dec * 2 + ($bit eq "1" ? 1 : 0);
	}

	$dec;
}

sub state_funcunit() {
	my $val = $_[0];

	if (defined $::funcunit) {
		&funcunit_end();
	}

	$::funcunit = $val;
	undef $::error;
	&funcunit_begin($::funcunit);
}

sub state_desc() {
	my $desc = $_[0];

	&error_init();

	&errout("\t/* $desc */\n\t{\n");
}

sub state_error() {
	$::error = $_[0];
	&error_begin($::error);
}

sub state_mask_on() {
	@::mask_on = map { tr/[a-z]/[A-Z]/; $_; } split(/,\s*/, $_[0]);
}

sub state_mask_off() {
	my @mask_off = map { tr/[a-z]/[A-Z]/; $_; } split(/,\s*/, $_[0]);

	&errout(&print_bits("mask", @::mask_on, @mask_off));
	&errout(&print_bits("mask_res", @::mask_on));
}

sub state_code() {
	my ($ext, $type, $pp, $t, $r4, $addr, $ii, $ll, $tt) =
	    split(/\s+/, $_[0]);

	my %tt_values = ( instr => 1, data => 1, gen => 1, '-' => 1 );
	my %ll_values = ( l0 => 1, l1 => 1, l2 => 1, lg => 1 );

	my %r4_values = (
		'err' => 'err',
		'rd' => 'rd',
		'wr' => 'wr',
		'drd' => 'drd',
		'dwr' => 'dwr',
		'ird' => 'ird',
		'pf' => 'prefetch',
		'ev' => 'evict',
		'snp' => 'snoop',
	        '-' => '-');

	my %pp_values = (
		'src' => 'src',
		'res' => 'res',
		'obs' => 'obs',
		'gen' => 'gen',
		'-' => '-' );

	my %t_values = ( 0 => 1, 1 => 1, '-' => 1 );

	my %ii_values = (
		'mem' => 'mem',
		'io' => 'io',
		'gen' => 'gen',
		'-' => '-' );

	my $instance = &code_lines();
	if ($instance > 0) {
		&error_dup($::codeoutlen);	# dup info thus far
	}

	if (!defined $tt_values{$tt}) {
		&parsebail("unknown tt value `$tt'");
	}

	if (!defined $ll_values{$ll}) {
		&parsebail("unknown ll value `$ll'");
	}

	my @r4 = &field_burst($r4, \%r4_values, "r4", "AO_MCA_R4_BIT");

	my @pp = ($pp eq '-') ? () :
	    &field_burst($pp, \%pp_values, "pp", "AO_MCA_PP_BIT");

	if (!defined $t_values{$t}) {
		&parsebail("unknown t value `$t'");
	}

	my @ii = ($ii eq '-') ? () :
	    &field_burst($ii, \%ii_values, "ii", "AO_MCA_II_BIT");

	map {
		tr/[a-z]/[A-Z]/;
	} ($ii, $ll, $tt);

	if ($type eq "bus") {
		if ($pp eq "-" || $t eq "-" || $r4 eq "-" || $ii eq "-" ||
		    $ll eq "-" ||
		    $tt ne "-") {
			&parsebail("invalid members for bus code type");
		}

		$::codeoutlen += &errout_N($instance, "\tAMD_ERRCODE_MKBUS(" .
		    "0, " . # pp
		    "MCAX86_ERRCODE_T_" . ($t ? "TIMEOUT" : "NONE") . ", " .
		    "0, " . # r4
		    "0, " . # ii
		    "MCAX86_ERRCODE_LL_$ll),\n");

	} elsif ($type eq "mem") {
		if ($r4 eq "-" || $tt eq "-" || $ll eq "-" ||
		    $pp ne "-" || $t ne "-" || $ii ne "-") {
			&parsebail("invalid members for mem code type");
		}

		$::codeoutlen += &errout_N($instance, "\tAMD_ERRCODE_MKMEM(" .
		    "0, " . # r4
		    "MCAX86_ERRCODE_TT_$tt, " .
		    "MCAX86_ERRCODE_LL_$ll),\n");

	} elsif ($type eq "tlb") {
		if ($tt eq "-" || $ll eq "-" ||
		    $r4 ne "-" || $pp ne "-" || $t ne "-" || $ii ne "-") {
			&parsebail("invalid members for tlb code type");
		}

		$::codeoutlen += &errout_N($instance, "\tAMD_ERRCODE_MKTLB(" .
		    "MCAX86_ERRCODE_TT_$tt, " .
		    "MCAX86_ERRCODE_LL_$ll),\n");
	} else {
		&parsebail("unknown code type `$type'");
	}

	$::codeoutlen += &errout_N($instance, "\t" . &bin2dec($ext) .
	    ", /* ext code $ext */\n");

	$::codeoutlen += &errout_N($instance, &print_bits("pp_bits", @pp));
	$::codeoutlen += &errout_N($instance, &print_bits("ii_bits", @ii));
	$::codeoutlen += &errout_N($instance, &print_bits("r4_bits", @r4));

	my $valid_hi;
	my $valid_lo;

	if ($addr eq "none") {
		$valid_hi = $valid_lo = 0;
	} elsif ($addr =~ /<(\d+):(\d+)>/) {
		$valid_hi = $1;
		$valid_lo = $2;
	} else {
		&parsebail("invalid addr specification");
	}
	$::codeoutlen += &errout_N($instance, "\t" . $valid_hi .
	    ", /* addr valid hi */\n");
	$::codeoutlen += &errout_N($instance, "\t" . $valid_lo .
	    ", /* addr valid lo */\n");
}

sub state_panic() {
	my @vals = split(/,\s*/, $_[0]);

	if ($#vals < 0) {
		&errout("\t0, /* panic_when */\n");
	} else {
		@vals = map { tr/[a-z]/[A-Z]/; "AO_AED_PANIC_" . $_; } @vals;
		&errout(&print_bits("panic_when", @vals));
	}
}

sub state_flags() {
	my @flags = split(/,\s*/, $_[0]);

	@flags = map { tr/[a-z]/[A-Z]/; "AO_AED_F_" . $_; } @flags;

	&errout(&print_bits("flags", @flags));
}

sub state_errtype() {
	my @types = split(/,\s*/, $_[0]);

	@types = map { tr/[a-z]/[A-Z]/; "AO_AED_ET_" . $_; } @types;

	&errout(&print_bits("errtype", @types));
}

my %stateparse = (
	funcunit	=> [ \&state_funcunit, 'desc' ],
	desc		=> [ \&state_desc, 'error' ],
	error		=> [ \&state_error, 'mask on' ],
	'mask on'	=> [ \&state_mask_on, 'mask off' ],
	'mask off'	=> [ \&state_mask_off, 'code' ],
	code		=> [ \&state_code, 'code|panic' ],
	panic		=> [ \&state_panic, 'flags' ],
	flags		=> [ \&state_flags, 'errtype' ],
	errtype		=> [ \&state_errtype, 'initial' ]
);

usage unless (@ARGV == 1);

my $infile = $ARGV[0];
open(INFILE, "<$infile") || &bail("failed to open $infile: $!");

&print_header();

while (<INFILE>) {
	chop;

	/^#/ && next;
	/^$/ && next;

	if (!/^\s*(\S[^=]*\S)\s*=\s*(\S.*)?$/) {
		&parsebail("failed to parse");
	}

	my ($keyword, $val) = ($1, $2);

	if ($state eq "initial") {
		if ($keyword eq "funcunit") {
			$state = "funcunit";
		} elsif ($keyword eq "desc") {
			$state = "desc";
		} else {
			&parsebail("unexpected keyword $keyword between " .
			    "errors");
		}

	} elsif ($state eq "desc") {
		if ($keyword eq "funcunit") {
			$state = "funcunit";
		}
	}

	if (!($keyword =~ /$state/)) {
		&parsebail("keyword `$keyword' invalid here; expected " .
		    "`$state'");
	}
	$state = $keyword;	# disambiguate between multiple legal states

	if (!defined $stateparse{$state}) {
		&parsebail("attempt to transition to invalid state `$state'");
	}

	my ($handler, $next) = @{$stateparse{$state}};

	&{$handler}($val);

	$state = $next;

	if ($state eq "initial") {
		&error_end();
	}
}

close(INFILE);

if ($state ne "initial" && $state ne "desc") {
	&bail("input file ended prematurely");
}

if (defined $::funcunit) {
	&funcunit_end();
} else {
	&bail("no functional units defined");
}

&print_footer;