generic_cpu - zygaena/hammerhead

root / base / usr / src / uts / i86pc / cpu / generic_cpu
generic_cpu Plain Text 3196 lines 87.1 KB
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright (c) 2006, 2010, Oracle and/or its affiliates. All rights reserved.
 * Copyright (c) 2018, Joyent, Inc.
 */

#ifndef _GCPU_H
#define	_GCPU_H

#include <sys/types.h>
#include <sys/cpu_module_impl.h>
#include <sys/cpu_module_ms.h>
#include <sys/ksynch.h>
#include <sys/systm.h>
#include <sys/fm/util.h>

#ifdef __cplusplus
extern "C" {
#endif

#define	GCPU_MCA_ERRS_PERCPU	10	/* errorq slots per cpu */
#define	GCPU_MCA_MIN_ERRORS	30	/* minimum total errorq slots */
#define	GCPU_MCA_MAX_ERRORS	100	/* maximum total errorq slots */

typedef struct gcpu_data gcpu_data_t;

#define	GCPU_ERRCODE_MASK_ALL		0xffff

typedef struct gcpu_error_disp {
	const char *ged_class_fmt;	/* ereport class formatter (last bit) */
	const char *ged_compound_fmt;	/* compound error formatter */
	uint64_t ged_ereport_members;	/* ereport payload members */
	uint16_t ged_errcode_mask_on;	/* errcode bits that must be set ... */
	uint16_t ged_errcode_mask_off;	/* ... and must be clear for a match */
} gcpu_error_disp_t;

/*
 * For errorq_dispatch we need to have a single contiguous structure
 * capturing all our logout data.  We do not know in advance how many
 * error detector banks there are in this cpu model, so we'll manually
 * allocate additional space for the gcl_banks array below.
 */
typedef struct gcpu_bank_logout {
	uint64_t gbl_status;		/* MCi_STATUS value */
	uint64_t gbl_addr;		/* MCi_ADDR value */
	uint64_t gbl_misc;		/* MCi_MISC value */
	uint64_t gbl_disp;		/* Error disposition for this bank */
	uint32_t gbl_clrdefcnt;		/* Count of deferred status clears */
} gcpu_bank_logout_t;

/*
 * The data structure we "logout" all error telemetry from all banks of
 * a cpu to.  The gcl_data array declared with 1 member below will actually
 * have gcl_nbanks members - variable with the actual cpu model present.
 * After the gcl_data array there is a further model-specific array that
 * may be allocated, and gcl_ms_logout will point to that if present.
 * This cpu logout data must form one contiguous chunk of memory for
 * dispatch with errorq_dispatch.
 */
typedef struct gcpu_logout {
	gcpu_data_t *gcl_gcpu;		/* pointer to per-cpu gcpu_data_t */
	uintptr_t gcl_ip;		/* instruction pointer from #mc trap */
	uint64_t gcl_timestamp;		/* gethrtime() at logout */
	uint64_t gcl_mcg_status;	/* MCG_STATUS register value */
	uint64_t gcl_flags;		/* Flags */
	pc_t gcl_stack[FM_STK_DEPTH];	/* saved stack trace, if any */
	int gcl_stackdepth;		/* saved stack trace depth */
	int ismc;			/* is a machine check flag */
	int gcl_nbanks;			/* number of banks in array below */
	void *gcl_ms_logout;		/* Model-specific area after gcl_data */
	gcpu_bank_logout_t gcl_data[1];	/* Bank logout areas - must be last */
} gcpu_logout_t;

/*
 * gcl_flag values
 */
#define	GCPU_GCL_F_PRIV		0x1	/* #MC during privileged code */
#define	GCPU_GCL_F_TES_P	0x2	/* MCG_CAP indicates TES_P */

struct gcpu_bios_bankcfg {
	uint64_t bios_bank_ctl;
	uint64_t bios_bank_status;
	uint64_t bios_bank_addr;
	uint64_t bios_bank_misc;
};

struct gcpu_bios_cfg {
	uint64_t bios_mcg_cap;
	uint64_t bios_mcg_ctl;
	struct gcpu_bios_bankcfg *bios_bankcfg;
};

/*
 * Events types in poll trace records.  Keep these in sync with
 * the generic cpu mdb module names for each (see gcpu_mpt_dump in mdb).
 */
#define	GCPU_MPT_WHAT_CYC_ERR		0	/* cyclic-induced poll */
#define	GCPU_MPT_WHAT_POKE_ERR		1	/* manually-induced poll */
#define	GCPU_MPT_WHAT_UNFAULTING	2	/* discarded error state */
#define	GCPU_MPT_WHAT_MC_ERR		3	/* MC# */
#define	GCPU_MPT_WHAT_CMCI_ERR		4	/* CMCI interrupt */
#define	GCPU_MPT_WHAT_XPV_VIRQ		5	/* MCA_VIRQ in dom0 */
#define	GCPU_MPT_WHAT_XPV_VIRQ_LOGOUT	6	/* MCA_VIRQ logout complete */

typedef struct gcpu_poll_trace {
	hrtime_t mpt_when;		/* timestamp of event */
	uint8_t mpt_what;		/* GCPU_MPT_WHAT_* (which event?) */
	uint8_t mpt_nerr;		/* number of errors discovered */
	uint16_t mpt_pad1;
	uint32_t mpt_pad2;
} gcpu_poll_trace_t;

typedef struct gcpu_poll_trace_ctl {
	gcpu_poll_trace_t *mptc_tbufs;	/* trace buffers */
	uint_t mptc_curtrace;			/* last buffer filled */
} gcpu_poll_trace_ctl_t;


/*
 * For counting some of the important number or time for runtime
 * cmci enable/disable
 */
typedef struct gcpu_mca_cmci {
	uint32_t cmci_cap;	/* cmci capability for this bank */
	uint32_t ncmci;		/* number of correctable errors between polls */
	uint32_t drtcmci;	/* duration of no cmci when cmci is disabled */
	uint32_t cmci_enabled;	/* cmci enable/disable status for this bank */
} gcpu_mca_cmci_t;

/* Index for gcpu_mca_logout array below */
#define	GCPU_MCA_LOGOUT_EXCEPTION	0	/* area for #MC */
#define	GCPU_MCA_LOGOUT_POLLER_1	1	/* next/prev poll area */
#define	GCPU_MCA_LOGOUT_POLLER_2	2	/* prev/next poll area */
#define	GCPU_MCA_LOGOUT_NUM		3

typedef struct gcpu_mca {
	gcpu_logout_t *gcpu_mca_logout[GCPU_MCA_LOGOUT_NUM];
	uint32_t gcpu_mca_nextpoll_idx;	/* logout area for next poll */
	struct gcpu_bios_cfg gcpu_mca_bioscfg;
	uint_t gcpu_mca_nbanks;
	size_t gcpu_mca_lgsz;		/* size of gcpu_mca_logout structs */
	uint_t gcpu_mca_flags;		/* GCPU_MCA_F_* */
	hrtime_t gcpu_mca_lastpoll;
	gcpu_poll_trace_ctl_t gcpu_polltrace;
	uint32_t gcpu_mca_first_poll_cmci_enabled; /* cmci on in first poll */
	gcpu_mca_cmci_t *gcpu_bank_cmci;
} gcpu_mca_t;

typedef struct gcpu_mce_status {
	uint_t mce_nerr;	/* total errors found in logout of all banks */
	uint64_t mce_disp;	/* Disposition information */
	uint_t mce_npcc;	/* number of errors with PCC */
	uint_t mce_npcc_ok;	/* PCC with CMS_ERRSCOPE_CURCONTEXT_OK */
	uint_t mce_nuc;		/* number of errors with UC */
	uint_t mce_nuc_ok;	/* UC with CMS_ERRSCOPE_CLEARED_UC */
	uint_t mce_nuc_poisoned; /* UC with CMS_ERRSCOPE_POISONED */
	uint_t mce_forcefatal;	/* CMS_ERRSCOPE_FORCE_FATAL */
	uint_t mce_ignored;	/* CMS_ERRSCOPE_IGNORE_ERR */
} gcpu_mce_status_t;

/*
 * Flags for gcpu_mca_flags
 */
#define	GCPU_MCA_F_UNFAULTING		0x1	/* CPU exiting faulted state */
#define	GCPU_MCA_F_CMCI_CAPABLE		0x2	/* CPU supports CMCI */
#define	GCPU_MCA_F_CMCI_ENABLE		0x4	/* CPU CMCI enabled */

/*
 * State shared by all cpus on a chip
 */
struct gcpu_chipshared {
	kmutex_t gcpus_cfglock;		/* serial MCA config from chip cores */
	kmutex_t gcpus_poll_lock;	/* serialize pollers on the same chip */
	uint32_t gcpus_actv_banks;	/* MCA bank numbers active on chip */
	volatile uint32_t gcpus_actv_cnt; /* active cpu count in this chip */
	char *gcpus_ident;		/* ident string, if available */
};

struct gcpu_data {
	gcpu_mca_t gcpu_mca;			/* MCA state for this CPU */
	cmi_hdl_t gcpu_hdl;			/* associated handle */
	struct gcpu_chipshared *gcpu_shared;	/* Shared state for the chip */
};

#ifdef _KERNEL

struct regs;

/*
 * CMI implementation
 */
extern int gcpu_init(cmi_hdl_t, void **);
extern void gcpu_fini(cmi_hdl_t);
extern void gcpu_post_startup(cmi_hdl_t);
extern void gcpu_post_mpstartup(cmi_hdl_t);
extern void gcpu_faulted_enter(cmi_hdl_t);
extern void gcpu_faulted_exit(cmi_hdl_t);
extern void gcpu_mca_init(cmi_hdl_t);
extern void gcpu_mca_fini(cmi_hdl_t);
extern void gcpu_mca_cmci_enable(cmi_hdl_t);
extern cmi_errno_t gcpu_msrinject(cmi_hdl_t, cmi_mca_regs_t *, uint_t, int);
#ifndef __xpv
extern uint64_t gcpu_mca_trap(cmi_hdl_t, struct regs *);
extern void gcpu_cmci_trap(cmi_hdl_t);
extern void gcpu_hdl_poke(cmi_hdl_t);
#else
extern void gcpu_xpv_panic_callback(void);
#endif

/*
 * Local functions
 */
extern void gcpu_mca_poll_init(cmi_hdl_t);
extern void gcpu_mca_poll_fini(cmi_hdl_t);
extern void gcpu_mca_poll_start(cmi_hdl_t);
extern void gcpu_poll_trace_init(gcpu_poll_trace_ctl_t *);
extern void gcpu_poll_trace(gcpu_poll_trace_ctl_t *, uint8_t, uint8_t);
extern void gcpu_mca_logout(cmi_hdl_t, struct regs *, uint64_t,
    gcpu_mce_status_t *, boolean_t, int);
#ifdef __xpv
extern void gcpu_xpv_mca_init(int);
#endif /* __xpv */

#endif /* _KERNEL */

#ifdef __cplusplus
}
#endif

#endif /* _GCPU_H */
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 * Copyright (c) 2018, Joyent, Inc.
 */
/*
 * Copyright (c) 2010, Intel Corporation.
 * All rights reserved.
 */

/*
 * Copyright (c) 2018, Joyent, Inc.
 * Copyright 2020 RackTop Systems, Inc.
 */

/*
 * Generic x86 CPU Module
 *
 * This CPU module is used for generic x86 CPUs when Solaris has no other
 * CPU-specific support module available.  Code in this module should be the
 * absolute bare-bones support and must be cognizant of both Intel and AMD etc.
 */

#include <sys/types.h>
#include <sys/cpu_module_impl.h>
#include <sys/cpuvar.h>
#include <sys/kmem.h>
#include <sys/modctl.h>
#include <sys/pghw.h>
#include <sys/x86_archext.h>

#include "gcpu.h"

/*
 * Prevent generic cpu support from loading.
 */
int gcpu_disable = 0;

#define	GCPU_MAX_CHIPID		32
static struct gcpu_chipshared *gcpu_shared[GCPU_MAX_CHIPID];
#ifdef	DEBUG
int gcpu_id_disable = 0;
static const char *gcpu_id_override[GCPU_MAX_CHIPID] = { NULL };
#endif

#ifndef	__xpv

/*
 * The purpose of this is to construct a unique identifier for a given processor
 * that can be used by things like FMA to determine when a FRU has been
 * replaced. It is supported on Intel Xeon Platforms since Ivy Bridge and AMD
 * 17h processors since Rome. See cpuid_pass1_ppin() for how we determine if a
 * CPU is supported.
 *
 * The protected processor inventory number (PPIN) can be used to create a
 * unique identifier when combined with the processor's cpuid signature. We
 * create a versioned, synthetic ID using the following scheme for the
 * identifier: iv0-<vendor>-<signature>-<PPIN>. The iv0 is the illumos version
 * zero of the ID. If we have a new scheme for a new generation of processors,
 * then that should rev the version field, otherwise for a given processor, this
 * synthetic ID should not change.
 *
 * We use the string "INTC" for Intel and "AMD" for AMD. None of these or the
 * formatting of the values can change without changing the version string.
 */
static char *
gcpu_init_ident_ppin(cmi_hdl_t hdl)
{
	uint_t ppin_ctl_msr, ppin_msr;
	uint64_t value;
	const char *vendor;

	/*
	 * This list should be extended as new Intel Xeon family processors come
	 * out.
	 */
	switch (cmi_hdl_vendor(hdl)) {
	case X86_VENDOR_Intel:
		ppin_ctl_msr = MSR_PPIN_CTL_INTC;
		ppin_msr = MSR_PPIN_INTC;
		vendor = "INTC";
		break;
	case X86_VENDOR_AMD:
		ppin_ctl_msr = MSR_PPIN_CTL_AMD;
		ppin_msr = MSR_PPIN_AMD;
		vendor = "AMD";
		break;
	default:
		return (NULL);
	}

	if (cmi_hdl_rdmsr(hdl, ppin_ctl_msr, &value) != CMI_SUCCESS) {
		return (NULL);
	}

	/*
	 * If the PPIN is not enabled and not locked, attempt to enable it.
	 * Note: in some environments such as Amazon EC2 the PPIN appears
	 * to be disabled and unlocked but our attempts to enable it don't
	 * stick, and when we attempt to read the PPIN we get an uncaught
	 * #GP. To avoid that happening we read the MSR back and verify it
	 * has taken the new value.
	 */
	if ((value & MSR_PPIN_CTL_ENABLED) == 0) {
		if ((value & MSR_PPIN_CTL_LOCKED) != 0) {
			return (NULL);
		}

		if (cmi_hdl_wrmsr(hdl, ppin_ctl_msr, MSR_PPIN_CTL_ENABLED) !=
		    CMI_SUCCESS) {
			return (NULL);
		}

		if (cmi_hdl_rdmsr(hdl, ppin_ctl_msr, &value) != CMI_SUCCESS) {
			return (NULL);
		}

		if ((value & MSR_PPIN_CTL_ENABLED) == 0) {
			return (NULL);
		}
	}

	if (cmi_hdl_rdmsr(hdl, ppin_msr, &value) != CMI_SUCCESS) {
		return (NULL);
	}

	/*
	 * Now that we've read data, lock the PPIN. Don't worry about success or
	 * failure of this part, as we will have gotten everything that we need.
	 * It is possible that it locked open, for example.
	 */
	if (cmi_hdl_wrmsr(hdl, ppin_ctl_msr, MSR_PPIN_CTL_DISABLED) ==
	    CMI_SUCCESS) {
		(void) cmi_hdl_wrmsr(hdl, ppin_ctl_msr, MSR_PPIN_CTL_LOCKED);
	}

	return (kmem_asprintf("iv0-%s-%x-%llx", vendor, cmi_hdl_chipsig(hdl),
	    value));
}
#endif	/* __xpv */

static void
gcpu_init_ident(cmi_hdl_t hdl, struct gcpu_chipshared *sp)
{
#ifdef	DEBUG
	uint_t chipid;

	/*
	 * On debug, allow a developer to override the string to more
	 * easily test CPU autoreplace without needing to physically
	 * replace a CPU.
	 */
	if (gcpu_id_disable != 0) {
		return;
	}

	chipid = cmi_hdl_chipid(hdl);
	if (gcpu_id_override[chipid] != NULL) {
		sp->gcpus_ident = strdup(gcpu_id_override[chipid]);
		return;
	}
#endif

#ifndef __xpv
	if (is_x86_feature(x86_featureset, X86FSET_PPIN)) {
		sp->gcpus_ident = gcpu_init_ident_ppin(hdl);
	}
#endif	/* __xpv */
}

/*
 * Our cmi_init entry point, called during startup of each cpu instance.
 */
int
gcpu_init(cmi_hdl_t hdl, void **datap)
{
	uint_t chipid = cmi_hdl_chipid(hdl);
	struct gcpu_chipshared *sp, *osp;
	gcpu_data_t *gcpu;

	if (gcpu_disable || chipid >= GCPU_MAX_CHIPID)
		return (ENOTSUP);

	/*
	 * Allocate the state structure for this cpu.  We will only
	 * allocate the bank logout areas in gcpu_mca_init once we
	 * know how many banks there are.
	 */
	gcpu = *datap = kmem_zalloc(sizeof (gcpu_data_t), KM_SLEEP);
	cmi_hdl_hold(hdl);	/* release in gcpu_fini */
	gcpu->gcpu_hdl = hdl;

	/*
	 * Allocate a chipshared structure if no sibling cpu has already
	 * allocated it, but allow for the fact that a sibling core may
	 * be starting up in parallel.
	 */
	if ((sp = gcpu_shared[chipid]) == NULL) {
		sp = kmem_zalloc(sizeof (struct gcpu_chipshared), KM_SLEEP);
		mutex_init(&sp->gcpus_poll_lock, NULL, MUTEX_DRIVER, NULL);
		mutex_init(&sp->gcpus_cfglock, NULL, MUTEX_DRIVER, NULL);
		osp = atomic_cas_ptr(&gcpu_shared[chipid], NULL, sp);
		if (osp != NULL) {
			mutex_destroy(&sp->gcpus_cfglock);
			mutex_destroy(&sp->gcpus_poll_lock);
			kmem_free(sp, sizeof (struct gcpu_chipshared));
			sp = osp;
		} else {
			gcpu_init_ident(hdl, sp);
		}
	}

	atomic_inc_32(&sp->gcpus_actv_cnt);
	gcpu->gcpu_shared = sp;

	return (0);
}

/*
 * deconfigure gcpu_init()
 */
void
gcpu_fini(cmi_hdl_t hdl)
{
	uint_t chipid = cmi_hdl_chipid(hdl);
	gcpu_data_t *gcpu = cmi_hdl_getcmidata(hdl);
	struct gcpu_chipshared *sp;

	if (gcpu_disable || chipid >= GCPU_MAX_CHIPID)
		return;

	gcpu_mca_fini(hdl);

	/*
	 * Keep shared data in cache for reuse.
	 */
	sp = gcpu_shared[chipid];
	ASSERT(sp != NULL);
	atomic_dec_32(&sp->gcpus_actv_cnt);

	if (gcpu != NULL)
		kmem_free(gcpu, sizeof (gcpu_data_t));

	/* Release reference count held in gcpu_init(). */
	cmi_hdl_rele(hdl);
}

void
gcpu_post_startup(cmi_hdl_t hdl)
{
	gcpu_data_t *gcpu = cmi_hdl_getcmidata(hdl);

	if (gcpu_disable)
		return;

	if (gcpu != NULL)
		cms_post_startup(hdl);
#ifdef __xpv
	/*
	 * All cpu handles are initialized so we can begin polling now.
	 * Furthermore, our virq mechanism requires that everything
	 * be run on cpu 0 so we can assure that by starting from here.
	 */
	gcpu_mca_poll_start(hdl);
#else
	/*
	 * The boot CPU has a bit of a chicken and egg problem for CMCI. Its MCA
	 * initialization is run before we have initialized the PSM module that
	 * we would use for enabling CMCI. Therefore, we use this as a chance to
	 * enable CMCI for the boot CPU. For all other CPUs, this chicken and
	 * egg problem will have already been solved.
	 */
	gcpu_mca_cmci_enable(hdl);
#endif
}

void
gcpu_post_mpstartup(cmi_hdl_t hdl)
{
	if (gcpu_disable)
		return;

	cms_post_mpstartup(hdl);

#ifndef __xpv
	/*
	 * All cpu handles are initialized only once all cpus are started, so we
	 * can begin polling post mp startup.
	 */
	gcpu_mca_poll_start(hdl);
#endif
}

const char *
gcpu_ident(cmi_hdl_t hdl)
{
	uint_t chipid;
	struct gcpu_chipshared *sp;

	if (gcpu_disable)
		return (NULL);

	chipid = cmi_hdl_chipid(hdl);
	if (chipid >= GCPU_MAX_CHIPID)
		return (NULL);

	if (cmi_hdl_getcmidata(hdl) == NULL)
		return (NULL);

	sp = gcpu_shared[cmi_hdl_chipid(hdl)];
	return (sp->gcpus_ident);
}

#ifdef __xpv
#define	GCPU_OP(ntvop, xpvop)	xpvop
#else
#define	GCPU_OP(ntvop, xpvop)	ntvop
#endif

cmi_api_ver_t _cmi_api_version = CMI_API_VERSION_3;

const cmi_ops_t _cmi_ops = {
	gcpu_init,				/* cmi_init */
	gcpu_post_startup,			/* cmi_post_startup */
	gcpu_post_mpstartup,			/* cmi_post_mpstartup */
	gcpu_faulted_enter,			/* cmi_faulted_enter */
	gcpu_faulted_exit,			/* cmi_faulted_exit */
	gcpu_mca_init,				/* cmi_mca_init */
	GCPU_OP(gcpu_mca_trap, NULL),		/* cmi_mca_trap */
	GCPU_OP(gcpu_cmci_trap, NULL),		/* cmi_cmci_trap */
	gcpu_msrinject,				/* cmi_msrinject */
	GCPU_OP(gcpu_hdl_poke, NULL),		/* cmi_hdl_poke */
	gcpu_fini,				/* cmi_fini */
	GCPU_OP(NULL, gcpu_xpv_panic_callback),	/* cmi_panic_callback */
	gcpu_ident				/* cmi_ident */
};

static struct modlcpu modlcpu = {
	&mod_cpuops,
	"Generic x86 CPU Module"
};

static struct modlinkage modlinkage = {
	MODREV_1,
	(void *)&modlcpu,
	NULL
};

int
_init(void)
{
	return (mod_install(&modlinkage));
}

int
_info(struct modinfo *modinfop)
{
	return (mod_info(&modlinkage, modinfop));
}

int
_fini(void)
{
	return (mod_remove(&modlinkage));
}
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright (c) 2006, 2010, Oracle and/or its affiliates. All rights reserved.
 * Copyright (c) 2018, Joyent, Inc.
 * Copyright 2022 Oxide Computer Co.
 */
/*
 * Copyright (c) 2010, Intel Corporation.
 * All rights reserved.
 */

#include <sys/mca_x86.h>
#include <sys/cpu_module_impl.h>
#include <sys/cpu_module_ms.h>
#include <sys/cmn_err.h>
#include <sys/cpuvar.h>
#include <sys/pghw.h>
#include <sys/x86_archext.h>
#include <sys/sysmacros.h>
#include <sys/regset.h>
#include <sys/privregs.h>
#include <sys/systm.h>
#include <sys/types.h>
#include <sys/log.h>
#include <sys/psw.h>
#include <sys/fm/protocol.h>
#include <sys/fm/util.h>
#include <sys/errorq.h>
#include <sys/mca_x86.h>
#include <sys/fm/cpu/GMCA.h>
#include <sys/fm/smb/fmsmb.h>
#include <sys/sysevent.h>
#include <sys/ontrap.h>
#include <sys/smp_impldefs.h>

#include "gcpu.h"

extern int x86gentopo_legacy;	/* x86 generic topology support */

static uint_t gcpu_force_addr_in_payload = 0;

/*
 * Clear to log telemetry found at initialization.  While processor docs
 * say you should process this telemetry on all but Intel family 0x6
 * there are way too many exceptions and we want to avoid bogus
 * diagnoses.
 */
int gcpu_suppress_log_on_init = 1;

/*
 * gcpu_mca_stack_flag is a debug assist option to capture a stack trace at
 * error logout time.  The stack will be included in the ereport if the
 * error type selects stack inclusion, or in all cases if
 * gcpu_mca_stack_ereport_include is nonzero.
 */
int gcpu_mca_stack_flag = 0;
int gcpu_mca_stack_ereport_include = 0;

/*
 * The number of times to re-read MCA telemetry to try to obtain a
 * consistent snapshot if we find it to be changing under our feet.
 */
int gcpu_mca_telemetry_retries = 5;

#ifndef __xpv
int gcpu_mca_cmci_throttling_threshold = 10;
int gcpu_mca_cmci_reenable_threshold = 1000;

/*
 * This is used to determine whether or not we have registered the CMCI CPU
 * setup function. This is protected by cpu_lock.
 */
static boolean_t gcpu_mca_cpu_registered = B_FALSE;
#endif

static gcpu_error_disp_t gcpu_errtypes[] = {

	/*
	 * Unclassified
	 */
	{
		FM_EREPORT_CPU_GENERIC_UNCLASSIFIED,
		NULL,
		FM_EREPORT_PAYLOAD_FLAGS_COMMON,
		MCAX86_SIMPLE_UNCLASSIFIED_MASKON,
		MCAX86_SIMPLE_UNCLASSIFIED_MASKOFF
	},

	/*
	 * Microcode ROM Parity Error
	 */
	{
		FM_EREPORT_CPU_GENERIC_MC_CODE_PARITY,
		NULL,
		FM_EREPORT_PAYLOAD_FLAGS_COMMON,
		MCAX86_SIMPLE_MC_CODE_PARITY_MASKON,
		MCAX86_SIMPLE_MC_CODE_PARITY_MASKOFF
	},

	/*
	 * External - BINIT# from another processor during power-on config
	 */
	{
		FM_EREPORT_CPU_GENERIC_EXTERNAL,
		NULL,
		FM_EREPORT_PAYLOAD_FLAGS_COMMON,
		MCAX86_SIMPLE_EXTERNAL_MASKON,
		MCAX86_SIMPLE_EXTERNAL_MASKOFF
	},

	/*
	 * Functional redundancy check master/slave error
	 */
	{
		FM_EREPORT_CPU_GENERIC_FRC,
		NULL,
		FM_EREPORT_PAYLOAD_FLAGS_COMMON,
		MCAX86_SIMPLE_FRC_MASKON,
		MCAX86_SIMPLE_FRC_MASKOFF
	},

	/*
	 * Internal parity error
	 */
	{
		FM_EREPORT_CPU_GENERIC_INTERNAL_PARITY,
		NULL,
		FM_EREPORT_PAYLOAD_FLAGS_COMMON,
		MCAX86_SIMPLE_INTERNAL_PARITY_MASKON,
		MCAX86_SIMPLE_INTERNAL_PARITY_MASKOFF
	},


	/*
	 * Internal timer error
	 */
	{
		FM_EREPORT_CPU_GENERIC_INTERNAL_TIMER,
		NULL,
		FM_EREPORT_PAYLOAD_FLAGS_COMMON,
		MCAX86_SIMPLE_INTERNAL_TIMER_MASKON,
		MCAX86_SIMPLE_INTERNAL_TIMER_MASKOFF
	},

	/*
	 * Internal unclassified
	 */
	{
		FM_EREPORT_CPU_GENERIC_INTERNAL_UNCLASS,
		NULL,
		FM_EREPORT_PAYLOAD_FLAGS_COMMON,
		MCAX86_SIMPLE_INTERNAL_UNCLASS_MASK_MASKON,
		MCAX86_SIMPLE_INTERNAL_UNCLASS_MASK_MASKOFF
	},

	/*
	 * Compound error codes - generic memory hierarchy
	 */
	{
		FM_EREPORT_CPU_GENERIC_GENMEMHIER,
		NULL,
		FM_EREPORT_PAYLOAD_FLAGS_COMMON, /* yes, no compound name */
		MCAX86_COMPOUND_GENERIC_MEMHIER_MASKON,
		MCAX86_COMPOUND_GENERIC_MEMHIER_MASKOFF
	},

	/*
	 * Compound error codes - TLB errors
	 */
	{
		FM_EREPORT_CPU_GENERIC_TLB,
		"%1$s" "TLB" "%2$s" "_ERR",
		FM_EREPORT_PAYLOAD_FLAGS_COMPOUND_ERR,
		MCAX86_COMPOUND_TLB_MASKON,
		MCAX86_COMPOUND_TLB_MASKOFF
	},

	/*
	 * Compound error codes - memory hierarchy
	 */
	{
		FM_EREPORT_CPU_GENERIC_MEMHIER,
		"%1$s" "CACHE" "%2$s" "_" "%3$s" "_ERR",
		FM_EREPORT_PAYLOAD_FLAGS_COMPOUND_ERR,
		MCAX86_COMPOUND_MEMHIER_MASKON,
		MCAX86_COMPOUND_MEMHIER_MASKOFF
	},

	/*
	 * Compound error codes - bus and interconnect errors
	 */
	{
		FM_EREPORT_CPU_GENERIC_BUS_INTERCONNECT,
		"BUS" "%2$s" "_" "%4$s" "_" "%3$s" "_" "%5$s" "_" "%6$s" "_ERR",
		FM_EREPORT_PAYLOAD_FLAGS_COMPOUND_ERR,
		MCAX86_COMPOUND_BUS_INTERCONNECT_MASKON,
		MCAX86_COMPOUND_BUS_INTERCONNECT_MASKOFF
	},
	/*
	 * Compound error codes - memory controller errors
	 */
	{
		FM_EREPORT_CPU_GENERIC_MEMORY_CONTROLLER,
		"MC" "_" "%8$s" "_" "%9$s" "_ERR",
		FM_EREPORT_PAYLOAD_FLAGS_COMPOUND_ERR,
		MCAX86_COMPOUND_MEMORY_CONTROLLER_MASKON,
		MCAX86_COMPOUND_MEMORY_CONTROLLER_MASKOFF
	},
};

static gcpu_error_disp_t gcpu_unknown = {
	FM_EREPORT_CPU_GENERIC_UNKNOWN,
	"UNKNOWN",
	FM_EREPORT_PAYLOAD_FLAGS_COMMON,
	0,
	0
};

static errorq_t *gcpu_mca_queue;
static kmutex_t gcpu_mca_queue_lock;

#ifdef __xpv
static int isxpv = 1;
#else
static int isxpv = 0;
#endif

static const gcpu_error_disp_t *
gcpu_disp_match(uint16_t code)
{
	const gcpu_error_disp_t *ged = gcpu_errtypes;
	int i;

	for (i = 0; i < sizeof (gcpu_errtypes) / sizeof (gcpu_error_disp_t);
	    i++, ged++) {
		uint16_t on = ged->ged_errcode_mask_on;
		uint16_t off = ged->ged_errcode_mask_off;

		if ((code & on) == on && (code & off) == 0)
			return (ged);
	}

	return (NULL);
}

static uint16_t
bit_strip(uint16_t code, uint16_t mask, uint16_t shift)
{
	return ((code & mask) >> shift);
}

#define	BIT_STRIP(code, name) \
	bit_strip(code, MCAX86_ERRCODE_##name##_MASK, \
	MCAX86_ERRCODE_##name##_SHIFT)

#define	GCPU_MNEMONIC_UNDEF	"undefined"
#define	GCPU_MNEMONIC_RESVD	"reserved"

/*
 * Mappings of TT, LL, RRRR, PP, II and T values to compound error name
 * mnemonics and to ereport class name components.
 */

struct gcpu_mnexp {
	const char *mne_compound;	/* used in expanding compound errname */
	const char *mne_ereport;	/* used in expanding ereport class */
};

static struct gcpu_mnexp gcpu_TT_mnemonics[] = { /* MCAX86_ERRCODE_TT_* */
	{ "I", FM_EREPORT_CPU_GENERIC_TT_INSTR },		/* INSTR */
	{ "D", FM_EREPORT_CPU_GENERIC_TT_DATA },		/* DATA */
	{ "G", FM_EREPORT_CPU_GENERIC_TT_GEN },			/* GEN */
	{ GCPU_MNEMONIC_UNDEF, "" }
};

static struct gcpu_mnexp gcpu_LL_mnemonics[] = { /* MCAX86_ERRCODE_LL_* */
	{ "LO", FM_EREPORT_CPU_GENERIC_LL_L0 },			/* L0 */
	{ "L1",	FM_EREPORT_CPU_GENERIC_LL_L1 },			/* L1 */
	{ "L2",	FM_EREPORT_CPU_GENERIC_LL_L2 },			/* L2 */
	{ "LG", FM_EREPORT_CPU_GENERIC_LL_LG }			/* LG */
};

static struct gcpu_mnexp gcpu_RRRR_mnemonics[] = { /* MCAX86_ERRCODE_RRRR_* */
	{ "ERR", FM_EREPORT_CPU_GENERIC_RRRR_ERR },		/* ERR */
	{ "RD",	FM_EREPORT_CPU_GENERIC_RRRR_RD },		/* RD */
	{ "WR", FM_EREPORT_CPU_GENERIC_RRRR_WR },		/* WR */
	{ "DRD", FM_EREPORT_CPU_GENERIC_RRRR_DRD },		/* DRD */
	{ "DWR", FM_EREPORT_CPU_GENERIC_RRRR_DWR },		/* DWR */
	{ "IRD", FM_EREPORT_CPU_GENERIC_RRRR_IRD },		/* IRD */
	{ "PREFETCH", FM_EREPORT_CPU_GENERIC_RRRR_PREFETCH },	/* PREFETCH */
	{ "EVICT", FM_EREPORT_CPU_GENERIC_RRRR_EVICT },		/* EVICT */
	{ "SNOOP", FM_EREPORT_CPU_GENERIC_RRRR_SNOOP },		/* SNOOP */
};

static struct gcpu_mnexp gcpu_PP_mnemonics[] = { /* MCAX86_ERRCODE_PP_* */
	{ "SRC", FM_EREPORT_CPU_GENERIC_PP_SRC },		/* SRC */
	{ "RES", FM_EREPORT_CPU_GENERIC_PP_RES },		/* RES */
	{ "OBS", FM_EREPORT_CPU_GENERIC_PP_OBS },		/* OBS */
	{ "", FM_EREPORT_CPU_GENERIC_PP_GEN }			/* GEN */
};

static struct gcpu_mnexp gcpu_II_mnemonics[] = { /* MCAX86_ERRCODE_II_* */
	{ "M", FM_EREPORT_CPU_GENERIC_II_MEM },			/* MEM */
	{ GCPU_MNEMONIC_RESVD, "" },
	{ "IO", FM_EREPORT_CPU_GENERIC_II_IO },			/* IO */
	{ "", FM_EREPORT_CPU_GENERIC_II_GEN }			/* GEN */
};

static struct gcpu_mnexp gcpu_T_mnemonics[] = {	 /* MCAX86_ERRCODE_T_* */
	{ "NOTIMEOUT", FM_EREPORT_CPU_GENERIC_T_NOTIMEOUT },	/* NONE */
	{ "TIMEOUT", FM_EREPORT_CPU_GENERIC_T_TIMEOUT }		/* TIMEOUT */
};

static struct gcpu_mnexp gcpu_CCCC_mnemonics[] = { /* MCAX86_ERRCODE_CCCC_* */
	{ "CH0", FM_EREPORT_CPU_GENERIC_CCCC },		/* CH0 */
	{ "CH1", FM_EREPORT_CPU_GENERIC_CCCC },		/* CH1 */
	{ "CH2", FM_EREPORT_CPU_GENERIC_CCCC },		/* CH2 */
	{ "CH3", FM_EREPORT_CPU_GENERIC_CCCC },		/* CH3 */
	{ "CH4", FM_EREPORT_CPU_GENERIC_CCCC },		/* CH4 */
	{ "CH5", FM_EREPORT_CPU_GENERIC_CCCC },		/* CH5 */
	{ "CH6", FM_EREPORT_CPU_GENERIC_CCCC },		/* CH6 */
	{ "CH7", FM_EREPORT_CPU_GENERIC_CCCC },		/* CH7 */
	{ "CH8", FM_EREPORT_CPU_GENERIC_CCCC },		/* CH8 */
	{ "CH9", FM_EREPORT_CPU_GENERIC_CCCC },		/* CH9 */
	{ "CH10", FM_EREPORT_CPU_GENERIC_CCCC },	/* CH10 */
	{ "CH11", FM_EREPORT_CPU_GENERIC_CCCC },	/* CH11 */
	{ "CH12", FM_EREPORT_CPU_GENERIC_CCCC },	/* CH12 */
	{ "CH13", FM_EREPORT_CPU_GENERIC_CCCC },	/* CH13 */
	{ "CH14", FM_EREPORT_CPU_GENERIC_CCCC },	/* CH14 */
	{ "CH", FM_EREPORT_CPU_GENERIC_CCCC }		/* GEN */
};

static struct gcpu_mnexp gcpu_MMM_mnemonics[] = { /* MCAX86_ERRCODE_MMM_* */
	{ "GEN", FM_EREPORT_CPU_GENERIC_MMM_ERR },	/* GEN ERR */
	{ "RD", FM_EREPORT_CPU_GENERIC_MMM_RD },	/* READ  */
	{ "WR", FM_EREPORT_CPU_GENERIC_MMM_WR },	/* WRITE  */
	{ "ADDR_CMD", FM_EREPORT_CPU_GENERIC_MMM_ADRCMD },	/* ADDR, CMD  */
	{ "SCRUB", FM_EREPORT_CPU_GENERIC_MMM_SCRUB },
	{ GCPU_MNEMONIC_RESVD, ""},			/* RESERVED  */
	{ GCPU_MNEMONIC_RESVD, ""},			/* RESERVED  */
	{ GCPU_MNEMONIC_RESVD, ""}			/* RESERVED  */
};

enum gcpu_mn_namespace {
	GCPU_MN_NAMESPACE_COMPOUND,
	GCPU_MN_NAMESPACE_EREPORT
};

static const char *
gcpu_mnemonic(const struct gcpu_mnexp *tbl, size_t tbl_sz, uint16_t val,
    enum gcpu_mn_namespace nspace)
{
	if (val >= tbl_sz || val > 0xff)
		return (GCPU_MNEMONIC_UNDEF);	/* for all namespaces */

	switch (nspace) {
	case GCPU_MN_NAMESPACE_COMPOUND:
		return (tbl[val].mne_compound);
		/*NOTREACHED*/

	case GCPU_MN_NAMESPACE_EREPORT:
		return (tbl[val].mne_ereport);
		/*NOTREACHED*/

	default:
		return (GCPU_MNEMONIC_UNDEF);
		/*NOTREACHED*/
	}
}

/*
 * The ereport class leaf component is either a simple string with no
 * format specifiers, or a string with one or more embedded %n$s specifiers -
 * positional selection for string arguments.  The kernel snprintf does
 * not support %n$ (and teaching it to do so is too big a headache) so
 * we will expand this restricted format string ourselves.
 */

#define	GCPU_CLASS_VARCOMPS	9

#define	GCPU_MNEMONIC(code, name, nspace) \
	gcpu_mnemonic(gcpu_##name##_mnemonics, \
	sizeof (gcpu_##name##_mnemonics) / sizeof (struct gcpu_mnexp), \
	BIT_STRIP(code, name), nspace)

static void
gcpu_mn_fmt(const char *fmt, char *buf, size_t buflen, uint64_t status,
    enum gcpu_mn_namespace nspace)
{
	uint16_t code = MCAX86_ERRCODE(status);
	const char *mn[GCPU_CLASS_VARCOMPS];
	char *p = buf;			/* current position in buf */
	char *q = buf + buflen;		/* pointer past last char in buf */
	int which, expfmtchar, error;
	char c;

	mn[0] = GCPU_MNEMONIC(code, TT, nspace);
	mn[1] = GCPU_MNEMONIC(code, LL, nspace);
	mn[2] = GCPU_MNEMONIC(code, RRRR, nspace);
	mn[3] = GCPU_MNEMONIC(code, PP, nspace);
	mn[4] = GCPU_MNEMONIC(code, II, nspace);
	mn[5] = GCPU_MNEMONIC(code, T, nspace);
	mn[6] = (status & MSR_MC_STATUS_UC) ? "_uc" : "";
	mn[7] = GCPU_MNEMONIC(code, CCCC, nspace);
	mn[8] = GCPU_MNEMONIC(code, MMM, nspace);

	while (p < q - 1 && (c = *fmt++) != '\0') {
		if (c != '%') {
			/* not the beginning of a format specifier - copy */
			*p++ = c;
			continue;
		}

		error = 0;
		which = -1;
		expfmtchar = -1;

nextfmt:
		if ((c = *fmt++) == '\0')
			break;	/* early termination of fmt specifier */

		switch (c) {
		case '1':
		case '2':
		case '3':
		case '4':
		case '5':
		case '6':
		case '7':
		case '8':
		case '9':
			if (which != -1) { /* allow only one positional digit */
				error++;
				break;
			}
			which = c - '1';
			goto nextfmt;
			/*NOTREACHED*/

		case '$':
			if (which == -1) { /* no position specified */
				error++;
				break;
			}
			expfmtchar = 's';
			goto nextfmt;
			/*NOTREACHED*/

		case 's':
			if (expfmtchar != 's') {
				error++;
				break;
			}
			(void) snprintf(p, (uintptr_t)q - (uintptr_t)p, "%s",
			    mn[which]);
			p += strlen(p);
			break;

		default:
			error++;
			break;
		}

		if (error)
			break;
	}

	*p = '\0';	/* NUL termination */
}

static void
gcpu_erpt_clsfmt(const char *fmt, char *buf, size_t buflen, uint64_t status,
    const char *cpuclass, const char *leafclass)
{
	char *p = buf;			/* current position in buf */
	char *q = buf + buflen;		/* pointer past last char in buf */

	(void) snprintf(buf, (uintptr_t)q - (uintptr_t)p, "%s.%s.",
	    FM_ERROR_CPU, cpuclass ? cpuclass : FM_EREPORT_CPU_GENERIC);

	p += strlen(p);
	if (p >= q)
		return;

	if (leafclass == NULL) {
		gcpu_mn_fmt(fmt, p, (uintptr_t)q - (uintptr_t)p, status,
		    GCPU_MN_NAMESPACE_EREPORT);
	} else {
		(void) snprintf(p, (uintptr_t)q - (uintptr_t)p, "%s",
		    leafclass);
	}
}

/*
 * Create an "hc" scheme FMRI identifying the given cpu with
 * motherboard/chip/core/strand instance numbers.
 */
static nvlist_t *
gcpu_fmri_create(cmi_hdl_t hdl, nv_alloc_t *nva)
{
	nvlist_t *nvl, *fmri;

	if ((nvl = fm_nvlist_create(nva)) == NULL)
		return (NULL);

	if (!x86gentopo_legacy) {
		fmri = cmi_hdl_smb_bboard(hdl);
		if (fmri == NULL)
			return (NULL);

		fm_fmri_hc_create(nvl, FM_HC_SCHEME_VERSION,
		    NULL, NULL, fmri, 3,
		    "chip", cmi_hdl_smb_chipid(hdl),
		    "core", cmi_hdl_coreid(hdl),
		    "strand", cmi_hdl_strandid(hdl));
	} else {
		fm_fmri_hc_set(nvl, FM_HC_SCHEME_VERSION, NULL, NULL, 4,
		    "motherboard", 0,
		    "chip", cmi_hdl_chipid(hdl),
		    "core", cmi_hdl_coreid(hdl),
		    "strand", cmi_hdl_strandid(hdl));
	}

	return (nvl);
}

int gcpu_bleat_count_thresh = 5;
hrtime_t gcpu_bleat_min_interval = 10 * 1000000000ULL;

/*
 * Called when we are unable to propogate a logout structure onto an
 * errorq for subsequent ereport preparation and logging etc.  The caller
 * should usually only decide to call this for severe errors - those we
 * suspect we may need to panic for.
 */
static void
gcpu_bleat(cmi_hdl_t hdl, gcpu_logout_t *gcl)
{
	hrtime_t now  = gethrtime_waitfree();
	static hrtime_t gcpu_last_bleat;
	gcpu_bank_logout_t *gbl;
	static int bleatcount;
	int i;

	/*
	 * Throttle spamming of the console.  The first gcpu_bleat_count_thresh
	 * can come as fast as we like, but once we've spammed that many
	 * to the console we require a minimum interval to pass before
	 * any more complaints.
	 */
	if (++bleatcount > gcpu_bleat_count_thresh) {
		if (now - gcpu_last_bleat < gcpu_bleat_min_interval)
			return;
		else
			bleatcount = 0;
	}
	gcpu_last_bleat = now;

	cmn_err(CE_WARN,
	    "Machine-Check Errors unlogged on chip %d core %d strand %d, "
	    "raw dump follows", cmi_hdl_chipid(hdl), cmi_hdl_coreid(hdl),
	    cmi_hdl_strandid(hdl));
	cmn_err(CE_WARN, "MCG_STATUS 0x%016llx",
	    (u_longlong_t)gcl->gcl_mcg_status);
	for (i = 0, gbl = &gcl->gcl_data[0]; i < gcl->gcl_nbanks; i++, gbl++) {
		uint64_t status = gbl->gbl_status;

		if (!(status & MSR_MC_STATUS_VAL))
			continue;

		/* Force ADDRV for AMD Family 0xf and above */
		if (gcpu_force_addr_in_payload)
			status = status | MSR_MC_STATUS_ADDRV;

		switch (status & (MSR_MC_STATUS_ADDRV | MSR_MC_STATUS_MISCV)) {
		case MSR_MC_STATUS_ADDRV | MSR_MC_STATUS_MISCV:
			cmn_err(CE_WARN, "Bank %d (offset 0x%llx) "
			    "STAT 0x%016llx ADDR 0x%016llx MISC 0x%016llx",
			    i, IA32_MSR_MC(i, STATUS),
			    (u_longlong_t)gbl->gbl_status,
			    (u_longlong_t)gbl->gbl_addr,
			    (u_longlong_t)gbl->gbl_misc);
			break;

		case MSR_MC_STATUS_ADDRV:
			cmn_err(CE_WARN, "Bank %d (offset 0x%llx) "
			    "STAT 0x%016llx ADDR 0x%016llx",
			    i, IA32_MSR_MC(i, STATUS),
			    (u_longlong_t)gbl->gbl_status,
			    (u_longlong_t)gbl->gbl_addr);
			break;

		case MSR_MC_STATUS_MISCV:
			cmn_err(CE_WARN, "Bank %d (offset 0x%llx) "
			    "STAT 0x%016llx MISC 0x%016llx",
			    i, IA32_MSR_MC(i, STATUS),
			    (u_longlong_t)gbl->gbl_status,
			    (u_longlong_t)gbl->gbl_misc);
			break;

		default:
			cmn_err(CE_WARN, "Bank %d (offset 0x%llx) "
			    "STAT 0x%016llx",
			    i, IA32_MSR_MC(i, STATUS),
			    (u_longlong_t)gbl->gbl_status);
			break;

		}
	}
}

#define	_GCPU_BSTATUS(status, what) \
	FM_EREPORT_PAYLOAD_NAME_MC_STATUS_##what, DATA_TYPE_BOOLEAN_VALUE, \
	(status) & MSR_MC_STATUS_##what ? B_TRUE : B_FALSE

static void
gcpu_ereport_add_logout(nvlist_t *ereport, const gcpu_logout_t *gcl,
    uint_t bankno, const gcpu_error_disp_t *ged, uint16_t code)
{
	uint64_t members = ged ? ged->ged_ereport_members :
	    FM_EREPORT_PAYLOAD_FLAGS_COMMON;
	uint64_t mcg = gcl->gcl_mcg_status;
	int mcip = mcg & MCG_STATUS_MCIP;
	const gcpu_bank_logout_t *gbl = &gcl->gcl_data[bankno];
	uint64_t bstat = gbl->gbl_status;

	/*
	 * Include the compound error name if requested and if this
	 * is a compound error type.
	 */
	if (members & FM_EREPORT_PAYLOAD_FLAG_COMPOUND_ERR && ged &&
	    ged->ged_compound_fmt != NULL) {
		char buf[FM_MAX_CLASS];

		gcpu_mn_fmt(ged->ged_compound_fmt, buf, sizeof (buf), code,
		    GCPU_MN_NAMESPACE_COMPOUND);
		fm_payload_set(ereport, FM_EREPORT_PAYLOAD_NAME_COMPOUND_ERR,
		    DATA_TYPE_STRING, buf, NULL);
	}

	/*
	 * Include disposition information for this error
	 */
	if (members & FM_EREPORT_PAYLOAD_FLAG_DISP &&
	    gbl->gbl_disp != 0) {
		int i, empty = 1;
		char buf[128];
		char *p = buf, *q = buf + 128;
		static struct _gcpu_disp_name {
			uint64_t dv;
			const char *dn;
		} disp_names[] = {
			{ CMI_ERRDISP_CURCTXBAD,
			    "processor_context_corrupt" },
			{ CMI_ERRDISP_RIPV_INVALID,
			    "return_ip_invalid" },
			{ CMI_ERRDISP_UC_UNCONSTRAINED,
			    "unconstrained" },
			{ CMI_ERRDISP_FORCEFATAL,
			    "forcefatal" },
			{ CMI_ERRDISP_IGNORED,
			    "ignored" },
			{ CMI_ERRDISP_PCC_CLEARED,
			    "corrupt_context_cleared" },
			{ CMI_ERRDISP_UC_CLEARED,
			    "uncorrected_data_cleared" },
			{ CMI_ERRDISP_POISONED,
			    "poisoned" },
			{ CMI_ERRDISP_INCONSISTENT,
			    "telemetry_unstable" },
		};

		for (i = 0; i < sizeof (disp_names) /
		    sizeof (struct _gcpu_disp_name); i++) {
			if ((gbl->gbl_disp & disp_names[i].dv) == 0)
				continue;

			(void) snprintf(p, (uintptr_t)q - (uintptr_t)p,
			    "%s%s", empty ? "" : ",", disp_names[i].dn);
			p += strlen(p);
			empty = 0;
		}

		if (p != buf)
			fm_payload_set(ereport, FM_EREPORT_PAYLOAD_NAME_DISP,
			    DATA_TYPE_STRING, buf, NULL);
	}

	/*
	 * If MCG_STATUS is included add that and an indication of whether
	 * this ereport was the result of a machine check or poll.
	 */
	if (members & FM_EREPORT_PAYLOAD_FLAG_MCG_STATUS) {
		fm_payload_set(ereport, FM_EREPORT_PAYLOAD_NAME_MCG_STATUS,
		    DATA_TYPE_UINT64, mcg, NULL);

		fm_payload_set(ereport, FM_EREPORT_PAYLOAD_NAME_MCG_STATUS_MCIP,
		    DATA_TYPE_BOOLEAN_VALUE, mcip ? B_TRUE : B_FALSE, NULL);
	}

	/*
	 * If an instruction pointer is to be included add one provided
	 * MCG_STATUS indicated it is valid; meaningless for polled events.
	 */
	if (mcip && members & FM_EREPORT_PAYLOAD_FLAG_IP &&
	    mcg & MCG_STATUS_EIPV) {
		fm_payload_set(ereport, FM_EREPORT_PAYLOAD_NAME_IP,
		    DATA_TYPE_UINT64, gcl->gcl_ip, NULL);
	}

	/*
	 * Add an indication of whether the trap occured during privileged code.
	 */
	if (mcip && members & FM_EREPORT_PAYLOAD_FLAG_PRIV) {
		fm_payload_set(ereport, FM_EREPORT_PAYLOAD_NAME_PRIV,
		    DATA_TYPE_BOOLEAN_VALUE,
		    gcl->gcl_flags & GCPU_GCL_F_PRIV ? B_TRUE : B_FALSE, NULL);
	}

	/*
	 * If requested, add the index of the MCA bank.  This indicates the
	 * n'th bank of 4 MCA registers, and does not necessarily correspond
	 * to MCi_* - use the bank offset to correlate
	 */
	if (members & FM_EREPORT_PAYLOAD_FLAG_BANK_NUM) {
		fm_payload_set(ereport,
		    /* Bank number */
		    FM_EREPORT_PAYLOAD_NAME_BANK_NUM, DATA_TYPE_UINT8, bankno,
		    /* Offset of MCi_CTL */
		    FM_EREPORT_PAYLOAD_NAME_BANK_MSR_OFFSET, DATA_TYPE_UINT64,
		    IA32_MSR_MC(bankno, CTL),
		    NULL);
	}

	/*
	 * Add MCi_STATUS if requested, and decode it.
	 */
	if (members & FM_EREPORT_PAYLOAD_FLAG_MC_STATUS) {
		const char *tbes[] = {
			"No tracking",			/* 00 */
			"Green - below threshold",	/* 01 */
			"Yellow - above threshold",	/* 10 */
			"Reserved"			/* 11 */
		};

		fm_payload_set(ereport,
		    /* Bank MCi_STATUS */
		    FM_EREPORT_PAYLOAD_NAME_MC_STATUS, DATA_TYPE_UINT64, bstat,
		    /* Overflow? */
		    _GCPU_BSTATUS(bstat, OVER),
		    /* Uncorrected? */
		    _GCPU_BSTATUS(bstat, UC),
		    /* Enabled? */
		    _GCPU_BSTATUS(bstat, EN),
		    /* Processor context corrupt? */
		    _GCPU_BSTATUS(bstat, PCC),
		    /* Error code */
		    FM_EREPORT_PAYLOAD_NAME_MC_STATUS_ERRCODE,
		    DATA_TYPE_UINT16, MCAX86_ERRCODE(bstat),
		    /* Model-specific error code */
		    FM_EREPORT_PAYLOAD_NAME_MC_STATUS_EXTERRCODE,
		    DATA_TYPE_UINT16, MCAX86_MSERRCODE(bstat),
		    NULL);

		/*
		 * If MCG_CAP.TES_P indicates that that thresholding info
		 * is present in the architural component of the bank status
		 * then include threshold information for this bank.
		 */
		if (gcl->gcl_flags & GCPU_GCL_F_TES_P) {
			fm_payload_set(ereport,
			    FM_EREPORT_PAYLOAD_NAME_MC_STATUS_TES,
			    DATA_TYPE_STRING, tbes[MCAX86_TBES_VALUE(bstat)],
			    NULL);
		}
	}

	/*
	 * Add MCi_ADDR info if requested and valid. We force addition of
	 * MCi_ADDR, even if its not valid on AMD family 0xf and above,
	 * to aid in analysis of ereports, for WatchDog errors.
	 */
	if (members & FM_EREPORT_PAYLOAD_FLAG_MC_ADDR &&
	    ((bstat & MSR_MC_STATUS_ADDRV) ||
	    gcpu_force_addr_in_payload)) {
		fm_payload_set(ereport, FM_EREPORT_PAYLOAD_NAME_MC_ADDR,
		    DATA_TYPE_UINT64, gbl->gbl_addr, NULL);
	}

	/*
	 * MCi_MISC if requested and MCi_STATUS.MISCV).
	 */
	if (members & FM_EREPORT_PAYLOAD_FLAG_MC_MISC &&
	    bstat & MSR_MC_STATUS_MISCV) {
		fm_payload_set(ereport, FM_EREPORT_PAYLOAD_NAME_MC_MISC,
		    DATA_TYPE_UINT64, gbl->gbl_misc, NULL);
	}

}

/*
 * Construct and post an ereport based on the logout information from a
 * single MCA bank.  We are not necessarily running on the cpu that
 * detected the error.
 */
static void
gcpu_ereport_post(const gcpu_logout_t *gcl, int bankidx,
    const gcpu_error_disp_t *ged, cms_cookie_t mscookie, uint64_t status)
{
	gcpu_data_t *gcpu = gcl->gcl_gcpu;
	cmi_hdl_t hdl = gcpu->gcpu_hdl;
	const gcpu_bank_logout_t *gbl = &gcl->gcl_data[bankidx];
	const char *cpuclass = NULL, *leafclass = NULL;
	uint16_t code = MCAX86_ERRCODE(status);
	errorq_elem_t *eqep, *scr_eqep;
	nvlist_t *ereport, *detector;
	char buf[FM_MAX_CLASS];
	const char *classfmt;
	nv_alloc_t *nva;

	if (panicstr) {
		if ((eqep = errorq_reserve(ereport_errorq)) == NULL)
			return;
		ereport = errorq_elem_nvl(ereport_errorq, eqep);

		/*
		 * Allocate another element for scratch space, but fallback
		 * to the one we have if that fails.  We'd like to use the
		 * additional scratch space for nvlist construction.
		 */
		if ((scr_eqep = errorq_reserve(ereport_errorq)) != NULL)
			nva = errorq_elem_nva(ereport_errorq, scr_eqep);
		else
			nva = errorq_elem_nva(ereport_errorq, eqep);
	} else {
		ereport = fm_nvlist_create(NULL);
		nva = NULL;
		eqep = NULL;
		scr_eqep = NULL;
	}

	if (ereport == NULL)
		return;

	/*
	 * Common payload data required by the protocol:
	 *	- ereport class
	 *	- detector
	 *	- ENA
	 */

	/*
	 * Ereport class - call into model-specific support to allow it to
	 * provide a cpu class or leaf class, otherwise calculate our own.
	 */
	cms_ereport_class(hdl, mscookie, &cpuclass, &leafclass);
	classfmt = ged ?  ged->ged_class_fmt : FM_EREPORT_CPU_GENERIC_UNKNOWN;
	gcpu_erpt_clsfmt(classfmt, buf, sizeof (buf), status, cpuclass,
	    leafclass);

	/*
	 * The detector FMRI.
	 */
	if ((detector = cms_ereport_detector(hdl, bankidx, mscookie,
	    nva)) == NULL)
		detector = gcpu_fmri_create(hdl, nva);

	/*
	 * Should we define a new ENA format 3?? for chip/core/strand?
	 * It will be better when virtualized.
	 */
	fm_ereport_set(ereport, FM_EREPORT_VERSION, buf,
	    fm_ena_generate_cpu(gcl->gcl_timestamp,
	    cmi_hdl_chipid(hdl) << 6 | cmi_hdl_coreid(hdl) << 3 |
	    cmi_hdl_strandid(hdl), FM_ENA_FMT1), detector, NULL);

	if (panicstr) {
		fm_nvlist_destroy(detector, FM_NVA_RETAIN);
		nv_alloc_reset(nva);
	} else {
		fm_nvlist_destroy(detector, FM_NVA_FREE);
	}

	/*
	 * Add the architectural ereport class-specific payload data.
	 */
	gcpu_ereport_add_logout(ereport, gcl, bankidx, ged, code);

	/*
	 * Allow model-specific code to add ereport members.
	 */
	cms_ereport_add_logout(hdl, ereport, nva, bankidx, gbl->gbl_status,
	    gbl->gbl_addr, gbl->gbl_misc, gcl->gcl_ms_logout, mscookie);

	/*
	 * Include stack if options is turned on and either selected in
	 * the payload member bitmask or inclusion is forced.
	 */
	if (gcpu_mca_stack_flag &&
	    (cms_ereport_includestack(hdl, mscookie) ==
	    B_TRUE || gcpu_mca_stack_ereport_include)) {
		fm_payload_stack_add(ereport, gcl->gcl_stack,
		    gcl->gcl_stackdepth);
	}

	/*
	 * If injection has taken place anytime in the past then note this
	 * on the ereport.
	 */
	if (cmi_inj_tainted() == B_TRUE) {
		fm_payload_set(ereport, "__injected", DATA_TYPE_BOOLEAN_VALUE,
		    B_TRUE, NULL);
	}

	/*
	 * Post ereport.
	 */
	if (panicstr) {
		errorq_commit(ereport_errorq, eqep, ERRORQ_SYNC);
		if (scr_eqep)
			errorq_cancel(ereport_errorq, scr_eqep);
	} else {
		(void) fm_ereport_post(ereport, EVCH_TRYHARD);
		fm_nvlist_destroy(ereport, FM_NVA_FREE);
	}

}

/*ARGSUSED*/
void
gcpu_mca_drain(void *ignored, const void *data, const errorq_elem_t *eqe)
{
	const gcpu_logout_t *gcl = data;
	const gcpu_bank_logout_t *gbl;
	int ismc;
	int i;

	ismc = gcl->ismc;
	for (i = 0, gbl = &gcl->gcl_data[0]; i < gcl->gcl_nbanks; i++, gbl++) {
		const gcpu_error_disp_t *gened;
		cms_cookie_t mscookie;

		if (gbl->gbl_status & MSR_MC_STATUS_VAL &&
		    !(gbl->gbl_disp & CMI_ERRDISP_INCONSISTENT)) {
			uint16_t code = MCAX86_ERRCODE(gbl->gbl_status);

			/*
			 * Perform a match based on IA32 MCA architectural
			 * components alone.
			 */
			gened = gcpu_disp_match(code); /* may be NULL */

			/*
			 * Now see if an model-specific match can be made.
			 */
			mscookie = cms_disp_match(gcl->gcl_gcpu->gcpu_hdl, ismc,
			    i, gbl->gbl_status, gbl->gbl_addr, gbl->gbl_misc,
			    gcl->gcl_ms_logout);

			/*
			 * Prepare and dispatch an ereport for logging and
			 * diagnosis.
			 */
			gcpu_ereport_post(gcl, i, gened, mscookie,
			    gbl->gbl_status);
		} else if (gbl->gbl_status & MSR_MC_STATUS_VAL &&
		    (gbl->gbl_disp & CMI_ERRDISP_INCONSISTENT)) {
			/*
			 * Telemetry kept changing as we tried to read
			 * it.  Force an unknown ereport leafclass but
			 * keep the telemetry unchanged for logging.
			 */
			gcpu_ereport_post(gcl, i, &gcpu_unknown, NULL,
			    gbl->gbl_status);
		}
	}
}

static size_t gcpu_mca_queue_datasz = 0;

/*
 * The following code is ready to make a weak attempt at growing the
 * errorq structure size.  Since it is not foolproof (we don't know
 * who may already be producing to the outgoing errorq) our caller
 * instead assures that we'll always be called with no greater data
 * size than on our first call.
 */
static void
gcpu_errorq_init(size_t datasz)
{
	int slots;

	mutex_enter(&gcpu_mca_queue_lock);

	if (gcpu_mca_queue_datasz >= datasz) {
		mutex_exit(&gcpu_mca_queue_lock);
		return;
	}

	membar_producer();
	if (gcpu_mca_queue) {
		gcpu_mca_queue_datasz = 0;
		errorq_destroy(gcpu_mca_queue);
	}

	slots = MAX(GCPU_MCA_ERRS_PERCPU * max_ncpus, GCPU_MCA_MIN_ERRORS);
	slots = MIN(slots, GCPU_MCA_MAX_ERRORS);

	gcpu_mca_queue = errorq_create("gcpu_mca_queue", gcpu_mca_drain,
	    NULL, slots, datasz, 1, ERRORQ_VITAL);

	if (gcpu_mca_queue != NULL)
		gcpu_mca_queue_datasz = datasz;

	mutex_exit(&gcpu_mca_queue_lock);
}

/*
 * Perform MCA initialization as described in section 14.6 of Intel 64
 * and IA-32 Architectures Software Developer's Manual Volume 3A.
 */

static uint_t global_nbanks;

#ifndef __xpv
/*ARGSUSED*/
int
gcpu_cmci_cpu_setup(cpu_setup_t what, int cpuid, void *arg)
{
	/*
	 * In general, we'd expect that in a multi-socket configuration, either
	 * all CPUs would support CMCI or none of them would.  Unfortunately,
	 * that may not be the case in the wild.  While we'd rather check the
	 * handle's enablement state here, that itself is a bit complicated. We
	 * don't have a guarantee in a heterogenous situation that the CPU in
	 * question is using the generic CPU module or not, even though we've
	 * been registered. As such, we allow the interrupt to be registered and
	 * written to the local apic anyways. We won't have a CMCI interrupt
	 * generated anyways because the MCA banks will not be programmed as
	 * such for that CPU by the polling thread.
	 */
	switch (what) {
	case CPU_ON:
		psm_cmci_setup(cpuid, B_TRUE);
		break;
	case CPU_OFF:
		psm_cmci_setup(cpuid, B_FALSE);
		break;
	default:
		break;
	}

	return (0);
}

void
gcpu_mca_cmci_enable(cmi_hdl_t hdl)
{
	gcpu_data_t *gcpu = cmi_hdl_getcmidata(hdl);
	gcpu_mca_t *mca = &gcpu->gcpu_mca;

	/*
	 * If this CPU doesn't support CMCI, don't do anything.
	 */
	if ((mca->gcpu_mca_flags & GCPU_MCA_F_CMCI_CAPABLE) == 0)
		return;

	/*
	 * If we don't have support from the PSM module, then there's nothing we
	 * can do. Note that this changes as we start up the system. The only
	 * case where it may be mistakenly NULL is for the boot CPU. The boot
	 * CPU will have this taken care of for it in gcpu_post_startup(), once
	 * we know for certain whether or not the PSM module supports CMCI.
	 */
	if (psm_cmci_setup == NULL) {
		return;
	}

	mca->gcpu_mca_flags |= GCPU_MCA_F_CMCI_ENABLE;
	if (MUTEX_HELD(&cpu_lock)) {
		if (!gcpu_mca_cpu_registered) {
			register_cpu_setup_func(gcpu_cmci_cpu_setup, NULL);
			gcpu_mca_cpu_registered = B_TRUE;
		}
	} else {
		mutex_enter(&cpu_lock);
		if (!gcpu_mca_cpu_registered) {
			register_cpu_setup_func(gcpu_cmci_cpu_setup, NULL);
			gcpu_mca_cpu_registered = B_TRUE;
		}
		mutex_exit(&cpu_lock);
	}

	/*
	 * Call the PSM op to make sure that we initialize things on
	 * this CPU.
	 */
	psm_cmci_setup(cmi_hdl_logical_id(hdl), B_TRUE);
}
#endif	/* !__xpv */

void
gcpu_mca_init(cmi_hdl_t hdl)
{
	gcpu_data_t *gcpu = cmi_hdl_getcmidata(hdl);
	uint64_t cap;
	uint_t vendor = cmi_hdl_vendor(hdl);
	uint_t family = cmi_hdl_family(hdl);
	gcpu_mca_t *mca = &gcpu->gcpu_mca;
	int mcg_ctl_present;
	uint_t nbanks;
	uint32_t ctl_skip_mask = 0;
	uint32_t status_skip_mask = 0;
	size_t mslsz;
	int i;
#ifndef __xpv
	int mcg_ctl2_present;
	uint32_t cmci_capable = 0;
#endif
	if (gcpu == NULL)
		return;

	/* We add MCi_ADDR always for AMD Family 0xf and above */
	if (family >= 0xf)
		gcpu_force_addr_in_payload = 1;

	/*
	 * Protect from some silly /etc/system settings.
	 */
	if (gcpu_mca_telemetry_retries < 0 || gcpu_mca_telemetry_retries > 100)
		gcpu_mca_telemetry_retries = 5;

	if (cmi_hdl_rdmsr(hdl, IA32_MSR_MCG_CAP, &cap) != CMI_SUCCESS)
		return;

	/*
	 * CPU startup code only calls cmi_mca_init if x86_featureset indicates
	 * both MCA and MCE support (i.e., X86FSET_MCA).  P5, K6, and earlier
	 * processors, which have their own more primitive way of doing
	 * machine checks, will not have cmi_mca_init called since their
	 * CPUID information will not indicate both MCA and MCE features.
	 */
	ASSERT(is_x86_feature(x86_featureset, X86FSET_MCA));

	/*
	 * Determine whether the IA32_MCG_CTL register is present.  If it
	 * is we will enable all features by writing -1 to it towards
	 * the end of this initialization;  if it is absent then volume 3A
	 * says we must nonetheless continue to initialize the individual
	 * banks.
	 */
	mcg_ctl_present = cap & MCG_CAP_CTL_P;
#ifndef __xpv
	mcg_ctl2_present = cap & MCG_CAP_CTL2_P;
#endif

	/*
	 * We squirell values away for inspection/debugging.
	 */
	mca->gcpu_mca_bioscfg.bios_mcg_cap = cap;
	if (mcg_ctl_present)
		(void) cmi_hdl_rdmsr(hdl, IA32_MSR_MCG_CTL,
		    &mca->gcpu_mca_bioscfg.bios_mcg_ctl);

	/*
	 * Determine the number of error-reporting banks implemented.
	 */
	mca->gcpu_mca_nbanks = nbanks = cap & MCG_CAP_COUNT_MASK;

	if (nbanks != 0 && global_nbanks == 0)
		global_nbanks = nbanks;	/* no race - BSP will get here first */

	/*
	 * If someone is hiding the number of banks (perhaps we are fully
	 * virtualized?) or if this processor has more banks than the
	 * first to set global_nbanks then bail.  The latter requirement
	 * is because we need to size our errorq data structure and we
	 * don't want to have to grow the errorq (destroy and recreate)
	 * which may just lose some telemetry.
	 */
	if (nbanks == 0 || nbanks > global_nbanks)
		return;

	mca->gcpu_mca_bioscfg.bios_bankcfg = kmem_zalloc(nbanks *
	    sizeof (struct gcpu_bios_bankcfg), KM_SLEEP);

	/*
	 * Calculate the size we need to allocate for a gcpu_logout_t
	 * with a gcl_data array big enough for all banks of this cpu.
	 * Add any space requested by the model-specific logout support.
	 */
	mslsz = cms_logout_size(hdl);
	mca->gcpu_mca_lgsz = sizeof (gcpu_logout_t) +
	    (nbanks - 1) * sizeof (gcpu_bank_logout_t) + mslsz;

	for (i = 0; i < GCPU_MCA_LOGOUT_NUM; i++) {
		gcpu_logout_t *gcl;

		mca->gcpu_mca_logout[i] = gcl =
		    kmem_zalloc(mca->gcpu_mca_lgsz, KM_SLEEP);
		gcl->gcl_gcpu = gcpu;
		gcl->gcl_nbanks = nbanks;
		gcl->gcl_ms_logout = (mslsz == 0) ? NULL :
		    (char *)(&gcl->gcl_data[0]) + nbanks *
		    sizeof (gcpu_bank_logout_t);

	}

#ifdef __xpv
	gcpu_xpv_mca_init(nbanks);
#endif

	mca->gcpu_mca_nextpoll_idx = GCPU_MCA_LOGOUT_POLLER_1;

#ifndef __xpv
	mca->gcpu_bank_cmci = kmem_zalloc(sizeof (gcpu_mca_cmci_t) * nbanks,
	    KM_SLEEP);
#endif

	/*
	 * Create our errorq to transport the logout structures.  This
	 * can fail so users of gcpu_mca_queue must be prepared for NULL.
	 */
	gcpu_errorq_init(mca->gcpu_mca_lgsz);

	/*
	 * Not knowing which, if any, banks are shared between cores we
	 * assure serialization of MCA bank initialization by each cpu
	 * on the chip.  On chip architectures in which some banks are
	 * shared this will mean the shared resource is initialized more
	 * than once - we're simply aiming to avoid simultaneous MSR writes
	 * to the shared resource.
	 *
	 * Even with these precautions, some platforms may yield a GP fault
	 * if a core other than a designated master tries to write anything
	 * but all 0's to MCi_{STATUS,ADDR,CTL}.  So we will perform
	 * those writes under on_trap protection.
	 */
	mutex_enter(&gcpu->gcpu_shared->gcpus_cfglock);

	/*
	 * Initialize poller data, but don't start polling yet.
	 */
	gcpu_mca_poll_init(hdl);

	/*
	 * Work out which MCA banks we will initialize.  In MCA logout
	 * code we will only read those banks which we initialize here.
	 */
	for (i = 0; i < nbanks; i++) {
		boolean_t skipctl = cms_bankctl_skipinit(hdl, i);
		boolean_t skipstatus = cms_bankstatus_skipinit(hdl, i);

		if (!cms_present(hdl)) {
			/*
			 * Model-specific support is not present, try to use
			 * sane defaults.
			 *
			 * On AMD family 6 processors, reports about spurious
			 * machine checks indicate that bank 0 should be
			 * skipped.
			 *
			 * On Intel family 6 processors, the documentation tells
			 * us not to write to MC0_CTL.
			 *
			 */
			if (i == 0 && family == 6) {
				switch (vendor) {
				case X86_VENDOR_AMD:
					skipstatus = B_TRUE;
					/*FALLTHRU*/
				case X86_VENDOR_Intel:
					skipctl = B_TRUE;
					break;
				}
			}
		}

		ctl_skip_mask |= skipctl << i;
		status_skip_mask |= skipstatus << i;

		if (skipctl && skipstatus)
			continue;

		/*
		 * Record which MCA banks were enabled, from the point of view
		 * of the whole chip (if some cores share a bank we must be
		 * sure either can logout from it).
		 */
		atomic_or_32(&gcpu->gcpu_shared->gcpus_actv_banks, 1 << i);

#ifndef __xpv
		/*
		 * check CMCI capability
		 */
		if (mcg_ctl2_present) {
			uint64_t ctl2;
			uint32_t cap = 0;
			(void) cmi_hdl_rdmsr(hdl, IA32_MSR_MC_CTL2(i), &ctl2);
			if (ctl2 & MSR_MC_CTL2_EN)
				continue;
			ctl2 |= MSR_MC_CTL2_EN;
			(void) cmi_hdl_wrmsr(hdl, IA32_MSR_MC_CTL2(i), ctl2);
			(void) cmi_hdl_rdmsr(hdl, IA32_MSR_MC_CTL2(i), &ctl2);
			mca->gcpu_bank_cmci[i].cmci_cap = cap =
			    (ctl2 & MSR_MC_CTL2_EN) ? 1 : 0;
			if (cap)
				cmci_capable ++;
			/*
			 * Set threshold to 1 while unset the en field, to avoid
			 * CMCI trigged before APIC LVT entry init.
			 */
			ctl2 = (ctl2 & (~MSR_MC_CTL2_EN)) | 1;
			(void) cmi_hdl_wrmsr(hdl, IA32_MSR_MC_CTL2(i), ctl2);

			/*
			 * init cmci related count
			 */
			mca->gcpu_bank_cmci[i].cmci_enabled = 0;
			mca->gcpu_bank_cmci[i].drtcmci = 0;
			mca->gcpu_bank_cmci[i].ncmci = 0;
		}
#endif
	}

#ifndef __xpv
	if (cmci_capable) {
		mca->gcpu_mca_flags |= GCPU_MCA_F_CMCI_CAPABLE;
		gcpu_mca_cmci_enable(hdl);
	}
#endif

#ifndef __xpv
	/*
	 * Log any valid telemetry lurking in the MCA banks, but do not
	 * clear the status registers.  Ignore the disposition returned -
	 * we have already paniced or reset for any nasty errors found here.
	 *
	 * Intel vol 3A says that we should not do this on family 0x6,
	 * and that for any extended family the BIOS clears things
	 * on power-on reset so you'll only potentially find valid telemetry
	 * on warm reset (we do it for both - on power-on reset we should
	 * just see zeroes).
	 *
	 * AMD docs since K7 say we should process anything we find here.
	 */
	if (!gcpu_suppress_log_on_init &&
	    ((vendor == X86_VENDOR_Intel && family >= 0xf) ||
	    vendor == X86_VENDOR_AMD ||
	    vendor == X86_VENDOR_HYGON))
		gcpu_mca_logout(hdl, NULL, -1ULL, NULL, B_FALSE,
		    GCPU_MPT_WHAT_POKE_ERR);

	/*
	 * Initialize all MCi_CTL and clear all MCi_STATUS, allowing the
	 * model-specific module the power of veto.
	 */
	for (i = 0; i < nbanks; i++) {
		struct gcpu_bios_bankcfg *bcfgp =
		    mca->gcpu_mca_bioscfg.bios_bankcfg + i;

		/*
		 * Stash inherited bank MCA state, even for banks we will
		 * not initialize ourselves.  Do not read the MISC register
		 * unconditionally - on some processors that will #GP on
		 * banks that do not implement the MISC register (would be
		 * caught by on_trap, anyway).
		 */
		(void) cmi_hdl_rdmsr(hdl, IA32_MSR_MC(i, CTL),
		    &bcfgp->bios_bank_ctl);

		(void) cmi_hdl_rdmsr(hdl, IA32_MSR_MC(i, STATUS),
		    &bcfgp->bios_bank_status);

		if ((bcfgp->bios_bank_status & MSR_MC_STATUS_ADDRV) ||
		    gcpu_force_addr_in_payload) {
			(void) cmi_hdl_rdmsr(hdl, IA32_MSR_MC(i, ADDR),
			    &bcfgp->bios_bank_addr);
		}

		/*
		 * In some old BIOS the status value after boot can indicate
		 * MISCV when there is actually no MISC register for
		 * that bank.  The following read could therefore
		 * aggravate a general protection fault.  This should be
		 * caught by on_trap, but the #GP fault handler is busted
		 * and can suffer a double fault even before we get to
		 * trap() to check for on_trap protection.  Until that
		 * issue is fixed we remove the one access that we know
		 * can cause a #GP.
		 *
		 * if (bcfgp->bios_bank_status & MSR_MC_STATUS_MISCV)
		 *	(void) cmi_hdl_rdmsr(hdl, IA32_MSR_MC(i, MISC),
		 *	    &bcfgp->bios_bank_misc);
		 */
		bcfgp->bios_bank_misc = 0;

		if (!(ctl_skip_mask & (1 << i))) {
			(void) cmi_hdl_wrmsr(hdl, IA32_MSR_MC(i, CTL),
			    cms_bankctl_val(hdl, i, -1ULL));
		}

		if (!(status_skip_mask & (1 << i))) {
			(void) cmi_hdl_wrmsr(hdl, IA32_MSR_MC(i, STATUS),
			    cms_bankstatus_val(hdl, i, 0ULL));
		}
	}
#endif
	/*
	 * Now let the model-specific support perform further initialization
	 * of non-architectural features.
	 */
	cms_mca_init(hdl, nbanks);

#ifndef __xpv
	(void) cmi_hdl_wrmsr(hdl, IA32_MSR_MCG_STATUS, 0ULL);
	membar_producer();

	/* enable all machine-check features */
	if (mcg_ctl_present)
		(void) cmi_hdl_wrmsr(hdl, IA32_MSR_MCG_CTL,
		    cms_mcgctl_val(hdl, nbanks, -1ULL));
#endif

	mutex_exit(&gcpu->gcpu_shared->gcpus_cfglock);

#ifndef __xpv
	/* enable machine-check exception in CR4 */
	cmi_hdl_enable_mce(hdl);
#endif
}

static uint64_t
gcpu_mca_process(cmi_hdl_t hdl, struct regs *rp, int nerr, gcpu_data_t *gcpu,
    gcpu_logout_t *gcl, int ismc, gcpu_mce_status_t *mcesp)
{
	int curctxbad = 0, unconstrained = 0, forcefatal = 0;
	gcpu_mca_t *mca = &gcpu->gcpu_mca;
	int nbanks = mca->gcpu_mca_nbanks;
	gcpu_mce_status_t mce;
	gcpu_bank_logout_t *gbl;
	uint64_t disp = 0;
	int i;

	if (mcesp == NULL)
		mcesp = &mce;

	mcesp->mce_nerr = nerr;

	mcesp->mce_npcc = mcesp->mce_npcc_ok = mcesp->mce_nuc =
	    mcesp->mce_nuc_ok = mcesp->mce_nuc_poisoned =
	    mcesp->mce_forcefatal = mcesp->mce_ignored = 0;

	/*
	 * If this a machine check then if the return instruction pointer
	 * is not valid the current context is lost.
	 */
	if (ismc && !(gcl->gcl_mcg_status & MCG_STATUS_RIPV))
		disp |= CMI_ERRDISP_RIPV_INVALID;
	gcl->ismc = ismc;

	for (i = 0, gbl = &gcl->gcl_data[0]; i < nbanks; i++, gbl++) {
		uint64_t mcistatus = gbl->gbl_status;
		uint32_t ms_scope;
		int pcc, uc;
		int poisoned;

		if (!(mcistatus & MSR_MC_STATUS_VAL))
			continue;

		if (gbl->gbl_disp & CMI_ERRDISP_INCONSISTENT)
			continue;

		pcc = (mcistatus & MSR_MC_STATUS_PCC) != 0;
		uc = (mcistatus & MSR_MC_STATUS_UC) != 0;
		mcesp->mce_npcc += pcc;
		mcesp->mce_nuc += uc;

		ms_scope = cms_error_action(hdl, ismc, i, mcistatus,
		    gbl->gbl_addr, gbl->gbl_misc, gcl->gcl_ms_logout);

		if (pcc && ms_scope & CMS_ERRSCOPE_CURCONTEXT_OK) {
			pcc = 0;
			mcesp->mce_npcc_ok++;
			gbl->gbl_disp |= CMI_ERRDISP_PCC_CLEARED;
		}

		if (uc && ms_scope & CMS_ERRSCOPE_CLEARED_UC) {
			uc = 0;
			mcesp->mce_nuc_ok++;
			gbl->gbl_disp |= CMI_ERRDISP_UC_CLEARED;
		}

		if (uc) {
			poisoned = (ms_scope & CMS_ERRSCOPE_POISONED) != 0;
			if (poisoned) {
				mcesp->mce_nuc_poisoned++;
				gbl->gbl_disp |= CMI_ERRDISP_POISONED;
			}
		}

		if ((ms_scope & CMS_ERRSCOPE_IGNORE_ERR) == 0) {
			/*
			 * We're not being instructed to ignore the error,
			 * so apply our standard disposition logic to it.
			 */
			if (uc && !poisoned) {
				unconstrained++;
				gbl->gbl_disp |= disp |
				    CMI_ERRDISP_UC_UNCONSTRAINED;
			}

			if (pcc && ismc) {
				curctxbad++;
				gbl->gbl_disp |= disp |
				    CMI_ERRDISP_CURCTXBAD;
			}

			/*
			 * Even if the above may not indicate that the error
			 * is terminal, model-specific support may insist
			 * that we treat it as such.  Such errors wil be
			 * fatal even if discovered via poll.
			 */
			if (ms_scope & CMS_ERRSCOPE_FORCE_FATAL) {
				forcefatal++;
				mcesp->mce_forcefatal++;
				gbl->gbl_disp |= disp |
				    CMI_ERRDISP_FORCEFATAL;
			}
		} else {
			mcesp->mce_ignored++;
			gbl->gbl_disp |= disp | CMI_ERRDISP_IGNORED;
		}
	}

	if (unconstrained > 0)
		disp |= CMI_ERRDISP_UC_UNCONSTRAINED;

	if (curctxbad > 0)
		disp |= CMI_ERRDISP_CURCTXBAD;

	if (forcefatal > 0)
		disp |= CMI_ERRDISP_FORCEFATAL;

	if (gcpu_mca_queue != NULL) {
		int how;

		if (ismc) {
			how = cmi_mce_response(rp, disp) ?
			    ERRORQ_ASYNC :	/* no panic, so arrange drain */
			    ERRORQ_SYNC;	/* panic flow will drain */
		} else {
			how = (disp & CMI_ERRDISP_FORCEFATAL &&
			    cmi_panic_on_ue()) ?
			    ERRORQ_SYNC :	/* poller will panic */
			    ERRORQ_ASYNC;	/* no panic */
		}

		errorq_dispatch(gcpu_mca_queue, gcl, mca->gcpu_mca_lgsz, how);
	} else if (disp != 0) {
		gcpu_bleat(hdl, gcl);
	}

	mcesp->mce_disp = disp;

	return (disp);
}

/*
 * Gather error telemetry from our source, and then submit it for
 * processing.
 */

#define	IS_MCE_CANDIDATE(status) (((status) & MSR_MC_STATUS_EN) != 0 && \
	((status) & (MSR_MC_STATUS_UC | MSR_MC_STATUS_PCC)) != 0)

#define	STATUS_EQV(s1, s2) \
	(((s1) & ~MSR_MC_STATUS_OVER) == ((s2) & ~MSR_MC_STATUS_OVER))

static uint32_t gcpu_deferrred_polled_clears;

#ifndef __xpv
static void
gcpu_cmci_logout(cmi_hdl_t hdl, int bank, gcpu_mca_cmci_t *bank_cmci_p,
    uint64_t status, int what)
{
	uint64_t ctl2;

	if (bank_cmci_p->cmci_cap && (what == GCPU_MPT_WHAT_CYC_ERR) &&
	    (!(status & MSR_MC_STATUS_VAL) || ((status & MSR_MC_STATUS_VAL) &&
	    !(status & MSR_MC_STATUS_CEC_MASK)))) {

		if (!(bank_cmci_p->cmci_enabled)) {
			/*
			 * when cmci is disabled, and the bank has no error or
			 * no corrected error for
			 * gcpu_mca_cmci_reenable_threshold consecutive polls,
			 * turn on this bank's cmci.
			 */

			bank_cmci_p->drtcmci ++;

			if (bank_cmci_p->drtcmci >=
			    gcpu_mca_cmci_reenable_threshold) {

				/* turn on cmci */

				(void) cmi_hdl_rdmsr(hdl,
				    IA32_MSR_MC_CTL2(bank), &ctl2);
				ctl2 |= MSR_MC_CTL2_EN;
				(void) cmi_hdl_wrmsr(hdl,
				    IA32_MSR_MC_CTL2(bank), ctl2);

				/* reset counter and set flag */
				bank_cmci_p->drtcmci = 0;
				bank_cmci_p->cmci_enabled = 1;
			}
		} else {
			/*
			 * when cmci is enabled,if is in cyclic poll and the
			 * bank has no error or no corrected error, reset ncmci
			 * counter
			 */
			bank_cmci_p->ncmci = 0;
		}
	}
}

static void
gcpu_cmci_throttle(cmi_hdl_t hdl, int bank, gcpu_mca_cmci_t *bank_cmci_p,
    int what)
{
	uint64_t ctl2 = 0;

	/*
	 * if cmci of this bank occurred beyond
	 * gcpu_mca_cmci_throttling_threshold between 2 polls,
	 * turn off this bank's CMCI;
	 */
	if (bank_cmci_p->cmci_enabled && what == GCPU_MPT_WHAT_CMCI_ERR) {

		/* if it is cmci trap, increase the count */
		bank_cmci_p->ncmci++;

		if (bank_cmci_p->ncmci >= gcpu_mca_cmci_throttling_threshold) {

			/* turn off cmci */

			(void) cmi_hdl_rdmsr(hdl, IA32_MSR_MC_CTL2(bank),
			    &ctl2);
			ctl2 &= ~MSR_MC_CTL2_EN;
			(void) cmi_hdl_wrmsr(hdl, IA32_MSR_MC_CTL2(bank),
			    ctl2);

			/* clear the flag and count */

			bank_cmci_p->cmci_enabled = 0;
			bank_cmci_p->ncmci = 0;
		}
	}
}
#endif

static void
clear_mc(int first, int last, int ismc, boolean_t clrstatus,
    cmi_hdl_t hdl, gcpu_logout_t *gcl, gcpu_logout_t *pgcl)
{
	int i;
	gcpu_bank_logout_t *gbl, *pgbl;
	uint64_t status;

	if (first < 0 || last < 0)
		return;

	for (i = first, gbl = &gcl->gcl_data[first]; i <= last; i++, gbl++) {
		status = gbl->gbl_status;
		if (status == 0)
			continue;
		if (clrstatus == B_FALSE)
			goto serialize;

		/*
		 * For i86xpv we always clear status in order to invalidate
		 * the interposed telemetry.
		 *
		 * For native machine checks we always clear status here.  For
		 * native polls we must be a little more cautious since there
		 * is an outside chance that we may clear telemetry from a
		 * shared MCA bank on which a sibling core is machine checking.
		 *
		 * For polled observations of errors that look like they may
		 * produce a machine check (UC/PCC and ENabled, although these
		 * do not guarantee a machine check on error occurence)
		 * we will not clear the status at this wakeup unless
		 * we saw the same status at the previous poll.	 We will
		 * always process and log the current observations - it
		 * is only the clearing of MCi_STATUS which may be
		 * deferred until the next wakeup.
		 */
		if (isxpv || ismc || !IS_MCE_CANDIDATE(status)) {
			(void) cmi_hdl_wrmsr(hdl, IA32_MSR_MC(i, STATUS), 0ULL);
			goto serialize;
		}

		/*
		 * We have a polled observation of a machine check
		 * candidate.  If we saw essentially the same status at the
		 * last poll then clear the status now since this appears
		 * not to be a #MC candidate after all.	 If we see quite
		 * different status now then do not clear, but reconsider at
		 * the next poll.  In no actual machine check clears
		 * the status in the interim then the status should not
		 * keep changing forever (meaning we'd never clear it)
		 * since before long we'll simply have latched the highest-
		 * priority error and set the OVerflow bit.  Nonetheless
		 * we count how many times we defer clearing and after
		 * a while insist on clearing the status.
		 */
		pgbl = &pgcl->gcl_data[i];
		if (pgbl->gbl_clrdefcnt != 0) {
			/* We deferred clear on this bank at last wakeup */
			if (STATUS_EQV(status, pgcl->gcl_data[i].gbl_status) ||
			    pgbl->gbl_clrdefcnt > 5) {
				/*
				 * Status is unchanged so clear it now and,
				 * since we have already logged this info,
				 * avoid logging it again.
				 */
				gbl->gbl_status = 0;
				(void) cmi_hdl_wrmsr(hdl,
				    IA32_MSR_MC(i, STATUS), 0ULL);
			} else {
				/* Record deferral for next wakeup */
				gbl->gbl_clrdefcnt = pgbl->gbl_clrdefcnt + 1;
			}
		} else {
			/* Record initial deferral for next wakeup */
			gbl->gbl_clrdefcnt = 1;
			gcpu_deferrred_polled_clears++;
		}

serialize:
		{
#ifdef __xpv
			;
#else
			/*
			 * Intel Vol 3A says to execute a serializing
			 * instruction here, ie CPUID.	Well WRMSR is also
			 * defined to be serializing, so the status clear above
			 * should suffice.  To be a good citizen, and since
			 * some clears are deferred, we'll execute a CPUID
			 * instruction here.
			 */
			struct cpuid_regs tmp;
			(void) __cpuid_insn(&tmp);
#endif
		}
	}
}

/*ARGSUSED5*/
void
gcpu_mca_logout(cmi_hdl_t hdl, struct regs *rp, uint64_t bankmask,
    gcpu_mce_status_t *mcesp, boolean_t clrstatus, int what)
{
	gcpu_data_t *gcpu = cmi_hdl_getcmidata(hdl);
	gcpu_mca_t *mca = &gcpu->gcpu_mca;
	int nbanks = mca->gcpu_mca_nbanks;
	gcpu_bank_logout_t *gbl, *pgbl;
	gcpu_logout_t *gcl, *pgcl;
	int ismc = (rp != NULL);
	int ispoll = !ismc;
	int i, nerr = 0;
	cmi_errno_t err;
	uint64_t mcg_status;
	uint64_t disp;
	uint64_t cap;
	int first = -1;
	int last = -1;
	int willpanic = 0;

	if (cmi_hdl_rdmsr(hdl, IA32_MSR_MCG_STATUS, &mcg_status) !=
	    CMI_SUCCESS || cmi_hdl_rdmsr(hdl, IA32_MSR_MCG_CAP, &cap) !=
	    CMI_SUCCESS) {
		if (mcesp != NULL)
			mcesp->mce_nerr = mcesp->mce_disp = 0;
		return;
	}

	if (ismc) {
		gcl = mca->gcpu_mca_logout[GCPU_MCA_LOGOUT_EXCEPTION];
		pgcl = NULL;
	} else {
		int pidx = mca->gcpu_mca_nextpoll_idx;
		int ppidx = (pidx == GCPU_MCA_LOGOUT_POLLER_1) ?
		    GCPU_MCA_LOGOUT_POLLER_2 : GCPU_MCA_LOGOUT_POLLER_1;

		gcl = mca->gcpu_mca_logout[pidx];	/* current logout */
		pgcl = mca->gcpu_mca_logout[ppidx];	/* previous logout */
		mca->gcpu_mca_nextpoll_idx = ppidx;	/* switch next time */
	}

	gcl->gcl_timestamp = gethrtime_waitfree();
	gcl->gcl_mcg_status = mcg_status;
	gcl->gcl_ip = rp ? rp->r_pc : 0;

	gcl->gcl_flags = (rp && USERMODE(rp->r_cs)) ? GCPU_GCL_F_PRIV : 0;
	if (cap & MCG_CAP_TES_P)
		gcl->gcl_flags |= GCPU_GCL_F_TES_P;

	for (i = 0, gbl = &gcl->gcl_data[0]; i < nbanks; i++, gbl++) {
		uint64_t status, status2, addr, misc;
		int retries = gcpu_mca_telemetry_retries;

		gbl->gbl_status = 0;
		gbl->gbl_disp = 0;
		gbl->gbl_clrdefcnt = 0;

		/*
		 * Only logout from MCA banks we have initialized from at
		 * least one core.  If a core shares an MCA bank with another
		 * but perhaps lost the race to initialize it, then it must
		 * still be allowed to logout from the shared bank.
		 */
		if (!(gcpu->gcpu_shared->gcpus_actv_banks & 1 << i))
			continue;

		/*
		 * On a poll look only at the banks we've been asked to check.
		 */
		if (rp == NULL && !(bankmask & 1 << i))
			continue;


		if (cmi_hdl_rdmsr(hdl, IA32_MSR_MC(i, STATUS), &status) !=
		    CMI_SUCCESS)
			continue;

#ifndef __xpv
		gcpu_cmci_logout(hdl, i, &mca->gcpu_bank_cmci[i], status, what);
#endif

retry:
		if (!(status & MSR_MC_STATUS_VAL))
			continue;

		/* First and last bank that have valid status */
		if (first < 0)
			first = i;
		last = i;

		addr = -1;
		misc = 0;

		if ((status & MSR_MC_STATUS_ADDRV) ||
		    gcpu_force_addr_in_payload)
			(void) cmi_hdl_rdmsr(hdl, IA32_MSR_MC(i, ADDR), &addr);

		if (status & MSR_MC_STATUS_MISCV)
			(void) cmi_hdl_rdmsr(hdl, IA32_MSR_MC(i, MISC), &misc);

#ifndef __xpv
		gcpu_cmci_throttle(hdl, i, &mca->gcpu_bank_cmci[i], what);
#endif

		/*
		 * Allow the model-specific code to extract bank telemetry.
		 */
		cms_bank_logout(hdl, i, status, addr, misc, gcl->gcl_ms_logout);

		/*
		 * Not all cpu models assure us that the status/address/misc
		 * data will not change during the above sequence of MSR reads,
		 * or that it can only change by the addition of the OVerflow
		 * bit to the status register.  If the status has changed
		 * other than in the overflow bit then we attempt to reread
		 * for a consistent snapshot, but eventually give up and
		 * go with what we've got.  We only perform this check
		 * for a poll - a further #MC during a #MC will reset, and
		 * polled errors should not overwrite higher-priority
		 * trapping errors (but could set the overflow bit).
		 */
		if (ispoll && (err = cmi_hdl_rdmsr(hdl, IA32_MSR_MC(i, STATUS),
		    &status2)) == CMI_SUCCESS) {
			if (!STATUS_EQV(status, status2)) {
				if (retries-- > 0) {
					status = status2;
					goto retry;
				} else {
					gbl->gbl_disp |=
					    CMI_ERRDISP_INCONSISTENT;
				}
			}
		} else if (ispoll && err != CMI_SUCCESS) {
			gbl->gbl_disp |= CMI_ERRDISP_INCONSISTENT;
		}

		nerr++;
		gbl->gbl_status = status;
		gbl->gbl_addr = addr;
		gbl->gbl_misc = misc;

		/*
		 * For polled observation, if the count of deferred status
		 * clears updated in the clear_mc() is nonzero and the
		 * MCi_STATUS has not changed, the last wakeup has produced
		 * the ereport of the error. Therefore, clear the status in
		 * this wakeup to avoid duplicate ereport.
		 */
		pgbl = &pgcl->gcl_data[i];
		if (!isxpv && ispoll && IS_MCE_CANDIDATE(status) &&
		    pgbl->gbl_clrdefcnt != 0) {
			if (STATUS_EQV(status, pgcl->gcl_data[i].gbl_status)) {
				gbl->gbl_status = 0;
				(void) cmi_hdl_wrmsr(hdl,
				    IA32_MSR_MC(i, STATUS), 0ULL);
			}
		}
	}

	if (gcpu_mca_stack_flag)
		gcl->gcl_stackdepth = getpcstack(gcl->gcl_stack, FM_STK_DEPTH);
	else
		gcl->gcl_stackdepth = 0;

	/*
	 * Decide our disposition for this error or errors, and submit for
	 * logging and subsequent diagnosis.
	 */
	if (nerr != 0) {
		disp = gcpu_mca_process(hdl, rp, nerr, gcpu, gcl, ismc, mcesp);

		willpanic = (ismc && cmi_mce_response(rp, disp) == 0);

		if (!willpanic)
			clear_mc(first, last, ismc, clrstatus, hdl, gcl, pgcl);
	} else {
		disp = 0;
		if (mcesp) {
			mcesp->mce_nerr = mcesp->mce_disp = 0;
		}
	}

	/*
	 * Clear MCG_STATUS if MCIP is set (machine check in progress).
	 * If a second #MC had occured before now the system would have
	 * reset.  We can only do thise once gcpu_mca_process has copied
	 * the logout structure.
	 */
	if (ismc && mcg_status & MCG_STATUS_MCIP)
		(void) cmi_hdl_wrmsr(hdl, IA32_MSR_MCG_STATUS, 0);

	/*
	 * At this point we have read and logged all telemetry that is visible
	 * under the MCA.  On architectures for which the NorthBridge is
	 * on-chip this may include NB-observed errors, but where the NB
	 * is off chip it may have been the source of the #MC request and
	 * so we must call into the memory-controller driver to give it
	 * a chance to log errors.
	 */
	if (ismc) {
		cmi_mc_logout(hdl, 1, willpanic);
	}
}

#ifndef __xpv
int gcpu_mca_trap_vomit_summary = 0;

/*
 * On a native machine check exception we come here from mcetrap via
 * cmi_mca_trap.  A machine check on one cpu of a chip does not trap others
 * cpus of the chip, so it is possible that another cpu on this chip could
 * initiate a poll while we're in the #mc handler;  it is also possible that
 * this trap has occured during a poll on this cpu.  So we must acquire
 * the chip-wide poll lock, but be careful to avoid deadlock.
 *
 * The 'data' pointer cannot be NULL due to init order.
 */
uint64_t
gcpu_mca_trap(cmi_hdl_t hdl, struct regs *rp)
{
	gcpu_data_t *gcpu = cmi_hdl_getcmidata(hdl);
	kmutex_t *poll_lock = NULL;
	gcpu_mce_status_t mce;
	uint64_t mcg_status;
	int tooklock = 0;

	if (cmi_hdl_rdmsr(hdl, IA32_MSR_MCG_STATUS, &mcg_status) !=
	    CMI_SUCCESS || !(mcg_status & MCG_STATUS_MCIP))
		return (0);

	/*
	 * Synchronize with any poller from another core that may happen
	 * to share access to one or more of the MCA banks.
	 */
	if (gcpu->gcpu_shared != NULL)
		poll_lock = &gcpu->gcpu_shared->gcpus_poll_lock;

	if (poll_lock != NULL && !mutex_owned(poll_lock)) {
		/*
		 * The lock is not owned by the thread we have
		 * interrupted.  Spin for this adaptive lock.
		 */
		while (!mutex_tryenter(poll_lock)) {
			while (mutex_owner(poll_lock) != NULL)
				;
		}
		tooklock = 1;
	}

	gcpu_mca_logout(hdl, rp, 0, &mce, B_TRUE, GCPU_MPT_WHAT_MC_ERR);

	if (tooklock)
		mutex_exit(poll_lock);

	/*
	 * gcpu_mca_trap_vomit_summary may be set for debug assistance.
	 */
	if (mce.mce_nerr != 0 && gcpu_mca_trap_vomit_summary) {
		cmn_err(CE_WARN, "MCE: %u errors, disp=0x%llx, "
		    "%u PCC (%u ok), "
		    "%u UC (%d ok, %u poisoned), "
		    "%u forcefatal, %u ignored",
		    mce.mce_nerr, (u_longlong_t)mce.mce_disp,
		    mce.mce_npcc, mce.mce_npcc_ok,
		    mce.mce_nuc, mce.mce_nuc_ok, mce.mce_nuc_poisoned,
		    mce.mce_forcefatal, mce.mce_ignored);
	}

	return (mce.mce_disp);
}
#endif

/*ARGSUSED*/
void
gcpu_faulted_enter(cmi_hdl_t hdl)
{
	/* Nothing to do here */
}

/*ARGSUSED*/
void
gcpu_faulted_exit(cmi_hdl_t hdl)
{
	gcpu_data_t *gcpu = cmi_hdl_getcmidata(hdl);

	gcpu->gcpu_mca.gcpu_mca_flags |= GCPU_MCA_F_UNFAULTING;
}

/*
 * Write the requested values to the indicated MSRs.  Having no knowledge
 * of the model-specific requirements for writing to these model-specific
 * registers, we will only blindly write to those MSRs if the 'force'
 * argument is nonzero.  That option should only be used in prototyping
 * and debugging.
 */
/*ARGSUSED*/
cmi_errno_t
gcpu_msrinject(cmi_hdl_t hdl, cmi_mca_regs_t *regs, uint_t nregs,
    int force)
{
	int i, errs = 0;

	for (i = 0; i < nregs; i++) {
		uint_t msr = regs[i].cmr_msrnum;
		uint64_t val = regs[i].cmr_msrval;

		if (cms_present(hdl)) {
			if (cms_msrinject(hdl, msr, val) != CMS_SUCCESS)
				errs++;
		} else if (force) {
			errs += (cmi_hdl_wrmsr(hdl, msr, val) != CMI_SUCCESS);
		} else {
			errs++;
		}
	}

	return (errs == 0 ? CMI_SUCCESS : CMIERR_UNKNOWN);
}

/* deconfigure gcpu_mca_init() */
void
gcpu_mca_fini(cmi_hdl_t hdl)
{
	gcpu_data_t *gcpu = cmi_hdl_getcmidata(hdl);
	gcpu_mca_t *mca = &gcpu->gcpu_mca;
	int i;

	/*
	 * CPU startup code only calls cmi_mca_init if x86_featureset indicates
	 * both MCA and MCE support (i.e., X86FSET_MCA).  P5, K6, and earlier
	 * processors, which have their own more primitive way of doing
	 * machine checks, will not have cmi_mca_init called since their
	 * CPUID information will not indicate both MCA and MCE features.
	 */
	if (!is_x86_feature(x86_featureset, X86FSET_MCA))
		return;
#ifndef __xpv
	/*
	 * disable machine check in CR4
	 */
	cmi_ntv_hwdisable_mce(hdl);
#endif
	mutex_enter(&gcpu->gcpu_shared->gcpus_cfglock);
	gcpu_mca_poll_fini(hdl);
	mutex_exit(&gcpu->gcpu_shared->gcpus_cfglock);

	/*
	 * free resources allocated during init
	 */
	if (mca->gcpu_bank_cmci != NULL) {
		kmem_free(mca->gcpu_bank_cmci, sizeof (gcpu_mca_cmci_t) *
		    mca->gcpu_mca_nbanks);
	}

	for (i = 0; i < GCPU_MCA_LOGOUT_NUM; i++) {
		if (mca->gcpu_mca_logout[i] != NULL) {
			kmem_free(mca->gcpu_mca_logout[i], mca->gcpu_mca_lgsz);
		}
	}

	if (mca->gcpu_mca_bioscfg.bios_bankcfg != NULL) {
		kmem_free(mca->gcpu_mca_bioscfg.bios_bankcfg,
		    sizeof (struct gcpu_bios_bankcfg) * mca->gcpu_mca_nbanks);
	}
}
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 * Copyright (c) 2018, Joyent, Inc.
 */
/*
 * Copyright (c) 2010, Intel Corporation.
 * All rights reserved.
 */

/*
 * Native MCA polling.  We establish an ommipresent cyclic to fire on all
 * online cpus to check their MCA state and log any valid errors for
 * diagnosis.
 */

#include <sys/types.h>
#include <sys/atomic.h>
#include <sys/cyclic.h>
#include <sys/x86_archext.h>
#include <sys/mca_x86.h>

#include "gcpu.h"

hrtime_t gcpu_mca_poll_interval = NANOSEC * 10ULL;	/* tuneable */
static cyclic_id_t gcpu_mca_poll_cycid;
static volatile uint_t gcpu_mca_poll_inits;
extern int gcpu_poll_trace_always;
extern uint_t gcpu_poll_trace_nent;

/*
 * Return nonzero of the given handle should poll the MCH.  We stick with
 * the same handle as before unless the timestamp has not been updated
 * for a while.  There is no need to keep a hold on the mch_poll_owner
 * handle.
 */

static kmutex_t mch_poll_lock;
static hrtime_t mch_poll_timestamp;
static cmi_hdl_t mch_poll_owner;

static int
mch_pollowner(cmi_hdl_t hdl)
{
	hrtime_t now = gethrtime_waitfree();
	int dopoll = 0;

	mutex_enter(&mch_poll_lock);
	if (now - mch_poll_timestamp > 2 * gcpu_mca_poll_interval ||
	    mch_poll_timestamp == 0) {
		mch_poll_owner = hdl;
		dopoll = 1;
	} else if (mch_poll_owner == hdl) {
		dopoll = 1;
	}

	if (dopoll)
		mch_poll_timestamp = now;

	mutex_exit(&mch_poll_lock);
	return (dopoll);
}


static void
gcpu_ntv_mca_poll(cmi_hdl_t hdl, int what)
{
	gcpu_data_t *gcpu = cmi_hdl_getcmidata(hdl);
	gcpu_mca_t *mca = &gcpu->gcpu_mca;
	gcpu_mce_status_t mce;
	int willpanic;
	uint64_t bankmask;

	ASSERT(MUTEX_HELD(&gcpu->gcpu_shared->gcpus_poll_lock));

	/* Enable CMCI in first poll if is supported */
	if ((mca->gcpu_mca_flags & GCPU_MCA_F_CMCI_ENABLE) != 0 &&
	    (!mca->gcpu_mca_first_poll_cmci_enabled)) {
		int i;
		uint64_t ctl2;

		for (i = 0; i < mca->gcpu_mca_nbanks; i++) {
			if (mca->gcpu_bank_cmci[i].cmci_cap) {
				(void) cmi_hdl_rdmsr(hdl, IA32_MSR_MC_CTL2(i),
				    &ctl2);
				ctl2 |= MSR_MC_CTL2_EN;
				(void) cmi_hdl_wrmsr(hdl, IA32_MSR_MC_CTL2(i),
				    ctl2);
				mca->gcpu_bank_cmci[i].cmci_enabled = 1;
			}
		}
		mca->gcpu_mca_first_poll_cmci_enabled = 1;
	}

	if (mca->gcpu_mca_flags & GCPU_MCA_F_UNFAULTING) {
		int i;

		mca->gcpu_mca_flags &= ~GCPU_MCA_F_UNFAULTING;
		gcpu_poll_trace(&gcpu->gcpu_mca.gcpu_polltrace,
		    GCPU_MPT_WHAT_UNFAULTING, 0);

		/*
		 * On the first cyclic poll after unfaulting a CPU we
		 * clear the status registers; see gcpu_faulted_exit
		 * for details.  We don't do this if the poll was
		 * initiated manually (presumably from some injection
		 * activity).
		 */
		if (what == GCPU_MPT_WHAT_CYC_ERR) {
			for (i = 0; i < mca->gcpu_mca_nbanks; i++) {
				(void) cmi_hdl_wrmsr(hdl,
				    IA32_MSR_MC(i, STATUS), 0ULL);
			}
			return;
		}
	}

	/*
	 * Logout errors of the MCA banks of this cpu.
	 */
	if (what == GCPU_MPT_WHAT_CMCI_ERR) {
		/*
		 * for CMCI, all banks should be scanned for log out
		 */
		bankmask = -1ULL;
	} else {
		bankmask = cms_poll_ownermask(hdl, gcpu_mca_poll_interval);
	}
	gcpu_mca_logout(hdl, NULL, bankmask, &mce, B_TRUE, what);

	if (mce.mce_nerr != 0)
		gcpu_poll_trace(&gcpu->gcpu_mca.gcpu_polltrace, what,
		    mce.mce_nerr);

	mca->gcpu_mca_lastpoll = gethrtime_waitfree();

	willpanic = mce.mce_disp & CMI_ERRDISP_FORCEFATAL && cmi_panic_on_ue();

	if (what != GCPU_MPT_WHAT_CMCI_ERR) {
		/*
		 * Call to the memory-controller driver which may report some
		 * errors not visible under the MCA (for off-chip NB).
		 * Since there is typically a single MCH we arrange that
		 * just one cpu perform this task at each cyclic fire.
		 */
		if (mch_pollowner(hdl))
			cmi_mc_logout(hdl, 0, willpanic);
	}

	/*
	 * In the common case any polled error is considered non-fatal,
	 * even if it indicates PCC or UC etc.  The only condition on which
	 * we will panic for a polled error is if model-specific support
	 * forces the error to be terminal regardless of how it is
	 * encountered.
	 */
	if (willpanic) {
#ifdef DEBUG
		cmn_err(CE_WARN, "MCA Poll: %u errors, disp=0x%llx, "
		    "%u PCC (%u ok), "
		    "%u UC (%u ok, %u poisoned), "
		    "%u forcefatal, %u ignored",
		    mce.mce_nerr, (u_longlong_t)mce.mce_disp,
		    mce.mce_npcc, mce.mce_npcc_ok,
		    mce.mce_nuc, mce.mce_nuc_ok, mce.mce_nuc_poisoned,
		    mce.mce_forcefatal, mce.mce_ignored);

#endif
		fm_panic("Unrecoverable Machine-Check Exception (Polled)");
	}
}

/*
 * See gcpu_mca_trap for an explanation of why preemption is disabled here.
 * Note that we disable preemption and then contend for an adaptive mutex -
 * we could block during the mutex operation, but once we return with the
 * mutex held we nust perform no operation that can block and we cannot
 * be preempted so we will stay on cpu for the duration.  The disabling
 * of preemption also means we cannot migrate cpus once we have returned
 * with the mutex held - cyclic invocations can't migrate, anyway, but
 * others could if they have failed to bind before this point.
 */
static void
gcpu_ntv_mca_poll_wrapper(cmi_hdl_t hdl, int what)
{
	gcpu_data_t *gcpu;

	if (hdl == NULL || (gcpu = cmi_hdl_getcmidata(hdl)) == NULL ||
	    gcpu->gcpu_mca.gcpu_mca_lgsz == 0)
		return;

	kpreempt_disable();
	mutex_enter(&gcpu->gcpu_shared->gcpus_poll_lock);
	gcpu_ntv_mca_poll(hdl, what);
	mutex_exit(&gcpu->gcpu_shared->gcpus_poll_lock);
	kpreempt_enable();
}

static void
gcpu_ntv_mca_poll_cyclic(void *arg)
{
	gcpu_ntv_mca_poll_wrapper((cmi_hdl_t)arg, GCPU_MPT_WHAT_CYC_ERR);
}

/*ARGSUSED*/
static void
gcpu_ntv_mca_poll_online(void *arg, cpu_t *cp, cyc_handler_t *cyh,
    cyc_time_t *cyt)
{
	cmi_hdl_t hdl;

	/*
	 * Lookup and hold a handle for this cpu (any hold released in
	 * our offline function).  If we chose not to initialize a handle
	 * for this cpu back at cmi_init time then this lookup will return
	 * NULL, so the cyh_func we appoint must be prepared for that.
	 */
	hdl = cmi_hdl_lookup(CMI_HDL_NATIVE, cmi_ntv_hwchipid(cp),
	    cmi_ntv_hwcoreid(cp), cmi_ntv_hwstrandid(cp));

	cyt->cyt_when = 0;
	cyt->cyt_interval = gcpu_mca_poll_interval;
	cyh->cyh_func = gcpu_ntv_mca_poll_cyclic;
	cyh->cyh_arg = (void *)hdl;
	cyh->cyh_level = CY_LOW_LEVEL;
}

/*ARGSUSED*/
static void
gcpu_ntv_mca_poll_offline(void *arg, cpu_t *cpu, void *cyh_arg)
{
	cmi_hdl_t hdl = (cmi_hdl_t)cyh_arg;

	if (hdl != NULL)
		cmi_hdl_rele(hdl);
}

static void
gcpu_ntv_mca_poll_start(void)
{
	cyc_omni_handler_t cyo;

	if (gcpu_mca_poll_interval == 0 || gcpu_mca_poll_inits == 0)
		return;

	cyo.cyo_online = gcpu_ntv_mca_poll_online;
	cyo.cyo_offline = gcpu_ntv_mca_poll_offline;
	cyo.cyo_arg = NULL;

	mutex_enter(&cpu_lock);
	gcpu_mca_poll_cycid = cyclic_add_omni(&cyo);
	mutex_exit(&cpu_lock);
}

/*
 * gcpu_mca_poll_init is called from gcpu_mca_init for each cpu handle
 * that we initialize for.  It should prepare for polling by allocating
 * control structures and the like, but must not kick polling off yet.
 */

void
gcpu_mca_poll_init(cmi_hdl_t hdl)
{
	gcpu_data_t *gcpu = cmi_hdl_getcmidata(hdl);
	gcpu_poll_trace_ctl_t *ptc = &gcpu->gcpu_mca.gcpu_polltrace;

	ASSERT(cmi_hdl_class(hdl) == CMI_HDL_NATIVE);

	gcpu_poll_trace_init(ptc);

	atomic_inc_uint(&gcpu_mca_poll_inits);
}

/* deconfigure gcpu_mca_poll_init() */
void
gcpu_mca_poll_fini(cmi_hdl_t hdl)
{
	gcpu_data_t *gcpu = cmi_hdl_getcmidata(hdl);
	gcpu_poll_trace_ctl_t *ptc = &gcpu->gcpu_mca.gcpu_polltrace;

	ASSERT(cmi_hdl_class(hdl) == CMI_HDL_NATIVE);

	if (gcpu_poll_trace_always && (ptc->mptc_tbufs != NULL)) {
		kmem_free(ptc->mptc_tbufs, sizeof (gcpu_poll_trace_t) *
		    gcpu_poll_trace_nent);
	}

	atomic_dec_uint(&gcpu_mca_poll_inits);
}

void
gcpu_mca_poll_start(cmi_hdl_t hdl)
{
	ASSERT(cmi_hdl_class(hdl) == CMI_HDL_NATIVE);
	gcpu_ntv_mca_poll_start();
}

void
gcpu_hdl_poke(cmi_hdl_t hdl)
{
	ASSERT(cmi_hdl_class(hdl) == CMI_HDL_NATIVE);
	gcpu_ntv_mca_poll_wrapper(hdl, GCPU_MPT_WHAT_POKE_ERR);
}

void
gcpu_cmci_trap(cmi_hdl_t hdl)
{
	gcpu_ntv_mca_poll_wrapper(hdl, GCPU_MPT_WHAT_CMCI_ERR);
}
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

/*
 * Generic x86 CPU MCA poller - support functions for native and xpv pollers.
 */

#include <sys/types.h>
#include <sys/sysmacros.h>
#include <sys/sdt.h>
#include <sys/cmn_err.h>

#include "gcpu.h"

uint_t gcpu_poll_trace_nent = 100;

#ifdef DEBUG
int gcpu_poll_trace_always = 1;
#else
int gcpu_poll_trace_always = 0;
#endif

void
gcpu_poll_trace(gcpu_poll_trace_ctl_t *ptc, uint8_t what, uint8_t nerr)
{
	gcpu_poll_trace_t *pt;
	uint_t next;

	DTRACE_PROBE2(gcpu__mca__poll__trace, uint32_t, what, uint32_t, nerr);

	if (ptc->mptc_tbufs == NULL)
		return; /* poll trace buffer is disabled */

	next = (ptc->mptc_curtrace + 1) % gcpu_poll_trace_nent;
	pt = &ptc->mptc_tbufs[next];

	pt->mpt_when = 0;
	pt->mpt_what = what;

	pt->mpt_nerr = MIN(nerr, UINT8_MAX);

	pt->mpt_when = gethrtime_waitfree();
	ptc->mptc_curtrace = next;
}

void
gcpu_poll_trace_init(gcpu_poll_trace_ctl_t *ptc)
{
	gcpu_poll_trace_t *tbufs = NULL;

	if (gcpu_poll_trace_always) {
		tbufs = kmem_zalloc(sizeof (gcpu_poll_trace_t) *
		    gcpu_poll_trace_nent, KM_SLEEP);
	}

	ptc->mptc_tbufs = tbufs;
	ptc->mptc_curtrace = 0;
}