os - zygaena/hammerhead

root / base / usr / src / uts / intel / os
os Plain Text 28547 lines 1.1 MB
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 *
 * Copyright 2018 Joyent, Inc.
 */

/*
 * Kernel/Debugger Interface (KDI) routines.  Called during debugger under
 * various system states (boot, while running, while the debugger has control).
 * Functions intended for use while the debugger has control may not grab any
 * locks or perform any functions that assume the availability of other system
 * services.
 */

#include <sys/systm.h>
#include <sys/x86_archext.h>
#include <sys/kdi_impl.h>
#include <sys/smp_impldefs.h>
#include <sys/psm_types.h>
#include <sys/segments.h>
#include <sys/archsystm.h>
#include <sys/controlregs.h>
#include <sys/trap.h>
#include <sys/kobj.h>
#include <sys/kobj_impl.h>
#include <sys/clock_impl.h>

static void
kdi_system_claim(void)
{
	lbolt_debug_entry();

	psm_notifyf(PSM_DEBUG_ENTER);
}

static void
kdi_system_release(void)
{
	psm_notifyf(PSM_DEBUG_EXIT);

	lbolt_debug_return();
}

static cpu_t *
kdi_gdt2cpu(uintptr_t gdtbase)
{
	cpu_t *cp = cpu_list;

	if (cp == NULL)
		return (NULL);

	do {
		if (gdtbase == (uintptr_t)cp->cpu_gdt)
			return (cp);
	} while ((cp = cp->cpu_next) != cpu_list);

	return (NULL);
}

uintptr_t
kdi_gdt2gsbase(uintptr_t gdtbase)
{
	return ((uintptr_t)kdi_gdt2cpu(gdtbase));
}

static uintptr_t
kdi_get_userlimit(void)
{
	return (_userlimit);
}

static int
kdi_get_cpuinfo(uint_t *vendorp, uint_t *familyp, uint_t *modelp)
{
	desctbr_t gdtr;
	cpu_t *cpu;

	/*
	 * CPU doesn't work until the GDT and gs/GSBASE have been set up.
	 * Boot-loaded kmdb will call us well before then, so we have to
	 * find the current cpu_t the hard way.
	 */
	rd_gdtr(&gdtr);
	if ((cpu = kdi_gdt2cpu(gdtr.dtr_base)) == NULL ||
	    !cpuid_checkpass(cpu, 1))
		return (EAGAIN); /* cpuid isn't done yet */

	*vendorp = cpuid_getvendor(cpu);
	*familyp = cpuid_getfamily(cpu);
	*modelp = cpuid_getmodel(cpu);

	return (0);
}

void
kdi_idtr_set(gate_desc_t *idt, size_t limit)
{
	desctbr_t idtr;

	/*
	 * This rare case could happen if we entered kmdb whilst still on the
	 * fake CPU set up by boot_kdi_tmpinit().  We're trying to restore the
	 * kernel's IDT that we saved on entry, but it was from the fake cpu_t
	 * rather than the real IDT (which is still boot's).  It's unpleasant,
	 * but we just encode knowledge that it's idt0 we want to restore.
	 */
	if (idt == NULL)
		idt = idt0;

	CPU->cpu_m.mcpu_idt = idt;
	idtr.dtr_base = (uintptr_t)idt;
	idtr.dtr_limit = limit;
	kdi_idtr_write(&idtr);
}

static void
kdi_plat_call(void (*platfn)(void))
{
	if (platfn != NULL)
		platfn();
}

/*
 * On Intel, most of these are shared between i86*, so this is really an
 * arch_kdi_init().
 */
void
mach_kdi_init(kdi_t *kdi)
{
	kdi->kdi_plat_call = kdi_plat_call;
	kdi->kdi_kmdb_enter = kmdb_enter;
	kdi->mkdi_activate = kdi_activate;
	kdi->mkdi_deactivate = kdi_deactivate;
	kdi->mkdi_idt_switch = kdi_idt_switch;
	kdi->mkdi_update_drreg = kdi_update_drreg;
	kdi->mkdi_get_userlimit = kdi_get_userlimit;
	kdi->mkdi_get_cpuinfo = kdi_get_cpuinfo;
	kdi->mkdi_stop_slaves = kdi_stop_slaves;
	kdi->mkdi_start_slaves = kdi_start_slaves;
	kdi->mkdi_slave_wait = kdi_slave_wait;
	kdi->mkdi_memrange_add = kdi_memrange_add;
	kdi->mkdi_reboot = kdi_reboot;
}

void
plat_kdi_init(kdi_t *kdi)
{
	kdi->pkdi_system_claim = kdi_system_claim;
	kdi->pkdi_system_release = kdi_system_release;
}
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved.
 */

/*	Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T	*/
/*	  All Rights Reserved	*/
/*
 * Copyright (c) 2018, Joyent, Inc.
 * Copyright 2012 Nexenta Systems, Inc.  All rights reserved.
 * Copyright 2023 Oxide Computer Company
 */

#include <sys/param.h>
#include <sys/types.h>
#include <sys/vmparam.h>
#include <sys/systm.h>
#include <sys/signal.h>
#include <sys/stack.h>
#include <sys/regset.h>
#include <sys/privregs.h>
#include <sys/frame.h>
#include <sys/proc.h>
#include <sys/psw.h>
#include <sys/siginfo.h>
#include <sys/cpuvar.h>
#include <sys/asm_linkage.h>
#include <sys/kmem.h>
#include <sys/errno.h>
#include <sys/bootconf.h>
#include <sys/archsystm.h>
#include <sys/debug.h>
#include <sys/elf.h>
#include <sys/spl.h>
#include <sys/time.h>
#include <sys/atomic.h>
#include <sys/sysmacros.h>
#include <sys/cmn_err.h>
#include <sys/modctl.h>
#include <sys/kobj.h>
#include <sys/panic.h>
#include <sys/reboot.h>
#include <sys/time.h>
#include <sys/fp.h>
#include <sys/x86_archext.h>
#include <sys/auxv.h>
#include <sys/auxv_386.h>
#include <sys/dtrace.h>
#include <sys/brand.h>
#include <sys/machbrand.h>
#include <sys/cmn_err.h>

/*
 * Map an fnsave-formatted save area into an fxsave-formatted save area.
 *
 * Most fields are the same width, content and semantics.  However
 * the tag word is compressed.
 */
static void
fnsave_to_fxsave(const struct fnsave_state *fn, struct fxsave_state *fx)
{
	uint_t i, tagbits;

	fx->fx_fcw = fn->f_fcw;
	fx->fx_fsw = fn->f_fsw;

	/*
	 * copy element by element (because of holes)
	 */
	for (i = 0; i < 8; i++)
		bcopy(&fn->f_st[i].fpr_16[0], &fx->fx_st[i].fpr_16[0],
		    sizeof (fn->f_st[0].fpr_16)); /* 80-bit x87-style floats */

	/*
	 * synthesize compressed tag bits
	 */
	fx->fx_fctw = 0;
	for (tagbits = fn->f_ftw, i = 0; i < 8; i++, tagbits >>= 2)
		if ((tagbits & 3) != 3)
			fx->fx_fctw |= (1 << i);

	fx->fx_fop = fn->f_fop;

	fx->fx_rip = (uint64_t)fn->f_eip;
	fx->fx_rdp = (uint64_t)fn->f_dp;
}

/*
 * Map from an fxsave-format save area to an fnsave-format save area.
 */
static void
fxsave_to_fnsave(const struct fxsave_state *fx, struct fnsave_state *fn)
{
	uint_t i, top, tagbits;

	fn->f_fcw = fx->fx_fcw;
	fn->__f_ign0 = 0;
	fn->f_fsw = fx->fx_fsw;
	fn->__f_ign1 = 0;

	top = (fx->fx_fsw & FPS_TOP) >> 11;

	/*
	 * copy element by element (because of holes)
	 */
	for (i = 0; i < 8; i++)
		bcopy(&fx->fx_st[i].fpr_16[0], &fn->f_st[i].fpr_16[0],
		    sizeof (fn->f_st[0].fpr_16)); /* 80-bit x87-style floats */

	/*
	 * synthesize uncompressed tag bits
	 */
	fn->f_ftw = 0;
	for (tagbits = fx->fx_fctw, i = 0; i < 8; i++, tagbits >>= 1) {
		uint_t ibit, expo;
		const uint16_t *fpp;
		static const uint16_t zero[5] = { 0, 0, 0, 0, 0 };

		if ((tagbits & 1) == 0) {
			fn->f_ftw |= 3 << (i << 1);	/* empty */
			continue;
		}

		/*
		 * (tags refer to *physical* registers)
		 */
		fpp = &fx->fx_st[(i - top + 8) & 7].fpr_16[0];
		ibit = fpp[3] >> 15;
		expo = fpp[4] & 0x7fff;

		if (ibit && expo != 0 && expo != 0x7fff)
			continue;			/* valid fp number */

		if (bcmp(fpp, &zero, sizeof (zero)))
			fn->f_ftw |= 2 << (i << 1);	/* NaN */
		else
			fn->f_ftw |= 1 << (i << 1);	/* fp zero */
	}

	fn->f_fop = fx->fx_fop;

	fn->__f_ign2 = 0;
	fn->f_eip = (uint32_t)fx->fx_rip;
	fn->f_cs = U32CS_SEL;
	fn->f_dp = (uint32_t)fx->fx_rdp;
	fn->f_ds = UDS_SEL;
	fn->__f_ign3 = 0;
}

/*
 * Map from an fpregset_t into an fxsave-format save area
 */
static void
fpregset_to_fxsave(const fpregset_t *fp, struct fxsave_state *fx)
{
	bcopy(fp, fx, sizeof (*fx));
	/*
	 * avoid useless #gp exceptions - mask reserved bits
	 */
	fx->fx_mxcsr &= sse_mxcsr_mask;
}

/*
 * Map from an fxsave-format save area into a fpregset_t
 */
static void
fxsave_to_fpregset(const struct fxsave_state *fx, fpregset_t *fp)
{
	bcopy(fx, fp, sizeof (*fx));
}

#if defined(_SYSCALL32_IMPL)
static void
fpregset32_to_fxsave(const fpregset32_t *fp, struct fxsave_state *fx)
{
	const struct fpchip32_state *fc = &fp->fp_reg_set.fpchip_state;

	fnsave_to_fxsave((const struct fnsave_state *)fc, fx);
	/*
	 * avoid useless #gp exceptions - mask reserved bits
	 */
	fx->fx_mxcsr = sse_mxcsr_mask & fc->mxcsr;
	bcopy(&fc->xmm[0], &fx->fx_xmm[0], sizeof (fc->xmm));
}

static void
fxsave_to_fpregset32(const struct fxsave_state *fx, fpregset32_t *fp)
{
	struct fpchip32_state *fc = &fp->fp_reg_set.fpchip_state;

	fxsave_to_fnsave(fx, (struct fnsave_state *)fc);
	fc->mxcsr = fx->fx_mxcsr;
	bcopy(&fx->fx_xmm[0], &fc->xmm[0], sizeof (fc->xmm));
}

static void
fpregset_nto32(const fpregset_t *src, fpregset32_t *dst)
{
	fxsave_to_fpregset32((struct fxsave_state *)src, dst);
	dst->fp_reg_set.fpchip_state.status =
	    src->fp_reg_set.fpchip_state.status;
	dst->fp_reg_set.fpchip_state.xstatus =
	    src->fp_reg_set.fpchip_state.xstatus;
}

static void
fpregset_32ton(const fpregset32_t *src, fpregset_t *dst)
{
	fpregset32_to_fxsave(src, (struct fxsave_state *)dst);
	dst->fp_reg_set.fpchip_state.status =
	    src->fp_reg_set.fpchip_state.status;
	dst->fp_reg_set.fpchip_state.xstatus =
	    src->fp_reg_set.fpchip_state.xstatus;
}
#endif

/*
 * Set floating-point registers from a native fpregset_t.
 */
void
setfpregs(klwp_t *lwp, fpregset_t *fp)
{
	fpu_set_fpregset(lwp, fp);
}

/*
 * Get floating-point registers into a native fpregset_t.
 */
void
getfpregs(klwp_t *lwp, fpregset_t *fp)
{
	bzero(fp, sizeof (*fp));
	fpu_get_fpregset(lwp, fp);
}

#if defined(_SYSCALL32_IMPL)

/*
 * Set floating-point registers from an fpregset32_t.
 */
void
setfpregs32(klwp_t *lwp, fpregset32_t *fp)
{
	fpregset_t fpregs;

	fpregset_32ton(fp, &fpregs);
	setfpregs(lwp, &fpregs);
}

/*
 * Get floating-point registers into an fpregset32_t.
 */
void
getfpregs32(klwp_t *lwp, fpregset32_t *fp)
{
	fpregset_t fpregs;

	getfpregs(lwp, &fpregs);
	fpregset_nto32(&fpregs, fp);
}

#endif	/* _SYSCALL32_IMPL */

/*
 * Return the general registers
 */
void
getgregs(klwp_t *lwp, gregset_t grp)
{
	struct regs *rp = lwptoregs(lwp);
	struct pcb *pcb = &lwp->lwp_pcb;
	int thisthread = lwptot(lwp) == curthread;

	grp[REG_RDI] = rp->r_rdi;
	grp[REG_RSI] = rp->r_rsi;
	grp[REG_RDX] = rp->r_rdx;
	grp[REG_RCX] = rp->r_rcx;
	grp[REG_R8] = rp->r_r8;
	grp[REG_R9] = rp->r_r9;
	grp[REG_RAX] = rp->r_rax;
	grp[REG_RBX] = rp->r_rbx;
	grp[REG_RBP] = rp->r_rbp;
	grp[REG_R10] = rp->r_r10;
	grp[REG_R11] = rp->r_r11;
	grp[REG_R12] = rp->r_r12;
	grp[REG_R13] = rp->r_r13;
	grp[REG_R14] = rp->r_r14;
	grp[REG_R15] = rp->r_r15;
	grp[REG_FSBASE] = pcb->pcb_fsbase;
	grp[REG_GSBASE] = pcb->pcb_gsbase;
	if (thisthread)
		kpreempt_disable();
	if (PCB_NEED_UPDATE_SEGS(pcb)) {
		grp[REG_DS] = pcb->pcb_ds;
		grp[REG_ES] = pcb->pcb_es;
		grp[REG_FS] = pcb->pcb_fs;
		grp[REG_GS] = pcb->pcb_gs;
	} else {
		grp[REG_DS] = rp->r_ds;
		grp[REG_ES] = rp->r_es;
		grp[REG_FS] = rp->r_fs;
		grp[REG_GS] = rp->r_gs;
	}
	if (thisthread)
		kpreempt_enable();
	grp[REG_TRAPNO] = rp->r_trapno;
	grp[REG_ERR] = rp->r_err;
	grp[REG_RIP] = rp->r_rip;
	grp[REG_CS] = rp->r_cs;
	grp[REG_SS] = rp->r_ss;
	grp[REG_RFL] = rp->r_rfl;
	grp[REG_RSP] = rp->r_rsp;
}

#if defined(_SYSCALL32_IMPL)

void
getgregs32(klwp_t *lwp, gregset32_t grp)
{
	struct regs *rp = lwptoregs(lwp);
	struct pcb *pcb = &lwp->lwp_pcb;
	int thisthread = lwptot(lwp) == curthread;

	if (thisthread)
		kpreempt_disable();
	if (PCB_NEED_UPDATE_SEGS(pcb)) {
		grp[GS] = (uint16_t)pcb->pcb_gs;
		grp[FS] = (uint16_t)pcb->pcb_fs;
		grp[DS] = (uint16_t)pcb->pcb_ds;
		grp[ES] = (uint16_t)pcb->pcb_es;
	} else {
		grp[GS] = (uint16_t)rp->r_gs;
		grp[FS] = (uint16_t)rp->r_fs;
		grp[DS] = (uint16_t)rp->r_ds;
		grp[ES] = (uint16_t)rp->r_es;
	}
	if (thisthread)
		kpreempt_enable();
	grp[EDI] = (greg32_t)rp->r_rdi;
	grp[ESI] = (greg32_t)rp->r_rsi;
	grp[EBP] = (greg32_t)rp->r_rbp;
	grp[ESP] = 0;
	grp[EBX] = (greg32_t)rp->r_rbx;
	grp[EDX] = (greg32_t)rp->r_rdx;
	grp[ECX] = (greg32_t)rp->r_rcx;
	grp[EAX] = (greg32_t)rp->r_rax;
	grp[TRAPNO] = (greg32_t)rp->r_trapno;
	grp[ERR] = (greg32_t)rp->r_err;
	grp[EIP] = (greg32_t)rp->r_rip;
	grp[CS] = (uint16_t)rp->r_cs;
	grp[EFL] = (greg32_t)rp->r_rfl;
	grp[UESP] = (greg32_t)rp->r_rsp;
	grp[SS] = (uint16_t)rp->r_ss;
}

void
ucontext_32ton(const ucontext32_t *src, ucontext_t *dst)
{
	mcontext_t *dmc = &dst->uc_mcontext;
	const mcontext32_t *smc = &src->uc_mcontext;

	bzero(dst, sizeof (*dst));
	dst->uc_flags = src->uc_flags;
	dst->uc_link = (ucontext_t *)(uintptr_t)src->uc_link;

	bcopy(&src->uc_sigmask, &dst->uc_sigmask, sizeof (dst->uc_sigmask));

	dst->uc_stack.ss_sp = (void *)(uintptr_t)src->uc_stack.ss_sp;
	dst->uc_stack.ss_size = (size_t)src->uc_stack.ss_size;
	dst->uc_stack.ss_flags = src->uc_stack.ss_flags;

	dmc->gregs[REG_GS] = (greg_t)(uint32_t)smc->gregs[GS];
	dmc->gregs[REG_FS] = (greg_t)(uint32_t)smc->gregs[FS];
	dmc->gregs[REG_ES] = (greg_t)(uint32_t)smc->gregs[ES];
	dmc->gregs[REG_DS] = (greg_t)(uint32_t)smc->gregs[DS];
	dmc->gregs[REG_RDI] = (greg_t)(uint32_t)smc->gregs[EDI];
	dmc->gregs[REG_RSI] = (greg_t)(uint32_t)smc->gregs[ESI];
	dmc->gregs[REG_RBP] = (greg_t)(uint32_t)smc->gregs[EBP];
	dmc->gregs[REG_RBX] = (greg_t)(uint32_t)smc->gregs[EBX];
	dmc->gregs[REG_RDX] = (greg_t)(uint32_t)smc->gregs[EDX];
	dmc->gregs[REG_RCX] = (greg_t)(uint32_t)smc->gregs[ECX];
	dmc->gregs[REG_RAX] = (greg_t)(uint32_t)smc->gregs[EAX];
	dmc->gregs[REG_TRAPNO] = (greg_t)(uint32_t)smc->gregs[TRAPNO];
	dmc->gregs[REG_ERR] = (greg_t)(uint32_t)smc->gregs[ERR];
	dmc->gregs[REG_RIP] = (greg_t)(uint32_t)smc->gregs[EIP];
	dmc->gregs[REG_CS] = (greg_t)(uint32_t)smc->gregs[CS];
	dmc->gregs[REG_RFL] = (greg_t)(uint32_t)smc->gregs[EFL];
	dmc->gregs[REG_RSP] = (greg_t)(uint32_t)smc->gregs[UESP];
	dmc->gregs[REG_SS] = (greg_t)(uint32_t)smc->gregs[SS];

	/*
	 * A valid fpregs is only copied in if uc.uc_flags has UC_FPU set
	 * otherwise there is no guarantee that anything in fpregs is valid.
	 */
	if (src->uc_flags & UC_FPU)
		fpregset_32ton(&src->uc_mcontext.fpregs,
		    &dst->uc_mcontext.fpregs);

	if (src->uc_flags & UC_XSAVE) {
		dst->uc_xsave = (long)(uint32_t)src->uc_xsave;
	} else {
		dst->uc_xsave = 0;
	}
}

#endif	/* _SYSCALL32_IMPL */

/*
 * Return the user-level PC.
 * If in a system call, return the address of the syscall trap.
 */
greg_t
getuserpc(void)
{
	greg_t upc = lwptoregs(ttolwp(curthread))->r_pc;
	uint32_t insn;

	if (curthread->t_sysnum == 0)
		return (upc);

	/*
	 * We might've gotten here from sysenter (0xf 0x34),
	 * syscall (0xf 0x5) or lcall (0x9a 0 0 0 0 0x27 0).
	 *
	 * Go peek at the binary to figure it out..
	 */
	if (fuword32((void *)(upc - 2), &insn) != -1 &&
	    (insn & 0xffff) == 0x340f || (insn & 0xffff) == 0x050f)
		return (upc - 2);
	return (upc - 7);
}

/*
 * Protect segment registers from non-user privilege levels and GDT selectors
 * other than USER_CS, USER_DS and lwp FS and GS values.  If the segment
 * selector is non-null and not USER_CS/USER_DS, we make sure that the
 * TI bit is set to point into the LDT and that the RPL is set to 3.
 *
 * Since struct regs stores each 16-bit segment register as a 32-bit greg_t, we
 * also explicitly zero the top 16 bits since they may be coming from the
 * user's address space via setcontext(2) or /proc.
 *
 * Note about null selector. When running on the hypervisor if we allow a
 * process to set its %cs to null selector with RPL of 0 the hypervisor will
 * crash the domain. If running on bare metal we would get a #gp fault and
 * be able to kill the process and continue on. Therefore we make sure to
 * force RPL to SEL_UPL even for null selector when setting %cs.
 */

#if defined(IS_CS) || defined(IS_NOT_CS)
#error	"IS_CS and IS_NOT_CS already defined"
#endif

#define	IS_CS		1
#define	IS_NOT_CS	0

/*ARGSUSED*/
static greg_t
fix_segreg(greg_t sr, int iscs, model_t datamodel)
{
	switch (sr &= 0xffff) {

	case 0:
		if (iscs == IS_CS)
			return (0 | SEL_UPL);
		else
			return (0);

	/*
	 * If lwp attempts to switch data model then force their
	 * code selector to be null selector.
	 */
	case U32CS_SEL:
		if (datamodel == DATAMODEL_NATIVE)
			return (0 | SEL_UPL);
		else
			return (sr);

	case UCS_SEL:
		if (datamodel == DATAMODEL_ILP32)
			return (0 | SEL_UPL);
	/*FALLTHROUGH*/
	case UDS_SEL:
	case LWPFS_SEL:
	case LWPGS_SEL:
	case SEL_UPL:
		return (sr);
	default:
		break;
	}

	/*
	 * Force it into the LDT in ring 3 for 32-bit processes, which by
	 * default do not have an LDT, so that any attempt to use an invalid
	 * selector will reference the (non-existant) LDT, and cause a #gp
	 * fault for the process.
	 *
	 * 64-bit processes get the null gdt selector since they
	 * are not allowed to have a private LDT.
	 */
	if (datamodel == DATAMODEL_ILP32) {
		return (sr | SEL_TI_LDT | SEL_UPL);
	} else {
		if (iscs == IS_CS)
			return (0 | SEL_UPL);
		else
			return (0);
	}

}

/*
 * Set general registers.
 */
void
setgregs(klwp_t *lwp, gregset_t grp)
{
	struct regs *rp = lwptoregs(lwp);
	model_t	datamodel = lwp_getdatamodel(lwp);

	struct pcb *pcb = &lwp->lwp_pcb;
	int thisthread = lwptot(lwp) == curthread;

	if (datamodel == DATAMODEL_NATIVE) {
		if (thisthread)
			(void) save_syscall_args();	/* copy the args */

		rp->r_rdi = grp[REG_RDI];
		rp->r_rsi = grp[REG_RSI];
		rp->r_rdx = grp[REG_RDX];
		rp->r_rcx = grp[REG_RCX];
		rp->r_r8 = grp[REG_R8];
		rp->r_r9 = grp[REG_R9];
		rp->r_rax = grp[REG_RAX];
		rp->r_rbx = grp[REG_RBX];
		rp->r_rbp = grp[REG_RBP];
		rp->r_r10 = grp[REG_R10];
		rp->r_r11 = grp[REG_R11];
		rp->r_r12 = grp[REG_R12];
		rp->r_r13 = grp[REG_R13];
		rp->r_r14 = grp[REG_R14];
		rp->r_r15 = grp[REG_R15];
		rp->r_trapno = grp[REG_TRAPNO];
		rp->r_err = grp[REG_ERR];
		rp->r_rip = grp[REG_RIP];
		/*
		 * Setting %cs or %ss to anything else is quietly but
		 * quite definitely forbidden!
		 */
		rp->r_cs = UCS_SEL;
		rp->r_ss = UDS_SEL;
		rp->r_rsp = grp[REG_RSP];

		if (thisthread)
			kpreempt_disable();

		pcb->pcb_ds = UDS_SEL;
		pcb->pcb_es = UDS_SEL;

		/*
		 * 64-bit processes -are- allowed to set their fsbase/gsbase
		 * values directly, but only if they're using the segment
		 * selectors that allow that semantic.
		 *
		 * (32-bit processes must use lwp_set_private().)
		 */
		pcb->pcb_fsbase = grp[REG_FSBASE];
		pcb->pcb_gsbase = grp[REG_GSBASE];
		pcb->pcb_fs = fix_segreg(grp[REG_FS], IS_NOT_CS, datamodel);
		pcb->pcb_gs = fix_segreg(grp[REG_GS], IS_NOT_CS, datamodel);

		/*
		 * Ensure that we go out via update_sregs
		 */
		PCB_SET_UPDATE_SEGS(pcb);
		lwptot(lwp)->t_post_sys = 1;
		if (thisthread)
			kpreempt_enable();
#if defined(_SYSCALL32_IMPL)
	} else {
		rp->r_rdi = (uint32_t)grp[REG_RDI];
		rp->r_rsi = (uint32_t)grp[REG_RSI];
		rp->r_rdx = (uint32_t)grp[REG_RDX];
		rp->r_rcx = (uint32_t)grp[REG_RCX];
		rp->r_rax = (uint32_t)grp[REG_RAX];
		rp->r_rbx = (uint32_t)grp[REG_RBX];
		rp->r_rbp = (uint32_t)grp[REG_RBP];
		rp->r_trapno = (uint32_t)grp[REG_TRAPNO];
		rp->r_err = (uint32_t)grp[REG_ERR];
		rp->r_rip = (uint32_t)grp[REG_RIP];

		rp->r_cs = fix_segreg(grp[REG_CS], IS_CS, datamodel);
		rp->r_ss = fix_segreg(grp[REG_DS], IS_NOT_CS, datamodel);

		rp->r_rsp = (uint32_t)grp[REG_RSP];

		if (thisthread)
			kpreempt_disable();

		pcb->pcb_ds = fix_segreg(grp[REG_DS], IS_NOT_CS, datamodel);
		pcb->pcb_es = fix_segreg(grp[REG_ES], IS_NOT_CS, datamodel);

		/*
		 * (See fsbase/gsbase commentary above)
		 */
		pcb->pcb_fs = fix_segreg(grp[REG_FS], IS_NOT_CS, datamodel);
		pcb->pcb_gs = fix_segreg(grp[REG_GS], IS_NOT_CS, datamodel);

		/*
		 * Ensure that we go out via update_sregs
		 */
		PCB_SET_UPDATE_SEGS(pcb);
		lwptot(lwp)->t_post_sys = 1;
		if (thisthread)
			kpreempt_enable();
#endif
	}

	/*
	 * Only certain bits of the flags register can be modified.
	 */
	rp->r_rfl = (rp->r_rfl & ~PSL_USERMASK) |
	    (grp[REG_RFL] & PSL_USERMASK);
}

/*
 * Determine whether eip is likely to have an interrupt frame
 * on the stack.  We do this by comparing the address to the
 * range of addresses spanned by several well-known routines.
 */
extern void _interrupt();
extern void _allsyscalls();
extern void _cmntrap();
extern void fakesoftint();

extern size_t _interrupt_size;
extern size_t _allsyscalls_size;
extern size_t _cmntrap_size;
extern size_t _fakesoftint_size;

/*
 * Get a pc-only stacktrace.  Used for kmem_alloc() buffer ownership tracking.
 * Returns MIN(current stack depth, pcstack_limit).
 */
int
getpcstack(pc_t *pcstack, int pcstack_limit)
{
	struct frame *fp = (struct frame *)getfp();
	struct frame *nextfp, *minfp, *stacktop;
	int depth = 0;
	int on_intr;
	uintptr_t pc;

	if ((on_intr = CPU_ON_INTR(CPU)) != 0)
		stacktop = (struct frame *)(CPU->cpu_intr_stack + SA(MINFRAME));
	else
		stacktop = (struct frame *)curthread->t_stk;
	minfp = fp;

	pc = ((struct regs *)fp)->r_pc;

	while (depth < pcstack_limit) {
		nextfp = (struct frame *)fp->fr_savfp;
		pc = fp->fr_savpc;
		if (nextfp <= minfp || nextfp >= stacktop) {
			if (on_intr) {
				/*
				 * Hop from interrupt stack to thread stack.
				 */
				stacktop = (struct frame *)curthread->t_stk;
				minfp = (struct frame *)curthread->t_stkbase;
				on_intr = 0;
				continue;
			}
			break;
		}
		pcstack[depth++] = (pc_t)pc;
		fp = nextfp;
		minfp = fp;
	}
	return (depth);
}

/*
 * The following ELF header fields are defined as processor-specific
 * in the V8 ABI:
 *
 *	e_ident[EI_DATA]	encoding of the processor-specific
 *				data in the object file
 *	e_machine		processor identification
 *	e_flags			processor-specific flags associated
 *				with the file
 */

/*
 * The value of at_flags reflects a platform's cpu module support.
 * at_flags is used to check for allowing a binary to execute and
 * is passed as the value of the AT_FLAGS auxiliary vector.
 */
int at_flags = 0;

/*
 * Check the processor-specific fields of an ELF header.
 *
 * returns 1 if the fields are valid, 0 otherwise
 */
/*ARGSUSED2*/
int
elfheadcheck(
	unsigned char e_data,
	Elf32_Half e_machine,
	Elf32_Word e_flags)
{
	if (e_data != ELFDATA2LSB)
		return (0);
	if (e_machine == EM_AMD64)
		return (1);
	return (e_machine == EM_386);
}

uint_t auxv_hwcap_include = 0;	/* patch to enable unrecognized features */
uint_t auxv_hwcap_include_2 = 0;	/* second word */
uint_t auxv_hwcap_exclude = 0;	/* patch for broken cpus, debugging */
uint_t auxv_hwcap_exclude_2 = 0;	/* second word */
#if defined(_SYSCALL32_IMPL)
uint_t auxv_hwcap32_include = 0;	/* ditto for 32-bit apps */
uint_t auxv_hwcap32_include_2 = 0;	/* ditto for 32-bit apps */
uint_t auxv_hwcap32_exclude = 0;	/* ditto for 32-bit apps */
uint_t auxv_hwcap32_exclude_2 = 0;	/* ditto for 32-bit apps */
#endif

/*
 * Gather information about the processor and place it into auxv_hwcap
 * so that it can be exported to the linker via the aux vector.
 *
 * We use this seemingly complicated mechanism so that we can ensure
 * that /etc/system can be used to override what the system can or
 * cannot discover for itself. Due to a lack of use, this has not
 * been extended to the 3rd word.
 */
void
bind_hwcap(void)
{
	uint_t cpu_hwcap_flags[3];
	cpuid_execpass(NULL, CPUID_PASS_RESOLVE, cpu_hwcap_flags);

	auxv_hwcap = (auxv_hwcap_include | cpu_hwcap_flags[0]) &
	    ~auxv_hwcap_exclude;
	auxv_hwcap_2 = (auxv_hwcap_include_2 | cpu_hwcap_flags[1]) &
	    ~auxv_hwcap_exclude_2;
	auxv_hwcap_3 = cpu_hwcap_flags[2];

	/*
	 * On AMD processors, sysenter just doesn't work at all
	 * when the kernel is in long mode.  On IA-32e processors
	 * it does, but there's no real point in all the alternate
	 * mechanism when syscall works on both.
	 *
	 * Besides, the kernel's sysenter handler is expecting a
	 * 32-bit lwp ...
	 */
	auxv_hwcap &= ~AV_386_SEP;

	if (auxv_hwcap_include || auxv_hwcap_exclude || auxv_hwcap_include_2 ||
	    auxv_hwcap_exclude_2) {
		/*
		 * The below assignment is regrettably required to get lint
		 * to accept the validity of our format string.  The format
		 * string is in fact valid, but whatever intelligence in lint
		 * understands the cmn_err()-specific %b appears to have an
		 * off-by-one error:  it (mistakenly) complains about bit
		 * number 32 (even though this is explicitly permitted).
		 * Normally, one would will away such warnings with a "LINTED"
		 * directive, but for reasons unclear and unknown, lint
		 * refuses to be assuaged in this case.  Fortunately, lint
		 * doesn't pretend to have solved the Halting Problem --
		 * and as soon as the format string is programmatic, it
		 * knows enough to shut up.
		 */
		char *fmt = "?user ABI extensions: %b\n";
		cmn_err(CE_CONT, fmt, auxv_hwcap, FMT_AV_386);
		fmt = "?user ABI extensions (word 2): %b\n";
		cmn_err(CE_CONT, fmt, auxv_hwcap_2, FMT_AV_386_2);
		fmt = "?user ABI extensions (word 2): %b\n";
		cmn_err(CE_CONT, fmt, auxv_hwcap_3, FMT_AV_386_3);
	}

#if defined(_SYSCALL32_IMPL)
	auxv_hwcap32 = (auxv_hwcap32_include | cpu_hwcap_flags[0]) &
	    ~auxv_hwcap32_exclude;
	auxv_hwcap32_2 = (auxv_hwcap32_include_2 | cpu_hwcap_flags[1]) &
	    ~auxv_hwcap32_exclude_2;
	auxv_hwcap32_3 = auxv_hwcap_3;

	/*
	 * If this is an amd64 architecture machine from Intel, then
	 * syscall -doesn't- work in compatibility mode, only sysenter does.
	 *
	 * Sigh.
	 */
	if (!cpuid_syscall32_insn(NULL))
		auxv_hwcap32 &= ~AV_386_AMD_SYSC;

	/*
	 * 32-bit processes can -always- use the lahf/sahf instructions
	 */
	auxv_hwcap32 |= AV_386_AHF;

	/*
	 * 32-bit processes can -never- use fsgsbase instructions.
	 */
	auxv_hwcap32_2 &= ~AV_386_2_FSGSBASE;

	if (auxv_hwcap32_include || auxv_hwcap32_exclude ||
	    auxv_hwcap32_include_2 || auxv_hwcap32_exclude_2) {
		/*
		 * See the block comment in the cmn_err() of auxv_hwcap, above.
		 */
		char *fmt = "?32-bit user ABI extensions: %b\n";
		cmn_err(CE_CONT, fmt, auxv_hwcap32, FMT_AV_386);
		fmt = "?32-bit user ABI extensions (word 2): %b\n";
		cmn_err(CE_CONT, fmt, auxv_hwcap32_2, FMT_AV_386_2);
		fmt = "?32-bit user ABI extensions (word 3): %b\n";
		cmn_err(CE_CONT, fmt, auxv_hwcap32_3, FMT_AV_386_3);
	}
#endif
}

/*
 *	sync_icache() - this is called
 *	in proc/fs/prusrio.c. x86 has an unified cache and therefore
 *	this is a nop.
 */
/* ARGSUSED */
void
sync_icache(caddr_t addr, uint_t len)
{
	/* Do nothing for now */
}

/*ARGSUSED*/
void
sync_data_memory(caddr_t va, size_t len)
{
	/* Not implemented for this platform */
}

int
__ipltospl(int ipl)
{
	return (ipltospl(ipl));
}

/*
 * The panic code invokes panic_saveregs() to record the contents of a
 * regs structure into the specified panic_data structure for debuggers.
 */
void
panic_saveregs(panic_data_t *pdp, struct regs *rp)
{
	panic_nv_t *pnv = PANICNVGET(pdp);

	struct cregs	creg;

	getcregs(&creg);

	PANICNVADD(pnv, "rdi", rp->r_rdi);
	PANICNVADD(pnv, "rsi", rp->r_rsi);
	PANICNVADD(pnv, "rdx", rp->r_rdx);
	PANICNVADD(pnv, "rcx", rp->r_rcx);
	PANICNVADD(pnv, "r8", rp->r_r8);
	PANICNVADD(pnv, "r9", rp->r_r9);
	PANICNVADD(pnv, "rax", rp->r_rax);
	PANICNVADD(pnv, "rbx", rp->r_rbx);
	PANICNVADD(pnv, "rbp", rp->r_rbp);
	PANICNVADD(pnv, "r10", rp->r_r10);
	PANICNVADD(pnv, "r11", rp->r_r11);
	PANICNVADD(pnv, "r12", rp->r_r12);
	PANICNVADD(pnv, "r13", rp->r_r13);
	PANICNVADD(pnv, "r14", rp->r_r14);
	PANICNVADD(pnv, "r15", rp->r_r15);
	PANICNVADD(pnv, "fsbase", rdmsr(MSR_AMD_FSBASE));
	PANICNVADD(pnv, "gsbase", rdmsr(MSR_AMD_GSBASE));
	PANICNVADD(pnv, "ds", rp->r_ds);
	PANICNVADD(pnv, "es", rp->r_es);
	PANICNVADD(pnv, "fs", rp->r_fs);
	PANICNVADD(pnv, "gs", rp->r_gs);
	PANICNVADD(pnv, "trapno", rp->r_trapno);
	PANICNVADD(pnv, "err", rp->r_err);
	PANICNVADD(pnv, "rip", rp->r_rip);
	PANICNVADD(pnv, "cs", rp->r_cs);
	PANICNVADD(pnv, "rflags", rp->r_rfl);
	PANICNVADD(pnv, "rsp", rp->r_rsp);
	PANICNVADD(pnv, "ss", rp->r_ss);
	PANICNVADD(pnv, "gdt_hi", (uint64_t)(creg.cr_gdt._l[3]));
	PANICNVADD(pnv, "gdt_lo", (uint64_t)(creg.cr_gdt._l[0]));
	PANICNVADD(pnv, "idt_hi", (uint64_t)(creg.cr_idt._l[3]));
	PANICNVADD(pnv, "idt_lo", (uint64_t)(creg.cr_idt._l[0]));

	PANICNVADD(pnv, "ldt", creg.cr_ldt);
	PANICNVADD(pnv, "task", creg.cr_task);
	PANICNVADD(pnv, "cr0", creg.cr_cr0);
	PANICNVADD(pnv, "cr2", creg.cr_cr2);
	PANICNVADD(pnv, "cr3", creg.cr_cr3);
	if (creg.cr_cr4)
		PANICNVADD(pnv, "cr4", creg.cr_cr4);

	PANICNVSET(pdp, pnv);
}

#define	TR_ARG_MAX 6	/* Max args to print, same as SPARC */


/*
 * Print a stack backtrace using the specified frame pointer.  We delay two
 * seconds before continuing, unless this is the panic traceback.
 * If we are in the process of panicking, we also attempt to write the
 * stack backtrace to a staticly assigned buffer, to allow the panic
 * code to find it and write it in to uncompressed pages within the
 * system crash dump.
 * Note that the frame for the starting stack pointer value is omitted because
 * the corresponding %eip is not known.
 */

extern char *dump_stack_scratch;


void
traceback(caddr_t fpreg)
{
	struct frame	*fp = (struct frame *)fpreg;
	struct frame	*nextfp;
	uintptr_t	pc, nextpc;
	ulong_t		off;
	char		args[TR_ARG_MAX * 2 + 16], *sym;
	uint_t	  offset = 0;
	uint_t	  next_offset = 0;
	char	    stack_buffer[1024];

	if (!panicstr)
		printf("traceback: %%fp = %p\n", (void *)fp);

	if (panicstr && !dump_stack_scratch) {
		printf("Warning - stack not written to the dump buffer\n");
	}

	fp = (struct frame *)plat_traceback(fpreg);
	if ((uintptr_t)fp < KERNELBASE)
		goto out;

	pc = fp->fr_savpc;
	fp = (struct frame *)fp->fr_savfp;

	while ((uintptr_t)fp >= KERNELBASE) {
		/*
		 * XX64 Until port is complete tolerate 8-byte aligned
		 * frame pointers but flag with a warning so they can
		 * be fixed.
		 */
		if (((uintptr_t)fp & (STACK_ALIGN - 1)) != 0) {
			if (((uintptr_t)fp & (8 - 1)) == 0) {
				printf("  >> warning! 8-byte"
				    " aligned %%fp = %p\n", (void *)fp);
			} else {
				printf(
				    "  >> mis-aligned %%fp = %p\n", (void *)fp);
				break;
			}
		}

		args[0] = '\0';
		nextpc = (uintptr_t)fp->fr_savpc;
		nextfp = (struct frame *)fp->fr_savfp;
		if ((sym = kobj_getsymname(pc, &off)) != NULL) {
			printf("%016lx %s:%s+%lx (%s)\n", (uintptr_t)fp,
			    mod_containing_pc((caddr_t)pc), sym, off, args);
			(void) snprintf(stack_buffer, sizeof (stack_buffer),
			    "%s:%s+%lx (%s) | ",
			    mod_containing_pc((caddr_t)pc), sym, off, args);
		} else {
			printf("%016lx %lx (%s)\n",
			    (uintptr_t)fp, pc, args);
			(void) snprintf(stack_buffer, sizeof (stack_buffer),
			    "%lx (%s) | ", pc, args);
		}

		if (panicstr && dump_stack_scratch) {
			next_offset = offset + strlen(stack_buffer);
			if (next_offset < STACK_BUF_SIZE) {
				bcopy(stack_buffer, dump_stack_scratch + offset,
				    strlen(stack_buffer));
				offset = next_offset;
			} else {
				/*
				 * In attempting to save the panic stack
				 * to the dumpbuf we have overflowed that area.
				 * Print a warning and continue to printf the
				 * stack to the msgbuf
				 */
				printf("Warning: stack in the dump buffer"
				    " may be incomplete\n");
				offset = next_offset;
			}
		}

		pc = nextpc;
		fp = nextfp;
	}
out:
	if (!panicstr) {
		printf("end of traceback\n");
		DELAY(2 * MICROSEC);
	} else if (dump_stack_scratch) {
		dump_stack_scratch[offset] = '\0';
	}
}


/*
 * Generate a stack backtrace from a saved register set.
 */
void
traceregs(struct regs *rp)
{
	traceback((caddr_t)rp->r_fp);
}

void
exec_set_sp(size_t stksize)
{
	klwp_t *lwp = ttolwp(curthread);

	lwptoregs(lwp)->r_sp = (uintptr_t)curproc->p_usrstack - stksize;
}

hrtime_t
gethrtime_waitfree(void)
{
	return (dtrace_gethrtime());
}

hrtime_t
gethrtime(void)
{
	return (gethrtimef());
}

hrtime_t
gethrtime_unscaled(void)
{
	return (gethrtimeunscaledf());
}

void
scalehrtime(hrtime_t *hrt)
{
	scalehrtimef(hrt);
}

uint64_t
unscalehrtime(hrtime_t nsecs)
{
	return (unscalehrtimef(nsecs));
}

void
gethrestime(timespec_t *tp)
{
	gethrestimef(tp);
}

/*
 * Part of the implementation of hres_tick(); this routine is
 * easier in C than assembler .. called with the hres_lock held.
 *
 * XX64	Many of these timekeeping variables need to be extern'ed in a header
 */

#include <sys/time.h>
#include <sys/machlock.h>

extern int one_sec;
extern int max_hres_adj;

void
__adj_hrestime(void)
{
	long long adj;

	if (hrestime_adj == 0)
		adj = 0;
	else if (hrestime_adj > 0) {
		if (hrestime_adj < max_hres_adj)
			adj = hrestime_adj;
		else
			adj = max_hres_adj;
	} else {
		if (hrestime_adj < -max_hres_adj)
			adj = -max_hres_adj;
		else
			adj = hrestime_adj;
	}

	timedelta -= adj;
	hrestime_adj = timedelta;
	hrestime.tv_nsec += adj;

	while (hrestime.tv_nsec >= NANOSEC) {
		one_sec++;
		hrestime.tv_sec++;
		hrestime.tv_nsec -= NANOSEC;
	}
}

/*
 * Wrapper functions to maintain backwards compability
 */
int
xcopyin(const void *uaddr, void *kaddr, size_t count)
{
	return (xcopyin_nta(uaddr, kaddr, count, UIO_COPY_CACHED));
}

int
xcopyout(const void *kaddr, void *uaddr, size_t count)
{
	return (xcopyout_nta(kaddr, uaddr, count, UIO_COPY_CACHED));
}
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License, Version 1.0 only
 * (the "License").  You may not use this file except in compliance
 * with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#include <sys/modctl.h>
#include <sys/sunddi.h>

/* internal global data */
static struct modlmisc modlmisc = {
	&mod_miscops, "bootdev misc module"
};

static struct modlinkage modlinkage = {
	MODREV_1, (void *)&modlmisc, NULL
};

int
_init()
{
	return (mod_install(&modlinkage));
}

int
_fini()
{
	return (mod_remove(&modlinkage));
}

int
_info(struct modinfo *modinfop)
{
	return (mod_info(&modlinkage, modinfop));
}

/*
 * convert a prom device path to an equivalent path in /devices
 * Does not deal with aliases.  Does deal with pathnames which
 * are not fully qualified.  This routine is generalized
 * to work across several flavors of OBP
 */
int
i_promname_to_devname(char *prom_name, char *ret_buf)
{
	if (prom_name == NULL || ret_buf == NULL ||
	    (strlen(prom_name) >= MAXPATHLEN)) {
		return (EINVAL);
	}
	if (i_ddi_prompath_to_devfspath(prom_name, ret_buf) != DDI_SUCCESS)
		return (EINVAL);

	return (0);
}

/*
 * If bootstring contains a device path, we need to convert to a format
 * the prom will understand.  To do so, we convert the existing path to
 * a prom-compatible path and return the value of new_path.  If the
 * caller specifies new_path as NULL, we allocate an appropriately
 * sized new_path on behalf of the caller.  If the caller invokes this
 * function with new_path = NULL, they must do so from a context in
 * which it is safe to perform a sleeping memory allocation.
 *
 * NOTE: Intel does not have a real PROM, so the implementation
 *       simply returns a copy of the string passed in.
 */
char *
i_convert_boot_device_name(char *cur_path, char *new_path, size_t *len)
{
	if (new_path != NULL) {
		(void) snprintf(new_path, *len, "%s", cur_path);
		return (new_path);
	} else {
		*len = strlen(cur_path) + 1;
		new_path = kmem_alloc(*len, KM_SLEEP);
		(void) snprintf(new_path, *len, "%s", cur_path);
		return (new_path);
	}
}
#
# Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
# Use is subject to license terms.
#
# CDDL HEADER START
#
# The contents of this file are subject to the terms of the
# Common Development and Distribution License (the "License").
# You may not use this file except in compliance with the License.
#
# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
# or http://www.opensolaris.org/os/licensing.
# See the License for the specific language governing permissions
# and limitations under the License.
#
# When distributing Covered Code, include this CDDL HEADER in each
# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
# If applicable, add the following below this CDDL HEADER, with the
# fields enclosed by brackets "[]" replaced with your own identifying
# information: Portions Copyright [yyyy] [name of copyright owner]
#
# CDDL HEADER END
#
#	bootenv.rc -- boot "environment variables"
#
setprop keyboard-layout Unknown
setprop ata-dma-enabled 1
setprop atapi-cd-dma-enabled 1
setprop ttyd-rts-dtr-off false
setprop ttyd-ignore-cd true
setprop ttyc-rts-dtr-off false
setprop ttyc-ignore-cd true
setprop ttyb-rts-dtr-off false
setprop ttyb-ignore-cd true
setprop ttya-rts-dtr-off false
setprop ttya-ignore-cd true
setprop ttyd-mode 9600,8,n,1,-
setprop ttyc-mode 9600,8,n,1,-
setprop ttyb-mode 9600,8,n,1,-
setprop ttya-mode 9600,8,n,1,-
setprop lba-access-ok 1
/*
 * This file and its contents are supplied under the terms of the
 * Common Development and Distribution License ("CDDL"), version 1.0.
 * You may only use this file in accordance with the terms of version
 * 1.0 of the CDDL.
 *
 * A full copy of the text of the CDDL should have accompanied this
 * source.  A copy of the CDDL is also available via the Internet at
 * http://www.illumos.org/license/CDDL.
 */

/*
 * Copyright 2016 Joyent, Inc.
 * Copyright 2025 Oxide Computer Company
 */


#include <sys/types.h>
#include <sys/thread.h>
#include <sys/proc.h>
#include <sys/mman.h>
#include <sys/vmsystm.h>
#include <vm/as.h>
#include <vm/seg_umap.h>

#if !defined(__xpv)
#include <sys/comm_page.h>
#endif /* !defined(__xpv) */

/*
 * Map in the comm page.
 *
 * The contents of the comm page are only defined on non-xpv x86 at this time.
 * Furthermore, the data is only valid in userspace (32-bit or 64-bit) when
 * mapped from a 64-bit kernel.
 * See: "uts/i86pc/sys/comm_page.h"
 */
caddr_t
comm_page_mapin()
{
#if !defined(__xpv)
	proc_t *p = curproc;
	caddr_t addr = NULL;
	const size_t len = sizeof (comm_page_t);
	const uint_t prot = PROT_USER | PROT_READ;

	map_addr(&addr, len, (offset_t)0, 1, 0);
	if (addr == NULL || valid_usr_range(addr, len, prot, p->p_as,
	    p->p_as->a_userlimit) != RANGE_OKAY) {
		return (NULL);
	}

	segumap_crargs_t suarg = {
		.kaddr = (caddr_t)&comm_page,
		.prot = prot,
		.maxprot = prot,
	};
	if (as_map(p->p_as, addr, len, segumap_create, &suarg) != 0) {
		return (NULL);
	}
	return (addr);
#else /* !defined(__xpv) */
	return (NULL);
#endif /* !defined(__xpv) */
}
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License, Version 1.0 only
 * (the "License").  You may not use this file except in compliance
 * with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2004 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

/*
 * Miscellaneous C routines for copying data around without
 * descending into assembler.  Compilers are pretty good at
 * scheduling instructions, and humans are pretty hopeless at
 * writing correct assembler.
 */

#include <sys/types.h>
#include <sys/systm.h>
#include <sys/errno.h>
#include <sys/param.h>

/*
 * copyinstr_noerr and copyoutstr_noerr can be implemented completely
 * in C on machines with shared user and kernel context.
 */
static int
copystr_nofault(const char *src, char *dst, size_t maxlength,
    size_t *lencopied)
{
	int error = 0;
	size_t leftover;

	if ((leftover = maxlength) == 0)
		error = ENAMETOOLONG;
	else
		do {
			leftover--;
			if ((*dst++ = *src++) == '\0')
				break;
			if (leftover == 0) {
				error = ENAMETOOLONG;
				break;
			}
		/*CONSTCOND*/
		} while (1);

	if (lencopied)
		*lencopied = maxlength - leftover;
	return (error);
}


int
copyinstr_noerr(const char *uaddr, char *kaddr, size_t maxlength,
    size_t *lencopied)
{
	char *ua = (char *)uaddr;

	ASSERT((uintptr_t)kaddr > kernelbase);

	if ((uintptr_t)ua > kernelbase) {
		/*
		 * force fault at kernelbase
		 */
		ua = (char *)kernelbase;
	}
	return (copystr_nofault(ua, kaddr, maxlength, lencopied));
}

int
copyoutstr_noerr(const char *kaddr, char *uaddr, size_t maxlength,
    size_t *lencopied)
{
	char *ua = (char *)uaddr;

	ASSERT((uintptr_t)kaddr > kernelbase);

	if ((uintptr_t)ua > kernelbase) {
		/*
		 * force fault at kernelbase
		 */
		ua = (char *)kernelbase;
	}
	return (copystr_nofault(kaddr, ua, maxlength, lencopied));
}
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright (c) 1999, 2010, Oracle and/or its affiliates. All rights reserved.
 * Copyright 2021 Joyent, Inc.
 */

/*
 * x86-specific routines used by the CPU Performance counter driver.
 */

#include <sys/types.h>
#include <sys/time.h>
#include <sys/atomic.h>
#include <sys/regset.h>
#include <sys/privregs.h>
#include <sys/x86_archext.h>
#include <sys/cpuvar.h>
#include <sys/machcpuvar.h>
#include <sys/archsystm.h>
#include <sys/cpc_pcbe.h>
#include <sys/cpc_impl.h>
#include <sys/x_call.h>
#include <sys/cmn_err.h>
#include <sys/cmt.h>
#include <sys/spl.h>
#include <sys/apic.h>

static kcpc_ctx_t *(*overflow_intr_handler)(caddr_t);

/* Do threads share performance monitoring hardware? */
static int strands_perfmon_shared = 0;

int kcpc_hw_overflow_intr_installed;		/* set by APIC code */
extern kcpc_ctx_t *kcpc_overflow_intr(caddr_t arg, uint64_t bitmap);

extern int kcpc_counts_include_idle; /* Project Private /etc/system variable */

void (*kcpc_hw_enable_cpc_intr)(void);		/* set by APIC code */

int
kcpc_hw_add_ovf_intr(kcpc_ctx_t *(*handler)(caddr_t))
{
	if (x86_type != X86_TYPE_P6)
		return (0);
	overflow_intr_handler = handler;
	return (ipltospl(APIC_PCINT_IPL));
}

void
kcpc_hw_rem_ovf_intr(void)
{
	overflow_intr_handler = NULL;
}

/*
 * Hook used on P4 systems to catch online/offline events.
 */
/*ARGSUSED*/
static int
kcpc_cpu_setup(cpu_setup_t what, int cpuid, void *arg)
{
	pg_cmt_t	*chip_pg;
	int		active_cpus_cnt;

	if (what != CPU_ON)
		return (0);

	/*
	 * If any CPU-bound contexts exist, we don't need to invalidate
	 * anything, as no per-LWP contexts can coexist.
	 */
	if (kcpc_cpuctx || dtrace_cpc_in_use)
		return (0);

	/*
	 * If this chip now has more than 1 active cpu, we must invalidate all
	 * contexts in the system.
	 */
	chip_pg = (pg_cmt_t *)pghw_find_pg(cpu[cpuid], PGHW_CHIP);
	if (chip_pg != NULL) {
		active_cpus_cnt = GROUP_SIZE(&chip_pg->cmt_cpus_actv);
		if (active_cpus_cnt > 1)
			kcpc_invalidate_all();
	}

	return (0);
}

static kmutex_t cpu_setup_lock;	/* protects setup_registered */
static int setup_registered;


void
kcpc_hw_init(cpu_t *cp)
{
	kthread_t *t = cp->cpu_idle_thread;
	uint32_t versionid;
	struct cpuid_regs cpuid;

	strands_perfmon_shared = 0;
	if (is_x86_feature(x86_featureset, X86FSET_HTT)) {
		if (cpuid_getvendor(cpu[0]) == X86_VENDOR_Intel) {
			/*
			 * Intel processors that support Architectural
			 * Performance Monitoring Version 3 have per strand
			 * performance monitoring hardware.
			 * Hence we can allow use of performance counters on
			 * multiple strands on the same core simultaneously.
			 */
			cpuid.cp_eax = 0x0;
			(void) __cpuid_insn(&cpuid);
			if (cpuid.cp_eax < 0xa) {
				strands_perfmon_shared = 1;
			} else {
				cpuid.cp_eax = 0xa;
				(void) __cpuid_insn(&cpuid);

				versionid = cpuid.cp_eax & 0xFF;
				if (versionid < 3) {
					strands_perfmon_shared = 1;
				}
			}
		} else if (cpuid_getvendor(cpu[0]) == X86_VENDOR_AMD ||
		    cpuid_getvendor(cpu[0]) == X86_VENDOR_HYGON) {
			/*
			 * On AMD systems with HT, all of the performance
			 * monitors exist on a per-logical CPU basis.
			 */
			strands_perfmon_shared = 0;
		} else {
			strands_perfmon_shared = 1;
		}
	}

	if (strands_perfmon_shared) {
		mutex_enter(&cpu_setup_lock);
		if (setup_registered == 0) {
			mutex_enter(&cpu_lock);
			register_cpu_setup_func(kcpc_cpu_setup, NULL);
			mutex_exit(&cpu_lock);
			setup_registered = 1;
		}
		mutex_exit(&cpu_setup_lock);
	}

	mutex_init(&cp->cpu_cpc_ctxlock, "cpu_cpc_ctxlock", MUTEX_DEFAULT, 0);

	if (kcpc_counts_include_idle)
		return;

	kcpc_idle_ctxop_install(t, cp);
}

void
kcpc_hw_fini(cpu_t *cp)
{
	ASSERT(cp->cpu_idle_thread == NULL);

	mutex_destroy(&cp->cpu_cpc_ctxlock);
}

#define	BITS(v, u, l)	\
	(((v) >> (l)) & ((1 << (1 + (u) - (l))) - 1))

#define	PCBE_NAMELEN 30	/* Enough Room for pcbe.manuf.model.family.stepping */

/*
 * Examine the processor and load an appropriate PCBE.
 */
int
kcpc_hw_load_pcbe(void)
{
	return (kcpc_pcbe_tryload(cpuid_getvendorstr(CPU), cpuid_getfamily(CPU),
	    cpuid_getmodel(CPU), cpuid_getstep(CPU)));
}

/*
 * Called by the generic framework to check if it's OK to bind a set to a CPU.
 */
int
kcpc_hw_cpu_hook(processorid_t cpuid, ulong_t *kcpc_cpumap)
{
	cpu_t		*cpu, *p;
	pg_t		*chip_pg;
	pg_cpu_itr_t	itr;

	if (!strands_perfmon_shared)
		return (0);

	/*
	 * Only one logical CPU on each Pentium 4 HT CPU may be bound to at
	 * once.
	 *
	 * This loop is protected by holding cpu_lock, in order to properly
	 * access the cpu_t of the desired cpu.
	 */
	mutex_enter(&cpu_lock);
	if ((cpu = cpu_get(cpuid)) == NULL) {
		mutex_exit(&cpu_lock);
		return (-1);
	}

	chip_pg = (pg_t *)pghw_find_pg(cpu, PGHW_CHIP);

	PG_CPU_ITR_INIT(chip_pg, itr);
	while ((p = pg_cpu_next(&itr)) != NULL) {
		if (p == cpu)
			continue;
		if (BT_TEST(kcpc_cpumap, p->cpu_id)) {
			mutex_exit(&cpu_lock);
			return (-1);
		}
	}

	mutex_exit(&cpu_lock);
	return (0);
}

/*
 * Called by the generic framework to check if it's OK to bind a set to an LWP.
 */
int
kcpc_hw_lwp_hook(void)
{
	pg_cmt_t	*chip;
	group_t		*chips;
	group_iter_t	i;

	if (!strands_perfmon_shared)
		return (0);

	/*
	 * Only one CPU per chip may be online.
	 */
	mutex_enter(&cpu_lock);

	chips = pghw_set_lookup(PGHW_CHIP);
	if (chips == NULL) {
		mutex_exit(&cpu_lock);
		return (0);
	}

	group_iter_init(&i);
	while ((chip = group_iterate(chips, &i)) != NULL) {
		if (GROUP_SIZE(&chip->cmt_cpus_actv) > 1) {
			mutex_exit(&cpu_lock);
			return (-1);
		}
	}

	mutex_exit(&cpu_lock);
	return (0);
}
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

/*
 * cpr functions for supported sparc platforms
 */
#include <sys/types.h>
#include <sys/systm.h>
#include <sys/cpr.h>
#include <sys/kmem.h>
#include <sys/errno.h>

/*
 * setup the original and new sets of property names/values
 * Not relevant to S3, which is all we support for now.
 */
/*ARGSUSED*/
int
cpr_default_setup(int alloc)
{
	return (0);
}

void
cpr_send_notice(void)
{
	static char cstr[] = "\014" "\033[1P" "\033[18;21H";

	prom_printf(cstr);
	prom_printf("Saving System State. Please Wait... ");
}

void
cpr_spinning_bar(void)
{
	static char *spin_strings[] = { "|\b", "/\b", "-\b", "\\\b" };
	static int idx;

	prom_printf(spin_strings[idx]);
	if (++idx == 4)
		idx = 0;
}

void
cpr_resume_notice(void)
{
	static char cstr[] = "\014" "\033[1P" "\033[18;21H";

	prom_printf(cstr);
	prom_printf("Restoring System State. Please Wait... ");
}
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
 * Copyright (c) 2011, 2016 by Delphix. All rights reserved.
 * Copyright 2013 Nexenta Systems, Inc. All rights reserved.
 * Copyright 2014 Josef "Jeff" Sipek <jeffpc@josefsipek.net>
 * Copyright 2020 Joyent, Inc.
 * Copyright 2025 Oxide Computer Company
 * Copyright 2024 MNX Cloud, Inc.
 * Copyright 2025 Edgecast Cloud LLC.
 */
/*
 * Copyright (c) 2010, Intel Corporation.
 * All rights reserved.
 */
/*
 * Portions Copyright 2009 Advanced Micro Devices, Inc.
 */

/*
 * CPU Identification logic
 *
 * The purpose of this file and its companion, cpuid_subr.c, is to help deal
 * with the identification of CPUs, their features, and their topologies. More
 * specifically, this file helps drive the following:
 *
 * 1. Enumeration of features of the processor which are used by the kernel to
 *    determine what features to enable or disable. These may be instruction set
 *    enhancements or features that we use.
 *
 * 2. Enumeration of instruction set architecture (ISA) additions that userland
 *    will be told about through the auxiliary vector.
 *
 * 3. Understanding the physical topology of the CPU such as the number of
 *    caches, how many cores it has, whether or not it supports symmetric
 *    multi-processing (SMT), etc.
 *
 * ------------------------
 * CPUID History and Basics
 * ------------------------
 *
 * The cpuid instruction was added by Intel roughly around the time that the
 * original Pentium was introduced. The purpose of cpuid was to tell in a
 * programmatic fashion information about the CPU that previously was guessed
 * at. For example, an important part of cpuid is that we can know what
 * extensions to the ISA exist. If you use an invalid opcode you would get a
 * #UD, so this method allows a program (whether a user program or the kernel)
 * to determine what exists without crashing or getting a SIGILL. Of course,
 * this was also during the era of the clones and the AMD Am5x86. The vendor
 * name shows up first in cpuid for a reason.
 *
 * cpuid information is broken down into ranges called a 'leaf'. Each leaf puts
 * unique values into the registers %eax, %ebx, %ecx, and %edx and each leaf has
 * its own meaning. The different leaves are broken down into different regions:
 *
 *	[ 0, 7fffffff ]			This region is called the 'basic'
 *					region. This region is generally defined
 *					by Intel, though some of the original
 *					portions have different meanings based
 *					on the manufacturer. These days, Intel
 *					adds most new features to this region.
 *					AMD adds non-Intel compatible
 *					information in the third, extended
 *					region. Intel uses this for everything
 *					including ISA extensions, CPU
 *					features, cache information, topology,
 *					and more.
 *
 *					There is a hole carved out of this
 *					region which is reserved for
 *					hypervisors.
 *
 *	[ 40000000, 4fffffff ]		This region, which is found in the
 *					middle of the previous region, is
 *					explicitly promised to never be used by
 *					CPUs. Instead, it is used by hypervisors
 *					to communicate information about
 *					themselves to the operating system. The
 *					values and details are unique for each
 *					hypervisor.
 *
 *	[ 80000000, ffffffff ]		This region is called the 'extended'
 *					region. Some of the low leaves mirror
 *					parts of the basic leaves. This region
 *					has generally been used by AMD for
 *					various extensions. For example, AMD-
 *					specific information about caches,
 *					features, and topology are found in this
 *					region.
 *
 * To specify a range, you place the desired leaf into %eax, zero %ebx, %ecx,
 * and %edx, and then issue the cpuid instruction. At the first leaf in each of
 * the ranges, one of the primary things returned is the maximum valid leaf in
 * that range. This allows for discovery of what range of CPUID is valid.
 *
 * The CPUs have potentially surprising behavior when using an invalid leaf or
 * unimplemented leaf. If the requested leaf is within the valid basic or
 * extended range, but is unimplemented, then %eax, %ebx, %ecx, and %edx will be
 * set to zero. However, if you specify a leaf that is outside of a valid range,
 * then instead it will be filled with the last valid _basic_ leaf. For example,
 * if the maximum basic value is on leaf 0x3, then issuing a cpuid for leaf 4 or
 * an invalid extended leaf will return the information for leaf 3.
 *
 * Some leaves are broken down into sub-leaves. This means that the value
 * depends on both the leaf asked for in %eax and a secondary register. For
 * example, Intel uses the value in %ecx on leaf 7 to indicate a sub-leaf to get
 * additional information. Or when getting topology information in leaf 0xb, the
 * initial value in %ecx changes which level of the topology that you are
 * getting information about.
 *
 * cpuid values are always kept to 32 bits regardless of whether or not the
 * program is in 64-bit mode. When executing in 64-bit mode, the upper
 * 32 bits of the register are always set to zero so that way the values are the
 * same regardless of execution mode.
 *
 * ----------------------
 * Identifying Processors
 * ----------------------
 *
 * We can identify a processor in two steps. The first step looks at cpuid leaf
 * 0. Leaf 0 contains the processor's vendor information. This is done by
 * putting a 12 character string in %ebx, %ecx, and %edx. On AMD, it is
 * 'AuthenticAMD' and on Intel it is 'GenuineIntel'.
 *
 * From there, a processor is identified by a combination of three different
 * values:
 *
 *  1. Family
 *  2. Model
 *  3. Stepping
 *
 * Each vendor uses the family and model to uniquely identify a processor. The
 * way that family and model are changed depends on the vendor. For example,
 * Intel has been using family 0x6 for almost all of their processor since the
 * Pentium Pro/Pentium II era, often called the P6. The model is used to
 * identify the exact processor. Different models are often used for the client
 * (consumer) and server parts. Even though each processor often has major
 * architectural differences, they still are considered the same family by
 * Intel.
 *
 * On the other hand, each major AMD architecture generally has its own family.
 * For example, the K8 is family 0x10, Bulldozer 0x15, and Zen 0x17. Within it
 * the model number is used to help identify specific processors.  As AMD's
 * product lines have expanded, they have started putting a mixed bag of
 * processors into the same family, with each processor under a single
 * identifying banner (e.g., Milan, Cezanne) using a range of model numbers.  We
 * refer to each such collection as a processor family, distinct from cpuid
 * family.  Importantly, each processor family has a BIOS and Kernel Developer's
 * Guide (BKDG, older parts) or Processor Programming Reference (PPR) that
 * defines the processor family's non-architectural features.  In general, we'll
 * use "family" here to mean the family number reported by the cpuid instruction
 * and distinguish the processor family from it where appropriate.
 *
 * The stepping is used to refer to a revision of a specific microprocessor. The
 * term comes from equipment used to produce masks that are used to create
 * integrated circuits.
 *
 * The information is present in leaf 1, %eax. In technical documentation you
 * will see the terms extended model and extended family. The original family,
 * model, and stepping fields were each 4 bits wide. If the values in either
 * are 0xf, then one is to consult the extended model and extended family, which
 * take previously reserved bits and allow for a larger number of models and add
 * 0xf to them.
 *
 * When we process this information, we store the full family, model, and
 * stepping in the struct cpuid_info members cpi_family, cpi_model, and
 * cpi_step, respectively. Whenever you are performing comparisons with the
 * family, model, and stepping, you should use these members and not the raw
 * values from cpuid. If you must use the raw values from cpuid directly, you
 * must make sure that you add the extended model and family to the base model
 * and family.
 *
 * In general, we do not use information about the family, model, and stepping
 * to determine whether or not a feature is present; that is generally driven by
 * specific leaves. However, when something we care about on the processor is
 * not considered 'architectural' meaning that it is specific to a set of
 * processors and not promised in the architecture model to be consistent from
 * generation to generation, then we will fall back on this information. The
 * most common cases where this comes up is when we have to workaround errata in
 * the processor, are dealing with processor-specific features such as CPU
 * performance counters, or we want to provide additional information for things
 * such as fault management.
 *
 * While processors also do have a brand string, which is the name that people
 * are familiar with when buying the processor, they are not meant for
 * programmatic consumption. That is what the family, model, and stepping are
 * for.
 *
 * We use the x86_chiprev_t to encode a combination of vendor, processor family,
 * and stepping(s) that refer to a single or very closely related set of silicon
 * implementations; while there are sometimes more specific ways to learn of the
 * presence or absence of a particular erratum or workaround, one may generally
 * assume that all processors of the same chiprev have the same errata and we
 * have chosen to represent them this way precisely because that is how AMD
 * groups them in their revision guides (errata documentation).  The processor
 * family (x86_processor_family_t) may be extracted from the chiprev if that
 * level of detail is not needed.  Processor families are considered unordered
 * but revisions within a family may be compared for either an exact match or at
 * least as recent as a reference revision.  See the chiprev_xxx() functions
 * below.
 *
 * Similarly, each processor family implements a particular microarchitecture,
 * which itself may have multiple revisions.  In general, non-architectural
 * features are specific to a processor family, but some may exist across
 * families containing cores that implement the same microarchitectural revision
 * (and, such cores share common bugs, too).  We provide utility routines
 * analogous to those for extracting and comparing chiprevs for
 * microarchitectures as well; see the uarch_xxx() functions.
 *
 * Both chiprevs and uarchrevs are defined in x86_archext.h and both are at
 * present used and available only for AMD and AMD-like processors.
 *
 * ------------
 * CPUID Passes
 * ------------
 *
 * As part of performing feature detection, we break this into several different
 * passes. There used to be a pass 0 that was done from assembly in locore.s to
 * support processors that have a missing or broken cpuid instruction (notably
 * certain Cyrix processors) but those were all 32-bit processors which are no
 * longer supported. Passes are no longer numbered explicitly to make it easier
 * to break them up or move them around as needed; however, they still have a
 * well-defined execution ordering enforced by the definition of cpuid_pass_t in
 * x86_archext.h. The external interface to execute a cpuid pass or determine
 * whether a pass has been completed consists of cpuid_execpass() and
 * cpuid_checkpass() respectively.  The passes now, in that execution order,
 * are as follows:
 *
 *	PRELUDE		This pass does not have any dependencies on system
 *			setup; in particular, unlike all subsequent passes it is
 *			guaranteed not to require PCI config space access.  It
 *			sets the flag indicating that the processor we are
 *			running on supports the cpuid instruction, which all
 *			64-bit processors do.  This would also be the place to
 *			add any other basic state that is required later on and
 *			can be learned without dependencies.
 *
 *	IDENT		Determine which vendor manufactured the CPU, the family,
 *			model, and stepping information, and compute basic
 *			identifying tags from those values.  This is done first
 *			so that machine-dependent code can control the features
 *			the cpuid instruction will report during subsequent
 *			passes if needed, and so that any intervening
 *			machine-dependent code that needs basic identity will
 *			have it available.  This includes synthesised
 *			identifiers such as chiprev and uarchrev as well as the
 *			values obtained directly from cpuid.  Prior to executing
 *			this pass, machine-depedent boot code is responsible for
 *			ensuring that the PCI configuration space access
 *			functions have been set up and, if necessary, that
 *			determine_platform() has been called.
 *
 *	BASIC		This is the primary pass and is responsible for doing a
 *			large number of different things:
 *
 *			1. Gathering a large number of feature flags to
 *			determine which features the CPU support and which
 *			indicate things that we need to do other work in the OS
 *			to enable. Features detected this way are added to the
 *			x86_featureset which can be queried to
 *			determine what we should do. This includes processing
 *			all of the basic and extended CPU features that we care
 *			about.
 *
 *			2. Determining the CPU's topology. This includes
 *			information about how many cores and threads are present
 *			in the package. It also is responsible for figuring out
 *			which logical CPUs are potentially part of the same core
 *			and what other resources they might share. For more
 *			information see the 'Topology' section.
 *
 *			3. Determining the set of CPU security-specific features
 *			that we need to worry about and determine the
 *			appropriate set of workarounds.
 *
 *			Pass 1 on the boot CPU occurs before KMDB is started.
 *
 *	EXTENDED	The second pass is done after startup(). Here, we check
 *			other miscellaneous features. Most of this is gathering
 *			additional basic and extended features that we'll use in
 *			later passes or for debugging support.
 *
 *	DYNAMIC		The third pass occurs after the kernel memory allocator
 *			has been fully initialized. This gathers information
 *			where we might need dynamic memory available for our
 *			uses. This includes several varying width leaves that
 *			have cache information and the processor's brand string.
 *
 *	RESOLVE		The fourth and final normal pass is performed after the
 *			kernel has brought most everything online. This is
 *			invoked from post_startup(). In this pass, we go through
 *			the set of features that we have enabled and turn that
 *			into the hardware auxiliary vector features that
 *			userland receives. This is used by userland, primarily
 *			by the run-time link-editor (RTLD), though userland
 *			software could also refer to it directly.
 *
 * The function that performs a pass is currently assumed to be infallible, and
 * all existing implementation are.  This simplifies callers by allowing
 * cpuid_execpass() to return void. Similarly, implementers do not need to check
 * for a NULL CPU argument; the current CPU's cpu_t is substituted if necessary.
 * Both of these assumptions can be relaxed if needed by future developments.
 * Tracking of completed states is handled by cpuid_execpass(). It is programmer
 * error to attempt to execute a pass before all previous passes have been
 * completed on the specified CPU, or to request cpuid information before the
 * pass that captures it has been executed.  These conditions can be tested
 * using cpuid_checkpass().
 *
 * ---------
 * Microcode
 * ---------
 *
 * Microcode updates may be applied by the firmware (BIOS/UEFI) and/or by the
 * operating system and may result in architecturally visible changes (e.g.,
 * changed MSR or CPUID bits). As such, we want to apply any updates as early
 * as possible during the boot process -- right after the IDENT pass.
 *
 * Microcode may also be updated at runtime via ucodeadm(8), after which we do
 * a selective rescan of the cpuid leaves to determine what features have
 * changed. Microcode updates can provide more details about security related
 * features to deal with issues like Spectre and L1TF. On occasion, vendors have
 * violated their contract and removed bits. However, we don't try to detect
 * that because that puts us in a situation that we really can't deal with. As
 * such, the only thing we rescan are security related features today. See
 * cpuid_pass_ucode(). This is not a pass in the same sense as the others and
 * is run on demand, via cpuid_post_ucodeadm().
 *
 *
 * All of the passes are run on all CPUs. However, for the most part we only
 * care about what the boot CPU says about this information and use the other
 * CPUs as a rough guide to sanity check that we have the same feature set.
 *
 * We do not support running multiple logical CPUs with disjoint, let alone
 * different, feature sets.
 *
 * ------------------
 * Processor Topology
 * ------------------
 *
 * One of the important things that we need to do is to understand the topology
 * of the underlying processor. When we say topology in this case, we're trying
 * to understand the relationship between the logical CPUs that the operating
 * system sees and the underlying physical layout. Different logical CPUs may
 * share different resources which can have important consequences for the
 * performance of the system. For example, they may share caches, execution
 * units, and more.
 *
 * The topology of the processor changes from generation to generation and
 * vendor to vendor.  Along with that, different vendors use different
 * terminology, and the operating system itself uses occasionally overlapping
 * terminology. It's important to understand what this topology looks like so
 * one can understand the different things that we try to calculate and
 * determine.
 *
 * To get started, let's talk about a little bit of terminology that we've used
 * so far, is used throughout this file, and is fairly generic across multiple
 * vendors:
 *
 * CPU
 *	A central processing unit (CPU) refers to a logical and/or virtual
 *	entity that the operating system can execute instructions on. The
 *	underlying resources for this CPU may be shared between multiple
 *	entities; however, to the operating system it is a discrete unit.
 *
 * PROCESSOR and PACKAGE
 *
 *	Generally, when we use the term 'processor' on its own, we are referring
 *	to the physical entity that one buys and plugs into a board. However,
 *	because processor has been overloaded and one might see it used to mean
 *	multiple different levels, we will instead use the term 'package' for
 *	the rest of this file. The term package comes from the electrical
 *	engineering side and refers to the physical entity that encloses the
 *	electronics inside. Strictly speaking the package can contain more than
 *	just the CPU, for example, on many processors it may also have what's
 *	called an 'integrated graphical processing unit (GPU)'. Because the
 *	package can encapsulate multiple units, it is the largest physical unit
 *	that we refer to.
 *
 * SOCKET
 *
 *	A socket refers to unit on a system board (generally the motherboard)
 *	that can receive a package. A single package, or processor, is plugged
 *	into a single socket. A system may have multiple sockets. Often times,
 *	the term socket is used interchangeably with package and refers to the
 *	electrical component that has plugged in, and not the receptacle itself.
 *
 * CORE
 *
 *	A core refers to the physical instantiation of a CPU, generally, with a
 *	full set of hardware resources available to it. A package may contain
 *	multiple cores inside of it or it may just have a single one. A
 *	processor with more than one core is often referred to as 'multi-core'.
 *	In illumos, we will use the feature X86FSET_CMP to refer to a system
 *	that has 'multi-core' processors.
 *
 *	A core may expose a single logical CPU to the operating system, or it
 *	may expose multiple CPUs, which we call threads, defined below.
 *
 *	Some resources may still be shared by cores in the same package. For
 *	example, many processors will share the level 3 cache between cores.
 *	Some AMD generations share hardware resources between cores. For more
 *	information on that see the section 'AMD Topology'.
 *
 * THREAD and STRAND
 *
 *	In this file, generally a thread refers to a hardware resources and not
 *	the operating system's logical abstraction. A thread is always exposed
 *	as an independent logical CPU to the operating system. A thread belongs
 *	to a specific core. A core may have more than one thread. When that is
 *	the case, the threads that are part of the same core are often referred
 *	to as 'siblings'.
 *
 *	When multiple threads exist, this is generally referred to as
 *	simultaneous multi-threading (SMT). When Intel introduced this in their
 *	processors they called it hyper-threading (HT). When multiple threads
 *	are active in a core, they split the resources of the core. For example,
 *	two threads may share the same set of hardware execution units.
 *
 *	The operating system often uses the term 'strand' to refer to a thread.
 *	This helps disambiguate it from the software concept.
 *
 * CHIP
 *
 *	Unfortunately, the term 'chip' is dramatically overloaded. At its most
 *	base meaning, it is used to refer to a single integrated circuit, which
 *	may or may not be the only thing in the package. In illumos, when you
 *	see the term 'chip' it is almost always referring to the same thing as
 *	the 'package'. However, many vendors may use chip to refer to one of
 *	many integrated circuits that have been placed in the package. As an
 *	example, see the subsequent definition.
 *
 *	To try and keep things consistent, we will only use chip when referring
 *	to the entire integrated circuit package, with the exception of the
 *	definition of multi-chip module (because it is in the name) and use the
 *	term 'die' when we want the more general, potential sub-component
 *	definition.
 *
 * DIE
 *
 *	A die refers to an integrated circuit. Inside of the package there may
 *	be a single die or multiple dies. This is sometimes called a 'chip' in
 *	vendor's parlance, but in this file, we use the term die to refer to a
 *	subcomponent.
 *
 * MULTI-CHIP MODULE
 *
 *	A multi-chip module (MCM) refers to putting multiple distinct chips that
 *	are connected together in the same package. When a multi-chip design is
 *	used, generally each chip is manufactured independently and then joined
 *	together in the package. For example, on AMD's Zen microarchitecture
 *	(family 0x17), the package contains several dies (the second meaning of
 *	chip from above) that are connected together.
 *
 * CACHE
 *
 *	A cache is a part of the processor that maintains copies of recently
 *	accessed memory. Caches are split into levels and then into types.
 *	Commonly there are one to three levels, called level one, two, and
 *	three. The lower the level, the smaller it is, the closer it is to the
 *	execution units of the CPU, and the faster it is to access. The layout
 *	and design of the cache come in many different flavors, consult other
 *	resources for a discussion of those.
 *
 *	Caches are generally split into two types, the instruction and data
 *	cache. The caches contain what their names suggest, the instruction
 *	cache has executable program text, while the data cache has all other
 *	memory that the processor accesses. As of this writing, data is kept
 *	coherent between all of the caches on x86, so if one modifies program
 *	text before it is executed, that will be in the data cache, and the
 *	instruction cache will be synchronized with that change when the
 *	processor actually executes those instructions. This coherency also
 *	covers the fact that data could show up in multiple caches.
 *
 *	Generally, the lowest level caches are specific to a core. However, the
 *	last layer cache is shared between some number of cores. The number of
 *	CPUs sharing this last level cache is important. This has implications
 *	for the choices that the scheduler makes, as accessing memory that might
 *	be in a remote cache after thread migration can be quite expensive.
 *
 *	Sometimes, the word cache is abbreviated with a '$', because in US
 *	English the word cache is pronounced the same as cash. So L1D$ refers to
 *	the L1 data cache, and L2$ would be the L2 cache. This will not be used
 *	in the rest of this theory statement for clarity.
 *
 * MEMORY CONTROLLER
 *
 *	The memory controller is a component that provides access to DRAM. Each
 *	memory controller can access a set number of DRAM channels. Each channel
 *	can have a number of DIMMs (sticks of memory) associated with it. A
 *	given package may have more than one memory controller. The association
 *	of the memory controller to a group of cores is important as it is
 *	cheaper to access memory on the controller that you are associated with.
 *
 * NUMA
 *
 *	NUMA or non-uniform memory access, describes a way that systems are
 *	built. On x86, any processor core can address all of the memory in the
 *	system. However, When using multiple sockets or possibly within a
 *	multi-chip module, some of that memory is physically closer and some of
 *	it is further. Memory that is further away is more expensive to access.
 *	Consider the following image of multiple sockets with memory:
 *
 *	+--------+                                                +--------+
 *	| DIMM A |         +----------+      +----------+         | DIMM D |
 *	+--------+-+       |          |      |          |       +-+------+-+
 *	  | DIMM B |=======| Socket 0 |======| Socket 1 |=======| DIMM E |
 *	  +--------+-+     |          |      |          |     +-+------+-+
 *	    | DIMM C |     +----------+      +----------+     | DIMM F |
 *	    +--------+                                        +--------+
 *
 *	In this example, Socket 0 is closer to DIMMs A-C while Socket 1 is
 *	closer to DIMMs D-F. This means that it is cheaper for socket 0 to
 *	access DIMMs A-C and more expensive to access D-F as it has to go
 *	through Socket 1 to get there. The inverse is true for Socket 1. DIMMs
 *	D-F are cheaper than A-C. While the socket form is the most common, when
 *	using multi-chip modules, this can also sometimes occur. For another
 *	example of this that's more involved, see the AMD topology section.
 *
 *
 * Intel Topology
 * --------------
 *
 * Most Intel processors since Nehalem, (as of this writing the current gen
 * is Skylake / Cannon Lake) follow a fairly similar pattern. The CPU portion of
 * the package is a single monolithic die. MCMs currently aren't used. Most
 * parts have three levels of caches, with the L3 cache being shared between
 * all of the cores on the package. The L1/L2 cache is generally specific to
 * an individual core. The following image shows at a simplified level what
 * this looks like. The memory controller is commonly part of something called
 * the 'Uncore', that used to be separate physical chips that were not a part of
 * the package, but are now part of the same chip.
 *
 *  +-----------------------------------------------------------------------+
 *  | Package                                                               |
 *  |  +-------------------+  +-------------------+  +-------------------+  |
 *  |  | Core              |  | Core              |  | Core              |  |
 *  |  |  +--------+ +---+ |  |  +--------+ +---+ |  |  +--------+ +---+ |  |
 *  |  |  | Thread | | L | |  |  | Thread | | L | |  |  | Thread | | L | |  |
 *  |  |  +--------+ | 1 | |  |  +--------+ | 1 | |  |  +--------+ | 1 | |  |
 *  |  |  +--------+ |   | |  |  +--------+ |   | |  |  +--------+ |   | |  |
 *  |  |  | Thread | |   | |  |  | Thread | |   | |  |  | Thread | |   | |  |
 *  |  |  +--------+ +---+ |  |  +--------+ +---+ |  |  +--------+ +---+ |  |
 *  |  |  +--------------+ |  |  +--------------+ |  |  +--------------+ |  |
 *  |  |  | L2 Cache     | |  |  | L2 Cache     | |  |  | L2 Cache     | |  |
 *  |  |  +--------------+ |  |  +--------------+ |  |  +--------------+ |  |
 *  |  +-------------------+  +-------------------+  +-------------------+  |
 *  | +-------------------------------------------------------------------+ |
 *  | |                         Shared L3 Cache                           | |
 *  | +-------------------------------------------------------------------+ |
 *  | +-------------------------------------------------------------------+ |
 *  | |                        Memory Controller                          | |
 *  | +-------------------------------------------------------------------+ |
 *  +-----------------------------------------------------------------------+
 *
 * A side effect of this current architecture is that what we care about from a
 * scheduling and topology perspective, is simplified. In general we care about
 * understanding which logical CPUs are part of the same core and socket.
 *
 * To determine the relationship between threads and cores, Intel initially used
 * the identifier in the advanced programmable interrupt controller (APIC). They
 * also added cpuid leaf 4 to give additional information about the number of
 * threads and CPUs in the processor. With the addition of x2apic (which
 * increased the number of addressable logical CPUs from 8-bits to 32-bits), an
 * additional cpuid topology leaf 0xB was added.
 *
 * AMD Topology
 * ------------
 *
 * When discussing AMD topology, we want to break this into three distinct
 * generations of topology. There's the basic topology that has been used in
 * family 0xf+ (Opteron, Athlon64), there's the topology that was introduced
 * with family 0x15 (Bulldozer), and there's the topology that was introduced
 * with family 0x17 (Zen), evolved more dramatically in Zen 2 (still family
 * 0x17), and tweaked slightly in Zen 3 (family 19h). AMD also has some
 * additional terminology that's worth talking about.
 *
 * Until the introduction of family 0x17 (Zen), AMD did not implement something
 * that they considered SMT. Whether or not the AMD processors have SMT
 * influences many things including scheduling and reliability, availability,
 * and serviceability (RAS) features.
 *
 * NODE
 *
 *	AMD uses the term node to refer to a die that contains a number of cores
 *	and I/O resources. Depending on the processor family and model, more
 *	than one node can be present in the package. When there is more than one
 *	node this indicates a multi-chip module. Usually each node has its own
 *	access to memory and I/O devices. This is important and generally
 *	different from the corresponding Intel Nehalem-Skylake+ processors. As a
 *	result, we track this relationship in the operating system.
 *
 *	In processors with an L3 cache, the L3 cache is generally shared across
 *	the entire node, though the way this is carved up varies from generation
 *	to generation.
 *
 * BULLDOZER
 *
 *	Starting with the Bulldozer family (0x15) and continuing until the
 *	introduction of the Zen microarchitecture, AMD introduced the idea of a
 *	compute unit. In a compute unit, two traditional cores share a number of
 *	hardware resources. Critically, they share the FPU, L1 instruction
 *	cache, and the L2 cache. Several compute units were then combined inside
 *	of a single node.  Because the integer execution units, L1 data cache,
 *	and some other resources were not shared between the cores, AMD never
 *	considered this to be SMT.
 *
 * ZEN
 *
 *	The Zen family (0x17) uses a multi-chip module (MCM) design, the module
 *	is called Zeppelin. These modules are similar to the idea of nodes used
 *	previously. Each of these nodes has two DRAM channels which all of the
 *	cores in the node can access uniformly. These nodes are linked together
 *	in the package, creating a NUMA environment.
 *
 *	The Zeppelin die itself contains two different 'core complexes'. Each
 *	core complex consists of four cores which each have two threads, for a
 *	total of 8 logical CPUs per complex. Unlike other generations,
 *	where all the logical CPUs in a given node share the L3 cache, here each
 *	core complex has its own shared L3 cache.
 *
 *	A further thing that we need to consider is that in some configurations,
 *	particularly with the Threadripper line of processors, not every die
 *	actually has its memory controllers wired up to actual memory channels.
 *	This means that some cores have memory attached to them and others
 *	don't.
 *
 *	To put Zen in perspective, consider the following images:
 *
 *      +--------------------------------------------------------+
 *      | Core Complex                                           |
 *      | +-------------------+    +-------------------+  +---+  |
 *      | | Core       +----+ |    | Core       +----+ |  |   |  |
 *      | | +--------+ | L2 | |    | +--------+ | L2 | |  |   |  |
 *      | | | Thread | +----+ |    | | Thread | +----+ |  |   |  |
 *      | | +--------+-+ +--+ |    | +--------+-+ +--+ |  | L |  |
 *      | |   | Thread | |L1| |    |   | Thread | |L1| |  | 3 |  |
 *      | |   +--------+ +--+ |    |   +--------+ +--+ |  |   |  |
 *      | +-------------------+    +-------------------+  | C |  |
 *      | +-------------------+    +-------------------+  | a |  |
 *      | | Core       +----+ |    | Core       +----+ |  | c |  |
 *      | | +--------+ | L2 | |    | +--------+ | L2 | |  | h |  |
 *      | | | Thread | +----+ |    | | Thread | +----+ |  | e |  |
 *      | | +--------+-+ +--+ |    | +--------+-+ +--+ |  |   |  |
 *      | |   | Thread | |L1| |    |   | Thread | |L1| |  |   |  |
 *      | |   +--------+ +--+ |    |   +--------+ +--+ |  |   |  |
 *      | +-------------------+    +-------------------+  +---+  |
 *      |                                                        |
 *	+--------------------------------------------------------+
 *
 *  This first image represents a single Zen core complex that consists of four
 *  cores.
 *
 *
 *	+--------------------------------------------------------+
 *	| Zeppelin Die                                           |
 *	|  +--------------------------------------------------+  |
 *	|  |         I/O Units (PCIe, SATA, USB, etc.)        |  |
 *	|  +--------------------------------------------------+  |
 *      |                           HH                           |
 *	|          +-----------+    HH    +-----------+          |
 *	|          |           |    HH    |           |          |
 *	|          |    Core   |==========|    Core   |          |
 *	|          |  Complex  |==========|  Complex  |          |
 *	|          |           |    HH    |           |          |
 *	|          +-----------+    HH    +-----------+          |
 *      |                           HH                           |
 *	|  +--------------------------------------------------+  |
 *	|  |                Memory Controller                 |  |
 *	|  +--------------------------------------------------+  |
 *      |                                                        |
 *	+--------------------------------------------------------+
 *
 *  This image represents a single Zeppelin Die. Note how both cores are
 *  connected to the same memory controller and I/O units. While each core
 *  complex has its own L3 cache as seen in the first image, they both have
 *  uniform access to memory.
 *
 *
 *                      PP                     PP
 *                      PP                     PP
 *           +----------PP---------------------PP---------+
 *           |          PP                     PP         |
 *           |    +-----------+          +-----------+    |
 *           |    |           |          |           |    |
 *       MMMMMMMMM|  Zeppelin |==========|  Zeppelin |MMMMMMMMM
 *       MMMMMMMMM|    Die    |==========|    Die    |MMMMMMMMM
 *           |    |           |          |           |    |
 *           |    +-----------+ooo    ...+-----------+    |
 *           |          HH      ooo  ...       HH         |
 *           |          HH        oo..         HH         |
 *           |          HH        ..oo         HH         |
 *           |          HH      ...  ooo       HH         |
 *           |    +-----------+...    ooo+-----------+    |
 *           |    |           |          |           |    |
 *       MMMMMMMMM|  Zeppelin |==========|  Zeppelin |MMMMMMMMM
 *       MMMMMMMMM|    Die    |==========|    Die    |MMMMMMMMM
 *           |    |           |          |           |    |
 *           |    +-----------+          +-----------+    |
 *           |          PP                     PP         |
 *           +----------PP---------------------PP---------+
 *                      PP                     PP
 *                      PP                     PP
 *
 *  This image represents a single Zen package. In this example, it has four
 *  Zeppelin dies, though some configurations only have a single one. In this
 *  example, each die is directly connected to the next. Also, each die is
 *  represented as being connected to memory by the 'M' character and connected
 *  to PCIe devices and other I/O, by the 'P' character. Because each Zeppelin
 *  die is made up of two core complexes, we have multiple different NUMA
 *  domains that we care about for these systems.
 *
 * ZEN 2
 *
 *	Zen 2 changes things in a dramatic way from Zen 1. Whereas in Zen 1
 *	each Zeppelin Die had its own I/O die, that has been moved out of the
 *	core complex in Zen 2. The actual core complex looks pretty similar, but
 *	now the die actually looks much simpler:
 *
 *      +--------------------------------------------------------+
 *      | Zen 2 Core Complex Die    HH                           |
 *      |                           HH                           |
 *      |          +-----------+    HH    +-----------+          |
 *      |          |           |    HH    |           |          |
 *      |          |    Core   |==========|    Core   |          |
 *      |          |  Complex  |==========|  Complex  |          |
 *      |          |           |    HH    |           |          |
 *      |          +-----------+    HH    +-----------+          |
 *      |                           HH                           |
 *      |                           HH                           |
 *      +--------------------------------------------------------+
 *
 *	From here, when we add the central I/O die, this changes things a bit.
 *	Each die is connected to the I/O die, rather than trying to interconnect
 *	them directly. The following image takes the same Zen 1 image that we
 *	had earlier and shows what it looks like with the I/O die instead:
 *
 *                                 PP    PP
 *                                 PP    PP
 *           +---------------------PP----PP---------------------+
 *           |                     PP    PP                     |
 *           |  +-----------+      PP    PP      +-----------+  |
 *           |  |           |      PP    PP      |           |  |
 *           |  |   Zen 2   |    +-PP----PP-+    |   Zen 2   |  |
 *           |  |    Die   _|    | PP    PP |    |_   Die    |  |
 *           |  |         |o|oooo|          |oooo|o|         |  |
 *           |  +-----------+    |          |    +-----------+  |
 *           |                   |   I/O    |                   |
 *       MMMMMMMMMMMMMMMMMMMMMMMMMM  Die   MMMMMMMMMMMMMMMMMMMMMMMMMM
 *       MMMMMMMMMMMMMMMMMMMMMMMMMM        MMMMMMMMMMMMMMMMMMMMMMMMMM
 *           |                   |          |                   |
 *       MMMMMMMMMMMMMMMMMMMMMMMMMM        MMMMMMMMMMMMMMMMMMMMMMMMMM
 *       MMMMMMMMMMMMMMMMMMMMMMMMMM        MMMMMMMMMMMMMMMMMMMMMMMMMM
 *           |                   |          |                   |
 *           |  +-----------+    |          |    +-----------+  |
 *           |  |         |o|oooo| PP    PP |oooo|o|         |  |
 *           |  |   Zen 2  -|    +-PP----PP-+    |-  Zen 2   |  |
 *           |  |    Die    |      PP    PP      |    Die    |  |
 *           |  |           |      PP    PP      |           |  |
 *           |  +-----------+      PP    PP      +-----------+  |
 *           |                     PP    PP                     |
 *           +---------------------PP----PP---------------------+
 *                                 PP    PP
 *                                 PP    PP
 *
 *	The above has four core complex dies installed, though the Zen 2 EPYC
 *	and ThreadRipper parts allow for up to eight, while the Ryzen parts
 *	generally only have one to two. The more notable difference here is how
 *	everything communicates. Note that memory and PCIe come out of the
 *	central die. This changes the way that one die accesses a resource. It
 *	basically always has to go to the I/O die, where as in Zen 1 it may have
 *	satisfied it locally. In general, this ends up being a better strategy
 *	for most things, though it is possible to still treat everything in four
 *	distinct NUMA domains with each Zen 2 die slightly closer to some memory
 *	and PCIe than otherwise. This also impacts the 'amdzen' nexus driver as
 *	now there is only one 'node' present.
 *
 * ZEN 3
 *
 *	From an architectural perspective, Zen 3 is a much smaller change from
 *	Zen 2 than Zen 2 was from Zen 1, though it makes up for most of that in
 *	its microarchitectural changes. The biggest thing for us is how the die
 *	changes. In Zen 1 and Zen 2, each core complex still had its own L3
 *	cache. However, in Zen 3, the L3 is now shared between the entire core
 *	complex die and is no longer partitioned between each core complex. This
 *	means that all cores on the die can share the same L3 cache. Otherwise,
 *	the general layout of the overall package with various core complexes
 *	and an I/O die stays the same. Here's what the Core Complex Die looks
 *	like in a bit more detail:
 *
 *               +-------------------------------------------------+
 *               | Zen 3 Core Complex Die                          |
 *               | +-------------------+    +-------------------+  |
 *               | | Core       +----+ |    | Core       +----+ |  |
 *               | | +--------+ | L2 | |    | +--------+ | L2 | |  |
 *               | | | Thread | +----+ |    | | Thread | +----+ |  |
 *               | | +--------+-+ +--+ |    | +--------+-+ +--+ |  |
 *               | |   | Thread | |L1| |    |   | Thread | |L1| |  |
 *               | |   +--------+ +--+ |    |   +--------+ +--+ |  |
 *               | +-------------------+    +-------------------+  |
 *               | +-------------------+    +-------------------+  |
 *               | | Core       +----+ |    | Core       +----+ |  |
 *               | | +--------+ | L2 | |    | +--------+ | L2 | |  |
 *               | | | Thread | +----+ |    | | Thread | +----+ |  |
 *               | | +--------+-+ +--+ |    | +--------+-+ +--+ |  |
 *               | |   | Thread | |L1| |    |   | Thread | |L1| |  |
 *               | |   +--------+ +--+ |    |   +--------+ +--+ |  |
 *               | +-------------------+    +-------------------+  |
 *               |                                                 |
 *               | +--------------------------------------------+  |
 *               | |                 L3 Cache                   |  |
 *               | +--------------------------------------------+  |
 *               |                                                 |
 *               | +-------------------+    +-------------------+  |
 *               | | Core       +----+ |    | Core       +----+ |  |
 *               | | +--------+ | L2 | |    | +--------+ | L2 | |  |
 *               | | | Thread | +----+ |    | | Thread | +----+ |  |
 *               | | +--------+-+ +--+ |    | +--------+-+ +--+ |  |
 *               | |   | Thread | |L1| |    |   | Thread | |L1| |  |
 *               | |   +--------+ +--+ |    |   +--------+ +--+ |  |
 *               | +-------------------+    +-------------------+  |
 *               | +-------------------+    +-------------------+  |
 *               | | Core       +----+ |    | Core       +----+ |  |
 *               | | +--------+ | L2 | |    | +--------+ | L2 | |  |
 *               | | | Thread | +----+ |    | | Thread | +----+ |  |
 *               | | +--------+-+ +--+ |    | +--------+-+ +--+ |  |
 *               | |   | Thread | |L1| |    |   | Thread | |L1| |  |
 *               | |   +--------+ +--+ |    |   +--------+ +--+ |  |
 *               | +-------------------+    +-------------------+  |
 *               +-------------------------------------------------+
 *
 *	While it is not pictured, there are connections from the die to the
 *	broader data fabric and additional functional blocks to support that
 *	communication and coherency.
 *
 * CPUID LEAVES
 *
 * There are a few different CPUID leaves that we can use to try and understand
 * the actual state of the world. As part of the introduction of family 0xf, AMD
 * added CPUID leaf 0x80000008. This leaf tells us the number of logical
 * processors that are in the system. Because families before Zen didn't have
 * SMT, this was always the number of cores that were in the system. However, it
 * should always be thought of as the number of logical threads to be consistent
 * between generations. In addition we also get the size of the APIC ID that is
 * used to represent the number of logical processors. This is important for
 * deriving topology information.
 *
 * In the Bulldozer family, AMD added leaf 0x8000001E. The information varies a
 * bit between Bulldozer and later families, but it is quite useful in
 * determining the topology information. Because this information has changed
 * across family generations, it's worth calling out what these mean
 * explicitly. The registers have the following meanings:
 *
 *	%eax	The APIC ID. The entire register is defined to have a 32-bit
 *		APIC ID, even though on systems without x2apic support, it will
 *		be limited to 8 bits.
 *
 *	%ebx	On Bulldozer-era systems this contains information about the
 *		number of cores that are in a compute unit (cores that share
 *		resources). It also contains a per-package compute unit ID that
 *		identifies which compute unit the logical CPU is a part of.
 *
 *		On Zen-era systems this instead contains the number of threads
 *		per core and the ID of the core that the logical CPU is a part
 *		of. Note, this ID is unique only to the package, it is not
 *		globally unique across the entire system.
 *
 *	%ecx	This contains the number of nodes that exist in the package. It
 *		also contains an ID that identifies which node the logical CPU
 *		is a part of.
 *
 * Finally, we also use cpuid leaf 0x8000001D to determine information about the
 * cache layout to determine which logical CPUs are sharing which caches.
 *
 * illumos Topology
 * ----------------
 *
 * Based on the above we synthesize the information into several different
 * variables that we store in the 'struct cpuid_info'. We'll go into the details
 * of what each member is supposed to represent and their uniqueness. In
 * general, there are two levels of uniqueness that we care about. We care about
 * an ID that is globally unique. That means that it will be unique across all
 * entities in the system. For example, the default logical CPU ID is globally
 * unique. On the other hand, there is some information that we only care about
 * being unique within the context of a single package / socket. Here are the
 * variables that we keep track of and their meaning.
 *
 * Several of the values that are asking for an identifier, with the exception
 * of cpi_apicid, are allowed to be synthetic.
 *
 *
 * cpi_apicid
 *
 *	This is the value of the CPU's APIC id. This should be the full 32-bit
 *	ID if the CPU is using the x2apic. Otherwise, it should be the 8-bit
 *	APIC ID. This value is globally unique between all logical CPUs across
 *	all packages. This is usually required by the APIC.
 *
 * cpi_chipid
 *
 *	This value indicates the ID of the package that the logical CPU is a
 *	part of. This value is allowed to be synthetic. It is usually derived by
 *	taking the CPU's APIC ID and determining how many bits are used to
 *	represent CPU cores in the package. All logical CPUs that are part of
 *	the same package must have the same value.
 *
 * cpi_coreid
 *
 *	This represents the ID of a CPU core. Two logical CPUs should only have
 *	the same cpi_coreid value if they are part of the same core. These
 *	values may be synthetic. On systems that support SMT, this value is
 *	usually derived from the APIC ID, otherwise it is often synthetic and
 *	just set to the value of the cpu_id in the cpu_t.
 *
 * cpi_pkgcoreid
 *
 *	This is similar to the cpi_coreid in that logical CPUs that are part of
 *	the same core should have the same ID. The main difference is that these
 *	values are only required to be unique to a given socket.
 *
 * cpi_clogid
 *
 *	This represents the logical ID of a logical CPU. This value should be
 *	unique within a given socket for each logical CPU. This is allowed to be
 *	synthetic, though it is usually based off of the CPU's apic ID. The
 *	broader system expects that logical CPUs that have are part of the same
 *	core have contiguous numbers. For example, if there were two threads per
 *	core, then the core IDs divided by two should be the same and the first
 *	modulus two should be zero and the second one. For example, IDs 4 and 5
 *	indicate two logical CPUs that are part of the same core. But IDs 5 and
 *	6 represent two logical CPUs that are part of different cores.
 *
 *	While it is common for the cpi_coreid and the cpi_clogid to be derived
 *	from the same source, strictly speaking, they don't have to be and the
 *	two values should be considered logically independent. One should not
 *	try to compare a logical CPU's cpi_coreid and cpi_clogid to determine
 *	some kind of relationship. While this is tempting, we've seen cases on
 *	AMD family 0xf where the system's cpu id is not related to its APIC ID.
 *
 * cpi_ncpu_per_chip
 *
 *	This value indicates the total number of logical CPUs that exist in the
 *	physical package. Critically, this is not the number of logical CPUs
 *	that exist for just the single core.
 *
 *	This value should be the same for all logical CPUs in the same package.
 *
 * cpi_ncore_per_chip
 *
 *	This value indicates the total number of physical CPU cores that exist
 *	in the package. The system compares this value with cpi_ncpu_per_chip to
 *	determine if simultaneous multi-threading (SMT) is enabled. When
 *	cpi_ncpu_per_chip equals cpi_ncore_per_chip, then there is no SMT and
 *	the X86FSET_HTT feature is not set. If this value is greater than one,
 *	than we consider the processor to have the feature X86FSET_CMP, to
 *	indicate that there is support for more than one core.
 *
 *	This value should be the same for all logical CPUs in the same package.
 *
 * cpi_procnodes_per_pkg
 *
 *	This value indicates the number of 'nodes' that exist in the package.
 *	When processors are actually a multi-chip module, this represents the
 *	number of such modules that exist in the package. Currently, on Intel
 *	based systems this member is always set to 1.
 *
 *	This value should be the same for all logical CPUs in the same package.
 *
 * cpi_procnodeid
 *
 *	This value indicates the ID of the node that the logical CPU is a part
 *	of. All logical CPUs that are in the same node must have the same value
 *	here. This value must be unique across all of the packages in the
 *	system.  On Intel based systems, this is currently set to the value in
 *	cpi_chipid because there is only one node.
 *
 * cpi_cores_per_compunit
 *
 *	This value indicates the number of cores that are part of a compute
 *	unit. See the AMD topology section for this. This member only has real
 *	meaning currently for AMD Bulldozer family processors. For all other
 *	processors, this should currently be set to 1.
 *
 * cpi_compunitid
 *
 *	This indicates the compute unit that the logical CPU belongs to. For
 *	processors without AMD Bulldozer-style compute units this should be set
 *	to the value of cpi_coreid.
 *
 * cpi_ncpu_shr_last_cache
 *
 *	This indicates the number of logical CPUs that are sharing the same last
 *	level cache. This value should be the same for all CPUs that are sharing
 *	that cache. The last cache refers to the cache that is closest to memory
 *	and furthest away from the CPU.
 *
 * cpi_last_lvl_cacheid
 *
 *	This indicates the ID of the last cache that the logical CPU uses. This
 *	cache is often shared between multiple logical CPUs and is the cache
 *	that is closest to memory and furthest away from the CPU. This value
 *	should be the same for a group of logical CPUs only if they actually
 *	share the same last level cache. IDs should not overlap between
 *	packages.
 *
 * cpi_ncore_bits
 *
 *	This indicates the number of bits that are required to represent all of
 *	the cores in the system. As cores are derived based on their APIC IDs,
 *	we aren't guaranteed a run of APIC IDs starting from zero. It's OK for
 *	this value to be larger than the actual number of IDs that are present
 *	in the system. This is used to size tables by the CMI framework. It is
 *	only filled in for Intel and AMD CPUs.
 *
 * cpi_nthread_bits
 *
 *	This indicates the number of bits required to represent all of the IDs
 *	that cover the logical CPUs that exist on a given core. It's OK for this
 *	value to be larger than the actual number of IDs that are present in the
 *	system.  This is used to size tables by the CMI framework. It is
 *	only filled in for Intel and AMD CPUs.
 *
 * -----------
 * Hypervisors
 * -----------
 *
 * If trying to manage the differences between vendors wasn't bad enough, it can
 * get worse thanks to our friend hardware virtualization. Hypervisors are given
 * the ability to interpose on all cpuid instructions and change them to suit
 * their purposes. In general, this is necessary as the hypervisor wants to be
 * able to present a more uniform set of features or not necessarily give the
 * guest operating system kernel knowledge of all features so it can be
 * more easily migrated between systems.
 *
 * When it comes to trying to determine topology information, this can be a
 * double edged sword. When a hypervisor doesn't actually implement a cpuid
 * leaf, it'll often return all zeros. Because of that, you'll often see various
 * checks scattered about fields being non-zero before we assume we can use
 * them.
 *
 * When it comes to topology information, the hypervisor is often incentivized
 * to lie to you about topology. This is because it doesn't always actually
 * guarantee that topology at all. The topology path we take in the system
 * depends on how the CPU advertises itself. If it advertises itself as an Intel
 * or AMD CPU, then we basically do our normal path. However, when they don't
 * use an actual vendor, then that usually turns into multiple one-core CPUs
 * that we enumerate that are often on different sockets. The actual behavior
 * depends greatly on what the hypervisor actually exposes to us.
 *
 * --------------------
 * Exposing Information
 * --------------------
 *
 * We expose CPUID information in three different forms in the system.
 *
 * The first is through the x86_featureset variable. This is used in conjunction
 * with the is_x86_feature() function. This is queried by x86-specific functions
 * to determine which features are or aren't present in the system and to make
 * decisions based upon them. For example, users of this include everything from
 * parts of the system dedicated to reliability, availability, and
 * serviceability (RAS), to making decisions about how to handle security
 * mitigations, to various x86-specific drivers. General purpose or
 * architecture independent drivers should never be calling this function.
 *
 * The second means is through the auxiliary vector. The auxiliary vector is a
 * series of tagged data that the kernel passes down to a user program when it
 * begins executing. This information is used to indicate to programs what
 * instruction set extensions are present. For example, information about the
 * CPU supporting the machine check architecture (MCA) wouldn't be passed down
 * since user programs cannot make use of it. However, things like the AVX
 * instruction sets are. Programs use this information to make run-time
 * decisions about what features they should use. As an example, the run-time
 * link-editor (rtld) can relocate different functions depending on the hardware
 * support available.
 *
 * The final form is through a series of accessor functions that all have the
 * form cpuid_get*. This is used by a number of different subsystems in the
 * kernel to determine more detailed information about what we're running on,
 * topology information, etc. Some of these subsystems include processor groups
 * (uts/common/os/pg.c.), CPU Module Interface (uts/i86pc/os/cmi.c), ACPI,
 * microcode, and performance monitoring. These functions all ASSERT that the
 * CPU they're being called on has reached a certain cpuid pass. If the passes
 * are rearranged, then this needs to be adjusted.
 *
 * -----------------------------------------------
 * Speculative Execution CPU Side Channel Security
 * -----------------------------------------------
 *
 * With the advent of the Spectre and Meltdown attacks which exploit speculative
 * execution in the CPU to create side channels there have been a number of
 * different attacks and corresponding issues that the operating system needs to
 * mitigate against. The following list is some of the common, but not
 * exhaustive, set of issues that we know about and have done some or need to do
 * more work in the system to mitigate against:
 *
 *   - Spectre v1
 *   - swapgs (Spectre v1 variant)
 *   - Spectre v2
 *     - Branch History Injection (BHI).
 *   - Meltdown (Spectre v3)
 *   - Rogue Register Read (Spectre v3a)
 *   - Speculative Store Bypass (Spectre v4)
 *   - ret2spec, SpectreRSB
 *   - L1 Terminal Fault (L1TF)
 *   - Microarchitectural Data Sampling (MDS)
 *   - Register File Data Sampling (RFDS)
 *
 * Each of these requires different sets of mitigations and has different attack
 * surfaces. For the most part, this discussion is about protecting the kernel
 * from non-kernel executing environments such as user processes and hardware
 * virtual machines. Unfortunately, there are a number of user vs. user
 * scenarios that exist with these. The rest of this section will describe the
 * overall approach that the system has taken to address these as well as their
 * shortcomings. Unfortunately, not all of the above have been handled today.
 *
 * SPECTRE v2, ret2spec, SpectreRSB
 *
 * The second variant of the spectre attack focuses on performing branch target
 * injection. This generally impacts indirect call instructions in the system.
 * There are four different ways to mitigate this issue that are commonly
 * described today:
 *
 *  1. Using Indirect Branch Restricted Speculation (IBRS).
 *  2. Using Retpolines and RSB Stuffing
 *  3. Using Enhanced Indirect Branch Restricted Speculation (eIBRS)
 *  4. Using Automated Indirect Branch Restricted Speculation (AIBRS)
 *
 * IBRS uses a feature added to microcode to restrict speculation, among other
 * things. This form of mitigation has not been used as it has been generally
 * seen as too expensive and requires reactivation upon various transitions in
 * the system.
 *
 * As a less impactful alternative to IBRS, retpolines were developed by
 * Google. These basically require one to replace indirect calls with a specific
 * trampoline that will cause speculation to fail and break the attack.
 * Retpolines require compiler support. We always build with retpolines in the
 * external thunk mode. This means that a traditional indirect call is replaced
 * with a call to one of the __x86_indirect_thunk_<reg> functions. A side effect
 * of this is that all indirect function calls are performed through a register.
 *
 * We have to use a common external location of the thunk and not inline it into
 * the callsite so that way we can have a single place to patch these functions.
 * As it turns out, we currently have two different forms of retpolines that
 * exist in the system:
 *
 *  1. A full retpoline
 *  2. A no-op version
 *
 * The first one is used in the general case. Historically, there was an
 * AMD-specific optimized retopoline variant that was based around using a
 * serializing lfence instruction; however, in March 2022 it was announced that
 * this was actually still vulnerable to Spectre v2 and therefore we no longer
 * use it and it is no longer available in the system.
 *
 * The third form described above is the most curious. It turns out that the way
 * that retpolines are implemented is that they rely on how speculation is
 * performed on a 'ret' instruction. Intel has continued to optimize this
 * process (which is partly why we need to have return stack buffer stuffing,
 * but more on that in a bit) and in processors starting with Cascade Lake
 * on the server side, it's dangerous to rely on retpolines. Instead, a new
 * mechanism has been introduced called Enhanced IBRS (eIBRS).
 *
 * Unlike IBRS, eIBRS is designed to be enabled once at boot and left on each
 * physical core. However, if this is the case, we don't want to use retpolines
 * any more. Therefore if eIBRS is present, we end up turning each retpoline
 * function (called a thunk) into a jmp instruction. This means that we're still
 * paying the cost of an extra jump to the external thunk, but it gives us
 * flexibility and the ability to have a single kernel image that works across a
 * wide variety of systems and hardware features.
 *
 * Unfortunately, this alone is insufficient. First, Skylake systems have
 * additional speculation for the Return Stack Buffer (RSB) which is used to
 * return from call instructions which retpolines take advantage of. However,
 * this problem is not just limited to Skylake and is actually more pernicious.
 * The SpectreRSB paper introduces several more problems that can arise with
 * dealing with this. The RSB can be poisoned just like the indirect branch
 * predictor. This means that one needs to clear the RSB when transitioning
 * between two different privilege domains. Some examples include:
 *
 *  - Switching between two different user processes
 *  - Going between user land and the kernel
 *  - Returning to the kernel from a hardware virtual machine
 *
 * Mitigating this involves combining a couple of different things. The first is
 * SMEP (supervisor mode execution protection) which was introduced in Ivy
 * Bridge. When an RSB entry refers to a user address and we're executing in the
 * kernel, speculation through it will be stopped when SMEP is enabled. This
 * protects against a number of the different cases that we would normally be
 * worried about such as when we enter the kernel from user land.
 *
 * To prevent against additional manipulation of the RSB from other contexts
 * such as a non-root VMX context attacking the kernel we first look to
 * enhanced IBRS. When eIBRS is present and enabled, then there should be
 * nothing else that we need to do to protect the kernel at this time.
 *
 * Unfortunately, not all eIBRS implementations are sufficient to guard
 * against RSB manipulations, so we still need to manually overwrite the
 * contents of the return stack buffer unless the hardware specifies we are
 * covered. We do this through the x86_rsb_stuff() function.  Currently this
 * is employed on context switch and vmx_exit. The x86_rsb_stuff() function is
 * disabled only when mitigations in general are, or if we have hardware
 * indicating no need for post-barrier RSB protections, either in one place
 * (old hardware), or on both (newer hardware).
 *
 * If SMEP is not present, then we would have to stuff the RSB every time we
 * transitioned from user mode to the kernel, which isn't very practical right
 * now.
 *
 * To fully protect user to user and vmx to vmx attacks from these classes of
 * issues, we would also need to allow them to opt into performing an Indirect
 * Branch Prediction Barrier (IBPB) on switch. This is not currently wired up.
 *
 * The fourth form of mitigation here is specific to AMD and is called Automated
 * IBRS (AIBRS). This is similar in spirit to eIBRS; however rather than set the
 * IBRS bit in MSR_IA32_SPEC_CTRL (0x48) we instead set a bit in the EFER
 * (extended feature enable register) MSR. This bit basically says that IBRS
 * acts as though it is always active when executing at CPL0 and when executing
 * in the 'host' context when SEV-SNP is enabled.
 *
 * When this is active, AMD states that the RSB is cleared on VMEXIT and
 * therefore it is unnecessary. While this handles RSB stuffing attacks from SVM
 * to the kernel, we must still consider the remaining cases that exist, just
 * like above. While traditionally AMD employed a 32 entry RSB allowing the
 * traditional technique to work, this is not true on all CPUs. While a write to
 * IBRS would clear the RSB if the processor supports more than 32 entries (but
 * not otherwise), AMD states that as long as at leat a single 4 KiB unmapped
 * guard page is present between user and kernel address spaces and SMEP is
 * enabled, then there is no need to clear the RSB at all.
 *
 * By default, the system will enable RSB stuffing and the required variant of
 * retpolines and store that information in the x86_spectrev2_mitigation value.
 * This will be evaluated after a microcode update as well, though it is
 * expected that microcode updates will not take away features. This may mean
 * that a late loaded microcode may not end up in the optimal configuration
 * (though this should be rare).
 *
 * Currently we do not build kmdb with retpolines or perform any additional side
 * channel security mitigations for it. One complication with kmdb is that it
 * requires its own retpoline thunks and it would need to adjust itself based on
 * what the kernel does. The threat model of kmdb is more limited and therefore
 * it may make more sense to investigate using prediction barriers as the whole
 * system is only executing a single instruction at a time while in kmdb.
 *
 * Branch History Injection (BHI)
 *
 * BHI is a specific form of SPECTREv2 where an attacker may manipulate branch
 * history before transitioning from user to supervisor mode (or from VMX
 * non-root/guest to root mode). The attacker can then exploit certain
 * compiler-generated code-sequences ("gadgets") to disclose information from
 * other contexts or domains.  Recent (late-2023/early-2024) research in
 * object code analysis discovered many more potential gadgets than what was
 * initially reported (which previously was confined to Linux use of
 * unprivileged eBPF).
 *
 * The BHI threat doesn't exist in processsors that predate eIBRS, or in AMD
 * ones. Some eIBRS processors have the ability to disable branch history in
 * certain (but not all) cases using an MSR write. eIBRS processors that don't
 * have the ability to disable must use a software sequence to scrub the
 * branch history buffer.
 *
 * BHI_DIS_S (the aforementioned MSR) prevents ring 0 from ring 3 (VMX guest
 * or VMX root). It does not protect different user processes from each other,
 * or ring 3 VMX guest from ring 3 VMX root or vice versa.
 *
 * The BHI clearing sequence prevents user exploiting kernel gadgets, and user
 * A's use of user B's gadgets.
 *
 * SMEP and eIBRS are a continuing defense-in-depth measure protecting the
 * kernel.
 *
 * SPECTRE v1, v4
 *
 * The v1 and v4 variants of spectre are not currently mitigated in the
 * system and require other classes of changes to occur in the code.
 *
 * SPECTRE v1 (SWAPGS VARIANT)
 *
 * The class of Spectre v1 vulnerabilities aren't all about bounds checks, but
 * can generally affect any branch-dependent code. The swapgs issue is one
 * variant of this. If we are coming in from userspace, we can have code like
 * this:
 *
 *	cmpw	$KCS_SEL, REGOFF_CS(%rsp)
 *	je	1f
 *	movq	$0, REGOFF_SAVFP(%rsp)
 *	swapgs
 *	1:
 *	movq	%gs:CPU_THREAD, %rax
 *
 * If an attacker can cause a mis-speculation of the branch here, we could skip
 * the needed swapgs, and use the /user/ %gsbase as the base of the %gs-based
 * load. If subsequent code can act as the usual Spectre cache gadget, this
 * would potentially allow KPTI bypass. To fix this, we need an lfence prior to
 * any use of the %gs override.
 *
 * The other case is also an issue: if we're coming into a trap from kernel
 * space, we could mis-speculate and swapgs the user %gsbase back in prior to
 * using it. AMD systems are not vulnerable to this version, as a swapgs is
 * serializing with respect to subsequent uses. But as AMD /does/ need the other
 * case, and the fix is the same in both cases (an lfence at the branch target
 * 1: in this example), we'll just do it unconditionally.
 *
 * Note that we don't enable user-space "wrgsbase" via CR4_FSGSBASE, making it
 * harder for user-space to actually set a useful %gsbase value: although it's
 * not clear, it might still be feasible via lwp_setprivate(), though, so we
 * mitigate anyway.
 *
 * MELTDOWN
 *
 * Meltdown, or spectre v3, allowed a user process to read any data in their
 * address space regardless of whether or not the page tables in question
 * allowed the user to have the ability to read them. The solution to meltdown
 * is kernel page table isolation. In this world, there are two page tables that
 * are used for a process, one in user land and one in the kernel. To implement
 * this we use per-CPU page tables and switch between the user and kernel
 * variants when entering and exiting the kernel.  For more information about
 * this process and how the trampolines work, please see the big theory
 * statements and additional comments in:
 *
 *  - uts/i86pc/ml/kpti_trampolines.s
 *  - uts/i86pc/vm/hat_i86.c
 *
 * While Meltdown only impacted Intel systems and there are also Intel systems
 * that have Meltdown fixed (called Rogue Data Cache Load), we always have
 * kernel page table isolation enabled. While this may at first seem weird, an
 * important thing to remember is that you can't speculatively read an address
 * if it's never in your page table at all. Having user processes without kernel
 * pages present provides us with an important layer of defense in the kernel
 * against any other side channel attacks that exist and have yet to be
 * discovered. As such, kernel page table isolation (KPTI) is always enabled by
 * default, no matter the x86 system.
 *
 * L1 TERMINAL FAULT
 *
 * L1 Terminal Fault (L1TF) takes advantage of an issue in how speculative
 * execution uses page table entries. Effectively, it is two different problems.
 * The first is that it ignores the not present bit in the page table entries
 * when performing speculative execution. This means that something can
 * speculatively read the listed physical address if it's present in the L1
 * cache under certain conditions (see Intel's documentation for the full set of
 * conditions). Secondly, this can be used to bypass hardware virtualization
 * extended page tables (EPT) that are part of Intel's hardware virtual machine
 * instructions.
 *
 * For the non-hardware virtualized case, this is relatively easy to deal with.
 * We must make sure that all unmapped pages have an address of zero. This means
 * that they could read the first 4k of physical memory; however, we never use
 * that first page in the operating system and always skip putting it in our
 * memory map, even if firmware tells us we can use it in our memory map. While
 * other systems try to put extra metadata in the address and reserved bits,
 * which led to this being problematic in those cases, we do not.
 *
 * For hardware virtual machines things are more complicated. Because they can
 * construct their own page tables, it isn't hard for them to perform this
 * attack against any physical address. The one wrinkle is that this physical
 * address must be in the L1 data cache. Thus Intel added an MSR that we can use
 * to flush the L1 data cache. We wrap this up in the function
 * spec_uarch_flush(). This function is also used in the mitigation of
 * microarchitectural data sampling (MDS) discussed later on. Kernel based
 * hypervisors such as KVM or bhyve are responsible for performing this before
 * entering the guest.
 *
 * Because this attack takes place in the L1 cache, there's another wrinkle
 * here. The L1 cache is shared between all logical CPUs in a core in most Intel
 * designs. This means that when a thread enters a hardware virtualized context
 * and flushes the L1 data cache, the other thread on the processor may then go
 * ahead and put new data in it that can be potentially attacked. While one
 * solution is to disable SMT on the system, another option that is available is
 * to use a feature for hardware virtualization called 'SMT exclusion'. This
 * goes through and makes sure that if a HVM is being scheduled on one thread,
 * then the thing on the other thread is from the same hardware virtual machine.
 * If an interrupt comes in or the guest exits to the broader system, then the
 * other SMT thread will be kicked out.
 *
 * L1TF can be fully mitigated by hardware. If the RDCL_NO feature is set in the
 * architecture capabilities MSR (MSR_IA32_ARCH_CAPABILITIES), then we will not
 * perform L1TF related mitigations.
 *
 * MICROARCHITECTURAL DATA SAMPLING
 *
 * Microarchitectural data sampling (MDS) is a combination of four discrete
 * vulnerabilities that are similar issues affecting various parts of the CPU's
 * microarchitectural implementation around load, store, and fill buffers.
 * Specifically it is made up of the following subcomponents:
 *
 *  1. Microarchitectural Store Buffer Data Sampling (MSBDS)
 *  2. Microarchitectural Fill Buffer Data Sampling (MFBDS)
 *  3. Microarchitectural Load Port Data Sampling (MLPDS)
 *  4. Microarchitectural Data Sampling Uncacheable Memory (MDSUM)
 *
 * To begin addressing these, Intel has introduced another feature in microcode
 * called MD_CLEAR. This changes the verw instruction to operate in a different
 * way. This allows us to execute the verw instruction in a particular way to
 * flush the state of the affected parts. The L1TF L1D flush mechanism is also
 * updated when this microcode is present to flush this state.
 *
 * Primarily we need to flush this state whenever we transition from the kernel
 * to a less privileged context such as user mode or an HVM guest. MSBDS is a
 * little bit different. Here the structures are statically sized when a logical
 * CPU is in use and resized when it goes to sleep. Therefore, we also need to
 * flush the microarchitectural state before the CPU goes idles by calling hlt,
 * mwait, or another ACPI method. To perform these flushes, we call
 * x86_md_clear() at all of these transition points.
 *
 * If hardware enumerates RDCL_NO, indicating that it is not vulnerable to L1TF,
 * then we change the spec_uarch_flush() function to point to x86_md_clear(). If
 * MDS_NO has been set, then this is fully mitigated and x86_md_clear() becomes
 * a no-op.
 *
 * Unfortunately, with this issue hyperthreading rears its ugly head. In
 * particular, everything we've discussed above is only valid for a single
 * thread executing on a core. In the case where you have hyper-threading
 * present, this attack can be performed between threads. The theoretical fix
 * for this is to ensure that both threads are always in the same security
 * domain. This means that they are executing in the same ring and mutually
 * trust each other. Practically speaking, this would mean that a system call
 * would have to issue an inter-processor interrupt (IPI) to the other thread.
 * Rather than implement this, we recommend that one disables hyper-threading
 * through the use of psradm -aS.
 *
 * TSX ASYNCHRONOUS ABORT
 *
 * TSX Asynchronous Abort (TAA) is another side-channel vulnerability that
 * behaves like MDS, but leverages Intel's transactional instructions as another
 * vector. Effectively, when a transaction hits one of these cases (unmapped
 * page, various cache snoop activity, etc.) then the same data can be exposed
 * as in the case of MDS. This means that you can attack your twin.
 *
 * Intel has described that there are two different ways that we can mitigate
 * this problem on affected processors:
 *
 *   1) We can use the same techniques used to deal with MDS. Flushing the
 *      microarchitectural buffers and disabling hyperthreading will mitigate
 *      this in the same way.
 *
 *   2) Using microcode to disable TSX.
 *
 * Now, most processors that are subject to MDS (as in they don't have MDS_NO in
 * the IA32_ARCH_CAPABILITIES MSR) will not receive microcode to disable TSX.
 * That's OK as we're already doing all such mitigations. On the other hand,
 * processors with MDS_NO are all supposed to receive microcode updates that
 * enumerate support for disabling TSX. In general, we'd rather use this method
 * when available as it doesn't require disabling hyperthreading to be
 * effective. Currently we basically are relying on microcode for processors
 * that enumerate MDS_NO.
 *
 * Another MDS-variant in a few select Intel Atom CPUs is Register File Data
 * Sampling: RFDS. This allows an attacker to sample values that were in any
 * of integer, floating point, or vector registers. This was discovered by
 * Intel during internal validation work.  The existence of the RFDS_NO
 * capability, or the LACK of a RFDS_CLEAR capability, means we do not have to
 * act. Intel has said some CPU models immune to RFDS MAY NOT enumerate
 * RFDS_NO. If RFDS_NO is not set, but RFDS_CLEAR is, we must set x86_md_clear,
 * and make sure it's using VERW. Unlike MDS, RFDS can't be helped by the
 * MSR that L1D uses.
 *
 * The microcode features are enumerated as part of the IA32_ARCH_CAPABILITIES.
 * When bit 7 (IA32_ARCH_CAP_TSX_CTRL) is present, then we are given two
 * different powers. The first allows us to cause all transactions to
 * immediately abort. The second gives us a means of disabling TSX completely,
 * which includes removing it from cpuid. If we have support for this in
 * microcode during the first cpuid pass, then we'll disable TSX completely such
 * that user land never has a chance to observe the bit. However, if we are late
 * loading the microcode, then we must use the functionality to cause
 * transactions to automatically abort. This is necessary for user land's sake.
 * Once a program sees a cpuid bit, it must not be taken away.
 *
 * We track whether or not we should do this based on what cpuid pass we're in.
 * Whenever we hit cpuid_scan_security() on the boot CPU and we're still on pass
 * 1 of the cpuid logic, then we can completely turn off TSX. Notably this
 * should happen twice. Once in the normal cpuid_pass_basic() code and then a
 * second time after we do the initial microcode update.  As a result we need to
 * be careful in cpuid_apply_tsx() to only use the MSR if we've loaded a
 * suitable microcode on the current CPU (which happens prior to
 * cpuid_pass_ucode()).
 *
 * If TAA has been fixed, then it will be enumerated in IA32_ARCH_CAPABILITIES
 * as TAA_NO. In such a case, we will still disable TSX: it's proven to be an
 * unfortunate feature in a number of ways, and taking the opportunity to
 * finally be able to turn it off is likely to be of benefit in the future.
 *
 * SUMMARY
 *
 * The following table attempts to summarize the mitigations for various issues
 * and what's done in various places:
 *
 *  - Spectre v1: Not currently mitigated
 *  - swapgs: lfences after swapgs paths
 *  - Spectre v2: Retpolines/RSB Stuffing or eIBRS/AIBRS if HW support
 *  - Meltdown: Kernel Page Table Isolation
 *  - Spectre v3a: Updated CPU microcode
 *  - Spectre v4: Not currently mitigated
 *  - SpectreRSB: SMEP and RSB Stuffing
 *  - L1TF: spec_uarch_flush, SMT exclusion, requires microcode
 *  - MDS: x86_md_clear, requires microcode, disabling SMT
 *  - TAA: x86_md_clear and disabling SMT OR microcode and disabling TSX
 *  - RFDS: microcode with x86_md_clear if RFDS_CLEAR set and RFDS_NO not.
 *  - BHI: software sequence, and use of BHI_DIS_S if microcode has it.
 *
 * The following table indicates the x86 feature set bits that indicate that a
 * given problem has been solved or a notable feature is present:
 *
 *  - RDCL_NO: Meltdown, L1TF, MSBDS subset of MDS
 *  - MDS_NO: All forms of MDS
 *  - TAA_NO: TAA
 *  - RFDS_NO: RFDS
 *  - BHI_NO: BHI
 */

#include <sys/types.h>
#include <sys/archsystm.h>
#include <sys/x86_archext.h>
#include <sys/kmem.h>
#include <sys/systm.h>
#include <sys/cmn_err.h>
#include <sys/sunddi.h>
#include <sys/sunndi.h>
#include <sys/cpuvar.h>
#include <sys/processor.h>
#include <sys/stdbool.h>
#include <sys/sysmacros.h>
#include <sys/pg.h>
#include <sys/fp.h>
#include <sys/controlregs.h>
#include <sys/bitmap.h>
#include <sys/auxv_386.h>
#include <sys/memnode.h>
#include <sys/pci_cfgspace.h>
#include <sys/comm_page.h>
#include <sys/mach_mmu.h>
#include <sys/ucode.h>
#include <sys/tsc.h>
#include <sys/kobj.h>
#include <sys/asm_misc.h>
#include <sys/bitmap.h>

#ifdef __xpv
#include <sys/hypervisor.h>
#else
#include <sys/ontrap.h>
#endif

uint_t x86_vendor = X86_VENDOR_IntelClone;
uint_t x86_type = X86_TYPE_OTHER;
uint_t x86_clflush_size = 0;

#if defined(__xpv)
int x86_use_pcid = 0;
int x86_use_invpcid = 0;
#else
int x86_use_pcid = -1;
int x86_use_invpcid = -1;
#endif

typedef enum {
	X86_SPECTREV2_RETPOLINE,
	X86_SPECTREV2_ENHANCED_IBRS,
	X86_SPECTREV2_AUTO_IBRS,
	X86_SPECTREV2_DISABLED
} x86_spectrev2_mitigation_t;

uint_t x86_disable_spectrev2 = 0;
static x86_spectrev2_mitigation_t x86_spectrev2_mitigation =
    X86_SPECTREV2_RETPOLINE;

/*
 * The mitigation status for TAA:
 * X86_TAA_NOTHING -- no mitigation available for TAA side-channels
 * X86_TAA_DISABLED -- mitigation disabled via x86_disable_taa
 * X86_TAA_MD_CLEAR -- MDS mitigation also suffices for TAA
 * X86_TAA_TSX_FORCE_ABORT -- transactions are forced to abort
 * X86_TAA_TSX_DISABLE -- force abort transactions and hide from CPUID
 * X86_TAA_HW_MITIGATED -- TSX potentially active but H/W not TAA-vulnerable
 */
typedef enum {
	X86_TAA_NOTHING,
	X86_TAA_DISABLED,
	X86_TAA_MD_CLEAR,
	X86_TAA_TSX_FORCE_ABORT,
	X86_TAA_TSX_DISABLE,
	X86_TAA_HW_MITIGATED
} x86_taa_mitigation_t;

uint_t x86_disable_taa = 0;
static x86_taa_mitigation_t x86_taa_mitigation = X86_TAA_NOTHING;

uint_t pentiumpro_bug4046376;

uchar_t x86_featureset[BT_SIZEOFMAP(NUM_X86_FEATURES)];

static char *x86_feature_names[NUM_X86_FEATURES] = {
	"lgpg",
	"tsc",
	"msr",
	"mtrr",
	"pge",
	"de",
	"cmov",
	"mmx",
	"mca",
	"pae",
	"cv8",
	"pat",
	"sep",
	"sse",
	"sse2",
	"htt",
	"asysc",
	"nx",
	"sse3",
	"cx16",
	"cmp",
	"tscp",
	"mwait",
	"sse4a",
	"cpuid",
	"ssse3",
	"sse4_1",
	"sse4_2",
	"1gpg",
	"clfsh",
	"64",
	"aes",
	"pclmulqdq",
	"xsave",
	"avx",
	"vmx",
	"svm",
	"topoext",
	"f16c",
	"rdrand",
	"x2apic",
	"avx2",
	"bmi1",
	"bmi2",
	"fma",
	"smep",
	"smap",
	"adx",
	"rdseed",
	"mpx",
	"avx512f",
	"avx512dq",
	"avx512pf",
	"avx512er",
	"avx512cd",
	"avx512bw",
	"avx512vl",
	"avx512fma",
	"avx512vbmi",
	"avx512_vpopcntdq",
	"avx512_4vnniw",
	"avx512_4fmaps",
	"xsaveopt",
	"xsavec",
	"xsaves",
	"sha",
	"umip",
	"pku",
	"ospke",
	"pcid",
	"invpcid",
	"ibrs",
	"ibpb",
	"stibp",
	"ssbd",
	"ssbd_virt",
	"rdcl_no",
	"ibrs_all",
	"rsba",
	"ssb_no",
	"stibp_all",
	"flush_cmd",
	"l1d_vmentry_no",
	"fsgsbase",
	"clflushopt",
	"clwb",
	"monitorx",
	"clzero",
	"xop",
	"fma4",
	"tbm",
	"avx512_vnni",
	"amd_pcec",
	"md_clear",
	"mds_no",
	"core_thermal",
	"pkg_thermal",
	"tsx_ctrl",
	"taa_no",
	"ppin",
	"vaes",
	"vpclmulqdq",
	"lfence_serializing",
	"gfni",
	"avx512_vp2intersect",
	"avx512_bitalg",
	"avx512_vbmi2",
	"avx512_bf16",
	"auto_ibrs",
	"rfds_no",
	"rfds_clear",
	"pbrsb_no",
	"bhi_no",
	"bhi_clear"
};

boolean_t
is_x86_feature(void *featureset, uint_t feature)
{
	ASSERT(feature < NUM_X86_FEATURES);
	return (BT_TEST((ulong_t *)featureset, feature));
}

void
add_x86_feature(void *featureset, uint_t feature)
{
	ASSERT(feature < NUM_X86_FEATURES);
	BT_SET((ulong_t *)featureset, feature);
}

void
remove_x86_feature(void *featureset, uint_t feature)
{
	ASSERT(feature < NUM_X86_FEATURES);
	BT_CLEAR((ulong_t *)featureset, feature);
}

boolean_t
compare_x86_featureset(void *setA, void *setB)
{
	/*
	 * We assume that the unused bits of the bitmap are always zero.
	 */
	if (memcmp(setA, setB, BT_SIZEOFMAP(NUM_X86_FEATURES)) == 0) {
		return (B_TRUE);
	} else {
		return (B_FALSE);
	}
}

void
print_x86_featureset(void *featureset)
{
	uint_t i;

	for (i = 0; i < NUM_X86_FEATURES; i++) {
		if (is_x86_feature(featureset, i)) {
			cmn_err(CE_CONT, "?x86_feature: %s\n",
			    x86_feature_names[i]);
		}
	}
}

/* Note: This is the maximum size for the CPU, not the size of the structure. */
static size_t xsave_state_size = 0;
uint64_t xsave_bv_all = (XFEATURE_LEGACY_FP | XFEATURE_SSE);
boolean_t xsave_force_disable = B_FALSE;
extern int disable_smap;

/*
 * This is set to platform type we are running on.
 */
static int platform_type = -1;

#if !defined(__xpv)
/*
 * Variable to patch if hypervisor platform detection needs to be
 * disabled (e.g. platform_type will always be HW_NATIVE if this is 0).
 */
int enable_platform_detection = 1;
#endif

/*
 * monitor/mwait info.
 *
 * size_actual and buf_actual are the real address and size allocated to get
 * proper mwait_buf alignement.  buf_actual and size_actual should be passed
 * to kmem_free().  Currently kmem_alloc() and mwait happen to both use
 * processor cache-line alignment, but this is not guarantied in the furture.
 */
struct mwait_info {
	size_t		mon_min;	/* min size to avoid missed wakeups */
	size_t		mon_max;	/* size to avoid false wakeups */
	size_t		size_actual;	/* size actually allocated */
	void		*buf_actual;	/* memory actually allocated */
	uint32_t	support;	/* processor support of monitor/mwait */
};

/*
 * xsave/xrestor info.
 *
 * This structure contains HW feature bits and the size of the xsave save area.
 * Note: the kernel declares a fixed size (AVX_XSAVE_SIZE) structure
 * (xsave_state) to describe the xsave layout. However, at runtime the
 * per-lwp xsave area is dynamically allocated based on xsav_max_size. The
 * xsave_state structure simply represents the legacy layout of the beginning
 * of the xsave area.
 */
struct xsave_info {
	uint32_t	xsav_hw_features_low;   /* Supported HW features */
	uint32_t	xsav_hw_features_high;  /* Supported HW features */
	size_t		xsav_max_size;  /* max size save area for HW features */
	size_t		ymm_size;	/* AVX: size of ymm save area */
	size_t		ymm_offset;	/* AVX: offset for ymm save area */
	size_t		bndregs_size;	/* MPX: size of bndregs save area */
	size_t		bndregs_offset;	/* MPX: offset for bndregs save area */
	size_t		bndcsr_size;	/* MPX: size of bndcsr save area */
	size_t		bndcsr_offset;	/* MPX: offset for bndcsr save area */
	size_t		opmask_size;	/* AVX512: size of opmask save */
	size_t		opmask_offset;	/* AVX512: offset for opmask save */
	size_t		zmmlo_size;	/* AVX512: size of zmm 256 save */
	size_t		zmmlo_offset;	/* AVX512: offset for zmm 256 save */
	size_t		zmmhi_size;	/* AVX512: size of zmm hi reg save */
	size_t		zmmhi_offset;	/* AVX512: offset for zmm hi reg save */
	size_t		pkru_size;	/* PKRU size */
	size_t		pkru_offset;	/* PKRU offset */
};


/*
 * These constants determine how many of the elements of the
 * cpuid we cache in the cpuid_info data structure; the
 * remaining elements are accessible via the cpuid instruction.
 */

#define	NMAX_CPI_STD	8		/* eax = 0 .. 7 */
#define	NMAX_CPI_EXTD	0x22		/* eax = 0x80000000 .. 0x80000021 */
#define	NMAX_CPI_TOPO	0x10		/* Sanity check on leaf 8X26, 1F */

/*
 * See the big theory statement for a more detailed explanation of what some of
 * these members mean.
 */
struct cpuid_info {
	uint_t cpi_pass;		/* last pass completed */
	/*
	 * standard function information
	 */
	uint_t cpi_maxeax;		/* fn 0: %eax */
	char cpi_vendorstr[13];		/* fn 0: %ebx:%ecx:%edx */
	uint_t cpi_vendor;		/* enum of cpi_vendorstr */

	uint_t cpi_family;		/* fn 1: extended family */
	uint_t cpi_model;		/* fn 1: extended model */
	uint_t cpi_step;		/* fn 1: stepping */
	chipid_t cpi_chipid;		/* fn 1: %ebx:  Intel: chip # */
					/*		AMD: package/socket # */
	uint_t cpi_brandid;		/* fn 1: %ebx: brand ID */
	int cpi_clogid;			/* fn 1: %ebx: thread # */
	uint_t cpi_ncpu_per_chip;	/* fn 1: %ebx: logical cpu count */
	uint8_t cpi_cacheinfo[16];	/* fn 2: intel-style cache desc */
	uint_t cpi_ncache;		/* fn 2: number of elements */
	uint_t cpi_ncpu_shr_last_cache;	/* fn 4: %eax: ncpus sharing cache */
	id_t cpi_last_lvl_cacheid;	/* fn 4: %eax: derived cache id */
	uint_t cpi_cache_leaf_size;	/* Number of cache elements */
					/* Intel fn: 4, AMD fn: 8000001d */
	struct cpuid_regs **cpi_cache_leaves;	/* Actual leaves from above */
	struct cpuid_regs cpi_std[NMAX_CPI_STD];	/* 0 .. 7 */
	struct cpuid_regs cpi_sub7[2];	/* Leaf 7, sub-leaves 1-2 */
	/*
	 * extended function information
	 */
	uint_t cpi_xmaxeax;		/* fn 0x80000000: %eax */
	char cpi_brandstr[49];		/* fn 0x8000000[234] */
	uint8_t cpi_pabits;		/* fn 0x80000006: %eax */
	uint8_t	cpi_vabits;		/* fn 0x80000006: %eax */
	uint8_t cpi_fp_amd_save;	/* AMD: FP error pointer save rqd. */
	struct	cpuid_regs cpi_extd[NMAX_CPI_EXTD];	/* 0x800000XX */

	id_t cpi_coreid;		/* same coreid => strands share core */
	int cpi_pkgcoreid;		/* core number within single package */
	uint_t cpi_ncore_per_chip;	/* AMD: fn 0x80000008: %ecx[7-0] */
					/* Intel: fn 4: %eax[31-26] */

	/*
	 * These values represent the number of bits that are required to store
	 * information about the number of cores and threads.
	 */
	uint_t cpi_ncore_bits;
	uint_t cpi_nthread_bits;
	/*
	 * supported feature information
	 */
	uint32_t cpi_support[6];
#define	STD_EDX_FEATURES	0
#define	AMD_EDX_FEATURES	1
#define	TM_EDX_FEATURES		2
#define	STD_ECX_FEATURES	3
#define	AMD_ECX_FEATURES	4
#define	STD_EBX_FEATURES	5
	/*
	 * Synthesized information, where known.
	 */
	x86_chiprev_t cpi_chiprev;	/* See X86_CHIPREV_* in x86_archext.h */
	const char *cpi_chiprevstr;	/* May be NULL if chiprev unknown */
	uint32_t cpi_socket;		/* Chip package/socket type */
	x86_uarchrev_t cpi_uarchrev;	/* Microarchitecture and revision */

	struct mwait_info cpi_mwait;	/* fn 5: monitor/mwait info */
	uint32_t cpi_apicid;
	uint_t cpi_procnodeid;		/* AMD: nodeID on HT, Intel: chipid */
	uint_t cpi_procnodes_per_pkg;	/* AMD: # of nodes in the package */
					/* Intel: 1 */
	uint_t cpi_compunitid;		/* AMD: ComputeUnit ID, Intel: coreid */
	uint_t cpi_cores_per_compunit;	/* AMD: # of cores in the ComputeUnit */

	struct xsave_info cpi_xsave;	/* fn D: xsave/xrestor info */

	/*
	 * AMD and Intel extended topology information. Leaf 8X26 (AMD) and
	 * eventually leaf 0x1F (Intel).
	 */
	uint_t cpi_topo_nleaves;
	struct cpuid_regs cpi_topo[NMAX_CPI_TOPO];
};


static struct cpuid_info cpuid_info0;

/*
 * These bit fields are defined by the Intel Application Note AP-485
 * "Intel Processor Identification and the CPUID Instruction"
 */
#define	CPI_FAMILY_XTD(cpi)	BITX((cpi)->cpi_std[1].cp_eax, 27, 20)
#define	CPI_MODEL_XTD(cpi)	BITX((cpi)->cpi_std[1].cp_eax, 19, 16)
#define	CPI_TYPE(cpi)		BITX((cpi)->cpi_std[1].cp_eax, 13, 12)
#define	CPI_FAMILY(cpi)		BITX((cpi)->cpi_std[1].cp_eax, 11, 8)
#define	CPI_STEP(cpi)		BITX((cpi)->cpi_std[1].cp_eax, 3, 0)
#define	CPI_MODEL(cpi)		BITX((cpi)->cpi_std[1].cp_eax, 7, 4)

#define	CPI_FEATURES_EDX(cpi)		((cpi)->cpi_std[1].cp_edx)
#define	CPI_FEATURES_ECX(cpi)		((cpi)->cpi_std[1].cp_ecx)
#define	CPI_FEATURES_XTD_EDX(cpi)	((cpi)->cpi_extd[1].cp_edx)
#define	CPI_FEATURES_XTD_ECX(cpi)	((cpi)->cpi_extd[1].cp_ecx)
#define	CPI_FEATURES_7_0_EBX(cpi)	((cpi)->cpi_std[7].cp_ebx)
#define	CPI_FEATURES_7_0_ECX(cpi)	((cpi)->cpi_std[7].cp_ecx)
#define	CPI_FEATURES_7_0_EDX(cpi)	((cpi)->cpi_std[7].cp_edx)
#define	CPI_FEATURES_7_1_EAX(cpi)	((cpi)->cpi_sub7[0].cp_eax)
#define	CPI_FEATURES_7_2_EDX(cpi)	((cpi)->cpi_sub7[1].cp_edx)

#define	CPI_BRANDID(cpi)	BITX((cpi)->cpi_std[1].cp_ebx, 7, 0)
#define	CPI_CHUNKS(cpi)		BITX((cpi)->cpi_std[1].cp_ebx, 15, 7)
#define	CPI_CPU_COUNT(cpi)	BITX((cpi)->cpi_std[1].cp_ebx, 23, 16)
#define	CPI_APIC_ID(cpi)	BITX((cpi)->cpi_std[1].cp_ebx, 31, 24)

#define	CPI_MAXEAX_MAX		0x100		/* sanity control */
#define	CPI_XMAXEAX_MAX		0x80000100
#define	CPI_FN4_ECX_MAX		0x20		/* sanity: max fn 4 levels */
#define	CPI_FNB_ECX_MAX		0x20		/* sanity: max fn B levels */

/*
 * Function 4 (Deterministic Cache Parameters) macros
 * Defined by Intel Application Note AP-485
 */
#define	CPI_NUM_CORES(regs)		BITX((regs)->cp_eax, 31, 26)
#define	CPI_NTHR_SHR_CACHE(regs)	BITX((regs)->cp_eax, 25, 14)
#define	CPI_FULL_ASSOC_CACHE(regs)	BITX((regs)->cp_eax, 9, 9)
#define	CPI_SELF_INIT_CACHE(regs)	BITX((regs)->cp_eax, 8, 8)
#define	CPI_CACHE_LVL(regs)		BITX((regs)->cp_eax, 7, 5)
#define	CPI_CACHE_TYPE(regs)		BITX((regs)->cp_eax, 4, 0)
#define	CPI_CACHE_TYPE_DONE	0
#define	CPI_CACHE_TYPE_DATA	1
#define	CPI_CACHE_TYPE_INSTR	2
#define	CPI_CACHE_TYPE_UNIFIED	3
#define	CPI_CPU_LEVEL_TYPE(regs)	BITX((regs)->cp_ecx, 15, 8)

#define	CPI_CACHE_WAYS(regs)		BITX((regs)->cp_ebx, 31, 22)
#define	CPI_CACHE_PARTS(regs)		BITX((regs)->cp_ebx, 21, 12)
#define	CPI_CACHE_COH_LN_SZ(regs)	BITX((regs)->cp_ebx, 11, 0)

#define	CPI_CACHE_SETS(regs)		BITX((regs)->cp_ecx, 31, 0)

#define	CPI_PREFCH_STRIDE(regs)		BITX((regs)->cp_edx, 9, 0)


/*
 * A couple of shorthand macros to identify "later" P6-family chips
 * like the Pentium M and Core.  First, the "older" P6-based stuff
 * (loosely defined as "pre-Pentium-4"):
 * P6, PII, Mobile PII, PII Xeon, PIII, Mobile PIII, PIII Xeon
 */
#define	IS_LEGACY_P6(cpi) (			\
	cpi->cpi_family == 6 &&			\
		(cpi->cpi_model == 1 ||		\
		cpi->cpi_model == 3 ||		\
		cpi->cpi_model == 5 ||		\
		cpi->cpi_model == 6 ||		\
		cpi->cpi_model == 7 ||		\
		cpi->cpi_model == 8 ||		\
		cpi->cpi_model == 0xA ||	\
		cpi->cpi_model == 0xB)		\
)

/* A "new F6" is everything with family 6 that's not the above */
#define	IS_NEW_F6(cpi) ((cpi->cpi_family == 6) && !IS_LEGACY_P6(cpi))

/* Extended family/model support */
#define	IS_EXTENDED_MODEL_INTEL(cpi) (cpi->cpi_family == 0x6 || \
	cpi->cpi_family >= 0xf)

/*
 * Info for monitor/mwait idle loop.
 *
 * See cpuid section of "Intel 64 and IA-32 Architectures Software Developer's
 * Manual Volume 2A: Instruction Set Reference, A-M" #25366-022US, November
 * 2006.
 * See MONITOR/MWAIT section of "AMD64 Architecture Programmer's Manual
 * Documentation Updates" #33633, Rev 2.05, December 2006.
 */
#define	MWAIT_SUPPORT		(0x00000001)	/* mwait supported */
#define	MWAIT_EXTENSIONS	(0x00000002)	/* extenstion supported */
#define	MWAIT_ECX_INT_ENABLE	(0x00000004)	/* ecx 1 extension supported */
#define	MWAIT_SUPPORTED(cpi)	((cpi)->cpi_std[1].cp_ecx & CPUID_INTC_ECX_MON)
#define	MWAIT_INT_ENABLE(cpi)	((cpi)->cpi_std[5].cp_ecx & 0x2)
#define	MWAIT_EXTENSION(cpi)	((cpi)->cpi_std[5].cp_ecx & 0x1)
#define	MWAIT_SIZE_MIN(cpi)	BITX((cpi)->cpi_std[5].cp_eax, 15, 0)
#define	MWAIT_SIZE_MAX(cpi)	BITX((cpi)->cpi_std[5].cp_ebx, 15, 0)
/*
 * Number of sub-cstates for a given c-state.
 */
#define	MWAIT_NUM_SUBC_STATES(cpi, c_state)			\
	BITX((cpi)->cpi_std[5].cp_edx, c_state + 3, c_state)

/*
 * XSAVE leaf 0xD enumeration
 */
#define	CPUID_LEAFD_2_YMM_OFFSET	576
#define	CPUID_LEAFD_2_YMM_SIZE		256

/*
 * Common extended leaf names to cut down on typos.
 */
#define	CPUID_LEAF_EXT_0		0x80000000
#define	CPUID_LEAF_EXT_8		0x80000008
#define	CPUID_LEAF_EXT_1d		0x8000001d
#define	CPUID_LEAF_EXT_1e		0x8000001e
#define	CPUID_LEAF_EXT_21		0x80000021
#define	CPUID_LEAF_EXT_26		0x80000026

/*
 * Functions we consume from cpuid_subr.c;  don't publish these in a header
 * file to try and keep people using the expected cpuid_* interfaces.
 */
extern uint32_t _cpuid_skt(uint_t, uint_t, uint_t, uint_t);
extern const char *_cpuid_sktstr(uint_t, uint_t, uint_t, uint_t);
extern x86_chiprev_t _cpuid_chiprev(uint_t, uint_t, uint_t, uint_t);
extern const char *_cpuid_chiprevstr(uint_t, uint_t, uint_t, uint_t);
extern x86_uarchrev_t _cpuid_uarchrev(uint_t, uint_t, uint_t, uint_t);
extern uint_t _cpuid_vendorstr_to_vendorcode(char *);

/*
 * Apply up various platform-dependent restrictions where the
 * underlying platform restrictions mean the CPU can be marked
 * as less capable than its cpuid instruction would imply.
 */
#if defined(__xpv)
static void
platform_cpuid_mangle(uint_t vendor, uint32_t eax, struct cpuid_regs *cp)
{
	switch (eax) {
	case 1: {
		uint32_t mcamask = DOMAIN_IS_INITDOMAIN(xen_info) ?
		    0 : CPUID_INTC_EDX_MCA;
		cp->cp_edx &=
		    ~(mcamask |
		    CPUID_INTC_EDX_PSE |
		    CPUID_INTC_EDX_VME | CPUID_INTC_EDX_DE |
		    CPUID_INTC_EDX_SEP | CPUID_INTC_EDX_MTRR |
		    CPUID_INTC_EDX_PGE | CPUID_INTC_EDX_PAT |
		    CPUID_AMD_EDX_SYSC | CPUID_INTC_EDX_SEP |
		    CPUID_INTC_EDX_PSE36 | CPUID_INTC_EDX_HTT);
		break;
	}

	case 0x80000001:
		cp->cp_edx &=
		    ~(CPUID_AMD_EDX_PSE |
		    CPUID_INTC_EDX_VME | CPUID_INTC_EDX_DE |
		    CPUID_AMD_EDX_MTRR | CPUID_AMD_EDX_PGE |
		    CPUID_AMD_EDX_PAT | CPUID_AMD_EDX_PSE36 |
		    CPUID_AMD_EDX_SYSC | CPUID_INTC_EDX_SEP |
		    CPUID_AMD_EDX_TSCP);
		cp->cp_ecx &= ~CPUID_AMD_ECX_CMP_LGCY;
		break;
	default:
		break;
	}

	switch (vendor) {
	case X86_VENDOR_Intel:
		switch (eax) {
		case 4:
			/*
			 * Zero out the (ncores-per-chip - 1) field
			 */
			cp->cp_eax &= 0x03fffffff;
			break;
		default:
			break;
		}
		break;
	case X86_VENDOR_AMD:
	case X86_VENDOR_HYGON:
		switch (eax) {

		case 0x80000001:
			cp->cp_ecx &= ~CPUID_AMD_ECX_CR8D;
			break;

		case CPUID_LEAF_EXT_8:
			/*
			 * Zero out the (ncores-per-chip - 1) field
			 */
			cp->cp_ecx &= 0xffffff00;
			break;
		default:
			break;
		}
		break;
	default:
		break;
	}
}
#else
#define	platform_cpuid_mangle(vendor, eax, cp)	/* nothing */
#endif

/*
 *  Some undocumented ways of patching the results of the cpuid
 *  instruction to permit running Solaris 10 on future cpus that
 *  we don't currently support.  Could be set to non-zero values
 *  via settings in eeprom.
 */

uint32_t cpuid_feature_ecx_include;
uint32_t cpuid_feature_ecx_exclude;
uint32_t cpuid_feature_edx_include;
uint32_t cpuid_feature_edx_exclude;

/*
 * Allocate space for mcpu_cpi in the machcpu structure for all non-boot CPUs.
 */
void
cpuid_alloc_space(cpu_t *cpu)
{
	/*
	 * By convention, cpu0 is the boot cpu, which is set up
	 * before memory allocation is available.  All other cpus get
	 * their cpuid_info struct allocated here.
	 */
	ASSERT(cpu->cpu_id != 0);
	ASSERT(cpu->cpu_m.mcpu_cpi == NULL);
	cpu->cpu_m.mcpu_cpi =
	    kmem_zalloc(sizeof (*cpu->cpu_m.mcpu_cpi), KM_SLEEP);
}

void
cpuid_free_space(cpu_t *cpu)
{
	struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
	int i;

	ASSERT(cpi != NULL);
	ASSERT(cpi != &cpuid_info0);

	/*
	 * Free up any cache leaf related dynamic storage. The first entry was
	 * cached from the standard cpuid storage, so we should not free it.
	 */
	for (i = 1; i < cpi->cpi_cache_leaf_size; i++)
		kmem_free(cpi->cpi_cache_leaves[i], sizeof (struct cpuid_regs));
	if (cpi->cpi_cache_leaf_size > 0)
		kmem_free(cpi->cpi_cache_leaves,
		    cpi->cpi_cache_leaf_size * sizeof (struct cpuid_regs *));

	kmem_free(cpi, sizeof (*cpi));
	cpu->cpu_m.mcpu_cpi = NULL;
}

#if !defined(__xpv)
/*
 * Determine the type of the underlying platform. This is used to customize
 * initialization of various subsystems (e.g. TSC). determine_platform() must
 * only ever be called once to prevent two processors from seeing different
 * values of platform_type. Must be called before cpuid_pass_ident(), the
 * earliest consumer to execute; the identification pass will call
 * synth_amd_info() to compute the chiprev, which in turn calls get_hwenv().
 */
void
determine_platform(void)
{
	struct cpuid_regs cp;
	uint32_t base;
	uint32_t regs[4];
	char *hvstr = (char *)regs;

	ASSERT(platform_type == -1);

	platform_type = HW_NATIVE;

	if (!enable_platform_detection)
		return;

	/*
	 * If Hypervisor CPUID bit is set, try to determine hypervisor
	 * vendor signature, and set platform type accordingly.
	 *
	 * References:
	 * http://lkml.org/lkml/2008/10/1/246
	 * http://kb.vmware.com/kb/1009458
	 */
	cp.cp_eax = 0x1;
	(void) __cpuid_insn(&cp);
	if ((cp.cp_ecx & CPUID_INTC_ECX_HV) != 0) {
		cp.cp_eax = 0x40000000;
		(void) __cpuid_insn(&cp);
		regs[0] = cp.cp_ebx;
		regs[1] = cp.cp_ecx;
		regs[2] = cp.cp_edx;
		regs[3] = 0;
		if (strcmp(hvstr, HVSIG_XEN_HVM) == 0) {
			platform_type = HW_XEN_HVM;
			return;
		}
		if (strcmp(hvstr, HVSIG_VMWARE) == 0) {
			platform_type = HW_VMWARE;
			return;
		}
		if (strcmp(hvstr, HVSIG_KVM) == 0) {
			platform_type = HW_KVM;
			return;
		}
		if (strcmp(hvstr, HVSIG_BHYVE) == 0) {
			platform_type = HW_BHYVE;
			return;
		}
		if (strcmp(hvstr, HVSIG_MICROSOFT) == 0) {
			platform_type = HW_MICROSOFT;
			return;
		}
		if (strcmp(hvstr, HVSIG_QEMU_TCG) == 0) {
			platform_type = HW_QEMU_TCG;
			return;
		}
		if (strcmp(hvstr, HVSIG_VIRTUALBOX) == 0) {
			platform_type = HW_VIRTUALBOX;
			return;
		}
		if (strcmp(hvstr, HVSIG_ACRN) == 0) {
			platform_type = HW_ACRN;
			return;
		}
	} else {
		/*
		 * Check older VMware hardware versions. VMware hypervisor is
		 * detected by performing an IN operation to VMware hypervisor
		 * port and checking that value returned in %ebx is VMware
		 * hypervisor magic value.
		 *
		 * References: http://kb.vmware.com/kb/1009458
		 */
		vmware_port(VMWARE_HVCMD_GETVERSION, regs);
		if (regs[1] == VMWARE_HVMAGIC) {
			platform_type = HW_VMWARE;
			return;
		}
	}

	/*
	 * Check Xen hypervisor. In a fully virtualized domain,
	 * Xen's pseudo-cpuid function returns a string representing the
	 * Xen signature in %ebx, %ecx, and %edx. %eax contains the maximum
	 * supported cpuid function. We need at least a (base + 2) leaf value
	 * to do what we want to do. Try different base values, since the
	 * hypervisor might use a different one depending on whether Hyper-V
	 * emulation is switched on by default or not.
	 */
	for (base = 0x40000000; base < 0x40010000; base += 0x100) {
		cp.cp_eax = base;
		(void) __cpuid_insn(&cp);
		regs[0] = cp.cp_ebx;
		regs[1] = cp.cp_ecx;
		regs[2] = cp.cp_edx;
		regs[3] = 0;
		if (strcmp(hvstr, HVSIG_XEN_HVM) == 0 &&
		    cp.cp_eax >= (base + 2)) {
			platform_type &= ~HW_NATIVE;
			platform_type |= HW_XEN_HVM;
			return;
		}
	}
}

int
get_hwenv(void)
{
	ASSERT(platform_type != -1);
	return (platform_type);
}

int
is_controldom(void)
{
	return (0);
}

#else

int
get_hwenv(void)
{
	return (HW_XEN_PV);
}

int
is_controldom(void)
{
	return (DOMAIN_IS_INITDOMAIN(xen_info));
}

#endif	/* __xpv */

/*
 * Gather the extended topology information. This should be the same for both
 * AMD leaf 8X26 and Intel leaf 0x1F (though the data interpretation varies).
 */
static void
cpuid_gather_ext_topo_leaf(struct cpuid_info *cpi, uint32_t leaf)
{
	uint_t i;

	for (i = 0; i < ARRAY_SIZE(cpi->cpi_topo); i++) {
		struct cpuid_regs *regs = &cpi->cpi_topo[i];

		bzero(regs, sizeof (struct cpuid_regs));
		regs->cp_eax = leaf;
		regs->cp_ecx = i;

		(void) __cpuid_insn(regs);
		if (CPUID_AMD_8X26_ECX_TYPE(regs->cp_ecx) ==
		    CPUID_AMD_8X26_TYPE_DONE) {
			break;
		}
	}

	cpi->cpi_topo_nleaves = i;
}

/*
 * Make sure that we have gathered all of the CPUID leaves that we might need to
 * determine topology. We assume that the standard leaf 1 has already been done
 * and that xmaxeax has already been calculated.
 */
static void
cpuid_gather_amd_topology_leaves(cpu_t *cpu)
{
	struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;

	if (cpi->cpi_xmaxeax >= CPUID_LEAF_EXT_8) {
		struct cpuid_regs *cp;

		cp = &cpi->cpi_extd[8];
		cp->cp_eax = CPUID_LEAF_EXT_8;
		(void) __cpuid_insn(cp);
		platform_cpuid_mangle(cpi->cpi_vendor, CPUID_LEAF_EXT_8, cp);
	}

	if (is_x86_feature(x86_featureset, X86FSET_TOPOEXT) &&
	    cpi->cpi_xmaxeax >= CPUID_LEAF_EXT_1e) {
		struct cpuid_regs *cp;

		cp = &cpi->cpi_extd[0x1e];
		cp->cp_eax = CPUID_LEAF_EXT_1e;
		(void) __cpuid_insn(cp);
	}

	if (cpi->cpi_xmaxeax >= CPUID_LEAF_EXT_26) {
		cpuid_gather_ext_topo_leaf(cpi, CPUID_LEAF_EXT_26);
	}
}

/*
 * Get the APIC ID for this processor. If Leaf B is present and valid, we prefer
 * it to everything else. If not, and we're on an AMD system where 8000001e is
 * valid, then we use that. Othewrise, we fall back to the default value for the
 * APIC ID in leaf 1.
 */
static uint32_t
cpuid_gather_apicid(struct cpuid_info *cpi)
{
	/*
	 * Leaf B changes based on the arguments to it. Because we don't cache
	 * it, we need to gather it again.
	 */
	if (cpi->cpi_maxeax >= 0xB) {
		struct cpuid_regs regs;
		struct cpuid_regs *cp;

		cp = &regs;
		cp->cp_eax = 0xB;
		cp->cp_edx = cp->cp_ebx = cp->cp_ecx = 0;
		(void) __cpuid_insn(cp);

		if (cp->cp_ebx != 0) {
			return (cp->cp_edx);
		}
	}

	if ((cpi->cpi_vendor == X86_VENDOR_AMD ||
	    cpi->cpi_vendor == X86_VENDOR_HYGON) &&
	    is_x86_feature(x86_featureset, X86FSET_TOPOEXT) &&
	    cpi->cpi_xmaxeax >= CPUID_LEAF_EXT_1e) {
		return (cpi->cpi_extd[0x1e].cp_eax);
	}

	return (CPI_APIC_ID(cpi));
}

/*
 * For AMD processors, attempt to calculate the number of chips and cores that
 * exist. The way that we do this varies based on the generation, because the
 * generations themselves have changed dramatically.
 *
 * If cpuid leaf 0x80000008 exists, that generally tells us the number of cores.
 * However, with the advent of family 17h (Zen) it actually tells us the number
 * of threads, so we need to look at leaf 0x8000001e if available to determine
 * its value. Otherwise, for all prior families, the number of enabled cores is
 * the same as threads.
 *
 * If we do not have leaf 0x80000008, then we assume that this processor does
 * not have anything. AMD's older CPUID specification says there's no reason to
 * fall back to leaf 1.
 *
 * In some virtualization cases we will not have leaf 8000001e or it will be
 * zero. When that happens we assume the number of threads is one.
 */
static void
cpuid_amd_ncores(struct cpuid_info *cpi, uint_t *ncpus, uint_t *ncores)
{
	uint_t nthreads, nthread_per_core;

	nthreads = nthread_per_core = 1;

	if (cpi->cpi_xmaxeax >= CPUID_LEAF_EXT_8) {
		nthreads = BITX(cpi->cpi_extd[8].cp_ecx, 7, 0) + 1;
	} else if ((cpi->cpi_std[1].cp_edx & CPUID_INTC_EDX_HTT) != 0) {
		nthreads = CPI_CPU_COUNT(cpi);
	}

	/*
	 * For us to have threads, and know about it, we have to be at least at
	 * family 17h and have the cpuid bit that says we have extended
	 * topology.
	 */
	if (cpi->cpi_family >= 0x17 &&
	    is_x86_feature(x86_featureset, X86FSET_TOPOEXT) &&
	    cpi->cpi_xmaxeax >= CPUID_LEAF_EXT_1e) {
		nthread_per_core = BITX(cpi->cpi_extd[0x1e].cp_ebx, 15, 8) + 1;
	}

	*ncpus = nthreads;
	*ncores = nthreads / nthread_per_core;
}

/*
 * Seed the initial values for the cores and threads for an Intel based
 * processor. These values will be overwritten if we detect that the processor
 * supports CPUID leaf 0xb.
 */
static void
cpuid_intel_ncores(struct cpuid_info *cpi, uint_t *ncpus, uint_t *ncores)
{
	/*
	 * Only seed the number of physical cores from the first level leaf 4
	 * information. The number of threads there indicate how many share the
	 * L1 cache, which may or may not have anything to do with the number of
	 * logical CPUs per core.
	 */
	if (cpi->cpi_maxeax >= 4) {
		*ncores = BITX(cpi->cpi_std[4].cp_eax, 31, 26) + 1;
	} else {
		*ncores = 1;
	}

	if ((cpi->cpi_std[1].cp_edx & CPUID_INTC_EDX_HTT) != 0) {
		*ncpus = CPI_CPU_COUNT(cpi);
	} else {
		*ncpus = *ncores;
	}
}

static boolean_t
cpuid_leafB_getids(cpu_t *cpu)
{
	struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
	struct cpuid_regs regs;
	struct cpuid_regs *cp;

	if (cpi->cpi_maxeax < 0xB)
		return (B_FALSE);

	cp = &regs;
	cp->cp_eax = 0xB;
	cp->cp_edx = cp->cp_ebx = cp->cp_ecx = 0;

	(void) __cpuid_insn(cp);

	/*
	 * Check CPUID.EAX=0BH, ECX=0H:EBX is non-zero, which
	 * indicates that the extended topology enumeration leaf is
	 * available.
	 */
	if (cp->cp_ebx != 0) {
		uint32_t x2apic_id = 0;
		uint_t coreid_shift = 0;
		uint_t ncpu_per_core = 1;
		uint_t chipid_shift = 0;
		uint_t ncpu_per_chip = 1;
		uint_t i;
		uint_t level;

		for (i = 0; i < CPI_FNB_ECX_MAX; i++) {
			cp->cp_eax = 0xB;
			cp->cp_ecx = i;

			(void) __cpuid_insn(cp);
			level = CPI_CPU_LEVEL_TYPE(cp);

			if (level == 1) {
				x2apic_id = cp->cp_edx;
				coreid_shift = BITX(cp->cp_eax, 4, 0);
				ncpu_per_core = BITX(cp->cp_ebx, 15, 0);
			} else if (level == 2) {
				x2apic_id = cp->cp_edx;
				chipid_shift = BITX(cp->cp_eax, 4, 0);
				ncpu_per_chip = BITX(cp->cp_ebx, 15, 0);
			}
		}

		/*
		 * cpi_apicid is taken care of in cpuid_gather_apicid.
		 */
		cpi->cpi_ncpu_per_chip = ncpu_per_chip;
		cpi->cpi_ncore_per_chip = ncpu_per_chip /
		    ncpu_per_core;
		cpi->cpi_chipid = x2apic_id >> chipid_shift;
		cpi->cpi_clogid = x2apic_id & ((1 << chipid_shift) - 1);
		cpi->cpi_coreid = x2apic_id >> coreid_shift;
		cpi->cpi_pkgcoreid = cpi->cpi_clogid >> coreid_shift;
		cpi->cpi_procnodeid = cpi->cpi_chipid;
		cpi->cpi_compunitid = cpi->cpi_coreid;

		if (coreid_shift > 0 && chipid_shift > coreid_shift) {
			cpi->cpi_nthread_bits = coreid_shift;
			cpi->cpi_ncore_bits = chipid_shift - coreid_shift;
		}

		return (B_TRUE);
	} else {
		return (B_FALSE);
	}
}

static void
cpuid_intel_getids(cpu_t *cpu, void *feature)
{
	uint_t i;
	uint_t chipid_shift = 0;
	uint_t coreid_shift = 0;
	struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;

	/*
	 * There are no compute units or processor nodes currently on Intel.
	 * Always set these to one.
	 */
	cpi->cpi_procnodes_per_pkg = 1;
	cpi->cpi_cores_per_compunit = 1;

	/*
	 * If cpuid Leaf B is present, use that to try and get this information.
	 * It will be the most accurate for Intel CPUs.
	 */
	if (cpuid_leafB_getids(cpu))
		return;

	/*
	 * In this case, we have the leaf 1 and leaf 4 values for ncpu_per_chip
	 * and ncore_per_chip. These represent the largest power of two values
	 * that we need to cover all of the IDs in the system. Therefore, we use
	 * those values to seed the number of bits needed to cover information
	 * in the case when leaf B is not available. These values will probably
	 * be larger than required, but that's OK.
	 */
	cpi->cpi_nthread_bits = ddi_fls(cpi->cpi_ncpu_per_chip);
	cpi->cpi_ncore_bits = ddi_fls(cpi->cpi_ncore_per_chip);

	for (i = 1; i < cpi->cpi_ncpu_per_chip; i <<= 1)
		chipid_shift++;

	cpi->cpi_chipid = cpi->cpi_apicid >> chipid_shift;
	cpi->cpi_clogid = cpi->cpi_apicid & ((1 << chipid_shift) - 1);

	if (is_x86_feature(feature, X86FSET_CMP)) {
		/*
		 * Multi-core (and possibly multi-threaded)
		 * processors.
		 */
		uint_t ncpu_per_core = 0;

		if (cpi->cpi_ncore_per_chip == 1)
			ncpu_per_core = cpi->cpi_ncpu_per_chip;
		else if (cpi->cpi_ncore_per_chip > 1)
			ncpu_per_core = cpi->cpi_ncpu_per_chip /
			    cpi->cpi_ncore_per_chip;
		/*
		 * 8bit APIC IDs on dual core Pentiums
		 * look like this:
		 *
		 * +-----------------------+------+------+
		 * | Physical Package ID   |  MC  |  HT  |
		 * +-----------------------+------+------+
		 * <------- chipid -------->
		 * <------- coreid --------------->
		 *			   <--- clogid -->
		 *			   <------>
		 *			   pkgcoreid
		 *
		 * Where the number of bits necessary to
		 * represent MC and HT fields together equals
		 * to the minimum number of bits necessary to
		 * store the value of cpi->cpi_ncpu_per_chip.
		 * Of those bits, the MC part uses the number
		 * of bits necessary to store the value of
		 * cpi->cpi_ncore_per_chip.
		 */
		for (i = 1; i < ncpu_per_core; i <<= 1)
			coreid_shift++;
		cpi->cpi_coreid = cpi->cpi_apicid >> coreid_shift;
		cpi->cpi_pkgcoreid = cpi->cpi_clogid >> coreid_shift;
	} else if (is_x86_feature(feature, X86FSET_HTT)) {
		/*
		 * Single-core multi-threaded processors.
		 */
		cpi->cpi_coreid = cpi->cpi_chipid;
		cpi->cpi_pkgcoreid = 0;
	} else {
		/*
		 * Single-core single-thread processors.
		 */
		cpi->cpi_coreid = cpu->cpu_id;
		cpi->cpi_pkgcoreid = 0;
	}
	cpi->cpi_procnodeid = cpi->cpi_chipid;
	cpi->cpi_compunitid = cpi->cpi_coreid;
}

/*
 * Historically, AMD has had CMP chips with only a single thread per core.
 * However, starting in family 17h (Zen), this has changed and they now have
 * multiple threads. Our internal core id needs to be a unique value.
 *
 * To determine the core id of an AMD system, if we're from a family before 17h,
 * then we just use the cpu id, as that gives us a good value that will be
 * unique for each core. If instead, we're on family 17h or later, then we need
 * to do something more complicated. CPUID leaf 0x8000001e can tell us
 * how many threads are in the system. Based on that, we'll shift the APIC ID.
 * We can't use the normal core id in that leaf as it's only unique within the
 * socket, which is perfect for cpi_pkgcoreid, but not us.
 */
static id_t
cpuid_amd_get_coreid(cpu_t *cpu)
{
	struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;

	if (cpi->cpi_family >= 0x17 &&
	    is_x86_feature(x86_featureset, X86FSET_TOPOEXT) &&
	    cpi->cpi_xmaxeax >= CPUID_LEAF_EXT_1e) {
		uint_t nthreads = BITX(cpi->cpi_extd[0x1e].cp_ebx, 15, 8) + 1;
		if (nthreads > 1) {
			VERIFY3U(nthreads, ==, 2);
			return (cpi->cpi_apicid >> 1);
		}
	}

	return (cpu->cpu_id);
}

/*
 * IDs on AMD is a more challenging task. This is notable because of the
 * following two facts:
 *
 *  1. Before family 0x17 (Zen), there was no support for SMT and there was
 *     also no way to get an actual unique core id from the system. As such, we
 *     synthesize this case by using cpu->cpu_id.  This scheme does not,
 *     however, guarantee that sibling cores of a chip will have sequential
 *     coreids starting at a multiple of the number of cores per chip - that is
 *     usually the case, but if the APIC IDs have been set up in a different
 *     order then we need to perform a few more gymnastics for the pkgcoreid.
 *
 *  2. In families 0x15 and 16x (Bulldozer and co.) the cores came in groups
 *     called compute units. These compute units share the L1I cache, L2 cache,
 *     and the FPU. To deal with this, a new topology leaf was added in
 *     0x8000001e. However, parts of this leaf have different meanings
 *     once we get to family 0x17.
 */

static void
cpuid_amd_getids(cpu_t *cpu, uchar_t *features)
{
	int i, first_half, coreidsz;
	uint32_t nb_caps_reg;
	uint_t node2_1;
	struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
	struct cpuid_regs *cp;

	/*
	 * Calculate the core id (this comes from hardware in family 0x17 if it
	 * hasn't been stripped by virtualization). We always set the compute
	 * unit id to the same value. Also, initialize the default number of
	 * cores per compute unit and nodes per package. This will be
	 * overwritten when we know information about a particular family.
	 */
	cpi->cpi_coreid = cpuid_amd_get_coreid(cpu);
	cpi->cpi_compunitid = cpi->cpi_coreid;
	cpi->cpi_cores_per_compunit = 1;
	cpi->cpi_procnodes_per_pkg = 1;

	/*
	 * To construct the logical ID, we need to determine how many APIC IDs
	 * are dedicated to the cores and threads. This is provided for us in
	 * 0x80000008. However, if it's not present (say due to virtualization),
	 * then we assume it's one. This should be present on all 64-bit AMD
	 * processors.  It was added in family 0xf (Hammer).
	 */
	if (cpi->cpi_xmaxeax >= CPUID_LEAF_EXT_8) {
		coreidsz = BITX((cpi)->cpi_extd[8].cp_ecx, 15, 12);

		/*
		 * In AMD parlance chip is really a node while illumos
		 * uses chip as equivalent to socket/package.
		 */
		if (coreidsz == 0) {
			/* Use legacy method */
			for (i = 1; i < cpi->cpi_ncore_per_chip; i <<= 1)
				coreidsz++;
			if (coreidsz == 0)
				coreidsz = 1;
		}
	} else {
		/* Assume single-core part */
		coreidsz = 1;
	}
	cpi->cpi_clogid = cpi->cpi_apicid & ((1 << coreidsz) - 1);

	/*
	 * The package core ID varies depending on the family. While it may be
	 * tempting to use the CPUID_LEAF_EXT_1e %ebx core id, unfortunately,
	 * this value is the core id in the given node. For non-virtualized
	 * family 17h, we need to take the logical core id and shift off the
	 * threads like we do when getting the core id.  Otherwise, we can use
	 * the clogid as is. When family 17h is virtualized, the clogid should
	 * be sufficient as if we don't have valid data in the leaf, then we
	 * won't think we have SMT, in which case the cpi_clogid should be
	 * sufficient.
	 */
	if (cpi->cpi_family >= 0x17 &&
	    is_x86_feature(x86_featureset, X86FSET_TOPOEXT) &&
	    cpi->cpi_xmaxeax >= CPUID_LEAF_EXT_1e &&
	    cpi->cpi_extd[0x1e].cp_ebx != 0) {
		uint_t nthreads = BITX(cpi->cpi_extd[0x1e].cp_ebx, 15, 8) + 1;
		if (nthreads > 1) {
			VERIFY3U(nthreads, ==, 2);
			cpi->cpi_pkgcoreid = cpi->cpi_clogid >> 1;
		} else {
			cpi->cpi_pkgcoreid = cpi->cpi_clogid;
		}
	} else {
		cpi->cpi_pkgcoreid = cpi->cpi_clogid;
	}

	/*
	 * Obtain the node ID and compute unit IDs. If we're on family 0x15
	 * (bulldozer) or newer, then we can derive all of this from leaf
	 * CPUID_LEAF_EXT_1e. Otherwise, the method varies by family.
	 */
	if (is_x86_feature(x86_featureset, X86FSET_TOPOEXT) &&
	    cpi->cpi_xmaxeax >= CPUID_LEAF_EXT_1e) {
		cp = &cpi->cpi_extd[0x1e];

		cpi->cpi_procnodes_per_pkg = BITX(cp->cp_ecx, 10, 8) + 1;
		cpi->cpi_procnodeid = BITX(cp->cp_ecx, 7, 0);

		/*
		 * For Bulldozer-era CPUs, recalculate the compute unit
		 * information.
		 */
		if (cpi->cpi_family >= 0x15 && cpi->cpi_family < 0x17) {
			cpi->cpi_cores_per_compunit =
			    BITX(cp->cp_ebx, 15, 8) + 1;
			cpi->cpi_compunitid = BITX(cp->cp_ebx, 7, 0) +
			    (cpi->cpi_ncore_per_chip /
			    cpi->cpi_cores_per_compunit) *
			    (cpi->cpi_procnodeid /
			    cpi->cpi_procnodes_per_pkg);
		}
	} else if (cpi->cpi_family == 0xf || cpi->cpi_family >= 0x11) {
		cpi->cpi_procnodeid = (cpi->cpi_apicid >> coreidsz) & 7;
	} else if (cpi->cpi_family == 0x10) {
		/*
		 * See if we are a multi-node processor.
		 * All processors in the system have the same number of nodes
		 */
		nb_caps_reg =  pci_getl_func(0, 24, 3, 0xe8);
		if ((cpi->cpi_model < 8) || BITX(nb_caps_reg, 29, 29) == 0) {
			/* Single-node */
			cpi->cpi_procnodeid = BITX(cpi->cpi_apicid, 5,
			    coreidsz);
		} else {

			/*
			 * Multi-node revision D (2 nodes per package
			 * are supported)
			 */
			cpi->cpi_procnodes_per_pkg = 2;

			first_half = (cpi->cpi_pkgcoreid <=
			    (cpi->cpi_ncore_per_chip/2 - 1));

			if (cpi->cpi_apicid == cpi->cpi_pkgcoreid) {
				/* We are BSP */
				cpi->cpi_procnodeid = (first_half ? 0 : 1);
			} else {

				/* We are AP */
				/* NodeId[2:1] bits to use for reading F3xe8 */
				node2_1 = BITX(cpi->cpi_apicid, 5, 4) << 1;

				nb_caps_reg =
				    pci_getl_func(0, 24 + node2_1, 3, 0xe8);

				/*
				 * Check IntNodeNum bit (31:30, but bit 31 is
				 * always 0 on dual-node processors)
				 */
				if (BITX(nb_caps_reg, 30, 30) == 0)
					cpi->cpi_procnodeid = node2_1 +
					    !first_half;
				else
					cpi->cpi_procnodeid = node2_1 +
					    first_half;
			}
		}
	} else {
		cpi->cpi_procnodeid = 0;
	}

	cpi->cpi_chipid =
	    cpi->cpi_procnodeid / cpi->cpi_procnodes_per_pkg;

	cpi->cpi_ncore_bits = coreidsz;
	cpi->cpi_nthread_bits = ddi_fls(cpi->cpi_ncpu_per_chip /
	    cpi->cpi_ncore_per_chip);
}

static void
spec_uarch_flush_noop(void)
{
}

/*
 * When microcode is present that mitigates MDS, this wrmsr will also flush the
 * MDS-related micro-architectural state that would normally happen by calling
 * x86_md_clear().
 */
static void
spec_uarch_flush_msr(void)
{
	wrmsr(MSR_IA32_FLUSH_CMD, IA32_FLUSH_CMD_L1D);
}

/*
 * This function points to a function that will flush certain
 * micro-architectural state on the processor. This flush is used to mitigate
 * three different classes of Intel CPU vulnerabilities: L1TF, MDS, and RFDS.
 * This function can point to one of three functions:
 *
 * - A noop which is done because we either are vulnerable, but do not have
 *   microcode available to help deal with a fix, or because we aren't
 *   vulnerable.
 *
 * - spec_uarch_flush_msr which will issue an L1D flush and if microcode to
 *   mitigate MDS is present, also perform the equivalent of the MDS flush;
 *   however, it only flushes the MDS related micro-architectural state on the
 *   current hyperthread, it does not do anything for the twin.
 *
 * - x86_md_clear which will flush the MDS related state. This is done when we
 *   have a processor that is vulnerable to MDS, but is not vulnerable to L1TF
 *   (RDCL_NO is set); or if the CPU is vulnerable to RFDS and indicates VERW
 *   can clear it (RFDS_CLEAR is set).
 */
void (*spec_uarch_flush)(void) = spec_uarch_flush_noop;

static void
cpuid_update_md_clear(cpu_t *cpu, uchar_t *featureset)
{
	struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;

	/* Non-Intel doesn't concern us here. */
	if (cpi->cpi_vendor != X86_VENDOR_Intel)
		return;

	/*
	 * While RDCL_NO indicates that one of the MDS vulnerabilities (MSBDS)
	 * has been fixed in hardware, it doesn't cover everything related to
	 * MDS. Therefore we can only rely on MDS_NO to determine that we don't
	 * need to mitigate this.
	 *
	 * We must ALSO check the case of RFDS_NO and if RFDS_CLEAR is set,
	 * because of the small cases of RFDS.
	 */

	if ((!is_x86_feature(featureset, X86FSET_MDS_NO) &&
	    is_x86_feature(featureset, X86FSET_MD_CLEAR)) ||
	    (!is_x86_feature(featureset, X86FSET_RFDS_NO) &&
	    is_x86_feature(featureset, X86FSET_RFDS_CLEAR))) {
		const uint8_t nop = NOP_INSTR;
		uint8_t *md = (uint8_t *)x86_md_clear;

		*md = nop;
	}

	membar_producer();
}

static void
cpuid_update_l1d_flush(cpu_t *cpu, uchar_t *featureset)
{
	boolean_t need_l1d, need_mds, need_rfds;
	struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;

	/*
	 * If we're not on Intel or we've mitigated all of RDCL, MDS, and RFDS
	 * in hardware, then there's nothing left for us to do for enabling
	 * the flush. We can also go ahead and say that SMT exclusion is
	 * unnecessary.
	 */
	if (cpi->cpi_vendor != X86_VENDOR_Intel ||
	    (is_x86_feature(featureset, X86FSET_RDCL_NO) &&
	    is_x86_feature(featureset, X86FSET_MDS_NO) &&
	    is_x86_feature(featureset, X86FSET_RFDS_NO))) {
		extern int smt_exclusion;
		smt_exclusion = 0;
		spec_uarch_flush = spec_uarch_flush_noop;
		membar_producer();
		return;
	}

	/*
	 * The locations where we need to perform an L1D flush are required both
	 * for mitigating L1TF and MDS. When verw support is present in
	 * microcode, then the L1D flush will take care of doing that as well.
	 * However, if we have a system where RDCL_NO is present, but we don't
	 * have MDS_NO, then we need to do a verw (x86_md_clear) and not a full
	 * L1D flush.
	 */
	if (!is_x86_feature(featureset, X86FSET_RDCL_NO) &&
	    is_x86_feature(featureset, X86FSET_FLUSH_CMD) &&
	    !is_x86_feature(featureset, X86FSET_L1D_VM_NO)) {
		need_l1d = B_TRUE;
	} else {
		need_l1d = B_FALSE;
	}

	if (!is_x86_feature(featureset, X86FSET_MDS_NO) &&
	    is_x86_feature(featureset, X86FSET_MD_CLEAR)) {
		need_mds = B_TRUE;
	} else {
		need_mds = B_FALSE;
	}

	if (!is_x86_feature(featureset, X86FSET_RFDS_NO) &&
	    is_x86_feature(featureset, X86FSET_RFDS_CLEAR)) {
		need_rfds = B_TRUE;
	} else {
		need_rfds = B_FALSE;
	}

	if (need_l1d) {
		/*
		 * As of Feb, 2024, no CPU needs L1D *and* RFDS mitigation
		 * together. If the following VERIFY trips, we need to add
		 * further fixes here.
		 */
		VERIFY(!need_rfds);
		spec_uarch_flush = spec_uarch_flush_msr;
	} else if (need_mds || need_rfds) {
		spec_uarch_flush = x86_md_clear;
	} else {
		/*
		 * We have no hardware mitigations available to us.
		 */
		spec_uarch_flush = spec_uarch_flush_noop;
	}
	membar_producer();
}

/*
 * Branch History Injection (BHI) mitigations.
 *
 * Intel has provided a software sequence that will scrub the BHB. Like RSB
 * (below) we can scribble a return at the beginning to avoid if if the CPU
 * is modern enough. We can also scribble a return if the CPU is old enough
 * to not have an RSB (pre-eIBRS).
 */
typedef enum {
	X86_BHI_TOO_OLD_OR_DISABLED,	/* Pre-eIBRS or disabled */
	X86_BHI_NEW_ENOUGH,		/* AMD, or Intel with BHI_NO set */
	X86_BHI_DIS_S,			/* BHI_NO == 0, but BHI_DIS_S avail. */
	/* NOTE: BHI_DIS_S above will still need the software sequence. */
	X86_BHI_SOFTWARE_SEQUENCE,	/* Use software sequence */
} x86_native_bhi_mitigation_t;

x86_native_bhi_mitigation_t x86_bhi_mitigation = X86_BHI_SOFTWARE_SEQUENCE;

static void
cpuid_enable_bhi_dis_s(void)
{
	uint64_t val;

	val = rdmsr(MSR_IA32_SPEC_CTRL);
	val |= IA32_SPEC_CTRL_BHI_DIS_S;
	wrmsr(MSR_IA32_SPEC_CTRL, val);
}

/*
 * This function scribbles RET into the first instruction of x86_bhb_clear()
 * if SPECTREV2 mitigations are disabled, the CPU is too old, the CPU is new
 * enough to fix (which includes non-Intel CPUs), or the CPU has an explicit
 * disable-Branch-History control.
 */
static x86_native_bhi_mitigation_t
cpuid_learn_and_patch_bhi(x86_spectrev2_mitigation_t v2mit, cpu_t *cpu,
    uchar_t *featureset)
{
	struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
	const uint8_t ret = RET_INSTR;
	uint8_t *bhb_clear = (uint8_t *)x86_bhb_clear;

	ASSERT0(cpu->cpu_id);

	/* First check for explicitly disabled... */
	if (v2mit == X86_SPECTREV2_DISABLED) {
		*bhb_clear = ret;
		return (X86_BHI_TOO_OLD_OR_DISABLED);
	}

	/*
	 * Then check for BHI_NO, which means the CPU doesn't have this bug,
	 * or if it's non-Intel, in which case this mitigation mechanism
	 * doesn't apply.
	 */
	if (cpi->cpi_vendor != X86_VENDOR_Intel ||
	    is_x86_feature(featureset, X86FSET_BHI_NO)) {
		*bhb_clear = ret;
		return (X86_BHI_NEW_ENOUGH);
	}

	/*
	 * Now check for the BHI_CTRL MSR, and then set it if available.
	 * We will still need to use the software sequence, however.
	 */
	if (is_x86_feature(featureset, X86FSET_BHI_CTRL)) {
		cpuid_enable_bhi_dis_s();
		return (X86_BHI_DIS_S);
	}

	/*
	 * Finally, check if we are too old to bother with RSB:
	 */
	if (v2mit == X86_SPECTREV2_RETPOLINE) {
		*bhb_clear = ret;
		return (X86_BHI_TOO_OLD_OR_DISABLED);
	}

	ASSERT(*bhb_clear != ret);
	return (X86_BHI_SOFTWARE_SEQUENCE);
}

/*
 * We default to enabling Return Stack Buffer (RSB) mitigations.
 *
 * We used to skip RSB mitigations with Intel eIBRS, but developments around
 * post-barrier RSB (PBRSB) guessing suggests we should enable Intel RSB
 * mitigations always unless explicitly bypassed, or unless hardware indicates
 * the bug has been fixed.
 *
 * The current decisions for using, or ignoring, a RSB software stuffing
 * sequence are expressed by the following table:
 *
 * +-------+------------+-----------------+--------+
 * | eIBRS |  PBRSB_NO  |  context switch | vmexit |
 * +-------+------------+-----------------+--------+
 * |   Yes |     No     |  stuff          | stuff  |
 * |   Yes |     Yes    |  ignore         | ignore |
 * |   No  |     No     |  stuff          | ignore |
 * +-------+------------+-----------------+--------+
 *
 * Note that if an Intel CPU has no eIBRS, it will never enumerate PBRSB_NO,
 * because machines with no eIBRS do not have a problem with PBRSB overflow.
 * See the Intel document cited below for details.
 *
 * Also note that AMD AUTO_IBRS has no PBRSB problem, so it is not included in
 * the table above, and that there is no situation where vmexit stuffing is
 * needed, but context-switch stuffing isn't.
 */

/* BEGIN CSTYLED */
/*
 * https://www.intel.com/content/www/us/en/developer/articles/technical/software-security-guidance/advisory-guidance/post-barrier-return-stack-buffer-predictions.html
 */
/* END CSTYLED */

/*
 * AMD indicates that when Automatic IBRS is enabled we do not need to implement
 * return stack buffer clearing for VMEXIT as it takes care of it. The manual
 * also states that as long as SMEP and we maintain at least one page between
 * the kernel and user space (we have much more of a red zone), then we do not
 * need to clear the RSB. We constrain this to only when Automatic IRBS is
 * present.
 */
static void
cpuid_patch_rsb(x86_spectrev2_mitigation_t mit, bool intel_pbrsb_no)
{
	const uint8_t ret = RET_INSTR;
	uint8_t *stuff = (uint8_t *)x86_rsb_stuff;
	uint8_t *vmx_stuff = (uint8_t *)x86_rsb_stuff_vmexit;

	switch (mit) {
	case X86_SPECTREV2_AUTO_IBRS:
	case X86_SPECTREV2_DISABLED:
		/* Don't bother with any RSB stuffing! */
		*stuff = ret;
		*vmx_stuff = ret;
		break;
	case X86_SPECTREV2_RETPOLINE:
		/*
		 * The Intel document on Post-Barrier RSB says that processors
		 * without eIBRS do not have PBRSB problems upon VMEXIT.
		 */
		VERIFY(!intel_pbrsb_no);
		VERIFY3U(*stuff, !=, ret);
		*vmx_stuff = ret;
		break;
	default:
		/*
		 * eIBRS is all that's left.  If CPU claims PBRSB is fixed,
		 * don't use the RSB mitigation in either case.  Otherwise
		 * both vmexit and context-switching require the software
		 * mitigation.
		 */
		if (intel_pbrsb_no) {
			/* CPU claims PBRSB problems are fixed. */
			*stuff = ret;
			*vmx_stuff = ret;
		}
		VERIFY3U(*stuff, ==, *vmx_stuff);
		break;
	}
}

static void
cpuid_patch_retpolines(x86_spectrev2_mitigation_t mit)
{
	const char *thunks[] = { "_rax", "_rbx", "_rcx", "_rdx", "_rdi",
	    "_rsi", "_rbp", "_r8", "_r9", "_r10", "_r11", "_r12", "_r13",
	    "_r14", "_r15" };
	const uint_t nthunks = ARRAY_SIZE(thunks);
	const char *type;
	uint_t i;

	if (mit == x86_spectrev2_mitigation)
		return;

	switch (mit) {
	case X86_SPECTREV2_RETPOLINE:
		type = "gen";
		break;
	case X86_SPECTREV2_AUTO_IBRS:
	case X86_SPECTREV2_ENHANCED_IBRS:
	case X86_SPECTREV2_DISABLED:
		type = "jmp";
		break;
	default:
		panic("asked to update retpoline state with unknown state!");
	}

	for (i = 0; i < nthunks; i++) {
		uintptr_t source, dest;
		int ssize, dsize;
		char sourcebuf[64], destbuf[64];

		(void) snprintf(destbuf, sizeof (destbuf),
		    "__x86_indirect_thunk%s", thunks[i]);
		(void) snprintf(sourcebuf, sizeof (sourcebuf),
		    "__x86_indirect_thunk_%s%s", type, thunks[i]);

		source = kobj_getelfsym(sourcebuf, NULL, &ssize);
		dest = kobj_getelfsym(destbuf, NULL, &dsize);
		VERIFY3U(source, !=, 0);
		VERIFY3U(dest, !=, 0);
		VERIFY3S(dsize, >=, ssize);
		bcopy((void *)source, (void *)dest, ssize);
	}
}

static void
cpuid_enable_enhanced_ibrs(void)
{
	uint64_t val;

	val = rdmsr(MSR_IA32_SPEC_CTRL);
	val |= IA32_SPEC_CTRL_IBRS;
	wrmsr(MSR_IA32_SPEC_CTRL, val);
}

static void
cpuid_enable_auto_ibrs(void)
{
	uint64_t val;

	val = rdmsr(MSR_AMD_EFER);
	val |= AMD_EFER_AIBRSE;
	wrmsr(MSR_AMD_EFER, val);
}

/*
 * AMD Zen 5 processors are affected by a defect where the 16- and 32-bit
 * forms of the RDSEED instruction may return 0 despite indicating success
 * (CF=1) - See AMD-SB-7055 / CVE-2025-62626.
 *
 * This table records the minimum microcode revision for each affected CPU
 * at which RDSEED is considered reliable and may be exposed. On all other
 * Zen 5 parts, or when running below the listed revision, RDSEED is masked
 * from CPUID leaf 7 feature reporting.
 *
 * The model field is required to distinguish between Krackan and Krackan2,
 * which otherwise share the same chip revision identifier.
 */
static struct cpuid_fwrev {
	const x86_chiprev_t	cf_chiprev;
	const uint_t		cf_model;
	const uint32_t		cf_minfwrev;
} cpuid_amd_zen5_rdseed_good[] = {
	{ X86_CHIPREV_AMD_TURIN_C1,		0x02,	0x0b00215a },
	{ X86_CHIPREV_AMD_DENSE_TURIN_B0,	0x11,	0x0b101054 },
	{ X86_CHIPREV_AMD_STRIX_B0,		0x24,	0x0b204037 },
	{ X86_CHIPREV_AMD_GRANITE_RIDGE_B0,	0x44,	0x0b404035 },
	{ X86_CHIPREV_AMD_GRANITE_RIDGE_B1,	0x44,	0x0b404108 },
	{ X86_CHIPREV_AMD_KRACKAN_A0,		0x60,	0x0b600037 },
	{ X86_CHIPREV_AMD_KRACKAN_A0,		0x68,	0x0b608038 },
	{ X86_CHIPREV_AMD_STRIX_HALO_A0,	0x70,	0x0b700037 },
	{ X86_CHIPREV_AMD_SHIMADA_PEAK_C1,	0x08,	0x0b008121 },
};

static void
cpuid_evaluate_amd_rdseed(cpu_t *cpu, uchar_t *featureset)
{
	struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
	struct cpuid_regs *ecp = &cpi->cpi_std[7];
	uint32_t rev = cpu->cpu_m.mcpu_ucode_info->cui_rev;

	ASSERT3U(cpi->cpi_vendor, ==, X86_VENDOR_AMD);
	ASSERT(ecp->cp_ebx & CPUID_INTC_EBX_7_0_RDSEED);

	/* This erratum only applies to the Zen 5 uarch */
	if (uarchrev_uarch(cpi->cpi_uarchrev) != X86_UARCH_AMD_ZEN5)
		return;

	/*
	 * If the CPU microcode is new enough then this issue is mitigated.
	 * Unfortunately there is not a bit that indicates this so we need to
	 * check the version explicitly against a table of known good versions.
	 */
	for (size_t i = 0; i < ARRAY_SIZE(cpuid_amd_zen5_rdseed_good); i++) {
		const struct cpuid_fwrev *cf = &cpuid_amd_zen5_rdseed_good[i];

		if (chiprev_matches(cpi->cpi_chiprev, cf->cf_chiprev) &&
		    cpi->cpi_model == cf->cf_model && rev >= cf->cf_minfwrev) {
			/* Mitigated, leave enabled. */
			return;
		}
	}

	/*
	 * Go ahead and disable RDSEED on this boot.
	 * In addition to removing it from the feature set and cached value, we
	 * also need to remove it from the features returned by CPUID7 so that
	 * userland programs performing their own feature detection will
	 * determine it is not available.
	 */
	if (cpu->cpu_id == 0)
		cmn_err(CE_WARN, "Masking unreliable RDSEED on this hardware");

	remove_x86_feature(featureset, X86FSET_RDSEED);
	ecp->cp_ebx &= ~CPUID_INTC_EBX_7_0_RDSEED;

	/*
	 * Some hypervisors that expose RDSEED do not emulate this MSR and so
	 * we guard against a trap here.
	 */
#ifndef __xpv
	on_trap_data_t otd;

	if (!on_trap(&otd, OT_DATA_ACCESS)) {
		uint64_t val;

		val = rdmsr(MSR_AMD_CPUID7_FEATURES);
		val &= ~MSR_AMD_CPUID7_FEATURES_RDSEED;
		wrmsr(MSR_AMD_CPUID7_FEATURES, val);
	}
	no_trap();
#endif
}

/*
 * Determine how we should mitigate TAA or if we need to. Regardless of TAA, if
 * we can disable TSX, we do so.
 *
 * This determination is done only on the boot CPU, potentially after loading
 * updated microcode.
 */
static void
cpuid_update_tsx(cpu_t *cpu, uchar_t *featureset)
{
	struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;

	VERIFY(cpu->cpu_id == 0);

	if (cpi->cpi_vendor != X86_VENDOR_Intel) {
		x86_taa_mitigation = X86_TAA_HW_MITIGATED;
		return;
	}

	if (x86_disable_taa) {
		x86_taa_mitigation = X86_TAA_DISABLED;
		return;
	}

	/*
	 * If we do not have the ability to disable TSX, then our only
	 * mitigation options are in hardware (TAA_NO), or by using our existing
	 * MDS mitigation as described above.  The latter relies upon us having
	 * configured MDS mitigations correctly! This includes disabling SMT if
	 * we want to cross-CPU-thread protection.
	 */
	if (!is_x86_feature(featureset, X86FSET_TSX_CTRL)) {
		/*
		 * It's not clear whether any parts will enumerate TAA_NO
		 * *without* TSX_CTRL, but let's mark it as such if we see this.
		 */
		if (is_x86_feature(featureset, X86FSET_TAA_NO)) {
			x86_taa_mitigation = X86_TAA_HW_MITIGATED;
			return;
		}

		if (is_x86_feature(featureset, X86FSET_MD_CLEAR) &&
		    !is_x86_feature(featureset, X86FSET_MDS_NO)) {
			x86_taa_mitigation = X86_TAA_MD_CLEAR;
		} else {
			x86_taa_mitigation = X86_TAA_NOTHING;
		}
		return;
	}

	/*
	 * We have TSX_CTRL, but we can only fully disable TSX if we're early
	 * enough in boot.
	 *
	 * Otherwise, we'll fall back to causing transactions to abort as our
	 * mitigation. TSX-using code will always take the fallback path.
	 */
	if (cpi->cpi_pass < 4) {
		x86_taa_mitigation = X86_TAA_TSX_DISABLE;
	} else {
		x86_taa_mitigation = X86_TAA_TSX_FORCE_ABORT;
	}
}

/*
 * As mentioned, we should only touch the MSR when we've got a suitable
 * microcode loaded on this CPU.
 */
static void
cpuid_apply_tsx(x86_taa_mitigation_t taa, uchar_t *featureset)
{
	uint64_t val;

	switch (taa) {
	case X86_TAA_TSX_DISABLE:
		if (!is_x86_feature(featureset, X86FSET_TSX_CTRL))
			return;
		val = rdmsr(MSR_IA32_TSX_CTRL);
		val |= IA32_TSX_CTRL_CPUID_CLEAR | IA32_TSX_CTRL_RTM_DISABLE;
		wrmsr(MSR_IA32_TSX_CTRL, val);
		break;
	case X86_TAA_TSX_FORCE_ABORT:
		if (!is_x86_feature(featureset, X86FSET_TSX_CTRL))
			return;
		val = rdmsr(MSR_IA32_TSX_CTRL);
		val |= IA32_TSX_CTRL_RTM_DISABLE;
		wrmsr(MSR_IA32_TSX_CTRL, val);
		break;
	case X86_TAA_HW_MITIGATED:
	case X86_TAA_MD_CLEAR:
	case X86_TAA_DISABLED:
	case X86_TAA_NOTHING:
		break;
	}
}

static void
cpuid_scan_security(cpu_t *cpu, uchar_t *featureset)
{
	struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
	x86_spectrev2_mitigation_t v2mit;

	if ((cpi->cpi_vendor == X86_VENDOR_AMD ||
	    cpi->cpi_vendor == X86_VENDOR_HYGON) &&
	    cpi->cpi_xmaxeax >= CPUID_LEAF_EXT_8) {
		if (cpi->cpi_extd[8].cp_ebx & CPUID_AMD_EBX_IBPB)
			add_x86_feature(featureset, X86FSET_IBPB);
		if (cpi->cpi_extd[8].cp_ebx & CPUID_AMD_EBX_IBRS)
			add_x86_feature(featureset, X86FSET_IBRS);
		if (cpi->cpi_extd[8].cp_ebx & CPUID_AMD_EBX_STIBP)
			add_x86_feature(featureset, X86FSET_STIBP);
		if (cpi->cpi_extd[8].cp_ebx & CPUID_AMD_EBX_STIBP_ALL)
			add_x86_feature(featureset, X86FSET_STIBP_ALL);
		if (cpi->cpi_extd[8].cp_ebx & CPUID_AMD_EBX_SSBD)
			add_x86_feature(featureset, X86FSET_SSBD);
		if (cpi->cpi_extd[8].cp_ebx & CPUID_AMD_EBX_VIRT_SSBD)
			add_x86_feature(featureset, X86FSET_SSBD_VIRT);
		if (cpi->cpi_extd[8].cp_ebx & CPUID_AMD_EBX_SSB_NO)
			add_x86_feature(featureset, X86FSET_SSB_NO);

		/*
		 * Rather than Enhanced IBRS, AMD has a different feature that
		 * is a bit in EFER that can be enabled and will basically do
		 * the right thing while executing in the kernel.
		 */
		if (cpi->cpi_vendor == X86_VENDOR_AMD &&
		    (cpi->cpi_extd[8].cp_ebx & CPUID_AMD_EBX_PREFER_IBRS) &&
		    cpi->cpi_xmaxeax >= CPUID_LEAF_EXT_21 &&
		    (cpi->cpi_extd[0x21].cp_eax & CPUID_AMD_8X21_EAX_AIBRS)) {
			add_x86_feature(featureset, X86FSET_AUTO_IBRS);
		}

	} else if (cpi->cpi_vendor == X86_VENDOR_Intel &&
	    cpi->cpi_maxeax >= 7) {
		struct cpuid_regs *ecp;
		ecp = &cpi->cpi_std[7];

		if (ecp->cp_edx & CPUID_INTC_EDX_7_0_MD_CLEAR) {
			add_x86_feature(featureset, X86FSET_MD_CLEAR);
		}

		if (ecp->cp_edx & CPUID_INTC_EDX_7_0_SPEC_CTRL) {
			add_x86_feature(featureset, X86FSET_IBRS);
			add_x86_feature(featureset, X86FSET_IBPB);
		}

		if (ecp->cp_edx & CPUID_INTC_EDX_7_0_STIBP) {
			add_x86_feature(featureset, X86FSET_STIBP);
		}

		/*
		 * Some prediction controls are enumerated by subleaf 2 of
		 * leaf 7.
		 */
		if (CPI_FEATURES_7_2_EDX(cpi) & CPUID_INTC_EDX_7_2_BHI_CTRL) {
			add_x86_feature(featureset, X86FSET_BHI_CTRL);
		}

		/*
		 * Don't read the arch caps MSR on xpv where we lack the
		 * on_trap().
		 */
#ifndef __xpv
		if (ecp->cp_edx & CPUID_INTC_EDX_7_0_ARCH_CAPS) {
			on_trap_data_t otd;

			/*
			 * Be paranoid and assume we'll get a #GP.
			 */
			if (!on_trap(&otd, OT_DATA_ACCESS)) {
				uint64_t reg;

				reg = rdmsr(MSR_IA32_ARCH_CAPABILITIES);
				if (reg & IA32_ARCH_CAP_RDCL_NO) {
					add_x86_feature(featureset,
					    X86FSET_RDCL_NO);
				}
				if (reg & IA32_ARCH_CAP_IBRS_ALL) {
					add_x86_feature(featureset,
					    X86FSET_IBRS_ALL);
				}
				if (reg & IA32_ARCH_CAP_RSBA) {
					add_x86_feature(featureset,
					    X86FSET_RSBA);
				}
				if (reg & IA32_ARCH_CAP_SKIP_L1DFL_VMENTRY) {
					add_x86_feature(featureset,
					    X86FSET_L1D_VM_NO);
				}
				if (reg & IA32_ARCH_CAP_SSB_NO) {
					add_x86_feature(featureset,
					    X86FSET_SSB_NO);
				}
				if (reg & IA32_ARCH_CAP_MDS_NO) {
					add_x86_feature(featureset,
					    X86FSET_MDS_NO);
				}
				if (reg & IA32_ARCH_CAP_TSX_CTRL) {
					add_x86_feature(featureset,
					    X86FSET_TSX_CTRL);
				}
				if (reg & IA32_ARCH_CAP_TAA_NO) {
					add_x86_feature(featureset,
					    X86FSET_TAA_NO);
				}
				if (reg & IA32_ARCH_CAP_RFDS_NO) {
					add_x86_feature(featureset,
					    X86FSET_RFDS_NO);
				}
				if (reg & IA32_ARCH_CAP_RFDS_CLEAR) {
					add_x86_feature(featureset,
					    X86FSET_RFDS_CLEAR);
				}
				if (reg & IA32_ARCH_CAP_PBRSB_NO) {
					add_x86_feature(featureset,
					    X86FSET_PBRSB_NO);
				}
				if (reg & IA32_ARCH_CAP_BHI_NO) {
					add_x86_feature(featureset,
					    X86FSET_BHI_NO);
				}
			}
			no_trap();
		}
#endif	/* !__xpv */

		if (ecp->cp_edx & CPUID_INTC_EDX_7_0_SSBD)
			add_x86_feature(featureset, X86FSET_SSBD);

		if (ecp->cp_edx & CPUID_INTC_EDX_7_0_FLUSH_CMD)
			add_x86_feature(featureset, X86FSET_FLUSH_CMD);
	}

	/*
	 * Take care of certain mitigations on the non-boot CPU. The boot CPU
	 * will have already run this function and determined what we need to
	 * do. This gives us a hook for per-HW thread mitigations such as
	 * enhanced IBRS, or disabling TSX.
	 */
	if (cpu->cpu_id != 0) {
		switch (x86_spectrev2_mitigation) {
		case X86_SPECTREV2_ENHANCED_IBRS:
			cpuid_enable_enhanced_ibrs();
			break;
		case X86_SPECTREV2_AUTO_IBRS:
			cpuid_enable_auto_ibrs();
			break;
		default:
			break;
		}

		/* If we're committed to BHI_DIS_S, set it for this core. */
		if (x86_bhi_mitigation == X86_BHI_DIS_S)
			cpuid_enable_bhi_dis_s();

		cpuid_apply_tsx(x86_taa_mitigation, featureset);
		return;
	}

	/*
	 * Go through and initialize various security mechanisms that we should
	 * only do on a single CPU. This includes Spectre V2, L1TF, MDS, and
	 * TAA.
	 */

	/*
	 * By default we've come in with retpolines enabled. Check whether we
	 * should disable them or enable enhanced or automatic IBRS.
	 *
	 * Note, we do not allow the use of AMD optimized retpolines as it was
	 * disclosed by AMD in March 2022 that they were still
	 * vulnerable. Prior to that point, we used them.
	 */
	if (x86_disable_spectrev2 != 0) {
		v2mit = X86_SPECTREV2_DISABLED;
	} else if (is_x86_feature(featureset, X86FSET_AUTO_IBRS)) {
		cpuid_enable_auto_ibrs();
		v2mit = X86_SPECTREV2_AUTO_IBRS;
	} else if (is_x86_feature(featureset, X86FSET_IBRS_ALL)) {
		cpuid_enable_enhanced_ibrs();
		v2mit = X86_SPECTREV2_ENHANCED_IBRS;
	} else {
		v2mit = X86_SPECTREV2_RETPOLINE;
	}

	cpuid_patch_retpolines(v2mit);
	cpuid_patch_rsb(v2mit, is_x86_feature(featureset, X86FSET_PBRSB_NO));
	x86_bhi_mitigation = cpuid_learn_and_patch_bhi(v2mit, cpu, featureset);
	x86_spectrev2_mitigation = v2mit;
	membar_producer();

	/*
	 * We need to determine what changes are required for mitigating L1TF
	 * and MDS. If the CPU suffers from either of them, then SMT exclusion
	 * is required.
	 *
	 * If any of these are present, then we need to flush u-arch state at
	 * various points. For MDS, we need to do so whenever we change to a
	 * lesser privilege level or we are halting the CPU. For L1TF we need to
	 * flush the L1D cache at VM entry. When we have microcode that handles
	 * MDS, the L1D flush also clears the other u-arch state that the
	 * md_clear does.
	 */

	/*
	 * Update whether or not we need to be taking explicit action against
	 * MDS or RFDS.
	 */
	cpuid_update_md_clear(cpu, featureset);

	/*
	 * Determine whether SMT exclusion is required and whether or not we
	 * need to perform an l1d flush.
	 */
	cpuid_update_l1d_flush(cpu, featureset);

	/*
	 * Determine what our mitigation strategy should be for TAA and then
	 * also apply TAA mitigations.
	 */
	cpuid_update_tsx(cpu, featureset);
	cpuid_apply_tsx(x86_taa_mitigation, featureset);
}

/*
 * Setup XFeature_Enabled_Mask register. Required by xsave feature.
 */
void
setup_xfem(void)
{
	uint64_t flags = XFEATURE_LEGACY_FP;

	ASSERT(is_x86_feature(x86_featureset, X86FSET_XSAVE));

	if (is_x86_feature(x86_featureset, X86FSET_SSE))
		flags |= XFEATURE_SSE;

	if (is_x86_feature(x86_featureset, X86FSET_AVX))
		flags |= XFEATURE_AVX;

	if (is_x86_feature(x86_featureset, X86FSET_AVX512F))
		flags |= XFEATURE_AVX512;

	set_xcr(XFEATURE_ENABLED_MASK, flags);

	xsave_bv_all = flags;
}

static void
cpuid_basic_topology(cpu_t *cpu, uchar_t *featureset)
{
	struct cpuid_info *cpi;

	cpi = cpu->cpu_m.mcpu_cpi;

	if (cpi->cpi_vendor == X86_VENDOR_AMD ||
	    cpi->cpi_vendor == X86_VENDOR_HYGON) {
		cpuid_gather_amd_topology_leaves(cpu);
	}

	cpi->cpi_apicid = cpuid_gather_apicid(cpi);

	/*
	 * Before we can calculate the IDs that we should assign to this
	 * processor, we need to understand how many cores and threads it has.
	 */
	switch (cpi->cpi_vendor) {
	case X86_VENDOR_Intel:
		cpuid_intel_ncores(cpi, &cpi->cpi_ncpu_per_chip,
		    &cpi->cpi_ncore_per_chip);
		break;
	case X86_VENDOR_AMD:
	case X86_VENDOR_HYGON:
		cpuid_amd_ncores(cpi, &cpi->cpi_ncpu_per_chip,
		    &cpi->cpi_ncore_per_chip);
		break;
	default:
		/*
		 * If we have some other x86 compatible chip, it's not clear how
		 * they would behave. The most common case is virtualization
		 * today, though there are also 64-bit VIA chips. Assume that
		 * all we can get is the basic Leaf 1 HTT information.
		 */
		if ((cpi->cpi_std[1].cp_edx & CPUID_INTC_EDX_HTT) != 0) {
			cpi->cpi_ncore_per_chip = 1;
			cpi->cpi_ncpu_per_chip = CPI_CPU_COUNT(cpi);
		}
		break;
	}

	/*
	 * Based on the calculated number of threads and cores, potentially
	 * assign the HTT and CMT features.
	 */
	if (cpi->cpi_ncore_per_chip > 1) {
		add_x86_feature(featureset, X86FSET_CMP);
	}

	if (cpi->cpi_ncpu_per_chip > 1 &&
	    cpi->cpi_ncpu_per_chip != cpi->cpi_ncore_per_chip) {
		add_x86_feature(featureset, X86FSET_HTT);
	}

	/*
	 * Now that has been set up, we need to go through and calculate all of
	 * the rest of the parameters that exist. If we think the CPU doesn't
	 * have either SMT (HTT) or CMP, then we basically go through and fake
	 * up information in some way. The most likely case for this is
	 * virtualization where we have a lot of partial topology information.
	 */
	if (!is_x86_feature(featureset, X86FSET_HTT) &&
	    !is_x86_feature(featureset, X86FSET_CMP)) {
		/*
		 * This is a single core, single-threaded processor.
		 */
		cpi->cpi_procnodes_per_pkg = 1;
		cpi->cpi_cores_per_compunit = 1;
		cpi->cpi_compunitid = 0;
		cpi->cpi_chipid = -1;
		cpi->cpi_clogid = 0;
		cpi->cpi_coreid = cpu->cpu_id;
		cpi->cpi_pkgcoreid = 0;
		if (cpi->cpi_vendor == X86_VENDOR_AMD ||
		    cpi->cpi_vendor == X86_VENDOR_HYGON) {
			cpi->cpi_procnodeid = BITX(cpi->cpi_apicid, 3, 0);
		} else {
			cpi->cpi_procnodeid = cpi->cpi_chipid;
		}
	} else {
		switch (cpi->cpi_vendor) {
		case X86_VENDOR_Intel:
			cpuid_intel_getids(cpu, featureset);
			break;
		case X86_VENDOR_AMD:
		case X86_VENDOR_HYGON:
			cpuid_amd_getids(cpu, featureset);
			break;
		default:
			/*
			 * In this case, it's hard to say what we should do.
			 * We're going to model them to the OS as single core
			 * threads. We don't have a good identifier for them, so
			 * we're just going to use the cpu id all on a single
			 * chip.
			 *
			 * This case has historically been different from the
			 * case above where we don't have HTT or CMP. While they
			 * could be combined, we've opted to keep it separate to
			 * minimize the risk of topology changes in weird cases.
			 */
			cpi->cpi_procnodes_per_pkg = 1;
			cpi->cpi_cores_per_compunit = 1;
			cpi->cpi_chipid = 0;
			cpi->cpi_coreid = cpu->cpu_id;
			cpi->cpi_clogid = cpu->cpu_id;
			cpi->cpi_pkgcoreid = cpu->cpu_id;
			cpi->cpi_procnodeid = cpi->cpi_chipid;
			cpi->cpi_compunitid = cpi->cpi_coreid;
			break;
		}
	}
}

/*
 * Gather relevant CPU features from leaf 6 which covers thermal information. We
 * always gather leaf 6 if it's supported; however, we only look for features on
 * Intel systems as AMD does not currently define any of the features we look
 * for below.
 */
static void
cpuid_basic_thermal(cpu_t *cpu, uchar_t *featureset)
{
	struct cpuid_regs *cp;
	struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;

	if (cpi->cpi_maxeax < 6) {
		return;
	}

	cp = &cpi->cpi_std[6];
	cp->cp_eax = 6;
	cp->cp_ebx = cp->cp_ecx = cp->cp_edx = 0;
	(void) __cpuid_insn(cp);
	platform_cpuid_mangle(cpi->cpi_vendor, 6, cp);

	if (cpi->cpi_vendor != X86_VENDOR_Intel) {
		return;
	}

	if ((cp->cp_eax & CPUID_INTC_EAX_DTS) != 0) {
		add_x86_feature(featureset, X86FSET_CORE_THERMAL);
	}

	if ((cp->cp_eax & CPUID_INTC_EAX_PTM) != 0) {
		add_x86_feature(featureset, X86FSET_PKG_THERMAL);
	}
}

/*
 * This is used when we discover that we have AVX support in cpuid. This
 * proceeds to scan for the rest of the AVX derived features.
 */
static void
cpuid_basic_avx(cpu_t *cpu, uchar_t *featureset)
{
	struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;

	/*
	 * If we don't have AVX, don't bother with most of this.
	 */
	if ((cpi->cpi_std[1].cp_ecx & CPUID_INTC_ECX_AVX) == 0)
		return;

	add_x86_feature(featureset, X86FSET_AVX);

	/*
	 * Intel says we can't check these without also
	 * checking AVX.
	 */
	if (cpi->cpi_std[1].cp_ecx & CPUID_INTC_ECX_F16C)
		add_x86_feature(featureset, X86FSET_F16C);

	if (cpi->cpi_std[1].cp_ecx & CPUID_INTC_ECX_FMA)
		add_x86_feature(featureset, X86FSET_FMA);

	if (cpi->cpi_std[7].cp_ebx & CPUID_INTC_EBX_7_0_BMI1)
		add_x86_feature(featureset, X86FSET_BMI1);

	if (cpi->cpi_std[7].cp_ebx & CPUID_INTC_EBX_7_0_BMI2)
		add_x86_feature(featureset, X86FSET_BMI2);

	if (cpi->cpi_std[7].cp_ebx & CPUID_INTC_EBX_7_0_AVX2)
		add_x86_feature(featureset, X86FSET_AVX2);

	if (cpi->cpi_std[7].cp_ecx & CPUID_INTC_ECX_7_0_VAES)
		add_x86_feature(featureset, X86FSET_VAES);

	if (cpi->cpi_std[7].cp_ecx & CPUID_INTC_ECX_7_0_VPCLMULQDQ)
		add_x86_feature(featureset, X86FSET_VPCLMULQDQ);

	/*
	 * The rest of the AVX features require AVX512. Do not check them unless
	 * it is present.
	 */
	if ((cpi->cpi_std[7].cp_ebx & CPUID_INTC_EBX_7_0_AVX512F) == 0)
		return;
	add_x86_feature(featureset, X86FSET_AVX512F);

	if (cpi->cpi_std[7].cp_ebx & CPUID_INTC_EBX_7_0_AVX512DQ)
		add_x86_feature(featureset, X86FSET_AVX512DQ);

	if (cpi->cpi_std[7].cp_ebx & CPUID_INTC_EBX_7_0_AVX512IFMA)
		add_x86_feature(featureset, X86FSET_AVX512FMA);

	if (cpi->cpi_std[7].cp_ebx & CPUID_INTC_EBX_7_0_AVX512PF)
		add_x86_feature(featureset, X86FSET_AVX512PF);

	if (cpi->cpi_std[7].cp_ebx & CPUID_INTC_EBX_7_0_AVX512ER)
		add_x86_feature(featureset, X86FSET_AVX512ER);

	if (cpi->cpi_std[7].cp_ebx & CPUID_INTC_EBX_7_0_AVX512CD)
		add_x86_feature(featureset, X86FSET_AVX512CD);

	if (cpi->cpi_std[7].cp_ebx & CPUID_INTC_EBX_7_0_AVX512BW)
		add_x86_feature(featureset, X86FSET_AVX512BW);

	if (cpi->cpi_std[7].cp_ebx & CPUID_INTC_EBX_7_0_AVX512VL)
		add_x86_feature(featureset, X86FSET_AVX512VL);

	if (cpi->cpi_std[7].cp_ecx & CPUID_INTC_ECX_7_0_AVX512VBMI)
		add_x86_feature(featureset, X86FSET_AVX512VBMI);

	if (cpi->cpi_std[7].cp_ecx & CPUID_INTC_ECX_7_0_AVX512VBMI2)
		add_x86_feature(featureset, X86FSET_AVX512_VBMI2);

	if (cpi->cpi_std[7].cp_ecx & CPUID_INTC_ECX_7_0_AVX512VNNI)
		add_x86_feature(featureset, X86FSET_AVX512VNNI);

	if (cpi->cpi_std[7].cp_ecx & CPUID_INTC_ECX_7_0_AVX512BITALG)
		add_x86_feature(featureset, X86FSET_AVX512_BITALG);

	if (cpi->cpi_std[7].cp_ecx & CPUID_INTC_ECX_7_0_AVX512VPOPCDQ)
		add_x86_feature(featureset, X86FSET_AVX512VPOPCDQ);

	if (cpi->cpi_std[7].cp_edx & CPUID_INTC_EDX_7_0_AVX5124NNIW)
		add_x86_feature(featureset, X86FSET_AVX512NNIW);

	if (cpi->cpi_std[7].cp_edx & CPUID_INTC_EDX_7_0_AVX5124FMAPS)
		add_x86_feature(featureset, X86FSET_AVX512FMAPS);

	/*
	 * More features here are in Leaf 7, subleaf 1. Don't bother checking if
	 * we don't need to.
	 */
	if (cpi->cpi_std[7].cp_eax < 1)
		return;

	if (cpi->cpi_sub7[0].cp_eax & CPUID_INTC_EAX_7_1_AVX512_BF16)
		add_x86_feature(featureset, X86FSET_AVX512_BF16);
}

/*
 * PPIN is the protected processor inventory number. On AMD this is an actual
 * feature bit. However, on Intel systems we need to read the platform
 * information MSR if we're on a specific model.
 */
#if !defined(__xpv)
static void
cpuid_basic_ppin(cpu_t *cpu, uchar_t *featureset)
{
	on_trap_data_t otd;
	struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;

	switch (cpi->cpi_vendor) {
	case X86_VENDOR_AMD:
		/*
		 * This leaf will have already been gathered in the topology
		 * functions.
		 */
		if (cpi->cpi_xmaxeax >= CPUID_LEAF_EXT_8) {
			if (cpi->cpi_extd[8].cp_ebx & CPUID_AMD_EBX_PPIN) {
				add_x86_feature(featureset, X86FSET_PPIN);
			}
		}
		break;
	case X86_VENDOR_Intel:
		if (cpi->cpi_family != 6)
			break;
		switch (cpi->cpi_model) {
		case INTC_MODEL_IVYBRIDGE_XEON:
		case INTC_MODEL_HASWELL_XEON:
		case INTC_MODEL_BROADWELL_XEON:
		case INTC_MODEL_BROADWELL_XEON_D:
		case INTC_MODEL_SKYLAKE_XEON:
		case INTC_MODEL_ICELAKE_XEON:
			if (!on_trap(&otd, OT_DATA_ACCESS)) {
				uint64_t value;

				value = rdmsr(MSR_PLATFORM_INFO);
				if ((value & MSR_PLATFORM_INFO_PPIN) != 0) {
					add_x86_feature(featureset,
					    X86FSET_PPIN);
				}
			}
			no_trap();
			break;
		default:
			break;
		}
		break;
	default:
		break;
	}
}
#endif	/* ! __xpv */

static void
cpuid_pass_prelude(cpu_t *cpu, void *arg)
{
	uchar_t *featureset = (uchar_t *)arg;

	/*
	 * We don't run on any processor that doesn't have cpuid, and could not
	 * possibly have arrived here.
	 */
	add_x86_feature(featureset, X86FSET_CPUID);
}

static void
cpuid_pass_ident(cpu_t *cpu, void *arg __unused)
{
	struct cpuid_info *cpi;
	struct cpuid_regs *cp;

	/*
	 * We require that virtual/native detection be complete and that PCI
	 * config space access has been set up; at present there is no reliable
	 * way to determine the latter.
	 */
#if !defined(__xpv)
	ASSERT3S(platform_type, !=, -1);
#endif	/* !__xpv */

	cpi = cpu->cpu_m.mcpu_cpi;
	ASSERT(cpi != NULL);

	cp = &cpi->cpi_std[0];
	cp->cp_eax = 0;
	cpi->cpi_maxeax = __cpuid_insn(cp);
	{
		uint32_t *iptr = (uint32_t *)cpi->cpi_vendorstr;
		*iptr++ = cp->cp_ebx;
		*iptr++ = cp->cp_edx;
		*iptr++ = cp->cp_ecx;
		*(char *)&cpi->cpi_vendorstr[12] = '\0';
	}

	cpi->cpi_vendor = _cpuid_vendorstr_to_vendorcode(cpi->cpi_vendorstr);
	x86_vendor = cpi->cpi_vendor; /* for compatibility */

	/*
	 * Limit the range in case of weird hardware
	 */
	if (cpi->cpi_maxeax > CPI_MAXEAX_MAX)
		cpi->cpi_maxeax = CPI_MAXEAX_MAX;
	if (cpi->cpi_maxeax < 1)
		return;

	cp = &cpi->cpi_std[1];
	cp->cp_eax = 1;
	(void) __cpuid_insn(cp);

	/*
	 * Extract identifying constants for easy access.
	 */
	cpi->cpi_model = CPI_MODEL(cpi);
	cpi->cpi_family = CPI_FAMILY(cpi);

	if (cpi->cpi_family == 0xf)
		cpi->cpi_family += CPI_FAMILY_XTD(cpi);

	/*
	 * Beware: AMD uses "extended model" iff base *FAMILY* == 0xf.
	 * Intel, and presumably everyone else, uses model == 0xf, as
	 * one would expect (max value means possible overflow).  Sigh.
	 */

	switch (cpi->cpi_vendor) {
	case X86_VENDOR_Intel:
		if (IS_EXTENDED_MODEL_INTEL(cpi))
			cpi->cpi_model += CPI_MODEL_XTD(cpi) << 4;
		break;
	case X86_VENDOR_AMD:
		if (CPI_FAMILY(cpi) == 0xf)
			cpi->cpi_model += CPI_MODEL_XTD(cpi) << 4;
		break;
	case X86_VENDOR_HYGON:
		cpi->cpi_model += CPI_MODEL_XTD(cpi) << 4;
		break;
	default:
		if (cpi->cpi_model == 0xf)
			cpi->cpi_model += CPI_MODEL_XTD(cpi) << 4;
		break;
	}

	cpi->cpi_step = CPI_STEP(cpi);
	cpi->cpi_brandid = CPI_BRANDID(cpi);

	/*
	 * Synthesize chip "revision" and socket type
	 */
	cpi->cpi_chiprev = _cpuid_chiprev(cpi->cpi_vendor, cpi->cpi_family,
	    cpi->cpi_model, cpi->cpi_step);
	cpi->cpi_chiprevstr = _cpuid_chiprevstr(cpi->cpi_vendor,
	    cpi->cpi_family, cpi->cpi_model, cpi->cpi_step);
	cpi->cpi_socket = _cpuid_skt(cpi->cpi_vendor, cpi->cpi_family,
	    cpi->cpi_model, cpi->cpi_step);
	cpi->cpi_uarchrev = _cpuid_uarchrev(cpi->cpi_vendor, cpi->cpi_family,
	    cpi->cpi_model, cpi->cpi_step);
}

static void
cpuid_pass_basic(cpu_t *cpu, void *arg)
{
	uchar_t *featureset = (uchar_t *)arg;
	uint32_t mask_ecx, mask_edx;
	struct cpuid_info *cpi;
	struct cpuid_regs *cp;
	int xcpuid;
#if !defined(__xpv)
	extern int idle_cpu_prefer_mwait;
#endif

	cpi = cpu->cpu_m.mcpu_cpi;
	ASSERT(cpi != NULL);

	if (cpi->cpi_maxeax < 1)
		return;

	/*
	 * This was filled during the identification pass.
	 */
	cp = &cpi->cpi_std[1];

	/*
	 * *default* assumptions:
	 * - believe %edx feature word
	 * - ignore %ecx feature word
	 * - 32-bit virtual and physical addressing
	 */
	mask_edx = 0xffffffff;
	mask_ecx = 0;

	cpi->cpi_pabits = cpi->cpi_vabits = 32;

	switch (cpi->cpi_vendor) {
	case X86_VENDOR_Intel:
		if (cpi->cpi_family == 5)
			x86_type = X86_TYPE_P5;
		else if (IS_LEGACY_P6(cpi)) {
			x86_type = X86_TYPE_P6;
			pentiumpro_bug4046376 = 1;
			/*
			 * Clear the SEP bit when it was set erroneously
			 */
			if (cpi->cpi_model < 3 && cpi->cpi_step < 3)
				cp->cp_edx &= ~CPUID_INTC_EDX_SEP;
		} else if (IS_NEW_F6(cpi) || cpi->cpi_family == 0xf) {
			x86_type = X86_TYPE_P4;
			/*
			 * We don't currently depend on any of the %ecx
			 * features until Prescott, so we'll only check
			 * this from P4 onwards.  We might want to revisit
			 * that idea later.
			 */
			mask_ecx = 0xffffffff;
		} else if (cpi->cpi_family > 0xf)
			mask_ecx = 0xffffffff;
		/*
		 * We don't support MONITOR/MWAIT if leaf 5 is not available
		 * to obtain the monitor linesize.
		 */
		if (cpi->cpi_maxeax < 5)
			mask_ecx &= ~CPUID_INTC_ECX_MON;
		break;
	case X86_VENDOR_IntelClone:
	default:
		break;
	case X86_VENDOR_AMD:
#if defined(OPTERON_ERRATUM_108)
		if (cpi->cpi_family == 0xf && cpi->cpi_model == 0xe) {
			cp->cp_eax = (0xf0f & cp->cp_eax) | 0xc0;
			cpi->cpi_model = 0xc;
		} else
#endif
		if (cpi->cpi_family == 5) {
			/*
			 * AMD K5 and K6
			 *
			 * These CPUs have an incomplete implementation
			 * of MCA/MCE which we mask away.
			 */
			mask_edx &= ~(CPUID_INTC_EDX_MCE | CPUID_INTC_EDX_MCA);

			/*
			 * Model 0 uses the wrong (APIC) bit
			 * to indicate PGE.  Fix it here.
			 */
			if (cpi->cpi_model == 0) {
				if (cp->cp_edx & 0x200) {
					cp->cp_edx &= ~0x200;
					cp->cp_edx |= CPUID_INTC_EDX_PGE;
				}
			}

			/*
			 * Early models had problems w/ MMX; disable.
			 */
			if (cpi->cpi_model < 6)
				mask_edx &= ~CPUID_INTC_EDX_MMX;
		}

		/*
		 * For newer families, SSE3 and CX16, at least, are valid;
		 * enable all
		 */
		if (cpi->cpi_family >= 0xf)
			mask_ecx = 0xffffffff;
		/*
		 * We don't support MONITOR/MWAIT if leaf 5 is not available
		 * to obtain the monitor linesize.
		 */
		if (cpi->cpi_maxeax < 5)
			mask_ecx &= ~CPUID_INTC_ECX_MON;

#if !defined(__xpv)
		/*
		 * AMD has not historically used MWAIT in the CPU's idle loop.
		 * Pre-family-10h Opterons do not have the MWAIT instruction. We
		 * know for certain that in at least family 17h, per AMD, mwait
		 * is preferred. Families in-between are less certain.
		 */
		if (cpi->cpi_family < 0x17) {
			idle_cpu_prefer_mwait = 0;
		}
#endif

		break;
	case X86_VENDOR_HYGON:
		/* Enable all for Hygon Dhyana CPU */
		mask_ecx = 0xffffffff;
		break;
	case X86_VENDOR_TM:
		/*
		 * workaround the NT workaround in CMS 4.1
		 */
		if (cpi->cpi_family == 5 && cpi->cpi_model == 4 &&
		    (cpi->cpi_step == 2 || cpi->cpi_step == 3))
			cp->cp_edx |= CPUID_INTC_EDX_CX8;
		break;
	case X86_VENDOR_Centaur:
		/*
		 * workaround the NT workarounds again
		 */
		if (cpi->cpi_family == 6)
			cp->cp_edx |= CPUID_INTC_EDX_CX8;
		break;
	case X86_VENDOR_Cyrix:
		/*
		 * We rely heavily on the probing in locore
		 * to actually figure out what parts, if any,
		 * of the Cyrix cpuid instruction to believe.
		 */
		switch (x86_type) {
		case X86_TYPE_CYRIX_486:
			mask_edx = 0;
			break;
		case X86_TYPE_CYRIX_6x86:
			mask_edx = 0;
			break;
		case X86_TYPE_CYRIX_6x86L:
			mask_edx =
			    CPUID_INTC_EDX_DE |
			    CPUID_INTC_EDX_CX8;
			break;
		case X86_TYPE_CYRIX_6x86MX:
			mask_edx =
			    CPUID_INTC_EDX_DE |
			    CPUID_INTC_EDX_MSR |
			    CPUID_INTC_EDX_CX8 |
			    CPUID_INTC_EDX_PGE |
			    CPUID_INTC_EDX_CMOV |
			    CPUID_INTC_EDX_MMX;
			break;
		case X86_TYPE_CYRIX_GXm:
			mask_edx =
			    CPUID_INTC_EDX_MSR |
			    CPUID_INTC_EDX_CX8 |
			    CPUID_INTC_EDX_CMOV |
			    CPUID_INTC_EDX_MMX;
			break;
		case X86_TYPE_CYRIX_MediaGX:
			break;
		case X86_TYPE_CYRIX_MII:
		case X86_TYPE_VIA_CYRIX_III:
			mask_edx =
			    CPUID_INTC_EDX_DE |
			    CPUID_INTC_EDX_TSC |
			    CPUID_INTC_EDX_MSR |
			    CPUID_INTC_EDX_CX8 |
			    CPUID_INTC_EDX_PGE |
			    CPUID_INTC_EDX_CMOV |
			    CPUID_INTC_EDX_MMX;
			break;
		default:
			break;
		}
		break;
	}

#if defined(__xpv)
	/*
	 * Do not support MONITOR/MWAIT under a hypervisor
	 */
	mask_ecx &= ~CPUID_INTC_ECX_MON;
	/*
	 * Do not support XSAVE under a hypervisor for now
	 */
	xsave_force_disable = B_TRUE;

#endif	/* __xpv */

	if (xsave_force_disable) {
		mask_ecx &= ~CPUID_INTC_ECX_XSAVE;
		mask_ecx &= ~CPUID_INTC_ECX_AVX;
		mask_ecx &= ~CPUID_INTC_ECX_F16C;
		mask_ecx &= ~CPUID_INTC_ECX_FMA;
	}

	/*
	 * Now we've figured out the masks that determine
	 * which bits we choose to believe, apply the masks
	 * to the feature words, then map the kernel's view
	 * of these feature words into its feature word.
	 */
	cp->cp_edx &= mask_edx;
	cp->cp_ecx &= mask_ecx;

	/*
	 * apply any platform restrictions (we don't call this
	 * immediately after __cpuid_insn here, because we need the
	 * workarounds applied above first)
	 */
	platform_cpuid_mangle(cpi->cpi_vendor, 1, cp);

	/*
	 * In addition to ecx and edx, Intel and AMD are storing a bunch of
	 * instruction set extensions in leaf 7's ebx, ecx, and edx. Note, leaf
	 * 7 has sub-leaves determined by ecx.
	 */
	if (cpi->cpi_maxeax >= 7) {
		struct cpuid_regs *ecp;
		ecp = &cpi->cpi_std[7];
		ecp->cp_eax = 7;
		ecp->cp_ecx = 0;
		(void) __cpuid_insn(ecp);

		/*
		 * If XSAVE has been disabled, just ignore all of the
		 * extended-save-area dependent flags here. By removing most of
		 * the leaf 7, sub-leaf 0 flags, that will ensure that we don't
		 * end up looking at additional xsave dependent leaves right
		 * now.
		 */
		if (xsave_force_disable) {
			ecp->cp_ebx &= ~CPUID_INTC_EBX_7_0_BMI1;
			ecp->cp_ebx &= ~CPUID_INTC_EBX_7_0_BMI2;
			ecp->cp_ebx &= ~CPUID_INTC_EBX_7_0_AVX2;
			ecp->cp_ebx &= ~CPUID_INTC_EBX_7_0_MPX;
			ecp->cp_ebx &= ~CPUID_INTC_EBX_7_0_ALL_AVX512;
			ecp->cp_ecx &= ~CPUID_INTC_ECX_7_0_ALL_AVX512;
			ecp->cp_edx &= ~CPUID_INTC_EDX_7_0_ALL_AVX512;
			ecp->cp_ecx &= ~CPUID_INTC_ECX_7_0_VAES;
			ecp->cp_ecx &= ~CPUID_INTC_ECX_7_0_VPCLMULQDQ;
			ecp->cp_ecx &= ~CPUID_INTC_ECX_7_0_GFNI;
		}

		if (ecp->cp_ebx & CPUID_INTC_EBX_7_0_SMEP)
			add_x86_feature(featureset, X86FSET_SMEP);

		/*
		 * We check disable_smap here in addition to in startup_smap()
		 * to ensure CPUs that aren't the boot CPU don't accidentally
		 * include it in the feature set and thus generate a mismatched
		 * x86 feature set across CPUs.
		 */
		if (ecp->cp_ebx & CPUID_INTC_EBX_7_0_SMAP &&
		    disable_smap == 0)
			add_x86_feature(featureset, X86FSET_SMAP);

		if (ecp->cp_ebx & CPUID_INTC_EBX_7_0_RDSEED) {
			add_x86_feature(featureset, X86FSET_RDSEED);
			if (cpi->cpi_vendor == X86_VENDOR_AMD)
				cpuid_evaluate_amd_rdseed(cpu, featureset);
		}

		if (ecp->cp_ebx & CPUID_INTC_EBX_7_0_ADX)
			add_x86_feature(featureset, X86FSET_ADX);

		if (ecp->cp_ebx & CPUID_INTC_EBX_7_0_FSGSBASE)
			add_x86_feature(featureset, X86FSET_FSGSBASE);

		if (ecp->cp_ebx & CPUID_INTC_EBX_7_0_CLFLUSHOPT)
			add_x86_feature(featureset, X86FSET_CLFLUSHOPT);

		if (ecp->cp_ebx & CPUID_INTC_EBX_7_0_INVPCID)
			add_x86_feature(featureset, X86FSET_INVPCID);

		if (ecp->cp_ecx & CPUID_INTC_ECX_7_0_UMIP)
			add_x86_feature(featureset, X86FSET_UMIP);
		if (ecp->cp_ecx & CPUID_INTC_ECX_7_0_PKU)
			add_x86_feature(featureset, X86FSET_PKU);
		if (ecp->cp_ecx & CPUID_INTC_ECX_7_0_OSPKE)
			add_x86_feature(featureset, X86FSET_OSPKE);
		if (ecp->cp_ecx & CPUID_INTC_ECX_7_0_GFNI)
			add_x86_feature(featureset, X86FSET_GFNI);

		if (ecp->cp_ebx & CPUID_INTC_EBX_7_0_CLWB)
			add_x86_feature(featureset, X86FSET_CLWB);

		if (cpi->cpi_vendor == X86_VENDOR_Intel) {
			if (ecp->cp_ebx & CPUID_INTC_EBX_7_0_MPX)
				add_x86_feature(featureset, X86FSET_MPX);
		}

		/*
		 * If we have subleaf 1 or 2 available, grab and store
		 * that. This is used for more AVX and related features.
		 */
		if (ecp->cp_eax >= 1) {
			struct cpuid_regs *c71;
			c71 = &cpi->cpi_sub7[0];
			c71->cp_eax = 7;
			c71->cp_ecx = 1;
			(void) __cpuid_insn(c71);
		}

		/* Subleaf 2 has certain security indicators in it. */
		if (ecp->cp_eax >= 2) {
			struct cpuid_regs *c72;
			c72 = &cpi->cpi_sub7[1];
			c72->cp_eax = 7;
			c72->cp_ecx = 2;
			(void) __cpuid_insn(c72);
		}
	}

	/*
	 * fold in overrides from the "eeprom" mechanism
	 */
	cp->cp_edx |= cpuid_feature_edx_include;
	cp->cp_edx &= ~cpuid_feature_edx_exclude;

	cp->cp_ecx |= cpuid_feature_ecx_include;
	cp->cp_ecx &= ~cpuid_feature_ecx_exclude;

	if (cp->cp_edx & CPUID_INTC_EDX_PSE) {
		add_x86_feature(featureset, X86FSET_LARGEPAGE);
	}
	if (cp->cp_edx & CPUID_INTC_EDX_TSC) {
		add_x86_feature(featureset, X86FSET_TSC);
	}
	if (cp->cp_edx & CPUID_INTC_EDX_MSR) {
		add_x86_feature(featureset, X86FSET_MSR);
	}
	if (cp->cp_edx & CPUID_INTC_EDX_MTRR) {
		add_x86_feature(featureset, X86FSET_MTRR);
	}
	if (cp->cp_edx & CPUID_INTC_EDX_PGE) {
		add_x86_feature(featureset, X86FSET_PGE);
	}
	if (cp->cp_edx & CPUID_INTC_EDX_CMOV) {
		add_x86_feature(featureset, X86FSET_CMOV);
	}
	if (cp->cp_edx & CPUID_INTC_EDX_MMX) {
		add_x86_feature(featureset, X86FSET_MMX);
	}
	if ((cp->cp_edx & CPUID_INTC_EDX_MCE) != 0 &&
	    (cp->cp_edx & CPUID_INTC_EDX_MCA) != 0) {
		add_x86_feature(featureset, X86FSET_MCA);
	}
	if (cp->cp_edx & CPUID_INTC_EDX_PAE) {
		add_x86_feature(featureset, X86FSET_PAE);
	}
	if (cp->cp_edx & CPUID_INTC_EDX_CX8) {
		add_x86_feature(featureset, X86FSET_CX8);
	}
	if (cp->cp_ecx & CPUID_INTC_ECX_CX16) {
		add_x86_feature(featureset, X86FSET_CX16);
	}
	if (cp->cp_edx & CPUID_INTC_EDX_PAT) {
		add_x86_feature(featureset, X86FSET_PAT);
	}
	if (cp->cp_edx & CPUID_INTC_EDX_SEP) {
		add_x86_feature(featureset, X86FSET_SEP);
	}
	if (cp->cp_edx & CPUID_INTC_EDX_FXSR) {
		/*
		 * In our implementation, fxsave/fxrstor
		 * are prerequisites before we'll even
		 * try and do SSE things.
		 */
		if (cp->cp_edx & CPUID_INTC_EDX_SSE) {
			add_x86_feature(featureset, X86FSET_SSE);
		}
		if (cp->cp_edx & CPUID_INTC_EDX_SSE2) {
			add_x86_feature(featureset, X86FSET_SSE2);
		}
		if (cp->cp_ecx & CPUID_INTC_ECX_SSE3) {
			add_x86_feature(featureset, X86FSET_SSE3);
		}
		if (cp->cp_ecx & CPUID_INTC_ECX_SSSE3) {
			add_x86_feature(featureset, X86FSET_SSSE3);
		}
		if (cp->cp_ecx & CPUID_INTC_ECX_SSE4_1) {
			add_x86_feature(featureset, X86FSET_SSE4_1);
		}
		if (cp->cp_ecx & CPUID_INTC_ECX_SSE4_2) {
			add_x86_feature(featureset, X86FSET_SSE4_2);
		}
		if (cp->cp_ecx & CPUID_INTC_ECX_AES) {
			add_x86_feature(featureset, X86FSET_AES);
		}
		if (cp->cp_ecx & CPUID_INTC_ECX_PCLMULQDQ) {
			add_x86_feature(featureset, X86FSET_PCLMULQDQ);
		}

		if (cpi->cpi_std[7].cp_ebx & CPUID_INTC_EBX_7_0_SHA)
			add_x86_feature(featureset, X86FSET_SHA);

		if (cp->cp_ecx & CPUID_INTC_ECX_XSAVE) {
			add_x86_feature(featureset, X86FSET_XSAVE);

			/* We only test AVX & AVX512 when there is XSAVE */
			cpuid_basic_avx(cpu, featureset);
		}
	}

	if (cp->cp_ecx & CPUID_INTC_ECX_PCID) {
		add_x86_feature(featureset, X86FSET_PCID);
	}

	if (cp->cp_ecx & CPUID_INTC_ECX_X2APIC) {
		add_x86_feature(featureset, X86FSET_X2APIC);
	}
	if (cp->cp_edx & CPUID_INTC_EDX_DE) {
		add_x86_feature(featureset, X86FSET_DE);
	}
#if !defined(__xpv)
	if (cp->cp_ecx & CPUID_INTC_ECX_MON) {

		/*
		 * We require the CLFLUSH instruction for erratum workaround
		 * to use MONITOR/MWAIT.
		 */
		if (cp->cp_edx & CPUID_INTC_EDX_CLFSH) {
			cpi->cpi_mwait.support |= MWAIT_SUPPORT;
			add_x86_feature(featureset, X86FSET_MWAIT);
		} else {
			extern int idle_cpu_assert_cflush_monitor;

			/*
			 * All processors we are aware of which have
			 * MONITOR/MWAIT also have CLFLUSH.
			 */
			if (idle_cpu_assert_cflush_monitor) {
				ASSERT((cp->cp_ecx & CPUID_INTC_ECX_MON) &&
				    (cp->cp_edx & CPUID_INTC_EDX_CLFSH));
			}
		}
	}
#endif	/* __xpv */

	if (cp->cp_ecx & CPUID_INTC_ECX_VMX) {
		add_x86_feature(featureset, X86FSET_VMX);
	}

	if (cp->cp_ecx & CPUID_INTC_ECX_RDRAND)
		add_x86_feature(featureset, X86FSET_RDRAND);

	/*
	 * Only need it first time, rest of the cpus would follow suit.
	 * we only capture this for the bootcpu.
	 */
	if (cp->cp_edx & CPUID_INTC_EDX_CLFSH) {
		add_x86_feature(featureset, X86FSET_CLFSH);
		x86_clflush_size = (BITX(cp->cp_ebx, 15, 8) * 8);
	}
	if (is_x86_feature(featureset, X86FSET_PAE))
		cpi->cpi_pabits = 36;

	if (cpi->cpi_maxeax >= 0xD && !xsave_force_disable) {
		struct cpuid_regs r, *ecp;

		ecp = &r;
		ecp->cp_eax = 0xD;
		ecp->cp_ecx = 1;
		ecp->cp_edx = ecp->cp_ebx = 0;
		(void) __cpuid_insn(ecp);

		if (ecp->cp_eax & CPUID_INTC_EAX_D_1_XSAVEOPT)
			add_x86_feature(featureset, X86FSET_XSAVEOPT);
		if (ecp->cp_eax & CPUID_INTC_EAX_D_1_XSAVEC)
			add_x86_feature(featureset, X86FSET_XSAVEC);
		if (ecp->cp_eax & CPUID_INTC_EAX_D_1_XSAVES)
			add_x86_feature(featureset, X86FSET_XSAVES);

		/*
		 * Zen 2 family processors suffer from erratum 1386 that causes
		 * xsaves to not function correctly in some circumstances. There
		 * are no supervisor states in Zen 2 and earlier. Practically
		 * speaking this has no impact for us as we currently do not
		 * leverage compressed xsave formats. To safeguard against
		 * issues in the future where we may opt to using it, we remove
		 * it from the feature set now. While Matisse has a microcode
		 * update available with a fix, not all Zen 2 CPUs do so it's
		 * simpler for the moment to unconditionally remove it.
		 */
		if (cpi->cpi_vendor == X86_VENDOR_AMD &&
		    uarchrev_uarch(cpi->cpi_uarchrev) <= X86_UARCH_AMD_ZEN2) {
			remove_x86_feature(featureset, X86FSET_XSAVES);
		}
	}

	/*
	 * Work on the "extended" feature information, doing
	 * some basic initialization to be used in the extended pass.
	 */
	xcpuid = 0;
	switch (cpi->cpi_vendor) {
	case X86_VENDOR_Intel:
		/*
		 * On KVM we know we will have proper support for extended
		 * cpuid.
		 */
		if (IS_NEW_F6(cpi) || cpi->cpi_family >= 0xf ||
		    (get_hwenv() == HW_KVM && cpi->cpi_family == 6 &&
		    (cpi->cpi_model == 6 || cpi->cpi_model == 2)))
			xcpuid++;
		break;
	case X86_VENDOR_AMD:
		if (cpi->cpi_family > 5 ||
		    (cpi->cpi_family == 5 && cpi->cpi_model >= 1))
			xcpuid++;
		break;
	case X86_VENDOR_Cyrix:
		/*
		 * Only these Cyrix CPUs are -known- to support
		 * extended cpuid operations.
		 */
		if (x86_type == X86_TYPE_VIA_CYRIX_III ||
		    x86_type == X86_TYPE_CYRIX_GXm)
			xcpuid++;
		break;
	case X86_VENDOR_HYGON:
	case X86_VENDOR_Centaur:
	case X86_VENDOR_TM:
	default:
		xcpuid++;
		break;
	}

	if (xcpuid) {
		cp = &cpi->cpi_extd[0];
		cp->cp_eax = CPUID_LEAF_EXT_0;
		cpi->cpi_xmaxeax = __cpuid_insn(cp);
	}

	if (cpi->cpi_xmaxeax & CPUID_LEAF_EXT_0) {

		if (cpi->cpi_xmaxeax > CPI_XMAXEAX_MAX)
			cpi->cpi_xmaxeax = CPI_XMAXEAX_MAX;

		switch (cpi->cpi_vendor) {
		case X86_VENDOR_Intel:
		case X86_VENDOR_AMD:
		case X86_VENDOR_HYGON:
			if (cpi->cpi_xmaxeax < 0x80000001)
				break;
			cp = &cpi->cpi_extd[1];
			cp->cp_eax = 0x80000001;
			(void) __cpuid_insn(cp);

			if (cpi->cpi_vendor == X86_VENDOR_AMD &&
			    cpi->cpi_family == 5 &&
			    cpi->cpi_model == 6 &&
			    cpi->cpi_step == 6) {
				/*
				 * K6 model 6 uses bit 10 to indicate SYSC
				 * Later models use bit 11. Fix it here.
				 */
				if (cp->cp_edx & 0x400) {
					cp->cp_edx &= ~0x400;
					cp->cp_edx |= CPUID_AMD_EDX_SYSC;
				}
			}

			platform_cpuid_mangle(cpi->cpi_vendor, 0x80000001, cp);

			/*
			 * Compute the additions to the kernel's feature word.
			 */
			if (cp->cp_edx & CPUID_AMD_EDX_NX) {
				add_x86_feature(featureset, X86FSET_NX);
			}

			/*
			 * Regardless whether or not we boot 64-bit,
			 * we should have a way to identify whether
			 * the CPU is capable of running 64-bit.
			 */
			if (cp->cp_edx & CPUID_AMD_EDX_LM) {
				add_x86_feature(featureset, X86FSET_64);
			}

			/* 1 GB large page - enable only for 64 bit kernel */
			if (cp->cp_edx & CPUID_AMD_EDX_1GPG) {
				add_x86_feature(featureset, X86FSET_1GPG);
			}

			if ((cpi->cpi_vendor == X86_VENDOR_AMD ||
			    cpi->cpi_vendor == X86_VENDOR_HYGON) &&
			    (cpi->cpi_std[1].cp_edx & CPUID_INTC_EDX_FXSR) &&
			    (cp->cp_ecx & CPUID_AMD_ECX_SSE4A)) {
				add_x86_feature(featureset, X86FSET_SSE4A);
			}

			/*
			 * It's really tricky to support syscall/sysret in
			 * the i386 kernel; we rely on sysenter/sysexit
			 * instead.  In the amd64 kernel, things are -way-
			 * better.
			 */
			if (cp->cp_edx & CPUID_AMD_EDX_SYSC) {
				add_x86_feature(featureset, X86FSET_ASYSC);
			}

			/*
			 * While we're thinking about system calls, note
			 * that AMD processors don't support sysenter
			 * in long mode at all, so don't try to program them.
			 */
			if (x86_vendor == X86_VENDOR_AMD ||
			    x86_vendor == X86_VENDOR_HYGON) {
				remove_x86_feature(featureset, X86FSET_SEP);
			}

			if (cp->cp_edx & CPUID_AMD_EDX_TSCP) {
				add_x86_feature(featureset, X86FSET_TSCP);
			}

			if (cp->cp_ecx & CPUID_AMD_ECX_SVM) {
				add_x86_feature(featureset, X86FSET_SVM);
			}

			if (cp->cp_ecx & CPUID_AMD_ECX_TOPOEXT) {
				add_x86_feature(featureset, X86FSET_TOPOEXT);
			}

			if (cp->cp_ecx & CPUID_AMD_ECX_PCEC) {
				add_x86_feature(featureset, X86FSET_AMD_PCEC);
			}

			if (cp->cp_ecx & CPUID_AMD_ECX_XOP) {
				add_x86_feature(featureset, X86FSET_XOP);
			}

			if (cp->cp_ecx & CPUID_AMD_ECX_FMA4) {
				add_x86_feature(featureset, X86FSET_FMA4);
			}

			if (cp->cp_ecx & CPUID_AMD_ECX_TBM) {
				add_x86_feature(featureset, X86FSET_TBM);
			}

			if (cp->cp_ecx & CPUID_AMD_ECX_MONITORX) {
				add_x86_feature(featureset, X86FSET_MONITORX);
			}
			break;
		default:
			break;
		}

		/*
		 * Get CPUID data about processor cores and hyperthreads.
		 */
		switch (cpi->cpi_vendor) {
		case X86_VENDOR_Intel:
			if (cpi->cpi_maxeax >= 4) {
				cp = &cpi->cpi_std[4];
				cp->cp_eax = 4;
				cp->cp_ecx = 0;
				(void) __cpuid_insn(cp);
				platform_cpuid_mangle(cpi->cpi_vendor, 4, cp);
			}
			/*FALLTHROUGH*/
		case X86_VENDOR_AMD:
		case X86_VENDOR_HYGON:
			if (cpi->cpi_xmaxeax < CPUID_LEAF_EXT_8)
				break;
			cp = &cpi->cpi_extd[8];
			cp->cp_eax = CPUID_LEAF_EXT_8;
			(void) __cpuid_insn(cp);
			platform_cpuid_mangle(cpi->cpi_vendor, CPUID_LEAF_EXT_8,
			    cp);

			/*
			 * AMD uses ebx for some extended functions.
			 */
			if (cpi->cpi_vendor == X86_VENDOR_AMD ||
			    cpi->cpi_vendor == X86_VENDOR_HYGON) {
				/*
				 * While we're here, check for the AMD "Error
				 * Pointer Zero/Restore" feature. This can be
				 * used to setup the FP save handlers
				 * appropriately.
				 */
				if (cp->cp_ebx & CPUID_AMD_EBX_ERR_PTR_ZERO) {
					cpi->cpi_fp_amd_save = 0;
				} else {
					cpi->cpi_fp_amd_save = 1;
				}

				if (cp->cp_ebx & CPUID_AMD_EBX_CLZERO) {
					add_x86_feature(featureset,
					    X86FSET_CLZERO);
				}
			}

			/*
			 * Virtual and physical address limits from
			 * cpuid override previously guessed values.
			 */
			cpi->cpi_pabits = BITX(cp->cp_eax, 7, 0);
			cpi->cpi_vabits = BITX(cp->cp_eax, 15, 8);
			break;
		default:
			break;
		}

		/*
		 * Get CPUID data about TSC Invariance in Deep C-State.
		 */
		switch (cpi->cpi_vendor) {
		case X86_VENDOR_Intel:
		case X86_VENDOR_AMD:
		case X86_VENDOR_HYGON:
			if (cpi->cpi_maxeax >= 7) {
				cp = &cpi->cpi_extd[7];
				cp->cp_eax = 0x80000007;
				cp->cp_ecx = 0;
				(void) __cpuid_insn(cp);
			}
			break;
		default:
			break;
		}
	}

	/*
	 * cpuid_basic_ppin assumes that cpuid_basic_topology has already been
	 * run and thus gathered some of its dependent leaves.
	 */
	cpuid_basic_topology(cpu, featureset);
	cpuid_basic_thermal(cpu, featureset);
#if !defined(__xpv)
	cpuid_basic_ppin(cpu, featureset);
#endif

	if (cpi->cpi_vendor == X86_VENDOR_AMD ||
	    cpi->cpi_vendor == X86_VENDOR_HYGON) {
		if (cpi->cpi_xmaxeax >= CPUID_LEAF_EXT_8 &&
		    cpi->cpi_extd[8].cp_ebx & CPUID_AMD_EBX_ERR_PTR_ZERO) {
			/* Special handling for AMD FP not necessary. */
			cpi->cpi_fp_amd_save = 0;
		} else {
			cpi->cpi_fp_amd_save = 1;
		}
	}

	/*
	 * Check (and potentially set) if lfence is serializing.
	 * This is useful for accurate rdtsc measurements and AMD retpolines.
	 */
	if ((cpi->cpi_vendor == X86_VENDOR_AMD ||
	    cpi->cpi_vendor == X86_VENDOR_HYGON) &&
	    is_x86_feature(featureset, X86FSET_SSE2)) {
		/*
		 * The AMD white paper Software Techniques For Managing
		 * Speculation on AMD Processors details circumstances for when
		 * lfence instructions are serializing.
		 *
		 * On family 0xf and 0x11, it is inherently so.  On family 0x10
		 * and later (excluding 0x11), a bit in the DE_CFG MSR
		 * determines the lfence behavior.  Per that whitepaper, AMD has
		 * committed to supporting that MSR on all later CPUs.
		 */
		if (cpi->cpi_family == 0xf || cpi->cpi_family == 0x11) {
			add_x86_feature(featureset, X86FSET_LFENCE_SER);
		} else if (cpi->cpi_family >= 0x10) {
#if !defined(__xpv)
			uint64_t val;

			/*
			 * Be careful when attempting to enable the bit, and
			 * verify that it was actually set in case we are
			 * running in a hypervisor which is less than faithful
			 * about its emulation of this feature.
			 */
			on_trap_data_t otd;
			if (!on_trap(&otd, OT_DATA_ACCESS)) {
				val = rdmsr(MSR_AMD_DE_CFG);
				val |= AMD_DE_CFG_LFENCE_DISPATCH;
				wrmsr(MSR_AMD_DE_CFG, val);
				val = rdmsr(MSR_AMD_DE_CFG);
			} else {
				val = 0;
			}
			no_trap();

			if ((val & AMD_DE_CFG_LFENCE_DISPATCH) != 0) {
				add_x86_feature(featureset, X86FSET_LFENCE_SER);
			}
#endif
		}
	} else if (cpi->cpi_vendor == X86_VENDOR_Intel &&
	    is_x86_feature(featureset, X86FSET_SSE2)) {
		/*
		 * Documentation and other OSes indicate that lfence is always
		 * serializing on Intel CPUs.
		 */
		add_x86_feature(featureset, X86FSET_LFENCE_SER);
	}


	/*
	 * Check the processor leaves that are used for security features. Grab
	 * any additional processor-specific leaves that we may not have yet.
	 */
	switch (cpi->cpi_vendor) {
	case X86_VENDOR_AMD:
	case X86_VENDOR_HYGON:
		if (cpi->cpi_xmaxeax >= CPUID_LEAF_EXT_21) {
			cp = &cpi->cpi_extd[0x21];
			cp->cp_eax = CPUID_LEAF_EXT_21;
			cp->cp_ecx = 0;
			(void) __cpuid_insn(cp);
		}
		break;
	default:
		break;
	}

	cpuid_scan_security(cpu, featureset);
}

/*
 * Make copies of the cpuid table entries we depend on, in
 * part for ease of parsing now, in part so that we have only
 * one place to correct any of it, in part for ease of
 * later export to userland, and in part so we can look at
 * this stuff in a crash dump.
 */

static void
cpuid_pass_extended(cpu_t *cpu, void *_arg __unused)
{
	uint_t n, nmax;
	int i;
	struct cpuid_regs *cp;
	uint8_t *dp;
	uint32_t *iptr;
	struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;

	if (cpi->cpi_maxeax < 1)
		return;

	if ((nmax = cpi->cpi_maxeax + 1) > NMAX_CPI_STD)
		nmax = NMAX_CPI_STD;
	/*
	 * (We already handled n == 0 and n == 1 in the basic pass)
	 */
	for (n = 2, cp = &cpi->cpi_std[2]; n < nmax; n++, cp++) {
		/*
		 * leaves 6 and 7 were handled in the basic pass
		 */
		if (n == 6 || n == 7)
			continue;

		cp->cp_eax = n;

		/*
		 * CPUID function 4 expects %ecx to be initialized
		 * with an index which indicates which cache to return
		 * information about. The OS is expected to call function 4
		 * with %ecx set to 0, 1, 2, ... until it returns with
		 * EAX[4:0] set to 0, which indicates there are no more
		 * caches.
		 *
		 * Here, populate cpi_std[4] with the information returned by
		 * function 4 when %ecx == 0, and do the rest in a later pass
		 * when dynamic memory allocation becomes available.
		 *
		 * Note: we need to explicitly initialize %ecx here, since
		 * function 4 may have been previously invoked.
		 */
		if (n == 4)
			cp->cp_ecx = 0;

		(void) __cpuid_insn(cp);
		platform_cpuid_mangle(cpi->cpi_vendor, n, cp);
		switch (n) {
		case 2:
			/*
			 * "the lower 8 bits of the %eax register
			 * contain a value that identifies the number
			 * of times the cpuid [instruction] has to be
			 * executed to obtain a complete image of the
			 * processor's caching systems."
			 *
			 * How *do* they make this stuff up?
			 */
			cpi->cpi_ncache = sizeof (*cp) *
			    BITX(cp->cp_eax, 7, 0);
			if (cpi->cpi_ncache == 0)
				break;
			cpi->cpi_ncache--;	/* skip count byte */

			/*
			 * Well, for now, rather than attempt to implement
			 * this slightly dubious algorithm, we just look
			 * at the first 15 ..
			 */
			if (cpi->cpi_ncache > (sizeof (*cp) - 1))
				cpi->cpi_ncache = sizeof (*cp) - 1;

			dp = cpi->cpi_cacheinfo;
			if (BITX(cp->cp_eax, 31, 31) == 0) {
				uint8_t *p = (void *)&cp->cp_eax;
				for (i = 1; i < 4; i++)
					if (p[i] != 0)
						*dp++ = p[i];
			}
			if (BITX(cp->cp_ebx, 31, 31) == 0) {
				uint8_t *p = (void *)&cp->cp_ebx;
				for (i = 0; i < 4; i++)
					if (p[i] != 0)
						*dp++ = p[i];
			}
			if (BITX(cp->cp_ecx, 31, 31) == 0) {
				uint8_t *p = (void *)&cp->cp_ecx;
				for (i = 0; i < 4; i++)
					if (p[i] != 0)
						*dp++ = p[i];
			}
			if (BITX(cp->cp_edx, 31, 31) == 0) {
				uint8_t *p = (void *)&cp->cp_edx;
				for (i = 0; i < 4; i++)
					if (p[i] != 0)
						*dp++ = p[i];
			}
			break;

		case 3:	/* Processor serial number, if PSN supported */
			break;

		case 4:	/* Deterministic cache parameters */
			break;

		case 5:	/* Monitor/Mwait parameters */
		{
			size_t mwait_size;

			/*
			 * check cpi_mwait.support which was set in
			 * cpuid_pass_basic()
			 */
			if (!(cpi->cpi_mwait.support & MWAIT_SUPPORT))
				break;

			/*
			 * Protect ourself from insane mwait line size.
			 * Workaround for incomplete hardware emulator(s).
			 */
			mwait_size = (size_t)MWAIT_SIZE_MAX(cpi);
			if (mwait_size < sizeof (uint32_t) ||
			    !ISP2(mwait_size)) {
#if DEBUG
				cmn_err(CE_NOTE, "Cannot handle cpu %d mwait "
				    "size %ld", cpu->cpu_id, (long)mwait_size);
#endif
				break;
			}

			cpi->cpi_mwait.mon_min = (size_t)MWAIT_SIZE_MIN(cpi);
			cpi->cpi_mwait.mon_max = mwait_size;
			if (MWAIT_EXTENSION(cpi)) {
				cpi->cpi_mwait.support |= MWAIT_EXTENSIONS;
				if (MWAIT_INT_ENABLE(cpi))
					cpi->cpi_mwait.support |=
					    MWAIT_ECX_INT_ENABLE;
			}
			break;
		}
		default:
			break;
		}
	}

	/*
	 * XSAVE enumeration
	 */
	if (cpi->cpi_maxeax >= 0xD) {
		struct cpuid_regs regs;
		boolean_t cpuid_d_valid = B_TRUE;

		cp = &regs;
		cp->cp_eax = 0xD;
		cp->cp_edx = cp->cp_ebx = cp->cp_ecx = 0;

		(void) __cpuid_insn(cp);

		/*
		 * Sanity checks for debug
		 */
		if ((cp->cp_eax & XFEATURE_LEGACY_FP) == 0 ||
		    (cp->cp_eax & XFEATURE_SSE) == 0) {
			cpuid_d_valid = B_FALSE;
		}

		cpi->cpi_xsave.xsav_hw_features_low = cp->cp_eax;
		cpi->cpi_xsave.xsav_hw_features_high = cp->cp_edx;
		cpi->cpi_xsave.xsav_max_size = cp->cp_ecx;

		/*
		 * If the hw supports AVX, get the size and offset in the save
		 * area for the ymm state.
		 */
		if (cpi->cpi_xsave.xsav_hw_features_low & XFEATURE_AVX) {
			cp->cp_eax = 0xD;
			cp->cp_ecx = 2;
			cp->cp_edx = cp->cp_ebx = 0;

			(void) __cpuid_insn(cp);

			if (cp->cp_ebx != CPUID_LEAFD_2_YMM_OFFSET ||
			    cp->cp_eax != CPUID_LEAFD_2_YMM_SIZE) {
				cpuid_d_valid = B_FALSE;
			}

			cpi->cpi_xsave.ymm_size = cp->cp_eax;
			cpi->cpi_xsave.ymm_offset = cp->cp_ebx;
		}

		/*
		 * If the hw supports MPX, get the size and offset in the
		 * save area for BNDREGS and BNDCSR.
		 */
		if (cpi->cpi_xsave.xsav_hw_features_low & XFEATURE_MPX) {
			cp->cp_eax = 0xD;
			cp->cp_ecx = 3;
			cp->cp_edx = cp->cp_ebx = 0;

			(void) __cpuid_insn(cp);

			cpi->cpi_xsave.bndregs_size = cp->cp_eax;
			cpi->cpi_xsave.bndregs_offset = cp->cp_ebx;

			cp->cp_eax = 0xD;
			cp->cp_ecx = 4;
			cp->cp_edx = cp->cp_ebx = 0;

			(void) __cpuid_insn(cp);

			cpi->cpi_xsave.bndcsr_size = cp->cp_eax;
			cpi->cpi_xsave.bndcsr_offset = cp->cp_ebx;
		}

		/*
		 * If the hw supports AVX512, get the size and offset in the
		 * save area for the opmask registers and zmm state.
		 */
		if (cpi->cpi_xsave.xsav_hw_features_low & XFEATURE_AVX512) {
			cp->cp_eax = 0xD;
			cp->cp_ecx = 5;
			cp->cp_edx = cp->cp_ebx = 0;

			(void) __cpuid_insn(cp);

			cpi->cpi_xsave.opmask_size = cp->cp_eax;
			cpi->cpi_xsave.opmask_offset = cp->cp_ebx;

			cp->cp_eax = 0xD;
			cp->cp_ecx = 6;
			cp->cp_edx = cp->cp_ebx = 0;

			(void) __cpuid_insn(cp);

			cpi->cpi_xsave.zmmlo_size = cp->cp_eax;
			cpi->cpi_xsave.zmmlo_offset = cp->cp_ebx;

			cp->cp_eax = 0xD;
			cp->cp_ecx = 7;
			cp->cp_edx = cp->cp_ebx = 0;

			(void) __cpuid_insn(cp);

			cpi->cpi_xsave.zmmhi_size = cp->cp_eax;
			cpi->cpi_xsave.zmmhi_offset = cp->cp_ebx;
		}

		if (cpi->cpi_xsave.xsav_hw_features_low & XFEATURE_PKRU) {
			cp->cp_eax = 0xD;
			cp->cp_ecx = 9;
			cp->cp_edx = cp->cp_ebx = 0;

			(void) __cpuid_insn(cp);

			cpi->cpi_xsave.pkru_size = cp->cp_eax;
			cpi->cpi_xsave.pkru_offset = cp->cp_ebx;
		}

		if (is_x86_feature(x86_featureset, X86FSET_XSAVE)) {
			xsave_state_size = 0;
		} else if (cpuid_d_valid) {
			xsave_state_size = cpi->cpi_xsave.xsav_max_size;
		} else {
			/* Broken CPUID 0xD, probably in HVM */
			cmn_err(CE_WARN, "cpu%d: CPUID.0xD returns invalid "
			    "value: hw_low = %d, hw_high = %d, xsave_size = %d"
			    ", ymm_size = %d, ymm_offset = %d\n",
			    cpu->cpu_id, cpi->cpi_xsave.xsav_hw_features_low,
			    cpi->cpi_xsave.xsav_hw_features_high,
			    (int)cpi->cpi_xsave.xsav_max_size,
			    (int)cpi->cpi_xsave.ymm_size,
			    (int)cpi->cpi_xsave.ymm_offset);

			if (xsave_state_size != 0) {
				/*
				 * This must be a non-boot CPU. We cannot
				 * continue, because boot cpu has already
				 * enabled XSAVE.
				 */
				ASSERT(cpu->cpu_id != 0);
				cmn_err(CE_PANIC, "cpu%d: we have already "
				    "enabled XSAVE on boot cpu, cannot "
				    "continue.", cpu->cpu_id);
			} else {
				/*
				 * If we reached here on the boot CPU, it's also
				 * almost certain that we'll reach here on the
				 * non-boot CPUs. When we're here on a boot CPU
				 * we should disable the feature, on a non-boot
				 * CPU we need to confirm that we have.
				 */
				if (cpu->cpu_id == 0) {
					remove_x86_feature(x86_featureset,
					    X86FSET_XSAVE);
					remove_x86_feature(x86_featureset,
					    X86FSET_AVX);
					remove_x86_feature(x86_featureset,
					    X86FSET_F16C);
					remove_x86_feature(x86_featureset,
					    X86FSET_BMI1);
					remove_x86_feature(x86_featureset,
					    X86FSET_BMI2);
					remove_x86_feature(x86_featureset,
					    X86FSET_FMA);
					remove_x86_feature(x86_featureset,
					    X86FSET_AVX2);
					remove_x86_feature(x86_featureset,
					    X86FSET_MPX);
					remove_x86_feature(x86_featureset,
					    X86FSET_AVX512F);
					remove_x86_feature(x86_featureset,
					    X86FSET_AVX512DQ);
					remove_x86_feature(x86_featureset,
					    X86FSET_AVX512PF);
					remove_x86_feature(x86_featureset,
					    X86FSET_AVX512ER);
					remove_x86_feature(x86_featureset,
					    X86FSET_AVX512CD);
					remove_x86_feature(x86_featureset,
					    X86FSET_AVX512BW);
					remove_x86_feature(x86_featureset,
					    X86FSET_AVX512VL);
					remove_x86_feature(x86_featureset,
					    X86FSET_AVX512FMA);
					remove_x86_feature(x86_featureset,
					    X86FSET_AVX512VBMI);
					remove_x86_feature(x86_featureset,
					    X86FSET_AVX512VNNI);
					remove_x86_feature(x86_featureset,
					    X86FSET_AVX512VPOPCDQ);
					remove_x86_feature(x86_featureset,
					    X86FSET_AVX512NNIW);
					remove_x86_feature(x86_featureset,
					    X86FSET_AVX512FMAPS);
					remove_x86_feature(x86_featureset,
					    X86FSET_VAES);
					remove_x86_feature(x86_featureset,
					    X86FSET_VPCLMULQDQ);
					remove_x86_feature(x86_featureset,
					    X86FSET_GFNI);
					remove_x86_feature(x86_featureset,
					    X86FSET_AVX512_VP2INT);
					remove_x86_feature(x86_featureset,
					    X86FSET_AVX512_BITALG);
					remove_x86_feature(x86_featureset,
					    X86FSET_AVX512_VBMI2);
					remove_x86_feature(x86_featureset,
					    X86FSET_AVX512_BF16);

					xsave_force_disable = B_TRUE;
				} else {
					VERIFY(is_x86_feature(x86_featureset,
					    X86FSET_XSAVE) == B_FALSE);
				}
			}
		}
	}


	if ((cpi->cpi_xmaxeax & CPUID_LEAF_EXT_0) == 0)
		return;

	if ((nmax = cpi->cpi_xmaxeax - CPUID_LEAF_EXT_0 + 1) > NMAX_CPI_EXTD)
		nmax = NMAX_CPI_EXTD;
	/*
	 * Copy the extended properties, fixing them as we go. While we start at
	 * 2 because we've already handled a few cases in the basic pass, the
	 * rest we let ourselves just grab again (e.g. 0x8, 0x21).
	 */
	iptr = (void *)cpi->cpi_brandstr;
	for (n = 2, cp = &cpi->cpi_extd[2]; n < nmax; cp++, n++) {
		cp->cp_eax = CPUID_LEAF_EXT_0 + n;
		(void) __cpuid_insn(cp);
		platform_cpuid_mangle(cpi->cpi_vendor, CPUID_LEAF_EXT_0 + n,
		    cp);
		switch (n) {
		case 2:
		case 3:
		case 4:
			/*
			 * Extract the brand string
			 */
			*iptr++ = cp->cp_eax;
			*iptr++ = cp->cp_ebx;
			*iptr++ = cp->cp_ecx;
			*iptr++ = cp->cp_edx;
			break;
		case 5:
			switch (cpi->cpi_vendor) {
			case X86_VENDOR_AMD:
				/*
				 * The Athlon and Duron were the first
				 * parts to report the sizes of the
				 * TLB for large pages. Before then,
				 * we don't trust the data.
				 */
				if (cpi->cpi_family < 6 ||
				    (cpi->cpi_family == 6 &&
				    cpi->cpi_model < 1))
					cp->cp_eax = 0;
				break;
			default:
				break;
			}
			break;
		case 6:
			switch (cpi->cpi_vendor) {
			case X86_VENDOR_AMD:
				/*
				 * The Athlon and Duron were the first
				 * AMD parts with L2 TLB's.
				 * Before then, don't trust the data.
				 */
				if (cpi->cpi_family < 6 ||
				    (cpi->cpi_family == 6 &&
				    cpi->cpi_model < 1))
					cp->cp_eax = cp->cp_ebx = 0;
				/*
				 * AMD Duron rev A0 reports L2
				 * cache size incorrectly as 1K
				 * when it is really 64K
				 */
				if (cpi->cpi_family == 6 &&
				    cpi->cpi_model == 3 &&
				    cpi->cpi_step == 0) {
					cp->cp_ecx &= 0xffff;
					cp->cp_ecx |= 0x400000;
				}
				break;
			case X86_VENDOR_Cyrix:	/* VIA C3 */
				/*
				 * VIA C3 processors are a bit messed
				 * up w.r.t. encoding cache sizes in %ecx
				 */
				if (cpi->cpi_family != 6)
					break;
				/*
				 * model 7 and 8 were incorrectly encoded
				 *
				 * xxx is model 8 really broken?
				 */
				if (cpi->cpi_model == 7 ||
				    cpi->cpi_model == 8)
					cp->cp_ecx =
					    BITX(cp->cp_ecx, 31, 24) << 16 |
					    BITX(cp->cp_ecx, 23, 16) << 12 |
					    BITX(cp->cp_ecx, 15, 8) << 8 |
					    BITX(cp->cp_ecx, 7, 0);
				/*
				 * model 9 stepping 1 has wrong associativity
				 */
				if (cpi->cpi_model == 9 && cpi->cpi_step == 1)
					cp->cp_ecx |= 8 << 12;
				break;
			case X86_VENDOR_Intel:
				/*
				 * Extended L2 Cache features function.
				 * First appeared on Prescott.
				 */
			default:
				break;
			}
			break;
		default:
			break;
		}
	}
}

static const char *
intel_cpubrand(const struct cpuid_info *cpi)
{
	int i;

	ASSERT(is_x86_feature(x86_featureset, X86FSET_CPUID));

	switch (cpi->cpi_family) {
	case 5:
		return ("Intel Pentium(r)");
	case 6:
		switch (cpi->cpi_model) {
			uint_t celeron, xeon;
			const struct cpuid_regs *cp;
		case 0:
		case 1:
		case 2:
			return ("Intel Pentium(r) Pro");
		case 3:
		case 4:
			return ("Intel Pentium(r) II");
		case 6:
			return ("Intel Celeron(r)");
		case 5:
		case 7:
			celeron = xeon = 0;
			cp = &cpi->cpi_std[2];	/* cache info */

			for (i = 1; i < 4; i++) {
				uint_t tmp;

				tmp = (cp->cp_eax >> (8 * i)) & 0xff;
				if (tmp == 0x40)
					celeron++;
				if (tmp >= 0x44 && tmp <= 0x45)
					xeon++;
			}

			for (i = 0; i < 2; i++) {
				uint_t tmp;

				tmp = (cp->cp_ebx >> (8 * i)) & 0xff;
				if (tmp == 0x40)
					celeron++;
				else if (tmp >= 0x44 && tmp <= 0x45)
					xeon++;
			}

			for (i = 0; i < 4; i++) {
				uint_t tmp;

				tmp = (cp->cp_ecx >> (8 * i)) & 0xff;
				if (tmp == 0x40)
					celeron++;
				else if (tmp >= 0x44 && tmp <= 0x45)
					xeon++;
			}

			for (i = 0; i < 4; i++) {
				uint_t tmp;

				tmp = (cp->cp_edx >> (8 * i)) & 0xff;
				if (tmp == 0x40)
					celeron++;
				else if (tmp >= 0x44 && tmp <= 0x45)
					xeon++;
			}

			if (celeron)
				return ("Intel Celeron(r)");
			if (xeon)
				return (cpi->cpi_model == 5 ?
				    "Intel Pentium(r) II Xeon(tm)" :
				    "Intel Pentium(r) III Xeon(tm)");
			return (cpi->cpi_model == 5 ?
			    "Intel Pentium(r) II or Pentium(r) II Xeon(tm)" :
			    "Intel Pentium(r) III or Pentium(r) III Xeon(tm)");
		default:
			break;
		}
	default:
		break;
	}

	/* BrandID is present if the field is nonzero */
	if (cpi->cpi_brandid != 0) {
		static const struct {
			uint_t bt_bid;
			const char *bt_str;
		} brand_tbl[] = {
			{ 0x1,	"Intel(r) Celeron(r)" },
			{ 0x2,	"Intel(r) Pentium(r) III" },
			{ 0x3,	"Intel(r) Pentium(r) III Xeon(tm)" },
			{ 0x4,	"Intel(r) Pentium(r) III" },
			{ 0x6,	"Mobile Intel(r) Pentium(r) III" },
			{ 0x7,	"Mobile Intel(r) Celeron(r)" },
			{ 0x8,	"Intel(r) Pentium(r) 4" },
			{ 0x9,	"Intel(r) Pentium(r) 4" },
			{ 0xa,	"Intel(r) Celeron(r)" },
			{ 0xb,	"Intel(r) Xeon(tm)" },
			{ 0xc,	"Intel(r) Xeon(tm) MP" },
			{ 0xe,	"Mobile Intel(r) Pentium(r) 4" },
			{ 0xf,	"Mobile Intel(r) Celeron(r)" },
			{ 0x11, "Mobile Genuine Intel(r)" },
			{ 0x12, "Intel(r) Celeron(r) M" },
			{ 0x13, "Mobile Intel(r) Celeron(r)" },
			{ 0x14, "Intel(r) Celeron(r)" },
			{ 0x15, "Mobile Genuine Intel(r)" },
			{ 0x16,	"Intel(r) Pentium(r) M" },
			{ 0x17, "Mobile Intel(r) Celeron(r)" }
		};
		uint_t btblmax = sizeof (brand_tbl) / sizeof (brand_tbl[0]);
		uint_t sgn;

		sgn = (cpi->cpi_family << 8) |
		    (cpi->cpi_model << 4) | cpi->cpi_step;

		for (i = 0; i < btblmax; i++)
			if (brand_tbl[i].bt_bid == cpi->cpi_brandid)
				break;
		if (i < btblmax) {
			if (sgn == 0x6b1 && cpi->cpi_brandid == 3)
				return ("Intel(r) Celeron(r)");
			if (sgn < 0xf13 && cpi->cpi_brandid == 0xb)
				return ("Intel(r) Xeon(tm) MP");
			if (sgn < 0xf13 && cpi->cpi_brandid == 0xe)
				return ("Intel(r) Xeon(tm)");
			return (brand_tbl[i].bt_str);
		}
	}

	return (NULL);
}

static const char *
amd_cpubrand(const struct cpuid_info *cpi)
{
	ASSERT(is_x86_feature(x86_featureset, X86FSET_CPUID));

	switch (cpi->cpi_family) {
	case 5:
		switch (cpi->cpi_model) {
		case 0:
		case 1:
		case 2:
		case 3:
		case 4:
		case 5:
			return ("AMD-K5(r)");
		case 6:
		case 7:
			return ("AMD-K6(r)");
		case 8:
			return ("AMD-K6(r)-2");
		case 9:
			return ("AMD-K6(r)-III");
		default:
			return ("AMD (family 5)");
		}
	case 6:
		switch (cpi->cpi_model) {
		case 1:
			return ("AMD-K7(tm)");
		case 0:
		case 2:
		case 4:
			return ("AMD Athlon(tm)");
		case 3:
		case 7:
			return ("AMD Duron(tm)");
		case 6:
		case 8:
		case 10:
			/*
			 * Use the L2 cache size to distinguish
			 */
			return ((cpi->cpi_extd[6].cp_ecx >> 16) >= 256 ?
			    "AMD Athlon(tm)" : "AMD Duron(tm)");
		default:
			return ("AMD (family 6)");
		}
	default:
		break;
	}

	if (cpi->cpi_family == 0xf && cpi->cpi_model == 5 &&
	    cpi->cpi_brandid != 0) {
		switch (BITX(cpi->cpi_brandid, 7, 5)) {
		case 3:
			return ("AMD Opteron(tm) UP 1xx");
		case 4:
			return ("AMD Opteron(tm) DP 2xx");
		case 5:
			return ("AMD Opteron(tm) MP 8xx");
		default:
			return ("AMD Opteron(tm)");
		}
	}

	return (NULL);
}

static const char *
cyrix_cpubrand(struct cpuid_info *cpi, uint_t type)
{
	ASSERT(is_x86_feature(x86_featureset, X86FSET_CPUID));

	switch (type) {
	case X86_TYPE_CYRIX_6x86:
		return ("Cyrix 6x86");
	case X86_TYPE_CYRIX_6x86L:
		return ("Cyrix 6x86L");
	case X86_TYPE_CYRIX_6x86MX:
		return ("Cyrix 6x86MX");
	case X86_TYPE_CYRIX_GXm:
		return ("Cyrix GXm");
	case X86_TYPE_CYRIX_MediaGX:
		return ("Cyrix MediaGX");
	case X86_TYPE_CYRIX_MII:
		return ("Cyrix M2");
	case X86_TYPE_VIA_CYRIX_III:
		return ("VIA Cyrix M3");
	default:
		/*
		 * Have another wild guess ..
		 */
		if (cpi->cpi_family == 4 && cpi->cpi_model == 9)
			return ("Cyrix 5x86");
		else if (cpi->cpi_family == 5) {
			switch (cpi->cpi_model) {
			case 2:
				return ("Cyrix 6x86");	/* Cyrix M1 */
			case 4:
				return ("Cyrix MediaGX");
			default:
				break;
			}
		} else if (cpi->cpi_family == 6) {
			switch (cpi->cpi_model) {
			case 0:
				return ("Cyrix 6x86MX"); /* Cyrix M2? */
			case 5:
			case 6:
			case 7:
			case 8:
			case 9:
				return ("VIA C3");
			default:
				break;
			}
		}
		break;
	}
	return (NULL);
}

/*
 * This only gets called in the case that the CPU extended
 * feature brand string (0x80000002, 0x80000003, 0x80000004)
 * aren't available, or contain null bytes for some reason.
 */
static void
fabricate_brandstr(struct cpuid_info *cpi)
{
	const char *brand = NULL;

	switch (cpi->cpi_vendor) {
	case X86_VENDOR_Intel:
		brand = intel_cpubrand(cpi);
		break;
	case X86_VENDOR_AMD:
		brand = amd_cpubrand(cpi);
		break;
	case X86_VENDOR_Cyrix:
		brand = cyrix_cpubrand(cpi, x86_type);
		break;
	case X86_VENDOR_NexGen:
		if (cpi->cpi_family == 5 && cpi->cpi_model == 0)
			brand = "NexGen Nx586";
		break;
	case X86_VENDOR_Centaur:
		if (cpi->cpi_family == 5)
			switch (cpi->cpi_model) {
			case 4:
				brand = "Centaur C6";
				break;
			case 8:
				brand = "Centaur C2";
				break;
			case 9:
				brand = "Centaur C3";
				break;
			default:
				break;
			}
		break;
	case X86_VENDOR_Rise:
		if (cpi->cpi_family == 5 &&
		    (cpi->cpi_model == 0 || cpi->cpi_model == 2))
			brand = "Rise mP6";
		break;
	case X86_VENDOR_SiS:
		if (cpi->cpi_family == 5 && cpi->cpi_model == 0)
			brand = "SiS 55x";
		break;
	case X86_VENDOR_TM:
		if (cpi->cpi_family == 5 && cpi->cpi_model == 4)
			brand = "Transmeta Crusoe TM3x00 or TM5x00";
		break;
	case X86_VENDOR_NSC:
	case X86_VENDOR_UMC:
	default:
		break;
	}
	if (brand) {
		(void) strcpy((char *)cpi->cpi_brandstr, brand);
		return;
	}

	/*
	 * If all else fails ...
	 */
	(void) snprintf(cpi->cpi_brandstr, sizeof (cpi->cpi_brandstr),
	    "%s %d.%d.%d", cpi->cpi_vendorstr, cpi->cpi_family,
	    cpi->cpi_model, cpi->cpi_step);
}

/*
 * This routine is called just after kernel memory allocation
 * becomes available on cpu0, and as part of mp_startup() on
 * the other cpus.
 *
 * Fixup the brand string, and collect any information from cpuid
 * that requires dynamically allocated storage to represent.
 */

static void
cpuid_pass_dynamic(cpu_t *cpu, void *_arg __unused)
{
	int	i, max, shft, level, size;
	struct cpuid_regs regs;
	struct cpuid_regs *cp;
	struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;

	/*
	 * Deterministic cache parameters
	 *
	 * Intel uses leaf 0x4 for this, while AMD uses leaf 0x8000001d. The
	 * values that are present are currently defined to be the same. This
	 * means we can use the same logic to parse it as long as we use the
	 * appropriate leaf to get the data. If you're updating this, make sure
	 * you're careful about which vendor supports which aspect.
	 *
	 * Take this opportunity to detect the number of threads sharing the
	 * last level cache, and construct a corresponding cache id. The
	 * respective cpuid_info members are initialized to the default case of
	 * "no last level cache sharing".
	 */
	cpi->cpi_ncpu_shr_last_cache = 1;
	cpi->cpi_last_lvl_cacheid = cpu->cpu_id;

	if ((cpi->cpi_maxeax >= 4 && cpi->cpi_vendor == X86_VENDOR_Intel) ||
	    ((cpi->cpi_vendor == X86_VENDOR_AMD ||
	    cpi->cpi_vendor == X86_VENDOR_HYGON) &&
	    cpi->cpi_xmaxeax >= CPUID_LEAF_EXT_1d &&
	    is_x86_feature(x86_featureset, X86FSET_TOPOEXT))) {
		uint32_t leaf;

		if (cpi->cpi_vendor == X86_VENDOR_Intel) {
			leaf = 4;
		} else {
			leaf = CPUID_LEAF_EXT_1d;
		}

		/*
		 * Find the # of elements (size) returned by the leaf and along
		 * the way detect last level cache sharing details.
		 */
		bzero(&regs, sizeof (regs));
		cp = &regs;
		for (i = 0, max = 0; i < CPI_FN4_ECX_MAX; i++) {
			cp->cp_eax = leaf;
			cp->cp_ecx = i;

			(void) __cpuid_insn(cp);

			if (CPI_CACHE_TYPE(cp) == 0)
				break;
			level = CPI_CACHE_LVL(cp);
			if (level > max) {
				max = level;
				cpi->cpi_ncpu_shr_last_cache =
				    CPI_NTHR_SHR_CACHE(cp) + 1;
			}
		}
		cpi->cpi_cache_leaf_size = size = i;

		/*
		 * Allocate the cpi_cache_leaves array. The first element
		 * references the regs for the corresponding leaf with %ecx set
		 * to 0. This was gathered in cpuid_pass_extended().
		 */
		if (size > 0) {
			cpi->cpi_cache_leaves =
			    kmem_alloc(size * sizeof (cp), KM_SLEEP);
			if (cpi->cpi_vendor == X86_VENDOR_Intel) {
				cpi->cpi_cache_leaves[0] = &cpi->cpi_std[4];
			} else {
				cpi->cpi_cache_leaves[0] = &cpi->cpi_extd[0x1d];
			}

			/*
			 * Allocate storage to hold the additional regs
			 * for the leaf, %ecx == 1 .. cpi_cache_leaf_size.
			 *
			 * The regs for the leaf, %ecx == 0 has already
			 * been allocated as indicated above.
			 */
			for (i = 1; i < size; i++) {
				cp = cpi->cpi_cache_leaves[i] =
				    kmem_zalloc(sizeof (regs), KM_SLEEP);
				cp->cp_eax = leaf;
				cp->cp_ecx = i;

				(void) __cpuid_insn(cp);
			}
		}
		/*
		 * Determine the number of bits needed to represent
		 * the number of CPUs sharing the last level cache.
		 *
		 * Shift off that number of bits from the APIC id to
		 * derive the cache id.
		 */
		shft = 0;
		for (i = 1; i < cpi->cpi_ncpu_shr_last_cache; i <<= 1)
			shft++;
		cpi->cpi_last_lvl_cacheid = cpi->cpi_apicid >> shft;
	}

	/*
	 * Now fixup the brand string
	 */
	if ((cpi->cpi_xmaxeax & CPUID_LEAF_EXT_0) == 0) {
		fabricate_brandstr(cpi);
	} else {

		/*
		 * If we successfully extracted a brand string from the cpuid
		 * instruction, clean it up by removing leading spaces and
		 * similar junk.
		 */
		if (cpi->cpi_brandstr[0]) {
			size_t maxlen = sizeof (cpi->cpi_brandstr);
			char *src, *dst;

			dst = src = (char *)cpi->cpi_brandstr;
			src[maxlen - 1] = '\0';
			/*
			 * strip leading spaces
			 */
			while (*src == ' ')
				src++;
			/*
			 * Remove any 'Genuine' or "Authentic" prefixes
			 */
			if (strncmp(src, "Genuine ", 8) == 0)
				src += 8;
			if (strncmp(src, "Authentic ", 10) == 0)
				src += 10;

			/*
			 * Now do an in-place copy.
			 * Map (R) to (r) and (TM) to (tm).
			 * The era of teletypes is long gone, and there's
			 * -really- no need to shout.
			 */
			while (*src != '\0') {
				if (src[0] == '(') {
					if (strncmp(src + 1, "R)", 2) == 0) {
						(void) strncpy(dst, "(r)", 3);
						src += 3;
						dst += 3;
						continue;
					}
					if (strncmp(src + 1, "TM)", 3) == 0) {
						(void) strncpy(dst, "(tm)", 4);
						src += 4;
						dst += 4;
						continue;
					}
				}
				*dst++ = *src++;
			}
			*dst = '\0';

			/*
			 * Finally, remove any trailing spaces
			 */
			while (--dst > cpi->cpi_brandstr)
				if (*dst == ' ')
					*dst = '\0';
				else
					break;
		} else
			fabricate_brandstr(cpi);
	}
}

typedef struct {
	uint32_t avm_av;
	uint32_t avm_feat;
} av_feat_map_t;

/*
 * These arrays are used to map features that we should add based on x86
 * features that are present. As a large number depend on kernel features,
 * rather than rechecking and clearing CPUID everywhere, we simply map these.
 * There is an array of these for each hwcap word. Some features aren't tracked
 * in the kernel x86 featureset and that's ok. They will not show up in here.
 */
static const av_feat_map_t x86fset_to_av1[] = {
	{ AV_386_CX8, X86FSET_CX8 },
	{ AV_386_SEP, X86FSET_SEP },
	{ AV_386_AMD_SYSC, X86FSET_ASYSC },
	{ AV_386_CMOV, X86FSET_CMOV },
	{ AV_386_FXSR, X86FSET_SSE },
	{ AV_386_SSE, X86FSET_SSE },
	{ AV_386_SSE2, X86FSET_SSE2 },
	{ AV_386_SSE3, X86FSET_SSE3 },
	{ AV_386_CX16, X86FSET_CX16 },
	{ AV_386_TSCP, X86FSET_TSCP },
	{ AV_386_AMD_SSE4A, X86FSET_SSE4A },
	{ AV_386_SSSE3, X86FSET_SSSE3 },
	{ AV_386_SSE4_1, X86FSET_SSE4_1 },
	{ AV_386_SSE4_2, X86FSET_SSE4_2 },
	{ AV_386_AES, X86FSET_AES },
	{ AV_386_PCLMULQDQ, X86FSET_PCLMULQDQ },
	{ AV_386_XSAVE, X86FSET_XSAVE },
	{ AV_386_AVX, X86FSET_AVX },
	{ AV_386_VMX, X86FSET_VMX },
	{ AV_386_AMD_SVM, X86FSET_SVM }
};

static const av_feat_map_t x86fset_to_av2[] = {
	{ AV_386_2_F16C, X86FSET_F16C },
	{ AV_386_2_RDRAND, X86FSET_RDRAND },
	{ AV_386_2_BMI1, X86FSET_BMI1 },
	{ AV_386_2_BMI2, X86FSET_BMI2 },
	{ AV_386_2_FMA, X86FSET_FMA },
	{ AV_386_2_AVX2, X86FSET_AVX2 },
	{ AV_386_2_ADX, X86FSET_ADX },
	{ AV_386_2_RDSEED, X86FSET_RDSEED },
	{ AV_386_2_AVX512F, X86FSET_AVX512F },
	{ AV_386_2_AVX512DQ, X86FSET_AVX512DQ },
	{ AV_386_2_AVX512IFMA, X86FSET_AVX512FMA },
	{ AV_386_2_AVX512PF, X86FSET_AVX512PF },
	{ AV_386_2_AVX512ER, X86FSET_AVX512ER },
	{ AV_386_2_AVX512CD, X86FSET_AVX512CD },
	{ AV_386_2_AVX512BW, X86FSET_AVX512BW },
	{ AV_386_2_AVX512VL, X86FSET_AVX512VL },
	{ AV_386_2_AVX512VBMI, X86FSET_AVX512VBMI },
	{ AV_386_2_AVX512VPOPCDQ, X86FSET_AVX512VPOPCDQ },
	{ AV_386_2_SHA, X86FSET_SHA },
	{ AV_386_2_FSGSBASE, X86FSET_FSGSBASE },
	{ AV_386_2_CLFLUSHOPT, X86FSET_CLFLUSHOPT },
	{ AV_386_2_CLWB, X86FSET_CLWB },
	{ AV_386_2_MONITORX, X86FSET_MONITORX },
	{ AV_386_2_CLZERO, X86FSET_CLZERO },
	{ AV_386_2_AVX512_VNNI, X86FSET_AVX512VNNI },
	{ AV_386_2_VPCLMULQDQ, X86FSET_VPCLMULQDQ },
	{ AV_386_2_VAES, X86FSET_VAES },
	{ AV_386_2_GFNI, X86FSET_GFNI },
	{ AV_386_2_AVX512_VP2INT, X86FSET_AVX512_VP2INT },
	{ AV_386_2_AVX512_BITALG, X86FSET_AVX512_BITALG }
};

static const av_feat_map_t x86fset_to_av3[] = {
	{ AV_386_3_AVX512_VBMI2, X86FSET_AVX512_VBMI2 },
	{ AV_386_3_AVX512_BF16, X86FSET_AVX512_BF16 }
};

/*
 * This routine is called out of bind_hwcap() much later in the life
 * of the kernel (post_startup()).  The job of this routine is to resolve
 * the hardware feature support and kernel support for those features into
 * what we're actually going to tell applications via the aux vector.
 *
 * Most of the aux vector is derived from the x86_featureset array vector where
 * a given feature indicates that an aux vector should be plumbed through. This
 * allows the kernel to use one tracking mechanism for these based on whether or
 * not it has the required hardware support (most often xsave). Most newer
 * features are added there in case we need them in the kernel. Otherwise,
 * features are evaluated based on looking at the cpuid features that remain. If
 * you find yourself wanting to clear out cpuid features for some reason, they
 * should instead be driven by the feature set so we have a consistent view.
 */

static void
cpuid_pass_resolve(cpu_t *cpu, void *arg)
{
	uint_t *hwcap_out = (uint_t *)arg;
	struct cpuid_info *cpi;
	uint_t hwcap_flags = 0, hwcap_flags_2 = 0, hwcap_flags_3 = 0;

	cpi = cpu->cpu_m.mcpu_cpi;

	for (uint_t i = 0; i < ARRAY_SIZE(x86fset_to_av1); i++) {
		if (is_x86_feature(x86_featureset,
		    x86fset_to_av1[i].avm_feat)) {
			hwcap_flags |= x86fset_to_av1[i].avm_av;
		}
	}

	for (uint_t i = 0; i < ARRAY_SIZE(x86fset_to_av2); i++) {
		if (is_x86_feature(x86_featureset,
		    x86fset_to_av2[i].avm_feat)) {
			hwcap_flags_2 |= x86fset_to_av2[i].avm_av;
		}
	}

	for (uint_t i = 0; i < ARRAY_SIZE(x86fset_to_av3); i++) {
		if (is_x86_feature(x86_featureset,
		    x86fset_to_av3[i].avm_feat)) {
			hwcap_flags_3 |= x86fset_to_av3[i].avm_av;
		}
	}

	/*
	 * From here on out we're working through features that don't have
	 * corresponding kernel feature flags for various reasons that are
	 * mostly just due to the historical implementation.
	 */
	if (cpi->cpi_maxeax >= 1) {
		uint32_t *edx = &cpi->cpi_support[STD_EDX_FEATURES];
		uint32_t *ecx = &cpi->cpi_support[STD_ECX_FEATURES];

		*edx = CPI_FEATURES_EDX(cpi);
		*ecx = CPI_FEATURES_ECX(cpi);

		/*
		 * [no explicit support required beyond x87 fp context]
		 */
		if (!fpu_exists)
			*edx &= ~(CPUID_INTC_EDX_FPU | CPUID_INTC_EDX_MMX);

		/*
		 * Now map the supported feature vector to things that we
		 * think userland will care about.
		 */
		if (*ecx & CPUID_INTC_ECX_MOVBE)
			hwcap_flags |= AV_386_MOVBE;

		if (*ecx & CPUID_INTC_ECX_POPCNT)
			hwcap_flags |= AV_386_POPCNT;
		if (*edx & CPUID_INTC_EDX_FPU)
			hwcap_flags |= AV_386_FPU;
		if (*edx & CPUID_INTC_EDX_MMX)
			hwcap_flags |= AV_386_MMX;
		if (*edx & CPUID_INTC_EDX_TSC)
			hwcap_flags |= AV_386_TSC;
	}

	/*
	 * Check a few miscellaneous features.
	 */
	if (cpi->cpi_xmaxeax < 0x80000001)
		goto resolve_done;

	switch (cpi->cpi_vendor) {
		uint32_t *edx, *ecx;

	case X86_VENDOR_Intel:
		/*
		 * Seems like Intel duplicated what we necessary
		 * here to make the initial crop of 64-bit OS's work.
		 * Hopefully, those are the only "extended" bits
		 * they'll add.
		 */
		/*FALLTHROUGH*/

	case X86_VENDOR_AMD:
	case X86_VENDOR_HYGON:
		edx = &cpi->cpi_support[AMD_EDX_FEATURES];
		ecx = &cpi->cpi_support[AMD_ECX_FEATURES];

		*edx = CPI_FEATURES_XTD_EDX(cpi);
		*ecx = CPI_FEATURES_XTD_ECX(cpi);

		/*
		 * [no explicit support required beyond
		 * x87 fp context and exception handlers]
		 */
		if (!fpu_exists)
			*edx &= ~(CPUID_AMD_EDX_MMXamd |
			    CPUID_AMD_EDX_3DNow | CPUID_AMD_EDX_3DNowx);

		/*
		 * Now map the supported feature vector to
		 * things that we think userland will care about.
		 */
		if (*edx & CPUID_AMD_EDX_MMXamd)
			hwcap_flags |= AV_386_AMD_MMX;
		if (*edx & CPUID_AMD_EDX_3DNow)
			hwcap_flags |= AV_386_AMD_3DNow;
		if (*edx & CPUID_AMD_EDX_3DNowx)
			hwcap_flags |= AV_386_AMD_3DNowx;

		switch (cpi->cpi_vendor) {
		case X86_VENDOR_AMD:
		case X86_VENDOR_HYGON:
			if (*ecx & CPUID_AMD_ECX_AHF64)
				hwcap_flags |= AV_386_AHF;
			if (*ecx & CPUID_AMD_ECX_LZCNT)
				hwcap_flags |= AV_386_AMD_LZCNT;
			break;

		case X86_VENDOR_Intel:
			if (*ecx & CPUID_AMD_ECX_LZCNT)
				hwcap_flags |= AV_386_AMD_LZCNT;
			/*
			 * Aarrgh.
			 * Intel uses a different bit in the same word.
			 */
			if (*ecx & CPUID_INTC_ECX_AHF64)
				hwcap_flags |= AV_386_AHF;
			break;
		default:
			break;
		}
		break;

	default:
		break;
	}

resolve_done:
	if (hwcap_out != NULL) {
		hwcap_out[0] = hwcap_flags;
		hwcap_out[1] = hwcap_flags_2;
		hwcap_out[2] = hwcap_flags_3;
	}
}


/*
 * Simulate the cpuid instruction using the data we previously
 * captured about this CPU.  We try our best to return the truth
 * about the hardware, independently of kernel support.
 */
uint32_t
cpuid_insn(cpu_t *cpu, struct cpuid_regs *cp)
{
	struct cpuid_info *cpi;
	struct cpuid_regs *xcp;

	if (cpu == NULL)
		cpu = CPU;
	cpi = cpu->cpu_m.mcpu_cpi;

	ASSERT(cpuid_checkpass(cpu, CPUID_PASS_DYNAMIC));

	/*
	 * CPUID data is cached in two separate places: cpi_std for standard
	 * CPUID leaves , and cpi_extd for extended CPUID leaves.
	 */
	if (cp->cp_eax <= cpi->cpi_maxeax && cp->cp_eax < NMAX_CPI_STD) {
		xcp = &cpi->cpi_std[cp->cp_eax];
	} else if (cp->cp_eax >= CPUID_LEAF_EXT_0 &&
	    cp->cp_eax <= cpi->cpi_xmaxeax &&
	    cp->cp_eax < CPUID_LEAF_EXT_0 + NMAX_CPI_EXTD) {
		xcp = &cpi->cpi_extd[cp->cp_eax - CPUID_LEAF_EXT_0];
	} else {
		/*
		 * The caller is asking for data from an input parameter which
		 * the kernel has not cached.  In this case we go fetch from
		 * the hardware and return the data directly to the user.
		 */
		return (__cpuid_insn(cp));
	}

	cp->cp_eax = xcp->cp_eax;
	cp->cp_ebx = xcp->cp_ebx;
	cp->cp_ecx = xcp->cp_ecx;
	cp->cp_edx = xcp->cp_edx;
	return (cp->cp_eax);
}

boolean_t
cpuid_checkpass(const cpu_t *const cpu, const cpuid_pass_t pass)
{
	return (cpu != NULL && cpu->cpu_m.mcpu_cpi != NULL &&
	    cpu->cpu_m.mcpu_cpi->cpi_pass >= pass);
}

int
cpuid_getbrandstr(cpu_t *cpu, char *s, size_t n)
{
	ASSERT(cpuid_checkpass(cpu, CPUID_PASS_DYNAMIC));

	return (snprintf(s, n, "%s", cpu->cpu_m.mcpu_cpi->cpi_brandstr));
}

int
cpuid_is_cmt(cpu_t *cpu)
{
	if (cpu == NULL)
		cpu = CPU;

	ASSERT(cpuid_checkpass(cpu, CPUID_PASS_BASIC));

	return (cpu->cpu_m.mcpu_cpi->cpi_chipid >= 0);
}

/*
 * AMD and Intel both implement the 64-bit variant of the syscall
 * instruction (syscallq), so if there's -any- support for syscall,
 * cpuid currently says "yes, we support this".
 *
 * However, Intel decided to -not- implement the 32-bit variant of the
 * syscall instruction, so we provide a predicate to allow our caller
 * to test that subtlety here.
 *
 * XXPV	Currently, 32-bit syscall instructions don't work via the hypervisor,
 *	even in the case where the hardware would in fact support it.
 */
/*ARGSUSED*/
int
cpuid_syscall32_insn(cpu_t *cpu)
{
	ASSERT(cpuid_checkpass((cpu == NULL ? CPU : cpu), CPUID_PASS_BASIC));

#if !defined(__xpv)
	if (cpu == NULL)
		cpu = CPU;

	/*CSTYLED*/
	{
		struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;

		if ((cpi->cpi_vendor == X86_VENDOR_AMD ||
		    cpi->cpi_vendor == X86_VENDOR_HYGON) &&
		    cpi->cpi_xmaxeax >= 0x80000001 &&
		    (CPI_FEATURES_XTD_EDX(cpi) & CPUID_AMD_EDX_SYSC))
			return (1);
	}
#endif
	return (0);
}

int
cpuid_getidstr(cpu_t *cpu, char *s, size_t n)
{
	struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;

	static const char fmt[] =
	    "x86 (%s %X family %d model %d step %d clock %d MHz)";
	static const char fmt_ht[] =
	    "x86 (chipid 0x%x %s %X family %d model %d step %d clock %d MHz)";

	ASSERT(cpuid_checkpass(cpu, CPUID_PASS_BASIC));

	if (cpuid_is_cmt(cpu))
		return (snprintf(s, n, fmt_ht, cpi->cpi_chipid,
		    cpi->cpi_vendorstr, cpi->cpi_std[1].cp_eax,
		    cpi->cpi_family, cpi->cpi_model,
		    cpi->cpi_step, cpu->cpu_type_info.pi_clock));
	return (snprintf(s, n, fmt,
	    cpi->cpi_vendorstr, cpi->cpi_std[1].cp_eax,
	    cpi->cpi_family, cpi->cpi_model,
	    cpi->cpi_step, cpu->cpu_type_info.pi_clock));
}

const char *
cpuid_getvendorstr(cpu_t *cpu)
{
	ASSERT(cpuid_checkpass(cpu, CPUID_PASS_IDENT));
	return ((const char *)cpu->cpu_m.mcpu_cpi->cpi_vendorstr);
}

uint_t
cpuid_getvendor(cpu_t *cpu)
{
	ASSERT(cpuid_checkpass(cpu, CPUID_PASS_IDENT));
	return (cpu->cpu_m.mcpu_cpi->cpi_vendor);
}

uint_t
cpuid_getfamily(cpu_t *cpu)
{
	ASSERT(cpuid_checkpass(cpu, CPUID_PASS_IDENT));
	return (cpu->cpu_m.mcpu_cpi->cpi_family);
}

uint_t
cpuid_getmodel(cpu_t *cpu)
{
	ASSERT(cpuid_checkpass(cpu, CPUID_PASS_IDENT));
	return (cpu->cpu_m.mcpu_cpi->cpi_model);
}

uint_t
cpuid_get_ncpu_per_chip(cpu_t *cpu)
{
	ASSERT(cpuid_checkpass(cpu, CPUID_PASS_BASIC));
	return (cpu->cpu_m.mcpu_cpi->cpi_ncpu_per_chip);
}

uint_t
cpuid_get_ncore_per_chip(cpu_t *cpu)
{
	ASSERT(cpuid_checkpass(cpu, CPUID_PASS_BASIC));
	return (cpu->cpu_m.mcpu_cpi->cpi_ncore_per_chip);
}

uint_t
cpuid_get_ncpu_sharing_last_cache(cpu_t *cpu)
{
	ASSERT(cpuid_checkpass(cpu, CPUID_PASS_EXTENDED));
	return (cpu->cpu_m.mcpu_cpi->cpi_ncpu_shr_last_cache);
}

id_t
cpuid_get_last_lvl_cacheid(cpu_t *cpu)
{
	ASSERT(cpuid_checkpass(cpu, CPUID_PASS_EXTENDED));
	return (cpu->cpu_m.mcpu_cpi->cpi_last_lvl_cacheid);
}

uint_t
cpuid_getstep(cpu_t *cpu)
{
	ASSERT(cpuid_checkpass(cpu, CPUID_PASS_IDENT));
	return (cpu->cpu_m.mcpu_cpi->cpi_step);
}

uint_t
cpuid_getsig(struct cpu *cpu)
{
	ASSERT(cpuid_checkpass(cpu, CPUID_PASS_IDENT));
	return (cpu->cpu_m.mcpu_cpi->cpi_std[1].cp_eax);
}

x86_chiprev_t
cpuid_getchiprev(struct cpu *cpu)
{
	ASSERT(cpuid_checkpass(cpu, CPUID_PASS_IDENT));
	return (cpu->cpu_m.mcpu_cpi->cpi_chiprev);
}

const char *
cpuid_getchiprevstr(struct cpu *cpu)
{
	ASSERT(cpuid_checkpass(cpu, CPUID_PASS_IDENT));
	return (cpu->cpu_m.mcpu_cpi->cpi_chiprevstr);
}

uint32_t
cpuid_getsockettype(struct cpu *cpu)
{
	ASSERT(cpuid_checkpass(cpu, CPUID_PASS_IDENT));
	return (cpu->cpu_m.mcpu_cpi->cpi_socket);
}

const char *
cpuid_getsocketstr(cpu_t *cpu)
{
	static const char *socketstr = NULL;
	struct cpuid_info *cpi;

	ASSERT(cpuid_checkpass(cpu, CPUID_PASS_IDENT));
	cpi = cpu->cpu_m.mcpu_cpi;

	/* Assume that socket types are the same across the system */
	if (socketstr == NULL)
		socketstr = _cpuid_sktstr(cpi->cpi_vendor, cpi->cpi_family,
		    cpi->cpi_model, cpi->cpi_step);


	return (socketstr);
}

x86_uarchrev_t
cpuid_getuarchrev(cpu_t *cpu)
{
	return (cpu->cpu_m.mcpu_cpi->cpi_uarchrev);
}

int
cpuid_get_chipid(cpu_t *cpu)
{
	ASSERT(cpuid_checkpass(cpu, CPUID_PASS_BASIC));

	if (cpuid_is_cmt(cpu))
		return (cpu->cpu_m.mcpu_cpi->cpi_chipid);
	return (cpu->cpu_id);
}

id_t
cpuid_get_coreid(cpu_t *cpu)
{
	ASSERT(cpuid_checkpass(cpu, CPUID_PASS_BASIC));
	return (cpu->cpu_m.mcpu_cpi->cpi_coreid);
}

int
cpuid_get_pkgcoreid(cpu_t *cpu)
{
	ASSERT(cpuid_checkpass(cpu, CPUID_PASS_BASIC));
	return (cpu->cpu_m.mcpu_cpi->cpi_pkgcoreid);
}

int
cpuid_get_clogid(cpu_t *cpu)
{
	ASSERT(cpuid_checkpass(cpu, CPUID_PASS_BASIC));
	return (cpu->cpu_m.mcpu_cpi->cpi_clogid);
}

int
cpuid_get_cacheid(cpu_t *cpu)
{
	ASSERT(cpuid_checkpass(cpu, CPUID_PASS_BASIC));
	return (cpu->cpu_m.mcpu_cpi->cpi_last_lvl_cacheid);
}

uint_t
cpuid_get_procnodeid(cpu_t *cpu)
{
	ASSERT(cpuid_checkpass(cpu, CPUID_PASS_BASIC));
	return (cpu->cpu_m.mcpu_cpi->cpi_procnodeid);
}

uint_t
cpuid_get_procnodes_per_pkg(cpu_t *cpu)
{
	ASSERT(cpuid_checkpass(cpu, CPUID_PASS_BASIC));
	return (cpu->cpu_m.mcpu_cpi->cpi_procnodes_per_pkg);
}

uint_t
cpuid_get_compunitid(cpu_t *cpu)
{
	ASSERT(cpuid_checkpass(cpu, CPUID_PASS_BASIC));
	return (cpu->cpu_m.mcpu_cpi->cpi_compunitid);
}

uint_t
cpuid_get_cores_per_compunit(cpu_t *cpu)
{
	ASSERT(cpuid_checkpass(cpu, CPUID_PASS_BASIC));
	return (cpu->cpu_m.mcpu_cpi->cpi_cores_per_compunit);
}

uint32_t
cpuid_get_apicid(cpu_t *cpu)
{
	ASSERT(cpuid_checkpass(cpu, CPUID_PASS_BASIC));
	if (cpu->cpu_m.mcpu_cpi->cpi_maxeax < 1) {
		return (UINT32_MAX);
	} else {
		return (cpu->cpu_m.mcpu_cpi->cpi_apicid);
	}
}

void
cpuid_get_addrsize(cpu_t *cpu, uint_t *pabits, uint_t *vabits)
{
	struct cpuid_info *cpi;

	if (cpu == NULL)
		cpu = CPU;
	cpi = cpu->cpu_m.mcpu_cpi;

	ASSERT(cpuid_checkpass(cpu, CPUID_PASS_BASIC));

	if (pabits)
		*pabits = cpi->cpi_pabits;
	if (vabits)
		*vabits = cpi->cpi_vabits;
}

/*
 * Export information about known offsets to the kernel. We only care about
 * things we have actually enabled support for in %xcr0.
 */
void
cpuid_get_xsave_info(uint64_t bit, size_t *sizep, size_t *offp)
{
	size_t size, off;

	VERIFY3U(bit & xsave_bv_all, !=, 0);

	if (sizep == NULL)
		sizep = &size;
	if (offp == NULL)
		offp = &off;

	switch (bit) {
	case XFEATURE_LEGACY_FP:
	case XFEATURE_SSE:
		*sizep = sizeof (struct fxsave_state);
		*offp = 0;
		break;
	case XFEATURE_AVX:
		*sizep = cpuid_info0.cpi_xsave.ymm_size;
		*offp = cpuid_info0.cpi_xsave.ymm_offset;
		break;
	case XFEATURE_AVX512_OPMASK:
		*sizep = cpuid_info0.cpi_xsave.opmask_size;
		*offp = cpuid_info0.cpi_xsave.opmask_offset;
		break;
	case XFEATURE_AVX512_ZMM:
		*sizep = cpuid_info0.cpi_xsave.zmmlo_size;
		*offp = cpuid_info0.cpi_xsave.zmmlo_offset;
		break;
	case XFEATURE_AVX512_HI_ZMM:
		*sizep = cpuid_info0.cpi_xsave.zmmhi_size;
		*offp = cpuid_info0.cpi_xsave.zmmhi_offset;
		break;
	default:
		panic("asked for unsupported xsave feature: 0x%lx", bit);
	}
}

/*
 * Use our supported-features indicators (xsave_bv_all) to return the XSAVE
 * size of our supported-features that need saving. Some CPUs' maximum save
 * size (stored in cpuid_info0.cpi_xsave.xsav_max_size) includes
 * unsupported-by-us features (e.g. Intel AMX) which we MAY be able to safely
 * dismiss if the supported XSAVE data's offset + length are before the
 * unsupported feature.
 */
size_t
cpuid_get_xsave_size(void)
{
	size_t furthest_out = sizeof (struct xsave_state);
	uint_t shift = 0;

	VERIFY(xsave_bv_all != 0);

	for (uint64_t current = xsave_bv_all; current != 0;
	    current >>= 1, shift++) {
		uint64_t testbit = 1UL << shift;
		size_t size, offset;

		if ((testbit & xsave_bv_all) == 0)
			continue;

		cpuid_get_xsave_info(testbit, &size, &offset);
		furthest_out = MAX(furthest_out, offset + size);
	}

	return (furthest_out);
}

/*
 * Return true if the CPUs on this system require 'pointer clearing' for the
 * floating point error pointer exception handling. In the past, this has been
 * true for all AMD K7 & K8 CPUs, although newer AMD CPUs have been changed to
 * behave the same as Intel. This is checked via the CPUID_AMD_EBX_ERR_PTR_ZERO
 * feature bit and is reflected in the cpi_fp_amd_save member.
 */
boolean_t
cpuid_need_fp_excp_handling(void)
{
	return (cpuid_info0.cpi_vendor == X86_VENDOR_AMD &&
	    cpuid_info0.cpi_fp_amd_save != 0);
}

/*
 * Returns the number of data TLB entries for a corresponding
 * pagesize.  If it can't be computed, or isn't known, the
 * routine returns zero.  If you ask about an architecturally
 * impossible pagesize, the routine will panic (so that the
 * hat implementor knows that things are inconsistent.)
 */
uint_t
cpuid_get_dtlb_nent(cpu_t *cpu, size_t pagesize)
{
	struct cpuid_info *cpi;
	uint_t dtlb_nent = 0;

	if (cpu == NULL)
		cpu = CPU;
	cpi = cpu->cpu_m.mcpu_cpi;

	ASSERT(cpuid_checkpass(cpu, CPUID_PASS_BASIC));

	/*
	 * Check the L2 TLB info
	 */
	if (cpi->cpi_xmaxeax >= 0x80000006) {
		struct cpuid_regs *cp = &cpi->cpi_extd[6];

		switch (pagesize) {

		case 4 * 1024:
			/*
			 * All zero in the top 16 bits of the register
			 * indicates a unified TLB. Size is in low 16 bits.
			 */
			if ((cp->cp_ebx & 0xffff0000) == 0)
				dtlb_nent = cp->cp_ebx & 0x0000ffff;
			else
				dtlb_nent = BITX(cp->cp_ebx, 27, 16);
			break;

		case 2 * 1024 * 1024:
			if ((cp->cp_eax & 0xffff0000) == 0)
				dtlb_nent = cp->cp_eax & 0x0000ffff;
			else
				dtlb_nent = BITX(cp->cp_eax, 27, 16);
			break;

		default:
			panic("unknown L2 pagesize");
			/*NOTREACHED*/
		}
	}

	if (dtlb_nent != 0)
		return (dtlb_nent);

	/*
	 * No L2 TLB support for this size, try L1.
	 */
	if (cpi->cpi_xmaxeax >= 0x80000005) {
		struct cpuid_regs *cp = &cpi->cpi_extd[5];

		switch (pagesize) {
		case 4 * 1024:
			dtlb_nent = BITX(cp->cp_ebx, 23, 16);
			break;
		case 2 * 1024 * 1024:
			dtlb_nent = BITX(cp->cp_eax, 23, 16);
			break;
		default:
			panic("unknown L1 d-TLB pagesize");
			/*NOTREACHED*/
		}
	}

	return (dtlb_nent);
}

/*
 * Return 0 if the erratum is not present or not applicable, positive
 * if it is, and negative if the status of the erratum is unknown.
 *
 * See "Revision Guide for AMD Athlon(tm) 64 and AMD Opteron(tm)
 * Processors" #25759, Rev 3.57, August 2005
 */
int
cpuid_opteron_erratum(cpu_t *cpu, uint_t erratum)
{
	struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
	uint_t eax;

	/*
	 * Bail out if this CPU isn't an AMD CPU, or if it's
	 * a legacy (32-bit) AMD CPU.
	 */
	if (cpi->cpi_vendor != X86_VENDOR_AMD ||
	    cpi->cpi_family == 4 || cpi->cpi_family == 5 ||
	    cpi->cpi_family == 6) {
		return (0);
	}

	eax = cpi->cpi_std[1].cp_eax;

#define	SH_B0(eax)	(eax == 0xf40 || eax == 0xf50)
#define	SH_B3(eax)	(eax == 0xf51)
#define	B(eax)		(SH_B0(eax) || SH_B3(eax))

#define	SH_C0(eax)	(eax == 0xf48 || eax == 0xf58)

#define	SH_CG(eax)	(eax == 0xf4a || eax == 0xf5a || eax == 0xf7a)
#define	DH_CG(eax)	(eax == 0xfc0 || eax == 0xfe0 || eax == 0xff0)
#define	CH_CG(eax)	(eax == 0xf82 || eax == 0xfb2)
#define	CG(eax)		(SH_CG(eax) || DH_CG(eax) || CH_CG(eax))

#define	SH_D0(eax)	(eax == 0x10f40 || eax == 0x10f50 || eax == 0x10f70)
#define	DH_D0(eax)	(eax == 0x10fc0 || eax == 0x10ff0)
#define	CH_D0(eax)	(eax == 0x10f80 || eax == 0x10fb0)
#define	D0(eax)		(SH_D0(eax) || DH_D0(eax) || CH_D0(eax))

#define	SH_E0(eax)	(eax == 0x20f50 || eax == 0x20f40 || eax == 0x20f70)
#define	JH_E1(eax)	(eax == 0x20f10)	/* JH8_E0 had 0x20f30 */
#define	DH_E3(eax)	(eax == 0x20fc0 || eax == 0x20ff0)
#define	SH_E4(eax)	(eax == 0x20f51 || eax == 0x20f71)
#define	BH_E4(eax)	(eax == 0x20fb1)
#define	SH_E5(eax)	(eax == 0x20f42)
#define	DH_E6(eax)	(eax == 0x20ff2 || eax == 0x20fc2)
#define	JH_E6(eax)	(eax == 0x20f12 || eax == 0x20f32)
#define	EX(eax)		(SH_E0(eax) || JH_E1(eax) || DH_E3(eax) || \
			    SH_E4(eax) || BH_E4(eax) || SH_E5(eax) || \
			    DH_E6(eax) || JH_E6(eax))

#define	DR_AX(eax)	(eax == 0x100f00 || eax == 0x100f01 || eax == 0x100f02)
#define	DR_B0(eax)	(eax == 0x100f20)
#define	DR_B1(eax)	(eax == 0x100f21)
#define	DR_BA(eax)	(eax == 0x100f2a)
#define	DR_B2(eax)	(eax == 0x100f22)
#define	DR_B3(eax)	(eax == 0x100f23)
#define	RB_C0(eax)	(eax == 0x100f40)

	switch (erratum) {
	case 1:
		return (cpi->cpi_family < 0x10);
	case 51:	/* what does the asterisk mean? */
		return (B(eax) || SH_C0(eax) || CG(eax));
	case 52:
		return (B(eax));
	case 57:
		return (cpi->cpi_family <= 0x11);
	case 58:
		return (B(eax));
	case 60:
		return (cpi->cpi_family <= 0x11);
	case 61:
	case 62:
	case 63:
	case 64:
	case 65:
	case 66:
	case 68:
	case 69:
	case 70:
	case 71:
		return (B(eax));
	case 72:
		return (SH_B0(eax));
	case 74:
		return (B(eax));
	case 75:
		return (cpi->cpi_family < 0x10);
	case 76:
		return (B(eax));
	case 77:
		return (cpi->cpi_family <= 0x11);
	case 78:
		return (B(eax) || SH_C0(eax));
	case 79:
		return (B(eax) || SH_C0(eax) || CG(eax) || D0(eax) || EX(eax));
	case 80:
	case 81:
	case 82:
		return (B(eax));
	case 83:
		return (B(eax) || SH_C0(eax) || CG(eax));
	case 85:
		return (cpi->cpi_family < 0x10);
	case 86:
		return (SH_C0(eax) || CG(eax));
	case 88:
		return (B(eax) || SH_C0(eax));
	case 89:
		return (cpi->cpi_family < 0x10);
	case 90:
		return (B(eax) || SH_C0(eax) || CG(eax));
	case 91:
	case 92:
		return (B(eax) || SH_C0(eax));
	case 93:
		return (SH_C0(eax));
	case 94:
		return (B(eax) || SH_C0(eax) || CG(eax));
	case 95:
		return (B(eax) || SH_C0(eax));
	case 96:
		return (B(eax) || SH_C0(eax) || CG(eax));
	case 97:
	case 98:
		return (SH_C0(eax) || CG(eax));
	case 99:
		return (B(eax) || SH_C0(eax) || CG(eax) || D0(eax));
	case 100:
		return (B(eax) || SH_C0(eax));
	case 101:
	case 103:
		return (B(eax) || SH_C0(eax) || CG(eax) || D0(eax));
	case 104:
		return (SH_C0(eax) || CG(eax) || D0(eax));
	case 105:
	case 106:
	case 107:
		return (B(eax) || SH_C0(eax) || CG(eax) || D0(eax));
	case 108:
		return (DH_CG(eax));
	case 109:
		return (SH_C0(eax) || CG(eax) || D0(eax));
	case 110:
		return (D0(eax) || EX(eax));
	case 111:
		return (CG(eax));
	case 112:
		return (B(eax) || SH_C0(eax) || CG(eax) || D0(eax) || EX(eax));
	case 113:
		return (eax == 0x20fc0);
	case 114:
		return (SH_E0(eax) || JH_E1(eax) || DH_E3(eax));
	case 115:
		return (SH_E0(eax) || JH_E1(eax));
	case 116:
		return (SH_E0(eax) || JH_E1(eax) || DH_E3(eax));
	case 117:
		return (B(eax) || SH_C0(eax) || CG(eax) || D0(eax));
	case 118:
		return (SH_E0(eax) || JH_E1(eax) || SH_E4(eax) || BH_E4(eax) ||
		    JH_E6(eax));
	case 121:
		return (B(eax) || SH_C0(eax) || CG(eax) || D0(eax) || EX(eax));
	case 122:
		return (cpi->cpi_family < 0x10 || cpi->cpi_family == 0x11);
	case 123:
		return (JH_E1(eax) || BH_E4(eax) || JH_E6(eax));
	case 131:
		return (cpi->cpi_family < 0x10);
	case 6336786:

		/*
		 * Test for AdvPowerMgmtInfo.TscPStateInvariant
		 * if this is a K8 family or newer processor. We're testing for
		 * this 'erratum' to determine whether or not we have a constant
		 * TSC.
		 *
		 * Our current fix for this is to disable the C1-Clock ramping.
		 * However, this doesn't work on newer processor families nor
		 * does it work when virtualized as those devices don't exist.
		 */
		if (cpi->cpi_family >= 0x12 || get_hwenv() != HW_NATIVE) {
			return (0);
		}

		if (CPI_FAMILY(cpi) == 0xf) {
			struct cpuid_regs regs;
			regs.cp_eax = 0x80000007;
			(void) __cpuid_insn(&regs);
			return (!(regs.cp_edx & 0x100));
		}
		return (0);
	case 147:
		/*
		 * This erratum (K8 #147) is not present on family 10 and newer.
		 */
		if (cpi->cpi_family >= 0x10) {
			return (0);
		}
		return (((((eax >> 12) & 0xff00) + (eax & 0xf00)) |
		    (((eax >> 4) & 0xf) | ((eax >> 12) & 0xf0))) < 0xf40);

	case 6671130:
		/*
		 * check for processors (pre-Shanghai) that do not provide
		 * optimal management of 1gb ptes in its tlb.
		 */
		return (cpi->cpi_family == 0x10 && cpi->cpi_model < 4);

	case 298:
		return (DR_AX(eax) || DR_B0(eax) || DR_B1(eax) || DR_BA(eax) ||
		    DR_B2(eax) || RB_C0(eax));

	case 721:
		return (cpi->cpi_family == 0x10 || cpi->cpi_family == 0x12);

	default:
		return (-1);

	}
}

/*
 * Determine if specified erratum is present via OSVW (OS Visible Workaround).
 * Return 1 if erratum is present, 0 if not present and -1 if indeterminate.
 */
int
osvw_opteron_erratum(cpu_t *cpu, uint_t erratum)
{
	struct cpuid_info	*cpi;
	uint_t			osvwid;
	static int		osvwfeature = -1;
	uint64_t		osvwlength;


	cpi = cpu->cpu_m.mcpu_cpi;

	/* confirm OSVW supported */
	if (osvwfeature == -1) {
		osvwfeature = cpi->cpi_extd[1].cp_ecx & CPUID_AMD_ECX_OSVW;
	} else {
		/* assert that osvw feature setting is consistent on all cpus */
		ASSERT(osvwfeature ==
		    (cpi->cpi_extd[1].cp_ecx & CPUID_AMD_ECX_OSVW));
	}
	if (!osvwfeature)
		return (-1);

	osvwlength = rdmsr(MSR_AMD_OSVW_ID_LEN) & OSVW_ID_LEN_MASK;

	switch (erratum) {
	case 298:	/* osvwid is 0 */
		osvwid = 0;
		if (osvwlength <= (uint64_t)osvwid) {
			/* osvwid 0 is unknown */
			return (-1);
		}

		/*
		 * Check the OSVW STATUS MSR to determine the state
		 * of the erratum where:
		 *   0 - fixed by HW
		 *   1 - BIOS has applied the workaround when BIOS
		 *   workaround is available. (Or for other errata,
		 *   OS workaround is required.)
		 * For a value of 1, caller will confirm that the
		 * erratum 298 workaround has indeed been applied by BIOS.
		 *
		 * A 1 may be set in cpus that have a HW fix
		 * in a mixed cpu system. Regarding erratum 298:
		 *   In a multiprocessor platform, the workaround above
		 *   should be applied to all processors regardless of
		 *   silicon revision when an affected processor is
		 *   present.
		 */

		return (rdmsr(MSR_AMD_OSVW_STATUS +
		    (osvwid / OSVW_ID_CNT_PER_MSR)) &
		    (1ULL << (osvwid % OSVW_ID_CNT_PER_MSR)));

	default:
		return (-1);
	}
}

static const char assoc_str[] = "associativity";
static const char line_str[] = "line-size";
static const char size_str[] = "size";

static void
add_cache_prop(dev_info_t *devi, const char *label, const char *type,
    uint32_t val)
{
	char buf[128];

	/*
	 * ndi_prop_update_int() is used because it is desirable for
	 * DDI_PROP_HW_DEF and DDI_PROP_DONTSLEEP to be set.
	 */
	if (snprintf(buf, sizeof (buf), "%s-%s", label, type) < sizeof (buf))
		(void) ndi_prop_update_int(DDI_DEV_T_NONE, devi, buf, val);
}

/*
 * Intel-style cache/tlb description
 *
 * Standard cpuid level 2 gives a randomly ordered
 * selection of tags that index into a table that describes
 * cache and tlb properties.
 */

static const char l1_icache_str[] = "l1-icache";
static const char l1_dcache_str[] = "l1-dcache";
static const char l2_cache_str[] = "l2-cache";
static const char l3_cache_str[] = "l3-cache";
static const char itlb4k_str[] = "itlb-4K";
static const char dtlb4k_str[] = "dtlb-4K";
static const char itlb2M_str[] = "itlb-2M";
static const char itlb4M_str[] = "itlb-4M";
static const char dtlb4M_str[] = "dtlb-4M";
static const char dtlb24_str[] = "dtlb0-2M-4M";
static const char itlb424_str[] = "itlb-4K-2M-4M";
static const char itlb24_str[] = "itlb-2M-4M";
static const char dtlb44_str[] = "dtlb-4K-4M";
static const char sl1_dcache_str[] = "sectored-l1-dcache";
static const char sl2_cache_str[] = "sectored-l2-cache";
static const char itrace_str[] = "itrace-cache";
static const char sl3_cache_str[] = "sectored-l3-cache";
static const char sh_l2_tlb4k_str[] = "shared-l2-tlb-4k";

static const struct cachetab {
	uint8_t		ct_code;
	uint8_t		ct_assoc;
	uint16_t	ct_line_size;
	size_t		ct_size;
	const char	*ct_label;
} intel_ctab[] = {
	/*
	 * maintain descending order!
	 *
	 * Codes ignored - Reason
	 * ----------------------
	 * 40H - intel_cpuid_4_cache_info() disambiguates l2/l3 cache
	 * f0H/f1H - Currently we do not interpret prefetch size by design
	 */
	{ 0xe4, 16, 64, 8*1024*1024, l3_cache_str},
	{ 0xe3, 16, 64, 4*1024*1024, l3_cache_str},
	{ 0xe2, 16, 64, 2*1024*1024, l3_cache_str},
	{ 0xde, 12, 64, 6*1024*1024, l3_cache_str},
	{ 0xdd, 12, 64, 3*1024*1024, l3_cache_str},
	{ 0xdc, 12, 64, ((1*1024*1024)+(512*1024)), l3_cache_str},
	{ 0xd8, 8, 64, 4*1024*1024, l3_cache_str},
	{ 0xd7, 8, 64, 2*1024*1024, l3_cache_str},
	{ 0xd6, 8, 64, 1*1024*1024, l3_cache_str},
	{ 0xd2, 4, 64, 2*1024*1024, l3_cache_str},
	{ 0xd1, 4, 64, 1*1024*1024, l3_cache_str},
	{ 0xd0, 4, 64, 512*1024, l3_cache_str},
	{ 0xca, 4, 0, 512, sh_l2_tlb4k_str},
	{ 0xc0, 4, 0, 8, dtlb44_str },
	{ 0xba, 4, 0, 64, dtlb4k_str },
	{ 0xb4, 4, 0, 256, dtlb4k_str },
	{ 0xb3, 4, 0, 128, dtlb4k_str },
	{ 0xb2, 4, 0, 64, itlb4k_str },
	{ 0xb0, 4, 0, 128, itlb4k_str },
	{ 0x87, 8, 64, 1024*1024, l2_cache_str},
	{ 0x86, 4, 64, 512*1024, l2_cache_str},
	{ 0x85, 8, 32, 2*1024*1024, l2_cache_str},
	{ 0x84, 8, 32, 1024*1024, l2_cache_str},
	{ 0x83, 8, 32, 512*1024, l2_cache_str},
	{ 0x82, 8, 32, 256*1024, l2_cache_str},
	{ 0x80, 8, 64, 512*1024, l2_cache_str},
	{ 0x7f, 2, 64, 512*1024, l2_cache_str},
	{ 0x7d, 8, 64, 2*1024*1024, sl2_cache_str},
	{ 0x7c, 8, 64, 1024*1024, sl2_cache_str},
	{ 0x7b, 8, 64, 512*1024, sl2_cache_str},
	{ 0x7a, 8, 64, 256*1024, sl2_cache_str},
	{ 0x79, 8, 64, 128*1024, sl2_cache_str},
	{ 0x78, 8, 64, 1024*1024, l2_cache_str},
	{ 0x73, 8, 0, 64*1024, itrace_str},
	{ 0x72, 8, 0, 32*1024, itrace_str},
	{ 0x71, 8, 0, 16*1024, itrace_str},
	{ 0x70, 8, 0, 12*1024, itrace_str},
	{ 0x68, 4, 64, 32*1024, sl1_dcache_str},
	{ 0x67, 4, 64, 16*1024, sl1_dcache_str},
	{ 0x66, 4, 64, 8*1024, sl1_dcache_str},
	{ 0x60, 8, 64, 16*1024, sl1_dcache_str},
	{ 0x5d, 0, 0, 256, dtlb44_str},
	{ 0x5c, 0, 0, 128, dtlb44_str},
	{ 0x5b, 0, 0, 64, dtlb44_str},
	{ 0x5a, 4, 0, 32, dtlb24_str},
	{ 0x59, 0, 0, 16, dtlb4k_str},
	{ 0x57, 4, 0, 16, dtlb4k_str},
	{ 0x56, 4, 0, 16, dtlb4M_str},
	{ 0x55, 0, 0, 7, itlb24_str},
	{ 0x52, 0, 0, 256, itlb424_str},
	{ 0x51, 0, 0, 128, itlb424_str},
	{ 0x50, 0, 0, 64, itlb424_str},
	{ 0x4f, 0, 0, 32, itlb4k_str},
	{ 0x4e, 24, 64, 6*1024*1024, l2_cache_str},
	{ 0x4d, 16, 64, 16*1024*1024, l3_cache_str},
	{ 0x4c, 12, 64, 12*1024*1024, l3_cache_str},
	{ 0x4b, 16, 64, 8*1024*1024, l3_cache_str},
	{ 0x4a, 12, 64, 6*1024*1024, l3_cache_str},
	{ 0x49, 16, 64, 4*1024*1024, l3_cache_str},
	{ 0x48, 12, 64, 3*1024*1024, l2_cache_str},
	{ 0x47, 8, 64, 8*1024*1024, l3_cache_str},
	{ 0x46, 4, 64, 4*1024*1024, l3_cache_str},
	{ 0x45, 4, 32, 2*1024*1024, l2_cache_str},
	{ 0x44, 4, 32, 1024*1024, l2_cache_str},
	{ 0x43, 4, 32, 512*1024, l2_cache_str},
	{ 0x42, 4, 32, 256*1024, l2_cache_str},
	{ 0x41, 4, 32, 128*1024, l2_cache_str},
	{ 0x3e, 4, 64, 512*1024, sl2_cache_str},
	{ 0x3d, 6, 64, 384*1024, sl2_cache_str},
	{ 0x3c, 4, 64, 256*1024, sl2_cache_str},
	{ 0x3b, 2, 64, 128*1024, sl2_cache_str},
	{ 0x3a, 6, 64, 192*1024, sl2_cache_str},
	{ 0x39, 4, 64, 128*1024, sl2_cache_str},
	{ 0x30, 8, 64, 32*1024, l1_icache_str},
	{ 0x2c, 8, 64, 32*1024, l1_dcache_str},
	{ 0x29, 8, 64, 4096*1024, sl3_cache_str},
	{ 0x25, 8, 64, 2048*1024, sl3_cache_str},
	{ 0x23, 8, 64, 1024*1024, sl3_cache_str},
	{ 0x22, 4, 64, 512*1024, sl3_cache_str},
	{ 0x0e, 6, 64, 24*1024, l1_dcache_str},
	{ 0x0d, 4, 32, 16*1024, l1_dcache_str},
	{ 0x0c, 4, 32, 16*1024, l1_dcache_str},
	{ 0x0b, 4, 0, 4, itlb4M_str},
	{ 0x0a, 2, 32, 8*1024, l1_dcache_str},
	{ 0x08, 4, 32, 16*1024, l1_icache_str},
	{ 0x06, 4, 32, 8*1024, l1_icache_str},
	{ 0x05, 4, 0, 32, dtlb4M_str},
	{ 0x04, 4, 0, 8, dtlb4M_str},
	{ 0x03, 4, 0, 64, dtlb4k_str},
	{ 0x02, 4, 0, 2, itlb4M_str},
	{ 0x01, 4, 0, 32, itlb4k_str},
	{ 0 }
};

static const struct cachetab cyrix_ctab[] = {
	{ 0x70, 4, 0, 32, "tlb-4K" },
	{ 0x80, 4, 16, 16*1024, "l1-cache" },
	{ 0 }
};

/*
 * Search a cache table for a matching entry
 */
static const struct cachetab *
find_cacheent(const struct cachetab *ct, uint_t code)
{
	if (code != 0) {
		for (; ct->ct_code != 0; ct++)
			if (ct->ct_code <= code)
				break;
		if (ct->ct_code == code)
			return (ct);
	}
	return (NULL);
}

/*
 * Populate cachetab entry with L2 or L3 cache-information using
 * cpuid function 4. This function is called from intel_walk_cacheinfo()
 * when descriptor 0x49 is encountered. It returns 0 if no such cache
 * information is found.
 */
static int
intel_cpuid_4_cache_info(struct cachetab *ct, struct cpuid_info *cpi)
{
	uint32_t level, i;
	int ret = 0;

	for (i = 0; i < cpi->cpi_cache_leaf_size; i++) {
		level = CPI_CACHE_LVL(cpi->cpi_cache_leaves[i]);

		if (level == 2 || level == 3) {
			ct->ct_assoc =
			    CPI_CACHE_WAYS(cpi->cpi_cache_leaves[i]) + 1;
			ct->ct_line_size =
			    CPI_CACHE_COH_LN_SZ(cpi->cpi_cache_leaves[i]) + 1;
			ct->ct_size = ct->ct_assoc *
			    (CPI_CACHE_PARTS(cpi->cpi_cache_leaves[i]) + 1) *
			    ct->ct_line_size *
			    (cpi->cpi_cache_leaves[i]->cp_ecx + 1);

			if (level == 2) {
				ct->ct_label = l2_cache_str;
			} else if (level == 3) {
				ct->ct_label = l3_cache_str;
			}
			ret = 1;
		}
	}

	return (ret);
}

/*
 * Walk the cacheinfo descriptor, applying 'func' to every valid element
 * The walk is terminated if the walker returns non-zero.
 */
static void
intel_walk_cacheinfo(struct cpuid_info *cpi,
    void *arg, int (*func)(void *, const struct cachetab *))
{
	const struct cachetab *ct;
	struct cachetab des_49_ct, des_b1_ct;
	uint8_t *dp;
	int i;

	if ((dp = cpi->cpi_cacheinfo) == NULL)
		return;
	for (i = 0; i < cpi->cpi_ncache; i++, dp++) {
		/*
		 * For overloaded descriptor 0x49 we use cpuid function 4
		 * if supported by the current processor, to create
		 * cache information.
		 * For overloaded descriptor 0xb1 we use X86_PAE flag
		 * to disambiguate the cache information.
		 */
		if (*dp == 0x49 && cpi->cpi_maxeax >= 0x4 &&
		    intel_cpuid_4_cache_info(&des_49_ct, cpi) == 1) {
				ct = &des_49_ct;
		} else if (*dp == 0xb1) {
			des_b1_ct.ct_code = 0xb1;
			des_b1_ct.ct_assoc = 4;
			des_b1_ct.ct_line_size = 0;
			if (is_x86_feature(x86_featureset, X86FSET_PAE)) {
				des_b1_ct.ct_size = 8;
				des_b1_ct.ct_label = itlb2M_str;
			} else {
				des_b1_ct.ct_size = 4;
				des_b1_ct.ct_label = itlb4M_str;
			}
			ct = &des_b1_ct;
		} else {
			if ((ct = find_cacheent(intel_ctab, *dp)) == NULL) {
				continue;
			}
		}

		if (func(arg, ct) != 0) {
			break;
		}
	}
}

/*
 * (Like the Intel one, except for Cyrix CPUs)
 */
static void
cyrix_walk_cacheinfo(struct cpuid_info *cpi,
    void *arg, int (*func)(void *, const struct cachetab *))
{
	const struct cachetab *ct;
	uint8_t *dp;
	int i;

	if ((dp = cpi->cpi_cacheinfo) == NULL)
		return;
	for (i = 0; i < cpi->cpi_ncache; i++, dp++) {
		/*
		 * Search Cyrix-specific descriptor table first ..
		 */
		if ((ct = find_cacheent(cyrix_ctab, *dp)) != NULL) {
			if (func(arg, ct) != 0)
				break;
			continue;
		}
		/*
		 * .. else fall back to the Intel one
		 */
		if ((ct = find_cacheent(intel_ctab, *dp)) != NULL) {
			if (func(arg, ct) != 0)
				break;
			continue;
		}
	}
}

/*
 * A cacheinfo walker that adds associativity, line-size, and size properties
 * to the devinfo node it is passed as an argument.
 */
static int
add_cacheent_props(void *arg, const struct cachetab *ct)
{
	dev_info_t *devi = arg;

	add_cache_prop(devi, ct->ct_label, assoc_str, ct->ct_assoc);
	if (ct->ct_line_size != 0)
		add_cache_prop(devi, ct->ct_label, line_str,
		    ct->ct_line_size);
	add_cache_prop(devi, ct->ct_label, size_str, ct->ct_size);
	return (0);
}


static const char fully_assoc[] = "fully-associative?";

/*
 * AMD style cache/tlb description
 *
 * Extended functions 5 and 6 directly describe properties of
 * tlbs and various cache levels.
 */
static void
add_amd_assoc(dev_info_t *devi, const char *label, uint_t assoc)
{
	switch (assoc) {
	case 0:	/* reserved; ignore */
		break;
	default:
		add_cache_prop(devi, label, assoc_str, assoc);
		break;
	case 0xff:
		add_cache_prop(devi, label, fully_assoc, 1);
		break;
	}
}

static void
add_amd_tlb(dev_info_t *devi, const char *label, uint_t assoc, uint_t size)
{
	if (size == 0)
		return;
	add_cache_prop(devi, label, size_str, size);
	add_amd_assoc(devi, label, assoc);
}

static void
add_amd_cache(dev_info_t *devi, const char *label,
    uint_t size, uint_t assoc, uint_t lines_per_tag, uint_t line_size)
{
	if (size == 0 || line_size == 0)
		return;
	add_amd_assoc(devi, label, assoc);
	/*
	 * Most AMD parts have a sectored cache. Multiple cache lines are
	 * associated with each tag. A sector consists of all cache lines
	 * associated with a tag. For example, the AMD K6-III has a sector
	 * size of 2 cache lines per tag.
	 */
	if (lines_per_tag != 0)
		add_cache_prop(devi, label, "lines-per-tag", lines_per_tag);
	add_cache_prop(devi, label, line_str, line_size);
	add_cache_prop(devi, label, size_str, size * 1024);
}

static void
add_amd_l2_assoc(dev_info_t *devi, const char *label, uint_t assoc)
{
	switch (assoc) {
	case 0:	/* off */
		break;
	case 1:
	case 2:
	case 4:
		add_cache_prop(devi, label, assoc_str, assoc);
		break;
	case 6:
		add_cache_prop(devi, label, assoc_str, 8);
		break;
	case 8:
		add_cache_prop(devi, label, assoc_str, 16);
		break;
	case 0xf:
		add_cache_prop(devi, label, fully_assoc, 1);
		break;
	default: /* reserved; ignore */
		break;
	}
}

static void
add_amd_l2_tlb(dev_info_t *devi, const char *label, uint_t assoc, uint_t size)
{
	if (size == 0 || assoc == 0)
		return;
	add_amd_l2_assoc(devi, label, assoc);
	add_cache_prop(devi, label, size_str, size);
}

static void
add_amd_l2_cache(dev_info_t *devi, const char *label,
    uint_t size, uint_t assoc, uint_t lines_per_tag, uint_t line_size)
{
	if (size == 0 || assoc == 0 || line_size == 0)
		return;
	add_amd_l2_assoc(devi, label, assoc);
	if (lines_per_tag != 0)
		add_cache_prop(devi, label, "lines-per-tag", lines_per_tag);
	add_cache_prop(devi, label, line_str, line_size);
	add_cache_prop(devi, label, size_str, size * 1024);
}

static void
amd_cache_info(struct cpuid_info *cpi, dev_info_t *devi)
{
	struct cpuid_regs *cp;

	if (cpi->cpi_xmaxeax < 0x80000005)
		return;
	cp = &cpi->cpi_extd[5];

	/*
	 * 4M/2M L1 TLB configuration
	 *
	 * We report the size for 2M pages because AMD uses two
	 * TLB entries for one 4M page.
	 */
	add_amd_tlb(devi, "dtlb-2M",
	    BITX(cp->cp_eax, 31, 24), BITX(cp->cp_eax, 23, 16));
	add_amd_tlb(devi, "itlb-2M",
	    BITX(cp->cp_eax, 15, 8), BITX(cp->cp_eax, 7, 0));

	/*
	 * 4K L1 TLB configuration
	 */

	switch (cpi->cpi_vendor) {
		uint_t nentries;
	case X86_VENDOR_TM:
		if (cpi->cpi_family >= 5) {
			/*
			 * Crusoe processors have 256 TLB entries, but
			 * cpuid data format constrains them to only
			 * reporting 255 of them.
			 */
			if ((nentries = BITX(cp->cp_ebx, 23, 16)) == 255)
				nentries = 256;
			/*
			 * Crusoe processors also have a unified TLB
			 */
			add_amd_tlb(devi, "tlb-4K", BITX(cp->cp_ebx, 31, 24),
			    nentries);
			break;
		}
		/*FALLTHROUGH*/
	default:
		add_amd_tlb(devi, itlb4k_str,
		    BITX(cp->cp_ebx, 31, 24), BITX(cp->cp_ebx, 23, 16));
		add_amd_tlb(devi, dtlb4k_str,
		    BITX(cp->cp_ebx, 15, 8), BITX(cp->cp_ebx, 7, 0));
		break;
	}

	/*
	 * data L1 cache configuration
	 */

	add_amd_cache(devi, l1_dcache_str,
	    BITX(cp->cp_ecx, 31, 24), BITX(cp->cp_ecx, 23, 16),
	    BITX(cp->cp_ecx, 15, 8), BITX(cp->cp_ecx, 7, 0));

	/*
	 * code L1 cache configuration
	 */

	add_amd_cache(devi, l1_icache_str,
	    BITX(cp->cp_edx, 31, 24), BITX(cp->cp_edx, 23, 16),
	    BITX(cp->cp_edx, 15, 8), BITX(cp->cp_edx, 7, 0));

	if (cpi->cpi_xmaxeax < 0x80000006)
		return;
	cp = &cpi->cpi_extd[6];

	/* Check for a unified L2 TLB for large pages */

	if (BITX(cp->cp_eax, 31, 16) == 0)
		add_amd_l2_tlb(devi, "l2-tlb-2M",
		    BITX(cp->cp_eax, 15, 12), BITX(cp->cp_eax, 11, 0));
	else {
		add_amd_l2_tlb(devi, "l2-dtlb-2M",
		    BITX(cp->cp_eax, 31, 28), BITX(cp->cp_eax, 27, 16));
		add_amd_l2_tlb(devi, "l2-itlb-2M",
		    BITX(cp->cp_eax, 15, 12), BITX(cp->cp_eax, 11, 0));
	}

	/* Check for a unified L2 TLB for 4K pages */

	if (BITX(cp->cp_ebx, 31, 16) == 0) {
		add_amd_l2_tlb(devi, "l2-tlb-4K",
		    BITX(cp->cp_eax, 15, 12), BITX(cp->cp_eax, 11, 0));
	} else {
		add_amd_l2_tlb(devi, "l2-dtlb-4K",
		    BITX(cp->cp_eax, 31, 28), BITX(cp->cp_eax, 27, 16));
		add_amd_l2_tlb(devi, "l2-itlb-4K",
		    BITX(cp->cp_eax, 15, 12), BITX(cp->cp_eax, 11, 0));
	}

	add_amd_l2_cache(devi, l2_cache_str,
	    BITX(cp->cp_ecx, 31, 16), BITX(cp->cp_ecx, 15, 12),
	    BITX(cp->cp_ecx, 11, 8), BITX(cp->cp_ecx, 7, 0));
}

/*
 * There are two basic ways that the x86 world describes it cache
 * and tlb architecture - Intel's way and AMD's way.
 *
 * Return which flavor of cache architecture we should use
 */
static int
x86_which_cacheinfo(struct cpuid_info *cpi)
{
	switch (cpi->cpi_vendor) {
	case X86_VENDOR_Intel:
		if (cpi->cpi_maxeax >= 2)
			return (X86_VENDOR_Intel);
		break;
	case X86_VENDOR_AMD:
		/*
		 * The K5 model 1 was the first part from AMD that reported
		 * cache sizes via extended cpuid functions.
		 */
		if (cpi->cpi_family > 5 ||
		    (cpi->cpi_family == 5 && cpi->cpi_model >= 1))
			return (X86_VENDOR_AMD);
		break;
	case X86_VENDOR_HYGON:
		return (X86_VENDOR_AMD);
	case X86_VENDOR_TM:
		if (cpi->cpi_family >= 5)
			return (X86_VENDOR_AMD);
		/*FALLTHROUGH*/
	default:
		/*
		 * If they have extended CPU data for 0x80000005
		 * then we assume they have AMD-format cache
		 * information.
		 *
		 * If not, and the vendor happens to be Cyrix,
		 * then try our-Cyrix specific handler.
		 *
		 * If we're not Cyrix, then assume we're using Intel's
		 * table-driven format instead.
		 */
		if (cpi->cpi_xmaxeax >= 0x80000005)
			return (X86_VENDOR_AMD);
		else if (cpi->cpi_vendor == X86_VENDOR_Cyrix)
			return (X86_VENDOR_Cyrix);
		else if (cpi->cpi_maxeax >= 2)
			return (X86_VENDOR_Intel);
		break;
	}
	return (-1);
}

void
cpuid_set_cpu_properties(void *dip, processorid_t cpu_id,
    struct cpuid_info *cpi)
{
	dev_info_t *cpu_devi;
	int create;

	cpu_devi = (dev_info_t *)dip;

	/* device_type */
	(void) ndi_prop_update_string(DDI_DEV_T_NONE, cpu_devi,
	    "device_type", "cpu");

	/* reg */
	(void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
	    "reg", cpu_id);

	/* cpu-mhz, and clock-frequency */
	if (cpu_freq > 0) {
		long long mul;

		(void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
		    "cpu-mhz", cpu_freq);
		if ((mul = cpu_freq * 1000000LL) <= INT_MAX)
			(void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
			    "clock-frequency", (int)mul);
	}

	ASSERT(is_x86_feature(x86_featureset, X86FSET_CPUID));

	/* vendor-id */
	(void) ndi_prop_update_string(DDI_DEV_T_NONE, cpu_devi,
	    "vendor-id", cpi->cpi_vendorstr);

	if (cpi->cpi_maxeax == 0) {
		return;
	}

	/*
	 * family, model, and step
	 */
	(void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
	    "family", CPI_FAMILY(cpi));
	(void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
	    "cpu-model", CPI_MODEL(cpi));
	(void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
	    "stepping-id", CPI_STEP(cpi));

	/* type */
	switch (cpi->cpi_vendor) {
	case X86_VENDOR_Intel:
		create = 1;
		break;
	default:
		create = 0;
		break;
	}
	if (create)
		(void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
		    "type", CPI_TYPE(cpi));

	/* ext-family */
	switch (cpi->cpi_vendor) {
	case X86_VENDOR_Intel:
	case X86_VENDOR_AMD:
		create = cpi->cpi_family >= 0xf;
		break;
	case X86_VENDOR_HYGON:
		create = 1;
		break;
	default:
		create = 0;
		break;
	}
	if (create)
		(void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
		    "ext-family", CPI_FAMILY_XTD(cpi));

	/* ext-model */
	switch (cpi->cpi_vendor) {
	case X86_VENDOR_Intel:
		create = IS_EXTENDED_MODEL_INTEL(cpi);
		break;
	case X86_VENDOR_AMD:
		create = CPI_FAMILY(cpi) == 0xf;
		break;
	case X86_VENDOR_HYGON:
		create = 1;
		break;
	default:
		create = 0;
		break;
	}
	if (create)
		(void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
		    "ext-model", CPI_MODEL_XTD(cpi));

	/* generation */
	switch (cpi->cpi_vendor) {
	case X86_VENDOR_AMD:
	case X86_VENDOR_HYGON:
		/*
		 * AMD K5 model 1 was the first part to support this
		 */
		create = cpi->cpi_xmaxeax >= 0x80000001;
		break;
	default:
		create = 0;
		break;
	}
	if (create)
		(void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
		    "generation", BITX((cpi)->cpi_extd[1].cp_eax, 11, 8));

	/* brand-id */
	switch (cpi->cpi_vendor) {
	case X86_VENDOR_Intel:
		/*
		 * brand id first appeared on Pentium III Xeon model 8,
		 * and Celeron model 8 processors and Opteron
		 */
		create = cpi->cpi_family > 6 ||
		    (cpi->cpi_family == 6 && cpi->cpi_model >= 8);
		break;
	case X86_VENDOR_AMD:
		create = cpi->cpi_family >= 0xf;
		break;
	case X86_VENDOR_HYGON:
		create = 1;
		break;
	default:
		create = 0;
		break;
	}
	if (create && cpi->cpi_brandid != 0) {
		(void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
		    "brand-id", cpi->cpi_brandid);
	}

	/* chunks, and apic-id */
	switch (cpi->cpi_vendor) {
		/*
		 * first available on Pentium IV and Opteron (K8)
		 */
	case X86_VENDOR_Intel:
		create = IS_NEW_F6(cpi) || cpi->cpi_family >= 0xf;
		break;
	case X86_VENDOR_AMD:
		create = cpi->cpi_family >= 0xf;
		break;
	case X86_VENDOR_HYGON:
		create = 1;
		break;
	default:
		create = 0;
		break;
	}
	if (create) {
		(void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
		    "chunks", CPI_CHUNKS(cpi));
		(void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
		    "apic-id", cpi->cpi_apicid);
		if (cpi->cpi_chipid >= 0) {
			(void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
			    "chip#", cpi->cpi_chipid);
			(void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
			    "clog#", cpi->cpi_clogid);
		}
	}

	/* cpuid-features */
	(void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
	    "cpuid-features", CPI_FEATURES_EDX(cpi));


	/* cpuid-features-ecx */
	switch (cpi->cpi_vendor) {
	case X86_VENDOR_Intel:
		create = IS_NEW_F6(cpi) || cpi->cpi_family >= 0xf;
		break;
	case X86_VENDOR_AMD:
		create = cpi->cpi_family >= 0xf;
		break;
	case X86_VENDOR_HYGON:
		create = 1;
		break;
	default:
		create = 0;
		break;
	}
	if (create)
		(void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
		    "cpuid-features-ecx", CPI_FEATURES_ECX(cpi));

	/* ext-cpuid-features */
	switch (cpi->cpi_vendor) {
	case X86_VENDOR_Intel:
	case X86_VENDOR_AMD:
	case X86_VENDOR_HYGON:
	case X86_VENDOR_Cyrix:
	case X86_VENDOR_TM:
	case X86_VENDOR_Centaur:
		create = cpi->cpi_xmaxeax >= 0x80000001;
		break;
	default:
		create = 0;
		break;
	}
	if (create) {
		(void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
		    "ext-cpuid-features", CPI_FEATURES_XTD_EDX(cpi));
		(void) ndi_prop_update_int(DDI_DEV_T_NONE, cpu_devi,
		    "ext-cpuid-features-ecx", CPI_FEATURES_XTD_ECX(cpi));
	}

	/*
	 * Brand String first appeared in Intel Pentium IV, AMD K5
	 * model 1, and Cyrix GXm.  On earlier models we try and
	 * simulate something similar .. so this string should always
	 * same -something- about the processor, however lame.
	 */
	(void) ndi_prop_update_string(DDI_DEV_T_NONE, cpu_devi,
	    "brand-string", cpi->cpi_brandstr);

	/*
	 * Finally, cache and tlb information
	 */
	switch (x86_which_cacheinfo(cpi)) {
	case X86_VENDOR_Intel:
		intel_walk_cacheinfo(cpi, cpu_devi, add_cacheent_props);
		break;
	case X86_VENDOR_Cyrix:
		cyrix_walk_cacheinfo(cpi, cpu_devi, add_cacheent_props);
		break;
	case X86_VENDOR_AMD:
		amd_cache_info(cpi, cpu_devi);
		break;
	default:
		break;
	}
}

struct l2info {
	int *l2i_csz;
	int *l2i_lsz;
	int *l2i_assoc;
	int l2i_ret;
};

/*
 * A cacheinfo walker that fetches the size, line-size and associativity
 * of the L2 cache
 */
static int
intel_l2cinfo(void *arg, const struct cachetab *ct)
{
	struct l2info *l2i = arg;
	int *ip;

	if (ct->ct_label != l2_cache_str &&
	    ct->ct_label != sl2_cache_str)
		return (0);	/* not an L2 -- keep walking */

	if ((ip = l2i->l2i_csz) != NULL)
		*ip = ct->ct_size;
	if ((ip = l2i->l2i_lsz) != NULL)
		*ip = ct->ct_line_size;
	if ((ip = l2i->l2i_assoc) != NULL)
		*ip = ct->ct_assoc;
	l2i->l2i_ret = ct->ct_size;
	return (1);		/* was an L2 -- terminate walk */
}

/*
 * AMD L2/L3 Cache and TLB Associativity Field Definition:
 *
 *	Unlike the associativity for the L1 cache and tlb where the 8 bit
 *	value is the associativity, the associativity for the L2 cache and
 *	tlb is encoded in the following table. The 4 bit L2 value serves as
 *	an index into the amd_afd[] array to determine the associativity.
 *	-1 is undefined. 0 is fully associative.
 */

static int amd_afd[] =
	{-1, 1, 2, -1, 4, -1, 8, -1, 16, -1, 32, 48, 64, 96, 128, 0};

static void
amd_l2cacheinfo(struct cpuid_info *cpi, struct l2info *l2i)
{
	struct cpuid_regs *cp;
	uint_t size, assoc;
	int i;
	int *ip;

	if (cpi->cpi_xmaxeax < 0x80000006)
		return;
	cp = &cpi->cpi_extd[6];

	if ((i = BITX(cp->cp_ecx, 15, 12)) != 0 &&
	    (size = BITX(cp->cp_ecx, 31, 16)) != 0) {
		uint_t cachesz = size * 1024;
		assoc = amd_afd[i];

		ASSERT(assoc != -1);

		if ((ip = l2i->l2i_csz) != NULL)
			*ip = cachesz;
		if ((ip = l2i->l2i_lsz) != NULL)
			*ip = BITX(cp->cp_ecx, 7, 0);
		if ((ip = l2i->l2i_assoc) != NULL)
			*ip = assoc;
		l2i->l2i_ret = cachesz;
	}
}

int
getl2cacheinfo(cpu_t *cpu, int *csz, int *lsz, int *assoc)
{
	struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
	struct l2info __l2info, *l2i = &__l2info;

	l2i->l2i_csz = csz;
	l2i->l2i_lsz = lsz;
	l2i->l2i_assoc = assoc;
	l2i->l2i_ret = -1;

	switch (x86_which_cacheinfo(cpi)) {
	case X86_VENDOR_Intel:
		intel_walk_cacheinfo(cpi, l2i, intel_l2cinfo);
		break;
	case X86_VENDOR_Cyrix:
		cyrix_walk_cacheinfo(cpi, l2i, intel_l2cinfo);
		break;
	case X86_VENDOR_AMD:
		amd_l2cacheinfo(cpi, l2i);
		break;
	default:
		break;
	}
	return (l2i->l2i_ret);
}

#if !defined(__xpv)

uint32_t *
cpuid_mwait_alloc(cpu_t *cpu)
{
	uint32_t	*ret;
	size_t		mwait_size;

	ASSERT(cpuid_checkpass(CPU, CPUID_PASS_EXTENDED));

	mwait_size = CPU->cpu_m.mcpu_cpi->cpi_mwait.mon_max;
	if (mwait_size == 0)
		return (NULL);

	/*
	 * kmem_alloc() returns cache line size aligned data for mwait_size
	 * allocations.  mwait_size is currently cache line sized.  Neither
	 * of these implementation details are guarantied to be true in the
	 * future.
	 *
	 * First try allocating mwait_size as kmem_alloc() currently returns
	 * correctly aligned memory.  If kmem_alloc() does not return
	 * mwait_size aligned memory, then use mwait_size ROUNDUP.
	 *
	 * Set cpi_mwait.buf_actual and cpi_mwait.size_actual in case we
	 * decide to free this memory.
	 */
	ret = kmem_zalloc(mwait_size, KM_SLEEP);
	if (ret == (uint32_t *)P2ROUNDUP((uintptr_t)ret, mwait_size)) {
		cpu->cpu_m.mcpu_cpi->cpi_mwait.buf_actual = ret;
		cpu->cpu_m.mcpu_cpi->cpi_mwait.size_actual = mwait_size;
		*ret = MWAIT_RUNNING;
		return (ret);
	} else {
		kmem_free(ret, mwait_size);
		ret = kmem_zalloc(mwait_size * 2, KM_SLEEP);
		cpu->cpu_m.mcpu_cpi->cpi_mwait.buf_actual = ret;
		cpu->cpu_m.mcpu_cpi->cpi_mwait.size_actual = mwait_size * 2;
		ret = (uint32_t *)P2ROUNDUP((uintptr_t)ret, mwait_size);
		*ret = MWAIT_RUNNING;
		return (ret);
	}
}

void
cpuid_mwait_free(cpu_t *cpu)
{
	if (cpu->cpu_m.mcpu_cpi == NULL) {
		return;
	}

	if (cpu->cpu_m.mcpu_cpi->cpi_mwait.buf_actual != NULL &&
	    cpu->cpu_m.mcpu_cpi->cpi_mwait.size_actual > 0) {
		kmem_free(cpu->cpu_m.mcpu_cpi->cpi_mwait.buf_actual,
		    cpu->cpu_m.mcpu_cpi->cpi_mwait.size_actual);
	}

	cpu->cpu_m.mcpu_cpi->cpi_mwait.buf_actual = NULL;
	cpu->cpu_m.mcpu_cpi->cpi_mwait.size_actual = 0;
}

void
patch_tsc_read(int flag)
{
	size_t cnt;

	switch (flag) {
	case TSC_NONE:
		cnt = &_no_rdtsc_end - &_no_rdtsc_start;
		(void) memcpy((void *)tsc_read, (void *)&_no_rdtsc_start, cnt);
		break;
	case TSC_RDTSC_LFENCE:
		cnt = &_tsc_lfence_end - &_tsc_lfence_start;
		(void) memcpy((void *)tsc_read,
		    (void *)&_tsc_lfence_start, cnt);
		break;
	case TSC_TSCP:
		cnt = &_tscp_end - &_tscp_start;
		(void) memcpy((void *)tsc_read, (void *)&_tscp_start, cnt);
		break;
	default:
		/* Bail for unexpected TSC types. (TSC_NONE covers 0) */
		cmn_err(CE_PANIC, "Unrecogized TSC type: %d", flag);
		break;
	}
	tsc_type = flag;
}

int
cpuid_deep_cstates_supported(void)
{
	struct cpuid_info *cpi;
	struct cpuid_regs regs;

	ASSERT(cpuid_checkpass(CPU, CPUID_PASS_BASIC));
	ASSERT(is_x86_feature(x86_featureset, X86FSET_CPUID));

	cpi = CPU->cpu_m.mcpu_cpi;

	switch (cpi->cpi_vendor) {
	case X86_VENDOR_Intel:
	case X86_VENDOR_AMD:
	case X86_VENDOR_HYGON:
		if (cpi->cpi_xmaxeax < 0x80000007)
			return (0);

		/*
		 * Does TSC run at a constant rate in all C-states?
		 */
		regs.cp_eax = 0x80000007;
		(void) __cpuid_insn(&regs);
		return (regs.cp_edx & CPUID_TSC_CSTATE_INVARIANCE);

	default:
		return (0);
	}
}

#endif	/* !__xpv */

void
post_startup_cpu_fixups(void)
{
#ifndef __xpv
	/*
	 * Some AMD processors support C1E state. Entering this state will
	 * cause the local APIC timer to stop, which we can't deal with at
	 * this time.
	 */
	if (cpuid_getvendor(CPU) == X86_VENDOR_AMD) {
		on_trap_data_t otd;
		uint64_t reg;

		if (!on_trap(&otd, OT_DATA_ACCESS)) {
			reg = rdmsr(MSR_AMD_INT_PENDING_CMP_HALT);
			/* Disable C1E state if it is enabled by BIOS */
			if ((reg >> AMD_ACTONCMPHALT_SHIFT) &
			    AMD_ACTONCMPHALT_MASK) {
				reg &= ~(AMD_ACTONCMPHALT_MASK <<
				    AMD_ACTONCMPHALT_SHIFT);
				wrmsr(MSR_AMD_INT_PENDING_CMP_HALT, reg);
			}
		}
		no_trap();
	}
#endif	/* !__xpv */
}

void
enable_pcid(void)
{
	if (x86_use_pcid == -1)
		x86_use_pcid = is_x86_feature(x86_featureset, X86FSET_PCID);

	if (x86_use_invpcid == -1) {
		x86_use_invpcid = is_x86_feature(x86_featureset,
		    X86FSET_INVPCID);
	}

	if (!x86_use_pcid)
		return;

	/*
	 * Intel say that on setting PCIDE, it immediately starts using the PCID
	 * bits; better make sure there's nothing there.
	 */
	ASSERT((getcr3() & MMU_PAGEOFFSET) == PCID_NONE);

	setcr4(getcr4() | CR4_PCIDE);
}

/*
 * Setup necessary registers to enable XSAVE feature on this processor.
 * This function needs to be called early enough, so that no xsave/xrstor
 * ops will execute on the processor before the MSRs are properly set up.
 *
 * Current implementation has the following assumption:
 * - cpuid_pass_basic() is done, so that X86 features are known.
 * - fpu_probe() is done, so that fp_save_mech is chosen.
 */
void
xsave_setup_msr(cpu_t *cpu)
{
	ASSERT(cpuid_checkpass(cpu, CPUID_PASS_BASIC));
	ASSERT(fp_save_mech == FP_XSAVE);
	ASSERT(is_x86_feature(x86_featureset, X86FSET_XSAVE));

	/* Enable OSXSAVE in CR4. */
	setcr4(getcr4() | CR4_OSXSAVE);
	/*
	 * Update SW copy of ECX, so that /dev/cpu/self/cpuid will report
	 * correct value.
	 */
	cpu->cpu_m.mcpu_cpi->cpi_std[1].cp_ecx |= CPUID_INTC_ECX_OSXSAVE;
	setup_xfem();
}

/*
 * Starting with the Westmere processor the local
 * APIC timer will continue running in all C-states,
 * including the deepest C-states.
 */
int
cpuid_arat_supported(void)
{
	struct cpuid_info *cpi;
	struct cpuid_regs regs;

	ASSERT(cpuid_checkpass(CPU, CPUID_PASS_BASIC));
	ASSERT(is_x86_feature(x86_featureset, X86FSET_CPUID));

	cpi = CPU->cpu_m.mcpu_cpi;

	switch (cpi->cpi_vendor) {
	case X86_VENDOR_Intel:
	case X86_VENDOR_AMD:
	case X86_VENDOR_HYGON:
		/*
		 * Always-running Local APIC Timer is
		 * indicated by CPUID.6.EAX[2].
		 */
		if (cpi->cpi_maxeax >= 6) {
			regs.cp_eax = 6;
			(void) cpuid_insn(NULL, &regs);
			return (regs.cp_eax & CPUID_INTC_EAX_ARAT);
		} else {
			return (0);
		}
	default:
		return (0);
	}
}

/*
 * Check support for Intel ENERGY_PERF_BIAS feature
 */
int
cpuid_iepb_supported(struct cpu *cp)
{
	struct cpuid_info *cpi = cp->cpu_m.mcpu_cpi;
	struct cpuid_regs regs;

	ASSERT(cpuid_checkpass(cp, CPUID_PASS_BASIC));
	ASSERT(is_x86_feature(x86_featureset, X86FSET_CPUID));

	if (!(is_x86_feature(x86_featureset, X86FSET_MSR))) {
		return (0);
	}

	/*
	 * Intel ENERGY_PERF_BIAS MSR is indicated by
	 * capability bit CPUID.6.ECX.3
	 */
	if ((cpi->cpi_vendor != X86_VENDOR_Intel) || (cpi->cpi_maxeax < 6))
		return (0);

	regs.cp_eax = 0x6;
	(void) cpuid_insn(NULL, &regs);
	return (regs.cp_ecx & CPUID_INTC_ECX_PERFBIAS);
}

/*
 * Check support for TSC deadline timer
 *
 * TSC deadline timer provides a superior software programming
 * model over local APIC timer that eliminates "time drifts".
 * Instead of specifying a relative time, software specifies an
 * absolute time as the target at which the processor should
 * generate a timer event.
 */
int
cpuid_deadline_tsc_supported(void)
{
	struct cpuid_info *cpi = CPU->cpu_m.mcpu_cpi;
	struct cpuid_regs regs;

	ASSERT(cpuid_checkpass(CPU, CPUID_PASS_BASIC));
	ASSERT(is_x86_feature(x86_featureset, X86FSET_CPUID));

	switch (cpi->cpi_vendor) {
	case X86_VENDOR_Intel:
		if (cpi->cpi_maxeax >= 1) {
			regs.cp_eax = 1;
			(void) cpuid_insn(NULL, &regs);
			return (regs.cp_ecx & CPUID_DEADLINE_TSC);
		} else {
			return (0);
		}
	default:
		return (0);
	}
}

#if !defined(__xpv)
/*
 * Patch in versions of bcopy for high performance Intel Nhm processors
 * and later...
 */
void
patch_memops(uint_t vendor)
{
	size_t cnt, i;
	caddr_t to, from;

	if ((vendor == X86_VENDOR_Intel) &&
	    is_x86_feature(x86_featureset, X86FSET_SSE4_2)) {
		cnt = &bcopy_patch_end - &bcopy_patch_start;
		to = &bcopy_ck_size;
		from = &bcopy_patch_start;
		for (i = 0; i < cnt; i++) {
			*to++ = *from++;
		}
	}
}
#endif  /*  !__xpv */

/*
 * We're being asked to tell the system how many bits are required to represent
 * the various thread and strand IDs. While it's tempting to derive this based
 * on the values in cpi_ncore_per_chip and cpi_ncpu_per_chip, that isn't quite
 * correct. Instead, this needs to be based on the number of bits that the APIC
 * allows for these different configurations. We only update these to a larger
 * value if we find one.
 */
void
cpuid_get_ext_topo(cpu_t *cpu, uint_t *core_nbits, uint_t *strand_nbits)
{
	struct cpuid_info *cpi;

	VERIFY(cpuid_checkpass(CPU, CPUID_PASS_BASIC));
	cpi = cpu->cpu_m.mcpu_cpi;

	if (cpi->cpi_ncore_bits > *core_nbits) {
		*core_nbits = cpi->cpi_ncore_bits;
	}

	if (cpi->cpi_nthread_bits > *strand_nbits) {
		*strand_nbits = cpi->cpi_nthread_bits;
	}
}

void
cpuid_pass_ucode(cpu_t *cpu, uchar_t *fset)
{
	struct cpuid_info *cpi = cpu->cpu_m.mcpu_cpi;
	struct cpuid_regs cp;

	/*
	 * Reread the CPUID portions that we need for various security
	 * information.
	 */
	switch (cpi->cpi_vendor) {
	case X86_VENDOR_Intel:
		/*
		 * Check if we now have leaf 7 available to us.
		 */
		if (cpi->cpi_maxeax < 7) {
			bzero(&cp, sizeof (cp));
			cp.cp_eax = 0;
			cpi->cpi_maxeax = __cpuid_insn(&cp);
			if (cpi->cpi_maxeax < 7)
				break;
		}

		bzero(&cp, sizeof (cp));
		cp.cp_eax = 7;
		cp.cp_ecx = 0;
		(void) __cpuid_insn(&cp);
		cpi->cpi_std[7] = cp;
		break;

	case X86_VENDOR_AMD:
	case X86_VENDOR_HYGON:
		/* No xcpuid support */
		if (cpi->cpi_family < 5 ||
		    (cpi->cpi_family == 5 && cpi->cpi_model < 1))
			break;

		if (cpi->cpi_xmaxeax < CPUID_LEAF_EXT_8) {
			bzero(&cp, sizeof (cp));
			cp.cp_eax = CPUID_LEAF_EXT_0;
			cpi->cpi_xmaxeax = __cpuid_insn(&cp);
			if (cpi->cpi_xmaxeax < CPUID_LEAF_EXT_8)
				break;
		}

		/*
		 * Most AMD features are in leaf 8. Automatic IBRS was added in
		 * leaf 0x21. So we also check that.
		 */
		bzero(&cp, sizeof (cp));
		cp.cp_eax = CPUID_LEAF_EXT_8;
		(void) __cpuid_insn(&cp);
		platform_cpuid_mangle(cpi->cpi_vendor, CPUID_LEAF_EXT_8, &cp);
		cpi->cpi_extd[8] = cp;

		if (cpi->cpi_xmaxeax < CPUID_LEAF_EXT_21)
			break;

		bzero(&cp, sizeof (cp));
		cp.cp_eax = CPUID_LEAF_EXT_21;
		(void) __cpuid_insn(&cp);
		platform_cpuid_mangle(cpi->cpi_vendor, CPUID_LEAF_EXT_21, &cp);
		cpi->cpi_extd[0x21] = cp;
		break;

	default:
		/*
		 * Nothing to do here. Return an empty set which has already
		 * been zeroed for us.
		 */
		return;
	}

	cpuid_scan_security(cpu, fset);
}

/* ARGSUSED */
static int
cpuid_post_ucodeadm_xc(xc_arg_t arg0, xc_arg_t arg1, xc_arg_t arg2)
{
	uchar_t *fset;
	boolean_t first_pass = (boolean_t)arg1;

	fset = (uchar_t *)(arg0 + sizeof (x86_featureset) * CPU->cpu_id);
	if (first_pass && CPU->cpu_id != 0)
		return (0);
	if (!first_pass && CPU->cpu_id == 0)
		return (0);
	cpuid_pass_ucode(CPU, fset);

	return (0);
}

/*
 * After a microcode update where the version has changed, then we need to
 * rescan CPUID. To do this we check every CPU to make sure that they have the
 * same microcode. Then we perform a cross call to all such CPUs. It's the
 * caller's job to make sure that no one else can end up doing an update while
 * this is going on.
 *
 * We assume that the system is microcode capable if we're called.
 */
void
cpuid_post_ucodeadm(void)
{
	uint32_t rev;
	int i;
	struct cpu *cpu;
	cpuset_t cpuset;
	void *argdata;
	uchar_t *f0;

	argdata = kmem_zalloc(sizeof (x86_featureset) * NCPU, KM_SLEEP);

	mutex_enter(&cpu_lock);
	cpu = cpu_get(0);
	rev = cpu->cpu_m.mcpu_ucode_info->cui_rev;
	CPUSET_ONLY(cpuset, 0);
	for (i = 1; i < max_ncpus; i++) {
		if ((cpu = cpu_get(i)) == NULL)
			continue;

		if (cpu->cpu_m.mcpu_ucode_info->cui_rev != rev) {
			panic("post microcode update CPU %d has differing "
			    "microcode revision (%u) from CPU 0 (%u)",
			    i, cpu->cpu_m.mcpu_ucode_info->cui_rev, rev);
		}
		CPUSET_ADD(cpuset, i);
	}

	/*
	 * We do the cross calls in two passes. The first pass is only for the
	 * boot CPU. The second pass is for all of the other CPUs. This allows
	 * the boot CPU to go through and change behavior related to patching or
	 * whether or not Enhanced IBRS needs to be enabled and then allow all
	 * other CPUs to follow suit.
	 */
	kpreempt_disable();
	xc_sync((xc_arg_t)argdata, B_TRUE, 0, CPUSET2BV(cpuset),
	    cpuid_post_ucodeadm_xc);
	xc_sync((xc_arg_t)argdata, B_FALSE, 0, CPUSET2BV(cpuset),
	    cpuid_post_ucodeadm_xc);
	kpreempt_enable();

	/*
	 * OK, now look at each CPU and see if their feature sets are equal.
	 */
	f0 = argdata;
	for (i = 1; i < max_ncpus; i++) {
		uchar_t *fset;
		if (!CPU_IN_SET(cpuset, i))
			continue;

		fset = (uchar_t *)((uintptr_t)argdata +
		    sizeof (x86_featureset) * i);

		if (!compare_x86_featureset(f0, fset)) {
			panic("Post microcode update CPU %d has "
			    "differing security feature (%p) set from CPU 0 "
			    "(%p), not appending to feature set", i,
			    (void *)fset, (void *)f0);
		}
	}

	mutex_exit(&cpu_lock);

	for (i = 0; i < NUM_X86_FEATURES; i++) {
		cmn_err(CE_CONT, "?post-ucode x86_feature: %s\n",
		    x86_feature_names[i]);
		if (is_x86_feature(f0, i)) {
			add_x86_feature(x86_featureset, i);
		}
	}
	kmem_free(argdata, sizeof (x86_featureset) * NCPU);
}

typedef void (*cpuid_pass_f)(cpu_t *, void *);

typedef struct cpuid_pass_def {
	cpuid_pass_t cpd_pass;
	cpuid_pass_f cpd_func;
} cpuid_pass_def_t;

/*
 * See block comment at the top; note that cpuid_pass_ucode is not a pass in the
 * normal sense and should not appear here.
 */
static const cpuid_pass_def_t cpuid_pass_defs[] = {
	{ CPUID_PASS_PRELUDE, cpuid_pass_prelude },
	{ CPUID_PASS_IDENT, cpuid_pass_ident },
	{ CPUID_PASS_BASIC, cpuid_pass_basic },
	{ CPUID_PASS_EXTENDED, cpuid_pass_extended },
	{ CPUID_PASS_DYNAMIC, cpuid_pass_dynamic },
	{ CPUID_PASS_RESOLVE, cpuid_pass_resolve },
};

void
cpuid_execpass(cpu_t *cp, cpuid_pass_t pass, void *arg)
{
	VERIFY3S(pass, !=, CPUID_PASS_NONE);

	if (cp == NULL)
		cp = CPU;

	/*
	 * Space statically allocated for BSP, ensure pointer is set
	 */
	if (cp->cpu_id == 0 && cp->cpu_m.mcpu_cpi == NULL)
		cp->cpu_m.mcpu_cpi = &cpuid_info0;

	ASSERT(cpuid_checkpass(cp, pass - 1));

	for (uint_t i = 0; i < ARRAY_SIZE(cpuid_pass_defs); i++) {
		if (cpuid_pass_defs[i].cpd_pass == pass) {
			cpuid_pass_defs[i].cpd_func(cp, arg);
			cp->cpu_m.mcpu_cpi->cpi_pass = pass;
			return;
		}
	}

	panic("unable to execute invalid cpuid pass %d on cpu%d\n",
	    pass, cp->cpu_id);
}

/*
 * Extract the processor family from a chiprev.  Processor families are not the
 * same as cpuid families; see comments above and in x86_archext.h.
 */
x86_processor_family_t
chiprev_family(const x86_chiprev_t cr)
{
	return ((x86_processor_family_t)_X86_CHIPREV_FAMILY(cr));
}

/*
 * A chiprev matches its template if the vendor and family are identical and the
 * revision of the chiprev matches one of the bits set in the template.  Callers
 * may bitwise-OR together chiprevs of the same vendor and family to form the
 * template, or use the _ANY variant.  It is not possible to match chiprevs of
 * multiple vendors or processor families with a single call.  Note that this
 * function operates on processor families, not cpuid families.
 */
boolean_t
chiprev_matches(const x86_chiprev_t cr, const x86_chiprev_t template)
{
	return (_X86_CHIPREV_VENDOR(cr) == _X86_CHIPREV_VENDOR(template) &&
	    _X86_CHIPREV_FAMILY(cr) == _X86_CHIPREV_FAMILY(template) &&
	    (_X86_CHIPREV_REV(cr) & _X86_CHIPREV_REV(template)) != 0);
}

/*
 * A chiprev is at least min if the vendor and family are identical and the
 * revision of the chiprev is at least as recent as that of min.  Processor
 * families are considered unordered and cannot be compared using this function.
 * Note that this function operates on processor families, not cpuid families.
 * Use of the _ANY chiprev variant with this function is not useful; it will
 * always return B_FALSE if the _ANY variant is supplied as the minimum
 * revision.  To determine only whether a chiprev is of a given processor
 * family, test the return value of chiprev_family() instead.
 */
boolean_t
chiprev_at_least(const x86_chiprev_t cr, const x86_chiprev_t min)
{
	return (_X86_CHIPREV_VENDOR(cr) == _X86_CHIPREV_VENDOR(min) &&
	    _X86_CHIPREV_FAMILY(cr) == _X86_CHIPREV_FAMILY(min) &&
	    _X86_CHIPREV_REV(cr) >= _X86_CHIPREV_REV(min));
}

/*
 * The uarch functions operate in a manner similar to the chiprev functions
 * above.  While it is tempting to allow these to operate on microarchitectures
 * produced by a specific vendor in an ordered fashion (e.g., ZEN3 is "newer"
 * than ZEN2), we elect not to do so because a manufacturer may supply
 * processors of multiple different microarchitecture families each of which may
 * be internally ordered but unordered with respect to those of other families.
 */
x86_uarch_t
uarchrev_uarch(const x86_uarchrev_t ur)
{
	return ((x86_uarch_t)_X86_UARCHREV_UARCH(ur));
}

boolean_t
uarchrev_matches(const x86_uarchrev_t ur, const x86_uarchrev_t template)
{
	return (_X86_UARCHREV_VENDOR(ur) == _X86_UARCHREV_VENDOR(template) &&
	    _X86_UARCHREV_UARCH(ur) == _X86_UARCHREV_UARCH(template) &&
	    (_X86_UARCHREV_REV(ur) & _X86_UARCHREV_REV(template)) != 0);
}

boolean_t
uarchrev_at_least(const x86_uarchrev_t ur, const x86_uarchrev_t min)
{
	return (_X86_UARCHREV_VENDOR(ur) == _X86_UARCHREV_VENDOR(min) &&
	    _X86_UARCHREV_UARCH(ur) == _X86_UARCHREV_UARCH(min) &&
	    _X86_UARCHREV_REV(ur) >= _X86_UARCHREV_REV(min));
}

/*
 * Topology cache related information. This is yet another cache interface that
 * we're exposing out intended to be used when we have either Intel Leaf 4 or
 * AMD Leaf 8x1D (introduced with Zen 1).
 */
static boolean_t
cpuid_cache_topo_sup(const struct cpuid_info *cpi)
{
	switch (cpi->cpi_vendor) {
	case X86_VENDOR_Intel:
		if (cpi->cpi_maxeax >= 4) {
			return (B_TRUE);
		}
		break;
	case X86_VENDOR_AMD:
	case X86_VENDOR_HYGON:
		if (cpi->cpi_xmaxeax >= CPUID_LEAF_EXT_1d &&
		    is_x86_feature(x86_featureset, X86FSET_TOPOEXT)) {
			return (B_TRUE);
		}
		break;
	default:
		break;
	}

	return (B_FALSE);
}

int
cpuid_getncaches(struct cpu *cpu, uint32_t *ncache)
{
	const struct cpuid_info *cpi;

	ASSERT(cpuid_checkpass(cpu, CPUID_PASS_DYNAMIC));
	cpi = cpu->cpu_m.mcpu_cpi;

	if (!cpuid_cache_topo_sup(cpi)) {
		return (ENOTSUP);
	}

	*ncache = cpi->cpi_cache_leaf_size;
	return (0);
}

int
cpuid_getcache(struct cpu *cpu, uint32_t cno, x86_cache_t *cache)
{
	const struct cpuid_info *cpi;
	const struct cpuid_regs *cp;

	ASSERT(cpuid_checkpass(cpu, CPUID_PASS_DYNAMIC));
	cpi = cpu->cpu_m.mcpu_cpi;

	if (!cpuid_cache_topo_sup(cpi)) {
		return (ENOTSUP);
	}

	if (cno >= cpi->cpi_cache_leaf_size) {
		return (EINVAL);
	}

	bzero(cache, sizeof (x86_cache_t));
	cp = cpi->cpi_cache_leaves[cno];
	switch (CPI_CACHE_TYPE(cp)) {
	case CPI_CACHE_TYPE_DATA:
		cache->xc_type = X86_CACHE_TYPE_DATA;
		break;
	case CPI_CACHE_TYPE_INSTR:
		cache->xc_type = X86_CACHE_TYPE_INST;
		break;
	case CPI_CACHE_TYPE_UNIFIED:
		cache->xc_type = X86_CACHE_TYPE_UNIFIED;
		break;
	case CPI_CACHE_TYPE_DONE:
	default:
		return (EINVAL);
	}
	cache->xc_level = CPI_CACHE_LVL(cp);
	if (CPI_FULL_ASSOC_CACHE(cp) != 0) {
		cache->xc_flags |= X86_CACHE_F_FULL_ASSOC;
	}
	cache->xc_nparts = CPI_CACHE_PARTS(cp) + 1;
	/*
	 * The number of sets is reserved on AMD if the CPU is tagged as fully
	 * associative, where as it is considered valid on Intel.
	 */
	if (cpi->cpi_vendor == X86_VENDOR_AMD &&
	    CPI_FULL_ASSOC_CACHE(cp) != 0) {
		cache->xc_nsets = 1;
	} else {
		cache->xc_nsets = CPI_CACHE_SETS(cp) + 1;
	}
	cache->xc_nways = CPI_CACHE_WAYS(cp) + 1;
	cache->xc_line_size = CPI_CACHE_COH_LN_SZ(cp) + 1;
	cache->xc_size = cache->xc_nparts * cache->xc_nsets * cache->xc_nways *
	    cache->xc_line_size;
	/*
	 * We're looking for the number of bits to cover the number of CPUs that
	 * are being shared. Normally this would be the value - 1, but the CPUID
	 * value is encoded as the actual value minus one, so we don't modify
	 * this at all.
	 */
	cache->xc_apic_shift = highbit(CPI_NTHR_SHR_CACHE(cp));

	/*
	 * To construct a unique ID we construct a uint64_t that looks as
	 * follows:
	 *
	 * [47:40] cache level
	 * [39:32] CPUID cache type
	 * [31:00] shifted APIC ID
	 *
	 * The shifted APIC ID gives us a guarantee that a given cache entry is
	 * unique within its peers. The other two numbers give us something that
	 * ensures that something is unique within the CPU. If we just had the
	 * APIC ID shifted over by the indicated number of bits we'd end up with
	 * an ID of zero for the L1I, L1D, L2, and L3.
	 *
	 * The format of this ID is private to the system and can change across
	 * a reboot for the time being.
	 */
	cache->xc_id = (uint64_t)cache->xc_level << 40;
	cache->xc_id |= (uint64_t)cache->xc_type << 32;
	cache->xc_id |= (uint64_t)cpi->cpi_apicid >> cache->xc_apic_shift;

	return (0);
}
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 *
 * Copyright 2012 Nexenta Systems, Inc. All rights reserved.
 */

/*
 * Portions Copyright 2009 Advanced Micro Devices, Inc.
 */

/*
 * Copyright 2012 Jens Elkner <jel+illumos@cs.uni-magdeburg.de>
 * Copyright 2012 Hans Rosenfeld <rosenfeld@grumpf.hope-2000.org>
 * Copyright 2019 Joyent, Inc.
 * Copyright 2025 Oxide Computer Company
 */

/*
 * Support functions that interpret CPUID and similar information.
 * These should not be used from anywhere other than cpuid.c and
 * cmi_hw.c - as such we will not list them in any header file
 * such as x86_archext.h.
 *
 * In cpuid.c we process CPUID information for each cpu_t instance
 * we're presented with, and stash this raw information and material
 * derived from it in per-cpu_t structures.
 *
 * If we are virtualized then the CPUID information derived from CPUID
 * instructions executed in the guest is based on whatever the hypervisor
 * wanted to make things look like, and the cpu_t are not necessarily in 1:1
 * or fixed correspondence with real processor execution resources.  In cmi_hw.c
 * we are interested in the native properties of a processor - for fault
 * management (and potentially other, such as power management) purposes;
 * it will tunnel through to real hardware information, and use the
 * functionality provided in this file to process it.
 */

#include <sys/types.h>
#include <sys/systm.h>
#include <sys/bitmap.h>
#include <sys/x86_archext.h>
#include <sys/pci_cfgspace.h>
#include <sys/sysmacros.h>
#ifdef __xpv
#include <sys/hypervisor.h>
#endif

/*
 * AMD socket types.
 * First index defines a processor family; see notes inline.  The second index
 * selects the socket type by either (model & 0x3) for family 0fh or the CPUID
 * pkg bits (Fn8000_0001_EBX[31:28]) for later families.
 */
static uint32_t amd_skts[][16] = {
	/*
	 * Family 0xf revisions B through E
	 */
#define	A_SKTS_0			0
	{
		[0] = X86_SOCKET_754,
		[1] = X86_SOCKET_940,
		[2] = X86_SOCKET_754,
		[3] = X86_SOCKET_939,
	},
	/*
	 * Family 0xf revisions F and G
	 */
#define	A_SKTS_1			1
	{
		[0] = X86_SOCKET_S1g1,
		[1] = X86_SOCKET_F1207,
		[3] = X86_SOCKET_AM2
	},
	/*
	 * Family 0x10
	 */
#define	A_SKTS_2			2
	{
		[0] = X86_SOCKET_F1207,
		[1] = X86_SOCKET_AM2R2,
		[2] = X86_SOCKET_S1g3,
		[3] = X86_SOCKET_G34,
		[4] = X86_SOCKET_ASB2,
		[5] = X86_SOCKET_C32
	},

	/*
	 * Family 0x11
	 */
#define	A_SKTS_3			3
	{
		[2] = X86_SOCKET_S1g2
	},

	/*
	 * Family 0x12
	 */
#define	A_SKTS_4			4
	{
		[1] = X86_SOCKET_FS1,
		[2] = X86_SOCKET_FM1
	},

	/*
	 * Family 0x14
	 */
#define	A_SKTS_5			5
	{
		[0] = X86_SOCKET_FT1
	},

	/*
	 * Family 0x15 models 00 - 0f
	 */
#define	A_SKTS_6			6
	{
		[1] = X86_SOCKET_AM3R2,
		[3] = X86_SOCKET_G34,
		[5] = X86_SOCKET_C32
	},

	/*
	 * Family 0x15 models 10 - 1f
	 */
#define	A_SKTS_7			7
	{
		[0] = X86_SOCKET_FP2,
		[1] = X86_SOCKET_FS1R2,
		[2] = X86_SOCKET_FM2
	},

	/*
	 * Family 0x15 models 30-3f
	 */
#define	A_SKTS_8			8
	{
		[0] = X86_SOCKET_FP3,
		[1] = X86_SOCKET_FM2R2
	},

	/*
	 * Family 0x15 models 60-6f
	 */
#define	A_SKTS_9			9
	{
		[0] = X86_SOCKET_FP4,
		[2] = X86_SOCKET_AM4,
		[3] = X86_SOCKET_FM2R2
	},

	/*
	 * Family 0x15 models 70-7f
	 */
#define	A_SKTS_10			10
	{
		[0] = X86_SOCKET_FP4,
		[2] = X86_SOCKET_AM4,
		[4] = X86_SOCKET_FT4
	},

	/*
	 * Family 0x16 models 00-0f
	 */
#define	A_SKTS_11			11
	{
		[0] = X86_SOCKET_FT3,
		[1] = X86_SOCKET_FS1B
	},

	/*
	 * Family 0x16 models 30-3f
	 */
#define	A_SKTS_12			12
	{
		[0] = X86_SOCKET_FT3B,
		[3] = X86_SOCKET_FP4
	},

	/*
	 * Family 0x17 models 00-0f	(Zen 1 - Naples, Ryzen)
	 */
#define	A_SKTS_NAPLES			13
	{
		[2] = X86_SOCKET_AM4,
		[4] = X86_SOCKET_SP3,
		[7] = X86_SOCKET_SP3R2
	},

	/*
	 * Family 0x17 models 10-2f	(Zen 1 - APU: Raven Ridge)
	 *				(Zen 1 - APU: Banded Kestrel)
	 *				(Zen 1 - APU: Dali)
	 */
#define	A_SKTS_RAVEN			14
	{
		[0] = X86_SOCKET_FP5,
		[2] = X86_SOCKET_AM4
	},

	/*
	 * Family 0x17 models 30-3f	(Zen 2 - Rome)
	 */
#define	A_SKTS_ROME			15
	{
		[4] = X86_SOCKET_SP3,
		[7] = X86_SOCKET_SP3R2
	},

	/*
	 * Family 0x17 models 60-6f	(Zen 2 - Renoir)
	 */
#define	A_SKTS_RENOIR			16
	{
		[0] = X86_SOCKET_FP6,
		[2] = X86_SOCKET_AM4
	},

	/*
	 * Family 0x17 models 70-7f	(Zen 2 - Matisse)
	 */
#define	A_SKTS_MATISSE			17
	{
		[2] = X86_SOCKET_AM4,
	},

	/*
	 * Family 0x18 models 00-0f	(Dhyana)
	 */
#define	A_SKTS_DHYANA			18
	{
		[4] = X86_SOCKET_SL1,
		[6] = X86_SOCKET_DM1,
		[7] = X86_SOCKET_SL1R2
	},

	/*
	 * Family 0x19 models 00-0f	(Zen 3 - Milan)
	 */
#define	A_SKTS_MILAN			19
	{
		[4] = X86_SOCKET_SP3,
		[7] = X86_SOCKET_STRX4
	},

	/*
	 * Family 0x19 models 20-2f	(Zen 3 - Vermeer)
	 */
#define	A_SKTS_VERMEER			20
	{
		[2] = X86_SOCKET_AM4,
	},

	/*
	 * Family 0x19 models 50-5f	(Zen 3 - Cezanne)
	 */
#define	A_SKTS_CEZANNE			21
	{
		[0] = X86_SOCKET_FP6,
		[2] = X86_SOCKET_AM4
	},

	/*
	 * Family 0x19 models 10-1f	(Zen 4 - Genoa)
	 */
#define	A_SKTS_GENOA			22
	{
		[4] = X86_SOCKET_SP5,
		[8] = X86_SOCKET_TR5
	},

	/*
	 * Family 0x19 models 40-4f	(Zen 3 - Rembrandt)
	 */
#define	A_SKTS_REMBRANDT			23
	{
		[0] = X86_SOCKET_AM5,
		[1] = X86_SOCKET_FP7,
		[2] = X86_SOCKET_FP7R2
	},

	/*
	 * Family 0x19 models 60-6f	(Zen 4 - Raphael)
	 */
#define	A_SKTS_RAPHAEL			24
	{
		[0] = X86_SOCKET_AM5,
		[1] = X86_SOCKET_FL1
	},

	/*
	 * The always-unknown socket group, used for undocumented parts.  It
	 * need not be last; the position is arbitrary. The default initializer
	 * for this is zero which is x86 socket unknown.
	 */
#define	A_SKTS_UNKNOWN			25
	{
	},
	/*
	 * Family 0x17 models 90-97	(Zen 2 - Van Gogh)
	 */
#define	A_SKTS_VANGOGH			26
	{
		[3] = X86_SOCKET_FF3
	},
	/*
	 * Family 0x17 models a0-af	(Zen 2 - Mendocino)
	 */
#define	A_SKTS_MENDOCINO			27
	{
		[1] = X86_SOCKET_FT6
	},

	/*
	 * Family 0x19 models 70-7f	(Zen 4 - Phoenix)
	 */
#define	A_SKTS_PHOENIX			28
	{
		[0] = X86_SOCKET_AM5,
		[1] = X86_SOCKET_FP8,
		[4] = X86_SOCKET_FP7,
		[5] = X86_SOCKET_FP7R2,
	},

	/*
	 * Family 0x19 models a0-af	(Zen 4c - Bergamo/Siena)
	 */
#define	A_SKTS_BERGAMO			29
	{
		[4] = X86_SOCKET_SP5,
		[8] = X86_SOCKET_SP6
	},

	/*
	 * Family 0x1a models 00-1f	(Zen 5[c] - Turin)
	 */
#define	A_SKTS_TURIN			30
	{
		[4] = X86_SOCKET_SP5,
	},

	/*
	 * Family 0x1a model 08		(Zen 5 - Shimada Peak)
	 */
#define	A_SKTS_SHIMADA_PEAK		31
	{
		[7] = X86_SOCKET_TR5
	},

	/*
	 * Family 0x1a models 20-2f	(Zen 5 - Strix)
	 * Family 0x1a models 60-6f	(Zen 5 - Krackan)
	 */
#define	A_SKTS_STRIX			32
	{
		[0] = X86_SOCKET_AM5,
		[1] = X86_SOCKET_FP8
	},

	/*
	 * Family 0x1a models 40-4f	(Zen 5 - Granite Ridge)
	 */
#define	A_SKTS_GRANITE_RIDGE		33
	{
		[0] = X86_SOCKET_AM5,
		[1] = X86_SOCKET_FL1
	},

	/*
	 * Family 0x1a models 70-77	(Zen 5 - Strix Halo)
	 */
#define	A_SKTS_STRIX_HALO		34
	{
		[1] = X86_SOCKET_FP11
	}
};

struct amd_sktmap_s {
	uint32_t	skt_code;
	char		sktstr[16];
};
static struct amd_sktmap_s amd_sktmap_strs[] = {
	{ X86_SOCKET_754,	"754" },
	{ X86_SOCKET_939,	"939" },
	{ X86_SOCKET_940,	"940" },
	{ X86_SOCKET_S1g1,	"S1g1" },
	{ X86_SOCKET_AM2,	"AM2" },
	{ X86_SOCKET_F1207,	"F(1207)" },
	{ X86_SOCKET_S1g2,	"S1g2" },
	{ X86_SOCKET_S1g3,	"S1g3" },
	{ X86_SOCKET_AM,	"AM" },
	{ X86_SOCKET_AM2R2,	"AM2r2" },
	{ X86_SOCKET_AM3,	"AM3" },
	{ X86_SOCKET_G34,	"G34" },
	{ X86_SOCKET_ASB2,	"ASB2" },
	{ X86_SOCKET_C32,	"C32" },
	{ X86_SOCKET_S1g4,	"S1g4" },
	{ X86_SOCKET_FT1,	"FT1" },
	{ X86_SOCKET_FM1,	"FM1" },
	{ X86_SOCKET_FS1,	"FS1" },
	{ X86_SOCKET_AM3R2,	"AM3r2" },
	{ X86_SOCKET_FP2,	"FP2" },
	{ X86_SOCKET_FS1R2,	"FS1r2" },
	{ X86_SOCKET_FM2,	"FM2" },
	{ X86_SOCKET_FP3,	"FP3" },
	{ X86_SOCKET_FM2R2,	"FM2r2" },
	{ X86_SOCKET_FP4,	"FP4" },
	{ X86_SOCKET_AM4,	"AM4" },
	{ X86_SOCKET_FT3,	"FT3" },
	{ X86_SOCKET_FT4,	"FT4" },
	{ X86_SOCKET_FS1B,	"FS1b" },
	{ X86_SOCKET_FT3B,	"FT3b" },
	{ X86_SOCKET_SP3,	"SP3" },
	{ X86_SOCKET_SP3R2,	"SP3r2" },
	{ X86_SOCKET_FP5,	"FP5" },
	{ X86_SOCKET_FP6,	"FP6" },
	{ X86_SOCKET_STRX4,	"sTRX4" },
	{ X86_SOCKET_SL1,	"SL1" },
	{ X86_SOCKET_SL1R2,	"SL1R2" },
	{ X86_SOCKET_DM1,	"DM1" },
	{ X86_SOCKET_SP5,	"SP5" },
	{ X86_SOCKET_AM5,	"AM5" },
	{ X86_SOCKET_FP7,	"FP7" },
	{ X86_SOCKET_FP7R2,	"FP7r2" },
	{ X86_SOCKET_FF3,	"FF3" },
	{ X86_SOCKET_FT6,	"FT6" },
	{ X86_SOCKET_FP8,	"FP8" },
	{ X86_SOCKET_FL1,	"FL1" },
	{ X86_SOCKET_SP6,	"SP6" },
	{ X86_SOCKET_TR5,	"TR5" },
	{ X86_SOCKET_FP11,	"FP11" },
	{ X86_SOCKET_UNKNOWN,	"Unknown" }	/* Must be last! */
};

/* Keep the array above in sync with the definitions in x86_archext.h. */
CTASSERT(ARRAY_SIZE(amd_sktmap_strs) == X86_NUM_SOCKETS + 1);

/*
 * Table for mapping AMD family/model/stepping ranges onto three derived items:
 *
 * * The "chiprev" and associated string, which is generally the AMD silicon
 * revision along with a symbolic representation of the marketing (not cpuid)
 * family.  In line with the overall cpuid usage, we refer to this as a
 * processor family.
 * * The uarch, which is analogous to the chiprev and provides the
 * microarchitecture/core generation and silicon revision.  Note that this is
 * distinct from the package-level silicon/product revision and is often common
 * to multiple product lines offered at a given time.
 * * The socket map selector, used to translate this collection of products'
 * last 4 model bits (for family 0xf only) or Fn8000_0001_EBX[30:28] into a
 * socket ID.
 *
 * The first member of this array that matches a given family, extended model
 * plus model range, and stepping range will be considered a match.  This allows
 * us to end each cpuid family and/or processor family with a catchall that
 * while less specific than we might like still allows us to provide a fair
 * amount of detail to both other kernel consumers and userland.
 */
static const struct amd_rev_mapent {
	uint_t rm_family;
	uint_t rm_modello;
	uint_t rm_modelhi;
	uint_t rm_steplo;
	uint_t rm_stephi;
	x86_chiprev_t rm_chiprev;
	const char *rm_chiprevstr;
	x86_uarchrev_t rm_uarchrev;
	uint_t rm_sktidx;
} amd_revmap[] = {
	/*
	 * =============== AuthenticAMD Family 0xf ===============
	 */

	/*
	 * Rev B includes model 0x4 stepping 0 and model 0x5 stepping 0 and 1.
	 */
	{ 0xf, 0x04, 0x04, 0x0, 0x0, X86_CHIPREV_AMD_LEGACY_F_REV_B, "B",
	    X86_UARCHREV_AMD_LEGACY, A_SKTS_0 },
	{ 0xf, 0x05, 0x05, 0x0, 0x1, X86_CHIPREV_AMD_LEGACY_F_REV_B, "B",
	    X86_UARCHREV_AMD_LEGACY, A_SKTS_0 },
	/*
	 * Rev C0 includes model 0x4 stepping 8 and model 0x5 stepping 8
	 */
	{ 0xf, 0x04, 0x05, 0x8, 0x8, X86_CHIPREV_AMD_LEGACY_F_REV_C0, "C0",
	    X86_UARCHREV_AMD_LEGACY, A_SKTS_0 },
	/*
	 * Rev CG is the rest of extended model 0x0 - i.e., everything
	 * but the rev B and C0 combinations covered above.
	 */
	{ 0xf, 0x00, 0x0f, 0x0, 0xf, X86_CHIPREV_AMD_LEGACY_F_REV_CG, "CG",
	    X86_UARCHREV_AMD_LEGACY, A_SKTS_0 },
	/*
	 * Rev D has extended model 0x1.
	 */
	{ 0xf, 0x10, 0x1f, 0x0, 0xf, X86_CHIPREV_AMD_LEGACY_F_REV_D, "D",
	    X86_UARCHREV_AMD_LEGACY, A_SKTS_0 },
	/*
	 * Rev E has extended model 0x2.
	 * Extended model 0x3 is unused but available to grow into.
	 */
	{ 0xf, 0x20, 0x3f, 0x0, 0xf, X86_CHIPREV_AMD_LEGACY_F_REV_E, "E",
	    X86_UARCHREV_AMD_LEGACY, A_SKTS_0 },
	/*
	 * Rev F has extended models 0x4 and 0x5.
	 */
	{ 0xf, 0x40, 0x5f, 0x0, 0xf, X86_CHIPREV_AMD_LEGACY_F_REV_F, "F",
	    X86_UARCHREV_AMD_LEGACY, A_SKTS_1 },
	/*
	 * Rev G has extended model 0x6.
	 */
	{ 0xf, 0x60, 0x6f, 0x0, 0xf, X86_CHIPREV_AMD_LEGACY_F_REV_G, "G",
	    X86_UARCHREV_AMD_LEGACY, A_SKTS_1 },

	/*
	 * =============== AuthenticAMD Family 0x10 ===============
	 */

	/*
	 * Rev A has model 0 and stepping 0/1/2 for DR-{A0,A1,A2}.
	 * Give all of model 0 stepping range to rev A.
	 */
	{ 0x10, 0x00, 0x00, 0x0, 0x2, X86_CHIPREV_AMD_LEGACY_10_REV_A, "A",
	    X86_UARCHREV_AMD_LEGACY, A_SKTS_2 },

	/*
	 * Rev B has model 2 and steppings 0/1/0xa/2 for DR-{B0,B1,BA,B2}.
	 * Give all of model 2 stepping range to rev B.
	 */
	{ 0x10, 0x02, 0x02, 0x0, 0xf, X86_CHIPREV_AMD_LEGACY_10_REV_B, "B",
	    X86_UARCHREV_AMD_LEGACY, A_SKTS_2 },

	/*
	 * Rev C has models 4-6 (depending on L3 cache configuration)
	 * Give all of models 4-6 stepping range 0-2 to rev C2.
	 */
	{ 0x10, 0x4, 0x6, 0x0, 0x2, X86_CHIPREV_AMD_LEGACY_10_REV_C2, "C2",
	    X86_UARCHREV_AMD_LEGACY, A_SKTS_2 },

	/*
	 * Rev C has models 4-6 (depending on L3 cache configuration)
	 * Give all of models 4-6 stepping range >= 3 to rev C3.
	 */
	{ 0x10, 0x4, 0x6, 0x3, 0xf, X86_CHIPREV_AMD_LEGACY_10_REV_C3, "C3",
	    X86_UARCHREV_AMD_LEGACY, A_SKTS_2 },

	/*
	 * Rev D has models 8 and 9
	 * Give all of model 8 and 9 stepping 0 to rev D0.
	 */
	{ 0x10, 0x8, 0x9, 0x0, 0x0, X86_CHIPREV_AMD_LEGACY_10_REV_D0, "D0",
	    X86_UARCHREV_AMD_LEGACY, A_SKTS_2 },

	/*
	 * Rev D has models 8 and 9
	 * Give all of model 8 and 9 stepping range >= 1 to rev D1.
	 */
	{ 0x10, 0x8, 0x9, 0x1, 0xf, X86_CHIPREV_AMD_LEGACY_10_REV_D1, "D1",
	    X86_UARCHREV_AMD_LEGACY, A_SKTS_2 },

	/*
	 * Rev E has models A and stepping 0
	 * Give all of model A stepping range to rev E.
	 */
	{ 0x10, 0xA, 0xA, 0x0, 0xf, X86_CHIPREV_AMD_LEGACY_10_REV_E, "E",
	    X86_UARCHREV_AMD_LEGACY, A_SKTS_2 },

	{ 0x10, 0x0, 0xff, 0x0, 0xf, X86_CHIPREV_AMD_LEGACY_10_UNKNOWN, "??",
	    X86_UARCHREV_AMD_LEGACY, A_SKTS_2 },

	/*
	 * =============== AuthenticAMD Family 0x11 ===============
	 */
	{ 0x11, 0x03, 0x03, 0x0, 0xf, X86_CHIPREV_AMD_LEGACY_11_REV_B, "B",
	    X86_UARCHREV_AMD_LEGACY, A_SKTS_3 },
	{ 0x11, 0x00, 0xff, 0x0, 0xf, X86_CHIPREV_AMD_LEGACY_11_UNKNOWN, "??",
	    X86_UARCHREV_AMD_LEGACY, A_SKTS_3 },

	/*
	 * =============== AuthenticAMD Family 0x12 ===============
	 */
	{ 0x12, 0x01, 0x01, 0x0, 0xf, X86_CHIPREV_AMD_LEGACY_12_REV_B, "B",
	    X86_UARCHREV_AMD_LEGACY, A_SKTS_4 },
	{ 0x12, 0x00, 0x00, 0x0, 0xf, X86_CHIPREV_AMD_LEGACY_12_UNKNOWN, "??",
	    X86_UARCHREV_AMD_LEGACY, A_SKTS_4 },

	/*
	 * =============== AuthenticAMD Family 0x14 ===============
	 */
	{ 0x14, 0x01, 0x01, 0x0, 0xf, X86_CHIPREV_AMD_LEGACY_14_REV_B, "B",
	    X86_UARCHREV_AMD_LEGACY, A_SKTS_5 },
	{ 0x14, 0x02, 0x02, 0x0, 0xf, X86_CHIPREV_AMD_LEGACY_14_REV_C, "C",
	    X86_UARCHREV_AMD_LEGACY, A_SKTS_5 },
	{ 0x14, 0x00, 0xff, 0x0, 0xf, X86_CHIPREV_AMD_LEGACY_14_UNKNOWN, "??",
	    X86_UARCHREV_AMD_LEGACY, A_SKTS_5 },

	/*
	 * =============== AuthenticAMD Family 0x15 ===============
	 */
	{ 0x15, 0x01, 0x01, 0x2, 0x2, X86_CHIPREV_AMD_OROCHI_REV_B2, "OR-B2",
	    X86_UARCHREV_AMD_LEGACY, A_SKTS_6 },
	{ 0x15, 0x02, 0x02, 0x0, 0x0, X86_CHIPREV_AMD_OROCHI_REV_C0, "OR-C0",
	    X86_UARCHREV_AMD_LEGACY, A_SKTS_6 },
	{ 0x15, 0x00, 0x0f, 0x0, 0xf, X86_CHIPREV_AMD_OROCHI_UNKNOWN, "OR-??",
	    X86_UARCHREV_AMD_LEGACY, A_SKTS_6 },

	{ 0x15, 0x10, 0x10, 0x1, 0x1, X86_CHIPREV_AMD_TRINITY_REV_A1, "TN-A1",
	    X86_UARCHREV_AMD_LEGACY, A_SKTS_7 },
	{ 0x15, 0x10, 0x1f, 0x0, 0xf, X86_CHIPREV_AMD_TRINITY_UNKNOWN, "TN-??",
	    X86_UARCHREV_AMD_LEGACY, A_SKTS_7 },

	{ 0x15, 0x30, 0x30, 0x1, 0x1, X86_CHIPREV_AMD_KAVERI_REV_A1, "KV-A1",
	    X86_UARCHREV_AMD_LEGACY, A_SKTS_8 },
	{ 0x15, 0x30, 0x3f, 0x0, 0xf, X86_CHIPREV_AMD_KAVERI_UNKNOWN, "KV-??",
	    X86_UARCHREV_AMD_LEGACY, A_SKTS_8 },

	/*
	 * The Carrizo rev guide mentions A0 as having an ID of "00600F00h" but
	 * this appears to be a typo as elsewhere it's given as "00660F00h".  We
	 * assume the latter is correct.
	 */
	{ 0x15, 0x60, 0x60, 0x0, 0x0, X86_CHIPREV_AMD_CARRIZO_REV_A0, "CZ-A0",
	    X86_UARCHREV_AMD_LEGACY, A_SKTS_9 },
	{ 0x15, 0x60, 0x60, 0x1, 0x1, X86_CHIPREV_AMD_CARRIZO_REV_A1, "CZ-A1",
	    X86_UARCHREV_AMD_LEGACY, A_SKTS_9 },
	/*
	 * CZ-DDR4 and BR-A1 are indistinguishable via cpuid; the rev guide
	 * indicates that they should be distinguished by the contents of the
	 * OSVW MSR, but this register is just a software scratch space which
	 * means the actual method of distinguishing the two is not documented
	 * and on PCs will be done by a BIOS.  In the extremely unlikely event
	 * it becomes necessary to distinguish these, an OSVW-driven fixup can
	 * be added.
	 */
	{ 0x15, 0x65, 0x65, 0x1, 0x1, X86_CHIPREV_AMD_CARRIZO_REV_DDR4,
	    "CZ-DDR4", X86_UARCHREV_AMD_LEGACY, A_SKTS_9 },
	{ 0x15, 0x60, 0x6f, 0x0, 0xf, X86_CHIPREV_AMD_CARRIZO_UNKNOWN, "CZ-??",
	    X86_UARCHREV_AMD_LEGACY, A_SKTS_9 },

	{ 0x15, 0x70, 0x70, 0x0, 0x0, X86_CHIPREV_AMD_STONEY_RIDGE_REV_A0,
	    "ST-A0", X86_UARCHREV_AMD_LEGACY, A_SKTS_10 },
	{ 0x15, 0x70, 0x7f, 0x0, 0xf, X86_CHIPREV_AMD_STONEY_RIDGE_UNKNOWN,
	    "ST-??", X86_UARCHREV_AMD_LEGACY, A_SKTS_10 },

	/*
	 * =============== AuthenticAMD Family 0x16 ===============
	 */
	{ 0x16, 0x00, 0x00, 0x1, 0x1, X86_CHIPREV_AMD_KABINI_A1, "KB-A1",
	    X86_UARCHREV_AMD_LEGACY, A_SKTS_11 },
	{ 0x16, 0x00, 0x0f, 0x0, 0xf, X86_CHIPREV_AMD_KABINI_UNKNOWN, "KB-??",
	    X86_UARCHREV_AMD_LEGACY, A_SKTS_11 },

	{ 0x16, 0x30, 0x30, 0x1, 0x1, X86_CHIPREV_AMD_MULLINS_A1, "ML-A1",
	    X86_UARCHREV_AMD_LEGACY, A_SKTS_12 },
	{ 0x16, 0x30, 0x3f, 0x0, 0xf, X86_CHIPREV_AMD_MULLINS_UNKNOWN, "ML-??",
	    X86_UARCHREV_AMD_LEGACY, A_SKTS_12 },

	/*
	 * =============== AuthenticAMD Family 0x17 ===============
	 */
	/* Naples == Zeppelin == ZP */
	{ 0x17, 0x00, 0x00, 0x0, 0x0, X86_CHIPREV_AMD_NAPLES_A0, "ZP-A0",
	    X86_UARCHREV_AMD_ZEN1, A_SKTS_NAPLES },
	{ 0x17, 0x01, 0x01, 0x1, 0x1, X86_CHIPREV_AMD_NAPLES_B1, "ZP-B1",
	    X86_UARCHREV_AMD_ZEN1, A_SKTS_NAPLES },
	{ 0x17, 0x01, 0x01, 0x2, 0x2, X86_CHIPREV_AMD_NAPLES_B2, "ZP-B2",
	    X86_UARCHREV_AMD_ZEN1, A_SKTS_NAPLES },
	{ 0x17, 0x00, 0x07, 0x0, 0xf, X86_CHIPREV_AMD_NAPLES_UNKNOWN, "ZP-??",
	    X86_UARCHREV_AMD_ZEN1, A_SKTS_NAPLES },
	{ 0x17, 0x08, 0x08, 0x2, 0x2, X86_CHIPREV_AMD_PINNACLE_RIDGE_B2,
	    "PiR-B2", X86_UARCHREV_AMD_ZENPLUS, A_SKTS_NAPLES },
	{ 0x17, 0x08, 0x0f, 0x0, 0xf, X86_CHIPREV_AMD_PINNACLE_RIDGE_UNKNOWN,
	    "PiR-??", X86_UARCHREV_AMD_ZENPLUS, A_SKTS_NAPLES },

	{ 0x17, 0x11, 0x11, 0x0, 0x0, X86_CHIPREV_AMD_RAVEN_RIDGE_B0,
	    "RV-B0", X86_UARCHREV_AMD_ZEN1, A_SKTS_RAVEN },
	{ 0x17, 0x11, 0x11, 0x1, 0x1, X86_CHIPREV_AMD_RAVEN_RIDGE_B1,
	    "RV-B1", X86_UARCHREV_AMD_ZEN1, A_SKTS_RAVEN },
	{ 0x17, 0x10, 0x17, 0x0, 0xf, X86_CHIPREV_AMD_RAVEN_RIDGE_UNKNOWN,
	    "RV-??", X86_UARCHREV_AMD_ZEN1, A_SKTS_RAVEN },
	{ 0x17, 0x18, 0x18, 0x1, 0x1, X86_CHIPREV_AMD_PICASSO_B1, "PCO-B1",
	    X86_UARCHREV_AMD_ZENPLUS, A_SKTS_RAVEN },
	{ 0x17, 0x18, 0x1f, 0x0, 0xf, X86_CHIPREV_AMD_PICASSO_UNKNOWN, "PCO-??",
	    X86_UARCHREV_AMD_ZENPLUS, A_SKTS_RAVEN },

	{ 0x17, 0x20, 0x20, 0x1, 0x1, X86_CHIPREV_AMD_DALI_A1, "RV2X-A1",
	    X86_UARCHREV_AMD_ZEN1, A_SKTS_RAVEN },
	{ 0x17, 0x20, 0x2f, 0x0, 0xf, X86_CHIPREV_AMD_DALI_UNKNOWN, "RV2X-??",
	    X86_UARCHREV_AMD_ZEN1, A_SKTS_RAVEN },

	/* Rome == Starship == SSP */
	{ 0x17, 0x30, 0x30, 0x0, 0x0, X86_CHIPREV_AMD_ROME_A0, "SSP-A0",
	    X86_UARCHREV_AMD_ZEN2_A0, A_SKTS_ROME },
	{ 0x17, 0x31, 0x31, 0x0, 0x0, X86_CHIPREV_AMD_ROME_B0, "SSP-B0",
	    X86_UARCHREV_AMD_ZEN2_B0, A_SKTS_ROME },
	{ 0x17, 0x30, 0x3f, 0x0, 0xf, X86_CHIPREV_AMD_ROME_UNKNOWN, "SSP-??",
	    X86_UARCHREV_AMD_ZEN2_UNKNOWN, A_SKTS_ROME },

	{ 0x17, 0x60, 0x60, 0x1, 0x1, X86_CHIPREV_AMD_RENOIR_A1, "RN-A1",
	    X86_UARCHREV_AMD_ZEN2_B0, A_SKTS_RENOIR },
	{ 0x17, 0x60, 0x67, 0x0, 0xf, X86_CHIPREV_AMD_RENOIR_UNKNOWN, "RN-??",
	    X86_UARCHREV_AMD_ZEN2_UNKNOWN, A_SKTS_RENOIR },
	{ 0x17, 0x68, 0x68, 0x1, 0x1, X86_CHIPREV_AMD_RENOIR_LCN_A1, "LCN-A1",
	    X86_UARCHREV_AMD_ZEN2_B0, A_SKTS_RENOIR },
	{ 0x17, 0x68, 0x6f, 0x0, 0xf, X86_CHIPREV_AMD_RENOIR_UNKNOWN, "LCN-??",
	    X86_UARCHREV_AMD_ZEN2_UNKNOWN, A_SKTS_RENOIR },

	{ 0x17, 0x71, 0x71, 0x0, 0x0, X86_CHIPREV_AMD_MATISSE_B0, "MTS-B0",
	    X86_UARCHREV_AMD_ZEN2_B0, A_SKTS_MATISSE },
	{ 0x17, 0x70, 0x7f, 0x0, 0xf, X86_CHIPREV_AMD_MATISSE_UNKNOWN, "MTS-??",
	    X86_UARCHREV_AMD_ZEN2_UNKNOWN, A_SKTS_MATISSE },

	{ 0x17, 0x90, 0x97, 0x0, 0xf, X86_CHIPREV_AMD_VAN_GOGH_UNKNOWN, "??",
	    X86_UARCHREV_AMD_ZEN2_UNKNOWN, A_SKTS_VANGOGH },
	{ 0x17, 0x98, 0x9f, 0x0, 0xf, X86_CHIPREV_AMD_VAN_GOGH_UNKNOWN, "??",
	    X86_UARCHREV_AMD_ZEN2_UNKNOWN, A_SKTS_UNKNOWN },

	{ 0x17, 0xa0, 0xaf, 0x0, 0xf, X86_CHIPREV_AMD_MENDOCINO_UNKNOWN, "??",
	    X86_UARCHREV_AMD_ZEN2_UNKNOWN, A_SKTS_MENDOCINO },

	/*
	 * =============== HygonGenuine Family 0x18 ===============
	 */
	{ 0x18, 0x00, 0x00, 0x1, 0x1, X86_CHIPREV_HYGON_DHYANA_A1, "DN_A1",
	    X86_UARCHREV_AMD_ZEN1, A_SKTS_DHYANA },
	{ 0x18, 0x00, 0x0f, 0x0, 0xf, X86_CHIPREV_HYGON_DHYANA_UNKNOWN, "DN_??",
	    X86_UARCHREV_AMD_ZEN1, A_SKTS_DHYANA },

	/*
	 * =============== AuthenticAMD Family 0x19 ===============
	 */
	/* Milan == Genesis == GN */
	{ 0x19, 0x00, 0x00, 0x0, 0x0, X86_CHIPREV_AMD_MILAN_A0, "GN-A0",
	    X86_UARCHREV_AMD_ZEN3_A0, A_SKTS_MILAN },
	{ 0x19, 0x01, 0x01, 0x0, 0x0, X86_CHIPREV_AMD_MILAN_B0, "GN-B0",
	    X86_UARCHREV_AMD_ZEN3_B0, A_SKTS_MILAN },
	{ 0x19, 0x01, 0x01, 0x1, 0x1, X86_CHIPREV_AMD_MILAN_B1, "GN-B1",
	    X86_UARCHREV_AMD_ZEN3_B1, A_SKTS_MILAN },
	/* Marketed as Milan-X but still GN */
	{ 0x19, 0x01, 0x01, 0x2, 0x2, X86_CHIPREV_AMD_MILAN_B2, "GN-B2",
	    X86_UARCHREV_AMD_ZEN3_B2, A_SKTS_MILAN },
	{ 0x19, 0x00, 0x0f, 0x0, 0xf, X86_CHIPREV_AMD_MILAN_UNKNOWN, "GN-??",
	    X86_UARCHREV_AMD_ZEN3_UNKNOWN, A_SKTS_MILAN },

	/* Genoa == Stones == RS */
	{ 0x19, 0x10, 0x10, 0x0, 0x0, X86_CHIPREV_AMD_GENOA_A0, "RS-A0",
	    X86_UARCHREV_AMD_ZEN4_A0, A_SKTS_GENOA },
	/* RS-A0 & RS-A1 both map to Zen 4 uarch A0 */
	{ 0x19, 0x10, 0x10, 0x1, 0x1, X86_CHIPREV_AMD_GENOA_A1, "RS-A1",
	    X86_UARCHREV_AMD_ZEN4_A0, A_SKTS_GENOA },
	{ 0x19, 0x11, 0x11, 0x0, 0x0, X86_CHIPREV_AMD_GENOA_B0, "RS-B0",
	    X86_UARCHREV_AMD_ZEN4_B0, A_SKTS_GENOA },
	{ 0x19, 0x11, 0x11, 0x1, 0x1, X86_CHIPREV_AMD_GENOA_B1, "RS-B1",
	    X86_UARCHREV_AMD_ZEN4_B1, A_SKTS_GENOA },
	{ 0x19, 0x10, 0x1f, 0x0, 0xf, X86_CHIPREV_AMD_GENOA_UNKNOWN, "RS-??",
	    X86_UARCHREV_AMD_ZEN4_UNKNOWN, A_SKTS_GENOA },

	{ 0x19, 0x20, 0x20, 0x0, 0x0, X86_CHIPREV_AMD_VERMEER_A0, "VMR-A0",
	    X86_UARCHREV_AMD_ZEN3_A0, A_SKTS_VERMEER },
	{ 0x19, 0x21, 0x21, 0x0, 0x0, X86_CHIPREV_AMD_VERMEER_B0, "VMR-B0",
	    X86_UARCHREV_AMD_ZEN3_B0, A_SKTS_VERMEER },
	{ 0x19, 0x21, 0x21, 0x2, 0x2, X86_CHIPREV_AMD_VERMEER_B2, "VMR-B2",
	    X86_UARCHREV_AMD_ZEN3_B2, A_SKTS_VERMEER },
	{ 0x19, 0x20, 0x2f, 0x0, 0xf, X86_CHIPREV_AMD_VERMEER_UNKNOWN, "VMR-??",
	    X86_UARCHREV_AMD_ZEN3_UNKNOWN, A_SKTS_VERMEER },

	/* Rev guide is missing AM5 information, including A0 and B0 */
	{ 0x19, 0x40, 0x40, 0x0, 0x0, X86_CHIPREV_AMD_REMBRANDT_A0, "RMB-A0",
	    X86_UARCHREV_AMD_ZEN3_B0, A_SKTS_REMBRANDT },
	{ 0x19, 0x44, 0x44, 0x0, 0x0, X86_CHIPREV_AMD_REMBRANDT_B0, "RMB-B0",
	    X86_UARCHREV_AMD_ZEN3_B0, A_SKTS_REMBRANDT },
	{ 0x19, 0x44, 0x44, 0x1, 0x1, X86_CHIPREV_AMD_REMBRANDT_B1, "RMB-B1",
	    X86_UARCHREV_AMD_ZEN3_B0, A_SKTS_REMBRANDT },
	{ 0x19, 0x40, 0x4f, 0x0, 0xf, X86_CHIPREV_AMD_REMBRANDT_UNKNOWN,
	    "RMB-??", X86_UARCHREV_AMD_ZEN3_UNKNOWN, A_SKTS_REMBRANDT },

	/* Cezanne */
	{ 0x19, 0x50, 0x50, 0x0, 0x0, X86_CHIPREV_AMD_CEZANNE_A0, "CZN-A0",
	    X86_UARCHREV_AMD_ZEN3_B0, A_SKTS_CEZANNE },
	{ 0x19, 0x50, 0x5f, 0x0, 0xf, X86_CHIPREV_AMD_CEZANNE_UNKNOWN, "CZN-??",
	    X86_UARCHREV_AMD_ZEN3_UNKNOWN, A_SKTS_CEZANNE },

	/* Raphael */
	{ 0x19, 0x61, 0x61, 0x2, 0x2, X86_CHIPREV_AMD_RAPHAEL_B2, "RPL-B2",
	    X86_UARCHREV_AMD_ZEN4_B2, A_SKTS_RAPHAEL },
	{ 0x19, 0x60, 0x6f, 0x0, 0xf, X86_CHIPREV_AMD_RAPHAEL_UNKNOWN, "RPL-??",
	    X86_UARCHREV_AMD_ZEN4_UNKNOWN, A_SKTS_RAPHAEL },

	/* Phoenix */
	{ 0x19, 0x74, 0x74, 0x1, 0x1, X86_CHIPREV_AMD_PHOENIX_A1, "PHX-A1",
	    X86_UARCHREV_AMD_ZEN4_A1, A_SKTS_PHOENIX },
	{ 0x19, 0x78, 0x78, 0x0, 0x0, X86_CHIPREV_AMD_PHOENIX_A1, "PHX2-A0",
	    X86_UARCHREV_AMD_ZEN4_A1, A_SKTS_PHOENIX },
	{ 0x19, 0x7C, 0x7C, 0x0, 0x0, X86_CHIPREV_AMD_PHOENIX_A1, "HPT2-A0",
	    X86_UARCHREV_AMD_ZEN4_A1, A_SKTS_PHOENIX },
	{ 0x19, 0x70, 0x7f, 0x0, 0xf, X86_CHIPREV_AMD_PHOENIX_UNKNOWN, "PHX-??",
	    X86_UARCHREV_AMD_ZEN4_UNKNOWN, A_SKTS_PHOENIX },

	/* Bergamo / Siena */
	{ 0x19, 0xa0, 0xaf, 0x0, 0x0, X86_CHIPREV_AMD_BERGAMO_A0, "RSDN-A0",
	    X86_UARCHREV_AMD_ZEN4_A0, A_SKTS_BERGAMO },
	{ 0x19, 0xa0, 0xaf, 0x1, 0x1, X86_CHIPREV_AMD_BERGAMO_A1, "RSDN-A1",
	    X86_UARCHREV_AMD_ZEN4_A1, A_SKTS_BERGAMO },
	{ 0x19, 0xa0, 0xaf, 0x2, 0x2, X86_CHIPREV_AMD_BERGAMO_A2, "RSDN-A2",
	    X86_UARCHREV_AMD_ZEN4_A2, A_SKTS_BERGAMO },
	{ 0x19, 0xa0, 0xaf, 0x0, 0xf, X86_CHIPREV_AMD_BERGAMO_UNKNOWN, "???",
	    X86_UARCHREV_AMD_ZEN4_UNKNOWN, A_SKTS_BERGAMO },

	/*
	 * =============== AuthenticAMD Family 0x1a ===============
	 */
	/* Turin */
	{ 0x1a, 0x00, 0x00, 0x0, 0x0, X86_CHIPREV_AMD_TURIN_A0, "BRH-A0",
	    X86_UARCHREV_AMD_ZEN5_A0, A_SKTS_TURIN },
	/* BRH-A0 & BRH-B0 both map to Zen 5 uarch A0 */
	{ 0x1a, 0x01, 0x01, 0x0, 0x0, X86_CHIPREV_AMD_TURIN_B0, "BRH-B0",
	    X86_UARCHREV_AMD_ZEN5_A0, A_SKTS_TURIN },
	/* BRH-B1 maps to Zen 5 uarch B0 */
	{ 0x1a, 0x01, 0x01, 0x1, 0x1, X86_CHIPREV_AMD_TURIN_B1, "BRH-B1",
	    X86_UARCHREV_AMD_ZEN5_B0, A_SKTS_TURIN },
	{ 0x1a, 0x02, 0x02, 0x0, 0x0, X86_CHIPREV_AMD_TURIN_C0, "BRH-C0",
	    X86_UARCHREV_AMD_ZEN5_C0, A_SKTS_TURIN },
	{ 0x1a, 0x02, 0x02, 0x1, 0x1, X86_CHIPREV_AMD_TURIN_C1, "BRH-C1",
	    X86_UARCHREV_AMD_ZEN5_C1, A_SKTS_TURIN },
	{ 0x1a, 0x08, 0x08, 0x0, 0x0, X86_CHIPREV_AMD_SHIMADA_PEAK_C0,
	    "SHP-C0", X86_UARCHREV_AMD_ZEN5_C0, A_SKTS_SHIMADA_PEAK },
	{ 0x1a, 0x08, 0x08, 0x1, 0x1, X86_CHIPREV_AMD_SHIMADA_PEAK_C1,
	    "SHP-C1", X86_UARCHREV_AMD_ZEN5_C1, A_SKTS_SHIMADA_PEAK },
	{ 0x1a, 0x08, 0x08, 0x0, 0xf, X86_CHIPREV_AMD_SHIMADA_PEAK_UNKNOWN,
	    "SHP-???", X86_UARCHREV_AMD_ZEN5_UNKNOWN, A_SKTS_SHIMADA_PEAK },
	{ 0x1a, 0x00, 0x0f, 0x0, 0xf, X86_CHIPREV_AMD_TURIN_UNKNOWN, "BRH-???",
	    X86_UARCHREV_AMD_ZEN5_UNKNOWN, A_SKTS_TURIN },
	{ 0x1a, 0x10, 0x10, 0x0, 0x0, X86_CHIPREV_AMD_DENSE_TURIN_A0,
	    "BRHD-A0", X86_UARCHREV_AMD_ZEN5_A0, A_SKTS_TURIN },
	{ 0x1a, 0x11, 0x11, 0x0, 0x0, X86_CHIPREV_AMD_DENSE_TURIN_B0,
	    "BRHD-B0", X86_UARCHREV_AMD_ZEN5_B0, A_SKTS_TURIN },
	/* BRHD-B0 & BRHD-B1 both map to Zen 5 uarch B0 */
	{ 0x1a, 0x11, 0x11, 0x1, 0x1, X86_CHIPREV_AMD_DENSE_TURIN_B1,
	    "BRHD-B1", X86_UARCHREV_AMD_ZEN5_B0, A_SKTS_TURIN },
	{ 0x1a, 0x10, 0x1f, 0x0, 0xf, X86_CHIPREV_AMD_DENSE_TURIN_UNKNOWN,
	    "BRHD-???", X86_UARCHREV_AMD_ZEN5_UNKNOWN, A_SKTS_TURIN },

	/* Strix and Krackan */
	{ 0x1a, 0x24, 0x24, 0x0, 0x0, X86_CHIPREV_AMD_STRIX_B0,
	    "STX-B0", X86_UARCHREV_AMD_ZEN5_B0, A_SKTS_STRIX },
	{ 0x1a, 0x20, 0x2f, 0x0, 0xf, X86_CHIPREV_AMD_STRIX_UNKNOWN,
	    "STX-???", X86_UARCHREV_AMD_ZEN5_UNKNOWN, A_SKTS_STRIX },
	{ 0x1a, 0x60, 0x60, 0x0, 0x0, X86_CHIPREV_AMD_KRACKAN_A0,
	    "KRK-A0", X86_UARCHREV_AMD_ZEN5_A0, A_SKTS_STRIX },
	{ 0x1a, 0x68, 0x68, 0x0, 0x0, X86_CHIPREV_AMD_KRACKAN_A0,
	    "KRK2-A0", X86_UARCHREV_AMD_ZEN5_A0, A_SKTS_STRIX },
	{ 0x1a, 0x60, 0x6f, 0x0, 0xf, X86_CHIPREV_AMD_KRACKAN_UNKNOWN,
	    "KRK-???", X86_UARCHREV_AMD_ZEN5_UNKNOWN, A_SKTS_STRIX },

	/* Granite Ridge */
	{ 0x1a, 0x44, 0x44, 0x0, 0x0, X86_CHIPREV_AMD_GRANITE_RIDGE_B0,
	    "GNR-B0", X86_UARCHREV_AMD_ZEN5_B0, A_SKTS_GRANITE_RIDGE },
	{ 0x1a, 0x44, 0x44, 0x1, 0x1, X86_CHIPREV_AMD_GRANITE_RIDGE_B1,
	    "GNR-B1", X86_UARCHREV_AMD_ZEN5_B1, A_SKTS_GRANITE_RIDGE },
	{ 0x1a, 0x40, 0x4f, 0x0, 0xf, X86_CHIPREV_AMD_GRANITE_RIDGE_UNKNOWN,
	    "GNR-???", X86_UARCHREV_AMD_ZEN5_UNKNOWN, A_SKTS_GRANITE_RIDGE },

	{ 0x1a, 0x70, 0x70, 0x0, 0x0, X86_CHIPREV_AMD_STRIX_HALO_A0,
	    "STXH-A0", X86_UARCHREV_AMD_ZEN5_A0, A_SKTS_STRIX_HALO },
	{ 0x1a, 0x70, 0x77, 0x0, 0xf, X86_CHIPREV_AMD_STRIX_HALO_UNKNOWN,
	    "STXH-???", X86_UARCHREV_AMD_ZEN5_UNKNOWN, A_SKTS_STRIX_HALO }
};

/*
 * AMD keeps the socket type in CPUID Fn8000_0001_EBX, bits 31:28.
 */
static uint32_t
synth_amd_skt_cpuid(uint_t family, uint_t sktid)
{
	struct cpuid_regs cp;
	uint_t idx;

	cp.cp_eax = 0x80000001;
	(void) __cpuid_insn(&cp);

	/* PkgType bits */
	idx = BITX(cp.cp_ebx, 31, 28);

	if (family == 0x10) {
		uint32_t val;

		val = pci_getl_func(0, 24, 2, 0x94);
		if (BITX(val, 8, 8)) {
			if (amd_skts[sktid][idx] == X86_SOCKET_AM2R2) {
				return (X86_SOCKET_AM3);
			} else if (amd_skts[sktid][idx] == X86_SOCKET_S1g3) {
				return (X86_SOCKET_S1g4);
			}
		}
	}

	return (amd_skts[sktid][idx]);
}

static void
synth_amd_info(uint_t family, uint_t model, uint_t step,
    uint32_t *skt_p, x86_chiprev_t *chiprev_p, const char **chiprevstr_p,
    x86_uarchrev_t *uarchrev_p)
{
	const struct amd_rev_mapent *rmp;
	int found = 0;
	int i;

	if (family < 0xf)
		return;

	for (i = 0, rmp = amd_revmap; i < ARRAY_SIZE(amd_revmap); i++, rmp++) {
		if (family == rmp->rm_family &&
		    model >= rmp->rm_modello && model <= rmp->rm_modelhi &&
		    step >= rmp->rm_steplo && step <= rmp->rm_stephi) {
			found = 1;
			break;
		}
	}

	if (found) {
		if (chiprev_p != NULL)
			*chiprev_p = rmp->rm_chiprev;
		if (chiprevstr_p != NULL)
			*chiprevstr_p = rmp->rm_chiprevstr;
		if (uarchrev_p != NULL)
			*uarchrev_p = rmp->rm_uarchrev;
	}

	if (skt_p != NULL) {
		int platform;

#ifdef __xpv
		/* PV guest */
		if (!is_controldom()) {
			*skt_p = X86_SOCKET_UNKNOWN;
			return;
		}
#endif
		platform = get_hwenv();

		if ((platform & HW_VIRTUAL) != 0) {
			*skt_p = X86_SOCKET_UNKNOWN;
			return;
		}

		if (!found)
			return;

		if (family == 0xf) {
			*skt_p = amd_skts[rmp->rm_sktidx][model & 0x3];
		} else {
			*skt_p = synth_amd_skt_cpuid(family, rmp->rm_sktidx);
		}
	}
}

uint32_t
_cpuid_skt(uint_t vendor, uint_t family, uint_t model, uint_t step)
{
	uint32_t skt = X86_SOCKET_UNKNOWN;

	switch (vendor) {
	case X86_VENDOR_AMD:
	case X86_VENDOR_HYGON:
		synth_amd_info(family, model, step, &skt, NULL, NULL, NULL);
		break;

	default:
		break;

	}

	return (skt);
}

const char *
_cpuid_sktstr(uint_t vendor, uint_t family, uint_t model, uint_t step)
{
	const char *sktstr = "Unknown";
	struct amd_sktmap_s *sktmapp;
	uint32_t skt = X86_SOCKET_UNKNOWN;

	switch (vendor) {
	case X86_VENDOR_AMD:
	case X86_VENDOR_HYGON:
		synth_amd_info(family, model, step, &skt, NULL, NULL, NULL);

		sktmapp = amd_sktmap_strs;
		while (sktmapp->skt_code != X86_SOCKET_UNKNOWN) {
			if (sktmapp->skt_code == skt)
				break;
			sktmapp++;
		}
		sktstr = sktmapp->sktstr;
		break;

	default:
		break;

	}

	return (sktstr);
}

x86_chiprev_t
_cpuid_chiprev(uint_t vendor, uint_t family, uint_t model, uint_t step)
{
	x86_chiprev_t chiprev = X86_CHIPREV_UNKNOWN;

	switch (vendor) {
	case X86_VENDOR_AMD:
	case X86_VENDOR_HYGON:
		synth_amd_info(family, model, step, NULL, &chiprev, NULL, NULL);
		break;

	default:
		break;

	}

	return (chiprev);
}

x86_uarchrev_t
_cpuid_uarchrev(uint_t vendor, uint_t family, uint_t model, uint_t step)
{
	x86_uarchrev_t uarchrev = X86_UARCHREV_UNKNOWN;

	switch (vendor) {
	case X86_VENDOR_AMD:
	case X86_VENDOR_HYGON:
		synth_amd_info(family, model, step, NULL, NULL, NULL,
		    &uarchrev);
		break;

	default:
		break;

	}

	return (uarchrev);
}

const char *
_cpuid_chiprevstr(uint_t vendor, uint_t family, uint_t model, uint_t step)
{
	const char *revstr = "Unknown";

	switch (vendor) {
	case X86_VENDOR_AMD:
	case X86_VENDOR_HYGON:
		synth_amd_info(family, model, step, NULL, NULL, &revstr, NULL);
		break;

	default:
		break;

	}

	return (revstr);

}

/*
 * Map the vendor string to a type code
 */
uint_t
_cpuid_vendorstr_to_vendorcode(char *vendorstr)
{
	if (strcmp(vendorstr, X86_VENDORSTR_Intel) == 0)
		return (X86_VENDOR_Intel);
	else if (strcmp(vendorstr, X86_VENDORSTR_AMD) == 0)
		return (X86_VENDOR_AMD);
	else if (strcmp(vendorstr, X86_VENDORSTR_HYGON) == 0)
		return (X86_VENDOR_HYGON);
	else if (strcmp(vendorstr, X86_VENDORSTR_TM) == 0)
		return (X86_VENDOR_TM);
	else if (strcmp(vendorstr, X86_VENDORSTR_CYRIX) == 0)
		return (X86_VENDOR_Cyrix);
	else if (strcmp(vendorstr, X86_VENDORSTR_UMC) == 0)
		return (X86_VENDOR_UMC);
	else if (strcmp(vendorstr, X86_VENDORSTR_NexGen) == 0)
		return (X86_VENDOR_NexGen);
	else if (strcmp(vendorstr, X86_VENDORSTR_Centaur) == 0)
		return (X86_VENDOR_Centaur);
	else if (strcmp(vendorstr, X86_VENDORSTR_Rise) == 0)
		return (X86_VENDOR_Rise);
	else if (strcmp(vendorstr, X86_VENDORSTR_SiS) == 0)
		return (X86_VENDOR_SiS);
	else if (strcmp(vendorstr, X86_VENDORSTR_NSC) == 0)
		return (X86_VENDOR_NSC);
	else
		return (X86_VENDOR_IntelClone);
}
#
# CDDL HEADER START
#
# The contents of this file are subject to the terms of the
# Common Development and Distribution License (the "License").
# You may not use this file except in compliance with the License.
#
# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
# or http://www.opensolaris.org/os/licensing.
# See the License for the specific language governing permissions
# and limitations under the License.
#
# When distributing Covered Code, include this CDDL HEADER in each
# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
# If applicable, add the following below this CDDL HEADER, with the
# fields enclosed by brackets "[]" replaced with your own identifying
# information: Portions Copyright [yyyy] [name of copyright owner]
#
# CDDL HEADER END
#
#
# Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
# Use is subject to license terms.
#
#
# NOTICE: This file contains important KERNEL STATE.  Do not edit this file.
# Its format and contents are subject to change in future releases of Solaris.
#

#
# keyboard and mouse configuration rules
#
# Configure and/or unconfigure a keyboard into the keyboard console stream.
# pushmod is the STREAMS module to be pushed on the minor node specified by
# driver-minorname.  The pushmod varies based on the keyboard hardware. 
#
driver-minorname="hid:internal_keyboard" consconfig_dacf:kb_config post-attach - pushmod="usbkbm"
driver-minorname="hid:internal_keyboard" consconfig_dacf:kb_config pre-detach - pushmod="usbkbm"

#
# Configure and/or unconfigure a mouse into the mouse console stream.  pushmod
# is the STREAMS module to be pushed on the minor node specified by
# driver-minorname.  The pushmod varies based on the mouse hardware. 
#
driver-minorname="hid:internal_mouse" consconfig_dacf:ms_config post-attach - pushmod="usbms"
driver-minorname="hid:internal_mouse" consconfig_dacf:ms_config pre-detach - pushmod="usbms"

#
# Devices directly supporting the keyboard API need no device-specific module,
# but do need to be linked to the console stream.
#
minor-nodetype="ddi_keyboard" consconfig_dacf:kb_config post-attach -
minor-nodetype="ddi_keyboard" consconfig_dacf:kb_config pre-detach -

#
# Devices directly supporting the keyboard API need no device-specific module,
# but do need to be linked to the console stream.
#
driver-minorname="mouse8042:internal_mouse" consconfig_dacf:ms_config post-attach - pushmod="vuid3ps2"
driver-minorname="mouse8042:internal_mouse" consconfig_dacf:ms_config pre-detach - pushmod="vuid3ps2"

#
# Configure and/or unconfigure DDI_NT_NET devices.
#
minor-nodetype="ddi_network" net_dacf:net_config post-attach -
minor-nodetype="ddi_network" net_dacf:net_config pre-detach -
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

/*
 * This file contains ddi functions common to intel architectures
 */

#include <sys/archsystm.h>
#include <sys/types.h>
#include <sys/dditypes.h>
#include <sys/ddi_impldefs.h>
#include <sys/sunddi.h>
#include <sys/cpu.h>

/*
 * DDI Mapping
 */

/*
 * i_ddi_bus_map:
 * Generic bus_map entry point, for byte addressable devices
 * conforming to the reg/range addressing model with no HAT layer
 * to be programmed at this level.
 */

int
i_ddi_bus_map(dev_info_t *dip, dev_info_t *rdip, ddi_map_req_t *mp,
	off_t offset, off_t len, caddr_t *vaddrp)
{
	struct regspec tmp_reg, *rp;
	ddi_map_req_t mr = *mp;		/* Get private copy of request */
	int error;

	mp = &mr;

	/*
	 * First, if given an rnumber, convert it to a regspec...
	 */

	if (mp->map_type == DDI_MT_RNUMBER)  {

		int rnumber = mp->map_obj.rnumber;
#ifdef	DDI_MAP_DEBUG
		static char *out_of_range =
		    "i_ddi_bus_map: Out of range rnumber <%d>, device <%s>";
#endif	/* DDI_MAP_DEBUG */

		rp = i_ddi_rnumber_to_regspec(rdip, rnumber);
		if (rp == (struct regspec *)0)  {
#ifdef	DDI_MAP_DEBUG
			cmn_err(CE_WARN, out_of_range, rnumber,
			    ddi_get_name(rdip));
#endif	/* DDI_MAP_DEBUG */
			return (DDI_ME_RNUMBER_RANGE);
		}

		/*
		 * Convert the given ddi_map_req_t from rnumber to regspec...
		 */

		mp->map_type = DDI_MT_REGSPEC;
		mp->map_obj.rp = rp;
	}

	/*
	 * Adjust offset and length correspnding to called values...
	 * XXX: A non-zero length means override the one in the regspec.
	 * XXX: (Regardless of what's in the parent's range)
	 */

	tmp_reg = *(mp->map_obj.rp);		/* Preserve underlying data */
	rp = mp->map_obj.rp = &tmp_reg;		/* Use tmp_reg in request */

#ifdef	DDI_MAP_DEBUG
	cmn_err(CE_CONT,
	    "i_ddi_bus_map: <%s,%s> <0x%x, 0x%x, 0x%d> "
	    "offset %d len %d handle 0x%x\n",
	    ddi_get_name(dip), ddi_get_name(rdip),
	    rp->regspec_bustype, rp->regspec_addr, rp->regspec_size,
	    offset, len, mp->map_handlep);
#endif	/* DDI_MAP_DEBUG */

	/*
	 * I/O or memory mapping
	 *
	 *	<bustype=0, addr=x, len=x>: memory
	 *	<bustype=1, addr=x, len=x>: i/o
	 *	<bustype>1, addr=0, len=x>: x86-compatibility i/o
	 */

	if (rp->regspec_bustype > 1 && rp->regspec_addr != 0) {
		cmn_err(CE_WARN, "<%s,%s>: invalid register spec"
		    " <0x%x, 0x%x, 0x%x>\n", ddi_get_name(dip),
		    ddi_get_name(rdip), rp->regspec_bustype,
		    rp->regspec_addr, rp->regspec_size);
		return (DDI_ME_INVAL);
	}

	if (rp->regspec_bustype > 1 && rp->regspec_addr == 0) {
		/*
		 * compatibility i/o mapping
		 */
		rp->regspec_bustype += (uint_t)offset;
	} else {
		/*
		 * Normal memory or i/o mapping
		 */
		rp->regspec_addr += (uint_t)offset;
	}

	if (len != 0)
		rp->regspec_size = (uint_t)len;

#ifdef	DDI_MAP_DEBUG
	cmn_err(CE_CONT,
	    "               <%s,%s> <0x%x, 0x%x, 0x%d> "
	    "offset %d len %d\n",
	    ddi_get_name(dip), ddi_get_name(rdip),
	    rp->regspec_bustype, rp->regspec_addr, rp->regspec_size,
	    offset, len);
#endif	/* DDI_MAP_DEBUG */

	/*
	 * If we had an MMU, this is where you'd program the MMU and hat layer.
	 * Since we're using the default function here, we do not have an MMU
	 * to program.
	 */

	/*
	 * Apply any parent ranges at this level, if applicable.
	 * (This is where nexus specific regspec translation takes place.
	 * Use of this function is implicit agreement that translation is
	 * provided via ddi_apply_range.)  Note that we assume that
	 * the request is within the parents limits.
	 */

#ifdef	DDI_MAP_DEBUG
	ddi_map_debug("applying range of parent <%s> to child <%s>...\n",
	    ddi_get_name(dip), ddi_get_name(rdip));
#endif	/* DDI_MAP_DEBUG */

	if ((error = i_ddi_apply_range(dip, rdip, mp->map_obj.rp)) != 0)
		return (error);

	/*
	 * Call my parents bus_map function with modified values...
	 */

	return (ddi_map(dip, mp, (off_t)0, (off_t)0, vaddrp));
}

/*
 * Creating register mappings and handling interrupts:
 */

struct regspec *
i_ddi_rnumber_to_regspec(dev_info_t *dip, int rnumber)
{
	if (rnumber >= sparc_pd_getnreg(DEVI(dip)))
		return ((struct regspec *)0);

	return (sparc_pd_getreg(DEVI(dip), rnumber));
}

/*
 * Static function to determine if a reg prop is enclosed within
 * a given a range spec.  (For readability: only used by i_ddi_aply_range.).
 */
static int
reg_is_enclosed_in_range(struct regspec *rp, struct rangespec *rangep)
{
	if (rp->regspec_bustype != rangep->rng_cbustype)
		return (0);

	if (rp->regspec_addr < rangep->rng_coffset)
		return (0);

	if (rangep->rng_size == 0)
		return (1);	/* size is really 2**(bits_per_word) */

	if ((rp->regspec_addr + rp->regspec_size - 1) <=
	    (rangep->rng_coffset + rangep->rng_size - 1))
		return (1);

	return (0);
}

/*
 * i_ddi_apply_range:
 * Apply range of dp to struct regspec *rp, if applicable.
 * If there's any range defined, it gets applied.
 */

int
i_ddi_apply_range(dev_info_t *dp, dev_info_t *rdip, struct regspec *rp)
{
	int nrange, b;
	struct rangespec *rangep;
	static char *out_of_range =
	    "Out of range register specification from device node <%s>\n";

	nrange = sparc_pd_getnrng(dp);
	if (nrange == 0)  {
#ifdef	DDI_MAP_DEBUG
		ddi_map_debug("    No range.\n");
#endif	/* DDI_MAP_DEBUG */
		return (0);
	}

	/*
	 * Find a match, making sure the regspec is within the range
	 * of the parent, noting that a size of zero in a range spec
	 * really means a size of 2**(bitsperword).
	 */

	for (b = 0, rangep = sparc_pd_getrng(dp, 0); b < nrange; ++b, ++rangep)
		if (reg_is_enclosed_in_range(rp, rangep))
			break;		/* found a match */

	if (b == nrange)  {
		cmn_err(CE_WARN, out_of_range, ddi_get_name(rdip));
		return (DDI_ME_REGSPEC_RANGE);
	}

#ifdef	DDI_MAP_DEBUG
	ddi_map_debug("    Input:  %x.%x.%x\n", rp->regspec_bustype,
	    rp->regspec_addr, rp->regspec_size);
	ddi_map_debug("    Range:  %x.%x %x.%x %x\n",
	    rangep->rng_cbustype, rangep->rng_coffset,
	    rangep->rng_bustype, rangep->rng_offset, rangep->rng_size);
#endif	/* DDI_MAP_DEBUG */

	rp->regspec_bustype = rangep->rng_bustype;
	rp->regspec_addr += rangep->rng_offset - rangep->rng_coffset;

#ifdef	DDI_MAP_DEBUG
	ddi_map_debug("    Return: %x.%x.%x\n", rp->regspec_bustype,
	    rp->regspec_addr, rp->regspec_size);
#endif	/* DDI_MAP_DEBUG */

	return (0);
}

/*
 * i_ddi_map_fault: wrapper for bus_map_fault.
 */
int
i_ddi_map_fault(dev_info_t *dip, dev_info_t *rdip,
	struct hat *hat, struct seg *seg, caddr_t addr,
	struct devpage *dp, pfn_t pfn, uint_t prot, uint_t lock)
{
	dev_info_t *pdip;

	if (dip == NULL)
		return (DDI_FAILURE);

	pdip = (dev_info_t *)DEVI(dip)->devi_bus_map_fault;

	/* request appropriate parent to map fault */
	return ((*(DEVI(pdip)->devi_ops->devo_bus_ops->bus_map_fault))(pdip,
	    rdip, hat, seg, addr, dp, pfn, prot, lock));
}

/*
 * Return an integer in native machine format from an OBP 1275 integer
 * representation, which is big-endian, with no particular alignment
 * guarantees.  intp points to the OBP data, and n the number of bytes.
 *
 * Byte-swapping is needed on intel.
 */
int
impl_ddi_prop_int_from_prom(uchar_t *intp, int n)
{
	int	i = 0;

	ASSERT(n > 0 && n <= 4);

	intp += n;
	while (n-- > 0) {
		i = (i << 8) | *(--intp);
	}

	return (i);
}


int drv_usec_coarse_timing = 0;

/*
 * Time delay function called by drivers
 */
void
drv_usecwait(clock_t count)
{
	int tens = 0;
	extern int gethrtime_hires;

	if (gethrtime_hires) {
		hrtime_t start, end;
		hrtime_t waittime;

		if (drv_usec_coarse_timing) {
			/* revert to the wait time as before using tsc */
			/* in case there are callers depending on the */
			/* old behaviour */
			waittime = ((count > 10) ?
			    (((hrtime_t)count / 10) + 1) : 1) *
			    10 * (NANOSEC / MICROSEC);
		} else  {
			waittime = (hrtime_t)count * (NANOSEC / MICROSEC);
		}
		start = end =  gethrtime();
		while ((end - start) < waittime) {
			SMT_PAUSE();
			end = gethrtime();
		}
		return;

	}

	if (count > 10)
		tens = count/10;
	tens++;			/* roundup; wait at least 10 microseconds */
	while (tens > 0) {
		tenmicrosec();
		tens--;
	}
}
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */
/*
 * Copyright 2014 Garrett D'Amore <garrett@damore.org>
 */

#include <sys/conf.h>
#include <sys/kmem.h>
#include <sys/ddi_impldefs.h>
#include <sys/ddi.h>
#include <sys/sunddi.h>
#include <sys/ddifm.h>
#include <sys/fm/io/ddi.h>
#include <sys/fm/protocol.h>
#include <sys/ontrap.h>


/*
 * DDI DMA Engine functions for x86.
 * These functions are more naturally generic, but do not apply to SPARC.
 */

int
ddi_dmae_alloc(dev_info_t *dip, int chnl, int (*dmae_waitfp)(), caddr_t arg)
{
	return (ddi_dma_mctl(dip, dip, 0, DDI_DMA_E_ACQUIRE,
	    (off_t *)dmae_waitfp, (size_t *)arg,
	    (caddr_t *)(uintptr_t)chnl, 0));
}

int
ddi_dmae_release(dev_info_t *dip, int chnl)
{
	return (ddi_dma_mctl(dip, dip, 0, DDI_DMA_E_FREE, 0, 0,
	    (caddr_t *)(uintptr_t)chnl, 0));
}

int
ddi_dmae_getattr(dev_info_t *dip, ddi_dma_attr_t *attrp)
{
	return (ddi_dma_mctl(dip, dip, 0, DDI_DMA_E_GETATTR, 0, 0,
	    (caddr_t *)attrp, 0));
}

int
ddi_dmae_1stparty(dev_info_t *dip, int chnl)
{
	return (ddi_dma_mctl(dip, dip, 0, DDI_DMA_E_1STPTY, 0, 0,
	    (caddr_t *)(uintptr_t)chnl, 0));
}

int
ddi_dmae_prog(dev_info_t *dip, struct ddi_dmae_req *dmaereqp,
    ddi_dma_cookie_t *cookiep, int chnl)
{
	return (ddi_dma_mctl(dip, dip, 0, DDI_DMA_E_PROG, (off_t *)dmaereqp,
	    (size_t *)cookiep, (caddr_t *)(uintptr_t)chnl, 0));
}

int
ddi_dmae_swsetup(dev_info_t *dip, struct ddi_dmae_req *dmaereqp,
    ddi_dma_cookie_t *cookiep, int chnl)
{
	return (ddi_dma_mctl(dip, dip, 0, DDI_DMA_E_SWSETUP, (off_t *)dmaereqp,
	    (size_t *)cookiep, (caddr_t *)(uintptr_t)chnl, 0));
}

int
ddi_dmae_swstart(dev_info_t *dip, int chnl)
{
	return (ddi_dma_mctl(dip, dip, 0, DDI_DMA_E_SWSTART, 0, 0,
	    (caddr_t *)(uintptr_t)chnl, 0));
}

int
ddi_dmae_stop(dev_info_t *dip, int chnl)
{
	return (ddi_dma_mctl(dip, dip, 0, DDI_DMA_E_STOP, 0, 0,
	    (caddr_t *)(uintptr_t)chnl, 0));
}

int
ddi_dmae_enable(dev_info_t *dip, int chnl)
{
	return (ddi_dma_mctl(dip, dip, 0, DDI_DMA_E_ENABLE, 0, 0,
	    (caddr_t *)(uintptr_t)chnl, 0));
}

int
ddi_dmae_disable(dev_info_t *dip, int chnl)
{
	return (ddi_dma_mctl(dip, dip, 0, DDI_DMA_E_DISABLE, 0, 0,
	    (caddr_t *)(uintptr_t)chnl, 0));
}

int
ddi_dmae_getcnt(dev_info_t *dip, int chnl, int *countp)
{
	return (ddi_dma_mctl(dip, dip, 0, DDI_DMA_E_GETCNT, 0, (size_t *)countp,
	    (caddr_t *)(uintptr_t)chnl, 0));
}

/*
 * implementation specific access handle and routines:
 */

static uintptr_t impl_acc_hdl_id = 0;

/*
 * access handle allocator
 */
ddi_acc_hdl_t *
impl_acc_hdl_get(ddi_acc_handle_t hdl)
{
	/*
	 * recast to ddi_acc_hdl_t instead of
	 * casting to ddi_acc_impl_t and then return the ah_platform_private
	 *
	 * this optimization based on the ddi_acc_hdl_t is the
	 * first member of the ddi_acc_impl_t.
	 */
	return ((ddi_acc_hdl_t *)hdl);
}

ddi_acc_handle_t
impl_acc_hdl_alloc(int (*waitfp)(caddr_t), caddr_t arg)
{
	ddi_acc_impl_t *hp;
	on_trap_data_t *otp;
	int sleepflag;

	sleepflag = ((waitfp == (int (*)())KM_SLEEP) ? KM_SLEEP : KM_NOSLEEP);
	/*
	 * Allocate and initialize the data access handle and error status.
	 */
	if ((hp = kmem_zalloc(sizeof (ddi_acc_impl_t), sleepflag)) == NULL)
		goto fail;
	if ((hp->ahi_err = (ndi_err_t *)kmem_zalloc(
	    sizeof (ndi_err_t), sleepflag)) == NULL) {
		kmem_free(hp, sizeof (ddi_acc_impl_t));
		goto fail;
	}
	if ((otp = (on_trap_data_t *)kmem_zalloc(
	    sizeof (on_trap_data_t), sleepflag)) == NULL) {
		kmem_free(hp->ahi_err, sizeof (ndi_err_t));
		kmem_free(hp, sizeof (ddi_acc_impl_t));
		goto fail;
	}
	hp->ahi_err->err_ontrap = otp;
	hp->ahi_common.ah_platform_private = (void *)hp;

	return ((ddi_acc_handle_t)hp);
fail:
	if ((waitfp != (int (*)())KM_SLEEP) &&
	    (waitfp != (int (*)())KM_NOSLEEP))
		ddi_set_callback(waitfp, arg, &impl_acc_hdl_id);
	return (NULL);
}

void
impl_acc_hdl_free(ddi_acc_handle_t handle)
{
	ddi_acc_impl_t *hp;

	/*
	 * The supplied (ddi_acc_handle_t) is actually a (ddi_acc_impl_t *),
	 * because that's what we allocated in impl_acc_hdl_alloc() above.
	 */
	hp = (ddi_acc_impl_t *)handle;
	if (hp) {
		kmem_free(hp->ahi_err->err_ontrap, sizeof (on_trap_data_t));
		kmem_free(hp->ahi_err, sizeof (ndi_err_t));
		kmem_free(hp, sizeof (ddi_acc_impl_t));
		if (impl_acc_hdl_id)
			ddi_run_callback(&impl_acc_hdl_id);
	}
}

/*
 * Function used to check if a given access handle owns the failing address.
 * Called by ndi_fmc_error, when we detect a PIO error.
 */
/* ARGSUSED */
static int
impl_acc_check(dev_info_t *dip, const void *handle, const void *addr,
    const void *not_used)
{
	pfn_t pfn, fault_pfn;
	ddi_acc_hdl_t *hp;

	hp = impl_acc_hdl_get((ddi_acc_handle_t)handle);

	ASSERT(hp);

	if (addr != NULL) {
		pfn = hp->ah_pfn;
		fault_pfn = mmu_btop(*(uint64_t *)addr);
		if (fault_pfn >= pfn && fault_pfn < (pfn + hp->ah_pnum))
			return (DDI_FM_NONFATAL);
	}
	return (DDI_FM_UNKNOWN);
}

void
impl_acc_err_init(ddi_acc_hdl_t *handlep)
{
	int fmcap;
	ndi_err_t *errp;
	on_trap_data_t *otp;
	ddi_acc_impl_t *hp = (ddi_acc_impl_t *)handlep;

	fmcap = ddi_fm_capable(handlep->ah_dip);

	if (handlep->ah_acc.devacc_attr_version < DDI_DEVICE_ATTR_V1 ||
	    !DDI_FM_ACC_ERR_CAP(fmcap)) {
		handlep->ah_acc.devacc_attr_access = DDI_DEFAULT_ACC;
	} else if (handlep->ah_acc.devacc_attr_access == DDI_FLAGERR_ACC &&
	    hp->ahi_scan == NULL) {
		handlep->ah_acc.devacc_attr_access = DDI_DEFAULT_ACC;
	} else if (DDI_FM_ACC_ERR_CAP(fmcap)) {
		if (handlep->ah_acc.devacc_attr_access == DDI_DEFAULT_ACC) {
			if (handlep->ah_xfermodes)
				return;
			i_ddi_drv_ereport_post(handlep->ah_dip, DVR_EFMCAP,
			    NULL, DDI_NOSLEEP);
		} else {
			errp = hp->ahi_err;
			otp = (on_trap_data_t *)errp->err_ontrap;
			otp->ot_handle = (void *)(hp);
			otp->ot_prot = OT_DATA_ACCESS;
			errp->err_status = DDI_FM_OK;
			errp->err_expected = DDI_FM_ERR_UNEXPECTED;
			errp->err_cf = impl_acc_check;
		}
	}
}

/* ARGSUSED */
int
impl_dma_check(dev_info_t *dip, const void *handle, const void *pci_hdl,
    const void *not_used)
{
	return (DDI_FM_UNKNOWN);
}

void
impl_acc_hdl_init(ddi_acc_hdl_t *handlep)
{
	ddi_acc_impl_t *hp;
	int fmcap;
	int devacc_attr_access;

	if (!handlep)
		return;
	fmcap = ddi_fm_capable(handlep->ah_dip);
	if (handlep->ah_acc.devacc_attr_version < DDI_DEVICE_ATTR_V1 ||
	    !DDI_FM_ACC_ERR_CAP(fmcap))
		devacc_attr_access = DDI_DEFAULT_ACC;
	else
		devacc_attr_access = handlep->ah_acc.devacc_attr_access;

	hp = (ddi_acc_impl_t *)handlep->ah_platform_private;

	/*
	 * Can only do FLAGERR if scan callback is set up. This should
	 * also guarantee that the peekpoke_mutex and err_mutex are defined.
	 */
	if (devacc_attr_access == DDI_FLAGERR_ACC && hp->ahi_scan == NULL)
		devacc_attr_access = DDI_DEFAULT_ACC;

	switch (devacc_attr_access) {
	case DDI_CAUTIOUS_ACC:
		hp->ahi_get8 = i_ddi_caut_get8;
		hp->ahi_put8 = i_ddi_caut_put8;
		hp->ahi_rep_get8 = i_ddi_caut_rep_get8;
		hp->ahi_rep_put8 = i_ddi_caut_rep_put8;
		hp->ahi_get16 = i_ddi_caut_get16;
		hp->ahi_get32 = i_ddi_caut_get32;
		hp->ahi_put16 = i_ddi_caut_put16;
		hp->ahi_put32 = i_ddi_caut_put32;
		hp->ahi_rep_get16 = i_ddi_caut_rep_get16;
		hp->ahi_rep_get32 = i_ddi_caut_rep_get32;
		hp->ahi_rep_put16 = i_ddi_caut_rep_put16;
		hp->ahi_rep_put32 = i_ddi_caut_rep_put32;
		hp->ahi_get64 = i_ddi_caut_get64;
		hp->ahi_put64 = i_ddi_caut_put64;
		hp->ahi_rep_get64 = i_ddi_caut_rep_get64;
		hp->ahi_rep_put64 = i_ddi_caut_rep_put64;
		break;
	case DDI_FLAGERR_ACC:
		if (hp->ahi_acc_attr & DDI_ACCATTR_IO_SPACE) {
			hp->ahi_get8 = i_ddi_prot_io_get8;
			hp->ahi_put8 = i_ddi_prot_io_put8;
			hp->ahi_rep_get8 = i_ddi_prot_io_rep_get8;
			hp->ahi_rep_put8 = i_ddi_prot_io_rep_put8;

			/* temporary set these 64 functions to no-ops */
			hp->ahi_get64 = i_ddi_io_get64;
			hp->ahi_put64 = i_ddi_io_put64;
			hp->ahi_rep_get64 = i_ddi_io_rep_get64;
			hp->ahi_rep_put64 = i_ddi_io_rep_put64;

			/*
			 * check for BIG endian access
			 */
			if (handlep->ah_acc.devacc_attr_endian_flags ==
			    DDI_STRUCTURE_BE_ACC) {
				hp->ahi_get16 = i_ddi_prot_io_swap_get16;
				hp->ahi_get32 = i_ddi_prot_io_swap_get32;
				hp->ahi_put16 = i_ddi_prot_io_swap_put16;
				hp->ahi_put32 = i_ddi_prot_io_swap_put32;
				hp->ahi_rep_get16 =
				    i_ddi_prot_io_swap_rep_get16;
				hp->ahi_rep_get32 =
				    i_ddi_prot_io_swap_rep_get32;
				hp->ahi_rep_put16 =
				    i_ddi_prot_io_swap_rep_put16;
				hp->ahi_rep_put32 =
				    i_ddi_prot_io_swap_rep_put32;
			} else {
				hp->ahi_acc_attr |= DDI_ACCATTR_DIRECT;
				hp->ahi_get16 = i_ddi_prot_io_get16;
				hp->ahi_get32 = i_ddi_prot_io_get32;
				hp->ahi_put16 = i_ddi_prot_io_put16;
				hp->ahi_put32 = i_ddi_prot_io_put32;
				hp->ahi_rep_get16 = i_ddi_prot_io_rep_get16;
				hp->ahi_rep_get32 = i_ddi_prot_io_rep_get32;
				hp->ahi_rep_put16 = i_ddi_prot_io_rep_put16;
				hp->ahi_rep_put32 = i_ddi_prot_io_rep_put32;
			}

		} else if (hp->ahi_acc_attr & DDI_ACCATTR_CPU_VADDR) {

			hp->ahi_get8 = i_ddi_prot_vaddr_get8;
			hp->ahi_put8 = i_ddi_prot_vaddr_put8;
			hp->ahi_rep_get8 = i_ddi_prot_vaddr_rep_get8;
			hp->ahi_rep_put8 = i_ddi_prot_vaddr_rep_put8;

			/*
			 * check for BIG endian access
			 */
			if (handlep->ah_acc.devacc_attr_endian_flags ==
			    DDI_STRUCTURE_BE_ACC) {

				hp->ahi_get16 = i_ddi_prot_vaddr_swap_get16;
				hp->ahi_get32 = i_ddi_prot_vaddr_swap_get32;
				hp->ahi_get64 = i_ddi_prot_vaddr_swap_get64;
				hp->ahi_put16 = i_ddi_prot_vaddr_swap_put16;
				hp->ahi_put32 = i_ddi_prot_vaddr_swap_put32;
				hp->ahi_put64 = i_ddi_prot_vaddr_swap_put64;
				hp->ahi_rep_get16 =
				    i_ddi_prot_vaddr_swap_rep_get16;
				hp->ahi_rep_get32 =
				    i_ddi_prot_vaddr_swap_rep_get32;
				hp->ahi_rep_get64 =
				    i_ddi_prot_vaddr_swap_rep_get64;
				hp->ahi_rep_put16 =
				    i_ddi_prot_vaddr_swap_rep_put16;
				hp->ahi_rep_put32 =
				    i_ddi_prot_vaddr_swap_rep_put32;
				hp->ahi_rep_put64 =
				    i_ddi_prot_vaddr_swap_rep_put64;
			} else {
				hp->ahi_acc_attr |= DDI_ACCATTR_DIRECT;
				hp->ahi_get16 = i_ddi_prot_vaddr_get16;
				hp->ahi_get32 = i_ddi_prot_vaddr_get32;
				hp->ahi_get64 = i_ddi_prot_vaddr_get64;
				hp->ahi_put16 = i_ddi_prot_vaddr_put16;
				hp->ahi_put32 = i_ddi_prot_vaddr_put32;
				hp->ahi_put64 = i_ddi_prot_vaddr_put64;
				hp->ahi_rep_get16 = i_ddi_prot_vaddr_rep_get16;
				hp->ahi_rep_get32 = i_ddi_prot_vaddr_rep_get32;
				hp->ahi_rep_get64 = i_ddi_prot_vaddr_rep_get64;
				hp->ahi_rep_put16 = i_ddi_prot_vaddr_rep_put16;
				hp->ahi_rep_put32 = i_ddi_prot_vaddr_rep_put32;
				hp->ahi_rep_put64 = i_ddi_prot_vaddr_rep_put64;
			}
		}
		break;
	case DDI_DEFAULT_ACC:
		if (hp->ahi_acc_attr & DDI_ACCATTR_IO_SPACE) {
			hp->ahi_get8 = i_ddi_io_get8;
			hp->ahi_put8 = i_ddi_io_put8;
			hp->ahi_rep_get8 = i_ddi_io_rep_get8;
			hp->ahi_rep_put8 = i_ddi_io_rep_put8;

			/* temporary set these 64 functions to no-ops */
			hp->ahi_get64 = i_ddi_io_get64;
			hp->ahi_put64 = i_ddi_io_put64;
			hp->ahi_rep_get64 = i_ddi_io_rep_get64;
			hp->ahi_rep_put64 = i_ddi_io_rep_put64;

			/*
			 * check for BIG endian access
			 */
			if (handlep->ah_acc.devacc_attr_endian_flags ==
			    DDI_STRUCTURE_BE_ACC) {
				hp->ahi_get16 = i_ddi_io_swap_get16;
				hp->ahi_get32 = i_ddi_io_swap_get32;
				hp->ahi_put16 = i_ddi_io_swap_put16;
				hp->ahi_put32 = i_ddi_io_swap_put32;
				hp->ahi_rep_get16 = i_ddi_io_swap_rep_get16;
				hp->ahi_rep_get32 = i_ddi_io_swap_rep_get32;
				hp->ahi_rep_put16 = i_ddi_io_swap_rep_put16;
				hp->ahi_rep_put32 = i_ddi_io_swap_rep_put32;
			} else {
				hp->ahi_acc_attr |= DDI_ACCATTR_DIRECT;
				hp->ahi_get16 = i_ddi_io_get16;
				hp->ahi_get32 = i_ddi_io_get32;
				hp->ahi_put16 = i_ddi_io_put16;
				hp->ahi_put32 = i_ddi_io_put32;
				hp->ahi_rep_get16 = i_ddi_io_rep_get16;
				hp->ahi_rep_get32 = i_ddi_io_rep_get32;
				hp->ahi_rep_put16 = i_ddi_io_rep_put16;
				hp->ahi_rep_put32 = i_ddi_io_rep_put32;
			}

		} else if (hp->ahi_acc_attr & DDI_ACCATTR_CPU_VADDR) {

			hp->ahi_get8 = i_ddi_vaddr_get8;
			hp->ahi_put8 = i_ddi_vaddr_put8;
			hp->ahi_rep_get8 = i_ddi_vaddr_rep_get8;
			hp->ahi_rep_put8 = i_ddi_vaddr_rep_put8;

			/*
			 * check for BIG endian access
			 */
			if (handlep->ah_acc.devacc_attr_endian_flags ==
			    DDI_STRUCTURE_BE_ACC) {

				hp->ahi_get16 = i_ddi_vaddr_swap_get16;
				hp->ahi_get32 = i_ddi_vaddr_swap_get32;
				hp->ahi_get64 = i_ddi_vaddr_swap_get64;
				hp->ahi_put16 = i_ddi_vaddr_swap_put16;
				hp->ahi_put32 = i_ddi_vaddr_swap_put32;
				hp->ahi_put64 = i_ddi_vaddr_swap_put64;
				hp->ahi_rep_get16 = i_ddi_vaddr_swap_rep_get16;
				hp->ahi_rep_get32 = i_ddi_vaddr_swap_rep_get32;
				hp->ahi_rep_get64 = i_ddi_vaddr_swap_rep_get64;
				hp->ahi_rep_put16 = i_ddi_vaddr_swap_rep_put16;
				hp->ahi_rep_put32 = i_ddi_vaddr_swap_rep_put32;
				hp->ahi_rep_put64 = i_ddi_vaddr_swap_rep_put64;
			} else {
				hp->ahi_acc_attr |= DDI_ACCATTR_DIRECT;
				hp->ahi_get16 = i_ddi_vaddr_get16;
				hp->ahi_get32 = i_ddi_vaddr_get32;
				hp->ahi_get64 = i_ddi_vaddr_get64;
				hp->ahi_put16 = i_ddi_vaddr_put16;
				hp->ahi_put32 = i_ddi_vaddr_put32;
				hp->ahi_put64 = i_ddi_vaddr_put64;
				hp->ahi_rep_get16 = i_ddi_vaddr_rep_get16;
				hp->ahi_rep_get32 = i_ddi_vaddr_rep_get32;
				hp->ahi_rep_get64 = i_ddi_vaddr_rep_get64;
				hp->ahi_rep_put16 = i_ddi_vaddr_rep_put16;
				hp->ahi_rep_put32 = i_ddi_vaddr_rep_put32;
				hp->ahi_rep_put64 = i_ddi_vaddr_rep_put64;
			}
		}
		break;
	}
	hp->ahi_fault_check = i_ddi_acc_fault_check;
	hp->ahi_fault_notify = i_ddi_acc_fault_notify;
	hp->ahi_fault = 0;
	impl_acc_err_init(handlep);
}

/*
 * The followings are low-level routines for data access.
 *
 * All of these routines should be implemented in assembly. Those
 * that have been rewritten be found in ~ml/ddi_i86_asm.s
 */

/*ARGSUSED*/
uint16_t
i_ddi_vaddr_swap_get16(ddi_acc_impl_t *hdlp, uint16_t *addr)
{
	return (ddi_swap16(*addr));
}

/*ARGSUSED*/
uint16_t
i_ddi_io_swap_get16(ddi_acc_impl_t *hdlp, uint16_t *addr)
{
	return (ddi_swap16(inw((uintptr_t)addr)));
}

/*ARGSUSED*/
uint32_t
i_ddi_vaddr_swap_get32(ddi_acc_impl_t *hdlp, uint32_t *addr)
{
	return (ddi_swap32(*addr));
}

/*ARGSUSED*/
uint32_t
i_ddi_io_swap_get32(ddi_acc_impl_t *hdlp, uint32_t *addr)
{
	return (ddi_swap32(inl((uintptr_t)addr)));
}

/*ARGSUSED*/
uint64_t
i_ddi_vaddr_swap_get64(ddi_acc_impl_t *hdlp, uint64_t *addr)
{
	return (ddi_swap64(*addr));
}

/*ARGSUSED*/
void
i_ddi_vaddr_swap_put16(ddi_acc_impl_t *hdlp, uint16_t *addr, uint16_t value)
{
	*addr = ddi_swap16(value);
}

/*ARGSUSED*/
void
i_ddi_io_swap_put16(ddi_acc_impl_t *hdlp, uint16_t *addr, uint16_t value)
{
	outw((uintptr_t)addr, ddi_swap16(value));
}

/*ARGSUSED*/
void
i_ddi_vaddr_swap_put32(ddi_acc_impl_t *hdlp, uint32_t *addr, uint32_t value)
{
	*addr = ddi_swap32(value);
}

/*ARGSUSED*/
void
i_ddi_io_swap_put32(ddi_acc_impl_t *hdlp, uint32_t *addr, uint32_t value)
{
	outl((uintptr_t)addr, ddi_swap32(value));
}

/*ARGSUSED*/
void
i_ddi_vaddr_swap_put64(ddi_acc_impl_t *hdlp, uint64_t *addr, uint64_t value)
{
	*addr = ddi_swap64(value);
}

/*ARGSUSED*/
void
i_ddi_vaddr_rep_get8(ddi_acc_impl_t *hdlp, uint8_t *host_addr,
    uint8_t *dev_addr, size_t repcount, uint_t flags)
{
	uint8_t	*h, *d;

	h = host_addr;
	d = dev_addr;

	if (flags == DDI_DEV_AUTOINCR)
		for (; repcount; repcount--)
			*h++ = *d++;
	else
		for (; repcount; repcount--)
			*h++ = *d;
}

/*ARGSUSED*/
void
i_ddi_vaddr_rep_get16(ddi_acc_impl_t *hdlp, uint16_t *host_addr,
    uint16_t *dev_addr, size_t repcount, uint_t flags)
{
	uint16_t *h, *d;

	h = host_addr;
	d = dev_addr;

	if (flags == DDI_DEV_AUTOINCR)
		for (; repcount; repcount--)
			*h++ = *d++;
	else
		for (; repcount; repcount--)
			*h++ = *d;
}

/*ARGSUSED*/
void
i_ddi_vaddr_swap_rep_get16(ddi_acc_impl_t *hdlp, uint16_t *host_addr,
    uint16_t *dev_addr, size_t repcount, uint_t flags)
{
	uint16_t *h, *d;

	h = host_addr;
	d = dev_addr;

	if (flags == DDI_DEV_AUTOINCR)
		for (; repcount; repcount--)
			*h++ = ddi_swap16(*d++);
	else
		for (; repcount; repcount--)
			*h++ = ddi_swap16(*d);
}

/*ARGSUSED*/
void
i_ddi_io_swap_rep_get16(ddi_acc_impl_t *hdlp, uint16_t *host_addr,
    uint16_t *dev_addr, size_t repcount, uint_t flags)
{
	uint16_t *h;
	uintptr_t port;

	h = host_addr;
	port = (uintptr_t)dev_addr;

	if (flags == DDI_DEV_AUTOINCR)
		for (; repcount; repcount--, port += 2)
			*h++ = ddi_swap16(inw(port));
	else
		for (; repcount; repcount--)
			*h++ = ddi_swap16(inw(port));
}

/*ARGSUSED*/
void
i_ddi_vaddr_rep_get32(ddi_acc_impl_t *hdlp, uint32_t *host_addr,
    uint32_t *dev_addr, size_t repcount, uint_t flags)
{
	uint32_t *h, *d;

	h = host_addr;
	d = dev_addr;

	if (flags == DDI_DEV_AUTOINCR)
		for (; repcount; repcount--)
			*h++ = *d++;
	else
		for (; repcount; repcount--)
			*h++ = *d;
}

/*ARGSUSED*/
void
i_ddi_vaddr_swap_rep_get32(ddi_acc_impl_t *hdlp, uint32_t *host_addr,
    uint32_t *dev_addr, size_t repcount, uint_t flags)
{
	uint32_t *h, *d;

	h = host_addr;
	d = dev_addr;

	if (flags == DDI_DEV_AUTOINCR)
		for (; repcount; repcount--)
			*h++ = ddi_swap32(*d++);
	else
		for (; repcount; repcount--)
			*h++ = ddi_swap32(*d);
}

/*ARGSUSED*/
void
i_ddi_io_swap_rep_get32(ddi_acc_impl_t *hdlp, uint32_t *host_addr,
    uint32_t *dev_addr, size_t repcount, uint_t flags)
{
	uint32_t *h;
	uintptr_t port;

	h = host_addr;
	port = (uintptr_t)dev_addr;

	if (flags == DDI_DEV_AUTOINCR)
		for (; repcount; repcount--, port += 4)
			*h++ = ddi_swap32(inl(port));
	else
		for (; repcount; repcount--)
			*h++ = ddi_swap32(inl(port));
}

/*ARGSUSED*/
void
i_ddi_vaddr_rep_get64(ddi_acc_impl_t *hdlp, uint64_t *host_addr,
    uint64_t *dev_addr, size_t repcount, uint_t flags)
{
	uint64_t *h, *d;

	h = host_addr;
	d = dev_addr;

	if (flags == DDI_DEV_AUTOINCR)
		for (; repcount; repcount--)
			*h++ = *d++;
	else
		for (; repcount; repcount--)
			*h++ = *d;
}

/*ARGSUSED*/
void
i_ddi_vaddr_swap_rep_get64(ddi_acc_impl_t *hdlp, uint64_t *host_addr,
    uint64_t *dev_addr, size_t repcount, uint_t flags)
{
	uint64_t *h, *d;

	h = host_addr;
	d = dev_addr;

	if (flags == DDI_DEV_AUTOINCR)
		for (; repcount; repcount--)
			*h++ = ddi_swap64(*d++);
	else
		for (; repcount; repcount--)
			*h++ = ddi_swap64(*d);
}

/*ARGSUSED*/
void
i_ddi_vaddr_rep_put8(ddi_acc_impl_t *hdlp, uint8_t *host_addr,
    uint8_t *dev_addr, size_t repcount, uint_t flags)
{
	uint8_t	*h, *d;

	h = host_addr;
	d = dev_addr;

	if (flags == DDI_DEV_AUTOINCR)
		for (; repcount; repcount--)
			*d++ = *h++;
	else
		for (; repcount; repcount--)
			*d = *h++;
}

/*ARGSUSED*/
void
i_ddi_vaddr_rep_put16(ddi_acc_impl_t *hdlp, uint16_t *host_addr,
    uint16_t *dev_addr, size_t repcount, uint_t flags)
{
	uint16_t *h, *d;

	h = host_addr;
	d = dev_addr;

	if (flags == DDI_DEV_AUTOINCR)
		for (; repcount; repcount--)
			*d++ = *h++;
	else
		for (; repcount; repcount--)
			*d = *h++;
}

/*ARGSUSED*/
void
i_ddi_vaddr_swap_rep_put16(ddi_acc_impl_t *hdlp, uint16_t *host_addr,
    uint16_t *dev_addr, size_t repcount, uint_t flags)
{
	uint16_t *h, *d;

	h = host_addr;
	d = dev_addr;

	if (flags == DDI_DEV_AUTOINCR)
		for (; repcount; repcount--)
			*d++ = ddi_swap16(*h++);
	else
		for (; repcount; repcount--)
			*d = ddi_swap16(*h++);
}

/*ARGSUSED*/
void
i_ddi_io_swap_rep_put16(ddi_acc_impl_t *hdlp, uint16_t *host_addr,
    uint16_t *dev_addr, size_t repcount, uint_t flags)
{
	uint16_t *h;
	uintptr_t port;

	h = host_addr;
	port = (uintptr_t)dev_addr;

	if (flags == DDI_DEV_AUTOINCR)
		for (; repcount; repcount--, port += 2)
			outw(port, ddi_swap16(*h++));
	else
		for (; repcount; repcount--)
			outw(port, ddi_swap16(*h++));
}

/*ARGSUSED*/
void
i_ddi_vaddr_rep_put32(ddi_acc_impl_t *hdlp, uint32_t *host_addr,
    uint32_t *dev_addr, size_t repcount, uint_t flags)
{
	uint32_t *h, *d;

	h = host_addr;
	d = dev_addr;

	if (flags == DDI_DEV_AUTOINCR)
		for (; repcount; repcount--)
			*d++ = *h++;
	else
		for (; repcount; repcount--)
			*d = *h++;
}

/*ARGSUSED*/
void
i_ddi_vaddr_swap_rep_put32(ddi_acc_impl_t *hdlp, uint32_t *host_addr,
    uint32_t *dev_addr, size_t repcount, uint_t flags)
{
	uint32_t *h, *d;

	h = host_addr;
	d = dev_addr;

	if (flags == DDI_DEV_AUTOINCR)
		for (; repcount; repcount--)
			*d++ = ddi_swap32(*h++);
	else
		for (; repcount; repcount--)
			*d = ddi_swap32(*h++);
}

/*ARGSUSED*/
void
i_ddi_io_swap_rep_put32(ddi_acc_impl_t *hdlp, uint32_t *host_addr,
    uint32_t *dev_addr, size_t repcount, uint_t flags)
{
	uint32_t *h;
	uintptr_t port;

	h = host_addr;
	port = (uintptr_t)dev_addr;

	if (flags == DDI_DEV_AUTOINCR)
		for (; repcount; repcount--, port += 4)
			outl(port, ddi_swap32(*h++));
	else
		for (; repcount; repcount--)
			outl(port, ddi_swap32(*h++));
}

/*ARGSUSED*/
void
i_ddi_vaddr_rep_put64(ddi_acc_impl_t *hdlp, uint64_t *host_addr,
    uint64_t *dev_addr, size_t repcount, uint_t flags)
{
	uint64_t *h, *d;

	h = host_addr;
	d = dev_addr;

	if (flags == DDI_DEV_AUTOINCR)
		for (; repcount; repcount--)
			*d++ = *h++;
	else
		for (; repcount; repcount--)
			*d = *h++;
}

/*ARGSUSED*/
void
i_ddi_vaddr_swap_rep_put64(ddi_acc_impl_t *hdlp, uint64_t *host_addr,
    uint64_t *dev_addr, size_t repcount, uint_t flags)
{
	uint64_t *h, *d;

	h = host_addr;
	d = dev_addr;

	if (flags == DDI_DEV_AUTOINCR)
		for (; repcount; repcount--)
			*d++ = ddi_swap64(*h++);
	else
		for (; repcount; repcount--)
			*d = ddi_swap64(*h++);
}

/*ARGSUSED*/
uint64_t
i_ddi_io_get64(ddi_acc_impl_t *hdlp, uint64_t *addr)
{
	panic("ddi_get64 from i/o space");
	/*NOTREACHED*/
	return (0);
}

/*ARGSUSED*/
void
i_ddi_io_put64(ddi_acc_impl_t *hdlp, uint64_t *host_addr, uint64_t value)
{
	panic("ddi_put64 to i/o space");
	/*NOTREACHED*/
}

void
do_scan(ddi_acc_impl_t *hdlp)
{
	ddi_fm_error_t de;
	ndi_err_t *errp = (ndi_err_t *)hdlp->ahi_err;

	bzero(&de, sizeof (ddi_fm_error_t));
	de.fme_version = DDI_FME_VERSION;
	de.fme_ena = fm_ena_generate(0, FM_ENA_FMT1);
	de.fme_flag = DDI_FM_ERR_UNEXPECTED;

	mutex_enter(hdlp->ahi_err_mutexp);
	hdlp->ahi_scan(hdlp->ahi_scan_dip, &de);
	if (de.fme_status != DDI_FM_OK) {
		errp->err_ena = de.fme_ena;
		errp->err_expected = de.fme_flag;
		errp->err_status = DDI_FM_NONFATAL;
	}
	mutex_exit(hdlp->ahi_err_mutexp);
}

/*ARGSUSED*/
uint8_t
i_ddi_prot_vaddr_get8(ddi_acc_impl_t *hdlp, uint8_t *addr)
{
	uint8_t val;

	mutex_enter(hdlp->ahi_peekpoke_mutexp);
	val = *addr;
	if (val == 0xff)
		do_scan(hdlp);
	mutex_exit(hdlp->ahi_peekpoke_mutexp);

	return (val);
}

/*ARGSUSED*/
uint16_t
i_ddi_prot_vaddr_get16(ddi_acc_impl_t *hdlp, uint16_t *addr)
{
	uint16_t val;

	mutex_enter(hdlp->ahi_peekpoke_mutexp);
	val = *addr;
	if (val == 0xffff)
		do_scan(hdlp);
	mutex_exit(hdlp->ahi_peekpoke_mutexp);

	return (val);
}

/*ARGSUSED*/
uint32_t
i_ddi_prot_vaddr_get32(ddi_acc_impl_t *hdlp, uint32_t *addr)
{
	uint32_t val;

	mutex_enter(hdlp->ahi_peekpoke_mutexp);
	val = *addr;
	if (val == 0xffffffff)
		do_scan(hdlp);
	mutex_exit(hdlp->ahi_peekpoke_mutexp);

	return (val);
}

/*ARGSUSED*/
uint64_t
i_ddi_prot_vaddr_get64(ddi_acc_impl_t *hdlp, uint64_t *addr)
{
	uint64_t val;

	mutex_enter(hdlp->ahi_peekpoke_mutexp);
	val = *addr;
	if (val == 0xffffffffffffffff)
		do_scan(hdlp);
	mutex_exit(hdlp->ahi_peekpoke_mutexp);

	return (val);
}

/*ARGSUSED*/
uint8_t
i_ddi_prot_io_get8(ddi_acc_impl_t *hdlp, uint8_t *addr)
{
	uint8_t val;

	mutex_enter(hdlp->ahi_peekpoke_mutexp);
	val = inb((uintptr_t)addr);
	if (val == 0xff)
		do_scan(hdlp);
	mutex_exit(hdlp->ahi_peekpoke_mutexp);

	return (val);
}

/*ARGSUSED*/
uint16_t
i_ddi_prot_io_get16(ddi_acc_impl_t *hdlp, uint16_t *addr)
{
	uint16_t val;

	mutex_enter(hdlp->ahi_peekpoke_mutexp);
	val = inw((uintptr_t)addr);
	if (val == 0xffff)
		do_scan(hdlp);
	mutex_exit(hdlp->ahi_peekpoke_mutexp);

	return (val);
}

/*ARGSUSED*/
uint32_t
i_ddi_prot_io_get32(ddi_acc_impl_t *hdlp, uint32_t *addr)
{
	uint32_t val;

	mutex_enter(hdlp->ahi_peekpoke_mutexp);
	val = inl((uintptr_t)addr);
	if (val == 0xffffffff)
		do_scan(hdlp);
	mutex_exit(hdlp->ahi_peekpoke_mutexp);

	return (val);
}

/*ARGSUSED*/
uint16_t
i_ddi_prot_vaddr_swap_get16(ddi_acc_impl_t *hdlp, uint16_t *addr)
{
	uint16_t val;

	mutex_enter(hdlp->ahi_peekpoke_mutexp);
	val = ddi_swap16(*addr);
	if (val == 0xffff)
		do_scan(hdlp);
	mutex_exit(hdlp->ahi_peekpoke_mutexp);

	return (val);
}

/*ARGSUSED*/
uint16_t
i_ddi_prot_io_swap_get16(ddi_acc_impl_t *hdlp, uint16_t *addr)
{
	uint16_t val;

	mutex_enter(hdlp->ahi_peekpoke_mutexp);
	val = ddi_swap16(inw((uintptr_t)addr));
	if (val == 0xffff)
		do_scan(hdlp);
	mutex_exit(hdlp->ahi_peekpoke_mutexp);

	return (val);
}

/*ARGSUSED*/
uint32_t
i_ddi_prot_vaddr_swap_get32(ddi_acc_impl_t *hdlp, uint32_t *addr)
{
	uint32_t val;

	mutex_enter(hdlp->ahi_peekpoke_mutexp);
	val = ddi_swap32(*addr);
	if (val == 0xffffffff)
		do_scan(hdlp);
	mutex_exit(hdlp->ahi_peekpoke_mutexp);

	return (val);
}

/*ARGSUSED*/
uint32_t
i_ddi_prot_io_swap_get32(ddi_acc_impl_t *hdlp, uint32_t *addr)
{
	uint32_t val;

	mutex_enter(hdlp->ahi_peekpoke_mutexp);
	val = ddi_swap32(inl((uintptr_t)addr));
	if (val == 0xffffffff)
		do_scan(hdlp);
	mutex_exit(hdlp->ahi_peekpoke_mutexp);

	return (val);
}

/*ARGSUSED*/
uint64_t
i_ddi_prot_vaddr_swap_get64(ddi_acc_impl_t *hdlp, uint64_t *addr)
{
	uint64_t val;

	mutex_enter(hdlp->ahi_peekpoke_mutexp);
	val = ddi_swap64(*addr);
	if (val == 0xffffffffffffffff)
		do_scan(hdlp);
	mutex_exit(hdlp->ahi_peekpoke_mutexp);

	return (val);
}

/*ARGSUSED*/
void
i_ddi_prot_vaddr_put8(ddi_acc_impl_t *hdlp, uint8_t *addr, uint8_t value)
{
	mutex_enter(hdlp->ahi_peekpoke_mutexp);
	*addr = value;
	mutex_exit(hdlp->ahi_peekpoke_mutexp);
}

/*ARGSUSED*/
void
i_ddi_prot_io_put8(ddi_acc_impl_t *hdlp, uint8_t *addr, uint8_t value)
{
	mutex_enter(hdlp->ahi_peekpoke_mutexp);
	outb((uintptr_t)addr, value);
	mutex_exit(hdlp->ahi_peekpoke_mutexp);
}

/*ARGSUSED*/
void
i_ddi_prot_vaddr_put16(ddi_acc_impl_t *hdlp, uint16_t *addr, uint16_t value)
{
	mutex_enter(hdlp->ahi_peekpoke_mutexp);
	*addr = value;
	mutex_exit(hdlp->ahi_peekpoke_mutexp);
}

/*ARGSUSED*/
void
i_ddi_prot_io_put16(ddi_acc_impl_t *hdlp, uint16_t *addr, uint16_t value)
{
	mutex_enter(hdlp->ahi_peekpoke_mutexp);
	outw((uintptr_t)addr, value);
	mutex_exit(hdlp->ahi_peekpoke_mutexp);
}

/*ARGSUSED*/
void
i_ddi_prot_vaddr_put32(ddi_acc_impl_t *hdlp, uint32_t *addr,
    uint32_t value)
{
	mutex_enter(hdlp->ahi_peekpoke_mutexp);
	*addr = value;
	mutex_exit(hdlp->ahi_peekpoke_mutexp);
}

/*ARGSUSED*/
void
i_ddi_prot_io_put32(ddi_acc_impl_t *hdlp, uint32_t *addr, uint32_t value)
{
	mutex_enter(hdlp->ahi_peekpoke_mutexp);
	outl((uintptr_t)addr, value);
	mutex_exit(hdlp->ahi_peekpoke_mutexp);
}

/*ARGSUSED*/
void
i_ddi_prot_vaddr_put64(ddi_acc_impl_t *hdlp, uint64_t *addr,
    uint64_t value)
{
	mutex_enter(hdlp->ahi_peekpoke_mutexp);
	*addr = value;
	mutex_exit(hdlp->ahi_peekpoke_mutexp);
}

/*ARGSUSED*/
void
i_ddi_prot_vaddr_swap_put16(ddi_acc_impl_t *hdlp, uint16_t *addr,
    uint16_t value)
{
	mutex_enter(hdlp->ahi_peekpoke_mutexp);
	*addr = ddi_swap16(value);
	mutex_exit(hdlp->ahi_peekpoke_mutexp);
}

/*ARGSUSED*/
void
i_ddi_prot_io_swap_put16(ddi_acc_impl_t *hdlp, uint16_t *addr, uint16_t value)
{
	mutex_enter(hdlp->ahi_peekpoke_mutexp);
	outw((uintptr_t)addr, ddi_swap16(value));
	mutex_exit(hdlp->ahi_peekpoke_mutexp);
}

/*ARGSUSED*/
void
i_ddi_prot_vaddr_swap_put32(ddi_acc_impl_t *hdlp, uint32_t *addr,
    uint32_t value)
{
	mutex_enter(hdlp->ahi_peekpoke_mutexp);
	*addr = ddi_swap32(value);
	mutex_exit(hdlp->ahi_peekpoke_mutexp);
}

/*ARGSUSED*/
void
i_ddi_prot_io_swap_put32(ddi_acc_impl_t *hdlp, uint32_t *addr, uint32_t value)
{
	mutex_enter(hdlp->ahi_peekpoke_mutexp);
	outl((uintptr_t)addr, ddi_swap32(value));
	mutex_exit(hdlp->ahi_peekpoke_mutexp);
}

/*ARGSUSED*/
void
i_ddi_prot_vaddr_swap_put64(ddi_acc_impl_t *hdlp, uint64_t *addr,
    uint64_t value)
{
	mutex_enter(hdlp->ahi_peekpoke_mutexp);
	*addr = ddi_swap64(value);
	mutex_exit(hdlp->ahi_peekpoke_mutexp);
}

/*ARGSUSED*/
void
i_ddi_prot_io_rep_get8(ddi_acc_impl_t *hdlp, uint8_t *host_addr,
    uint8_t *dev_addr, size_t repcount, uint_t flags)
{
	int fail = 0;
	uint8_t	*h;
	uintptr_t port;

	h = host_addr;
	port = (uintptr_t)dev_addr;

	mutex_enter(hdlp->ahi_peekpoke_mutexp);
	if (flags == DDI_DEV_AUTOINCR) {
		for (; repcount; repcount--, port++)
			if ((*h++ = inb(port)) == 0xff)
				fail = 1;
	} else {
		for (; repcount; repcount--)
			if ((*h++ = inb(port)) == 0xff)
				fail = 1;
	}
	if (fail == 1)
		do_scan(hdlp);
	mutex_exit(hdlp->ahi_peekpoke_mutexp);
}

/*ARGSUSED*/
void
i_ddi_prot_io_rep_get16(ddi_acc_impl_t *hdlp, uint16_t *host_addr,
    uint16_t *dev_addr, size_t repcount, uint_t flags)
{
	int fail = 0;
	uint16_t *h;
	uintptr_t port;

	h = host_addr;
	port = (uintptr_t)dev_addr;

	mutex_enter(hdlp->ahi_peekpoke_mutexp);
	if (flags == DDI_DEV_AUTOINCR) {
		for (; repcount; repcount--, port += 2)
			if ((*h++ = inw(port)) == 0xffff)
				fail = 1;
	} else {
		for (; repcount; repcount--)
			if ((*h++ = inw(port)) == 0xffff)
				fail = 1;
	}
	if (fail == 1)
		do_scan(hdlp);
	mutex_exit(hdlp->ahi_peekpoke_mutexp);
}

/*ARGSUSED*/
void
i_ddi_prot_io_rep_get32(ddi_acc_impl_t *hdlp, uint32_t *host_addr,
    uint32_t *dev_addr, size_t repcount, uint_t flags)
{
	int fail = 0;
	uint32_t *h;
	uintptr_t port;

	h = host_addr;
	port = (uintptr_t)dev_addr;

	mutex_enter(hdlp->ahi_peekpoke_mutexp);
	if (flags == DDI_DEV_AUTOINCR) {
		for (; repcount; repcount--, port += 4)
			if ((*h++ = inl(port)) == 0xffffffff)
				fail = 1;
	} else {
		for (; repcount; repcount--)
			if ((*h++ = inl(port)) == 0xffffffff)
				fail = 1;
	}
	if (fail == 1)
		do_scan(hdlp);
	mutex_exit(hdlp->ahi_peekpoke_mutexp);
}

/*ARGSUSED*/
void
i_ddi_prot_vaddr_rep_get8(ddi_acc_impl_t *hdlp, uint8_t *host_addr,
    uint8_t *dev_addr, size_t repcount, uint_t flags)
{
	int fail = 0;
	uint8_t	*h, *d;

	h = host_addr;
	d = dev_addr;

	mutex_enter(hdlp->ahi_peekpoke_mutexp);
	if (flags == DDI_DEV_AUTOINCR) {
		for (; repcount; repcount--)
			if ((*h++ = *d++) == 0xff)
				fail = 1;
	} else {
		for (; repcount; repcount--)
			if ((*h++ = *d) == 0xff)
				fail = 1;
	}
	if (fail == 1)
		do_scan(hdlp);
	mutex_exit(hdlp->ahi_peekpoke_mutexp);
}

/*ARGSUSED*/
void
i_ddi_prot_vaddr_rep_get16(ddi_acc_impl_t *hdlp, uint16_t *host_addr,
    uint16_t *dev_addr, size_t repcount, uint_t flags)
{
	int fail = 0;
	uint16_t *h, *d;

	h = host_addr;
	d = dev_addr;

	mutex_enter(hdlp->ahi_peekpoke_mutexp);
	if (flags == DDI_DEV_AUTOINCR) {
		for (; repcount; repcount--)
			if ((*h++ = *d++) == 0xffff)
				fail = 1;
	} else {
		for (; repcount; repcount--)
			if ((*h++ = *d) == 0xffff)
				fail = 1;
	}
	if (fail == 1)
		do_scan(hdlp);
	mutex_exit(hdlp->ahi_peekpoke_mutexp);
}

/*ARGSUSED*/
void
i_ddi_prot_vaddr_swap_rep_get16(ddi_acc_impl_t *hdlp, uint16_t *host_addr,
    uint16_t *dev_addr, size_t repcount, uint_t flags)
{
	int fail = 0;
	uint16_t *h, *d;

	h = host_addr;
	d = dev_addr;

	mutex_enter(hdlp->ahi_peekpoke_mutexp);
	if (flags == DDI_DEV_AUTOINCR) {
		for (; repcount; repcount--)
			if ((*h++ = ddi_swap16(*d++)) == 0xffff)
				fail = 1;
	} else {
		for (; repcount; repcount--)
			if ((*h++ = ddi_swap16(*d)) == 0xffff)
				fail = 1;
	}
	if (fail == 1)
		do_scan(hdlp);
	mutex_exit(hdlp->ahi_peekpoke_mutexp);
}

/*ARGSUSED*/
void
i_ddi_prot_io_swap_rep_get16(ddi_acc_impl_t *hdlp, uint16_t *host_addr,
    uint16_t *dev_addr, size_t repcount, uint_t flags)
{
	int fail = 0;
	uint16_t *h;
	uintptr_t port;

	h = host_addr;
	port = (uintptr_t)dev_addr;

	mutex_enter(hdlp->ahi_peekpoke_mutexp);
	if (flags == DDI_DEV_AUTOINCR) {
		for (; repcount; repcount--, port += 2)
			if ((*h++ = ddi_swap16(inw(port))) == 0xffff)
				fail = 1;
	} else {
		for (; repcount; repcount--)
			if ((*h++ = ddi_swap16(inw(port))) == 0xffff)
				fail = 1;
	}
	if (fail == 1)
		do_scan(hdlp);
	mutex_exit(hdlp->ahi_peekpoke_mutexp);
}

/*ARGSUSED*/
void
i_ddi_prot_vaddr_rep_get32(ddi_acc_impl_t *hdlp, uint32_t *host_addr,
    uint32_t *dev_addr, size_t repcount, uint_t flags)
{
	int fail = 0;
	uint32_t *h, *d;

	h = host_addr;
	d = dev_addr;

	mutex_enter(hdlp->ahi_peekpoke_mutexp);
	if (flags == DDI_DEV_AUTOINCR) {
		for (; repcount; repcount--)
			if ((*h++ = *d++) == 0xffffffff)
				fail = 1;
	} else {
		for (; repcount; repcount--)
			if ((*h++ = *d) == 0xffffffff)
				fail = 1;
	}
	if (fail == 1)
		do_scan(hdlp);
	mutex_exit(hdlp->ahi_peekpoke_mutexp);
}

/*ARGSUSED*/
void
i_ddi_prot_vaddr_swap_rep_get32(ddi_acc_impl_t *hdlp, uint32_t *host_addr,
    uint32_t *dev_addr, size_t repcount, uint_t flags)
{
	int fail = 0;
	uint32_t *h, *d;

	h = host_addr;
	d = dev_addr;

	mutex_enter(hdlp->ahi_peekpoke_mutexp);
	if (flags == DDI_DEV_AUTOINCR) {
		for (; repcount; repcount--)
			if ((*h++ = ddi_swap32(*d++)) == 0xffffffff)
				fail = 1;
	} else {
		for (; repcount; repcount--)
			if ((*h++ = ddi_swap32(*d)) == 0xffffffff)
				fail = 1;
	}
	if (fail == 1)
		do_scan(hdlp);
	mutex_exit(hdlp->ahi_peekpoke_mutexp);
}

/*ARGSUSED*/
void
i_ddi_prot_io_swap_rep_get32(ddi_acc_impl_t *hdlp, uint32_t *host_addr,
    uint32_t *dev_addr, size_t repcount, uint_t flags)
{
	int fail = 0;
	uint32_t *h;
	uintptr_t port;

	h = host_addr;
	port = (uintptr_t)dev_addr;

	mutex_enter(hdlp->ahi_peekpoke_mutexp);
	if (flags == DDI_DEV_AUTOINCR) {
		for (; repcount; repcount--, port += 4)
			if ((*h++ = ddi_swap32(inl(port))) == 0xffffffff)
				fail = 1;
	} else {
		for (; repcount; repcount--)
			if ((*h++ = ddi_swap32(inl(port))) == 0xffffffff)
				fail = 1;
	}
	if (fail == 1)
		do_scan(hdlp);
	mutex_exit(hdlp->ahi_peekpoke_mutexp);
}

/*ARGSUSED*/
void
i_ddi_prot_vaddr_rep_get64(ddi_acc_impl_t *hdlp, uint64_t *host_addr,
    uint64_t *dev_addr, size_t repcount, uint_t flags)
{
	int fail = 0;
	uint64_t *h, *d;

	h = host_addr;
	d = dev_addr;

	mutex_enter(hdlp->ahi_peekpoke_mutexp);
	if (flags == DDI_DEV_AUTOINCR) {
		for (; repcount; repcount--)
			if ((*h++ = *d++) == 0xffffffffffffffff)
				fail = 1;
	} else {
		for (; repcount; repcount--)
			if ((*h++ = *d) == 0xffffffffffffffff)
				fail = 1;
	}
	if (fail == 1)
		do_scan(hdlp);
	mutex_exit(hdlp->ahi_peekpoke_mutexp);
}

/*ARGSUSED*/
void
i_ddi_prot_vaddr_swap_rep_get64(ddi_acc_impl_t *hdlp, uint64_t *host_addr,
    uint64_t *dev_addr, size_t repcount, uint_t flags)
{
	int fail = 0;
	uint64_t *h, *d;

	h = host_addr;
	d = dev_addr;

	mutex_enter(hdlp->ahi_peekpoke_mutexp);
	if (flags == DDI_DEV_AUTOINCR) {
		for (; repcount; repcount--)
			if ((*h++ = ddi_swap64(*d++)) == 0xffffffffffffffff)
				fail = 1;
	} else {
		for (; repcount; repcount--)
			if ((*h++ = ddi_swap64(*d)) == 0xffffffffffffffff)
				fail = 1;
	}
	if (fail == 1)
		do_scan(hdlp);
	mutex_exit(hdlp->ahi_peekpoke_mutexp);
}

/*ARGSUSED*/
void
i_ddi_prot_vaddr_rep_put8(ddi_acc_impl_t *hdlp, uint8_t *host_addr,
    uint8_t *dev_addr, size_t repcount, uint_t flags)
{
	uint8_t	*h, *d;

	h = host_addr;
	d = dev_addr;

	mutex_enter(hdlp->ahi_peekpoke_mutexp);
	if (flags == DDI_DEV_AUTOINCR)
		for (; repcount; repcount--)
			*d++ = *h++;
	else
		for (; repcount; repcount--)
			*d = *h++;
	mutex_exit(hdlp->ahi_peekpoke_mutexp);
}

/*ARGSUSED*/
void
i_ddi_prot_io_rep_put8(ddi_acc_impl_t *hdlp, uint8_t *host_addr,
    uint8_t *dev_addr, size_t repcount, uint_t flags)
{
	uint8_t	*h;
	uintptr_t port;

	h = host_addr;
	port = (uintptr_t)dev_addr;

	mutex_enter(hdlp->ahi_peekpoke_mutexp);
	if (flags == DDI_DEV_AUTOINCR)
		for (; repcount; repcount--, port++)
			outb(port, *h++);
	else
		for (; repcount; repcount--)
			outb(port, *h++);
	mutex_exit(hdlp->ahi_peekpoke_mutexp);
}

/*ARGSUSED*/
void
i_ddi_prot_vaddr_rep_put16(ddi_acc_impl_t *hdlp, uint16_t *host_addr,
    uint16_t *dev_addr, size_t repcount, uint_t flags)
{
	uint16_t *h, *d;

	h = host_addr;
	d = dev_addr;

	mutex_enter(hdlp->ahi_peekpoke_mutexp);
	if (flags == DDI_DEV_AUTOINCR)
		for (; repcount; repcount--)
			*d++ = *h++;
	else
		for (; repcount; repcount--)
			*d = *h++;
	mutex_exit(hdlp->ahi_peekpoke_mutexp);
}

/*ARGSUSED*/
void
i_ddi_prot_io_rep_put16(ddi_acc_impl_t *hdlp, uint16_t *host_addr,
    uint16_t *dev_addr, size_t repcount, uint_t flags)
{
	uint16_t *h;
	uintptr_t port;

	h = host_addr;
	port = (uintptr_t)dev_addr;

	mutex_enter(hdlp->ahi_peekpoke_mutexp);
	if (flags == DDI_DEV_AUTOINCR)
		for (; repcount; repcount--, port += 2)
			outw(port, *h++);
	else
		for (; repcount; repcount--)
			outw(port, *h++);
	mutex_exit(hdlp->ahi_peekpoke_mutexp);
}

/*ARGSUSED*/
void
i_ddi_prot_vaddr_swap_rep_put16(ddi_acc_impl_t *hdlp, uint16_t *host_addr,
    uint16_t *dev_addr, size_t repcount, uint_t flags)
{
	uint16_t *h, *d;

	h = host_addr;
	d = dev_addr;

	mutex_enter(hdlp->ahi_peekpoke_mutexp);
	if (flags == DDI_DEV_AUTOINCR)
		for (; repcount; repcount--)
			*d++ = ddi_swap16(*h++);
	else
		for (; repcount; repcount--)
			*d = ddi_swap16(*h++);
	mutex_exit(hdlp->ahi_peekpoke_mutexp);
}

/*ARGSUSED*/
void
i_ddi_prot_io_swap_rep_put16(ddi_acc_impl_t *hdlp, uint16_t *host_addr,
    uint16_t *dev_addr, size_t repcount, uint_t flags)
{
	uint16_t *h;
	uintptr_t port;

	h = host_addr;
	port = (uintptr_t)dev_addr;

	mutex_enter(hdlp->ahi_peekpoke_mutexp);
	if (flags == DDI_DEV_AUTOINCR)
		for (; repcount; repcount--, port += 2)
			outw(port, ddi_swap16(*h++));
	else
		for (; repcount; repcount--)
			outw(port, ddi_swap16(*h++));
	mutex_exit(hdlp->ahi_peekpoke_mutexp);
}

/*ARGSUSED*/
void
i_ddi_prot_vaddr_rep_put32(ddi_acc_impl_t *hdlp, uint32_t *host_addr,
    uint32_t *dev_addr, size_t repcount, uint_t flags)
{
	uint32_t *h, *d;

	h = host_addr;
	d = dev_addr;

	mutex_enter(hdlp->ahi_peekpoke_mutexp);
	if (flags == DDI_DEV_AUTOINCR)
		for (; repcount; repcount--)
			*d++ = *h++;
	else
		for (; repcount; repcount--)
			*d = *h++;
	mutex_exit(hdlp->ahi_peekpoke_mutexp);
}

/*ARGSUSED*/
void
i_ddi_prot_io_rep_put32(ddi_acc_impl_t *hdlp, uint32_t *host_addr,
    uint32_t *dev_addr, size_t repcount, uint_t flags)
{
	uint32_t *h;
	uintptr_t port;

	h = host_addr;
	port = (uintptr_t)dev_addr;

	mutex_enter(hdlp->ahi_peekpoke_mutexp);
	if (flags == DDI_DEV_AUTOINCR)
		for (; repcount; repcount--, port += 4)
			outl(port, *h++);
	else
		for (; repcount; repcount--)
			outl(port, *h++);
	mutex_exit(hdlp->ahi_peekpoke_mutexp);
}

/*ARGSUSED*/
void
i_ddi_prot_vaddr_swap_rep_put32(ddi_acc_impl_t *hdlp, uint32_t *host_addr,
    uint32_t *dev_addr, size_t repcount, uint_t flags)
{
	uint32_t *h, *d;

	h = host_addr;
	d = dev_addr;

	mutex_enter(hdlp->ahi_peekpoke_mutexp);
	if (flags == DDI_DEV_AUTOINCR)
		for (; repcount; repcount--)
			*d++ = ddi_swap32(*h++);
	else
		for (; repcount; repcount--)
			*d = ddi_swap32(*h++);
	mutex_exit(hdlp->ahi_peekpoke_mutexp);
}

/*ARGSUSED*/
void
i_ddi_prot_io_swap_rep_put32(ddi_acc_impl_t *hdlp, uint32_t *host_addr,
    uint32_t *dev_addr, size_t repcount, uint_t flags)
{
	uint32_t *h;
	uintptr_t port;

	h = host_addr;
	port = (uintptr_t)dev_addr;

	mutex_enter(hdlp->ahi_peekpoke_mutexp);
	if (flags == DDI_DEV_AUTOINCR)
		for (; repcount; repcount--, port += 4)
			outl(port, ddi_swap32(*h++));
	else
		for (; repcount; repcount--)
			outl(port, ddi_swap32(*h++));
	mutex_exit(hdlp->ahi_peekpoke_mutexp);
}

/*ARGSUSED*/
void
i_ddi_prot_vaddr_rep_put64(ddi_acc_impl_t *hdlp, uint64_t *host_addr,
    uint64_t *dev_addr, size_t repcount, uint_t flags)
{
	uint64_t *h, *d;

	h = host_addr;
	d = dev_addr;

	mutex_enter(hdlp->ahi_peekpoke_mutexp);
	if (flags == DDI_DEV_AUTOINCR)
		for (; repcount; repcount--)
			*d++ = *h++;
	else
		for (; repcount; repcount--)
			*d = *h++;
	mutex_exit(hdlp->ahi_peekpoke_mutexp);
}

/*ARGSUSED*/
void
i_ddi_prot_vaddr_swap_rep_put64(ddi_acc_impl_t *hdlp, uint64_t *host_addr,
    uint64_t *dev_addr, size_t repcount, uint_t flags)
{
	uint64_t *h, *d;

	h = host_addr;
	d = dev_addr;

	mutex_enter(hdlp->ahi_peekpoke_mutexp);
	if (flags == DDI_DEV_AUTOINCR)
		for (; repcount; repcount--)
			*d++ = ddi_swap64(*h++);
	else
		for (; repcount; repcount--)
			*d = ddi_swap64(*h++);
	mutex_exit(hdlp->ahi_peekpoke_mutexp);
}

void
ddi_io_rep_get8(ddi_acc_handle_t handle,
    uint8_t *host_addr, uint8_t *dev_addr, size_t repcount)
{
	(((ddi_acc_impl_t *)handle)->ahi_rep_get8)
	    ((ddi_acc_impl_t *)handle, host_addr, dev_addr,
	    repcount, DDI_DEV_NO_AUTOINCR);
}

void
ddi_io_rep_get16(ddi_acc_handle_t handle,
    uint16_t *host_addr, uint16_t *dev_addr, size_t repcount)
{
	(((ddi_acc_impl_t *)handle)->ahi_rep_get16)
	    ((ddi_acc_impl_t *)handle, host_addr, dev_addr,
	    repcount, DDI_DEV_NO_AUTOINCR);
}

void
ddi_io_rep_get32(ddi_acc_handle_t handle,
    uint32_t *host_addr, uint32_t *dev_addr, size_t repcount)
{
	(((ddi_acc_impl_t *)handle)->ahi_rep_get32)
	    ((ddi_acc_impl_t *)handle, host_addr, dev_addr,
	    repcount, DDI_DEV_NO_AUTOINCR);
}

/*ARGSUSED*/
void
i_ddi_io_rep_get64(ddi_acc_impl_t *hdlp, uint64_t *host_addr,
    uint64_t *dev_addr, size_t repcount, uint_t flags)
{
	cmn_err(CE_PANIC, "ddi_rep_get64 from i/o space");
}

void
ddi_io_rep_put8(ddi_acc_handle_t handle,
    uint8_t *host_addr, uint8_t *dev_addr, size_t repcount)
{
	(((ddi_acc_impl_t *)handle)->ahi_rep_put8)
	    ((ddi_acc_impl_t *)handle, host_addr, dev_addr,
	    repcount, DDI_DEV_NO_AUTOINCR);
}

void
ddi_io_rep_put16(ddi_acc_handle_t handle,
    uint16_t *host_addr, uint16_t *dev_addr, size_t repcount)
{
	(((ddi_acc_impl_t *)handle)->ahi_rep_put16)
	    ((ddi_acc_impl_t *)handle, host_addr, dev_addr,
	    repcount, DDI_DEV_NO_AUTOINCR);
}

void
ddi_io_rep_put32(ddi_acc_handle_t handle,
    uint32_t *host_addr, uint32_t *dev_addr, size_t repcount)
{
	(((ddi_acc_impl_t *)handle)->ahi_rep_put32)
	    ((ddi_acc_impl_t *)handle, host_addr, dev_addr,
	    repcount, DDI_DEV_NO_AUTOINCR);
}

/*ARGSUSED*/
void
i_ddi_io_rep_put64(ddi_acc_impl_t *hdlp, uint64_t *host_addr,
    uint64_t *dev_addr, size_t repcount, uint_t flags)
{
	cmn_err(CE_PANIC, "ddi_rep_put64 to i/o space");
}

/*
 * These next two functions could be translated into assembler someday
 */
int
ddi_check_acc_handle(ddi_acc_handle_t handle)
{
	ddi_acc_impl_t *hdlp = (ddi_acc_impl_t *)handle;
	return (((*hdlp->ahi_fault_check)(hdlp) == DDI_SUCCESS) ? DDI_SUCCESS :
	    DDI_FAILURE);
}

int
i_ddi_acc_fault_check(ddi_acc_impl_t *hdlp)
{
	/* Default version, just returns flag value */
	return (hdlp->ahi_fault);
}

/*ARGSUSED*/
void
i_ddi_acc_fault_notify(ddi_acc_impl_t *hdlp)
{
	/* Default version, does nothing for now */
}

void
i_ddi_acc_set_fault(ddi_acc_handle_t handle)
{
	ddi_acc_impl_t *hdlp = (ddi_acc_impl_t *)handle;

	if (!hdlp->ahi_fault) {
		hdlp->ahi_fault = 1;
		(*hdlp->ahi_fault_notify)(hdlp);
	}
}

void
i_ddi_acc_clr_fault(ddi_acc_handle_t handle)
{
	ddi_acc_impl_t *hdlp = (ddi_acc_impl_t *)handle;

	if (hdlp->ahi_fault) {
		hdlp->ahi_fault = 0;
		(*hdlp->ahi_fault_notify)(hdlp);
	}
}
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
 */

/*
 * Copyright 2018 Joyent, Inc. All rights reserved.
 * Copyright 2022 Oxide Computer Compnay
 */

/*
 * Copyright (c) 1992 Terrence R. Lambert.
 * Copyright (c) 1990 The Regents of the University of California.
 * All rights reserved.
 *
 * This code is derived from software contributed to Berkeley by
 * William Jolitz.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. All advertising materials mentioning features or use of this software
 *    must display the following acknowledgement:
 *	This product includes software developed by the University of
 *	California, Berkeley and its contributors.
 * 4. Neither the name of the University nor the names of its contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 *	from: @(#)machdep.c	7.4 (Berkeley) 6/3/91
 */

#include <sys/types.h>
#include <sys/sysmacros.h>
#include <sys/tss.h>
#include <sys/segments.h>
#include <sys/trap.h>
#include <sys/cpuvar.h>
#include <sys/bootconf.h>
#include <sys/x86_archext.h>
#include <sys/controlregs.h>
#include <sys/archsystm.h>
#include <sys/machsystm.h>
#include <sys/kobj.h>
#include <sys/cmn_err.h>
#include <sys/reboot.h>
#include <sys/kdi.h>
#include <sys/mach_mmu.h>
#include <sys/systm.h>
#include <sys/note.h>

#ifdef __xpv
#include <sys/hypervisor.h>
#include <vm/as.h>
#endif

#include <sys/promif.h>
#include <sys/bootinfo.h>
#include <vm/kboot_mmu.h>
#include <vm/hat_pte.h>

/*
 * cpu0 and default tables and structures.
 */
user_desc_t	*gdt0;
#if !defined(__xpv)
desctbr_t	gdt0_default_r;
#endif

gate_desc_t	*idt0;		/* interrupt descriptor table */

tss_t		*ktss0;			/* kernel task state structure */


user_desc_t	zero_udesc;		/* base zero user desc native procs */
user_desc_t	null_udesc;		/* null user descriptor */
system_desc_t	null_sdesc;		/* null system descriptor */

user_desc_t	zero_u32desc;		/* 32-bit compatibility procs */

user_desc_t	ucs_on;
user_desc_t	ucs_off;
user_desc_t	ucs32_on;
user_desc_t	ucs32_off;

/*
 * If the size of this is changed, you must update hat_pcp_setup() and the
 * definitions in exception.s
 */
extern char dblfault_stack0[DEFAULTSTKSZ];
extern char nmi_stack0[DEFAULTSTKSZ];
extern char mce_stack0[DEFAULTSTKSZ];

extern void	fast_null(void);
extern hrtime_t	get_hrtime(void);
extern hrtime_t	gethrvtime(void);
extern hrtime_t	get_hrestime(void);
extern uint64_t	getlgrp(void);

void (*(fasttable[]))(void) = {
	fast_null,			/* T_FNULL routine */
	fast_null,			/* T_FGETFP routine (initially null) */
	fast_null,			/* T_FSETFP routine (initially null) */
	(void (*)())(uintptr_t)get_hrtime,	/* T_GETHRTIME */
	(void (*)())(uintptr_t)gethrvtime,	/* T_GETHRVTIME */
	(void (*)())(uintptr_t)get_hrestime,	/* T_GETHRESTIME */
	(void (*)())(uintptr_t)getlgrp		/* T_GETLGRP */
};

/*
 * Structure containing pre-computed descriptors to allow us to temporarily
 * interpose on a standard handler.
 */
struct interposing_handler {
	int ih_inum;
	gate_desc_t ih_interp_desc;
	gate_desc_t ih_default_desc;
};

/*
 * The brand infrastructure interposes on two handlers, and we use one as a
 * NULL signpost.
 */
static struct interposing_handler brand_tbl[2];

/*
 * software prototypes for default local descriptor table
 */

/*
 * Routines for loading segment descriptors in format the hardware
 * can understand.
 */

/*
 * In long mode we have the new L or long mode attribute bit
 * for code segments. Only the conforming bit in type is used along
 * with descriptor priority and present bits. Default operand size must
 * be zero when in long mode. In 32-bit compatibility mode all fields
 * are treated as in legacy mode. For data segments while in long mode
 * only the present bit is loaded.
 */
void
set_usegd(user_desc_t *dp, uint_t lmode, void *base, uint32_t size,
    uint_t type, uint_t dpl, uint_t gran, uint_t defopsz)
{
	ASSERT(lmode == SDP_SHORT || lmode == SDP_LONG);
	/* This should never be a "system" segment. */
	ASSERT3U(type & SDT_S, !=, 0);
	ASSERT3P(dp, !=, NULL);

	/*
	 * 64-bit long mode.
	 */
	if (lmode == SDP_LONG)
		dp->usd_def32 = 0;		/* 32-bit operands only */
	else
		/*
		 * 32-bit compatibility mode.
		 */
		dp->usd_def32 = defopsz;	/* 0 = 16, 1 = 32-bit ops */

	/*
	 * We should always set the "accessed" bit (SDT_A), otherwise the CPU
	 * will write to the GDT whenever we change segment registers around.
	 * With KPTI on, the GDT is read-only in the user page table, which
	 * causes crashes if we don't set this.
	 */
	ASSERT3U(type & SDT_A, !=, 0);

	dp->usd_long = lmode;	/* 64-bit mode */
	dp->usd_type = type;
	dp->usd_dpl = dpl;
	dp->usd_p = 1;
	dp->usd_gran = gran;		/* 0 = bytes, 1 = pages */

	dp->usd_lobase = (uintptr_t)base;
	dp->usd_midbase = (uintptr_t)base >> 16;
	dp->usd_hibase = (uintptr_t)base >> (16 + 8);
	dp->usd_lolimit = size;
	dp->usd_hilimit = (uintptr_t)size >> 16;
}

/*
 * Install system segment descriptor for LDT and TSS segments.
 */

void
set_syssegd(system_desc_t *dp, void *base, size_t size, uint_t type,
    uint_t dpl)
{
	dp->ssd_lolimit = size;
	dp->ssd_hilimit = (uintptr_t)size >> 16;

	dp->ssd_lobase = (uintptr_t)base;
	dp->ssd_midbase = (uintptr_t)base >> 16;
	dp->ssd_hibase = (uintptr_t)base >> (16 + 8);
	dp->ssd_hi64base = (uintptr_t)base >> (16 + 8 + 8);

	dp->ssd_type = type;
	dp->ssd_zero1 = 0;	/* must be zero */
	dp->ssd_zero2 = 0;
	dp->ssd_dpl = dpl;
	dp->ssd_p = 1;
	dp->ssd_gran = 0;	/* force byte units */
}

void *
get_ssd_base(system_desc_t *dp)
{
	uintptr_t	base;

	base = (uintptr_t)dp->ssd_lobase |
	    (uintptr_t)dp->ssd_midbase << 16 |
	    (uintptr_t)dp->ssd_hibase << (16 + 8) |
	    (uintptr_t)dp->ssd_hi64base << (16 + 8 + 8);
	return ((void *)base);
}

/*
 * Install gate segment descriptor for interrupt, trap, call and task gates.
 *
 * For 64 bit native if we have KPTI enabled, we use the IST stack mechanism on
 * all interrupts.  We have different ISTs for each class of exceptions that are
 * most likely to occur while handling an existing exception; while many of
 * these are just going to panic, it's nice not to trample on the existing
 * exception state for debugging purposes.
 *
 * Normal interrupts are all redirected unconditionally to the KPTI trampoline
 * stack space. This unifies the trampoline handling between user and kernel
 * space (and avoids the need to touch %gs).
 *
 * The KDI IDT *all* uses the DBG IST: consider single stepping tr_pftrap, when
 * we do a read from KMDB that cause another #PF.  Without its own IST, this
 * would stomp on the kernel's mcpu_kpti_flt frame.
 */
uint_t
idt_vector_to_ist(uint_t vector)
{
#if defined(__xpv)
	_NOTE(ARGUNUSED(vector));
	return (IST_NONE);
#else
	switch (vector) {
	/* These should always use IST even without KPTI enabled. */
	case T_DBLFLT:
		return (IST_DF);
	case T_NMIFLT:
		return (IST_NMI);
	case T_MCE:
		return (IST_MCE);

	case T_BPTFLT:
	case T_SGLSTP:
		if (kpti_enable == 1) {
			return (IST_DBG);
		}
		return (IST_NONE);
	case T_STKFLT:
	case T_GPFLT:
	case T_PGFLT:
		if (kpti_enable == 1) {
			return (IST_NESTABLE);
		}
		return (IST_NONE);
	default:
		if (kpti_enable == 1) {
			return (IST_DEFAULT);
		}
		return (IST_NONE);
	}
#endif
}

void
set_gatesegd(gate_desc_t *dp, void (*func)(void), selector_t sel,
    uint_t type, uint_t dpl, uint_t ist)
{
	dp->sgd_looffset = (uintptr_t)func;
	dp->sgd_hioffset = (uintptr_t)func >> 16;
	dp->sgd_hi64offset = (uintptr_t)func >> (16 + 16);
	dp->sgd_selector =  (uint16_t)sel;
	dp->sgd_ist = ist;
	dp->sgd_type = type;
	dp->sgd_dpl = dpl;
	dp->sgd_p = 1;
}

/*
 * Updates a single user descriptor in the the GDT of the current cpu.
 * Caller is responsible for preventing cpu migration.
 */

void
gdt_update_usegd(uint_t sidx, user_desc_t *udp)
{
#if defined(DEBUG)
	/* This should never be a "system" segment, but it might be null. */
	if (udp->usd_p != 0 || udp->usd_type != 0) {
		ASSERT3U(udp->usd_type & SDT_S, !=, 0);
	}
	/*
	 * We should always set the "accessed" bit (SDT_A), otherwise the CPU
	 * will write to the GDT whenever we change segment registers around.
	 * With KPTI on, the GDT is read-only in the user page table, which
	 * causes crashes if we don't set this.
	 */
	if (udp->usd_p != 0 || udp->usd_type != 0) {
		ASSERT3U(udp->usd_type & SDT_A, !=, 0);
	}
#endif

#if defined(__xpv)
	uint64_t dpa = CPU->cpu_m.mcpu_gdtpa + sizeof (*udp) * sidx;

	if (HYPERVISOR_update_descriptor(pa_to_ma(dpa), *(uint64_t *)udp))
		panic("gdt_update_usegd: HYPERVISOR_update_descriptor");

#else	/* __xpv */
	CPU->cpu_gdt[sidx] = *udp;
#endif	/* __xpv */
}

/*
 * Writes single descriptor pointed to by udp into a processes
 * LDT entry pointed to by ldp.
 */
int
ldt_update_segd(user_desc_t *ldp, user_desc_t *udp)
{
#if defined(DEBUG)
	/* This should never be a "system" segment, but it might be null. */
	if (udp->usd_p != 0 || udp->usd_type != 0) {
		ASSERT3U(udp->usd_type & SDT_S, !=, 0);
	}
	/*
	 * We should always set the "accessed" bit (SDT_A), otherwise the CPU
	 * will write to the LDT whenever we change segment registers around.
	 * With KPTI on, the LDT is read-only in the user page table, which
	 * causes crashes if we don't set this.
	 */
	if (udp->usd_p != 0 || udp->usd_type != 0) {
		ASSERT3U(udp->usd_type & SDT_A, !=, 0);
	}
#endif

#if defined(__xpv)
	uint64_t dpa;

	dpa = mmu_ptob(hat_getpfnum(kas.a_hat, (caddr_t)ldp)) |
	    ((uintptr_t)ldp & PAGEOFFSET);

	/*
	 * The hypervisor is a little more restrictive about what it
	 * supports in the LDT.
	 */
	if (HYPERVISOR_update_descriptor(pa_to_ma(dpa), *(uint64_t *)udp) != 0)
		return (EINVAL);

#else	/* __xpv */
	*ldp = *udp;

#endif	/* __xpv */
	return (0);
}

#if defined(__xpv)

/*
 * Converts hw format gate descriptor into pseudo-IDT format for the hypervisor.
 * Returns true if a valid entry was written.
 */
int
xen_idt_to_trap_info(uint_t vec, gate_desc_t *sgd, void *ti_arg)
{
	trap_info_t *ti = ti_arg;	/* XXPV	Aargh - segments.h comment */

	/*
	 * skip holes in the IDT
	 */
	if (GATESEG_GETOFFSET(sgd) == 0)
		return (0);

	ASSERT(sgd->sgd_type == SDT_SYSIGT);
	ti->vector = vec;
	TI_SET_DPL(ti, sgd->sgd_dpl);

	/*
	 * Is this an interrupt gate?
	 */
	if (sgd->sgd_type == SDT_SYSIGT) {
		/* LINTED */
		TI_SET_IF(ti, 1);
	}
	ti->cs = sgd->sgd_selector;
	ti->cs |= SEL_KPL;	/* force into ring 3. see KCS_SEL  */
	ti->address = GATESEG_GETOFFSET(sgd);
	return (1);
}

/*
 * Convert a single hw format gate descriptor and write it into our virtual IDT.
 */
void
xen_idt_write(gate_desc_t *sgd, uint_t vec)
{
	trap_info_t trapinfo[2];

	bzero(trapinfo, sizeof (trapinfo));
	if (xen_idt_to_trap_info(vec, sgd, &trapinfo[0]) == 0)
		return;
	if (xen_set_trap_table(trapinfo) != 0)
		panic("xen_idt_write: xen_set_trap_table() failed");
}

#endif	/* __xpv */


/*
 * Build kernel GDT.
 */

static void
init_gdt_common(user_desc_t *gdt)
{
	int i;

	ASSERT3P(gdt, !=, NULL);

	init_boot_gdt(gdt);

	/*
	 * 64-bit kernel code segment.
	 */
	set_usegd(&gdt[GDT_KCODE], SDP_LONG, NULL, 0, SDT_MEMERA, SEL_KPL,
	    SDP_PAGES, SDP_OP32);

	/*
	 * 64-bit kernel data segment. The limit attribute is ignored in 64-bit
	 * mode, but we set it here to SDP_LIMIT_MAX so that we can use the
	 * SYSRET instruction to return from system calls back to 32-bit
	 * applications.  SYSRET doesn't update the base, limit, or attributes
	 * of %ss or %ds descriptors. We therefore must ensure that the kernel
	 * uses something, though it will be ignored by hardware, that is
	 * compatible with 32-bit apps. For the same reason we must set the
	 * default op size of this descriptor to 32-bit operands.
	 */
	set_usegd(&gdt[GDT_KDATA], SDP_LONG, NULL, SDP_LIMIT_MAX, SDT_MEMRWA,
	    SEL_KPL, SDP_PAGES, SDP_OP32);
	gdt[GDT_KDATA].usd_def32 = 1;

	/*
	 * 64-bit user code segment.
	 */
	set_usegd(&gdt[GDT_UCODE], SDP_LONG, NULL, 0, SDT_MEMERA, SEL_UPL,
	    SDP_PAGES, SDP_OP32);

	/*
	 * 32-bit user code segment.
	 */
	set_usegd(&gdt[GDT_U32CODE], SDP_SHORT, NULL, SDP_LIMIT_MAX, SDT_MEMERA,
	    SEL_UPL, SDP_PAGES, SDP_OP32);

	/*
	 * See gdt_ucode32() and gdt_ucode_native().
	 */
	ucs_on = ucs_off = gdt[GDT_UCODE];
	ucs_off.usd_p = 0;	/* forces #np fault */

	ucs32_on = ucs32_off = gdt[GDT_U32CODE];
	ucs32_off.usd_p = 0;	/* forces #np fault */

	/*
	 * 32 and 64 bit data segments can actually share the same descriptor.
	 * In long mode only the present bit is checked but all other fields
	 * are loaded. But in compatibility mode all fields are interpreted
	 * as in legacy mode so they must be set correctly for a 32-bit data
	 * segment.
	 */
	set_usegd(&gdt[GDT_UDATA], SDP_SHORT, NULL, SDP_LIMIT_MAX, SDT_MEMRWA,
	    SEL_UPL, SDP_PAGES, SDP_OP32);

#if !defined(__xpv)

	/*
	 * The 64-bit kernel has no default LDT. By default, the LDT descriptor
	 * in the GDT is 0.
	 */

	/*
	 * Kernel TSS
	 */
	set_syssegd((system_desc_t *)&gdt[GDT_KTSS], ktss0,
	    sizeof (*ktss0) - 1, SDT_SYSTSS, SEL_KPL);

#endif	/* !__xpv */

	/*
	 * Initialize fs and gs descriptors for 32 bit processes.
	 * Only attributes and limits are initialized, the effective
	 * base address is programmed via fsbase/gsbase.
	 */
	set_usegd(&gdt[GDT_LWPFS], SDP_SHORT, NULL, SDP_LIMIT_MAX, SDT_MEMRWA,
	    SEL_UPL, SDP_PAGES, SDP_OP32);
	set_usegd(&gdt[GDT_LWPGS], SDP_SHORT, NULL, SDP_LIMIT_MAX, SDT_MEMRWA,
	    SEL_UPL, SDP_PAGES, SDP_OP32);

	/*
	 * Initialize the descriptors set aside for brand usage.
	 * Only attributes and limits are initialized.
	 */
	for (i = GDT_BRANDMIN; i <= GDT_BRANDMAX; i++)
		set_usegd(&gdt0[i], SDP_SHORT, NULL, SDP_LIMIT_MAX, SDT_MEMRWA,
		    SEL_UPL, SDP_PAGES, SDP_OP32);

	/*
	 * Initialize convenient zero base user descriptors for clearing
	 * lwp private %fs and %gs descriptors in GDT. See setregs() for
	 * an example.
	 */
	set_usegd(&zero_udesc, SDP_LONG, 0, 0, SDT_MEMRWA, SEL_UPL,
	    SDP_BYTES, SDP_OP32);
	set_usegd(&zero_u32desc, SDP_SHORT, 0, SDP_LIMIT_MAX, SDT_MEMRWA,
	    SEL_UPL, SDP_PAGES, SDP_OP32);
}

#if defined(__xpv)

static user_desc_t *
init_gdt(void)
{
	uint64_t gdtpa;
	ulong_t ma[1];		/* XXPV should be a memory_t */
	ulong_t addr;

	/*
	 * Our gdt is never larger than a single page.
	 */
	ASSERT((sizeof (*gdt0) * NGDT) <= PAGESIZE);
	gdt0 = (user_desc_t *)BOP_ALLOC(bootops, (caddr_t)GDT_VA,
	    PAGESIZE, PAGESIZE);
	ASSERT3P(gdt0, !=, NULL);
	bzero(gdt0, PAGESIZE);

	init_gdt_common(gdt0);

	/*
	 * XXX Since we never invoke kmdb until after the kernel takes
	 * over the descriptor tables why not have it use the kernel's
	 * selectors?
	 */
	if (boothowto & RB_DEBUG) {
		set_usegd(&gdt0[GDT_B32DATA], SDP_LONG, NULL, SDP_LIMIT_MAX,
		    SDT_MEMRWA, SEL_KPL, SDP_PAGES, SDP_OP32);
		set_usegd(&gdt0[GDT_B64CODE], SDP_LONG, NULL, SDP_LIMIT_MAX,
		    SDT_MEMERA, SEL_KPL, SDP_PAGES, SDP_OP32);
	}

	/*
	 * Clear write permission for page containing the gdt and install it.
	 */
	gdtpa = pfn_to_pa(va_to_pfn(gdt0));
	ma[0] = (ulong_t)(pa_to_ma(gdtpa) >> PAGESHIFT);
	kbm_read_only((uintptr_t)gdt0, gdtpa);
	xen_set_gdt(ma, NGDT);

	/*
	 * Reload the segment registers to use the new GDT.
	 * On 64-bit, fixup KCS_SEL to be in ring 3.
	 * See KCS_SEL in segments.h.
	 */
	load_segment_registers((KCS_SEL | SEL_KPL), KFS_SEL, KGS_SEL, KDS_SEL);

	/*
	 *  setup %gs for kernel
	 */
	xen_set_segment_base(SEGBASE_GS_KERNEL, (ulong_t)&cpus[0]);

	/*
	 * XX64 We should never dereference off "other gsbase" or
	 * "fsbase".  So, we should arrange to point FSBASE and
	 * KGSBASE somewhere truly awful e.g. point it at the last
	 * valid address below the hole so that any attempts to index
	 * off them cause an exception.
	 *
	 * For now, point it at 8G -- at least it should be unmapped
	 * until some 64-bit processes run.
	 */
	addr = 0x200000000ul;
	xen_set_segment_base(SEGBASE_FS, addr);
	xen_set_segment_base(SEGBASE_GS_USER, addr);
	xen_set_segment_base(SEGBASE_GS_USER_SEL, 0);

	return (gdt0);
}

#else	/* __xpv */

static user_desc_t *
init_gdt(void)
{
	/*
	 * Our gdt is never larger than a single page.
	 */
	ASSERT((sizeof (*gdt0) * NGDT) <= PAGESIZE);
	gdt0 = (user_desc_t *)BOP_ALLOC(bootops, (caddr_t)GDT_VA,
	    PAGESIZE, PAGESIZE);
	bzero(gdt0, PAGESIZE);

	init_gdt_common(gdt0);

	/*
	 * Install our new GDT
	 */
	gdt0_default_r.dtr_limit = (sizeof (*gdt0) * NGDT) - 1;
	gdt0_default_r.dtr_base = (uintptr_t)gdt0;
	wr_gdtr(&gdt0_default_r);

	/*
	 * Reload the segment registers to use the new GDT
	 */
	load_segment_registers(KCS_SEL, KFS_SEL, KGS_SEL, KDS_SEL);

	/*
	 *  setup %gs for kernel
	 */
	wrmsr(MSR_AMD_GSBASE, (uint64_t)&cpus[0]);

	/*
	 * XX64 We should never dereference off "other gsbase" or
	 * "fsbase".  So, we should arrange to point FSBASE and
	 * KGSBASE somewhere truly awful e.g. point it at the last
	 * valid address below the hole so that any attempts to index
	 * off them cause an exception.
	 *
	 * For now, point it at 8G -- at least it should be unmapped
	 * until some 64-bit processes run.
	 */
	wrmsr(MSR_AMD_FSBASE, 0x200000000ul);
	wrmsr(MSR_AMD_KGSBASE, 0x200000000ul);
	return (gdt0);
}

#endif	/* __xpv */


/*
 * Build kernel IDT.
 *
 * Note that for amd64 we pretty much require every gate to be an interrupt
 * gate which blocks interrupts atomically on entry; that's because of our
 * dependency on using 'swapgs' every time we come into the kernel to find
 * the cpu structure. If we get interrupted just before doing that, %cs could
 * be in kernel mode (so that the trap prolog doesn't do a swapgs), but
 * %gsbase is really still pointing at something in userland. Bad things will
 * ensue. We also use interrupt gates for i386 as well even though this is not
 * required for some traps.
 *
 * Perhaps they should have invented a trap gate that does an atomic swapgs?
 */
static void
init_idt_common(gate_desc_t *idt)
{
	set_gatesegd(&idt[T_ZERODIV],
	    (kpti_enable == 1) ? &tr_div0trap : &div0trap,
	    KCS_SEL, SDT_SYSIGT, TRP_KPL, idt_vector_to_ist(T_ZERODIV));
	set_gatesegd(&idt[T_SGLSTP],
	    (kpti_enable == 1) ? &tr_dbgtrap : &dbgtrap,
	    KCS_SEL, SDT_SYSIGT, TRP_KPL, idt_vector_to_ist(T_SGLSTP));
	set_gatesegd(&idt[T_NMIFLT],
	    (kpti_enable == 1) ? &tr_nmiint : &nmiint,
	    KCS_SEL, SDT_SYSIGT, TRP_KPL, idt_vector_to_ist(T_NMIFLT));
	set_gatesegd(&idt[T_BPTFLT],
	    (kpti_enable == 1) ? &tr_brktrap : &brktrap,
	    KCS_SEL, SDT_SYSIGT, TRP_UPL, idt_vector_to_ist(T_BPTFLT));
	set_gatesegd(&idt[T_OVFLW],
	    (kpti_enable == 1) ? &tr_ovflotrap : &ovflotrap,
	    KCS_SEL, SDT_SYSIGT, TRP_UPL, idt_vector_to_ist(T_OVFLW));
	set_gatesegd(&idt[T_BOUNDFLT],
	    (kpti_enable == 1) ? &tr_boundstrap : &boundstrap,
	    KCS_SEL, SDT_SYSIGT, TRP_KPL, idt_vector_to_ist(T_BOUNDFLT));
	set_gatesegd(&idt[T_ILLINST],
	    (kpti_enable == 1) ? &tr_invoptrap : &invoptrap,
	    KCS_SEL, SDT_SYSIGT, TRP_KPL, idt_vector_to_ist(T_ILLINST));
	set_gatesegd(&idt[T_NOEXTFLT],
	    (kpti_enable == 1) ? &tr_ndptrap : &ndptrap,
	    KCS_SEL, SDT_SYSIGT, TRP_KPL, idt_vector_to_ist(T_NOEXTFLT));

	/*
	 * double fault handler.
	 *
	 * Note that on the hypervisor a guest does not receive #df faults.
	 * Instead a failsafe event is injected into the guest if its selectors
	 * and/or stack is in a broken state. See xen_failsafe_callback.
	 */
#if !defined(__xpv)
	set_gatesegd(&idt[T_DBLFLT],
	    (kpti_enable == 1) ? &tr_syserrtrap : &syserrtrap,
	    KCS_SEL, SDT_SYSIGT, TRP_KPL, idt_vector_to_ist(T_DBLFLT));
#endif	/* !__xpv */

	/*
	 * T_EXTOVRFLT coprocessor-segment-overrun not supported.
	 */
	set_gatesegd(&idt[T_TSSFLT],
	    (kpti_enable == 1) ? &tr_invtsstrap : &invtsstrap,
	    KCS_SEL, SDT_SYSIGT, TRP_KPL, idt_vector_to_ist(T_TSSFLT));
	set_gatesegd(&idt[T_SEGFLT],
	    (kpti_enable == 1) ? &tr_segnptrap : &segnptrap,
	    KCS_SEL, SDT_SYSIGT, TRP_KPL, idt_vector_to_ist(T_SEGFLT));
	set_gatesegd(&idt[T_STKFLT],
	    (kpti_enable == 1) ? &tr_stktrap : &stktrap,
	    KCS_SEL, SDT_SYSIGT, TRP_KPL, idt_vector_to_ist(T_STKFLT));
	set_gatesegd(&idt[T_GPFLT],
	    (kpti_enable == 1) ? &tr_gptrap : &gptrap,
	    KCS_SEL, SDT_SYSIGT, TRP_KPL, idt_vector_to_ist(T_GPFLT));
	set_gatesegd(&idt[T_PGFLT],
	    (kpti_enable == 1) ? &tr_pftrap : &pftrap,
	    KCS_SEL, SDT_SYSIGT, TRP_KPL, idt_vector_to_ist(T_PGFLT));
	set_gatesegd(&idt[T_EXTERRFLT],
	    (kpti_enable == 1) ? &tr_ndperr : &ndperr,
	    KCS_SEL, SDT_SYSIGT, TRP_KPL, idt_vector_to_ist(T_EXTERRFLT));
	set_gatesegd(&idt[T_ALIGNMENT],
	    (kpti_enable == 1) ? &tr_achktrap : &achktrap,
	    KCS_SEL, SDT_SYSIGT, TRP_KPL, idt_vector_to_ist(T_ALIGNMENT));
	set_gatesegd(&idt[T_MCE],
	    (kpti_enable == 1) ? &tr_mcetrap : &mcetrap,
	    KCS_SEL, SDT_SYSIGT, TRP_KPL, idt_vector_to_ist(T_MCE));
	set_gatesegd(&idt[T_SIMDFPE],
	    (kpti_enable == 1) ? &tr_xmtrap : &xmtrap,
	    KCS_SEL, SDT_SYSIGT, TRP_KPL, idt_vector_to_ist(T_SIMDFPE));

	/*
	 * install fast trap handler at 210.
	 */
	set_gatesegd(&idt[T_FASTTRAP],
	    (kpti_enable == 1) ? &tr_fasttrap : &fasttrap,
	    KCS_SEL, SDT_SYSIGT, TRP_UPL, idt_vector_to_ist(T_FASTTRAP));

	/*
	 * System call handler.
	 */
	set_gatesegd(&idt[T_SYSCALLINT],
	    (kpti_enable == 1) ? &tr_sys_syscall_int : &sys_syscall_int,
	    KCS_SEL, SDT_SYSIGT, TRP_UPL, idt_vector_to_ist(T_SYSCALLINT));

	/*
	 * Install the DTrace interrupt handler for the pid provider.
	 */
	set_gatesegd(&idt[T_DTRACE_RET],
	    (kpti_enable == 1) ? &tr_dtrace_ret : &dtrace_ret,
	    KCS_SEL, SDT_SYSIGT, TRP_UPL, idt_vector_to_ist(T_DTRACE_RET));

	/*
	 * Prepare interposing descriptor for the syscall handler
	 * and cache copy of the default descriptor.
	 */
	brand_tbl[0].ih_inum = T_SYSCALLINT;
	brand_tbl[0].ih_default_desc = idt0[T_SYSCALLINT];

	set_gatesegd(&(brand_tbl[0].ih_interp_desc),
	    (kpti_enable == 1) ? &tr_brand_sys_syscall_int :
	    &brand_sys_syscall_int, KCS_SEL, SDT_SYSIGT, TRP_UPL,
	    idt_vector_to_ist(T_SYSCALLINT));

	brand_tbl[1].ih_inum = 0;
}

#if defined(__xpv)

static void
init_idt(gate_desc_t *idt)
{
	init_idt_common(idt);
}

#else	/* __xpv */

static void
init_idt(gate_desc_t *idt)
{
	char	ivctname[80];
	void	(*ivctptr)(void);
	int	i;

	/*
	 * Initialize entire table with 'reserved' trap and then overwrite
	 * specific entries. T_EXTOVRFLT (9) is unsupported and reserved
	 * since it can only be generated on a 386 processor. 15 is also
	 * unsupported and reserved.
	 */
	for (i = 0; i < NIDT; i++) {
		set_gatesegd(&idt[i],
		    (kpti_enable == 1) ? &tr_resvtrap : &resvtrap,
		    KCS_SEL, SDT_SYSIGT, TRP_KPL,
		    idt_vector_to_ist(T_RESVTRAP));
	}

	/*
	 * 20-31 reserved
	 */
	for (i = 20; i < 32; i++) {
		set_gatesegd(&idt[i],
		    (kpti_enable == 1) ? &tr_invaltrap : &invaltrap,
		    KCS_SEL, SDT_SYSIGT, TRP_KPL,
		    idt_vector_to_ist(T_INVALTRAP));
	}

	/*
	 * interrupts 32 - 255
	 */
	for (i = 32; i < 256; i++) {
		(void) snprintf(ivctname, sizeof (ivctname),
		    (kpti_enable == 1) ? "tr_ivct%d" : "ivct%d", i);
		ivctptr = (void (*)(void))kobj_getsymvalue(ivctname, 0);
		if (ivctptr == NULL)
			panic("kobj_getsymvalue(%s) failed", ivctname);

		set_gatesegd(&idt[i], ivctptr, KCS_SEL, SDT_SYSIGT, TRP_KPL,
		    idt_vector_to_ist(i));
	}

	/*
	 * Now install the common ones. Note that it will overlay some
	 * entries installed above like T_SYSCALLINT, T_FASTTRAP etc.
	 */
	init_idt_common(idt);
}

#endif	/* __xpv */

/*
 * The kernel does not deal with LDTs unless a user explicitly creates
 * one. Under normal circumstances, the LDTR contains 0. Any process attempting
 * to reference the LDT will therefore cause a #gp. System calls made via the
 * obsolete lcall mechanism are emulated by the #gp fault handler.
 */
static void
init_ldt(void)
{
#if defined(__xpv)
	xen_set_ldt(NULL, 0);
#else
	wr_ldtr(0);
#endif
}

#if !defined(__xpv)

static void
init_tss(void)
{
	extern struct cpu cpus[];

	/*
	 * tss_rsp0 is dynamically filled in by resume() (in swtch.s) on each
	 * context switch but it'll be overwritten with this same value anyway.
	 */
	if (kpti_enable == 1) {
		ktss0->tss_rsp0 = (uint64_t)&cpus->cpu_m.mcpu_kpti.kf_tr_rsp;
	}

	/* Set up the IST stacks for double fault, NMI, MCE. */
	ktss0->tss_ist1 = (uintptr_t)&dblfault_stack0[sizeof (dblfault_stack0)];
	ktss0->tss_ist2 = (uintptr_t)&nmi_stack0[sizeof (nmi_stack0)];
	ktss0->tss_ist3 = (uintptr_t)&mce_stack0[sizeof (mce_stack0)];

	/*
	 * This IST stack is used for #DB,#BP (debug) interrupts (when KPTI is
	 * enabled), and also for KDI (always).
	 */
	ktss0->tss_ist4 = (uint64_t)&cpus->cpu_m.mcpu_kpti_dbg.kf_tr_rsp;

	if (kpti_enable == 1) {
		/* This IST stack is used for #GP,#PF,#SS (fault) interrupts. */
		ktss0->tss_ist5 =
		    (uint64_t)&cpus->cpu_m.mcpu_kpti_flt.kf_tr_rsp;

		/* This IST stack is used for all other intrs (for KPTI). */
		ktss0->tss_ist6 = (uint64_t)&cpus->cpu_m.mcpu_kpti.kf_tr_rsp;
	}

	/*
	 * Set I/O bit map offset equal to size of TSS segment limit
	 * for no I/O permission map. This will force all user I/O
	 * instructions to generate #gp fault.
	 */
	ktss0->tss_bitmapbase = sizeof (*ktss0);

	/*
	 * Point %tr to descriptor for ktss0 in gdt.
	 */
	wr_tsr(KTSS_SEL);
}

#endif	/* !__xpv */

#if defined(__xpv)

void
init_desctbls(void)
{
	uint_t vec;
	user_desc_t *gdt;

	/*
	 * Setup and install our GDT.
	 */
	gdt = init_gdt();

	/*
	 * Store static pa of gdt to speed up pa_to_ma() translations
	 * on lwp context switches.
	 */
	ASSERT(IS_P2ALIGNED((uintptr_t)gdt, PAGESIZE));
	CPU->cpu_gdt = gdt;
	CPU->cpu_m.mcpu_gdtpa = pfn_to_pa(va_to_pfn(gdt));

	/*
	 * Setup and install our IDT.
	 */
	ASSERT(NIDT * sizeof (*idt0) <= PAGESIZE);
	idt0 = (gate_desc_t *)BOP_ALLOC(bootops, (caddr_t)IDT_VA,
	    PAGESIZE, PAGESIZE);
	bzero(idt0, PAGESIZE);
	init_idt(idt0);
	for (vec = 0; vec < NIDT; vec++)
		xen_idt_write(&idt0[vec], vec);

	CPU->cpu_idt = idt0;

	/*
	 * set default kernel stack
	 */
	xen_stack_switch(KDS_SEL,
	    (ulong_t)&dblfault_stack0[sizeof (dblfault_stack0)]);

	xen_init_callbacks();

	init_ldt();
}

#else	/* __xpv */

void
init_desctbls(void)
{
	user_desc_t *gdt;
	desctbr_t idtr;

	/*
	 * Allocate IDT and TSS structures on unique pages for better
	 * performance in virtual machines.
	 */
	ASSERT(NIDT * sizeof (*idt0) <= PAGESIZE);
	idt0 = (gate_desc_t *)BOP_ALLOC(bootops, (caddr_t)IDT_VA,
	    PAGESIZE, PAGESIZE);
	bzero(idt0, PAGESIZE);
	ASSERT(sizeof (*ktss0) <= PAGESIZE);
	ktss0 = (tss_t *)BOP_ALLOC(bootops, (caddr_t)KTSS_VA,
	    PAGESIZE, PAGESIZE);
	bzero(ktss0, PAGESIZE);


	/*
	 * Setup and install our GDT.
	 */
	gdt = init_gdt();
	ASSERT(IS_P2ALIGNED((uintptr_t)gdt, PAGESIZE));
	CPU->cpu_gdt = gdt;

	/*
	 * Initialize this CPU's LDT.
	 */
	CPU->cpu_m.mcpu_ldt = BOP_ALLOC(bootops, (caddr_t)LDT_VA,
	    LDT_CPU_SIZE, PAGESIZE);
	bzero(CPU->cpu_m.mcpu_ldt, LDT_CPU_SIZE);
	CPU->cpu_m.mcpu_ldt_len = 0;

	/*
	 * Setup and install our IDT.
	 */
	init_idt(idt0);

	idtr.dtr_base = (uintptr_t)idt0;
	idtr.dtr_limit = (NIDT * sizeof (*idt0)) - 1;
	wr_idtr(&idtr);
	CPU->cpu_idt = idt0;


	init_tss();
	CPU->cpu_tss = ktss0;
	init_ldt();

	/* Stash this so that the NMI,MCE,#DF and KDI handlers can use it. */
	kpti_safe_cr3 = (uint64_t)getcr3();
}

#endif	/* __xpv */

#ifndef __xpv
/*
 * As per Intel Vol 3 27.5.2, the GDTR limit is reset to 64Kb on a VM exit, so
 * we have to manually fix it up ourselves.
 *
 * The caller may still need to make sure that it can't go off-CPU with the
 * incorrect limit, before calling this (such as disabling pre-emption).
 */
void
reset_gdtr_limit(void)
{
	ulong_t flags = intr_clear();
	desctbr_t gdtr;

	rd_gdtr(&gdtr);
	gdtr.dtr_limit = (sizeof (user_desc_t) * NGDT) - 1;
	wr_gdtr(&gdtr);

	intr_restore(flags);
}
#endif /* __xpv */

/*
 * We need a GDT owned by the kernel and not the bootstrap relatively
 * early in kernel initialization (e.g., to have segments we can reliably
 * catch an exception on).
 *
 * Initializes a GDT with segments normally defined in the boot loader.
 */
void
init_boot_gdt(user_desc_t *bgdt)
{
	ASSERT3P(bgdt, !=, NULL);

#ifdef	__xpv
	/* XXX: It is unclear why this 32-bit data segment is marked long. */
	set_usegd(&bgdt[GDT_B32DATA], SDP_LONG, NULL, SDP_LIMIT_MAX, SDT_MEMRWA,
	    SEL_KPL, SDP_PAGES, SDP_OP32);
#else
	/*
	 * Reset boot segments.  These ostensibly come from the boot loader,
	 * but we reset them to match our expectations, particulary if we
	 * are not using that loader.
	 */
	set_usegd(&bgdt[GDT_B32DATA], SDP_SHORT, NULL, SDP_LIMIT_MAX,
	    SDT_MEMRWA, SEL_KPL, SDP_PAGES, SDP_OP32);
	set_usegd(&bgdt[GDT_B32CODE], SDP_SHORT, NULL, SDP_LIMIT_MAX,
	    SDT_MEMERA, SEL_KPL, SDP_PAGES, SDP_OP32);

	/*
	 * 16-bit segments for making BIOS calls (not applicable on all
	 * architectures).
	 */
	set_usegd(&bgdt[GDT_B16CODE], SDP_SHORT, NULL, SDP_LIMIT_MAX,
	    SDT_MEMERA, SEL_KPL, 0, 0);
	/*
	 * XXX: SDP_OP32 makes this a 32-bit segment, which seems wrong
	 * here, but that's what boot_gdt.s used.
	 */
	set_usegd(&bgdt[GDT_B16DATA], SDP_SHORT, NULL, SDP_LIMIT_MAX,
	    SDT_MEMRWA, SEL_KPL, 0, SDP_OP32);
#endif	/* __xpv */

	/*
	 * A 64-bit code segment used in early boot.  Early IDTs refer to this.
	 */
	set_usegd(&bgdt[GDT_B64CODE], SDP_LONG, NULL, SDP_LIMIT_MAX, SDT_MEMERA,
	    SEL_KPL, SDP_PAGES, SDP_OP32);
}

/*
 * Enable interpositioning on the system call path by rewriting the
 * sys{call|enter} MSRs and the syscall-related entries in the IDT to use
 * the branded entry points.
 */
void
brand_interpositioning_enable(void *arg __unused)
{
	gate_desc_t	*idt = CPU->cpu_idt;
	int		i;

	ASSERT(curthread->t_preempt != 0 || getpil() >= DISP_LEVEL);

	for (i = 0; brand_tbl[i].ih_inum; i++) {
		idt[brand_tbl[i].ih_inum] = brand_tbl[i].ih_interp_desc;
#if defined(__xpv)
		xen_idt_write(&idt[brand_tbl[i].ih_inum],
		    brand_tbl[i].ih_inum);
#endif
	}

#if defined(__xpv)

	/*
	 * Currently the hypervisor only supports 64-bit syscalls via
	 * syscall instruction. The 32-bit syscalls are handled by
	 * interrupt gate above.
	 */
	xen_set_callback(brand_sys_syscall, CALLBACKTYPE_syscall,
	    CALLBACKF_mask_events);

#else

	if (is_x86_feature(x86_featureset, X86FSET_ASYSC)) {
		if (kpti_enable == 1) {
			wrmsr(MSR_AMD_LSTAR, (uintptr_t)tr_brand_sys_syscall);
			wrmsr(MSR_AMD_CSTAR, (uintptr_t)tr_brand_sys_syscall32);
		} else {
			wrmsr(MSR_AMD_LSTAR, (uintptr_t)brand_sys_syscall);
			wrmsr(MSR_AMD_CSTAR, (uintptr_t)brand_sys_syscall32);
		}
	}

#endif

	if (is_x86_feature(x86_featureset, X86FSET_SEP)) {
		if (kpti_enable == 1) {
			wrmsr(MSR_INTC_SEP_EIP,
			    (uintptr_t)tr_brand_sys_sysenter);
		} else {
			wrmsr(MSR_INTC_SEP_EIP, (uintptr_t)brand_sys_sysenter);
		}
	}
}

/*
 * Disable interpositioning on the system call path by rewriting the
 * sys{call|enter} MSRs and the syscall-related entries in the IDT to use
 * the standard entry points, which bypass the interpositioning hooks.
 */
void
brand_interpositioning_disable(void *arg __unused)
{
	gate_desc_t	*idt = CPU->cpu_idt;
	int i;

	ASSERT(curthread->t_preempt != 0 || getpil() >= DISP_LEVEL);

	for (i = 0; brand_tbl[i].ih_inum; i++) {
		idt[brand_tbl[i].ih_inum] = brand_tbl[i].ih_default_desc;
#if defined(__xpv)
		xen_idt_write(&idt[brand_tbl[i].ih_inum],
		    brand_tbl[i].ih_inum);
#endif
	}

#if defined(__xpv)

	/*
	 * See comment above in brand_interpositioning_enable.
	 */
	xen_set_callback(sys_syscall, CALLBACKTYPE_syscall,
	    CALLBACKF_mask_events);

#else

	if (is_x86_feature(x86_featureset, X86FSET_ASYSC)) {
		if (kpti_enable == 1) {
			wrmsr(MSR_AMD_LSTAR, (uintptr_t)tr_sys_syscall);
			wrmsr(MSR_AMD_CSTAR, (uintptr_t)tr_sys_syscall32);
		} else {
			wrmsr(MSR_AMD_LSTAR, (uintptr_t)sys_syscall);
			wrmsr(MSR_AMD_CSTAR, (uintptr_t)sys_syscall32);
		}
	}

#endif

	if (is_x86_feature(x86_featureset, X86FSET_SEP)) {
		if (kpti_enable == 1) {
			wrmsr(MSR_INTC_SEP_EIP, (uintptr_t)tr_sys_sysenter);
		} else {
			wrmsr(MSR_INTC_SEP_EIP, (uintptr_t)sys_sysenter);
		}
	}
}
#
# Device policy configuration file.   When devices are opened the
# additional access controls in this file are enforced.
#
# The format of this file is subject to change without notice.
#
# Default open privileges, must be first entry in the file.
#

*		read_priv_set=none		write_priv_set=none
md:admin					write_priv_set=sys_config
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License, Version 1.0 only
 * (the "License").  You may not use this file except in compliance
 * with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#include <sys/types.h>
#include <sys/systm.h>
#include <sys/door.h>
#include <sys/proc.h>
#include <sys/thread.h>
#include <sys/stack.h>
#include <sys/privregs.h>

int
door_finish_dispatch(caddr_t newsp)
{
	void lwp_setsp(klwp_t *, caddr_t);

	/*
	 * If being traced, need to copy in syscall arguments for /proc
	 * before changing the sp.  In most cases, this will have no
	 * effect, since we'll have already done this in door_return().
	 */
	if (curthread->t_post_sys && PTOU(ttoproc(curthread))->u_systrap)
		(void) save_syscall_args();
	lwp_setsp(ttolwp(curthread), newsp);
	lwptoregs(ttolwp(curthread))->r_fp = 0;	/* stack ends here */

	return (0);
}

/*ARGSUSED*/
uintptr_t
door_final_sp(uintptr_t resultsp, size_t align, int datamodel)
{
	return (resultsp);
}
asy "pci11c1,480"
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#include <sys/types.h>
#include <sys/time.h>
#include <sys/nvpair.h>
#include <sys/cmn_err.h>
#include <sys/fm/util.h>
#include <sys/fm/protocol.h>
#include <sys/smbios.h>
#include <sys/smbios_impl.h>

/*
 * Variable used to determine if the x86 generic topology enumerator will
 * revert to legacy enumeration. I.E. Big Kill Switch... tunable via
 * /etc/system
 */
int x86gentopo_legacy = 0;

#define	MC		0
#define	PROC		1
#define	MAX_PAIRS	20
#define	MAX_CONT	40

typedef struct bbindex  {
	int count;
	uint16_t index[MAX_PAIRS];
} bbindex_t;

/*
 * the enum values come from DMTF
 */
typedef enum baseb {
	BB_BAD = 0,		/* There is no bb value 0 */
	BB_UNKNOWN,		/* Unknown */
	BB_OTHER,		/* Other */
	BB_BLADE,		/* Server Blade */
	BB_CONNSW,		/* Connectivity Switch */
	BB_SMM,			/* System Management Module */
	BB_PROCMOD,		/* Processor Module */
	BB_IOMOD,		/* I/O Module */
	BB_MEMMOD,		/* Memory Module */
	BB_DBOARD,		/* Daughter Board */
	BB_MBOARD,		/* Motherboard */
	BB_PROCMMOD,		/* Processor/Memory Module */
	BB_PROCIOMOD,		/* Processor/IO Module */
	BB_ICONNBD		/* Interconnect Board */
} bbd_t;

static struct bboard_type {
	bbd_t		baseb;
	const char	*name;
} bbd_type[] = {
	{BB_BAD,		NULL},
	{BB_UNKNOWN,		"unknown"},
	{BB_OTHER,		"other"},
	{BB_BLADE,		"systemboard"},
	{BB_CONNSW,		"connswitch"},
	{BB_SMM,		"smmodule"},
	{BB_PROCMOD,		"cpuboard"},
	{BB_IOMOD,		"ioboard"},
	{BB_MEMMOD,		"memboard"},
	{BB_DBOARD,		"systemboard"},
	{BB_MBOARD,		"motherboard"},
	{BB_PROCMMOD,		"systemboard"},
	{BB_PROCIOMOD,		"systemboard"},
	{BB_ICONNBD,		"systemboard"}
};

typedef struct smbs_con_ids {
	int id;
	int inst;
	int cont_count;
	uint16_t **cont_ids;
	int cont_by_id;
	int visited;
} smbs_con_ids_t;

typedef struct smbs_cnt {
	int type;			/* SMBIOS stucture type */
	int count;			/* number of table entries */
	smbs_con_ids_t **ids;		/* SMBIOS table entry id(s) */
} smbs_cnt_t;

/*
 * dynamically allocate the storage for the smbs_cnt_t
 */
static smbs_cnt_t *
smb_create_strcnt(int count)
{
	smbs_cnt_t *types = NULL;
	int i, j;

	types = kmem_zalloc(sizeof (smbs_cnt_t), KM_SLEEP);

	types->ids = (smbs_con_ids_t **)kmem_zalloc(
	    count * sizeof (smbs_con_ids_t *), KM_SLEEP);

	for (i = 0; i < count; i++) {
		types->ids[i] = (smbs_con_ids_t *)kmem_zalloc(
		    sizeof (smbs_con_ids_t), KM_SLEEP);
	}

	for (i = 0; i < count; i++) {
		types->ids[i]->cont_ids = (uint16_t **)kmem_zalloc(
		    MAX_CONT * sizeof (uint16_t *), KM_SLEEP);
	}

	for (i = 0; i < count; i++) {
		for (j = 0; j < MAX_CONT; j++) {
			types->ids[i]->cont_ids[j] = (uint16_t *)kmem_zalloc(
			    sizeof (uint16_t), KM_SLEEP);
		}
	}
	return (types);
}

/*
 * free the smbs_cnt_t memory
 */
static void
smb_free_strcnt(smbs_cnt_t *types, int count)
{
	int i, j;

	if (types == NULL)
		return;

	for (i = 0; i < count; i++) {
		for (j = 0; j < MAX_CONT; j++) {
			if (types->ids[i]->cont_ids[j] != NULL)
				kmem_free(types->ids[i]->cont_ids[j],
				    sizeof (uint16_t));
		}
	}

	for (i = 0; i < count; i++) {
		if (types->ids[i]->cont_ids != NULL)
			kmem_free(types->ids[i]->cont_ids,
			    MAX_CONT * sizeof (uint16_t *));
	}

	for (i = 0; i < count; i++) {
		if (types->ids[i] != NULL)
			kmem_free(types->ids[i], sizeof (smbs_con_ids_t));
	}

	if (types->ids != NULL)
		kmem_free(types->ids, count * sizeof (smbs_con_ids_t *));

	if (types != NULL)
		kmem_free(types, sizeof (smbs_cnt_t));

}

/*
 * count number of the structure type in the ksmbios
 */
static int
smb_cnttypes(smbios_hdl_t *shp, int type)
{
	const smb_struct_t *sp = shp->sh_structs;
	int nstructs = shp->sh_nstructs;
	int i;
	int cnt = 0;

	for (i = 0, cnt = 0; i < nstructs; i++, sp++) {
		if (sp->smbst_hdr->smbh_type == type)
			cnt++;
	}
	return (cnt);
}

static void
smb_strcnt(smbios_hdl_t *shp, smbs_cnt_t *stype)
{
	const smb_struct_t *sp = shp->sh_structs;
	int nstructs = shp->sh_nstructs;
	smbios_bboard_t bb;
	int i, cnt;
	int mb_cnt = 0;
	int cpub_cnt = 0;
	int sysb_cnt = 0;
	int memb_cnt = 0;
	int iob_cnt = 0;
	int inst = 0;
	int rc = 0;

	for (i = 0, cnt = 0; i < nstructs; i++, sp++) {
		if (sp->smbst_hdr->smbh_type == stype->type) {
			stype->ids[cnt]->id = sp->smbst_hdr->smbh_hdl;
			stype->ids[cnt]->inst = cnt;
			stype->ids[cnt]->visited = 0;
			stype->ids[cnt]->cont_by_id = -1;
			if (stype->type == SMB_TYPE_BASEBOARD) {
				rc = smbios_info_bboard(shp,
				    stype->ids[cnt]->id, &bb);
				if (rc == 0) {
					switch (bb.smbb_type) {
						case SMB_BBT_PROC :
							inst = cpub_cnt++;
							break;
						case SMB_BBT_IO :
							inst = iob_cnt++;
							break;
						case SMB_BBT_MEM :
							inst = memb_cnt++;
							break;
						case SMB_BBT_MOTHER :
							inst = mb_cnt++;
							break;
						default:
							/*
							 * SMB_BBT_UNKNOWN
							 * SMB_BBT_OTHER
							 * SMB_BBT_SBLADE
							 * SMB_BBT_CSWITCH
							 * SMB_BBT_SMM
							 * SMB_BBT_DAUGHTER
							 * SMB_BBT_PROCMEM
							 * SMB_BBT_PROCIO
							 * SMB_BBT_INTER
							 */
							inst = sysb_cnt++;
							break;
					}
					stype->ids[cnt]->inst = inst;
				}
			}
			cnt++;
		}
	}
	stype->count = cnt;
}

/*
 * Go through the smbios structures looking for type 2. Fill in
 * the cont_id and cont_by_id for each type 2
 *
 */
static void
smb_bb_contains(smbios_hdl_t *shp, smbs_cnt_t *stype)
{
	int i, j, cnt, c;
	uint_t cont_count;
	const smb_struct_t *spt;
	smbios_bboard_t smb_bb;
	uint16_t bb_id, cont_id;
	uint_t cont_len;
	id_t *cont_hdl = NULL;
	int rc;

	for (cnt = 0; cnt < stype->count; cnt++) {
		bb_id = stype->ids[cnt]->id;
		(void) smbios_info_bboard(shp, stype->ids[cnt]->id, &smb_bb);
		cont_count = (uint_t)smb_bb.smbb_contn;
		if (cont_count == 0) {
			continue;
		}

		cont_len = sizeof (id_t);
		cont_hdl = kmem_zalloc(cont_count * cont_len, KM_SLEEP);
		if (cont_hdl == NULL)
			continue;

		rc = smbios_info_contains(shp, stype->ids[cnt]->id,
		    cont_count, cont_hdl);
		if (rc > SMB_CONT_MAX) {
			kmem_free(cont_hdl, cont_count * cont_len);
			continue;
		}
		cont_count = MIN(rc, cont_count);

		/*
		 * fill in the type 2 and type 4 ids which are
		 * contained in this type 2
		 */
		c = 0;
		for (j = 0; j < cont_count; j++) {
			cont_id = (uint16_t)cont_hdl[j];
			spt = smb_lookup_id(shp, cont_id);
			if (spt->smbst_hdr->smbh_type == SMB_TYPE_BASEBOARD ||
			    spt->smbst_hdr->smbh_type == SMB_TYPE_PROCESSOR) {
				*stype->ids[cnt]->cont_ids[c] = cont_id;
				c++;
			}

			if (spt->smbst_hdr->smbh_type == SMB_TYPE_BASEBOARD) {
				for (i = 0; i < stype->count; i++) {
					if (stype->ids[i]->id == cont_id) {
						stype->ids[i]->cont_by_id =
						    bb_id;
					}
				}
			}

		}
		stype->ids[cnt]->cont_count = c;
		if (cont_hdl != NULL)
			kmem_free(cont_hdl, cont_count * cont_len);
	}
}

/*
 * Verify SMBIOS structures for x86 generic topology.
 *
 * Return (0) on success.
 */
static int
fm_smb_check(smbios_hdl_t *shp)
{
	int i, j;
	int bb_cnt = 0;
	int pr_cnt = 0;
	int expr_cnt = 0;
	int ma_cnt = 0;
	int exma_cnt = 0;
	int mdev_cnt = 0;
	int exmdev_cnt = 0;
	uint16_t bb_id;
	uint16_t pr_id, expr_id;
	uint16_t ma_id, exma_id;
	uint16_t mdev_id, exmdev_id;
	uint16_t *sys_ma;
	smbios_bboard_t bb;
	smbios_processor_ext_t exproc;
	smbios_memarray_t ma;
	smbios_memarray_ext_t exma;
	smbios_memdevice_t mdev;
	smbios_memdevice_ext_t exmdev;
	smbs_cnt_t *bb_stype;
	smbs_cnt_t *pr_stype, *expr_stype;
	smbs_cnt_t *ma_stype, *exma_stype;
	smbs_cnt_t *mdev_stype, *exmdev_stype;

	/*
	 * Verify the existance of the requuired extended OEM-Specific
	 * structures and they coincide with the structures they extend
	 * (e.g. the number of extended processor structures equal the
	 * number of processor structures).
	 */
	pr_cnt = smb_cnttypes(shp, SMB_TYPE_PROCESSOR);
	expr_cnt = smb_cnttypes(shp, SUN_OEM_EXT_PROCESSOR);
	ma_cnt = smb_cnttypes(shp, SMB_TYPE_MEMARRAY);
	exma_cnt = smb_cnttypes(shp, SUN_OEM_EXT_MEMARRAY);
	mdev_cnt = smb_cnttypes(shp, SMB_TYPE_MEMDEVICE);
	exmdev_cnt = smb_cnttypes(shp, SUN_OEM_EXT_MEMDEVICE);
	if (expr_cnt == 0 || exma_cnt == 0 || exmdev_cnt == 0 ||
	    expr_cnt != pr_cnt || exma_cnt > ma_cnt ||
	    exmdev_cnt > mdev_cnt) {
#ifdef	DEBUG
		cmn_err(CE_NOTE, "!Structure mismatch: ext_proc (%d) "
		    "proc (%d) ext_ma (%d) ma (%d) ext_mdev (%d) mdev (%d)\n",
		    expr_cnt, pr_cnt, exma_cnt, ma_cnt, exmdev_cnt,
		    mdev_cnt);
#endif	/* DEBUG */
		return (-1);
	}

	/*
	 * Verify the OEM-Specific structrures are correctly
	 * linked to the SMBIOS structure types they extend.
	 */

	/* allocate processor stypes */
	pr_stype = smb_create_strcnt(pr_cnt);
	expr_stype = smb_create_strcnt(expr_cnt);

	/* fill in stypes */
	pr_stype->type = SMB_TYPE_PROCESSOR;
	smb_strcnt(shp, pr_stype);
	expr_stype->type = SUN_OEM_EXT_PROCESSOR;
	smb_strcnt(shp, expr_stype);

	/* verify the ext proc struct belong to the proc struct */
	for (i = 0; i < pr_cnt; i++) {
		pr_id = pr_stype->ids[i]->id;
		expr_id = expr_stype->ids[i]->id;
		(void) smbios_info_extprocessor(shp, expr_id, &exproc);
		if (exproc.smbpe_processor != pr_id) {
#ifdef	DEBUG
			cmn_err(CE_NOTE, "!Processor struct linkage (%d)", i);
#endif	/* DEBUG */
			smb_free_strcnt(pr_stype, pr_cnt);
			smb_free_strcnt(expr_stype, expr_cnt);
			return (-1);
		}
	}

	/* free stypes */
	smb_free_strcnt(pr_stype, pr_cnt);
	smb_free_strcnt(expr_stype, expr_cnt);

	/* allocate memory array stypes */
	ma_stype = smb_create_strcnt(ma_cnt);
	exma_stype = smb_create_strcnt(exma_cnt);
	sys_ma = kmem_zalloc(sizeof (uint16_t) * ma_cnt, KM_SLEEP);

	/* fill in stypes */
	ma_stype->type = SMB_TYPE_MEMARRAY;
	smb_strcnt(shp, ma_stype);
	exma_stype->type = SUN_OEM_EXT_MEMARRAY;
	smb_strcnt(shp, exma_stype);

	/* verify linkage from ext memarray struct to memarray struct */
	for (i = 0; i < ma_cnt; i++) {
		sys_ma[i] = (uint16_t)-1;
		ma_id = ma_stype->ids[i]->id;
		(void) smbios_info_memarray(shp, ma_id, &ma);
		if (ma.smbma_use != SMB_MAU_SYSTEM)
			continue;
		/* this memarray is system memory */
		sys_ma[i] = ma_id;
		exma_id = exma_stype->ids[i]->id;
		(void) smbios_info_extmemarray(shp, exma_id, &exma);
		if (exma.smbmae_ma != ma_id) {
#ifdef	DEBUG
			cmn_err(CE_NOTE,
			    "!Memory Array struct linkage (%d)", i);
#endif	/* DEBUG */
			smb_free_strcnt(ma_stype, ma_cnt);
			smb_free_strcnt(exma_stype, exma_cnt);
			kmem_free(sys_ma, sizeof (uint16_t) * ma_cnt);
			return (-1);
		}
	}

	/* free stypes */
	smb_free_strcnt(ma_stype, ma_cnt);
	smb_free_strcnt(exma_stype, exma_cnt);

	/* allocate memory device stypes */
	mdev_stype = smb_create_strcnt(mdev_cnt);
	exmdev_stype = smb_create_strcnt(exmdev_cnt);

	/* fill in stypes */
	mdev_stype->type = SMB_TYPE_MEMDEVICE;
	smb_strcnt(shp, mdev_stype);
	exmdev_stype->type = SUN_OEM_EXT_MEMDEVICE;
	smb_strcnt(shp, exmdev_stype);

	/* verify linkage */
	for (i = 0; i < mdev_cnt; i++) {
		mdev_id = mdev_stype->ids[i]->id;
		(void) smbios_info_memdevice(shp, mdev_id, &mdev);
		/* only check system memory devices */
		for (j = 0; j < ma_cnt; j++) {
			if (sys_ma[j] == mdev.smbmd_array)
				break;
		}
		if (j == ma_cnt)
			continue;
		exmdev_id = exmdev_stype->ids[i]->id;
		(void) smbios_info_extmemdevice(shp, exmdev_id, &exmdev);
		if (exmdev.smbmdeve_md != mdev_id) {
#ifdef	DEBUG
			cmn_err(CE_NOTE, "!Memory Device struct linkage (%d)",
			    i);
#endif	/* DEBUG */
			smb_free_strcnt(mdev_stype, mdev_cnt);
			smb_free_strcnt(exmdev_stype, exmdev_cnt);
			kmem_free(sys_ma, sizeof (uint16_t) * ma_cnt);
			return (-1);
		}
	}

	/* free stypes */
	smb_free_strcnt(mdev_stype, mdev_cnt);
	smb_free_strcnt(exmdev_stype, exmdev_cnt);
	kmem_free(sys_ma, sizeof (uint16_t) * ma_cnt);

	/*
	 * Verify the presece of contained handles if there are more
	 * than one Type-2 (Base Board) structures.
	 */
	bb_cnt = smb_cnttypes(shp, SMB_TYPE_BASEBOARD);
	if (bb_cnt > 1) {
		/* allocate base board stypes */
		bb_stype = smb_create_strcnt(bb_cnt);

		/* fill in stypes */
		bb_stype->type = SMB_TYPE_BASEBOARD;
		smb_strcnt(shp, bb_stype);

		/* verify contained handles */
		for (i = 0; i < bb_cnt; i++) {
			bb_id = bb_stype->ids[i]->id;
			(void) smbios_info_bboard(shp, bb_id, &bb);
			if (bb.smbb_contn == 0) {
#ifdef	DEBUG
				cmn_err(CE_NOTE, "!No contained hanldes (%d)",
				    i);
#endif	/* DEBUG */
				smb_free_strcnt(bb_stype, bb_cnt);
				return (-1);
			}
		}

		/* free stypes */
		smb_free_strcnt(bb_stype, bb_cnt);
	}

	return (0);
}

void
fm_smb_fmacompat()
{
	int i, j;
	int id;
	int cnt;
	const char **oem_strings = NULL;
	smbs_cnt_t *oemstypes;
	smbios_hdl_t *shp;
	int strcnt;
	int compat = 0;

	/* check for BKS */
	if (x86gentopo_legacy == 1) {
		return;
	}

	shp = ksmbios;
	if (shp == NULL) {
		goto bad;
	}

	/* OEM strings (Type 11) */
	strcnt = smb_cnttypes(shp, SMB_TYPE_OEMSTR);
	if (strcnt == 0)
		goto bad;

	oemstypes = smb_create_strcnt(strcnt);
	if (oemstypes == NULL)
		goto bad;

	oemstypes->type = SMB_TYPE_OEMSTR;
	smb_strcnt(shp, oemstypes);

	for (i = 0; i < oemstypes->count && compat == 0; i++) {
		id = oemstypes->ids[i]->id;
		cnt = smbios_info_strtab(shp, id, 0, NULL);
		if (cnt > 0) {
			oem_strings = kmem_zalloc(sizeof (char *) * cnt,
			    KM_SLEEP);
			(void) smbios_info_strtab(shp, id, cnt, oem_strings);

			for (j = 0; j < cnt; j++) {
				if (strncmp(oem_strings[j], SMB_PRMS1,
				    strlen(SMB_PRMS1) + 1) == 0) {
					compat = 1;
					break;
				}
			}
			kmem_free(oem_strings, sizeof (char *) * cnt);
		}
	}
	smb_free_strcnt(oemstypes, strcnt);

	/* sanity check SMBIOS structures */
	if ((compat != 0) && (fm_smb_check(shp) == 0))
		return;

bad:
	/* not compatible with x86gentopo; revert to legacy enumeration */
#ifdef	DEBUG
	cmn_err(CE_NOTE,
	    "!SMBIOS is not compatible with x86 generic topology.");
	cmn_err(CE_NOTE, "!Invoking legacy x86 topology enumeration.");
#endif	/* DEBUG */
	x86gentopo_legacy = 1;
}

static int
find_matching_apic(smbios_hdl_t *shp, uint16_t proc_id, uint_t strand_apicid)
{
	uint16_t ext_id;
	int i, j;
	smbios_processor_ext_t ep;
	smbs_cnt_t *pstypes;
	int strcnt;

	strcnt = smb_cnttypes(shp, SUN_OEM_EXT_PROCESSOR);
	if (strcnt == 0)
		return (0);

	pstypes = smb_create_strcnt(strcnt);
	if (pstypes == NULL)
		return (0);

	pstypes->type = SUN_OEM_EXT_PROCESSOR;
	smb_strcnt(shp, pstypes);
	for (i = 0; i < pstypes->count; i++) {
		ext_id = pstypes->ids[i]->id;
		(void) smbios_info_extprocessor(shp, ext_id, &ep);
		if (ep.smbpe_processor == proc_id) {
			for (j = 0; j < ep.smbpe_n; j++) {
				if (ep.smbpe_apicid[j] == strand_apicid) {
					smb_free_strcnt(pstypes, strcnt);
					return (1);
				}
			}
		}
	}
	smb_free_strcnt(pstypes, strcnt);
	return (0);
}

/*
 * go throught the type 2 structure contained_ids looking for
 * the type 4 which  has strand_apicid == this strand_apicid
 */
static int
find_matching_proc(smbios_hdl_t *shp, uint_t strand_apicid,
    uint16_t bb_id, uint16_t proc_hdl, int is_proc)
{
	int n;
	const smb_struct_t *sp;
	smbios_bboard_t bb;
	uint_t cont_count, cont_len;
	uint16_t cont_id;
	id_t *cont_hdl = NULL;
	int rc;


	(void) smbios_info_bboard(shp, bb_id, &bb);
	cont_count = (uint_t)bb.smbb_contn;
	if (cont_count == 0)
		return (0);

	cont_len = sizeof (id_t);
	cont_hdl = kmem_zalloc(cont_count * cont_len, KM_SLEEP);
	if (cont_hdl == NULL)
		return (0);

	rc = smbios_info_contains(shp, bb_id, cont_count, cont_hdl);
	if (rc > SMB_CONT_MAX) {
		kmem_free(cont_hdl, cont_count * cont_len);
		return (0);
	}
	cont_count = MIN(rc, cont_count);

	for (n = 0; n < cont_count; n++) {
		cont_id = (uint16_t)cont_hdl[n];
		sp = smb_lookup_id(shp, cont_id);
		if (sp->smbst_hdr->smbh_type == SMB_TYPE_PROCESSOR) {
			if (is_proc) {
				if (find_matching_apic(shp, cont_id,
				    strand_apicid)) {
					kmem_free(cont_hdl,
					    cont_count * cont_len);
					return (1);
				}
			} else {
				if (cont_id == proc_hdl) {
					kmem_free(cont_hdl,
					    cont_count * cont_len);
					return (1);
				}
			}
		}
	}
	if (cont_hdl != NULL)
		kmem_free(cont_hdl, cont_count * cont_len);

	return (0);
}

void
get_bboard_index(smbs_cnt_t *bbstypes, uint_t bb_id, bbindex_t *bb_idx)
{
	int curr_id, tmp_id;
	int i, j, nb;
	bbindex_t tmp_idx;

	for (i = 0; i < MAX_PAIRS; i++)
		tmp_idx.index[i] = 0;

	tmp_idx.count = 0;

	curr_id = bb_id;
	for (nb = bbstypes->count-1, i = 0; nb >= 0; nb--) {
		tmp_id = bbstypes->ids[nb]->id;
		if (tmp_id == curr_id) {
			tmp_idx.index[i] = nb;
			tmp_idx.count++;
			curr_id = bbstypes->ids[nb]->cont_by_id;
			if (curr_id == -1)
				break;
			i++;
		}
	}

	for (i = tmp_idx.count - 1, j = 0; i >= 0; i--) {
		bb_idx->index[j] = tmp_idx.index[i];
		j++;
	}

	bb_idx->count = tmp_idx.count;
}

int
get_chassis_inst(smbios_hdl_t *shp, uint16_t *chassis_inst,
    uint16_t bb_id, int *chcnt)
{
	int ch_strcnt;
	smbs_cnt_t *chstypes;
	uint16_t chassis_id, tmp_id;
	smbios_bboard_t bb;
	int rc = 0;
	int i;

	rc = smbios_info_bboard(shp, bb_id, &bb);
	if (rc != 0) {
		return (-1);
	}

	chassis_id = bb.smbb_chassis;

	ch_strcnt = smb_cnttypes(shp, SMB_TYPE_CHASSIS);

	if (ch_strcnt == 0)
		return (-1);

	chstypes = smb_create_strcnt(ch_strcnt);
	if (chstypes == NULL)
		return (-1);

	chstypes->type = SMB_TYPE_CHASSIS;
	smb_strcnt(shp, chstypes);

	for (i = 0; i < chstypes->count; i++) {
		tmp_id = chstypes->ids[i]->id;
		if (tmp_id == chassis_id) {
			*chassis_inst = chstypes->ids[i]->inst;
			if (chstypes->ids[i]->inst != 0)
				*chcnt = 2;
			else
				*chcnt = 1;
			smb_free_strcnt(chstypes, ch_strcnt);
			return (0);
		}
	}

	smb_free_strcnt(chstypes, ch_strcnt);
	return (-1);
}

int
smb_get_bb_fmri(smbios_hdl_t *shp, nvlist_t *fmri,  uint_t parent,
    smbs_cnt_t *bbstypes)
{
	int rc = 0;
	int i, j, n, cnt;
	int id, index;
	nvlist_t *pairs[MAX_PAIRS];
	smbios_bboard_t bb;
	uint16_t chassis_inst, mch_inst;
	char name[40];
	char idstr[11];
	bbindex_t bb_idx;
	uint16_t bbid;
	int chcnt = 0;

	for (n = 0; n < MAX_PAIRS; n++) {
		bb_idx.index[n] = 0;
		pairs[n] = NULL;
	}
	bb_idx.count = 0;

	get_bboard_index(bbstypes, parent, &bb_idx);

	index = bb_idx.index[0];
	bbid = bbstypes->ids[index]->id;

	rc = get_chassis_inst(shp, &chassis_inst, bbid, &chcnt);

	if (rc != 0) {
		return (rc);
	}

	if ((bb_idx.count + chcnt) > MAX_PAIRS) {
		return (-1);
	}

	i = 0;
	if (chcnt > 1) {
		/*
		 * create main chassis pair
		 */
		pairs[i] = fm_nvlist_create(NULL);
		if (pairs[i] == NULL) {
			return (-1);
		}
		mch_inst = 0;
		(void) snprintf(idstr, sizeof (idstr), "%u", mch_inst);
		if ((nvlist_add_string(pairs[i], FM_FMRI_HC_NAME,
		    "chassis") != 0) ||
		    (nvlist_add_string(pairs[i], FM_FMRI_HC_ID, idstr)) != 0) {
			fm_nvlist_destroy(pairs[i], FM_NVA_FREE);
			return (-1);
		}
		i++;
	}

	/*
	 * create chassis pair
	 */
	pairs[i] = fm_nvlist_create(NULL);
	if (pairs[i] == NULL) {
		for (n = 0; n < MAX_PAIRS; n++) {
			if (pairs[n] != NULL)
				fm_nvlist_destroy(pairs[n], FM_NVA_FREE);
		}
		return (-1);
	}
	(void) snprintf(idstr, sizeof (idstr), "%u", chassis_inst);
	if ((nvlist_add_string(pairs[i], FM_FMRI_HC_NAME, "chassis") != 0) ||
	    (nvlist_add_string(pairs[i], FM_FMRI_HC_ID, idstr) != 0)) {
		for (n = 0; n < MAX_PAIRS; n++) {
			if (pairs[n] != NULL)
				fm_nvlist_destroy(pairs[n], FM_NVA_FREE);
		}
		return (-1);
	}

	for (j = 0, i = chcnt, cnt = chcnt; j < bb_idx.count; j++) {
		index = bb_idx.index[j];
		bbid = bbstypes->ids[index]->id;
		rc =  smbios_info_bboard(shp, bbid, &bb);
		if (rc != 0) {
			rc = -1;
			break;
		}

		pairs[i] = fm_nvlist_create(NULL);
		if (pairs[i] == NULL) {
			rc = -1;
			break;
		}

		id = bbstypes->ids[index]->inst;
		(void) snprintf(idstr, sizeof (idstr), "%u", id);
		(void) strncpy(name, bbd_type[bb.smbb_type].name,
		    sizeof (name));
		cnt++;

		if (nvlist_add_string(pairs[i], FM_FMRI_HC_NAME, name) != 0 ||
		    nvlist_add_string(pairs[i], FM_FMRI_HC_ID, idstr)
		    != 0) {
			rc = -1;
			break;
		}
		i++;
	}

	if (rc != -1) {
		if (nvlist_add_nvlist_array(fmri, FM_FMRI_HC_LIST,
		    pairs, cnt) != 0) {
			rc = -1;
		}
	}

	for (n = 0; n < cnt; n++) {
		if (pairs[n] != NULL)
			fm_nvlist_destroy(pairs[n], FM_NVA_FREE);
	}

	return (rc);
}

/*
 * pass in strand_apic id
 * return chip's bboards list which has strand_apicid == passed
 * in strand_apic id
 */
static nvlist_t *
smb_bboard(uint_t strand_apicid, uint16_t proc_hdl, int is_proc)
{
	smbios_hdl_t *shp;
	smbs_cnt_t *bbstypes;
	int nb;
	int bb_smbid;
	nvlist_t *fmri = NULL;
	int rc = 0;
	int bb_strcnt;

	if (x86gentopo_legacy)
		return (NULL);

	shp = ksmbios;
	if (shp == NULL) {
		goto bad;
	}

	/*
	 * Type 2 structs : "base board"
	 */
	bb_strcnt = smb_cnttypes(shp, SMB_TYPE_BASEBOARD);
	if (bb_strcnt == 0) {
		goto bad;
	}

	bbstypes = smb_create_strcnt(bb_strcnt);
	if (bbstypes == NULL)  {
		goto bad;
	}

	bbstypes->type = SMB_TYPE_BASEBOARD;
	smb_strcnt(shp, bbstypes);
	smb_bb_contains(shp, bbstypes);

	for (nb = 0; nb < bbstypes->count; nb++) {
		if (bbstypes->ids[nb]->visited) {
			continue;
		}

		bbstypes->ids[nb]->visited = 1;
		bb_smbid = bbstypes->ids[nb]->id;

		/*
		 * check if there is a matching  processor under
		 * this board. If found, find base board(s) of this proc
		 * If proc is not in contained handle of a base board and
		 * there is only one base board in the system, treat that base
		 * board as the parent of the proc
		 */
		if (find_matching_proc(shp, strand_apicid,
		    bb_smbid, proc_hdl, is_proc) || (bbstypes->count == 1)) {
			fmri = fm_nvlist_create(NULL);
			if (fmri == NULL) {
				smb_free_strcnt(bbstypes, bb_strcnt);
				goto bad;
			}
			/*
			 * find parent by walking the cont_by_id
			 */
			rc = smb_get_bb_fmri(shp, fmri, bb_smbid, bbstypes);
			smb_free_strcnt(bbstypes, bb_strcnt);
			if (rc == 0) {
				return (fmri);
			} else
				goto bad;
		}

	}

	smb_free_strcnt(bbstypes, bb_strcnt);
bad:
	/* revert to legacy enumeration */
	x86gentopo_legacy = 1;

	return (NULL);
}

nvlist_t *
fm_smb_bboard(uint_t strand_apicid)
{
	return (smb_bboard(strand_apicid, 0, PROC));
}

int
fm_smb_chipinst(uint_t strand_apicid, uint_t *chip_inst, uint16_t *smbiosid)
{
	int n;
	smbios_hdl_t *shp;
	uint16_t proc_id;
	smbs_cnt_t *pstypes;
	int strcnt;

	if (x86gentopo_legacy)
		return (-1);

	shp = ksmbios;
	if (shp == NULL) {
		goto bad;
	}

	strcnt = smb_cnttypes(shp, SMB_TYPE_PROCESSOR);
	if (strcnt == 0)
		goto bad;

	pstypes = smb_create_strcnt(strcnt);
	if (pstypes == NULL)
		goto bad;

	pstypes->type = SMB_TYPE_PROCESSOR;
	smb_strcnt(shp, pstypes);
	for (n = 0; n < pstypes->count; n++) {
		proc_id = pstypes->ids[n]->id;
		if (find_matching_apic(shp, proc_id, strand_apicid)) {
			*chip_inst = pstypes->ids[n]->inst;
			*smbiosid = pstypes->ids[n]->id;
			smb_free_strcnt(pstypes, strcnt);
			return (0);
		}
	}
	smb_free_strcnt(pstypes, strcnt);
bad:
	/* revert to legacy enumerarion */
	x86gentopo_legacy = 1;

	return (-1);
}

nvlist_t *
fm_smb_mc_bboards(uint_t bdf)
{

	int i;
	smbios_hdl_t *shp;
	uint16_t ext_id;
	smbios_memarray_ext_t em;
	nvlist_t *fmri = NULL;
	smbs_cnt_t *mastypes;
	int strcnt;

	if (x86gentopo_legacy)
		return (NULL);

	shp = ksmbios;
	if (shp == NULL) {
		goto bad;
	}

	strcnt = smb_cnttypes(shp, SUN_OEM_EXT_MEMARRAY);
	if (strcnt == 0)
		goto bad;

	mastypes = smb_create_strcnt(strcnt);
	if (mastypes == NULL)
		goto bad;

	mastypes->type = SUN_OEM_EXT_MEMARRAY;
	smb_strcnt(shp, mastypes);
	for (i = 0; i < mastypes->count; i++) {
		ext_id = mastypes->ids[i]->id;
		(void) smbios_info_extmemarray(shp, ext_id, &em);
		if (em.smbmae_bdf == bdf) {
			fmri = smb_bboard(0, em.smbmae_comp, MC);
			smb_free_strcnt(mastypes, strcnt);
			return (fmri);
		}
	}
	smb_free_strcnt(mastypes, strcnt);
bad:
	/* revert to legacy enumerarion */
	x86gentopo_legacy = 1;

	return (NULL);
}

int
fm_smb_mc_chipinst(uint_t bdf, uint_t *chip_inst)
{

	int i, j;
	smbios_hdl_t *shp;
	smbios_memarray_ext_t em;
	uint16_t ext_id, proc_id;
	smbs_cnt_t *mastypes;
	smbs_cnt_t *pstypes;
	int ma_strcnt, p_strcnt;

	if (x86gentopo_legacy)
		return (-1);

	shp = ksmbios;
	if (shp == NULL) {
		goto bad;
	}

	ma_strcnt = smb_cnttypes(shp, SUN_OEM_EXT_MEMARRAY);
	if (ma_strcnt == 0)
		goto bad;

	mastypes = smb_create_strcnt(ma_strcnt);
	if (mastypes == NULL)
		goto bad;

	mastypes->type = SUN_OEM_EXT_MEMARRAY;
	smb_strcnt(shp, mastypes);
	pstypes = NULL;
	p_strcnt = 0;
	for (i = 0; i < mastypes->count; i++) {
		ext_id = mastypes->ids[i]->id;
		(void) smbios_info_extmemarray(shp, ext_id, &em);
		if (em.smbmae_bdf == bdf) {
			p_strcnt = smb_cnttypes(shp, SMB_TYPE_PROCESSOR);
			if (p_strcnt == 0) {
				smb_free_strcnt(mastypes, ma_strcnt);
				goto bad;
			}

			pstypes = smb_create_strcnt(p_strcnt);
			if (pstypes == NULL) {
				smb_free_strcnt(mastypes, ma_strcnt);
				goto bad;
			}

			pstypes->type = SMB_TYPE_PROCESSOR;
			smb_strcnt(shp, pstypes);
			for (j = 0; j < pstypes->count; j++) {
				proc_id = pstypes->ids[j]->id;
				if (proc_id == em.smbmae_comp) {
					*chip_inst = pstypes->ids[j]->inst;
					smb_free_strcnt(mastypes, ma_strcnt);
					smb_free_strcnt(pstypes, p_strcnt);
					return (0);
				}
			}
		}
	}
	smb_free_strcnt(mastypes, ma_strcnt);
	smb_free_strcnt(pstypes, p_strcnt);
bad:
	/* revert to legacy enumeration */
	x86gentopo_legacy = 1;

	return (-1);
}
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved.
 * Copyright 2021 Joyent, Inc.
 * Copyright 2021 RackTop Systems, Inc.
 * Copyright 2023 Oxide Computer Company
 * Copyright 2025 Edgecast Cloud LLC.
 */

/*	Copyright (c) 1990, 1991 UNIX System Laboratories, Inc. */
/*	Copyright (c) 1984, 1986, 1987, 1988, 1989, 1990 AT&T   */
/*		All Rights Reserved				*/

/*	Copyright (c) 1987, 1988 Microsoft Corporation		*/
/*		All Rights Reserved				*/

/*
 * Copyright (c) 2009, Intel Corporation.
 * All rights reserved.
 */

#include <sys/types.h>
#include <sys/param.h>
#include <sys/signal.h>
#include <sys/regset.h>
#include <sys/privregs.h>
#include <sys/psw.h>
#include <sys/trap.h>
#include <sys/fault.h>
#include <sys/systm.h>
#include <sys/user.h>
#include <sys/file.h>
#include <sys/proc.h>
#include <sys/pcb.h>
#include <sys/lwp.h>
#include <sys/cpuvar.h>
#include <sys/thread.h>
#include <sys/disp.h>
#include <sys/fp.h>
#include <sys/siginfo.h>
#include <sys/archsystm.h>
#include <sys/kmem.h>
#include <sys/debug.h>
#include <sys/x86_archext.h>
#include <sys/sysmacros.h>
#include <sys/cmn_err.h>
#include <sys/kfpu.h>
#include <sys/stdbool.h>
#include <sys/stdalign.h>
#include <sys/procfs_isa.h>
#include <sys/sunddi.h>

/*
 * FPU Management Overview
 * -----------------------
 *
 * The x86 FPU has evolved substantially since its days as the x87 coprocessor;
 * however, many aspects of its life as a coprocessor are still around in x86.
 *
 * Today, when we refer to the 'FPU', we don't just mean the original x87 FPU.
 * While that state still exists, there is much more that is covered by the FPU.
 * Today, this includes not just traditional FPU state, but also supervisor only
 * state. The following state is currently managed and covered logically by the
 * idea of the FPU registers and more generally is called the Extended Processor
 * States:
 *
 *    o Traditional x87 FPU
 *    o Vector Registers (%xmm, %ymm, %zmm)
 *    o Memory Protection Extensions (MPX) Bounds Registers
 *    o Protected Key Rights Registers (PKRU)
 *    o Processor Trace data
 *    o Control-Flow Enforcement state
 *    o Hardware Duty Cycle
 *    o Hardware P-states
 *
 * The rest of this covers how the FPU is managed and controlled, how state is
 * saved and restored between threads, interactions with hypervisors, and other
 * information exported to userland through aux vectors. A lot of background
 * information is here to synthesize major parts of the Intel SDM, but
 * unfortunately, it is not a replacement for reading it.
 *
 * FPU Control Registers
 * ---------------------
 *
 * Because the x87 FPU began its life as a co-processor and the FPU was
 * optional there are several bits that show up in %cr0 that we have to
 * manipulate when dealing with the FPU. These are:
 *
 *   o CR0.ET	The 'extension type' bit. This was used originally to indicate
 *		that the FPU co-processor was present. Now it is forced on for
 *		compatibility. This is often used to verify whether or not the
 *		FPU is present.
 *
 *   o CR0.NE	The 'native error' bit. Used to indicate that native error
 *		mode should be enabled. This indicates that we should take traps
 *		on FPU errors. The OS enables this early in boot.
 *
 *   o CR0.MP	The 'Monitor Coprocessor' bit. Used to control whether or not
 *		wait/fwait instructions generate a #NM if CR0.TS is set.
 *
 *   o CR0.EM	The 'Emulation' bit. This is used to cause floating point
 *		operations (x87 through SSE4) to trap with a #UD so they can be
 *		emulated. The system never sets this bit, but makes sure it is
 *		clear on processor start up.
 *
 *   o CR0.TS	The 'Task Switched' bit. When this is turned on, a floating
 *		point operation will generate a #NM. An fwait will as well,
 *		depending on the value in CR0.MP.
 *
 * Our general policy is that CR0.ET, CR0.NE, and CR0.MP are always set by
 * the system. Similarly CR0.EM is always unset by the system. CR0.TS has a more
 * complicated role. Historically it has been used to allow running systems to
 * restore the FPU registers lazily. This will be discussed in greater depth
 * later on.
 *
 * %cr4 is also used as part of the FPU control. Specifically we need to worry
 * about the following bits in the system:
 *
 *   o CR4.OSFXSR	This bit is used to indicate that the OS understands and
 *			supports the execution of the fxsave and fxrstor
 *			instructions. This bit is required to be set to enable
 *			the use of the SSE->SSE4 instructions.
 *
 *   o CR4.OSXMMEXCPT	This bit is used to indicate that the OS can understand
 *			and take a SIMD floating point exception (#XM). This bit
 *			is always enabled by the system.
 *
 *   o CR4.OSXSAVE	This bit is used to indicate that the OS understands and
 *			supports the execution of the xsave and xrstor family of
 *			instructions. This bit is required to use any of the AVX
 *			and newer feature sets.
 *
 * Because all supported processors are 64-bit, they'll always support the XMM
 * extensions and we will enable both CR4.OXFXSR and CR4.OSXMMEXCPT in boot.
 * CR4.OSXSAVE will be enabled and used whenever xsave is reported in cpuid.
 *
 * %xcr0 is used to manage the behavior of the xsave feature set and is only
 * present on the system if xsave is supported. %xcr0 is read and written to
 * through by the xgetbv and xsetbv instructions. This register is present
 * whenever the xsave feature set is supported. Each bit in %xcr0 refers to a
 * different component of the xsave state and controls whether or not that
 * information is saved and restored. For newer feature sets like AVX and MPX,
 * it also controls whether or not the corresponding instructions can be
 * executed (much like CR0.OSFXSR does for the SSE feature sets).
 *
 * Everything in %xcr0 is around features available to users. There is also the
 * IA32_XSS MSR which is used to control supervisor-only features that are still
 * part of the xsave state. Bits that can be set in %xcr0 are reserved in
 * IA32_XSS and vice versa. This is an important property that is particularly
 * relevant to how the xsave instructions operate.
 *
 * Save Mechanisms
 * ---------------
 *
 * When switching between running threads the FPU state needs to be saved and
 * restored by the OS. If this state was not saved, users would rightfully
 * complain about corrupt state. There are three mechanisms that exist on the
 * processor for saving and restoring these state images:
 *
 *   o fsave
 *   o fxsave
 *   o xsave
 *
 * fsave saves and restores only the x87 FPU and is the oldest of these
 * mechanisms. This mechanism is never used in the kernel today because we are
 * always running on systems that support fxsave.
 *
 * The fxsave and fxrstor mechanism allows the x87 FPU and the SSE register
 * state to be saved and restored to and from a struct fxsave_state. This is the
 * default mechanism that is used to save and restore the FPU on amd64. An
 * important aspect of fxsave that was different from the original i386 fsave
 * mechanism is that the restoring of FPU state with pending exceptions will not
 * generate an exception, it will be deferred to the next use of the FPU.
 *
 * The final and by far the most complex mechanism is that of the xsave set.
 * xsave allows for saving and restoring all of the traditional x86 pieces (x87
 * and SSE), while allowing for extensions that will save the %ymm, %zmm, etc.
 * registers.
 *
 * Data is saved and restored into and out of a struct xsave_state. The first
 * part of the struct xsave_state is equivalent to the struct fxsave_state.
 * After that, there is a header which is used to describe the remaining
 * portions of the state. The header is a 64-byte value of which the first two
 * uint64_t values are defined and the rest are reserved and must be zero. The
 * first uint64_t is the xstate_bv member. This describes which values in the
 * xsave_state are actually valid and present. This is updated on a save and
 * used on restore. The second member is the xcomp_bv member. Its last bit
 * determines whether or not a compressed version of the structure is used.
 *
 * When the uncompressed structure is used (currently the only format we
 * support), then each state component is at a fixed offset in the structure,
 * even if it is not being used. For example, if you only saved the AVX related
 * state, but did not save the MPX related state, the offset would not change
 * for any component. With the compressed format, components that aren't used
 * are all elided (though the x87 and SSE state are always there).
 *
 * Unlike fxsave which saves all state, the xsave family does not always save
 * and restore all the state that could be covered by the xsave_state. The
 * instructions all take an argument which is a mask of what to consider. This
 * is the same mask that will be used in the xstate_bv vector and it is also the
 * same values that are present in %xcr0 and IA32_XSS. Though IA32_XSS is only
 * considered with the xsaves and xrstors instructions.
 *
 * When a save or restore is requested, a bitwise and is performed between the
 * requested bits and those that have been enabled in %xcr0. Only the bits that
 * match that are then saved or restored. Others will be silently ignored by
 * the processor. This idea is used often in the OS. We will always request that
 * we save and restore all of the state, but only those portions that are
 * actually enabled in %xcr0 will be touched.
 *
 * If a feature has been asked to be restored that is not set in the xstate_bv
 * feature vector of the save state, then it will be set to its initial state by
 * the processor (usually zeros). Also, when asked to save state, the processor
 * may not write out data that is in its initial state as an optimization. This
 * optimization only applies to saving data and not to restoring data.
 *
 * There are a few different variants of the xsave and xrstor instruction. They
 * are:
 *
 *   o xsave	This is the original save instruction. It will save all of the
 *		requested data in the xsave state structure. It only saves data
 *		in the uncompressed (xcomp_bv[63] is zero) format. It may be
 *		executed at all privilege levels.
 *
 *   o xrstor	This is the original restore instruction. It will restore all of
 *		the requested data. The xrstor function can handle both the
 *		compressed and uncompressed formats. It may be executed at all
 *		privilege levels.
 *
 *   o xsaveopt	This is a variant of the xsave instruction that employs
 *		optimizations to try and only write out state that has been
 *		modified since the last time an xrstor instruction was called.
 *		The processor tracks a tuple of information about the last
 *		xrstor and tries to ensure that the same buffer is being used
 *		when this optimization is being used. However, because of the
 *		way that it tracks the xrstor buffer based on the address of it,
 *		it is not suitable for use if that buffer can be easily reused.
 *		The most common case is trying to save data to the stack in
 *		rtld. It may be executed at all privilege levels.
 *
 *   o xsavec	This is a variant of the xsave instruction that writes out the
 *		compressed form of the xsave_state. Otherwise it behaves as
 *		xsave. It may be executed at all privilege levels.
 *
 *   o xsaves	This is a variant of the xsave instruction. It is similar to
 *		xsavec in that it always writes the compressed form of the
 *		buffer. Unlike all the other forms, this instruction looks at
 *		both the user (%xcr0) and supervisor (IA32_XSS MSR) to determine
 *		what to save and restore. xsaves also implements the same
 *		optimization that xsaveopt does around modified pieces. User
 *		land may not execute the instruction.
 *
 *   o xrstors	This is a variant of the xrstor instruction. Similar to xsaves
 *		it can save and restore both the user and privileged states.
 *		Unlike xrstor it can only operate on the compressed form.
 *		User land may not execute the instruction.
 *
 * Based on all of these, the kernel has a precedence for what it will use.
 * Basically, xsaves (not supported) is preferred to xsaveopt, which is
 * preferred to xsave. A similar scheme is used when informing rtld (more later)
 * about what it should use. xsavec is preferred to xsave. xsaveopt is not
 * recommended due to the modified optimization not being appropriate for this
 * use.
 *
 * Finally, there is one last gotcha with the xsave state. Importantly some AMD
 * processors did not always save and restore some of the FPU exception state in
 * some cases like Intel did. In those cases the OS will make up for this fact
 * itself.
 *
 * FPU Initialization
 * ------------------
 *
 * One difference with the FPU registers is that not all threads have FPU state,
 * only those that have an lwp. Generally this means kernel threads, which all
 * share p0 and its lwp, do not have FPU state. Though there are definitely
 * exceptions such as kcfpoold. In the rest of this discussion we'll use thread
 * and lwp interchangeably, just think of thread meaning a thread that has a
 * lwp.
 *
 * Each lwp has its FPU state allocated in its pcb (process control block). The
 * actual storage comes from the fpsave_cachep kmem cache. This cache is sized
 * dynamically at start up based on the save mechanism that we're using and the
 * amount of memory required for it. This is dynamic because the xsave_state
 * size varies based on the supported feature set.
 *
 * The hardware side of the FPU is initialized early in boot before we mount the
 * root file system. This is effectively done in fpu_probe(). This is where we
 * make the final decision about what the save and restore mechanisms we should
 * use are, create the fpsave_cachep kmem cache, and initialize a number of
 * function pointers that use save and restoring logic.
 *
 * The thread/lwp side is a a little more involved. There are two different
 * things that we need to concern ourselves with. The first is how the FPU
 * resources are allocated and the second is how the FPU state is initialized
 * for a given lwp.
 *
 * We allocate the FPU save state from our kmem cache as part of lwp_fp_init().
 * This is always called unconditionally by the system as part of creating an
 * LWP.
 *
 * There are three different initialization paths that we deal with. The first
 * is when we are executing a new process. As part of exec all of the register
 * state is reset. The exec case is particularly important because init is born
 * like Athena, sprouting from the head of the kernel, without any true parent
 * to fork from. The second is used whenever we fork or create a new lwp.  The
 * third is to deal with special lwps like the agent lwp.
 *
 * During exec, we will call fp_exec() which will initialize and set up the FPU
 * state for the process. That will fill in the initial state for the FPU and
 * also set that state in the FPU itself. As part of fp_exec() we also install a
 * thread context operations vector that takes care of dealing with the saving
 * and restoring of the FPU. These context handlers will also be called whenever
 * an lwp is created or forked. In those cases, to initialize the FPU we will
 * call fp_new_lwp(). Like fp_exec(), fp_new_lwp() will install a context
 * operations vector for the new thread.
 *
 * Next we'll end up in the context operation fp_new_lwp(). This saves the
 * current thread's state, initializes the new thread's state, and copies over
 * the relevant parts of the originating thread's state. It's as this point that
 * we also install the FPU context operations into the new thread, which ensures
 * that all future threads that are descendants of the current one get the
 * thread context operations (unless they call exec).
 *
 * To deal with some things like the agent lwp, we double check the state of the
 * FPU in sys_rtt_common() to make sure that it has been enabled before
 * returning to userland. In general, this path should be rare, but it's useful
 * for the odd lwp here and there.
 *
 * The FPU state will remain valid most of the time. There are times that
 * the state will be rewritten. For example in restorecontext, due to /proc, or
 * the lwp calls exec(). Whether the context is being freed or we are resetting
 * the state, we will call fp_free() to disable the FPU and our context.
 *
 * Finally, when the lwp is destroyed, it will actually destroy and free the FPU
 * state by calling fp_lwp_cleanup().
 *
 * Kernel FPU Multiplexing
 * -----------------------
 *
 * Just as the kernel has to maintain all of the general purpose registers when
 * switching between scheduled threads, the same is true of the FPU registers.
 *
 * When a thread has FPU state, it also has a set of context operations
 * installed. These context operations take care of making sure that the FPU is
 * properly saved and restored during a context switch (fpsave_ctxt and
 * fprestore_ctxt respectively). This means that the current implementation of
 * the FPU is 'eager', when a thread is running the CPU will have its FPU state
 * loaded. While this is always true when executing in userland, there are a few
 * cases where this is not true in the kernel.
 *
 * This was not always the case. Traditionally on x86 a 'lazy' FPU restore was
 * employed. This meant that the FPU would be saved on a context switch and the
 * CR0.TS bit would be set. When a thread next tried to use the FPU, it would
 * then take a #NM trap, at which point we would restore the FPU from the save
 * area and return to userland. Given the frequency of use of the FPU alone by
 * libc, there's no point returning to userland just to trap again.
 *
 * There are a few cases though where the FPU state may need to be changed for a
 * thread on its behalf. The most notable cases are in the case of processes
 * using /proc, restorecontext, forking, etc. In all of these cases the kernel
 * will force a threads FPU state to be saved into the PCB through the fp_save()
 * function. Whenever the FPU is saved, then the FPU_VALID flag is set on the
 * pcb. This indicates that the save state holds currently valid data. As a side
 * effect of this, CR0.TS will be set. To make sure that all of the state is
 * updated before returning to userland, in these cases, we set a flag on the
 * PCB that says the FPU needs to be updated. This will make sure that we take
 * the slow path out of a system call to fix things up for the thread. Due to
 * the fact that this is a rather rare case, effectively setting the equivalent
 * of t_postsys is acceptable.
 *
 * CR0.TS will be set after a save occurs and cleared when a restore occurs.
 * Generally this means it will be cleared immediately by the new thread that is
 * running in a context switch. However, this isn't the case for kernel threads.
 * They currently operate with CR0.TS set as no kernel state is restored for
 * them. This means that using the FPU will cause a #NM and panic.
 *
 * The FPU_VALID flag on the currently executing thread's pcb is meant to track
 * what the value of CR0.TS should be. If it is set, then CR0.TS will be set.
 * However, because we eagerly restore, the only time that CR0.TS should be set
 * for a non-kernel thread is during operations where it will be cleared before
 * returning to userland and importantly, the only data that is in it is its
 * own.
 *
 * Kernel FPU Usage
 * ----------------
 *
 * Traditionally the kernel never used the FPU since it had no need for
 * floating point operations. However, modern FPU hardware supports a variety
 * of SIMD extensions which can speed up code such as parity calculations or
 * encryption.
 *
 * To allow the kernel to take advantage of these features, the
 * kernel_fpu_begin() and kernel_fpu_end() functions should be wrapped
 * around any usage of the FPU by the kernel to ensure that user-level context
 * is properly saved/restored, as well as to properly setup the FPU for use by
 * the kernel. There are a variety of ways this wrapping can be used, as
 * discussed in this section below.
 *
 * When kernel_fpu_begin() and kernel_fpu_end() are used for extended
 * operations, the kernel_fpu_alloc() function should be used to allocate a
 * kfpu_state_t structure that is used to save/restore the thread's kernel FPU
 * state. This structure is not tied to any thread. That is, different threads
 * can reuse the same kfpu_state_t structure, although not concurrently. A
 * kfpu_state_t structure is freed by the kernel_fpu_free() function.
 *
 * In some cases, the kernel may need to use the FPU for a short operation
 * without the overhead to manage a kfpu_state_t structure and without
 * allowing for a context switch off the FPU. In this case the KFPU_NO_STATE
 * bit can be set in the kernel_fpu_begin() and kernel_fpu_end() flags
 * parameter. This indicates that there is no kfpu_state_t. When used this way,
 * kernel preemption should be disabled by the caller (kpreempt_disable) before
 * calling kernel_fpu_begin(), and re-enabled after calling kernel_fpu_end().
 * For this usage, it is important to limit the kernel's FPU use to short
 * operations. The tradeoff between using the FPU without a kfpu_state_t
 * structure vs. the overhead of allowing a context switch while using the FPU
 * should be carefully considered on a case by case basis.
 *
 * In other cases, kernel threads have an LWP, but never execute in user space.
 * In this situation, the LWP's pcb_fpu area can be used to save/restore the
 * kernel's FPU state if the thread is context switched, instead of having to
 * allocate and manage a kfpu_state_t structure. The KFPU_USE_LWP bit in the
 * kernel_fpu_begin() and kernel_fpu_end() flags parameter is used to
 * enable this behavior. It is the caller's responsibility to ensure that this
 * is only used for a kernel thread which never executes in user space.
 *
 * FPU Exceptions
 * --------------
 *
 * Certain operations can cause the kernel to take traps due to FPU activity.
 * Generally these events will cause a user process to receive a SIGFPU and if
 * the kernel receives it in kernel context, we will die. Traditionally the #NM
 * (Device Not Available / No Math) exception generated by CR0.TS would have
 * caused us to restore the FPU. Now it is a fatal event regardless of whether
 * or not userland causes it.
 *
 * While there are some cases where the kernel uses the FPU, it is up to the
 * kernel to use the FPU in a way such that it cannot receive a trap or to use
 * the appropriate trap protection mechanisms.
 *
 * Hypervisors
 * -----------
 *
 * When providing support for hypervisors things are a little bit more
 * complicated because the FPU is not virtualized at all. This means that they
 * need to save and restore the FPU and %xcr0 across entry and exit to the
 * guest. To facilitate this, we provide a series of APIs in <sys/hma.h>. These
 * allow us to use the full native state to make sure that we are always saving
 * and restoring the full FPU that the host sees, even when the guest is using a
 * subset.
 *
 * One tricky aspect of this is that the guest may be using a subset of %xcr0
 * and therefore changing our %xcr0 on the fly. It is vital that when we're
 * saving and restoring the FPU that we always use the largest %xcr0 contents
 * otherwise we will end up leaving behind data in it.
 *
 * ELF PLT Support
 * ---------------
 *
 * rtld has to preserve a subset of the FPU when it is saving and restoring
 * registers due to the amd64 SYS V ABI. See cmd/sgs/rtld/amd64/boot_elf.s for
 * more information. As a result, we set up an aux vector that contains
 * information about what save and restore mechanisms it should be using and
 * the sizing thereof based on what the kernel supports. This is passed down in
 * a series of aux vectors SUN_AT_FPTYPE and SUN_AT_FPSIZE. This information is
 * initialized in fpu_subr.c.
 *
 * Signal Handling and the ucontext_t
 * ----------------------------------
 *
 * One of the many gifts that signals give us is the twofold fact that when a
 * signal occurs, the signal handler is allowed to change the CPU's state
 * arbitrarily and when the signal handler is done executing, we must restore it
 * back to the original state. However, the second part of this is that the
 * signal handler is actually allowed to modify the state that the thread will
 * return to! To create this facade, the kernel will create a full ucontext_t
 * state, effectively calling getcontext(2) on the thread's behalf, and a
 * pointer to that is given to the signal handler (the void * argument for the
 * sa_sigaction function pointer in sigaction(2)). When libc is done with a
 * signal, it will call setcontext(2) with that same ucontext_t.
 *
 * Now, the ucontext_t has a fixed ABI for both ILP32 and LP64 environments and
 * it's often declared on the stack itself, with the signal handler spilling all
 * this state to the stack. The ucontext_t machine portion was broken into the
 * general purpose and floating point registers. In 64-bit code, the floating
 * point registers were mostly the same as the results of the fxsave instruction
 * (i.e. struct fxsave_state). While the 64-bit kernel still uses the equivalent
 * starting point for information, it is transformed into a different shape to
 * deal with the history of the 32-bit SYS V ABI.
 *
 * While this worked, if you're reading this, you're aware that the x86 FPU and
 * extended register states didn't stop at the initial 16 128-bit %xmm
 * registers. Since then we have added 256-bit %ymm, 512-bit %zmm, and the %k
 * opmask registers. None of these fit inside the standard ucontext_t; however,
 * they must all be preserved and restored across a signal. While the various
 * x86 platform-specific ABIs all suggest that these registers are not preserved
 * across a function call, receiving a signal is not a function call and must be
 * thought of like a process receiving an interrupt. In other words, this
 * extended state must be preserved.
 *
 * To facilitate this, we have extended the ucontext_t structure with an
 * additional flag, UC_XSAVE, which indicates that the traditional padding
 * member, uc_xsave, actually is a pointer to the extended state. While this is
 * accessible outside of a signal handling context through the combination of
 * ucontext_alloc(3C) and getcontext_extd(2), our design around saving this
 * state is focused on signal handling. Signal handling spills all this state to
 * the stack and if we cannot spill the entire state to the stack then our
 * inability to deliver the signal results in the process being killed! While
 * there are separate efforts to ensure that the signal stack sizing that is
 * used for the minimum and maximum signal sizes are sufficient, we still need
 * to do our part to minimize the likelihood here.
 *
 * In designing this, we make the following observations which have helped us
 * focus our design:
 *
 *   o While the start of an xsave area is the traditional 512-byte fxsave XMM
 *     region, we already have that in the fpregs. Thus there is no reason to
 *     duplicate it. This not only saves 512 bytes of additional stack space,
 *     but it also means we don't have to ask which of the version of it to take
 *     if they were to differ.
 *
 *   o Many applications out there aren't necessarily using the extended vectors
 *     and even when we do make libc and others take advantage of it, it will
 *     behoove us to ensure that they are put back into their initial state
 *     after use. This leads us to expect that in a number of cases, the actual
 *     extended register state will be in its initial state.
 *
 *   o While the signal handler does allow contents to be modified, we are
 *     starting with making the interface private and thus allowing us to excise
 *     components that are in their initial state.
 *
 *   o There are similarities to what we want to create with the compressed
 *     xsave format; however, because we don't always have support for the
 *     compressed format, we can't just arbitrarily say let's do a compressed
 *     save to the user stack.
 *
 *   o Because we are not handing this state directly to and from hardware, we
 *     don't need to meet some of the constraints of the compressed xsave format
 *     around wanting alignment for the initial save or additional components.
 *
 * All of the above lead us to our own unique format for this data. When the
 * UC_XSAVE flag is set in the ucontext_t, the uc_xsave member points to a
 * uc_xsave_t structure which has a magic version number, a 32-bit length of the
 * overall structure, and the 64-bit state bit-vector to represent which
 * components are valid. Following this 8-byte header, each component that is
 * present in the bit vector is immediately written out in roughly ascending bit
 * order (the order is determined based on the order of the fpu_xsave_info
 * array).
 *
 * This makes the rough logic that we have here when taking a signal and writing
 * out this state as:
 *
 *   1. Ensure that the FPU is saved and that the contents of the pcb save area
 *      are valid. That is, call fp_save() if the state is not already flagged
 *      with FPU_VALID.
 *
 *   2. Copy the bit-vector from the save area and remove the XFEATURE_LEGACY_FP
 *      and XFEATURE_SSE bits as these will be placed in the xsave area.
 *
 *   3. Initialize the uc_xsave_t by setting our version field, initializing the
 *      length to the length of the current structure, and then setting the
 *      modified bit vector above.
 *
 *   4. Walk each remaining bit of the bit-vector. For each set bit, copy out
 *      its extended state starting at the current length in the header and then
 *      increase the header size by that length.
 *
 *   5. Finally write out the final uc_xsave_t structure.
 *
 * The above process is also used when someone manually calls getcontext_extd(2)
 * to get this state. The main difference between the two is which copyout
 * function we use. This deserves some explanation. Our main starting point for
 * all the logic here is fpu_signal_copyout(). It takes a copyfunc that allows
 * the signal handling context to operate with a different copyout than we
 * normally use in say getcontext_extd(2).
 *
 * When we've received a signal, we're at the intersection of several different
 * gotchas. Normal copyout (or ddi_copyout()) will trigger watchpoints. That is,
 * the watchpoints effectively set a copyout override function (t_copyops) that
 * we end up vectoring to rather than a normal copyout. This allows the data to
 * be modified and for the watchpoint to fire. While this is all well and good
 * normally, it is problematic if we are trying to handle a signal. The signal
 * deliver logic, sendsig(), goes through and disables the watchpoint for the
 * region of the stack that we are copying out to. However, disabling
 * watchpoints is not sufficient, we also need to use the copyout_noerr
 * variants.
 *
 * These variants also require the use of on_fault() and no_fault() for error
 * handling. While it is tempting to try and on_fault() the entire
 * fpu_signal_copyout() operation, that is actually fraught for a few reasons.
 * The first is that we don't want to disable faults during the entire operation
 * as if the kernel messes up we will treat that as a user error. That isn't
 * theoretical and happened during development. The second and perhaps more
 * important issue is that correctly bounding the on_fault() / no_fault() means
 * being careful about state. For example, kernel pre-emption is often disabled
 * during parts of these operations, but it needs to be re-enabled when we're
 * done. This would require tracking in some volatile variable that this had
 * been enabled and disabled and tracking that.
 *
 * Instead, this is why fpu_signal_copyout() takes a copy out function as an
 * argument. When we're in signal handling context, the function will use
 * coypout_noerr() and wrap it in the appropriate on_fault() mechanisms.
 *
 * RESTORING STATE
 *
 * Copying out our current state is the easier half of this problem. When the
 * kernel is done with a signal it calls setcontext(2) with the ucontext_t we
 * assembled for it as described above. setcontext(2) isn't just used for
 * returning from signals.
 *
 * The process for this goes in two steps. The first step is to copy in,
 * validate, and transform the ucontext_t UC_XSAVE that we created above into an
 * equivalent xsave format that we can use the appropriate xrstor function on.
 * This first phase is implemented in fpu_signal_copyin(). Once that is done, we
 * come back through a second phase that is driven out of restorecontext() and
 * is implemented in fpu_set_xsave().
 *
 * Let's start by discussing the second part of this, which is more
 * straightforward. In particular, the second phase assumes that all of the
 * validation and error handling has been done by the first phase. This means
 * here, we have a buffer that is already the appropriate size
 * (cpuid_get_xsave_size()) and all we need to do is make sure that we can
 * replace the actual save state with the current one.
 *
 * The only piece of shenanigans we have to do is around the kernel provided
 * notion of 'status' and 'xstatus', which are cached versions of the x87 and
 * SSE exception vectors. These are part of the fpregset ABI and therefore we
 * need to propagate them from the temporary storage that part 1 sets up in the
 * ignored region of the fxsave data. We use that because it is not persisted by
 * the CPU, so clobbering it is generally alright.
 *
 * Once that is done, we simply note that we need a PCB update to occur to
 * refresh the FPU state before we return to userland. Given that someone has
 * called setcontext(2), this was always going to happen because we have to
 * update segment registers and related, so this isn't so bad. With that, let's
 * move onto the more nuanced part (1).
 *
 * When we're handling a setcontext(2) we have, in userland, a data structure
 * that should match one we serialized out, though we cannot assume that a user
 * has not modified it either accidentally or maliciously. Our goal is to set up
 * the appropriate xsave state that can be passed to the CPU's xrstor. The first
 * problem we have to deal with is where do we actually put this state?
 *
 * While not many programs actually call setcontext(2) of their own volition,
 * this is going to get hit every time we take a signal. The first thought was
 * to re-use the existing thread's save area; however, that's a bit challenging
 * for a few reasons. In particular, we would need to ensure that we don't go
 * off-CPU for any reason, which we cannot assume with a copyin from a user
 * address space. In particular, it is trivial for us to hit a case where the
 * stack has been paged out for some reason, which eschews that path.
 *
 * Instead, whenever a thread first calls setcontext(2), generally from signal
 * context, we will at that time allocate another entry from the 'fpsave_cachep'
 * kmem cache, giving us a buffer of the appropriate space to handle this. Once
 * this buffer has been allocated, we leave it assigned to the thread's pcb and
 * only tear it down when the thread itself finally exits. We reason that a
 * thread that takes a signal once is either going to have the process exit
 * shortly thereafter or is much more likely to take a signal again in the
 * future. Many daemons and other processes set things up so signals are
 * dispatched via one location, masking signals in other thread, using
 * sigsuspend(2), signalfd(3C), or something similar.
 *
 * With this buffer in hand, we begin our task of reassembling state. Note, all
 * of this is conditional on UC_XSAVE being set in the uc_flags member of the
 * ucontext_t. If it is not set, then we assume that there is no extended state
 * and will use the traditional path of setting the fpregset_t into the system
 * via setfpregs().
 *
 * We first will copyin and validate the uc_xsave_t. In particular, we need to
 * make sure the version makes sense, that the xsave component bit-vector
 * doesn't have anything unexpected and more importantly unsupported in it, and
 * that the addresses we've been given are within the user address space. At
 * this point we can walk through our table of implemented bits and process
 * them.
 *
 * For most components in here, the processing is straightforward. We continue
 * walking our cursor and copy data into the kernel and place it in the
 * appropriate place in our xsave state. If a xsave state component bit-vector
 * isn't set, then we must ensure that we have the item in the initial state,
 * which for everything other than the x87/SSE state is the memory being zeroed.
 *
 * The most unique case in the copyin state is that of the x87/SSE state. You
 * might recall that we didn't copy it out explicitly as part of the uc_xsave_t,
 * but instead have opted to use the single definition in the fpregset_t. Thus
 * here, we copy it out of the fpregset_t, which the kernel has helpfully
 * already unified into the 64-bit fxsave version prior to calling us, and
 * install that into the save area we're building up.
 *
 * As part of this, there are two important pieces to be aware of. The first is
 * that because the fpregset_t has both the status and xstatus members
 * mentioned earlier, we temporarily copy them to the software-usable ignored
 * areas of the fxsave state so we can corral this extra state into part (2)
 * without needing to allocate additional space. The second piece is that when
 * we're done processing this we explicitly remove the UC_FPU flag that would
 * tell the kernel to proceed with updating that region. The problem is that
 * that goes directly into the pcb's save area and not to the intermediate
 * buffer as it uses the same entry point as /proc, mainly setfpregs().
 *
 * We don't do much validation of the actual contents of the registers that are
 * being set with the exception of ensuring that no reserved bits of the mxcsr
 * are used. This is not as strict as /proc, but failure here means the process
 * is likely going to die (returning from setcontext() in a signal handler is
 * fatal).
 *
 * /proc xregs
 * -----------
 *
 * Observability of the state of the extended registers is important for
 * understanding the system. While on the surface this is similar to signal
 * handling, it is crucially different in a number of ways:
 *
 *   o In signal handling, we're trying to conserve every byte of stack that we
 *     can.
 *   o The /proc xregs file will end up in core files, which means that we need
 *     a way of knowing what components are present and not present in it,
 *     because this will vary from CPU to CPU due to the addition of
 *     architectural features. For example, some CPUs support AVX-512, but
 *     others do not.
 *
 *   o The signal handling structure (uc_xsave_t) is private and we're not
 *     trying to have software modify it, on the other hand, the /proc
 *     interfaces that we support we do want software to be able to interrogate
 *     and manipulate. These need to be something that we can introduce
 *     additional components into and make other changes that still allow it to
 *     work.
 *
 * The x86 xregs format is documented in proc(5). The short form is that the
 * prxregset_hdr_t has a number of information entries, which are of the type
 * prxregset_info_t. Each of the information headers has a type, size, and
 * offset which indicate where to find the additional data.
 *
 * Each entry is described as one of the entries in the fpu_xsave_info[]. These
 * items either are a 1:1 correspondence with a xsave related feature (e.g.
 * there is one entry for each of the three AVX-512 components) or it is
 * something synthetic that we provide as additional information such as the
 * PRX_INFO_XCR, which is a way of getting information about the system such as
 * what is enabled in %xcr0 out there.
 *
 * Unlike signal handling, we are given the buffer to place everything that
 * needs to be written out. This is partially the design of the /proc APIs. That
 * is, we will always assemble everything into the entire buffer that /proc asks
 * us to, and then it will use as much or as little of it as is required.
 * Similarly, when setting things, we don't have to worry about copying in
 * information in the same way as signal handling does, because /proc takes care
 * of it and always hands us a full buffer. Sizing that is a little nuanced, but
 * is all handled in prmachdep.c.
 *
 * When someone performs a read of the xregs and thus is asking us for the
 * current state, there is a little bit of nuance that we need to deal with.
 * The first, is whether or not the FPU is enabled and the second is if the FPU
 * is enabled, whether a given component is noted as being in its initial state.
 * This basically gives us three possible states for a given component:
 *
 *   1. FPU_EN is not set and FPU_VALID is not set. This means we need to take
 *      the illumos FPU default for an item. More on that in a moment.
 *   2. The saved xsave state indicates that the bit for a given component is
 *      zero -- specifically the xsh_xstate_bv member of the struct xsave_state.
 *      In this case, we must take the CPU's default for an item. This is
 *      usually the same as illumos, but not always.
 *   3. The saved xsave state indicates that a given component's state bit is
 *      valid. The simplest of our cases. We can just take what we have from the
 *      xsave state.
 *
 * The CPU's default state for most components other than the x87/SSE state is
 * to have it be zeroed. This is what we treat as our default state as well. The
 * primary difference is in the initialization of the x87/SSE state. The SYS V
 * ABI requires that we enable a different floating point control word then the
 * hardware default. This means that when we're dealing with case (1) for
 * x87/SSE we have to be more careful than the other components. Thankfully for
 * everything else this is just keeping it zeroed.
 *
 * A reasonable question would be why not just skip components that aren't
 * marked as present. There are a few reasons we take a different approach and
 * always include them. Both of these are to make lives simpler for consumers.
 * In the first case, when someone is performing a read and wants to reassemble
 * and answer the question of 'what is the value of %ymm0 or %zmm15', they have
 * to combine multiple disparate parts. If one knows that the data we put into
 * there is always valid and represents what is in hardware and doesn't have to
 * keep track of what are the defaults in different circumstances, then that
 * greatly simplifies consumers lives. It also helps us for core files and other
 * observability cases because the answer to what is the operating system's
 * default may change over time.
 *
 * Similarly, including all the possible structures means that we have
 * simplified writes. Writes are always setting the full state of a thread,
 * meaning that if someone wants to modify only a single register they must do a
 * read, modify, and write. By including everything that they might need, it
 * makes it easier for consumers to do this and not have to cons up the whole
 * structure on their own.
 *
 * When we're setting state, things change around a little bit. We have a few
 * constraints that are laid out in proc(5). In particular, we require that the
 * PRX_INFO_XSAVE component always be present to tell us which other components
 * we expect to be here and which ones we don't. We also are much stricter about
 * writes in several ways. Of all the components, the PRX_INFO_XCR is read-only
 * and may not be modified by a calling process. In addition, when we have
 * 32-bit applications which have reserved registers in the %ymm, %zmm, etc.
 * components, if they are being written to and have modifications, then we will
 * indicate an error there.
 *
 * Because we are given the entire buffer from userland and don't need to have
 * an intermediate place to copy it in, we will validate the entire thing in
 * advance. Once it has been validated and we consider it legal, then we will
 * translate each entry into its corresponding entry in pcb's normal floating
 * point state. This is different from signal handling mostly because of the
 * fact that we are not using copyin, and once we get to this point, there is
 * no more validation, so we don't have the same concerns around blocking while
 * pre-emption is disabled.
 *
 * The Wrinkle with fpregs
 * -----------------------
 *
 * When we instead turn our attention to the fpregs, whether we're gathering
 * them as part of the ucontext_t or as part of /proc, there are a few
 * complications that we need to be aware of when we're operating on a kernel
 * that is using xsave as the save mechanism. When we're using fxsave as the
 * save mechanism, the CPU will always save the entire 512-byte fxsave region.
 * The fpregs ABI that the kernel expects is basically this structure itself,
 * which is transformed into a 32-bit compatible form in archdep.c.
 *
 * But xsave makes this much more complex and has historically been a source of
 * bugs in the system. In particular, unlike fxsave, xsave has its component bit
 * vector that is written out to indicate validity. This means that blindly
 * copying the fxsave area without checking those bits will lead us to do the
 * wrong thing. The XMM state flag mostly covers the 16 128-bit %xmm registers,
 * while the x87 legacy fp flag covers the rest of the state. This is all good,
 * aside from the MCXSR.
 *
 * One of the more complicated pieces of xsave state management is correctly
 * answering the question of when the MXCSR is written out to xsave_state. In
 * practice, this is rather convoluted and varies. If either the XMM or AVX
 * feature bits are set then the CPU will write out the MXCSR and its mask
 * register into the traditional fxsave state region. This behavior is dependent
 * on the type of save function that we use. xsave and xsaveopt will look at the
 * AVX feature bit; however, xsavec does not and only considers the SSE feature
 * bit. This means that when we're retrieving things, we need to check both of
 * those bits to determine if we should use the initial state or the value
 * written out.
 *
 * When we come to someone trying to set the fpregs through /proc, the main
 * question we have is what happens to the extended registers. We have opted to
 * implement and document it such that a write to the fpregs only impacts the
 * fpregs. Put differently, we will save the FPU state with fp_save() ahead of
 * copying the data into the save area, set the state bits for x87 and XMM
 * state, and then set the FPU to be restored. All in all, this basically means
 * that writing to fpregs does not touch any of the %ymm, %zmm, or other state
 * that we might have present.
 *
 * Forward Looking: Adding Intel AMX Support
 * -----------------------------------------
 *
 * Nothing can stop the march of features being added into the FPU. One of the
 * larger chunks that we will need to wrangle with is Intel's Advanced Matrix
 * Extensions (AMX), which add a large chunk of xsave state to each process.
 * While things like AVX and AVX-512 have been enabled by default, the broader
 * OS community has not been wanting to do this for AMX ,because of the size of
 * the state which exceeds 8 KiB. While the signal handling state went out of
 * its way to minimize the size it wrote to the stack, if this is used, it would
 * need to be preserved.
 *
 * To deal with this reality and the fact that folks don't really want to
 * enable it by default for all purposes when its use will be quite special
 * purpose, Intel has also added a MSR around extended feature disable or xfd.
 * This is what we represent in the PRX_INFO_XCR prx_xfd member. Our starting
 * assumption, and the reason that so much of the /proc and signal logic ensures
 * that we have the thread and process around, taking as an example the unused
 * process argument in fpu_proc_xregs_info(), is that we will follow suit and
 * default to having support disabled, but that a process will be able to opt
 * into it, which will result in several different assumptions around signal
 * stack sizing and cause us to reallocate and extend the pcb's FPU save state.
 *
 * The following is a list of items to pay attention to for future folks who
 * work on this:
 *
 *   o We will want to confirm whether other systems have opted to make this
 *     process-wide or thread-wide. Assuming process-wide, we will need to do a
 *     hold of all lwps while making a change. The interface for that probably
 *     doesn't want to be /proc, as a process probably doesn't want to write to
 *     its own control file. Changing it for another process could be done
 *     through the agent-lwp.
 *   o Opting into this should probably be a one-way street.
 *   o Opting into this will need to evaluate all threads and in particular
 *     stack sizes to confirm they adhere to the new minimum.
 *   o We will need to make sure that setting and clearing the xfd MSR is part
 *     of the FPU context ops and something we set by default on every CPU.
 *   o We will need to add a new interface to allow opting into this feature.
 *   o We will need to ensure that all subsequently created signal stacks adhere
 *     to a required minimum size that we communicate through libc.
 *   o We will need to make sure that both rtld and libc no longer rely on a
 *     static value of the AT_SUN_FPSIZE, but rather realize that this can be
 *     dynamic. At that time, we should evaluate if we can get away with not
 *     needing to save this for rtld, even though signal handlers should assume
 *     they will.
 *   o The various components (because there is more than one) will want to be
 *     added to the fpu_xsave_info[]. Consulting the processes's xfd will be
 *     required and probably require logic changes.
 *
 * The above is not exhaustive. We'll probably have some other issues and fun
 * while doing this.
 */

/*
 * The kind of FPU we advertise to rtld so it knows what to do when working
 * through the PLT.
 */
int fp_elf = AT_386_FPINFO_FXSAVE;

/*
 * Mechanism to save FPU state.
 */
int fp_save_mech = FP_FXSAVE;

/*
 * See section 10.5.1 in the Intel 64 and IA-32 Architectures Software
 * Developer's Manual, Volume 1.
 */
#define	FXSAVE_ALIGN	16

/*
 * See section 13.4 in the Intel 64 and IA-32 Architectures Software
 * Developer's Manual, Volume 1.
 */
#define	XSAVE_ALIGN	64

kmem_cache_t *fpsave_cachep;

/* Legacy fxsave layout + xsave header + ymm */
#define	AVX_XSAVE_SIZE		(512 + 64 + 256)

/*
 * Various sanity checks.
 */
CTASSERT(sizeof (struct fxsave_state) == 512);
CTASSERT(sizeof (struct fnsave_state) == 108);
CTASSERT((offsetof(struct fxsave_state, fx_xmm[0]) & 0xf) == 0);
CTASSERT(sizeof (struct xsave_state) >= AVX_XSAVE_SIZE);

/*
 * Basic architectural alignment information.
 */
#define	FPU_ALIGN_XMM	16
#define	FPU_ALIGN_YMM	32
#define	FPU_ALIGN_ZMM	64

/*
 * This structure is the x86 implementation of the kernel FPU that is defined in
 * uts/common/sys/kfpu.h.
 */

typedef enum kfpu_flags {
	/*
	 * This indicates that the save state has initial FPU data.
	 */
	KFPU_F_INITIALIZED = 0x01
} kfpu_flags_t;

struct kfpu_state {
	fpu_ctx_t	kfpu_ctx;
	kfpu_flags_t	kfpu_flags;
	kthread_t	*kfpu_curthread;
};

/*
 * Initial kfpu state for SSE/SSE2 used by fpinit()
 */
const struct fxsave_state sse_initial = {
	FPU_CW_INIT,	/* fx_fcw */
	0,		/* fx_fsw */
	0,		/* fx_fctw */
	0,		/* fx_fop */
	0,		/* fx_rip */
	0,		/* fx_rdp */
	SSE_MXCSR_INIT	/* fx_mxcsr */
	/* rest of structure is zero */
};

/*
 * Initial kfpu state for AVX used by fpinit()
 */
const struct xsave_state avx_initial = {
	/*
	 * The definition below needs to be identical with sse_initial
	 * defined above.
	 */
	.xs_fxsave = {
		.fx_fcw = FPU_CW_INIT,
		.fx_mxcsr = SSE_MXCSR_INIT,
	},
	.xs_header = {
		/*
		 * bit0 = 1 for XSTATE_BV to indicate that legacy fields are
		 * valid, and CPU should initialize XMM/YMM.
		 */
		.xsh_xstate_bv = 1,
		.xsh_xcomp_bv = 0,
	},
};

/*
 * mxcsr_mask value (possibly reset in fpu_probe); used to avoid
 * the #gp exception caused by setting unsupported bits in the
 * MXCSR register
 */
uint32_t sse_mxcsr_mask = SSE_MXCSR_MASK_DEFAULT;

/*
 * This vector is patched to xsave_ctxt() or xsaveopt_ctxt() if we discover we
 * have an XSAVE-capable chip in fpu_probe.
 */
void (*fpsave_ctxt)(void *) = fpxsave_ctxt;
void (*fprestore_ctxt)(void *) = fpxrestore_ctxt;

/*
 * This function pointer is changed to xsaveopt if the CPU is xsaveopt capable.
 */
void (*xsavep)(struct xsave_state *, uint64_t) = xsave;

static int fpe_sicode(uint_t);
static int fpe_simd_sicode(uint_t);
static void fp_new_lwp(void *, void *);
static void fp_free_ctx(void *, int);

static struct ctxop *
fp_ctxop_allocate(struct fpu_ctx *fp)
{
	const struct ctxop_template tpl = {
		.ct_rev		= CTXOP_TPL_REV,
		.ct_save	= fpsave_ctxt,
		.ct_restore	= fprestore_ctxt,
		.ct_fork	= fp_new_lwp,
		.ct_lwp_create	= fp_new_lwp,
		.ct_free	= fp_free_ctx,
	};
	return (ctxop_allocate(&tpl, fp));
}

/*
 * Copy the state of parent lwp's floating point context into the new lwp.
 * Invoked for both fork() and lwp_create().
 *
 * Note that we inherit -only- the control state (e.g. exception masks,
 * rounding, precision control, etc.); the FPU registers are otherwise
 * reset to their initial state.
 */
static void
fp_new_lwp(void *parent, void *child)
{
	kthread_id_t t = parent, ct = child;
	struct fpu_ctx *fp;		/* parent fpu context */
	struct fpu_ctx *cfp;		/* new fpu context */
	struct fxsave_state *fx, *cfx;
	struct xsave_state *cxs;

	ASSERT(fp_kind != FP_NO);

	fp = &t->t_lwp->lwp_pcb.pcb_fpu;
	cfp = &ct->t_lwp->lwp_pcb.pcb_fpu;

	/*
	 * If the parent FPU state is still in the FPU hw then save it;
	 * conveniently, fp_save() already does this for us nicely.
	 */
	fp_save(fp);

	cfp->fpu_flags = FPU_EN | FPU_VALID;
	cfp->fpu_regs.kfpu_status = 0;
	cfp->fpu_regs.kfpu_xstatus = 0;

	/*
	 * Make sure that the child's FPU is cleaned up and made ready for user
	 * land.
	 */
	PCB_SET_UPDATE_FPU(&ct->t_lwp->lwp_pcb);

	switch (fp_save_mech) {
	case FP_FXSAVE:
		fx = fp->fpu_regs.kfpu_u.kfpu_fx;
		cfx = cfp->fpu_regs.kfpu_u.kfpu_fx;
		bcopy(&sse_initial, cfx, sizeof (*cfx));
		cfx->fx_mxcsr = fx->fx_mxcsr & ~SSE_MXCSR_EFLAGS;
		cfx->fx_fcw = fx->fx_fcw;
		break;

	case FP_XSAVE:
		cfp->fpu_xsave_mask = fp->fpu_xsave_mask;

		VERIFY(fp->fpu_regs.kfpu_u.kfpu_xs != NULL);

		fx = &fp->fpu_regs.kfpu_u.kfpu_xs->xs_fxsave;
		cxs = cfp->fpu_regs.kfpu_u.kfpu_xs;
		cfx = &cxs->xs_fxsave;

		bcopy(&avx_initial, cxs, sizeof (*cxs));
		cfx->fx_mxcsr = fx->fx_mxcsr & ~SSE_MXCSR_EFLAGS;
		cfx->fx_fcw = fx->fx_fcw;
		cxs->xs_header.xsh_xstate_bv |=
		    (get_xcr(XFEATURE_ENABLED_MASK) & XFEATURE_FP_INITIAL);
		break;
	default:
		panic("Invalid fp_save_mech");
		/*NOTREACHED*/
	}

	/*
	 * Mark that both the parent and child need to have the FPU cleaned up
	 * before returning to userland.
	 */

	ctxop_attach(ct, fp_ctxop_allocate(cfp));
}

/*
 * Free any state associated with floating point context.
 * Fp_free can be called in three cases:
 * 1) from reaper -> thread_free -> freectx-> fp_free
 *	fp context belongs to a thread on deathrow
 *	nothing to do,  thread will never be resumed
 *	thread calling ctxfree is reaper
 *
 * 2) from exec -> freectx -> fp_free
 *	fp context belongs to the current thread
 *	must disable fpu, thread calling ctxfree is curthread
 *
 * 3) from restorecontext -> setfpregs -> fp_free
 *	we have a modified context in the memory (lwp->pcb_fpu)
 *	disable fpu and release the fp context for the CPU
 *
 */
void
fp_free(struct fpu_ctx *fp)
{
	ASSERT(fp_kind != FP_NO);

	if (fp->fpu_flags & FPU_VALID)
		return;

	kpreempt_disable();
	/*
	 * We want to do fpsave rather than fpdisable so that we can
	 * keep the fpu_flags as FPU_VALID tracking the CR0_TS bit
	 */
	fp->fpu_flags |= FPU_VALID;
	/* If for current thread disable FP to track FPU_VALID */
	if (curthread->t_lwp && fp == &curthread->t_lwp->lwp_pcb.pcb_fpu) {
		/* Clear errors if any to prevent frstor from complaining */
		(void) fperr_reset();
		if (fp_kind & __FP_SSE)
			(void) fpxerr_reset();
		fpdisable();
	}
	kpreempt_enable();
}

/*
 * Wrapper for freectx to make the types line up for fp_free()
 */
static void
fp_free_ctx(void *arg, int isexec __unused)
{
	fp_free((struct fpu_ctx *)arg);
}

/*
 * Store the floating point state and disable the floating point unit.
 */
void
fp_save(struct fpu_ctx *fp)
{
	ASSERT(fp_kind != FP_NO);

	kpreempt_disable();
	if (!fp || fp->fpu_flags & FPU_VALID ||
	    (fp->fpu_flags & FPU_EN) == 0) {
		kpreempt_enable();
		return;
	}
	ASSERT(curthread->t_lwp && fp == &curthread->t_lwp->lwp_pcb.pcb_fpu);

	switch (fp_save_mech) {
	case FP_FXSAVE:
		fpxsave(fp->fpu_regs.kfpu_u.kfpu_fx);
		break;

	case FP_XSAVE:
		xsavep(fp->fpu_regs.kfpu_u.kfpu_xs, fp->fpu_xsave_mask);
		break;
	default:
		panic("Invalid fp_save_mech");
		/*NOTREACHED*/
	}

	fp->fpu_flags |= FPU_VALID;

	/*
	 * We save the FPU as part of forking, execing, modifications via /proc,
	 * restorecontext, etc. As such, we need to make sure that we return to
	 * userland with valid state in the FPU. If we're context switched out
	 * before we hit sys_rtt_common() we'll end up having restored the FPU
	 * as part of the context ops operations. The restore logic always makes
	 * sure that FPU_VALID is set before doing a restore so we don't restore
	 * it a second time.
	 */
	PCB_SET_UPDATE_FPU(&curthread->t_lwp->lwp_pcb);

	kpreempt_enable();
}

/*
 * Restore the FPU context for the thread:
 * The possibilities are:
 *	1. No active FPU context: Load the new context into the FPU hw
 *	   and enable the FPU.
 */
void
fp_restore(struct fpu_ctx *fp)
{
	switch (fp_save_mech) {
	case FP_FXSAVE:
		fpxrestore(fp->fpu_regs.kfpu_u.kfpu_fx);
		break;

	case FP_XSAVE:
		xrestore(fp->fpu_regs.kfpu_u.kfpu_xs, fp->fpu_xsave_mask);
		break;
	default:
		panic("Invalid fp_save_mech");
		/*NOTREACHED*/
	}

	fp->fpu_flags &= ~FPU_VALID;
}

/*
 * Reset the FPU such that it is in a valid state for a new thread that is
 * coming out of exec. The FPU will be in a usable state at this point. At this
 * point we know that the FPU state has already been allocated and if this
 * wasn't an init process, then it will have had fp_free() previously called.
 */
void
fp_exec(void)
{
	struct fpu_ctx *fp = &ttolwp(curthread)->lwp_pcb.pcb_fpu;

	if (fp_save_mech == FP_XSAVE) {
		fp->fpu_xsave_mask = XFEATURE_FP_ALL;
	}

	struct ctxop *ctx = fp_ctxop_allocate(fp);
	/*
	 * Make sure that we're not preempted in the middle of initializing the
	 * FPU on CPU.
	 */
	kpreempt_disable();
	ctxop_attach(curthread, ctx);
	fpinit();
	fp->fpu_flags = FPU_EN;
	kpreempt_enable();
}


/*
 * Seeds the initial state for the current thread.  The possibilities are:
 *      1. Another process has modified the FPU state before we have done any
 *         initialization: Load the FPU state from the LWP state.
 *      2. The FPU state has not been externally modified:  Load a clean state.
 */
void
fp_seed(void)
{
	struct fpu_ctx *fp = &ttolwp(curthread)->lwp_pcb.pcb_fpu;

	ASSERT(curthread->t_preempt >= 1);
	ASSERT((fp->fpu_flags & FPU_EN) == 0);

	/*
	 * Always initialize a new context and initialize the hardware.
	 */
	if (fp_save_mech == FP_XSAVE) {
		fp->fpu_xsave_mask = XFEATURE_FP_ALL;
	}

	ctxop_attach(curthread, fp_ctxop_allocate(fp));
	fpinit();

	/*
	 * If FPU_VALID is set, it means someone has modified registers via
	 * /proc.  In this case, restore the current lwp's state.
	 */
	if (fp->fpu_flags & FPU_VALID)
		fp_restore(fp);

	ASSERT((fp->fpu_flags & FPU_VALID) == 0);
	fp->fpu_flags = FPU_EN;
}

/*
 * When using xsave/xrstor, these three functions are used by the lwp code to
 * manage the memory for the xsave area.
 */
void
fp_lwp_init(klwp_t *lwp)
{
	struct fpu_ctx *fp = &lwp->lwp_pcb.pcb_fpu;

	/*
	 * We keep a copy of the pointer in lwp_fpu so that we can restore the
	 * value in forklwp() after we duplicate the parent's LWP state.
	 */
	lwp->lwp_fpu = fp->fpu_regs.kfpu_u.kfpu_generic =
	    kmem_cache_alloc(fpsave_cachep, KM_SLEEP);
	fp->fpu_signal = NULL;

	if (fp_save_mech == FP_XSAVE) {
		/*
		 *
		 * We bzero since the fpinit() code path will only
		 * partially initialize the xsave area using avx_inital.
		 */
		ASSERT(cpuid_get_xsave_size() >= sizeof (struct xsave_state));
		bzero(fp->fpu_regs.kfpu_u.kfpu_xs, cpuid_get_xsave_size());
	}
}

void
fp_lwp_cleanup(klwp_t *lwp)
{
	struct fpu_ctx *fp = &lwp->lwp_pcb.pcb_fpu;

	if (fp->fpu_regs.kfpu_u.kfpu_generic != NULL) {
		kmem_cache_free(fpsave_cachep,
		    fp->fpu_regs.kfpu_u.kfpu_generic);
		lwp->lwp_fpu = fp->fpu_regs.kfpu_u.kfpu_generic = NULL;
	}

	if (fp->fpu_signal != NULL) {
		kmem_cache_free(fpsave_cachep, fp->fpu_signal);
		fp->fpu_signal = NULL;
	}
}

/*
 * Called during the process of forklwp(). The kfpu_u pointer will have been
 * overwritten while copying the parent's LWP structure. We have a valid copy
 * stashed in the child's lwp_fpu which we use to restore the correct value.
 */
void
fp_lwp_dup(klwp_t *lwp)
{
	void *xp = lwp->lwp_fpu;
	size_t sz;

	switch (fp_save_mech) {
	case FP_FXSAVE:
		sz = sizeof (struct fxsave_state);
		break;
	case FP_XSAVE:
		sz = cpuid_get_xsave_size();
		break;
	default:
		panic("Invalid fp_save_mech");
		/*NOTREACHED*/
	}

	/* copy the parent's values into the new lwp's struct */
	bcopy(lwp->lwp_pcb.pcb_fpu.fpu_regs.kfpu_u.kfpu_generic, xp, sz);
	/* now restore the pointer */
	lwp->lwp_pcb.pcb_fpu.fpu_regs.kfpu_u.kfpu_generic = xp;
	/* Ensure that we don't inherit our parent's signal state */
	lwp->lwp_pcb.pcb_fpu.fpu_signal = NULL;
}

/*
 * Handle a processor extension error fault
 * Returns non zero for error.
 */

/*ARGSUSED*/
int
fpexterrflt(struct regs *rp)
{
	uint32_t fpcw, fpsw;
	fpu_ctx_t *fp = &ttolwp(curthread)->lwp_pcb.pcb_fpu;

	ASSERT(fp_kind != FP_NO);

	/*
	 * Now we can enable the interrupts.
	 * (NOTE: x87 fp exceptions come thru interrupt gate)
	 */
	sti();

	if (!fpu_exists)
		return (FPE_FLTINV);

	/*
	 * Do an unconditional save of the FP state.  If it's dirty (TS=0),
	 * it'll be saved into the fpu context area passed in (that of the
	 * current thread).  If it's not dirty (it may not be, due to
	 * an intervening save due to a context switch between the sti(),
	 * above and here, then it's safe to just use the stored values in
	 * the context save area to determine the cause of the fault.
	 */
	fp_save(fp);

	/* clear exception flags in saved state, as if by fnclex */
	switch (fp_save_mech) {
	case FP_FXSAVE:
		fpsw = fp->fpu_regs.kfpu_u.kfpu_fx->fx_fsw;
		fpcw = fp->fpu_regs.kfpu_u.kfpu_fx->fx_fcw;
		fp->fpu_regs.kfpu_u.kfpu_fx->fx_fsw &= ~FPS_SW_EFLAGS;
		break;

	case FP_XSAVE:
		fpsw = fp->fpu_regs.kfpu_u.kfpu_xs->xs_fxsave.fx_fsw;
		fpcw = fp->fpu_regs.kfpu_u.kfpu_xs->xs_fxsave.fx_fcw;
		fp->fpu_regs.kfpu_u.kfpu_xs->xs_fxsave.fx_fsw &= ~FPS_SW_EFLAGS;
		/*
		 * Always set LEGACY_FP as it may have been cleared by XSAVE
		 * instruction
		 */
		fp->fpu_regs.kfpu_u.kfpu_xs->xs_header.xsh_xstate_bv |=
		    XFEATURE_LEGACY_FP;
		break;
	default:
		panic("Invalid fp_save_mech");
		/*NOTREACHED*/
	}

	fp->fpu_regs.kfpu_status = fpsw;

	if ((fpsw & FPS_ES) == 0)
		return (0);		/* No exception */

	/*
	 * "and" the exception flags with the complement of the mask
	 * bits to determine which exception occurred
	 */
	return (fpe_sicode(fpsw & ~fpcw & 0x3f));
}

/*
 * Handle an SSE/SSE2 precise exception.
 * Returns a non-zero sicode for error.
 */
/*ARGSUSED*/
int
fpsimderrflt(struct regs *rp)
{
	uint32_t mxcsr, xmask;
	fpu_ctx_t *fp = &ttolwp(curthread)->lwp_pcb.pcb_fpu;

	ASSERT(fp_kind & __FP_SSE);

	/*
	 * NOTE: Interrupts are disabled during execution of this
	 * function.  They are enabled by the caller in trap.c.
	 */

	/*
	 * The only way we could have gotten here if there is no FP unit
	 * is via a user executing an INT $19 instruction, so there is
	 * no fault in that case.
	 */
	if (!fpu_exists)
		return (0);

	/*
	 * Do an unconditional save of the FP state.  If it's dirty (TS=0),
	 * it'll be saved into the fpu context area passed in (that of the
	 * current thread).  If it's not dirty, then it's safe to just use
	 * the stored values in the context save area to determine the
	 * cause of the fault.
	 */
	fp_save(fp);		/* save the FPU state */

	if (fp_save_mech == FP_XSAVE) {
		mxcsr = fp->fpu_regs.kfpu_u.kfpu_xs->xs_fxsave.fx_mxcsr;
		fp->fpu_regs.kfpu_status =
		    fp->fpu_regs.kfpu_u.kfpu_xs->xs_fxsave.fx_fsw;
	} else {
		mxcsr = fp->fpu_regs.kfpu_u.kfpu_fx->fx_mxcsr;
		fp->fpu_regs.kfpu_status = fp->fpu_regs.kfpu_u.kfpu_fx->fx_fsw;
	}
	fp->fpu_regs.kfpu_xstatus = mxcsr;

	/*
	 * compute the mask that determines which conditions can cause
	 * a #xm exception, and use this to clean the status bits so that
	 * we can identify the true cause of this one.
	 */
	xmask = (mxcsr >> 7) & SSE_MXCSR_EFLAGS;
	return (fpe_simd_sicode((mxcsr & SSE_MXCSR_EFLAGS) & ~xmask));
}

/*
 * In the unlikely event that someone is relying on this subcode being
 * FPE_FLTILL for denormalize exceptions, it can always be patched back
 * again to restore old behaviour.
 */
int fpe_fltden = FPE_FLTDEN;

/*
 * Map from the FPU status word to the FP exception si_code.
 */
static int
fpe_sicode(uint_t sw)
{
	if (sw & FPS_IE)
		return (FPE_FLTINV);
	if (sw & FPS_ZE)
		return (FPE_FLTDIV);
	if (sw & FPS_DE)
		return (fpe_fltden);
	if (sw & FPS_OE)
		return (FPE_FLTOVF);
	if (sw & FPS_UE)
		return (FPE_FLTUND);
	if (sw & FPS_PE)
		return (FPE_FLTRES);
	return (FPE_FLTINV);	/* default si_code for other exceptions */
}

/*
 * Map from the SSE status word to the FP exception si_code.
 */
static int
fpe_simd_sicode(uint_t sw)
{
	if (sw & SSE_IE)
		return (FPE_FLTINV);
	if (sw & SSE_ZE)
		return (FPE_FLTDIV);
	if (sw & SSE_DE)
		return (FPE_FLTDEN);
	if (sw & SSE_OE)
		return (FPE_FLTOVF);
	if (sw & SSE_UE)
		return (FPE_FLTUND);
	if (sw & SSE_PE)
		return (FPE_FLTRES);
	return (FPE_FLTINV);	/* default si_code for other exceptions */
}

/*
 * This routine is invoked as part of libc's __fpstart implementation
 * via sysi86(2).
 *
 * It may be called -before- any context has been assigned in which case
 * we try and avoid touching the hardware.  Or it may be invoked well
 * after the context has been assigned and fiddled with, in which case
 * just tweak it directly.
 */
void
fpsetcw(uint16_t fcw, uint32_t mxcsr)
{
	struct fpu_ctx *fp = &curthread->t_lwp->lwp_pcb.pcb_fpu;
	struct fxsave_state *fx;

	if (!fpu_exists || fp_kind == FP_NO)
		return;

	if ((fp->fpu_flags & FPU_EN) == 0) {
		if (fcw == FPU_CW_INIT && mxcsr == SSE_MXCSR_INIT) {
			/*
			 * Common case.  Floating point unit not yet
			 * enabled, and kernel already intends to initialize
			 * the hardware the way the caller wants.
			 */
			return;
		}
		/*
		 * Hmm.  Userland wants a different default.
		 * Do a fake "first trap" to establish the context, then
		 * handle as if we already had a context before we came in.
		 */
		kpreempt_disable();
		fp_seed();
		kpreempt_enable();
	}

	/*
	 * Ensure that the current hardware state is flushed back to the
	 * pcb, then modify that copy.  Next use of the fp will
	 * restore the context.
	 */
	fp_save(fp);

	switch (fp_save_mech) {
	case FP_FXSAVE:
		fx = fp->fpu_regs.kfpu_u.kfpu_fx;
		fx->fx_fcw = fcw;
		fx->fx_mxcsr = sse_mxcsr_mask & mxcsr;
		break;

	case FP_XSAVE:
		fx = &fp->fpu_regs.kfpu_u.kfpu_xs->xs_fxsave;
		fx->fx_fcw = fcw;
		fx->fx_mxcsr = sse_mxcsr_mask & mxcsr;
		/*
		 * Always set LEGACY_FP as it may have been cleared by XSAVE
		 * instruction
		 */
		fp->fpu_regs.kfpu_u.kfpu_xs->xs_header.xsh_xstate_bv |=
		    XFEATURE_LEGACY_FP;
		break;
	default:
		panic("Invalid fp_save_mech");
		/*NOTREACHED*/
	}
}

static void
kernel_fpu_fpstate_init(kfpu_state_t *kfpu)
{
	struct xsave_state *xs;

	switch (fp_save_mech) {
	case FP_FXSAVE:
		bcopy(&sse_initial, kfpu->kfpu_ctx.fpu_regs.kfpu_u.kfpu_fx,
		    sizeof (struct fxsave_state));
		kfpu->kfpu_ctx.fpu_xsave_mask = 0;
		break;
	case FP_XSAVE:
		xs = kfpu->kfpu_ctx.fpu_regs.kfpu_u.kfpu_xs;
		bzero(xs, cpuid_get_xsave_size());
		bcopy(&avx_initial, xs, sizeof (*xs));
		xs->xs_header.xsh_xstate_bv = XFEATURE_LEGACY_FP | XFEATURE_SSE;
		kfpu->kfpu_ctx.fpu_xsave_mask = XFEATURE_FP_ALL;
		break;
	default:
		panic("invalid fp_save_mech");
	}

	/*
	 * Set the corresponding flags that the system expects on the FPU state
	 * to indicate that this is our state. The FPU_EN flag is required to
	 * indicate that FPU usage is allowed. The FPU_KERN flag is explicitly
	 * not set below as it represents that this state is being suppressed
	 * by the kernel.
	 */
	kfpu->kfpu_ctx.fpu_flags = FPU_EN | FPU_VALID;
	kfpu->kfpu_flags |= KFPU_F_INITIALIZED;
}

kfpu_state_t *
kernel_fpu_alloc(int kmflags)
{
	kfpu_state_t *kfpu;

	if ((kfpu = kmem_zalloc(sizeof (kfpu_state_t), kmflags)) == NULL) {
		return (NULL);
	}

	kfpu->kfpu_ctx.fpu_regs.kfpu_u.kfpu_generic =
	    kmem_cache_alloc(fpsave_cachep, kmflags);
	if (kfpu->kfpu_ctx.fpu_regs.kfpu_u.kfpu_generic == NULL) {
		kmem_free(kfpu, sizeof (kfpu_state_t));
		return (NULL);
	}

	kernel_fpu_fpstate_init(kfpu);

	return (kfpu);
}

void
kernel_fpu_free(kfpu_state_t *kfpu)
{
	kmem_cache_free(fpsave_cachep,
	    kfpu->kfpu_ctx.fpu_regs.kfpu_u.kfpu_generic);
	kmem_free(kfpu, sizeof (kfpu_state_t));
}

static void
kernel_fpu_ctx_save(void *arg)
{
	kfpu_state_t *kfpu = arg;
	fpu_ctx_t *pf;

	if (kfpu == NULL) {
		/*
		 * A NULL kfpu implies this is a kernel thread with an LWP and
		 * no user-level FPU usage. Use the lwp fpu save area.
		 */
		pf = &curthread->t_lwp->lwp_pcb.pcb_fpu;

		ASSERT(curthread->t_procp->p_flag & SSYS);
		ASSERT3U(pf->fpu_flags & FPU_VALID, ==, 0);

		fp_save(pf);
	} else {
		pf = &kfpu->kfpu_ctx;

		ASSERT3P(kfpu->kfpu_curthread, ==, curthread);
		ASSERT3U(pf->fpu_flags & FPU_VALID, ==, 0);

		/*
		 * Note, we can't use fp_save because it assumes that we're
		 * saving to the thread's PCB and not somewhere else. Because
		 * this is a different FPU context, we instead have to do this
		 * ourselves.
		 */
		switch (fp_save_mech) {
		case FP_FXSAVE:
			fpxsave(pf->fpu_regs.kfpu_u.kfpu_fx);
			break;
		case FP_XSAVE:
			xsavep(pf->fpu_regs.kfpu_u.kfpu_xs, pf->fpu_xsave_mask);
			break;
		default:
			panic("Invalid fp_save_mech");
		}

		/*
		 * Because we have saved context here, our save state is no
		 * longer valid and therefore needs to be reinitialized.
		 */
		kfpu->kfpu_flags &= ~KFPU_F_INITIALIZED;
	}

	pf->fpu_flags |= FPU_VALID;

	/*
	 * Clear KFPU flag. This allows swtch to check for improper kernel
	 * usage of the FPU (i.e. switching to a new thread while the old
	 * thread was in the kernel and using the FPU, but did not perform a
	 * context save).
	 */
	curthread->t_flag &= ~T_KFPU;
}

static void
kernel_fpu_ctx_restore(void *arg)
{
	kfpu_state_t *kfpu = arg;
	fpu_ctx_t *pf;

	if (kfpu == NULL) {
		/*
		 * A NULL kfpu implies this is a kernel thread with an LWP and
		 * no user-level FPU usage. Use the lwp fpu save area.
		 */
		pf = &curthread->t_lwp->lwp_pcb.pcb_fpu;

		ASSERT(curthread->t_procp->p_flag & SSYS);
		ASSERT3U(pf->fpu_flags & FPU_VALID, !=, 0);
	} else {
		pf = &kfpu->kfpu_ctx;

		ASSERT3P(kfpu->kfpu_curthread, ==, curthread);
		ASSERT3U(pf->fpu_flags & FPU_VALID, !=, 0);
	}

	fp_restore(pf);
	curthread->t_flag |= T_KFPU;
}

/*
 * Validate that the thread is not switching off-cpu while actively using the
 * FPU within the kernel.
 */
void
kernel_fpu_no_swtch(void)
{
	if ((curthread->t_flag & T_KFPU) != 0) {
		panic("curthread swtch-ing while the kernel is using the FPU");
	}
}

static const struct ctxop_template kfpu_ctxop_tpl = {
	.ct_rev		= CTXOP_TPL_REV,
	.ct_save	= kernel_fpu_ctx_save,
	.ct_restore	= kernel_fpu_ctx_restore,
};

void
kernel_fpu_begin(kfpu_state_t *kfpu, uint_t flags)
{
	klwp_t *pl = curthread->t_lwp;
	struct ctxop *ctx;

	if ((curthread->t_flag & T_KFPU) != 0) {
		panic("curthread attempting to nest kernel FPU states");
	}

	/* KFPU_USE_LWP and KFPU_NO_STATE are mutually exclusive. */
	ASSERT((flags & (KFPU_USE_LWP | KFPU_NO_STATE)) !=
	    (KFPU_USE_LWP | KFPU_NO_STATE));

	if ((flags & KFPU_NO_STATE) == KFPU_NO_STATE) {
		/*
		 * Since we don't have a kfpu_state or usable lwp pcb_fpu to
		 * hold our kernel FPU context, we depend on the caller doing
		 * kpreempt_disable for the duration of our FPU usage. This
		 * should only be done for very short periods of time.
		 */
		ASSERT(curthread->t_preempt > 0);
		ASSERT(kfpu == NULL);

		if (pl != NULL) {
			/*
			 * We might have already saved once so FPU_VALID could
			 * be set. This is handled in fp_save.
			 */
			fp_save(&pl->lwp_pcb.pcb_fpu);
			pl->lwp_pcb.pcb_fpu.fpu_flags |= FPU_KERNEL;
		}

		curthread->t_flag |= T_KFPU;

		/* Always restore the fpu to the initial state. */
		fpinit();

		return;
	}

	/*
	 * We either have a kfpu, or are using the LWP pcb_fpu for context ops.
	 */

	if ((flags & KFPU_USE_LWP) == 0) {
		if (kfpu->kfpu_curthread != NULL)
			panic("attempting to reuse kernel FPU state at %p when "
			    "another thread already is using", kfpu);

		if ((kfpu->kfpu_flags & KFPU_F_INITIALIZED) == 0)
			kernel_fpu_fpstate_init(kfpu);

		kfpu->kfpu_curthread = curthread;
	}

	/*
	 * Not all threads may have an active LWP. If they do and we're not
	 * going to re-use the LWP, then we should go ahead and save the state.
	 * We must also note that the fpu is now being used by the kernel and
	 * therefore we do not want to manage the fpu state via the user-level
	 * thread's context handlers.
	 *
	 * We might have already saved once (due to a prior use of the kernel
	 * FPU or another code path) so FPU_VALID could be set. This is handled
	 * by fp_save, as is the FPU_EN check.
	 */
	ctx = ctxop_allocate(&kfpu_ctxop_tpl, kfpu);
	kpreempt_disable();
	if (pl != NULL) {
		if ((flags & KFPU_USE_LWP) == 0)
			fp_save(&pl->lwp_pcb.pcb_fpu);
		pl->lwp_pcb.pcb_fpu.fpu_flags |= FPU_KERNEL;
	}

	/*
	 * Set the context operations for kernel FPU usage.  Because kernel FPU
	 * setup and ctxop attachment needs to happen under the protection of
	 * kpreempt_disable(), we allocate the ctxop outside the guard so its
	 * sleeping allocation will not cause a voluntary swtch().  This allows
	 * the rest of the initialization to proceed, ensuring valid state for
	 * the ctxop handlers.
	 */
	ctxop_attach(curthread, ctx);
	curthread->t_flag |= T_KFPU;

	if ((flags & KFPU_USE_LWP) == KFPU_USE_LWP) {
		/*
		 * For pure kernel threads with an LWP, we can use the LWP's
		 * pcb_fpu to save/restore context.
		 */
		fpu_ctx_t *pf = &pl->lwp_pcb.pcb_fpu;

		VERIFY(curthread->t_procp->p_flag & SSYS);
		VERIFY(kfpu == NULL);
		ASSERT((pf->fpu_flags & FPU_EN) == 0);

		/* Always restore the fpu to the initial state. */
		if (fp_save_mech == FP_XSAVE)
			pf->fpu_xsave_mask = XFEATURE_FP_ALL;
		fpinit();
		pf->fpu_flags = FPU_EN | FPU_KERNEL;
	} else {
		/* initialize the kfpu state */
		kernel_fpu_ctx_restore(kfpu);
	}
	kpreempt_enable();
}

void
kernel_fpu_end(kfpu_state_t *kfpu, uint_t flags)
{
	if ((curthread->t_flag & T_KFPU) == 0) {
		panic("curthread attempting to clear kernel FPU state "
		    "without using it");
	}

	/*
	 * General comments on why the rest of this function is structured the
	 * way it is. Be aware that there is a lot of subtlety here.
	 *
	 * If a user-level thread ever uses the fpu while in the kernel, then
	 * we cannot call fpdisable since that does STTS. That will set the
	 * ts bit in %cr0 which will cause an exception if anything touches the
	 * fpu. However, the user-level context switch handler (fpsave_ctxt)
	 * needs to access the fpu to save the registers into the pcb.
	 * fpsave_ctxt relies on CLTS having been done to clear the ts bit in
	 * fprestore_ctxt when the thread context switched onto the CPU.
	 *
	 * Calling fpdisable only effects the current CPU's %cr0 register.
	 *
	 * During ctxop_remove and kpreempt_enable, we can voluntarily context
	 * switch, so the CPU we were on when we entered this function might
	 * not be the same one we're on when we return from ctxop_remove or end
	 * the function. Note there can be user-level context switch handlers
	 * still installed if this is a user-level thread.
	 *
	 * We also must be careful in the unlikely chance we're running in an
	 * interrupt thread, since we can't leave the CPU's %cr0 TS state set
	 * incorrectly for the "real" thread to resume on this CPU.
	 */

	if ((flags & KFPU_NO_STATE) == 0) {
		kpreempt_disable();
	} else {
		ASSERT(curthread->t_preempt > 0);
	}

	curthread->t_flag &= ~T_KFPU;

	/*
	 * When we are ending things, we explicitly don't save the current
	 * kernel FPU state back to the temporary state. The kfpu API is not
	 * intended to be a permanent save location.
	 *
	 * If this is a user-level thread and we were to context switch
	 * before returning to user-land, fpsave_ctxt will be a no-op since we
	 * already saved the user-level FPU state the first time we run
	 * kernel_fpu_begin (i.e. we won't save the bad kernel fpu state over
	 * the user-level fpu state). The fpsave_ctxt functions only save if
	 * FPU_VALID is not already set. fp_save also set PCB_SET_UPDATE_FPU so
	 * fprestore_ctxt will be done in sys_rtt_common when the thread
	 * finally returns to user-land.
	 */

	if ((curthread->t_procp->p_flag & SSYS) != 0 &&
	    curthread->t_intr == NULL) {
		/*
		 * A kernel thread which is not an interrupt thread, so we
		 * STTS now.
		 */
		fpdisable();
	}

	if ((flags & KFPU_NO_STATE) == 0) {
		ctxop_remove(curthread, &kfpu_ctxop_tpl, kfpu);

		if (kfpu != NULL) {
			if (kfpu->kfpu_curthread != curthread) {
				panic("attempting to end kernel FPU state "
				    "for %p, but active thread is not "
				    "curthread", kfpu);
			} else {
				kfpu->kfpu_curthread = NULL;
			}
		}

		kpreempt_enable();
	}

	if (curthread->t_lwp != NULL) {
		uint_t f;

		if (flags & KFPU_USE_LWP) {
			f = FPU_EN | FPU_KERNEL;
		} else {
			f = FPU_KERNEL;
		}
		curthread->t_lwp->lwp_pcb.pcb_fpu.fpu_flags &= ~f;
	}
}

void
fpu_save_cache_init(void)
{
	switch (fp_save_mech) {
	case FP_FXSAVE:
		fpsave_cachep = kmem_cache_create("fxsave_cache",
		    sizeof (struct fxsave_state), FXSAVE_ALIGN,
		    NULL, NULL, NULL, NULL, NULL, 0);
		break;
	case FP_XSAVE:
		fpsave_cachep = kmem_cache_create("xsave_cache",
		    cpuid_get_xsave_size(), XSAVE_ALIGN,
		    NULL, NULL, NULL, NULL, NULL, 0);
		break;
	default:
		panic("Invalid fp_save_mech");
	}
}

/*
 * Fill in FPU information that is required by exec.
 */
void
fpu_auxv_info(int *typep, size_t *lenp)
{
	*typep = fp_elf;
	switch (fp_save_mech) {
	case FP_FXSAVE:
		*lenp = sizeof (struct fxsave_state);
		break;
	case FP_XSAVE:
		*lenp = cpuid_get_xsave_size();
		break;
	default:
		*lenp = 0;
		break;
	}
}

/*
 * This function exists to transform an xsave_state into an fxsave_state. The
 * way that we have to do this is nuanced. We assume that callers have already
 * handled FPU_EN and thus we only need to consider the xsave_state and its
 * component vector itself. This results in the following cases that we need to
 * consider:
 *
 *   o Neither the x87 / XMM state bits are set. We use the hardware default and
 *     need to ensure to copy the xsave header.
 *   o Both x87 / XMM state bits are set. We can copy everything.
 *   o Only the x87 bit is set. We need to copy the x87 state but make the XMM
 *     state be in the initial case.
 *   o Only the XMM bit is set. The reverse of the above case.
 *
 * The illumos and hardware defaults in 'sse_initial' and 'avx_initial' are
 * generally the same; however, the default floating point control word is
 * different.
 *
 * Finally, we have the complication of the MXCSR and MCXSR_MASK registers.
 * Because we are using xsave and xsaveopt in the kernel right now and not
 * xsavec, the hardware may write out the MXCSR and MXCSR_MASK registers if the
 * XFEATURE_AVX bit is set. Therefore if we don't have the XMM bit set but AVX
 * is set, we must also come back and copy out the MXCSR register. Sorry, we
 * don't make the rules.
 */
static void
fpu_xsave_to_fxsave(const struct xsave_state *xsave, struct fxsave_state *fx)
{
	const uint64_t comps = xsave->xs_header.xsh_xstate_bv;

	switch (comps & (XFEATURE_LEGACY_FP | XFEATURE_SSE)) {
	case XFEATURE_LEGACY_FP | XFEATURE_SSE:
		bcopy(xsave, fx, sizeof (*fx));
		return;
	case XFEATURE_LEGACY_FP:
		bcopy(xsave, fx, offsetof(struct fxsave_state, fx_xmm));
		fx->fx_mxcsr = SSE_MXCSR_INIT;
		fx->fx_mxcsr_mask = 0;
		break;
	case XFEATURE_SSE:
		bcopy(&sse_initial, fx, offsetof(struct fxsave_state,
		    fx_mxcsr));

		fx->fx_fcw = FPU_CW_INIT_HW;
		fx->fx_mxcsr = xsave->xs_fxsave.fx_mxcsr;
		fx->fx_mxcsr_mask = xsave->xs_fxsave.fx_mxcsr_mask;
		bcopy(xsave->xs_fxsave.fx_xmm, fx->fx_xmm, sizeof (fx->fx_xmm));
		break;
	default:
		bcopy(&sse_initial, fx, sizeof (*fx));
		fx->fx_fcw = FPU_CW_INIT_HW;
		break;
	}

	/*
	 * Account for the AVX causing MXCSR to be valid.
	 */
	if ((xsave->xs_header.xsh_xstate_bv & XFEATURE_AVX) != 0 &&
	    (xsave->xs_header.xsh_xstate_bv & XFEATURE_SSE) == 0) {
		fx->fx_mxcsr = xsave->xs_fxsave.fx_mxcsr;
		fx->fx_mxcsr_mask = xsave->xs_fxsave.fx_mxcsr_mask;
	}
}

/*
 * This function is designed to answer the question of are we using any xsave
 * family of instructions in context switch and therefore we have this state.
 * This should still remain true if we are using xsavec or xsaves in the kernel
 * in the future.
 */
boolean_t
fpu_xsave_enabled(void)
{
	return (fp_save_mech == FP_XSAVE);
}

/*
 * The following structure is used to track and manage the programmatic
 * construction of /proc and signal stack spilling of xsave information. All
 * known xsave types that the kernel supports must be included here.
 */
typedef struct xsave_proc_info {
	/*
	 * This matches the /proc xregs type that this data represents. This s
	 * used for /proc only.
	 */
	uint32_t xi_type;
	/*
	 * This indicates the size of the /proc data that we're operating on.
	 * This is only used for /proc.
	 */
	size_t	xi_size;
	/*
	 * This indicates the alignment that we want to have for the member when
	 * we're writing out. This is not used when setting data. This is only
	 * used for /proc.
	 */
	size_t	xi_align;
	/*
	 * This indicates whether this member must always be considered or not.
	 * This is used in both /proc and context/signal handling.
	 */
	bool	xi_always;
	/*
	 * This contains the corresponding bits in the xsave bit vector that
	 * corresponds to this entry. This is used for both /proc and
	 * context/signal handling.
	 */
	uint64_t xi_bits;
	/*
	 * The xi_fill function pointer is used to write out the /proc regset
	 * data (e.g. when a user reads xregs). This is only used for the /proc
	 * handling. The xi_valid function pointer is used instead to validate a
	 * given set of data that we've read in, while the xi_set pointer is
	 * used to actually transform the data in the underlying fpu save area.
	 */
	void	(*xi_fill)(const fpu_ctx_t *, const struct xsave_proc_info *,
	    void *);
	bool	(*xi_valid)(model_t, const void *);
	void	(*xi_set)(fpu_ctx_t *, const struct xsave_proc_info *,
	    uint64_t, const void *);
	/*
	 * The xi_signal_in and xi_signal_out function pointers are used for
	 * extended context and signal handling information. They are used when
	 * reading in data from a ucontext_t and writing it out respectively.
	 * These are only used for context/signal handling.
	 */
	int	(*xi_signal_in)(const struct xsave_proc_info *,
	    const ucontext_t *, const uc_xsave_t *, void *, uintptr_t *,
	    const uintptr_t);
	int	(*xi_signal_out)(const struct xsave_proc_info *, fpu_copyout_f,
	    uc_xsave_t *, const void *fpup, uintptr_t);
} xsave_proc_info_t;

static bool
fpu_proc_xregs_initial_state(const fpu_ctx_t *fpu, uint64_t feats)
{
	const struct xsave_state *xs = fpu->fpu_regs.kfpu_u.kfpu_xs;

	if ((fpu->fpu_flags & (FPU_EN | FPU_VALID)) == 0) {
		return (true);
	}

	return ((xs->xs_header.xsh_xstate_bv & feats) == 0);
}

static void
fpu_proc_xregs_xcr_fill(const fpu_ctx_t *fpu, const xsave_proc_info_t *info,
    void *datap)
{
	prxregset_xcr_t *xcr = datap;

	xcr->prx_xcr_xcr0 = xsave_bv_all;
}

/*
 * Unlike other instruction portions, we treat the xsave header and the legacy
 * XMM section together as both are somewhat tied at the instruction hip. Unlike
 * the when dealing with other xsave regions like the ymm and zmm components,
 * the initial state here is much more nuanced as it has to match what we actual
 * do in the OS and depends on the components that are present.
 */
static void
fpu_proc_xregs_xsave_fill(const fpu_ctx_t *fpu, const xsave_proc_info_t *info,
    void *datap)
{
	prxregset_xsave_t *prxsave = datap;
	const struct xsave_state *xsave = fpu->fpu_regs.kfpu_u.kfpu_xs;
	size_t hdr_off;

	/*
	 * In the x87/XMM case, the no device vs. initial state is different
	 * because the initial state case still wants us to copy the real xsave
	 * header. It's also worth calling out that the actual illumos default
	 * fxsave state is not the same as what Intel documents. The main
	 * difference is in what the x87 FPU control word is. This results in
	 * the following different cases that we need to think about:
	 *
	 *   o FPU_EN is not set. So we use the illumos default.
	 */
	if ((fpu->fpu_flags & FPU_EN) == 0) {
		bcopy(&avx_initial, prxsave, sizeof (*prxsave));
		return;
	}

	/*
	 * Convert all the fxsave region while taking into account the validity
	 * of the xsave bits. The prxregset_xsave_t structure is the same as the
	 * xsave structure in our ABI and Intel designed the xsave header to
	 * begin with the 512-bit fxsave structure.
	 */
	fpu_xsave_to_fxsave(xsave, (struct fxsave_state *)prxsave);

	/*
	 * Now that we've dealt with the x87 and XMM state, take care of the
	 * header.
	 */
	hdr_off = offsetof(prxregset_xsave_t, prx_xsh_xstate_bv);
	bcopy((const void *)((uintptr_t)xsave + hdr_off),
	    (void *)((uintptr_t)prxsave + hdr_off),
	    sizeof (struct xsave_header));
}

static void
fpu_proc_xregs_std_fill(const fpu_ctx_t *fpu, const xsave_proc_info_t *info,
    void *datap)
{
	if (!fpu_proc_xregs_initial_state(fpu, info->xi_bits)) {
		size_t size, off;
		const void *xsave_off;

		cpuid_get_xsave_info(info->xi_bits, &size, &off);
		ASSERT3U(size, ==, info->xi_size);
		xsave_off = (void *)((uintptr_t)fpu->fpu_regs.kfpu_u.kfpu_xs +
		    off);
		bcopy(xsave_off, datap, info->xi_size);
	}
}

/*
 * Users are not allowed to actually set the xcr information this way. However,
 * to make it easier for someone to just do a read, modify, write, of the xregs
 * data, if it is identical, then we will accept it (and do nothing).
 */
static bool
fpu_proc_xregs_xcr_valid(model_t model, const void *datap)
{
	const prxregset_xcr_t *xcr = datap;

	return (xcr->prx_xcr_xcr0 == xsave_bv_all && xcr->prx_xcr_xfd == 0 &&
	    xcr->prx_xcr_pad[0] == 0 && xcr->prx_xcr_pad[1] == 0);
}

/*
 * To match traditional /proc semantics, we do not error if reserved bits of
 * MXCSR are set, they will be masked off when writing data. We do not allow
 * someone to indicate that they are asking for compressed xsave data, hence the
 * check that prx_xsh_comp_bv is zero. Separately, in fpu_proc_xregs_set() we
 * check that each component that was indicated in the xstate_bv is actually
 * present.
 */
static bool
fpu_proc_xregs_xsave_valid(model_t model, const void *datap)
{
	const prxregset_xsave_t *xsave = datap;
	uint64_t rsvd[6] = { 0 };

	if (bcmp(rsvd, xsave->prx_xsh_reserved, sizeof (rsvd)) != 0 ||
	    xsave->prx_xsh_xcomp_bv != 0) {
		return (false);
	}

	if ((xsave->prx_xsh_xstate_bv & ~xsave_bv_all) != 0) {
		return (false);
	}

	return (true);
}

/*
 * The YMM, ZMM, and Hi-ZMM registers are all valid when in an LP64 environment
 * on x86; however, when operating in ILP32, subsets are reserved. We require
 * that all reserved portions are set to zero.
 */
static bool
fpu_proc_xregs_ymm_valid(model_t model, const void *datap)
{
	upad128_t ymm_zero[8];
	const prxregset_ymm_t *ymm = datap;

	if (model == DATAMODEL_LP64) {
		return (true);
	}

	bzero(&ymm_zero, sizeof (ymm_zero));
	return (bcmp(&ymm->prx_ymm[8], &ymm_zero, sizeof (ymm_zero)) == 0);
}

static bool
fpu_proc_xregs_zmm_valid(model_t model, const void *datap)
{
	upad256_t zmm_zero[8];
	const prxregset_zmm_t *zmm = datap;

	if (model == DATAMODEL_LP64) {
		return (true);
	}

	bzero(&zmm_zero, sizeof (zmm_zero));
	return (bcmp(&zmm->prx_zmm[8], &zmm_zero, sizeof (zmm_zero)) == 0);
}

static bool
fpu_proc_xregs_hi_zmm_valid(model_t model, const void *datap)
{
	prxregset_hi_zmm_t hi_zmm_zero;
	const prxregset_hi_zmm_t *hi_zmm = datap;

	if (model == DATAMODEL_LP64) {
		return (true);
	}

	bzero(&hi_zmm_zero, sizeof (hi_zmm_zero));
	return (bcmp(hi_zmm, &hi_zmm_zero, sizeof (hi_zmm_zero)) == 0);
}

/*
 * The xsave state consists of the first 512 bytes of the XMM state and then the
 * xsave header itself. Because of the xsave header, this structure is marked
 * with xi_always, so we must always process and consider it.
 *
 * Semantically if either of the bits around SSE / x87 is set, then we will copy
 * the entire thing. This may mean that we end up copying a region that is not
 * valid into the save area; however, that should be OK as we still have the
 * specific bit flags that indicate what we should consider or not.
 *
 * There is one additional wrinkle we need to consider and honor here. The CPU
 * will load the MXCSR values if the AVX bit is set in an xrstor regardless of
 * anything else. So if this is set and we do not have a valid x87/XMM bits
 * set then we will set the MXCSR to its default state in case the processor
 * tries to load it. For reference see:
 *
 *   o Intel SDM Volume 1: 13.8.1 Standard Form of XRSTOR
 *   o AMD64 Volume 2: Section 11.5.9 MXCSR State Management
 *
 * Note, the behavior around this changes depending on whether using the
 * compressed xrstor or not. We are not, but it's worth being aware of. We do
 * not worry about MXCSR_MASK because the instructions ignore it.
 */
static void
fpu_proc_xregs_xsave_set(fpu_ctx_t *fpu, const xsave_proc_info_t *info,
    uint64_t xsave_bv, const void *datap)
{
	const struct xsave_state *src_xs = datap;
	struct xsave_state *targ_xs = fpu->fpu_regs.kfpu_u.kfpu_xs;

	if ((xsave_bv & info->xi_bits) != 0) {
		bcopy(&src_xs->xs_fxsave, &targ_xs->xs_fxsave,
		    sizeof (struct fxsave_state));
	} else if ((xsave_bv & XFEATURE_AVX) != 0) {
		targ_xs->xs_fxsave.fx_mxcsr = SSE_MXCSR_INIT;
	}

	bcopy(&src_xs->xs_header, &targ_xs->xs_header,
	    sizeof (struct xsave_header));
	targ_xs->xs_fxsave.fx_mxcsr &= sse_mxcsr_mask;
}

static void
fpu_proc_xregs_std_set(fpu_ctx_t *fpu, const xsave_proc_info_t *info,
    uint64_t xsave_bv, const void *datap)
{
	size_t size, off;
	void *xsave_off;

	cpuid_get_xsave_info(info->xi_bits, &size, &off);
	xsave_off = (void *)((uintptr_t)fpu->fpu_regs.kfpu_u.kfpu_xs +
	    off);
	bcopy(datap, xsave_off, size);
}

/*
 * Dealing with XMM data is a little more annoying in signal context. If UC_FPU
 * is set, the ucontext_t's fpregset_t contains a copy of the XMM region. That
 * must take priority over an XMM region that showed up in the uc_xsave_t data.
 * In the signal copyout code we do not save XMM region in the uc_xsave_t or set
 * it as a present component because of it being kept in the fpregset_t. Because
 * of this behavior, if we find the XMM (or x87) state bits present, we treat
 * that as an error.
 *
 * The system has always gone through and cleaned up the reserved bits in the
 * fxsave state when someone calls setcontext(). Therefore we need to do the
 * same thing which is why you see the masking of the mxcsr below.
 *
 * Finally, there is one last wrinkle here that we need to consider. The
 * fpregset_t has two private words which cache the status/exception
 * information. Therefore, we well...  cheat. Intel has left bytes 464 (0x1d0)
 * through 511 (0x1ff) available for us to do what we want. So we will pass this
 * through that for the moment to help us pass this state around without too
 * much extra allocation.
 */
static int
fpu_signal_copyin_xmm(const xsave_proc_info_t *info, const ucontext_t *kuc,
    const uc_xsave_t *ucx, void *fpup, uintptr_t *udatap,
    const uintptr_t max_udata)
{
	struct xsave_state *xsave = fpup;

	if ((ucx->ucx_bv & info->xi_bits) != 0) {
		return (EINVAL);
	}

	if ((kuc->uc_flags & UC_FPU) != 0) {
		bcopy(&kuc->uc_mcontext.fpregs, &xsave->xs_fxsave,
		    sizeof (struct fxsave_state));
		xsave->xs_fxsave.__fx_ign2[3]._l[0] =
		    kuc->uc_mcontext.fpregs.fp_reg_set.fpchip_state.status;
		xsave->xs_fxsave.__fx_ign2[3]._l[1] =
		    kuc->uc_mcontext.fpregs.fp_reg_set.fpchip_state.xstatus;
		xsave->xs_fxsave.fx_mxcsr &= sse_mxcsr_mask;
		xsave->xs_header.xsh_xstate_bv |= info->xi_bits;
	}

	return (0);
}

static int
fpu_signal_copyin_std(const xsave_proc_info_t *info, const ucontext_t *kuc,
    const uc_xsave_t *ucx, void *fpup, uintptr_t *udatap,
    const uintptr_t max_udata)
{
	size_t len, xsave_off;
	void *copy_to;
	struct xsave_state *xsave = fpup;

	cpuid_get_xsave_info(info->xi_bits, &len, &xsave_off);
	if (*udatap + len > max_udata) {
		return (EOVERFLOW);
	}

	copy_to = (void *)((uintptr_t)fpup + xsave_off);
	if (ddi_copyin((void *)*udatap, copy_to, len, 0) != 0) {
		return (EFAULT);
	}

	xsave->xs_header.xsh_xstate_bv |= info->xi_bits;
	*udatap = *udatap + len;

	return (0);
}

static int
fpu_signal_copyout_std(const xsave_proc_info_t *info, fpu_copyout_f copyfunc,
    uc_xsave_t *ucx, const void *fpup, uintptr_t udatap)
{
	size_t len, xsave_off;
	const void *copy_from;
	void *copy_to;
	int ret;

	cpuid_get_xsave_info(info->xi_bits, &len, &xsave_off);
	copy_from = (void *)(uintptr_t)fpup + xsave_off;
	copy_to = (void *)(udatap + ucx->ucx_len);

	ret = copyfunc(copy_from, copy_to, len);
	if (ret != 0) {
		return (ret);
	}

	ucx->ucx_len += len;
	ucx->ucx_bv |= info->xi_bits;
	return (0);
}

/*
 * This table contains information about the extended FPU states and synthetic
 * information we create for /proc, the ucontext_t, and signal handling. The
 * definition of the xsave_proc_info_t describes how each member is used.
 *
 * In general, this table is expected to be in the order of the xsave data
 * structure itself. Synthetic elements that we create can go anywhere and new
 * ones should be inserted at the end. This structure is walked in order to
 * produce the /proc and signal handling logic, so changing the order is
 * meaningful for those and should not be done lightly.
 */
static const xsave_proc_info_t fpu_xsave_info[] = { {
	.xi_type = PRX_INFO_XCR,
	.xi_size = sizeof (prxregset_xcr_t),
	.xi_align = alignof (prxregset_xcr_t),
	.xi_always = true,
	.xi_bits = 0,
	.xi_fill = fpu_proc_xregs_xcr_fill,
	.xi_valid = fpu_proc_xregs_xcr_valid
}, {
	/*
	 * The XSAVE entry covers both the xsave header and the %xmm registers.
	 * Note, there is no signal copyout information for the %xmm registers
	 * because it is expected that that data is already in the fpregset_t.
	 */
	.xi_type = PRX_INFO_XSAVE,
	.xi_size = sizeof (prxregset_xsave_t),
	.xi_align = FPU_ALIGN_XMM,
	.xi_always = true,
	.xi_bits = XFEATURE_LEGACY_FP | XFEATURE_SSE,
	.xi_fill = fpu_proc_xregs_xsave_fill,
	.xi_set = fpu_proc_xregs_xsave_set,
	.xi_valid = fpu_proc_xregs_xsave_valid,
	.xi_signal_in = fpu_signal_copyin_xmm
}, {
	.xi_type = PRX_INFO_YMM,
	.xi_size = sizeof (prxregset_ymm_t),
	.xi_align = FPU_ALIGN_YMM,
	.xi_always = false,
	.xi_bits = XFEATURE_AVX,
	.xi_fill = fpu_proc_xregs_std_fill,
	.xi_set = fpu_proc_xregs_std_set,
	.xi_signal_in = fpu_signal_copyin_std,
	.xi_valid = fpu_proc_xregs_ymm_valid,
	.xi_signal_out = fpu_signal_copyout_std
}, {
	/*
	 * There is no /proc validation function for the mask registers because
	 * they are the same in ILP32 / LP64 and there is nothing for us to
	 * actually validate.
	 */
	.xi_type = PRX_INFO_OPMASK,
	.xi_size = sizeof (prxregset_opmask_t),
	.xi_align = alignof (prxregset_opmask_t),
	.xi_always = false,
	.xi_bits = XFEATURE_AVX512_OPMASK,
	.xi_fill = fpu_proc_xregs_std_fill,
	.xi_set = fpu_proc_xregs_std_set,
	.xi_signal_in = fpu_signal_copyin_std,
	.xi_signal_out = fpu_signal_copyout_std
}, {
	.xi_type = PRX_INFO_ZMM,
	.xi_size = sizeof (prxregset_zmm_t),
	.xi_align = FPU_ALIGN_ZMM,
	.xi_always = false,
	.xi_bits = XFEATURE_AVX512_ZMM,
	.xi_fill = fpu_proc_xregs_std_fill,
	.xi_set = fpu_proc_xregs_std_set,
	.xi_valid = fpu_proc_xregs_zmm_valid,
	.xi_signal_in = fpu_signal_copyin_std,
	.xi_signal_out = fpu_signal_copyout_std
}, {
	.xi_type = PRX_INFO_HI_ZMM,
	.xi_size = sizeof (prxregset_hi_zmm_t),
	.xi_align = FPU_ALIGN_ZMM,
	.xi_always = false,
	.xi_bits = XFEATURE_AVX512_HI_ZMM,
	.xi_fill = fpu_proc_xregs_std_fill,
	.xi_set = fpu_proc_xregs_std_set,
	.xi_valid = fpu_proc_xregs_hi_zmm_valid,
	.xi_signal_in = fpu_signal_copyin_std,
	.xi_signal_out = fpu_signal_copyout_std
} };

static bool
fpu_proc_xregs_include(const xsave_proc_info_t *infop)
{
	return (infop->xi_always || (xsave_bv_all & infop->xi_bits) != 0);
}

void
fpu_proc_xregs_info(struct proc *p __unused, uint32_t *ninfop, uint32_t *sizep,
    uint32_t *dstart)
{
	size_t ret = sizeof (prxregset_hdr_t);
	uint32_t ninfo = 0;

	ASSERT(fpu_xsave_enabled());

	/*
	 * Right now the set of flags that are enabled in the FPU is global.
	 * That is, while the pcb's fcpu_ctx_t has the fpu_xsave_mask, the
	 * actual things that might show up and we care about are all about what
	 * is set up in %xcr0 which is stored in the global xsave_bv_all. If we
	 * move to per-process FPU enablement which is likely to come with AMX,
	 * then this will need the proc_t to look at, hence why we've set things
	 * up with the unused variable above.
	 *
	 * We take two passes through the array. The first is just to count up
	 * how many informational entries we need.
	 */
	for (size_t i = 0; i < ARRAY_SIZE(fpu_xsave_info); i++) {
		if (!fpu_proc_xregs_include(&fpu_xsave_info[i]))
			continue;
		ninfo++;
	}

	ASSERT3U(ninfo, >, 0);
	ret += sizeof (prxregset_info_t) * ninfo;

	for (size_t i = 0; i < ARRAY_SIZE(fpu_xsave_info); i++) {
		size_t curphase;
		if (!fpu_proc_xregs_include(&fpu_xsave_info[i]))
			continue;

		curphase = ret % fpu_xsave_info[i].xi_align;
		if (ret < fpu_xsave_info[i].xi_align) {
			ret = fpu_xsave_info[i].xi_align;
		} else if (curphase != 0) {
			ret += curphase;
		}

		if (i == 0 && dstart != NULL) {
			*dstart = ret;
		}

		ret += fpu_xsave_info[i].xi_size;
	}

	VERIFY3U(ret, <=, UINT32_MAX);
	if (sizep != NULL) {
		*sizep = ret;
	}

	if (ninfop != NULL) {
		*ninfop = ninfo;
	}
}

/*
 * This function supports /proc. Because /proc does not have a process locked
 * while processing a PCSXREG, this tries to establish an upper bound that we
 * will validate later in fpu_proc_xregs_set(). We basically say that if you
 * take the maximum xsave size and add 1 KiB that is a good enough approximation
 * for the maximum size. The 1 KiB is us basically trying to rationalize the
 * overhead of our structures that we're adding right, while being cognisant of
 * differing alignments and the fact that the full xsave size is in some cases
 * (when supervisor states or features we don't support are present) going to be
 * larger than we would need for this.
 */
size_t
fpu_proc_xregs_max_size(void)
{
	VERIFY(fpu_xsave_enabled());
	return (cpuid_get_xsave_size() + 0x1000);
}

/*
 * This functions supports /proc. In particular, it's meant to perform the
 * following:
 *
 *  o Potentially save the current thread's registers.
 *  o Write out the x86 xsave /proc xregs format data from the xsave data we
 *    actually have. Note, this can be a little weird for cases where the FPU is
 *    not actually enabled, which happens for system processes.
 */
void
fpu_proc_xregs_get(klwp_t *lwp, void *buf)
{
	uint32_t size, ninfo, curinfo, dstart;
	fpu_ctx_t *fpu = &lwp->lwp_pcb.pcb_fpu;
	prxregset_hdr_t *hdr = buf;

	ASSERT(fpu_xsave_enabled());
	fpu_proc_xregs_info(lwp->lwp_procp, &ninfo, &size, &dstart);

	/*
	 * Before we get going, defensively zero out all the data buffer so that
	 * the rest of the fill functions can assume a specific base.
	 */
	bzero(buf, size);

	kpreempt_disable();
	if ((fpu->fpu_flags & (FPU_EN | FPU_VALID)) == FPU_EN) {
		/*
		 * This case suggests that thread in question doesn't have a
		 * valid FPU save state which should only happen when it is on
		 * CPU. If this is the case, we must ensure that we save the
		 * current FPU state before proceeding. We also sanity check
		 * several things here before doing this as using /proc on
		 * yourself is always exciting. fp_save() will ensure that the
		 * thread is flagged to go back to being an eager FPU before
		 * returning back to userland.
		 */
		VERIFY3P(curthread, ==, lwptot(lwp));
		VERIFY0(lwptot(lwp)->t_flag & T_KFPU);
		fp_save(fpu);
	}
	kpreempt_enable();

	hdr->pr_type = PR_TYPE_XSAVE;
	hdr->pr_size = size;
	hdr->pr_flags = hdr->pr_pad[0] = hdr->pr_pad[1] = hdr->pr_pad[2] =
	    hdr->pr_pad[3] = 0;
	hdr->pr_ninfo = ninfo;

	curinfo = 0;
	for (size_t i = 0; i < ARRAY_SIZE(fpu_xsave_info); i++) {
		void *startp;
		uint32_t phase;

		if (!fpu_proc_xregs_include(&fpu_xsave_info[i]))
			continue;

		phase = dstart % fpu_xsave_info[i].xi_align;
		if (dstart < fpu_xsave_info[i].xi_align) {
			ASSERT3U(i, !=, 0);
			dstart = fpu_xsave_info[i].xi_align;
		} else if (phase != 0) {
			ASSERT3U(i, !=, 0);
			dstart += phase;
		}

		hdr->pr_info[curinfo].pri_type = fpu_xsave_info[i].xi_type;
		hdr->pr_info[curinfo].pri_flags = 0;
		hdr->pr_info[curinfo].pri_size = fpu_xsave_info[i].xi_size;
		hdr->pr_info[curinfo].pri_offset = dstart;

		startp = (void *)((uintptr_t)buf + dstart);
		fpu_xsave_info[i].xi_fill(fpu, &fpu_xsave_info[i], startp);
		dstart += fpu_xsave_info[i].xi_size;
		ASSERT3U(curinfo, <=, ninfo);
		curinfo++;
	}
}

/*
 * We have been asked to set the data in the FPU for a given thread. Our
 * prmachdep code has already validated that the raw semantics of the data that
 * we have are valid (that is the appropriate sizes, offsets, and flags). We now
 * apply additional checking here:
 *
 *   o The xsave structure is present and only valid bits are set.
 *   o If the xsave component bit-vector is set, we have the corresponding proc
 *     info item.
 *   o Read-only items are ignored if and only if they actually match what we
 *     gave the user mostly as a courtesy to simplify things here.
 *   o ILP32 processes which can't support many of the regions are allowed to
 *     have the items here (as we likely gave them to them), but they must be
 *     zero if they are set.
 *
 * We take a first pass through all the data, validating it makes sense for the
 * FPU. Only after that point do we ensure that we have the FPU data in question
 * and then we clobber all the FPU data. Part of the semantics of setting this
 * is that we're setting the entire extended FPU.
 */
int
fpu_proc_xregs_set(klwp_t *lwp, void *buf)
{
	prxregset_hdr_t *prx = buf;
	model_t model = lwp_getdatamodel(lwp);
	uint64_t bv_found = 0;
	const prxregset_xsave_t *xsave = NULL;
	fpu_ctx_t *fpu = &lwp->lwp_pcb.pcb_fpu;

	VERIFY(fpu_xsave_enabled());

	/*
	 * First, walk each note info header that we have from the user and
	 * proceed to validate it. The prmachdep code has already validated that
	 * the size, type, and offset information is valid, but it has not
	 * validated the semantic contents of this or if someone is trying to
	 * write something they shouldn't.
	 *
	 * While we walk this, we keep track of where the xsave header is. We
	 * also track all of the bits that we have found along the way so we can
	 * match up and ensure that everything that was set has a corresponding
	 * bit in the xsave bitmap. If we have something in the xsave bitmap,
	 * but not its corresponding data, then that is an error. However, we
	 * allow folks to write data regions without the bit set in the xsave
	 * data to make the read, modify, write process simpler.
	 */
	for (uint32_t i = 0; i < prx->pr_ninfo; i++) {
		const prxregset_info_t *info = &prx->pr_info[i];
		bool found = false;

		for (size_t pt = 0; pt < ARRAY_SIZE(fpu_xsave_info); pt++) {
			void *data;
			if (info->pri_type != fpu_xsave_info[pt].xi_type)
				continue;

			found = true;
			data = (void *)((uintptr_t)buf + info->pri_offset);
			if (fpu_xsave_info[pt].xi_valid != NULL &&
			    !fpu_xsave_info[pt].xi_valid(model, data)) {
				return (EINVAL);
			}

			if (info->pri_type == PRX_INFO_XSAVE) {
				xsave = data;
			}
			bv_found |= fpu_xsave_info[pt].xi_bits;
			break;
		}

		if (!found) {
			return (EINVAL);
		}
	}

	/*
	 * No xsave data, no dice.
	 */
	if (xsave == NULL) {
		return (EINVAL);
	}

	/*
	 * If anything is set in the xsave header that was not found as we
	 * walked structures, then that is an error. The opposite is not true as
	 * discussed above.
	 */
	if ((xsave->prx_xsh_xstate_bv & ~bv_found) != 0) {
		return (EINVAL);
	}

	/*
	 * At this point, we consider all the data actually valid. Now we must
	 * set up this information in the save area. If this is our own lwp, we
	 * must disable it first. Otherwise, we expect that it is already valid.
	 * To try to sanitize this, we will defensively zero the entire region
	 * as we are setting everything that will result in here.
	 */
	kpreempt_disable();
	if ((fpu->fpu_flags & (FPU_EN | FPU_VALID)) == FPU_EN) {
		/*
		 * This case suggests that thread in question doesn't have a
		 * valid FPU save state which should only happen when it is on
		 * CPU. If this is the case, we explicitly disable the FPU, but
		 * do not save it before proceeding. We also sanity check
		 * several things here before doing this as using /proc on
		 * yourself is always exciting. Unlike fp_save(), fp_free() does
		 * not signal that an update is required, so we unconditionally
		 * set that for all threads.
		 */
		VERIFY3P(curthread, ==, lwptot(lwp));
		VERIFY0(lwptot(lwp)->t_flag & T_KFPU);
		fp_free(fpu);
	}
	PCB_SET_UPDATE_FPU(&lwp->lwp_pcb);
	bzero(lwp->lwp_pcb.pcb_fpu.fpu_regs.kfpu_u.kfpu_generic,
	    cpuid_get_xsave_size());

	for (uint32_t i = 0; i < prx->pr_ninfo; i++) {
		const prxregset_info_t *info = &prx->pr_info[i];
		bool found = false;

		for (size_t pt = 0; pt < ARRAY_SIZE(fpu_xsave_info); pt++) {
			const void *data;
			if (info->pri_type != fpu_xsave_info[pt].xi_type)
				continue;

			/*
			 * Check if we have a set function and if we should
			 * include this. We may not if this is something like
			 * PRX_INFO_XCR which is read-only.
			 *
			 * We may not include a given entry as it may not have
			 * been set in the actual xsave state that we have been
			 * asked to restore, in which case to not break the
			 * xsaveopt logic, we must leave it in its initial
			 * state, e.g. zeroed (generally). XMM data initial
			 * state is not zeroed, but is marked with xi_always to
			 * help account for this.
			 */
			found = true;
			if (fpu_xsave_info[pt].xi_set == NULL)
				break;
			if (!fpu_xsave_info[pt].xi_always &&
			    (xsave->prx_xsh_xstate_bv &
			    fpu_xsave_info[pt].xi_bits) !=
			    fpu_xsave_info[pt].xi_bits) {
				break;
			}

			data = (void *)((uintptr_t)buf + info->pri_offset);
			fpu_xsave_info[pt].xi_set(fpu, &fpu_xsave_info[pt],
			    xsave->prx_xsh_xstate_bv, data);
		}

		VERIFY(found);
	}
	kpreempt_enable();

	return (0);
}

/*
 * To be included in the signal copyout logic we must have a copy function and
 * the bit in question must be included. Note, we don't consult xi_always here
 * as that is really part of what is always present for xsave logic and
 * therefore isn't really pertinent here because of our custom format. See the
 * big theory statement for more info.
 */
static bool
fpu_signal_include(const xsave_proc_info_t *infop, uint64_t xs_bv)
{
	return ((infop->xi_bits & xs_bv) == infop->xi_bits &&
	    infop->xi_signal_out != NULL);
}

/*
 * We need to fill out the xsave related data into the ucontext_t that we've
 * been given. We should have a valid user pointer at this point in the uc_xsave
 * member. This is much simpler than the copyin that we have. Here are the
 * current assumptions:
 *
 *   o This is being called for the current thread. This is not meant to operate
 *     on an arbitrary thread's state.
 *   o We cannot assume whether the FPU is valid in the pcb or not. While most
 *     callers will have just called getfpregs() which saved the state, don't
 *     assume that.
 *   o We assume that the user address has the requisite required space for this
 *     to be copied out.
 *   o We assume that copyfunc() will ensure we are not copying into a kernel
 *     address.
 *
 * For more information on the format of the data, see the 'Signal Handling and
 * the ucontext_t' portion of the big theory statement. We copy out all the
 * constituent parts and then come back and write out the actual final header
 * information.
 */
int
fpu_signal_copyout(klwp_t *lwp, uintptr_t uaddr, fpu_copyout_f copyfunc)
{
	struct fpu_ctx *fpu = &lwp->lwp_pcb.pcb_fpu;
	uint64_t xs_bv;
	uc_xsave_t ucx;
	int ret;

	VERIFY3P(curthread, ==, lwptot(lwp));
	VERIFY0(lwptot(lwp)->t_flag & T_KFPU);
	VERIFY3U(fpu->fpu_flags & FPU_EN, ==, FPU_EN);

	if (!fpu_xsave_enabled()) {
		return (ENOTSUP);
	}

	/*
	 * Unlike when we're dealing with /proc, we can unconditionally call
	 * fp_save() because this is always called in the context where the lwp
	 * we're operating on is always the one on CPU (which is what fp_save()
	 * asserts).
	 */
	fp_save(fpu);

	bzero(&ucx, sizeof (ucx));
	ucx.ucx_vers = UC_XSAVE_VERS;
	ucx.ucx_len += sizeof (uc_xsave_t);

	xs_bv = fpu->fpu_regs.kfpu_u.kfpu_xs->xs_header.xsh_xstate_bv;
	for (size_t i = 0; i < ARRAY_SIZE(fpu_xsave_info); i++) {
		const xsave_proc_info_t *info = &fpu_xsave_info[i];

		if (!fpu_signal_include(&fpu_xsave_info[i], xs_bv))
			continue;
		ret = info->xi_signal_out(info, copyfunc, &ucx,
		    lwp->lwp_pcb.pcb_fpu.fpu_regs.kfpu_u.kfpu_generic,
		    uaddr);
		if (ret != 0) {
			kpreempt_enable();
			return (ret);
		}
	}

	/*
	 * Now that everything has been copied out, we should have an accurate
	 * value in the uc_xsave_t header and we can copy that out at the start
	 * of the user data.
	 */
	ret = copyfunc(&ucx, (void *)uaddr, sizeof (ucx));
	return (ret);
}

/*
 * Here we've been given a ucontext_t which potentially has a user pointer to
 * xsave state that we've copied out previously. In this case we need to do the
 * following, assuming UC_XSAVE is present:
 *
 *   o Copy in our header and validate it.
 *   o Allocate an fpu context to use as a holding ground for all this data.
 *   o If UC_FPU is set, override the xsave structure with the saved XMM state,
 *     clear UC_FPU, and make sure that the correct xsave_bv bits are set.
 *
 * Currently we always allocate the additional state as a holding ground for the
 * FPU. What we're copying in may not be valid and we don't want to clobber the
 * existing FPU state or deal with merging it until we believe it's reasonable
 * enough. The proc_t is here to set us up for when we have per-process settings
 * in the extended feature disable MSRs.
 */
int
fpu_signal_copyin(klwp_t *lwp, ucontext_t *kuc)
{
	uc_xsave_t ucx;
	uint64_t bv;
	uintptr_t data, max_data;
	void *fpu;
	proc_t *p = lwp->lwp_procp;
	size_t ksize;

	/*
	 * Because this has been opaque filler and the kernel has never
	 * historically looked at it, we don't really care about the uc_xsave
	 * pointer being garbage in the case that the flag is not set. While
	 * this isn't perhaps the most sporting choice in some cases, this is on
	 * the other hand, pragmatic.
	 */
	if ((kuc->uc_flags & UC_XSAVE) != 0) {
		if (kuc->uc_xsave == 0) {
			return (EINVAL);
		}

		if (!fpu_xsave_enabled()) {
			return (ENOTSUP);
		}
	} else {
		return (0);
	}

	if (ddi_copyin((const void *)kuc->uc_xsave, &ucx, sizeof (ucx), 0) !=
	    0) {
		return (EFAULT);
	}

	ksize = cpuid_get_xsave_size();
	if (ucx.ucx_vers != UC_XSAVE_VERS || ucx.ucx_len < sizeof (ucx) ||
	    ucx.ucx_len > ksize ||
	    (ucx.ucx_bv & ~xsave_bv_all) != 0 ||
	    (uintptr_t)p->p_as->a_userlimit - ucx.ucx_len <
	    (uintptr_t)kuc->uc_xsave) {
		return (EINVAL);
	}

	/*
	 * OK, our goal right now is to recreate a valid xsave_state structure
	 * that we'll ultimately end up having to merge with our existing one in
	 * the FPU save state. The reason we describe this as a merge is to help
	 * future us when we want to retain supervisor state which will never be
	 * part of userland signal state. The design of the userland signal
	 * state is basically to compress it as much as we can. This is done for
	 * two reasons:
	 *
	 *   1) We currently consider this a private interface.
	 *   2) We really want to minimize the actual amount of stack space we
	 *	use as much as possible. Most applications aren't using AVX-512
	 *	right now, so doing our own compression style is worthwhile. If
	 *	libc adopts AVX-512 routines, we may want to change this.
	 *
	 * On the allocation below, our assumption is that if a thread has taken
	 * a signal, then it is likely to take a signal again in the future (or
	 * be shortly headed to its demise). As such, when that happens we will
	 * leave the allocated signal stack around for the process. Most
	 * applications don't allow all threads to take signals, so this should
	 * hopefully help amortize the cost of the allocation.
	 */
	max_data = (uintptr_t)kuc->uc_xsave + ucx.ucx_len;
	data = (uintptr_t)kuc->uc_xsave + sizeof (ucx);
	bv = ucx.ucx_bv;
	if (lwp->lwp_pcb.pcb_fpu.fpu_signal == NULL) {
		lwp->lwp_pcb.pcb_fpu.fpu_signal =
		    kmem_cache_alloc(fpsave_cachep, KM_SLEEP);
	}
	fpu = lwp->lwp_pcb.pcb_fpu.fpu_signal;

	/*
	 * Unconditionally initialize the memory we get in here to ensure that
	 * it is in a reasonable state for ourselves. This ensures that unused
	 * regions are mostly left in their initial state (the main exception
	 * here is the x87/XMM state, but that should be OK). We don't fill in
	 * the initial xsave state as we expect that to happen as part of our
	 * processing.
	 */
	bzero(fpu, ksize);

	for (size_t i = 0; i < ARRAY_SIZE(fpu_xsave_info); i++) {
		int ret;
		const xsave_proc_info_t *info = &fpu_xsave_info[i];
		if (!info->xi_always && (info->xi_bits & bv) == 0)
			continue;
		bv &= ~info->xi_bits;

		if (info->xi_signal_in == NULL)
			continue;
		ret = info->xi_signal_in(info, kuc, &ucx, fpu, &data, max_data);
		if (ret != 0) {
			return (ret);
		}
	}
	ASSERT0(bv);

	/*
	 * As described in the big theory statement section 'Signal Handling and
	 * the ucontext_t', we always remove UC_FPU from here as we've taken
	 * care of reassembling it ourselves.
	 */
	kuc->uc_flags &= ~UC_FPU;
	kuc->uc_xsave = (uintptr_t)fpu;

	return (0);
}

/*
 * This determines the size of the signal stack that we need for our custom form
 * of the xsave state.
 */
size_t
fpu_signal_size(klwp_t *lwp)
{
	struct fpu_ctx *fpu = &lwp->lwp_pcb.pcb_fpu;
	size_t len = sizeof (uc_xsave_t);
	uint64_t xs_bv;

	VERIFY3P(curthread, ==, lwptot(lwp));
	VERIFY0(lwptot(lwp)->t_flag & T_KFPU);
	VERIFY3U(fpu->fpu_flags & FPU_EN, ==, FPU_EN);

	if (!fpu_xsave_enabled()) {
		return (0);
	}

	kpreempt_disable();
	if ((fpu->fpu_flags & (FPU_EN | FPU_VALID)) == FPU_EN) {
		fp_save(fpu);
	}

	xs_bv = fpu->fpu_regs.kfpu_u.kfpu_xs->xs_header.xsh_xstate_bv;
	for (size_t i = 0; i < ARRAY_SIZE(fpu_xsave_info); i++) {
		size_t comp_size;

		if (!fpu_signal_include(&fpu_xsave_info[i], xs_bv))
			continue;

		cpuid_get_xsave_info(fpu_xsave_info[i].xi_bits, &comp_size,
		    NULL);
		len += comp_size;
	}

	kpreempt_enable();
	return (len);
}

/*
 * This function is used in service of restorecontext() to set the specified
 * thread's extended FPU state to the passed in data. Our assumptions at this
 * point from the system are:
 *
 *   o Someone has already verified that the actual xsave header is correct.
 *   o Any traditional XMM state that causes a #gp has been clamped.
 *   o That data is basically the correct sized xsave state structure. Right now
 *     that means it is not compressed and follows the CPUID-based rules for
 *     constructing and laying out data.
 *   o That the lwp argument refers to the current thread.
 *
 * Our primary purpose here is to merge the current FPU state with what exists
 * here. Right now, "merge", strictly speaking is just "replace". We can get
 * away with just replacing everything because all we currently save are user
 * states. If we start saving kernel states in here, this will get more nuanced
 * and we will need to be more careful about how we store data here.
 */
void
fpu_set_xsave(klwp_t *lwp, const void *data)
{
	struct fpu_ctx *fpu = &lwp->lwp_pcb.pcb_fpu;
	uint32_t status, xstatus;
	struct xsave_state *dst_xsave;

	VERIFY(fpu_xsave_enabled());
	VERIFY3P(curthread, ==, lwptot(lwp));
	VERIFY0(lwptot(lwp)->t_flag & T_KFPU);
	ASSERT3U(fpu->fpu_flags & FPU_EN, ==, FPU_EN);

	/*
	 * We use fp_save() here rather than a stock fpdisable() so we can
	 * attempt to honor our invariants that when the thread state has been
	 * saved, the valid flag is set, even though we're going to be
	 * overwriting it shortly. If we just called fpdisable() then we would
	 * basically be asking for trouble.
	 *
	 * Because we are modifying the state here and we don't want the system
	 * to end up in an odd state, we are being a little paranoid and
	 * disabling preemption across this operation. In particular, once the
	 * state is properly tagged with FPU_VALID, there should be no other way
	 * that this thread can return to userland and get cleared out because
	 * we're resetting its context; however, we let paranoia win out.
	 */
	kpreempt_disable();
	if ((fpu->fpu_flags & (FPU_EN | FPU_VALID)) == FPU_EN) {
		fp_save(fpu);
	}

	bcopy(data, lwp->lwp_pcb.pcb_fpu.fpu_regs.kfpu_u.kfpu_generic,
	    cpuid_get_xsave_size());
	dst_xsave = lwp->lwp_pcb.pcb_fpu.fpu_regs.kfpu_u.kfpu_generic;
	status = dst_xsave->xs_fxsave.__fx_ign2[3]._l[0];
	xstatus = dst_xsave->xs_fxsave.__fx_ign2[3]._l[1];
	dst_xsave->xs_fxsave.__fx_ign2[3]._l[0] = 0;
	dst_xsave->xs_fxsave.__fx_ign2[3]._l[1] = 0;

	/*
	 * These two status words are information that the kernel itself uses to
	 * track additional information and is part of the traditional fpregset,
	 * but is not part of our xregs information. Because we are setting this
	 * state, we leave it up to the rest of the kernel to determine whether
	 * this came from an fpregset_t or is being reset to the default of 0.
	 */
	fpu->fpu_regs.kfpu_status = status;
	fpu->fpu_regs.kfpu_xstatus = xstatus;

	fpu->fpu_flags |= FPU_VALID;
	PCB_SET_UPDATE_FPU(&lwp->lwp_pcb);
	kpreempt_enable();
}

/*
 * Convert the current FPU state to the traditional fpregset_t. In the 64-bit
 * kernel, this is just an fxsave_state with additional values for the status
 * and xstatus members.
 *
 * This has the same nuance as the xregs cases discussed above, but is simpler
 * in that we only need to handle the fxsave state, but more complicated because
 * we need to check our save mechanism.
 */
void
fpu_get_fpregset(klwp_t *lwp, fpregset_t *fp)
{
	struct fpu_ctx *fpu = &lwp->lwp_pcb.pcb_fpu;

	kpreempt_disable();
	fp->fp_reg_set.fpchip_state.status = fpu->fpu_regs.kfpu_status;
	fp->fp_reg_set.fpchip_state.xstatus = fpu->fpu_regs.kfpu_xstatus;

	if ((fpu->fpu_flags & (FPU_EN | FPU_VALID)) == FPU_EN) {
		/*
		 * If we're requesting the fpregs of a thread that isn't
		 * currently valid and isn't the one that we're executing, then
		 * we consider getting this information to be a best-effort and
		 * we will not stop the thread in question to serialize it,
		 * which means possibly getting stale data. This is the
		 * traditional semantics that the system has used to service
		 * this for /proc.
		 */
		if (curthread == lwptot(lwp)) {
			VERIFY0(lwptot(lwp)->t_flag & T_KFPU);
			fp_save(fpu);
		}
	}

	/*
	 * If the FPU is not enabled and the state isn't valid (due to someone
	 * else setting it), just copy the initial state.
	 */
	if ((fpu->fpu_flags & (FPU_EN | FPU_VALID)) == 0) {
		bcopy(&sse_initial, fp, sizeof (sse_initial));
		kpreempt_enable();
		return;
	}

	/*
	 * Given that we have an enabled FPU, we must look at the type of FPU
	 * save mechanism to clean this up. In particular, while we can just
	 * copy the save area with FXSAVE, with XSAVE we must carefully copy
	 * only the bits that are valid and reset the rest to their default
	 * state.
	 */
	switch (fp_save_mech) {
	case FP_FXSAVE:
		bcopy(fpu->fpu_regs.kfpu_u.kfpu_fx, fp,
		    sizeof (struct fxsave_state));
		break;
	case FP_XSAVE:
		fpu_xsave_to_fxsave(fpu->fpu_regs.kfpu_u.kfpu_xs,
		    (struct fxsave_state *)fp);
		break;
	default:
		panic("Invalid fp_save_mech");
	}

	kpreempt_enable();
}

/*
 * This is a request to set the ABI fpregset_t into our actual hardware state.
 * In the 64-bit kernel the first 512 bytes of the fpregset_t is the same as the
 * 512-byte fxsave area.
 */
void
fpu_set_fpregset(klwp_t *lwp, const fpregset_t *fp)
{
	struct fpu_ctx *fpu = &lwp->lwp_pcb.pcb_fpu;

	kpreempt_disable();
	if ((fpu->fpu_flags & (FPU_EN | FPU_VALID)) == FPU_EN) {
		/*
		 * We always save the entire FPU. This is required if we're
		 * using xsave. If we're using fxsave, we could skip the
		 * 512-byte write and instead just disable the FPU since we'd be
		 * replacing it all. For now we don't bother with more
		 * conditional logic.
		 */
		VERIFY3P(curthread, ==, lwptot(lwp));
		VERIFY0(lwptot(lwp)->t_flag & T_KFPU);
		fp_save(fpu);
	}

	fpu->fpu_regs.kfpu_xstatus = fp->fp_reg_set.fpchip_state.xstatus;
	fpu->fpu_regs.kfpu_status = fp->fp_reg_set.fpchip_state.status;
	switch (fp_save_mech) {
	case FP_FXSAVE:
		bcopy(fp, fpu->fpu_regs.kfpu_u.kfpu_fx,
		    sizeof (struct fxsave_state));
		break;
	case FP_XSAVE:
		bcopy(fp, fpu->fpu_regs.kfpu_u.kfpu_xs,
		    sizeof (struct fxsave_state));
		fpu->fpu_regs.kfpu_u.kfpu_xs->xs_header.xsh_xstate_bv |=
		    XFEATURE_LEGACY_FP | XFEATURE_SSE;
		break;
	default:
		panic("Invalid fp_save_mech");
	}

	fpu->fpu_flags |= FPU_VALID;
	PCB_SET_UPDATE_FPU(&lwp->lwp_pcb);
	kpreempt_enable();
}
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
 * Copyright (c) 2018, Joyent, Inc.
 */

/*
 * Floating point configuration.
 */

#include <sys/types.h>
#include <sys/regset.h>
#include <sys/privregs.h>
#include <sys/x86_archext.h>
#include <sys/archsystm.h>
#include <sys/fp.h>
#include <sys/cmn_err.h>
#include <sys/exec.h>

#define	XMM_ALIGN	16

/*
 * If fpu_exists is non-zero, fpu_probe will attempt to use any
 * hardware FPU (subject to other constraints, see below).  If
 * fpu_exists is zero, fpu_probe will report that there is no
 * FPU even if there is one.
 */
int fpu_exists = 1;

int fp_kind = FP_387;

/*
 * The variable fpu_ignored is provided to allow other code to
 * determine whether emulation is being done because there is
 * no FPU or because of an override requested via /etc/system.
 */
int fpu_ignored = 0;

/*
 * Used by ppcopy and ppzero to determine whether or not to use the
 * SSE-based pagecopy and pagezero routines
 */
int use_sse_pagecopy = 0;
int use_sse_pagezero = 0;
int use_sse_copy = 0;

#if defined(__xpv)

/*
 * Use of SSE or otherwise is forcibly configured for us by the hypervisor.
 */

#define	ENABLE_SSE()
#define	DISABLE_SSE()

#else	/* __xpv */

#define	ENABLE_SSE()	setcr4(CR4_ENABLE_SSE_FLAGS(getcr4()))
#define	DISABLE_SSE()	setcr4(CR4_DISABLE_SSE_FLAGS(getcr4()))

#endif	/* __xpv */

/*
 * Try and figure out what kind of FP capabilities we have, and
 * set up the control registers accordingly.
 */
void
fpu_probe(void)
{
	if (fpu_initial_probe() != 0)
		goto nofpu;

	if (fpu_exists == 0) {
		fpu_ignored = 1;
		goto nofpu;
	}

#ifndef __xpv
	/*
	 * Check and see if the fpu is present by looking
	 * at the "extension type" bit.  (While this used to
	 * indicate a 387DX coprocessor in days gone by,
	 * it's forced on by modern implementations for
	 * compatibility.)
	 */
	if ((getcr0() & CR0_ET) == 0)
		goto nofpu;
#endif

	/* Use the more complex exception clearing code if necessary */
	if (cpuid_need_fp_excp_handling())
		fpsave_ctxt = fpxsave_excp_clr_ctxt;

	/*
	 * SSE and SSE2 are required for the 64-bit ABI.
	 *
	 * If they're not present, we can in principal run
	 * 32-bit userland, though 64-bit processes will be hosed.
	 *
	 * (Perhaps we should complain more about this case!)
	 */
	if (is_x86_feature(x86_featureset, X86FSET_SSE) &&
	    is_x86_feature(x86_featureset, X86FSET_SSE2)) {
		fp_kind |= __FP_SSE;
		ENABLE_SSE();

		if (is_x86_feature(x86_featureset, X86FSET_AVX)) {
			ASSERT(is_x86_feature(x86_featureset, X86FSET_XSAVE));
			fp_kind |= __FP_AVX;
		}

		if (is_x86_feature(x86_featureset, X86FSET_XSAVE)) {
			fp_save_mech = FP_XSAVE;
			fp_elf = AT_386_FPINFO_XSAVE;
			if (is_x86_feature(x86_featureset, X86FSET_XSAVEOPT)) {
				/*
				 * Use the more complex exception
				 * clearing code if necessary.
				 */
				if (cpuid_need_fp_excp_handling()) {
					fpsave_ctxt = xsaveopt_excp_clr_ctxt;
					fp_elf = AT_386_FPINFO_XSAVE_AMD;
				} else {
					fpsave_ctxt = xsaveopt_ctxt;
				}
				xsavep = xsaveopt;
			} else {
				/*
				 * Use the more complex exception
				 * clearing code if necessary.
				 */
				if (cpuid_need_fp_excp_handling()) {
					fpsave_ctxt = xsave_excp_clr_ctxt;
					fp_elf = AT_386_FPINFO_XSAVE_AMD;
				} else {
					fpsave_ctxt = xsave_ctxt;
				}
			}
			fprestore_ctxt = xrestore_ctxt;
		} else {
			/* fp_save_mech defaults to FP_FXSAVE */
			fp_elf = AT_386_FPINFO_FXSAVE;
		}
	}

	if (is_x86_feature(x86_featureset, X86FSET_SSE2)) {
		use_sse_pagecopy = use_sse_pagezero = use_sse_copy = 1;
	}

	if (fp_kind & __FP_SSE) {
		struct fxsave_state *fx;
		uint8_t fxsave_state[sizeof (struct fxsave_state) + XMM_ALIGN];

		/*
		 * Extract the mxcsr mask from our first fxsave
		 */
		fx = (void *)(((uintptr_t)(&fxsave_state[0]) +
		    XMM_ALIGN) & ~(XMM_ALIGN - 1ul));

		fx->fx_mxcsr_mask = 0;
		fxsave_insn(fx);
		if (fx->fx_mxcsr_mask != 0) {
			/*
			 * Override default mask initialized in fpu.c
			 */
			sse_mxcsr_mask = fx->fx_mxcsr_mask;
		}
	}

	setcr0(CR0_ENABLE_FPU_FLAGS(getcr0()));
	return;

	/*
	 * No FPU hardware present
	 */
nofpu:
	setcr0(CR0_DISABLE_FPU_FLAGS(getcr0()));
	DISABLE_SSE();
	fp_kind = FP_NO;
	fpu_exists = 0;
}
/*
 * This file and its contents are supplied under the terms of the
 * Common Development and Distribution License ("CDDL"), version 1.0.
 * You may only use this file in accordance with the terms of version
 * 1.0 of the CDDL.
 *
 * A full copy of the text of the CDDL should have accompanied this
 * source.  A copy of the CDDL is also available via the Internet at
 * http://www.illumos.org/license/CDDL.
 */

/*
 * Copyright 2019 Joyent, Inc.
 * Copyright 2020 OmniOS Community Edition (OmniOSce) Association.
 * Copyright 2024 Oxide Computer Company
 */

#include <sys/cpuvar.h>
#include <sys/types.h>
#include <sys/errno.h>
#include <sys/machsystm.h>
#include <sys/archsystm.h>
#include <sys/controlregs.h>
#include <sys/x86_archext.h>
#include <sys/id_space.h>
#include <sys/hma.h>
#include <sys/cmn_err.h>
#include <vm/hat.h>
#include <vm/as.h>

struct hma_reg {
	const char	*hr_name;
	list_node_t	hr_node;
};

static kmutex_t hma_lock;
static list_t hma_registrations;
static boolean_t hma_exclusive = B_FALSE;
int hma_disable = 0;

typedef enum hma_cpu_status {
	HCS_UNINITIALIZED = 0,
	HCS_READY,
	HCS_ERROR
} hma_cpu_status_t;

/*
 * When both host and guest want simultaneous use of the CPU performance
 * counters, which should take priority?
 *
 * Defer to the guest by default, making its activity invisible to
 * host-configured CPC measurements.  This is necessary since the Capacity &
 * Utilization system keeps the CPCs active at all times when not in use by
 * libcpc or dtrace users.
 */
typedef enum hma_cpc_priority {
	HCP_HOST_WINS = 0,
	HCP_GUEST_WINS = 1,
} hma_cpc_priority_t;
static hma_cpc_priority_t hma_cpc_priority = HCP_GUEST_WINS;

/*
 * VMX-specific per-CPU data
 */
typedef struct hma_vmx_cpu {
	void		*hvc_vmxon_page;
	uintptr_t	hvc_vmxon_pa;

} hma_vmx_cpu_t;

/*
 * SVM-specific per-CPU data
 */
typedef struct hma_svm_cpu {
	void		*hsc_hsave_page;
	uintptr_t	hsc_hsave_pa;
	hma_svm_asid_t	hsc_asid;
	uint_t		hsc_gif_disabled;
	/*
	 * hsc_cpc_saved_flags stores the state of guest performance counters
	 * while inside the hma_svm_cpc_enter/hma_svm_cpc_exit critical section.
	 *
	 * If, due to the state of host counters, requested guest counters, and
	 * hma_cpc_priority, the guest counters are _not_ loaded during
	 * hma_svm_cpc_enter(), then this field will hold HCF_DISABLED,
	 * indicating that no state restoration is required during
	 * hma_svm_cpc_exit().
	 *
	 * When hsc_cpc_saved_flags is not HCF_DISABLED, then hsc_cpc_host_regs
	 * will hold the saved host CPC state while the guest state occupies
	 * those registers in the CPU.
	 */
	hma_cpc_flags_t	hsc_cpc_saved_flags;
	hma_cpc_t	hsc_cpc_host_regs[6];
} hma_svm_cpu_t;

/*
 * Combined per-CPU state data
 *
 * The bulk of HMA state (VMX & SVM) is protected by cpu_lock, rather than a
 * mutex specific to the module.  It (cpu_lock) is already required for the
 * state needed to perform setup on all CPUs, so it was a natural fit to
 * protect this data too.
 */
struct hma_cpu {
	union {
		struct hma_vmx_cpu vmx;
		struct hma_svm_cpu svm;
	} hc_u;
	hma_cpu_status_t	hc_status;
	uintptr_t		_hc_padding[6];
} hma_cpu[NCPU];

/* Keep per-CPU state aligned to cache line size to avoid false sharing */
CTASSERT(sizeof (struct hma_cpu) % _CACHE_LINE_SIZE == 0);


static boolean_t hma_vmx_ready = B_FALSE;
static const char *hma_vmx_error = NULL;
static id_space_t *hma_vmx_vpid;

/* HMA-internal tracking of optional VMX capabilities */
typedef enum {
	HVC_EPT		= (1 << 0),
	HVC_VPID	= (1 << 1),
	HVC_INVEPT_ONE	= (1 << 2),
	HVC_INVEPT_ALL	= (1 << 3),
} hma_vmx_capab_t;

static uint32_t hma_vmx_revision;
static hma_vmx_capab_t hma_vmx_capabs = 0;

static boolean_t hma_svm_ready = B_FALSE;
static const char *hma_svm_error = NULL;
static uint32_t hma_svm_features;
static uint32_t hma_svm_max_asid;
static hma_cpc_flags_t hma_svm_cpc_allowed = HCF_DISABLED;

static int hma_vmx_init(void);
static int hma_svm_init(void);

/* Helpers from ml/hma_asm.s */
int hma_vmx_do_invept(int, uintptr_t);
int hma_vmx_vmxon(uintptr_t);

void
hma_init(void)
{
	mutex_init(&hma_lock, NULL, MUTEX_DEFAULT, NULL);
	list_create(&hma_registrations, sizeof (struct hma_reg),
	    offsetof(struct hma_reg, hr_node));

	if (hma_disable != 0) {
		cmn_err(CE_CONT, "?hma_init: disabled");
		return;
	}

	switch (cpuid_getvendor(CPU)) {
	case X86_VENDOR_Intel:
		(void) hma_vmx_init();
		break;
	case X86_VENDOR_AMD:
	case X86_VENDOR_HYGON:
		(void) hma_svm_init();
		break;
	default:
		break;
	}
}

static hma_reg_t *
hma_register_backend(const char *name)
{
	struct hma_reg *reg;
	boolean_t is_ready;

	ASSERT(MUTEX_HELD(&hma_lock));

	switch (cpuid_getvendor(CPU)) {
	case X86_VENDOR_Intel:
		is_ready = hma_vmx_ready;
		break;
	case X86_VENDOR_AMD:
	case X86_VENDOR_HYGON:
		is_ready = hma_svm_ready;
		break;
	default:
		is_ready = B_FALSE;
		break;
	}

	if (!is_ready)
		return (NULL);

	reg = kmem_zalloc(sizeof (*reg), KM_SLEEP);
	reg->hr_name = name;
	list_insert_tail(&hma_registrations, reg);

	return (reg);
}

hma_reg_t *
hma_register(const char *name)
{
	struct hma_reg *reg = NULL;

	VERIFY(name != NULL);

	mutex_enter(&hma_lock);

	if (!hma_exclusive)
		reg = hma_register_backend(name);

	mutex_exit(&hma_lock);

	return (reg);
}

hma_reg_t *
hma_register_exclusive(const char *name)
{
	struct hma_reg *reg = NULL;

	VERIFY(name != NULL);

	mutex_enter(&hma_lock);

	if (list_is_empty(&hma_registrations)) {
		reg = hma_register_backend(name);
		if (reg != NULL)
			hma_exclusive = B_TRUE;
	}

	mutex_exit(&hma_lock);

	return (reg);
}

void
hma_unregister(hma_reg_t *reg)
{
	VERIFY(reg != NULL);
	VERIFY(!list_is_empty(&hma_registrations));

	mutex_enter(&hma_lock);
	list_remove(&hma_registrations, reg);
	if (hma_exclusive && list_is_empty(&hma_registrations))
		hma_exclusive = B_FALSE;
	mutex_exit(&hma_lock);
	kmem_free(reg, sizeof (*reg));
}

static __inline hma_vmx_cpu_t *
hma_vmx_cpu(processorid_t id)
{
	return (&hma_cpu[id].hc_u.vmx);
}

static __inline hma_svm_cpu_t *
hma_svm_cpu(processorid_t id)
{
	return (&hma_cpu[id].hc_u.svm);
}

/*
 * VPID 0 is reserved for instances where VPID is disabled.  Some hypervisors
 * (read: bhyve) reserve lower-order VPIDs for use in fallback behavior if
 * unique VPIDs could not be allocated for all the vCPUs belonging to a VM.
 */
#define	HMA_VPID_RESERVED	NCPU

uint16_t
hma_vmx_vpid_alloc(void)
{
	id_t res;

	/* Do not bother if the CPU lacks support */
	if ((hma_vmx_capabs & HVC_VPID) == 0) {
		return (0);
	}

	res = id_alloc_nosleep(hma_vmx_vpid);
	if (res == -1) {
		return (0);
	} else {
		ASSERT(res > HMA_VPID_RESERVED && res <= UINT16_MAX);
		return (res);
	}
}

void
hma_vmx_vpid_free(uint16_t vpid)
{
	VERIFY(vpid > HMA_VPID_RESERVED);
	id_free(hma_vmx_vpid, (id_t)vpid);
}

#define	INVEPT_SINGLE_CONTEXT	1
#define	INVEPT_ALL_CONTEXTS	2

static int
hma_vmx_invept_xcall(xc_arg_t arg1, xc_arg_t arg2, xc_arg_t arg3 __unused)
{
	int flag = (int)arg1;
	uintptr_t eptp = (uintptr_t)arg2;

	ASSERT(flag == INVEPT_SINGLE_CONTEXT || flag == INVEPT_ALL_CONTEXTS);

	VERIFY0(hma_vmx_do_invept(flag, eptp));
	return (0);
}

void
hma_vmx_invept_allcpus(uintptr_t eptp)
{
	int flag = -1;
	cpuset_t set;

	if ((hma_vmx_capabs & HVC_INVEPT_ONE) != 0) {
		flag = INVEPT_SINGLE_CONTEXT;
	} else if ((hma_vmx_capabs & HVC_INVEPT_ALL) != 0) {
		flag = INVEPT_ALL_CONTEXTS;
		eptp = 0;
	} else {
		return;
	}

	cpuset_zero(&set);
	mutex_enter(&cpu_lock);

	cpuset_or(&set, &cpu_active_set);
	xc_call((xc_arg_t)flag, (xc_arg_t)eptp, 0, CPUSET2BV(set),
	    hma_vmx_invept_xcall);

	mutex_exit(&cpu_lock);
}

static int
hma_vmx_cpu_vmxon(xc_arg_t arg1 __unused, xc_arg_t arg2 __unused,
    xc_arg_t arg3 __unused)
{
	uint64_t fctrl;
	const processorid_t id = CPU->cpu_seqid;
	hma_vmx_cpu_t *vmx_cpu = hma_vmx_cpu(id);

	VERIFY(vmx_cpu->hvc_vmxon_page != NULL);
	VERIFY(vmx_cpu->hvc_vmxon_pa != 0);

	/*
	 * Ensure that the VMX support and lock bits are enabled in the
	 * feature-control MSR.
	 */
	fctrl = rdmsr(MSR_IA32_FEAT_CTRL);
	if ((fctrl & IA32_FEAT_CTRL_LOCK) == 0 ||
	    (fctrl & IA32_FEAT_CTRL_VMX_EN) == 0) {
		fctrl = fctrl | IA32_FEAT_CTRL_VMX_EN | IA32_FEAT_CTRL_LOCK;
		wrmsr(MSR_IA32_FEAT_CTRL, fctrl);
	}

	setcr4(getcr4() | CR4_VMXE);

	if (hma_vmx_vmxon(vmx_cpu->hvc_vmxon_pa) == 0) {
		hma_cpu[id].hc_status = HCS_READY;
	} else {
		hma_cpu[id].hc_status = HCS_ERROR;

		/*
		 * If VMX has already been marked active and available for the
		 * system, then failure to perform VMXON on a newly-onlined CPU
		 * represents a fatal problem.  Continuing on would mean
		 * failure for any hypervisor thread which landed here.
		 */
		if (hma_vmx_ready) {
			panic("VMXON failure after VMX marked ready");
		}
	}
	return (0);
}

static int
hma_vmx_cpu_setup(cpu_setup_t what, int id, void *arg __unused)
{
	hma_vmx_cpu_t *vmx_cpu = hma_vmx_cpu(id);

	ASSERT(MUTEX_HELD(&cpu_lock));
	ASSERT(id >= 0 && id < NCPU);

	if (what != CPU_ON) {
		/*
		 * For the purposes of VMX setup, only the CPU_ON event is of
		 * interest.  Letting VMX state linger on an offline CPU should
		 * not cause any harm.
		 *
		 * This logic assumes that any offlining activity is strictly
		 * administrative in nature and will not alter any existing
		 * configuration (such as %cr4 bits previously set).
		 */
		return (0);
	}

	const hma_cpu_status_t status = hma_cpu[id].hc_status;
	if (status == HCS_ERROR) {
		return (-1);
	}

	/* Allocate the VMXON page for this CPU, if not already done */
	if (vmx_cpu->hvc_vmxon_page == NULL) {
		caddr_t va;
		pfn_t pfn;

		va = kmem_alloc(PAGESIZE, KM_SLEEP);
		VERIFY0((uintptr_t)va & PAGEOFFSET);
		vmx_cpu->hvc_vmxon_page = va;

		/* Initialize the VMX revision field as expected */
		bcopy(&hma_vmx_revision, va, sizeof (hma_vmx_revision));

		/*
		 * Cache the physical address of the VMXON page rather than
		 * looking it up later when the potential blocking of
		 * hat_getpfnum would be less acceptable.
		 */
		pfn = hat_getpfnum(kas.a_hat, va);
		vmx_cpu->hvc_vmxon_pa = (pfn << PAGESHIFT);
	} else {
		VERIFY(vmx_cpu->hvc_vmxon_pa != 0);
	}

	if (status == HCS_UNINITIALIZED) {
		cpuset_t set;

		/* Activate VMX on this CPU */
		cpuset_zero(&set);
		cpuset_add(&set, id);
		xc_call(0, 0, 0, CPUSET2BV(set), hma_vmx_cpu_vmxon);
	} else {
		VERIFY3U(status, ==, HCS_READY);

		/*
		 * If an already-initialized CPU is going back online, perform
		 * an all-contexts invept to eliminate the possibility of
		 * cached EPT state causing issues.
		 */
		if ((hma_vmx_capabs & HVC_INVEPT_ALL) != 0) {
			cpuset_t set;

			cpuset_zero(&set);
			cpuset_add(&set, id);
			xc_call((xc_arg_t)INVEPT_ALL_CONTEXTS, 0, 0,
			    CPUSET2BV(set), hma_vmx_invept_xcall);
		}
	}

	return (hma_cpu[id].hc_status != HCS_READY);
}

/*
 * Determining the availability of VM execution controls is somewhat different
 * from conventional means, where one simply checks for asserted bits in the
 * MSR value.  Instead, these execution control MSRs are split into two halves:
 * the lower 32-bits indicating capabilities which can be zeroed in the VMCS
 * field and the upper 32-bits indicating capabilities which can be set to one.
 *
 * It is described in detail in Appendix A.3 of SDM volume 3.
 */
#define	VMX_CTL_ONE_SETTING(val, flag)	\
	(((val) & ((uint64_t)(flag) << 32)) != 0)

static const char *
hma_vmx_query_details(void)
{
	boolean_t query_true_ctl = B_FALSE;
	uint64_t msr;

	/* The basic INS/OUTS functionality is cited as a necessary prereq */
	msr = rdmsr(MSR_IA32_VMX_BASIC);
	if ((msr & IA32_VMX_BASIC_INS_OUTS) == 0) {
		return ("VMX does not support INS/OUTS");
	}

	/* Record the VMX revision for later VMXON usage */
	hma_vmx_revision = (uint32_t)msr;

	/*
	 * Bit 55 in the VMX_BASIC MSR determines how VMX control information
	 * can be queried.
	 */
	query_true_ctl = (msr & IA32_VMX_BASIC_TRUE_CTRLS) != 0;

	/* Check for EPT and VPID support */
	msr = rdmsr(query_true_ctl ?
	    MSR_IA32_VMX_TRUE_PROCBASED_CTLS : MSR_IA32_VMX_PROCBASED_CTLS);
	if (VMX_CTL_ONE_SETTING(msr, IA32_VMX_PROCBASED_2ND_CTLS)) {
		msr = rdmsr(MSR_IA32_VMX_PROCBASED2_CTLS);
		if (VMX_CTL_ONE_SETTING(msr, IA32_VMX_PROCBASED2_EPT)) {
			hma_vmx_capabs |= HVC_EPT;
		}
		if (VMX_CTL_ONE_SETTING(msr, IA32_VMX_PROCBASED2_VPID)) {
			hma_vmx_capabs |= HVC_VPID;
		}
	}

	/* Check for INVEPT support */
	if ((hma_vmx_capabs & HVC_EPT) != 0) {
		msr = rdmsr(MSR_IA32_VMX_EPT_VPID_CAP);
		if ((msr & IA32_VMX_EPT_VPID_INVEPT) != 0) {
			if ((msr & IA32_VMX_EPT_VPID_INVEPT_SINGLE) != 0) {
				hma_vmx_capabs |= HVC_INVEPT_ONE;
			}
			if ((msr & IA32_VMX_EPT_VPID_INVEPT_ALL) != 0) {
				hma_vmx_capabs |= HVC_INVEPT_ALL;
			}
		}
	}

	return (NULL);
}

static int
hma_vmx_init(void)
{
	cpu_t *cp;
	uint64_t msr;
	int err = 0;
	const char *msg = NULL;

	if (!is_x86_feature(x86_featureset, X86FSET_VMX)) {
		msg = "CPU does not support VMX";
		goto bail;
	}

	/* Has the BIOS set the feature-control lock bit without VMX enabled? */
	msr = rdmsr(MSR_IA32_FEAT_CTRL);
	if ((msr & IA32_FEAT_CTRL_LOCK) != 0 &&
	    (msr & IA32_FEAT_CTRL_VMX_EN) == 0) {
		msg = "VMX support disabled by BIOS";
		goto bail;
	}

	msg = hma_vmx_query_details();
	if (msg != NULL) {
		goto bail;
	}

	mutex_enter(&cpu_lock);
	/* Perform VMX configuration for already-online CPUs. */
	cp = cpu_active;
	do {
		err = hma_vmx_cpu_setup(CPU_ON, cp->cpu_seqid, NULL);
		if (err != 0) {
			msg = "failure during VMXON setup";
			mutex_exit(&cpu_lock);
			goto bail;
		}
	} while ((cp = cp->cpu_next_onln) != cpu_active);

	/*
	 * Register callback for later-onlined CPUs and perform other remaining
	 * resource allocation.
	 */
	register_cpu_setup_func(hma_vmx_cpu_setup, NULL);
	mutex_exit(&cpu_lock);

	hma_vmx_vpid = id_space_create("hma_vmx_vpid", HMA_VPID_RESERVED + 1,
	    UINT16_MAX);
	hma_vmx_ready = B_TRUE;

	return (0);

bail:
	hma_vmx_error = msg;
	cmn_err(CE_NOTE, "!hma_vmx_init: %s", msg);
	return (-1);
}

#define	VMCB_FLUSH_NOTHING	0x0
#define	VMCB_FLUSH_ALL		0x1
#define	VMCB_FLUSH_ASID		0x3

void
hma_svm_asid_init(hma_svm_asid_t *vcp)
{
	/*
	 * Initialize the generation to 0, forcing an ASID allocation on first
	 * entry.  Leave the ASID at 0, so if the host forgoes the call to
	 * hma_svm_asid_update(), SVM will bail on the invalid vcpu state.
	 */
	vcp->hsa_gen = 0;
	vcp->hsa_asid = 0;
}

uint8_t
hma_svm_asid_update(hma_svm_asid_t *vcp, boolean_t flush_by_asid,
    boolean_t npt_flush)
{
	/*
	 * Most ASID resource updates are expected to be performed as part of
	 * VMM entry into guest context, where interrupts would be disabled for
	 * the sake of state consistency.
	 *
	 * We demand this be the case, even though other situations which might
	 * incur an ASID update, such as userspace manipulation of guest vCPU
	 * state, may not require such consistency.
	 */
	ASSERT(!interrupts_enabled());

	/*
	 * If NPT changes dictate a TLB flush and by-ASID flushing is not
	 * supported/used, force a fresh ASID allocation.
	 */
	if (npt_flush && !flush_by_asid) {
		vcp->hsa_gen = 0;
	}

	hma_svm_asid_t *hcp = &(hma_svm_cpu(CPU->cpu_seqid)->hsc_asid);
	if (vcp->hsa_gen != hcp->hsa_gen) {
		hcp->hsa_asid++;

		if (hcp->hsa_asid >= hma_svm_max_asid) {
			/* Keep the ASID properly constrained */
			hcp->hsa_asid = 1;
			hcp->hsa_gen++;
			if (hcp->hsa_gen == 0) {
				/*
				 * Stay clear of the '0' sentinel value for
				 * generation, if wrapping around.
				 */
				hcp->hsa_gen = 1;
			}
		}
		vcp->hsa_gen = hcp->hsa_gen;
		vcp->hsa_asid = hcp->hsa_asid;

		ASSERT(vcp->hsa_asid != 0);
		ASSERT3U(vcp->hsa_asid, <, hma_svm_max_asid);

		if (flush_by_asid) {
			return (VMCB_FLUSH_ASID);
		} else {
			return (VMCB_FLUSH_ALL);
		}
	} else if (npt_flush) {
		ASSERT(flush_by_asid);
		return (VMCB_FLUSH_ASID);
	}

	return (VMCB_FLUSH_NOTHING);
}

void
hma_svm_gif_disable(void)
{
	/*
	 * Clear the GIF (masking interrupts) first, so the subsequent
	 * housekeeping can be done under its protection.
	 */
	__asm__ __volatile__("clgi");

	hma_svm_cpu_t *svm_cpu = hma_svm_cpu(CPU->cpu_seqid);
	const uint_t old_gif = atomic_swap_uint(&svm_cpu->hsc_gif_disabled, 1);

	if (old_gif != 0) {
		panic("GIF disable is set when expected to be clear");
	}
}

void
hma_svm_gif_enable(void)
{
	hma_svm_cpu_t *svm_cpu = hma_svm_cpu(CPU->cpu_seqid);
	const uint_t old_gif = atomic_swap_uint(&svm_cpu->hsc_gif_disabled, 0);

	if (old_gif == 0) {
		panic("GIF disable is clear when expected to be set");
	}

	/*
	 * Set the GIF last (un-masking interrupts) last, so the housekeeping
	 * will have been completed under its protection.
	 */
	__asm__ __volatile__("stgi");
}

boolean_t
hma_svm_gif_is_disabled(void)
{
	hma_svm_cpu_t *svm_cpu = hma_svm_cpu(CPU->cpu_seqid);

	/*
	 * At the time of this writing, there exists no mechanism by which the
	 * state of the GIF on a CPU can be directly queried.  Rather than
	 * attempting an indirect means of checking its state, we track it
	 * manually through the HMA disable/enable functions.
	 */
	return (svm_cpu->hsc_gif_disabled != 0);
}

#define	EVTSEL_EN(evt) (((evt) & AMD_PERF_EVTSEL_CTR_EN) != 0)
#define	CPC_BASE_REGS	4
#define	CPC_EXTD_REGS	6
#define	MSR_CPC_EXTD_EVTSEL(idx)	(MSR_AMD_F15H_PERF_EVTSEL0 + (idx * 2))
#define	MSR_CPC_EXTD_CTR(idx)		(MSR_AMD_F15H_PERF_CTR0 + (idx * 2))

/*
 * AMD CPU Performance Counter Support
 *
 * This provides a means of safely saving/loading host CPC state, along with
 * loading/saving guest CPC state upon guest entry/exit (respectively).
 * Currently, this only supports the 6 "extended" performance counters
 * (in MSRs C0010200h - C001020bh).  It pays no head to any other CPC state such
 * as the Northbridge counters or PerfMonV2 registers.
 */

hma_svm_cpc_res_t
hma_svm_cpc_enter(struct hma_svm_cpc_state *cpc_state)
{
	hma_svm_cpu_t *svm_cpu = hma_svm_cpu(CPU->cpu_seqid);

	ASSERT(!interrupts_enabled());

	svm_cpu->hsc_cpc_saved_flags = HCF_DISABLED;

	const hma_cpc_flags_t req_flags =
	    cpc_state->hscs_flags & hma_svm_cpc_allowed;
	if (req_flags == HCF_DISABLED) {
		return (HSCR_EMPTY);
	}

	/* Extended regs should not be enabled without base */
	IMPLY((req_flags & HCF_EN_EXTD) != 0, (req_flags & HCF_EN_BASE) != 0);

	const uint_t max_guest_reg =
	    (req_flags & HCF_EN_EXTD) != 0 ? CPC_EXTD_REGS : CPC_BASE_REGS;
	uint_t guest_active = 0;
	for (uint_t i = 0; i < max_guest_reg; i++) {
		if (EVTSEL_EN(cpc_state->hscs_regs[i].hc_evtsel)) {
			guest_active++;
		}
	}

	/*
	 * Guest is not currently measuring with any of the CPCs, so leave any
	 * host counters in place.
	 */
	if (guest_active == 0) {
		return (HSCR_EMPTY);
	}

	/*
	 * Read (and save) the host evtsel values, counting the number of
	 * registers in active use
	 */
	uint_t host_active = 0;
	for (uint_t i = 0; i < CPC_EXTD_REGS; i++) {
		const uint64_t evtsel = rdmsr(MSR_CPC_EXTD_EVTSEL(i));

		svm_cpu->hsc_cpc_host_regs[i].hc_evtsel = evtsel;
		if (EVTSEL_EN(evtsel)) {
			host_active++;
		}
	}

	if (host_active != 0) {
		if (hma_cpc_priority == HCP_HOST_WINS) {
			/*
			 * Host has priority access to the perf counters over
			 * the guest, so just leave everything in place.
			 */
			DTRACE_PROBE2(hma_svm__guest_deferred,
			    processorid_t, CPU->cpu_seqid,
			    uint_t, guest_active);
			return (HSCR_EMPTY);
		}

		DTRACE_PROBE2(hma_svm__host_deferred,
		    processorid_t, CPU->cpu_seqid, uint_t, host_active);

		/*
		 * Disable any active host counters, trying to do so in as
		 * consistent a manner as possible.
		 */
		for (uint_t i = 0; i < CPC_EXTD_REGS; i++) {
			const uint64_t evtsel =
			    svm_cpu->hsc_cpc_host_regs[i].hc_evtsel;
			wrmsr(MSR_CPC_EXTD_EVTSEL(i),
			    evtsel & ~AMD_PERF_EVTSEL_CTR_EN);
		}
	}

	/*
	 * With any active host counters stopped from collecting new events,
	 * save the counter values themselves before loading guest state.
	 */
	for (uint_t i = 0; i < CPC_EXTD_REGS; i++) {
		svm_cpu->hsc_cpc_host_regs[i].hc_ctr =
		    rdmsr(MSR_CPC_EXTD_CTR(i));
	}

	/*
	 * Now load the guest state, fixing it up with the flag necessary to
	 * collect events only while in guest context.
	 */
	for (uint_t i = 0; i < max_guest_reg; i++) {
		uint64_t evtsel = cpc_state->hscs_regs[i].hc_evtsel;

		/*
		 * Clear any existing HG flags, as well as any request for
		 * interrupt enable. (Trapping the interrupt from guest counters
		 * is not presently supported.)
		 */
		evtsel &= ~(AMD_PERF_EVTSEL_HG_MASK | AMD_PERF_EVTSEL_INT_EN);
		/* And indicate guest-only event tracking */
		evtsel |= AMD_PERF_EVTSEL_HG_GUEST;

		wrmsr(MSR_CPC_EXTD_EVTSEL(i), evtsel);
		wrmsr(MSR_CPC_EXTD_CTR(i), cpc_state->hscs_regs[i].hc_ctr);
	}

	svm_cpu->hsc_cpc_saved_flags = req_flags;
	return (HSCR_ACCESS_RDPMC | HSCR_ACCESS_CTR_MSR);
}

void
hma_svm_cpc_exit(struct hma_svm_cpc_state *cpc_state)
{
	ASSERT(!interrupts_enabled());

	hma_svm_cpu_t *svm_cpu = hma_svm_cpu(CPU->cpu_seqid);

	const hma_cpc_flags_t saved_flags = svm_cpu->hsc_cpc_saved_flags;
	if (saved_flags == HCF_DISABLED) {
		return;
	}

	/* Save the guest counter values. */
	const uint_t max_guest_reg =
	    (saved_flags & HCF_EN_EXTD) != 0 ? CPC_EXTD_REGS : CPC_BASE_REGS;
	for (uint_t i = 0; i < max_guest_reg; i++) {
		cpc_state->hscs_regs[i].hc_ctr = rdmsr(MSR_CPC_EXTD_CTR(i));
	}

	/*
	 * Load the host values back, once again taking care to toggle the
	 * counter enable state as a separate step in an attempt to keep
	 * readings as consistent as possible
	 */
	uint_t host_active = 0;
	for (uint_t i = 0; i < CPC_EXTD_REGS; i++) {
		const uint64_t evtsel = svm_cpu->hsc_cpc_host_regs[i].hc_evtsel;

		if (EVTSEL_EN(evtsel)) {
			host_active++;
		}
		wrmsr(MSR_CPC_EXTD_EVTSEL(i), evtsel & ~AMD_PERF_EVTSEL_CTR_EN);
		wrmsr(MSR_CPC_EXTD_CTR(i),
		    svm_cpu->hsc_cpc_host_regs[i].hc_ctr);
	}

	/*
	 * Allow any enabled host counters to collect events, now that all of
	 * the other state is loaded.
	 */
	if (host_active != 0) {
		for (uint_t i = 0; i < CPC_EXTD_REGS; i++) {
			wrmsr(MSR_CPC_EXTD_EVTSEL(i),
			    svm_cpu->hsc_cpc_host_regs[i].hc_evtsel);
		}
	}
}

static int
hma_svm_cpu_activate(xc_arg_t arg1 __unused, xc_arg_t arg2 __unused,
    xc_arg_t arg3 __unused)
{
	const processorid_t id = CPU->cpu_seqid;
	const uintptr_t hsave_pa = hma_svm_cpu(id)->hsc_hsave_pa;
	uint64_t efer;

	VERIFY(hsave_pa != 0);

	/* Enable SVM via EFER */
	efer = rdmsr(MSR_AMD_EFER);
	efer |= AMD_EFER_SVME;
	wrmsr(MSR_AMD_EFER, efer);

	/* Setup hsave area */
	wrmsr(MSR_AMD_VM_HSAVE_PA, hsave_pa);

	hma_cpu[id].hc_status = HCS_READY;
	return (0);
}

static int
hma_svm_cpu_setup(cpu_setup_t what, int id, void *arg __unused)
{
	hma_svm_cpu_t *svm_cpu = hma_svm_cpu(id);

	ASSERT(MUTEX_HELD(&cpu_lock));
	ASSERT(id >= 0 && id < NCPU);

	switch (what) {
	case CPU_CONFIG:
	case CPU_ON:
	case CPU_INIT:
		break;
	default:
		/*
		 * Other events, such as CPU offlining, are of no interest.
		 * Letting the SVM state linger should not cause any harm.
		 *
		 * This logic assumes that any offlining activity is strictly
		 * administrative in nature and will not alter any existing
		 * configuration (such as EFER bits previously set).
		 */
		return (0);
	}

	/* Perform initialization if it has not been previously attempted. */
	if (hma_cpu[id].hc_status != HCS_UNINITIALIZED) {
		return ((hma_cpu[id].hc_status == HCS_READY) ? 0 : -1);
	}

	/* Allocate the hsave page for this CPU */
	if (svm_cpu->hsc_hsave_page == NULL) {
		caddr_t va;
		pfn_t pfn;

		va = kmem_alloc(PAGESIZE, KM_SLEEP);
		VERIFY0((uintptr_t)va & PAGEOFFSET);
		svm_cpu->hsc_hsave_page = va;

		/*
		 * Cache the physical address of the hsave page rather than
		 * looking it up later when the potential blocking of
		 * hat_getpfnum would be less acceptable.
		 */
		pfn = hat_getpfnum(kas.a_hat, va);
		svm_cpu->hsc_hsave_pa = (pfn << PAGESHIFT);
	} else {
		VERIFY(svm_cpu->hsc_hsave_pa != 0);
	}

	kpreempt_disable();
	if (CPU->cpu_seqid == id) {
		/* Perform svm setup directly if this CPU is the target */
		(void) hma_svm_cpu_activate(0, 0, 0);
		kpreempt_enable();
	} else {
		cpuset_t set;

		/* Use a cross-call if a remote CPU is the target */
		kpreempt_enable();
		cpuset_zero(&set);
		cpuset_add(&set, id);
		xc_call(0, 0, 0, CPUSET2BV(set), hma_svm_cpu_activate);
	}

	return (hma_cpu[id].hc_status != HCS_READY);
}

static int
hma_svm_init(void)
{
	uint64_t msr;
	const char *msg = NULL;
	struct cpuid_regs regs;
	cpu_t *cp;

	if (!is_x86_feature(x86_featureset, X86FSET_SVM)) {
		msg = "CPU does not support SVM";
		goto bail;
	}

	msr = rdmsr(MSR_AMD_VM_CR);
	if ((msr & AMD_VM_CR_SVMDIS) != 0) {
		msg = "SVM disabled by BIOS";
		goto bail;
	}

	regs.cp_eax = 0x8000000a;
	(void) cpuid_insn(NULL, &regs);
	const uint32_t nasid = regs.cp_ebx;
	const uint32_t feat = regs.cp_edx;

	if (nasid == 0) {
		msg = "Not enough ASIDs for guests";
		goto bail;
	}
	if ((feat & CPUID_AMD_EDX_NESTED_PAGING) == 0) {
		msg = "CPU does not support nested paging";
		goto bail;
	}
	if ((feat & CPUID_AMD_EDX_NRIPS) == 0) {
		msg = "CPU does not support NRIP save";
		goto bail;
	}

	hma_svm_features = feat;
	hma_svm_max_asid = nasid;

	mutex_enter(&cpu_lock);
	/* Perform SVM configuration for already-online CPUs. */
	cp = cpu_active;
	do {
		int err = hma_svm_cpu_setup(CPU_ON, cp->cpu_seqid, NULL);
		if (err != 0) {
			msg = "failure during SVM setup";
			mutex_exit(&cpu_lock);
			goto bail;
		}
	} while ((cp = cp->cpu_next_onln) != cpu_active);

	/*
	 * Register callback for later-onlined CPUs and perform other remaining
	 * resource allocation.
	 */
	register_cpu_setup_func(hma_svm_cpu_setup, NULL);
	mutex_exit(&cpu_lock);

	/* Initialize per-CPU ASID state. */
	for (uint_t i = 0; i < NCPU; i++) {
		/*
		 * Skip past sentinel 0 value for generation.  Doing so for
		 * ASID is unneeded, since it will be incremented during the
		 * first allocation.
		 */
		hma_svm_asid_t *cpu_asid = &hma_svm_cpu(i)->hsc_asid;
		cpu_asid->hsa_gen = 1;
		cpu_asid->hsa_asid = 0;
	}

	/*
	 * For now, only expose performance counter support if the host supports
	 * "extended" counters.  This makes MSR access more consistent for logic
	 * handling that state.
	 */
	if (is_x86_feature(x86_featureset, X86FSET_AMD_PCEC)) {
		hma_svm_cpc_allowed = HCF_EN_BASE | HCF_EN_EXTD;
	}

	hma_svm_ready = B_TRUE;
	return (0);

bail:
	hma_svm_error = msg;
	cmn_err(CE_NOTE, "!hma_svm_init: %s", msg);
	return (-1);
}
/*
 * This file and its contents are supplied under the terms of the
 * Common Development and Distribution License ("CDDL"), version 1.0.
 * You may only use this file in accordance with the terms of version
 * 1.0 of the CDDL.
 *
 * A full copy of the text of the CDDL should have accompanied this
 * source.  A copy of the CDDL is also available via the Internet at
 * http://www.illumos.org/license/CDDL.
 */

/*
 * Copyright (c) 2018, Joyent, Inc.
 * Copyright 2022 Oxide Computer Company
 */

/*
 * This implements the hypervisor multiplexor FPU API. Its purpose is to make it
 * easy to switch between the host and guest hypervisor while hiding all the
 * details about CR0.TS and how to save the host's state as required.
 */

#include <sys/pcb.h>
#include <sys/kmem.h>
#include <sys/debug.h>
#include <sys/cmn_err.h>
#include <sys/ddi.h>
#include <sys/sunddi.h>
#include <sys/hma.h>
#include <sys/x86_archext.h>
#include <sys/archsystm.h>
#include <sys/controlregs.h>
#include <sys/sysmacros.h>
#include <sys/stdbool.h>
#include <sys/ontrap.h>
#include <sys/cpuvar.h>
#include <sys/disp.h>

struct hma_fpu {
	fpu_ctx_t	hf_guest_fpu;
	kthread_t	*hf_curthread;
	boolean_t	hf_inguest;
};

int
hma_fpu_init(hma_fpu_t *fpu)
{
	struct xsave_state *xs;

	ASSERT0(fpu->hf_inguest);

	switch (fp_save_mech) {
	case FP_FXSAVE:
		bcopy(&sse_initial, fpu->hf_guest_fpu.fpu_regs.kfpu_u.kfpu_fx,
		    sizeof (struct fxsave_state));
		fpu->hf_guest_fpu.fpu_xsave_mask = 0;
		break;
	case FP_XSAVE:
		/*
		 * Zero everything in the xsave case as we may have data in
		 * the structure that's not part of the initial value (which
		 * only really deals with a small portion of the xsave state).
		 */
		xs = fpu->hf_guest_fpu.fpu_regs.kfpu_u.kfpu_xs;
		bzero(xs, cpuid_get_xsave_size());
		bcopy(&avx_initial, xs, sizeof (*xs));
		xs->xs_header.xsh_xstate_bv = XFEATURE_LEGACY_FP | XFEATURE_SSE;
		fpu->hf_guest_fpu.fpu_xsave_mask = XFEATURE_FP_ALL;
		break;
	default:
		panic("Invalid fp_save_mech");
	}

	fpu->hf_guest_fpu.fpu_flags = FPU_EN | FPU_VALID;

	return (0);
}

void
hma_fpu_free(hma_fpu_t *fpu)
{
	if (fpu == NULL)
		return;

	ASSERT3P(fpu->hf_guest_fpu.fpu_regs.kfpu_u.kfpu_generic, !=, NULL);
	kmem_cache_free(fpsave_cachep,
	    fpu->hf_guest_fpu.fpu_regs.kfpu_u.kfpu_generic);
	kmem_free(fpu, sizeof (*fpu));
}

hma_fpu_t *
hma_fpu_alloc(int kmflag)
{
	hma_fpu_t *fpu;

	fpu = kmem_zalloc(sizeof (hma_fpu_t), kmflag);
	if (fpu == NULL)
		return (NULL);

	fpu->hf_guest_fpu.fpu_regs.kfpu_u.kfpu_generic =
	    kmem_cache_alloc(fpsave_cachep, kmflag);
	if (fpu->hf_guest_fpu.fpu_regs.kfpu_u.kfpu_generic == NULL) {
		kmem_free(fpu, sizeof (hma_fpu_t));
		return (NULL);
	}
	fpu->hf_inguest = B_FALSE;

	/*
	 * Make sure the entire structure is zero.
	 */
	switch (fp_save_mech) {
	case FP_FXSAVE:
		bzero(fpu->hf_guest_fpu.fpu_regs.kfpu_u.kfpu_generic,
		    sizeof (struct fxsave_state));
		break;
	case FP_XSAVE:
		bzero(fpu->hf_guest_fpu.fpu_regs.kfpu_u.kfpu_generic,
		    cpuid_get_xsave_size());
		break;
	default:
		panic("Invalid fp_save_mech");
	}

	return (fpu);
}

void
hma_fpu_start_guest(hma_fpu_t *fpu)
{
	/*
	 * Note, we don't check / assert whether or not t_prempt is true because
	 * there are contexts where this is safe to call (from a context op)
	 * where t_preempt may not be set.
	 */
	ASSERT3S(fpu->hf_inguest, ==, B_FALSE);
	ASSERT3P(fpu->hf_curthread, ==, NULL);
	ASSERT3P(curthread->t_lwp, !=, NULL);
	ASSERT3U(fpu->hf_guest_fpu.fpu_flags & FPU_EN, !=, 0);
	ASSERT3U(fpu->hf_guest_fpu.fpu_flags & FPU_VALID, !=, 0);

	fpu->hf_inguest = B_TRUE;
	fpu->hf_curthread = curthread;


	fp_save(&curthread->t_lwp->lwp_pcb.pcb_fpu);
	fp_restore(&fpu->hf_guest_fpu);
	fpu->hf_guest_fpu.fpu_flags &= ~FPU_VALID;
}

/*
 * Since fp_save() assumes a thread-centric view of the FPU usage -- it will
 * assert if attempting to save elsewhere than the thread PCB, and will elide
 * action if the FPU is not enabled -- we cannot use it for the manual saving of
 * FPU contents.  To work around that, we call the save mechanism directly.
 */
static void
do_fp_save(fpu_ctx_t *fpu)
{
	/*
	 * For our manual saving, we expect that the thread PCB never be the
	 * landing zone for the data.
	 */
	ASSERT(curthread->t_lwp == NULL ||
	    fpu != &curthread->t_lwp->lwp_pcb.pcb_fpu);

	switch (fp_save_mech) {
	case FP_FXSAVE:
		fpxsave(fpu->fpu_regs.kfpu_u.kfpu_fx);
		break;
	case FP_XSAVE:
		xsavep(fpu->fpu_regs.kfpu_u.kfpu_xs, fpu->fpu_xsave_mask);
		break;
	default:
		panic("Invalid fp_save_mech");
	}
	fpu->fpu_flags |= FPU_VALID;
}


void
hma_fpu_stop_guest(hma_fpu_t *fpu)
{
	ASSERT3S(fpu->hf_inguest, ==, B_TRUE);
	ASSERT3P(fpu->hf_curthread, ==, curthread);
	ASSERT3U(fpu->hf_guest_fpu.fpu_flags & FPU_EN, !=, 0);
	ASSERT3U(fpu->hf_guest_fpu.fpu_flags & FPU_VALID, ==, 0);

	do_fp_save(&fpu->hf_guest_fpu);

	fp_restore(&curthread->t_lwp->lwp_pcb.pcb_fpu);

	fpu->hf_inguest = B_FALSE;
	fpu->hf_curthread = NULL;
}

/*
 * Will output up to `ndesc` records into `descp`.  The required size for an
 * XSAVE area containing all of the data fields supported by the host will be
 * placed in `req_sizep` (if non-NULL).  Returns the number of feature bits
 * supported by the host.
 */
uint_t
hma_fpu_describe_xsave_state(hma_xsave_state_desc_t *descp, uint_t ndesc,
    size_t *req_sizep)
{
	uint64_t features;

	switch (fp_save_mech) {
	case FP_FXSAVE:
		/*
		 * Even without xsave support, the FPU will have legacy x87
		 * float and SSE state contained within.
		 */
		features = XFEATURE_LEGACY_FP | XFEATURE_SSE;
		break;
	case FP_XSAVE:
		features = get_xcr(XFEATURE_ENABLED_MASK);
		break;
	default:
		panic("Invalid fp_save_mech");
	}

	uint_t count, pos;
	uint_t max_size = MIN_XSAVE_SIZE;
	for (count = 0, pos = 0; pos <= 63; pos++) {
		const uint64_t bit = (1 << pos);
		uint32_t size, off;

		if ((features & bit) == 0) {
			continue;
		}

		if (bit == XFEATURE_LEGACY_FP || bit == XFEATURE_SSE) {
			size = sizeof (struct fxsave_state);
			off = 0;
		} else {
			/*
			 * Size and position of data types within the XSAVE area
			 * is described in leaf 0xD in the subfunction
			 * corresponding to the bit position (for pos > 1).
			 */
			struct cpuid_regs regs = {
				.cp_eax = 0xD,
				.cp_ecx = pos,
			};

			ASSERT3U(pos, >, 1);

			(void) __cpuid_insn(&regs);
			size = regs.cp_eax;
			off = regs.cp_ebx;
		}
		max_size = MAX(max_size, off + size);

		if (count < ndesc) {
			hma_xsave_state_desc_t *desc = &descp[count];

			desc->hxsd_bit = bit;
			desc->hxsd_size = size;
			desc->hxsd_off = off;
		}
		count++;
	}
	if (req_sizep != NULL) {
		*req_sizep = max_size;
	}
	return (count);
}

hma_fpu_xsave_result_t
hma_fpu_get_xsave_state(const hma_fpu_t *fpu, void *buf, size_t len)
{
	ASSERT(!fpu->hf_inguest);

	size_t valid_len;
	switch (fp_save_mech) {
	case FP_FXSAVE: {
		if (len < MIN_XSAVE_SIZE) {
			return (HFXR_NO_SPACE);
		}
		bcopy(fpu->hf_guest_fpu.fpu_regs.kfpu_u.kfpu_generic, buf,
		    sizeof (struct fxsave_state));

		struct xsave_header hdr = {
			.xsh_xstate_bv = XFEATURE_LEGACY_FP | XFEATURE_SSE,
		};
		bcopy(&hdr, buf + sizeof (struct fxsave_state), sizeof (hdr));

		break;
	}
	case FP_XSAVE:
		(void) hma_fpu_describe_xsave_state(NULL,  0, &valid_len);
		if (len < valid_len) {
			return (HFXR_NO_SPACE);
		}
		bcopy(fpu->hf_guest_fpu.fpu_regs.kfpu_u.kfpu_generic, buf,
		    valid_len);
		break;
	default:
		panic("Invalid fp_save_mech");
	}

	return (HFXR_OK);
}

hma_fpu_xsave_result_t
hma_fpu_set_xsave_state(hma_fpu_t *fpu, void *buf, size_t len)
{
	ASSERT(!fpu->hf_inguest);

	if (len < MIN_XSAVE_SIZE) {
		return (HFXR_NO_SPACE);
	}
	/* 64-byte alignment is demanded of the FPU-related operations */
	if (((uintptr_t)buf & 63) != 0) {
		return (HFXR_BAD_ALIGN);
	}

	struct xsave_header *hdr = buf + sizeof (struct fxsave_state);
	if (hdr->xsh_xcomp_bv != 0) {
		/* XSAVEC formatting not supported at this time */
		return (HFXR_UNSUP_FMT);
	}

	uint64_t allowed_bits;
	size_t save_area_size;
	switch (fp_save_mech) {
	case FP_FXSAVE:
		allowed_bits = XFEATURE_LEGACY_FP | XFEATURE_SSE;
		save_area_size = sizeof (struct fxsave_state);
		break;
	case FP_XSAVE:
		allowed_bits = get_xcr(XFEATURE_ENABLED_MASK);
		save_area_size = cpuid_get_xsave_size();
		break;
	default:
		panic("Invalid fp_save_mech");
	}
	if ((hdr->xsh_xstate_bv & ~(allowed_bits)) != 0) {
		return (HFXR_UNSUP_FEAT);
	}

	/*
	 * We validate the incoming state with the FPU itself prior to saving it
	 * into the guest FPU context area.  In order to preserve any state
	 * currently housed in the FPU, we save it to a temporarily allocated
	 * FPU context. It is important to note that we are not following the
	 * normal rules around state management detailed in uts/intel/os/fpu.c.
	 * This saving is unconditional, uncaring about the state in the FPU or
	 * the value of CR0_TS, simplifying our process before returning to the
	 * caller (without needing to chcek of an lwp, etc).  To prevent
	 * interrupting threads from encountering this unusual FPU state, we
	 * keep interrupts disabled for the duration.
	 */
	fpu_ctx_t temp_ctx = {
		.fpu_xsave_mask = XFEATURE_FP_ALL,
	};
	temp_ctx.fpu_regs.kfpu_u.kfpu_generic =
	    kmem_cache_alloc(fpsave_cachep, KM_SLEEP);
	bzero(temp_ctx.fpu_regs.kfpu_u.kfpu_generic, save_area_size);

	ulong_t iflag;
	iflag = intr_clear();
	bool disable_when_done = (getcr0() & CR0_TS) != 0;
	do_fp_save(&temp_ctx);

	/*
	 * If the provided data is invalid, it will cause a #GP when we attempt
	 * to load it into the FPU, so protect against that with on_trap().
	 * Should the data load successfully, we can then be confident that its
	 * later use in via hma_fpu_start_guest() will be safe.
	 */
	on_trap_data_t otd;
	volatile hma_fpu_xsave_result_t res = HFXR_OK;
	if (on_trap(&otd, OT_DATA_EC) != 0) {
		res = HFXR_INVALID_DATA;
		goto done;
	}

	switch (fp_save_mech) {
	case FP_FXSAVE:
		if (hdr->xsh_xstate_bv == 0) {
			/*
			 * An empty xstate_bv means we can simply load the
			 * legacy FP/SSE area with their initial state.
			 */
			bcopy(&sse_initial,
			    fpu->hf_guest_fpu.fpu_regs.kfpu_u.kfpu_fx,
			    sizeof (sse_initial));
		} else {
			fpxrestore(buf);
			fpxsave(fpu->hf_guest_fpu.fpu_regs.kfpu_u.kfpu_fx);
		}
		break;
	case FP_XSAVE:
		xrestore(buf, XFEATURE_FP_ALL);
		xsavep(fpu->hf_guest_fpu.fpu_regs.kfpu_u.kfpu_xs,
		    fpu->hf_guest_fpu.fpu_xsave_mask);
		break;
	default:
		panic("Invalid fp_save_mech");
	}

done:
	no_trap();
	fp_restore(&temp_ctx);
	if (disable_when_done) {
		fpdisable();
	}
	intr_restore(iflag);
	kmem_cache_free(fpsave_cachep, temp_ctx.fpu_regs.kfpu_u.kfpu_generic);

	return (res);
}

void
hma_fpu_get_fxsave_state(const hma_fpu_t *fpu, struct fxsave_state *fx)
{
	const struct fxsave_state *guest;

	ASSERT3S(fpu->hf_inguest, ==, B_FALSE);

	guest = fpu->hf_guest_fpu.fpu_regs.kfpu_u.kfpu_fx;
	bcopy(guest, fx, sizeof (*fx));
}

int
hma_fpu_set_fxsave_state(hma_fpu_t *fpu, const struct fxsave_state *fx)
{
	struct fxsave_state *gfx;
	struct xsave_state *gxs;

	ASSERT3S(fpu->hf_inguest, ==, B_FALSE);

	/*
	 * If reserved bits are set in fx_mxcsr, then we will take a #GP when
	 * we restore them. Reject this outright.
	 *
	 * We do not need to check if we are dealing with state that has pending
	 * exceptions. This was only the case with the original FPU save and
	 * restore mechanisms (fsave/frstor). When using fxsave/fxrstor and
	 * xsave/xrstor they will be deferred to the user using the FPU, which
	 * is what we'd want here (they'd be used in guest context).
	 */
	if ((fx->fx_mxcsr & ~sse_mxcsr_mask) != 0)
		return (EINVAL);

	switch (fp_save_mech) {
	case FP_FXSAVE:
		gfx = fpu->hf_guest_fpu.fpu_regs.kfpu_u.kfpu_fx;
		bcopy(fx, gfx, sizeof (*fx));
		break;
	case FP_XSAVE:
		gxs = fpu->hf_guest_fpu.fpu_regs.kfpu_u.kfpu_xs;
		bzero(gxs, cpuid_get_xsave_size());
		bcopy(fx, &gxs->xs_fxsave, sizeof (*fx));
		gxs->xs_header.xsh_xstate_bv =
		    XFEATURE_LEGACY_FP | XFEATURE_SSE;
		break;
	default:
		panic("Invalid fp_save_mech");
	}

	return (0);
}
# CDDL HEADER START
#
# The contents of this file are subject to the terms of the
# Common Development and Distribution License (the "License").
# You may not use this file except in compliance with the License.
#
# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
# or http://www.opensolaris.org/os/licensing.
# See the License for the specific language governing permissions
# and limitations under the License.
#
# When distributing Covered Code, include this CDDL HEADER in each
# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
# If applicable, add the following below this CDDL HEADER, with the
# fields enclosed by brackets "[]" replaced with your own identifying
# information: Portions Copyright [yyyy] [name of copyright owner]
#
# CDDL HEADER END
#

#
# Copyright (c) 1993, 2010, Oracle and/or its affiliates. All rights reserved.
#

#
# CAUTION!  The order of modules specified here is very important. If the
# order is not correct it can result in unexpected system behavior. The
# loading of modules is in the reverse order specified here (i.e. the last
# entry is loaded first and the first entry loaded last).
#
pcplusmp
apix
xpv_psm

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 *
 * Copyright 2012 Nexenta Systems, Inc. All rights reserved.
 * Copyright (c) 2018, Joyent, Inc.
 * Copyright 2021 OmniOS Community Edition (OmniOSce) Association.
 * Copyright 2025 Oxide Computer Company
 */

#include <sys/bootconf.h>
#include <sys/cmn_err.h>
#include <sys/controlregs.h>
#include <sys/utsname.h>
#include <sys/debug.h>
#include <sys/kobj.h>
#include <sys/kobj_impl.h>
#include <sys/ontrap.h>
#include <sys/stdbool.h>
#include <sys/systeminfo.h>
#include <sys/systm.h>
#include <sys/ucode.h>
#include <sys/x86_archext.h>
#include <sys/x_call.h>

/*
 * mcpu_ucode_info for the boot CPU.  Statically allocated.
 */
static struct cpu_ucode_info cpu_ucode_info0;
static const ucode_source_t *ucode;
static char *ucodepath;
static kmutex_t ucode_lock;
static bool ucode_cleanup_done = false;

/*
 * Flag for use by microcode impls to determine if they can use kmem.  Note this
 * is meant primarily for gating use of functions like kobj_open_file() which
 * allocate internally with kmem.  ucode_zalloc() and ucode_free() should
 * otherwise be used.
 */
bool ucode_use_kmem = false;

static const char ucode_failure_fmt[] =
	"cpu%d: failed to update microcode from version 0x%x to 0x%x";
static const char ucode_success_fmt[] =
	"?cpu%d: microcode has been updated from version 0x%x to 0x%x\n";
static const char ucode_fallback_fmt[] =
	"?cpu%d: using older fallback microcode; update the system firmware";

static const char ucode_path_fmt[] = "/platform/%s/ucode";

SET_DECLARE(ucode_source_set, ucode_source_t);

/*
 * Force flag.  If set, the first microcode binary that matches
 * signature and platform id will be used for microcode update,
 * regardless of version.  Should only be used for debugging.
 */
int ucode_force_update = 0;

void
ucode_init(void)
{
	ucode_source_t **src;

	mutex_init(&ucode_lock, NULL, MUTEX_DEFAULT, NULL);

	/* Set up function pointers */
	SET_FOREACH(src, ucode_source_set) {
		if ((*src)->us_select(CPU)) {
			ucode = *src;
			break;
		}
	}

	if (ucode == NULL)
		return;

#ifdef DEBUG
	cmn_err(CE_CONT, "?ucode: selected %s\n", ucode->us_name);

	if (!ucode->us_capable(CPU)) {
		cmn_err(CE_CONT,
		    "?ucode: microcode update not supported on CPU\n");
		return;
	}
#endif
}

/*
 * Allocate space for mcpu_ucode_info in the machcpu structure
 * for all non-boot CPUs.
 */
void
ucode_alloc_space(cpu_t *cp)
{
	ASSERT(cp->cpu_id != 0);
	ASSERT(cp->cpu_m.mcpu_ucode_info == NULL);
	cp->cpu_m.mcpu_ucode_info =
	    kmem_zalloc(sizeof (*cp->cpu_m.mcpu_ucode_info), KM_SLEEP);
}

void
ucode_free_space(cpu_t *cp)
{
	ASSERT(cp->cpu_m.mcpu_ucode_info != NULL);
	ASSERT(cp->cpu_m.mcpu_ucode_info != &cpu_ucode_info0);
	kmem_free(cp->cpu_m.mcpu_ucode_info,
	    sizeof (*cp->cpu_m.mcpu_ucode_info));
	cp->cpu_m.mcpu_ucode_info = NULL;
}

const char *
ucode_path(void)
{
	ASSERT(ucodepath != NULL);
	return (ucodepath);
}

/*
 * Allocate/free a buffer used to hold ucode data. Space allocated before kmem
 * is available is allocated with BOP_ALLOC() and does not require a free.
 */
void *
ucode_zalloc(size_t size)
{
	if (ucode_use_kmem)
		return (kmem_zalloc(size, KM_NOSLEEP));

	/* BOP_ALLOC() failure results in panic */
	return (BOP_ALLOC(bootops, NULL, size, MMU_PAGESIZE));
}

void
ucode_free(void *buf, size_t size)
{
	if (ucode_use_kmem && buf != NULL)
		kmem_free(buf, size);
}

/*
 * Called to free up space allocated for the microcode file. This is called
 * from start_other_cpus() after an update attempt has been performed on all
 * CPUs.
 */
void
ucode_cleanup(void)
{
	mutex_enter(&ucode_lock);
	if (ucode != NULL)
		ucode->us_file_reset();
	ucode_cleanup_done = true;
	mutex_exit(&ucode_lock);

	/*
	 * We purposefully do not free 'ucodepath' here so that it persists for
	 * any future callers to ucode_locate(), such as could occur on systems
	 * that support DR.
	 */
}

static int
ucode_write(xc_arg_t arg1, xc_arg_t unused2, xc_arg_t unused3)
{
	ucode_update_t *uusp = (ucode_update_t *)arg1;
	cpu_ucode_info_t *uinfop = CPU->cpu_m.mcpu_ucode_info;
	on_trap_data_t otd;

	ASSERT(ucode != NULL);
	ASSERT(uusp->ucodep != NULL);

	/*
	 * Check one more time to see if it is really necessary to update
	 * microcode just in case this is a hyperthreaded processor where
	 * the threads share the same microcode.
	 */
	if (!ucode_force_update) {
		ucode->us_read_rev(uinfop);
		uusp->new_rev = uinfop->cui_rev;
		if (uinfop->cui_rev >= uusp->expected_rev)
			return (0);
	}

	if (!on_trap(&otd, OT_DATA_ACCESS)) {
		if (ucode->us_invalidate) {
			/*
			 * On some platforms a cache invalidation is required
			 * for the ucode update to be successful due to the
			 * parts of the processor that the microcode is
			 * updating.
			 */
			invalidate_cache();
		}
		wrmsr(ucode->us_write_msr, (uintptr_t)uusp->ucodep);
	}

	no_trap();
	ucode->us_read_rev(uinfop);
	uusp->new_rev = uinfop->cui_rev;

	return (0);
}

/*
 * Entry points to microcode update from the 'ucode' driver.
 */

ucode_errno_t
ucode_validate(uint8_t *ucodep, size_t size)
{
	if (ucode == NULL)
		return (EM_NOTSUP);
	return (ucode->us_validate(ucodep, size));
}

ucode_errno_t
ucode_update(uint8_t *ucodep, size_t size)
{
	bool		found = false;
	ucode_update_t	cached = { 0 };
	ucode_update_t	*cachedp = NULL;
	ucode_errno_t	rc = EM_OK;
	ucode_errno_t	search_rc = EM_NOMATCH; /* search result */
	cpuset_t cpuset;

	ASSERT(ucode != 0);
	ASSERT(ucodep != 0);
	CPUSET_ZERO(cpuset);

	if (!ucode->us_capable(CPU))
		return (EM_NOTSUP);

	mutex_enter(&cpu_lock);

	for (processorid_t id = 0; id < max_ncpus; id++) {
		cpu_t *cpu;
		ucode_update_t uus = { 0 };
		ucode_update_t *uusp = &uus;

		/*
		 * If there is no such CPU or it is not xcall ready, skip it.
		 */
		if ((cpu = cpu_get(id)) == NULL ||
		    !(cpu->cpu_flags & CPU_READY)) {
			continue;
		}

		uusp->sig = cpuid_getsig(cpu);
		bcopy(cpu->cpu_m.mcpu_ucode_info, &uusp->info,
		    sizeof (uusp->info));

		/*
		 * If the current CPU has the same signature and platform
		 * id as the previous one we processed, reuse the information.
		 */
		if (cachedp && cachedp->sig == cpuid_getsig(cpu) &&
		    cachedp->info.cui_platid == uusp->info.cui_platid) {
			uusp->ucodep = cachedp->ucodep;
			uusp->expected_rev = cachedp->expected_rev;
			/*
			 * Intuitively we should check here to see whether the
			 * running microcode rev is >= the expected rev, and
			 * quit if it is.  But we choose to proceed with the
			 * xcall regardless of the running version so that
			 * the other threads in an HT processor can update
			 * the cpu_ucode_info structure in machcpu.
			 */
		} else if ((search_rc = ucode->us_extract(uusp, ucodep, size))
		    == EM_OK) {
			bcopy(uusp, &cached, sizeof (cached));
			cachedp = &cached;
			found = true;
		}

		/* Nothing to do */
		if (uusp->ucodep == NULL)
			continue;

		CPUSET_ADD(cpuset, id);
		kpreempt_disable();
		xc_sync((xc_arg_t)uusp, 0, 0, CPUSET2BV(cpuset), ucode_write);
		kpreempt_enable();
		CPUSET_DEL(cpuset, id);

		if (uusp->new_rev != 0 && uusp->info.cui_rev == uusp->new_rev &&
		    !ucode_force_update) {
			rc = EM_HIGHERREV;
		} else if ((uusp->new_rev == 0) || (uusp->expected_rev != 0 &&
		    uusp->expected_rev != uusp->new_rev)) {
			cmn_err(CE_WARN, ucode_failure_fmt,
			    id, uusp->info.cui_rev, uusp->expected_rev);
			rc = EM_UPDATE;
		} else {
			cmn_err(CE_CONT, ucode_success_fmt,
			    id, uusp->info.cui_rev, uusp->new_rev);
		}
	}

	mutex_exit(&cpu_lock);

	if (!found) {
		rc = search_rc;
	} else if (rc == EM_OK) {
		cpuid_post_ucodeadm();
	}

	return (rc);
}

/*
 * Called when starting up non-boot CPUs from mp_startup() to read the current
 * microcode revision before the control CPU calls ucode_locate().
 */
void
ucode_read_rev(cpu_t *cp)
{
	cpu_ucode_info_t *uinfop;

	ASSERT3P(cp, !=, NULL);

	if (ucode == NULL || !ucode->us_capable(cp))
		return;

	uinfop = cp->cpu_m.mcpu_ucode_info;
	ASSERT3P(uinfop, !=, NULL);

	ucode->us_read_rev(uinfop);
}

/*
 * Called by the control CPU when starting up non-boot CPUs to find any
 * applicable microcode updates. Initializes mcpu_ucode_info, which will contain
 * the relevant update to be applied, via ucode_apply(), if one is found.
 * ucode_read_rev() must be called before this function on the target CPU.
 */
void
ucode_locate(cpu_t *cp)
{
	cpu_ucode_info_t *uinfop;
	ucode_errno_t rc;
	size_t sz;

	ASSERT3P(cp, !=, NULL);
	ASSERT(ucode_use_kmem);

	mutex_enter(&ucode_lock);

	if (ucode == NULL || !ucode->us_capable(cp))
		goto out;

	if (ucodepath == NULL) {
		sz = snprintf(NULL, 0, ucode_path_fmt, platform) + 1;
		ucodepath = kmem_zalloc(sz, KM_NOSLEEP);
		if (ucodepath == NULL) {
			cmn_err(CE_WARN,
			    "ucode: could not allocate memory for path");
			goto out;
		}
		(void) snprintf(ucodepath, sz, ucode_path_fmt, platform);
	}

	uinfop = cp->cpu_m.mcpu_ucode_info;
	ASSERT3P(uinfop, !=, NULL);

	/*
	 * Search for any applicable updates.
	 *
	 * A return value of EM_HIGHERREV indicates that no update was applied
	 * due to the CPU already being at that or a higher revision, but both
	 * EM_HIGHERREV and EM_OK indicate that some microcode that matches the
	 * CPU was successfully located. In either of these cases it's worth
	 * keeping it around in case it's useful for the next CPU -- and if it
	 * isn't it will end up being discarded. In all other cases we clear it
	 * out just in case we have read in a partial or invalid file.
	 *
	 * Architectural note:
	 *   Depending on the platform, the cpu_t being processed may represent
	 *   a thread within a CPU core. If updating one thread's microcode
	 *   implicitly updates all sibling threads in the core, it's normal to
	 *   see a mix of EM_OK and EM_HIGHERREV when iterating over those
	 *   threads.
	 *
	 * There's one additional consideration. If we are here after
	 * ucode_cleanup() has been called, such as could occur with CPU
	 * hotplug, we also clear the memory and reset the data structure as
	 * nothing else will call ucode_cleanup() and we don't need to cache
	 * the data as we do during boot when starting the APs.
	 */
	rc = ucode->us_locate(cp, uinfop);
	if ((rc != EM_OK && rc != EM_HIGHERREV) || ucode_cleanup_done)
		ucode->us_file_reset();

out:
	mutex_exit(&ucode_lock);
}

/*
 * Called when starting up non-boot CPUs to load any pending microcode updates
 * found in ucode_locate().  Note this is called very early in the startup
 * process (before CPU_READY is set and while CPU_QUIESCED is) so we must be
 * careful about what we do here, e.g., no kmem_free or anything that might call
 * hat_unload; no kmem_alloc or anything which may cause thread context switch.
 * We also don't take the ucode_lock here for similar reasons (if contended
 * the idle thread will spin with CPU_QUIESCED set). This is fine though since
 * we should not be updating any shared ucode state.
 */
void
ucode_apply(cpu_t *cp)
{
	cpu_ucode_info_t *uinfop;

	ASSERT3P(cp, !=, NULL);

	if (ucode == NULL || !ucode->us_capable(cp))
		return;

	uinfop = cp->cpu_m.mcpu_ucode_info;
	ASSERT3P(uinfop, !=, NULL);

	/*
	 * No pending update -- nothing to do.
	 */
	if (uinfop->cui_pending_ucode == NULL)
		return;

	/*
	 * Apply pending update.
	 */
	uinfop->cui_boot_rev = uinfop->cui_rev;
	ucode->us_load(uinfop);
	ucode->us_read_rev(uinfop);
}

/*
 * Called when starting up non-boot CPUs to free any pending microcode updates
 * found in ucode_locate() and print the result of the attempting to load it in
 * ucode_apply().  This is separate from ucode_apply() as we can't yet call
 * kmem_free() at that point in the startup process.
 */
void
ucode_finish(cpu_t *cp)
{
	cpu_ucode_info_t *uinfop;
	uint32_t old_rev, new_rev;

	ASSERT3P(cp, !=, NULL);

	if (ucode == NULL || !ucode->us_capable(cp))
		return;

	uinfop = cp->cpu_m.mcpu_ucode_info;
	ASSERT3P(uinfop, !=, NULL);

	/*
	 * No pending update -- nothing to do.
	 */
	if (uinfop->cui_pending_ucode == NULL)
		return;

	old_rev = uinfop->cui_boot_rev;
	new_rev = uinfop->cui_pending_rev;

	if (uinfop->cui_rev != new_rev) {
		ASSERT3U(uinfop->cui_rev, ==, old_rev);
		cmn_err(CE_WARN, ucode_failure_fmt, cp->cpu_id, old_rev,
		    new_rev);
	} else {
		cmn_err(CE_CONT, ucode_success_fmt, cp->cpu_id, old_rev,
		    new_rev);
	}

	ucode_free(uinfop->cui_pending_ucode, uinfop->cui_pending_size);
	uinfop->cui_pending_ucode = NULL;
	uinfop->cui_pending_size = 0;
	uinfop->cui_pending_rev = 0;
}

/*
 * Entry point to microcode update from mlsetup() for boot CPU.
 * Initialize mcpu_ucode_info, and perform microcode update if necessary.
 * cpuid_info must be initialized before we can be called.
 */
void
ucode_check_boot(void)
{
	cpu_t *cp = CPU;
	cpu_ucode_info_t *uinfop;
	const char *prop;
	char *plat;
	int prop_len;
	size_t path_len;

	ASSERT3U(cp->cpu_id, ==, 0);
	ASSERT(!ucode_use_kmem);

	mutex_enter(&ucode_lock);

	/* Space statically allocated for BSP; ensure pointer is set */
	ASSERT3P(cp->cpu_m.mcpu_ucode_info, ==, NULL);
	uinfop = cp->cpu_m.mcpu_ucode_info = &cpu_ucode_info0;

	if (ucode == NULL || !ucode->us_capable(cp))
		goto out;

	ASSERT3P(ucodepath, ==, NULL);

	prop = "impl-arch-name";
	prop_len = BOP_GETPROPLEN(bootops, prop);
	if (prop_len <= 0) {
		cmn_err(CE_WARN, "ucode: could not find %s property", prop);
		goto out;
	}

	/*
	 * We're running on the boot CPU before kmem is available so we make use
	 * of BOP_ALLOC() -- which panics on failure -- to allocate any memory
	 * we need.  That also means we don't need to explicity free it.
	 */
	plat = BOP_ALLOC(bootops, NULL, prop_len + 1, MMU_PAGESIZE);
	(void) BOP_GETPROP(bootops, prop, plat);
	if (plat[0] == '\0') {
		/*
		 * If we can't determine the architecture name,
		 * we cannot find microcode files for it.
		 * Return without setting 'ucodepath'.
		 */
		cmn_err(CE_WARN, "ucode: could not determine arch");
		goto out;
	}

	path_len = snprintf(NULL, 0, ucode_path_fmt, plat) + 1;
	ucodepath = BOP_ALLOC(bootops, NULL, path_len, MMU_PAGESIZE);
	(void) snprintf(ucodepath, path_len, ucode_path_fmt, plat);

	/*
	 * Check to see if we need ucode update
	 */
	ucode->us_read_rev(uinfop);
	if (ucode->us_locate(cp, uinfop) == EM_OK) {
		uint32_t old_rev, new_rev;
		bool fallback = false;

		old_rev = uinfop->cui_boot_rev = uinfop->cui_rev;

retry:
		new_rev = uinfop->cui_pending_rev;
		ucode->us_load(uinfop);
		ucode->us_read_rev(uinfop);

		if (uinfop->cui_rev != new_rev) {
			ASSERT3U(uinfop->cui_rev, ==, old_rev);

			cmn_err(CE_WARN, ucode_failure_fmt, cp->cpu_id,
			    old_rev, new_rev);

			/*
			 * If the updater supports attempting a fallback
			 * microcode version, try that.
			 */
			if (!fallback && ucode->us_locate_fallback != NULL) {
				ucode->us_file_reset();
				uinfop->cui_pending_ucode = NULL;
				uinfop->cui_pending_size = 0;
				uinfop->cui_pending_rev = 0;
				if (ucode->us_locate_fallback(cp, uinfop) ==
				    EM_OK) {
					cmn_err(CE_WARN, ucode_fallback_fmt,
					    cp->cpu_id);
					fallback = true;
					goto retry;
				}
			}
		} else {
			cmn_err(CE_CONT, ucode_success_fmt, cp->cpu_id,
			    old_rev, new_rev);
		}
	}

	/*
	 * Regardless of whether we found a match or not, since the scratch
	 * memory for holding the microcode for the boot CPU came from
	 * BOP_ALLOC, we will reset the data structure as if we never did the
	 * allocation so we don't have to keep track of this special chunk of
	 * memory.
	 */
	ucode->us_file_reset();

	/*
	 * Similarly clear any pending update that may have been found.
	 */
	uinfop->cui_pending_ucode = NULL;
	uinfop->cui_pending_size = 0;
	uinfop->cui_pending_rev = 0;

out:
	/*
	 * Discard the memory that came from BOP_ALLOC and was used to build the
	 * ucode path.  Subsequent CPUs will be handled via ucode_locate() at
	 * which point kmem is available and we can cache the path.
	 */
	ucodepath = NULL;
	ucode_use_kmem = true;

	mutex_exit(&ucode_lock);
}

/*
 * Returns microcode revision from the machcpu structure.
 */
ucode_errno_t
ucode_get_rev(uint32_t *revp)
{
	int i;

	ASSERT(revp != NULL);

	if (ucode == NULL || !ucode->us_capable(CPU))
		return (EM_NOTSUP);

	mutex_enter(&cpu_lock);
	for (i = 0; i < max_ncpus; i++) {
		cpu_t *cpu;

		if ((cpu = cpu_get(i)) == NULL)
			continue;

		revp[i] = cpu->cpu_m.mcpu_ucode_info->cui_rev;
	}
	mutex_exit(&cpu_lock);

	return (EM_OK);
}
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 *
 * Copyright 2012 Nexenta Systems, Inc. All rights reserved.
 * Copyright (c) 2018, Joyent, Inc.
 * Copyright 2021 OmniOS Community Edition (OmniOSce) Association.
 * Copyright 2025 Oxide Computer Company
 */

#include <sys/stdbool.h>
#include <sys/cmn_err.h>
#include <sys/controlregs.h>
#include <sys/kobj.h>
#include <sys/kobj_impl.h>
#include <sys/machparam.h>
#include <sys/ontrap.h>
#include <sys/sysmacros.h>
#include <sys/systm.h>
#include <sys/ucode.h>
#include <sys/ucode_amd.h>
#include <ucode/ucode_errno.h>
#include <ucode/ucode_utils_amd.h>
#include <sys/x86_archext.h>

extern void *ucode_zalloc(size_t);
extern void ucode_free(void *, size_t);
extern const char *ucode_path(void);
extern int ucode_force_update;
extern bool ucode_use_kmem;

static ucode_file_amd_t *amd_ucodef;
static size_t amd_ucodef_len, amd_ucodef_buflen;
static ucode_eqtbl_amd_t *ucode_eqtbl_amd;
static uint_t ucode_eqtbl_amd_entries;
static bool ucode_amd_fallback = false;

/*
 * Check whether this module can be used for microcode updates on this
 * platform.
 */
static bool
ucode_select_amd(cpu_t *cp)
{
	if ((get_hwenv() & HW_VIRTUAL) != 0)
		return (false);

	return (cpuid_getvendor(cp) == X86_VENDOR_AMD);
}

/*
 * Check whether or not a processor is capable of microcode operations
 *
 * At this point we only support microcode update for:
 * - AMD processors family 0x10 and above.
 */
static bool
ucode_capable_amd(cpu_t *cp)
{
	return (cpuid_getfamily(cp) >= 0x10);
}

/*
 * Called when it is no longer necessary to keep the microcode around,
 * or when the cached microcode doesn't match the CPU being processed.
 */
static void
ucode_file_reset_amd(void)
{
	if (amd_ucodef == NULL)
		return;

	ucode_free(amd_ucodef, amd_ucodef_buflen);
	amd_ucodef = NULL;
	amd_ucodef_buflen = amd_ucodef_len = 0;
}

/*
 * Find the equivalent CPU id in the equivalence table.
 */
static ucode_errno_t
ucode_equiv_cpu_amd(cpu_t *cp, uint16_t *eq_sig)
{
	char *name = NULL;
	int cpi_sig = cpuid_getsig(cp);
	ucode_errno_t ret = EM_OK;

	if (ucode_eqtbl_amd == NULL) {
		name = ucode_zalloc(MAXPATHLEN);
		if (name == NULL)
			return (EM_NOMEM);

		(void) snprintf(name, MAXPATHLEN, "%s/%s/%s",
		    ucode_path(), cpuid_getvendorstr(cp),
		    UCODE_AMD_EQUIVALENCE_TABLE_NAME);
	}

	if (!ucode_use_kmem) {
		/*
		 * No kmem_zalloc() etc. available yet.
		 */
		ucode_eqtbl_amd_t eqtbl;
		int count, offset = 0;
		intptr_t fd;

		ASSERT3P(name, !=, NULL);

		if ((fd = kobj_open(name)) == -1) {
			ret = EM_OPENFILE;
			goto out;
		}
		do {
			count = kobj_read(fd, (int8_t *)&eqtbl,
			    sizeof (eqtbl), offset);
			if (count != sizeof (eqtbl)) {
				(void) kobj_close(fd);
				ret = EM_HIGHERREV;
				goto out;
			}
			offset += count;
		} while (eqtbl.ue_inst_cpu != 0 &&
		    eqtbl.ue_inst_cpu != cpi_sig);
		(void) kobj_close(fd);
		*eq_sig = eqtbl.ue_equiv_cpu;
	} else {
		ucode_eqtbl_amd_t *eqtbl;

		/*
		 * If not already done, load the equivalence table.
		 */
		if (ucode_eqtbl_amd == NULL) {
			struct _buf *eq;
			uint64_t size;
			int count;

			ASSERT3P(name, !=, NULL);

			if ((eq = kobj_open_file(name)) == (struct _buf *)-1) {
				ret = EM_OPENFILE;
				goto out;
			}

			if (kobj_get_filesize(eq, &size) < 0) {
				kobj_close_file(eq);
				ret = EM_OPENFILE;
				goto out;
			}

			if (size == 0 ||
			    size % sizeof (*ucode_eqtbl_amd) != 0) {
				kobj_close_file(eq);
				ret = EM_HIGHERREV;
				goto out;
			}

			ucode_eqtbl_amd = kmem_zalloc(size, KM_NOSLEEP);
			if (ucode_eqtbl_amd == NULL) {
				kobj_close_file(eq);
				ret = EM_NOMEM;
				goto out;
			}
			count = kobj_read_file(eq, (char *)ucode_eqtbl_amd,
			    size, 0);
			kobj_close_file(eq);

			if (count != size) {
				ucode_eqtbl_amd_entries = 0;
				ret = EM_FILESIZE;
				goto out;
			}

			ucode_eqtbl_amd_entries =
			    size / sizeof (*ucode_eqtbl_amd);
		}

		eqtbl = ucode_eqtbl_amd;
		*eq_sig = 0;
		for (uint_t i = 0; i < ucode_eqtbl_amd_entries; i++, eqtbl++) {
			if (eqtbl->ue_inst_cpu == 0) {
				/* End of table */
				ret = EM_HIGHERREV;
				goto out;
			}
			if (eqtbl->ue_inst_cpu == cpi_sig) {
				*eq_sig = eqtbl->ue_equiv_cpu;
				ret = EM_OK;
				goto out;
			}
		}
		/*
		 * No equivalent CPU id found, assume outdated microcode file.
		 */
		ret = EM_HIGHERREV;
	}

out:
	ucode_free(name, MAXPATHLEN);

	return (ret);
}

static ucode_errno_t
ucode_match_amd(uint16_t eq_sig, cpu_ucode_info_t *uinfop,
    ucode_file_amd_t *ucodefp, size_t size)
{
	ucode_header_amd_t *uh;

	if (ucodefp == NULL || size < sizeof (ucode_header_amd_t))
		return (EM_NOMATCH);

	uh = &ucodefp->uf_header;

	/*
	 * Don't even think about loading patches that would require code
	 * execution. Does not apply to patches for family 0x14 and beyond.
	 */
	if (uh->uh_cpu_rev < 0x5000 &&
	    size > offsetof(ucode_file_amd_t, uf_code_present) &&
	    ucodefp->uf_code_present) {
		return (EM_NOMATCH);
	}

	if (eq_sig != uh->uh_cpu_rev)
		return (EM_NOMATCH);

	if (uh->uh_nb_id) {
		cmn_err(CE_WARN, "ignoring northbridge-specific ucode: "
		    "chipset id %x, revision %x", uh->uh_nb_id, uh->uh_nb_rev);
		return (EM_NOMATCH);
	}

	if (uh->uh_sb_id) {
		cmn_err(CE_WARN, "ignoring southbridge-specific ucode: "
		    "chipset id %x, revision %x", uh->uh_sb_id, uh->uh_sb_rev);
		return (EM_NOMATCH);
	}

	if (uh->uh_patch_id <= uinfop->cui_rev && !ucode_force_update)
		return (EM_HIGHERREV);

	return (EM_OK);
}

/*
 * Copy the given ucode into cpu_ucode_info_t in preparation for loading onto
 * the corresponding CPU via ucode_load_amd().
 */
static ucode_errno_t
ucode_copy_amd(cpu_ucode_info_t *uinfop, const ucode_file_amd_t *ucodefp,
    size_t size)
{
	ASSERT3P(uinfop->cui_pending_ucode, ==, NULL);
	ASSERT3U(size, <=, UCODE_AMD_MAXSIZE);

	uinfop->cui_pending_ucode = ucode_zalloc(size);
	if (uinfop->cui_pending_ucode == NULL)
		return (EM_NOMEM);

	(void) memcpy(uinfop->cui_pending_ucode, ucodefp, size);
	uinfop->cui_pending_size = size;
	uinfop->cui_pending_rev = ucodefp->uf_header.uh_patch_id;

	return (EM_OK);
}

/*
 * Populate the ucode file structure from the microcode file corresponding to
 * this CPU, if exists.
 *
 * Return EM_OK on success, corresponding error code on failure.
 */
static ucode_errno_t
i_ucode_locate_amd(cpu_t *cp, cpu_ucode_info_t *uinfop, bool fallback)
{
	uint16_t eq_sig;
	ucode_errno_t rc;

	/* get equivalent CPU id */
	eq_sig = 0;
	if ((rc = ucode_equiv_cpu_amd(cp, &eq_sig)) != EM_OK)
		return (rc);

	/*
	 * Allocate a buffer for the microcode patch. If the buffer has been
	 * allocated before, check for a matching microcode to avoid loading
	 * the file again.
	 */
	if (amd_ucodef == NULL) {
		size_t len = PAGESIZE;

		amd_ucodef = ucode_zalloc(len);
		if (amd_ucodef == NULL)
			return (EM_NOMEM);
		amd_ucodef_buflen = len;
	} else {
		rc = ucode_match_amd(eq_sig, uinfop, amd_ucodef,
		    amd_ucodef_len);
		if (rc == EM_HIGHERREV)
			return (rc);
		if (rc == EM_OK) {
			return (ucode_copy_amd(uinfop, amd_ucodef,
			    amd_ucodef_len));
		}
	}

	/*
	 * Find the patch for this CPU. The patch files are named XXXX-YY, where
	 * XXXX is the equivalent CPU id and YY is the running patch number.
	 * Patches specific to certain chipsets are guaranteed to have lower
	 * numbers than less specific patches, so we can just load the first
	 * patch that matches.
	 */
	for (uint_t i = 0; i < 0xff; i++) {
		char name[MAXPATHLEN];
		intptr_t fd;
		int count;
		/* This is a uint_t to match the signature of kobj_read() */
		uint_t size;

		(void) snprintf(name, MAXPATHLEN, "%s/%s/%s%04X-%02X",
		    ucode_path(), cpuid_getvendorstr(cp),
		    fallback ? "fallback/" : "", eq_sig, i);

		if ((fd = kobj_open(name)) == -1)
			return (EM_NOMATCH);

		/*
		 * Since this code will run for the boot CPU before kmem is
		 * initialised we can't use the kobj_*_file() functions.
		 * In the case where the archive contains compressed files,
		 * kobj_fstat() will return the compressed size and so we must
		 * read the entire file through to determine its size.
		 */
		size = 0;
		do {
			count = kobj_read(fd, (char *)amd_ucodef,
			    amd_ucodef_buflen, size);
			if (count < 0) {
				(void) kobj_close(fd);
				return (EM_OPENFILE);
			}
			size += count;
		} while (count == amd_ucodef_buflen &&
		    size <= UCODE_AMD_MAXSIZE);

		if (size > UCODE_AMD_MAXSIZE) {
			(void) kobj_close(fd);
			cmn_err(CE_WARN, "ucode: microcode file %s is "
			    "too large (over 0x%x bytes)", name,
			    UCODE_AMD_MAXSIZE);
			return (EM_FILESIZE);
		}

		if (size > amd_ucodef_buflen) {
			size_t len = P2ROUNDUP(size, PAGESIZE);

			ucode_file_reset_amd();
			amd_ucodef = ucode_zalloc(len);
			if (amd_ucodef == NULL) {
				(void) kobj_close(fd);
				return (EM_NOMEM);
			}
			amd_ucodef_buflen = len;
		}

		count = kobj_read(fd, (char *)amd_ucodef, amd_ucodef_buflen, 0);
		(void) kobj_close(fd);
		if (count < 0 || count != size)
			return (EM_OPENFILE);

		amd_ucodef_len = count;

		rc = ucode_match_amd(eq_sig, uinfop, amd_ucodef,
		    amd_ucodef_len);
		if (rc == EM_HIGHERREV)
			return (rc);
		if (rc == EM_OK) {
			return (ucode_copy_amd(uinfop, amd_ucodef,
			    amd_ucodef_len));
		}
	}
	return (EM_NOMATCH);
}

ucode_errno_t
ucode_locate_amd(cpu_t *cp, cpu_ucode_info_t *uinfop)
{
	return (i_ucode_locate_amd(cp, uinfop, ucode_amd_fallback));
}

ucode_errno_t
ucode_locate_fallback_amd(cpu_t *cp, cpu_ucode_info_t *uinfop)
{
	/* Once we have switched to the fallback microcode, stick with it */
	ucode_amd_fallback = true;
	return (i_ucode_locate_amd(cp, uinfop, ucode_amd_fallback));
}

static void
ucode_read_rev_amd(cpu_ucode_info_t *uinfop)
{
	uinfop->cui_rev = rdmsr(MSR_AMD_PATCHLEVEL);
}

static void
ucode_load_amd(cpu_ucode_info_t *uinfop)
{
	ucode_file_amd_t *ucodefp = uinfop->cui_pending_ucode;
	on_trap_data_t otd;

	VERIFY3P(ucodefp, !=, NULL);
	VERIFY3U(ucodefp->uf_header.uh_patch_id, ==, uinfop->cui_pending_rev);

	kpreempt_disable();
	if (on_trap(&otd, OT_DATA_ACCESS)) {
		no_trap();
		goto out;
	}
	wrmsr(MSR_AMD_PATCHLOADER, (uintptr_t)ucodefp);
	no_trap();

out:
	kpreempt_enable();
}

static ucode_errno_t
ucode_extract_amd(ucode_update_t *uusp, uint8_t *ucodep, size_t size)
{
	uint32_t *ptr = (uint32_t *)ucodep;
	ucode_eqtbl_amd_t *eqtbl;
	ucode_file_amd_t *ufp;
	uint32_t count;
	bool higher = false;
	ucode_errno_t rc = EM_NOMATCH;
	uint16_t eq_sig;

	/* skip over magic number & equivalence table header */
	ptr += 2; size -= 8;

	count = *ptr++; size -= 4;
	for (eqtbl = (ucode_eqtbl_amd_t *)ptr;
	    eqtbl->ue_inst_cpu && eqtbl->ue_inst_cpu != uusp->sig;
	    eqtbl++)
		;

	eq_sig = eqtbl->ue_equiv_cpu;

	/* No equivalent CPU id found, assume outdated microcode file. */
	if (eq_sig == 0)
		return (EM_HIGHERREV);

	/* Use the first microcode patch that matches. */
	do {
		ptr += count >> 2; size -= count;

		if (size == 0)
			return (higher ? EM_HIGHERREV : EM_NOMATCH);

		ptr++; size -= 4;
		count = *ptr++; size -= 4;
		ufp = (ucode_file_amd_t *)ptr;

		rc = ucode_match_amd(eq_sig, &uusp->info, ufp, count);
		if (rc == EM_HIGHERREV)
			higher = true;
	} while (rc != EM_OK);

	uusp->ucodep = (uint8_t *)ufp;
	uusp->usize = count;
	uusp->expected_rev = ufp->uf_header.uh_patch_id;

	return (EM_OK);
}

static const ucode_source_t ucode_amd = {
	.us_name		= "AMD microcode updater",
	.us_write_msr		= MSR_AMD_PATCHLOADER,
	.us_invalidate		= false,
	.us_select		= ucode_select_amd,
	.us_capable		= ucode_capable_amd,
	.us_file_reset		= ucode_file_reset_amd,
	.us_read_rev		= ucode_read_rev_amd,
	.us_load		= ucode_load_amd,
	.us_validate		= ucode_validate_amd,
	.us_extract		= ucode_extract_amd,
	.us_locate		= ucode_locate_amd,
	.us_locate_fallback	= ucode_locate_fallback_amd
};
UCODE_SOURCE(ucode_amd);
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 *
 * Copyright 2012 Nexenta Systems, Inc. All rights reserved.
 * Copyright (c) 2018, Joyent, Inc.
 * Copyright 2021 OmniOS Community Edition (OmniOSce) Association.
 * Copyright 2025 Oxide Computer Company
 */

#include <sys/stdbool.h>
#include <sys/cmn_err.h>
#include <sys/controlregs.h>
#include <sys/kobj.h>
#include <sys/kobj_impl.h>
#include <sys/ontrap.h>
#include <sys/sysmacros.h>
#include <sys/systm.h>
#include <sys/ucode.h>
#include <sys/ucode_intel.h>
#include <ucode/ucode_errno.h>
#include <ucode/ucode_utils_intel.h>
#include <sys/x86_archext.h>

extern void *ucode_zalloc(size_t);
extern void ucode_free(void *, size_t);
extern const char *ucode_path(void);
extern int ucode_force_update;

static ucode_file_intel_t intel_ucodef;

/*
 * Check whether this module can be used for microcode updates on this
 * platform.
 */
static bool
ucode_select_intel(cpu_t *cp)
{
	if ((get_hwenv() & HW_VIRTUAL) != 0)
		return (false);

	return (cpuid_getvendor(cp) == X86_VENDOR_Intel);
}

/*
 * Check whether or not a processor is capable of microcode operations
 *
 * At this point we only support microcode update for:
 * - Intel processors family 6 and above.
 */
static bool
ucode_capable_intel(cpu_t *cp)
{
	return (cpuid_getfamily(cp) >= 6);
}

static void
ucode_file_reset_intel(void)
{
	ucode_file_intel_t *ucodefp = &intel_ucodef;
	int total_size, body_size;

	if (ucodefp->uf_header == NULL)
		return;

	total_size = UCODE_TOTAL_SIZE_INTEL(ucodefp->uf_header->uh_total_size);
	body_size = UCODE_BODY_SIZE_INTEL(ucodefp->uf_header->uh_body_size);

	if (ucodefp->uf_body != NULL) {
		ucode_free(ucodefp->uf_body, body_size);
		ucodefp->uf_body = NULL;
	}

	if (ucodefp->uf_ext_table != NULL) {
		int size = total_size - body_size - UCODE_HEADER_SIZE_INTEL;

		ucode_free(ucodefp->uf_ext_table, size);
		ucodefp->uf_ext_table = NULL;
	}

	ucode_free(ucodefp->uf_header, UCODE_HEADER_SIZE_INTEL);
	ucodefp->uf_header = NULL;
}

/*
 * Checks if the microcode is for this processor.
 */
static ucode_errno_t
ucode_match_intel(int cpi_sig, cpu_ucode_info_t *uinfop,
    ucode_header_intel_t *uhp, ucode_ext_table_intel_t *uetp)
{
	if (uhp == NULL)
		return (EM_NOMATCH);

	if (UCODE_MATCH_INTEL(cpi_sig, uhp->uh_signature,
	    uinfop->cui_platid, uhp->uh_proc_flags)) {

		if (uinfop->cui_rev >= uhp->uh_rev && !ucode_force_update)
			return (EM_HIGHERREV);

		return (EM_OK);
	}

	if (uetp != NULL) {
		for (uint_t i = 0; i < uetp->uet_count; i++) {
			ucode_ext_sig_intel_t *uesp;

			uesp = &uetp->uet_ext_sig[i];

			if (UCODE_MATCH_INTEL(cpi_sig, uesp->ues_signature,
			    uinfop->cui_platid, uesp->ues_proc_flags)) {

				if (uinfop->cui_rev >= uhp->uh_rev &&
				    !ucode_force_update)
					return (EM_HIGHERREV);

				return (EM_OK);
			}
		}
	}

	return (EM_NOMATCH);
}

/*
 * Copy the given ucode into cpu_ucode_info_t in preparation for loading onto
 * the corresponding CPU via ucode_load_intel().
 */
static ucode_errno_t
ucode_copy_intel(const ucode_file_intel_t *ucodefp, cpu_ucode_info_t *uinfop)
{
	ASSERT3P(ucodefp->uf_header, !=, NULL);
	ASSERT3P(ucodefp->uf_body, !=, NULL);
	ASSERT3P(uinfop->cui_pending_ucode, ==, NULL);

	/*
	 * Allocate memory for the pending microcode update and copy the body.
	 * We don't need the header or extended signature table which are only
	 * used for matching.
	 */
	size_t sz = UCODE_BODY_SIZE_INTEL(ucodefp->uf_header->uh_body_size);
	uinfop->cui_pending_ucode = ucode_zalloc(sz);
	if (uinfop->cui_pending_ucode == NULL)
		return (EM_NOMEM);
	memcpy(uinfop->cui_pending_ucode, ucodefp->uf_body, sz);

	uinfop->cui_pending_size = sz;
	uinfop->cui_pending_rev = ucodefp->uf_header->uh_rev;

	return (EM_OK);
}

static ucode_errno_t
ucode_locate_intel(cpu_t *cp, cpu_ucode_info_t *uinfop)
{
	char		name[MAXPATHLEN];
	intptr_t	fd;
	int		count;
	int		header_size = UCODE_HEADER_SIZE_INTEL;
	int		cpi_sig = cpuid_getsig(cp);
	ucode_errno_t	rc = EM_OK;
	ucode_file_intel_t *ucodefp = &intel_ucodef;

	/*
	 * If the cached microcode matches the CPU we are processing, use it.
	 */
	if (ucode_match_intel(cpi_sig, uinfop, ucodefp->uf_header,
	    ucodefp->uf_ext_table) == EM_OK && ucodefp->uf_body != NULL) {
		return (ucode_copy_intel(ucodefp, uinfop));
	}

	/*
	 * Look for microcode file with the right name.
	 */
	(void) snprintf(name, MAXPATHLEN, "%s/%s/%08X-%02X",
	    ucode_path(), cpuid_getvendorstr(cp), cpi_sig,
	    uinfop->cui_platid);
	if ((fd = kobj_open(name)) == -1) {
		return (EM_OPENFILE);
	}

	/*
	 * We found a microcode file for the CPU we are processing,
	 * reset the microcode data structure and read in the new
	 * file.
	 */
	ucode_file_reset_intel();

	ucodefp->uf_header = ucode_zalloc(header_size);
	if (ucodefp->uf_header == NULL)
		return (EM_NOMEM);

	count = kobj_read(fd, (char *)ucodefp->uf_header, header_size, 0);

	switch (count) {
	case UCODE_HEADER_SIZE_INTEL: {

		ucode_header_intel_t	*uhp = ucodefp->uf_header;
		uint32_t	offset = header_size;
		int		total_size, body_size, ext_size;
		uint32_t	sum = 0;

		/*
		 * Make sure that the header contains valid fields.
		 */
		if ((rc = ucode_header_validate_intel(uhp)) == EM_OK) {
			total_size = UCODE_TOTAL_SIZE_INTEL(uhp->uh_total_size);
			body_size = UCODE_BODY_SIZE_INTEL(uhp->uh_body_size);
			ucodefp->uf_body = ucode_zalloc(body_size);
			if (ucodefp->uf_body == NULL) {
				rc = EM_NOMEM;
				break;
			}

			if (kobj_read(fd, (char *)ucodefp->uf_body,
			    body_size, offset) != body_size)
				rc = EM_FILESIZE;
		}

		if (rc)
			break;

		sum = ucode_checksum_intel(0, header_size,
		    (uint8_t *)ucodefp->uf_header);
		if (ucode_checksum_intel(sum, body_size, ucodefp->uf_body)) {
			rc = EM_CHECKSUM;
			break;
		}

		/*
		 * Check to see if there is extended signature table.
		 */
		offset = body_size + header_size;
		ext_size = total_size - offset;

		if (ext_size <= 0)
			break;

		ucodefp->uf_ext_table = ucode_zalloc(ext_size);
		if (ucodefp->uf_ext_table == NULL) {
			rc = EM_NOMEM;
			break;
		}

		if (kobj_read(fd, (char *)ucodefp->uf_ext_table,
		    ext_size, offset) != ext_size) {
			rc = EM_FILESIZE;
		} else if (ucode_checksum_intel(0, ext_size,
		    (uint8_t *)(ucodefp->uf_ext_table))) {
			rc = EM_EXTCHECKSUM;
		} else {
			int i;

			for (i = 0; i < ucodefp->uf_ext_table->uet_count; i++) {
				ucode_ext_sig_intel_t *sig;

				sig = &ucodefp->uf_ext_table->uet_ext_sig[i];

				if (ucode_checksum_intel_extsig(uhp,
				    sig) != 0) {
					rc = EM_SIGCHECKSUM;
					break;
				}
			}
		}
		break;
	}

	default:
		rc = EM_FILESIZE;
		break;
	}

	kobj_close(fd);

	if (rc != EM_OK)
		return (rc);

	rc = ucode_match_intel(cpi_sig, uinfop, ucodefp->uf_header,
	    ucodefp->uf_ext_table);
	if (rc == EM_OK) {
		return (ucode_copy_intel(ucodefp, uinfop));
	}

	return (rc);
}

static void
ucode_read_rev_intel(cpu_ucode_info_t *uinfop)
{
	struct cpuid_regs crs;

	/*
	 * The Intel 64 and IA-32 Architecture Software Developer's Manual
	 * recommends that MSR_INTC_UCODE_REV be loaded with 0 first, then
	 * execute cpuid to guarantee the correct reading of this register.
	 */
	wrmsr(MSR_INTC_UCODE_REV, 0);
	(void) __cpuid_insn(&crs);
	uinfop->cui_rev = (rdmsr(MSR_INTC_UCODE_REV) >> INTC_UCODE_REV_SHIFT);

	/*
	 * The MSR_INTC_PLATFORM_ID is supported in Celeron and Xeon
	 * (Family 6, model 5 and above) and all processors after.
	 */
	if ((cpuid_getmodel(CPU) >= 5 || cpuid_getfamily(CPU) > 6)) {
		uinfop->cui_platid = 1 << ((rdmsr(MSR_INTC_PLATFORM_ID) >>
		    INTC_PLATFORM_ID_SHIFT) & INTC_PLATFORM_ID_MASK);
	}
}

static void
ucode_load_intel(cpu_ucode_info_t *uinfop)
{
	VERIFY3P(uinfop->cui_pending_ucode, !=, NULL);

	kpreempt_disable();
	/*
	 * On some platforms a cache invalidation is required for the
	 * ucode update to be successful due to the parts of the
	 * processor that the microcode is updating.
	 */
	invalidate_cache();
	wrmsr(MSR_INTC_UCODE_WRITE, (uintptr_t)uinfop->cui_pending_ucode);
	kpreempt_enable();
}

static ucode_errno_t
ucode_extract_intel(ucode_update_t *uusp, uint8_t *ucodep, size_t size)
{
	uint32_t	header_size = UCODE_HEADER_SIZE_INTEL;
	size_t		remaining;
	bool		found = false;
	ucode_errno_t	search_rc = EM_NOMATCH; /* search result */

	/*
	 * Go through the whole buffer in case there are
	 * multiple versions of matching microcode for this
	 * processor.
	 */
	for (remaining = size; remaining > 0; ) {
		uint32_t total_size, body_size, ext_size;
		uint8_t *curbuf = &ucodep[size - remaining];
		ucode_header_intel_t *uhp = (ucode_header_intel_t *)curbuf;
		ucode_ext_table_intel_t *uetp = NULL;
		ucode_errno_t tmprc;

		total_size = UCODE_TOTAL_SIZE_INTEL(uhp->uh_total_size);
		body_size = UCODE_BODY_SIZE_INTEL(uhp->uh_body_size);
		ext_size = total_size - (header_size + body_size);

		if (ext_size > 0) {
			uetp = (ucode_ext_table_intel_t *)
			    &curbuf[header_size + body_size];
		}

		tmprc = ucode_match_intel(uusp->sig, &uusp->info, uhp, uetp);

		/*
		 * Since we are searching through a big file
		 * containing microcode for pretty much all the
		 * processors, we are bound to get EM_NOMATCH
		 * at one point.  However, if we return
		 * EM_NOMATCH to users, it will really confuse
		 * them.  Therefore, if we ever find a match of
		 * a lower rev, we will set return code to
		 * EM_HIGHERREV.
		 */
		if (tmprc == EM_HIGHERREV)
			search_rc = EM_HIGHERREV;

		if (tmprc == EM_OK &&
		    uusp->expected_rev < uhp->uh_rev) {
			uusp->ucodep = (uint8_t *)&curbuf[header_size];
			uusp->usize =
			    UCODE_TOTAL_SIZE_INTEL(uhp->uh_total_size);
			uusp->expected_rev = uhp->uh_rev;
			found = true;
		}

		remaining -= total_size;
	}

	if (!found)
		return (search_rc);

	return (EM_OK);
}

static const ucode_source_t ucode_intel = {
	.us_name	= "Intel microcode updater",
	.us_write_msr	= MSR_INTC_UCODE_WRITE,
	.us_invalidate	= true,
	.us_select	= ucode_select_intel,
	.us_capable	= ucode_capable_intel,
	.us_file_reset	= ucode_file_reset_intel,
	.us_read_rev	= ucode_read_rev_intel,
	.us_load	= ucode_load_intel,
	.us_validate	= ucode_validate_intel,
	.us_extract	= ucode_extract_intel,
	.us_locate	= ucode_locate_intel
};
UCODE_SOURCE(ucode_intel);
devinfo:devinfo 0640 root sys
devinfo:devinfo,ro 0444 root sys
asy:* 0666 root sys
asy:*,cu 0600 uucp uucp
md:* 0640 root sys
md:admin 0644 root sys
md 85
devinfo 88
asy 106
did 239
nosys			0
rexit			1
psecflags		2
read			3
write			4
open			5
close			6
linkat			7
link			9
unlink			10
symlinkat		11
chdir			12
gtime			13
mknod			14
chmod			15
chown			16
brk			17
stat			18
lseek			19
getpid			20
mount			21
readlinkat		22
setuid			23
getuid			24
stime			25
pcsample		26
alarm			27
fstat			28
pause			29
stty			31
gtty			32
access			33
nice			34
statfs			35
syssync			36
kill			37
fstatfs			38
setpgrp			39
uucopystr		40
pipe			42
times			43
profil			44
faccessat		45
setgid			46
getgid			47
mknodat			48
msgsys			49
sysi86			50
sysacct			51
shmsys			52
semsys			53
ioctl			54
uadmin			55
fchownat		56
utssys			57
fdsync			58
exece			59
umask			60
chroot			61
fcntl			62
ulimit			63
renameat		64
unlinkat		65
fstatat			66
fstatat64		67
openat			68
openat64		69
tasksys			70
acctctl			71
exacctsys		72
getpagesizes		73
rctlsys			74
sidsys			75
lwp_park		77
sendfilev		78
rmdir			79
mkdir			80
getdents		81
privsys			82
ucredsys		83
sysfs			84
getmsg			85
putmsg			86
lstat			88
symlink			89
readlink		90
setgroups		91
getgroups		92
fchmod			93
fchown			94
sigprocmask		95
sigsuspend		96
sigaltstack		97
sigaction		98
sigpending		99
setcontext		100
fchmodat		101
mkdirat			102
statvfs			103
fstatvfs		104
getloadavg		105
nfs			106
waitsys			107
sigsendsys		108
hrtsys			109
utimesys		110
sigresend		111
priocntlsys		112
pathconf		113
mincore			114
mmap			115
mprotect		116
munmap			117
fpathconf		118
vfork			119
fchdir			120
readv			121
writev			122
preadv			123
pwritev			124
upanic			125
getrandom		126
mmapobj			127
setrlimit		128
getrlimit		129
lchown			130
memcntl			131
getpmsg			132
putpmsg			133
rename			134
uname			135
setegid			136
sysconfig		137
adjtime			138
systeminfo		139
sharefs			140
seteuid			141
forksys			142
sigwait			144
lwp_info		145
yield			146
lwp_sema_post		148
lwp_sema_trywait	149
lwp_detach		150
corectl			151
modctl			152
fchroot			153
vhangup			155
gettimeofday		156
getitimer		157
setitimer		158
lwp_create		159
lwp_exit		160
lwp_suspend		161
lwp_continue		162
lwp_kill		163
lwp_self		164
lwp_sigmask		165
lwp_wait		167
lwp_mutex_wakeup	168
lwp_cond_wait		170
lwp_cond_signal		171
lwp_cond_broadcast	172
pread			173
pwrite			174
llseek			175
inst_sync		176
brandsys		177
kaio			178
cpc			179
meminfosys		180
rusagesys		181
portfs			182
pollsys			183
labelsys		184
acl			185
c2audit			186
processor_bind		187
processor_info		188
p_online		189
sigqueue		190
clock_gettime		191
clock_settime		192
clock_getres		193
timer_create		194
timer_delete		195
timer_settime		196
timer_gettime		197
timer_getoverrun	198
nanosleep		199
facl			200
doorfs			201
setreuid		202
setregid		203
install_utrap		204
signotify		205
schedctl		206
pset			207
resolvepath		209
lwp_mutex_timedlock	210
lwp_sema_timedwait	211
lwp_rwlock_sys		212
getdents64		213
mmap64			214
stat64			215
lstat64			216
fstat64			217
statvfs64		218
fstatvfs64		219
setrlimit64		220
getrlimit64		221
pread64			222
pwrite64		223
open64			225
rpcmod			226
zone			227
autofs			228
getcwd			229
so_socket		230
so_socketpair		231
bind			232
listen			233
accept			234
connect			235
shutdown		236
recv			237
recvfrom		238
recvmsg			239
send			240
sendmsg			241
sendto			242
getpeername		243
getsockname		244
getsockopt		245
setsockopt		246
sockconfig		247
ntp_gettime		248
ntp_adjtime		249
lwp_mutex_unlock	250
lwp_mutex_trylock	251
lwp_mutex_register	252
cladm			253
uucopy			254
umount2			255
#path_to_inst_bootstrap_1
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

/*	Copyright (c) 1990, 1991 UNIX System Laboratories, Inc. */
/*	Copyright (c) 1984, 1986, 1987, 1988, 1989, 1990 AT&T   */
/*	All Rights Reserved   */

/*
 * Copyright 2023 Oxide Computer Company
 */

#include <sys/types.h>
#include <sys/param.h>
#include <sys/sysmacros.h>
#include <sys/signal.h>
#include <sys/systm.h>
#include <sys/user.h>
#include <sys/mman.h>
#include <sys/class.h>
#include <sys/proc.h>
#include <sys/procfs.h>
#include <sys/buf.h>
#include <sys/kmem.h>
#include <sys/cred.h>
#include <sys/archsystm.h>
#include <sys/vmparam.h>
#include <sys/prsystm.h>
#include <sys/reboot.h>
#include <sys/uadmin.h>
#include <sys/vfs.h>
#include <sys/vnode.h>
#include <sys/file.h>
#include <sys/session.h>
#include <sys/ucontext.h>
#include <sys/dnlc.h>
#include <sys/var.h>
#include <sys/cmn_err.h>
#include <sys/debugreg.h>
#include <sys/thread.h>
#include <sys/vtrace.h>
#include <sys/consdev.h>
#include <sys/psw.h>
#include <sys/regset.h>

#include <sys/privregs.h>

#include <sys/stack.h>
#include <sys/swap.h>
#include <vm/hat.h>
#include <vm/anon.h>
#include <vm/as.h>
#include <vm/page.h>
#include <vm/seg.h>
#include <vm/seg_kmem.h>
#include <vm/seg_map.h>
#include <vm/seg_vn.h>
#include <sys/exec.h>
#include <sys/acct.h>
#include <sys/core.h>
#include <sys/corectl.h>
#include <sys/modctl.h>
#include <sys/tuneable.h>
#include <c2/audit.h>
#include <sys/bootconf.h>
#include <sys/dumphdr.h>
#include <sys/promif.h>
#include <sys/systeminfo.h>
#include <sys/kdi.h>
#include <sys/contract_impl.h>
#include <sys/x86_archext.h>

/*
 * Construct the execution environment for the user's signal
 * handler and arrange for control to be given to it on return
 * to userland.  The library code now calls setcontext() to
 * clean up after the signal handler, so sigret() is no longer
 * needed.
 *
 * (The various 'volatile' declarations are need to ensure that values
 * are correct on the error return from on_fault().)
 */


/*
 * An amd64 signal frame looks like this on the stack:
 *
 * old %rsp:
 *		<128 bytes of untouched stack space>
 *		<a siginfo_t [optional]>
 *		<a ucontext_t>
 *		<a ucontext_t's xsave state>
 *		<siginfo_t *>                             ---+
 *		<signal number>                              | sigframe
 * new %rsp:	<return address (deliberately invalid)>   ---+
 *
 * The signal number and siginfo_t pointer are only pushed onto the stack in
 * order to allow stack backtraces.  The actual signal handling code expects the
 * arguments in registers.
 */

struct sigframe {
	caddr_t retaddr;
	long	signo;
	siginfo_t *sip;
};

int
sendsig(int sig, k_siginfo_t *sip, void (*hdlr)())
{
	volatile size_t minstacksz;
	boolean_t newstack;
	size_t xsave_size;
	int ret;
	label_t ljb;
	volatile caddr_t sp;
	caddr_t fp;
	volatile struct regs *rp;
	volatile greg_t upc;
	volatile proc_t *p = ttoproc(curthread);
	struct as *as = p->p_as;
	klwp_t *lwp = ttolwp(curthread);
	ucontext_t *volatile tuc = NULL;
	ucontext_t *uc;
	siginfo_t *sip_addr;
	volatile int watched;

	/*
	 * This routine is utterly dependent upon STACK_ALIGN being
	 * 16 and STACK_ENTRY_ALIGN being 8. Let's just acknowledge
	 * that and require it.
	 */

#if STACK_ALIGN != 16 || STACK_ENTRY_ALIGN != 8
#error "sendsig() amd64 did not find the expected stack alignments"
#endif

	rp = lwptoregs(lwp);
	upc = rp->r_pc;

	/*
	 * Since we're setting up to run the signal handler we have to
	 * arrange that the stack at entry to the handler is (only)
	 * STACK_ENTRY_ALIGN (i.e. 8) byte aligned so that when the handler
	 * executes its push of %rbp, the stack realigns to STACK_ALIGN
	 * (i.e. 16) correctly.
	 *
	 * The new sp will point to the sigframe and the ucontext_t. The
	 * above means that sp (and thus sigframe) will be 8-byte aligned,
	 * but not 16-byte aligned. ucontext_t, however, contains %xmm regs
	 * which must be 16-byte aligned. Because of this, for correct
	 * alignment, sigframe must be a multiple of 8-bytes in length, but
	 * not 16-bytes. This will place ucontext_t at a nice 16-byte boundary.
	 *
	 * When we move onto the xsave state, right now, we don't guarantee any
	 * alignment of the resulting data, but we will ensure that the
	 * resulting sp does have proper alignment. This will ensure that the
	 * guarantee on the ucontex_t is not violated.
	 */

	CTASSERT((sizeof (struct sigframe) % 16) == 8);

	minstacksz = sizeof (struct sigframe) + SA(sizeof (*uc));
	if (sip != NULL)
		minstacksz += SA(sizeof (siginfo_t));

	if (fpu_xsave_enabled()) {
		xsave_size = SA(fpu_signal_size(lwp));
		minstacksz += xsave_size;
	} else {
		xsave_size = 0;
	}

	ASSERT((minstacksz & (STACK_ENTRY_ALIGN - 1ul)) == 0);

	/*
	 * Figure out whether we will be handling this signal on
	 * an alternate stack specified by the user.  Then allocate
	 * and validate the stack requirements for the signal handler
	 * context.  on_fault will catch any faults.
	 */
	newstack = sigismember(&PTOU(curproc)->u_sigonstack, sig) &&
	    !(lwp->lwp_sigaltstack.ss_flags & (SS_ONSTACK|SS_DISABLE));

	if (newstack) {
		fp = (caddr_t)(SA((uintptr_t)lwp->lwp_sigaltstack.ss_sp) +
		    SA(lwp->lwp_sigaltstack.ss_size) - STACK_ALIGN);
	} else {
		/*
		 * Drop below the 128-byte reserved region of the stack frame
		 * we're interrupting.
		 */
		fp = (caddr_t)rp->r_sp - STACK_RESERVE;
	}

	/*
	 * Force proper stack pointer alignment, even in the face of a
	 * misaligned stack pointer from user-level before the signal.
	 */
	fp = (caddr_t)((uintptr_t)fp & ~(STACK_ENTRY_ALIGN - 1ul));

	/*
	 * Most of the time during normal execution, the stack pointer
	 * is aligned on a STACK_ALIGN (i.e. 16 byte) boundary.  However,
	 * (for example) just after a call instruction (which pushes
	 * the return address), the callers stack misaligns until the
	 * 'push %rbp' happens in the callee prolog.  So while we should
	 * expect the stack pointer to be always at least STACK_ENTRY_ALIGN
	 * aligned, we should -not- expect it to always be STACK_ALIGN aligned.
	 * We now adjust to ensure that the new sp is aligned to
	 * STACK_ENTRY_ALIGN but not to STACK_ALIGN.
	 */
	sp = fp - minstacksz;
	if (((uintptr_t)sp & (STACK_ALIGN - 1ul)) == 0) {
		sp -= STACK_ENTRY_ALIGN;
		minstacksz = fp - sp;
	}

	/*
	 * Now, make sure the resulting signal frame address is sane
	 */
	if (sp >= as->a_userlimit || fp >= as->a_userlimit) {
#ifdef DEBUG
		printf("sendsig: bad signal stack cmd=%s, pid=%d, sig=%d\n",
		    PTOU(p)->u_comm, p->p_pid, sig);
		printf("sigsp = 0x%p, action = 0x%p, upc = 0x%lx\n",
		    (void *)sp, (void *)hdlr, (uintptr_t)upc);
		printf("sp above USERLIMIT\n");
#endif
		return (0);
	}

	watched = watch_disable_addr((caddr_t)sp, minstacksz, S_WRITE);

	if (on_fault(&ljb))
		goto badstack;

	if (sip != NULL) {
		zoneid_t zoneid;

		fp -= SA(sizeof (siginfo_t));
		uzero(fp, sizeof (siginfo_t));
		if (SI_FROMUSER(sip) &&
		    (zoneid = p->p_zone->zone_id) != GLOBAL_ZONEID &&
		    zoneid != sip->si_zoneid) {
			k_siginfo_t sani_sip = *sip;

			sani_sip.si_pid = p->p_zone->zone_zsched->p_pid;
			sani_sip.si_uid = 0;
			sani_sip.si_ctid = -1;
			sani_sip.si_zoneid = zoneid;
			copyout_noerr(&sani_sip, fp, sizeof (sani_sip));
		} else
			copyout_noerr(sip, fp, sizeof (*sip));
		sip_addr = (siginfo_t *)fp;

		if (sig == SIGPROF &&
		    curthread->t_rprof != NULL &&
		    curthread->t_rprof->rp_anystate) {
			/*
			 * We stand on our head to deal with
			 * the real time profiling signal.
			 * Fill in the stuff that doesn't fit
			 * in a normal k_siginfo structure.
			 */
			int i = sip->si_nsysarg;

			while (--i >= 0)
				sulword_noerr(
				    (ulong_t *)&(sip_addr->si_sysarg[i]),
				    (ulong_t)lwp->lwp_arg[i]);
			copyout_noerr(curthread->t_rprof->rp_state,
			    sip_addr->si_mstate,
			    sizeof (curthread->t_rprof->rp_state));
		}
	} else
		sip_addr = NULL;

	no_fault();

	/*
	 * Save the current context on the user stack directly after the
	 * sigframe. Since sigframe is 8-byte-but-not-16-byte aligned, and since
	 * sizeof (struct sigframe) is 24, this guarantees 16-byte alignment for
	 * ucontext_t and its %xmm registers. The xsave state part of the
	 * ucontext_t may be inbetween these two. However, we have ensured that
	 * the size of the stack space is 16-byte aligned as the actual size may
	 * vary.
	 */
	tuc = kmem_alloc(sizeof (*tuc), KM_SLEEP);
	if (xsave_size != 0) {
		tuc->uc_xsave = (unsigned long)(sp + sizeof (struct sigframe));
	}
	uc = (ucontext_t *)(sp + sizeof (struct sigframe) + xsave_size);
	ret = savecontext(tuc, &lwp->lwp_sigoldmask, SAVECTXT_F_EXTD |
	    SAVECTXT_F_ONFAULT);
	if (ret != 0)
		goto postfault;
	if (on_fault(&ljb))
		goto badstack;
	copyout_noerr(tuc, uc, sizeof (*tuc));
	kmem_free(tuc, sizeof (*tuc));
	tuc = NULL;

	lwp->lwp_oldcontext = (uintptr_t)uc;

	if (newstack) {
		lwp->lwp_sigaltstack.ss_flags |= SS_ONSTACK;
		if (lwp->lwp_ustack)
			copyout_noerr(&lwp->lwp_sigaltstack,
			    (stack_t *)lwp->lwp_ustack, sizeof (stack_t));
	}

	/*
	 * Set up signal handler return and stack linkage
	 */
	{
		struct sigframe frame;

		/*
		 * ensure we never return "normally"
		 */
		frame.retaddr = (caddr_t)(uintptr_t)-1L;
		frame.signo = sig;
		frame.sip = sip_addr;
		copyout_noerr(&frame, sp, sizeof (frame));
	}

	no_fault();
	if (watched)
		watch_enable_addr((caddr_t)sp, minstacksz, S_WRITE);

	/*
	 * Set up user registers for execution of signal handler.
	 */
	rp->r_sp = (greg_t)sp;
	rp->r_pc = (greg_t)hdlr;
	rp->r_ps = PSL_USER | (rp->r_ps & PS_IOPL);

	rp->r_rdi = sig;
	rp->r_rsi = (uintptr_t)sip_addr;
	rp->r_rdx = (uintptr_t)uc;

	if ((rp->r_cs & 0xffff) != UCS_SEL ||
	    (rp->r_ss & 0xffff) != UDS_SEL) {
		/*
		 * Try our best to deliver the signal.
		 */
		rp->r_cs = UCS_SEL;
		rp->r_ss = UDS_SEL;
	}

	/*
	 * Don't set lwp_eosys here.  sendsig() is called via psig() after
	 * lwp_eosys is handled, so setting it here would affect the next
	 * system call.
	 */
	return (1);

badstack:
	no_fault();
postfault:
	if (watched)
		watch_enable_addr((caddr_t)sp, minstacksz, S_WRITE);
	if (tuc)
		kmem_free(tuc, sizeof (*tuc));
#ifdef DEBUG
	printf("sendsig: bad signal stack cmd=%s, pid=%d, sig=%d\n",
	    PTOU(p)->u_comm, p->p_pid, sig);
	printf("on fault, sigsp = 0x%p, action = 0x%p, upc = 0x%lx\n",
	    (void *)sp, (void *)hdlr, (uintptr_t)upc);
#endif
	return (0);
}

#ifdef _SYSCALL32_IMPL

/*
 * An i386 SVR4/ABI signal frame looks like this on the stack:
 *
 * old %esp:
 *		<a siginfo32_t [optional]>
 *		<a ucontext32_t>
 *		<a ucontext32_t's xsave state>
 *		<pointer to that ucontext32_t>
 *		<pointer to that siginfo32_t>
 *		<signo>
 * new %esp:	<return address (deliberately invalid)>
 */
struct sigframe32 {
	caddr32_t	retaddr;
	uint32_t	signo;
	caddr32_t	sip;
	caddr32_t	ucp;
};

int
sendsig32(int sig, k_siginfo_t *sip, void (*hdlr)())
{
	volatile size_t minstacksz;
	boolean_t newstack;
	size_t xsave_size;
	int ret;
	label_t ljb;
	volatile caddr_t sp;
	caddr_t fp;
	volatile struct regs *rp;
	volatile greg_t upc;
	volatile proc_t *p = ttoproc(curthread);
	klwp_t *lwp = ttolwp(curthread);
	ucontext32_t *volatile tuc = NULL;
	ucontext32_t *uc;
	siginfo32_t *sip_addr;
	volatile int watched;

	rp = lwptoregs(lwp);
	upc = rp->r_pc;

	minstacksz = SA32(sizeof (struct sigframe32)) + SA32(sizeof (*uc));
	if (sip != NULL)
		minstacksz += SA32(sizeof (siginfo32_t));

	if (fpu_xsave_enabled()) {
		xsave_size = SA32(fpu_signal_size(lwp));
		minstacksz += xsave_size;
	} else {
		xsave_size = 0;
	}
	ASSERT((minstacksz & (STACK_ALIGN32 - 1)) == 0);

	/*
	 * Figure out whether we will be handling this signal on
	 * an alternate stack specified by the user.  Then allocate
	 * and validate the stack requirements for the signal handler
	 * context.  on_fault will catch any faults.
	 */
	newstack = sigismember(&PTOU(curproc)->u_sigonstack, sig) &&
	    !(lwp->lwp_sigaltstack.ss_flags & (SS_ONSTACK|SS_DISABLE));

	if (newstack) {
		fp = (caddr_t)(SA32((uintptr_t)lwp->lwp_sigaltstack.ss_sp) +
		    SA32(lwp->lwp_sigaltstack.ss_size) - STACK_ALIGN32);
	} else if ((rp->r_ss & 0xffff) != UDS_SEL) {
		user_desc_t *ldt;
		/*
		 * If the stack segment selector is -not- pointing at
		 * the UDS_SEL descriptor and we have an LDT entry for
		 * it instead, add the base address to find the effective va.
		 */
		if ((ldt = p->p_ldt) != NULL)
			fp = (caddr_t)rp->r_sp +
			    USEGD_GETBASE(&ldt[SELTOIDX(rp->r_ss)]);
		else
			fp = (caddr_t)rp->r_sp;
	} else
		fp = (caddr_t)rp->r_sp;

	/*
	 * Force proper stack pointer alignment, even in the face of a
	 * misaligned stack pointer from user-level before the signal.
	 * Don't use the SA32() macro because that rounds up, not down.
	 */
	fp = (caddr_t)((uintptr_t)fp & ~(STACK_ALIGN32 - 1));
	sp = fp - minstacksz;

	/*
	 * Make sure lwp hasn't trashed its stack
	 */
	if (sp >= (caddr_t)(uintptr_t)USERLIMIT32 ||
	    fp >= (caddr_t)(uintptr_t)USERLIMIT32) {
#ifdef DEBUG
		printf("sendsig32: bad signal stack cmd=%s, pid=%d, sig=%d\n",
		    PTOU(p)->u_comm, p->p_pid, sig);
		printf("sigsp = 0x%p, action = 0x%p, upc = 0x%lx\n",
		    (void *)sp, (void *)hdlr, (uintptr_t)upc);
		printf("sp above USERLIMIT\n");
#endif
		return (0);
	}

	watched = watch_disable_addr((caddr_t)sp, minstacksz, S_WRITE);

	if (on_fault(&ljb))
		goto badstack;

	if (sip != NULL) {
		siginfo32_t si32;
		zoneid_t zoneid;

		siginfo_kto32(sip, &si32);
		if (SI_FROMUSER(sip) &&
		    (zoneid = p->p_zone->zone_id) != GLOBAL_ZONEID &&
		    zoneid != sip->si_zoneid) {
			si32.si_pid = p->p_zone->zone_zsched->p_pid;
			si32.si_uid = 0;
			si32.si_ctid = -1;
			si32.si_zoneid = zoneid;
		}
		fp -= SA32(sizeof (si32));
		uzero(fp, sizeof (si32));
		copyout_noerr(&si32, fp, sizeof (si32));
		sip_addr = (siginfo32_t *)fp;

		if (sig == SIGPROF &&
		    curthread->t_rprof != NULL &&
		    curthread->t_rprof->rp_anystate) {
			/*
			 * We stand on our head to deal with
			 * the real-time profiling signal.
			 * Fill in the stuff that doesn't fit
			 * in a normal k_siginfo structure.
			 */
			int i = sip->si_nsysarg;

			while (--i >= 0)
				suword32_noerr(&(sip_addr->si_sysarg[i]),
				    (uint32_t)lwp->lwp_arg[i]);
			copyout_noerr(curthread->t_rprof->rp_state,
			    sip_addr->si_mstate,
			    sizeof (curthread->t_rprof->rp_state));
		}
	} else
		sip_addr = NULL;
	no_fault();

	/* save the current context on the user stack */
	tuc = kmem_alloc(sizeof (*tuc), KM_SLEEP);
	fp -= SA32(sizeof (*tuc));
	uc = (ucontext32_t *)fp;
	if (xsave_size != 0) {
		fp -= xsave_size;
		tuc->uc_xsave = (int32_t)(uintptr_t)fp;
	}
	ret = savecontext32(tuc, &lwp->lwp_sigoldmask, SAVECTXT_F_EXTD |
	    SAVECTXT_F_ONFAULT);
	if (ret != 0)
		goto postfault;
	if (on_fault(&ljb))
		goto badstack;
	copyout_noerr(tuc, uc, sizeof (*tuc));
	kmem_free(tuc, sizeof (*tuc));
	tuc = NULL;

	lwp->lwp_oldcontext = (uintptr_t)uc;

	if (newstack) {
		lwp->lwp_sigaltstack.ss_flags |= SS_ONSTACK;
		if (lwp->lwp_ustack) {
			stack32_t stk32;

			stk32.ss_sp = (caddr32_t)(uintptr_t)
			    lwp->lwp_sigaltstack.ss_sp;
			stk32.ss_size = (size32_t)
			    lwp->lwp_sigaltstack.ss_size;
			stk32.ss_flags = (int32_t)
			    lwp->lwp_sigaltstack.ss_flags;
			copyout_noerr(&stk32,
			    (stack32_t *)lwp->lwp_ustack, sizeof (stk32));
		}
	}

	/*
	 * Set up signal handler arguments
	 */
	{
		struct sigframe32 frame32;

		frame32.sip = (caddr32_t)(uintptr_t)sip_addr;
		frame32.ucp = (caddr32_t)(uintptr_t)uc;
		frame32.signo = sig;
		frame32.retaddr = 0xffffffff;	/* never return! */
		copyout_noerr(&frame32, sp, sizeof (frame32));
	}

	no_fault();
	if (watched)
		watch_enable_addr((caddr_t)sp, minstacksz, S_WRITE);

	rp->r_sp = (greg_t)(uintptr_t)sp;
	rp->r_pc = (greg_t)(uintptr_t)hdlr;
	rp->r_ps = PSL_USER | (rp->r_ps & PS_IOPL);

	if ((rp->r_cs & 0xffff) != U32CS_SEL ||
	    (rp->r_ss & 0xffff) != UDS_SEL) {
		/*
		 * Try our best to deliver the signal.
		 */
		rp->r_cs = U32CS_SEL;
		rp->r_ss = UDS_SEL;
	}

	/*
	 * Don't set lwp_eosys here.  sendsig() is called via psig() after
	 * lwp_eosys is handled, so setting it here would affect the next
	 * system call.
	 */
	return (1);

badstack:
	no_fault();
postfault:
	if (watched)
		watch_enable_addr((caddr_t)sp, minstacksz, S_WRITE);
	if (tuc)
		kmem_free(tuc, sizeof (*tuc));
#ifdef DEBUG
	printf("sendsig32: bad signal stack cmd=%s pid=%d, sig=%d\n",
	    PTOU(p)->u_comm, p->p_pid, sig);
	printf("on fault, sigsp = 0x%p, action = 0x%p, upc = 0x%lx\n",
	    (void *)sp, (void *)hdlr, (uintptr_t)upc);
#endif
	return (0);
}

#endif	/* _SYSCALL32_IMPL */
/*
 * This file and its contents are supplied under the terms of the
 * Common Development and Distribution License ("CDDL"), version 1.0.
 * You may only use this file in accordance with the terms of version
 * 1.0 of the CDDL.
 *
 * A full copy of the text of the CDDL should have accompanied this
 * source.  A copy of the CDDL is also available via the Internet at
 * http://www.illumos.org/license/CDDL.
 */

/*
 * Copyright 2019 Joyent, Inc.
 */

/*
 * SMT exclusion: prevent a sibling in a hyper-threaded core from running in VMX
 * non-root guest mode, when certain threads are running on the other sibling.
 * This avoids speculation-based information leaks such as L1TF being available
 * to the untrusted guest.  The stance we take is that threads from the same
 * zone as the guest VPCU thread are considered safe to run alongside, but all
 * other threads (except the idle thread), and all interrupts, are unsafe.  Note
 * that due to the implementation here, there are significant sections of e.g.
 * the dispatcher code that can run concurrently with a guest, until the thread
 * reaches smt_mark().  This code assumes there are only two SMT threads per
 * core.
 *
 * The entry points are as follows:
 *
 * smt_mark_as_vcpu()
 *
 * All threads that enter guest mode (i.e. VCPU threads) need to call this at
 * least once, which sets TS_VCPU in ->t_schedflag.
 *
 * smt_mark()
 *
 * A new ->cpu_thread is now curthread (although interrupt threads have their
 * own separate handling).  After preventing any interrupts, we will take our
 * own CPU's spinlock and update our own state in mcpu_smt.
 *
 * If our sibling is poisoned (i.e. in guest mode or the little bit of code
 * around it), and we're not compatible (that is, same zone ID, or the idle
 * thread), then we need to smt_kick() that sibling.  smt_kick() itself waits
 * for the sibling to call smt_release(), and it will not re-enter guest mode
 * until allowed.
 *
 * Note that we ignore the fact a process can change its zone ID: poisoning
 * threads never do so, and we can ignore the other cases.
 *
 * smt_acquire()
 *
 * We are a VCPU thread about to start guest execution.  Interrupts are
 * disabled.  We must have already run smt_mark() to be in this code, so there's
 * no need to take our *own* spinlock in order to mark ourselves as CM_POISONED.
 * Instead, we take our sibling's lock to also mark ourselves as poisoned in the
 * sibling cpu_smt_t.  This is so smt_mark() will only ever need to look at its
 * local mcpu_smt.
 *
 * We'll loop here for up to smt_acquire_wait_time microseconds; this is mainly
 * to wait out any sibling interrupt: many of them will complete quicker than
 * this.
 *
 * Finally, if we succeeded in acquiring the core, we'll flush the L1 cache as
 * mitigation against L1TF: no incompatible thread will now be able to populate
 * the L1 cache until *we* smt_release().
 *
 * smt_release()
 *
 * Simply unpoison ourselves similarly to smt_acquire(); smt_kick() will wait
 * for this to happen if needed.
 *
 * smt_begin_intr()
 *
 * In an interrupt prolog.  We're either a hilevel interrupt, or a pinning
 * interrupt.  In both cases, we mark our interrupt depth, and potentially
 * smt_kick().  This enforces exclusion, but doesn't otherwise modify
 * ->cs_state: we want the dispatcher code to essentially ignore interrupts.
 *
 * smt_end_intr()
 *
 * In an interrupt epilogue *or* thread_unpin().  In the first case, we never
 * slept, and we can simply decrement our counter.  In the second case, we're an
 * interrupt thread about to sleep: we'll still just decrement our counter, and
 * henceforth treat the thread as a normal thread when it next gets scheduled,
 * until it finally gets to its epilogue.
 *
 * smt_mark_unsafe() / smt_mark_safe()
 *
 * Mark the current thread as temporarily unsafe (guests should not be executing
 * while a sibling is marked unsafe).  This can be used for a thread that's
 * otherwise considered safe, if it needs to handle potentially sensitive data.
 * Right now, this means certain I/O handling operations that reach down into
 * the networking and ZFS sub-systems.
 *
 * smt_should_run(thread, cpu)
 *
 * This is used by the dispatcher when making scheduling decisions: if the
 * sibling is compatible with the given thread, we return B_TRUE. This is
 * essentially trying to guess if any subsequent smt_acquire() will fail, by
 * peeking at the sibling CPU's state.  The peek is racy, but if we get things
 * wrong, the "only" consequence is that smt_acquire() may lose.
 *
 * smt_adjust_cpu_score()
 *
 * Used when scoring other CPUs in disp_lowpri_cpu().  If we shouldn't run here,
 * we'll add a small penalty to the score.  This also makes sure a VCPU thread
 * migration behaves properly.
 *
 * smt_init() / smt_late_init()
 *
 * Set up SMT handling. If smt_boot_disable is set, smt_late_init(), which runs
 * late enough to be able to do so, will offline and mark CPU_DISABLED all the
 * siblings. smt_disable() can also be called after boot via psradm -Ha.
 */

#include <sys/archsystm.h>
#include <sys/disp.h>
#include <sys/cmt.h>
#include <sys/systm.h>
#include <sys/cpu.h>
#include <sys/var.h>
#include <sys/xc_levels.h>
#include <sys/cmn_err.h>
#include <sys/sysmacros.h>
#include <sys/x86_archext.h>
#include <sys/esunddi.h>
#include <sys/promif.h>
#include <sys/policy.h>
#include <sys/smt.h>

#define	CS_SHIFT (8)
#define	CS_MASK ((1 << CS_SHIFT) - 1)
#define	CS_MARK(s) ((s) & CS_MASK)
#define	CS_ZONE(s) ((s) >> CS_SHIFT)
#define	CS_MK(s, z) ((s) | (z << CS_SHIFT))

typedef enum cs_mark {
	CM_IDLE = 0,	/* running CPU idle thread */
	CM_THREAD,	/* running general non-VCPU thread */
	CM_UNSAFE,	/* running ->t_unsafe thread */
	CM_VCPU,	/* running VCPU thread */
	CM_POISONED	/* running in guest */
} cs_mark_t;

/* Double-check our false-sharing padding. */
CTASSERT(offsetof(cpu_smt_t, cs_sib) == 64);
CTASSERT(CM_IDLE == 0);
CTASSERT(CM_POISONED < (1 << CS_SHIFT));
CTASSERT(CM_POISONED > CM_VCPU);
CTASSERT(CM_VCPU > CM_UNSAFE);

static uint_t empty_pil = XC_CPUPOKE_PIL;

/*
 * If disabled, no SMT exclusion is performed, and system is potentially
 * vulnerable to L1TF if hyper-threading is enabled, and we don't have the "not
 * vulnerable" CPUID bit.
 */
int smt_exclusion = 1;

/*
 * How long smt_acquire() will spin trying to acquire the core, in
 * micro-seconds.  This is enough time to wait out a significant proportion of
 * interrupts.
 */
clock_t smt_acquire_wait_time = 64;

/*
 * Did we request a disable of SMT at boot time?
 */
int smt_boot_disable;

/*
 * Whether SMT is enabled.
 */
int smt_enabled = 1;

/*
 * We're adding an interrupt handler of some kind at the given PIL.  If this
 * happens to be the same PIL as XC_CPUPOKE_PIL, then we need to disable our
 * pil_needs_kick() optimization, as there is now potentially an unsafe
 * interrupt handler at that PIL.  This typically won't occur, so we're not that
 * careful about what's actually getting added, which CPU it's on, or if it gets
 * removed.  This also presumes that softints can't cover our empty_pil.
 */
void
smt_intr_alloc_pil(uint_t pil)
{
	ASSERT(pil <= PIL_MAX);

	if (empty_pil == pil)
		empty_pil = PIL_MAX + 1;
}

/*
 * If our sibling is also a VCPU thread from a different zone, we need one of
 * them to give up, otherwise they will just battle each other for exclusion
 * until they exhaust their quantum.
 *
 * We arbitrate between them by dispatch priority: clearly, a higher-priority
 * thread deserves to win the acquisition.  However, under CPU load, it'll be
 * very common to see both threads with ->t_pri == 1.  If so, we'll break the
 * tie by cpu_id (which is hopefully arbitrary enough).
 *
 * If we lose, the VMM code will take this as a hint to call
 * thread_affinity_set(CPU_BEST), which will likely migrate the VCPU thread
 * somewhere else.
 *
 * Note that all of this state examination is racy, as we don't own any locks
 * here.
 */
static boolean_t
yield_to_vcpu(cpu_t *sib, zoneid_t zoneid)
{
	cpu_smt_t *sibsmt = &sib->cpu_m.mcpu_smt;
	uint64_t sibstate = sibsmt->cs_state;

	/*
	 * If we're likely just waiting for an interrupt, don't yield.
	 */
	if (sibsmt->cs_intr_depth != 0)
		return (B_FALSE);

	/*
	 * We're only interested in VCPUs from a different zone.
	 */
	if (CS_MARK(sibstate) < CM_VCPU || CS_ZONE(sibstate) == zoneid)
		return (B_FALSE);

	if (curthread->t_pri < sib->cpu_dispatch_pri)
		return (B_TRUE);

	if (curthread->t_pri == sib->cpu_dispatch_pri &&
	    CPU->cpu_id < sib->cpu_id)
		return (B_TRUE);

	return (B_FALSE);
}

static inline boolean_t
sibling_compatible(cpu_smt_t *sibsmt, zoneid_t zoneid)
{
	uint64_t sibstate = sibsmt->cs_state;

	if (sibsmt->cs_intr_depth != 0)
		return (B_FALSE);

	if (CS_MARK(sibstate) == CM_UNSAFE)
		return (B_FALSE);

	if (CS_MARK(sibstate) == CM_IDLE)
		return (B_TRUE);

	return (CS_ZONE(sibstate) == zoneid);
}

int
smt_acquire(void)
{
	clock_t wait = smt_acquire_wait_time;
	cpu_smt_t *smt = &CPU->cpu_m.mcpu_smt;
	zoneid_t zoneid = getzoneid();
	cpu_smt_t *sibsmt;
	int ret = 0;

	ASSERT(!interrupts_enabled());

	if (smt->cs_sib == NULL) {
		/* For the "sequential" L1TF case. */
		spec_uarch_flush();
		return (1);
	}

	sibsmt = &smt->cs_sib->cpu_m.mcpu_smt;

	/* A VCPU thread should never change zone. */
	ASSERT3U(CS_ZONE(smt->cs_state), ==, zoneid);
	ASSERT3U(CS_MARK(smt->cs_state), ==, CM_VCPU);
	ASSERT3U(curthread->t_preempt, >=, 1);
	ASSERT(curthread->t_schedflag & TS_VCPU);

	while (ret == 0 && wait > 0) {

		if (yield_to_vcpu(smt->cs_sib, zoneid)) {
			ret = -1;
			break;
		}

		if (sibling_compatible(sibsmt, zoneid)) {
			lock_set(&sibsmt->cs_lock);

			if (sibling_compatible(sibsmt, zoneid)) {
				smt->cs_state = CS_MK(CM_POISONED, zoneid);
				sibsmt->cs_sibstate = CS_MK(CM_POISONED,
				    zoneid);
				membar_enter();
				ret = 1;
			}

			lock_clear(&sibsmt->cs_lock);
		} else {
			drv_usecwait(10);
			wait -= 10;
		}
	}

	DTRACE_PROBE4(smt__acquire, int, ret, uint64_t, sibsmt->cs_state,
	    uint64_t, sibsmt->cs_intr_depth, clock_t, wait);

	if (ret == 1)
		spec_uarch_flush();

	return (ret);
}

void
smt_release(void)
{
	cpu_smt_t *smt = &CPU->cpu_m.mcpu_smt;
	zoneid_t zoneid = getzoneid();
	cpu_smt_t *sibsmt;

	ASSERT(!interrupts_enabled());

	if (smt->cs_sib == NULL)
		return;

	ASSERT3U(CS_ZONE(smt->cs_state), ==, zoneid);
	ASSERT3U(CS_MARK(smt->cs_state), ==, CM_POISONED);
	ASSERT3U(curthread->t_preempt, >=, 1);

	sibsmt = &smt->cs_sib->cpu_m.mcpu_smt;

	lock_set(&sibsmt->cs_lock);

	smt->cs_state = CS_MK(CM_VCPU, zoneid);
	sibsmt->cs_sibstate = CS_MK(CM_VCPU, zoneid);
	membar_producer();

	lock_clear(&sibsmt->cs_lock);
}

static void
smt_kick(cpu_smt_t *smt, zoneid_t zoneid)
{
	uint64_t sibstate;

	ASSERT(LOCK_HELD(&smt->cs_lock));
	ASSERT(!interrupts_enabled());

	poke_cpu(smt->cs_sib->cpu_id);

	membar_consumer();
	sibstate = smt->cs_sibstate;

	if (CS_MARK(sibstate) != CM_POISONED || CS_ZONE(sibstate) == zoneid)
		return;

	lock_clear(&smt->cs_lock);

	/*
	 * Spin until we can see the sibling has been kicked out or is otherwise
	 * OK.
	 */
	for (;;) {
		membar_consumer();
		sibstate = smt->cs_sibstate;

		if (CS_MARK(sibstate) != CM_POISONED ||
		    CS_ZONE(sibstate) == zoneid)
			break;

		SMT_PAUSE();
	}

	lock_set(&smt->cs_lock);
}

static boolean_t
pil_needs_kick(uint_t pil)
{
	return (pil != empty_pil);
}

void
smt_begin_intr(uint_t pil)
{
	ulong_t flags;
	cpu_smt_t *smt;

	ASSERT(pil <= PIL_MAX);

	flags = intr_clear();
	smt = &CPU->cpu_m.mcpu_smt;

	if (smt->cs_sib == NULL) {
		intr_restore(flags);
		return;
	}

	if (atomic_inc_64_nv(&smt->cs_intr_depth) == 1 && pil_needs_kick(pil)) {
		lock_set(&smt->cs_lock);

		membar_consumer();

		if (CS_MARK(smt->cs_sibstate) == CM_POISONED)
			smt_kick(smt, GLOBAL_ZONEID);

		lock_clear(&smt->cs_lock);
	}

	intr_restore(flags);
}

void
smt_end_intr(void)
{
	ulong_t flags;
	cpu_smt_t *smt;

	flags = intr_clear();
	smt = &CPU->cpu_m.mcpu_smt;

	if (smt->cs_sib == NULL) {
		intr_restore(flags);
		return;
	}

	ASSERT3U(smt->cs_intr_depth, >, 0);
	atomic_dec_64(&smt->cs_intr_depth);

	intr_restore(flags);
}

static inline boolean_t
smt_need_kick(cpu_smt_t *smt, zoneid_t zoneid)
{
	membar_consumer();

	if (CS_MARK(smt->cs_sibstate) != CM_POISONED)
		return (B_FALSE);

	if (CS_MARK(smt->cs_state) == CM_UNSAFE)
		return (B_TRUE);

	return (CS_ZONE(smt->cs_sibstate) != zoneid);
}

void
smt_mark(void)
{
	zoneid_t zoneid = getzoneid();
	kthread_t *t = curthread;
	ulong_t flags;
	cpu_smt_t *smt;
	cpu_t *cp;

	flags = intr_clear();

	cp = CPU;
	smt = &cp->cpu_m.mcpu_smt;

	if (smt->cs_sib == NULL) {
		intr_restore(flags);
		return;
	}

	lock_set(&smt->cs_lock);

	/*
	 * If we were a nested interrupt and went through the resume_from_intr()
	 * path, we can now be resuming to a pinning interrupt thread; in which
	 * case, skip marking, until we later resume to a "real" thread.
	 */
	if (smt->cs_intr_depth > 0) {
		ASSERT3P(t->t_intr, !=, NULL);

		if (smt_need_kick(smt, zoneid))
			smt_kick(smt, zoneid);
		goto out;
	}

	if (t == t->t_cpu->cpu_idle_thread) {
		ASSERT3U(zoneid, ==, GLOBAL_ZONEID);
		smt->cs_state = CS_MK(CM_IDLE, zoneid);
	} else {
		uint64_t state = CM_THREAD;

		if (t->t_unsafe)
			state = CM_UNSAFE;
		else if (t->t_schedflag & TS_VCPU)
			state = CM_VCPU;

		smt->cs_state = CS_MK(state, zoneid);

		if (smt_need_kick(smt, zoneid))
			smt_kick(smt, zoneid);
	}

out:
	membar_producer();
	lock_clear(&smt->cs_lock);
	intr_restore(flags);
}

void
smt_begin_unsafe(void)
{
	curthread->t_unsafe++;
	smt_mark();
}

void
smt_end_unsafe(void)
{
	ASSERT3U(curthread->t_unsafe, >, 0);
	curthread->t_unsafe--;
	smt_mark();
}

void
smt_mark_as_vcpu(void)
{
	thread_lock(curthread);
	curthread->t_schedflag |= TS_VCPU;
	smt_mark();
	thread_unlock(curthread);
}

boolean_t
smt_should_run(kthread_t *t, cpu_t *cp)
{
	uint64_t sibstate;
	cpu_t *sib;

	if (t == t->t_cpu->cpu_idle_thread)
		return (B_TRUE);

	if ((sib = cp->cpu_m.mcpu_smt.cs_sib) == NULL)
		return (B_TRUE);

	sibstate = sib->cpu_m.mcpu_smt.cs_state;

	if ((t->t_schedflag & TS_VCPU)) {
		if (CS_MARK(sibstate) == CM_IDLE)
			return (B_TRUE);
		if (CS_MARK(sibstate) == CM_UNSAFE)
			return (B_FALSE);
		return (CS_ZONE(sibstate) == ttozone(t)->zone_id);
	}

	if (CS_MARK(sibstate) < CM_VCPU)
		return (B_TRUE);

	return (CS_ZONE(sibstate) == ttozone(t)->zone_id);
}

pri_t
smt_adjust_cpu_score(kthread_t *t, struct cpu *cp, pri_t score)
{
	if (smt_should_run(t, cp))
		return (score);

	/*
	 * If we're a VCPU thread scoring our current CPU, we are most likely
	 * asking to be rescheduled elsewhere after losing smt_acquire().  In
	 * this case, the current CPU is not a good choice, most likely, and we
	 * should go elsewhere.
	 */
	if ((t->t_schedflag & TS_VCPU) && cp == t->t_cpu && score < 0)
		return ((v.v_maxsyspri + 1) * 2);

	return (score + 1);
}

static void
set_smt_prop(void)
{
	(void) e_ddi_prop_update_string(DDI_DEV_T_NONE, ddi_root_node(),
	    "smt_enabled", smt_enabled ? "true" : "false");
}

static cpu_t *
smt_find_sibling(cpu_t *cp)
{
	for (uint_t i = 0; i < GROUP_SIZE(&cp->cpu_pg->cmt_pgs); i++) {
		pg_cmt_t *pg = GROUP_ACCESS(&cp->cpu_pg->cmt_pgs, i);
		group_t *cg = &pg->cmt_pg.pghw_pg.pg_cpus;

		if (pg->cmt_pg.pghw_hw != PGHW_IPIPE)
			continue;

		if (GROUP_SIZE(cg) == 1)
			break;

		if (GROUP_SIZE(cg) != 2) {
			panic("%u SMT threads unsupported", GROUP_SIZE(cg));
		}

		if (GROUP_ACCESS(cg, 0) != cp)
			return (GROUP_ACCESS(cg, 0));

		VERIFY3P(GROUP_ACCESS(cg, 1), !=, cp);

		return (GROUP_ACCESS(cg, 1));
	}

	return (NULL);
}

/*
 * Offline all siblings and mark as CPU_DISABLED. Note that any siblings that
 * can't be offlined (if it would leave an empty partition, or it's a spare, or
 * whatever) will fail the whole operation.
 */
int
smt_disable(void)
{
	int error = 0;

	ASSERT(MUTEX_HELD(&cpu_lock));

	if (secpolicy_ponline(CRED()) != 0)
		return (EPERM);

	if (!smt_enabled)
		return (0);

	for (size_t i = 0; i < NCPU; i++) {
		cpu_t *sib;
		cpu_t *cp;

		if ((cp = cpu_get(i)) == NULL)
			continue;

		/* NB: we don't necessarily have .mcpu_smt to use here. */
		if ((sib = smt_find_sibling(cp)) == NULL)
			continue;

		if (cp->cpu_id < sib->cpu_id)
			continue;

		if (cp->cpu_flags & CPU_DISABLED) {
			VERIFY(cp->cpu_flags & CPU_OFFLINE);
			continue;
		}

		if (cp->cpu_flags & (CPU_FAULTED | CPU_SPARE)) {
			error = EINVAL;
			break;
		}

		if ((cp->cpu_flags & (CPU_READY | CPU_OFFLINE)) != CPU_READY) {
			cp->cpu_flags |= CPU_DISABLED;
			continue;
		}

		if ((error = cpu_offline(cp, CPU_FORCED)) != 0)
			break;

		cp->cpu_flags |= CPU_DISABLED;
		cpu_set_state(cp);
	}

	if (error != 0)
		return (error);

	smt_enabled = 0;
	set_smt_prop();
	cmn_err(CE_NOTE, "!SMT / hyper-threading explicitly disabled.");
	return (0);
}

boolean_t
smt_can_enable(cpu_t *cp, int flags)
{
	VERIFY(cp->cpu_flags & CPU_DISABLED);

	return (!smt_boot_disable && (flags & CPU_FORCED));
}

/*
 * If we force-onlined a CPU_DISABLED CPU, then we can no longer consider the
 * system to be SMT-disabled in toto.
 */
void
smt_force_enabled(void)
{
	VERIFY(!smt_boot_disable);

	if (!smt_enabled)
		cmn_err(CE_NOTE, "!Disabled SMT sibling forced on-line.");

	smt_enabled = 1;
	set_smt_prop();
}

/*
 * Initialize SMT links.  We have to be careful here not to race with
 * smt_begin/end_intr(), which also complicates trying to do this initialization
 * from a cross-call; hence the slightly odd approach below.
 *
 * If we're going to disable SMT via smt_late_init(), we will avoid paying the
 * price here at all (we can't do it here since we're still too early in
 * main()).
 */
void
smt_init(void)
{
	boolean_t found_sibling = B_FALSE;
	cpu_t *scp = CPU;
	cpu_t *cp = scp;
	ulong_t flags;

	if (!smt_exclusion || smt_boot_disable)
		return;

	mutex_enter(&cpu_lock);

	do {
		thread_affinity_set(curthread, cp->cpu_id);
		flags = intr_clear();

		cp->cpu_m.mcpu_smt.cs_intr_depth = 0;
		cp->cpu_m.mcpu_smt.cs_state = CS_MK(CM_THREAD, GLOBAL_ZONEID);
		cp->cpu_m.mcpu_smt.cs_sibstate = CS_MK(CM_THREAD,
		    GLOBAL_ZONEID);
		ASSERT3P(cp->cpu_m.mcpu_smt.cs_sib, ==, NULL);
		cp->cpu_m.mcpu_smt.cs_sib = smt_find_sibling(cp);

		if (cp->cpu_m.mcpu_smt.cs_sib != NULL)
			found_sibling = B_TRUE;

		intr_restore(flags);
		thread_affinity_clear(curthread);
	} while ((cp = cp->cpu_next_onln) != scp);

	mutex_exit(&cpu_lock);

	if (!found_sibling)
		smt_enabled = 0;
}

void
smt_late_init(void)
{
	if (smt_boot_disable) {
		int err;

		mutex_enter(&cpu_lock);

		err = smt_disable();

		/*
		 * We're early enough in boot that nothing should have stopped
		 * us from offlining the siblings. As we didn't prepare our
		 * L1TF mitigation in this case, we need to panic.
		 */
		if (err) {
			cmn_err(CE_PANIC, "smt_disable() failed with %d", err);
		}

		mutex_exit(&cpu_lock);
	}

	if (smt_enabled)
		cmn_err(CE_NOTE, "!SMT enabled\n");

	set_smt_prop();
}
/* XPM */
static char *splashimage2_xpm[] = {
/* width height num_colors chars_per_pixel */
"   640   480       15            1",
/* colors */
"  c None",
"` c #000100",
". c #ff430f",
"# c #e2e0e3",
"a c #999999",
"b c #ffaa11",
"c c #333333",
"d c #ff5f11",
"e c #1e1207",
"f c #9f3909",
"g c #ef3f0e",
"h c #472904",
"i c #cccccc",
"j c #601a06",
"k c #998844",
/* pixels */
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"`````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````jg...........................................................................................................................................................................................................................................................................................................................................................................................gj````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"```````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````jg...............................................................................................................................................................................................................................................................................................................................................................................................gj``````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"``````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````j...................................................................................................................................................................................................................................................................................................................................................................................................j`````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"`````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````j.....................................................................................................................................................................................................................................................................................................................................................................................................j````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"`````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````g.....................................................................................................................................................................................................................................................................................................................................................................................................g````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````j.......................................................................................................................................................................................................................................................................................................................................................................................................j```````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````g.......................................................................................................................................................................................................................................................................................................................................................................................................g```````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````.........................................................................................................................................................................................................................................................................................................................................................................................................```````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````.........................................................................................................................................................................................................................................................................................................................................................................................................```````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````.........................................................................................................................................................................................................................................................................................................................................................................................................```````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````.........................................................................................................................................................................................................................................................................................................................................................................................................```````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````g.......................................................................................................................................................................................................................................................................................................................................................................................................g```````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````j.......................................................................................................................................................................................................................................................................................................................................................................................................j```````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"`````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````g.....................................................................................................................................................................................................................................................................................................................................................................................................g````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"`````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````j.....................................................................................................................................................................................................................................................................................................................................................................................................j````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"``````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````j...................................................................................................................................................................................................................................................................................................................................................................................................j`````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"```````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````jg...............................................................................................................................................................................................................................................................................................................................................................................................gj``````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"`````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````jg...........................................................................................................................................................................................................................................................................................................................................................................................gj````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"``````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````he````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"``````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````hk````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"```````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````be```````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"`````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````h`kbe``````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"`````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````bhhbbe`````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"`````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````hbhbbbh``h`````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"``````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````bbbbbbf`gh````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"``````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````hbbbbbbbgg````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"``````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````hkbbbbbbd.h```````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"``````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````hbbbbbbbb.g```````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"```````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````fbbbbbbb..h``````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"```````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````ebbbbbbbd.g``````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````ebbbbbbb..h`````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"`````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````hbbbbbb..ge````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"`````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````ekbbbbbb..g````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"``````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````dbbbbbb...g```````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"``````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````ebbkbbbd...f``````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"```````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````hbbdbbb....f`````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````hbbbbbb....f````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````hfbbbbbd....h```````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````edhbbbbb.....f``````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"`````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````ebdbbbbd.....f`````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"``````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````ebbbbbb......f````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"```````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````edbbbbb......g```````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````efbbbbd......ge`````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````hfhfbbbd.......h```````````````````hfffffh``````````````````````````````````````````````````````````````````````````````````````````````````````",
"`````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````ddfffdbd.......f````````````````hg.......gfh```````````````````````````````````````````````````````````````````````````````````````````````````",
"``````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````dbbbbbbd.......gh`````````````f.........ffgf``````````````````````````````````````````````````````````````````````````````````````````````````",
"```````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````dbbbbbbd........f```````````g.......gh```````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````fbbbbbbd........ge````````g.......f`````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````eefbbbbbd........gh``````f......gh``````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````hdffdddddd.........fe```h......he```````````````````````````````````````````````````````````````````````````````````````````````````````````",
"`````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````hddddddddd.........gh``g.....f`````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"```````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````fddddddddd.........fh......e`````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````efdddddddd...............g``````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"`````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````hhe`ehfddddddd..............f``````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"``````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````h.dddddddddddd..............e`````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````ehfdddddddddd.............f`````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"`````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````hhfddddd.............f````````````aac```caa````````````````````````````````````````````````````````````````````````````````````````",
"``````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````efffdddddd..............f````````````a#a```c#a````````````````````````````````````````````````````````````````````````````````````````",
"```````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````hfgddddd...............f````````````a#a```c#a````````````````````````````````````````````````````````````````````````````````````````",
"`````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````h...............df``````i#a```a#a```c#a````````````````````````````````````````````````````````````````````````````````````````",
"```````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````ef...............ddh``````###```a#a```c#a````````````````````````````````````````````````````````````````````````````````````````",
"`````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````hg.................dde``````i#i```a#a```c#a````````````````````````````````````````````````````````````````````````````````````````",
"```````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````h...........ddd````````c````a#a```c#a````````````````````````````````````````````````````````````````````````````````````````",
"``````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````f............ddf`````````````a#a```c#a````````````````````````````````````````````````````````````````````````````````````````",
"```````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````ehgggffg........ddh`````````````a#a```c#a``````````````````````````cce``````ecce```````````ecce```````````ecc````````````````````",
"``````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````efhh``hg.........dd````````cac```a#a```c#a````aac```````aac```aac`a####ac``ca####ae``````eai####ic```````ci####ac`````````````````",
"`````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````efg...........dh````````a#a```a#a```c#a````##c```````a#a```##ci##ii###ec###a###i`````ci###ai###ie````a###ai###e````````````````",
"``````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````hhhf........dd`````````a#a```a#a```c#a````##c```````a#a```##i#ae``c##a##ae``a##c```e###ce``ea##ie``e##a```eaa`````````````````",
"`````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````g........dh`````````a#a```a#a```c#a````##c```````a#a```###a`````a###c`````i#a```a##c``````c##a``c##````````````````````````",
"`````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````.........f``````````a#a```a#a```c#a````##c```````a#a```###e`````c##a``````a#a``c##c````````c##e`c##e```````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````h........de``````````a#a```a#a```c#a````##c```````a#a```##a``````c##c``````a#a``a#i``````````##c`e##ic``````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````g........h```````````a#a```a#a```c#a````##c```````a#a```##c``````c##e``````a#a``a#a``````````a#a``c###ice```````````````````",
"```````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````f........f````````````a#a```a#a```c#a````##c```````a#a```##c``````c##```````a#a``a#a``````````a#a```ea####ae`````````````````",
"``````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````f.......gg`````````````a#a```a#a```c#a````##c```````i#a```##c``````c##```````a#a``a#a``````````i#a`````eci##i`````````````````",
"`````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````f.......ghe`````````````a#a```a#a```c#a````##c``````c##a```##c``````c##```````a#a``c##e````````e##c````````c##c````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````f.......ge```````````````a#a```a#a```c#a````##c``````a##a```##c``````c##```````a#a``e##a````````a##``````````i#a````````````````",
"``````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````hg.......ge````````````````a#a```a#a```c##````a#a`````a###a```##c``````c##```````a#a```a##c``````c##c``eae`````i#a````````````````",
"`````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````f........ge`````````````````a#a```a##ce`c##ac``c##accca#ii#a```##c``````c##```````a#a````i##ac`eca##a```a##ac`ca##c````````````````",
"```````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````fg........g```````````````````a#a```c###a``a###``ei######iea#a```##c``````c##```````a#a````ea########a````ci#######a`````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````efg....g.....e```````````````````cac````caac``ecaa```eai##ac``cac```aae``````eaa```````cac``````cai##aae```````cai##ac``````````````````",
"`````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````hfg....gfhg....h```````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"```````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````h....gfhe`hg....f````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"`````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````e`````ef..gg.g`````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"`````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````efg.gff.ge`````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"``````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````hfg..gf`h.ge``````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````fg...gf``f..h```````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"``````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````hg..gfh``ef..h````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"`````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````hgffe````eg.ge`````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"`````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````eg.ge``````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````eg.f````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````ggh`````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"```````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````hfe``````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````",
"````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````````"
};
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved.
 * Copyright 2021 Joyent, Inc.
 * Copyright 2021 Oxide Computer Company
 */

/*	Copyright (c) 1990, 1991 UNIX System Laboratories, Inc. */
/*	Copyright (c) 1984, 1986, 1987, 1988, 1989, 1990 AT&T   */
/*	All Rights Reserved   */

#include <sys/types.h>
#include <sys/stdbool.h>
#include <sys/param.h>
#include <sys/sysmacros.h>
#include <sys/signal.h>
#include <sys/systm.h>
#include <sys/user.h>
#include <sys/mman.h>
#include <sys/class.h>
#include <sys/proc.h>
#include <sys/procfs.h>
#include <sys/buf.h>
#include <sys/kmem.h>
#include <sys/cred.h>
#include <sys/archsystm.h>
#include <sys/vmparam.h>
#include <sys/prsystm.h>
#include <sys/reboot.h>
#include <sys/uadmin.h>
#include <sys/vfs.h>
#include <sys/vnode.h>
#include <sys/file.h>
#include <sys/session.h>
#include <sys/ucontext.h>
#include <sys/dnlc.h>
#include <sys/var.h>
#include <sys/cmn_err.h>
#include <sys/debugreg.h>
#include <sys/thread.h>
#include <sys/vtrace.h>
#include <sys/consdev.h>
#include <sys/psw.h>
#include <sys/regset.h>
#include <sys/privregs.h>
#include <sys/cpu.h>
#include <sys/stack.h>
#include <sys/swap.h>
#include <vm/hat.h>
#include <vm/anon.h>
#include <vm/as.h>
#include <vm/page.h>
#include <vm/seg.h>
#include <vm/seg_kmem.h>
#include <vm/seg_map.h>
#include <vm/seg_vn.h>
#include <sys/exec.h>
#include <sys/acct.h>
#include <sys/core.h>
#include <sys/corectl.h>
#include <sys/modctl.h>
#include <sys/tuneable.h>
#include <c2/audit.h>
#include <sys/bootconf.h>
#include <sys/brand.h>
#include <sys/dumphdr.h>
#include <sys/promif.h>
#include <sys/systeminfo.h>
#include <sys/kdi.h>
#include <sys/contract_impl.h>
#include <sys/x86_archext.h>
#include <sys/segments.h>
#include <sys/ontrap.h>
#include <sys/cpu.h>
#ifdef __xpv
#include <sys/hypervisor.h>
#endif

/*
 * Compare the version of boot that boot says it is against
 * the version of boot the kernel expects.
 */
int
check_boot_version(int boots_version)
{
	if (boots_version == BO_VERSION)
		return (0);

	prom_printf("Wrong boot interface - kernel needs v%d found v%d\n",
	    BO_VERSION, boots_version);
	prom_panic("halting");
	/*NOTREACHED*/
}

/*
 * Process the physical installed list for boot.
 * Finds:
 * 1) the pfn of the highest installed physical page,
 * 2) the number of pages installed
 * 3) the number of distinct contiguous regions these pages fall into.
 * 4) the number of contiguous memory ranges
 */
void
installed_top_size_ex(
	struct memlist *list,	/* pointer to start of installed list */
	pfn_t *high_pfn,	/* return ptr for top value */
	pgcnt_t *pgcnt,		/* return ptr for sum of installed pages */
	int	*ranges)	/* return ptr for the count of contig. ranges */
{
	pfn_t top = 0;
	pgcnt_t sumpages = 0;
	pfn_t highp;		/* high page in a chunk */
	int cnt = 0;

	for (; list; list = list->ml_next) {
		++cnt;
		highp = (list->ml_address + list->ml_size - 1) >> PAGESHIFT;
		if (top < highp)
			top = highp;
		sumpages += btop(list->ml_size);
	}

	*high_pfn = top;
	*pgcnt = sumpages;
	*ranges = cnt;
}

void
installed_top_size(
	struct memlist *list,	/* pointer to start of installed list */
	pfn_t *high_pfn,	/* return ptr for top value */
	pgcnt_t *pgcnt)		/* return ptr for sum of installed pages */
{
	int ranges;

	installed_top_size_ex(list, high_pfn, pgcnt, &ranges);
}

void
phys_install_has_changed(void)
{}

/*
 * Copy in a memory list from boot to kernel, with a filter function
 * to remove pages. The filter function can increase the address and/or
 * decrease the size to filter out pages.  It will also align addresses and
 * sizes to PAGESIZE.
 */
void
copy_memlist_filter(
	struct memlist *src,
	struct memlist **dstp,
	void (*filter)(uint64_t *, uint64_t *))
{
	struct memlist *dst, *prev;
	uint64_t addr;
	uint64_t size;
	uint64_t eaddr;

	dst = *dstp;
	prev = dst;

	/*
	 * Move through the memlist applying a filter against
	 * each range of memory. Note that we may apply the
	 * filter multiple times against each memlist entry.
	 */
	for (; src; src = src->ml_next) {
		addr = P2ROUNDUP(src->ml_address, PAGESIZE);
		eaddr = P2ALIGN(src->ml_address + src->ml_size, PAGESIZE);
		while (addr < eaddr) {
			size = eaddr - addr;
			if (filter != NULL)
				filter(&addr, &size);
			if (size == 0)
				break;
			dst->ml_address = addr;
			dst->ml_size = size;
			dst->ml_next = 0;
			if (prev == dst) {
				dst->ml_prev = 0;
				dst++;
			} else {
				dst->ml_prev = prev;
				prev->ml_next = dst;
				dst++;
				prev++;
			}
			addr += size;
		}
	}

	*dstp = dst;
}

/*
 * Kernel setup code, called from startup().
 */
void
kern_setup1(void)
{
	proc_t *pp;

	pp = &p0;

	proc_sched = pp;

	/*
	 * Initialize process 0 data structures
	 */
	pp->p_stat = SRUN;
	pp->p_flag = SSYS;

	pp->p_pidp = &pid0;
	pp->p_pgidp = &pid0;
	pp->p_sessp = &session0;
	pp->p_tlist = &t0;
	pid0.pid_pglink = pp;
	pid0.pid_pgtail = pp;

	/*
	 * XXX - we asssume that the u-area is zeroed out except for
	 * ttolwp(curthread)->lwp_regs.
	 */
	PTOU(curproc)->u_cmask = (mode_t)CMASK;

	thread_init();		/* init thread_free list */
	pid_init();		/* initialize pid (proc) table */
	contract_init();	/* initialize contracts */

	init_pages_pp_maximum();
}

/*
 * Load a procedure into a thread.
 */
void
thread_load(kthread_t *t, void (*start)(), caddr_t arg, size_t len)
{
	caddr_t sp;
	size_t framesz;
	caddr_t argp;
	long *p;
	extern void thread_start();

	/*
	 * Push a "c" call frame onto the stack to represent
	 * the caller of "start".
	 */
	sp = t->t_stk;
	ASSERT(((uintptr_t)t->t_stk & (STACK_ENTRY_ALIGN - 1)) == 0);
	if (len != 0) {
		/*
		 * the object that arg points at is copied into the
		 * caller's frame.
		 */
		framesz = SA(len);
		sp -= framesz;
		ASSERT(sp > t->t_stkbase);
		argp = sp + SA(MINFRAME);
		bcopy(arg, argp, len);
		arg = argp;
	}
	/*
	 * Set up arguments (arg and len) on the caller's stack frame.
	 */
	p = (long *)sp;

	*--p = 0;		/* fake call */
	*--p = 0;		/* null frame pointer terminates stack trace */
	*--p = (long)len;
	*--p = (intptr_t)arg;
	*--p = (intptr_t)start;

	/*
	 * initialize thread to resume at thread_start() which will
	 * turn around and invoke (*start)(arg, len).
	 */
	t->t_pc = (uintptr_t)thread_start;
	t->t_sp = (uintptr_t)p;

	ASSERT((t->t_sp & (STACK_ENTRY_ALIGN - 1)) == 0);
}

/*
 * load user registers into lwp.
 */
/*ARGSUSED2*/
void
lwp_load(klwp_t *lwp, gregset_t grp, uintptr_t thrptr)
{
	struct regs *rp = lwptoregs(lwp);

	setgregs(lwp, grp);
	rp->r_ps = PSL_USER;

	/*
	 * For 64-bit lwps, we allow one magic %fs selector value, and one
	 * magic %gs selector to point anywhere in the address space using
	 * %fsbase and %gsbase behind the scenes.  libc uses %fs to point
	 * at the ulwp_t structure.
	 *
	 * For 32-bit lwps, libc wedges its lwp thread pointer into the
	 * ucontext ESP slot (which is otherwise irrelevant to setting a
	 * ucontext) and LWPGS_SEL value into gregs[REG_GS].  This is so
	 * syslwp_create() can atomically setup %gs.
	 *
	 * See setup_context() in libc.
	 */
#ifdef _SYSCALL32_IMPL
	if (lwp_getdatamodel(lwp) == DATAMODEL_ILP32) {
		if (grp[REG_GS] == LWPGS_SEL)
			(void) lwp_setprivate(lwp, _LWP_GSBASE, thrptr);
	} else {
		/*
		 * See lwp_setprivate in kernel and setup_context in libc.
		 *
		 * Currently libc constructs a ucontext from whole cloth for
		 * every new (not main) lwp created.  For 64 bit processes
		 * %fsbase is directly set to point to current thread pointer.
		 * In the past (solaris 10) %fs was also set LWPFS_SEL to
		 * indicate %fsbase. Now we use the null GDT selector for
		 * this purpose. LWP[FS|GS]_SEL are only intended for 32 bit
		 * processes. To ease transition we support older libcs in
		 * the newer kernel by forcing %fs or %gs selector to null
		 * by calling lwp_setprivate if LWP[FS|GS]_SEL is passed in
		 * the ucontext.  This is should be ripped out at some future
		 * date.  Another fix would be for libc to do a getcontext
		 * and inherit the null %fs/%gs from the current context but
		 * that means an extra system call and could hurt performance.
		 */
		if (grp[REG_FS] == 0x1bb) /* hard code legacy LWPFS_SEL */
			(void) lwp_setprivate(lwp, _LWP_FSBASE,
			    (uintptr_t)grp[REG_FSBASE]);

		if (grp[REG_GS] == 0x1c3) /* hard code legacy LWPGS_SEL */
			(void) lwp_setprivate(lwp, _LWP_GSBASE,
			    (uintptr_t)grp[REG_GSBASE]);
	}
#else
	if (grp[GS] == LWPGS_SEL)
		(void) lwp_setprivate(lwp, _LWP_GSBASE, thrptr);
#endif

	lwp->lwp_eosys = JUSTRETURN;
	lwptot(lwp)->t_post_sys = 1;
}

/*
 * set syscall()'s return values for a lwp.
 */
void
lwp_setrval(klwp_t *lwp, int v1, int v2)
{
	lwptoregs(lwp)->r_ps &= ~PS_C;
	lwptoregs(lwp)->r_r0 = v1;
	lwptoregs(lwp)->r_r1 = v2;
}

/*
 * set syscall()'s return values for a lwp.
 */
void
lwp_setsp(klwp_t *lwp, caddr_t sp)
{
	lwptoregs(lwp)->r_sp = (intptr_t)sp;
}

/*
 * Copy regs from parent to child.
 */
void
lwp_forkregs(klwp_t *lwp, klwp_t *clwp)
{
	struct pcb *pcb = &clwp->lwp_pcb;
	struct regs *rp = lwptoregs(lwp);

	if (!PCB_NEED_UPDATE_SEGS(pcb)) {
		pcb->pcb_ds = rp->r_ds;
		pcb->pcb_es = rp->r_es;
		pcb->pcb_fs = rp->r_fs;
		pcb->pcb_gs = rp->r_gs;
		PCB_SET_UPDATE_SEGS(pcb);
		lwptot(clwp)->t_post_sys = 1;
	}
	ASSERT(lwptot(clwp)->t_post_sys);

	fp_lwp_dup(clwp);

	bcopy(lwp->lwp_regs, clwp->lwp_regs, sizeof (struct regs));
}

/*
 * This function is currently unused on x86.
 */
/*ARGSUSED*/
void
lwp_freeregs(klwp_t *lwp, int isexec)
{}

/*
 * This function is currently unused on x86.
 */
void
lwp_pcb_exit(void)
{}

/*
 * Lwp context ops for segment registers.
 */

/*
 * Every time we come into the kernel (syscall, interrupt or trap
 * but not fast-traps) we capture the current values of the user's
 * segment registers into the lwp's reg structure. This includes
 * lcall for i386 generic system call support since it is handled
 * as a segment-not-present trap.
 *
 * Here we save the current values from the lwp regs into the pcb
 * and or PCB_UPDATE_SEGS (1) in pcb->pcb_rupdate to tell the rest
 * of the kernel that the pcb copy of the segment registers is the
 * current one.  This ensures the lwp's next trip to user land via
 * update_sregs.  Finally we set t_post_sys to ensure that no
 * system call fast-path's its way out of the kernel via sysret.
 *
 * (This means that we need to have interrupts disabled when we
 * test t->t_post_sys in the syscall handlers; if the test fails,
 * we need to keep interrupts disabled until we return to userland
 * so we can't be switched away.)
 *
 * As a result of all this, we don't really have to do a whole lot
 * if the thread is just mucking about in the kernel, switching on
 * and off the cpu for whatever reason it feels like. And yet we
 * still preserve fast syscalls, cause if we -don't- get
 * descheduled, we never come here either.
 */

#define	VALID_LWP_DESC(udp) ((udp)->usd_type == SDT_MEMRWA && \
	    (udp)->usd_p == 1 && (udp)->usd_dpl == SEL_UPL)

/*ARGSUSED*/
void
lwp_segregs_save(void *arg)
{
	klwp_t *lwp = arg;
	pcb_t *pcb = &lwp->lwp_pcb;
	struct regs *rp;

	ASSERT(VALID_LWP_DESC(&pcb->pcb_fsdesc));
	ASSERT(VALID_LWP_DESC(&pcb->pcb_gsdesc));

	if (!PCB_NEED_UPDATE_SEGS(pcb)) {
		rp = lwptoregs(lwp);

		/*
		 * If there's no update already pending, capture the current
		 * %ds/%es/%fs/%gs values from lwp's regs in case the user
		 * changed them; %fsbase and %gsbase are privileged so the
		 * kernel versions of these registers in pcb_fsbase and
		 * pcb_gsbase are always up-to-date.
		 */
		pcb->pcb_ds = rp->r_ds;
		pcb->pcb_es = rp->r_es;
		pcb->pcb_fs = rp->r_fs;
		pcb->pcb_gs = rp->r_gs;
		PCB_SET_UPDATE_SEGS(pcb);
		lwp->lwp_thread->t_post_sys = 1;
	}

#if !defined(__xpv)	/* XXPV not sure if we can re-read gdt? */
	ASSERT(bcmp(&CPU->cpu_gdt[GDT_LWPFS], &lwp->lwp_pcb.pcb_fsdesc,
	    sizeof (lwp->lwp_pcb.pcb_fsdesc)) == 0);
	ASSERT(bcmp(&CPU->cpu_gdt[GDT_LWPGS], &lwp->lwp_pcb.pcb_gsdesc,
	    sizeof (lwp->lwp_pcb.pcb_gsdesc)) == 0);
#endif
}

/*
 * Update the segment registers with new values from the pcb.
 *
 * We have to do this carefully, and in the following order,
 * in case any of the selectors points at a bogus descriptor.
 * If they do, we'll catch trap with on_trap and return 1.
 * returns 0 on success.
 *
 * This is particularly tricky for %gs.
 * This routine must be executed under a cli.
 */
int
update_sregs(struct regs *rp,  klwp_t *lwp)
{
	pcb_t *pcb = &lwp->lwp_pcb;
	ulong_t	kgsbase;
	on_trap_data_t	otd;
	int rc;

	if (!on_trap(&otd, OT_SEGMENT_ACCESS)) {
		rc = 0;
#if defined(__xpv)
		/*
		 * On the hyervisor this is easy. The hypercall below will
		 * swapgs and load %gs with the user selector. If the user
		 * selector is bad the hypervisor will catch the fault and
		 * load %gs with the null selector instead. Either way the
		 * kernel's gsbase is not damaged.
		 */
		kgsbase = (ulong_t)CPU;
		if (HYPERVISOR_set_segment_base(SEGBASE_GS_USER_SEL,
		    pcb->pcb_gs) != 0) {
				no_trap();
				return (1);
		}

		rp->r_gs = pcb->pcb_gs;
		ASSERT((cpu_t *)kgsbase == CPU);

#else	/* __xpv */

		/*
		 * A little more complicated running native.
		 */
		kgsbase = (ulong_t)CPU;
		__set_gs(pcb->pcb_gs);

		/*
		 * If __set_gs fails it's because the new %gs is a bad %gs,
		 * we'll be taking a trap but with the original %gs and %gsbase
		 * undamaged (i.e. pointing at curcpu).
		 *
		 * We've just mucked up the kernel's gsbase.  Oops.  In
		 * particular we can't take any traps at all.  Make the newly
		 * computed gsbase be the hidden gs via swapgs, and fix
		 * the kernel's gsbase back again. Later, when we return to
		 * userland we'll swapgs again restoring gsbase just loaded
		 * above.
		 */
		__asm__ __volatile__("mfence; swapgs");

		rp->r_gs = pcb->pcb_gs;

		/*
		 * Restore kernel's gsbase. Note that this also serializes any
		 * attempted speculation from loading the user-controlled
		 * %gsbase.
		 */
		wrmsr(MSR_AMD_GSBASE, kgsbase);

#endif	/* __xpv */

		/*
		 * Only override the descriptor base address if
		 * r_gs == LWPGS_SEL or if r_gs == NULL. A note on
		 * NULL descriptors -- 32-bit programs take faults
		 * if they deference NULL descriptors; however,
		 * when 64-bit programs load them into %fs or %gs,
		 * they DONT fault -- only the base address remains
		 * whatever it was from the last load.   Urk.
		 *
		 * XXX - note that lwp_setprivate now sets %fs/%gs to the
		 * null selector for 64 bit processes. Whereas before
		 * %fs/%gs were set to LWP(FS|GS)_SEL regardless of
		 * the process's data model. For now we check for both
		 * values so that the kernel can also support the older
		 * libc. This should be ripped out at some point in the
		 * future.
		 */
		if (pcb->pcb_gs == LWPGS_SEL || pcb->pcb_gs == 0) {
#if defined(__xpv)
			if (HYPERVISOR_set_segment_base(SEGBASE_GS_USER,
			    pcb->pcb_gsbase)) {
				no_trap();
				return (1);
			}
#else
			wrmsr(MSR_AMD_KGSBASE, pcb->pcb_gsbase);
#endif
		}

		__set_ds(pcb->pcb_ds);
		rp->r_ds = pcb->pcb_ds;

		__set_es(pcb->pcb_es);
		rp->r_es = pcb->pcb_es;

		__set_fs(pcb->pcb_fs);
		rp->r_fs = pcb->pcb_fs;

		/*
		 * Same as for %gs
		 */
		if (pcb->pcb_fs == LWPFS_SEL || pcb->pcb_fs == 0) {
#if defined(__xpv)
			if (HYPERVISOR_set_segment_base(SEGBASE_FS,
			    pcb->pcb_fsbase)) {
				no_trap();
				return (1);
			}
#else
			wrmsr(MSR_AMD_FSBASE, pcb->pcb_fsbase);
#endif
		}

	} else {
		cli();
		rc = 1;
	}
	no_trap();
	return (rc);
}

/*
 * Make sure any stale selectors are cleared from the segment registers
 * by putting KDS_SEL (the kernel's default %ds gdt selector) into them.
 * This is necessary because the kernel itself does not use %es, %fs, nor
 * %ds. (%cs and %ss are necessary, and are set up by the kernel - along with
 * %gs - to point to the current cpu struct.) If we enter kmdb while in the
 * kernel and resume with a stale ldt or brandz selector sitting there in a
 * segment register, kmdb will #gp fault if the stale selector points to,
 * for example, an ldt in the context of another process.
 *
 * WARNING: Intel and AMD chips behave differently when storing
 * the null selector into %fs and %gs while in long mode. On AMD
 * chips fsbase and gsbase are not cleared. But on Intel chips, storing
 * a null selector into %fs or %gs has the side effect of clearing
 * fsbase or gsbase. For that reason we use KDS_SEL, which has
 * consistent behavor between AMD and Intel.
 *
 * Caller responsible for preventing cpu migration.
 */
void
reset_sregs(void)
{
	ulong_t kgsbase = (ulong_t)CPU;

	ASSERT(curthread->t_preempt != 0 || getpil() >= DISP_LEVEL);

	cli();
	__set_gs(KGS_SEL);

	/*
	 * restore kernel gsbase
	 */
#if defined(__xpv)
	xen_set_segment_base(SEGBASE_GS_KERNEL, kgsbase);
#else
	wrmsr(MSR_AMD_GSBASE, kgsbase);
#endif

	sti();

	__set_ds(KDS_SEL);
	__set_es(0 | SEL_KPL);	/* selector RPL not ring 0 on hypervisor */
	__set_fs(KFS_SEL);
}


#ifdef _SYSCALL32_IMPL

/*
 * Make it impossible for a process to change its data model.
 * We do this by toggling the present bits for the 32 and
 * 64-bit user code descriptors. That way if a user lwp attempts
 * to change its data model (by using the wrong code descriptor in
 * %cs) it will fault immediately. This also allows us to simplify
 * assertions and checks in the kernel.
 */

static void
gdt_ucode_model(model_t model)
{
	kpreempt_disable();
	if (model == DATAMODEL_NATIVE) {
		gdt_update_usegd(GDT_UCODE, &ucs_on);
		gdt_update_usegd(GDT_U32CODE, &ucs32_off);
	} else {
		gdt_update_usegd(GDT_U32CODE, &ucs32_on);
		gdt_update_usegd(GDT_UCODE, &ucs_off);
	}
	kpreempt_enable();
}

#endif	/* _SYSCALL32_IMPL */

/*
 * Restore lwp private fs and gs segment descriptors
 * on current cpu's GDT.
 */
static void
lwp_segregs_restore(void *arg)
{
	klwp_t *lwp = arg;
	pcb_t *pcb = &lwp->lwp_pcb;

	ASSERT(VALID_LWP_DESC(&pcb->pcb_fsdesc));
	ASSERT(VALID_LWP_DESC(&pcb->pcb_gsdesc));

#ifdef	_SYSCALL32_IMPL
	gdt_ucode_model(DATAMODEL_NATIVE);
#endif

	gdt_update_usegd(GDT_LWPFS, &pcb->pcb_fsdesc);
	gdt_update_usegd(GDT_LWPGS, &pcb->pcb_gsdesc);

}

#ifdef _SYSCALL32_IMPL

static void
lwp_segregs_restore32(void *arg)
{
	klwp_t *lwp = arg;
	pcb_t *pcb = &lwp->lwp_pcb;

	ASSERT(VALID_LWP_DESC(&lwp->lwp_pcb.pcb_fsdesc));
	ASSERT(VALID_LWP_DESC(&lwp->lwp_pcb.pcb_gsdesc));

	gdt_ucode_model(DATAMODEL_ILP32);
	gdt_update_usegd(GDT_LWPFS, &pcb->pcb_fsdesc);
	gdt_update_usegd(GDT_LWPGS, &pcb->pcb_gsdesc);
}

#endif	/* _SYSCALL32_IMPL */

static const struct ctxop_template brand_interpose_ctxop_tpl = {
	.ct_rev		= CTXOP_TPL_REV,
	.ct_save	= brand_interpositioning_disable,
	.ct_restore	= brand_interpositioning_enable,
	.ct_exit	= brand_interpositioning_disable,
};

/*
 * If this is a process in a branded zone, then we want it to use the brand
 * syscall entry points instead of the standard Solaris entry points.  This
 * routine must be called when a new lwp is created within a branded zone
 * or when an existing lwp moves into a branded zone via a zone_enter()
 * operation.
 */
void
lwp_attach_brand_hdlrs(klwp_t *lwp)
{
	kthread_t *t = lwptot(lwp);

	ASSERT(PROC_IS_BRANDED(lwptoproc(lwp)));

	/* Confirm that brand interposition ctxop is not already present */
	ASSERT0(ctxop_remove(t, &brand_interpose_ctxop_tpl, NULL));

	ctxop_install(t, &brand_interpose_ctxop_tpl, NULL);

	if (t == curthread) {
		kpreempt_disable();
		brand_interpositioning_enable(NULL);
		kpreempt_enable();
	}
}

/*
 * If this is a process in a branded zone, then we want it to disable the
 * brand syscall entry points.  This routine must be called when the last
 * lwp in a process is exiting in proc_exit().
 */
void
lwp_detach_brand_hdlrs(klwp_t *lwp)
{
	kthread_t *t = lwptot(lwp);

	ASSERT(PROC_IS_BRANDED(lwptoproc(lwp)));
	if (t == curthread)
		kpreempt_disable();

	/* Remove the original context handlers */
	ctxop_remove(t, &brand_interpose_ctxop_tpl, NULL);

	if (t == curthread) {
		/* Cleanup our MSR and IDT entries. */
		brand_interpositioning_disable(NULL);
		kpreempt_enable();
	}
}

static const struct ctxop_template sep_tpl = {
	.ct_rev		= CTXOP_TPL_REV,
	.ct_save	= sep_save,
	.ct_restore	= sep_restore,
};

/*
 * Add any lwp-associated context handlers to the lwp at the beginning
 * of the lwp's useful life.
 *
 * All paths which create lwp's invoke lwp_create(); lwp_create()
 * invokes lwp_stk_init() which initializes the stack, sets up
 * lwp_regs, and invokes this routine.
 *
 * All paths which destroy lwp's invoke lwp_exit() to rip the lwp
 * apart and put it on 'lwp_deathrow'; if the lwp is destroyed it
 * ends up in thread_free() which invokes freectx(t, 0) before
 * invoking lwp_stk_fini().  When the lwp is recycled from death
 * row, lwp_stk_fini() is invoked, then thread_free(), and thus
 * freectx(t, 0) as before.
 *
 * In the case of exec, the surviving lwp is thoroughly scrubbed
 * clean; exec invokes freectx(t, 1) to destroy associated contexts.
 * On the way back to the new image, it invokes setregs() which
 * in turn invokes this routine.
 */
void
lwp_installctx(klwp_t *lwp)
{
	kthread_t *t = lwptot(lwp);
	bool thisthread = (t == curthread);
	struct ctxop *ctx;

	const struct ctxop_template segreg_tpl = {
		.ct_rev		= CTXOP_TPL_REV,
		.ct_save	= lwp_segregs_save,
#ifdef _SYSCALL32_IMPL
		.ct_restore	= lwp_getdatamodel(lwp) == DATAMODEL_NATIVE ?
	    lwp_segregs_restore : lwp_segregs_restore32
#else
		.ct_restore	= lwp_segregs_restore;
#endif
	};

	/*
	 * Install the basic lwp context handlers on each lwp.
	 *
	 * On the amd64 kernel, the context handlers are responsible for
	 * virtualizing %ds, %es, %fs, and %gs to the lwp.  The register
	 * values are only ever changed via sys_rtt when the
	 * PCB_UPDATE_SEGS bit (1) is set in pcb->pcb_rupdate. Only
	 * sys_rtt gets to clear the bit.
	 *
	 * On the i386 kernel, the context handlers are responsible for
	 * virtualizing %gs/%fs to the lwp by updating the per-cpu GDTs
	 */
	ASSERT0(ctxop_remove(t, &segreg_tpl, lwp));

	ctx = ctxop_allocate(&segreg_tpl, lwp);
	if (thisthread) {
		kpreempt_disable();
	}
	ctxop_attach(t, ctx);
	if (thisthread) {
		/*
		 * Since we're the right thread, set the values in the GDT
		 */
		segreg_tpl.ct_restore(lwp);
		kpreempt_enable();
	}

	/*
	 * If we have sysenter/sysexit instructions enabled, we need
	 * to ensure that the hardware mechanism is kept up-to-date with the
	 * lwp's kernel stack pointer across context switches.
	 *
	 * sep_save zeros the sysenter stack pointer msr; sep_restore sets
	 * it to the lwp's kernel stack pointer (kstktop).
	 */
	if (is_x86_feature(x86_featureset, X86FSET_SEP)) {
		caddr_t kstktop = (caddr_t)lwp->lwp_regs;

		ASSERT0(ctxop_remove(t, &sep_tpl, kstktop));

		ctx = ctxop_allocate(&sep_tpl, kstktop);
		if (thisthread) {
			kpreempt_disable();
		}
		ctxop_attach(t, ctx);
		if (thisthread) {
			/*
			 * We're the right thread, so set the stack pointer
			 * for the first sysenter instruction to use
			 */
			sep_restore(kstktop);
			kpreempt_enable();
		}
	}

	if (PROC_IS_BRANDED(ttoproc(t)))
		lwp_attach_brand_hdlrs(lwp);
}

/*
 * Clear registers on exec(2).
 */
void
setregs(uarg_t *args)
{
	struct regs *rp;
	kthread_t *t = curthread;
	klwp_t *lwp = ttolwp(t);
	pcb_t *pcb = &lwp->lwp_pcb;
	greg_t sp;

	/*
	 * Initialize user registers
	 */
	(void) save_syscall_args();	/* copy args from registers first */
	rp = lwptoregs(lwp);
	sp = rp->r_sp;
	bzero(rp, sizeof (*rp));

	rp->r_ss = UDS_SEL;
	rp->r_sp = sp;
	rp->r_pc = args->entry;
	rp->r_ps = PSL_USER;

	pcb->pcb_fs = pcb->pcb_gs = 0;
	pcb->pcb_fsbase = pcb->pcb_gsbase = 0;

	if (ttoproc(t)->p_model == DATAMODEL_NATIVE) {

		rp->r_cs = UCS_SEL;

		/*
		 * Only allow 64-bit user code descriptor to be present.
		 */
		gdt_ucode_model(DATAMODEL_NATIVE);

		/*
		 * Arrange that the virtualized %fs and %gs GDT descriptors
		 * have a well-defined initial state (present, ring 3
		 * and of type data).
		 */
		pcb->pcb_fsdesc = pcb->pcb_gsdesc = zero_udesc;

		/*
		 * thrptr is either NULL or a value used by DTrace.
		 * 64-bit processes use %fs as their "thread" register.
		 */
		if (args->thrptr)
			(void) lwp_setprivate(lwp, _LWP_FSBASE, args->thrptr);

	} else {

		rp->r_cs = U32CS_SEL;
		rp->r_ds = rp->r_es = UDS_SEL;

		/*
		 * only allow 32-bit user code selector to be present.
		 */
		gdt_ucode_model(DATAMODEL_ILP32);

		pcb->pcb_fsdesc = pcb->pcb_gsdesc = zero_u32desc;

		/*
		 * thrptr is either NULL or a value used by DTrace.
		 * 32-bit processes use %gs as their "thread" register.
		 */
		if (args->thrptr)
			(void) lwp_setprivate(lwp, _LWP_GSBASE, args->thrptr);

	}

	pcb->pcb_ds = rp->r_ds;
	pcb->pcb_es = rp->r_es;
	PCB_SET_UPDATE_SEGS(pcb);

	lwp->lwp_eosys = JUSTRETURN;
	t->t_post_sys = 1;

	/*
	 * Add the lwp context handlers that virtualize segment registers,
	 * and/or system call stacks etc.
	 */
	lwp_installctx(lwp);

	/*
	 * Reset the FPU flags and then initialize the FPU for this lwp.
	 */
	fp_exec();
}

user_desc_t *
cpu_get_gdt(void)
{
	return (CPU->cpu_gdt);
}


#if !defined(lwp_getdatamodel)

/*
 * Return the datamodel of the given lwp.
 */
/*ARGSUSED*/
model_t
lwp_getdatamodel(klwp_t *lwp)
{
	return (lwp->lwp_procp->p_model);
}

#endif	/* !lwp_getdatamodel */

#if !defined(get_udatamodel)

model_t
get_udatamodel(void)
{
	return (curproc->p_model);
}

#endif	/* !get_udatamodel */
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved.
 * Copyright 2019 Joyent, Inc.
 */

#include <sys/param.h>
#include <sys/vmparam.h>
#include <sys/types.h>
#include <sys/sysmacros.h>
#include <sys/systm.h>
#include <sys/signal.h>
#include <sys/stack.h>
#include <sys/cred.h>
#include <sys/cmn_err.h>
#include <sys/user.h>
#include <sys/privregs.h>
#include <sys/psw.h>
#include <sys/debug.h>
#include <sys/errno.h>
#include <sys/proc.h>
#include <sys/modctl.h>
#include <sys/var.h>
#include <sys/inline.h>
#include <sys/syscall.h>
#include <sys/ucontext.h>
#include <sys/cpuvar.h>
#include <sys/siginfo.h>
#include <sys/trap.h>
#include <sys/vtrace.h>
#include <sys/sysinfo.h>
#include <sys/procfs.h>
#include <sys/prsystm.h>
#include <c2/audit.h>
#include <sys/modctl.h>
#include <sys/aio_impl.h>
#include <sys/copyops.h>
#include <sys/priv.h>
#include <sys/msacct.h>

int syscalltrace = 0;
#ifdef SYSCALLTRACE
static kmutex_t systrace_lock;		/* syscall tracing lock */
#else
#define	syscalltrace 0
#endif /* SYSCALLTRACE */

typedef	int64_t (*llfcn_t)();	/* function returning long long */

int pre_syscall(void);
void post_syscall(long rval1, long rval2);
static krwlock_t *lock_syscall(struct sysent *, uint_t);
void deferred_singlestep_trap(caddr_t);

#ifdef _SYSCALL32_IMPL
#define	LWP_GETSYSENT(lwp)	\
	(lwp_getdatamodel(lwp) == DATAMODEL_NATIVE ? sysent : sysent32)
#else
#define	LWP_GETSYSENT(lwp)	(sysent)
#endif

/*
 * If watchpoints are active, don't make copying in of
 * system call arguments take a read watchpoint trap.
 */
static int
copyin_args(struct regs *rp, long *ap, uint_t nargs)
{
	greg_t *sp = 1 + (greg_t *)rp->r_sp;		/* skip ret addr */

	ASSERT(nargs <= MAXSYSARGS);

	return (copyin_nowatch(sp, ap, nargs * sizeof (*sp)));
}

#if defined(_SYSCALL32_IMPL)
static int
copyin_args32(struct regs *rp, long *ap, uint_t nargs)
{
	greg32_t *sp = 1 + (greg32_t *)rp->r_sp;	/* skip ret addr */
	uint32_t a32[MAXSYSARGS];
	int rc;

	ASSERT(nargs <= MAXSYSARGS);

	if ((rc = copyin_nowatch(sp, a32, nargs * sizeof (*sp))) == 0) {
		uint32_t *a32p = &a32[0];

		while (nargs--)
			*ap++ = (ulong_t)*a32p++;
	}
	return (rc);
}
#define	COPYIN_ARGS32	copyin_args32
#else
#define	COPYIN_ARGS32	copyin_args
#endif

/*
 * Error handler for system calls where arg copy gets fault.
 */
static longlong_t
syscall_err()
{
	return (0);
}

/*
 * Corresponding sysent entry to allow syscall_entry caller
 * to invoke syscall_err.
 */
static struct sysent sysent_err =  {
	0, SE_32RVAL1, NULL, NULL, (llfcn_t)syscall_err
};

/*
 * Called from syscall() when a non-trivial 32-bit system call occurs.
 *	Sets up the args and returns a pointer to the handler.
 */
struct sysent *
syscall_entry(kthread_t *t, long *argp)
{
	klwp_t *lwp = ttolwp(t);
	struct regs *rp = lwptoregs(lwp);
	unsigned int code;
	struct sysent *callp;
	struct sysent *se = LWP_GETSYSENT(lwp);
	int error = 0;
	uint_t nargs;

	ASSERT(t == curthread && curthread->t_schedflag & TS_DONT_SWAP);

	lwp->lwp_ru.sysc++;
	lwp->lwp_eosys = NORMALRETURN;	/* assume this will be normal */

	/*
	 * Set lwp_ap to point to the args, even if none are needed for this
	 * system call.  This is for the loadable-syscall case where the
	 * number of args won't be known until the system call is loaded, and
	 * also maintains a non-NULL lwp_ap setup for get_syscall_args(). Note
	 * that lwp_ap MUST be set to a non-NULL value _BEFORE_ t_sysnum is
	 * set to non-zero; otherwise get_syscall_args(), seeing a non-zero
	 * t_sysnum for this thread, will charge ahead and dereference lwp_ap.
	 */
	lwp->lwp_ap = argp;		/* for get_syscall_args */

	code = rp->r_r0;
	t->t_sysnum = (short)code;
	callp = code >= NSYSCALL ? &nosys_ent : se + code;

	if ((t->t_pre_sys | syscalltrace) != 0) {
		error = pre_syscall();

		/*
		 * pre_syscall() has taken care so that lwp_ap is current;
		 * it either points to syscall-entry-saved amd64 regs,
		 * or it points to lwp_arg[], which has been re-copied from
		 * the ia32 ustack, but either way, it's a current copy after
		 * /proc has possibly mucked with the syscall args.
		 */

		if (error)
			return (&sysent_err);	/* use dummy handler */
	}

	/*
	 * Fetch the system call arguments to the kernel stack copy used
	 * for syscall handling.
	 * Note: for loadable system calls the number of arguments required
	 * may not be known at this point, and will be zero if the system call
	 * was never loaded.  Once the system call has been loaded, the number
	 * of args is not allowed to be changed.
	 */
	if ((nargs = (uint_t)callp->sy_narg) != 0 &&
	    COPYIN_ARGS32(rp, argp, nargs)) {
		(void) set_errno(EFAULT);
		return (&sysent_err);	/* use dummy handler */
	}

	return (callp);		/* return sysent entry for caller */
}

void
syscall_exit(kthread_t *t, long rval1, long rval2)
{
	/*
	 * Handle signals and other post-call events if necessary.
	 */
	if ((t->t_post_sys_ast | syscalltrace) == 0) {
		klwp_t *lwp = ttolwp(t);
		struct regs *rp = lwptoregs(lwp);

		/*
		 * Normal return.
		 * Clear error indication and set return values.
		 */
		rp->r_ps &= ~PS_C;	/* reset carry bit */
		rp->r_r0 = rval1;
		rp->r_r1 = rval2;
		lwp->lwp_state = LWP_USER;
	} else {
		post_syscall(rval1, rval2);
	}
	t->t_sysnum = 0;		/* invalidate args */
}

/*
 * Perform pre-system-call processing, including stopping for tracing,
 * auditing, etc.
 *
 * This routine is called only if the t_pre_sys flag is set. Any condition
 * requiring pre-syscall handling must set the t_pre_sys flag. If the
 * condition is persistent, this routine will repost t_pre_sys.
 */
int
pre_syscall()
{
	kthread_t *t = curthread;
	unsigned code = t->t_sysnum;
	klwp_t *lwp = ttolwp(t);
	proc_t *p = ttoproc(t);
	int	repost;

	t->t_pre_sys = repost = 0;	/* clear pre-syscall processing flag */

	ASSERT(t->t_schedflag & TS_DONT_SWAP);

#if defined(DEBUG)
	/*
	 * On the i386 kernel, lwp_ap points at the piece of the thread
	 * stack that we copy the users arguments into.
	 *
	 * On the amd64 kernel, the syscall arguments in the rdi..r9
	 * registers should be pointed at by lwp_ap.  If the args need to
	 * be copied so that those registers can be changed without losing
	 * the ability to get the args for /proc, they can be saved by
	 * save_syscall_args(), and lwp_ap will be restored by post_syscall().
	 */
	if (lwp_getdatamodel(lwp) == DATAMODEL_NATIVE) {
#if defined(_LP64)
		ASSERT(lwp->lwp_ap == (long *)&lwptoregs(lwp)->r_rdi);
	} else {
#endif
		ASSERT((caddr_t)lwp->lwp_ap > t->t_stkbase &&
		    (caddr_t)lwp->lwp_ap < t->t_stk);
	}
#endif	/* DEBUG */

	/*
	 * Make sure the thread is holding the latest credentials for the
	 * process.  The credentials in the process right now apply to this
	 * thread for the entire system call.
	 */
	if (t->t_cred != p->p_cred) {
		cred_t *oldcred = t->t_cred;
		/*
		 * DTrace accesses t_cred in probe context.  t_cred must
		 * always be either NULL, or point to a valid, allocated cred
		 * structure.
		 */
		t->t_cred = crgetcred();
		crfree(oldcred);
	}

	/*
	 * From the proc(5) manual page:
	 * When entry to a system call is being traced, the traced process
	 * stops after having begun the call to the system but before the
	 * system call arguments have been fetched from the process.
	 */
	if (PTOU(p)->u_systrap) {
		if (prismember(&PTOU(p)->u_entrymask, code)) {
			mutex_enter(&p->p_lock);
			/*
			 * Recheck stop condition, now that lock is held.
			 */
			if (PTOU(p)->u_systrap &&
			    prismember(&PTOU(p)->u_entrymask, code)) {
				stop(PR_SYSENTRY, code);

				/*
				 * /proc may have modified syscall args,
				 * either in regs for amd64 or on ustack
				 * for ia32.  Either way, arrange to
				 * copy them again, both for the syscall
				 * handler and for other consumers in
				 * post_syscall (like audit).  Here, we
				 * only do amd64, and just set lwp_ap
				 * back to the kernel-entry stack copy;
				 * the syscall ml code redoes
				 * move-from-regs to set up for the
				 * syscall handler after we return.  For
				 * ia32, save_syscall_args() below makes
				 * an lwp_ap-accessible copy.
				 */
#if defined(_LP64)
				if (lwp_getdatamodel(lwp) == DATAMODEL_NATIVE) {
					lwp->lwp_argsaved = 0;
					lwp->lwp_ap =
					    (long *)&lwptoregs(lwp)->r_rdi;
				}
#endif
			}
			mutex_exit(&p->p_lock);
		}
		repost = 1;
	}

	/*
	 * ia32 kernel, or ia32 proc on amd64 kernel: keep args in
	 * lwp_arg for post-syscall processing, regardless of whether
	 * they might have been changed in /proc above.
	 */
#if defined(_LP64)
	if (lwp_getdatamodel(lwp) != DATAMODEL_NATIVE)
#endif
		(void) save_syscall_args();

	if (lwp->lwp_sysabort) {
		/*
		 * lwp_sysabort may have been set via /proc while the process
		 * was stopped on PR_SYSENTRY.  If so, abort the system call.
		 * Override any error from the copyin() of the arguments.
		 */
		lwp->lwp_sysabort = 0;
		(void) set_errno(EINTR);	/* forces post_sys */
		t->t_pre_sys = 1;	/* repost anyway */
		return (1);		/* don't do system call, return EINTR */
	}

	/*
	 * begin auditing for this syscall if the c2audit module is loaded
	 * and auditing is enabled
	 */
	if (audit_active == C2AUDIT_LOADED) {
		uint32_t auditing = au_zone_getstate(NULL);

		if (auditing & AU_AUDIT_MASK) {
			int error;
			if (error = audit_start(T_SYSCALL, code, auditing, \
			    0, lwp)) {
				t->t_pre_sys = 1;	/* repost anyway */
				(void) set_errno(error);
				return (1);
			}
			repost = 1;
		}
	}

#ifdef SYSCALLTRACE
	if (syscalltrace) {
		int i;
		long *ap;
		char *cp;
		char *sysname;
		struct sysent *callp;

		if (code >= NSYSCALL)
			callp = &nosys_ent;	/* nosys has no args */
		else
			callp = LWP_GETSYSENT(lwp) + code;
		(void) save_syscall_args();
		mutex_enter(&systrace_lock);
		printf("%d: ", p->p_pid);
		if (code >= NSYSCALL) {
			printf("0x%x", code);
		} else {
			sysname = mod_getsysname(code);
			printf("%s[0x%x/0x%p]", sysname == NULL ? "NULL" :
			    sysname, code, callp->sy_callc);
		}
		cp = "(";
		for (i = 0, ap = lwp->lwp_ap; i < callp->sy_narg; i++, ap++) {
			printf("%s%lx", cp, *ap);
			cp = ", ";
		}
		if (i)
			printf(")");
		printf(" %s id=0x%p\n", PTOU(p)->u_comm, curthread);
		mutex_exit(&systrace_lock);
	}
#endif /* SYSCALLTRACE */

	/*
	 * If there was a continuing reason for pre-syscall processing,
	 * set the t_pre_sys flag for the next system call.
	 */
	if (repost)
		t->t_pre_sys = 1;
	lwp->lwp_error = 0;	/* for old drivers */
	lwp->lwp_badpriv = PRIV_NONE;
	return (0);
}


/*
 * Post-syscall processing.  Perform abnormal system call completion
 * actions such as /proc tracing, profiling, signals, preemption, etc.
 *
 * This routine is called only if t_post_sys, t_sig_check, or t_astflag is set.
 * Any condition requiring pre-syscall handling must set one of these.
 * If the condition is persistent, this routine will repost t_post_sys.
 */
void
post_syscall(long rval1, long rval2)
{
	kthread_t *t = curthread;
	klwp_t *lwp = ttolwp(t);
	proc_t *p = ttoproc(t);
	struct regs *rp = lwptoregs(lwp);
	uint_t	error;
	uint_t	code = t->t_sysnum;
	int	repost = 0;
	int	proc_stop = 0;		/* non-zero if stopping */
	int	sigprof = 0;		/* non-zero if sending SIGPROF */

	t->t_post_sys = 0;

	error = lwp->lwp_errno;

	/*
	 * Code can be zero if this is a new LWP returning after a forkall(),
	 * other than the one which matches the one in the parent which called
	 * forkall().  In these LWPs, skip most of post-syscall activity.
	 */
	if (code == 0)
		goto sig_check;
	/*
	 * If the trace flag is set, mark the lwp to take a single-step trap
	 * on return to user level (below). The x86 lcall interface and
	 * sysenter has already done this, and turned off the flag, but
	 * amd64 syscall interface has not.
	 */
	if (rp->r_ps & PS_T) {
		lwp->lwp_pcb.pcb_flags |= DEBUG_PENDING;
		rp->r_ps &= ~PS_T;
		aston(curthread);
	}

	/* put out audit record for this syscall */
	if (AU_AUDITING()) {
		rval_t	rval;

		/* XX64 -- truncation of 64-bit return values? */
		rval.r_val1 = (int)rval1;
		rval.r_val2 = (int)rval2;
		audit_finish(T_SYSCALL, code, error, &rval);
		repost = 1;
	}

	if (curthread->t_pdmsg != NULL) {
		char *m = curthread->t_pdmsg;

		uprintf("%s", m);
		kmem_free(m, strlen(m) + 1);
		curthread->t_pdmsg = NULL;
	}

	/*
	 * If we're going to stop for /proc tracing, set the flag and
	 * save the arguments so that the return values don't smash them.
	 */
	if (PTOU(p)->u_systrap) {
		if (prismember(&PTOU(p)->u_exitmask, code)) {
			if (lwp_getdatamodel(lwp) == DATAMODEL_LP64)
				(void) save_syscall_args();
			proc_stop = 1;
		}
		repost = 1;
	}

	/*
	 * Similarly check to see if SIGPROF might be sent.
	 */
	if (curthread->t_rprof != NULL &&
	    curthread->t_rprof->rp_anystate != 0) {
		if (lwp_getdatamodel(lwp) == DATAMODEL_LP64)
			(void) save_syscall_args();
		sigprof = 1;
	}

	if (lwp->lwp_eosys == NORMALRETURN) {
		if (error == 0) {
#ifdef SYSCALLTRACE
			if (syscalltrace) {
				mutex_enter(&systrace_lock);
				printf(
				    "%d: r_val1=0x%lx, r_val2=0x%lx, id 0x%p\n",
				    p->p_pid, rval1, rval2, curthread);
				mutex_exit(&systrace_lock);
			}
#endif /* SYSCALLTRACE */
			rp->r_ps &= ~PS_C;
			rp->r_r0 = rval1;
			rp->r_r1 = rval2;
		} else {
			int sig;
#ifdef SYSCALLTRACE
			if (syscalltrace) {
				mutex_enter(&systrace_lock);
				printf("%d: error=%d, id 0x%p\n",
				    p->p_pid, error, curthread);
				mutex_exit(&systrace_lock);
			}
#endif /* SYSCALLTRACE */
			if (error == EINTR && t->t_activefd.a_stale)
				error = EBADF;
			if (error == EINTR &&
			    (sig = lwp->lwp_cursig) != 0 &&
			    sigismember(&PTOU(p)->u_sigrestart, sig) &&
			    PTOU(p)->u_signal[sig - 1] != SIG_DFL &&
			    PTOU(p)->u_signal[sig - 1] != SIG_IGN)
				error = ERESTART;
			rp->r_r0 = error;
			rp->r_ps |= PS_C;
		}
	}

	/*
	 * From the proc(5) manual page:
	 * When exit from a system call is being traced, the traced process
	 * stops on completion of the system call just prior to checking for
	 * signals and returning to user level.  At this point all return
	 * values have been stored into the traced process's saved registers.
	 */
	if (proc_stop) {
		mutex_enter(&p->p_lock);
		if (PTOU(p)->u_systrap &&
		    prismember(&PTOU(p)->u_exitmask, code))
			stop(PR_SYSEXIT, code);
		mutex_exit(&p->p_lock);
	}

	/*
	 * If we are the parent returning from a successful
	 * vfork, wait for the child to exec or exit.
	 * This code must be here and not in the bowels of the system
	 * so that /proc can intercept exit from vfork in a timely way.
	 */
	if (t->t_flag & T_VFPARENT) {
		ASSERT(code == SYS_vfork || code == SYS_forksys);
		ASSERT(rp->r_r1 == 0 && error == 0);
		vfwait((pid_t)rval1);
		t->t_flag &= ~T_VFPARENT;
	}

	/*
	 * If profiling is active, bill the current PC in user-land
	 * and keep reposting until profiling is disabled.
	 */
	if (p->p_prof.pr_scale) {
		if (lwp->lwp_oweupc)
			profil_tick(rp->r_pc);
		repost = 1;
	}

sig_check:
	/*
	 * Reset flag for next time.
	 * We must do this after stopping on PR_SYSEXIT
	 * because /proc uses the information in lwp_eosys.
	 */
	lwp->lwp_eosys = NORMALRETURN;
	clear_stale_fd();
	t->t_flag &= ~T_FORKALL;

	if (t->t_astflag | t->t_sig_check) {
		/*
		 * Turn off the AST flag before checking all the conditions that
		 * may have caused an AST.  This flag is on whenever a signal or
		 * unusual condition should be handled after the next trap or
		 * syscall.
		 */
		astoff(t);
		/*
		 * If a single-step trap occurred on a syscall (see trap())
		 * recognize it now.  Do this before checking for signals
		 * because deferred_singlestep_trap() may generate a SIGTRAP to
		 * the LWP or may otherwise mark the LWP to call issig(FORREAL).
		 */
		if (lwp->lwp_pcb.pcb_flags & DEBUG_PENDING)
			deferred_singlestep_trap((caddr_t)rp->r_pc);

		t->t_sig_check = 0;

		/*
		 * The following check is legal for the following reasons:
		 *	1) The thread we are checking, is ourselves, so there is
		 *	   no way the proc can go away.
		 *	2) The only time we need to be protected by the
		 *	   lock is if the binding is changed.
		 *
		 *	Note we will still take the lock and check the binding
		 *	if the condition was true without the lock held.  This
		 *	prevents lock contention among threads owned by the
		 *	same proc.
		 */

		if (curthread->t_proc_flag & TP_CHANGEBIND) {
			mutex_enter(&p->p_lock);
			if (curthread->t_proc_flag & TP_CHANGEBIND) {
				timer_lwpbind();
				curthread->t_proc_flag &= ~TP_CHANGEBIND;
			}
			mutex_exit(&p->p_lock);
		}

		/*
		 * for kaio requests on the special kaio poll queue,
		 * copyout their results to user memory.
		 */
		if (p->p_aio)
			aio_cleanup(0);
		/*
		 * If this LWP was asked to hold, call holdlwp(), which will
		 * stop.  holdlwps() sets this up and calls pokelwps() which
		 * sets the AST flag.
		 *
		 * Also check TP_EXITLWP, since this is used by fresh new LWPs
		 * through lwp_rtt().  That flag is set if the lwp_create(2)
		 * syscall failed after creating the LWP.
		 */
		if (ISHOLD(p) || (t->t_proc_flag & TP_EXITLWP))
			holdlwp();

		/*
		 * All code that sets signals and makes ISSIG_PENDING
		 * evaluate true must set t_sig_check afterwards.
		 */
		if (ISSIG_PENDING(t, lwp, p)) {
			if (issig(FORREAL))
				psig();
			t->t_sig_check = 1;	/* recheck next time */
		}

		if (sigprof) {
			int nargs = (code > 0 && code < NSYSCALL)?
			    LWP_GETSYSENT(lwp)[code].sy_narg : 0;
			realsigprof(code, nargs, error);
			t->t_sig_check = 1;	/* recheck next time */
		}

		/*
		 * If a performance counter overflow interrupt was
		 * delivered *during* the syscall, then re-enable the
		 * AST so that we take a trip through trap() to cause
		 * the SIGEMT to be delivered.
		 */
		if (lwp->lwp_pcb.pcb_flags & CPC_OVERFLOW)
			aston(t);

		/*
		 * /proc can't enable/disable the trace bit itself
		 * because that could race with the call gate used by
		 * system calls via "lcall". If that happened, an
		 * invalid EFLAGS would result. prstep()/prnostep()
		 * therefore schedule an AST for the purpose.
		 */
		if (lwp->lwp_pcb.pcb_flags & REQUEST_STEP) {
			lwp->lwp_pcb.pcb_flags &= ~REQUEST_STEP;
			rp->r_ps |= PS_T;
		}
		if (lwp->lwp_pcb.pcb_flags & REQUEST_NOSTEP) {
			lwp->lwp_pcb.pcb_flags &= ~REQUEST_NOSTEP;
			rp->r_ps &= ~PS_T;
		}
	}

	lwp->lwp_errno = 0;		/* clear error for next time */

	/*
	 * Set state to LWP_USER here so preempt won't give us a kernel
	 * priority if it occurs after this point.  Call CL_TRAPRET() to
	 * restore the user-level priority.
	 *
	 * It is important that no locks (other than spinlocks) be entered
	 * after this point before returning to user mode (unless lwp_state
	 * is set back to LWP_SYS).
	 *
	 * XXX Sampled times past this point are charged to the user.
	 */
	lwp->lwp_state = LWP_USER;

	if (t->t_trapret) {
		t->t_trapret = 0;
		thread_lock(t);
		CL_TRAPRET(t);
		thread_unlock(t);
	}
	if (CPU->cpu_runrun || t->t_schedflag & TS_ANYWAITQ)
		preempt();
	prunstop();

	lwp->lwp_errno = 0;		/* clear error for next time */

	/*
	 * The thread lock must be held in order to clear sysnum and reset
	 * lwp_ap atomically with respect to other threads in the system that
	 * may be looking at the args via lwp_ap from get_syscall_args().
	 */

	thread_lock(t);
	t->t_sysnum = 0;		/* no longer in a system call */

	if (lwp_getdatamodel(lwp) == DATAMODEL_NATIVE) {
#if defined(_LP64)
		/*
		 * In case the args were copied to the lwp, reset the
		 * pointer so the next syscall will have the right
		 * lwp_ap pointer.
		 */
		lwp->lwp_ap = (long *)&rp->r_rdi;
	} else {
#endif
		lwp->lwp_ap = NULL;	/* reset on every syscall entry */
	}
	thread_unlock(t);

	lwp->lwp_argsaved = 0;

	/*
	 * If there was a continuing reason for post-syscall processing,
	 * set the t_post_sys flag for the next system call.
	 */
	if (repost)
		t->t_post_sys = 1;

	/*
	 * If there is a ustack registered for this lwp, and the stack rlimit
	 * has been altered, read in the ustack. If the saved stack rlimit
	 * matches the bounds of the ustack, update the ustack to reflect
	 * the new rlimit. If the new stack rlimit is RLIM_INFINITY, disable
	 * stack checking by setting the size to 0.
	 */
	if (lwp->lwp_ustack != 0 && lwp->lwp_old_stk_ctl != 0) {
		rlim64_t new_size;
		caddr_t top;
		stack_t stk;
		struct rlimit64 rl;

		mutex_enter(&p->p_lock);
		new_size = p->p_stk_ctl;
		top = p->p_usrstack;
		(void) rctl_rlimit_get(rctlproc_legacy[RLIMIT_STACK], p, &rl);
		mutex_exit(&p->p_lock);

		if (rl.rlim_cur == RLIM64_INFINITY)
			new_size = 0;

		if (copyin((stack_t *)lwp->lwp_ustack, &stk,
		    sizeof (stack_t)) == 0 &&
		    (stk.ss_size == lwp->lwp_old_stk_ctl ||
		    stk.ss_size == 0) &&
		    stk.ss_sp == top - stk.ss_size) {
			stk.ss_sp = (void *)((uintptr_t)stk.ss_sp +
			    stk.ss_size - (uintptr_t)new_size);
			stk.ss_size = new_size;

			(void) copyout(&stk, (stack_t *)lwp->lwp_ustack,
			    sizeof (stack_t));
		}

		lwp->lwp_old_stk_ctl = 0;
	}
}

/*
 * Called from post_syscall() when a deferred singlestep is to be taken.
 */
void
deferred_singlestep_trap(caddr_t pc)
{
	proc_t *p = ttoproc(curthread);
	klwp_t *lwp = ttolwp(curthread);
	pcb_t *pcb = &lwp->lwp_pcb;
	uint_t fault = 0;
	k_siginfo_t siginfo;

	bzero(&siginfo, sizeof (siginfo));

	/*
	 * If both NORMAL_STEP and WATCH_STEP are in
	 * effect, give precedence to WATCH_STEP.
	 * If neither is set, user must have set the
	 * PS_T bit in %efl; treat this as NORMAL_STEP.
	 */
	if ((fault = undo_watch_step(&siginfo)) == 0 &&
	    ((pcb->pcb_flags & NORMAL_STEP) ||
	    !(pcb->pcb_flags & WATCH_STEP))) {
		siginfo.si_signo = SIGTRAP;
		siginfo.si_code = TRAP_TRACE;
		siginfo.si_addr  = pc;
		fault = FLTTRACE;
	}
	pcb->pcb_flags &= ~(DEBUG_PENDING|NORMAL_STEP|WATCH_STEP);

	if (fault) {
		/*
		 * Remember the fault and fault adddress
		 * for real-time (SIGPROF) profiling.
		 */
		lwp->lwp_lastfault = fault;
		lwp->lwp_lastfaddr = siginfo.si_addr;
		/*
		 * If a debugger has declared this fault to be an
		 * event of interest, stop the lwp.  Otherwise just
		 * deliver the associated signal.
		 */
		if (prismember(&p->p_fltmask, fault) &&
		    stop_on_fault(fault, &siginfo) == 0)
			siginfo.si_signo = 0;
	}

	if (siginfo.si_signo)
		trapsig(&siginfo, 1);
}

/*
 * nonexistent system call-- signal lwp (may want to handle it)
 * flag error if lwp won't see signal immediately
 */
int64_t
nosys(void)
{
	tsignal(curthread, SIGSYS);
	return (set_errno(ENOSYS));
}

int
nosys32(void)
{
	return (nosys());
}

/*
 * Execute a 32-bit system call on behalf of the current thread.
 */
void
dosyscall(void)
{
	/*
	 * Need space on the stack to store syscall arguments.
	 */
	long		syscall_args[MAXSYSARGS];
	struct sysent	*se;
	int64_t		ret;

	syscall_mstate(LMS_TRAP, LMS_SYSTEM);

	ASSERT(curproc->p_model == DATAMODEL_ILP32);

	CPU_STATS_ENTER_K();
	CPU_STATS_ADDQ(CPU, sys, syscall, 1);
	CPU_STATS_EXIT_K();

	se = syscall_entry(curthread, syscall_args);

	/*
	 * syscall_entry() copied all 8 arguments into syscall_args.
	 */
	ret = se->sy_callc(syscall_args[0], syscall_args[1], syscall_args[2],
	    syscall_args[3], syscall_args[4], syscall_args[5], syscall_args[6],
	    syscall_args[7]);

	syscall_exit(curthread, (int)ret & 0xffffffffu, (int)(ret >> 32));
	syscall_mstate(LMS_SYSTEM, LMS_TRAP);
}

/*
 * Get the arguments to the current system call. See comment atop
 * save_syscall_args() regarding lwp_ap usage.
 */

uint_t
get_syscall_args(klwp_t *lwp, long *argp, int *nargsp)
{
	kthread_t	*t = lwptot(lwp);
	ulong_t	mask = 0xfffffffful;
	uint_t	code;
	long	*ap;
	int	nargs;

#if defined(_LP64)
	if (lwp_getdatamodel(lwp) == DATAMODEL_LP64)
		mask = 0xfffffffffffffffful;
#endif

	/*
	 * The thread lock must be held while looking at the arguments to ensure
	 * they don't go away via post_syscall().
	 * get_syscall_args() is the only routine to read them which is callable
	 * outside the LWP in question and hence the only one that must be
	 * synchronized in this manner.
	 */
	thread_lock(t);

	code = t->t_sysnum;
	ap = lwp->lwp_ap;

	thread_unlock(t);

	if (code != 0 && code < NSYSCALL) {
		nargs = LWP_GETSYSENT(lwp)[code].sy_narg;

		ASSERT(nargs <= MAXSYSARGS);

		*nargsp = nargs;
		while (nargs-- > 0)
			*argp++ = *ap++ & mask;
	} else {
		*nargsp = 0;
	}

	return (code);
}

#ifdef _SYSCALL32_IMPL
/*
 * Get the arguments to the current 32-bit system call.
 */
uint_t
get_syscall32_args(klwp_t *lwp, int *argp, int *nargsp)
{
	long args[MAXSYSARGS];
	uint_t i, code;

	code = get_syscall_args(lwp, args, nargsp);

	for (i = 0; i != *nargsp; i++)
		*argp++ = (int)args[i];
	return (code);
}
#endif

/*
 * Save the system call arguments in a safe place.
 *
 * On the i386 kernel:
 *
 *	Copy the users args prior to changing the stack or stack pointer.
 *	This is so /proc will be able to get a valid copy of the
 *	args from the user stack even after the user stack has been changed.
 *	Note that the kernel stack copy of the args may also have been
 *	changed by a system call handler which takes C-style arguments.
 *
 *	Note that this may be called by stop() from trap().  In that case
 *	t_sysnum will be zero (syscall_exit clears it), so no args will be
 *	copied.
 *
 * On the amd64 kernel:
 *
 *	For 64-bit applications, lwp->lwp_ap normally points to %rdi..%r9
 *	in the reg structure. If the user is going to change the argument
 *	registers, rax, or the stack and might want to get the args (for
 *	/proc tracing), it must copy the args elsewhere via save_syscall_args().
 *
 *	For 32-bit applications, lwp->lwp_ap normally points to a copy of
 *	the system call arguments on the kernel stack made from the user
 *	stack.  Copy the args prior to change the stack or stack pointer.
 *	This is so /proc will be able to get a valid copy of the args
 *	from the user stack even after that stack has been changed.
 *
 *	This may be called from stop() even when we're not in a system call.
 *	Since there's no easy way to tell, this must be safe (not panic).
 *	If the copyins get data faults, return non-zero.
 */
int
save_syscall_args()
{
	kthread_t	*t = curthread;
	klwp_t		*lwp = ttolwp(t);
	uint_t		code = t->t_sysnum;
	uint_t		nargs;

	if (lwp->lwp_argsaved || code == 0)
		return (0);		/* args already saved or not needed */

	if (code >= NSYSCALL) {
		nargs = 0;		/* illegal syscall */
	} else {
		struct sysent *se = LWP_GETSYSENT(lwp);
		struct sysent *callp = se + code;

		nargs = callp->sy_narg;
		if (LOADABLE_SYSCALL(callp) && nargs == 0) {
			krwlock_t	*module_lock;

			/*
			 * Find out how many arguments the system
			 * call uses.
			 *
			 * We have the property that loaded syscalls
			 * never change the number of arguments they
			 * use after they've been loaded once.  This
			 * allows us to stop for /proc tracing without
			 * holding the module lock.
			 * /proc is assured that sy_narg is valid.
			 */
			module_lock = lock_syscall(se, code);
			nargs = callp->sy_narg;
			rw_exit(module_lock);
		}
	}

	/*
	 * Fetch the system call arguments.
	 */
	if (nargs == 0)
		goto out;

	ASSERT(nargs <= MAXSYSARGS);

	if (lwp_getdatamodel(lwp) == DATAMODEL_NATIVE) {
#if defined(_LP64)
		struct regs *rp = lwptoregs(lwp);

		lwp->lwp_arg[0] = rp->r_rdi;
		lwp->lwp_arg[1] = rp->r_rsi;
		lwp->lwp_arg[2] = rp->r_rdx;
		lwp->lwp_arg[3] = rp->r_rcx;
		lwp->lwp_arg[4] = rp->r_r8;
		lwp->lwp_arg[5] = rp->r_r9;
		if (nargs > 6 && copyin_args(rp, &lwp->lwp_arg[6], nargs - 6))
			return (-1);
	} else {
#endif
		if (COPYIN_ARGS32(lwptoregs(lwp), lwp->lwp_arg, nargs))
			return (-1);
	}
out:
	lwp->lwp_ap = lwp->lwp_arg;
	lwp->lwp_argsaved = 1;
	t->t_post_sys = 1;	/* so lwp_ap will be reset */
	return (0);
}

void
reset_syscall_args(void)
{
	ttolwp(curthread)->lwp_argsaved = 0;
}

/*
 * Call a system call which takes a pointer to the user args struct and
 * a pointer to the return values.  This is a bit slower than the standard
 * C arg-passing method in some cases.
 */
int64_t
syscall_ap(void)
{
	uint_t	error;
	struct sysent *callp;
	rval_t	rval;
	kthread_t *t = curthread;
	klwp_t	*lwp = ttolwp(t);
	struct regs *rp = lwptoregs(lwp);

	callp = LWP_GETSYSENT(lwp) + t->t_sysnum;

	/*
	 * If the arguments don't fit in registers %rdi-%r9, make sure they
	 * have been copied to the lwp_arg array.
	 */
	if (callp->sy_narg > 6 && save_syscall_args())
		return ((int64_t)set_errno(EFAULT));

	rval.r_val1 = 0;
	rval.r_val2 = rp->r_r1;
	lwp->lwp_error = 0;	/* for old drivers */
	error = (*(callp->sy_call))(lwp->lwp_ap, &rval);
	if (error)
		return ((longlong_t)set_errno(error));
	return (rval.r_vals);
}

/*
 * Load system call module.
 *	Returns with pointer to held read lock for module.
 */
static krwlock_t *
lock_syscall(struct sysent *table, uint_t code)
{
	krwlock_t	*module_lock;
	struct modctl	*modp;
	int		id;
	struct sysent   *callp;

	callp = table + code;
	module_lock = callp->sy_lock;

	/*
	 * Optimization to only call modload if we don't have a loaded
	 * syscall.
	 */
	rw_enter(module_lock, RW_READER);
	if (LOADED_SYSCALL(callp))
		return (module_lock);
	rw_exit(module_lock);

	for (;;) {
		if ((id = modload("sys", syscallnames[code])) == -1)
			break;

		/*
		 * If we loaded successfully at least once, the modctl
		 * will still be valid, so we try to grab it by filename.
		 * If this call fails, it's because the mod_filename
		 * was changed after the call to modload() (mod_hold_by_name()
		 * is the likely culprit).  We can safely just take
		 * another lap if this is the case;  the modload() will
		 * change the mod_filename back to one by which we can
		 * find the modctl.
		 */
		modp = mod_find_by_filename("sys", syscallnames[code]);

		if (modp == NULL)
			continue;

		mutex_enter(&mod_lock);

		if (!modp->mod_installed) {
			mutex_exit(&mod_lock);
			continue;
		}
		break;
	}
	rw_enter(module_lock, RW_READER);

	if (id != -1)
		mutex_exit(&mod_lock);

	return (module_lock);
}

/*
 * Loadable syscall support.
 *	If needed, load the module, then reserve it by holding a read
 *	lock for the duration of the call.
 *	Later, if the syscall is not unloadable, it could patch the vector.
 */
/*ARGSUSED*/
int64_t
loadable_syscall(
    long a0, long a1, long a2, long a3,
    long a4, long a5, long a6, long a7)
{
	klwp_t *lwp = ttolwp(curthread);
	int64_t	rval;
	struct sysent *callp;
	struct sysent *se = LWP_GETSYSENT(lwp);
	krwlock_t *module_lock;
	int code, error = 0;

	code = curthread->t_sysnum;
	callp = se + code;

	/*
	 * Try to autoload the system call if necessary
	 */
	module_lock = lock_syscall(se, code);

	/*
	 * we've locked either the loaded syscall or nosys
	 */

	if (lwp_getdatamodel(lwp) == DATAMODEL_NATIVE) {
#if defined(_LP64)
		if (callp->sy_flags & SE_ARGC) {
			rval = (int64_t)(*callp->sy_call)(a0, a1, a2, a3,
			    a4, a5);
		} else {
			rval = syscall_ap();
		}
	} else {
#endif
		/*
		 * Now that it's loaded, make sure enough args were copied.
		 */
		if (COPYIN_ARGS32(lwptoregs(lwp), lwp->lwp_ap, callp->sy_narg))
			error = EFAULT;
		if (error) {
			rval = set_errno(error);
		} else if (callp->sy_flags & SE_ARGC) {
			rval = (int64_t)(*callp->sy_call)(lwp->lwp_ap[0],
			    lwp->lwp_ap[1], lwp->lwp_ap[2], lwp->lwp_ap[3],
			    lwp->lwp_ap[4], lwp->lwp_ap[5]);
		} else {
			rval = syscall_ap();
		}
	}

	rw_exit(module_lock);
	return (rval);
}

/*
 * Indirect syscall handled in libc on x86 architectures
 */
int64_t
indir()
{
	return (nosys());
}

/*
 * set_errno - set an error return from the current system call.
 *	This could be a macro.
 *	This returns the value it is passed, so that the caller can
 *	use tail-recursion-elimination and do return (set_errno(ERRNO));
 */
uint_t
set_errno(uint_t error)
{
	ASSERT(error != 0);		/* must not be used to clear errno */

	curthread->t_post_sys = 1;	/* have post_syscall do error return */
	return (ttolwp(curthread)->lwp_errno = error);
}

/*
 * set_proc_pre_sys - Set pre-syscall processing for entire process.
 */
void
set_proc_pre_sys(proc_t *p)
{
	kthread_t	*t;
	kthread_t	*first;

	ASSERT(MUTEX_HELD(&p->p_lock));

	t = first = p->p_tlist;
	do {
		t->t_pre_sys = 1;
	} while ((t = t->t_forw) != first);
}

/*
 * set_proc_post_sys - Set post-syscall processing for entire process.
 */
void
set_proc_post_sys(proc_t *p)
{
	kthread_t	*t;
	kthread_t	*first;

	ASSERT(MUTEX_HELD(&p->p_lock));

	t = first = p->p_tlist;
	do {
		t->t_post_sys = 1;
	} while ((t = t->t_forw) != first);
}

/*
 * set_proc_sys - Set pre- and post-syscall processing for entire process.
 */
void
set_proc_sys(proc_t *p)
{
	kthread_t	*t;
	kthread_t	*first;

	ASSERT(MUTEX_HELD(&p->p_lock));

	t = first = p->p_tlist;
	do {
		t->t_pre_sys = 1;
		t->t_post_sys = 1;
	} while ((t = t->t_forw) != first);
}

/*
 * set_all_proc_sys - set pre- and post-syscall processing flags for all
 * user processes.
 *
 * This is needed when auditing, tracing, or other facilities which affect
 * all processes are turned on.
 */
void
set_all_proc_sys()
{
	kthread_t	*t;
	kthread_t	*first;

	mutex_enter(&pidlock);
	t = first = curthread;
	do {
		t->t_pre_sys = 1;
		t->t_post_sys = 1;
	} while ((t = t->t_next) != first);
	mutex_exit(&pidlock);
}

/*
 * set_all_zone_usr_proc_sys - set pre- and post-syscall processing flags for
 * all user processes running in the zone of the current process
 *
 * This is needed when auditing, tracing, or other facilities which affect
 * all processes are turned on.
 */
void
set_all_zone_usr_proc_sys(zoneid_t zoneid)
{
	proc_t	    *p;
	kthread_t   *t;

	mutex_enter(&pidlock);
	for (p = practive; p != NULL; p = p->p_next) {
		/* skip kernel and incomplete processes */
		if (p->p_exec == NULLVP || p->p_as == &kas ||
		    p->p_stat == SIDL || p->p_stat == SZOMB ||
		    (p->p_flag & (SSYS | SEXITING | SEXITLWPS)))
			continue;
		/*
		 * Only processes in the given zone (eventually in
		 * all zones) are taken into account
		 */
		if (zoneid == ALL_ZONES || p->p_zone->zone_id == zoneid) {
			mutex_enter(&p->p_lock);
			if ((t = p->p_tlist) == NULL) {
				mutex_exit(&p->p_lock);
				continue;
			}
			/*
			 * Set pre- and post-syscall processing flags
			 * for all threads of the process
			 */
			do {
				t->t_pre_sys = 1;
				t->t_post_sys = 1;
			} while (p->p_tlist != (t = t->t_forw));
			mutex_exit(&p->p_lock);
		}
	}
	mutex_exit(&pidlock);
}

/*
 * set_proc_ast - Set asynchronous service trap (AST) flag for all
 * threads in process.
 */
void
set_proc_ast(proc_t *p)
{
	kthread_t	*t;
	kthread_t	*first;

	ASSERT(MUTEX_HELD(&p->p_lock));

	t = first = p->p_tlist;
	do {
		aston(t);
	} while ((t = t->t_forw) != first);
}
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved.
 * Copyright 2021 Joyent, Inc.
 */

/*	Copyright (c) 1990, 1991 UNIX System Laboratories, Inc.	*/
/*	Copyright (c) 1984, 1986, 1987, 1988, 1989, 1990 AT&T	*/
/*	  All Rights Reserved	*/

/*	Copyright (c) 1987, 1988 Microsoft Corporation	*/
/*	  All Rights Reserved	*/

#include <sys/param.h>
#include <sys/types.h>
#include <sys/sysmacros.h>
#include <sys/systm.h>
#include <sys/signal.h>
#include <sys/errno.h>
#include <sys/fault.h>
#include <sys/syscall.h>
#include <sys/cpuvar.h>
#include <sys/sysi86.h>
#include <sys/psw.h>
#include <sys/cred.h>
#include <sys/policy.h>
#include <sys/thread.h>
#include <sys/debug.h>
#include <sys/ontrap.h>
#include <sys/privregs.h>
#include <sys/x86_archext.h>
#include <sys/vmem.h>
#include <sys/kmem.h>
#include <sys/mman.h>
#include <sys/archsystm.h>
#include <vm/hat.h>
#include <vm/as.h>
#include <vm/seg.h>
#include <vm/seg_kmem.h>
#include <vm/faultcode.h>
#include <sys/fp.h>
#include <sys/cmn_err.h>
#include <sys/segments.h>
#include <sys/clock.h>
#include <vm/hat_i86.h>
#if defined(__xpv)
#include <sys/hypervisor.h>
#include <sys/note.h>
#endif

static void ldt_alloc(proc_t *, uint_t);
static void ldt_free(proc_t *);
static void ldt_dup(proc_t *, proc_t *);
static void ldt_grow(proc_t *, uint_t);

/*
 * sysi86 System Call
 */

/* ARGSUSED */
int
sysi86(short cmd, uintptr_t arg1, uintptr_t arg2, uintptr_t arg3)
{
	struct ssd ssd;
	int error = 0;
	int c;
	proc_t *pp = curproc;

	switch (cmd) {

	/*
	 * The SI86V86 subsystem call of the SYSI86 system call
	 * supports only one subcode -- V86SC_IOPL.
	 */
	case SI86V86:
		if (arg1 == V86SC_IOPL) {
			struct regs *rp = lwptoregs(ttolwp(curthread));
			greg_t oldpl = rp->r_ps & PS_IOPL;
			greg_t newpl = arg2 & PS_IOPL;

			/*
			 * Must be privileged to run this system call
			 * if giving more io privilege.
			 */
			if (newpl > oldpl && (error =
			    secpolicy_sys_config(CRED(), B_FALSE)) != 0)
				return (set_errno(error));
#if defined(__xpv)
			const struct ctxop_template xen_tpl = {
				.ct_rev		= CTXOP_TPL_REV,
				.ct_save	= xen_disable_user_iopl,
				.ct_restore	= xen_enable_user_iopl,
				.ct_exit	= xen_disable_user_iopl,
			};
			struct ctxop *ctx;

			ctx = ctxop_allocate(&xen_tpl, NULL);
			kpreempt_disable();
			ctxop_attach(curthread, ctx);
			xen_enable_user_iopl(NULL);
			kpreempt_enable();
#else
			rp->r_ps ^= oldpl ^ newpl;
#endif
		} else
			error = EINVAL;
		break;

	/*
	 * Set a segment descriptor
	 */
	case SI86DSCR:
		/*
		 * There are considerable problems here manipulating
		 * resources shared by many running lwps.  Get everyone
		 * into a safe state before changing the LDT.
		 */
		if (curthread != pp->p_agenttp && !holdlwps(SHOLDFORK1)) {
			error = EINTR;
			break;
		}

		if (get_udatamodel() == DATAMODEL_LP64) {
			error = EINVAL;
			break;
		}

		if (copyin((caddr_t)arg1, &ssd, sizeof (ssd)) < 0) {
			error = EFAULT;
			break;
		}

		error = setdscr(&ssd);

		mutex_enter(&pp->p_lock);
		if (curthread != pp->p_agenttp)
			continuelwps(pp);
		mutex_exit(&pp->p_lock);
		break;

	case SI86FPHW:
		c = fp_kind & 0xff;
		if (suword32((void *)arg1, c) == -1)
			error = EFAULT;
		break;

	case SI86FPSTART:
		/*
		 * arg1 is the address of _fp_hw
		 * arg2 is the desired x87 FCW value
		 * arg3 is the desired SSE MXCSR value
		 * a return value of one means SSE hardware, else none.
		 */
		c = fp_kind & 0xff;
		if (suword32((void *)arg1, c) == -1) {
			error = EFAULT;
			break;
		}
		fpsetcw((uint16_t)arg2, (uint32_t)arg3);
		return ((fp_kind & __FP_SSE) ? 1 : 0);

	/* real time clock management commands */

	case WTODC:
		if ((error = secpolicy_settime(CRED())) == 0) {
			timestruc_t ts;
			mutex_enter(&tod_lock);
			gethrestime(&ts);
			tod_set(ts);
			mutex_exit(&tod_lock);
		}
		break;

/* Give some timezone playing room */
#define	ONEWEEK	(7 * 24 * 60 * 60)

	case SGMTL:
		/*
		 * Called from 32 bit land, negative values
		 * are not sign extended, so we do that here
		 * by casting it to an int and back.  We also
		 * clamp the value to within reason and detect
		 * when a 64 bit call overflows an int.
		 */
		if ((error = secpolicy_settime(CRED())) == 0) {
			int newlag = (int)arg1;

#ifdef _SYSCALL32_IMPL
			if (get_udatamodel() == DATAMODEL_NATIVE &&
			    (long)newlag != (long)arg1) {
				error = EOVERFLOW;
			} else
#endif
			if (newlag >= -ONEWEEK && newlag <= ONEWEEK)
				sgmtl(newlag);
			else
				error = EOVERFLOW;
		}
		break;

	case GGMTL:
		if (get_udatamodel() == DATAMODEL_NATIVE) {
			if (sulword((void *)arg1, ggmtl()) == -1)
				error = EFAULT;
#ifdef _SYSCALL32_IMPL
		} else {
			time_t gmtl;

			if ((gmtl = ggmtl()) > INT32_MAX) {
				/*
				 * Since gmt_lag can at most be
				 * +/- 12 hours, something is
				 * *seriously* messed up here.
				 */
				error = EOVERFLOW;
			} else if (suword32((void *)arg1, (int32_t)gmtl) == -1)
				error = EFAULT;
#endif
		}
		break;

	case RTCSYNC:
		if ((error = secpolicy_settime(CRED())) == 0)
			rtcsync();
		break;

	/* END OF real time clock management commands */

	default:
		error = EINVAL;
		break;
	}
	return (error == 0 ? 0 : set_errno(error));
}

void
usd_to_ssd(user_desc_t *usd, struct ssd *ssd, selector_t sel)
{
	ssd->bo = USEGD_GETBASE(usd);
	ssd->ls = USEGD_GETLIMIT(usd);
	ssd->sel = sel;

	/*
	 * set type, dpl and present bits.
	 */
	ssd->acc1 = usd->usd_type;
	ssd->acc1 |= usd->usd_dpl << 5;
	ssd->acc1 |= usd->usd_p << (5 + 2);

	/*
	 * set avl, DB and granularity bits.
	 */
	ssd->acc2 = usd->usd_avl;

	ssd->acc2 |= usd->usd_long << 1;

	ssd->acc2 |= usd->usd_def32 << (1 + 1);
	ssd->acc2 |= usd->usd_gran << (1 + 1 + 1);
}

static void
ssd_to_usd(struct ssd *ssd, user_desc_t *usd)
{

	ASSERT(bcmp(usd, &null_udesc, sizeof (*usd)) == 0);

	USEGD_SETBASE(usd, ssd->bo);
	USEGD_SETLIMIT(usd, ssd->ls);

	/*
	 * Set type, dpl and present bits.
	 *
	 * Force the "accessed" bit to on so that we don't run afoul of
	 * KPTI.
	 */
	usd->usd_type = ssd->acc1 | SDT_A;
	usd->usd_dpl = ssd->acc1 >> 5;
	usd->usd_p = ssd->acc1 >> (5 + 2);

	ASSERT(usd->usd_type >= SDT_MEMRO);
	ASSERT(usd->usd_dpl == SEL_UPL);

	/*
	 * 64-bit code selectors are never allowed in the LDT.
	 * Reserved bit is always 0 on 32-bit systems.
	 */
	usd->usd_long = 0;

	/*
	 * set avl, DB and granularity bits.
	 */
	usd->usd_avl = ssd->acc2;
	usd->usd_def32 = ssd->acc2 >> (1 + 1);
	usd->usd_gran = ssd->acc2 >> (1 + 1 + 1);
}



/*
 * Load LDT register with the current process's LDT.
 */
static void
ldt_load(void)
{
#if defined(__xpv)
	xen_set_ldt(curproc->p_ldt, curproc->p_ldtlimit + 1);
#else
	size_t len;
	system_desc_t desc;

	/*
	 * Before we can use the LDT on this CPU, we must install the LDT in the
	 * user mapping table.
	 */
	len = (curproc->p_ldtlimit + 1) * sizeof (user_desc_t);
	bcopy(curproc->p_ldt, CPU->cpu_m.mcpu_ldt, len);
	CPU->cpu_m.mcpu_ldt_len = len;
	set_syssegd(&desc, CPU->cpu_m.mcpu_ldt, len - 1, SDT_SYSLDT, SEL_KPL);
	*((system_desc_t *)&CPU->cpu_gdt[GDT_LDT]) = desc;

	wr_ldtr(ULDT_SEL);
#endif
}

/*
 * Store a NULL selector in the LDTR. All subsequent illegal references to
 * the LDT will result in a #gp.
 */
void
ldt_unload(void)
{
#if defined(__xpv)
	xen_set_ldt(NULL, 0);
#else
	*((system_desc_t *)&CPU->cpu_gdt[GDT_LDT]) = null_sdesc;
	wr_ldtr(0);

	bzero(CPU->cpu_m.mcpu_ldt, CPU->cpu_m.mcpu_ldt_len);
	CPU->cpu_m.mcpu_ldt_len = 0;
#endif
}

/*ARGSUSED*/
static void
ldt_savectx(proc_t *p)
{
	ASSERT(p->p_ldt != NULL);
	ASSERT(p == curproc);

	/*
	 * The 64-bit kernel must be sure to clear any stale ldt
	 * selectors when context switching away from a process that
	 * has a private ldt. Consider the following example:
	 *
	 *	Wine creats a ldt descriptor and points a segment register
	 *	to it.
	 *
	 *	We then context switch away from wine lwp to kernel
	 *	thread and hit breakpoint in kernel with kmdb
	 *
	 *	When we continue and resume from kmdb we will #gp
	 *	fault since kmdb will have saved the stale ldt selector
	 *	from wine and will try to restore it but we are no longer in
	 *	the context of the wine process and do not have our
	 *	ldtr register pointing to the private ldt.
	 */
	reset_sregs();

	ldt_unload();
	cpu_fast_syscall_enable();
}

static void
ldt_restorectx(proc_t *p)
{
	ASSERT(p->p_ldt != NULL);
	ASSERT(p == curproc);

	ldt_load();
	cpu_fast_syscall_disable();
}

/*
 * At exec time, we need to clear up our LDT context and re-enable fast syscalls
 * for the new process image.
 *
 * The same is true for the other case, where we have:
 *
 * proc_exit()
 *  ->exitpctx()->ldt_savectx()
 *  ->freepctx()->ldt_freectx()
 *
 * Because pre-emption is not prevented between the two callbacks, we could have
 * come off CPU, and brought back LDT context when coming back on CPU via
 * ldt_restorectx().
 */
/* ARGSUSED */
static void
ldt_freectx(proc_t *p, int isexec)
{
	ASSERT(p->p_ldt != NULL);
	ASSERT(p == curproc);

	kpreempt_disable();
	ldt_free(p);
	cpu_fast_syscall_enable();
	kpreempt_enable();
}

/*
 * Install ctx op that ensures syscall/sysenter are disabled.
 * See comments below.
 *
 * When a thread with a private LDT forks, the new process
 * must have the LDT context ops installed.
 */
/* ARGSUSED */
static void
ldt_installctx(proc_t *p, proc_t *cp)
{
	proc_t		*targ = p;
	kthread_t	*t;

	/*
	 * If this is a fork, operate on the child process.
	 */
	if (cp != NULL) {
		targ = cp;
		ldt_dup(p, cp);
	}

	/*
	 * The process context ops expect the target process as their argument.
	 */
	ASSERT(removepctx(targ, targ, ldt_savectx, ldt_restorectx,
	    ldt_installctx, ldt_savectx, ldt_freectx) == 0);

	installpctx(targ, targ, ldt_savectx, ldt_restorectx,
	    ldt_installctx, ldt_savectx, ldt_freectx);

	/*
	 * We've just disabled fast system call and return instructions; take
	 * the slow path out to make sure we don't try to use one to return
	 * back to user. We must set t_post_sys for every thread in the
	 * process to make sure none of them escape out via fast return.
	 */

	mutex_enter(&targ->p_lock);
	t = targ->p_tlist;
	do {
		t->t_post_sys = 1;
	} while ((t = t->t_forw) != targ->p_tlist);
	mutex_exit(&targ->p_lock);
}

int
setdscr(struct ssd *ssd)
{
	ushort_t seli;		/* selector index */
	user_desc_t *ldp;	/* descriptor pointer */
	user_desc_t ndesc;	/* new descriptor */
	proc_t	*pp = curproc;
	int	rc = 0;

	/*
	 * LDT segments: executable and data at DPL 3 only.
	 */
	if (!SELISLDT(ssd->sel) || !SELISUPL(ssd->sel))
		return (EINVAL);

	/*
	 * check the selector index.
	 */
	seli = SELTOIDX(ssd->sel);
	if (seli >= MAXNLDT || seli < LDT_UDBASE)
		return (EINVAL);

	ndesc = null_udesc;
	mutex_enter(&pp->p_ldtlock);

	/*
	 * If this is the first time for this process then setup a
	 * private LDT for it.
	 */
	if (pp->p_ldt == NULL) {
		ldt_alloc(pp, seli);

		/*
		 * Now that this process has a private LDT, the use of
		 * the syscall/sysret and sysenter/sysexit instructions
		 * is forbidden for this processes because they destroy
		 * the contents of %cs and %ss segment registers.
		 *
		 * Explicity disable them here and add a context handler
		 * to the process. Note that disabling
		 * them here means we can't use sysret or sysexit on
		 * the way out of this system call - so we force this
		 * thread to take the slow path (which doesn't make use
		 * of sysenter or sysexit) back out.
		 */
		kpreempt_disable();
		ldt_installctx(pp, NULL);
		cpu_fast_syscall_disable();
		ASSERT(curthread->t_post_sys != 0);
		kpreempt_enable();

	} else if (seli > pp->p_ldtlimit) {
		ASSERT(pp->p_pctx != NULL);

		/*
		 * Increase size of ldt to include seli.
		 */
		ldt_grow(pp, seli);
	}

	ASSERT(seli <= pp->p_ldtlimit);
	ldp = &pp->p_ldt[seli];

	/*
	 * On the 64-bit kernel, this is where things get more subtle.
	 * Recall that in the 64-bit kernel, when we enter the kernel we
	 * deliberately -don't- reload the segment selectors we came in on
	 * for %ds, %es, %fs or %gs. Messing with selectors is expensive,
	 * and the underlying descriptors are essentially ignored by the
	 * hardware in long mode - except for the base that we override with
	 * the gsbase MSRs.
	 *
	 * However, there's one unfortunate issue with this rosy picture --
	 * a descriptor that's not marked as 'present' will still generate
	 * an #np when loading a segment register.
	 *
	 * Consider this case.  An lwp creates a harmless LDT entry, points
	 * one of it's segment registers at it, then tells the kernel (here)
	 * to delete it.  In the 32-bit kernel, the #np will happen on the
	 * way back to userland where we reload the segment registers, and be
	 * handled in kern_gpfault().  In the 64-bit kernel, the same thing
	 * will happen in the normal case too.  However, if we're trying to
	 * use a debugger that wants to save and restore the segment registers,
	 * and the debugger things that we have valid segment registers, we
	 * have the problem that the debugger will try and restore the
	 * segment register that points at the now 'not present' descriptor
	 * and will take a #np right there.
	 *
	 * We should obviously fix the debugger to be paranoid about
	 * -not- restoring segment registers that point to bad descriptors;
	 * however we can prevent the problem here if we check to see if any
	 * of the segment registers are still pointing at the thing we're
	 * destroying; if they are, return an error instead. (That also seems
	 * a lot better failure mode than SIGKILL and a core file
	 * from kern_gpfault() too.)
	 */
	if (SI86SSD_PRES(ssd) == 0) {
		kthread_t *t;
		int bad = 0;

		/*
		 * Look carefully at the segment registers of every lwp
		 * in the process (they're all stopped by our caller).
		 * If we're about to invalidate a descriptor that's still
		 * being referenced by *any* of them, return an error,
		 * rather than having them #gp on their way out of the kernel.
		 */
		ASSERT(pp->p_lwprcnt == 1);

		mutex_enter(&pp->p_lock);
		t = pp->p_tlist;
		do {
			klwp_t *lwp = ttolwp(t);
			struct regs *rp = lwp->lwp_regs;
			pcb_t *pcb = &lwp->lwp_pcb;

			if (ssd->sel == rp->r_cs || ssd->sel == rp->r_ss) {
				bad = 1;
				break;
			}

			if (PCB_NEED_UPDATE_SEGS(pcb)) {
				if (ssd->sel == pcb->pcb_ds ||
				    ssd->sel == pcb->pcb_es ||
				    ssd->sel == pcb->pcb_fs ||
				    ssd->sel == pcb->pcb_gs) {
					bad = 1;
					break;
				}
			} else {
				if (ssd->sel == rp->r_ds ||
				    ssd->sel == rp->r_es ||
				    ssd->sel == rp->r_fs ||
				    ssd->sel == rp->r_gs) {
					bad = 1;
					break;
				}
			}

		} while ((t = t->t_forw) != pp->p_tlist);
		mutex_exit(&pp->p_lock);

		if (bad) {
			mutex_exit(&pp->p_ldtlock);
			return (EBUSY);
		}
	}

	/*
	 * If acc1 is zero, clear the descriptor (including the 'present' bit).
	 * Make sure we update the CPU-private copy of the LDT.
	 */
	if (ssd->acc1 == 0) {
		rc  = ldt_update_segd(ldp, &null_udesc);
		kpreempt_disable();
		ldt_load();
		kpreempt_enable();
		mutex_exit(&pp->p_ldtlock);
		return (rc);
	}

	/*
	 * Check segment type, allow segment not present and
	 * only user DPL (3).
	 */
	if (SI86SSD_DPL(ssd) != SEL_UPL) {
		mutex_exit(&pp->p_ldtlock);
		return (EINVAL);
	}

	/*
	 * Do not allow 32-bit applications to create 64-bit mode code
	 * segments.
	 */
	if (SI86SSD_ISUSEG(ssd) && ((SI86SSD_TYPE(ssd) >> 3) & 1) == 1 &&
	    SI86SSD_ISLONG(ssd)) {
		mutex_exit(&pp->p_ldtlock);
		return (EINVAL);
	}

	/*
	 * Set up a code or data user segment descriptor, making sure to update
	 * the CPU-private copy of the LDT.
	 */
	if (SI86SSD_ISUSEG(ssd)) {
		ssd_to_usd(ssd, &ndesc);
		rc = ldt_update_segd(ldp, &ndesc);
		kpreempt_disable();
		ldt_load();
		kpreempt_enable();
		mutex_exit(&pp->p_ldtlock);
		return (rc);
	}

	mutex_exit(&pp->p_ldtlock);
	return (EINVAL);
}

/*
 * Allocate new LDT for process just large enough to contain seli.  Note we
 * allocate and grow LDT in PAGESIZE chunks. We do this to simplify the
 * implementation and because on the hypervisor it's required, since the LDT
 * must live on pages that have PROT_WRITE removed and which are given to the
 * hypervisor.
 *
 * Note that we don't actually load the LDT into the current CPU here: it's done
 * later by our caller.
 */
static void
ldt_alloc(proc_t *pp, uint_t seli)
{
	user_desc_t	*ldt;
	size_t		ldtsz;
	uint_t		nsels;

	ASSERT(MUTEX_HELD(&pp->p_ldtlock));
	ASSERT(pp->p_ldt == NULL);
	ASSERT(pp->p_ldtlimit == 0);

	/*
	 * Allocate new LDT just large enough to contain seli. The LDT must
	 * always be allocated in units of pages for KPTI.
	 */
	ldtsz = P2ROUNDUP((seli + 1) * sizeof (user_desc_t), PAGESIZE);
	nsels = ldtsz / sizeof (user_desc_t);
	ASSERT(nsels >= MINNLDT && nsels <= MAXNLDT);

	ldt = kmem_zalloc(ldtsz, KM_SLEEP);
	ASSERT(IS_P2ALIGNED(ldt, PAGESIZE));

#if defined(__xpv)
	if (xen_ldt_setprot(ldt, ldtsz, PROT_READ))
		panic("ldt_alloc:xen_ldt_setprot(PROT_READ) failed");
#endif

	pp->p_ldt = ldt;
	pp->p_ldtlimit = nsels - 1;
}

static void
ldt_free(proc_t *pp)
{
	user_desc_t	*ldt;
	size_t		ldtsz;

	ASSERT(pp->p_ldt != NULL);

	mutex_enter(&pp->p_ldtlock);
	ldt = pp->p_ldt;
	ldtsz = (pp->p_ldtlimit + 1) * sizeof (user_desc_t);

	ASSERT(IS_P2ALIGNED(ldtsz, PAGESIZE));

	pp->p_ldt = NULL;
	pp->p_ldtlimit = 0;
	mutex_exit(&pp->p_ldtlock);

	if (pp == curproc) {
		kpreempt_disable();
		ldt_unload();
		kpreempt_enable();
	}

#if defined(__xpv)
	/*
	 * We are not allowed to make the ldt writable until after
	 * we tell the hypervisor to unload it.
	 */
	if (xen_ldt_setprot(ldt, ldtsz, PROT_READ | PROT_WRITE))
		panic("ldt_free:xen_ldt_setprot(PROT_READ|PROT_WRITE) failed");
#endif

	kmem_free(ldt, ldtsz);
}

/*
 * On fork copy new ldt for child.
 */
static void
ldt_dup(proc_t *pp, proc_t *cp)
{
	size_t	ldtsz;

	ASSERT(pp->p_ldt != NULL);
	ASSERT(cp != curproc);

	/*
	 * I assume the parent's ldt can't increase since we're in a fork.
	 */
	mutex_enter(&pp->p_ldtlock);
	mutex_enter(&cp->p_ldtlock);

	ldtsz = (pp->p_ldtlimit + 1) * sizeof (user_desc_t);

	ldt_alloc(cp, pp->p_ldtlimit);

#if defined(__xpv)
	/*
	 * Make child's ldt writable so it can be copied into from
	 * parent's ldt. This works since ldt_alloc above did not load
	 * the ldt since its for the child process. If we tried to make
	 * an LDT writable that is loaded in hw the setprot operation
	 * would fail.
	 */
	if (xen_ldt_setprot(cp->p_ldt, ldtsz, PROT_READ | PROT_WRITE))
		panic("ldt_dup:xen_ldt_setprot(PROT_READ|PROT_WRITE) failed");
#endif

	bcopy(pp->p_ldt, cp->p_ldt, ldtsz);

#if defined(__xpv)
	if (xen_ldt_setprot(cp->p_ldt, ldtsz, PROT_READ))
		panic("ldt_dup:xen_ldt_setprot(PROT_READ) failed");
#endif
	mutex_exit(&cp->p_ldtlock);
	mutex_exit(&pp->p_ldtlock);

}

/*
 * Note that we don't actually load the LDT into the current CPU here: it's done
 * later by our caller - unless we take an error.  This works out because
 * ldt_load() does a copy of ->p_ldt instead of directly loading it into the GDT
 * (and therefore can't be using the freed old LDT), and by definition if the
 * new entry didn't pass validation, then the proc shouldn't be referencing an
 * entry in the extended region.
 */
static void
ldt_grow(proc_t *pp, uint_t seli)
{
	user_desc_t	*oldt, *nldt;
	uint_t		nsels;
	size_t		oldtsz, nldtsz;

	ASSERT(MUTEX_HELD(&pp->p_ldtlock));
	ASSERT(pp->p_ldt != NULL);
	ASSERT(pp->p_ldtlimit != 0);

	/*
	 * Allocate larger LDT just large enough to contain seli. The LDT must
	 * always be allocated in units of pages for KPTI.
	 */
	nldtsz = P2ROUNDUP((seli + 1) * sizeof (user_desc_t), PAGESIZE);
	nsels = nldtsz / sizeof (user_desc_t);
	ASSERT(nsels >= MINNLDT && nsels <= MAXNLDT);
	ASSERT(nsels > pp->p_ldtlimit);

	oldt = pp->p_ldt;
	oldtsz = (pp->p_ldtlimit + 1) * sizeof (user_desc_t);

	nldt = kmem_zalloc(nldtsz, KM_SLEEP);
	ASSERT(IS_P2ALIGNED(nldt, PAGESIZE));

	bcopy(oldt, nldt, oldtsz);

	/*
	 * unload old ldt.
	 */
	kpreempt_disable();
	ldt_unload();
	kpreempt_enable();

#if defined(__xpv)

	/*
	 * Make old ldt writable and new ldt read only.
	 */
	if (xen_ldt_setprot(oldt, oldtsz, PROT_READ | PROT_WRITE))
		panic("ldt_grow:xen_ldt_setprot(PROT_READ|PROT_WRITE) failed");

	if (xen_ldt_setprot(nldt, nldtsz, PROT_READ))
		panic("ldt_grow:xen_ldt_setprot(PROT_READ) failed");
#endif

	pp->p_ldt = nldt;
	pp->p_ldtlimit = nsels - 1;

	kmem_free(oldt, oldtsz);
}