kdi - zygaena/hammerhead

root / base / usr / src / uts / intel / kdi
kdi Plain Text 1530 lines 40.8 KB
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 *
 * Copyright 2019 Joyent, Inc.
 */

/*
 * Debugger entry and exit for both master and slave CPUs. kdi_idthdl.s contains
 * the IDT stubs that drop into here (mainly via kdi_cmnint).
 */

#include <sys/segments.h>
#include <sys/asm_linkage.h>
#include <sys/controlregs.h>
#include <sys/x86_archext.h>
#include <sys/privregs.h>
#include <sys/machprivregs.h>
#include <sys/kdi_regs.h>
#include <sys/psw.h>
#include <sys/uadmin.h>
#ifdef __xpv
#include <sys/hypervisor.h>
#endif
#include <kdi_assym.h>
#include <assym.h>

/* clobbers %rdx, %rcx, returns addr in %rax, CPU ID in %rbx */
#define	GET_CPUSAVE_ADDR \
	movzbq	%gs:CPU_ID, %rbx;		\
	movq	%rbx, %rax;			\
	movq	$KRS_SIZE, %rcx;		\
	mulq	%rcx;				\
	movq	$kdi_cpusave, %rdx;		\
	/*CSTYLED*/				\
	addq	(%rdx), %rax

/*
 * Save copies of the IDT and GDT descriptors.  Note that we only save the IDT
 * and GDT if the IDT isn't ours, as we may be legitimately re-entering the
 * debugger through the trap handler.  We don't want to clobber the saved IDT
 * in the process, as we'd end up resuming the world on our IDT.
 */
#define	SAVE_IDTGDT				\
	movq	%gs:CPU_IDT, %r11;		\
	leaq    kdi_idt(%rip), %rsi;		\
	cmpq	%rsi, %r11;			\
	je	1f;				\
	movq	%r11, KRS_IDT(%rax);		\
	movq	%gs:CPU_GDT, %r11;		\
	movq	%r11, KRS_GDT(%rax);		\
1:

#ifdef __xpv

/*
 * Already on kernel gsbase via the hypervisor.
 */
#define	SAVE_GSBASE(reg) /* nothing */
#define	RESTORE_GSBASE(reg) /* nothing */

#else

#define	SAVE_GSBASE(base)				\
	movl	$MSR_AMD_GSBASE, %ecx;			\
	rdmsr;						\
	shlq	$32, %rdx;				\
	orq	%rax, %rdx;				\
	movq	%rdx, REG_OFF(KDIREG_GSBASE)(base);	\
	movl	$MSR_AMD_KGSBASE, %ecx;			\
	rdmsr;						\
	shlq	$32, %rdx;				\
	orq	%rax, %rdx;				\
	movq	%rdx, REG_OFF(KDIREG_KGSBASE)(base)

/*
 * We shouldn't have stomped on KGSBASE, so don't try to restore it.
 */
#define	RESTORE_GSBASE(base)				\
	movq	REG_OFF(KDIREG_GSBASE)(base), %rdx;	\
	movq	%rdx, %rax;				\
	shrq	$32, %rdx;				\
	movl	$MSR_AMD_GSBASE, %ecx;			\
	wrmsr

#endif /* __xpv */

/*
 * %ss, %rsp, %rflags, %cs, %rip, %err, %trapno are already on the stack.
 */
#define	KDI_SAVE_REGS(base) \
	movq	%rdi, REG_OFF(KDIREG_RDI)(base);	\
	movq	%rsi, REG_OFF(KDIREG_RSI)(base);	\
	movq	%rdx, REG_OFF(KDIREG_RDX)(base);	\
	movq	%rcx, REG_OFF(KDIREG_RCX)(base);	\
	movq	%r8, REG_OFF(KDIREG_R8)(base);		\
	movq	%r9, REG_OFF(KDIREG_R9)(base);		\
	movq	%rax, REG_OFF(KDIREG_RAX)(base);	\
	movq	%rbx, REG_OFF(KDIREG_RBX)(base);	\
	movq	%rbp, REG_OFF(KDIREG_RBP)(base);	\
	movq	%r10, REG_OFF(KDIREG_R10)(base);	\
	movq	%r11, REG_OFF(KDIREG_R11)(base);	\
	movq	%r12, REG_OFF(KDIREG_R12)(base);	\
	movq	%r13, REG_OFF(KDIREG_R13)(base);	\
	movq	%r14, REG_OFF(KDIREG_R14)(base);	\
	movq	%r15, REG_OFF(KDIREG_R15)(base);	\
	movq	%rbp, REG_OFF(KDIREG_SAVFP)(base);	\
	movq	REG_OFF(KDIREG_RIP)(base), %rax;	\
	movq	%rax, REG_OFF(KDIREG_SAVPC)(base);	\
	movq	%cr2, %rax;				\
	movq	%rax, REG_OFF(KDIREG_CR2)(base);	\
	clrq	%rax;					\
	movw	%ds, %ax;				\
	movq	%rax, REG_OFF(KDIREG_DS)(base);		\
	movw	%es, %ax;				\
	movq	%rax, REG_OFF(KDIREG_ES)(base);		\
	movw	%fs, %ax;				\
	movq	%rax, REG_OFF(KDIREG_FS)(base);		\
	movw	%gs, %ax;				\
	movq	%rax, REG_OFF(KDIREG_GS)(base);		\
	SAVE_GSBASE(base)

#define	KDI_RESTORE_REGS(base) \
	movq	base, %rdi;				\
	RESTORE_GSBASE(%rdi);				\
	movq	REG_OFF(KDIREG_ES)(%rdi), %rax;		\
	movw	%ax, %es;				\
	movq	REG_OFF(KDIREG_DS)(%rdi), %rax;		\
	movw	%ax, %ds;				\
	movq	REG_OFF(KDIREG_CR2)(base), %rax;	\
	movq	%rax, %cr2;				\
	movq	REG_OFF(KDIREG_R15)(%rdi), %r15;	\
	movq	REG_OFF(KDIREG_R14)(%rdi), %r14;	\
	movq	REG_OFF(KDIREG_R13)(%rdi), %r13;	\
	movq	REG_OFF(KDIREG_R12)(%rdi), %r12;	\
	movq	REG_OFF(KDIREG_R11)(%rdi), %r11;	\
	movq	REG_OFF(KDIREG_R10)(%rdi), %r10;	\
	movq	REG_OFF(KDIREG_RBP)(%rdi), %rbp;	\
	movq	REG_OFF(KDIREG_RBX)(%rdi), %rbx;	\
	movq	REG_OFF(KDIREG_RAX)(%rdi), %rax;	\
	movq	REG_OFF(KDIREG_R9)(%rdi), %r9;		\
	movq	REG_OFF(KDIREG_R8)(%rdi), %r8;		\
	movq	REG_OFF(KDIREG_RCX)(%rdi), %rcx;	\
	movq	REG_OFF(KDIREG_RDX)(%rdi), %rdx;	\
	movq	REG_OFF(KDIREG_RSI)(%rdi), %rsi;	\
	movq	REG_OFF(KDIREG_RDI)(%rdi), %rdi

/*
 * Given the address of the current CPU's cpusave area in %rax, the following
 * macro restores the debugging state to said CPU.  Restored state includes
 * the debug registers from the global %dr variables.
 *
 * Takes the cpusave area in %rdi as a parameter.
 */
#define	KDI_RESTORE_DEBUGGING_STATE \
	pushq	%rdi;						\
	leaq	kdi_drreg(%rip), %r15;				\
	movl	$7, %edi;					\
	movq	DR_CTL(%r15), %rsi;				\
	call	kdi_dreg_set;					\
								\
	movl	$6, %edi;					\
	movq	$KDIREG_DRSTAT_RESERVED, %rsi;			\
	call	kdi_dreg_set;					\
								\
	movl	$0, %edi;					\
	movq	DRADDR_OFF(0)(%r15), %rsi;			\
	call	kdi_dreg_set;					\
	movl	$1, %edi;					\
	movq	DRADDR_OFF(1)(%r15), %rsi;			\
	call	kdi_dreg_set;					\
	movl	$2, %edi;					\
	movq	DRADDR_OFF(2)(%r15), %rsi;			\
	call	kdi_dreg_set;					\
	movl	$3, %edi;					\
	movq	DRADDR_OFF(3)(%r15), %rsi;			\
	call	kdi_dreg_set;					\
	popq	%rdi;

/*
 * Each cpusave buffer has an area set aside for a ring buffer of breadcrumbs.
 * The following macros manage the buffer.
 */

/* Advance the ring buffer */
#define	ADVANCE_CRUMB_POINTER(cpusave, tmp1, tmp2) \
	movq	KRS_CURCRUMBIDX(cpusave), tmp1;	\
	cmpq	$[KDI_NCRUMBS - 1], tmp1;	\
	jge	1f;				\
	/* Advance the pointer and index */	\
	addq	$1, tmp1;			\
	movq	tmp1, KRS_CURCRUMBIDX(cpusave);	\
	movq	KRS_CURCRUMB(cpusave), tmp1;	\
	addq	$KRM_SIZE, tmp1;		\
	jmp	2f;				\
1:	/* Reset the pointer and index */	\
	movq	$0, KRS_CURCRUMBIDX(cpusave);	\
	leaq	KRS_CRUMBS(cpusave), tmp1;	\
2:	movq	tmp1, KRS_CURCRUMB(cpusave);	\
	/* Clear the new crumb */		\
	movq	$KDI_NCRUMBS, tmp2;		\
3:	movq	$0, -4(tmp1, tmp2, 4);		\
	decq	tmp2;				\
	jnz	3b

/* Set a value in the current breadcrumb buffer */
#define	ADD_CRUMB(cpusave, offset, value, tmp) \
	movq	KRS_CURCRUMB(cpusave), tmp;	\
	movq	value, offset(tmp)

	/* XXX implement me */
	ENTRY_NP(kdi_nmiint)
	clrq	%rcx
	movq	(%rcx), %rcx
	SET_SIZE(kdi_nmiint)

	/*
	 * The main entry point for master CPUs.  It also serves as the trap
	 * handler for all traps and interrupts taken during single-step.
	 */
	ENTRY_NP(kdi_cmnint)
	ALTENTRY(kdi_master_entry)

	pushq	%rax
	CLI(%rax)
	popq	%rax

	/* Save current register state */
	subq	$REG_OFF(KDIREG_TRAPNO), %rsp
	KDI_SAVE_REGS(%rsp)

#ifdef __xpv
	/*
	 * Clear saved_upcall_mask in unused byte of cs slot on stack.
	 * It can only confuse things.
	 */
	movb	$0, REG_OFF(KDIREG_CS)+4(%rsp)
#endif

#if !defined(__xpv)
	/*
	 * Switch to the kernel's GSBASE.  Neither GSBASE nor the ill-named
	 * KGSBASE can be trusted, as the kernel may or may not have already
	 * done a swapgs.  All is not lost, as the kernel can divine the correct
	 * value for us.  Note that the previous GSBASE is saved in the
	 * KDI_SAVE_REGS macro to prevent a usermode process's GSBASE from being
	 * blown away.  On the hypervisor, we don't need to do this, since it's
	 * ensured we're on our requested kernel GSBASE already.
	 *
	 * No need to worry about swapgs speculation here as it's unconditional
	 * and via wrmsr anyway.
	 */
	subq	$10, %rsp
	sgdt	(%rsp)
	movq	2(%rsp), %rdi	/* gdt base now in %rdi */
	addq	$10, %rsp
	call	kdi_gdt2gsbase	/* returns kernel's GSBASE in %rax */

	movq	%rax, %rdx
	shrq	$32, %rdx
	movl	$MSR_AMD_GSBASE, %ecx
	wrmsr

	/*
	 * In the trampoline we stashed the incoming %cr3. Copy this into
	 * the kdiregs for restoration and later use.
	 */
	mov	%gs:(CPU_KPTI_DBG+KPTI_TR_CR3), %rdx
	mov	%rdx, REG_OFF(KDIREG_CR3)(%rsp)
	/*
	 * Switch to the kernel's %cr3. From the early interrupt handler
	 * until now we've been running on the "paranoid" %cr3 (that of kas
	 * from early in boot).
	 *
	 * If we took the interrupt from somewhere already on the kas/paranoid
	 * %cr3 though, don't change it (this could happen if kcr3 is corrupt
	 * and we took a gptrap earlier from this very code).
	 */
	cmpq	%rdx, kpti_safe_cr3
	je	.no_kcr3
	mov	%gs:CPU_KPTI_KCR3, %rdx
	cmpq	$0, %rdx
	je	.no_kcr3
	mov	%rdx, %cr3
.no_kcr3:

#endif	/* __xpv */

	GET_CPUSAVE_ADDR	/* %rax = cpusave, %rbx = CPU ID */

	ADVANCE_CRUMB_POINTER(%rax, %rcx, %rdx)

	ADD_CRUMB(%rax, KRM_CPU_STATE, $KDI_CPU_STATE_MASTER, %rdx)

	movq	REG_OFF(KDIREG_RIP)(%rsp), %rcx
	ADD_CRUMB(%rax, KRM_PC, %rcx, %rdx)
	ADD_CRUMB(%rax, KRM_SP, %rsp, %rdx)
	movq	REG_OFF(KDIREG_TRAPNO)(%rsp), %rcx
	ADD_CRUMB(%rax, KRM_TRAPNO, %rcx, %rdx)

	movq	%rsp, %rbp
	pushq	%rax

	/*
	 * Were we in the debugger when we took the trap (i.e. was %esp in one
	 * of the debugger's memory ranges)?
	 */
	leaq	kdi_memranges, %rcx
	movl	kdi_nmemranges, %edx
1:
	cmpq	MR_BASE(%rcx), %rsp
	jl	2f		/* below this range -- try the next one */
	cmpq	MR_LIM(%rcx), %rsp
	jg	2f		/* above this range -- try the next one */
	jmp	3f		/* matched within this range */

2:
	decl	%edx
	jz	kdi_save_common_state	/* %rsp not within debugger memory */
	addq	$MR_SIZE, %rcx
	jmp	1b

3:	/*
	 * The master is still set.  That should only happen if we hit a trap
	 * while running in the debugger.  Note that it may be an intentional
	 * fault.  kmdb_dpi_handle_fault will sort it all out.
	 */

	movq	REG_OFF(KDIREG_TRAPNO)(%rbp), %rdi
	movq	REG_OFF(KDIREG_RIP)(%rbp), %rsi
	movq	REG_OFF(KDIREG_RSP)(%rbp), %rdx
	movq	%rbx, %rcx		/* cpuid */

	call	kdi_dvec_handle_fault

	/*
	 * If we're here, we ran into a debugger problem, and the user
	 * elected to solve it by having the debugger debug itself.  The
	 * state we're about to save is that of the debugger when it took
	 * the fault.
	 */

	jmp	kdi_save_common_state

	SET_SIZE(kdi_master_entry)
	SET_SIZE(kdi_cmnint)

/*
 * The cross-call handler for slave CPUs.
 *
 * The debugger is single-threaded, so only one CPU, called the master, may be
 * running it at any given time.  The other CPUs, known as slaves, spin in a
 * busy loop until there's something for them to do.  This is the entry point
 * for the slaves - they'll be sent here in response to a cross-call sent by the
 * master.
 */

	ENTRY_NP(kdi_slave_entry)

	/*
	 * Cross calls are implemented as function calls, so our stack currently
	 * looks like one you'd get from a zero-argument function call.  That
	 * is, there's the return %rip at %rsp, and that's about it.  We need
	 * to make it look like an interrupt stack.  When we first save, we'll
	 * reverse the saved %ss and %rip, which we'll fix back up when we've
	 * freed up some general-purpose registers.  We'll also need to fix up
	 * the saved %rsp.
	 */

	pushq	%rsp		/* pushed value off by 8 */
	pushfq
	CLI(%rax)
	pushq	$KCS_SEL
	clrq	%rax
	movw	%ss, %ax
	pushq	%rax		/* rip should be here */
	pushq	$-1		/* phony trap error code */
	pushq	$-1		/* phony trap number */

	subq	$REG_OFF(KDIREG_TRAPNO), %rsp
	KDI_SAVE_REGS(%rsp)

	movq	%cr3, %rax
	movq	%rax, REG_OFF(KDIREG_CR3)(%rsp)

	movq	REG_OFF(KDIREG_SS)(%rsp), %rax
	movq	%rax, REG_OFF(KDIREG_SAVPC)(%rsp)
	xchgq	REG_OFF(KDIREG_RIP)(%rsp), %rax
	movq	%rax, REG_OFF(KDIREG_SS)(%rsp)

	movq	REG_OFF(KDIREG_RSP)(%rsp), %rax
	addq	$8, %rax
	movq	%rax, REG_OFF(KDIREG_RSP)(%rsp)

	/*
	 * We've saved all of the general-purpose registers, and have a stack
	 * that is irettable (after we strip down to the error code)
	 */

	GET_CPUSAVE_ADDR	/* %rax = cpusave, %rbx = CPU ID */

	ADVANCE_CRUMB_POINTER(%rax, %rcx, %rdx)

	ADD_CRUMB(%rax, KRM_CPU_STATE, $KDI_CPU_STATE_SLAVE, %rdx)

	movq	REG_OFF(KDIREG_RIP)(%rsp), %rcx
	ADD_CRUMB(%rax, KRM_PC, %rcx, %rdx)
	movq	REG_OFF(KDIREG_RSP)(%rsp), %rcx
	ADD_CRUMB(%rax, KRM_SP, %rcx, %rdx)
	ADD_CRUMB(%rax, KRM_TRAPNO, $-1, %rdx)

	movq    $KDI_CPU_STATE_SLAVE, KRS_CPU_STATE(%rax)

	pushq	%rax
	jmp	kdi_save_common_state

	SET_SIZE(kdi_slave_entry)

/*
 * The state of the world:
 *
 * The stack has a complete set of saved registers and segment
 * selectors, arranged in the kdi_regs.h order.  It also has a pointer
 * to our cpusave area.
 *
 * We need to save, into the cpusave area, a pointer to these saved
 * registers.  First we check whether we should jump straight back to
 * the kernel.  If not, we save a few more registers, ready the
 * machine for debugger entry, and enter the debugger.
 */

	ENTRY_NP(kdi_save_common_state)

	popq	%rdi			/* the cpusave area */
	movq	%rsp, KRS_GREGS(%rdi)	/* save ptr to current saved regs */

	pushq	%rdi
	call	kdi_trap_pass
	testq	%rax, %rax
	jnz	kdi_pass_to_kernel
	popq	%rax /* cpusave in %rax */

	SAVE_IDTGDT

#if !defined(__xpv)
	/* Save off %cr0, and clear write protect */
	movq	%cr0, %rcx
	movq	%rcx, KRS_CR0(%rax)
	andq	$_BITNOT(CR0_WP), %rcx
	movq	%rcx, %cr0
#endif

	/* Save the debug registers and disable any active watchpoints */

	movq	%rax, %r15		/* save cpusave area ptr */
	movl	$7, %edi
	call	kdi_dreg_get
	movq	%rax, KRS_DRCTL(%r15)

	andq	$_BITNOT(KDIREG_DRCTL_WPALLEN_MASK), %rax
	movq	%rax, %rsi
	movl	$7, %edi
	call	kdi_dreg_set

	movl	$6, %edi
	call	kdi_dreg_get
	movq	%rax, KRS_DRSTAT(%r15)

	movl	$0, %edi
	call	kdi_dreg_get
	movq	%rax, KRS_DROFF(0)(%r15)

	movl	$1, %edi
	call	kdi_dreg_get
	movq	%rax, KRS_DROFF(1)(%r15)

	movl	$2, %edi
	call	kdi_dreg_get
	movq	%rax, KRS_DROFF(2)(%r15)

	movl	$3, %edi
	call	kdi_dreg_get
	movq	%rax, KRS_DROFF(3)(%r15)

	movq	%r15, %rax	/* restore cpu save area to rax */

	clrq	%rbp		/* stack traces should end here */

	pushq	%rax
	movq	%rax, %rdi	/* cpusave */

	call	kdi_debugger_entry

	/* Pass cpusave to kdi_resume */
	popq	%rdi

	jmp	kdi_resume

	SET_SIZE(kdi_save_common_state)

/*
 * Resume the world.  The code that calls kdi_resume has already
 * decided whether or not to restore the IDT.
 */
	/* cpusave in %rdi */
	ENTRY_NP(kdi_resume)

	/*
	 * Send this CPU back into the world
	 */
#if !defined(__xpv)
	movq	KRS_CR0(%rdi), %rdx
	movq	%rdx, %cr0
#endif

	KDI_RESTORE_DEBUGGING_STATE

	movq	KRS_GREGS(%rdi), %rsp

#if !defined(__xpv)
	/*
	 * If we're going back via tr_iret_kdi, then we want to copy the
	 * final %cr3 we're going to back into the kpti_dbg area now.
	 *
	 * Since the trampoline needs to find the kpti_dbg too, we enter it
	 * with %r13 set to point at that. The real %r13 (to restore before
	 * the iret) we stash in the kpti_dbg itself.
	 */
	movq	%gs:CPU_SELF, %r13	/* can't leaq %gs:*, use self-ptr */
	addq	$CPU_KPTI_DBG, %r13

	movq	REG_OFF(KDIREG_R13)(%rsp), %rdx
	movq	%rdx, KPTI_R13(%r13)

	movq	REG_OFF(KDIREG_CR3)(%rsp), %rdx
	movq	%rdx, KPTI_TR_CR3(%r13)

	/* The trampoline will undo this later. */
	movq	%r13, REG_OFF(KDIREG_R13)(%rsp)
#endif

	KDI_RESTORE_REGS(%rsp)
	addq	$REG_OFF(KDIREG_RIP), %rsp	/* Discard state, trapno, err */
	/*
	 * The common trampoline code will restore %cr3 to the right value
	 * for either kernel or userland.
	 */
#if !defined(__xpv)
	jmp	tr_iret_kdi
#else
	IRET
#endif
	/*NOTREACHED*/
	SET_SIZE(kdi_resume)


	/*
	 * We took a trap that should be handled by the kernel, not KMDB.
	 *
	 * We're hard-coding the three cases where KMDB has installed permanent
	 * handlers, since after we KDI_RESTORE_REGS(), we don't have registers
	 * to work with; we can't use a global since other CPUs can easily pass
	 * through here at the same time.
	 *
	 * Note that we handle T_DBGENTR since userspace might have tried it.
	 *
	 * The trap handler will expect the stack to be in trap order, with %rip
	 * being the last entry, so we'll need to restore all our regs.  On
	 * i86xpv we'll need to compensate for XPV_TRAP_POP.
	 *
	 * %rax on entry is either 1 or 2, which is from kdi_trap_pass().
	 * kdi_cmnint stashed the original %cr3 into KDIREG_CR3, then (probably)
	 * switched us to the CPU's kf_kernel_cr3. But we're about to call, for
	 * example:
	 *
	 * dbgtrap->trap()->tr_iret_kernel
	 *
	 * which, unlike, tr_iret_kdi, doesn't restore the original %cr3, so
	 * we'll do so here if needed.
	 *
	 * This isn't just a matter of tidiness: for example, consider:
	 *
	 * hat_switch(oldhat=kas.a_hat, newhat=prochat)
	 *  setcr3()
	 *  reset_kpti()
	 *   *brktrap* due to fbt on reset_kpti:entry
	 *
	 * Here, we have the new hat's %cr3, but we haven't yet updated
	 * kf_kernel_cr3 (so its currently kas's). So if we don't restore here,
	 * we'll stay on kas's cr3 value on returning from the trap: not good if
	 * we fault on a userspace address.
	 */
	ENTRY_NP(kdi_pass_to_kernel)

	popq	%rdi /* cpusave */
	movq	$KDI_CPU_STATE_NONE, KRS_CPU_STATE(%rdi)
	movq	KRS_GREGS(%rdi), %rsp

	cmpq	$2, %rax
	jne	no_restore_cr3
	movq	REG_OFF(KDIREG_CR3)(%rsp), %r11
	movq	%r11, %cr3

no_restore_cr3:
	movq	REG_OFF(KDIREG_TRAPNO)(%rsp), %rdi

	cmpq	$T_SGLSTP, %rdi
	je	kdi_pass_dbgtrap
	cmpq	$T_BPTFLT, %rdi
	je	kdi_pass_brktrap
	cmpq	$T_DBGENTR, %rdi
	je	kdi_pass_invaltrap
	/*
	 * Hmm, unknown handler.  Somebody forgot to update this when they
	 * added a new trap interposition... try to drop back into kmdb.
	 */
	int	$T_DBGENTR

#define	CALL_TRAP_HANDLER(name) \
	KDI_RESTORE_REGS(%rsp); \
	/* Discard state, trapno, err */ \
	addq	$REG_OFF(KDIREG_RIP), %rsp; \
	XPV_TRAP_PUSH; \
	jmp	%cs:name

kdi_pass_dbgtrap:
	CALL_TRAP_HANDLER(dbgtrap)
	/*NOTREACHED*/
kdi_pass_brktrap:
	CALL_TRAP_HANDLER(brktrap)
	/*NOTREACHED*/
kdi_pass_invaltrap:
	CALL_TRAP_HANDLER(invaltrap)
	/*NOTREACHED*/

	SET_SIZE(kdi_pass_to_kernel)

	/*
	 * A minimal version of mdboot(), to be used by the master CPU only.
	 */
	ENTRY_NP(kdi_reboot)

	movl	$AD_BOOT, %edi
	movl	$A_SHUTDOWN, %esi
	call	*psm_shutdownf
#if defined(__xpv)
	movl	$SHUTDOWN_reboot, %edi
	call	HYPERVISOR_shutdown
#else
	call	reset
#endif
	/*NOTREACHED*/

	SET_SIZE(kdi_reboot)

	ENTRY_NP(kdi_cpu_debug_init)
	pushq	%rbp
	movq	%rsp, %rbp

	pushq	%rbx		/* macro will clobber %rbx */
	KDI_RESTORE_DEBUGGING_STATE
	popq	%rbx

	leave
	ret
	SET_SIZE(kdi_cpu_debug_init)

#define	GETDREG(name, r)	\
	ENTRY_NP(name);		\
	movq	r, %rax;	\
	ret;			\
	SET_SIZE(name)

#define	SETDREG(name, r)	\
	ENTRY_NP(name);		\
	movq	%rdi, r;	\
	ret;			\
	SET_SIZE(name)

	GETDREG(kdi_getdr0, %dr0)
	GETDREG(kdi_getdr1, %dr1)
	GETDREG(kdi_getdr2, %dr2)
	GETDREG(kdi_getdr3, %dr3)
	GETDREG(kdi_getdr6, %dr6)
	GETDREG(kdi_getdr7, %dr7)

	SETDREG(kdi_setdr0, %dr0)
	SETDREG(kdi_setdr1, %dr1)
	SETDREG(kdi_setdr2, %dr2)
	SETDREG(kdi_setdr3, %dr3)
	SETDREG(kdi_setdr6, %dr6)
	SETDREG(kdi_setdr7, %dr7)

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 *
 * Copyright 2018 Joyent, Inc.
 */

/*
 * Management of KMDB's IDT, which is installed upon KMDB activation.
 *
 * Debugger activation has two flavors, which cover the cases where KMDB is
 * loaded at boot, and when it is loaded after boot.  In brief, in both cases,
 * the KDI needs to interpose upon several handlers in the IDT.  When
 * mod-loaded KMDB is deactivated, we undo the IDT interposition, restoring the
 * handlers to what they were before we started.
 *
 * We also take over the entirety of IDT (except the double-fault handler) on
 * the active CPU when we're in kmdb so we can handle things like page faults
 * sensibly.
 *
 * Boot-loaded KMDB
 *
 * When we're first activated, we're running on boot's IDT.  We need to be able
 * to function in this world, so we'll install our handlers into boot's IDT.
 * This is a little complicated: we're using the fake cpu_t set up by
 * boot_kdi_tmpinit(), so we can't access cpu_idt directly.  Instead,
 * kdi_idt_write() notices that cpu_idt is NULL, and works around this problem.
 *
 * Later, when we're about to switch to the kernel's IDT, it'll call us via
 * kdi_idt_sync(), allowing us to add our handlers to the new IDT.  While
 * boot-loaded KMDB can't be unloaded, we still need to save the descriptors we
 * replace so we can pass traps back to the kernel as necessary.
 *
 * The last phase of boot-loaded KMDB activation occurs at non-boot CPU
 * startup.  We will be called on each non-boot CPU, thus allowing us to set up
 * any watchpoints that may have been configured on the boot CPU and interpose
 * on the given CPU's IDT.  We don't save the interposed descriptors in this
 * case -- see kdi_cpu_init() for details.
 *
 * Mod-loaded KMDB
 *
 * This style of activation is much simpler, as the CPUs are already running,
 * and are using their own copy of the kernel's IDT.  We simply interpose upon
 * each CPU's IDT.  We save the handlers we replace, both for deactivation and
 * for passing traps back to the kernel.  Note that for the hypervisors'
 * benefit, we need to xcall to the other CPUs to do this, since we need to
 * actively set the trap entries in its virtual IDT from that vcpu's context
 * rather than just modifying the IDT table from the CPU running kdi_activate().
 */

#include <sys/types.h>
#include <sys/segments.h>
#include <sys/trap.h>
#include <sys/cpuvar.h>
#include <sys/reboot.h>
#include <sys/sunddi.h>
#include <sys/archsystm.h>
#include <sys/kdi_impl.h>
#include <sys/x_call.h>
#include <sys/psw.h>
#include <vm/hat_i86.h>

#define	KDI_GATE_NVECS	3

#define	KDI_IDT_NOSAVE	0
#define	KDI_IDT_SAVE	1

#define	KDI_IDT_DTYPE_KERNEL	0
#define	KDI_IDT_DTYPE_BOOT	1

/* Solely to keep kdiregs_t in the CTF, otherwise unused. */
kdiregs_t kdi_regs;

kdi_cpusave_t *kdi_cpusave;
int kdi_ncpusave;

static kdi_main_t kdi_kmdb_main;

kdi_drreg_t kdi_drreg;


uintptr_t	kdi_kernel_handler;

int		kdi_trap_switch;

#define	KDI_MEMRANGES_MAX	2

kdi_memrange_t	kdi_memranges[KDI_MEMRANGES_MAX];
int		kdi_nmemranges;

typedef void idt_hdlr_f(void);

extern idt_hdlr_f kdi_trap0, kdi_trap1, kdi_int2, kdi_trap3, kdi_trap4;
extern idt_hdlr_f kdi_trap5, kdi_trap6, kdi_trap7, kdi_trap9;
extern idt_hdlr_f kdi_traperr10, kdi_traperr11, kdi_traperr12;
extern idt_hdlr_f kdi_traperr13, kdi_traperr14, kdi_trap16, kdi_traperr17;
extern idt_hdlr_f kdi_trap18, kdi_trap19, kdi_trap20, kdi_ivct32;
extern idt_hdlr_f kdi_invaltrap;
extern size_t kdi_ivct_size;

typedef struct kdi_gate_spec {
	uint_t kgs_vec;
	uint_t kgs_dpl;
} kdi_gate_spec_t;

/*
 * Beware: kdi_pass_to_kernel() has unpleasant knowledge of this list.
 */
static const kdi_gate_spec_t kdi_gate_specs[KDI_GATE_NVECS] = {
	{ T_SGLSTP, TRP_KPL },
	{ T_BPTFLT, TRP_UPL },
	{ T_DBGENTR, TRP_KPL }
};

static gate_desc_t kdi_kgates[KDI_GATE_NVECS];

extern gate_desc_t kdi_idt[NIDT];

struct idt_description {
	uint_t id_low;
	uint_t id_high;
	idt_hdlr_f *id_basehdlr;
	size_t *id_incrp;
} idt_description[] = {
	{ T_ZERODIV, 0,		kdi_trap0, NULL },
	{ T_SGLSTP, 0,		kdi_trap1, NULL },
	{ T_NMIFLT, 0,		kdi_int2, NULL },
	{ T_BPTFLT, 0,		kdi_trap3, NULL },
	{ T_OVFLW, 0,		kdi_trap4, NULL },
	{ T_BOUNDFLT, 0,	kdi_trap5, NULL },
	{ T_ILLINST, 0,		kdi_trap6, NULL },
	{ T_NOEXTFLT, 0,	kdi_trap7, NULL },
#if !defined(__xpv)
	{ T_DBLFLT, 0,		syserrtrap, NULL },
#endif
	{ T_EXTOVRFLT, 0,	kdi_trap9, NULL },
	{ T_TSSFLT, 0,		kdi_traperr10, NULL },
	{ T_SEGFLT, 0,		kdi_traperr11, NULL },
	{ T_STKFLT, 0,		kdi_traperr12, NULL },
	{ T_GPFLT, 0,		kdi_traperr13, NULL },
	{ T_PGFLT, 0,		kdi_traperr14, NULL },
	{ 15, 0,		kdi_invaltrap, NULL },
	{ T_EXTERRFLT, 0,	kdi_trap16, NULL },
	{ T_ALIGNMENT, 0,	kdi_traperr17, NULL },
	{ T_MCE, 0,		kdi_trap18, NULL },
	{ T_SIMDFPE, 0,		kdi_trap19, NULL },
	{ T_DBGENTR, 0,		kdi_trap20, NULL },
	{ 21, 31,		kdi_invaltrap, NULL },
	{ 32, 255,		kdi_ivct32, &kdi_ivct_size },
	{ 0, 0, NULL },
};

void
kdi_idt_init(selector_t sel)
{
	struct idt_description *id;
	int i;

	for (id = idt_description; id->id_basehdlr != NULL; id++) {
		uint_t high = id->id_high != 0 ? id->id_high : id->id_low;
		size_t incr = id->id_incrp != NULL ? *id->id_incrp : 0;

#if !defined(__xpv)
		if (kpti_enable && sel == KCS_SEL && id->id_low == T_DBLFLT)
			id->id_basehdlr = tr_syserrtrap;
#endif

		for (i = id->id_low; i <= high; i++) {
			caddr_t hdlr = (caddr_t)id->id_basehdlr +
			    incr * (i - id->id_low);
			set_gatesegd(&kdi_idt[i], (void (*)())hdlr, sel,
			    SDT_SYSIGT, TRP_KPL, IST_DBG);
		}
	}
}

static void
kdi_idt_gates_install(selector_t sel, int saveold)
{
	gate_desc_t gates[KDI_GATE_NVECS];
	int i;

	bzero(gates, sizeof (*gates));

	for (i = 0; i < KDI_GATE_NVECS; i++) {
		const kdi_gate_spec_t *gs = &kdi_gate_specs[i];
		uintptr_t func = GATESEG_GETOFFSET(&kdi_idt[gs->kgs_vec]);
		set_gatesegd(&gates[i], (void (*)())func, sel, SDT_SYSIGT,
		    gs->kgs_dpl, IST_DBG);
	}

	for (i = 0; i < KDI_GATE_NVECS; i++) {
		uint_t vec = kdi_gate_specs[i].kgs_vec;

		if (saveold)
			kdi_kgates[i] = CPU->cpu_m.mcpu_idt[vec];

		kdi_idt_write(&gates[i], vec);
	}
}

static void
kdi_idt_gates_restore(void)
{
	int i;

	for (i = 0; i < KDI_GATE_NVECS; i++)
		kdi_idt_write(&kdi_kgates[i], kdi_gate_specs[i].kgs_vec);
}

/*
 * Called when we switch to the kernel's IDT.  We need to interpose on the
 * kernel's IDT entries and stop using KMDBCODE_SEL.
 */
void
kdi_idt_sync(void)
{
	kdi_idt_init(KCS_SEL);
	kdi_idt_gates_install(KCS_SEL, KDI_IDT_SAVE);
}

void
kdi_update_drreg(kdi_drreg_t *drreg)
{
	kdi_drreg = *drreg;
}

void
kdi_memrange_add(caddr_t base, size_t len)
{
	kdi_memrange_t *mr = &kdi_memranges[kdi_nmemranges];

	ASSERT(kdi_nmemranges != KDI_MEMRANGES_MAX);

	mr->mr_base = base;
	mr->mr_lim = base + len - 1;
	kdi_nmemranges++;
}

void
kdi_idt_switch(kdi_cpusave_t *cpusave)
{
	if (cpusave == NULL)
		kdi_idtr_set(kdi_idt, sizeof (kdi_idt) - 1);
	else
		kdi_idtr_set(cpusave->krs_idt, (sizeof (*idt0) * NIDT) - 1);
}

/*
 * Activation for CPUs other than the boot CPU, called from that CPU's
 * mp_startup().  We saved the kernel's descriptors when we initialized the
 * boot CPU, so we don't want to do it again.  Saving the handlers from this
 * CPU's IDT would actually be dangerous with the CPU initialization method in
 * use at the time of this writing.  With that method, the startup code creates
 * the IDTs for slave CPUs by copying the one used by the boot CPU, which has
 * already been interposed upon by KMDB.  Were we to interpose again, we'd
 * replace the kernel's descriptors with our own in the save area.  By not
 * saving, but still overwriting, we'll work in the current world, and in any
 * future world where the IDT is generated from scratch.
 */
void
kdi_cpu_init(void)
{
	kdi_idt_gates_install(KCS_SEL, KDI_IDT_NOSAVE);
	/* Load the debug registers. */
	kdi_cpu_debug_init(&kdi_cpusave[CPU->cpu_id]);
}

/*
 * Activation for all CPUs for mod-loaded kmdb, i.e. a kmdb that wasn't
 * loaded at boot.
 */
static int
kdi_cpu_activate(xc_arg_t arg1 __unused, xc_arg_t arg2 __unused,
    xc_arg_t arg3 __unused)
{
	kdi_idt_gates_install(KCS_SEL, KDI_IDT_SAVE);
	return (0);
}

void
kdi_activate(kdi_main_t main, kdi_cpusave_t *cpusave, uint_t ncpusave)
{
	int i;
	cpuset_t cpuset;

	CPUSET_ALL(cpuset);

	kdi_cpusave = cpusave;
	kdi_ncpusave = ncpusave;

	kdi_kmdb_main = main;

	for (i = 0; i < kdi_ncpusave; i++) {
		kdi_cpusave[i].krs_cpu_id = i;

		kdi_cpusave[i].krs_curcrumb =
		    &kdi_cpusave[i].krs_crumbs[KDI_NCRUMBS - 1];
		kdi_cpusave[i].krs_curcrumbidx = KDI_NCRUMBS - 1;
	}

	if (boothowto & RB_KMDB)
		kdi_idt_init(KMDBCODE_SEL);
	else
		kdi_idt_init(KCS_SEL);

	kdi_memranges[0].mr_base = kdi_segdebugbase;
	kdi_memranges[0].mr_lim = kdi_segdebugbase + kdi_segdebugsize - 1;
	kdi_nmemranges = 1;

	kdi_drreg.dr_ctl = KDIREG_DRCTL_RESERVED;
	kdi_drreg.dr_stat = KDIREG_DRSTAT_RESERVED;

	if (boothowto & RB_KMDB) {
		kdi_idt_gates_install(KMDBCODE_SEL, KDI_IDT_NOSAVE);
	} else {
		xc_call(0, 0, 0, CPUSET2BV(cpuset), kdi_cpu_activate);
	}
}

static int
kdi_cpu_deactivate(xc_arg_t arg1 __unused, xc_arg_t arg2 __unused,
    xc_arg_t arg3 __unused)
{
	kdi_idt_gates_restore();
	return (0);
}

void
kdi_deactivate(void)
{
	cpuset_t cpuset;
	CPUSET_ALL(cpuset);

	xc_call(0, 0, 0, CPUSET2BV(cpuset), kdi_cpu_deactivate);
	kdi_nmemranges = 0;
}

/*
 * We receive all breakpoints and single step traps.  Some of them, including
 * those from userland and those induced by DTrace providers, are intended for
 * the kernel, and must be processed there.  We adopt this
 * ours-until-proven-otherwise position due to the painful consequences of
 * sending the kernel an unexpected breakpoint or single step.  Unless someone
 * can prove to us that the kernel is prepared to handle the trap, we'll assume
 * there's a problem and will give the user a chance to debug it.
 *
 * If we return 2, then the calling code should restore the trap-time %cr3: that
 * is, it really is a kernel-originated trap.
 */
int
kdi_trap_pass(kdi_cpusave_t *cpusave)
{
	greg_t tt = cpusave->krs_gregs[KDIREG_TRAPNO];
	greg_t pc = cpusave->krs_gregs[KDIREG_PC];
	greg_t cs = cpusave->krs_gregs[KDIREG_CS];

	if (USERMODE(cs))
		return (1);

	if (tt != T_BPTFLT && tt != T_SGLSTP)
		return (0);

	if (tt == T_BPTFLT && kdi_dtrace_get_state() ==
	    KDI_DTSTATE_DTRACE_ACTIVE)
		return (2);

	/*
	 * See the comments in the kernel's T_SGLSTP handler for why we need to
	 * do this.
	 */
#if !defined(__xpv)
	if (tt == T_SGLSTP &&
	    (pc == (greg_t)sys_sysenter || pc == (greg_t)brand_sys_sysenter ||
	    pc == (greg_t)tr_sys_sysenter ||
	    pc == (greg_t)tr_brand_sys_sysenter)) {
#else
	if (tt == T_SGLSTP &&
	    (pc == (greg_t)sys_sysenter || pc == (greg_t)brand_sys_sysenter)) {
#endif
		return (1);
	}

	return (0);
}

/*
 * State has been saved, and all CPUs are on the CPU-specific stacks.  All
 * CPUs enter here, and head off into the debugger proper.
 */
void
kdi_debugger_entry(kdi_cpusave_t *cpusave)
{
	/*
	 * BPTFLT gives us control with %eip set to the instruction *after*
	 * the int 3.  Back it off, so we're looking at the instruction that
	 * triggered the fault.
	 */
	if (cpusave->krs_gregs[KDIREG_TRAPNO] == T_BPTFLT)
		cpusave->krs_gregs[KDIREG_PC]--;

	kdi_kmdb_main(cpusave);
}
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 *
 * Copyright 2018 Joyent, Inc.
 * Copyright 2020 OmniOS Community Edition (OmniOSce) Association.
 */

/*
 * Companion to kdi_asm.s - the implementation of the trap and interrupt
 * handlers.  For the most part, these handlers do the same thing - they
 * push a trap number onto the stack, followed by a jump to kdi_cmnint.
 * Each trap and interrupt has its own handler because each one pushes a
 * different number.
 */

#include <sys/asm_linkage.h>
#include <sys/asm_misc.h>
#include <sys/machprivregs.h>
#include <sys/privregs.h>
#include <sys/kdi_regs.h>
#include <sys/trap.h>
#include <sys/param.h>

#include <kdi_assym.h>
#include <assym.h>

/*
 * The default ASM_ENTRY_ALIGN (16) wastes far too much space.
 */
#undef	ASM_ENTRY_ALIGN
#define	ASM_ENTRY_ALIGN	8

/*
 * Generic trap and interrupt handlers.
 */

#if defined(__xpv)

#define	INTERRUPT_TRAMPOLINE

#else

/*
 * If we're !xpv, then we will need to support KPTI (kernel page table
 * isolation), where we have separate page tables for user and kernel modes.
 * There's more detail about this in kpti_trampolines.s and hat_i86.c
 */

#define	INTERRUPT_TRAMPOLINE			\
	pushq	%r13;				\
	pushq	%r14;				\
	subq	$KPTI_R14, %rsp;		\
	/* Check for clobbering */		\
	cmpq	$0, KPTI_FLAG(%rsp);		\
	je	1f;				\
	/* Don't worry, this totally works */	\
	int	$8;				\
1:						\
	movq	$1, KPTI_FLAG(%rsp);		\
	/* Save current %cr3. */		\
	mov	%cr3, %r14;			\
	mov	%r14, KPTI_TR_CR3(%rsp);	\
	/* Switch to paranoid %cr3. */		\
	mov	kpti_safe_cr3, %r14;		\
	mov	%r14, %cr3;			\
						\
	cmpw	$KCS_SEL, KPTI_CS(%rsp);	\
	je	3f;				\
2:						\
	/* Get our cpu_t in %r13 */		\
	mov	%rsp, %r13;			\
	and	$(~(MMU_PAGESIZE - 1)), %r13;	\
	subq	$CPU_KPTI_START, %r13;		\
	/* Use top of the kthread stk */	\
	mov	CPU_THREAD(%r13), %r14;		\
	mov	T_STACK(%r14), %r14;		\
	addq	$REGSIZE+MINFRAME, %r14;	\
	jmp	5f;				\
3:						\
	/* Check the %rsp in the frame. */	\
	/* Is it above kernel base? */		\
	mov	kpti_kbase, %r14;		\
	cmp	%r14, KPTI_RSP(%rsp);		\
	jb	2b;				\
	/* Is it within the kpti_frame page? */	\
	mov	%rsp, %r13;			\
	and	$(~(MMU_PAGESIZE - 1)), %r13;	\
	mov	KPTI_RSP(%rsp), %r14;		\
	and	$(~(MMU_PAGESIZE - 1)), %r14;	\
	cmp	%r13, %r14;			\
	je	2b;				\
	/* Use the %rsp from the trap frame. */	\
	/* We already did %cr3. */		\
	mov	KPTI_RSP(%rsp), %r14;		\
	and	$(~0xf), %r14;			\
5:						\
	mov	%rsp, %r13;			\
	/* %r14 contains our destination stk */	\
	mov	%r14, %rsp;			\
	pushq	KPTI_SS(%r13);			\
	pushq	KPTI_RSP(%r13);			\
	pushq	KPTI_RFLAGS(%r13);		\
	pushq	KPTI_CS(%r13);			\
	pushq	KPTI_RIP(%r13);			\
	pushq	KPTI_ERR(%r13);			\
	mov	KPTI_R14(%r13), %r14;		\
	movq	$0, KPTI_FLAG(%r13);		\
	mov	KPTI_R13(%r13), %r13

#endif	/* !__xpv */


#define	MKIVCT(n) \
	ENTRY_NP(kdi_ivct##n);		\
	XPV_TRAP_POP;			\
	push	$0; /* err */		\
	INTERRUPT_TRAMPOLINE;		\
	push	$n;			\
	jmp	kdi_cmnint;		\
	SET_SIZE(kdi_ivct##n)

#define	MKTRAPHDLR(n) \
	ENTRY_NP(kdi_trap##n);		\
	XPV_TRAP_POP;			\
	push	$0; /* err */		\
	INTERRUPT_TRAMPOLINE;		\
	push	$n;			\
	jmp	kdi_cmnint;		\
	SET_SIZE(kdi_trap##n)

#define	MKTRAPERRHDLR(n) \
	ENTRY_NP(kdi_traperr##n);	\
	XPV_TRAP_POP;			\
	INTERRUPT_TRAMPOLINE;		\
	push	$n;			\
	jmp	kdi_cmnint;		\
	SET_SIZE(kdi_traperr##n)

#if !defined(__xpv)
#define	MKNMIHDLR \
	ENTRY_NP(kdi_int2);		\
	push	$0;			\
	push	$2;			\
	pushq	%r13;			\
	mov	kpti_safe_cr3, %r13;	\
	mov	%r13, %cr3;		\
	popq	%r13;			\
	jmp	kdi_nmiint;		\
	SET_SIZE(kdi_int2)

#define	MKMCEHDLR \
	ENTRY_NP(kdi_trap18);		\
	push	$0;			\
	push	$18;			\
	pushq	%r13;			\
	mov	kpti_safe_cr3, %r13;	\
	mov	%r13, %cr3;		\
	popq	%r13;			\
	jmp	kdi_cmnint;		\
	SET_SIZE(kdi_trap18)
#else
#define	MKNMIHDLR \
	ENTRY_NP(kdi_int2);		\
	push	$0;			\
	push	$2;			\
	jmp	kdi_nmiint;		\
	SET_SIZE(kdi_int2)

#define	MKMCEHDLR \
	ENTRY_NP(kdi_trap18);		\
	push	$0;			\
	push	$18;			\
	jmp	kdi_cmnint;		\
	SET_SIZE(kdi_trap18)
#endif

/*
 * The only way we should reach here is by an explicit "int 0x.." which is
 * defined not to push an error code.
 */
#define	MKINVALHDLR \
	ENTRY_NP(kdi_invaltrap);	\
	XPV_TRAP_POP;			\
	push	$0; /* err */		\
	INTERRUPT_TRAMPOLINE;		\
	push	$255;			\
	jmp	kdi_cmnint;		\
	SET_SIZE(kdi_invaltrap)

	.data
	DGDEF3(kdi_idt, 16 * NIDT, MMU_PAGESIZE)
	.fill	MMU_PAGESIZE, 1, 0

#if !defined(__xpv)
.section ".text"
.align MMU_PAGESIZE
.global kdi_isr_start
kdi_isr_start:
	nop

.global kpti_safe_cr3
.global kpti_kbase
#endif

/*
 * The handlers themselves
 */

	MKINVALHDLR
	MKTRAPHDLR(0)
	MKTRAPHDLR(1)
	MKNMIHDLR/*2*/
	MKTRAPHDLR(3)
	MKTRAPHDLR(4)
	MKTRAPHDLR(5)
	MKTRAPHDLR(6)
	MKTRAPHDLR(7)
	MKTRAPHDLR(9)
	MKTRAPHDLR(15)
	MKTRAPHDLR(16)
	MKMCEHDLR/*18*/
	MKTRAPHDLR(19)
	MKTRAPHDLR(20)

	MKTRAPERRHDLR(8)
	MKTRAPERRHDLR(10)
	MKTRAPERRHDLR(11)
	MKTRAPERRHDLR(12)
	MKTRAPERRHDLR(13)
	MKTRAPERRHDLR(14)
	MKTRAPERRHDLR(17)

	.globl	kdi_ivct_size
kdi_ivct_size:
	.NWORD [kdi_ivct33-kdi_ivct32]

	/* 10 billion and one interrupt handlers */
kdi_ivct_base:
	MKIVCT(32);	MKIVCT(33);	MKIVCT(34);	MKIVCT(35);
	MKIVCT(36);	MKIVCT(37);	MKIVCT(38);	MKIVCT(39);
	MKIVCT(40);	MKIVCT(41);	MKIVCT(42);	MKIVCT(43);
	MKIVCT(44);	MKIVCT(45);	MKIVCT(46);	MKIVCT(47);
	MKIVCT(48);	MKIVCT(49);	MKIVCT(50);	MKIVCT(51);
	MKIVCT(52);	MKIVCT(53);	MKIVCT(54);	MKIVCT(55);
	MKIVCT(56);	MKIVCT(57);	MKIVCT(58);	MKIVCT(59);
	MKIVCT(60);	MKIVCT(61);	MKIVCT(62);	MKIVCT(63);
	MKIVCT(64);	MKIVCT(65);	MKIVCT(66);	MKIVCT(67);
	MKIVCT(68);	MKIVCT(69);	MKIVCT(70);	MKIVCT(71);
	MKIVCT(72);	MKIVCT(73);	MKIVCT(74);	MKIVCT(75);
	MKIVCT(76);	MKIVCT(77);	MKIVCT(78);	MKIVCT(79);
	MKIVCT(80);	MKIVCT(81);	MKIVCT(82);	MKIVCT(83);
	MKIVCT(84);	MKIVCT(85);	MKIVCT(86);	MKIVCT(87);
	MKIVCT(88);	MKIVCT(89);	MKIVCT(90);	MKIVCT(91);
	MKIVCT(92);	MKIVCT(93);	MKIVCT(94);	MKIVCT(95);
	MKIVCT(96);	MKIVCT(97);	MKIVCT(98);	MKIVCT(99);
	MKIVCT(100);	MKIVCT(101);	MKIVCT(102);	MKIVCT(103);
	MKIVCT(104);	MKIVCT(105);	MKIVCT(106);	MKIVCT(107);
	MKIVCT(108);	MKIVCT(109);	MKIVCT(110);	MKIVCT(111);
	MKIVCT(112);	MKIVCT(113);	MKIVCT(114);	MKIVCT(115);
	MKIVCT(116);	MKIVCT(117);	MKIVCT(118);	MKIVCT(119);
	MKIVCT(120);	MKIVCT(121);	MKIVCT(122);	MKIVCT(123);
	MKIVCT(124);	MKIVCT(125);	MKIVCT(126);	MKIVCT(127);
	MKIVCT(128);	MKIVCT(129);	MKIVCT(130);	MKIVCT(131);
	MKIVCT(132);	MKIVCT(133);	MKIVCT(134);	MKIVCT(135);
	MKIVCT(136);	MKIVCT(137);	MKIVCT(138);	MKIVCT(139);
	MKIVCT(140);	MKIVCT(141);	MKIVCT(142);	MKIVCT(143);
	MKIVCT(144);	MKIVCT(145);	MKIVCT(146);	MKIVCT(147);
	MKIVCT(148);	MKIVCT(149);	MKIVCT(150);	MKIVCT(151);
	MKIVCT(152);	MKIVCT(153);	MKIVCT(154);	MKIVCT(155);
	MKIVCT(156);	MKIVCT(157);	MKIVCT(158);	MKIVCT(159);
	MKIVCT(160);	MKIVCT(161);	MKIVCT(162);	MKIVCT(163);
	MKIVCT(164);	MKIVCT(165);	MKIVCT(166);	MKIVCT(167);
	MKIVCT(168);	MKIVCT(169);	MKIVCT(170);	MKIVCT(171);
	MKIVCT(172);	MKIVCT(173);	MKIVCT(174);	MKIVCT(175);
	MKIVCT(176);	MKIVCT(177);	MKIVCT(178);	MKIVCT(179);
	MKIVCT(180);	MKIVCT(181);	MKIVCT(182);	MKIVCT(183);
	MKIVCT(184);	MKIVCT(185);	MKIVCT(186);	MKIVCT(187);
	MKIVCT(188);	MKIVCT(189);	MKIVCT(190);	MKIVCT(191);
	MKIVCT(192);	MKIVCT(193);	MKIVCT(194);	MKIVCT(195);
	MKIVCT(196);	MKIVCT(197);	MKIVCT(198);	MKIVCT(199);
	MKIVCT(200);	MKIVCT(201);	MKIVCT(202);	MKIVCT(203);
	MKIVCT(204);	MKIVCT(205);	MKIVCT(206);	MKIVCT(207);
	MKIVCT(208);	MKIVCT(209);	MKIVCT(210);	MKIVCT(211);
	MKIVCT(212);	MKIVCT(213);	MKIVCT(214);	MKIVCT(215);
	MKIVCT(216);	MKIVCT(217);	MKIVCT(218);	MKIVCT(219);
	MKIVCT(220);	MKIVCT(221);	MKIVCT(222);	MKIVCT(223);
	MKIVCT(224);	MKIVCT(225);	MKIVCT(226);	MKIVCT(227);
	MKIVCT(228);	MKIVCT(229);	MKIVCT(230);	MKIVCT(231);
	MKIVCT(232);	MKIVCT(233);	MKIVCT(234);	MKIVCT(235);
	MKIVCT(236);	MKIVCT(237);	MKIVCT(238);	MKIVCT(239);
	MKIVCT(240);	MKIVCT(241);	MKIVCT(242);	MKIVCT(243);
	MKIVCT(244);	MKIVCT(245);	MKIVCT(246);	MKIVCT(247);
	MKIVCT(248);	MKIVCT(249);	MKIVCT(250);	MKIVCT(251);
	MKIVCT(252);	MKIVCT(253);	MKIVCT(254);	MKIVCT(255);

#if !defined(__xpv)
.section ".text"
.align MMU_PAGESIZE
.global kdi_isr_end
kdi_isr_end:
	nop
#endif
\
\ Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
\ Use is subject to license terms.
\
\ Copyright 2018 Joyent, Inc.
\
\ CDDL HEADER START
\
\ The contents of this file are subject to the terms of the
\ Common Development and Distribution License (the "License").
\ You may not use this file except in compliance with the License.
\
\ You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
\ or http://www.opensolaris.org/os/licensing.
\ See the License for the specific language governing permissions
\ and limitations under the License.
\
\ When distributing Covered Code, include this CDDL HEADER in each
\ file and include the License file at usr/src/OPENSOLARIS.LICENSE.
\ If applicable, add the following below this CDDL HEADER, with the
\ fields enclosed by brackets "[]" replaced with your own identifying
\ information: Portions Copyright [yyyy] [name of copyright owner]
\
\ CDDL HEADER END
\
\ CPU-save structure offsets for use in assembly code.
\

#include <sys/cpuvar.h>
#include <sys/kdi_impl.h>

kdi_memrange_t	MR_SIZE
		mr_base
		mr_lim

kdi_crumb_t	KRM_SIZE
		krm_cpu_state
		krm_pc
		krm_sp
		krm_trapno
		krm_flag

kdi_drreg_t
		dr_ctl
		dr_stat
		dr_addr

kdi_cpusave_t	KRS_SIZE
		krs_gregs
		krs_dr
		krs_dr.dr_ctl			KRS_DRCTL
		krs_dr.dr_stat			KRS_DRSTAT
		krs_gdt
		krs_idt
		krs_cr0
		krs_cpu_state
		krs_curcrumbidx
		krs_curcrumb
		krs_crumbs

greg_t		KREG_SIZE

\#define	REG_SHIFT		3

\#define	DRADDR_IDX(num)		_CONST(_MUL(num, DR_ADDR_INCR))
\#define	DRADDR_OFF(num)		_CONST(DRADDR_IDX(num) + DR_ADDR)
\#define	KRS_DROFF(num)		_CONST(DRADDR_OFF(num) + KRS_DR)
\#define	REG_OFF(reg)		_CONST(_CONST(reg) << REG_SHIFT)
\#define	KDIREG_OFF(reg)		_CONST(_MUL(KREG_SIZE, reg) + KRS_GREGS)