ml - zygaena/hammerhead

root / base / usr / src / uts / i86pc / ml
ml Plain Text 5979 lines 147.0 KB
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

/*
 * Copyright 2019 Joyent, Inc.
 */

#include <sys/segments.h>
#include <sys/controlregs.h>

/*
 * Do a call into BIOS.  This goes down to 16 bit real mode and back again.
 */

/*
 * instruction prefix to change operand size in instruction
 */
#define DATASZ	.byte 0x66;

	.globl	_start
_start:

	/*
	 * Save caller registers
	 */
	movq	%rbp, save_rbp
	movq	%rsp, save_rsp
	movq	%rbx, save_rbx
	movq	%rsi, save_rsi
	movq	%r12, save_r12
	movq	%r13, save_r13
	movq	%r14, save_r14
	movq	%r15, save_r15

	/* Switch to a low memory stack */
	movq	$_start, %rsp

	/* put interrupt number in %bl */
	movq	%rdi, %rbx

	/* allocate space for args on stack */
	subq	$18, %rsp
	movq	%rsp, %rdi

	/* copy args from high memory to stack in low memory */
	cld
	movl	$18, %ecx
	rep
	movsb

	/*
	 * Save system registers
	 */
	sidt	save_idt
	sgdt	save_gdt
	str	save_tr
	movw	%cs, save_cs
	movw	%ds, save_ds
	movw	%ss, save_ss
	movw	%es, save_es
	movw	%fs, save_fs
	movw	%gs, save_gs
	movq	%cr4, %rax
	movq	%rax, save_cr4
	movq	%cr3, %rax
	movq	%rax, save_cr3
	movq	%cr0, %rax
	movq	%rax, save_cr0

	/*
	 * save/clear the extension parts of the fs/gs base registers and cr8
	 */
	movl	$MSR_AMD_FSBASE, %ecx
	rdmsr
	movl	%eax, save_fsbase
	movl	%edx, save_fsbase + 4
	xorl	%eax, %eax
	xorl	%edx, %edx
	wrmsr

	movl	$MSR_AMD_GSBASE, %ecx
	rdmsr
	movl	%eax, save_gsbase
	movl	%edx, save_gsbase + 4
	xorl	%eax, %eax
	xorl	%edx, %edx
	wrmsr

	movl	$MSR_AMD_KGSBASE, %ecx
	rdmsr
	movl	%eax, save_kgsbase
	movl	%edx, save_kgsbase + 4
	xorl	%eax, %eax
	xorl	%edx, %edx
	wrmsr

	movq	%cr8, %rax
	movq	%rax, save_cr8

	/*
	 * set offsets in 16 bit ljmp instructions below
	 */
	leaq	enter_real, %rax
	movw	%ax, enter_real_ljmp

	leaq	enter_protected, %rax
	movw	%ax, enter_protected_ljmp

	leaq	gdt_info, %rax
	movw	%ax, gdt_info_load

	/*
	 * insert BIOS interrupt number into later instruction
	 */
	movb    %bl, int_instr+1
	jmp     1f
1:

	/*
	 * zero out all the registers to make sure they're 16 bit clean
	 */
	xorq	%r8, %r8
	xorq	%r9, %r9
	xorq	%r10, %r10
	xorq	%r11, %r11
	xorq	%r12, %r12
	xorq	%r13, %r13
	xorq	%r14, %r14
	xorq	%r15, %r15
	xorl	%eax, %eax
	xorl	%ebx, %ebx
	xorl	%ecx, %ecx
	xorl	%edx, %edx
	xorl	%ebp, %ebp
	xorl	%esi, %esi
	xorl	%edi, %edi

	/*
	 * Load our own GDT/IDT
	 */
	lgdt	gdt_info
	lidt	idt_info

	/*
	 * Shut down 64 bit mode. First get into compatibility mode.
	 */
	movq	%rsp, %rax
	pushq	$B32DATA_SEL
	pushq	%rax
	pushf
	pushq	$B32CODE_SEL
	pushq	$1f
	iretq
1:
	.code32

	/*
	 * disable long mode by:
	 * - shutting down paging (bit 31 of cr0)
	 * - flushing the TLB
	 * - disabling LME (long made enable) in EFER (extended feature reg)
	 */
	movl	%cr0, %eax
	btcl	$31, %eax		/* disable paging */
	movl	%eax, %cr0
	ljmp	$B32CODE_SEL, $1f
1:

	xorl	%eax, %eax
	movl	%eax, %cr3		/* flushes TLB */

	movl	$MSR_AMD_EFER, %ecx	/* Extended Feature Enable */
	rdmsr
	btcl	$8, %eax		/* bit 8 Long Mode Enable bit */
	wrmsr

	/*
	 * ok.. now enter 16 bit mode, so we can shut down protected mode
	 *
	 * We'll have to act like we're still in a 32 bit section.
	 * So the code from this point has DATASZ in front of it to get 32 bit
	 * operands. If DATASZ is missing the operands will be 16 bit.
	 *
	 * Now shut down paging and protected (ie. segmentation) modes.
	 */
	ljmp	$B16CODE_SEL, $enter_16_bit
enter_16_bit:

	/*
	 * Make sure hidden parts of segment registers are 16 bit clean
	 */
	DATASZ	movl	$B16DATA_SEL, %eax
		movw    %ax, %ss
		movw    %ax, %ds
		movw    %ax, %es
		movw    %ax, %fs
		movw    %ax, %gs


	DATASZ	movl	$0x0, %eax	/* put us in real mode */
	DATASZ	movl	%eax, %cr0
	.byte	0xea			/* ljmp */
enter_real_ljmp:
	.value	0			/* addr (16 bit) */
	.value	0x0			/* value for %cs */
enter_real:

	/*
	 * zero out the remaining segment registers
	 */
	DATASZ	xorl	%eax, %eax
		movw    %ax, %ss
		movw    %ax, %ds
		movw    %ax, %es
		movw    %ax, %fs
		movw    %ax, %gs

	/*
	 * load the arguments to the BIOS call from the stack
	 */
	popl	%eax	/* really executes a 16 bit pop */
	popl	%ebx
	popl	%ecx
	popl	%edx
	popl	%esi
	popl	%edi
	popl	%ebp
	pop	%es
	pop	%ds

	/*
	 * do the actual BIOS call
	 */
	sti
int_instr:
	int	$0x10		/* this int number is overwritten */
	cli			/* ensure interrupts remain disabled */

	/*
	 * save results of the BIOS call
	 */
	pushf
	push	%ds
	push	%es
	pushl	%ebp		/* still executes as 16 bit */
	pushl	%edi
	pushl	%esi
	pushl	%edx
	pushl	%ecx
	pushl	%ebx
	pushl	%eax

	/*
	 * Restore protected mode and 32 bit execution
	 */
	push	$0			/* make sure %ds is zero before lgdt */
	pop	%ds
	.byte	0x0f, 0x01, 0x16	/* lgdt */
gdt_info_load:
	.value	0	/* temp GDT in currently addressible mem */

	DATASZ	movl	$0x1, %eax
	DATASZ	movl	%eax, %cr0

	.byte	0xea			/* ljmp */
enter_protected_ljmp:
	.value	0			/* addr (still in 16 bit) */
	.value	B32CODE_SEL		/* %cs value */
enter_protected:

	/*
	 * We are now back in a 32 bit code section, fix data/stack segments
	 */
	.code32
	movw	$B32DATA_SEL, %ax
	movw	%ax, %ds
	movw	%ax, %ss

	/*
	 * Re-enable paging. Note we only use 32 bit mov's to restore these
	 * control registers. That's OK as the upper 32 bits are always zero.
	 */
	movl	save_cr4, %eax
	movl	%eax, %cr4
	movl	save_cr3, %eax
	movl	%eax, %cr3

	/*
	 * re-enable long mode
	 */
	movl	$MSR_AMD_EFER, %ecx
	rdmsr
	btsl	$8, %eax
	wrmsr

	movl	save_cr0, %eax
	movl	%eax, %cr0
	jmp	enter_paging
enter_paging:


	/*
	 * transition back to 64 bit mode
	 */
	pushl	$B64CODE_SEL
	pushl	$longmode
	lret
longmode:
	.code64
	/*
	 * restore caller frame pointer and segment registers
	 */
	lgdt	save_gdt
	lidt	save_idt

	/*
	 * Before loading the task register we need to reset the busy bit
	 * in its corresponding GDT selector. The busy bit is the 2nd bit in
	 * the 5th byte of the selector.
	 */
	movzwq	save_tr, %rax
	addq	save_gdt+2, %rax
	btcl	$1, 5(%rax)
	ltr	save_tr
	movw	save_ds, %ds
	movw	save_ss, %ss
	movw	save_es, %es
	movw	save_fs, %fs
	movw	save_gs, %gs

	pushq	save_cs
	pushq	$.newcs
	lretq
.newcs:

	/*
	 * restore the hidden kernel segment base register values
	 */
	movl	save_fsbase, %eax
	movl	save_fsbase + 4, %edx
	movl	$MSR_AMD_FSBASE, %ecx
	wrmsr

	movl	save_gsbase, %eax
	movl	save_gsbase + 4, %edx
	movl	$MSR_AMD_GSBASE, %ecx
	wrmsr

	movl	save_kgsbase, %eax
	movl	save_kgsbase + 4, %edx
	movl	$MSR_AMD_KGSBASE, %ecx
	wrmsr

	movq	save_cr8, %rax
	cmpq	$0, %rax
	je	1f
	movq	%rax, %cr8
1:

	/*
	 * copy results to caller's location, then restore remaining registers
	 */
	movq    save_rsi, %rdi
	movq	%rsp, %rsi
	movq	$18, %rcx
	rep
	movsb
	movw	18(%rsp), %ax
	andq	$0xffff, %rax
	movq    save_r12, %r12
	movq    save_r13, %r13
	movq    save_r14, %r14
	movq    save_r15, %r15
	movq    save_rbx, %rbx
	movq    save_rbp, %rbp
	movq    save_rsp, %rsp
	ret


/*
 * Caller's registers to restore
 */
	.align 4
save_esi:
	.long	0
save_edi:
	.long	0
save_ebx:
	.long	0
save_ebp:
	.long	0
save_esp:
	.long	0

	.align 8
save_rsi:
	.quad	0
save_rbx:
	.quad	0
save_rbp:
	.quad	0
save_rsp:
	.quad	0
save_r12:
	.quad	0
save_r13:
	.quad	0
save_r14:
	.quad	0
save_r15:
	.quad	0
save_kgsbase:
	.quad	0
save_gsbase:
	.quad	0
save_fsbase:
	.quad	0
save_cr8:
	.quad	0

save_idt:
	.quad	0
	.quad	0

save_gdt:
	.quad	0
	.quad	0

save_cr0:
	.quad	0
save_cr3:
	.quad	0
save_cr4:
	.quad	0
save_cs:
	.quad	0
save_ss:
	.value	0
save_ds:
	.value	0
save_es:
	.value	0
save_fs:
	.value	0
save_gs:
	.value	0
save_tr:
	.value	0

idt_info:
	.value 0x3ff
	.quad 0


/*
 * We need to trampoline thru a gdt we have in low memory.
 */
#include "../boot/boot_gdt.s"

/*
 * This file and its contents are supplied under the terms of the
 * Common Development and Distribution License ("CDDL"), version 1.0.
 * You may only use this file in accordance with the terms of version
 * 1.0 of the CDDL.
 *
 * A full copy of the text of the CDDL should have accompanied this
 * source.  A copy of the CDDL is also available via the Internet at
 * http://www.illumos.org/license/CDDL.
 */

/*
 * Copyright 2016 Joyent, Inc.
 * Copyright 2025 Oxide Computer Company
 */

#include <sys/asm_linkage.h>
#include <sys/asm_misc.h>
#include <sys/param.h>
#include <sys/comm_page.h>
#include <sys/tsc.h>

#if defined(_GENCTF)

hrtime_t tsc_last;
hrtime_t tsc_resume_cap;
hrtime_t tsc_hrtime_base;
uint32_t tsc_max_delta;
volatile uint32_t hres_lock;
uint32_t tsc_type;
uint32_t nsec_scale;
int64_t hrestime_adj;
hrtime_t hres_last_tick;
uint32_t tsc_ncpu;
volatile timestruc_t hrestime;
hrtime_t tsc_sync_tick_delta[NCPU];

comm_page_t comm_page;

#else /* defined(_GENCTF) */

#include "assym.h"

/*
 * x86 Comm Page
 *
 * This is the definition for the comm page on x86.  The purpose of this struct
 * is to consolidate certain pieces of kernel state into one contiguous section
 * of memory in order for it to be exposed (read-only) to userspace.  The
 * struct contents are defined by hand so that member variables will maintain
 * their original symbols for use throughout the rest of the kernel.  This
 * layout must exactly match the C definition of comm_page_t.
 * See: "uts/i86pc/sys/comm_page.h"
 */

	.data
	DGDEF3(comm_page, COMM_PAGE_S_SIZE, 4096)
	DGDEF2(tsc_last, 8)
	.fill	1, 8, 0
	DGDEF2(tsc_hrtime_base, 8)
	.fill	1, 8, 0
	DGDEF2(tsc_resume_cap, 8)
	.fill	1, 8, 0
	DGDEF2(tsc_type, 4);
	.fill	1, 4, _CONST(TSC_RDTSC_CPUID)
	DGDEF2(tsc_max_delta, 4);
	.fill	1, 4, 0
	DGDEF2(hres_lock, 4);
	.fill	1, 4, 0
	DGDEF2(nsec_scale, 4);
	.fill	1, 4, 0
	DGDEF2(hrestime_adj, 8)
	.fill	1, 8, 0
	DGDEF2(hres_last_tick, 8)
	.fill	1, 8, 0
	DGDEF2(tsc_ncpu, 4)
	.fill	1, 4, 0
	/* _cp_pad0 */
	.fill	1, 4, 0
	DGDEF2(hrestime, _MUL(2, 8))
	.fill	2, 8, 0
	/* _cp_pad1 */
	.fill	502, 8, 0
	DGDEF2(tsc_sync_tick_delta, _MUL(NCPU, 8))
	.fill	_CONST(NCPU), 8, 0
	/* _cp_pad2 */
	.fill	_CONST(COMM_PAGE_S_SIZE - COMM_PAGE_PAD2), 1, 0

#endif /* defined(_GENCTF) */
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
 * Copyright 2019 Joyent, Inc.
 * Copyright 2020 OmniOS Community Edition (OmniOSce) Association.
 */

#include <sys/asm_linkage.h>
#include <sys/asm_misc.h>
#include <sys/regset.h>
#include <sys/privregs.h>
#include <sys/x86_archext.h>
#include <sys/cpr_wakecode.h>

#include <sys/segments.h>
#include "assym.h"

#ifdef  DEBUG
#define LED     1
#define SERIAL  1
#endif	/*	DEBUG	*/

#ifdef	DEBUG
#define	COM1	0x3f8
#define	COM2	0x2f8
#define	WC_COM	COM2	/* either COM1 or COM2			*/
#define	WC_LED	0x80    /* diagnostic led port ON motherboard	*/

/*
 * defined as offsets from the data register
 */
#define	DLL	0	/* divisor latch (lsb) */
#define	DLH	1	/* divisor latch (msb) */
#define	LCR	3	/* line control register		*/
#define	MCR	4	/* modem control register		*/


#define	DLAB	0x80    /* divisor latch access bit		*/
#define	B9600L	0X0c	/* lsb bit pattern for 9600 baud	*/
#define	B9600H	0X0	/* hsb bit pattern for 9600 baud	*/
#define	DTR	0x01    /* Data Terminal Ready			*/
#define	RTS	0x02    /* Request To Send			*/
#define	STOP1	0x00	/* 1 stop bit				*/
#define	BITS8	0x03    /* 8 bits per char			*/

#endif	/*	DEBUG	*/

/*
 *	This file contains the low level routines involved in getting
 *	into and out of ACPI S3, including those needed for restarting
 *	the non-boot cpus.
 *
 *	Our assumptions:
 *
 *	Our actions:
 *
 */

	ENTRY_NP(wc_save_context)

	movq	(%rsp), %rdx		/ return address
	movq	%rdx, WC_RETADDR(%rdi)
	pushq	%rbp
	movq	%rsp,%rbp

	movq    %rdi, WC_VIRTADDR(%rdi)
	movq    %rdi, WC_RDI(%rdi)

	movq    %rdx, WC_RDX(%rdi)

/ stash everything else we need
	sgdt	WC_GDT(%rdi)
	sidt	WC_IDT(%rdi)
	sldt	WC_LDT(%rdi)
	str	WC_TR(%rdi)

	movq	%cr0, %rdx
	movq	%rdx, WC_CR0(%rdi)
	movq	%cr3, %rdx
	movq	%rdx, WC_CR3(%rdi)
	movq	%cr4, %rdx
	movq	%rdx, WC_CR4(%rdi)
	movq	%cr8, %rdx
	movq	%rdx, WC_CR8(%rdi)

	movq    %r8, WC_R8(%rdi)
	movq    %r9, WC_R9(%rdi)
	movq    %r10, WC_R10(%rdi)
	movq    %r11, WC_R11(%rdi)
	movq    %r12, WC_R12(%rdi)
	movq    %r13, WC_R13(%rdi)
	movq    %r14, WC_R14(%rdi)
	movq    %r15, WC_R15(%rdi)
	movq    %rax, WC_RAX(%rdi)
	movq    %rbp, WC_RBP(%rdi)
	movq    %rbx, WC_RBX(%rdi)
	movq    %rcx, WC_RCX(%rdi)
	movq    %rsi, WC_RSI(%rdi)
	movq    %rsp, WC_RSP(%rdi)

	movw	%ss, WC_SS(%rdi)
	movw	%cs, WC_CS(%rdi)
	movw	%ds, WC_DS(%rdi)
	movw	%es, WC_ES(%rdi)

	movq	$0, %rcx		/ save %fs register
	movw    %fs, %cx
	movq    %rcx, WC_FS(%rdi)

	movl    $MSR_AMD_FSBASE, %ecx
	rdmsr
	movl    %eax, WC_FSBASE(%rdi)
	movl    %edx, WC_FSBASE+4(%rdi)

	movq	$0, %rcx		/ save %gs register
	movw    %gs, %cx
	movq    %rcx, WC_GS(%rdi)

	movl    $MSR_AMD_GSBASE, %ecx	/ save gsbase msr
	rdmsr
	movl    %eax, WC_GSBASE(%rdi)
	movl    %edx, WC_GSBASE+4(%rdi)

	movl    $MSR_AMD_KGSBASE, %ecx	/ save kgsbase msr
	rdmsr
	movl    %eax, WC_KGSBASE(%rdi)
	movl    %edx, WC_KGSBASE+4(%rdi)

	movq	%gs:CPU_ID, %rax	/ save current cpu id
	movq	%rax, WC_CPU_ID(%rdi)

	pushfq
	popq	WC_EFLAGS(%rdi)

	wbinvd				/ flush the cache
	mfence

	movq	$1, %rax		/ at suspend return 1

	leave

	ret

	SET_SIZE(wc_save_context)


/*
 *	Our assumptions:
 *		- We are running in real mode.
 *		- Interrupts are disabled.
 *
 *	Our actions:
 *		- We start using our GDT by loading correct values in the
 *		  selector registers (cs=KCS_SEL, ds=es=ss=KDS_SEL, fs=KFS_SEL,
 *		  gs=KGS_SEL).
 *		- We change over to using our IDT.
 *		- We load the default LDT into the hardware LDT register.
 *		- We load the default TSS into the hardware task register.
 *		- We restore registers
 *		- We return to original caller (a la setjmp)
 */

	ENTRY_NP(wc_rm_start)

	/*
	 * For the Sun Studio 10 assembler we needed to do a .code32 and
	 * mentally invert the meaning of the addr16 and data16 prefixes to
	 * get 32-bit access when generating code to be executed in 16-bit
	 * mode (sigh...)
	 *
	 * This code, despite always being built with GNU as, has inherited
	 * the conceptual damage.
	 */

	.code32

	cli
	movw		%cs, %ax
	movw		%ax, %ds		/ establish ds ...
	movw		%ax, %ss		/ ... and ss:esp
	D16 movl	$WC_STKSTART, %esp
/ using the following value blows up machines! - DO NOT USE
/	D16 movl	0xffc, %esp


#if     LED
	D16 movl        $WC_LED, %edx
	D16 movb        $0xd1, %al
	outb    (%dx)
#endif

#if     SERIAL
	D16 movl        $WC_COM, %edx
	D16 movb        $0x61, %al
	outb    (%dx)
#endif

	D16 call	cominit

	/*
	 * Enable protected-mode, write protect, and alignment mask
	 * %cr0 has already been initialsed to zero
	 */
	movl		%cr0, %eax
	D16 orl		$_CONST(CR0_PE|CR0_WP|CR0_AM), %eax
	movl		%eax, %cr0

	/*
	 * Do a jmp immediately after writing to cr0 when enabling protected
	 * mode to clear the real mode prefetch queue (per Intel's docs)
	 */
	jmp		pestart
pestart:

#if     LED
	D16 movl        $WC_LED, %edx
	D16 movb        $0xd2, %al
	outb    (%dx)
#endif

#if     SERIAL
	D16 movl        $WC_COM, %edx
	D16 movb        $0x62, %al
	outb    (%dx)
#endif

	/*
	 * 16-bit protected mode is now active, so prepare to turn on long
	 * mode
	 */

#if     LED
	D16 movl        $WC_LED, %edx
	D16 movb        $0xd3, %al
	outb    (%dx)
#endif

#if     SERIAL
	D16 movl        $WC_COM, %edx
	D16 movb        $0x63, %al
	outb    (%dx)
#endif

	/*
	 * Add any initial cr4 bits
	 */
	movl		%cr4, %eax
	A16 D16 orl	CR4OFF, %eax

	/*
	 * Enable PAE mode (CR4.PAE)
	 */
	D16 orl		$CR4_PAE, %eax
	movl		%eax, %cr4

#if     LED
	D16 movl        $WC_LED, %edx
	D16 movb        $0xd4, %al
	outb    (%dx)
#endif

#if     SERIAL
	D16 movl        $WC_COM, %edx
	D16 movb        $0x64, %al
	outb    (%dx)
#endif

	/*
	 * Point cr3 to the 64-bit long mode page tables.
	 *
	 * Note that these MUST exist in 32-bit space, as we don't have
	 * a way to load %cr3 with a 64-bit base address for the page tables
	 * until the CPU is actually executing in 64-bit long mode.
	 */
	A16 D16 movl	CR3OFF, %eax
	movl		%eax, %cr3

	/*
	 * Set long mode enable in EFER (EFER.LME = 1)
	 */
	D16 movl	$MSR_AMD_EFER, %ecx
	rdmsr

	D16 orl		$AMD_EFER_LME, %eax
	wrmsr

#if     LED
	D16 movl        $WC_LED, %edx
	D16 movb        $0xd5, %al
	outb    (%dx)
#endif

#if     SERIAL
	D16 movl        $WC_COM, %edx
	D16 movb        $0x65, %al
	outb    (%dx)
#endif

	/*
	 * Finally, turn on paging (CR0.PG = 1) to activate long mode.
	 */
	movl		%cr0, %eax
	D16 orl		$CR0_PG, %eax
	movl		%eax, %cr0

	/*
	 * The instruction after enabling paging in CR0 MUST be a branch.
	 */
	jmp		long_mode_active

long_mode_active:

#if     LED
	D16 movl        $WC_LED, %edx
	D16 movb        $0xd6, %al
	outb    (%dx)
#endif

#if     SERIAL
	D16 movl        $WC_COM, %edx
	D16 movb        $0x66, %al
	outb    (%dx)
#endif

	/*
	 * Long mode is now active but since we're still running with the
	 * original 16-bit CS we're actually in 16-bit compatability mode.
	 *
	 * We have to load an intermediate GDT and IDT here that we know are
	 * in 32-bit space before we can use the kernel's GDT and IDT, which
	 * may be in the 64-bit address space, and since we're in compatability
	 * mode, we only have access to 16 and 32-bit instructions at the
	 * moment.
	 */
	A16 D16 lgdt	TEMPGDTOFF	/* load temporary GDT */
	A16 D16 lidt	TEMPIDTOFF	/* load temporary IDT */


	/*
	 * Do a far transfer to 64-bit mode.  Set the CS selector to a 64-bit
	 * long mode selector (CS.L=1) in the temporary 32-bit GDT and jump
	 * to the real mode platter address of wc_long_mode_64 as until the
	 * 64-bit CS is in place we don't have access to 64-bit instructions
	 * and thus can't reference a 64-bit %rip.
	 */

#if     LED
	D16 movl        $WC_LED, %edx
	D16 movb        $0xd7, %al
	outb    (%dx)
#endif

#if     SERIAL
	D16 movl        $WC_COM, %edx
	D16 movb        $0x67, %al
	outb    (%dx)
#endif

	D16	pushl	$TEMP_CS64_SEL
	A16 D16 pushl	LM64OFF

	D16 lret


/*
 * Support routine to re-initialize VGA subsystem
 */
vgainit:
	D16 ret

/*
 * Support routine to re-initialize keyboard (which is USB - help!)
 */
kbdinit:
	D16 ret

/*
 * Support routine to re-initialize COM ports to something sane
 */
cominit:
	/ init COM1 & COM2

#if     DEBUG
/*
 * on debug kernels we need to initialize COM1 & COM2 here, so that
 * we can get debug output before the asy driver has resumed
 */

/ select COM1
	D16 movl	$_CONST(COM1+LCR), %edx
	D16 movb	$DLAB, %al		/ divisor latch
	outb	(%dx)

	D16 movl	$_CONST(COM1+DLL), %edx	/ divisor latch lsb
	D16 movb	$B9600L, %al		/ divisor latch
	outb	(%dx)

	D16 movl	$_CONST(COM1+DLH), %edx	/ divisor latch hsb
	D16 movb	$B9600H, %al		/ divisor latch
	outb	(%dx)

	D16 movl	$_CONST(COM1+LCR), %edx	/ select COM1
	D16 movb	$_CONST(STOP1|BITS8), %al	/ 1 stop bit, 8bit word len
	outb	(%dx)

	D16 movl	$_CONST(COM1+MCR), %edx	/ select COM1
	D16 movb	$_CONST(RTS|DTR), %al		/ data term ready & req to send
	outb	(%dx)

/ select COM2
	D16 movl	$_CONST(COM2+LCR), %edx
	D16 movb	$DLAB, %al		/ divisor latch
	outb	(%dx)

	D16 movl	$_CONST(COM2+DLL), %edx	/ divisor latch lsb
	D16 movb	$B9600L, %al		/ divisor latch
	outb	(%dx)

	D16 movl	$_CONST(COM2+DLH), %edx	/ divisor latch hsb
	D16 movb	$B9600H, %al		/ divisor latch
	outb	(%dx)

	D16 movl	$_CONST(COM2+LCR), %edx	/ select COM1
	D16 movb	$_CONST(STOP1|BITS8), %al	/ 1 stop bit, 8bit word len
	outb	(%dx)

	D16 movl	$_CONST(COM2+MCR), %edx	/ select COM1
	D16 movb	$_CONST(RTS|DTR), %al		/ data term ready & req to send
	outb	(%dx)
#endif	/*	DEBUG	*/

	D16 ret

	.code64

	.globl wc_long_mode_64
wc_long_mode_64:

#if     LED
	movw        $WC_LED, %dx
	movb        $0xd8, %al
	outb    (%dx)
#endif

#if     SERIAL
	movw        $WC_COM, %dx
	movb        $0x68, %al
	outb    (%dx)
#endif

	/*
	 * We are now running in long mode with a 64-bit CS (EFER.LMA=1,
	 * CS.L=1) so we now have access to 64-bit instructions.
	 *
	 * First, set the 64-bit GDT base.
	 */
	.globl	rm_platter_pa
	movl	rm_platter_pa, %eax

	lgdtq	GDTROFF(%rax)		/* load 64-bit GDT */

	/*
	 * Save the CPU number in %r11; get the value here since it's saved in
	 * the real mode platter.
	 */
/ JAN
/ the following is wrong! need to figure out MP systems
/	movl	CPUNOFF(%rax), %r11d

	/*
	 * Add rm_platter_pa to %rsp to point it to the same location as seen
	 * from 64-bit mode.
	 */
	addq	%rax, %rsp

	/*
	 * Now do an lretq to load CS with the appropriate selector for the
	 * kernel's 64-bit GDT and to start executing 64-bit setup code at the
	 * virtual address where boot originally loaded this code rather than
	 * the copy in the real mode platter's rm_code array as we've been
	 * doing so far.
	 */

#if     LED
	movw        $WC_LED, %dx
	movb        $0xd9, %al
	outb    (%dx)
#endif

/ JAN this should produce 'i' but we get 'g' instead ???
#if     SERIAL
	movw        $WC_COM, %dx
	movb        $0x69, %al
	outb    (%dx)
#endif

	pushq	$KCS_SEL
	pushq	$kernel_wc_code
	lretq

	.globl kernel_wc_code
kernel_wc_code:

#if     LED
	movw        $WC_LED, %dx
	movb        $0xda, %al
	outb    (%dx)
#endif

/ JAN this should produce 'j' but we get 'g' instead ???
#if     SERIAL
	movw        $WC_COM, %dx
	movb        $0x6a, %al
	outb    (%dx)
#endif

	/*
	 * Complete the balance of the setup we need to before executing
	 * 64-bit kernel code (namely init rsp, TSS, LGDT, FS and GS).
	 */
	.globl  rm_platter_va
	movq    rm_platter_va, %rbx
	addq	$WC_CPU, %rbx

#if     LED
	movw        $WC_LED, %dx
	movb        $0xdb, %al
	outb    (%dx)
#endif

#if     SERIAL
	movw        $WC_COM, %dx
	movw        $0x6b, %ax
	outb    (%dx)
#endif

	/*
	 * restore the rest of the registers
	 */

	lidtq	WC_IDT(%rbx)

#if     LED
	movw        $WC_LED, %dx
	movb        $0xdc, %al
	outb    (%dx)
#endif

#if     SERIAL
	movw        $WC_COM, %dx
	movw        $0x6c, %ax
	outb    (%dx)
#endif

	/*
	 * restore the rest of the registers
	 */

	movw    $KDS_SEL, %ax
	movw    %ax, %ds
	movw    %ax, %es
	movw    %ax, %ss

	/*
	 * Before proceeding, enable usage of the page table NX bit if
	 * that's how the page tables are set up.
	 */
	btl     $X86FSET_NX, x86_featureset(%rip)
	jnc     1f
	movl    $MSR_AMD_EFER, %ecx
	rdmsr
	orl     $AMD_EFER_NXE, %eax
	wrmsr
1:

	movq	WC_CR4(%rbx), %rax	/ restore full cr4 (with Global Enable)
	movq	%rax, %cr4

	lldt	WC_LDT(%rbx)
	movzwq	WC_TR(%rbx), %rax	/ clear TSS busy bit
	addq	WC_GDT+2(%rbx), %rax
	andl	$0xfffffdff, 4(%rax)
	movq	4(%rax), %rcx
	ltr	WC_TR(%rbx)

#if     LED
	movw        $WC_LED, %dx
	movb        $0xdd, %al
	outb    (%dx)
#endif

#if     SERIAL
	movw        $WC_COM, %dx
	movw        $0x6d, %ax
	outb    (%dx)
#endif

/ restore %fsbase %gsbase %kgbase registers using wrmsr instruction

	movq    WC_FS(%rbx), %rcx	/ restore fs register
	movw    %cx, %fs

	movl    $MSR_AMD_FSBASE, %ecx
	movl    WC_FSBASE(%rbx), %eax
	movl    WC_FSBASE+4(%rbx), %edx
	wrmsr

	movq    WC_GS(%rbx), %rcx	/ restore gs register
	movw    %cx, %gs

	movl    $MSR_AMD_GSBASE, %ecx	/ restore gsbase msr
	movl    WC_GSBASE(%rbx), %eax
	movl    WC_GSBASE+4(%rbx), %edx
	wrmsr

	movl    $MSR_AMD_KGSBASE, %ecx	/ restore kgsbase msr
	movl    WC_KGSBASE(%rbx), %eax
	movl    WC_KGSBASE+4(%rbx), %edx
	wrmsr

	movq	WC_CR0(%rbx), %rdx
	movq	%rdx, %cr0
	movq	WC_CR3(%rbx), %rdx
	movq	%rdx, %cr3
	movq	WC_CR8(%rbx), %rdx
	movq	%rdx, %cr8

#if     LED
	movw        $WC_LED, %dx
	movb        $0xde, %al
	outb    (%dx)
#endif

#if     SERIAL
	movw        $WC_COM, %dx
	movb        $0x6e, %al
	outb    (%dx)
#endif

	/*
	 * if we are not running on the boot CPU restore stack contents by
	 * calling i_cpr_restore_stack(curthread, save_stack);
	 */
	movq    %rsp, %rbp
	call	i_cpr_bootcpuid
	cmpl	%eax, WC_CPU_ID(%rbx)
	je	2f

	movq	%gs:CPU_THREAD, %rdi
	movq	WC_SAVED_STACK(%rbx), %rsi
	call	i_cpr_restore_stack
2:

	movq    WC_RSP(%rbx), %rsp	/ restore stack pointer

	/*
	 * APIC initialization
	 */
	movq    %rsp, %rbp

	/*
	 * skip iff function pointer is NULL
	 */
	cmpq	$0, ap_mlsetup
	je	3f
	leaq	ap_mlsetup, %rax
	INDIRECT_CALL_REG(rax)
3:

	leaq	cpr_start_cpu_func, %rax
	INDIRECT_CALL_REG(rax)

/ restore %rbx to the value it ahd before we called the functions above
	movq    rm_platter_va, %rbx
	addq	$WC_CPU, %rbx

	movq    WC_R8(%rbx), %r8
	movq    WC_R9(%rbx), %r9
	movq    WC_R10(%rbx), %r10
	movq    WC_R11(%rbx), %r11
	movq    WC_R12(%rbx), %r12
	movq    WC_R13(%rbx), %r13
	movq    WC_R14(%rbx), %r14
	movq    WC_R15(%rbx), %r15
/	movq    WC_RAX(%rbx), %rax
	movq    WC_RBP(%rbx), %rbp
	movq    WC_RCX(%rbx), %rcx
/	movq    WC_RDX(%rbx), %rdx
	movq    WC_RDI(%rbx), %rdi
	movq    WC_RSI(%rbx), %rsi


/ assume that %cs does not need to be restored
/ %ds, %es & %ss are ignored in 64bit mode
	movw	WC_SS(%rbx), %ss
	movw	WC_DS(%rbx), %ds
	movw	WC_ES(%rbx), %es

#if     LED
	movw        $WC_LED, %dx
	movb        $0xdf, %al
	outb    (%dx)
#endif

#if     SERIAL
	movw        $WC_COM, %dx
	movb        $0x6f, %al
	outb    (%dx)
#endif


	movq    WC_RBP(%rbx), %rbp
	movq    WC_RSP(%rbx), %rsp

#if     LED
	movw        $WC_LED, %dx
	movb        $0xe0, %al
	outb    (%dx)
#endif

#if     SERIAL
	movw        $WC_COM, %dx
	movb        $0x70, %al
	outb    (%dx)
#endif


	movq    WC_RCX(%rbx), %rcx

	pushq	WC_EFLAGS(%rbx)			/ restore flags
	popfq

#if     LED
	movw        $WC_LED, %dx
	movb        $0xe1, %al
	outb    (%dx)
#endif

#if     SERIAL
	movw        $WC_COM, %dx
	movb        $0x71, %al
	outb    (%dx)
#endif

/*
 * can not use outb after this point, because doing so would mean using
 * %dx which would modify %rdx which is restored here
 */

	movq	%rbx, %rax
	movq    WC_RDX(%rax), %rdx
	movq    WC_RBX(%rax), %rbx

	leave

	movq	WC_RETADDR(%rax), %rax
	movq	%rax, (%rsp)		/ return to caller of wc_save_context

	xorl	%eax, %eax			/ at wakeup return 0
	ret


	SET_SIZE(wc_rm_start)

	ENTRY_NP(asmspin)

	movl	%edi, %ecx
A1:
	loop	A1

	SET_SIZE(asmspin)

	.globl wc_rm_end
wc_rm_end:
	nop

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 * Copyright 2019 Joyent, Inc.
 */

#include <sys/asm_linkage.h>
#include <sys/asm_misc.h>
#include <sys/regset.h>
#include <sys/psw.h>

#include <sys/pcb.h>
#include <sys/trap.h>
#include <sys/ftrace.h>
#include <sys/traptrace.h>
#include <sys/clock.h>
#include <sys/panic.h>
#include <sys/privregs.h>

#include "assym.h"


/*
 * XX64: We are assuming that libc continues to expect the 64-bit value being
 * returned in %edx:%eax.  We further assume that it is safe to leave
 * the top 32-bit intact in %rax as they will be ignored by libc.  In
 * other words, if the 64-bit value is already in %rax, while we manually
 * manufacture a 64-bit value in %edx:%eax by setting %edx to be the high
 * 32 bits of %rax, we don't zero them out in %rax.
 * The following amd64 versions will need to be changed if the above
 * assumptions are not true.
 */

	.globl	gethrtimef
	ENTRY_NP(get_hrtime)
	FAST_INTR_PUSH
	movq	gethrtimef(%rip), %rax
	INDIRECT_CALL_REG(rax)
	movq	%rax, %rdx
	shrq	$32, %rdx			/* high 32-bit in %edx */
	FAST_INTR_POP
	FAST_INTR_RETURN
	SET_SIZE(get_hrtime)

	.globl	gethrestimef
	ENTRY_NP(get_hrestime)
	FAST_INTR_PUSH
	subq	$TIMESPEC_SIZE, %rsp
	movq	%rsp, %rdi
	movq	gethrestimef(%rip), %rax
	INDIRECT_CALL_REG(rax)
	movl	(%rsp), %eax
	movl	CLONGSIZE(%rsp), %edx
	addq	$TIMESPEC_SIZE, %rsp
	FAST_INTR_POP
	FAST_INTR_RETURN
	SET_SIZE(get_hrestime)

	/*
	 * In C this is
	 *
	 * klwp_t *lwp = ttolwp(curthread);
	 * struct mstate *ms = &lwp->lwp_mstate;
	 * return (gethrtime() - ms->ms_state_start + ms->ms_acct[LMS_USER]);
	 */
	ENTRY_NP(gethrvtime)
	FAST_INTR_PUSH
	call	gethrtime_unscaled		/* get time since boot */
	movq	%gs:CPU_LWP, %rcx		/* current lwp */
	subq	LWP_MS_STATE_START(%rcx), %rax	/* - ms->ms_state_start */
	addq	LWP_ACCT_USER(%rcx), %rax	/* add ms->ms_acct[LMS_USER] */
	subq	$16, %rsp
	movq	%rax, (%rsp)
	movq	%rsp, %rdi
	call	scalehrtime
	movq	(%rsp), %rax
	addq	$16, %rsp
	movq	%rax, %rdx
	shrq	$32, %rdx			/* high 32-bit in %rdx */
	FAST_INTR_POP
	FAST_INTR_RETURN
	SET_SIZE(gethrvtime)

	/*
	 * In C this is:
	 *
	 * return (((uint64_t)(curthread->t_lpl->lpl_lgrpid) << 32) |
	 *     curthread->t_cpu->cpu_id);
	 */
	ENTRY_NP(getlgrp)
	FAST_INTR_PUSH
	movq	%gs:CPU_THREAD, %rcx
	movq	T_LPL(%rcx), %rcx
	movl	LPL_LGRPID(%rcx), %edx
	movl	%gs:CPU_ID, %eax
	FAST_INTR_POP
	FAST_INTR_RETURN
	SET_SIZE(getlgrp)

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 * Copyright 2018 Joyent, Inc.
 */


#include <sys/asm_linkage.h>
#include <sys/segments.h>
#include <sys/controlregs.h>
#include <sys/machparam.h>
#include <sys/multiboot.h>
#include <sys/fastboot.h>
#include "assym.h"

/*
 * This code is to switch from 64-bit or 32-bit to protected mode.
 */

/*
 * For debugging with LEDs
 */
#define	FB_OUTB_ASM(val)	\
    movb	val, %al;	\
    outb	$0x80;


#define	DISABLE_PAGING							\
	movl	%cr4, %eax						;\
	btrl	$17, %eax	/* clear PCIDE bit */			;\
	movl	%eax, %cr4						;\
	movl	%cr0, %eax						;\
	btrl	$31, %eax	/* clear PG bit */			;\
	movl	%eax, %cr0

/*
 * This macro contains common code for 64/32-bit versions of copy_sections().
 * On entry:
 *	fbf points to the fboot_file_t
 *	snum contains the number of sections
 * Registers that would be clobbered:
 *	fbs, snum, %eax, %ecx, %edi, %esi.
 * NOTE: fb_dest_pa is supposed to be in the first 1GB,
 * therefore it is safe to use 32-bit register to hold it's value
 * even for 64-bit code.
 */

#define	COPY_SECT(fbf, fbs, snum)		\
	lea	FB_SECTIONS(fbf), fbs;		\
	xorl	%eax, %eax;			\
1:	movl	FB_DEST_PA(fbf), %esi;		\
	addl	FB_SEC_OFFSET(fbs), %esi;	\
	movl	FB_SEC_PADDR(fbs), %edi;	\
	movl	FB_SEC_SIZE(fbs), %ecx;		\
	rep					\
	  movsb;				\
	/* Zero BSS */				\
	movl	FB_SEC_BSS_SIZE(fbs), %ecx;	\
	rep					\
	  stosb;				\
	add	$FB_SECTIONS_INCR, fbs;		\
	dec	snum;				\
	jnz	1b


	.globl	_start
_start:

	/* Disable interrupts */
	cli

	/* Switch to a low memory stack */
	movq	$_start, %rsp
	addq	$FASTBOOT_STACK_OFFSET, %rsp

	/*
	 * Copy from old stack to new stack
	 * If the content before fi_valid gets bigger than 0x200 bytes,
	 * the reserved stack size above will need to be changed.
	 */
	movq	%rdi, %rsi	/* source from old stack */
	movq	%rsp, %rdi	/* destination on the new stack */
	movq	$FI_VALID, %rcx	/* size to copy */
	rep
	  smovb

	xorl	%eax, %eax
	xorl	%edx, %edx

	movl	$MSR_AMD_FSBASE, %ecx
	wrmsr

	movl	$MSR_AMD_GSBASE, %ecx
	wrmsr

	movl	$MSR_AMD_KGSBASE, %ecx
	wrmsr

	/*
	 * zero out all the registers to make sure they're 16 bit clean
	 */
	xorq	%r8, %r8
	xorq	%r9, %r9
	xorq	%r10, %r10
	xorq	%r11, %r11
	xorq	%r12, %r12
	xorq	%r13, %r13
	xorq	%r14, %r14
	xorq	%r15, %r15
	xorl	%eax, %eax
	xorl	%ebx, %ebx
	xorl	%ecx, %ecx
	xorl	%edx, %edx
	xorl	%ebp, %ebp

	/*
	 * Load our own GDT
	 */
	lgdt	gdt_info
	/*
	 * Load our own IDT
	 */
	lidt	idt_info

	/*
	 * Invalidate all TLB entries.
	 * Load temporary pagetables to copy kernel and boot-archive
	 */
	movq	%cr4, %rax
	andq	$_BITNOT(CR4_PGE), %rax
	movq	%rax, %cr4
	movq	FI_PAGETABLE_PA(%rsp), %rax
	movq	%rax, %cr3

	leaq	FI_FILES(%rsp), %rbx	/* offset to the files */

	/* copy unix to final destination */
	movq	FI_LAST_TABLE_PA(%rsp), %rsi	/* page table PA */
	leaq	_MUL(FASTBOOT_UNIX, FI_FILES_INCR)(%rbx), %rdi
	call	map_copy

	/* copy boot archive to final destination */
	movq	FI_LAST_TABLE_PA(%rsp), %rsi	/* page table PA */
	leaq	_MUL(FASTBOOT_BOOTARCHIVE, FI_FILES_INCR)(%rbx), %rdi
	call	map_copy

	/* Copy sections if there are any */
	leaq	_MUL(FASTBOOT_UNIX, FI_FILES_INCR)(%rbx), %rdi
	movl	FB_SECTCNT(%rdi), %esi
	cmpl	$0, %esi
	je	1f
	call	copy_sections
1:
	/*
	 * Shut down 64 bit mode. First get into compatiblity mode.
	 */
	movq	%rsp, %rax
	pushq	$B32DATA_SEL
	pushq	%rax
	pushf
	pushq	$B32CODE_SEL
	pushq	$1f
	iretq

	.code32
1:
	movl	$B32DATA_SEL, %eax
	movw	%ax, %ss
	movw	%ax, %ds
	movw	%ax, %es
	movw	%ax, %fs
	movw	%ax, %gs

	/*
	 * Disable long mode by:
	 * - shutting down paging (bit 31 of cr0).  This will flush the
	 *   TLBs.
	 * - turning off PCID in cr4
	 * - disabling LME (long mode enable) in EFER (extended feature reg)
	 */
	DISABLE_PAGING		/* clobbers %eax */

	ljmp	$B32CODE_SEL, $1f
1:

	/*
	 * Clear PGE, PAE and PSE flags as dboot expects them to be
	 * cleared.
	 */
	movl	%cr4, %eax
	andl	$_BITNOT(CR4_PGE | CR4_PAE | CR4_PSE), %eax
	movl	%eax, %cr4

	movl	$MSR_AMD_EFER, %ecx	/* Extended Feature Enable */
	rdmsr
	btcl	$8, %eax		/* bit 8 Long Mode Enable bit */
	wrmsr

dboot_jump:
	/* Jump to dboot */
	movl	$DBOOT_ENTRY_ADDRESS, %edi
	movl	FI_NEW_MBI_PA(%esp), %ebx
	movl	$MB_BOOTLOADER_MAGIC, %eax
	jmp	*%edi

	.code64
	ENTRY_NP(copy_sections)
	/*
	 * On entry
	 *	%rdi points to the fboot_file_t
	 *	%rsi contains number of sections
	 */
	movq	%rdi, %rdx
	movq	%rsi, %r9

	COPY_SECT(%rdx, %r8, %r9)
	ret
	SET_SIZE(copy_sections)

	ENTRY_NP(map_copy)
	/*
	 * On entry
	 *	%rdi points to the fboot_file_t
	 *	%rsi has FI_LAST_TABLE_PA(%rsp)
	 */

	movq	%rdi, %rdx
	movq	%rsi, %r8
	movq	FB_PTE_LIST_PA(%rdx), %rax	/* PA list of the source */
	movq	FB_DEST_PA(%rdx), %rdi		/* PA of the destination */

2:
	movq	(%rax), %rcx			/* Are we done? */
	cmpl	$FASTBOOT_TERMINATE, %ecx
	je	1f

	movq	%rcx, (%r8)
	movq	%cr3, %rsi		/* Reload cr3 */
	movq	%rsi, %cr3
	movq	FB_VA(%rdx), %rsi	/* Load from VA */
	movq	$PAGESIZE, %rcx
	shrq	$3, %rcx		/* 8-byte at a time */
	rep
	  smovq
	addq	$8, %rax 		/* Go to next PTE */
	jmp	2b
1:
	ret
	SET_SIZE(map_copy)

idt_info:
	.value	0x3ff
	.quad	0

/*
 * We need to trampoline thru a gdt we have in low memory.
 */
#include "../boot/boot_gdt.s"
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved.
 *
 * Copyright 2018 Joyent, Inc.
 */

#ifndef	_GENASSYM
#define	_GENASSYM
#endif

#define	exit	kern_exit

#include <sys/types.h>
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/elf_notes.h>
#include <sys/thread.h>
#include <sys/rwlock.h>
#include <sys/proc.h>
#include <sys/cpuvar.h>
#include <sys/clock.h>
#include <sys/trap.h>
#include <sys/modctl.h>
#include <sys/traptrace.h>
#include <vm/seg.h>
#include <sys/avintr.h>
#include <sys/pic.h>
#include <sys/pit.h>
#include <sys/fp.h>
#include <sys/disp.h>
#include <sys/archsystm.h>
#include <sys/x86_archext.h>
#include <sys/sunddi.h>
#include <sys/mach_mmu.h>

#if defined(__xpv)
#include <sys/hypervisor.h>
#endif

#undef	exit		/* unhide exit, see comment above */
extern void exit(int);

/*
 * Proactively discourage anyone from referring to structures or
 * member offsets in this program.
 */
#define	struct	struct...
#define	OFFSET	OFFSET...

int
main(int argc, char *argv[])
{
	printf("#define\tLOCK_LEVEL 0x%x\n", LOCK_LEVEL);
	printf("#define\tCLOCK_LEVEL 0x%x\n", CLOCK_LEVEL);
	printf("#define\tDISP_LEVEL 0x%x\n", DISP_LEVEL);
	printf("#define\tPIL_MAX 0x%x\n", PIL_MAX);
	printf("#define\tHIGH_LEVELS 0x%x\n", HIGH_LEVELS);
	printf("#define\tCPU_INTR_ACTV_HIGH_LEVEL_MASK 0x%x\n",
	    CPU_INTR_ACTV_HIGH_LEVEL_MASK);

	printf("#define\tPIC_NSEOI 0x%x\n", PIC_NSEOI);
	printf("#define\tPIC_SEOI_LVL7 0x%x\n", PIC_SEOI_LVL7);

	printf("#define\tNANOSEC 0x%llx\n", NANOSEC);
	printf("#define\tADJ_SHIFT 0x%x\n", ADJ_SHIFT);

	printf("#define\tSSLEEP 0x%x\n", SSLEEP);
	printf("#define\tSRUN 0x%x\n", SRUN);
	printf("#define\tSONPROC 0x%x\n", SONPROC);

	printf("#define\tT_INTR_THREAD 0x%x\n", T_INTR_THREAD);
	printf("#define\tFREE_THREAD 0x%x\n", TS_FREE);
	printf("#define\tTS_FREE 0x%x\n", TS_FREE);
	printf("#define\tTS_ZOMB 0x%x\n", TS_ZOMB);
	printf("#define\tTP_MSACCT 0x%x\n", TP_MSACCT);
	printf("#define\tTP_WATCHPT 0x%x\n", TP_WATCHPT);
	printf("#define\tONPROC_THREAD 0x%x\n", TS_ONPROC);

	printf("#define\tS_READ 0x%x\n", (int)S_READ);
	printf("#define\tS_WRITE 0x%x\n", (int)S_WRITE);
	printf("#define\tS_EXEC 0x%x\n", (int)S_EXEC);
	printf("#define\tS_OTHER 0x%x\n", (int)S_OTHER);

	printf("#define\tNORMALRETURN 0x%x\n", (int)NORMALRETURN);
	printf("#define\tLWP_USER 0x%x\n", LWP_USER);
	printf("#define\tLWP_SYS 0x%x\n", LWP_SYS);
	printf("#define\tLMS_USER 0x%x\n", LMS_USER);
	printf("#define\tLMS_SYSTEM 0x%x\n", LMS_SYSTEM);

	printf("#define\tSSE_MXCSR_EFLAGS 0x%x\n", SSE_MXCSR_EFLAGS);

	printf("#define\tFP_FXSAVE 0x%x\n", FP_FXSAVE);
	printf("#define\tFP_XSAVE 0x%x\n", FP_XSAVE);

	printf("#define\tAV_INT_SPURIOUS 0x%x\n", AV_INT_SPURIOUS);

	printf("#define\tCPU_READY 0x%x\n", CPU_READY);
	printf("#define\tCPU_QUIESCED 0x%x\n", CPU_QUIESCED);

	printf("#define\tMCMD_PORT 0x%x\n", MCMD_PORT);
	printf("#define\tSCMD_PORT 0x%x\n", SCMD_PORT);
	printf("#define\tMIMR_PORT 0x%x\n", MIMR_PORT);
	printf("#define\tSIMR_PORT 0x%x\n", SIMR_PORT);

	printf("#define\tDMP_NOSYNC 0x%x\n", DMP_NOSYNC);

	printf("#define\tRW_WRITER\t0x%x\n", RW_WRITER);
	printf("#define\tRW_READER\t0x%x\n", RW_READER);

	printf("#define\tNSYSCALL 0x%x\n", NSYSCALL);

	printf("#define\tSE_32RVAL1 0x%x\n", SE_32RVAL1);
	printf("#define\tSE_32RVAL2 0x%x\n", SE_32RVAL2);
	printf("#define\tSE_64RVAL 0x%x\n", SE_64RVAL);

	printf("#define\tMAXSYSARGS 0x%x\n", MAXSYSARGS);

	/* Hack value just to allow clock to be kicked */
	printf("#define\tNSEC_PER_CLOCK_TICK 0x%llx\n", NANOSEC / 100);

	printf("#define\tNSEC_PER_COUNTER_TICK 0x%llx\n", NANOSEC / PIT_HZ);

	printf("#define\tNBPW 0x%x\n", (uint_t)NBPW);

	printf("#define\tDDI_ACCATTR_IO_SPACE 0x%x\n", DDI_ACCATTR_IO_SPACE);
	printf("#define\tDDI_ACCATTR_DIRECT 0x%x\n", DDI_ACCATTR_DIRECT);
	printf("#define\tDDI_ACCATTR_CPU_VADDR 0x%x\n", DDI_ACCATTR_CPU_VADDR);
	printf("#define\tDDI_DEV_AUTOINCR 0x%x\n", DDI_DEV_AUTOINCR);

	printf("#define\tMMU_STD_PAGESIZE 0x%x\n", (uint_t)MMU_STD_PAGESIZE);
	printf("#define\tMMU_STD_PAGEMASK 0x%x\n", (uint_t)MMU_STD_PAGEMASK);
	printf("#define\tFOUR_MEG 0x%x\n", (uint_t)FOUR_MEG);

	printf("#define\tTRAPTR_NENT 0x%x\n", TRAPTR_NENT);

	printf("#define\tCPU_DTRACE_NOFAULT 0x%x\n", CPU_DTRACE_NOFAULT);
	printf("#define\tCPU_DTRACE_BADADDR 0x%x\n", CPU_DTRACE_BADADDR);
	printf("#define\tCPU_DTRACE_DIVZERO 0x%x\n", CPU_DTRACE_DIVZERO);
	printf("#define\tCPU_DTRACE_ILLOP 0x%x\n", CPU_DTRACE_ILLOP);

	printf("#define\tMODS_NOUNLOAD 0x%x\n", MODS_NOUNLOAD);
	printf("#define\tMODS_WEAK 0x%x\n", MODS_WEAK);
	printf("#define\tMODS_INSTALLED 0x%x\n", MODS_INSTALLED);

	printf("#define\tKPREEMPT_SYNC 0x%x\n", KPREEMPT_SYNC);

#if defined(__xpv)
	printf("#define\tSHUTDOWN_reboot 0x%x\n", SHUTDOWN_reboot);
	printf("#define\tSCHEDOP_block 0x%x\n", SCHEDOP_block);
	printf("#define\tVGCF_IN_KERNEL 0x%x\n", VGCF_IN_KERNEL);
#endif
	return (0);
}
/*
 * This file and its contents are supplied under the terms of the
 * Common Development and Distribution License ("CDDL"), version 1.0.
 * You may only use this file in accordance with the terms of version
 * 1.0 of the CDDL.
 *
 * A full copy of the text of the CDDL should have accompanied this
 * source.  A copy of the CDDL is also available via the Internet at
 * http://www.illumos.org/license/CDDL.
 */

/*
 * Copyright 2019 Joyent, Inc.
 */


#include <sys/asm_linkage.h>

	ENTRY_NP(hma_vmx_vmxon)
	push	%rbp
	movq	%rsp, %rbp
	pushq	%rdi

	xorl	%eax, %eax
	vmxon	-0x8(%rbp)
	ja	1f	/* CF=0, ZF=0 (success) */
	incl	%eax
1:

	leave
	ret
	SET_SIZE(hma_vmx_vmxon)

	ENTRY_NP(hma_vmx_do_invept)
	push	%rbp
	movq	%rsp, %rbp
	pushq	%rdi
	pushq	%rsi

	/* build INVEPT descriptor on stack */
	xorl	%eax, %eax
	pushq	%rax;
	pushq	%rsi

	invept	(%rsp), %rdi
	ja	1f	/* CF=0, ZF=0 (success) */
	incl	%eax
1:

	leave
	ret
	SET_SIZE(hma_vmx_do_invept)
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
 * Copyright 2019 Joyent, Inc.
 */

/*	Copyright (c) 1990, 1991 UNIX System Laboratories, Inc.	*/
/*	Copyright (c) 1984, 1986, 1987, 1988, 1989, 1990 AT&T	*/
/*	  All Rights Reserved					*/

/*	Copyright (c) 1987, 1988 Microsoft Corporation		*/
/*	  All Rights Reserved					*/

#include <sys/asm_linkage.h>
#include <sys/asm_misc.h>
#include <sys/regset.h>
#include <sys/psw.h>
#include <sys/x86_archext.h>

#include <sys/segments.h>
#include <sys/pcb.h>
#include <sys/trap.h>
#include <sys/ftrace.h>
#include <sys/traptrace.h>
#include <sys/clock.h>
#include <sys/panic.h>
#include "assym.h"

	/*
	 * Common register usage:
	 *
	 * %r12		trap trace pointer
	 */
	ENTRY_NP2(cmnint, _interrupt)

	INTR_PUSH
	INTGATE_INIT_KERNEL_FLAGS	/* (set kernel rflags values) */

	/*
	 * At the end of TRACE_PTR %r12 points to the current TRAPTRACE entry
	 */
	TRACE_PTR(%r12, %rax, %eax, %rdx, $TT_INTERRUPT)
						/* Uses labels 8 and 9 */
	TRACE_REGS(%r12, %rsp, %rax, %rbx)	/* Uses label 9 */
	TRACE_STAMP(%r12)		/* Clobbers %eax, %edx, uses 9 */

	movq	%rsp, %rbp

	TRACE_STACK(%r12)

#ifdef TRAPTRACE
	LOADCPU(%rbx)				/* &cpu */
	movl	CPU_PRI(%rbx), %r14d		/* old ipl */
	movl	$255, TTR_IPL(%r12)
	movl	%r14d, %edi
	movb	%dil, TTR_PRI(%r12)
	movl	CPU_BASE_SPL(%rbx), %edi
	movb	%dil, TTR_SPL(%r12)
	movb	$255, TTR_VECTOR(%r12)
	movq	%r12, %rsi		/* pass traptrace record pointer */
#endif

	movq	%rsp, %rdi		/* pass struct regs pointer */
	movq	do_interrupt_common, %rax
	INDIRECT_CALL_REG(rax)

	jmp	_sys_rtt_ints_disabled
	/*NOTREACHED*/

	SET_SIZE(cmnint)
	SET_SIZE(_interrupt)

/*
 * Declare a uintptr_t which has the size of _interrupt to enable stack
 * traceback code to know when a regs structure is on the stack.
 */
	.globl	_interrupt_size
	.align	CLONGSIZE
_interrupt_size:
	.NWORD	. - _interrupt
	.type	_interrupt_size, @object

	/*
	 * If we're here, we're being called from splx() to fake a soft
	 * interrupt (note that interrupts are still disabled from
	 * splx()).  We execute this code when a soft interrupt is
	 * posted at level higher than the CPU's current spl; when spl
	 * is lowered in splx(), it will see the softint and jump here.
	 * We'll do exactly what a trap would do:  push our flags, %cs,
	 * %rip, error code and trap number (T_SOFTINT).  The cmnint()
	 * code will see T_SOFTINT and branch to the dosoftint() code.
	 *
	 * iretq -always- pops all five regs. Imitate the 16-byte
	 * auto-align of the stack, and the zero-ed out %ss value.
	 */

	ENTRY_NP(fakesoftint)
	movq	%rsp, %r11
	andq	$-16, %rsp
	pushq	$KDS_SEL	/* %ss */
	pushq	%r11		/* %rsp */
	pushf			/* rflags */
#if defined(__xpv)
	popq	%r11
	EVENT_MASK_TO_IE(%rdi, %r11)
	pushq	%r11
#endif
	pushq	$KCS_SEL	/* %cs */
	leaq	fakesoftint_return(%rip), %r11
	pushq	%r11		/* %rip */
	pushq	$0		/* err */
	pushq	$T_SOFTINT	/* trap */
	jmp	cmnint
	ALTENTRY(fakesoftint_return)
	ret
	SET_SIZE(fakesoftint_return)
	SET_SIZE(fakesoftint)

/*
 * This file and its contents are supplied under the terms of the
 * Common Development and Distribution License ("CDDL"), version 1.0.
 * You may only use this file in accordance with the terms of version
 * 1.0 of the CDDL.
 *
 * A full copy of the text of the CDDL should have accompanied this
 * source.  A copy of the CDDL is also available via the Internet at
 * http://www.illumos.org/license/CDDL.
 */
/*
 * Copyright 2019 Joyent, Inc.
 * Copyright 2020 OmniOS Community Edition (OmniOSce) Association.
 */

/*
 * This file contains the trampolines that are used by KPTI in order to be
 * able to take interrupts/trap/etc while on the "user" page table.
 *
 * We don't map the full kernel text into the user page table: instead we
 * map this one small section of trampolines (which compiles to ~13 pages).
 * These trampolines are set in the IDT always (so they will run no matter
 * whether we're on the kernel or user page table), and their primary job is to
 * pivot us to the kernel %cr3 and %rsp without ruining everything.
 *
 * All of these interrupts use the amd64 IST feature when we have KPTI enabled,
 * meaning that they will execute with their %rsp set to a known location, even
 * if we take them in the kernel.
 *
 * Over in desctbls.c (for cpu0) and mp_pc.c (other cpus) we set up the IST
 * stack to point at &cpu->cpu_m.mcpu_kpti.kf_tr_rsp. You can see the mcpu_kpti
 * (a struct kpti_frame) defined in machcpuvar.h. This struct is set up to be
 * page-aligned, and we map the page it's on into both page tables. Using a
 * struct attached to the cpu_t also means that we can use %rsp-relative
 * addressing to find anything on the cpu_t, so we don't have to touch %gs or
 * GSBASE at all on incoming interrupt trampolines (which can get pretty hairy).
 *
 * This little struct is where the CPU will push the actual interrupt frame.
 * Then, in the trampoline, we change %cr3, then figure out our destination
 * stack pointer and "pivot" to it (set %rsp and re-push the CPU's interrupt
 * frame). Then we jump to the regular ISR in the kernel text and carry on as
 * normal.
 *
 * We leave the original frame and any spilled regs behind in the kpti_frame
 * lazily until we want to return to userland. Then, we clear any spilled
 * regs from it, and overwrite the rest with our iret frame. When switching
 * this cpu to a different process (in hat_switch), we bzero the whole region to
 * make sure nothing can leak between processes.
 *
 * When we're returning back to the original place we took the interrupt later
 * (especially if it was in userland), we have to jmp back to the "return
 * trampolines" here, since when we set %cr3 back to the user value, we need to
 * be executing from code here in these shared pages and not the main kernel
 * text again. Even though it should be fine to iret directly from kernel text
 * when returning to kernel code, we make things jmp to a trampoline here just
 * for consistency.
 *
 * Note that with IST, it's very important that we always must have pivoted
 * away from the IST stack before we could possibly take any other interrupt
 * on the same IST (unless it's an end-of-the-world fault and we don't care
 * about coming back from it ever).
 *
 * This is particularly relevant to the dbgtrap/brktrap trampolines, as they
 * regularly have to happen from within trampoline code (e.g. in the sysenter
 * single-step case) and then return to the world normally. As a result, these
 * two are IST'd to their own kpti_frame right above the normal one (in the same
 * page), so they don't clobber their parent interrupt.
 *
 * To aid with debugging, we also IST the page fault (#PF/pftrap), general
 * protection fault (#GP/gptrap) and stack fault (#SS/stktrap) interrupts to
 * their own separate kpti_frame. This ensures that if we take one of these
 * due to a bug in trampoline code, we preserve the original trampoline
 * state that caused the trap.
 *
 * NMI, MCE and dblfault interrupts also are taken on their own dedicated IST
 * stacks, since they can interrupt another ISR at any time. These stacks are
 * full-sized, however, and not a little kpti_frame struct. We only set %cr3 in
 * their trampolines (and do it unconditionally), and don't bother pivoting
 * away. We're either going into the panic() path, or we're going to return
 * straight away without rescheduling, so it's fine to not be on our real
 * kthread stack (and some of the state we want to go find it with might be
 * corrupt!)
 *
 * Finally, for these "special" interrupts (NMI/MCE/double fault) we use a
 * special %cr3 value we stash here in the text (kpti_safe_cr3). We set this to
 * point at the PML4 for kas early in boot and never touch it again. Hopefully
 * it survives whatever corruption brings down the rest of the kernel!
 *
 * Syscalls are different to interrupts (at least in the SYSENTER/SYSCALL64
 * cases) in that they do not push an interrupt frame (and also have some other
 * effects). In the syscall trampolines, we assume that we can only be taking
 * the call from userland and use swapgs and an unconditional overwrite of %cr3.
 * We do not do any stack pivoting for syscalls (and we leave SYSENTER's
 * existing %rsp pivot untouched) -- instead we spill registers into
 * %gs:CPU_KPTI_* as we need to.
 *
 * Note that the normal %cr3 values do not cause invalidations with PCIDE - see
 * hat_switch().
 */

/*
 * The macros here mostly line up with what's in kdi_idthdl.s, too, so if you
 * fix bugs here check to see if they should be fixed there as well.
 */

#include <sys/asm_linkage.h>
#include <sys/asm_misc.h>
#include <sys/regset.h>
#include <sys/privregs.h>
#include <sys/psw.h>
#include <sys/machbrand.h>
#include <sys/param.h>

#include <sys/segments.h>
#include <sys/pcb.h>
#include <sys/trap.h>
#include <sys/ftrace.h>
#include <sys/traptrace.h>
#include <sys/clock.h>
#include <sys/model.h>
#include <sys/panic.h>

#if defined(__xpv)
#include <sys/hypervisor.h>
#endif

#include "assym.h"

	.data
	DGDEF3(kpti_enable, 8, 8)
	.fill	1, 8, 1

#if DEBUG
	.data
_bad_ts_panic_msg:
	.string "kpti_trampolines.s: tr_iret_user but CR0.TS set"
#endif

.section ".text";
.align MMU_PAGESIZE

.global kpti_tramp_start
kpti_tramp_start:
	nop

/* This will be set by mlsetup, and then double-checked later */
.global kpti_safe_cr3
kpti_safe_cr3:
	.quad 0
	SET_SIZE(kpti_safe_cr3)

/* startup_kmem() will overwrite this */
.global kpti_kbase
kpti_kbase:
	.quad KERNELBASE
	SET_SIZE(kpti_kbase)

#define	SET_KERNEL_CR3(spillreg)		\
	mov	%cr3, spillreg;			\
	mov	spillreg, %gs:CPU_KPTI_TR_CR3;	\
	mov	%gs:CPU_KPTI_KCR3, spillreg;	\
	cmp	$0, spillreg;			\
	je	2f;				\
	mov	spillreg, %cr3;			\
2:

#if DEBUG
#define	SET_USER_CR3(spillreg)			\
	mov	%cr3, spillreg;			\
	mov	spillreg, %gs:CPU_KPTI_TR_CR3;	\
	mov	%gs:CPU_KPTI_UCR3, spillreg;	\
	mov	spillreg, %cr3
#else
#define	SET_USER_CR3(spillreg)			\
	mov	%gs:CPU_KPTI_UCR3, spillreg;	\
	mov	spillreg, %cr3
#endif

#define	PIVOT_KPTI_STK(spillreg)		\
	mov	%rsp, spillreg;			\
	mov	%gs:CPU_KPTI_RET_RSP, %rsp;	\
	pushq	T_FRAMERET_SS(spillreg);	\
	pushq	T_FRAMERET_RSP(spillreg);	\
	pushq	T_FRAMERET_RFLAGS(spillreg);	\
	pushq	T_FRAMERET_CS(spillreg);	\
	pushq	T_FRAMERET_RIP(spillreg)


#define	INTERRUPT_TRAMPOLINE_P(errpush)	\
	pushq	%r13;				\
	pushq	%r14;				\
	subq	$KPTI_R14, %rsp;		\
	/* Save current %cr3. */		\
	mov	%cr3, %r14;			\
	mov	%r14, KPTI_TR_CR3(%rsp);	\
						\
	cmpw	$KCS_SEL, KPTI_CS(%rsp);	\
	je	3f;				\
1:						\
	/* Change to the "kernel" %cr3 */	\
	mov	KPTI_KCR3(%rsp), %r14;		\
	cmp	$0, %r14;			\
	je	2f;				\
	mov	%r14, %cr3;			\
2:						\
	/* Get our cpu_t in %r13 */		\
	mov	%rsp, %r13;			\
	and	$(~(MMU_PAGESIZE - 1)), %r13;	\
	subq	$CPU_KPTI_START, %r13;		\
	/* Use top of the kthread stk */	\
	mov	CPU_THREAD(%r13), %r14;		\
	mov	T_STACK(%r14), %r14;		\
	addq	$REGSIZE+MINFRAME, %r14;	\
	jmp	4f;				\
3:						\
	/* Check the %rsp in the frame. */	\
	/* Is it above kernel base? */		\
	mov	kpti_kbase, %r14;		\
	cmp	%r14, KPTI_RSP(%rsp);		\
	jb	1b;				\
	/* Use the %rsp from the trap frame */	\
	mov	KPTI_RSP(%rsp), %r14;		\
	and	$(~0xf), %r14;			\
4:						\
	mov	%rsp, %r13;			\
	/* %r14 contains our destination stk */	\
	mov	%r14, %rsp;			\
	pushq	KPTI_SS(%r13);			\
	pushq	KPTI_RSP(%r13);			\
	pushq	KPTI_RFLAGS(%r13);		\
	pushq	KPTI_CS(%r13);			\
	pushq	KPTI_RIP(%r13);			\
	errpush;				\
	mov	KPTI_R14(%r13), %r14;		\
	mov	KPTI_R13(%r13), %r13

#define	INTERRUPT_TRAMPOLINE_NOERR		\
	INTERRUPT_TRAMPOLINE_P(/**/)

#define	INTERRUPT_TRAMPOLINE			\
	INTERRUPT_TRAMPOLINE_P(pushq KPTI_ERR(%r13))

/*
 * This is used for all interrupts that can plausibly be taken inside another
 * interrupt and are using a kpti_frame stack (so #BP, #DB, #GP, #PF, #SS).
 *
 * We also use this for #NP, even though it uses the standard IST: the
 * additional %rsp checks below will catch when we get an exception doing an
 * iret to userspace with a bad %cs/%ss.  This appears as a kernel trap, and
 * only later gets redirected via kern_gpfault().
 *
 * We check for whether we took the interrupt while in another trampoline, in
 * which case we need to use the kthread stack.
 */
#define	DBG_INTERRUPT_TRAMPOLINE_P(errpush)	\
	pushq	%r13;				\
	pushq	%r14;				\
	subq	$KPTI_R14, %rsp;		\
	/* Check for clobbering */		\
	cmpq	$0, KPTI_FLAG(%rsp);		\
	je	1f;				\
	/* Don't worry, this totally works */	\
	int	$8;				\
1:						\
	movq	$1, KPTI_FLAG(%rsp);		\
	/* Save current %cr3. */		\
	mov	%cr3, %r14;			\
	mov	%r14, KPTI_TR_CR3(%rsp);	\
						\
	cmpw	$KCS_SEL, KPTI_CS(%rsp);	\
	je	4f;				\
2:						\
	/* Change to the "kernel" %cr3 */	\
	mov	KPTI_KCR3(%rsp), %r14;		\
	cmp	$0, %r14;			\
	je	3f;				\
	mov	%r14, %cr3;			\
3:						\
	/* Get our cpu_t in %r13 */		\
	mov	%rsp, %r13;			\
	and	$(~(MMU_PAGESIZE - 1)), %r13;	\
	subq	$CPU_KPTI_START, %r13;		\
	/* Use top of the kthread stk */	\
	mov	CPU_THREAD(%r13), %r14;		\
	mov	T_STACK(%r14), %r14;		\
	addq	$REGSIZE+MINFRAME, %r14;	\
	jmp	6f;				\
4:						\
	/* Check the %rsp in the frame. */	\
	/* Is it above kernel base? */		\
	/* If not, treat as user. */		\
	mov	kpti_kbase, %r14;		\
	cmp	%r14, KPTI_RSP(%rsp);		\
	jb	2b;				\
	/* Is it within the kpti_frame page? */	\
	/* If it is, treat as user interrupt */	\
	mov	%rsp, %r13;			\
	and	$(~(MMU_PAGESIZE - 1)), %r13;	\
	mov	KPTI_RSP(%rsp), %r14;		\
	and	$(~(MMU_PAGESIZE - 1)), %r14;	\
	cmp	%r13, %r14;			\
	je	2b;				\
	/* Were we in trampoline code? */	\
	leaq	kpti_tramp_start, %r14;		\
	cmp	%r14, KPTI_RIP(%rsp);		\
	jb	5f;				\
	leaq	kpti_tramp_end, %r14;		\
	cmp	%r14, KPTI_RIP(%rsp);		\
	ja	5f;				\
	/* If we were, change %cr3: we might */	\
	/* have interrupted before it did. */	\
	mov	KPTI_KCR3(%rsp), %r14;		\
	mov	%r14, %cr3;			\
5:						\
	/* Use the %rsp from the trap frame */	\
	mov	KPTI_RSP(%rsp), %r14;		\
	and	$(~0xf), %r14;			\
6:						\
	mov	%rsp, %r13;			\
	/* %r14 contains our destination stk */	\
	mov	%r14, %rsp;			\
	pushq	KPTI_SS(%r13);			\
	pushq	KPTI_RSP(%r13);			\
	pushq	KPTI_RFLAGS(%r13);		\
	pushq	KPTI_CS(%r13);			\
	pushq	KPTI_RIP(%r13);			\
	errpush;				\
	mov	KPTI_R14(%r13), %r14;		\
	movq	$0, KPTI_FLAG(%r13);		\
	mov	KPTI_R13(%r13), %r13

#define	DBG_INTERRUPT_TRAMPOLINE_NOERR		\
	DBG_INTERRUPT_TRAMPOLINE_P(/**/)

#define	DBG_INTERRUPT_TRAMPOLINE		\
	DBG_INTERRUPT_TRAMPOLINE_P(pushq KPTI_ERR(%r13))

	/*
	 * These labels (_start and _end) are used by trap.c to determine if
	 * we took an interrupt like an NMI during the return process.
	 */
.global	tr_sysc_ret_start
tr_sysc_ret_start:

	/*
	 * Syscall return trampolines.
	 *
	 * These are expected to be called on the kernel %gs. tr_sysret[ql] are
	 * called after %rsp is changed back to the user value, so we have no
	 * stack to work with. tr_sysexit has a kernel stack (but has to
	 * preserve rflags, soooo).
	 */
	ENTRY_NP(tr_sysretq)
	cmpq	$1, kpti_enable
	jne	1f

	mov	%r13, %gs:CPU_KPTI_R13
	SET_USER_CR3(%r13)
	mov	%gs:CPU_KPTI_R13, %r13
	/* Zero these to make sure they didn't leak from a kernel trap */
	movq	$0, %gs:CPU_KPTI_R13
	movq	$0, %gs:CPU_KPTI_R14
1:
	swapgs
	sysretq
	SET_SIZE(tr_sysretq)

	ENTRY_NP(tr_sysretl)
	cmpq	$1, kpti_enable
	jne	1f

	mov	%r13, %gs:CPU_KPTI_R13
	SET_USER_CR3(%r13)
	mov	%gs:CPU_KPTI_R13, %r13
	/* Zero these to make sure they didn't leak from a kernel trap */
	movq	$0, %gs:CPU_KPTI_R13
	movq	$0, %gs:CPU_KPTI_R14
1:
	SWAPGS
	SYSRETL
	SET_SIZE(tr_sysretl)

	ENTRY_NP(tr_sysexit)
	/*
	 * Note: we want to preserve RFLAGS across this branch, since sysexit
	 * (unlike sysret above) does not restore RFLAGS for us.
	 *
	 * We still have the real kernel stack (sysexit does restore that), so
	 * we can use pushfq/popfq.
	 */
	pushfq

	cmpq	$1, kpti_enable
	jne	1f

	/* Have to pop it back off now before we change %cr3! */
	popfq
	mov	%r13, %gs:CPU_KPTI_R13
	SET_USER_CR3(%r13)
	mov	%gs:CPU_KPTI_R13, %r13
	/* Zero these to make sure they didn't leak from a kernel trap */
	movq	$0, %gs:CPU_KPTI_R13
	movq	$0, %gs:CPU_KPTI_R14
	jmp	2f
1:
	popfq
2:
	swapgs
	sti
	SYSEXITL
	SET_SIZE(tr_sysexit)

.global	tr_sysc_ret_end
tr_sysc_ret_end:

	/*
	 * Syscall entry trampolines.
	 */

#if DEBUG
#define	MK_SYSCALL_TRAMPOLINE(isr)		\
	ENTRY_NP(tr_##isr);			\
	swapgs;					\
	mov	%r13, %gs:CPU_KPTI_R13;		\
	mov	%cr3, %r13;			\
	mov	%r13, %gs:CPU_KPTI_TR_CR3;	\
	mov	%gs:CPU_KPTI_KCR3, %r13;	\
	mov	%r13, %cr3;			\
	mov	%gs:CPU_KPTI_R13, %r13;		\
	swapgs;					\
	jmp	isr;				\
	SET_SIZE(tr_##isr)
#else
#define	MK_SYSCALL_TRAMPOLINE(isr)		\
	ENTRY_NP(tr_##isr);			\
	swapgs;					\
	mov	%r13, %gs:CPU_KPTI_R13;		\
	mov	%gs:CPU_KPTI_KCR3, %r13;	\
	mov	%r13, %cr3;			\
	mov	%gs:CPU_KPTI_R13, %r13;		\
	swapgs;					\
	jmp	isr;				\
	SET_SIZE(tr_##isr)
#endif

	MK_SYSCALL_TRAMPOLINE(sys_syscall)
	MK_SYSCALL_TRAMPOLINE(sys_syscall32)
	MK_SYSCALL_TRAMPOLINE(brand_sys_syscall)
	MK_SYSCALL_TRAMPOLINE(brand_sys_syscall32)

	/*
	 * SYSENTER is special. The CPU is really not very helpful when it
	 * comes to preserving and restoring state with it, and as a result
	 * we have to do all of it by hand. So, since we want to preserve
	 * RFLAGS, we have to be very careful in these trampolines to not
	 * clobber any bits in it. That means no cmpqs or branches!
	 */
	ENTRY_NP(tr_sys_sysenter)
	swapgs
	mov	%r13, %gs:CPU_KPTI_R13
#if DEBUG
	mov	%cr3, %r13
	mov	%r13, %gs:CPU_KPTI_TR_CR3
#endif
	mov	%gs:CPU_KPTI_KCR3, %r13
	mov	%r13, %cr3
	mov	%gs:CPU_KPTI_R13, %r13
	jmp	_sys_sysenter_post_swapgs
	SET_SIZE(tr_sys_sysenter)

	ENTRY_NP(tr_brand_sys_sysenter)
	swapgs
	mov	%r13, %gs:CPU_KPTI_R13
#if DEBUG
	mov	%cr3, %r13
	mov	%r13, %gs:CPU_KPTI_TR_CR3
#endif
	mov	%gs:CPU_KPTI_KCR3, %r13
	mov	%r13, %cr3
	mov	%gs:CPU_KPTI_R13, %r13
	jmp	_brand_sys_sysenter_post_swapgs
	SET_SIZE(tr_brand_sys_sysenter)

#define	MK_SYSCALL_INT_TRAMPOLINE(isr)		\
	ENTRY_NP(tr_##isr);			\
	swapgs;					\
	mov	%r13, %gs:CPU_KPTI_R13;		\
	SET_KERNEL_CR3(%r13);			\
	mov	%gs:CPU_THREAD, %r13;		\
	mov	T_STACK(%r13), %r13;		\
	addq	$REGSIZE+MINFRAME, %r13;	\
	mov	%r13, %rsp;			\
	pushq	%gs:CPU_KPTI_SS;		\
	pushq	%gs:CPU_KPTI_RSP;		\
	pushq	%gs:CPU_KPTI_RFLAGS;		\
	pushq	%gs:CPU_KPTI_CS;		\
	pushq	%gs:CPU_KPTI_RIP;		\
	mov	%gs:CPU_KPTI_R13, %r13;		\
	swapgs;					\
	jmp	isr;				\
	SET_SIZE(tr_##isr)

	MK_SYSCALL_INT_TRAMPOLINE(brand_sys_syscall_int)
	MK_SYSCALL_INT_TRAMPOLINE(sys_syscall_int)

	/*
	 * Interrupt/trap return trampolines
	 */

.global	tr_intr_ret_start
tr_intr_ret_start:

	ENTRY_NP(tr_iret_auto)
	cmpq	$1, kpti_enable
	jne	tr_iret_kernel
	cmpw	$KCS_SEL, T_FRAMERET_CS(%rsp)
	je	tr_iret_kernel
	jmp	tr_iret_user
	SET_SIZE(tr_iret_auto)

	ENTRY_NP(tr_iret_kernel)
	/*
	 * Yes, this does nothing extra. But this way we know if we see iret
	 * elsewhere, then we've failed to properly consider trampolines there.
	 */
	iretq
	SET_SIZE(tr_iret_kernel)

	ENTRY_NP(tr_iret_user)
#if DEBUG
	/*
	 * Panic if we find CR0.TS set. We're still on the kernel stack and
	 * %cr3, but we do need to swap back to the kernel gs. (We don't worry
	 * about swapgs speculation here.)
	 */
	pushq	%rax
	mov	%cr0, %rax
	testq	$CR0_TS, %rax
	jz	1f
	swapgs
	popq	%rax
	leaq	_bad_ts_panic_msg(%rip), %rdi
	xorl	%eax, %eax
	pushq	%rbp
	movq	%rsp, %rbp
	call	panic
1:
	popq	%rax
#endif

	cmpq	$1, kpti_enable
	jne	1f

	/*
	 * KPTI enabled: we're on the user gsbase at this point, so we
	 * need to swap back so we can pivot stacks.
	 *
	 * The swapgs lfence mitigation is probably not needed here
	 * since a mis-speculation of the above branch would imply KPTI
	 * is disabled, but we'll do so anyway.
	 */
	swapgs
	lfence
	mov	%r13, %gs:CPU_KPTI_R13
	PIVOT_KPTI_STK(%r13)
	SET_USER_CR3(%r13)
	mov	%gs:CPU_KPTI_R13, %r13
	/* Zero these to make sure they didn't leak from a kernel trap. */
	movq	$0, %gs:CPU_KPTI_R13
	movq	$0, %gs:CPU_KPTI_R14
	/* And back to user gsbase again. */
	swapgs
1:
	iretq
	SET_SIZE(tr_iret_user)

	/*
	 * This special return trampoline is for KDI's use only (with kmdb).
	 *
	 * KDI/kmdb do not use swapgs -- they directly write the GSBASE MSR
	 * instead. This trampoline runs after GSBASE has already been changed
	 * back to the userland value (so we can't use %gs).
	 *
	 * Instead, the caller gives us a pointer to the kpti_dbg frame in %r13.
	 * The KPTI_R13 member in the kpti_dbg has already been set to what the
	 * real %r13 should be before we IRET.
	 *
	 * Additionally, KDI keeps a copy of the incoming %cr3 value when it
	 * took an interrupt, and has put that back in the kpti_dbg area for us
	 * to use, so we don't do any sniffing of %cs here. This is important
	 * so that debugging code that changes %cr3 is possible.
	 */
	ENTRY_NP(tr_iret_kdi)
	movq	%r14, KPTI_R14(%r13)	/* %r14 has to be preserved by us */

	movq	%rsp, %r14	/* original %rsp is pointing at IRET frame */
	leaq	KPTI_TOP(%r13), %rsp
	pushq	T_FRAMERET_SS(%r14)
	pushq	T_FRAMERET_RSP(%r14)
	pushq	T_FRAMERET_RFLAGS(%r14)
	pushq	T_FRAMERET_CS(%r14)
	pushq	T_FRAMERET_RIP(%r14)

	movq	KPTI_TR_CR3(%r13), %r14
	movq	%r14, %cr3

	movq	KPTI_R14(%r13), %r14
	movq	KPTI_R13(%r13), %r13	/* preserved by our caller */

	iretq
	SET_SIZE(tr_iret_kdi)

.global	tr_intr_ret_end
tr_intr_ret_end:

	/*
	 * Interrupt/trap entry trampolines
	 */

	/* CPU pushed an error code, and ISR wants one */
#define	MK_INTR_TRAMPOLINE(isr)			\
	ENTRY_NP(tr_##isr);			\
	INTERRUPT_TRAMPOLINE;			\
	jmp	isr;				\
	SET_SIZE(tr_##isr)

	/* CPU didn't push an error code, and ISR doesn't want one */
#define	MK_INTR_TRAMPOLINE_NOERR(isr)		\
	ENTRY_NP(tr_##isr);			\
	push	$0;				\
	INTERRUPT_TRAMPOLINE_NOERR;		\
	jmp	isr;				\
	SET_SIZE(tr_##isr)

	/* CPU pushed an error code, and ISR wants one */
#define	MK_DBG_INTR_TRAMPOLINE(isr)	\
	ENTRY_NP(tr_##isr);			\
	DBG_INTERRUPT_TRAMPOLINE;		\
	jmp	isr;				\
	SET_SIZE(tr_##isr)

	/* CPU didn't push an error code, and ISR doesn't want one */
#define	MK_DBG_INTR_TRAMPOLINE_NOERR(isr)	\
	ENTRY_NP(tr_##isr);			\
	push	$0;				\
	DBG_INTERRUPT_TRAMPOLINE_NOERR;		\
	jmp	isr;				\
	SET_SIZE(tr_##isr)


	MK_INTR_TRAMPOLINE_NOERR(div0trap)
	MK_DBG_INTR_TRAMPOLINE_NOERR(dbgtrap)
	MK_DBG_INTR_TRAMPOLINE_NOERR(brktrap)
	MK_INTR_TRAMPOLINE_NOERR(ovflotrap)
	MK_INTR_TRAMPOLINE_NOERR(boundstrap)
	MK_INTR_TRAMPOLINE_NOERR(invoptrap)
	MK_INTR_TRAMPOLINE_NOERR(ndptrap)
	MK_INTR_TRAMPOLINE(invtsstrap)
	MK_DBG_INTR_TRAMPOLINE(segnptrap)
	MK_DBG_INTR_TRAMPOLINE(stktrap)
	MK_DBG_INTR_TRAMPOLINE(gptrap)
	MK_DBG_INTR_TRAMPOLINE(pftrap)
	MK_INTR_TRAMPOLINE_NOERR(resvtrap)
	MK_INTR_TRAMPOLINE_NOERR(ndperr)
	MK_INTR_TRAMPOLINE(achktrap)
	MK_INTR_TRAMPOLINE_NOERR(xmtrap)
	MK_INTR_TRAMPOLINE_NOERR(invaltrap)
	MK_INTR_TRAMPOLINE_NOERR(fasttrap)
	MK_INTR_TRAMPOLINE_NOERR(dtrace_ret)

	/*
	 * These are special because they can interrupt other traps, and
	 * each other. We don't need to pivot their stacks, because they have
	 * dedicated IST stack space, but we need to change %cr3.
	 */
	ENTRY_NP(tr_nmiint)
	pushq	%r13
	mov	kpti_safe_cr3, %r13
	mov	%r13, %cr3
	popq	%r13
	jmp	nmiint
	SET_SIZE(tr_nmiint)

#if !defined(__xpv)
	ENTRY_NP(tr_syserrtrap)
	/*
	 * If we got here we should always have a zero error code pushed.
	 * The INT $0x8 instr doesn't seem to push one, though, which we use
	 * as an emergency panic in the other trampolines. So adjust things
	 * here.
	 */
	cmpq	$0, (%rsp)
	je	1f
	pushq	$0
1:
	pushq	%r13
	mov	kpti_safe_cr3, %r13
	mov	%r13, %cr3
	popq	%r13
	jmp	syserrtrap
	SET_SIZE(tr_syserrtrap)
#endif

	ENTRY_NP(tr_mcetrap)
	pushq	%r13
	mov	kpti_safe_cr3, %r13
	mov	%r13, %cr3
	popq	%r13
	jmp	mcetrap
	SET_SIZE(tr_mcetrap)

	/*
	 * Interrupts start at 32
	 */
#define MKIVCT(n)			\
	ENTRY_NP(tr_ivct##n)		\
	push	$0;			\
	INTERRUPT_TRAMPOLINE;		\
	push	$n - 0x20;		\
	jmp	cmnint;			\
	SET_SIZE(tr_ivct##n)

	MKIVCT(32);	MKIVCT(33);	MKIVCT(34);	MKIVCT(35);
	MKIVCT(36);	MKIVCT(37);	MKIVCT(38);	MKIVCT(39);
	MKIVCT(40);	MKIVCT(41);	MKIVCT(42);	MKIVCT(43);
	MKIVCT(44);	MKIVCT(45);	MKIVCT(46);	MKIVCT(47);
	MKIVCT(48);	MKIVCT(49);	MKIVCT(50);	MKIVCT(51);
	MKIVCT(52);	MKIVCT(53);	MKIVCT(54);	MKIVCT(55);
	MKIVCT(56);	MKIVCT(57);	MKIVCT(58);	MKIVCT(59);
	MKIVCT(60);	MKIVCT(61);	MKIVCT(62);	MKIVCT(63);
	MKIVCT(64);	MKIVCT(65);	MKIVCT(66);	MKIVCT(67);
	MKIVCT(68);	MKIVCT(69);	MKIVCT(70);	MKIVCT(71);
	MKIVCT(72);	MKIVCT(73);	MKIVCT(74);	MKIVCT(75);
	MKIVCT(76);	MKIVCT(77);	MKIVCT(78);	MKIVCT(79);
	MKIVCT(80);	MKIVCT(81);	MKIVCT(82);	MKIVCT(83);
	MKIVCT(84);	MKIVCT(85);	MKIVCT(86);	MKIVCT(87);
	MKIVCT(88);	MKIVCT(89);	MKIVCT(90);	MKIVCT(91);
	MKIVCT(92);	MKIVCT(93);	MKIVCT(94);	MKIVCT(95);
	MKIVCT(96);	MKIVCT(97);	MKIVCT(98);	MKIVCT(99);
	MKIVCT(100);	MKIVCT(101);	MKIVCT(102);	MKIVCT(103);
	MKIVCT(104);	MKIVCT(105);	MKIVCT(106);	MKIVCT(107);
	MKIVCT(108);	MKIVCT(109);	MKIVCT(110);	MKIVCT(111);
	MKIVCT(112);	MKIVCT(113);	MKIVCT(114);	MKIVCT(115);
	MKIVCT(116);	MKIVCT(117);	MKIVCT(118);	MKIVCT(119);
	MKIVCT(120);	MKIVCT(121);	MKIVCT(122);	MKIVCT(123);
	MKIVCT(124);	MKIVCT(125);	MKIVCT(126);	MKIVCT(127);
	MKIVCT(128);	MKIVCT(129);	MKIVCT(130);	MKIVCT(131);
	MKIVCT(132);	MKIVCT(133);	MKIVCT(134);	MKIVCT(135);
	MKIVCT(136);	MKIVCT(137);	MKIVCT(138);	MKIVCT(139);
	MKIVCT(140);	MKIVCT(141);	MKIVCT(142);	MKIVCT(143);
	MKIVCT(144);	MKIVCT(145);	MKIVCT(146);	MKIVCT(147);
	MKIVCT(148);	MKIVCT(149);	MKIVCT(150);	MKIVCT(151);
	MKIVCT(152);	MKIVCT(153);	MKIVCT(154);	MKIVCT(155);
	MKIVCT(156);	MKIVCT(157);	MKIVCT(158);	MKIVCT(159);
	MKIVCT(160);	MKIVCT(161);	MKIVCT(162);	MKIVCT(163);
	MKIVCT(164);	MKIVCT(165);	MKIVCT(166);	MKIVCT(167);
	MKIVCT(168);	MKIVCT(169);	MKIVCT(170);	MKIVCT(171);
	MKIVCT(172);	MKIVCT(173);	MKIVCT(174);	MKIVCT(175);
	MKIVCT(176);	MKIVCT(177);	MKIVCT(178);	MKIVCT(179);
	MKIVCT(180);	MKIVCT(181);	MKIVCT(182);	MKIVCT(183);
	MKIVCT(184);	MKIVCT(185);	MKIVCT(186);	MKIVCT(187);
	MKIVCT(188);	MKIVCT(189);	MKIVCT(190);	MKIVCT(191);
	MKIVCT(192);	MKIVCT(193);	MKIVCT(194);	MKIVCT(195);
	MKIVCT(196);	MKIVCT(197);	MKIVCT(198);	MKIVCT(199);
	MKIVCT(200);	MKIVCT(201);	MKIVCT(202);	MKIVCT(203);
	MKIVCT(204);	MKIVCT(205);	MKIVCT(206);	MKIVCT(207);
	MKIVCT(208);	MKIVCT(209);	MKIVCT(210);	MKIVCT(211);
	MKIVCT(212);	MKIVCT(213);	MKIVCT(214);	MKIVCT(215);
	MKIVCT(216);	MKIVCT(217);	MKIVCT(218);	MKIVCT(219);
	MKIVCT(220);	MKIVCT(221);	MKIVCT(222);	MKIVCT(223);
	MKIVCT(224);	MKIVCT(225);	MKIVCT(226);	MKIVCT(227);
	MKIVCT(228);	MKIVCT(229);	MKIVCT(230);	MKIVCT(231);
	MKIVCT(232);	MKIVCT(233);	MKIVCT(234);	MKIVCT(235);
	MKIVCT(236);	MKIVCT(237);	MKIVCT(238);	MKIVCT(239);
	MKIVCT(240);	MKIVCT(241);	MKIVCT(242);	MKIVCT(243);
	MKIVCT(244);	MKIVCT(245);	MKIVCT(246);	MKIVCT(247);
	MKIVCT(248);	MKIVCT(249);	MKIVCT(250);	MKIVCT(251);
	MKIVCT(252);	MKIVCT(253);	MKIVCT(254);	MKIVCT(255);

	/*
	 * We're PCIDE, but we don't have INVPCID.  The only way to invalidate a
	 * PCID other than the current one, then, is to load its cr3 then
	 * invlpg.  But loading kf_user_cr3 means we can longer access our
	 * caller's text mapping (or indeed, its stack).  So this little helper
	 * has to live within our trampoline text region.
	 *
	 * Called as tr_mmu_flush_user_range(addr, len, pgsz, cr3)
	 */
	ENTRY_NP(tr_mmu_flush_user_range)
	push	%rbx
	/* When we read cr3, it never has the NOINVL bit set. */
	mov	%cr3, %rax
	movq	$CR3_NOINVL_BIT, %rbx
	orq	%rbx, %rax

	mov	%rcx, %cr3
	add	%rdi, %rsi
.align	ASM_ENTRY_ALIGN
1:
	invlpg	(%rdi)
	add	%rdx, %rdi
	cmp	%rsi, %rdi
	jb	1b
	mov	%rax, %cr3
	pop	%rbx
	retq
	SET_SIZE(tr_mmu_flush_user_range)

.align MMU_PAGESIZE
.global kpti_tramp_end
kpti_tramp_end:
	nop
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved.
 */
/*
 * Copyright 2020 Joyent, Inc.
 * Copyright 2020 OmniOS Community Edition (OmniOSce) Association.
 */

/*	Copyright (c) 1990, 1991 UNIX System Laboratories, Inc.	*/
/*	Copyright (c) 1984, 1986, 1987, 1988, 1989, 1990 AT&T	*/
/*	  All Rights Reserved					*/

/*	Copyright (c) 1987, 1988 Microsoft Corporation		*/
/*	  All Rights Reserved					*/


#include <sys/asm_linkage.h>
#include <sys/asm_misc.h>
#include <sys/regset.h>
#include <sys/privregs.h>
#include <sys/psw.h>
#include <sys/reboot.h>
#include <sys/machparam.h>

#include <sys/segments.h>
#include <sys/pcb.h>
#include <sys/trap.h>
#include <sys/ftrace.h>
#include <sys/traptrace.h>
#include <sys/clock.h>
#include <sys/cmn_err.h>
#include <sys/pit.h>
#include <sys/panic.h>

#if defined(__xpv)
#include <sys/hypervisor.h>
#endif

#include "assym.h"

/*
 * Our assumptions:
 *	- We are running in protected-paged mode.
 *	- Interrupts are disabled.
 *	- The GDT and IDT are the callers; we need our copies.
 *	- The kernel's text, initialized data and bss are mapped.
 *
 * Our actions:
 *	- Save arguments
 *	- Initialize our stack pointer to the thread 0 stack (t0stack)
 *	  and leave room for a phony "struct regs".
 *	- Our GDT and IDT need to get munged.
 *	- Since we are using the boot's GDT descriptors, we need
 *	  to copy them into our GDT before we switch to ours.
 *	- We start using our GDT by loading correct values in the
 *	  selector registers (cs=KCS_SEL, ds=es=ss=KDS_SEL, fs=KFS_SEL,
 *	  gs=KGS_SEL).
 *	- The default LDT entry for syscall is set.
 *	- We load the default LDT into the hardware LDT register.
 *	- We load the default TSS into the hardware task register.
 *	- Check for cpu type, i.e. 486 vs. P5 vs. P6 etc.
 *	- mlsetup(%esp) gets called.
 *	- We change our appearance to look like the real thread 0.
 *	  (NOTE: making ourselves to be a real thread may be a noop)
 *	- main() gets called.  (NOTE: main() never returns).
 *
 * NOW, the real code!
 */
	/*
	 * The very first thing in the kernel's text segment must be a jump
	 * to the os/fakebop.c startup code.
	 */
	.text
	jmp     _start

	/*
	 * Globals:
	 */
	.globl	_locore_start
	.globl	mlsetup
	.globl	main
	.globl	panic
	.globl	t0stack
	.globl	t0
	.globl	sysp
	.globl	edata

	/*
	 * call back into boot - sysp (bootsvcs.h) and bootops (bootconf.h)
	 */
	.globl	bootops
	.globl	bootopsp

	/*
	 * NOTE: t0stack should be the first thing in the data section so that
	 * if it ever overflows, it will fault on the last kernel text page.
	 */
	.data
	.comm	t0stack, DEFAULTSTKSZ, 32
	.comm	t0, 4094, 32


	/*
	 * kobj_init() vectors us back to here with (note) a slightly different
	 * set of arguments than _start is given (see lint prototypes above).
	 *
	 * XXX	Make this less vile, please.
	 */
	ENTRY_NP(_locore_start)

	/*
	 * %rdi = boot services (should die someday)
	 * %rdx = bootops
	 * end
	 */

	leaq	edata(%rip), %rbp	/* reference edata for ksyms */
	movq	$0, (%rbp)		/* limit stack back trace */

	/*
	 * Initialize our stack pointer to the thread 0 stack (t0stack)
	 * and leave room for a "struct regs" for lwp0.  Note that the
	 * stack doesn't actually align to a 16-byte boundary until just
	 * before we call mlsetup because we want to use %rsp to point at
	 * our regs structure.
	 */
	leaq	t0stack(%rip), %rsp
	addq	$_CONST(DEFAULTSTKSZ - REGSIZE), %rsp
#if (REGSIZE & 15) == 0
	subq	$8, %rsp
#endif
	/*
	 * Save call back for special x86 boot services vector
	 */
	movq	%rdi, sysp(%rip)

	movq	%rdx, bootops(%rip)		/* save bootops */
	movq	$bootops, bootopsp(%rip)

	/*
	 * Save arguments and flags, if only for debugging ..
	 */
	movq	%rdi, REGOFF_RDI(%rsp)
	movq	%rsi, REGOFF_RSI(%rsp)
	movq	%rdx, REGOFF_RDX(%rsp)
	movq	%rcx, REGOFF_RCX(%rsp)
	movq	%r8, REGOFF_R8(%rsp)
	movq	%r9, REGOFF_R9(%rsp)
	pushf
	popq	%r11
	movq	%r11, REGOFF_RFL(%rsp)

#if !defined(__xpv)
	/*
	 * Enable write protect and alignment check faults.
	 */
	movq	%cr0, %rax
	orq	$_CONST(CR0_WP|CR0_AM), %rax
	andq	$_BITNOT(CR0_WT|CR0_CE), %rax
	movq	%rax, %cr0
#endif	/* __xpv */

	/*
	 * mlsetup() gets called with a struct regs as argument, while
	 * main takes no args and should never return.
	 */
	xorl	%ebp, %ebp
	movq	%rsp, %rdi
	pushq	%rbp
	/* (stack pointer now aligned on 16-byte boundary right here) */
	movq	%rsp, %rbp
	call	mlsetup
	call	main
	/* NOTREACHED */
	leaq	__return_from_main(%rip), %rdi
	xorl	%eax, %eax
	call	panic
	SET_SIZE(_locore_start)

__return_from_main:
	.string	"main() returned"
__unsupported_cpu:
	.string	"486 style cpu detected - no longer supported!"

#if defined(DEBUG)
_no_pending_updates:
	.string	"locore.s:%d lwp_rtt(lwp %p) but pcb_rupdate != 1"
#endif

/*
 *  For stack layout, see privregs.h
 *  When cmntrap gets called, the error code and trap number have been pushed.
 *  When cmntrap_pushed gets called, the entire struct regs has been pushed.
 */

	.globl	trap		/* C handler called below */

	ENTRY_NP2(cmntrap, _cmntrap)

	INTR_PUSH

	ALTENTRY(cmntrap_pushed)

	movq	%rsp, %rbp

	/*
	 * - if this is a #pf i.e. T_PGFLT, %r15 is live
	 *   and contains the faulting address i.e. a copy of %cr2
	 *
	 * - if this is a #db i.e. T_SGLSTP, %r15 is live
	 *   and contains the value of %db6
	 */

	TRACE_PTR(%rdi, %rbx, %ebx, %rcx, $TT_TRAP) /* Uses labels 8 and 9 */
	TRACE_REGS(%rdi, %rsp, %rbx, %rcx)	/* Uses label 9 */
	TRACE_STAMP(%rdi)		/* Clobbers %eax, %edx, uses 9 */

	/*
	 * We must first check if DTrace has set its NOFAULT bit.  This
	 * regrettably must happen before the trap stack is recorded, because
	 * this requires a call to getpcstack() and may induce recursion if an
	 * fbt::getpcstack: enabling is inducing the bad load.
	 */
	movl	%gs:CPU_ID, %eax
	shlq	$CPU_CORE_SHIFT, %rax
	leaq	cpu_core(%rip), %r8
	addq	%r8, %rax
	movw	CPUC_DTRACE_FLAGS(%rax), %cx
	testw	$CPU_DTRACE_NOFAULT, %cx
	jnz	.dtrace_induced

	TRACE_STACK(%rdi)

	movq	%rbp, %rdi
	movq	%r15, %rsi
	movl	%gs:CPU_ID, %edx

	/*
	 * We know that this isn't a DTrace non-faulting load; we can now safely
	 * reenable interrupts.  (In the case of pagefaults, we enter through an
	 * interrupt gate.)
	 */
	ENABLE_INTR_FLAGS

	call	trap		/* trap(rp, addr, cpuid) handles all traps */
	jmp	_sys_rtt

.dtrace_induced:
	cmpw	$KCS_SEL, REGOFF_CS(%rbp)	/* test CS for user-mode trap */
	jne	3f				/* if from user, panic */

	cmpl	$T_PGFLT, REGOFF_TRAPNO(%rbp)
	je	1f

	cmpl	$T_GPFLT, REGOFF_TRAPNO(%rbp)
	je	0f

	cmpl	$T_ILLINST, REGOFF_TRAPNO(%rbp)
	je	0f

	cmpl	$T_ZERODIV, REGOFF_TRAPNO(%rbp)
	jne	4f				/* if not PF/GP/UD/DE, panic */

	orw	$CPU_DTRACE_DIVZERO, %cx
	movw	%cx, CPUC_DTRACE_FLAGS(%rax)
	jmp	2f

	/*
	 * If we've taken a GPF, we don't (unfortunately) have the address that
	 * induced the fault.  So instead of setting the fault to BADADDR,
	 * we'll set the fault to ILLOP.
	 */
0:
	orw	$CPU_DTRACE_ILLOP, %cx
	movw	%cx, CPUC_DTRACE_FLAGS(%rax)
	jmp	2f
1:
	orw	$CPU_DTRACE_BADADDR, %cx
	movw	%cx, CPUC_DTRACE_FLAGS(%rax)	/* set fault to bad addr */
	movq	%r15, CPUC_DTRACE_ILLVAL(%rax)
					    /* fault addr is illegal value */
2:
	movq	REGOFF_RIP(%rbp), %rdi
	movq	%rdi, %r12
	call	dtrace_instr_size
	addq	%rax, %r12
	movq	%r12, REGOFF_RIP(%rbp)
	INTR_POP
	call	x86_md_clear
	jmp	tr_iret_auto
	/*NOTREACHED*/
3:
	leaq	dtrace_badflags(%rip), %rdi
	xorl	%eax, %eax
	call	panic
4:
	leaq	dtrace_badtrap(%rip), %rdi
	xorl	%eax, %eax
	call	panic
	SET_SIZE(cmntrap_pushed)
	SET_SIZE(cmntrap)
	SET_SIZE(_cmntrap)

/*
 * Declare a uintptr_t which has the size of _cmntrap to enable stack
 * traceback code to know when a regs structure is on the stack.
 */
	.globl	_cmntrap_size
	.align	CLONGSIZE
_cmntrap_size:
	.NWORD	. - _cmntrap
	.type	_cmntrap_size, @object

dtrace_badflags:
	.string "bad DTrace flags"

dtrace_badtrap:
	.string "bad DTrace trap"

	.globl	trap		/* C handler called below */

	ENTRY_NP(cmninttrap)

	INTR_PUSH
	INTGATE_INIT_KERNEL_FLAGS

	TRACE_PTR(%rdi, %rbx, %ebx, %rcx, $TT_TRAP) /* Uses labels 8 and 9 */
	TRACE_REGS(%rdi, %rsp, %rbx, %rcx)	/* Uses label 9 */
	TRACE_STAMP(%rdi)		/* Clobbers %eax, %edx, uses 9 */

	movq	%rsp, %rbp

	movl	%gs:CPU_ID, %edx
	xorl	%esi, %esi
	movq	%rsp, %rdi
	call	trap		/* trap(rp, addr, cpuid) handles all traps */
	jmp	_sys_rtt
	SET_SIZE(cmninttrap)

#if !defined(__xpv)
	/*
	 * Handle traps early in boot. Just revectors into C quickly as
	 * these are always fatal errors.
	 *
	 * Adjust %rsp to get same stack layout as in 32bit mode for bop_trap().
	 */
	ENTRY(bop_trap_handler)
	movq	%rsp, %rdi
	sub	$8, %rsp
	call	bop_trap
	SET_SIZE(bop_trap_handler)
#endif

	.globl	dtrace_user_probe

	ENTRY_NP(dtrace_trap)

	INTR_PUSH

	TRACE_PTR(%rdi, %rbx, %ebx, %rcx, $TT_TRAP) /* Uses labels 8 and 9 */
	TRACE_REGS(%rdi, %rsp, %rbx, %rcx)	/* Uses label 9 */
	TRACE_STAMP(%rdi)		/* Clobbers %eax, %edx, uses 9 */

	movq	%rsp, %rbp

	movl	%gs:CPU_ID, %edx
#if defined(__xpv)
	movq	%gs:CPU_VCPU_INFO, %rsi
	movq	VCPU_INFO_ARCH_CR2(%rsi), %rsi
#else
	movq	%cr2, %rsi
#endif
	movq	%rsp, %rdi

	ENABLE_INTR_FLAGS

	call	dtrace_user_probe /* dtrace_user_probe(rp, addr, cpuid) */
	jmp	_sys_rtt

	SET_SIZE(dtrace_trap)

/*
 * Return from _sys_trap routine.
 */

	ENTRY_NP(lwp_rtt_initial)
	movq	%gs:CPU_THREAD, %r15
	movq	T_STACK(%r15), %rsp	/* switch to the thread stack */
	movq	%rsp, %rbp
	call	__dtrace_probe___proc_start
	jmp	_lwp_rtt

	ENTRY_NP(lwp_rtt)

	/*
	 * r14	lwp
	 * rdx	lwp->lwp_procp
	 * r15	curthread
	 */

	movq	%gs:CPU_THREAD, %r15
	movq	T_STACK(%r15), %rsp	/* switch to the thread stack */
	movq	%rsp, %rbp
_lwp_rtt:
	call	__dtrace_probe___proc_lwp__start
	movq	%gs:CPU_LWP, %r14
	movq	LWP_PROCP(%r14), %rdx

	/*
	 * XX64	Is the stack misaligned correctly at this point?
	 *	If not, we need to do a push before calling anything ..
	 */

#if defined(DEBUG)
	/*
	 * If we were to run lwp_savectx at this point -without-
	 * pcb_rupdate being set to 1, we'd end up sampling the hardware
	 * state left by the previous running lwp, rather than setting
	 * the values requested by the lwp creator.  Bad.
	 */
	testb	$0x1, PCB_RUPDATE(%r14)
	jne	1f
	leaq	_no_pending_updates(%rip), %rdi
	movl	$__LINE__, %esi
	movq	%r14, %rdx
	xorl	%eax, %eax
	call	panic
1:
#endif

	/*
	 * If agent lwp, clear %fs and %gs
	 */
	cmpq	%r15, P_AGENTTP(%rdx)
	jne	1f
	xorl	%ecx, %ecx
	movq	%rcx, REGOFF_FS(%rsp)
	movq	%rcx, REGOFF_GS(%rsp)
	movw	%cx, LWP_PCB_FS(%r14)
	movw	%cx, LWP_PCB_GS(%r14)
1:
	call	dtrace_systrace_rtt
	movq	REGOFF_RDX(%rsp), %rsi
	movq	REGOFF_RAX(%rsp), %rdi
	call	post_syscall		/* post_syscall(rval1, rval2) */

	/*
	 * XXX - may want a fast path that avoids sys_rtt_common in the
	 * most common case.
	 */
	ALTENTRY(_sys_rtt)
	CLI(%rax)			/* disable interrupts */
	ALTENTRY(_sys_rtt_ints_disabled)
	movq	%rsp, %rdi		/* pass rp to sys_rtt_common */
	call	sys_rtt_common		/* do common sys_rtt tasks */
	testq	%rax, %rax		/* returning to userland? */
	jz	sr_sup

	/*
	 * Return to user
	 */
	ASSERT_UPCALL_MASK_IS_SET
	cmpw	$UCS_SEL, REGOFF_CS(%rsp) /* test for native (64-bit) lwp? */
	je	sys_rtt_syscall

	/*
	 * Return to 32-bit userland
	 */
	ALTENTRY(sys_rtt_syscall32)
	USER32_POP
	call	x86_md_clear
	jmp	tr_iret_user
	/*NOTREACHED*/

	ALTENTRY(sys_rtt_syscall)
	/*
	 * Return to 64-bit userland
	 */
	USER_POP
	ALTENTRY(nopop_sys_rtt_syscall)
	call	x86_md_clear
	jmp	tr_iret_user
	/*NOTREACHED*/
	SET_SIZE(nopop_sys_rtt_syscall)

	/*
	 * Return to supervisor
	 * NOTE: to make the check in trap() that tests if we are executing
	 * segment register fixup/restore code work properly, sr_sup MUST be
	 * after _sys_rtt .
	 */
	ALTENTRY(sr_sup)
	/*
	 * Restore regs before doing iretq to kernel mode
	 */
	INTR_POP
	jmp	tr_iret_kernel
	.globl	_sys_rtt_end
_sys_rtt_end:
	/*NOTREACHED*/
	SET_SIZE(sr_sup)
	SET_SIZE(_sys_rtt_end)
	SET_SIZE(lwp_rtt)
	SET_SIZE(lwp_rtt_initial)
	SET_SIZE(_sys_rtt_ints_disabled)
	SET_SIZE(_sys_rtt)
	SET_SIZE(sys_rtt_syscall)
	SET_SIZE(sys_rtt_syscall32)

	/*
	 * XX64 quick and dirty port from the i386 version. Since we
	 * believe the amd64 tsc is more reliable, could this code be
	 * simpler?
	 */
	ENTRY_NP(freq_tsc_pit)
	pushq	%rbp
	movq	%rsp, %rbp
	movq	%rdi, %r9	/* save pit_counter */
	pushq	%rbx

/ We have a TSC, but we have no way in general to know how reliable it is.
/ Usually a marginal TSC behaves appropriately unless not enough time
/ elapses between reads. A reliable TSC can be read as often and as rapidly
/ as desired. The simplistic approach of reading the TSC counter and
/ correlating to the PIT counter cannot be naively followed. Instead estimates
/ have to be taken to successively refine a guess at the speed of the cpu
/ and then the TSC and PIT counter are correlated. In practice very rarely
/ is more than one quick loop required for an estimate. Measures have to be
/ taken to prevent the PIT counter from wrapping beyond its resolution and for
/ measuring the clock rate of very fast processors.
/
/ The following constant can be tuned. It should be such that the loop does
/ not take too many nor too few PIT counts to execute. If this value is too
/ large, then on slow machines the loop will take a long time, or the PIT
/ counter may even wrap. If this value is too small, then on fast machines
/ the PIT counter may count so few ticks that the resolution of the PIT
/ itself causes a bad guess. Because this code is used in machines with
/ marginal TSC's and/or IO, if this value is too small on those, it may
/ cause the calculated cpu frequency to vary slightly from boot to boot.
/
/ In all cases even if this constant is set inappropriately, the algorithm
/ will still work and the caller should be able to handle variances in the
/ calculation of cpu frequency, but the calculation will be inefficient and
/ take a disproportionate amount of time relative to a well selected value.
/ As the slowest supported cpu becomes faster, this constant should be
/ carefully increased.

	movl	$0x8000, %ecx

	/ to make sure the instruction cache has been warmed
	clc

	jmp	freq_tsc_loop

/ The following block of code up to and including the latching of the PIT
/ counter after freq_tsc_perf_loop is very critical and very carefully
/ written, it should only be modified with great care. freq_tsc_loop to
/ freq_tsc_perf_loop fits exactly in 16 bytes as do the instructions in
/ freq_tsc_perf_loop up to the unlatching of the PIT counter.

	.align	32
freq_tsc_loop:
	/ save the loop count in %ebx
	movl	%ecx, %ebx

	/ initialize the PIT counter and start a count down
	movb	$PIT_LOADMODE, %al
	outb	$PITCTL_PORT
	movb	$0xff, %al
	outb	$PITCTR0_PORT
	outb	$PITCTR0_PORT

	/ read the TSC and store the TS in %edi:%esi
	rdtsc
	movl	%eax, %esi

freq_tsc_perf_loop:
	movl	%edx, %edi
	movl	%eax, %esi
	movl	%edx, %edi
	loop	freq_tsc_perf_loop

	/ read the TSC and store the LSW in %ecx
	rdtsc
	movl	%eax, %ecx

	/ latch the PIT counter and status
	movb	$_CONST(PIT_READBACK|PIT_READBACKC0), %al
	outb	$PITCTL_PORT

	/ remember if the icache has been warmed
	setc	%ah

	/ read the PIT status
	inb	$PITCTR0_PORT
	shll	$8, %eax

	/ read PIT count
	inb	$PITCTR0_PORT
	shll	$8, %eax
	inb	$PITCTR0_PORT
	bswap	%eax

	/ check to see if the PIT count was loaded into the CE
	btw	$_CONST(PITSTAT_NULLCNT+8), %ax
	jc	freq_tsc_increase_count

	/ check to see if PIT counter wrapped
	btw	$_CONST(PITSTAT_OUTPUT+8), %ax
	jnc	freq_tsc_pit_did_not_wrap

	/ halve count
	shrl	$1, %ebx
	movl	%ebx, %ecx

	/ the instruction cache has been warmed
	stc

	jmp	freq_tsc_loop

freq_tsc_increase_count:
	shll	$1, %ebx
	jc	freq_tsc_too_fast

	movl	%ebx, %ecx

	/ the instruction cache has been warmed
	stc

	jmp	freq_tsc_loop

freq_tsc_pit_did_not_wrap:
	roll	$16, %eax

	cmpw	$0x2000, %ax
	notw	%ax
	jb	freq_tsc_sufficient_duration

freq_tsc_calculate:
	/ in mode 0, the PIT loads the count into the CE on the first CLK pulse,
	/ then on the second CLK pulse the CE is decremented, therefore mode 0
	/ is really a (count + 1) counter, ugh
	xorl	%esi, %esi
	movw	%ax, %si
	incl	%esi

	movl	$0xf000, %eax
	mull	%ebx

	/ tuck away (target_pit_count * loop_count)
	movl	%edx, %ecx
	movl	%eax, %ebx

	movl	%esi, %eax
	movl	$0xffffffff, %edx
	mull	%edx

	addl	%esi, %eax
	adcl	$0, %edx

	cmpl	%ecx, %edx
	ja	freq_tsc_div_safe
	jb	freq_tsc_too_fast

	cmpl	%ebx, %eax
	jbe	freq_tsc_too_fast

freq_tsc_div_safe:
	movl	%ecx, %edx
	movl	%ebx, %eax

	movl	%esi, %ecx
	divl	%ecx

	movl	%eax, %ecx

	/ the instruction cache has been warmed
	stc

	jmp	freq_tsc_loop

freq_tsc_sufficient_duration:
	/ test to see if the icache has been warmed
	btl	$16, %eax
	jnc	freq_tsc_calculate

	/ recall mode 0 is a (count + 1) counter
	andl	$0xffff, %eax
	incl	%eax

	/ save the number of PIT counts
	movl	%eax, (%r9)

	/ calculate the number of TS's that elapsed
	movl	%ecx, %eax
	subl	%esi, %eax
	sbbl	%edi, %edx

	jmp	freq_tsc_end

freq_tsc_too_fast:
	/ return 0 as a 64 bit quantity
	xorl	%eax, %eax
	xorl	%edx, %edx

freq_tsc_end:
	shlq	$32, %rdx
	orq	%rdx, %rax

	popq	%rbx
	leaveq
	ret
	SET_SIZE(freq_tsc_pit)

/*
 * This file and its contents are supplied under the terms of the
 * Common Development and Distribution License ("CDDL"), version 1.0.
 * You may only use this file in accordance with the terms of version
 * 1.0 of the CDDL.
 *
 * A full copy of the text of the CDDL should have accompanied this
 * source.  A copy of the CDDL is also available via the Internet at
 * http://www.illumos.org/license/CDDL.
 */

/*
 * Copyright 2019 Joyent, Inc.
 */

#include <sys/asm_linkage.h>

/*
 * This ASM file contains various routines that are designed to flush
 * microarchitectural buffer state as part of dealing with the
 * microarchitectural data sampling (MDS) vulnerabilities.
 *
 * These are called from various points in the system ranging from interrupts,
 * before going idle, to returning from system calls. This means the following
 * is true about the state of the system:
 *
 *  o All register state is precious, we must not change register state upon
 *    entry or return from these functions.
 *
 *  o %ds is valid.
 *
 *  o %gs is arbitrary, it may be kernel or user. You cannot rely on it.
 *
 *  o Interrupts should be disabled by the caller.
 *
 *  o %cr3 is on the kernel-side and therefore we still have access to kernel
 *    text. In other words, we haven't switched back to the user page table.
 *
 *  o It is up to the caller to insure that a sufficient serializing instruction
 *    has been executed after this to make sure any pending speculations are
 *    captured. In general, this should be handled by the fact that callers of
 *    this are either going to change privilege levels or halt, which makes
 *    these operations safer.
 */

	/*
	 * By default, x86_md_clear is disabled until the system determines that
	 * it both needs MDS related mitigations and we have microcode that
	 * provides the needed functionality.
	 *
	 * The VERW instruction clobbers flags which is why it's important that
	 * we save and restore them here.
	 */
	ENTRY_NP(x86_md_clear)
	ret
	pushfq
	subq	$8, %rsp
	mov	%ds, (%rsp)
	verw	(%rsp)
	addq	$8, %rsp
	popfq
	ret
	SET_SIZE(x86_md_clear)
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved.
 */
/*
 * Copyright (c) 2010, Intel Corporation.
 * All rights reserved.
 *
 * Copyright 2019 Joyent, Inc.
 * Copyright 2020 OmniOS Community Edition (OmniOSce) Association.
 * Copyright 2023 Oxide Computer Co.
 */

#include <sys/asm_linkage.h>
#include <sys/asm_misc.h>
#include <sys/regset.h>
#include <sys/privregs.h>
#include <sys/x86_archext.h>

#include <sys/segments.h>
#include "assym.h"

/*
 *	Our assumptions:
 *		- We are running in real mode.
 *		- Interrupts are disabled.
 *		- Selectors are equal (cs == ds == ss) for all real mode code
 *		- The GDT, IDT, ktss and page directory has been built for us
 *
 *	Our actions:
 *	Start CPU:
 *		- We start using our GDT by loading correct values in the
 *		  selector registers (cs=KCS_SEL, ds=es=ss=KDS_SEL, fs=KFS_SEL,
 *		  gs=KGS_SEL).
 *		- We change over to using our IDT.
 *		- We load the default LDT into the hardware LDT register.
 *		- We load the default TSS into the hardware task register.
 *		- call mp_startup(void) indirectly through the T_PC
 *	Stop CPU:
 *		- Put CPU into halted state with interrupts disabled
 *
 */

	ENTRY_NP(real_mode_start_cpu)

	/*
	 * NOTE:  The GNU assembler automatically does the right thing to
	 *	  generate data size operand prefixes based on the code size
	 *	  generation mode (e.g. .code16, .code32, .code64) and as such
	 *	  prefixes need not be used on instructions EXCEPT in the case
	 *	  of address prefixes for code for which the reference is not
	 *	  automatically of the default operand size.
	 */
	.code16
	cli
	movw		%cs, %ax
	movw		%ax, %ds	/* load cs into ds */
	movw		%ax, %ss	/* and into ss */

	/*
	 * Helps in debugging by giving us the fault address.
	 *
	 * Remember to patch a hlt (0xf4) at cmntrap to get a good stack.
	 */
	movl		$0xffc, %esp
	movl		%cr0, %eax

	/*
	 * Enable protected-mode, write protect, and alignment mask
	 */
	orl		$(CR0_PE|CR0_WP|CR0_AM), %eax
	movl		%eax, %cr0

	/*
	 * Do a jmp immediately after writing to cr0 when enabling protected
	 * mode to clear the real mode prefetch queue (per Intel's docs)
	 */
	jmp		pestart

pestart:
	/*
	 * 16-bit protected mode is now active, so prepare to turn on long
	 * mode.
	 */

	/*
	 * Add any initial cr4 bits
	 */
	movl		%cr4, %eax
	addr32 orl	CR4OFF, %eax

	/*
	 * Enable PAE mode (CR4.PAE)
	 */
	orl		$CR4_PAE, %eax
	movl		%eax, %cr4

	/*
	 * Point cr3 to the 64-bit long mode page tables.
	 *
	 * Note that these MUST exist in 32-bit space, as we don't have
	 * a way to load %cr3 with a 64-bit base address for the page tables
	 * until the CPU is actually executing in 64-bit long mode.
	 */
	addr32 movl	CR3OFF, %eax
	movl		%eax, %cr3

	/*
	 * Set long mode enable in EFER (EFER.LME = 1)
	 */
	movl	$MSR_AMD_EFER, %ecx
	rdmsr
	orl	$AMD_EFER_LME, %eax
	wrmsr

	/*
	 * Finally, turn on paging (CR0.PG = 1) to activate long mode.
	 */
	movl	%cr0, %eax
	orl	$CR0_PG, %eax
	movl	%eax, %cr0

	/*
	 * The instruction after enabling paging in CR0 MUST be a branch.
	 */
	jmp	long_mode_active

long_mode_active:
	/*
	 * Long mode is now active but since we're still running with the
	 * original 16-bit CS we're actually in 16-bit compatability mode.
	 *
	 * We have to load an intermediate GDT and IDT here that we know are
	 * in 32-bit space before we can use the kernel's GDT and IDT, which
	 * may be in the 64-bit address space, and since we're in compatability
	 * mode, we only have access to 16 and 32-bit instructions at the
	 * moment.
	 */
	addr32 lgdtl	TEMPGDTOFF	/* load temporary GDT */
	addr32 lidtl	TEMPIDTOFF	/* load temporary IDT */

	/*
	 * Do a far transfer to 64-bit mode.  Set the CS selector to a 64-bit
	 * long mode selector (CS.L=1) in the temporary 32-bit GDT and jump
	 * to the real mode platter address of long_mode 64 as until the 64-bit
	 * CS is in place we don't have access to 64-bit instructions and thus
	 * can't reference a 64-bit %rip.
	 */
	pushl		$TEMP_CS64_SEL
	addr32 pushl	LM64OFF
	lretl

	.globl	long_mode_64
long_mode_64:
	.code64
	/*
	 * We are now running in long mode with a 64-bit CS (EFER.LMA=1,
	 * CS.L=1) so we now have access to 64-bit instructions.
	 *
	 * First, set the 64-bit GDT base.
	 */
	.globl	rm_platter_pa
	movl	rm_platter_pa, %eax
	lgdtq	GDTROFF(%rax)		/* load 64-bit GDT */

	/*
	 * Save the CPU number in %r11; get the value here since it's saved in
	 * the real mode platter.
	 */
	movl	CPUNOFF(%rax), %r11d

	/*
	 * Add rm_platter_pa to %rsp to point it to the same location as seen
	 * from 64-bit mode.
	 */
	addq	%rax, %rsp

	/*
	 * Now do an lretq to load CS with the appropriate selector for the
	 * kernel's 64-bit GDT and to start executing 64-bit setup code at the
	 * virtual address where boot originally loaded this code rather than
	 * the copy in the real mode platter's rm_code array as we've been
	 * doing so far.
	 */
	pushq	$KCS_SEL
	pushq	$kernel_cs_code
	lretq
	.globl real_mode_start_cpu_end
real_mode_start_cpu_end:
	nop

kernel_cs_code:
	/*
	 * Complete the balance of the setup we need to before executing
	 * 64-bit kernel code (namely init rsp, TSS, LGDT, FS and GS).
	 */
	.globl	rm_platter_va
	movq	rm_platter_va, %rax
	lidtq	IDTROFF(%rax)

	movw	$KDS_SEL, %ax
	movw	%ax, %ds
	movw	%ax, %es
	movw	%ax, %ss

	movw	$KTSS_SEL, %ax		/* setup kernel TSS */
	ltr	%ax

	xorw	%ax, %ax		/* clear LDTR */
	lldt	%ax

	/*
	 * Set GS to the address of the per-cpu structure as contained in
	 * cpu[cpu_number].
	 *
	 * Unfortunately there's no way to set the 64-bit gsbase with a mov,
	 * so we have to stuff the low 32 bits in %eax and the high 32 bits in
	 * %edx, then call wrmsr.
	 */
	leaq	cpu(%rip), %rdi
	movl	(%rdi, %r11, 8), %eax
	movl	4(%rdi, %r11, 8), %edx
	movl	$MSR_AMD_GSBASE, %ecx
	wrmsr

	/*
	 * Init FS and KernelGSBase.
	 *
	 * Based on code in mlsetup(), set them both to 8G (which shouldn't be
	 * valid until some 64-bit processes run); this will then cause an
	 * exception in any code that tries to index off them before they are
	 * properly setup.
	 */
	xorl	%eax, %eax		/* low 32 bits = 0 */
	movl	$2, %edx		/* high 32 bits = 2 */
	movl	$MSR_AMD_FSBASE, %ecx
	wrmsr

	movl	$MSR_AMD_KGSBASE, %ecx
	wrmsr

	/*
	 * Init %rsp to the exception stack set in tss_ist1 and create a legal
	 * AMD64 ABI stack frame
	 */
	movq	%gs:CPU_TSS, %rax
	movq	TSS_IST1(%rax), %rsp
	pushq	$0		/* null return address */
	pushq	$0		/* null frame pointer terminates stack trace */
	movq	%rsp, %rbp	/* stack aligned on 16-byte boundary */

	/*
	 * Get %cr0 into the state we (mostly) want, including turning on the
	 * caches.
	 */
	movq	%cr0, %rax
	andq    $~(CR0_CD|CR0_NW|CR0_TS|CR0_EM), %rax
	orq     $(CR0_MP|CR0_NE), %rax
	movq    %rax, %cr0		/* set machine status word */

	/*
	 * Before going any further, enable usage of page table NX bit if
	 * that's how our page tables are set up.
	 */
	btl	$X86FSET_NX, x86_featureset(%rip)
	jnc	1f
	movl	$MSR_AMD_EFER, %ecx
	rdmsr
	orl	$AMD_EFER_NXE, %eax
	wrmsr
1:

	/*
	 * Complete the rest of the setup and call mp_startup().
	 */
	movq	%gs:CPU_THREAD, %rax	/* get thread ptr */
	movq	T_PC(%rax), %rax
	INDIRECT_CALL_REG(rax)		/* call mp_startup_boot */
	/* not reached */
	int	$20			/* whoops, returned somehow! */

	SET_SIZE(real_mode_start_cpu)

	ENTRY_NP(real_mode_stop_cpu_stage1)

	/*
	 * NOTE:  The GNU assembler automatically does the right thing to
	 *	  generate data size operand prefixes based on the code size
	 *	  generation mode (e.g. .code16, .code32, .code64) and as such
	 *	  prefixes need not be used on instructions EXCEPT in the case
	 *	  of address prefixes for code for which the reference is not
	 *	  automatically of the default operand size.
	 */
	.code16
	cli
	movw		%cs, %ax
	movw		%ax, %ds	/* load cs into ds */
	movw		%ax, %ss	/* and into ss */

	/*
	 * Jump to the stage 2 code in the rm_platter_va->rm_cpu_halt_code
	 */
	movw		$CPUHALTCODEOFF, %ax
	jmp		*%ax

	.globl real_mode_stop_cpu_stage1_end
real_mode_stop_cpu_stage1_end:
	nop

	SET_SIZE(real_mode_stop_cpu_stage1)

	ENTRY_NP(real_mode_stop_cpu_stage2)

	movw		$0xdead, %ax
	movw		%ax, CPUHALTEDOFF

real_mode_stop_cpu_loop:
	/*
	 * Put CPU into halted state.
	 * Only INIT, SMI, NMI could break the loop.
	 */
	hlt
	jmp		real_mode_stop_cpu_loop

	.globl real_mode_stop_cpu_stage2_end
real_mode_stop_cpu_stage2_end:
	nop

	SET_SIZE(real_mode_stop_cpu_stage2)
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 *
 * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

/*
 * Copyright 2019 Joyent, Inc.
 */

#include <sys/elf_notes.h>

#include "assym.h"

/*
 * Tell the booter that we'd like to load unix on a large page.
 */
	.section        .note
	.align          4
	.4byte           .name1_end - .name1_begin
	.4byte           .desc1_end - .desc1_begin
	.4byte		ELF_NOTE_PAGESIZE_HINT
.name1_begin:
	.string         ELF_NOTE_SOLARIS
.name1_end:
	.align          4
.desc1_begin:
	.4byte		FOUR_MEG
.desc1_end:
	.align		4
\
\ Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
\ Copyright 2012 Garrett D'Amore <garrett@damore.org>.  All rights reserved.
\ Copyright 2019 Joyent, Inc.
\ Copyright 2025 Oxide Computer Company
\
\ CDDL HEADER START
\
\ The contents of this file are subject to the terms of the
\ Common Development and Distribution License (the "License").
\ You may not use this file except in compliance with the License.
\
\ You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
\ or http://www.opensolaris.org/os/licensing.
\ See the License for the specific language governing permissions
\ and limitations under the License.
\
\ When distributing Covered Code, include this CDDL HEADER in each
\ file and include the License file at usr/src/OPENSOLARIS.LICENSE.
\ If applicable, add the following below this CDDL HEADER, with the
\ fields enclosed by brackets "[]" replaced with your own identifying
\ information: Portions Copyright [yyyy] [name of copyright owner]
\
\ CDDL HEADER END
\


\
\ offsets.in: input file to produce assym.h using the ctfstabs program
\

#ifndef	_GENASSYM
#define	_GENASSYM
#endif

#define	SIZES	1

#include <sys/types.h>
#include <sys/bootsvcs.h>
#include <sys/systm.h>
#include <sys/sysinfo.h>
#include <sys/user.h>
#include <sys/thread.h>
#include <sys/proc.h>
#include <sys/cpuvar.h>
#include <sys/tss.h>
#include <sys/privregs.h>
#include <sys/segments.h>
#include <sys/devops.h>
#include <sys/ddi_impldefs.h>
#include <vm/as.h>
#include <sys/avintr.h>
#include <sys/pic.h>
#include <sys/rm_platter.h>
#include <sys/stream.h>
#include <sys/strsubr.h>
#include <sys/sunddi.h>
#include <sys/modctl.h>
#include <sys/traptrace.h>
#include <sys/ontrap.h>
#include <sys/lgrp.h>
#include <sys/dtrace.h>
#include <sys/brand.h>
#include <sys/fastboot.h>
#include <sys/cpr_wakecode.h>
#include <sys/comm_page.h>

proc		PROCSIZE
	p_link
	p_next
	p_child
	p_sibling
	p_sig
	p_flag
	p_tlist
	p_as
	p_lockp
	p_user
	p_model
	p_pctx
	p_agenttp
	p_zone
	p_brand
	p_brand_data

_kthread	THREAD_SIZE
	t_pcb			T_LABEL
	t_lock
	t_lockstat
	t_lockp
	t_lock_flush
	t_oldspl
	t_pri
	t_pil
	t_lwp
	t_procp
	t_link
	t_state
	t_mstate
	t_preempt_lk
	t_stk			T_STACK
	t_swap
	t_lwpchan.lc_wchan	T_WCHAN
	t_flag			T_FLAGS
	t_ctx
	t_lofault
	t_onfault
	t_ontrap
	t_cpu
	t_lpl
	t_bound_cpu
	t_intr
	t_forw
	t_back
	t_sig
	t_tid
	t_pre_sys
	t_preempt
	t_proc_flag
	t_startpc
	t_sysnum
	t_intr_start
	_tu._ts._t_astflag	T_ASTFLAG
	_tu._ts._t_post_sys	T_POST_SYS
	_tu._t_post_sys_ast	T_POST_SYS_AST
	t_copyops
	t_useracc

as
	a_hat

user	USIZEBYTES
	u_comm
	u_signal

_label_t
	val	LABEL_VAL

\#define	LABEL_PC	LABEL_VAL
\#define	LABEL_SP	_CONST(LABEL_VAL + LABEL_VAL_INCR)
\#define	T_PC		_CONST(T_LABEL + LABEL_PC)
\#define	T_SP		_CONST(T_LABEL + LABEL_SP)

_klwp
	lwp_thread
	lwp_procp
	lwp_brand
	lwp_eosys
	lwp_regs
	lwp_arg
	lwp_ap
	lwp_cursig
	lwp_state
	lwp_mstate.ms_acct	LWP_MS_ACCT
	lwp_mstate.ms_prev	LWP_MS_PREV
	lwp_mstate.ms_start	LWP_MS_START
	lwp_mstate.ms_state_start LWP_MS_STATE_START
	lwp_pcb
	lwp_ru.sysc		LWP_RU_SYSC

\#define	LWP_ACCT_USER	_CONST(LWP_MS_ACCT + _MUL(LMS_USER, LWP_MS_ACCT_INCR))
\#define	LWP_ACCT_SYSTEM	_CONST(LWP_MS_ACCT + _MUL(LMS_SYSTEM, LWP_MS_ACCT_INCR))

fpu_ctx
	fpu_regs		FPU_CTX_FPU_REGS
	fpu_flags		FPU_CTX_FPU_FLAGS
	fpu_xsave_mask		FPU_CTX_FPU_XSAVE_MASK

fxsave_state	FXSAVE_STATE_SIZE
	fx_fsw			FXSAVE_STATE_FSW
	fx_mxcsr_mask		FXSAVE_STATE_MXCSR_MASK


autovec		AUTOVECSIZE
	av_vector
	av_intarg1
	av_intarg2
	av_ticksp
	av_link
	av_prilevel
	av_dip

av_head
	avh_link
	avh_hi_pri
	avh_lo_pri

cpu
	cpu_id
	cpu_flags
	cpu_self
	cpu_thread
	cpu_thread_lock
	cpu_kprunrun
	cpu_lwp
	cpu_fpowner
	cpu_idle_thread
	cpu_intr_thread
	cpu_intr_actv
	cpu_base_spl
	cpu_intr_stack
	cpu_stats.sys.cpumigrate	CPU_STATS_SYS_CPUMIGRATE
	cpu_stats.sys.intr		CPU_STATS_SYS_INTR
	cpu_stats.sys.intrblk		CPU_STATS_SYS_INTRBLK
	cpu_stats.sys.syscall		CPU_STATS_SYS_SYSCALL
	cpu_profile_pc
	cpu_profile_upc
	cpu_profile_pil
	cpu_ftrace.ftd_state		CPU_FTRACE_STATE
	cpu_mstate
	cpu_intracct

\#define	CPU_INTR_ACTV_REF	_CONST(CPU_INTR_ACTV + 2)

cpu
	cpu_m.pil_high_start	CPU_PIL_HIGH_START
	cpu_m.intrstat		CPU_INTRSTAT
	cpu_m.mcpu_current_hat	CPU_CURRENT_HAT
	cpu_m.mcpu_gdt		CPU_GDT
	cpu_m.mcpu_idt		CPU_IDT
	cpu_m.mcpu_tss		CPU_TSS
	cpu_m.mcpu_softinfo	CPU_SOFTINFO
	cpu_m.mcpu_pri		CPU_PRI
#if defined(__xpv)
	cpu_m.mcpu_vcpu_info	CPU_VCPU_INFO
#endif

cpu
	cpu_m.mcpu_kpti.kf_kernel_cr3	CPU_KPTI_KCR3
	cpu_m.mcpu_kpti.kf_user_cr3	CPU_KPTI_UCR3
	cpu_m.mcpu_kpti.kf_tr_rsp	CPU_KPTI_TR_RSP
	cpu_m.mcpu_kpti.kf_tr_cr3	CPU_KPTI_TR_CR3
	cpu_m.mcpu_kpti.kf_r13		CPU_KPTI_R13
	cpu_m.mcpu_kpti.kf_r14		CPU_KPTI_R14
	cpu_m.mcpu_kpti.kf_tr_ret_rsp	CPU_KPTI_RET_RSP

	cpu_m.mcpu_kpti.kf_ss		CPU_KPTI_SS
	cpu_m.mcpu_kpti.kf_rsp		CPU_KPTI_RSP
	cpu_m.mcpu_kpti.kf_rflags	CPU_KPTI_RFLAGS
	cpu_m.mcpu_kpti.kf_cs		CPU_KPTI_CS
	cpu_m.mcpu_kpti.kf_rip		CPU_KPTI_RIP
	cpu_m.mcpu_kpti.kf_err		CPU_KPTI_ERR

	cpu_m.mcpu_pad2			CPU_KPTI_START
	cpu_m.mcpu_pad3			CPU_KPTI_END

	cpu_m.mcpu_kpti_dbg		CPU_KPTI_DBG

kpti_frame
	kf_r14		KPTI_R14
	kf_r13		KPTI_R13
	kf_err		KPTI_ERR
	kf_rip		KPTI_RIP
	kf_cs		KPTI_CS
	kf_rflags	KPTI_RFLAGS
	kf_rsp		KPTI_RSP
	kf_ss		KPTI_SS

	kf_tr_rsp	KPTI_TOP

	kf_kernel_cr3	KPTI_KCR3
	kf_user_cr3	KPTI_UCR3
	kf_tr_ret_rsp	KPTI_RET_RSP
	kf_tr_cr3	KPTI_TR_CR3

	kf_tr_flag	KPTI_FLAG

standard_pic
	c_curmask
	c_iplmask

ddi_dma_impl
	dmai_rflags
	dmai_rdip

dev_info
	devi_ops		DEVI_DEV_OPS
	devi_bus_ctl
	devi_bus_dma_ctl
	devi_bus_dma_allochdl
	devi_bus_dma_freehdl
	devi_bus_dma_bindhdl
	devi_bus_dma_unbindhdl
	devi_bus_dma_flush
	devi_bus_dma_win

dev_ops
	devo_bus_ops		DEVI_BUS_OPS

bus_ops
	bus_ctl			OPS_CTL
	bus_dma_map		OPS_MAP
	bus_dma_ctl		OPS_MCTL
	bus_dma_allochdl	OPS_ALLOCHDL
	bus_dma_freehdl		OPS_FREEHDL
	bus_dma_bindhdl		OPS_BINDHDL
	bus_dma_unbindhdl	OPS_UNBINDHDL
	bus_dma_flush		OPS_FLUSH
	bus_dma_win		OPS_WIN

sysent	SYSENT_SIZE	SYSENT_SIZE_SHIFT
	sy_callc
	sy_flags
	sy_narg

stdata
	sd_lock

queue
	q_flag
	q_next
	q_stream
	q_syncq
	q_qinfo

qinit
	qi_putp

syncq
	sq_flags
	sq_count
	sq_lock
	sq_wait

rm_platter
	rm_idt_lim		IDTROFF
	rm_gdt_lim		GDTROFF
	rm_pdbr			CR3OFF
	rm_cpu			CPUNOFF
	rm_cr4			CR4OFF
	rm_cpu_halt_code	CPUHALTCODEOFF
	rm_cpu_halted		CPUHALTEDOFF

ddi_acc_impl
	ahi_acc_attr	ACC_ATTR
	ahi_get8	ACC_GETB
	ahi_get16	ACC_GETW
	ahi_get32	ACC_GETL
	ahi_get64	ACC_GETLL
	ahi_put8	ACC_PUTB
	ahi_put16	ACC_PUTW
	ahi_put32	ACC_PUTL
	ahi_put64	ACC_PUTLL
	ahi_rep_get8	ACC_REP_GETB
	ahi_rep_get16	ACC_REP_GETW
	ahi_rep_get32	ACC_REP_GETL
	ahi_rep_get64	ACC_REP_GETLL
	ahi_rep_put8	ACC_REP_PUTB
	ahi_rep_put16	ACC_REP_PUTW
	ahi_rep_put32	ACC_REP_PUTL
	ahi_rep_put64	ACC_REP_PUTLL

on_trap_data
	ot_prot
	ot_trap
	ot_trampoline
	ot_jmpbuf
	ot_prev
	ot_handle
	ot_pad1

trap_trace_ctl_t	__TRAPTR_SIZE TRAPTR_SIZE_SHIFT
	ttc_next	TRAPTR_NEXT
	ttc_first	TRAPTR_FIRST
	ttc_limit	TRAPTR_LIMIT

trap_trace_rec_t	TRAP_ENT_SIZE
	ttr_cr2
	ttr_info.idt_entry.vector	TTR_VECTOR
	ttr_info.idt_entry.ipl		TTR_IPL
	ttr_info.idt_entry.spl		TTR_SPL
	ttr_info.idt_entry.pri		TTR_PRI
	ttr_info.gate_entry.sysnum	TTR_SYSNUM
	ttr_marker
	ttr_stamp
	ttr_curthread
	ttr_sdepth
	ttr_stack

lgrp_ld
	lpl_lgrpid

dtrace_id_t	DTRACE_IDSIZE

cpu_core	CPU_CORE_SIZE	CPU_CORE_SHIFT
	cpuc_dtrace_flags
	cpuc_dtrace_illval

timespec	TIMESPEC_SIZE

gate_desc	GATE_DESC_SIZE

desctbr_t	DESCTBR_SIZE
	dtr_limit
	dtr_base

mod_stub_info	MODS_SIZE
	mods_func_adr	MODS_INSTFCN
	mods_errfcn	MODS_RETFCN
	mods_flag

\#define	TRAP_TSIZE		_MUL(TRAP_ENT_SIZE, TRAPTR_NENT)

copyops
	cp_copyin
	cp_xcopyin
	cp_copyout
	cp_xcopyout
	cp_copyinstr
	cp_copyoutstr
	cp_fuword8
	cp_fuword16
	cp_fuword32
	cp_fuword64
	cp_suword8
	cp_suword16
	cp_suword32
	cp_suword64
	cp_physio

brand
	b_machops

brand_proc_data_t
	spd_handler

fastboot_file_t
	fb_va
	fb_pte_list_va
	fb_pte_list_pa
	fb_dest_pa
	fb_size
	fb_next_pa
	fb_sections
	fb_sectcnt

fastboot_section_t
	fb_sec_offset
	fb_sec_paddr
	fb_sec_size
	fb_sec_bss_size

fastboot_info_t
	fi_files
	fi_has_pae
	fi_pagetable_va
	fi_pagetable_pa
	fi_last_table_pa
	fi_new_mbi_pa
	fi_valid

zone
	zone_brand_data

wc_cpu	WC_CPU_SIZE
	wc_retaddr
	wc_virtaddr
	wc_cr0
	wc_cr3
	wc_cr4
	wc_cr8
	wc_fs
	wc_fsbase
	wc_gs
	wc_gsbase
	wc_kgsbase
	wc_r8
	wc_r9
	wc_r10
	wc_r11
	wc_r12
	wc_r13
	wc_r14
	wc_r15
	wc_rax
	wc_rbp
	wc_rbx
	wc_rcx
	wc_rdi
	wc_rdx
	wc_rsi
	wc_rsp
	wc_gdt_limit	WC_GDT
	wc_gdt_base
	wc_idt_limit	WC_IDT
	wc_idt_base
	wc_tr
	wc_ldt
	wc_eflags
	wc_ebx
	wc_edi
	wc_esi
	wc_ebp
	wc_esp
	wc_esp
	wc_ss
	wc_cs
	wc_ds
	wc_es
	wc_cpu_id
	wc_saved_stack

wc_wakecode
	wc_cpu

comm_page_s	COMM_PAGE_S_SIZE
	_cp_pad1	COMM_PAGE_PAD1
	cp_tsc_sync_tick_delta	COMM_PAGE_TSC_SYNC_TICK_DELTA
	_cp_pad2	COMM_PAGE_PAD2
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
 * Copyright 2019 Joyent, Inc.
 * Copyright (c) 2016 by Delphix. All rights reserved.
 * Copyright 2024 MNX Cloud, Inc.
 */

#include <sys/asm_linkage.h>
#include <sys/asm_misc.h>
#include <sys/regset.h>
#include <sys/privregs.h>
#include <sys/psw.h>
#include <sys/machbrand.h>

#include <sys/segments.h>
#include <sys/pcb.h>
#include <sys/trap.h>
#include <sys/ftrace.h>
#include <sys/traptrace.h>
#include <sys/clock.h>
#include <sys/model.h>
#include <sys/panic.h>

#if defined(__xpv)
#include <sys/hypervisor.h>
#endif

#include "assym.h"

/*
 * We implement five flavours of system call entry points
 *
 * -	syscall/sysretq		(amd64 generic)
 * -	syscall/sysretl		(i386 plus SYSC bit)
 * -	sysenter/sysexit	(i386 plus SEP bit)
 * -	int/iret		(i386 generic)
 * -	lcall/iret		(i386 generic)
 *
 * The current libc included in Solaris uses int/iret as the base unoptimized
 * kernel entry method. Older libc implementations and legacy binaries may use
 * the lcall call gate, so it must continue to be supported.
 *
 * System calls that use an lcall call gate are processed in trap() via a
 * segment-not-present trap, i.e. lcalls are extremely slow(!).
 *
 * The basic pattern used in the 32-bit SYSC handler at this point in time is
 * to have the bare minimum of assembler, and get to the C handlers as
 * quickly as possible.
 *
 * The 64-bit handler is much closer to the sparcv9 handler; that's
 * because of passing arguments in registers.  The 32-bit world still
 * passes arguments on the stack -- that makes that handler substantially
 * more complex.
 *
 * The two handlers share a few code fragments which are broken
 * out into preprocessor macros below.
 *
 * XX64	come back and speed all this up later.  The 32-bit stuff looks
 * especially easy to speed up the argument copying part ..
 *
 *
 * Notes about segment register usage (c.f. the 32-bit kernel)
 *
 * In the 32-bit kernel, segment registers are dutifully saved and
 * restored on all mode transitions because the kernel uses them directly.
 * When the processor is running in 64-bit mode, segment registers are
 * largely ignored.
 *
 * %cs and %ss
 *	controlled by the hardware mechanisms that make mode transitions
 *
 * The remaining segment registers have to either be pointing at a valid
 * descriptor i.e. with the 'present' bit set, or they can NULL descriptors
 *
 * %ds and %es
 *	always ignored
 *
 * %fs and %gs
 *	fsbase and gsbase are used to control the place they really point at.
 *	The kernel only depends on %gs, and controls its own gsbase via swapgs
 *
 * Note that loading segment registers is still costly because the GDT
 * lookup still happens (this is because the hardware can't know that we're
 * not setting up these segment registers for a 32-bit program).  Thus we
 * avoid doing this in the syscall path, and defer them to lwp context switch
 * handlers, so the register values remain virtualized to the lwp.
 */

#if defined(SYSCALLTRACE)
#define	ORL_SYSCALLTRACE(r32)		\
	orl	syscalltrace(%rip), r32
#else
#define	ORL_SYSCALLTRACE(r32)
#endif

/*
 * In the 32-bit kernel, we do absolutely nothing before getting into the
 * brand callback checks.  In 64-bit land, we do swapgs and then come here.
 * We assume that the %rsp- and %r15-stashing fields in the CPU structure
 * are still unused.
 *
 * Check if a brand_mach_ops callback is defined for the specified callback_id
 * type.  If so invoke it with the kernel's %gs value loaded and the following
 * data on the stack:
 *
 * stack:  --------------------------------------
 *      32 | callback pointer			|
 *    | 24 | user (or interrupt) stack pointer	|
 *    | 16 | lwp pointer			|
 *    v  8 | userland return address		|
 *       0 | callback wrapper return addr	|
 *         --------------------------------------
 *
 * Since we're pushing the userland return address onto the kernel stack
 * we need to get that address without accessing the user's stack (since we
 * can't trust that data).  There are different ways to get the userland
 * return address depending on how the syscall trap was made:
 *
 * a) For sys_syscall and sys_syscall32 the return address is in %rcx.
 * b) For sys_sysenter the return address is in %rdx.
 * c) For sys_int80 and sys_syscall_int (int91), upon entry into the macro,
 *    the stack pointer points at the state saved when we took the interrupt:
 *	 ------------------------
 *    |  | user's %ss		|
 *    |  | user's %esp		|
 *    |  | EFLAGS register	|
 *    v  | user's %cs		|
 *       | user's %eip		|
 *	 ------------------------
 *
 * The 2nd parameter to the BRAND_CALLBACK macro is either the
 * BRAND_URET_FROM_REG or BRAND_URET_FROM_INTR_STACK macro.  These macros are
 * used to generate the proper code to get the userland return address for
 * each syscall entry point.
 *
 * The interface to the brand callbacks on the 64-bit kernel assumes %r15
 * is available as a scratch register within the callback.  If the callback
 * returns within the kernel then this macro will restore %r15.  If the
 * callback is going to return directly to userland then it should restore
 * %r15 before returning to userland.
 */
#define BRAND_URET_FROM_REG(rip_reg)					\
	pushq	rip_reg			/* push the return address	*/

/*
 * The interrupt stack pointer we saved on entry to the BRAND_CALLBACK macro
 * is currently pointing at the user return address (%eip).
 */
#define BRAND_URET_FROM_INTR_STACK()					\
	movq	%gs:CPU_RTMP_RSP, %r15	/* grab the intr. stack pointer	*/ ;\
	pushq	(%r15)			/* push the return address	*/

#define	BRAND_CALLBACK(callback_id, push_userland_ret)			    \
	movq	%rsp, %gs:CPU_RTMP_RSP	/* save the stack pointer	*/ ;\
	movq	%r15, %gs:CPU_RTMP_R15	/* save %r15			*/ ;\
	movq	%gs:CPU_THREAD, %r15	/* load the thread pointer	*/ ;\
	movq	T_STACK(%r15), %rsp	/* switch to the kernel stack	*/ ;\
	subq	$16, %rsp		/* save space for 2 pointers	*/ ;\
	pushq	%r14			/* save %r14			*/ ;\
	movq	%gs:CPU_RTMP_RSP, %r14					   ;\
	movq	%r14, 8(%rsp)		/* stash the user stack pointer	*/ ;\
	popq	%r14			/* restore %r14			*/ ;\
	movq	T_LWP(%r15), %r15	/* load the lwp pointer		*/ ;\
	pushq	%r15			/* push the lwp pointer		*/ ;\
	movq	LWP_PROCP(%r15), %r15	/* load the proc pointer	*/ ;\
	movq	P_BRAND(%r15), %r15	/* load the brand pointer	*/ ;\
	movq	B_MACHOPS(%r15), %r15	/* load the machops pointer	*/ ;\
	movq	_CONST(_MUL(callback_id, CPTRSIZE))(%r15), %r15		   ;\
	cmpq	$0, %r15						   ;\
	je	1f							   ;\
	movq	%r15, 16(%rsp)		/* save the callback pointer	*/ ;\
	push_userland_ret		/* push the return address	*/ ;\
	movq	24(%rsp), %r15		/* load callback pointer	*/ ;\
	INDIRECT_CALL_REG(r15)		/* call callback		*/ ;\
1:	movq	%gs:CPU_RTMP_R15, %r15	/* restore %r15			*/ ;\
	movq	%gs:CPU_RTMP_RSP, %rsp	/* restore the stack pointer	*/

#define	MSTATE_TRANSITION(from, to)		\
	movl	$from, %edi;			\
	movl	$to, %esi;			\
	call	syscall_mstate

/*
 * Check to see if a simple (direct) return is possible i.e.
 *
 *	if (t->t_post_sys_ast | syscalltrace |
 *	    lwp->lwp_pcb.pcb_rupdate == 1)
 *		do full version	;
 *
 * Preconditions:
 * -	t is curthread
 * Postconditions:
 * -	condition code NE is set if post-sys is too complex
 * -	rtmp is zeroed if it isn't (we rely on this!)
 * -	ltmp is smashed
 */
#define	CHECK_POSTSYS_NE(t, ltmp, rtmp)			\
	movq	T_LWP(t), ltmp;				\
	movzbl	PCB_RUPDATE(ltmp), rtmp;		\
	ORL_SYSCALLTRACE(rtmp);				\
	orl	T_POST_SYS_AST(t), rtmp;		\
	cmpl	$0, rtmp

/*
 * Fix up the lwp, thread, and eflags for a successful return
 *
 * Preconditions:
 * -	zwreg contains zero
 */
#define	SIMPLE_SYSCALL_POSTSYS(t, lwp, zwreg)		\
	movb	$LWP_USER, LWP_STATE(lwp);		\
	movw	zwreg, T_SYSNUM(t);			\
	andb	$_CONST(0xffff - PS_C), REGOFF_RFL(%rsp)

/*
 * ASSERT(lwptoregs(lwp) == rp);
 *
 * This may seem obvious, but very odd things happen if this
 * assertion is false
 *
 * Preconditions:
 *	(%rsp is ready for normal call sequence)
 * Postconditions (if assertion is true):
 *	%r11 is smashed
 *
 * ASSERT(rp->r_cs == descnum)
 *
 * The code selector is written into the regs structure when the
 * lwp stack is created.  We use this ASSERT to validate that
 * the regs structure really matches how we came in.
 *
 * Preconditions:
 *	(%rsp is ready for normal call sequence)
 * Postconditions (if assertion is true):
 *	-none-
 *
 * ASSERT(lwp->lwp_pcb.pcb_rupdate == 0);
 *
 * If this is false, it meant that we returned to userland without
 * updating the segment registers as we were supposed to.
 *
 * Note that we must ensure no interrupts or other traps intervene
 * between entering privileged mode and performing the assertion,
 * otherwise we may perform a context switch on the thread, which
 * will end up setting pcb_rupdate to 1 again.
 *
 * ASSERT(%cr0 & CR0_TS == 0);
 * Preconditions:
 *	(%rsp is ready for normal call sequence)
 * Postconditions (if assertion is true):
 *      (specified register is clobbered)
 *
 * Check to make sure that we are returning to user land and that CR0.TS
 * is not set. This is required as part of the eager FPU (see
 * uts/intel/os/fpu.c for more information).
 */

#if defined(DEBUG)

__lwptoregs_msg:
	.string	"syscall_asm_amd64.s:%d lwptoregs(%p) [%p] != rp [%p]"

__codesel_msg:
	.string	"syscall_asm_amd64.s:%d rp->r_cs [%ld] != %ld"

__no_rupdate_msg:
	.string	"syscall_asm_amd64.s:%d lwp %p, pcb_rupdate != 0"

__bad_ts_msg:
	.string "syscall_asm_amd64.s:%d CR0.TS set on user return"

#define	ASSERT_LWPTOREGS(lwp, rp)			\
	movq	LWP_REGS(lwp), %r11;			\
	cmpq	rp, %r11;				\
	je	7f;					\
	leaq	__lwptoregs_msg(%rip), %rdi;		\
	movl	$__LINE__, %esi;			\
	movq	lwp, %rdx;				\
	movq	%r11, %rcx;				\
	movq	rp, %r8;				\
	xorl	%eax, %eax;				\
	call	panic;					\
7:

#define	ASSERT_NO_RUPDATE_PENDING(lwp)			\
	testb	$0x1, PCB_RUPDATE(lwp);			\
	je	8f;					\
	movq	lwp, %rdx;				\
	leaq	__no_rupdate_msg(%rip), %rdi;		\
	movl	$__LINE__, %esi;			\
	xorl	%eax, %eax;				\
	call	panic;					\
8:

#define	ASSERT_CR0TS_ZERO(reg)				\
	movq	%cr0, reg;				\
	testq	$CR0_TS, reg;				\
	jz	9f;					\
	leaq	__bad_ts_msg(%rip), %rdi;		\
	movl	$__LINE__, %esi;			\
	xorl	%eax, %eax;				\
	call	panic;					\
9:

#else
#define	ASSERT_LWPTOREGS(lwp, rp)
#define	ASSERT_NO_RUPDATE_PENDING(lwp)
#define	ASSERT_CR0TS_ZERO(reg)
#endif

/*
 * Do the traptrace thing and restore any registers we used
 * in situ.  Assumes that %rsp is pointing at the base of
 * the struct regs, obviously ..
 */
#ifdef TRAPTRACE
#define	SYSCALL_TRAPTRACE(ttype)				\
	TRACE_PTR(%rdi, %rbx, %ebx, %rcx, ttype);		\
	TRACE_REGS(%rdi, %rsp, %rbx, %rcx);			\
	TRACE_STAMP(%rdi);	/* rdtsc clobbers %eax, %edx */	\
	movq	REGOFF_RAX(%rsp), %rax;				\
	movq	REGOFF_RBX(%rsp), %rbx;				\
	movq	REGOFF_RCX(%rsp), %rcx;				\
	movq	REGOFF_RDX(%rsp), %rdx;				\
	movl	%eax, TTR_SYSNUM(%rdi);				\
	movq	REGOFF_RDI(%rsp), %rdi

#define	SYSCALL_TRAPTRACE32(ttype)				\
	SYSCALL_TRAPTRACE(ttype);				\
	/* paranoia: clean the top 32-bits of the registers */	\
	orl	%eax, %eax;					\
	orl	%ebx, %ebx;					\
	orl	%ecx, %ecx;					\
	orl	%edx, %edx;					\
	orl	%edi, %edi
#else	/* TRAPTRACE */
#define	SYSCALL_TRAPTRACE(ttype)
#define	SYSCALL_TRAPTRACE32(ttype)
#endif	/* TRAPTRACE */

/*
 * The 64-bit libc syscall wrapper does this:
 *
 * fn(<args>)
 * {
 *	movq	%rcx, %r10	-- because syscall smashes %rcx
 *	movl	$CODE, %eax
 *	syscall
 *	<error processing>
 * }
 *
 * Thus when we come into the kernel:
 *
 *	%rdi, %rsi, %rdx, %r10, %r8, %r9 contain first six args
 *	%rax is the syscall number
 *	%r12-%r15 contain caller state
 *
 * The syscall instruction arranges that:
 *
 *	%rcx contains the return %rip
 *	%r11d contains bottom 32-bits of %rflags
 *	%rflags is masked (as determined by the SFMASK msr)
 *	%cs is set to UCS_SEL (as determined by the STAR msr)
 *	%ss is set to UDS_SEL (as determined by the STAR msr)
 *	%rip is set to sys_syscall (as determined by the LSTAR msr)
 *
 * Or in other words, we have no registers available at all.
 * Only swapgs can save us!
 *
 * Under the hypervisor, the swapgs has happened already.  However, the
 * state of the world is very different from that we're familiar with.
 *
 * In particular, we have a stack structure like that for interrupt
 * gates, except that the %cs and %ss registers are modified for reasons
 * that are not entirely clear.  Critically, the %rcx/%r11 values do
 * *not* reflect the usage of those registers under a 'real' syscall[1];
 * the stack, therefore, looks like this:
 *
 *	0x0(rsp)	potentially junk %rcx
 *	0x8(rsp)	potentially junk %r11
 *	0x10(rsp)	user %rip
 *	0x18(rsp)	modified %cs
 *	0x20(rsp)	user %rflags
 *	0x28(rsp)	user %rsp
 *	0x30(rsp)	modified %ss
 *
 *
 * and before continuing on, we must load the %rip into %rcx and the
 * %rflags into %r11.
 *
 * [1] They used to, and we relied on it, but this was broken in 3.1.1.
 * Sigh.
 */
#if defined(__xpv)
#define	XPV_SYSCALL_PROD						\
	movq	0x10(%rsp), %rcx;					\
	movq	0x20(%rsp), %r11;					\
	movq	0x28(%rsp), %rsp
#else
#define	XPV_SYSCALL_PROD /* nothing */
#endif

	ENTRY_NP2(brand_sys_syscall,_allsyscalls)
	SWAPGS				/* kernel gsbase */
	XPV_SYSCALL_PROD
	BRAND_CALLBACK(BRAND_CB_SYSCALL, BRAND_URET_FROM_REG(%rcx))
	jmp	noprod_sys_syscall

	ALTENTRY(sys_syscall)
	SWAPGS				/* kernel gsbase */
	XPV_SYSCALL_PROD

noprod_sys_syscall:
	movq	%r15, %gs:CPU_RTMP_R15
	movq	%rsp, %gs:CPU_RTMP_RSP

	movq	%gs:CPU_THREAD, %r15
	movq	T_STACK(%r15), %rsp	/* switch from user to kernel stack */

	ASSERT_UPCALL_MASK_IS_SET

	movl	$UCS_SEL, REGOFF_CS(%rsp)
	movq	%rcx, REGOFF_RIP(%rsp)		/* syscall: %rip -> %rcx */
	movq	%r11, REGOFF_RFL(%rsp)		/* syscall: %rfl -> %r11d */
	movl	$UDS_SEL, REGOFF_SS(%rsp)

	movl	%eax, %eax			/* wrapper: sysc# -> %eax */
	movq	%rdi, REGOFF_RDI(%rsp)
	movq	%rsi, REGOFF_RSI(%rsp)
	movq	%rdx, REGOFF_RDX(%rsp)
	movq	%r10, REGOFF_RCX(%rsp)		/* wrapper: %rcx -> %r10 */
	movq	%r10, %rcx			/* arg[3] for direct calls */

	movq	%r8, REGOFF_R8(%rsp)
	movq	%r9, REGOFF_R9(%rsp)
	movq	%rax, REGOFF_RAX(%rsp)
	movq	%rbx, REGOFF_RBX(%rsp)

	movq	%rbp, REGOFF_RBP(%rsp)
	movq	%r10, REGOFF_R10(%rsp)
	movq	%gs:CPU_RTMP_RSP, %r11
	movq	%r11, REGOFF_RSP(%rsp)
	movq	%r12, REGOFF_R12(%rsp)

	movq	%r13, REGOFF_R13(%rsp)
	movq	%r14, REGOFF_R14(%rsp)
	movq	%gs:CPU_RTMP_R15, %r10
	movq	%r10, REGOFF_R15(%rsp)
	movq	$0, REGOFF_SAVFP(%rsp)
	movq	$0, REGOFF_SAVPC(%rsp)

	/*
	 * Copy these registers here in case we end up stopped with
	 * someone (like, say, /proc) messing with our register state.
	 * We don't -restore- them unless we have to in update_sregs.
	 *
	 * Since userland -can't- change fsbase or gsbase directly,
	 * and capturing them involves two serializing instructions,
	 * we don't bother to capture them here.
	 */
	xorl	%ebx, %ebx
	movw	%ds, %bx
	movq	%rbx, REGOFF_DS(%rsp)
	movw	%es, %bx
	movq	%rbx, REGOFF_ES(%rsp)
	movw	%fs, %bx
	movq	%rbx, REGOFF_FS(%rsp)
	movw	%gs, %bx
	movq	%rbx, REGOFF_GS(%rsp)

	/*
	 * If we're trying to use TRAPTRACE though, I take that back: we're
	 * probably debugging some problem in the SWAPGS logic and want to know
	 * what the incoming gsbase was.
	 *
	 * Since we already did SWAPGS, record the KGSBASE.
	 */
#if defined(DEBUG) && defined(TRAPTRACE) && !defined(__xpv)
	movl	$MSR_AMD_KGSBASE, %ecx
	rdmsr
	movl	%eax, REGOFF_GSBASE(%rsp)
	movl	%edx, REGOFF_GSBASE+4(%rsp)
#endif

	/*
	 * Machine state saved in the regs structure on the stack
	 * First six args in %rdi, %rsi, %rdx, %rcx, %r8, %r9
	 * %eax is the syscall number
	 * %rsp is the thread's stack, %r15 is curthread
	 * REG_RSP(%rsp) is the user's stack
	 */

	SYSCALL_TRAPTRACE($TT_SYSC64)

	movq	%rsp, %rbp

	movq	T_LWP(%r15), %r14
	ASSERT_NO_RUPDATE_PENDING(%r14)
	ENABLE_INTR_FLAGS

	MSTATE_TRANSITION(LMS_USER, LMS_SYSTEM)
	movl	REGOFF_RAX(%rsp), %eax	/* (%rax damaged by mstate call) */

	ASSERT_LWPTOREGS(%r14, %rsp)

	movb	$LWP_SYS, LWP_STATE(%r14)
	incq	LWP_RU_SYSC(%r14)
	movb	$NORMALRETURN, LWP_EOSYS(%r14)

	incq	%gs:CPU_STATS_SYS_SYSCALL

	movw	%ax, T_SYSNUM(%r15)
	movzbl	T_PRE_SYS(%r15), %ebx
	ORL_SYSCALLTRACE(%ebx)
	testl	%ebx, %ebx
	jne	_syscall_pre

_syscall_invoke:
	movq	REGOFF_RDI(%rbp), %rdi
	movq	REGOFF_RSI(%rbp), %rsi
	movq	REGOFF_RDX(%rbp), %rdx
	movq	REGOFF_RCX(%rbp), %rcx
	movq	REGOFF_R8(%rbp), %r8
	movq	REGOFF_R9(%rbp), %r9

	cmpl	$NSYSCALL, %eax
	jae	_syscall_ill
	shll	$SYSENT_SIZE_SHIFT, %eax
	leaq	sysent(%rax), %rbx

	movq	SY_CALLC(%rbx), %rax
	INDIRECT_CALL_REG(rax)

	movq	%rax, %r12
	movq	%rdx, %r13

	/*
	 * If the handler returns two ints, then we need to split the
	 * 64-bit return value into two 32-bit values.
	 */
	testw	$SE_32RVAL2, SY_FLAGS(%rbx)
	je	5f
	movq	%r12, %r13
	shrq	$32, %r13	/* upper 32-bits into %edx */
	movl	%r12d, %r12d	/* lower 32-bits into %eax */
5:
	/*
	 * Optimistically assume that there's no post-syscall
	 * work to do.  (This is to avoid having to call syscall_mstate()
	 * with interrupts disabled)
	 */
	MSTATE_TRANSITION(LMS_SYSTEM, LMS_USER)

	/*
	 * We must protect ourselves from being descheduled here;
	 * If we were, and we ended up on another cpu, or another
	 * lwp got in ahead of us, it could change the segment
	 * registers without us noticing before we return to userland.
	 */
	CLI(%r14)
	CHECK_POSTSYS_NE(%r15, %r14, %ebx)
	jne	_syscall_post

	/*
	 * We need to protect ourselves against non-canonical return values
	 * because Intel doesn't check for them on sysret (AMD does).  Canonical
	 * addresses on current amd64 processors only use 48-bits for VAs; an
	 * address is canonical if all upper bits (47-63) are identical. If we
	 * find a non-canonical %rip, we opt to go through the full
	 * _syscall_post path which takes us into an iretq which is not
	 * susceptible to the same problems sysret is.
	 *
	 * We're checking for a canonical address by first doing an arithmetic
	 * shift. This will fill in the remaining bits with the value of bit 63.
	 * If the address were canonical, the register would now have either all
	 * zeroes or all ones in it. Therefore we add one (inducing overflow)
	 * and compare against 1. A canonical address will either be zero or one
	 * at this point, hence the use of ja.
	 *
	 * At this point, r12 and r13 have the return value so we can't use
	 * those registers.
	 */
	movq	REGOFF_RIP(%rsp), %rcx
	sarq	$47, %rcx
	incq	%rcx
	cmpq	$1, %rcx
	ja	_syscall_post


	SIMPLE_SYSCALL_POSTSYS(%r15, %r14, %bx)

	movq	%r12, REGOFF_RAX(%rsp)
	movq	%r13, REGOFF_RDX(%rsp)

	/*
	 * Clobber %r11 as we check CR0.TS.
	 */
	ASSERT_CR0TS_ZERO(%r11)

	/*
	 * To get back to userland, we need the return %rip in %rcx and
	 * the return %rfl in %r11d.  The sysretq instruction also arranges
	 * to fix up %cs and %ss; everything else is our responsibility.
	 */
	movq	REGOFF_RDI(%rsp), %rdi
	movq	REGOFF_RSI(%rsp), %rsi
	movq	REGOFF_RDX(%rsp), %rdx
	/* %rcx used to restore %rip value */

	movq	REGOFF_R8(%rsp), %r8
	movq	REGOFF_R9(%rsp), %r9
	movq	REGOFF_RAX(%rsp), %rax
	movq	REGOFF_RBX(%rsp), %rbx

	movq	REGOFF_RBP(%rsp), %rbp
	movq	REGOFF_R10(%rsp), %r10
	/* %r11 used to restore %rfl value */
	movq	REGOFF_R12(%rsp), %r12

	movq	REGOFF_R13(%rsp), %r13
	movq	REGOFF_R14(%rsp), %r14
	movq	REGOFF_R15(%rsp), %r15

	movq	REGOFF_RIP(%rsp), %rcx
	movl	REGOFF_RFL(%rsp), %r11d

	/*
	 * Unlike other cases, because we need to restore the user stack pointer
	 * before exiting the kernel we must clear the microarch state before
	 * getting here. This should be safe because it means that the only
	 * values on the bus after this are based on the user's registers and
	 * potentially the addresses where we stored them. Given the constraints
	 * of sysret, that's how it has to be.
	 */
	call	x86_md_clear

#if defined(__xpv)
	addq	$REGOFF_RIP, %rsp
#else
	movq	REGOFF_RSP(%rsp), %rsp
#endif

        /*
         * There can be no instructions between the ALTENTRY below and
	 * SYSRET or we could end up breaking brand support. See label usage
         * in sn1_brand_syscall_callback for an example.
         */
	ASSERT_UPCALL_MASK_IS_SET
#if defined(__xpv)
	SYSRETQ
        ALTENTRY(nopop_sys_syscall_swapgs_sysretq)

	/*
	 * We can only get here after executing a brand syscall
	 * interposition callback handler and simply need to
	 * "sysretq" back to userland. On the hypervisor this
	 * involves the iret hypercall which requires us to construct
	 * just enough of the stack needed for the hypercall.
	 * (rip, cs, rflags, rsp, ss).
	 */
	movq    %rsp, %gs:CPU_RTMP_RSP		/* save user's rsp */
	movq	%gs:CPU_THREAD, %r11
	movq	T_STACK(%r11), %rsp

	movq	%rcx, REGOFF_RIP(%rsp)
	movl	$UCS_SEL, REGOFF_CS(%rsp)
	movq	%gs:CPU_RTMP_RSP, %r11
	movq	%r11, REGOFF_RSP(%rsp)
	pushfq
	popq	%r11				/* hypercall enables ints */
	movq	%r11, REGOFF_RFL(%rsp)
	movl	$UDS_SEL, REGOFF_SS(%rsp)
	addq	$REGOFF_RIP, %rsp
	/*
	 * XXPV: see comment in SYSRETQ definition for future optimization
	 *       we could take.
	 */
	ASSERT_UPCALL_MASK_IS_SET
	SYSRETQ
#else
        ALTENTRY(nopop_sys_syscall_swapgs_sysretq)
	jmp	tr_sysretq
#endif
        /*NOTREACHED*/
        SET_SIZE(nopop_sys_syscall_swapgs_sysretq)

_syscall_pre:
	call	pre_syscall
	movl	%eax, %r12d
	testl	%eax, %eax
	jne	_syscall_post_call
	/*
	 * Didn't abort, so reload the syscall args and invoke the handler.
	 */
	movzwl	T_SYSNUM(%r15), %eax
	jmp	_syscall_invoke

_syscall_ill:
	call	nosys
	movq	%rax, %r12
	movq	%rdx, %r13
	jmp	_syscall_post_call

_syscall_post:
	STI
	/*
	 * Sigh, our optimism wasn't justified, put it back to LMS_SYSTEM
	 * so that we can account for the extra work it takes us to finish.
	 */
	MSTATE_TRANSITION(LMS_USER, LMS_SYSTEM)
_syscall_post_call:
	movq	%r12, %rdi
	movq	%r13, %rsi
	call	post_syscall
	MSTATE_TRANSITION(LMS_SYSTEM, LMS_USER)
	jmp	_sys_rtt
	SET_SIZE(sys_syscall)
	SET_SIZE(brand_sys_syscall)

	ENTRY_NP(brand_sys_syscall32)
	SWAPGS				/* kernel gsbase */
	XPV_TRAP_POP
	BRAND_CALLBACK(BRAND_CB_SYSCALL32, BRAND_URET_FROM_REG(%rcx))
	jmp	nopop_sys_syscall32

	ALTENTRY(sys_syscall32)
	SWAPGS				/* kernel gsbase */
	XPV_TRAP_POP

nopop_sys_syscall32:
	movl	%esp, %r10d
	movq	%gs:CPU_THREAD, %r15
	movq	T_STACK(%r15), %rsp
	movl	%eax, %eax

	movl	$U32CS_SEL, REGOFF_CS(%rsp)
	movl	%ecx, REGOFF_RIP(%rsp)		/* syscall: %rip -> %rcx */
	movq	%r11, REGOFF_RFL(%rsp)		/* syscall: %rfl -> %r11d */
	movq	%r10, REGOFF_RSP(%rsp)
	movl	$UDS_SEL, REGOFF_SS(%rsp)

_syscall32_save:
	movl	%edi, REGOFF_RDI(%rsp)
	movl	%esi, REGOFF_RSI(%rsp)
	movl	%ebp, REGOFF_RBP(%rsp)
	movl	%ebx, REGOFF_RBX(%rsp)
	movl	%edx, REGOFF_RDX(%rsp)
	movl	%ecx, REGOFF_RCX(%rsp)
	movl	%eax, REGOFF_RAX(%rsp)		/* wrapper: sysc# -> %eax */
	movq	$0, REGOFF_SAVFP(%rsp)
	movq	$0, REGOFF_SAVPC(%rsp)

	/*
	 * Copy these registers here in case we end up stopped with
	 * someone (like, say, /proc) messing with our register state.
	 * We don't -restore- them unless we have to in update_sregs.
	 *
	 * Since userland -can't- change fsbase or gsbase directly,
	 * we don't bother to capture them here.
	 */
	xorl	%ebx, %ebx
	movw	%ds, %bx
	movq	%rbx, REGOFF_DS(%rsp)
	movw	%es, %bx
	movq	%rbx, REGOFF_ES(%rsp)
	movw	%fs, %bx
	movq	%rbx, REGOFF_FS(%rsp)
	movw	%gs, %bx
	movq	%rbx, REGOFF_GS(%rsp)

	/*
	 * If we're trying to use TRAPTRACE though, I take that back: we're
	 * probably debugging some problem in the SWAPGS logic and want to know
	 * what the incoming gsbase was.
	 *
	 * Since we already did SWAPGS, record the KGSBASE.
	 */
#if defined(DEBUG) && defined(TRAPTRACE) && !defined(__xpv)
	movl	$MSR_AMD_KGSBASE, %ecx
	rdmsr
	movl	%eax, REGOFF_GSBASE(%rsp)
	movl	%edx, REGOFF_GSBASE+4(%rsp)
#endif

	/*
	 * Application state saved in the regs structure on the stack
	 * %eax is the syscall number
	 * %rsp is the thread's stack, %r15 is curthread
	 * REG_RSP(%rsp) is the user's stack
	 */

	SYSCALL_TRAPTRACE32($TT_SYSC)

	movq	%rsp, %rbp

	movq	T_LWP(%r15), %r14
	ASSERT_NO_RUPDATE_PENDING(%r14)

	ENABLE_INTR_FLAGS

	MSTATE_TRANSITION(LMS_USER, LMS_SYSTEM)
	movl	REGOFF_RAX(%rsp), %eax	/* (%rax damaged by mstate call) */

	ASSERT_LWPTOREGS(%r14, %rsp)

	incq	 %gs:CPU_STATS_SYS_SYSCALL

	/*
	 * Make some space for MAXSYSARGS (currently 8) 32-bit args placed
	 * into 64-bit (long) arg slots, maintaining 16 byte alignment.  Or
	 * more succinctly:
	 *
	 *	SA(MAXSYSARGS * sizeof (long)) == 64
	 */
#define	SYS_DROP	64			/* drop for args */
	subq	$SYS_DROP, %rsp
	movb	$LWP_SYS, LWP_STATE(%r14)
	movq	%r15, %rdi
	movq	%rsp, %rsi
	call	syscall_entry

	/*
	 * Fetch the arguments copied onto the kernel stack and put
	 * them in the right registers to invoke a C-style syscall handler.
	 * %rax contains the handler address.
	 *
	 * Ideas for making all this go faster of course include simply
	 * forcibly fetching 6 arguments from the user stack under lofault
	 * protection, reverting to copyin_args only when watchpoints
	 * are in effect.
	 *
	 * (If we do this, make sure that exec and libthread leave
	 * enough space at the top of the stack to ensure that we'll
	 * never do a fetch from an invalid page.)
	 *
	 * Lots of ideas here, but they won't really help with bringup B-)
	 * Correctness can't wait, performance can wait a little longer ..
	 */

	movq	%rax, %rbx
	movl	0(%rsp), %edi
	movl	8(%rsp), %esi
	movl	0x10(%rsp), %edx
	movl	0x18(%rsp), %ecx
	movl	0x20(%rsp), %r8d
	movl	0x28(%rsp), %r9d

	movq	SY_CALLC(%rbx), %rax
	INDIRECT_CALL_REG(rax)

	movq	%rbp, %rsp	/* pop the args */

	/*
	 * amd64 syscall handlers -always- return a 64-bit value in %rax.
	 * On the 32-bit kernel, they always return that value in %eax:%edx
	 * as required by the 32-bit ABI.
	 *
	 * Simulate the same behaviour by unconditionally splitting the
	 * return value in the same way.
	 */
	movq	%rax, %r13
	shrq	$32, %r13	/* upper 32-bits into %edx */
	movl	%eax, %r12d	/* lower 32-bits into %eax */

	/*
	 * Optimistically assume that there's no post-syscall
	 * work to do.  (This is to avoid having to call syscall_mstate()
	 * with interrupts disabled)
	 */
	MSTATE_TRANSITION(LMS_SYSTEM, LMS_USER)

	/*
	 * We must protect ourselves from being descheduled here;
	 * If we were, and we ended up on another cpu, or another
	 * lwp got in ahead of us, it could change the segment
	 * registers without us noticing before we return to userland.
	 */
	CLI(%r14)
	CHECK_POSTSYS_NE(%r15, %r14, %ebx)
	jne	_full_syscall_postsys32
	SIMPLE_SYSCALL_POSTSYS(%r15, %r14, %bx)

	/*
	 * Clobber %r11 as we check CR0.TS.
	 */
	ASSERT_CR0TS_ZERO(%r11)

	/*
	 * To get back to userland, we need to put the return %rip in %rcx and
	 * the return %rfl in %r11d.  The sysret instruction also arranges
	 * to fix up %cs and %ss; everything else is our responsibility.
	 */

	movl	%r12d, %eax			/* %eax: rval1 */
	movl	REGOFF_RBX(%rsp), %ebx
	/* %ecx used for return pointer */
	movl	%r13d, %edx			/* %edx: rval2 */
	movl	REGOFF_RBP(%rsp), %ebp
	movl	REGOFF_RSI(%rsp), %esi
	movl	REGOFF_RDI(%rsp), %edi

	movl	REGOFF_RFL(%rsp), %r11d		/* %r11 -> eflags */
	movl	REGOFF_RIP(%rsp), %ecx		/* %ecx -> %eip */
	/*
	 * Unlike other cases, because we need to restore the user stack pointer
	 * before exiting the kernel we must clear the microarch state before
	 * getting here. This should be safe because it means that the only
	 * values on the bus after this are based on the user's registers and
	 * potentially the addresses where we stored them. Given the constraints
	 * of sysret, that's how it has to be.
	 */
	call	x86_md_clear

	movl	REGOFF_RSP(%rsp), %esp

	ASSERT_UPCALL_MASK_IS_SET
        ALTENTRY(nopop_sys_syscall32_swapgs_sysretl)
	jmp	tr_sysretl
        SET_SIZE(nopop_sys_syscall32_swapgs_sysretl)
	/*NOTREACHED*/

_full_syscall_postsys32:
	STI
	/*
	 * Sigh, our optimism wasn't justified, put it back to LMS_SYSTEM
	 * so that we can account for the extra work it takes us to finish.
	 */
	MSTATE_TRANSITION(LMS_USER, LMS_SYSTEM)
	movq	%r15, %rdi
	movq	%r12, %rsi			/* rval1 - %eax */
	movq	%r13, %rdx			/* rval2 - %edx */
	call	syscall_exit
	MSTATE_TRANSITION(LMS_SYSTEM, LMS_USER)
	jmp	_sys_rtt
	SET_SIZE(sys_syscall32)
	SET_SIZE(brand_sys_syscall32)

/*
 * System call handler via the sysenter instruction
 * Used only for 32-bit system calls on the 64-bit kernel.
 *
 * The caller in userland has arranged that:
 *
 * -	%eax contains the syscall number
 * -	%ecx contains the user %esp
 * -	%edx contains the return %eip
 * -	the user stack contains the args to the syscall
 *
 * Hardware and (privileged) initialization code have arranged that by
 * the time the sysenter instructions completes:
 *
 * - %rip is pointing to sys_sysenter (below).
 * - %cs and %ss are set to kernel text and stack (data) selectors.
 * - %rsp is pointing at the lwp's stack
 * - interrupts have been disabled.
 *
 * Note that we are unable to return both "rvals" to userland with
 * this call, as %edx is used by the sysexit instruction.
 *
 * One final complication in this routine is its interaction with
 * single-stepping in a debugger.  For most of the system call mechanisms, the
 * CPU automatically clears the single-step flag before we enter the kernel.
 * The sysenter mechanism does not clear the flag, so a user single-stepping
 * through a libc routine may suddenly find themself single-stepping through the
 * kernel.  To detect this, kmdb and trap() both compare the trap %pc to the
 * [brand_]sys_enter addresses on each single-step trap.  If it finds that we
 * have single-stepped to a sysenter entry point, it explicitly clears the flag
 * and executes the sys_sysenter routine.
 *
 * One final complication in this final complication is the fact that we have
 * two different entry points for sysenter: brand_sys_sysenter and sys_sysenter.
 * If we enter at brand_sys_sysenter and start single-stepping through the
 * kernel with kmdb, we will eventually hit the instruction at sys_sysenter.
 * kmdb cannot distinguish between that valid single-step and the undesirable
 * one mentioned above.  To avoid this situation, we simply add a jump over the
 * instruction at sys_sysenter to make it impossible to single-step to it.
 */

	ENTRY_NP(brand_sys_sysenter)
	SWAPGS				/* kernel gsbase */
	ALTENTRY(_brand_sys_sysenter_post_swapgs)

	BRAND_CALLBACK(BRAND_CB_SYSENTER, BRAND_URET_FROM_REG(%rdx))
	/*
	 * Jump over sys_sysenter to allow single-stepping as described
	 * above.
	 */
	jmp	_sys_sysenter_post_swapgs

	ALTENTRY(sys_sysenter)
	SWAPGS				/* kernel gsbase */
	ALTENTRY(_sys_sysenter_post_swapgs)

	movq	%gs:CPU_THREAD, %r15

	movl	$U32CS_SEL, REGOFF_CS(%rsp)
	movl	%ecx, REGOFF_RSP(%rsp)		/* wrapper: %esp -> %ecx */
	movl	%edx, REGOFF_RIP(%rsp)		/* wrapper: %eip -> %edx */
	/*
	 * NOTE: none of the instructions that run before we get here should
	 * clobber bits in (R)FLAGS! This includes the kpti trampoline.
	 */
	pushfq
	popq	%r10
	movl	$UDS_SEL, REGOFF_SS(%rsp)

	/*
	 * Set the interrupt flag before storing the flags to the
	 * flags image on the stack so we can return to user with
	 * interrupts enabled if we return via sys_rtt_syscall32
	 */
	orq	$PS_IE, %r10
	movq	%r10, REGOFF_RFL(%rsp)

	movl	%edi, REGOFF_RDI(%rsp)
	movl	%esi, REGOFF_RSI(%rsp)
	movl	%ebp, REGOFF_RBP(%rsp)
	movl	%ebx, REGOFF_RBX(%rsp)
	movl	%edx, REGOFF_RDX(%rsp)
	movl	%ecx, REGOFF_RCX(%rsp)
	movl	%eax, REGOFF_RAX(%rsp)		/* wrapper: sysc# -> %eax */
	movq	$0, REGOFF_SAVFP(%rsp)
	movq	$0, REGOFF_SAVPC(%rsp)

	/*
	 * Copy these registers here in case we end up stopped with
	 * someone (like, say, /proc) messing with our register state.
	 * We don't -restore- them unless we have to in update_sregs.
	 *
	 * Since userland -can't- change fsbase or gsbase directly,
	 * we don't bother to capture them here.
	 */
	xorl	%ebx, %ebx
	movw	%ds, %bx
	movq	%rbx, REGOFF_DS(%rsp)
	movw	%es, %bx
	movq	%rbx, REGOFF_ES(%rsp)
	movw	%fs, %bx
	movq	%rbx, REGOFF_FS(%rsp)
	movw	%gs, %bx
	movq	%rbx, REGOFF_GS(%rsp)

	/*
	 * If we're trying to use TRAPTRACE though, I take that back: we're
	 * probably debugging some problem in the SWAPGS logic and want to know
	 * what the incoming gsbase was.
	 *
	 * Since we already did SWAPGS, record the KGSBASE.
	 */
#if defined(DEBUG) && defined(TRAPTRACE) && !defined(__xpv)
	movl	$MSR_AMD_KGSBASE, %ecx
	rdmsr
	movl	%eax, REGOFF_GSBASE(%rsp)
	movl	%edx, REGOFF_GSBASE+4(%rsp)
#endif

	/*
	 * Application state saved in the regs structure on the stack
	 * %eax is the syscall number
	 * %rsp is the thread's stack, %r15 is curthread
	 * REG_RSP(%rsp) is the user's stack
	 */

	SYSCALL_TRAPTRACE($TT_SYSENTER)

	movq	%rsp, %rbp

	movq	T_LWP(%r15), %r14
	ASSERT_NO_RUPDATE_PENDING(%r14)

	ENABLE_INTR_FLAGS

	/*
	 * Catch 64-bit process trying to issue sysenter instruction
	 * on Nocona based systems.
	 */
	movq	LWP_PROCP(%r14), %rax
	cmpq	$DATAMODEL_ILP32, P_MODEL(%rax)
	je	7f

	/*
	 * For a non-32-bit process, simulate a #ud, since that's what
	 * native hardware does.  The traptrace entry (above) will
	 * let you know what really happened.
	 */
	movq	$T_ILLINST, REGOFF_TRAPNO(%rsp)
	movq	REGOFF_CS(%rsp), %rdi
	movq	%rdi, REGOFF_ERR(%rsp)
	movq	%rsp, %rdi
	movq	REGOFF_RIP(%rsp), %rsi
	movl	%gs:CPU_ID, %edx
	call	trap
	jmp	_sys_rtt
7:

	MSTATE_TRANSITION(LMS_USER, LMS_SYSTEM)
	movl	REGOFF_RAX(%rsp), %eax	/* (%rax damaged by mstate calls) */

	ASSERT_LWPTOREGS(%r14, %rsp)

	incq	%gs:CPU_STATS_SYS_SYSCALL

	/*
	 * Make some space for MAXSYSARGS (currently 8) 32-bit args
	 * placed into 64-bit (long) arg slots, plus one 64-bit
	 * (long) arg count, maintaining 16 byte alignment.
	 */
	subq	$SYS_DROP, %rsp
	movb	$LWP_SYS, LWP_STATE(%r14)
	movq	%r15, %rdi
	movq	%rsp, %rsi
	call	syscall_entry

	/*
	 * Fetch the arguments copied onto the kernel stack and put
	 * them in the right registers to invoke a C-style syscall handler.
	 * %rax contains the handler address.
	 */
	movq	%rax, %rbx
	movl	0(%rsp), %edi
	movl	8(%rsp), %esi
	movl	0x10(%rsp), %edx
	movl	0x18(%rsp), %ecx
	movl	0x20(%rsp), %r8d
	movl	0x28(%rsp), %r9d

	movq	SY_CALLC(%rbx), %rax
	INDIRECT_CALL_REG(rax)

	movq	%rbp, %rsp	/* pop the args */

	/*
	 * amd64 syscall handlers -always- return a 64-bit value in %rax.
	 * On the 32-bit kernel, the always return that value in %eax:%edx
	 * as required by the 32-bit ABI.
	 *
	 * Simulate the same behaviour by unconditionally splitting the
	 * return value in the same way.
	 */
	movq	%rax, %r13
	shrq	$32, %r13	/* upper 32-bits into %edx */
	movl	%eax, %r12d	/* lower 32-bits into %eax */

	/*
	 * Optimistically assume that there's no post-syscall
	 * work to do.  (This is to avoid having to call syscall_mstate()
	 * with interrupts disabled)
	 */
	MSTATE_TRANSITION(LMS_SYSTEM, LMS_USER)

	/*
	 * We must protect ourselves from being descheduled here;
	 * If we were, and we ended up on another cpu, or another
	 * lwp got int ahead of us, it could change the segment
	 * registers without us noticing before we return to userland.
	 *
	 * This cli is undone in the tr_sysexit trampoline code.
	 */
	cli
	CHECK_POSTSYS_NE(%r15, %r14, %ebx)
	jne	_full_syscall_postsys32
	SIMPLE_SYSCALL_POSTSYS(%r15, %r14, %bx)

	/*
	 * To get back to userland, load up the 32-bit registers and
	 * sysexit back where we came from.
	 */

	/*
	 * Interrupts will be turned on by the 'sti' executed just before
	 * sysexit.  The following ensures that restoring the user's rflags
	 * doesn't enable interrupts too soon.
	 */
	andq	$_BITNOT(PS_IE), REGOFF_RFL(%rsp)

	/*
	 * Clobber %r11 as we check CR0.TS.
	 */
	ASSERT_CR0TS_ZERO(%r11)

	/*
	 * (There's no point in loading up %edx because the sysexit
	 * mechanism smashes it.)
	 */
	movl	%r12d, %eax
	movl	REGOFF_RBX(%rsp), %ebx
	movl	REGOFF_RBP(%rsp), %ebp
	movl	REGOFF_RSI(%rsp), %esi
	movl	REGOFF_RDI(%rsp), %edi

	movl	REGOFF_RIP(%rsp), %edx	/* sysexit: %edx -> %eip */
	pushq	REGOFF_RFL(%rsp)
	popfq
	movl	REGOFF_RSP(%rsp), %ecx	/* sysexit: %ecx -> %esp */
        ALTENTRY(sys_sysenter_swapgs_sysexit)
	call	x86_md_clear
	jmp	tr_sysexit
	SET_SIZE(sys_sysenter_swapgs_sysexit)
	SET_SIZE(sys_sysenter)
	SET_SIZE(_sys_sysenter_post_swapgs)
	SET_SIZE(brand_sys_sysenter)

/*
 * This is the destination of the "int $T_SYSCALLINT" interrupt gate, used by
 * the generic i386 libc to do system calls. We do a small amount of setup
 * before jumping into the existing sys_syscall32 path.
 */

	ENTRY_NP(brand_sys_syscall_int)
	SWAPGS				/* kernel gsbase */
	XPV_TRAP_POP
	call	smap_enable
	BRAND_CALLBACK(BRAND_CB_INT91, BRAND_URET_FROM_INTR_STACK())
	jmp	nopop_syscall_int

	ALTENTRY(sys_syscall_int)
	SWAPGS				/* kernel gsbase */
	XPV_TRAP_POP
	call	smap_enable

nopop_syscall_int:
	movq	%gs:CPU_THREAD, %r15
	movq	T_STACK(%r15), %rsp
	movl	%eax, %eax
	/*
	 * Set t_post_sys on this thread to force ourselves out via the slow
	 * path. It might be possible at some later date to optimize this out
	 * and use a faster return mechanism.
	 */
	movb	$1, T_POST_SYS(%r15)
	CLEAN_CS
	jmp	_syscall32_save
	/*
	 * There should be no instructions between this label and SWAPGS/IRET
	 * or we could end up breaking branded zone support. See the usage of
	 * this label in lx_brand_int80_callback and sn1_brand_int91_callback
	 * for examples.
	 *
	 * We want to swapgs to maintain the invariant that all entries into
	 * tr_iret_user are done on the user gsbase.
	 */
	ALTENTRY(sys_sysint_swapgs_iret)
	call	x86_md_clear
	SWAPGS
	jmp	tr_iret_user
	/*NOTREACHED*/
	SET_SIZE(sys_sysint_swapgs_iret)
	SET_SIZE(sys_syscall_int)
	SET_SIZE(brand_sys_syscall_int)

/*
 * Legacy 32-bit applications and old libc implementations do lcalls;
 * we should never get here because the LDT entry containing the syscall
 * segment descriptor has the "segment present" bit cleared, which means
 * we end up processing those system calls in trap() via a not-present trap.
 *
 * We do it this way because a call gate unhelpfully does -nothing- to the
 * interrupt flag bit, so an interrupt can run us just after the lcall
 * completes, but just before the swapgs takes effect.   Thus the INTR_PUSH and
 * INTR_POP paths would have to be slightly more complex to dance around
 * this problem, and end up depending explicitly on the first
 * instruction of this handler being either swapgs or cli.
 */

	ENTRY_NP(sys_lcall32)
	SWAPGS				/* kernel gsbase */
	pushq	$0
	pushq	%rbp
	movq	%rsp, %rbp
	leaq	__lcall_panic_str(%rip), %rdi
	xorl	%eax, %eax
	call	panic
	SET_SIZE(sys_lcall32)

__lcall_panic_str:
	.string	"sys_lcall32: shouldn't be here!"

/*
 * Declare a uintptr_t which covers the entire pc range of syscall
 * handlers for the stack walkers that need this.
 */
	.align	CPTRSIZE
	.globl	_allsyscalls_size
	.type	_allsyscalls_size, @object
_allsyscalls_size:
	.NWORD	. - _allsyscalls
	SET_SIZE(_allsyscalls_size)

/*
 * These are the thread context handlers for lwps using sysenter/sysexit.
 */

	/*
	 * setting this value to zero as we switch away causes the
	 * stack-pointer-on-sysenter to be NULL, ensuring that we
	 * don't silently corrupt another (preempted) thread stack
	 * when running an lwp that (somehow) didn't get sep_restore'd
	 */
	ENTRY_NP(sep_save)
	xorl	%edx, %edx
	xorl	%eax, %eax
	movl	$MSR_INTC_SEP_ESP, %ecx
	wrmsr
	ret
	SET_SIZE(sep_save)

	/*
	 * Update the kernel stack pointer as we resume onto this cpu.
	 */
	ENTRY_NP(sep_restore)
	movq	%rdi, %rdx
	shrq	$32, %rdx
	movl	%edi, %eax
	movl	$MSR_INTC_SEP_ESP, %ecx
	wrmsr
	ret
	SET_SIZE(sep_restore)