/* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2007 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ #include #include #include /* * This file contains ACPI functions that are needed by the kernel before * the ACPI module is loaded. Any functions or definitions need to be * able to deal with the possibility that ACPI doesn't get loaded, or * doesn't contain the required method. */ int (*acpi_fp_setwake)(); /* * */ int acpi_ddi_setwake(dev_info_t *dip, int level) { if (acpi_fp_setwake == NULL) return (AE_ERROR); return ((*acpi_fp_setwake)(dip, level)); } /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2008 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #if defined(__xpv) #include #endif extern int prom_debug; /* hard code realmode memory address for now */ #define BIOS_RES_BUFFER_ADDR 0x7000 #define BIOSDEV_NUM 8 #define STARTING_DRVNUM 0x80 #define FP_OFF(fp) (((uintptr_t)(fp)) & 0xFFFF) #define FP_SEG(fp) ((((uintptr_t)(fp)) >> 16) & 0xFFFF) #ifdef DEBUG int biosdebug = 0; #define dprintf(fmt) \ if (biosdebug) \ prom_printf fmt #else #define dprintf(fmt) #endif biosdev_data_t biosdev_info[BIOSDEV_NUM]; /* from 0x80 to 0x87 */ static int bios_check_extension_present(uchar_t); static int get_dev_params(uchar_t); static int read_firstblock(uchar_t drivenum); static int drive_present(uchar_t drivenum); static void reset_disk(uchar_t drivenum); static int is_eltorito(uchar_t drivenum); #if !defined(__xpv) void startup_bios_disk() { uchar_t drivenum; int got_devparams = 0; int got_first_block = 0; uchar_t name[20]; dev_info_t *devi; int extensions; for (drivenum = 0x80; drivenum < (0x80 + BIOSDEV_NUM); drivenum++) { if (!drive_present(drivenum)) continue; extensions = bios_check_extension_present(drivenum); /* * If we're booting from an Eltorito CD/DVD image, there's * no need to get the device parameters or read the first block * because we'll never install onto this device. */ if (extensions && is_eltorito(drivenum)) continue; if (extensions && get_dev_params(drivenum)) got_devparams = 1; else got_devparams = 0; if ((got_first_block = read_firstblock(drivenum)) == 0) { /* retry */ got_first_block = read_firstblock(drivenum); } if (got_devparams || got_first_block) { (void) sprintf((char *)name, "biosdev-0x%x", drivenum); devi = ddi_root_node(); (void) e_ddi_prop_update_byte_array(DDI_DEV_T_NONE, devi, (char *)name, (uchar_t *)&biosdev_info[drivenum - 0x80], sizeof (biosdev_data_t)); } } } #endif static int bios_check_extension_present(uchar_t drivenum) { struct bop_regs rp = {0}; extern struct bootops *bootops; rp.eax.word.ax = 0x4100; rp.ebx.word.bx = 0x55AA; rp.edx.word.dx = drivenum; /* make sure we have extension support */ BOP_DOINT(bootops, 0x13, &rp); if (((rp.eflags & PS_C) != 0) || (rp.ebx.word.bx != 0xAA55)) { dprintf(("bios_check_extension_present int13 fn 41 " "failed %d bx = %x\n", rp.eflags, rp.ebx.word.bx)); return (0); } if ((rp.ecx.word.cx & 0x7) == 0) { dprintf(("bios_check_extension_present get device parameters " "not supported cx = %x\n", rp.ecx.word.cx)); return (0); } return (1); } static int get_dev_params(uchar_t drivenum) { struct bop_regs rp = {0}; fn48_t *bufp; extern struct bootops *bootops; int i; int index; uchar_t *tmp; dprintf(("In get_dev_params\n")); bufp = (fn48_t *)BIOS_RES_BUFFER_ADDR; /* * We cannot use bzero here as we're initializing data * at an address below kernel base. */ for (i = 0; i < sizeof (*bufp); i++) ((uchar_t *)bufp)[i] = 0; bufp->buflen = sizeof (*bufp); rp.eax.word.ax = 0x4800; rp.edx.byte.dl = drivenum; rp.esi.word.si = (uint16_t)FP_OFF((uint_t)(uintptr_t)bufp); rp.ds = FP_SEG((uint_t)(uintptr_t)bufp); BOP_DOINT(bootops, 0x13, &rp); if ((rp.eflags & PS_C) != 0) { dprintf(("EDD FAILED on drive eflag = %x ah= %x\n", rp.eflags, rp.eax.byte.ah)); return (0); } index = drivenum - 0x80; biosdev_info[index].edd_valid = 1; /* * Some compilers turn a structure copy into a call * to memcpy. Since we are copying data below kernel * base intentionally, and memcpy asserts that's not * the case, we do the copy manually here. */ tmp = (uchar_t *)&biosdev_info[index].fn48_dev_params; for (i = 0; i < sizeof (*bufp); i++) tmp[i] = ((uchar_t *)bufp)[i]; return (1); } static int drive_present(uchar_t drivenum) { struct bop_regs rp = {0}; rp.eax.byte.ah = 0x8; /* get params */ rp.edx.byte.dl = drivenum; BOP_DOINT(bootops, 0x13, &rp); if (((rp.eflags & PS_C) != 0) || rp.eax.byte.ah != 0) { dprintf(("drive not present drivenum %x eflag %x ah %x\n", drivenum, rp.eflags, rp.eax.byte.ah)); return (0); } dprintf(("drive-present %x\n", drivenum)); return (1); } static void reset_disk(uchar_t drivenum) { struct bop_regs rp = {0}; int status; rp.eax.byte.ah = 0x0; /* reset disk */ rp.edx.byte.dl = drivenum; BOP_DOINT(bootops, 0x13, &rp); status = rp.eax.byte.ah; if (((rp.eflags & PS_C) != 0) || status != 0) dprintf(("Bad disk reset driv %x, status %x\n", drivenum, status)); } /* Get first block */ static int read_firstblock(uchar_t drivenum) { struct bop_regs rp = {0}; caddr_t bufp; uchar_t status; int i, index; reset_disk(drivenum); bufp = (caddr_t)BIOS_RES_BUFFER_ADDR; rp.eax.byte.ah = 0x2; /* Read disk */ rp.eax.byte.al = 1; /* nsect */ rp.ecx.byte.ch = 0; /* cyl & 0xff */ rp.ecx.byte.cl = 1; /* cyl >> 2 & 0xc0 (sector number) */ rp.edx.byte.dh = 0; /* head */ rp.edx.byte.dl = drivenum; /* drivenum */ /* es:bx is buf address */ rp.ebx.word.bx = (uint16_t)FP_OFF((uint_t)(uintptr_t)bufp); rp.es = FP_SEG((uint_t)(uintptr_t)bufp); BOP_DOINT(bootops, 0x13, &rp); status = rp.eax.byte.ah; if (((rp.eflags & PS_C) != 0) || status != 0) { dprintf(("read_firstblock AH not clear %x \n", status)); return (0); } dprintf(("drivenum %x uid at 0x1b8 is %x\n", drivenum, *(uint32_t *)(bufp +0x1b8))); index = drivenum - 0x80; biosdev_info[index].first_block_valid = 1; for (i = 0; i < 512; i++) biosdev_info[index].first_block[i] = *((uchar_t *)bufp + i); return (1); } static int is_eltorito(uchar_t drivenum) { struct bop_regs rp = {0}; fn4b_t *bufp; extern struct bootops *bootops; int i; dprintf(("In is_eltorito\n")); bufp = (fn4b_t *)BIOS_RES_BUFFER_ADDR; /* * We cannot use bzero here as we're initializing data * at an address below kernel base. */ for (i = 0; i < sizeof (*bufp); i++) ((uchar_t *)bufp)[i] = 0; bufp->pkt_size = sizeof (*bufp); rp.eax.word.ax = 0x4b01; rp.edx.byte.dl = drivenum; rp.esi.word.si = (uint16_t)FP_OFF((uint_t)(uintptr_t)bufp); rp.ds = FP_SEG((uint_t)(uintptr_t)bufp); BOP_DOINT(bootops, 0x13, &rp); if ((rp.eflags & PS_C) != 0 || bufp->drivenum != drivenum) { dprintf(("fn 0x4b01 FAILED on drive " "eflags=%x ah=%x drivenum=%x\n", rp.eflags, rp.eax.byte.ah, bufp->drivenum)); return (0); } if (prom_debug) prom_printf("INT13 FN4B01 mtype => %x", bufp->boot_mtype); return (1); } /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2010 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. * Copyright (c) 2018, Joyent, Inc. */ /* * Public interface to routines implemented by CPU modules */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* * Set to force cmi_init to fail. */ int cmi_no_init = 0; /* * Set to avoid MCA initialization. */ int cmi_no_mca_init = 0; /* * If cleared for debugging we will not attempt to load a model-specific * cpu module but will load the generic cpu module instead. */ int cmi_force_generic = 0; /* * If cleared for debugging, we will suppress panicking on fatal hardware * errors. This should *only* be used for debugging; it use can and will * cause data corruption if actual hardware errors are detected by the system. */ int cmi_panic_on_uncorrectable_error = 1; /* * Subdirectory (relative to the module search path) in which we will * look for cpu modules. */ #define CPUMOD_SUBDIR "cpu" /* * CPU modules have a filenames such as "cpu.AuthenticAMD.15" and * "cpu.generic" - the "cpu" prefix is specified by the following. */ #define CPUMOD_PREFIX "cpu" /* * Structure used to keep track of cpu modules we have loaded and their ops */ typedef struct cmi { struct cmi *cmi_next; struct cmi *cmi_prev; const cmi_ops_t *cmi_ops; struct modctl *cmi_modp; uint_t cmi_refcnt; } cmi_t; static cmi_t *cmi_list; static const cmi_mc_ops_t *cmi_mc_global_ops; static void *cmi_mc_global_data; static kmutex_t cmi_load_lock; /* * Functions we need from cmi_hw.c that are not part of the cpu_module.h * interface. */ extern cmi_hdl_t cmi_hdl_create(enum cmi_hdl_class, uint_t, uint_t, uint_t); extern void cmi_hdl_destroy(cmi_hdl_t ophdl); extern void cmi_hdl_setcmi(cmi_hdl_t, void *, void *); extern void *cmi_hdl_getcmi(cmi_hdl_t); extern void cmi_hdl_setmc(cmi_hdl_t, const struct cmi_mc_ops *, void *); extern void cmi_hdl_inj_begin(cmi_hdl_t); extern void cmi_hdl_inj_end(cmi_hdl_t); extern void cmi_read_smbios(cmi_hdl_t); #define HDL2CMI(hdl) cmi_hdl_getcmi(hdl) #define CMI_OPS(cmi) (cmi)->cmi_ops #define CMI_OP_PRESENT(cmi, op) ((cmi) && CMI_OPS(cmi)->op != NULL) #define CMI_MATCH_VENDOR 0 /* Just match on vendor */ #define CMI_MATCH_FAMILY 1 /* Match down to family */ #define CMI_MATCH_MODEL 2 /* Match down to model */ #define CMI_MATCH_STEPPING 3 /* Match down to stepping */ static void cmi_link(cmi_t *cmi) { ASSERT(MUTEX_HELD(&cmi_load_lock)); cmi->cmi_prev = NULL; cmi->cmi_next = cmi_list; if (cmi_list != NULL) cmi_list->cmi_prev = cmi; cmi_list = cmi; } static void cmi_unlink(cmi_t *cmi) { ASSERT(MUTEX_HELD(&cmi_load_lock)); ASSERT(cmi->cmi_refcnt == 0); if (cmi->cmi_prev != NULL) cmi->cmi_prev = cmi->cmi_next; if (cmi->cmi_next != NULL) cmi->cmi_next->cmi_prev = cmi->cmi_prev; if (cmi_list == cmi) cmi_list = cmi->cmi_next; } /* * Hold the module in memory. We call to CPU modules without using the * stubs mechanism, so these modules must be manually held in memory. * The mod_ref acts as if another loaded module has a dependency on us. */ static void cmi_hold(cmi_t *cmi) { ASSERT(MUTEX_HELD(&cmi_load_lock)); mutex_enter(&mod_lock); cmi->cmi_modp->mod_ref++; mutex_exit(&mod_lock); cmi->cmi_refcnt++; } static void cmi_rele(cmi_t *cmi) { ASSERT(MUTEX_HELD(&cmi_load_lock)); mutex_enter(&mod_lock); cmi->cmi_modp->mod_ref--; mutex_exit(&mod_lock); if (--cmi->cmi_refcnt == 0) { cmi_unlink(cmi); kmem_free(cmi, sizeof (cmi_t)); } } static cmi_ops_t * cmi_getops(modctl_t *modp) { cmi_ops_t *ops; if ((ops = (cmi_ops_t *)modlookup_by_modctl(modp, "_cmi_ops")) == NULL) { cmn_err(CE_WARN, "cpu module '%s' is invalid: no _cmi_ops " "found", modp->mod_modname); return (NULL); } if (ops->cmi_init == NULL) { cmn_err(CE_WARN, "cpu module '%s' is invalid: no cmi_init " "entry point", modp->mod_modname); return (NULL); } return (ops); } static cmi_t * cmi_load_modctl(modctl_t *modp) { cmi_ops_t *ops; uintptr_t ver; cmi_t *cmi; cmi_api_ver_t apiver; ASSERT(MUTEX_HELD(&cmi_load_lock)); for (cmi = cmi_list; cmi != NULL; cmi = cmi->cmi_next) { if (cmi->cmi_modp == modp) return (cmi); } if ((ver = modlookup_by_modctl(modp, "_cmi_api_version")) == 0) { /* * Apparently a cpu module before versioning was introduced - * we call this version 0. */ apiver = CMI_API_VERSION_0; } else { apiver = *((cmi_api_ver_t *)ver); if (!CMI_API_VERSION_CHKMAGIC(apiver)) { cmn_err(CE_WARN, "cpu module '%s' is invalid: " "_cmi_api_version 0x%x has bad magic", modp->mod_modname, apiver); return (NULL); } } if (apiver != CMI_API_VERSION) { cmn_err(CE_WARN, "cpu module '%s' has API version %d, " "kernel requires API version %d", modp->mod_modname, CMI_API_VERSION_TOPRINT(apiver), CMI_API_VERSION_TOPRINT(CMI_API_VERSION)); return (NULL); } if ((ops = cmi_getops(modp)) == NULL) return (NULL); cmi = kmem_zalloc(sizeof (*cmi), KM_SLEEP); cmi->cmi_ops = ops; cmi->cmi_modp = modp; cmi_link(cmi); return (cmi); } static int cmi_cpu_match(cmi_hdl_t hdl1, cmi_hdl_t hdl2, int match) { if (match >= CMI_MATCH_VENDOR && cmi_hdl_vendor(hdl1) != cmi_hdl_vendor(hdl2)) return (0); if (match >= CMI_MATCH_FAMILY && cmi_hdl_family(hdl1) != cmi_hdl_family(hdl2)) return (0); if (match >= CMI_MATCH_MODEL && cmi_hdl_model(hdl1) != cmi_hdl_model(hdl2)) return (0); if (match >= CMI_MATCH_STEPPING && cmi_hdl_stepping(hdl1) != cmi_hdl_stepping(hdl2)) return (0); return (1); } static int cmi_search_list_cb(cmi_hdl_t whdl, void *arg1, void *arg2, void *arg3) { cmi_hdl_t thdl = (cmi_hdl_t)arg1; int match = *((int *)arg2); cmi_hdl_t *rsltp = (cmi_hdl_t *)arg3; if (cmi_cpu_match(thdl, whdl, match)) { cmi_hdl_hold(whdl); /* short-term hold */ *rsltp = whdl; return (CMI_HDL_WALK_DONE); } else { return (CMI_HDL_WALK_NEXT); } } static cmi_t * cmi_search_list(cmi_hdl_t hdl, int match) { cmi_hdl_t dhdl = NULL; cmi_t *cmi = NULL; ASSERT(MUTEX_HELD(&cmi_load_lock)); cmi_hdl_walk(cmi_search_list_cb, (void *)hdl, (void *)&match, &dhdl); if (dhdl) { cmi = HDL2CMI(dhdl); cmi_hdl_rele(dhdl); /* held in cmi_search_list_cb */ } return (cmi); } static cmi_t * cmi_load_module(cmi_hdl_t hdl, int match, int *chosenp) { modctl_t *modp; cmi_t *cmi; int modid; uint_t s[3]; ASSERT(MUTEX_HELD(&cmi_load_lock)); ASSERT(match == CMI_MATCH_STEPPING || match == CMI_MATCH_MODEL || match == CMI_MATCH_FAMILY || match == CMI_MATCH_VENDOR); /* * Have we already loaded a module for a cpu with the same * vendor/family/model/stepping? */ if ((cmi = cmi_search_list(hdl, match)) != NULL) { cmi_hold(cmi); return (cmi); } s[0] = cmi_hdl_family(hdl); s[1] = cmi_hdl_model(hdl); s[2] = cmi_hdl_stepping(hdl); modid = modload_qualified(CPUMOD_SUBDIR, CPUMOD_PREFIX, cmi_hdl_vendorstr(hdl), ".", s, match, chosenp); if (modid == -1) return (NULL); modp = mod_hold_by_id(modid); cmi = cmi_load_modctl(modp); if (cmi) cmi_hold(cmi); mod_release_mod(modp); return (cmi); } /* * Try to load a cpu module with specific support for this chip type. */ static cmi_t * cmi_load_specific(cmi_hdl_t hdl, void **datap) { cmi_t *cmi; int err; int i; ASSERT(MUTEX_HELD(&cmi_load_lock)); for (i = CMI_MATCH_STEPPING; i >= CMI_MATCH_VENDOR; i--) { int suffixlevel; if ((cmi = cmi_load_module(hdl, i, &suffixlevel)) == NULL) return (NULL); /* * A module has loaded and has a _cmi_ops structure, and the * module has been held for this instance. Call its cmi_init * entry point - we expect success (0) or ENOTSUP. */ if ((err = cmi->cmi_ops->cmi_init(hdl, datap)) == 0) { if (boothowto & RB_VERBOSE) { printf("initialized cpu module '%s' on " "chip %d core %d strand %d\n", cmi->cmi_modp->mod_modname, cmi_hdl_chipid(hdl), cmi_hdl_coreid(hdl), cmi_hdl_strandid(hdl)); } return (cmi); } else if (err != ENOTSUP) { cmn_err(CE_WARN, "failed to init cpu module '%s' on " "chip %d core %d strand %d: err=%d\n", cmi->cmi_modp->mod_modname, cmi_hdl_chipid(hdl), cmi_hdl_coreid(hdl), cmi_hdl_strandid(hdl), err); } /* * The module failed or declined to init, so release * it and update i to be equal to the number * of suffices actually used in the last module path. */ cmi_rele(cmi); i = suffixlevel; } return (NULL); } /* * Load the generic IA32 MCA cpu module, which may still supplement * itself with model-specific support through cpu model-specific modules. */ static cmi_t * cmi_load_generic(cmi_hdl_t hdl, void **datap) { modctl_t *modp; cmi_t *cmi; int modid; int err; ASSERT(MUTEX_HELD(&cmi_load_lock)); if ((modid = modload(CPUMOD_SUBDIR, CPUMOD_PREFIX ".generic")) == -1) return (NULL); modp = mod_hold_by_id(modid); cmi = cmi_load_modctl(modp); if (cmi) cmi_hold(cmi); mod_release_mod(modp); if (cmi == NULL) return (NULL); if ((err = cmi->cmi_ops->cmi_init(hdl, datap)) != 0) { if (err != ENOTSUP) cmn_err(CE_WARN, CPUMOD_PREFIX ".generic failed to " "init: err=%d", err); cmi_rele(cmi); return (NULL); } return (cmi); } cmi_hdl_t cmi_init(enum cmi_hdl_class class, uint_t chipid, uint_t coreid, uint_t strandid) { cmi_t *cmi = NULL; cmi_hdl_t hdl; void *data; if (cmi_no_init) { cmi_no_mca_init = 1; return (NULL); } mutex_enter(&cmi_load_lock); if ((hdl = cmi_hdl_create(class, chipid, coreid, strandid)) == NULL) { mutex_exit(&cmi_load_lock); cmn_err(CE_WARN, "There will be no MCA support on chip %d " "core %d strand %d (cmi_hdl_create returned NULL)\n", chipid, coreid, strandid); return (NULL); } if (!cmi_force_generic) cmi = cmi_load_specific(hdl, &data); if (cmi == NULL && (cmi = cmi_load_generic(hdl, &data)) == NULL) { cmn_err(CE_WARN, "There will be no MCA support on chip %d " "core %d strand %d\n", chipid, coreid, strandid); cmi_hdl_rele(hdl); mutex_exit(&cmi_load_lock); return (NULL); } cmi_hdl_setcmi(hdl, cmi, data); cms_init(hdl); cmi_read_smbios(hdl); mutex_exit(&cmi_load_lock); return (hdl); } /* * cmi_fini is called on DR deconfigure of a cpu resource. * It should not be called at simple offline of a cpu. */ void cmi_fini(cmi_hdl_t hdl) { cmi_t *cmi = HDL2CMI(hdl); if (cms_present(hdl)) cms_fini(hdl); if (CMI_OP_PRESENT(cmi, cmi_fini)) CMI_OPS(cmi)->cmi_fini(hdl); cmi_hdl_destroy(hdl); } /* * cmi_post_startup is called from post_startup for the boot cpu only (no * other cpus are started yet). */ void cmi_post_startup(void) { cmi_hdl_t hdl; cmi_t *cmi; if (cmi_no_mca_init != 0 || (hdl = cmi_hdl_any()) == NULL) /* short-term hold */ return; cmi = HDL2CMI(hdl); if (CMI_OP_PRESENT(cmi, cmi_post_startup)) CMI_OPS(cmi)->cmi_post_startup(hdl); cmi_hdl_rele(hdl); } /* * Called just once from start_other_cpus when all processors are started. * This will not be called for each cpu, so the registered op must not * assume it is called as such. We are not necessarily executing on * the boot cpu. */ void cmi_post_mpstartup(void) { cmi_hdl_t hdl; cmi_t *cmi; if (cmi_no_mca_init != 0 || (hdl = cmi_hdl_any()) == NULL) /* short-term hold */ return; cmi = HDL2CMI(hdl); if (CMI_OP_PRESENT(cmi, cmi_post_mpstartup)) CMI_OPS(cmi)->cmi_post_mpstartup(hdl); cmi_hdl_rele(hdl); } void cmi_faulted_enter(cmi_hdl_t hdl) { cmi_t *cmi = HDL2CMI(hdl); if (cmi_no_mca_init != 0) return; if (CMI_OP_PRESENT(cmi, cmi_faulted_enter)) CMI_OPS(cmi)->cmi_faulted_enter(hdl); } void cmi_faulted_exit(cmi_hdl_t hdl) { cmi_t *cmi = HDL2CMI(hdl); if (cmi_no_mca_init != 0) return; if (CMI_OP_PRESENT(cmi, cmi_faulted_exit)) CMI_OPS(cmi)->cmi_faulted_exit(hdl); } void cmi_mca_init(cmi_hdl_t hdl) { cmi_t *cmi; if (cmi_no_mca_init != 0) return; cmi = HDL2CMI(hdl); if (CMI_OP_PRESENT(cmi, cmi_mca_init)) CMI_OPS(cmi)->cmi_mca_init(hdl); } #define CMI_RESPONSE_PANIC 0x0 /* panic must have value 0 */ #define CMI_RESPONSE_NONE 0x1 #define CMI_RESPONSE_CKILL 0x2 #define CMI_RESPONSE_REBOOT 0x3 /* not implemented */ #define CMI_RESPONSE_ONTRAP_PROT 0x4 #define CMI_RESPONSE_LOFAULT_PROT 0x5 /* * Return 0 if we will panic in response to this machine check, otherwise * non-zero. If the caller is cmi_mca_trap in this file then the nonzero * return values are to be interpreted from CMI_RESPONSE_* above. * * This function must just return what will be done without actually * doing anything; this includes not changing the regs. */ int cmi_mce_response(struct regs *rp, uint64_t disp) { int panicrsp = cmi_panic_on_uncorrectable_error ? CMI_RESPONSE_PANIC : CMI_RESPONSE_NONE; on_trap_data_t *otp; ASSERT(rp != NULL); /* don't call for polling, only on #MC */ /* * If no bits are set in the disposition then there is nothing to * worry about and we do not need to trampoline to ontrap or * lofault handlers. */ if (disp == 0) return (CMI_RESPONSE_NONE); /* * Unconstrained errors cannot be forgiven, even by ontrap or * lofault protection. The data is not poisoned and may not * even belong to the trapped context - eg a writeback of * data that is found to be bad. */ if (disp & CMI_ERRDISP_UC_UNCONSTRAINED) return (panicrsp); /* * ontrap OT_DATA_EC and lofault protection forgive any disposition * other than unconstrained, even those normally forced fatal. */ if ((otp = curthread->t_ontrap) != NULL && otp->ot_prot & OT_DATA_EC) return (CMI_RESPONSE_ONTRAP_PROT); else if (curthread->t_lofault) return (CMI_RESPONSE_LOFAULT_PROT); /* * Forced-fatal errors are terminal even in user mode. */ if (disp & CMI_ERRDISP_FORCEFATAL) return (panicrsp); /* * If the trapped context is corrupt or we have no instruction pointer * to resume at (and aren't trampolining to a fault handler) * then in the kernel case we must panic and in usermode we * kill the affected contract. */ if (disp & (CMI_ERRDISP_CURCTXBAD | CMI_ERRDISP_RIPV_INVALID)) return (USERMODE(rp->r_cs) ? CMI_RESPONSE_CKILL : panicrsp); /* * Anything else is harmless */ return (CMI_RESPONSE_NONE); } int cma_mca_trap_panic_suppressed = 0; static void cmi_mca_panic(void) { if (cmi_panic_on_uncorrectable_error) { fm_panic("Unrecoverable Machine-Check Exception"); } else { cmn_err(CE_WARN, "suppressing panic from fatal #mc"); cma_mca_trap_panic_suppressed++; } } int cma_mca_trap_contract_kills = 0; int cma_mca_trap_ontrap_forgiven = 0; int cma_mca_trap_lofault_forgiven = 0; /* * Native #MC handler - we branch to here from mcetrap */ /*ARGSUSED*/ void cmi_mca_trap(struct regs *rp) { #ifndef __xpv cmi_hdl_t hdl = NULL; uint64_t disp; cmi_t *cmi; int s; if (cmi_no_mca_init != 0) return; /* * This function can call cmn_err, and the cpu module cmi_mca_trap * entry point may also elect to call cmn_err (e.g., if it can't * log the error onto an errorq, say very early in boot). * We need to let cprintf know that we must not block. */ s = spl8(); if ((hdl = cmi_hdl_lookup(CMI_HDL_NATIVE, cmi_ntv_hwchipid(CPU), cmi_ntv_hwcoreid(CPU), cmi_ntv_hwstrandid(CPU))) == NULL || (cmi = HDL2CMI(hdl)) == NULL || !CMI_OP_PRESENT(cmi, cmi_mca_trap)) { cmn_err(CE_WARN, "#MC exception on cpuid %d: %s", CPU->cpu_id, hdl ? "handle lookup ok but no #MC handler found" : "handle lookup failed"); if (hdl != NULL) cmi_hdl_rele(hdl); splx(s); return; } disp = CMI_OPS(cmi)->cmi_mca_trap(hdl, rp); switch (cmi_mce_response(rp, disp)) { default: cmn_err(CE_WARN, "Invalid response from cmi_mce_response"); /*FALLTHRU*/ case CMI_RESPONSE_PANIC: cmi_mca_panic(); break; case CMI_RESPONSE_NONE: break; case CMI_RESPONSE_CKILL: ttolwp(curthread)->lwp_pcb.pcb_flags |= ASYNC_HWERR; aston(curthread); cma_mca_trap_contract_kills++; break; case CMI_RESPONSE_ONTRAP_PROT: { on_trap_data_t *otp = curthread->t_ontrap; otp->ot_trap = OT_DATA_EC; rp->r_pc = otp->ot_trampoline; cma_mca_trap_ontrap_forgiven++; break; } case CMI_RESPONSE_LOFAULT_PROT: rp->r_r0 = EFAULT; rp->r_pc = curthread->t_lofault; cma_mca_trap_lofault_forgiven++; break; } cmi_hdl_rele(hdl); splx(s); #endif /* __xpv */ } void cmi_hdl_poke(cmi_hdl_t hdl) { cmi_t *cmi = HDL2CMI(hdl); if (!CMI_OP_PRESENT(cmi, cmi_hdl_poke)) return; CMI_OPS(cmi)->cmi_hdl_poke(hdl); } #ifndef __xpv void cmi_cmci_trap() { cmi_hdl_t hdl = NULL; cmi_t *cmi; if (cmi_no_mca_init != 0) return; if ((hdl = cmi_hdl_lookup(CMI_HDL_NATIVE, cmi_ntv_hwchipid(CPU), cmi_ntv_hwcoreid(CPU), cmi_ntv_hwstrandid(CPU))) == NULL || (cmi = HDL2CMI(hdl)) == NULL || !CMI_OP_PRESENT(cmi, cmi_cmci_trap)) { cmn_err(CE_WARN, "CMCI interrupt on cpuid %d: %s", CPU->cpu_id, hdl ? "handle lookup ok but no CMCI handler found" : "handle lookup failed"); if (hdl != NULL) cmi_hdl_rele(hdl); return; } CMI_OPS(cmi)->cmi_cmci_trap(hdl); cmi_hdl_rele(hdl); } #endif /* __xpv */ void cmi_mc_register(cmi_hdl_t hdl, const cmi_mc_ops_t *mcops, void *mcdata) { if (!cmi_no_mca_init) cmi_hdl_setmc(hdl, mcops, mcdata); } cmi_errno_t cmi_mc_register_global(const cmi_mc_ops_t *mcops, void *mcdata) { if (!cmi_no_mca_init) { if (cmi_mc_global_ops != NULL || cmi_mc_global_data != NULL || mcops == NULL || mcops->cmi_mc_patounum == NULL || mcops->cmi_mc_unumtopa == NULL) { return (CMIERR_UNKNOWN); } cmi_mc_global_data = mcdata; cmi_mc_global_ops = mcops; } return (CMI_SUCCESS); } void cmi_mc_sw_memscrub_disable(void) { memscrub_disable(); } cmi_errno_t cmi_mc_patounum(uint64_t pa, uint8_t valid_hi, uint8_t valid_lo, uint32_t synd, int syndtype, mc_unum_t *up) { const struct cmi_mc_ops *mcops; cmi_hdl_t hdl; cmi_errno_t rv; if (cmi_no_mca_init) return (CMIERR_MC_ABSENT); if (cmi_mc_global_ops != NULL) { if (cmi_mc_global_ops->cmi_mc_patounum == NULL) return (CMIERR_MC_NOTSUP); return (cmi_mc_global_ops->cmi_mc_patounum(cmi_mc_global_data, pa, valid_hi, valid_lo, synd, syndtype, up)); } if ((hdl = cmi_hdl_any()) == NULL) /* short-term hold */ return (CMIERR_MC_ABSENT); if ((mcops = cmi_hdl_getmcops(hdl)) == NULL || mcops->cmi_mc_patounum == NULL) { cmi_hdl_rele(hdl); return (CMIERR_MC_NOTSUP); } rv = mcops->cmi_mc_patounum(cmi_hdl_getmcdata(hdl), pa, valid_hi, valid_lo, synd, syndtype, up); cmi_hdl_rele(hdl); return (rv); } cmi_errno_t cmi_mc_unumtopa(mc_unum_t *up, nvlist_t *nvl, uint64_t *pap) { const struct cmi_mc_ops *mcops; cmi_hdl_t hdl; cmi_errno_t rv; nvlist_t *hcsp; if (up != NULL && nvl != NULL) return (CMIERR_API); /* convert from just one form */ if (cmi_no_mca_init) return (CMIERR_MC_ABSENT); if (cmi_mc_global_ops != NULL) { if (cmi_mc_global_ops->cmi_mc_unumtopa == NULL) return (CMIERR_MC_NOTSUP); return (cmi_mc_global_ops->cmi_mc_unumtopa(cmi_mc_global_data, up, nvl, pap)); } if ((hdl = cmi_hdl_any()) == NULL) /* short-term hold */ return (CMIERR_MC_ABSENT); if ((mcops = cmi_hdl_getmcops(hdl)) == NULL || mcops->cmi_mc_unumtopa == NULL) { cmi_hdl_rele(hdl); if (nvl != NULL && nvlist_lookup_nvlist(nvl, FM_FMRI_HC_SPECIFIC, &hcsp) == 0 && (nvlist_lookup_uint64(hcsp, "asru-" FM_FMRI_HC_SPECIFIC_PHYSADDR, pap) == 0 || nvlist_lookup_uint64(hcsp, FM_FMRI_HC_SPECIFIC_PHYSADDR, pap) == 0)) { return (CMIERR_MC_PARTIALUNUMTOPA); } else { return (mcops && mcops->cmi_mc_unumtopa == NULL ? CMIERR_MC_NOTSUP : CMIERR_MC_ABSENT); } } rv = mcops->cmi_mc_unumtopa(cmi_hdl_getmcdata(hdl), up, nvl, pap); cmi_hdl_rele(hdl); return (rv); } void cmi_mc_logout(cmi_hdl_t hdl, boolean_t ismc, boolean_t sync) { const struct cmi_mc_ops *mcops; if (cmi_no_mca_init) return; if (cmi_mc_global_ops != NULL) mcops = cmi_mc_global_ops; else mcops = cmi_hdl_getmcops(hdl); if (mcops != NULL && mcops->cmi_mc_logout != NULL) mcops->cmi_mc_logout(hdl, ismc, sync); } cmi_errno_t cmi_hdl_msrinject(cmi_hdl_t hdl, cmi_mca_regs_t *regs, uint_t nregs, int force) { cmi_t *cmi = cmi_hdl_getcmi(hdl); cmi_errno_t rc; if (!CMI_OP_PRESENT(cmi, cmi_msrinject)) return (CMIERR_NOTSUP); cmi_hdl_inj_begin(hdl); rc = CMI_OPS(cmi)->cmi_msrinject(hdl, regs, nregs, force); cmi_hdl_inj_end(hdl); return (rc); } boolean_t cmi_panic_on_ue(void) { return (cmi_panic_on_uncorrectable_error ? B_TRUE : B_FALSE); } void cmi_panic_callback(void) { cmi_hdl_t hdl; cmi_t *cmi; if (cmi_no_mca_init || (hdl = cmi_hdl_any()) == NULL) return; cmi = cmi_hdl_getcmi(hdl); if (CMI_OP_PRESENT(cmi, cmi_panic_callback)) CMI_OPS(cmi)->cmi_panic_callback(); cmi_hdl_rele(hdl); } const char * cmi_hdl_chipident(cmi_hdl_t hdl) { cmi_t *cmi = cmi_hdl_getcmi(hdl); if (!CMI_OP_PRESENT(cmi, cmi_ident)) return (NULL); return (CMI_OPS(cmi)->cmi_ident(hdl)); } /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2019, Joyent, Inc. * Copyright 2023 Oxide Computer Company */ /* * Copyright (c) 2010, Intel Corporation. * All rights reserved. */ /* * CPU Module Interface - hardware abstraction. */ #ifdef __xpv #include #endif #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* * Variable which determines if the SMBIOS supports x86 generic topology; or * if legacy topolgy enumeration will occur. */ extern int x86gentopo_legacy; /* * Outside of this file consumers use the opaque cmi_hdl_t. This * definition is duplicated in the generic_cpu mdb module, so keep * them in-sync when making changes. */ typedef struct cmi_hdl_impl { enum cmi_hdl_class cmih_class; /* Handle nature */ const struct cmi_hdl_ops *cmih_ops; /* Operations vector */ uint_t cmih_chipid; /* Chipid of cpu resource */ uint_t cmih_procnodeid; /* Nodeid of cpu resource */ uint_t cmih_coreid; /* Core within die */ uint_t cmih_strandid; /* Thread within core */ uint_t cmih_procnodes_per_pkg; /* Nodes in a processor */ boolean_t cmih_mstrand; /* cores are multithreaded */ volatile uint32_t *cmih_refcntp; /* Reference count pointer */ uint64_t cmih_msrsrc; /* MSR data source flags */ void *cmih_hdlpriv; /* cmi_hw.c private data */ void *cmih_spec; /* cmi_hdl_{set,get}_specific */ void *cmih_cmi; /* cpu mod control structure */ void *cmih_cmidata; /* cpu mod private data */ const struct cmi_mc_ops *cmih_mcops; /* Memory-controller ops */ void *cmih_mcdata; /* Memory-controller data */ uint64_t cmih_flags; /* See CMIH_F_* below */ uint16_t cmih_smbiosid; /* SMBIOS Type 4 struct ID */ uint_t cmih_smb_chipid; /* SMBIOS factored chipid */ nvlist_t *cmih_smb_bboard; /* SMBIOS bboard nvlist */ } cmi_hdl_impl_t; #define IMPLHDL(ophdl) ((cmi_hdl_impl_t *)ophdl) #define HDLOPS(hdl) ((hdl)->cmih_ops) #define CMIH_F_INJACTV 0x1ULL #define CMIH_F_DEAD 0x2ULL /* * Ops structure for handle operations. */ struct cmi_hdl_ops { /* * These ops are required in an implementation. */ uint_t (*cmio_vendor)(cmi_hdl_impl_t *); const char *(*cmio_vendorstr)(cmi_hdl_impl_t *); uint_t (*cmio_family)(cmi_hdl_impl_t *); uint_t (*cmio_model)(cmi_hdl_impl_t *); uint_t (*cmio_stepping)(cmi_hdl_impl_t *); uint_t (*cmio_chipid)(cmi_hdl_impl_t *); uint_t (*cmio_procnodeid)(cmi_hdl_impl_t *); uint_t (*cmio_coreid)(cmi_hdl_impl_t *); uint_t (*cmio_strandid)(cmi_hdl_impl_t *); uint_t (*cmio_procnodes_per_pkg)(cmi_hdl_impl_t *); uint_t (*cmio_strand_apicid)(cmi_hdl_impl_t *); x86_chiprev_t (*cmio_chiprev)(cmi_hdl_impl_t *); const char *(*cmio_chiprevstr)(cmi_hdl_impl_t *); uint32_t (*cmio_getsockettype)(cmi_hdl_impl_t *); const char *(*cmio_getsocketstr)(cmi_hdl_impl_t *); uint_t (*cmio_chipsig)(cmi_hdl_impl_t *); cmi_errno_t (*cmio_ncache)(cmi_hdl_impl_t *, uint32_t *); cmi_errno_t (*cmio_cache)(cmi_hdl_impl_t *, uint32_t, x86_cache_t *); id_t (*cmio_logical_id)(cmi_hdl_impl_t *); /* * These ops are optional in an implementation. */ ulong_t (*cmio_getcr4)(cmi_hdl_impl_t *); void (*cmio_setcr4)(cmi_hdl_impl_t *, ulong_t); cmi_errno_t (*cmio_rdmsr)(cmi_hdl_impl_t *, uint_t, uint64_t *); cmi_errno_t (*cmio_wrmsr)(cmi_hdl_impl_t *, uint_t, uint64_t); cmi_errno_t (*cmio_msrinterpose)(cmi_hdl_impl_t *, uint_t, uint64_t); void (*cmio_int)(cmi_hdl_impl_t *, int); int (*cmio_online)(cmi_hdl_impl_t *, int, int *); uint16_t (*cmio_smbiosid) (cmi_hdl_impl_t *); uint_t (*cmio_smb_chipid)(cmi_hdl_impl_t *); nvlist_t *(*cmio_smb_bboard)(cmi_hdl_impl_t *); }; static const struct cmi_hdl_ops cmi_hdl_ops; /* * Handles are looked up from contexts such as polling, injection etc * where the context is reasonably well defined (although a poller could * interrupt any old thread holding any old lock). They are also looked * up by machine check handlers, which may strike at inconvenient times * such as during handle initialization or destruction or during handle * lookup (which the #MC handler itself will also have to perform). * * So keeping handles in a linked list makes locking difficult when we * consider #MC handlers. Our solution is to have a look-up table indexed * by that which uniquely identifies a handle - chip/core/strand id - * with each entry a structure including a pointer to a handle * structure for the resource, and a reference count for the handle. * Reference counts are modified atomically. The public cmi_hdl_hold * always succeeds because this can only be used after handle creation * and before the call to destruct, so the hold count is already at least one. * In other functions that lookup a handle (cmi_hdl_lookup, cmi_hdl_any) * we must be certain that the count has not already decrmented to zero * before applying our hold. * * The table is an array of maximum number of chips defined in * CMI_CHIPID_ARR_SZ indexed by the chip id. If the chip is not present, the * entry is NULL. Each entry is a pointer to another array which contains a * list of all strands of the chip. This first level table is allocated when * first we want to populate an entry. The size of the latter (per chip) table * is CMI_MAX_STRANDS_PER_CHIP and it is populated when one of its cpus starts. * * Ideally we should only allocate to the actual number of chips, cores per * chip and strand per core. The number of chips is not available until all * of them are passed. The number of cores and strands are partially available. * For now we stick with the above approach. */ #define CMI_MAX_CHIPID_NBITS 6 /* max chipid of 63 */ #define CMI_MAX_CORES_PER_CHIP_NBITS 4 /* 16 cores per chip max */ #define CMI_MAX_STRANDS_PER_CORE_NBITS 3 /* 8 strands per core max */ #define CMI_MAX_CHIPID ((1 << (CMI_MAX_CHIPID_NBITS)) - 1) #define CMI_MAX_CORES_PER_CHIP(cbits) (1 << (cbits)) #define CMI_MAX_COREID(cbits) ((1 << (cbits)) - 1) #define CMI_MAX_STRANDS_PER_CORE(sbits) (1 << (sbits)) #define CMI_MAX_STRANDID(sbits) ((1 << (sbits)) - 1) #define CMI_MAX_STRANDS_PER_CHIP(cbits, sbits) \ (CMI_MAX_CORES_PER_CHIP(cbits) * CMI_MAX_STRANDS_PER_CORE(sbits)) #define CMI_CHIPID_ARR_SZ (1 << CMI_MAX_CHIPID_NBITS) typedef struct cmi_hdl_ent { volatile uint32_t cmae_refcnt; cmi_hdl_impl_t *cmae_hdlp; } cmi_hdl_ent_t; static cmi_hdl_ent_t *cmi_chip_tab[CMI_CHIPID_ARR_SZ]; /* * Default values for the number of core and strand bits. */ uint_t cmi_core_nbits = CMI_MAX_CORES_PER_CHIP_NBITS; uint_t cmi_strand_nbits = CMI_MAX_STRANDS_PER_CORE_NBITS; static int cmi_ext_topo_check = 0; /* * Controls where we will source PCI config space data. */ #define CMI_PCICFG_FLAG_RD_HWOK 0x0001 #define CMI_PCICFG_FLAG_RD_INTERPOSEOK 0X0002 #define CMI_PCICFG_FLAG_WR_HWOK 0x0004 #define CMI_PCICFG_FLAG_WR_INTERPOSEOK 0X0008 static uint64_t cmi_pcicfg_flags = CMI_PCICFG_FLAG_RD_HWOK | CMI_PCICFG_FLAG_RD_INTERPOSEOK | CMI_PCICFG_FLAG_WR_HWOK | CMI_PCICFG_FLAG_WR_INTERPOSEOK; /* * The flags for individual cpus are kept in their per-cpu handle cmih_msrsrc */ #define CMI_MSR_FLAG_RD_HWOK 0x0001 #define CMI_MSR_FLAG_RD_INTERPOSEOK 0x0002 #define CMI_MSR_FLAG_WR_HWOK 0x0004 #define CMI_MSR_FLAG_WR_INTERPOSEOK 0x0008 int cmi_call_func_ntv_tries = 3; static cmi_errno_t call_func_ntv(int cpuid, xc_func_t func, xc_arg_t arg1, xc_arg_t arg2) { cmi_errno_t rc = -1; int i; kpreempt_disable(); if (CPU->cpu_id == cpuid) { (*func)(arg1, arg2, (xc_arg_t)&rc); } else { /* * This should not happen for a #MC trap or a poll, so * this is likely an error injection or similar. * We will try to cross call with xc_trycall - we * can't guarantee success with xc_call because * the interrupt code in the case of a #MC may * already hold the xc mutex. */ for (i = 0; i < cmi_call_func_ntv_tries; i++) { cpuset_t cpus; CPUSET_ONLY(cpus, cpuid); xc_priority(arg1, arg2, (xc_arg_t)&rc, CPUSET2BV(cpus), func); if (rc != -1) break; DELAY(1); } } kpreempt_enable(); return (rc != -1 ? rc : CMIERR_DEADLOCK); } static uint64_t injcnt; void cmi_hdl_inj_begin(cmi_hdl_t ophdl) { cmi_hdl_impl_t *hdl = IMPLHDL(ophdl); if (hdl != NULL) hdl->cmih_flags |= CMIH_F_INJACTV; if (injcnt++ == 0) { cmn_err(CE_NOTE, "Hardware error injection/simulation " "activity noted"); } } void cmi_hdl_inj_end(cmi_hdl_t ophdl) { cmi_hdl_impl_t *hdl = IMPLHDL(ophdl); ASSERT(hdl == NULL || hdl->cmih_flags & CMIH_F_INJACTV); if (hdl != NULL) hdl->cmih_flags &= ~CMIH_F_INJACTV; } boolean_t cmi_inj_tainted(void) { return (injcnt != 0 ? B_TRUE : B_FALSE); } /* * ======================================================= * | MSR Interposition | * | ----------------- | * | | * ------------------------------------------------------- */ #define CMI_MSRI_HASHSZ 16 #define CMI_MSRI_HASHIDX(hdl, msr) \ ((((uintptr_t)(hdl) >> 3) + (msr)) % (CMI_MSRI_HASHSZ - 1)) struct cmi_msri_bkt { kmutex_t msrib_lock; struct cmi_msri_hashent *msrib_head; }; struct cmi_msri_hashent { struct cmi_msri_hashent *msrie_next; struct cmi_msri_hashent *msrie_prev; cmi_hdl_impl_t *msrie_hdl; uint_t msrie_msrnum; uint64_t msrie_msrval; }; #define CMI_MSRI_MATCH(ent, hdl, req_msr) \ ((ent)->msrie_hdl == (hdl) && (ent)->msrie_msrnum == (req_msr)) static struct cmi_msri_bkt msrihash[CMI_MSRI_HASHSZ]; static void msri_addent(cmi_hdl_impl_t *hdl, uint_t msr, uint64_t val) { int idx = CMI_MSRI_HASHIDX(hdl, msr); struct cmi_msri_bkt *hbp = &msrihash[idx]; struct cmi_msri_hashent *hep; mutex_enter(&hbp->msrib_lock); for (hep = hbp->msrib_head; hep != NULL; hep = hep->msrie_next) { if (CMI_MSRI_MATCH(hep, hdl, msr)) break; } if (hep != NULL) { hep->msrie_msrval = val; } else { hep = kmem_alloc(sizeof (*hep), KM_SLEEP); hep->msrie_hdl = hdl; hep->msrie_msrnum = msr; hep->msrie_msrval = val; if (hbp->msrib_head != NULL) hbp->msrib_head->msrie_prev = hep; hep->msrie_next = hbp->msrib_head; hep->msrie_prev = NULL; hbp->msrib_head = hep; } mutex_exit(&hbp->msrib_lock); } /* * Look for a match for the given hanlde and msr. Return 1 with valp * filled if a match is found, otherwise return 0 with valp untouched. */ static int msri_lookup(cmi_hdl_impl_t *hdl, uint_t msr, uint64_t *valp) { int idx = CMI_MSRI_HASHIDX(hdl, msr); struct cmi_msri_bkt *hbp = &msrihash[idx]; struct cmi_msri_hashent *hep; /* * This function is called during #MC trap handling, so we should * consider the possibility that the hash mutex is held by the * interrupted thread. This should not happen because interposition * is an artificial injection mechanism and the #MC is requested * after adding entries, but just in case of a real #MC at an * unlucky moment we'll use mutex_tryenter here. */ if (!mutex_tryenter(&hbp->msrib_lock)) return (0); for (hep = hbp->msrib_head; hep != NULL; hep = hep->msrie_next) { if (CMI_MSRI_MATCH(hep, hdl, msr)) { *valp = hep->msrie_msrval; break; } } mutex_exit(&hbp->msrib_lock); return (hep != NULL); } /* * Remove any interposed value that matches. */ static void msri_rment(cmi_hdl_impl_t *hdl, uint_t msr) { int idx = CMI_MSRI_HASHIDX(hdl, msr); struct cmi_msri_bkt *hbp = &msrihash[idx]; struct cmi_msri_hashent *hep; if (!mutex_tryenter(&hbp->msrib_lock)) return; for (hep = hbp->msrib_head; hep != NULL; hep = hep->msrie_next) { if (CMI_MSRI_MATCH(hep, hdl, msr)) { if (hep->msrie_prev != NULL) hep->msrie_prev->msrie_next = hep->msrie_next; if (hep->msrie_next != NULL) hep->msrie_next->msrie_prev = hep->msrie_prev; if (hbp->msrib_head == hep) hbp->msrib_head = hep->msrie_next; kmem_free(hep, sizeof (*hep)); break; } } mutex_exit(&hbp->msrib_lock); } /* * ======================================================= * | PCI Config Space Interposition | * | ------------------------------ | * | | * ------------------------------------------------------- */ /* * Hash for interposed PCI config space values. We lookup on bus/dev/fun/offset * and then record whether the value stashed was made with a byte, word or * doubleword access; we will only return a hit for an access of the * same size. If you access say a 32-bit register using byte accesses * and then attempt to read the full 32-bit value back you will not obtain * any sort of merged result - you get a lookup miss. */ #define CMI_PCII_HASHSZ 16 #define CMI_PCII_HASHIDX(b, d, f, o) \ (((b) + (d) + (f) + (o)) % (CMI_PCII_HASHSZ - 1)) struct cmi_pcii_bkt { kmutex_t pciib_lock; struct cmi_pcii_hashent *pciib_head; }; struct cmi_pcii_hashent { struct cmi_pcii_hashent *pcii_next; struct cmi_pcii_hashent *pcii_prev; int pcii_bus; int pcii_dev; int pcii_func; int pcii_reg; int pcii_asize; uint32_t pcii_val; }; #define CMI_PCII_MATCH(ent, b, d, f, r, asz) \ ((ent)->pcii_bus == (b) && (ent)->pcii_dev == (d) && \ (ent)->pcii_func == (f) && (ent)->pcii_reg == (r) && \ (ent)->pcii_asize == (asz)) static struct cmi_pcii_bkt pciihash[CMI_PCII_HASHSZ]; /* * Add a new entry to the PCI interpose hash, overwriting any existing * entry that is found. */ static void pcii_addent(int bus, int dev, int func, int reg, uint32_t val, int asz) { int idx = CMI_PCII_HASHIDX(bus, dev, func, reg); struct cmi_pcii_bkt *hbp = &pciihash[idx]; struct cmi_pcii_hashent *hep; cmi_hdl_inj_begin(NULL); mutex_enter(&hbp->pciib_lock); for (hep = hbp->pciib_head; hep != NULL; hep = hep->pcii_next) { if (CMI_PCII_MATCH(hep, bus, dev, func, reg, asz)) break; } if (hep != NULL) { hep->pcii_val = val; } else { hep = kmem_alloc(sizeof (*hep), KM_SLEEP); hep->pcii_bus = bus; hep->pcii_dev = dev; hep->pcii_func = func; hep->pcii_reg = reg; hep->pcii_asize = asz; hep->pcii_val = val; if (hbp->pciib_head != NULL) hbp->pciib_head->pcii_prev = hep; hep->pcii_next = hbp->pciib_head; hep->pcii_prev = NULL; hbp->pciib_head = hep; } mutex_exit(&hbp->pciib_lock); cmi_hdl_inj_end(NULL); } /* * Look for a match for the given bus/dev/func/reg; return 1 with valp * filled if a match is found, otherwise return 0 with valp untouched. */ static int pcii_lookup(int bus, int dev, int func, int reg, int asz, uint32_t *valp) { int idx = CMI_PCII_HASHIDX(bus, dev, func, reg); struct cmi_pcii_bkt *hbp = &pciihash[idx]; struct cmi_pcii_hashent *hep; if (!mutex_tryenter(&hbp->pciib_lock)) return (0); for (hep = hbp->pciib_head; hep != NULL; hep = hep->pcii_next) { if (CMI_PCII_MATCH(hep, bus, dev, func, reg, asz)) { *valp = hep->pcii_val; break; } } mutex_exit(&hbp->pciib_lock); return (hep != NULL); } static void pcii_rment(int bus, int dev, int func, int reg, int asz) { int idx = CMI_PCII_HASHIDX(bus, dev, func, reg); struct cmi_pcii_bkt *hbp = &pciihash[idx]; struct cmi_pcii_hashent *hep; mutex_enter(&hbp->pciib_lock); for (hep = hbp->pciib_head; hep != NULL; hep = hep->pcii_next) { if (CMI_PCII_MATCH(hep, bus, dev, func, reg, asz)) { if (hep->pcii_prev != NULL) hep->pcii_prev->pcii_next = hep->pcii_next; if (hep->pcii_next != NULL) hep->pcii_next->pcii_prev = hep->pcii_prev; if (hbp->pciib_head == hep) hbp->pciib_head = hep->pcii_next; kmem_free(hep, sizeof (*hep)); break; } } mutex_exit(&hbp->pciib_lock); } #ifndef __xpv /* * ======================================================= * | Native methods | * | -------------- | * | | * | These are used when we are running native on bare- | * | metal, or simply don't know any better. | * --------------------------------------------------------- */ #define HDLPRIV(hdl) ((cpu_t *)(hdl)->cmih_hdlpriv) static uint_t ntv_vendor(cmi_hdl_impl_t *hdl) { return (cpuid_getvendor(HDLPRIV(hdl))); } static const char * ntv_vendorstr(cmi_hdl_impl_t *hdl) { return (cpuid_getvendorstr(HDLPRIV(hdl))); } static uint_t ntv_family(cmi_hdl_impl_t *hdl) { return (cpuid_getfamily(HDLPRIV(hdl))); } static uint_t ntv_model(cmi_hdl_impl_t *hdl) { return (cpuid_getmodel(HDLPRIV(hdl))); } static uint_t ntv_stepping(cmi_hdl_impl_t *hdl) { return (cpuid_getstep(HDLPRIV(hdl))); } static uint_t ntv_chipid(cmi_hdl_impl_t *hdl) { return (hdl->cmih_chipid); } static uint_t ntv_procnodeid(cmi_hdl_impl_t *hdl) { return (hdl->cmih_procnodeid); } static uint_t ntv_procnodes_per_pkg(cmi_hdl_impl_t *hdl) { return (hdl->cmih_procnodes_per_pkg); } static uint_t ntv_coreid(cmi_hdl_impl_t *hdl) { return (hdl->cmih_coreid); } static uint_t ntv_strandid(cmi_hdl_impl_t *hdl) { return (hdl->cmih_strandid); } static uint_t ntv_strand_apicid(cmi_hdl_impl_t *hdl) { return (cpuid_get_apicid(HDLPRIV(hdl))); } static uint16_t ntv_smbiosid(cmi_hdl_impl_t *hdl) { return (hdl->cmih_smbiosid); } static uint_t ntv_smb_chipid(cmi_hdl_impl_t *hdl) { return (hdl->cmih_smb_chipid); } static nvlist_t * ntv_smb_bboard(cmi_hdl_impl_t *hdl) { return (hdl->cmih_smb_bboard); } static x86_chiprev_t ntv_chiprev(cmi_hdl_impl_t *hdl) { return (cpuid_getchiprev(HDLPRIV(hdl))); } static const char * ntv_chiprevstr(cmi_hdl_impl_t *hdl) { return (cpuid_getchiprevstr(HDLPRIV(hdl))); } static uint32_t ntv_getsockettype(cmi_hdl_impl_t *hdl) { return (cpuid_getsockettype(HDLPRIV(hdl))); } static const char * ntv_getsocketstr(cmi_hdl_impl_t *hdl) { return (cpuid_getsocketstr(HDLPRIV(hdl))); } static uint_t ntv_chipsig(cmi_hdl_impl_t *hdl) { return (cpuid_getsig(HDLPRIV(hdl))); } static cmi_errno_t cmi_cpuid_cache_to_cmi(int err) { switch (err) { case 0: return (CMI_SUCCESS); case ENOTSUP: return (CMIERR_C_NODATA); case EINVAL: return (CMIERR_C_BADCACHENO); default: return (CMIERR_UNKNOWN); } } static cmi_errno_t ntv_ncache(cmi_hdl_impl_t *hdl, uint32_t *ncache) { int ret = cpuid_getncaches(HDLPRIV(hdl), ncache); return (cmi_cpuid_cache_to_cmi(ret)); } static cmi_errno_t ntv_cache(cmi_hdl_impl_t *hdl, uint32_t cno, x86_cache_t *cachep) { int ret = cpuid_getcache(HDLPRIV(hdl), cno, cachep); return (cmi_cpuid_cache_to_cmi(ret)); } static id_t ntv_logical_id(cmi_hdl_impl_t *hdl) { return (HDLPRIV(hdl)->cpu_id); } /*ARGSUSED*/ static int ntv_getcr4_xc(xc_arg_t arg1, xc_arg_t arg2, xc_arg_t arg3) { ulong_t *dest = (ulong_t *)arg1; cmi_errno_t *rcp = (cmi_errno_t *)arg3; *dest = getcr4(); *rcp = CMI_SUCCESS; return (0); } static ulong_t ntv_getcr4(cmi_hdl_impl_t *hdl) { cpu_t *cp = HDLPRIV(hdl); ulong_t val; (void) call_func_ntv(cp->cpu_id, ntv_getcr4_xc, (xc_arg_t)&val, 0); return (val); } /*ARGSUSED*/ static int ntv_setcr4_xc(xc_arg_t arg1, xc_arg_t arg2, xc_arg_t arg3) { ulong_t val = (ulong_t)arg1; cmi_errno_t *rcp = (cmi_errno_t *)arg3; setcr4(val); *rcp = CMI_SUCCESS; return (0); } static void ntv_setcr4(cmi_hdl_impl_t *hdl, ulong_t val) { cpu_t *cp = HDLPRIV(hdl); (void) call_func_ntv(cp->cpu_id, ntv_setcr4_xc, (xc_arg_t)val, 0); } volatile uint32_t cmi_trapped_rdmsr; /*ARGSUSED*/ static int ntv_rdmsr_xc(xc_arg_t arg1, xc_arg_t arg2, xc_arg_t arg3) { uint_t msr = (uint_t)arg1; uint64_t *valp = (uint64_t *)arg2; cmi_errno_t *rcp = (cmi_errno_t *)arg3; on_trap_data_t otd; if (on_trap(&otd, OT_DATA_ACCESS) == 0) { if (checked_rdmsr(msr, valp) == 0) *rcp = CMI_SUCCESS; else *rcp = CMIERR_NOTSUP; } else { *rcp = CMIERR_MSRGPF; atomic_inc_32(&cmi_trapped_rdmsr); } no_trap(); return (0); } static cmi_errno_t ntv_rdmsr(cmi_hdl_impl_t *hdl, uint_t msr, uint64_t *valp) { cpu_t *cp = HDLPRIV(hdl); if (!(hdl->cmih_msrsrc & CMI_MSR_FLAG_RD_HWOK)) return (CMIERR_INTERPOSE); return (call_func_ntv(cp->cpu_id, ntv_rdmsr_xc, (xc_arg_t)msr, (xc_arg_t)valp)); } volatile uint32_t cmi_trapped_wrmsr; /*ARGSUSED*/ static int ntv_wrmsr_xc(xc_arg_t arg1, xc_arg_t arg2, xc_arg_t arg3) { uint_t msr = (uint_t)arg1; uint64_t val = *((uint64_t *)arg2); cmi_errno_t *rcp = (cmi_errno_t *)arg3; on_trap_data_t otd; if (on_trap(&otd, OT_DATA_ACCESS) == 0) { if (checked_wrmsr(msr, val) == 0) *rcp = CMI_SUCCESS; else *rcp = CMIERR_NOTSUP; } else { *rcp = CMIERR_MSRGPF; atomic_inc_32(&cmi_trapped_wrmsr); } no_trap(); return (0); } static cmi_errno_t ntv_wrmsr(cmi_hdl_impl_t *hdl, uint_t msr, uint64_t val) { cpu_t *cp = HDLPRIV(hdl); if (!(hdl->cmih_msrsrc & CMI_MSR_FLAG_WR_HWOK)) return (CMI_SUCCESS); return (call_func_ntv(cp->cpu_id, ntv_wrmsr_xc, (xc_arg_t)msr, (xc_arg_t)&val)); } static cmi_errno_t ntv_msrinterpose(cmi_hdl_impl_t *hdl, uint_t msr, uint64_t val) { msri_addent(hdl, msr, val); return (CMI_SUCCESS); } /*ARGSUSED*/ static int ntv_int_xc(xc_arg_t arg1, xc_arg_t arg2, xc_arg_t arg3) { cmi_errno_t *rcp = (cmi_errno_t *)arg3; int int_no = (int)arg1; if (int_no == T_MCE) int18(); else int_cmci(); *rcp = CMI_SUCCESS; return (0); } static void ntv_int(cmi_hdl_impl_t *hdl, int int_no) { cpu_t *cp = HDLPRIV(hdl); (void) call_func_ntv(cp->cpu_id, ntv_int_xc, (xc_arg_t)int_no, 0); } static int ntv_online(cmi_hdl_impl_t *hdl, int new_status, int *old_status) { int rc; processorid_t cpuid = HDLPRIV(hdl)->cpu_id; while (mutex_tryenter(&cpu_lock) == 0) { if (hdl->cmih_flags & CMIH_F_DEAD) return (EBUSY); delay(1); } rc = p_online_internal_locked(cpuid, new_status, old_status); mutex_exit(&cpu_lock); return (rc); } #else /* __xpv */ /* * ======================================================= * | xVM dom0 methods | * | ---------------- | * | | * | These are used when we are running as dom0 in | * | a Solaris xVM context. | * --------------------------------------------------------- */ #define HDLPRIV(hdl) ((xen_mc_lcpu_cookie_t)(hdl)->cmih_hdlpriv) extern uint_t _cpuid_vendorstr_to_vendorcode(char *); static uint_t xpv_vendor(cmi_hdl_impl_t *hdl) { return (_cpuid_vendorstr_to_vendorcode((char *)xen_physcpu_vendorstr( HDLPRIV(hdl)))); } static const char * xpv_vendorstr(cmi_hdl_impl_t *hdl) { return (xen_physcpu_vendorstr(HDLPRIV(hdl))); } static uint_t xpv_family(cmi_hdl_impl_t *hdl) { return (xen_physcpu_family(HDLPRIV(hdl))); } static uint_t xpv_model(cmi_hdl_impl_t *hdl) { return (xen_physcpu_model(HDLPRIV(hdl))); } static uint_t xpv_stepping(cmi_hdl_impl_t *hdl) { return (xen_physcpu_stepping(HDLPRIV(hdl))); } static uint_t xpv_chipid(cmi_hdl_impl_t *hdl) { return (hdl->cmih_chipid); } static uint_t xpv_procnodeid(cmi_hdl_impl_t *hdl) { return (hdl->cmih_procnodeid); } static uint_t xpv_procnodes_per_pkg(cmi_hdl_impl_t *hdl) { return (hdl->cmih_procnodes_per_pkg); } static uint_t xpv_coreid(cmi_hdl_impl_t *hdl) { return (hdl->cmih_coreid); } static uint_t xpv_strandid(cmi_hdl_impl_t *hdl) { return (hdl->cmih_strandid); } static uint_t xpv_strand_apicid(cmi_hdl_impl_t *hdl) { return (xen_physcpu_initial_apicid(HDLPRIV(hdl))); } static uint16_t xpv_smbiosid(cmi_hdl_impl_t *hdl) { return (hdl->cmih_smbiosid); } static uint_t xpv_smb_chipid(cmi_hdl_impl_t *hdl) { return (hdl->cmih_smb_chipid); } static nvlist_t * xpv_smb_bboard(cmi_hdl_impl_t *hdl) { return (hdl->cmih_smb_bboard); } extern x86_chiprev_t _cpuid_chiprev(uint_t, uint_t, uint_t, uint_t); static x86_chiprev_t xpv_chiprev(cmi_hdl_impl_t *hdl) { return (_cpuid_chiprev(xpv_vendor(hdl), xpv_family(hdl), xpv_model(hdl), xpv_stepping(hdl))); } extern const char *_cpuid_chiprevstr(uint_t, uint_t, uint_t, uint_t); static const char * xpv_chiprevstr(cmi_hdl_impl_t *hdl) { return (_cpuid_chiprevstr(xpv_vendor(hdl), xpv_family(hdl), xpv_model(hdl), xpv_stepping(hdl))); } extern uint32_t _cpuid_skt(uint_t, uint_t, uint_t, uint_t); static uint32_t xpv_getsockettype(cmi_hdl_impl_t *hdl) { return (_cpuid_skt(xpv_vendor(hdl), xpv_family(hdl), xpv_model(hdl), xpv_stepping(hdl))); } extern const char *_cpuid_sktstr(uint_t, uint_t, uint_t, uint_t); static const char * xpv_getsocketstr(cmi_hdl_impl_t *hdl) { return (_cpuid_sktstr(xpv_vendor(hdl), xpv_family(hdl), xpv_model(hdl), xpv_stepping(hdl))); } /* ARGSUSED */ static uint_t xpv_chipsig(cmi_hdl_impl_t *hdl) { return (0); } static cmi_errno_t xpv_ncache(cmi_hdl_impl_t *hdl, uint32_t *ncache) { return (CMIERR_NOTSUP); } static cmi_errno_t xpv_cache(cmi_hdl_impl_t *hdl, uint32_t cno, x86_cache_t *cachep) { return (CMIERR_NOTSUP); } static id_t xpv_logical_id(cmi_hdl_impl_t *hdl) { return (xen_physcpu_logical_id(HDLPRIV(hdl))); } static cmi_errno_t xpv_rdmsr(cmi_hdl_impl_t *hdl, uint_t msr, uint64_t *valp) { switch (msr) { case IA32_MSR_MCG_CAP: *valp = xen_physcpu_mcg_cap(HDLPRIV(hdl)); break; default: return (CMIERR_NOTSUP); } return (CMI_SUCCESS); } /* * Request the hypervisor to write an MSR for us. The hypervisor * will only accept MCA-related MSRs, as this is for MCA error * simulation purposes alone. We will pre-screen MSRs for injection * so we don't bother the HV with bogus requests. We will permit * injection to any MCA bank register, and to MCG_STATUS. */ #define IS_MCA_INJ_MSR(msr) \ (((msr) >= IA32_MSR_MC(0, CTL) && (msr) <= IA32_MSR_MC(10, MISC)) || \ (msr) == IA32_MSR_MCG_STATUS) static cmi_errno_t xpv_wrmsr_cmn(cmi_hdl_impl_t *hdl, uint_t msr, uint64_t val, boolean_t intpose) { xen_mc_t xmc; struct xen_mc_msrinject *mci = &xmc.u.mc_msrinject; if (!(hdl->cmih_flags & CMIH_F_INJACTV)) return (CMIERR_NOTSUP); /* for injection use only! */ if (!IS_MCA_INJ_MSR(msr)) return (CMIERR_API); if (panicstr) return (CMIERR_DEADLOCK); mci->mcinj_cpunr = xen_physcpu_logical_id(HDLPRIV(hdl)); mci->mcinj_flags = intpose ? MC_MSRINJ_F_INTERPOSE : 0; mci->mcinj_count = 1; /* learn to batch sometime */ mci->mcinj_msr[0].reg = msr; mci->mcinj_msr[0].value = val; return (HYPERVISOR_mca(XEN_MC_msrinject, &xmc) == 0 ? CMI_SUCCESS : CMIERR_NOTSUP); } static cmi_errno_t xpv_wrmsr(cmi_hdl_impl_t *hdl, uint_t msr, uint64_t val) { return (xpv_wrmsr_cmn(hdl, msr, val, B_FALSE)); } static cmi_errno_t xpv_msrinterpose(cmi_hdl_impl_t *hdl, uint_t msr, uint64_t val) { return (xpv_wrmsr_cmn(hdl, msr, val, B_TRUE)); } static void xpv_int(cmi_hdl_impl_t *hdl, int int_no) { xen_mc_t xmc; struct xen_mc_mceinject *mce = &xmc.u.mc_mceinject; if (!(hdl->cmih_flags & CMIH_F_INJACTV)) return; if (int_no != T_MCE) { cmn_err(CE_WARN, "xpv_int: int_no %d unimplemented\n", int_no); } mce->mceinj_cpunr = xen_physcpu_logical_id(HDLPRIV(hdl)); (void) HYPERVISOR_mca(XEN_MC_mceinject, &xmc); } static int xpv_online(cmi_hdl_impl_t *hdl, int new_status, int *old_status) { xen_sysctl_t xs; int op, rc, status; new_status &= ~P_FORCED; switch (new_status) { case P_STATUS: op = XEN_SYSCTL_CPU_HOTPLUG_STATUS; break; case P_FAULTED: case P_OFFLINE: op = XEN_SYSCTL_CPU_HOTPLUG_OFFLINE; break; case P_ONLINE: op = XEN_SYSCTL_CPU_HOTPLUG_ONLINE; break; default: return (-1); } xs.cmd = XEN_SYSCTL_cpu_hotplug; xs.interface_version = XEN_SYSCTL_INTERFACE_VERSION; xs.u.cpu_hotplug.cpu = xen_physcpu_logical_id(HDLPRIV(hdl)); xs.u.cpu_hotplug.op = op; if ((rc = HYPERVISOR_sysctl(&xs)) >= 0) { status = rc; rc = 0; switch (status) { case XEN_CPU_HOTPLUG_STATUS_NEW: *old_status = P_OFFLINE; break; case XEN_CPU_HOTPLUG_STATUS_OFFLINE: *old_status = P_FAULTED; break; case XEN_CPU_HOTPLUG_STATUS_ONLINE: *old_status = P_ONLINE; break; default: return (-1); } } return (-rc); } #endif /*ARGSUSED*/ static void * cpu_search(enum cmi_hdl_class class, uint_t chipid, uint_t coreid, uint_t strandid) { #ifdef __xpv xen_mc_lcpu_cookie_t cpi; for (cpi = xen_physcpu_next(NULL); cpi != NULL; cpi = xen_physcpu_next(cpi)) { if (xen_physcpu_chipid(cpi) == chipid && xen_physcpu_coreid(cpi) == coreid && xen_physcpu_strandid(cpi) == strandid) return ((void *)cpi); } return (NULL); #else /* __xpv */ cpu_t *cp, *startcp; kpreempt_disable(); cp = startcp = CPU; do { if (cmi_ntv_hwchipid(cp) == chipid && cmi_ntv_hwcoreid(cp) == coreid && cmi_ntv_hwstrandid(cp) == strandid) { kpreempt_enable(); return ((void *)cp); } cp = cp->cpu_next; } while (cp != startcp); kpreempt_enable(); return (NULL); #endif /* __ xpv */ } static boolean_t cpu_is_cmt(void *priv) { #ifdef __xpv return (xen_physcpu_is_cmt((xen_mc_lcpu_cookie_t)priv)); #else /* __xpv */ cpu_t *cp = (cpu_t *)priv; int strands_per_core = cpuid_get_ncpu_per_chip(cp) / cpuid_get_ncore_per_chip(cp); return (strands_per_core > 1); #endif /* __xpv */ } /* * Find the handle entry of a given cpu identified by a * tuple. */ static cmi_hdl_ent_t * cmi_hdl_ent_lookup(uint_t chipid, uint_t coreid, uint_t strandid) { int max_strands = CMI_MAX_STRANDS_PER_CHIP(cmi_core_nbits, cmi_strand_nbits); /* * Allocate per-chip table which contains a list of handle of * all strands of the chip. */ if (cmi_chip_tab[chipid] == NULL) { size_t sz; cmi_hdl_ent_t *pg; sz = max_strands * sizeof (cmi_hdl_ent_t); pg = kmem_zalloc(sz, KM_SLEEP); /* test and set the per-chip table if it is not allocated */ if (atomic_cas_ptr(&cmi_chip_tab[chipid], NULL, pg) != NULL) kmem_free(pg, sz); /* someone beats us */ } return (cmi_chip_tab[chipid] + ((((coreid) & CMI_MAX_COREID(cmi_core_nbits)) << cmi_strand_nbits) | ((strandid) & CMI_MAX_STRANDID(cmi_strand_nbits)))); } extern void cpuid_get_ext_topo(cpu_t *, uint_t *, uint_t *); cmi_hdl_t cmi_hdl_create(enum cmi_hdl_class class, uint_t chipid, uint_t coreid, uint_t strandid) { cmi_hdl_impl_t *hdl; void *priv; cmi_hdl_ent_t *ent; uint_t vendor; #ifdef __xpv ASSERT(class == CMI_HDL_SOLARIS_xVM_MCA); #else ASSERT(class == CMI_HDL_NATIVE); #endif if ((priv = cpu_search(class, chipid, coreid, strandid)) == NULL) return (NULL); /* * Assume all chips in the system are the same type. * For Intel, attempt to check if extended topology is available * CPUID.EAX=0xB. If so, get the number of core and strand bits. */ #ifdef __xpv vendor = _cpuid_vendorstr_to_vendorcode( (char *)xen_physcpu_vendorstr((xen_mc_lcpu_cookie_t)priv)); #else vendor = cpuid_getvendor((cpu_t *)priv); #endif switch (vendor) { case X86_VENDOR_Intel: case X86_VENDOR_AMD: case X86_VENDOR_HYGON: if (cmi_ext_topo_check == 0) { cpuid_get_ext_topo((cpu_t *)priv, &cmi_core_nbits, &cmi_strand_nbits); cmi_ext_topo_check = 1; } default: break; } if (chipid > CMI_MAX_CHIPID || coreid > CMI_MAX_COREID(cmi_core_nbits) || strandid > CMI_MAX_STRANDID(cmi_strand_nbits)) return (NULL); hdl = kmem_zalloc(sizeof (*hdl), KM_SLEEP); hdl->cmih_class = class; HDLOPS(hdl) = &cmi_hdl_ops; hdl->cmih_chipid = chipid; hdl->cmih_coreid = coreid; hdl->cmih_strandid = strandid; hdl->cmih_mstrand = cpu_is_cmt(priv); hdl->cmih_hdlpriv = priv; #ifdef __xpv hdl->cmih_msrsrc = CMI_MSR_FLAG_RD_INTERPOSEOK | CMI_MSR_FLAG_WR_INTERPOSEOK; /* * XXX: need hypervisor support for procnodeid, for now assume * single-node processors (procnodeid = chipid) */ hdl->cmih_procnodeid = xen_physcpu_chipid((xen_mc_lcpu_cookie_t)priv); hdl->cmih_procnodes_per_pkg = 1; #else /* __xpv */ hdl->cmih_msrsrc = CMI_MSR_FLAG_RD_HWOK | CMI_MSR_FLAG_RD_INTERPOSEOK | CMI_MSR_FLAG_WR_HWOK | CMI_MSR_FLAG_WR_INTERPOSEOK; hdl->cmih_procnodeid = cpuid_get_procnodeid((cpu_t *)priv); hdl->cmih_procnodes_per_pkg = cpuid_get_procnodes_per_pkg((cpu_t *)priv); #endif /* __xpv */ ent = cmi_hdl_ent_lookup(chipid, coreid, strandid); if (ent->cmae_refcnt != 0 || ent->cmae_hdlp != NULL) { /* * Somehow this (chipid, coreid, strandid) id tuple has * already been assigned! This indicates that the * callers logic in determining these values is busted, * or perhaps undermined by bad BIOS setup. Complain, * and refuse to initialize this tuple again as bad things * will happen. */ cmn_err(CE_NOTE, "cmi_hdl_create: chipid %d coreid %d " "strandid %d handle already allocated!", chipid, coreid, strandid); kmem_free(hdl, sizeof (*hdl)); return (NULL); } /* * Once we store a nonzero reference count others can find this * handle via cmi_hdl_lookup etc. This initial hold on the handle * is to be dropped only if some other part of cmi initialization * fails or, if it succeeds, at later cpu deconfigure. Note the * the module private data we hold in cmih_cmi and cmih_cmidata * is still NULL at this point (the caller will fill it with * cmi_hdl_setcmi if it initializes) so consumers of handles * should always be ready for that possibility. */ ent->cmae_hdlp = hdl; hdl->cmih_refcntp = &ent->cmae_refcnt; ent->cmae_refcnt = 1; return ((cmi_hdl_t)hdl); } void cmi_read_smbios(cmi_hdl_t ophdl) { uint_t strand_apicid = UINT_MAX; uint_t chip_inst = UINT_MAX; uint16_t smb_id = USHRT_MAX; int rc = 0; cmi_hdl_impl_t *hdl = IMPLHDL(ophdl); /* set x86gentopo compatibility */ fm_smb_fmacompat(); #ifndef __xpv strand_apicid = ntv_strand_apicid(hdl); #else strand_apicid = xpv_strand_apicid(hdl); #endif if (!x86gentopo_legacy) { /* * If fm_smb_chipinst() or fm_smb_bboard() fails, * topo reverts to legacy mode */ rc = fm_smb_chipinst(strand_apicid, &chip_inst, &smb_id); if (rc == 0) { hdl->cmih_smb_chipid = chip_inst; hdl->cmih_smbiosid = smb_id; } else { #ifdef DEBUG cmn_err(CE_NOTE, "!cmi reads smbios chip info failed"); #endif /* DEBUG */ return; } hdl->cmih_smb_bboard = fm_smb_bboard(strand_apicid); #ifdef DEBUG if (hdl->cmih_smb_bboard == NULL) cmn_err(CE_NOTE, "!cmi reads smbios base boards info failed"); #endif /* DEBUG */ } } void cmi_hdl_hold(cmi_hdl_t ophdl) { cmi_hdl_impl_t *hdl = IMPLHDL(ophdl); ASSERT(*hdl->cmih_refcntp != 0); /* must not be the initial hold */ atomic_inc_32(hdl->cmih_refcntp); } static int cmi_hdl_canref(cmi_hdl_ent_t *ent) { volatile uint32_t *refcntp; uint32_t refcnt; refcntp = &ent->cmae_refcnt; refcnt = *refcntp; if (refcnt == 0) { /* * Associated object never existed, is being destroyed, * or has been destroyed. */ return (0); } /* * We cannot use atomic increment here because once the reference * count reaches zero it must never be bumped up again. */ while (refcnt != 0) { if (atomic_cas_32(refcntp, refcnt, refcnt + 1) == refcnt) return (1); refcnt = *refcntp; } /* * Somebody dropped the reference count to 0 after our initial * check. */ return (0); } void cmi_hdl_rele(cmi_hdl_t ophdl) { cmi_hdl_impl_t *hdl = IMPLHDL(ophdl); ASSERT(*hdl->cmih_refcntp > 0); atomic_dec_32(hdl->cmih_refcntp); } void cmi_hdl_destroy(cmi_hdl_t ophdl) { cmi_hdl_impl_t *hdl = IMPLHDL(ophdl); cmi_hdl_ent_t *ent; /* Release the reference count held by cmi_hdl_create(). */ ASSERT(*hdl->cmih_refcntp > 0); atomic_dec_32(hdl->cmih_refcntp); hdl->cmih_flags |= CMIH_F_DEAD; ent = cmi_hdl_ent_lookup(hdl->cmih_chipid, hdl->cmih_coreid, hdl->cmih_strandid); /* * Use busy polling instead of condition variable here because * cmi_hdl_rele() may be called from #MC handler. */ while (cmi_hdl_canref(ent)) { cmi_hdl_rele(ophdl); delay(1); } ent->cmae_hdlp = NULL; kmem_free(hdl, sizeof (*hdl)); } void cmi_hdl_setspecific(cmi_hdl_t ophdl, void *arg) { IMPLHDL(ophdl)->cmih_spec = arg; } void * cmi_hdl_getspecific(cmi_hdl_t ophdl) { return (IMPLHDL(ophdl)->cmih_spec); } void cmi_hdl_setmc(cmi_hdl_t ophdl, const struct cmi_mc_ops *mcops, void *mcdata) { cmi_hdl_impl_t *hdl = IMPLHDL(ophdl); ASSERT(hdl->cmih_mcops == NULL && hdl->cmih_mcdata == NULL); hdl->cmih_mcops = mcops; hdl->cmih_mcdata = mcdata; } const struct cmi_mc_ops * cmi_hdl_getmcops(cmi_hdl_t ophdl) { return (IMPLHDL(ophdl)->cmih_mcops); } void * cmi_hdl_getmcdata(cmi_hdl_t ophdl) { return (IMPLHDL(ophdl)->cmih_mcdata); } cmi_hdl_t cmi_hdl_lookup(enum cmi_hdl_class class, uint_t chipid, uint_t coreid, uint_t strandid) { cmi_hdl_ent_t *ent; if (chipid > CMI_MAX_CHIPID || coreid > CMI_MAX_COREID(cmi_core_nbits) || strandid > CMI_MAX_STRANDID(cmi_strand_nbits)) return (NULL); ent = cmi_hdl_ent_lookup(chipid, coreid, strandid); if (class == CMI_HDL_NEUTRAL) #ifdef __xpv class = CMI_HDL_SOLARIS_xVM_MCA; #else class = CMI_HDL_NATIVE; #endif if (!cmi_hdl_canref(ent)) return (NULL); if (ent->cmae_hdlp->cmih_class != class) { cmi_hdl_rele((cmi_hdl_t)ent->cmae_hdlp); return (NULL); } return ((cmi_hdl_t)ent->cmae_hdlp); } cmi_hdl_t cmi_hdl_any(void) { int i, j; cmi_hdl_ent_t *ent; int max_strands = CMI_MAX_STRANDS_PER_CHIP(cmi_core_nbits, cmi_strand_nbits); for (i = 0; i < CMI_CHIPID_ARR_SZ; i++) { if (cmi_chip_tab[i] == NULL) continue; for (j = 0, ent = cmi_chip_tab[i]; j < max_strands; j++, ent++) { if (cmi_hdl_canref(ent)) return ((cmi_hdl_t)ent->cmae_hdlp); } } return (NULL); } void cmi_hdl_walk(int (*cbfunc)(cmi_hdl_t, void *, void *, void *), void *arg1, void *arg2, void *arg3) { int i, j; cmi_hdl_ent_t *ent; int max_strands = CMI_MAX_STRANDS_PER_CHIP(cmi_core_nbits, cmi_strand_nbits); for (i = 0; i < CMI_CHIPID_ARR_SZ; i++) { if (cmi_chip_tab[i] == NULL) continue; for (j = 0, ent = cmi_chip_tab[i]; j < max_strands; j++, ent++) { if (cmi_hdl_canref(ent)) { cmi_hdl_impl_t *hdl = ent->cmae_hdlp; if ((*cbfunc)((cmi_hdl_t)hdl, arg1, arg2, arg3) == CMI_HDL_WALK_DONE) { cmi_hdl_rele((cmi_hdl_t)hdl); return; } cmi_hdl_rele((cmi_hdl_t)hdl); } } } } void cmi_hdl_setcmi(cmi_hdl_t ophdl, void *cmi, void *cmidata) { IMPLHDL(ophdl)->cmih_cmidata = cmidata; IMPLHDL(ophdl)->cmih_cmi = cmi; } void * cmi_hdl_getcmi(cmi_hdl_t ophdl) { return (IMPLHDL(ophdl)->cmih_cmi); } void * cmi_hdl_getcmidata(cmi_hdl_t ophdl) { return (IMPLHDL(ophdl)->cmih_cmidata); } enum cmi_hdl_class cmi_hdl_class(cmi_hdl_t ophdl) { return (IMPLHDL(ophdl)->cmih_class); } #define CMI_HDL_OPFUNC(what, type) \ type \ cmi_hdl_##what(cmi_hdl_t ophdl) \ { \ return (HDLOPS(IMPLHDL(ophdl))-> \ cmio_##what(IMPLHDL(ophdl))); \ } /* BEGIN CSTYLED */ CMI_HDL_OPFUNC(vendor, uint_t) CMI_HDL_OPFUNC(vendorstr, const char *) CMI_HDL_OPFUNC(family, uint_t) CMI_HDL_OPFUNC(model, uint_t) CMI_HDL_OPFUNC(stepping, uint_t) CMI_HDL_OPFUNC(chipid, uint_t) CMI_HDL_OPFUNC(procnodeid, uint_t) CMI_HDL_OPFUNC(coreid, uint_t) CMI_HDL_OPFUNC(strandid, uint_t) CMI_HDL_OPFUNC(procnodes_per_pkg, uint_t) CMI_HDL_OPFUNC(strand_apicid, uint_t) CMI_HDL_OPFUNC(chiprev, x86_chiprev_t) CMI_HDL_OPFUNC(chiprevstr, const char *) CMI_HDL_OPFUNC(getsockettype, uint32_t) CMI_HDL_OPFUNC(getsocketstr, const char *) CMI_HDL_OPFUNC(logical_id, id_t) CMI_HDL_OPFUNC(smbiosid, uint16_t) CMI_HDL_OPFUNC(smb_chipid, uint_t) CMI_HDL_OPFUNC(smb_bboard, nvlist_t *) CMI_HDL_OPFUNC(chipsig, uint_t) /* END CSTYLED */ boolean_t cmi_hdl_is_cmt(cmi_hdl_t ophdl) { return (IMPLHDL(ophdl)->cmih_mstrand); } void cmi_hdl_int(cmi_hdl_t ophdl, int num) { if (HDLOPS(IMPLHDL(ophdl))->cmio_int == NULL) return; cmi_hdl_inj_begin(ophdl); HDLOPS(IMPLHDL(ophdl))->cmio_int(IMPLHDL(ophdl), num); cmi_hdl_inj_end(NULL); } int cmi_hdl_online(cmi_hdl_t ophdl, int new_status, int *old_status) { return (HDLOPS(IMPLHDL(ophdl))->cmio_online(IMPLHDL(ophdl), new_status, old_status)); } #ifndef __xpv /* * Return hardware chip instance; cpuid_get_chipid provides this directly. */ uint_t cmi_ntv_hwchipid(cpu_t *cp) { return (cpuid_get_chipid(cp)); } /* * Return hardware node instance; cpuid_get_procnodeid provides this directly. */ uint_t cmi_ntv_hwprocnodeid(cpu_t *cp) { return (cpuid_get_procnodeid(cp)); } /* * Return core instance within a single chip. */ uint_t cmi_ntv_hwcoreid(cpu_t *cp) { return (cpuid_get_pkgcoreid(cp)); } /* * Return strand number within a single core. cpuid_get_clogid numbers * all execution units (strands, or cores in unstranded models) sequentially * within a single chip. */ uint_t cmi_ntv_hwstrandid(cpu_t *cp) { int strands_per_core = cpuid_get_ncpu_per_chip(cp) / cpuid_get_ncore_per_chip(cp); return (cpuid_get_clogid(cp) % strands_per_core); } static void cmi_ntv_hwdisable_mce_xc(void) { ulong_t cr4; cr4 = getcr4(); cr4 = cr4 & (~CR4_MCE); setcr4(cr4); } void cmi_ntv_hwdisable_mce(cmi_hdl_t hdl) { cpuset_t set; cmi_hdl_impl_t *thdl = IMPLHDL(hdl); cpu_t *cp = HDLPRIV(thdl); if (CPU->cpu_id == cp->cpu_id) { cmi_ntv_hwdisable_mce_xc(); } else { CPUSET_ONLY(set, cp->cpu_id); xc_call(0, 0, 0, CPUSET2BV(set), (xc_func_t)cmi_ntv_hwdisable_mce_xc); } } #endif /* __xpv */ void cmi_hdlconf_rdmsr_nohw(cmi_hdl_t ophdl) { cmi_hdl_impl_t *hdl = IMPLHDL(ophdl); hdl->cmih_msrsrc &= ~CMI_MSR_FLAG_RD_HWOK; } void cmi_hdlconf_wrmsr_nohw(cmi_hdl_t ophdl) { cmi_hdl_impl_t *hdl = IMPLHDL(ophdl); hdl->cmih_msrsrc &= ~CMI_MSR_FLAG_WR_HWOK; } cmi_errno_t cmi_hdl_rdmsr(cmi_hdl_t ophdl, uint_t msr, uint64_t *valp) { cmi_hdl_impl_t *hdl = IMPLHDL(ophdl); /* * Regardless of the handle class, we first check for am * interposed value. In the xVM case you probably want to * place interposed values within the hypervisor itself, but * we still allow interposing them in dom0 for test and bringup * purposes. */ if ((hdl->cmih_msrsrc & CMI_MSR_FLAG_RD_INTERPOSEOK) && msri_lookup(hdl, msr, valp)) return (CMI_SUCCESS); if (HDLOPS(hdl)->cmio_rdmsr == NULL) return (CMIERR_NOTSUP); return (HDLOPS(hdl)->cmio_rdmsr(hdl, msr, valp)); } cmi_errno_t cmi_hdl_wrmsr(cmi_hdl_t ophdl, uint_t msr, uint64_t val) { cmi_hdl_impl_t *hdl = IMPLHDL(ophdl); /* Invalidate any interposed value */ msri_rment(hdl, msr); if (HDLOPS(hdl)->cmio_wrmsr == NULL) return (CMI_SUCCESS); /* pretend all is ok */ return (HDLOPS(hdl)->cmio_wrmsr(hdl, msr, val)); } void cmi_hdl_enable_mce(cmi_hdl_t ophdl) { cmi_hdl_impl_t *hdl = IMPLHDL(ophdl); ulong_t cr4; if (HDLOPS(hdl)->cmio_getcr4 == NULL || HDLOPS(hdl)->cmio_setcr4 == NULL) return; cr4 = HDLOPS(hdl)->cmio_getcr4(hdl); HDLOPS(hdl)->cmio_setcr4(hdl, cr4 | CR4_MCE); } void cmi_hdl_msrinterpose(cmi_hdl_t ophdl, cmi_mca_regs_t *regs, uint_t nregs) { cmi_hdl_impl_t *hdl = IMPLHDL(ophdl); int i; if (HDLOPS(hdl)->cmio_msrinterpose == NULL) return; cmi_hdl_inj_begin(ophdl); for (i = 0; i < nregs; i++, regs++) HDLOPS(hdl)->cmio_msrinterpose(hdl, regs->cmr_msrnum, regs->cmr_msrval); cmi_hdl_inj_end(ophdl); } /*ARGSUSED*/ void cmi_hdl_msrforward(cmi_hdl_t ophdl, cmi_mca_regs_t *regs, uint_t nregs) { #ifdef __xpv cmi_hdl_impl_t *hdl = IMPLHDL(ophdl); int i; for (i = 0; i < nregs; i++, regs++) msri_addent(hdl, regs->cmr_msrnum, regs->cmr_msrval); #endif } void cmi_pcird_nohw(void) { cmi_pcicfg_flags &= ~CMI_PCICFG_FLAG_RD_HWOK; } void cmi_pciwr_nohw(void) { cmi_pcicfg_flags &= ~CMI_PCICFG_FLAG_WR_HWOK; } static uint32_t cmi_pci_get_cmn(int bus, int dev, int func, int reg, int asz, int *interpose, ddi_acc_handle_t hdl) { uint32_t val; if (cmi_pcicfg_flags & CMI_PCICFG_FLAG_RD_INTERPOSEOK && pcii_lookup(bus, dev, func, reg, asz, &val)) { if (interpose) *interpose = 1; return (val); } if (interpose) *interpose = 0; if (!(cmi_pcicfg_flags & CMI_PCICFG_FLAG_RD_HWOK)) return (0); switch (asz) { case 1: if (hdl) val = pci_config_get8(hdl, (off_t)reg); else val = pci_cfgacc_get8(NULL, PCI_GETBDF(bus, dev, func), reg); break; case 2: if (hdl) val = pci_config_get16(hdl, (off_t)reg); else val = pci_cfgacc_get16(NULL, PCI_GETBDF(bus, dev, func), reg); break; case 4: if (hdl) val = pci_config_get32(hdl, (off_t)reg); else val = pci_cfgacc_get32(NULL, PCI_GETBDF(bus, dev, func), reg); break; default: val = 0; } return (val); } uint8_t cmi_pci_getb(int bus, int dev, int func, int reg, int *interpose, ddi_acc_handle_t hdl) { return ((uint8_t)cmi_pci_get_cmn(bus, dev, func, reg, 1, interpose, hdl)); } uint16_t cmi_pci_getw(int bus, int dev, int func, int reg, int *interpose, ddi_acc_handle_t hdl) { return ((uint16_t)cmi_pci_get_cmn(bus, dev, func, reg, 2, interpose, hdl)); } uint32_t cmi_pci_getl(int bus, int dev, int func, int reg, int *interpose, ddi_acc_handle_t hdl) { return (cmi_pci_get_cmn(bus, dev, func, reg, 4, interpose, hdl)); } void cmi_pci_interposeb(int bus, int dev, int func, int reg, uint8_t val) { pcii_addent(bus, dev, func, reg, val, 1); } void cmi_pci_interposew(int bus, int dev, int func, int reg, uint16_t val) { pcii_addent(bus, dev, func, reg, val, 2); } void cmi_pci_interposel(int bus, int dev, int func, int reg, uint32_t val) { pcii_addent(bus, dev, func, reg, val, 4); } static void cmi_pci_put_cmn(int bus, int dev, int func, int reg, int asz, ddi_acc_handle_t hdl, uint32_t val) { /* * If there is an interposed value for this register invalidate it. */ pcii_rment(bus, dev, func, reg, asz); if (!(cmi_pcicfg_flags & CMI_PCICFG_FLAG_WR_HWOK)) return; switch (asz) { case 1: if (hdl) pci_config_put8(hdl, (off_t)reg, (uint8_t)val); else pci_cfgacc_put8(NULL, PCI_GETBDF(bus, dev, func), reg, (uint8_t)val); break; case 2: if (hdl) pci_config_put16(hdl, (off_t)reg, (uint16_t)val); else pci_cfgacc_put16(NULL, PCI_GETBDF(bus, dev, func), reg, (uint16_t)val); break; case 4: if (hdl) pci_config_put32(hdl, (off_t)reg, val); else pci_cfgacc_put32(NULL, PCI_GETBDF(bus, dev, func), reg, val); break; default: break; } } void cmi_pci_putb(int bus, int dev, int func, int reg, ddi_acc_handle_t hdl, uint8_t val) { cmi_pci_put_cmn(bus, dev, func, reg, 1, hdl, val); } void cmi_pci_putw(int bus, int dev, int func, int reg, ddi_acc_handle_t hdl, uint16_t val) { cmi_pci_put_cmn(bus, dev, func, reg, 2, hdl, val); } void cmi_pci_putl(int bus, int dev, int func, int reg, ddi_acc_handle_t hdl, uint32_t val) { cmi_pci_put_cmn(bus, dev, func, reg, 4, hdl, val); } cmi_errno_t cmi_cache_ncaches(cmi_hdl_t hdl, uint32_t *ncache) { return (HDLOPS(IMPLHDL(hdl))->cmio_ncache(IMPLHDL(hdl), ncache)); } cmi_errno_t cmi_cache_info(cmi_hdl_t hdl, uint32_t cno, x86_cache_t *cachep) { return (HDLOPS(IMPLHDL(hdl))->cmio_cache(IMPLHDL(hdl), cno, cachep)); } static const struct cmi_hdl_ops cmi_hdl_ops = { #ifdef __xpv /* * CMI_HDL_SOLARIS_xVM_MCA - ops when we are an xVM dom0 */ xpv_vendor, /* cmio_vendor */ xpv_vendorstr, /* cmio_vendorstr */ xpv_family, /* cmio_family */ xpv_model, /* cmio_model */ xpv_stepping, /* cmio_stepping */ xpv_chipid, /* cmio_chipid */ xpv_procnodeid, /* cmio_procnodeid */ xpv_coreid, /* cmio_coreid */ xpv_strandid, /* cmio_strandid */ xpv_procnodes_per_pkg, /* cmio_procnodes_per_pkg */ xpv_strand_apicid, /* cmio_strand_apicid */ xpv_chiprev, /* cmio_chiprev */ xpv_chiprevstr, /* cmio_chiprevstr */ xpv_getsockettype, /* cmio_getsockettype */ xpv_getsocketstr, /* cmio_getsocketstr */ xpv_chipsig, /* cmio_chipsig */ xpv_ncache, /* cmio_ncache */ xpv_cache, /* cmio_cache */ xpv_logical_id, /* cmio_logical_id */ NULL, /* cmio_getcr4 */ NULL, /* cmio_setcr4 */ xpv_rdmsr, /* cmio_rdmsr */ xpv_wrmsr, /* cmio_wrmsr */ xpv_msrinterpose, /* cmio_msrinterpose */ xpv_int, /* cmio_int */ xpv_online, /* cmio_online */ xpv_smbiosid, /* cmio_smbiosid */ xpv_smb_chipid, /* cmio_smb_chipid */ xpv_smb_bboard /* cmio_smb_bboard */ #else /* __xpv */ /* * CMI_HDL_NATIVE - ops when apparently running on bare-metal */ ntv_vendor, /* cmio_vendor */ ntv_vendorstr, /* cmio_vendorstr */ ntv_family, /* cmio_family */ ntv_model, /* cmio_model */ ntv_stepping, /* cmio_stepping */ ntv_chipid, /* cmio_chipid */ ntv_procnodeid, /* cmio_procnodeid */ ntv_coreid, /* cmio_coreid */ ntv_strandid, /* cmio_strandid */ ntv_procnodes_per_pkg, /* cmio_procnodes_per_pkg */ ntv_strand_apicid, /* cmio_strand_apicid */ ntv_chiprev, /* cmio_chiprev */ ntv_chiprevstr, /* cmio_chiprevstr */ ntv_getsockettype, /* cmio_getsockettype */ ntv_getsocketstr, /* cmio_getsocketstr */ ntv_chipsig, /* cmio_chipsig */ ntv_ncache, /* cmio_ncache */ ntv_cache, /* cmio_cache */ ntv_logical_id, /* cmio_logical_id */ ntv_getcr4, /* cmio_getcr4 */ ntv_setcr4, /* cmio_setcr4 */ ntv_rdmsr, /* cmio_rdmsr */ ntv_wrmsr, /* cmio_wrmsr */ ntv_msrinterpose, /* cmio_msrinterpose */ ntv_int, /* cmio_int */ ntv_online, /* cmio_online */ ntv_smbiosid, /* cmio_smbiosid */ ntv_smb_chipid, /* cmio_smb_chipid */ ntv_smb_bboard /* cmio_smb_bboard */ #endif }; /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2016 by Delphix. All rights reserved. */ /* * Copyright (c) 2010, Intel Corporation. * All rights reserved. */ /* * Copyright (c) 2018, Joyent, Inc. */ #include #include #include #include #include #include #include #include #include #include /* * Set to prevent model-specific support from initialising. */ int cms_no_model_specific = 0; /* * Subdirectory (relative to the module search path) in which we will * look for model-specific modules. */ #define CPUMOD_MS_SUBDIR "cpu" /* * Cpu model-specific modules have filenames beginning with the following. */ #define CPUMOD_MS_PREFIX "cpu_ms" #define HDL2CMS(hdl) cms_hdl_getcms(hdl) #define CMS_OPS(cms) (cms)->cms_ops #define CMS_OP_PRESENT(cms, op) ((cms) && CMS_OPS(cms)->op != NULL) struct cms_cpuid { const char *vendor; uint_t family; uint_t model; uint_t stepping; }; #define CMS_MATCH_VENDOR 0 /* Just match on vendor */ #define CMS_MATCH_FAMILY 1 /* Match down to family */ #define CMS_MATCH_MODEL 2 /* Match down to model */ #define CMS_MATCH_STEPPING 3 /* Match down to stepping */ /* * Structure used to keep track of modules we have loaded. */ typedef struct cms { struct cms *cms_next; struct cms *cms_prev; const cms_ops_t *cms_ops; struct modctl *cms_modp; uint_t cms_refcnt; } cms_t; static cms_t *cms_list; static kmutex_t cms_load_lock; /* * We stash a cms_t and associated private data via cmi_hdl_setspecific. */ struct cms_ctl { cms_t *cs_cms; void *cs_cmsdata; }; static cms_t * cms_hdl_getcms(cmi_hdl_t hdl) { struct cms_ctl *cdp = cmi_hdl_getspecific(hdl); return (cdp != NULL ? cdp->cs_cms : NULL); } void * cms_hdl_getcmsdata(cmi_hdl_t hdl) { struct cms_ctl *cdp = cmi_hdl_getspecific(hdl); return (cdp != NULL ? cdp->cs_cmsdata : NULL); } static void cms_link(cms_t *cms) { ASSERT(MUTEX_HELD(&cms_load_lock)); cms->cms_prev = NULL; cms->cms_next = cms_list; if (cms_list != NULL) cms_list->cms_prev = cms; cms_list = cms; } static void cms_unlink(cms_t *cms) { ASSERT(MUTEX_HELD(&cms_load_lock)); ASSERT(cms->cms_refcnt == 0); if (cms->cms_prev != NULL) cms->cms_prev->cms_next = cms->cms_next; if (cms->cms_next != NULL) cms->cms_next->cms_prev = cms->cms_prev; if (cms_list == cms) cms_list = cms->cms_next; } /* * Hold the module in memory. We call to CPU modules without using the * stubs mechanism, so these modules must be manually held in memory. * The mod_ref acts as if another loaded module has a dependency on us. */ static void cms_hold(cms_t *cms) { ASSERT(MUTEX_HELD(&cms_load_lock)); mutex_enter(&mod_lock); cms->cms_modp->mod_ref++; mutex_exit(&mod_lock); cms->cms_refcnt++; } static void cms_rele(cms_t *cms) { ASSERT(MUTEX_HELD(&cms_load_lock)); mutex_enter(&mod_lock); cms->cms_modp->mod_ref--; mutex_exit(&mod_lock); if (--cms->cms_refcnt == 0) { cms_unlink(cms); kmem_free(cms, sizeof (cms_t)); } } static cms_ops_t * cms_getops(modctl_t *modp) { cms_ops_t *ops; if ((ops = (cms_ops_t *)modlookup_by_modctl(modp, "_cms_ops")) == NULL) { cmn_err(CE_WARN, "cpu_ms module '%s' is invalid: no _cms_ops " "found", modp->mod_modname); return (NULL); } if (ops->cms_init == NULL) { cmn_err(CE_WARN, "cpu_ms module '%s' is invalid: no cms_init " "entry point", modp->mod_modname); return (NULL); } return (ops); } static cms_t * cms_load_modctl(modctl_t *modp) { cms_ops_t *ops; uintptr_t ver; cms_t *cms; cms_api_ver_t apiver; ASSERT(MUTEX_HELD(&cms_load_lock)); for (cms = cms_list; cms != NULL; cms = cms->cms_next) { if (cms->cms_modp == modp) return (cms); } if ((ver = modlookup_by_modctl(modp, "_cms_api_version")) == 0) { cmn_err(CE_WARN, "cpu model-specific module '%s' is invalid: " "no _cms_api_version", modp->mod_modname); return (NULL); } else { apiver = *((cms_api_ver_t *)ver); if (!CMS_API_VERSION_CHKMAGIC(apiver)) { cmn_err(CE_WARN, "cpu model-specific module '%s' is " "invalid: _cms_api_version 0x%x has bad magic", modp->mod_modname, apiver); return (NULL); } } if (apiver != CMS_API_VERSION) { cmn_err(CE_WARN, "cpu model-specific module '%s' has API " "version %d, kernel requires API version %d", modp->mod_modname, CMS_API_VERSION_TOPRINT(apiver), CMS_API_VERSION_TOPRINT(CMS_API_VERSION)); return (NULL); } if ((ops = cms_getops(modp)) == NULL) return (NULL); cms = kmem_zalloc(sizeof (cms_t), KM_SLEEP); cms->cms_ops = ops; cms->cms_modp = modp; cms_link(cms); return (cms); } static int cms_cpu_match(cmi_hdl_t hdl1, cmi_hdl_t hdl2, int match) { if (match >= CMS_MATCH_VENDOR && cmi_hdl_vendor(hdl1) != cmi_hdl_vendor(hdl2)) return (0); if (match >= CMS_MATCH_FAMILY && cmi_hdl_family(hdl1) != cmi_hdl_family(hdl2)) return (0); if (match >= CMS_MATCH_MODEL && cmi_hdl_model(hdl1) != cmi_hdl_model(hdl2)) return (0); if (match >= CMS_MATCH_STEPPING && cmi_hdl_stepping(hdl1) != cmi_hdl_stepping(hdl2)) return (0); return (1); } static int cms_search_list_cb(cmi_hdl_t whdl, void *arg1, void *arg2, void *arg3) { cmi_hdl_t thdl = (cmi_hdl_t)arg1; int match = *((int *)arg2); cmi_hdl_t *rsltp = (cmi_hdl_t *)arg3; if (cms_cpu_match(thdl, whdl, match)) { cmi_hdl_hold(whdl); /* short-term hold */ *rsltp = whdl; return (CMI_HDL_WALK_DONE); } else { return (CMI_HDL_WALK_NEXT); } } /* * Look to see if we've already got a module loaded for a CPU just * like this one. If we do, then we'll re-use it. */ static cms_t * cms_search_list(cmi_hdl_t hdl, int match) { cmi_hdl_t dhdl = NULL; cms_t *cms = NULL; ASSERT(MUTEX_HELD(&cms_load_lock)); cmi_hdl_walk(cms_search_list_cb, (void *)hdl, (void *)&match, &dhdl); if (dhdl) { cms = HDL2CMS(dhdl); cmi_hdl_rele(dhdl); /* held in cms_search_list_cb */ } return (cms); } /* * Try to find or load a module that offers model-specific support for * this vendor/family/model/stepping combination. When attempting to load * a module we look in CPUMOD_MS_SUBDIR first for a match on * vendor/family/model/stepping, then on vendor/family/model (ignoring * stepping), then on vendor/family (ignoring model and stepping), then * on vendor alone. */ static cms_t * cms_load_module(cmi_hdl_t hdl, int match, int *chosenp) { modctl_t *modp; cms_t *cms; int modid; uint_t s[3]; ASSERT(MUTEX_HELD(&cms_load_lock)); ASSERT(match == CMS_MATCH_STEPPING || match == CMS_MATCH_MODEL || match == CMS_MATCH_FAMILY || match == CMS_MATCH_VENDOR); s[0] = cmi_hdl_family(hdl); s[1] = cmi_hdl_model(hdl); s[2] = cmi_hdl_stepping(hdl); /* * Have we already loaded a module for a cpu with the same * vendor/family/model/stepping? */ if ((cms = cms_search_list(hdl, match)) != NULL) { cms_hold(cms); return (cms); } modid = modload_qualified(CPUMOD_MS_SUBDIR, CPUMOD_MS_PREFIX, cmi_hdl_vendorstr(hdl), ".", s, match, chosenp); if (modid == -1) return (NULL); modp = mod_hold_by_id(modid); cms = cms_load_modctl(modp); if (cms) cms_hold(cms); mod_release_mod(modp); return (cms); } static cms_t * cms_load_specific(cmi_hdl_t hdl, void **datap) { cms_t *cms; int err; int i; ASSERT(MUTEX_HELD(&cms_load_lock)); for (i = CMS_MATCH_STEPPING; i >= CMS_MATCH_VENDOR; i--) { int suffixlevel; if ((cms = cms_load_module(hdl, i, &suffixlevel)) == NULL) return (NULL); /* * A module has loaded and has a _cms_ops structure, and the * module has been held for this instance. Call the cms_init * entry point - we expect success (0) or ENOTSUP. */ if ((err = cms->cms_ops->cms_init(hdl, datap)) == 0) { if (boothowto & RB_VERBOSE) { printf("initialized model-specific " "module '%s' on chip %d core %d " "strand %d\n", cms->cms_modp->mod_modname, cmi_hdl_chipid(hdl), cmi_hdl_coreid(hdl), cmi_hdl_strandid(hdl)); } return (cms); } else if (err != ENOTSUP) { cmn_err(CE_WARN, "failed to init model-specific " "module '%s' on chip %d core %d strand %d: err=%d", cms->cms_modp->mod_modname, cmi_hdl_chipid(hdl), cmi_hdl_coreid(hdl), cmi_hdl_strandid(hdl), err); } /* * The module failed or declined to init, so release * it and potentially change i to be equal to the number * of suffices actually used in the last module path. */ cms_rele(cms); i = suffixlevel; } return (NULL); } void cms_init(cmi_hdl_t hdl) { cms_t *cms; void *data; if (cms_no_model_specific != 0) return; mutex_enter(&cms_load_lock); if ((cms = cms_load_specific(hdl, &data)) != NULL) { struct cms_ctl *cdp; ASSERT(cmi_hdl_getspecific(hdl) == NULL); cdp = kmem_alloc(sizeof (*cdp), KM_SLEEP); cdp->cs_cms = cms; cdp->cs_cmsdata = data; cmi_hdl_setspecific(hdl, cdp); } mutex_exit(&cms_load_lock); } void cms_fini(cmi_hdl_t hdl) { cms_t *cms = HDL2CMS(hdl); struct cms_ctl *cdp; if (CMS_OP_PRESENT(cms, cms_fini)) CMS_OPS(cms)->cms_fini(hdl); mutex_enter(&cms_load_lock); cdp = (struct cms_ctl *)cmi_hdl_getspecific(hdl); if (cdp != NULL) { if (cdp->cs_cms != NULL) cms_rele(cdp->cs_cms); kmem_free(cdp, sizeof (*cdp)); } mutex_exit(&cms_load_lock); } boolean_t cms_present(cmi_hdl_t hdl) { return (HDL2CMS(hdl) != NULL ? B_TRUE : B_FALSE); } void cms_post_startup(cmi_hdl_t hdl) { cms_t *cms = HDL2CMS(hdl); if (CMS_OP_PRESENT(cms, cms_post_startup)) CMS_OPS(cms)->cms_post_startup(hdl); } void cms_post_mpstartup(cmi_hdl_t hdl) { cms_t *cms = HDL2CMS(hdl); if (CMS_OP_PRESENT(cms, cms_post_mpstartup)) CMS_OPS(cms)->cms_post_mpstartup(hdl); } size_t cms_logout_size(cmi_hdl_t hdl) { cms_t *cms = HDL2CMS(hdl); if (!CMS_OP_PRESENT(cms, cms_logout_size)) return (0); return (CMS_OPS(cms)->cms_logout_size(hdl)); } uint64_t cms_mcgctl_val(cmi_hdl_t hdl, int nbanks, uint64_t def) { cms_t *cms = HDL2CMS(hdl); if (!CMS_OP_PRESENT(cms, cms_mcgctl_val)) return (def); return (CMS_OPS(cms)->cms_mcgctl_val(hdl, nbanks, def)); } boolean_t cms_bankctl_skipinit(cmi_hdl_t hdl, int banknum) { cms_t *cms = HDL2CMS(hdl); if (!CMS_OP_PRESENT(cms, cms_bankctl_skipinit)) return (B_FALSE); return (CMS_OPS(cms)->cms_bankctl_skipinit(hdl, banknum)); } uint64_t cms_bankctl_val(cmi_hdl_t hdl, int banknum, uint64_t def) { cms_t *cms = HDL2CMS(hdl); if (!CMS_OP_PRESENT(cms, cms_bankctl_val)) return (def); return (CMS_OPS(cms)->cms_bankctl_val(hdl, banknum, def)); } boolean_t cms_bankstatus_skipinit(cmi_hdl_t hdl, int banknum) { cms_t *cms = HDL2CMS(hdl); if (!CMS_OP_PRESENT(cms, cms_bankstatus_skipinit)) return (B_FALSE); return (CMS_OPS(cms)->cms_bankstatus_skipinit(hdl, banknum)); } uint64_t cms_bankstatus_val(cmi_hdl_t hdl, int banknum, uint64_t def) { cms_t *cms = HDL2CMS(hdl); if (!CMS_OP_PRESENT(cms, cms_bankstatus_val)) return (def); return (CMS_OPS(cms)->cms_bankstatus_val(hdl, banknum, def)); } void cms_mca_init(cmi_hdl_t hdl, int nbanks) { cms_t *cms = HDL2CMS(hdl); if (CMS_OP_PRESENT(cms, cms_mca_init)) CMS_OPS(cms)->cms_mca_init(hdl, nbanks); } uint64_t cms_poll_ownermask(cmi_hdl_t hdl, hrtime_t poll_interval) { cms_t *cms = HDL2CMS(hdl); if (CMS_OP_PRESENT(cms, cms_poll_ownermask)) return (CMS_OPS(cms)->cms_poll_ownermask(hdl, poll_interval)); else return (-1ULL); /* poll all banks by default */ } void cms_bank_logout(cmi_hdl_t hdl, int banknum, uint64_t status, uint64_t addr, uint64_t misc, void *mslogout) { cms_t *cms = HDL2CMS(hdl); if (mslogout != NULL && CMS_OP_PRESENT(cms, cms_bank_logout)) CMS_OPS(cms)->cms_bank_logout(hdl, banknum, status, addr, misc, mslogout); } cms_errno_t cms_msrinject(cmi_hdl_t hdl, uint_t msr, uint64_t val) { cms_t *cms = HDL2CMS(hdl); if (CMS_OP_PRESENT(cms, cms_msrinject)) return (CMS_OPS(cms)->cms_msrinject(hdl, msr, val)); else return (CMSERR_NOTSUP); } uint32_t cms_error_action(cmi_hdl_t hdl, int ismc, int banknum, uint64_t status, uint64_t addr, uint64_t misc, void *mslogout) { cms_t *cms = HDL2CMS(hdl); if (CMS_OP_PRESENT(cms, cms_error_action)) return (CMS_OPS(cms)->cms_error_action(hdl, ismc, banknum, status, addr, misc, mslogout)); else return (0); } cms_cookie_t cms_disp_match(cmi_hdl_t hdl, int ismc, int banknum, uint64_t status, uint64_t addr, uint64_t misc, void *mslogout) { cms_t *cms = HDL2CMS(hdl); if (CMS_OP_PRESENT(cms, cms_disp_match)) return (CMS_OPS(cms)->cms_disp_match(hdl, ismc, banknum, status, addr, misc, mslogout)); else return (NULL); } void cms_ereport_class(cmi_hdl_t hdl, cms_cookie_t mscookie, const char **cpuclsp, const char **leafclsp) { cms_t *cms = HDL2CMS(hdl); if (cpuclsp == NULL || leafclsp == NULL) return; *cpuclsp = *leafclsp = NULL; if (CMS_OP_PRESENT(cms, cms_ereport_class)) { CMS_OPS(cms)->cms_ereport_class(hdl, mscookie, cpuclsp, leafclsp); } } nvlist_t * cms_ereport_detector(cmi_hdl_t hdl, int bankno, cms_cookie_t mscookie, nv_alloc_t *nva) { cms_t *cms = HDL2CMS(hdl); if (CMS_OP_PRESENT(cms, cms_ereport_detector)) return (CMS_OPS(cms)->cms_ereport_detector(hdl, bankno, mscookie, nva)); else return (NULL); } boolean_t cms_ereport_includestack(cmi_hdl_t hdl, cms_cookie_t mscookie) { cms_t *cms = HDL2CMS(hdl); if (CMS_OP_PRESENT(cms, cms_ereport_includestack)) { return (CMS_OPS(cms)->cms_ereport_includestack(hdl, mscookie)); } else { return (B_FALSE); } } void cms_ereport_add_logout(cmi_hdl_t hdl, nvlist_t *nvl, nv_alloc_t *nva, int banknum, uint64_t status, uint64_t addr, uint64_t misc, void *mslogout, cms_cookie_t mscookie) { cms_t *cms = HDL2CMS(hdl); if (CMS_OP_PRESENT(cms, cms_ereport_add_logout)) CMS_OPS(cms)->cms_ereport_add_logout(hdl, nvl, nva, banknum, status, addr, misc, mslogout, mscookie); } /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved. */ /* * Platform specific implementation code * Currently only suspend to RAM is supported (ACPI S3) */ #define SUNDDI_IMPL #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define AFMT "%lx" extern int flushes_require_xcalls; extern cpuset_t cpu_ready_set; extern void *wc_long_mode_64(void); extern int tsc_gethrtime_enable; extern void i_cpr_start_cpu(void); ushort_t cpr_mach_type = CPR_MACHTYPE_X86; void (*cpr_start_cpu_func)(void) = i_cpr_start_cpu; static wc_cpu_t *wc_other_cpus = NULL; static cpuset_t procset; static void init_real_mode_platter(int cpun, uint32_t offset, uint_t cr4, wc_desctbr_t gdt); static int i_cpr_platform_alloc(psm_state_request_t *req); static void i_cpr_platform_free(psm_state_request_t *req); static int i_cpr_save_apic(psm_state_request_t *req); static int i_cpr_restore_apic(psm_state_request_t *req); static int wait_for_set(cpuset_t *set, int who); static void i_cpr_save_stack(kthread_t *t, wc_cpu_t *wc_cpu); void i_cpr_restore_stack(kthread_t *t, greg_t *save_stack); #ifdef STACK_GROWTH_DOWN #define CPR_GET_STACK_START(t) ((t)->t_stkbase) #define CPR_GET_STACK_END(t) ((t)->t_stk) #else #define CPR_GET_STACK_START(t) ((t)->t_stk) #define CPR_GET_STACK_END(t) ((t)->t_stkbase) #endif /* STACK_GROWTH_DOWN */ /* * restart paused slave cpus */ void i_cpr_machdep_setup(void) { if (ncpus > 1) { CPR_DEBUG(CPR_DEBUG1, ("MP restarted...\n")); mutex_enter(&cpu_lock); start_cpus(); mutex_exit(&cpu_lock); } } /* * Stop all interrupt activities in the system */ void i_cpr_stop_intr(void) { (void) spl7(); } /* * Set machine up to take interrupts */ void i_cpr_enable_intr(void) { (void) spl0(); } /* * Save miscellaneous information which needs to be written to the * state file. This information is required to re-initialize * kernel/prom handshaking. */ void i_cpr_save_machdep_info(void) { int notcalled = 0; ASSERT(notcalled); } void i_cpr_set_tbr(void) { } processorid_t i_cpr_bootcpuid(void) { return (0); } /* * cpu0 should contain bootcpu info */ cpu_t * i_cpr_bootcpu(void) { ASSERT(MUTEX_HELD(&cpu_lock)); return (cpu_get(i_cpr_bootcpuid())); } /* * Save context for the specified CPU */ void * i_cpr_save_context(void *arg) { long index = (long)arg; psm_state_request_t *papic_state; int resuming; int ret; wc_cpu_t *wc_cpu = wc_other_cpus + index; PMD(PMD_SX, ("i_cpr_save_context() index = %ld\n", index)) ASSERT(index < NCPU); papic_state = &(wc_cpu)->wc_apic_state; ret = i_cpr_platform_alloc(papic_state); ASSERT(ret == 0); ret = i_cpr_save_apic(papic_state); ASSERT(ret == 0); i_cpr_save_stack(curthread, wc_cpu); /* * wc_save_context returns twice, once when susending and * once when resuming, wc_save_context() returns 0 when * suspending and non-zero upon resume */ resuming = (wc_save_context(wc_cpu) == 0); /* * do NOT call any functions after this point, because doing so * will modify the stack that we are running on */ if (resuming) { ret = i_cpr_restore_apic(papic_state); ASSERT(ret == 0); i_cpr_platform_free(papic_state); /* * Enable interrupts on this cpu. * Do not bind interrupts to this CPU's local APIC until * the CPU is ready to receive interrupts. */ ASSERT(CPU->cpu_id != i_cpr_bootcpuid()); mutex_enter(&cpu_lock); cpu_enable_intr(CPU); mutex_exit(&cpu_lock); /* * Setting the bit in cpu_ready_set must be the last operation * in processor initialization; the boot CPU will continue to * boot once it sees this bit set for all active CPUs. */ CPUSET_ATOMIC_ADD(cpu_ready_set, CPU->cpu_id); PMD(PMD_SX, ("i_cpr_save_context() resuming cpu %d in cpu_ready_set\n", CPU->cpu_id)) } else { /* * Disable interrupts on this CPU so that PSM knows not to bind * interrupts here on resume until the CPU has executed * cpu_enable_intr() (above) in the resume path. * We explicitly do not grab cpu_lock here because at this point * in the suspend process, the boot cpu owns cpu_lock and all * other cpus are also executing in the pause thread (only * modifying their respective CPU structure). */ (void) cpu_disable_intr(CPU); } PMD(PMD_SX, ("i_cpr_save_context: wc_save_context returns %d\n", resuming)) return (NULL); } static ushort_t *warm_reset_vector = NULL; static ushort_t * map_warm_reset_vector() { /*LINTED*/ if (!(warm_reset_vector = (ushort_t *)psm_map_phys(WARM_RESET_VECTOR, sizeof (ushort_t *), PROT_READ|PROT_WRITE))) return (NULL); /* * setup secondary cpu bios boot up vector */ *warm_reset_vector = (ushort_t)((caddr_t) /*LINTED*/ ((struct rm_platter *)rm_platter_va)->rm_code - rm_platter_va + ((ulong_t)rm_platter_va & 0xf)); warm_reset_vector++; *warm_reset_vector = (ushort_t)(rm_platter_pa >> 4); --warm_reset_vector; return (warm_reset_vector); } void i_cpr_pre_resume_cpus() { /* * this is a cut down version of start_other_cpus() * just do the initialization to wake the other cpus */ unsigned who; int boot_cpuid = i_cpr_bootcpuid(); uint32_t code_length = 0; caddr_t wakevirt = rm_platter_va; /*LINTED*/ wakecode_t *wp = (wakecode_t *)wakevirt; char *str = "i_cpr_pre_resume_cpus"; extern int get_tsc_ready(); int err; /*LINTED*/ rm_platter_t *real_mode_platter = (rm_platter_t *)rm_platter_va; /* * If startup wasn't able to find a page under 1M, we cannot * proceed. */ if (rm_platter_va == 0) { cmn_err(CE_WARN, "Cannot suspend the system because no " "memory below 1M could be found for processor startup"); return; } /* * Copy the real mode code at "real_mode_start" to the * page at rm_platter_va. */ warm_reset_vector = map_warm_reset_vector(); if (warm_reset_vector == NULL) { PMD(PMD_SX, ("i_cpr_pre_resume_cpus() returning #2\n")) return; } flushes_require_xcalls = 1; /* * We lock our affinity to the master CPU to ensure that all slave CPUs * do their TSC syncs with the same CPU. */ affinity_set(CPU_CURRENT); /* * Mark the boot cpu as being ready and in the procset, since we are * running on that cpu. */ CPUSET_ONLY(cpu_ready_set, boot_cpuid); CPUSET_ONLY(procset, boot_cpuid); for (who = 0; who < max_ncpus; who++) { wc_cpu_t *cpup = wc_other_cpus + who; wc_desctbr_t gdt; if (who == boot_cpuid) continue; if (!CPU_IN_SET(mp_cpus, who)) continue; PMD(PMD_SX, ("%s() waking up %d cpu\n", str, who)) bcopy(cpup, &(wp->wc_cpu), sizeof (wc_cpu_t)); gdt.base = cpup->wc_gdt_base; gdt.limit = cpup->wc_gdt_limit; code_length = (uint32_t)((uintptr_t)wc_long_mode_64 - (uintptr_t)wc_rm_start); init_real_mode_platter(who, code_length, cpup->wc_cr4, gdt); mutex_enter(&cpu_lock); err = mach_cpuid_start(who, rm_platter_va); mutex_exit(&cpu_lock); if (err != 0) { cmn_err(CE_WARN, "cpu%d: failed to start during " "suspend/resume error %d", who, err); continue; } PMD(PMD_SX, ("%s() #1 waiting for %d in procset\n", str, who)) if (!wait_for_set(&procset, who)) continue; PMD(PMD_SX, ("%s() %d cpu started\n", str, who)) PMD(PMD_SX, ("%s() tsc_ready = %d\n", str, get_tsc_ready())) if (tsc_gethrtime_enable) { PMD(PMD_SX, ("%s() calling tsc_sync_master\n", str)) tsc_sync_master(who); } PMD(PMD_SX, ("%s() waiting for %d in cpu_ready_set\n", str, who)) /* * Wait for cpu to declare that it is ready, we want the * cpus to start serially instead of in parallel, so that * they do not contend with each other in wc_rm_start() */ if (!wait_for_set(&cpu_ready_set, who)) continue; /* * do not need to re-initialize dtrace using dtrace_cpu_init * function */ PMD(PMD_SX, ("%s() cpu %d now ready\n", str, who)) } affinity_clear(); PMD(PMD_SX, ("%s() all cpus now ready\n", str)) } static void unmap_warm_reset_vector(ushort_t *warm_reset_vector) { psm_unmap_phys((caddr_t)warm_reset_vector, sizeof (ushort_t *)); } /* * We need to setup a 1:1 (virtual to physical) mapping for the * page containing the wakeup code. */ static struct as *save_as; /* when switching to kas */ static void unmap_wakeaddr_1to1(uint64_t wakephys) { uintptr_t wp = (uintptr_t)wakephys; hat_setup(save_as->a_hat, 0); /* switch back from kernel hat */ hat_unload(kas.a_hat, (caddr_t)wp, PAGESIZE, HAT_UNLOAD); } void i_cpr_post_resume_cpus() { uint64_t wakephys = rm_platter_pa; if (warm_reset_vector != NULL) unmap_warm_reset_vector(warm_reset_vector); hat_unload(kas.a_hat, (caddr_t)(uintptr_t)rm_platter_pa, MMU_PAGESIZE, HAT_UNLOAD); /* * cmi_post_mpstartup() is only required upon boot not upon * resume from RAM */ PT(PT_UNDO1to1); /* Tear down 1:1 mapping for wakeup code */ unmap_wakeaddr_1to1(wakephys); } /* ARGSUSED */ void i_cpr_handle_xc(int flag) { } int i_cpr_reusable_supported(void) { return (0); } static void map_wakeaddr_1to1(uint64_t wakephys) { uintptr_t wp = (uintptr_t)wakephys; hat_devload(kas.a_hat, (caddr_t)wp, PAGESIZE, btop(wakephys), (PROT_READ|PROT_WRITE|PROT_EXEC|HAT_STORECACHING_OK|HAT_NOSYNC), HAT_LOAD); save_as = curthread->t_procp->p_as; hat_setup(kas.a_hat, 0); /* switch to kernel-only hat */ } void prt_other_cpus() { int who; if (ncpus == 1) { PMD(PMD_SX, ("prt_other_cpus() other cpu table empty for " "uniprocessor machine\n")) return; } for (who = 0; who < max_ncpus; who++) { wc_cpu_t *cpup = wc_other_cpus + who; if (!CPU_IN_SET(mp_cpus, who)) continue; PMD(PMD_SX, ("prt_other_cpus() who = %d, gdt=%p:%x, " "idt=%p:%x, ldt=%lx, tr=%lx, kgsbase=" AFMT ", sp=%lx\n", who, (void *)cpup->wc_gdt_base, cpup->wc_gdt_limit, (void *)cpup->wc_idt_base, cpup->wc_idt_limit, (long)cpup->wc_ldt, (long)cpup->wc_tr, (long)cpup->wc_kgsbase, (long)cpup->wc_rsp)) } } /* * Power down the system. */ int i_cpr_power_down(int sleeptype) { caddr_t wakevirt = rm_platter_va; uint64_t wakephys = rm_platter_pa; ulong_t saved_intr; uint32_t code_length = 0; wc_desctbr_t gdt; /*LINTED*/ wakecode_t *wp = (wakecode_t *)wakevirt; /*LINTED*/ rm_platter_t *wcpp = (rm_platter_t *)wakevirt; wc_cpu_t *cpup = &(wp->wc_cpu); dev_info_t *ppm; int ret = 0; power_req_t power_req; char *str = "i_cpr_power_down"; /*LINTED*/ rm_platter_t *real_mode_platter = (rm_platter_t *)rm_platter_va; extern int cpr_suspend_succeeded; extern void kernel_wc_code(); ASSERT(sleeptype == CPR_TORAM); ASSERT(CPU->cpu_id == 0); if ((ppm = PPM(ddi_root_node())) == NULL) { PMD(PMD_SX, ("%s: root node not claimed\n", str)) return (ENOTTY); } PMD(PMD_SX, ("Entering %s()\n", str)) PT(PT_IC); saved_intr = intr_clear(); PT(PT_1to1); /* Setup 1:1 mapping for wakeup code */ map_wakeaddr_1to1(wakephys); PMD(PMD_SX, ("ncpus=%d\n", ncpus)) PMD(PMD_SX, ("wc_rm_end - wc_rm_start=%lx WC_CODESIZE=%x\n", ((size_t)((uintptr_t)wc_rm_end - (uintptr_t)wc_rm_start)), WC_CODESIZE)) PMD(PMD_SX, ("wakevirt=%p, wakephys=%x\n", (void *)wakevirt, (uint_t)wakephys)) ASSERT(((size_t)((uintptr_t)wc_rm_end - (uintptr_t)wc_rm_start)) < WC_CODESIZE); bzero(wakevirt, PAGESIZE); /* Copy code to rm_platter */ bcopy((caddr_t)wc_rm_start, wakevirt, (size_t)((uintptr_t)wc_rm_end - (uintptr_t)wc_rm_start)); prt_other_cpus(); PMD(PMD_SX, ("real_mode_platter->rm_cr4=%lx, getcr4()=%lx\n", (ulong_t)real_mode_platter->rm_cr4, (ulong_t)getcr4())) PMD(PMD_SX, ("real_mode_platter->rm_pdbr=%lx, getcr3()=%lx\n", (ulong_t)real_mode_platter->rm_pdbr, getcr3())) real_mode_platter->rm_cr4 = getcr4(); real_mode_platter->rm_pdbr = getcr3(); rmp_gdt_init(real_mode_platter); /* * Since the CPU needs to jump to protected mode using an identity * mapped address, we need to calculate it here. */ real_mode_platter->rm_longmode64_addr = rm_platter_pa + (uint32_t)((uintptr_t)wc_long_mode_64 - (uintptr_t)wc_rm_start); PMD(PMD_SX, ("real_mode_platter->rm_cr4=%lx, getcr4()=%lx\n", (ulong_t)real_mode_platter->rm_cr4, getcr4())) PMD(PMD_SX, ("real_mode_platter->rm_pdbr=%lx, getcr3()=%lx\n", (ulong_t)real_mode_platter->rm_pdbr, getcr3())) PMD(PMD_SX, ("real_mode_platter->rm_longmode64_addr=%lx\n", (ulong_t)real_mode_platter->rm_longmode64_addr)) PT(PT_SC); if (wc_save_context(cpup)) { ret = i_cpr_platform_alloc(&(wc_other_cpus->wc_apic_state)); if (ret != 0) return (ret); ret = i_cpr_save_apic(&(wc_other_cpus->wc_apic_state)); PMD(PMD_SX, ("%s: i_cpr_save_apic() returned %d\n", str, ret)) if (ret != 0) return (ret); PMD(PMD_SX, ("wakephys=%x, kernel_wc_code=%p\n", (uint_t)wakephys, (void *)&kernel_wc_code)) PMD(PMD_SX, ("virtaddr=%lx, retaddr=%lx\n", (long)cpup->wc_virtaddr, (long)cpup->wc_retaddr)) PMD(PMD_SX, ("ebx=%x, edi=%x, esi=%x, ebp=%x, esp=%x\n", cpup->wc_ebx, cpup->wc_edi, cpup->wc_esi, cpup->wc_ebp, cpup->wc_esp)) PMD(PMD_SX, ("cr0=%lx, cr3=%lx, cr4=%lx\n", (long)cpup->wc_cr0, (long)cpup->wc_cr3, (long)cpup->wc_cr4)) PMD(PMD_SX, ("cs=%x, ds=%x, es=%x, ss=%x, fs=%lx, gs=%lx, " "flgs=%lx\n", cpup->wc_cs, cpup->wc_ds, cpup->wc_es, cpup->wc_ss, (long)cpup->wc_fs, (long)cpup->wc_gs, (long)cpup->wc_eflags)) PMD(PMD_SX, ("gdt=%p:%x, idt=%p:%x, ldt=%lx, tr=%lx, " "kgbase=%lx\n", (void *)cpup->wc_gdt_base, cpup->wc_gdt_limit, (void *)cpup->wc_idt_base, cpup->wc_idt_limit, (long)cpup->wc_ldt, (long)cpup->wc_tr, (long)cpup->wc_kgsbase)) gdt.base = cpup->wc_gdt_base; gdt.limit = cpup->wc_gdt_limit; code_length = (uint32_t)((uintptr_t)wc_long_mode_64 - (uintptr_t)wc_rm_start); init_real_mode_platter(0, code_length, cpup->wc_cr4, gdt); PMD(PMD_SX, ("real_mode_platter->rm_cr4=%lx, getcr4()=%lx\n", (ulong_t)wcpp->rm_cr4, getcr4())) PMD(PMD_SX, ("real_mode_platter->rm_pdbr=%lx, getcr3()=%lx\n", (ulong_t)wcpp->rm_pdbr, getcr3())) PMD(PMD_SX, ("real_mode_platter->rm_longmode64_addr=%lx\n", (ulong_t)wcpp->rm_longmode64_addr)) PMD(PMD_SX, ("real_mode_platter->rm_temp_gdt[TEMPGDT_KCODE64]=%lx\n", (ulong_t)wcpp->rm_temp_gdt[TEMPGDT_KCODE64])) PMD(PMD_SX, ("gdt=%p:%x, idt=%p:%x, ldt=%lx, tr=%lx, " "kgsbase=%lx\n", (void *)wcpp->rm_gdt_base, wcpp->rm_gdt_lim, (void *)wcpp->rm_idt_base, wcpp->rm_idt_lim, (long)cpup->wc_ldt, (long)cpup->wc_tr, (long)cpup->wc_kgsbase)) power_req.request_type = PMR_PPM_ENTER_SX; power_req.req.ppm_power_enter_sx_req.sx_state = S3; power_req.req.ppm_power_enter_sx_req.test_point = cpr_test_point; power_req.req.ppm_power_enter_sx_req.wakephys = wakephys; PMD(PMD_SX, ("%s: pm_ctlops PMR_PPM_ENTER_SX\n", str)) PT(PT_PPMCTLOP); (void) pm_ctlops(ppm, ddi_root_node(), DDI_CTLOPS_POWER, &power_req, &ret); PMD(PMD_SX, ("%s: returns %d\n", str, ret)) /* * If it works, we get control back to the else branch below * If we get control back here, it didn't work. * XXX return EINVAL here? */ unmap_wakeaddr_1to1(wakephys); intr_restore(saved_intr); return (ret); } else { cpr_suspend_succeeded = 1; power_req.request_type = PMR_PPM_EXIT_SX; power_req.req.ppm_power_enter_sx_req.sx_state = S3; PMD(PMD_SX, ("%s: pm_ctlops PMR_PPM_EXIT_SX\n", str)) PT(PT_PPMCTLOP); (void) pm_ctlops(ppm, ddi_root_node(), DDI_CTLOPS_POWER, &power_req, &ret); PMD(PMD_SX, ("%s: returns %d\n", str, ret)) ret = i_cpr_restore_apic(&(wc_other_cpus->wc_apic_state)); /* * the restore should never fail, if the saved suceeded */ ASSERT(ret == 0); i_cpr_platform_free(&(wc_other_cpus->wc_apic_state)); /* * Enable interrupts on boot cpu. */ ASSERT(CPU->cpu_id == i_cpr_bootcpuid()); mutex_enter(&cpu_lock); cpu_enable_intr(CPU); mutex_exit(&cpu_lock); PT(PT_INTRRESTORE); intr_restore(saved_intr); PT(PT_CPU); return (ret); } } /* * Stop all other cpu's before halting or rebooting. We pause the cpu's * instead of sending a cross call. * Stolen from sun4/os/mp_states.c */ static int cpu_are_paused; /* sic */ void i_cpr_stop_other_cpus(void) { mutex_enter(&cpu_lock); if (cpu_are_paused) { mutex_exit(&cpu_lock); return; } pause_cpus(NULL, NULL); cpu_are_paused = 1; mutex_exit(&cpu_lock); } int i_cpr_is_supported(int sleeptype) { extern int cpr_supported_override; extern int cpr_platform_enable; extern int pm_S3_enabled; if (sleeptype != CPR_TORAM) return (0); /* * The next statement tests if a specific platform has turned off * cpr support. */ if (cpr_supported_override) return (0); /* * If a platform has specifically turned on cpr support ... */ if (cpr_platform_enable) return (1); return (pm_S3_enabled); } void i_cpr_bitmap_cleanup(void) { } void i_cpr_free_memory_resources(void) { } /* * Needed only for S3 so far */ static int i_cpr_platform_alloc(psm_state_request_t *req) { #ifdef DEBUG char *str = "i_cpr_platform_alloc"; #endif PMD(PMD_SX, ("cpu = %d, %s(%p) \n", CPU->cpu_id, str, (void *)req)) if (psm_state == NULL) { PMD(PMD_SX, ("%s() : psm_state == NULL\n", str)) return (0); } req->psr_cmd = PSM_STATE_ALLOC; return ((*psm_state)(req)); } /* * Needed only for S3 so far */ static void i_cpr_platform_free(psm_state_request_t *req) { #ifdef DEBUG char *str = "i_cpr_platform_free"; #endif PMD(PMD_SX, ("cpu = %d, %s(%p) \n", CPU->cpu_id, str, (void *)req)) if (psm_state == NULL) { PMD(PMD_SX, ("%s() : psm_state == NULL\n", str)) return; } req->psr_cmd = PSM_STATE_FREE; (void) (*psm_state)(req); } static int i_cpr_save_apic(psm_state_request_t *req) { #ifdef DEBUG char *str = "i_cpr_save_apic"; #endif if (psm_state == NULL) { PMD(PMD_SX, ("%s() : psm_state == NULL\n", str)) return (0); } req->psr_cmd = PSM_STATE_SAVE; return ((*psm_state)(req)); } static int i_cpr_restore_apic(psm_state_request_t *req) { #ifdef DEBUG char *str = "i_cpr_restore_apic"; #endif if (psm_state == NULL) { PMD(PMD_SX, ("%s() : psm_state == NULL\n", str)) return (0); } req->psr_cmd = PSM_STATE_RESTORE; return ((*psm_state)(req)); } static void init_real_mode_platter(int cpun, uint32_t offset, uint_t cr4, wc_desctbr_t gdt) { /*LINTED*/ rm_platter_t *real_mode_platter = (rm_platter_t *)rm_platter_va; /* * Fill up the real mode platter to make it easy for real mode code to * kick it off. This area should really be one passed by boot to kernel * and guaranteed to be below 1MB and aligned to 16 bytes. Should also * have identical physical and virtual address in paged mode. */ real_mode_platter->rm_pdbr = getcr3(); real_mode_platter->rm_cpu = cpun; real_mode_platter->rm_cr4 = cr4; real_mode_platter->rm_gdt_base = gdt.base; real_mode_platter->rm_gdt_lim = gdt.limit; if (getcr3() > 0xffffffffUL) panic("Cannot initialize CPUs; kernel's 64-bit page tables\n" "located above 4G in physical memory (@ 0x%llx).", (unsigned long long)getcr3()); /* * Setup pseudo-descriptors for temporary GDT and IDT for use ONLY * by code in real_mode_start(): * * GDT[0]: NULL selector * GDT[1]: 64-bit CS: Long = 1, Present = 1, bits 12, 11 = 1 * * Clear the IDT as interrupts will be off and a limit of 0 will cause * the CPU to triple fault and reset on an NMI, seemingly as reasonable * a course of action as any other, though it may cause the entire * platform to reset in some cases... */ real_mode_platter->rm_temp_gdt[0] = 0ULL; real_mode_platter->rm_temp_gdt[TEMPGDT_KCODE64] = 0x20980000000000ULL; real_mode_platter->rm_temp_gdt_lim = (ushort_t) (sizeof (real_mode_platter->rm_temp_gdt) - 1); real_mode_platter->rm_temp_gdt_base = rm_platter_pa + offsetof(rm_platter_t, rm_temp_gdt); real_mode_platter->rm_temp_idt_lim = 0; real_mode_platter->rm_temp_idt_base = 0; /* * Since the CPU needs to jump to protected mode using an identity * mapped address, we need to calculate it here. */ real_mode_platter->rm_longmode64_addr = rm_platter_pa + offset; /* return; */ } void i_cpr_start_cpu(void) { struct cpu *cp = CPU; char *str = "i_cpr_start_cpu"; extern void init_cpu_syscall(struct cpu *cp); PMD(PMD_SX, ("%s() called\n", str)) PMD(PMD_SX, ("%s() #0 cp->cpu_base_spl %d\n", str, cp->cpu_base_spl)) mutex_enter(&cpu_lock); if (cp == i_cpr_bootcpu()) { mutex_exit(&cpu_lock); PMD(PMD_SX, ("%s() called on bootcpu nothing to do!\n", str)) return; } mutex_exit(&cpu_lock); /* * We need to Sync PAT with cpu0's PAT. We have to do * this with interrupts disabled. */ pat_sync(); /* * If we use XSAVE, we need to restore XFEATURE_ENABLE_MASK register. */ if (fp_save_mech == FP_XSAVE) { setup_xfem(); } /* * Initialize this CPU's syscall handlers */ init_cpu_syscall(cp); PMD(PMD_SX, ("%s() #1 cp->cpu_base_spl %d\n", str, cp->cpu_base_spl)) /* * Do not need to call cpuid_pass2(), cpuid_pass3(), cpuid_pass4() or * init_cpu_info(), since the work that they do is only needed to * be done once at boot time */ mutex_enter(&cpu_lock); CPUSET_ADD(procset, cp->cpu_id); mutex_exit(&cpu_lock); PMD(PMD_SX, ("%s() #2 cp->cpu_base_spl %d\n", str, cp->cpu_base_spl)) if (tsc_gethrtime_enable) { PMD(PMD_SX, ("%s() calling tsc_sync_slave\n", str)) tsc_sync_slave(); } PMD(PMD_SX, ("%s() cp->cpu_id %d, cp->cpu_intr_actv %d\n", str, cp->cpu_id, cp->cpu_intr_actv)) PMD(PMD_SX, ("%s() #3 cp->cpu_base_spl %d\n", str, cp->cpu_base_spl)) (void) spl0(); /* enable interrupts */ PMD(PMD_SX, ("%s() #4 cp->cpu_base_spl %d\n", str, cp->cpu_base_spl)) /* * Set up the CPU module for this CPU. This can't be done before * this CPU is made CPU_READY, because we may (in heterogeneous systems) * need to go load another CPU module. The act of attempting to load * a module may trigger a cross-call, which will ASSERT unless this * cpu is CPU_READY. */ /* * cmi already been init'd (during boot), so do not need to do it again */ #ifdef PM_REINITMCAONRESUME if (is_x86_feature(x86_featureset, X86FSET_MCA)) cmi_mca_init(); #endif PMD(PMD_SX, ("%s() returning\n", str)) /* return; */ } void i_cpr_alloc_cpus(void) { char *str = "i_cpr_alloc_cpus"; PMD(PMD_SX, ("%s() CPU->cpu_id %d\n", str, CPU->cpu_id)) /* * we allocate this only when we actually need it to save on * kernel memory */ if (wc_other_cpus == NULL) { wc_other_cpus = kmem_zalloc(max_ncpus * sizeof (wc_cpu_t), KM_SLEEP); } } void i_cpr_free_cpus(void) { int index; wc_cpu_t *wc_cpu; if (wc_other_cpus != NULL) { for (index = 0; index < max_ncpus; index++) { wc_cpu = wc_other_cpus + index; if (wc_cpu->wc_saved_stack != NULL) { kmem_free(wc_cpu->wc_saved_stack, wc_cpu->wc_saved_stack_size); } } kmem_free((void *) wc_other_cpus, max_ncpus * sizeof (wc_cpu_t)); wc_other_cpus = NULL; } } /* * wrapper for acpica_ddi_save_resources() */ void i_cpr_save_configuration(dev_info_t *dip) { acpica_ddi_save_resources(dip); } /* * wrapper for acpica_ddi_restore_resources() */ void i_cpr_restore_configuration(dev_info_t *dip) { acpica_ddi_restore_resources(dip); } static int wait_for_set(cpuset_t *set, int who) { int delays; char *str = "wait_for_set"; for (delays = 0; !CPU_IN_SET(*set, who); delays++) { if (delays == 500) { /* * After five seconds, things are probably * looking a bit bleak - explain the hang. */ cmn_err(CE_NOTE, "cpu%d: started, " "but not running in the kernel yet", who); PMD(PMD_SX, ("%s() %d cpu started " "but not running in the kernel yet\n", str, who)) } else if (delays > 2000) { /* * We waited at least 20 seconds, bail .. */ cmn_err(CE_WARN, "cpu%d: timed out", who); PMD(PMD_SX, ("%s() %d cpu timed out\n", str, who)) return (0); } /* * wait at least 10ms, then check again.. */ drv_usecwait(10000); } return (1); } static void i_cpr_save_stack(kthread_t *t, wc_cpu_t *wc_cpu) { size_t stack_size; /* size of stack */ caddr_t start = CPR_GET_STACK_START(t); /* stack start */ caddr_t end = CPR_GET_STACK_END(t); /* stack end */ stack_size = (size_t)end - (size_t)start; if (wc_cpu->wc_saved_stack_size < stack_size) { if (wc_cpu->wc_saved_stack != NULL) { kmem_free(wc_cpu->wc_saved_stack, wc_cpu->wc_saved_stack_size); } wc_cpu->wc_saved_stack = kmem_zalloc(stack_size, KM_SLEEP); wc_cpu->wc_saved_stack_size = stack_size; } bcopy(start, wc_cpu->wc_saved_stack, stack_size); } void i_cpr_restore_stack(kthread_t *t, greg_t *save_stack) { size_t stack_size; /* size of stack */ caddr_t start = CPR_GET_STACK_START(t); /* stack start */ caddr_t end = CPR_GET_STACK_END(t); /* stack end */ stack_size = (size_t)end - (size_t)start; bcopy(save_stack, start, stack_size); } /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved. */ #include #include #include #include /* * List of the processor ACPI object types that are being used. */ typedef enum cpu_acpi_obj { PDC_OBJ = 0, PCT_OBJ, PSS_OBJ, PSD_OBJ, PPC_OBJ, PTC_OBJ, TSS_OBJ, TSD_OBJ, TPC_OBJ, CST_OBJ, CSD_OBJ, } cpu_acpi_obj_t; /* * Container to store object name. * Other attributes can be added in the future as necessary. */ typedef struct cpu_acpi_obj_attr { char *name; } cpu_acpi_obj_attr_t; /* * List of object attributes. * NOTE: Please keep the ordering of the list as same as cpu_acpi_obj_t. */ static cpu_acpi_obj_attr_t cpu_acpi_obj_attrs[] = { {"_PDC"}, {"_PCT"}, {"_PSS"}, {"_PSD"}, {"_PPC"}, {"_PTC"}, {"_TSS"}, {"_TSD"}, {"_TPC"}, {"_CST"}, {"_CSD"} }; /* * Cache the ACPI CPU control data objects. */ static int cpu_acpi_cache_ctrl_regs(cpu_acpi_handle_t handle, cpu_acpi_obj_t objtype, cpu_acpi_ctrl_regs_t *regs) { ACPI_STATUS astatus; ACPI_BUFFER abuf; ACPI_OBJECT *obj; AML_RESOURCE_GENERIC_REGISTER *greg; int ret = -1; int i; /* * Fetch the control registers (if present) for the CPU node. * Since they are optional, non-existence is not a failure * (we just consider it a fixed hardware case). */ abuf.Length = ACPI_ALLOCATE_BUFFER; abuf.Pointer = NULL; astatus = AcpiEvaluateObjectTyped(handle->cs_handle, cpu_acpi_obj_attrs[objtype].name, NULL, &abuf, ACPI_TYPE_PACKAGE); if (ACPI_FAILURE(astatus)) { if (astatus == AE_NOT_FOUND) { DTRACE_PROBE3(cpu_acpi__eval__err, int, handle->cs_id, int, objtype, int, astatus); regs[0].cr_addrspace_id = ACPI_ADR_SPACE_FIXED_HARDWARE; regs[1].cr_addrspace_id = ACPI_ADR_SPACE_FIXED_HARDWARE; return (1); } cmn_err(CE_NOTE, "!cpu_acpi: error %d evaluating %s package " "for CPU %d.", astatus, cpu_acpi_obj_attrs[objtype].name, handle->cs_id); goto out; } obj = abuf.Pointer; if (obj->Package.Count != 2) { cmn_err(CE_NOTE, "!cpu_acpi: %s package bad count %d for " "CPU %d.", cpu_acpi_obj_attrs[objtype].name, obj->Package.Count, handle->cs_id); goto out; } /* * Does the package look coherent? */ for (i = 0; i < obj->Package.Count; i++) { if (obj->Package.Elements[i].Type != ACPI_TYPE_BUFFER) { cmn_err(CE_NOTE, "!cpu_acpi: Unexpected data in " "%s package for CPU %d.", cpu_acpi_obj_attrs[objtype].name, handle->cs_id); goto out; } greg = (AML_RESOURCE_GENERIC_REGISTER *) obj->Package.Elements[i].Buffer.Pointer; if (greg->DescriptorType != ACPI_RESOURCE_NAME_GENERIC_REGISTER) { cmn_err(CE_NOTE, "!cpu_acpi: %s package has format " "error for CPU %d.", cpu_acpi_obj_attrs[objtype].name, handle->cs_id); goto out; } if (greg->ResourceLength != ACPI_AML_SIZE_LARGE(AML_RESOURCE_GENERIC_REGISTER)) { cmn_err(CE_NOTE, "!cpu_acpi: %s package not right " "size for CPU %d.", cpu_acpi_obj_attrs[objtype].name, handle->cs_id); goto out; } if (greg->AddressSpaceId != ACPI_ADR_SPACE_FIXED_HARDWARE && greg->AddressSpaceId != ACPI_ADR_SPACE_SYSTEM_IO) { cmn_err(CE_NOTE, "!cpu_apci: %s contains unsupported " "address space type %x for CPU %d.", cpu_acpi_obj_attrs[objtype].name, greg->AddressSpaceId, handle->cs_id); goto out; } } /* * Looks good! */ for (i = 0; i < obj->Package.Count; i++) { greg = (AML_RESOURCE_GENERIC_REGISTER *) obj->Package.Elements[i].Buffer.Pointer; regs[i].cr_addrspace_id = greg->AddressSpaceId; regs[i].cr_width = greg->BitWidth; regs[i].cr_offset = greg->BitOffset; regs[i].cr_asize = greg->AccessSize; regs[i].cr_address = greg->Address; } ret = 0; out: if (abuf.Pointer != NULL) AcpiOsFree(abuf.Pointer); return (ret); } /* * Cache the ACPI _PCT data. The _PCT data defines the interface to use * when making power level transitions (i.e., system IO ports, fixed * hardware port, etc). */ static int cpu_acpi_cache_pct(cpu_acpi_handle_t handle) { cpu_acpi_pct_t *pct; int ret; CPU_ACPI_OBJ_IS_NOT_CACHED(handle, CPU_ACPI_PCT_CACHED); pct = &CPU_ACPI_PCT(handle)[0]; if ((ret = cpu_acpi_cache_ctrl_regs(handle, PCT_OBJ, pct)) == 0) CPU_ACPI_OBJ_IS_CACHED(handle, CPU_ACPI_PCT_CACHED); return (ret); } /* * Cache the ACPI _PTC data. The _PTC data defines the interface to use * when making T-state transitions (i.e., system IO ports, fixed * hardware port, etc). */ static int cpu_acpi_cache_ptc(cpu_acpi_handle_t handle) { cpu_acpi_ptc_t *ptc; int ret; CPU_ACPI_OBJ_IS_NOT_CACHED(handle, CPU_ACPI_PTC_CACHED); ptc = &CPU_ACPI_PTC(handle)[0]; if ((ret = cpu_acpi_cache_ctrl_regs(handle, PTC_OBJ, ptc)) == 0) CPU_ACPI_OBJ_IS_CACHED(handle, CPU_ACPI_PTC_CACHED); return (ret); } /* * Cache the ACPI CPU state dependency data objects. */ static int cpu_acpi_cache_state_dependencies(cpu_acpi_handle_t handle, cpu_acpi_obj_t objtype, cpu_acpi_state_dependency_t *sd) { ACPI_STATUS astatus; ACPI_BUFFER abuf; ACPI_OBJECT *pkg, *elements; int number; int ret = -1; if (objtype == CSD_OBJ) { number = 6; } else { number = 5; } /* * Fetch the dependencies (if present) for the CPU node. * Since they are optional, non-existence is not a failure * (it's up to the caller to determine how to handle non-existence). */ abuf.Length = ACPI_ALLOCATE_BUFFER; abuf.Pointer = NULL; astatus = AcpiEvaluateObjectTyped(handle->cs_handle, cpu_acpi_obj_attrs[objtype].name, NULL, &abuf, ACPI_TYPE_PACKAGE); if (ACPI_FAILURE(astatus)) { if (astatus == AE_NOT_FOUND) { DTRACE_PROBE3(cpu_acpi__eval__err, int, handle->cs_id, int, objtype, int, astatus); return (1); } cmn_err(CE_NOTE, "!cpu_acpi: error %d evaluating %s package " "for CPU %d.", astatus, cpu_acpi_obj_attrs[objtype].name, handle->cs_id); goto out; } pkg = abuf.Pointer; if (((objtype != CSD_OBJ) && (pkg->Package.Count != 1)) || ((objtype == CSD_OBJ) && (pkg->Package.Count != 1) && (pkg->Package.Count != 2))) { cmn_err(CE_NOTE, "!cpu_acpi: %s unsupported package count %d " "for CPU %d.", cpu_acpi_obj_attrs[objtype].name, pkg->Package.Count, handle->cs_id); goto out; } /* * For C-state domain, we assume C2 and C3 have the same * domain information */ if (pkg->Package.Elements[0].Type != ACPI_TYPE_PACKAGE || pkg->Package.Elements[0].Package.Count != number) { cmn_err(CE_NOTE, "!cpu_acpi: Unexpected data in %s package " "for CPU %d.", cpu_acpi_obj_attrs[objtype].name, handle->cs_id); goto out; } elements = pkg->Package.Elements[0].Package.Elements; if (elements[0].Integer.Value != number || elements[1].Integer.Value != 0) { cmn_err(CE_NOTE, "!cpu_acpi: Unexpected %s revision for " "CPU %d.", cpu_acpi_obj_attrs[objtype].name, handle->cs_id); goto out; } sd->sd_entries = elements[0].Integer.Value; sd->sd_revision = elements[1].Integer.Value; sd->sd_domain = elements[2].Integer.Value; sd->sd_type = elements[3].Integer.Value; sd->sd_num = elements[4].Integer.Value; if (objtype == CSD_OBJ) { sd->sd_index = elements[5].Integer.Value; } ret = 0; out: if (abuf.Pointer != NULL) AcpiOsFree(abuf.Pointer); return (ret); } /* * Cache the ACPI _PSD data. The _PSD data defines P-state CPU dependencies * (think CPU domains). */ static int cpu_acpi_cache_psd(cpu_acpi_handle_t handle) { cpu_acpi_psd_t *psd; int ret; CPU_ACPI_OBJ_IS_NOT_CACHED(handle, CPU_ACPI_PSD_CACHED); psd = &CPU_ACPI_PSD(handle); ret = cpu_acpi_cache_state_dependencies(handle, PSD_OBJ, psd); if (ret == 0) CPU_ACPI_OBJ_IS_CACHED(handle, CPU_ACPI_PSD_CACHED); return (ret); } /* * Cache the ACPI _TSD data. The _TSD data defines T-state CPU dependencies * (think CPU domains). */ static int cpu_acpi_cache_tsd(cpu_acpi_handle_t handle) { cpu_acpi_tsd_t *tsd; int ret; CPU_ACPI_OBJ_IS_NOT_CACHED(handle, CPU_ACPI_TSD_CACHED); tsd = &CPU_ACPI_TSD(handle); ret = cpu_acpi_cache_state_dependencies(handle, TSD_OBJ, tsd); if (ret == 0) CPU_ACPI_OBJ_IS_CACHED(handle, CPU_ACPI_TSD_CACHED); return (ret); } /* * Cache the ACPI _CSD data. The _CSD data defines C-state CPU dependencies * (think CPU domains). */ static int cpu_acpi_cache_csd(cpu_acpi_handle_t handle) { cpu_acpi_csd_t *csd; int ret; CPU_ACPI_OBJ_IS_NOT_CACHED(handle, CPU_ACPI_CSD_CACHED); csd = &CPU_ACPI_CSD(handle); ret = cpu_acpi_cache_state_dependencies(handle, CSD_OBJ, csd); if (ret == 0) CPU_ACPI_OBJ_IS_CACHED(handle, CPU_ACPI_CSD_CACHED); return (ret); } static void cpu_acpi_cache_pstate(cpu_acpi_handle_t handle, ACPI_OBJECT *obj, int cnt) { cpu_acpi_pstate_t *pstate; ACPI_OBJECT *q, *l; int i, j; CPU_ACPI_PSTATES_COUNT(handle) = cnt; CPU_ACPI_PSTATES(handle) = kmem_zalloc(CPU_ACPI_PSTATES_SIZE(cnt), KM_SLEEP); pstate = (cpu_acpi_pstate_t *)CPU_ACPI_PSTATES(handle); for (i = 0, l = NULL; i < obj->Package.Count && cnt > 0; i++, l = q) { uint32_t *up; q = obj->Package.Elements[i].Package.Elements; /* * Skip duplicate entries. */ if (l != NULL && l[0].Integer.Value == q[0].Integer.Value) continue; up = (uint32_t *)pstate; for (j = 0; j < CPU_ACPI_PSS_CNT; j++) up[j] = q[j].Integer.Value; pstate++; cnt--; } } static void cpu_acpi_cache_tstate(cpu_acpi_handle_t handle, ACPI_OBJECT *obj, int cnt) { cpu_acpi_tstate_t *tstate; ACPI_OBJECT *q, *l; int i, j; CPU_ACPI_TSTATES_COUNT(handle) = cnt; CPU_ACPI_TSTATES(handle) = kmem_zalloc(CPU_ACPI_TSTATES_SIZE(cnt), KM_SLEEP); tstate = (cpu_acpi_tstate_t *)CPU_ACPI_TSTATES(handle); for (i = 0, l = NULL; i < obj->Package.Count && cnt > 0; i++, l = q) { uint32_t *up; q = obj->Package.Elements[i].Package.Elements; /* * Skip duplicate entries. */ if (l != NULL && l[0].Integer.Value == q[0].Integer.Value) continue; up = (uint32_t *)tstate; for (j = 0; j < CPU_ACPI_TSS_CNT; j++) up[j] = q[j].Integer.Value; tstate++; cnt--; } } /* * Cache the _PSS or _TSS data. */ static int cpu_acpi_cache_supported_states(cpu_acpi_handle_t handle, cpu_acpi_obj_t objtype, int fcnt) { ACPI_STATUS astatus; ACPI_BUFFER abuf; ACPI_OBJECT *obj, *q, *l; boolean_t eot = B_FALSE; int ret = -1; int cnt; int i, j; /* * Fetch the state data (if present) for the CPU node. */ abuf.Length = ACPI_ALLOCATE_BUFFER; abuf.Pointer = NULL; astatus = AcpiEvaluateObjectTyped(handle->cs_handle, cpu_acpi_obj_attrs[objtype].name, NULL, &abuf, ACPI_TYPE_PACKAGE); if (ACPI_FAILURE(astatus)) { if (astatus == AE_NOT_FOUND) { DTRACE_PROBE3(cpu_acpi__eval__err, int, handle->cs_id, int, objtype, int, astatus); return (1); } cmn_err(CE_NOTE, "!cpu_acpi: error %d evaluating %s package " "for CPU %d.", astatus, cpu_acpi_obj_attrs[objtype].name, handle->cs_id); goto out; } obj = abuf.Pointer; if (obj->Package.Count < 2) { cmn_err(CE_NOTE, "!cpu_acpi: %s package bad count %d for " "CPU %d.", cpu_acpi_obj_attrs[objtype].name, obj->Package.Count, handle->cs_id); goto out; } /* * Does the package look coherent? */ cnt = 0; for (i = 0, l = NULL; i < obj->Package.Count; i++, l = q) { if (obj->Package.Elements[i].Type != ACPI_TYPE_PACKAGE || obj->Package.Elements[i].Package.Count != fcnt) { cmn_err(CE_NOTE, "!cpu_acpi: Unexpected data in " "%s package for CPU %d.", cpu_acpi_obj_attrs[objtype].name, handle->cs_id); goto out; } q = obj->Package.Elements[i].Package.Elements; for (j = 0; j < fcnt; j++) { if (q[j].Type != ACPI_TYPE_INTEGER) { cmn_err(CE_NOTE, "!cpu_acpi: %s element " "invalid (type) for CPU %d.", cpu_acpi_obj_attrs[objtype].name, handle->cs_id); goto out; } } /* * Ignore duplicate entries. */ if (l != NULL && l[0].Integer.Value == q[0].Integer.Value) continue; /* * Some supported state tables are larger than required * and unused elements are filled with patterns * of 0xff. Simply check here for frequency = 0xffff * and stop counting if found. */ if (q[0].Integer.Value == 0xffff) { eot = B_TRUE; continue; } /* * We should never find a valid entry after we've hit * an the end-of-table entry. */ if (eot) { cmn_err(CE_NOTE, "!cpu_acpi: Unexpected data in %s " "package after eot for CPU %d.", cpu_acpi_obj_attrs[objtype].name, handle->cs_id); goto out; } /* * states must be defined in order from highest to lowest. */ if (l != NULL && l[0].Integer.Value < q[0].Integer.Value) { cmn_err(CE_NOTE, "!cpu_acpi: %s package state " "definitions out of order for CPU %d.", cpu_acpi_obj_attrs[objtype].name, handle->cs_id); goto out; } /* * This entry passes. */ cnt++; } if (cnt == 0) goto out; /* * Yes, fill in the structure. */ ASSERT(objtype == PSS_OBJ || objtype == TSS_OBJ); (objtype == PSS_OBJ) ? cpu_acpi_cache_pstate(handle, obj, cnt) : cpu_acpi_cache_tstate(handle, obj, cnt); ret = 0; out: if (abuf.Pointer != NULL) AcpiOsFree(abuf.Pointer); return (ret); } /* * Cache the _PSS data. The _PSS data defines the different power levels * supported by the CPU and the attributes associated with each power level * (i.e., frequency, voltage, etc.). The power levels are number from * highest to lowest. That is, the highest power level is _PSS entry 0 * and the lowest power level is the last _PSS entry. */ static int cpu_acpi_cache_pstates(cpu_acpi_handle_t handle) { int ret; CPU_ACPI_OBJ_IS_NOT_CACHED(handle, CPU_ACPI_PSS_CACHED); ret = cpu_acpi_cache_supported_states(handle, PSS_OBJ, CPU_ACPI_PSS_CNT); if (ret == 0) CPU_ACPI_OBJ_IS_CACHED(handle, CPU_ACPI_PSS_CACHED); return (ret); } /* * Cache the _TSS data. The _TSS data defines the different freq throttle * levels supported by the CPU and the attributes associated with each * throttle level (i.e., frequency throttle percentage, voltage, etc.). * The throttle levels are number from highest to lowest. */ static int cpu_acpi_cache_tstates(cpu_acpi_handle_t handle) { int ret; CPU_ACPI_OBJ_IS_NOT_CACHED(handle, CPU_ACPI_TSS_CACHED); ret = cpu_acpi_cache_supported_states(handle, TSS_OBJ, CPU_ACPI_TSS_CNT); if (ret == 0) CPU_ACPI_OBJ_IS_CACHED(handle, CPU_ACPI_TSS_CACHED); return (ret); } /* * Cache the ACPI CPU present capabilities data objects. */ static int cpu_acpi_cache_present_capabilities(cpu_acpi_handle_t handle, cpu_acpi_obj_t objtype, cpu_acpi_present_capabilities_t *pc) { ACPI_STATUS astatus; ACPI_BUFFER abuf; ACPI_OBJECT *obj; int ret = -1; /* * Fetch the present capabilites object (if present) for the CPU node. */ abuf.Length = ACPI_ALLOCATE_BUFFER; abuf.Pointer = NULL; astatus = AcpiEvaluateObject(handle->cs_handle, cpu_acpi_obj_attrs[objtype].name, NULL, &abuf); if (ACPI_FAILURE(astatus) && astatus != AE_NOT_FOUND) { cmn_err(CE_NOTE, "!cpu_acpi: error %d evaluating %s " "package for CPU %d.", astatus, cpu_acpi_obj_attrs[objtype].name, handle->cs_id); goto out; } if (astatus == AE_NOT_FOUND || abuf.Length == 0) { *pc = 0; return (1); } obj = (ACPI_OBJECT *)abuf.Pointer; *pc = obj->Integer.Value; ret = 0; out: if (abuf.Pointer != NULL) AcpiOsFree(abuf.Pointer); return (ret); } /* * Cache the _PPC data. The _PPC simply contains an integer value which * represents the highest power level that a CPU should transition to. * That is, it's an index into the array of _PSS entries and will be * greater than or equal to zero. */ void cpu_acpi_cache_ppc(cpu_acpi_handle_t handle) { cpu_acpi_ppc_t *ppc; int ret; CPU_ACPI_OBJ_IS_NOT_CACHED(handle, CPU_ACPI_PPC_CACHED); ppc = &CPU_ACPI_PPC(handle); ret = cpu_acpi_cache_present_capabilities(handle, PPC_OBJ, ppc); if (ret == 0) CPU_ACPI_OBJ_IS_CACHED(handle, CPU_ACPI_PPC_CACHED); } /* * Cache the _TPC data. The _TPC simply contains an integer value which * represents the throttle level that a CPU should transition to. * That is, it's an index into the array of _TSS entries and will be * greater than or equal to zero. */ void cpu_acpi_cache_tpc(cpu_acpi_handle_t handle) { cpu_acpi_tpc_t *tpc; int ret; CPU_ACPI_OBJ_IS_NOT_CACHED(handle, CPU_ACPI_TPC_CACHED); tpc = &CPU_ACPI_TPC(handle); ret = cpu_acpi_cache_present_capabilities(handle, TPC_OBJ, tpc); if (ret == 0) CPU_ACPI_OBJ_IS_CACHED(handle, CPU_ACPI_TPC_CACHED); } int cpu_acpi_verify_cstate(cpu_acpi_cstate_t *cstate) { uint32_t addrspaceid = cstate->cs_addrspace_id; if ((addrspaceid != ACPI_ADR_SPACE_FIXED_HARDWARE) && (addrspaceid != ACPI_ADR_SPACE_SYSTEM_IO)) { cmn_err(CE_NOTE, "!cpu_acpi: _CST unsupported address space id" ":C%d, type: %d\n", cstate->cs_type, addrspaceid); return (1); } return (0); } int cpu_acpi_cache_cst(cpu_acpi_handle_t handle) { ACPI_STATUS astatus; ACPI_BUFFER abuf; ACPI_OBJECT *obj; ACPI_INTEGER cnt, old_cnt; cpu_acpi_cstate_t *cstate, *p; size_t alloc_size; int i, count; int ret = 1; CPU_ACPI_OBJ_IS_NOT_CACHED(handle, CPU_ACPI_CST_CACHED); abuf.Length = ACPI_ALLOCATE_BUFFER; abuf.Pointer = NULL; /* * Fetch the C-state data (if present) for the CPU node. */ astatus = AcpiEvaluateObjectTyped(handle->cs_handle, "_CST", NULL, &abuf, ACPI_TYPE_PACKAGE); if (ACPI_FAILURE(astatus)) { if (astatus == AE_NOT_FOUND) { DTRACE_PROBE3(cpu_acpi__eval__err, int, handle->cs_id, int, CST_OBJ, int, astatus); return (1); } cmn_err(CE_NOTE, "!cpu_acpi: error %d evaluating _CST package " "for CPU %d.", astatus, handle->cs_id); goto out; } obj = (ACPI_OBJECT *)abuf.Pointer; if (obj->Package.Count < 2) { cmn_err(CE_NOTE, "!cpu_acpi: _CST unsupported package " "count %d for CPU %d.", obj->Package.Count, handle->cs_id); goto out; } /* * Does the package look coherent? */ cnt = obj->Package.Elements[0].Integer.Value; if (cnt < 1 || cnt != obj->Package.Count - 1) { cmn_err(CE_NOTE, "!cpu_acpi: _CST invalid element " "count %d != Package count %d for CPU %d", (int)cnt, (int)obj->Package.Count - 1, handle->cs_id); goto out; } /* * Reuse the old buffer if the number of C states is the same. */ if (CPU_ACPI_CSTATES(handle) && (old_cnt = CPU_ACPI_CSTATES_COUNT(handle)) != cnt) { kmem_free(CPU_ACPI_CSTATES(handle), CPU_ACPI_CSTATES_SIZE(old_cnt)); CPU_ACPI_CSTATES(handle) = NULL; } CPU_ACPI_CSTATES_COUNT(handle) = (uint32_t)cnt; alloc_size = CPU_ACPI_CSTATES_SIZE(cnt); if (CPU_ACPI_CSTATES(handle) == NULL) CPU_ACPI_CSTATES(handle) = kmem_zalloc(alloc_size, KM_SLEEP); cstate = (cpu_acpi_cstate_t *)CPU_ACPI_CSTATES(handle); p = cstate; for (i = 1, count = 1; i <= cnt; i++) { ACPI_OBJECT *pkg; AML_RESOURCE_GENERIC_REGISTER *reg; ACPI_OBJECT *element; pkg = &(obj->Package.Elements[i]); reg = (AML_RESOURCE_GENERIC_REGISTER *) pkg->Package.Elements[0].Buffer.Pointer; cstate->cs_addrspace_id = reg->AddressSpaceId; cstate->cs_address = reg->Address; element = &(pkg->Package.Elements[1]); cstate->cs_type = element->Integer.Value; element = &(pkg->Package.Elements[2]); cstate->cs_latency = element->Integer.Value; element = &(pkg->Package.Elements[3]); cstate->cs_power = element->Integer.Value; if (cpu_acpi_verify_cstate(cstate)) { /* * ignore this entry if it's not valid */ continue; } if (cstate == p) { cstate++; } else if (p->cs_type == cstate->cs_type) { /* * if there are duplicate entries, we keep the * last one. This fixes: * 1) some buggy BIOS have total duplicate entries. * 2) ACPI Spec allows the same cstate entry with * different power and latency, we use the one * with more power saving. */ (void) memcpy(p, cstate, sizeof (cpu_acpi_cstate_t)); } else { /* * we got a valid entry, cache it to the * cstate structure */ p = cstate++; count++; } } if (count < 2) { cmn_err(CE_NOTE, "!cpu_acpi: _CST invalid count %d < 2 for " "CPU %d", count, handle->cs_id); kmem_free(CPU_ACPI_CSTATES(handle), alloc_size); CPU_ACPI_CSTATES(handle) = NULL; CPU_ACPI_CSTATES_COUNT(handle) = (uint32_t)0; goto out; } cstate = (cpu_acpi_cstate_t *)CPU_ACPI_CSTATES(handle); if (cstate[0].cs_type != CPU_ACPI_C1) { cmn_err(CE_NOTE, "!cpu_acpi: _CST first element type not " "C1: %d for CPU %d", (int)cstate->cs_type, handle->cs_id); kmem_free(CPU_ACPI_CSTATES(handle), alloc_size); CPU_ACPI_CSTATES(handle) = NULL; CPU_ACPI_CSTATES_COUNT(handle) = (uint32_t)0; goto out; } if (count != cnt) { void *orig = CPU_ACPI_CSTATES(handle); CPU_ACPI_CSTATES_COUNT(handle) = (uint32_t)count; CPU_ACPI_CSTATES(handle) = kmem_zalloc( CPU_ACPI_CSTATES_SIZE(count), KM_SLEEP); (void) memcpy(CPU_ACPI_CSTATES(handle), orig, CPU_ACPI_CSTATES_SIZE(count)); kmem_free(orig, alloc_size); } CPU_ACPI_OBJ_IS_CACHED(handle, CPU_ACPI_CST_CACHED); ret = 0; out: if (abuf.Pointer != NULL) AcpiOsFree(abuf.Pointer); return (ret); } /* * Cache the _PCT, _PSS, _PSD and _PPC data. */ int cpu_acpi_cache_pstate_data(cpu_acpi_handle_t handle) { if (cpu_acpi_cache_pct(handle) < 0) { DTRACE_PROBE2(cpu_acpi__cache__err, int, handle->cs_id, int, PCT_OBJ); return (-1); } if (cpu_acpi_cache_pstates(handle) != 0) { DTRACE_PROBE2(cpu_acpi__cache__err, int, handle->cs_id, int, PSS_OBJ); return (-1); } if (cpu_acpi_cache_psd(handle) < 0) { DTRACE_PROBE2(cpu_acpi__cache__err, int, handle->cs_id, int, PSD_OBJ); return (-1); } cpu_acpi_cache_ppc(handle); return (0); } void cpu_acpi_free_pstate_data(cpu_acpi_handle_t handle) { if (handle != NULL) { if (CPU_ACPI_PSTATES(handle)) { kmem_free(CPU_ACPI_PSTATES(handle), CPU_ACPI_PSTATES_SIZE( CPU_ACPI_PSTATES_COUNT(handle))); CPU_ACPI_PSTATES(handle) = NULL; } } } /* * Cache the _PTC, _TSS, _TSD and _TPC data. */ int cpu_acpi_cache_tstate_data(cpu_acpi_handle_t handle) { int ret; if (cpu_acpi_cache_ptc(handle) < 0) { DTRACE_PROBE2(cpu_acpi__cache__err, int, handle->cs_id, int, PTC_OBJ); return (-1); } if ((ret = cpu_acpi_cache_tstates(handle)) != 0) { DTRACE_PROBE2(cpu_acpi__cache__err, int, handle->cs_id, int, TSS_OBJ); return (ret); } if (cpu_acpi_cache_tsd(handle) < 0) { DTRACE_PROBE2(cpu_acpi__cache__err, int, handle->cs_id, int, TSD_OBJ); return (-1); } cpu_acpi_cache_tpc(handle); return (0); } void cpu_acpi_free_tstate_data(cpu_acpi_handle_t handle) { if (handle != NULL) { if (CPU_ACPI_TSTATES(handle)) { kmem_free(CPU_ACPI_TSTATES(handle), CPU_ACPI_TSTATES_SIZE( CPU_ACPI_TSTATES_COUNT(handle))); CPU_ACPI_TSTATES(handle) = NULL; } } } /* * Cache the _CST data. */ int cpu_acpi_cache_cstate_data(cpu_acpi_handle_t handle) { int ret; if ((ret = cpu_acpi_cache_cst(handle)) != 0) { DTRACE_PROBE2(cpu_acpi__cache__err, int, handle->cs_id, int, CST_OBJ); return (ret); } if (cpu_acpi_cache_csd(handle) < 0) { DTRACE_PROBE2(cpu_acpi__cache__err, int, handle->cs_id, int, CSD_OBJ); return (-1); } return (0); } void cpu_acpi_free_cstate_data(cpu_acpi_handle_t handle) { if (handle != NULL) { if (CPU_ACPI_CSTATES(handle)) { kmem_free(CPU_ACPI_CSTATES(handle), CPU_ACPI_CSTATES_SIZE( CPU_ACPI_CSTATES_COUNT(handle))); CPU_ACPI_CSTATES(handle) = NULL; } } } /* * Register a handler for processor change notifications. */ void cpu_acpi_install_notify_handler(cpu_acpi_handle_t handle, ACPI_NOTIFY_HANDLER handler, void *ctx) { if (ACPI_FAILURE(AcpiInstallNotifyHandler(handle->cs_handle, ACPI_DEVICE_NOTIFY, handler, ctx))) cmn_err(CE_NOTE, "!cpu_acpi: Unable to register " "notify handler for CPU %d.", handle->cs_id); } /* * Remove a handler for processor change notifications. */ void cpu_acpi_remove_notify_handler(cpu_acpi_handle_t handle, ACPI_NOTIFY_HANDLER handler) { if (ACPI_FAILURE(AcpiRemoveNotifyHandler(handle->cs_handle, ACPI_DEVICE_NOTIFY, handler))) cmn_err(CE_NOTE, "!cpu_acpi: Unable to remove " "notify handler for CPU %d.", handle->cs_id); } /* * Write _PDC. */ int cpu_acpi_write_pdc(cpu_acpi_handle_t handle, uint32_t revision, uint32_t count, uint32_t *capabilities) { ACPI_STATUS astatus; ACPI_OBJECT obj; ACPI_OBJECT_LIST list = { 1, &obj}; uint32_t *buffer; uint32_t *bufptr; uint32_t bufsize; int i; int ret = 0; bufsize = (count + 2) * sizeof (uint32_t); buffer = kmem_zalloc(bufsize, KM_SLEEP); buffer[0] = revision; buffer[1] = count; bufptr = &buffer[2]; for (i = 0; i < count; i++) *bufptr++ = *capabilities++; obj.Type = ACPI_TYPE_BUFFER; obj.Buffer.Length = bufsize; obj.Buffer.Pointer = (void *)buffer; /* * Fetch the ??? (if present) for the CPU node. */ astatus = AcpiEvaluateObject(handle->cs_handle, "_PDC", &list, NULL); if (ACPI_FAILURE(astatus)) { if (astatus == AE_NOT_FOUND) { DTRACE_PROBE3(cpu_acpi__eval__err, int, handle->cs_id, int, PDC_OBJ, int, astatus); ret = 1; } else { cmn_err(CE_NOTE, "!cpu_acpi: error %d evaluating _PDC " "package for CPU %d.", astatus, handle->cs_id); ret = -1; } } kmem_free(buffer, bufsize); return (ret); } /* * Write to system IO port. */ int cpu_acpi_write_port(ACPI_IO_ADDRESS address, uint32_t value, uint32_t width) { if (ACPI_FAILURE(AcpiOsWritePort(address, value, width))) { cmn_err(CE_NOTE, "!cpu_acpi: error writing system IO port " "%lx.", (long)address); return (-1); } return (0); } /* * Read from a system IO port. */ int cpu_acpi_read_port(ACPI_IO_ADDRESS address, uint32_t *value, uint32_t width) { if (ACPI_FAILURE(AcpiOsReadPort(address, value, width))) { cmn_err(CE_NOTE, "!cpu_acpi: error reading system IO port " "%lx.", (long)address); return (-1); } return (0); } /* * Return supported frequencies. */ uint_t cpu_acpi_get_speeds(cpu_acpi_handle_t handle, int **speeds) { cpu_acpi_pstate_t *pstate; int *hspeeds; uint_t nspeeds; int i; nspeeds = CPU_ACPI_PSTATES_COUNT(handle); pstate = (cpu_acpi_pstate_t *)CPU_ACPI_PSTATES(handle); hspeeds = kmem_zalloc(nspeeds * sizeof (int), KM_SLEEP); for (i = 0; i < nspeeds; i++) { hspeeds[i] = CPU_ACPI_FREQ(pstate); pstate++; } *speeds = hspeeds; return (nspeeds); } /* * Free resources allocated by cpu_acpi_get_speeds(). */ void cpu_acpi_free_speeds(int *speeds, uint_t nspeeds) { kmem_free(speeds, nspeeds * sizeof (int)); } uint_t cpu_acpi_get_max_cstates(cpu_acpi_handle_t handle) { if (CPU_ACPI_CSTATES(handle)) return (CPU_ACPI_CSTATES_COUNT(handle)); else return (1); } void cpu_acpi_set_register(uint32_t bitreg, uint32_t value) { (void) AcpiWriteBitRegister(bitreg, value); } void cpu_acpi_get_register(uint32_t bitreg, uint32_t *value) { (void) AcpiReadBitRegister(bitreg, value); } /* * Map the dip to an ACPI handle for the device. */ cpu_acpi_handle_t cpu_acpi_init(cpu_t *cp) { cpu_acpi_handle_t handle; handle = kmem_zalloc(sizeof (cpu_acpi_state_t), KM_SLEEP); if (ACPI_FAILURE(acpica_get_handle_cpu(cp->cpu_id, &handle->cs_handle))) { kmem_free(handle, sizeof (cpu_acpi_state_t)); return (NULL); } handle->cs_id = cp->cpu_id; return (handle); } /* * Free any resources. */ void cpu_acpi_fini(cpu_acpi_handle_t handle) { if (handle) kmem_free(handle, sizeof (cpu_acpi_state_t)); } /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2009 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ /* * Copyright (c) 2009-2010, Intel Corporation. * All rights reserved. */ /* * Copyright 2019 Joyent, Inc. * Copyright 2025 Oxide Computer Company */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define CSTATE_USING_HPET 1 #define CSTATE_USING_LAT 2 #define CPU_IDLE_STOP_TIMEOUT 1000 extern void cpu_idle_adaptive(void); extern uint32_t cpupm_next_cstate(cma_c_state_t *cs_data, cpu_acpi_cstate_t *cstates, uint32_t cs_count, hrtime_t start); static int cpu_idle_init(cpu_t *); static void cpu_idle_fini(cpu_t *); static void cpu_idle_stop(cpu_t *); static boolean_t cpu_deep_idle_callb(void *arg, int code); static boolean_t cpu_idle_cpr_callb(void *arg, int code); static void acpi_cpu_cstate(cpu_acpi_cstate_t *cstate); static boolean_t cstate_use_timer(hrtime_t *lapic_expire, int timer); /* * the flag of always-running local APIC timer. * the flag of HPET Timer use in deep cstate. */ static boolean_t cpu_cstate_arat = B_FALSE; static boolean_t cpu_cstate_hpet = B_FALSE; /* * Interfaces for modules implementing Intel's deep c-state. */ cpupm_state_ops_t cpu_idle_ops = { "Generic ACPI C-state Support", cpu_idle_init, cpu_idle_fini, NULL, cpu_idle_stop }; static kmutex_t cpu_idle_callb_mutex; static callb_id_t cpu_deep_idle_callb_id; static callb_id_t cpu_idle_cpr_callb_id; static uint_t cpu_idle_cfg_state; static kmutex_t cpu_idle_mutex; cpu_idle_kstat_t cpu_idle_kstat = { { "address_space_id", KSTAT_DATA_STRING }, { "latency", KSTAT_DATA_UINT32 }, { "power", KSTAT_DATA_UINT32 }, }; /* * kstat update function of the c-state info */ static int cpu_idle_kstat_update(kstat_t *ksp, int flag) { cpu_acpi_cstate_t *cstate = ksp->ks_private; if (flag == KSTAT_WRITE) { return (EACCES); } if (cstate->cs_addrspace_id == ACPI_ADR_SPACE_FIXED_HARDWARE) { kstat_named_setstr(&cpu_idle_kstat.addr_space_id, "FFixedHW"); } else if (cstate->cs_addrspace_id == ACPI_ADR_SPACE_SYSTEM_IO) { kstat_named_setstr(&cpu_idle_kstat.addr_space_id, "SystemIO"); } else { kstat_named_setstr(&cpu_idle_kstat.addr_space_id, "Unsupported"); } cpu_idle_kstat.cs_latency.value.ui32 = cstate->cs_latency; cpu_idle_kstat.cs_power.value.ui32 = cstate->cs_power; return (0); } /* * Used during configuration callbacks to manage implementation specific * details of the hardware timer used during Deep C-state. */ boolean_t cstate_timer_callback(int code) { if (cpu_cstate_arat) { return (B_TRUE); } else if (cpu_cstate_hpet) { return (hpet.callback(code)); } return (B_FALSE); } /* * Some Local APIC Timers do not work during Deep C-states. * The Deep C-state idle function uses this function to ensure it is using a * hardware timer that works during Deep C-states. This function also * switches the timer back to the LACPI Timer after Deep C-state. */ static boolean_t cstate_use_timer(hrtime_t *lapic_expire, int timer) { if (cpu_cstate_arat) return (B_TRUE); /* * We have to return B_FALSE if no arat or hpet support */ if (!cpu_cstate_hpet) return (B_FALSE); switch (timer) { case CSTATE_USING_HPET: return (hpet.use_hpet_timer(lapic_expire)); case CSTATE_USING_LAT: hpet.use_lapic_timer(*lapic_expire); return (B_TRUE); default: return (B_FALSE); } } /* * c-state wakeup function. * Similar to cpu_wakeup and cpu_wakeup_mwait except this function deals * with CPUs asleep in MWAIT, HLT, or ACPI Deep C-State. */ void cstate_wakeup(cpu_t *cp, int bound) { struct machcpu *mcpu = &(cp->cpu_m); volatile uint32_t *mcpu_mwait = mcpu->mcpu_mwait; cpupart_t *cpu_part; uint_t cpu_found; processorid_t cpu_sid; cpu_part = cp->cpu_part; cpu_sid = cp->cpu_seqid; /* * Clear the halted bit for that CPU since it will be woken up * in a moment. */ if (bitset_in_set(&cpu_part->cp_haltset, cpu_sid)) { /* * Clear the halted bit for that CPU since it will be * poked in a moment. */ bitset_atomic_del(&cpu_part->cp_haltset, cpu_sid); /* * We may find the current CPU present in the halted cpuset * if we're in the context of an interrupt that occurred * before we had a chance to clear our bit in cpu_idle(). * Waking ourself is obviously unnecessary, since if * we're here, we're not halted. */ if (cp != CPU) { /* * Use correct wakeup mechanism */ if ((mcpu_mwait != NULL) && (*mcpu_mwait == MWAIT_HALTED)) MWAIT_WAKEUP(cp); else poke_cpu(cp->cpu_id); } return; } else { /* * This cpu isn't halted, but it's idle or undergoing a * context switch. No need to awaken anyone else. */ if (cp->cpu_thread == cp->cpu_idle_thread || cp->cpu_disp_flags & CPU_DISP_DONTSTEAL) return; } /* * No need to wake up other CPUs if the thread we just enqueued * is bound. */ if (bound) return; /* * See if there's any other halted CPUs. If there are, then * select one, and awaken it. * It's possible that after we find a CPU, somebody else * will awaken it before we get the chance. * In that case, look again. */ do { cpu_found = bitset_find(&cpu_part->cp_haltset); if (cpu_found == (uint_t)-1) return; } while (bitset_atomic_test_and_del(&cpu_part->cp_haltset, cpu_found) < 0); /* * Must use correct wakeup mechanism to avoid lost wakeup of * alternate cpu. */ if (cpu_found != CPU->cpu_seqid) { mcpu_mwait = cpu_seq[cpu_found]->cpu_m.mcpu_mwait; if ((mcpu_mwait != NULL) && (*mcpu_mwait == MWAIT_HALTED)) MWAIT_WAKEUP(cpu_seq[cpu_found]); else poke_cpu(cpu_seq[cpu_found]->cpu_id); } } /* * Function called by CPU idle notification framework to check whether CPU * has been awakened. It will be called with interrupt disabled. * If CPU has been awakened, call cpu_idle_exit() to notify CPU idle * notification framework. */ static void acpi_cpu_mwait_check_wakeup(void *arg) { volatile uint32_t *mcpu_mwait = (volatile uint32_t *)arg; ASSERT(arg != NULL); if (*mcpu_mwait != MWAIT_HALTED) { /* * CPU has been awakened, notify CPU idle notification system. */ cpu_idle_exit(CPU_IDLE_CB_FLAG_IDLE); } else { /* * Toggle interrupt flag to detect pending interrupts. * If interrupt happened, do_interrupt() will notify CPU idle * notification framework so no need to call cpu_idle_exit() * here. */ sti(); SMT_PAUSE(); cli(); } } static void acpi_cpu_mwait_ipi_check_wakeup(void *arg) { volatile uint32_t *mcpu_mwait = (volatile uint32_t *)arg; ASSERT(arg != NULL); if (*mcpu_mwait != MWAIT_WAKEUP_IPI) { /* * CPU has been awakened, notify CPU idle notification system. */ cpu_idle_exit(CPU_IDLE_CB_FLAG_IDLE); } else { /* * Toggle interrupt flag to detect pending interrupts. * If interrupt happened, do_interrupt() will notify CPU idle * notification framework so no need to call cpu_idle_exit() * here. */ sti(); SMT_PAUSE(); cli(); } } /*ARGSUSED*/ static void acpi_cpu_check_wakeup(void *arg) { /* * Toggle interrupt flag to detect pending interrupts. * If interrupt happened, do_interrupt() will notify CPU idle * notification framework so no need to call cpu_idle_exit() here. */ sti(); SMT_PAUSE(); cli(); } /* * Idle the current CPU via ACPI-defined System I/O read to an ACPI-specified * address. */ static void acpi_io_idle(uint32_t address) { uint32_t value; ACPI_TABLE_FADT *gbl_FADT; /* * Do we need to work around an ancient chipset bug in early ACPI * implementations that would result in a late STPCLK# assertion? * * Must be true when running on systems where the ACPI-indicated I/O * read to enter low-power states may resolve before actually stopping * the processor that initiated a low-power transition. On such systems, * it is possible the processor would proceed past the idle point and * *then* be stopped. * * An early workaround that has been carried forward is to read the ACPI * PM Timer after requesting a low-power transition. The timer read will * take long enough that we are certain the processor is safe to be * stopped. * * From some investigation, this was only ever necessary on older Intel * chipsets. Additionally, the timer read can take upwards of a thousand * CPU clocks, so for systems that work correctly, it's just a tarpit * for the CPU as it is woken back up. */ boolean_t need_stpclk_workaround = cpuid_getvendor(CPU) == X86_VENDOR_Intel; /* * The following call will cause us to halt which will cause the store * buffer to be repartitioned, potentially exposing us to the Intel CPU * vulnerability MDS. As such, we need to explicitly call that here. * The other idle methods do this automatically as part of the * implementation of i86_mwait(). */ x86_md_clear(); (void) cpu_acpi_read_port(address, &value, 8); if (need_stpclk_workaround) { acpica_get_global_FADT(&gbl_FADT); (void) cpu_acpi_read_port( gbl_FADT->XPmTimerBlock.Address, &value, 32); } } /* * enter deep c-state handler */ static void acpi_cpu_cstate(cpu_acpi_cstate_t *cstate) { volatile uint32_t *mcpu_mwait = CPU->cpu_m.mcpu_mwait; uint32_t mwait_idle_state; cpu_t *cpup = CPU; processorid_t cpu_sid = cpup->cpu_seqid; cpupart_t *cp = cpup->cpu_part; hrtime_t lapic_expire; uint8_t type = cstate->cs_addrspace_id; uint32_t cs_type = cstate->cs_type; int hset_update = 1; boolean_t using_timer; cpu_idle_check_wakeup_t check_func = &acpi_cpu_check_wakeup; /* * Set our mcpu_mwait here, so we can tell if anyone tries to * wake us between now and when we call mwait. No other cpu will * attempt to set our mcpu_mwait until we add ourself to the haltset. */ if (mcpu_mwait != NULL) { if (type == ACPI_ADR_SPACE_SYSTEM_IO) { mwait_idle_state = MWAIT_WAKEUP_IPI; check_func = &acpi_cpu_mwait_ipi_check_wakeup; } else { mwait_idle_state = MWAIT_HALTED; check_func = &acpi_cpu_mwait_check_wakeup; } *mcpu_mwait = mwait_idle_state; } else { /* * Initialize mwait_idle_state, but with mcpu_mwait NULL we'll * never actually use it here. "MWAIT_RUNNING" just * distinguishes from the "WAKEUP_IPI" and "HALTED" cases above. */ mwait_idle_state = MWAIT_RUNNING; } /* * If this CPU is online, and there are multiple CPUs * in the system, then we should note our halting * by adding ourselves to the partition's halted CPU * bitmap. This allows other CPUs to find/awaken us when * work becomes available. */ if (cpup->cpu_flags & CPU_OFFLINE || ncpus == 1) hset_update = 0; /* * Add ourselves to the partition's halted CPUs bitmask * and set our HALTED flag, if necessary. * * When a thread becomes runnable, it is placed on the queue * and then the halted cpuset is checked to determine who * (if anyone) should be awakened. We therefore need to first * add ourselves to the halted cpuset, and and then check if there * is any work available. * * Note that memory barriers after updating the HALTED flag * are not necessary since an atomic operation (updating the bitmap) * immediately follows. On x86 the atomic operation acts as a * memory barrier for the update of cpu_disp_flags. */ if (hset_update) { cpup->cpu_disp_flags |= CPU_DISP_HALTED; bitset_atomic_add(&cp->cp_haltset, cpu_sid); } /* * Check to make sure there's really nothing to do. Work destined for * this CPU may become available after this check. If we're * mwait-halting we'll be notified through the clearing of our bit in * the halted CPU bitmask, and a write to our mcpu_mwait. Otherwise, * we're hlt-based halting, and we'll be immediately woken by the * pending interrupt. * * disp_anywork() checks disp_nrunnable, so we do not have to later. */ if (disp_anywork()) { if (hset_update) { cpup->cpu_disp_flags &= ~CPU_DISP_HALTED; bitset_atomic_del(&cp->cp_haltset, cpu_sid); } return; } /* * We're on our way to being halted. * * The local APIC timer can stop in ACPI C2 and deeper c-states. * Try to program the HPET hardware to substitute for this CPU's * LAPIC timer. * cstate_use_timer() could disable the LAPIC Timer. Make sure * to start the LAPIC Timer again before leaving this function. * * Disable interrupts here so we will awaken immediately after halting * if someone tries to poke us between now and the time we actually * halt. */ cli(); using_timer = cstate_use_timer(&lapic_expire, CSTATE_USING_HPET); /* * We check for the presence of our bit after disabling interrupts. * If it's cleared, we'll return. If the bit is cleared after * we check then the cstate_wakeup() will pop us out of the halted * state. * * This means that the ordering of the cstate_wakeup() and the clearing * of the bit by cpu_wakeup is important. * cpu_wakeup() must clear our mc_haltset bit, and then call * cstate_wakeup(). * acpi_cpu_cstate() must disable interrupts, then check for the bit. */ if (hset_update && bitset_in_set(&cp->cp_haltset, cpu_sid) == 0) { (void) cstate_use_timer(&lapic_expire, CSTATE_USING_LAT); sti(); cpup->cpu_disp_flags &= ~CPU_DISP_HALTED; return; } /* * The check for anything locally runnable is here for performance * and isn't needed for correctness. disp_nrunnable ought to be * in our cache still, so it's inexpensive to check, and if there * is anything runnable we won't have to wait for the poke. */ if (cpup->cpu_disp->disp_nrunnable != 0) { (void) cstate_use_timer(&lapic_expire, CSTATE_USING_LAT); sti(); if (hset_update) { cpup->cpu_disp_flags &= ~CPU_DISP_HALTED; bitset_atomic_del(&cp->cp_haltset, cpu_sid); } return; } if (using_timer == B_FALSE) { (void) cstate_use_timer(&lapic_expire, CSTATE_USING_LAT); sti(); /* * We are currently unable to program the HPET to act as this * CPU's proxy LAPIC timer. This CPU cannot enter C2 or deeper * because no timer is set to wake it up while its LAPIC timer * stalls in deep C-States. * Enter C1 instead. * * cstate_wakeup() will wake this CPU with an IPI, which works * with either MWAIT or HLT. */ if (mcpu_mwait != NULL) { i86_monitor(mcpu_mwait, 0, 0); if (*mcpu_mwait == MWAIT_HALTED) { if (cpu_idle_enter(IDLE_STATE_C1, 0, check_func, (void *)mcpu_mwait) == 0) { if (*mcpu_mwait == MWAIT_HALTED) { i86_mwait(0, 0); } cpu_idle_exit(CPU_IDLE_CB_FLAG_IDLE); } } } else { if (cpu_idle_enter(cs_type, 0, check_func, NULL) == 0) { mach_cpu_idle(); cpu_idle_exit(CPU_IDLE_CB_FLAG_IDLE); } } /* * We're no longer halted */ if (hset_update) { cpup->cpu_disp_flags &= ~CPU_DISP_HALTED; bitset_atomic_del(&cp->cp_haltset, cpu_sid); } return; } /* * Tell the cpu idle framework we're going to try idling. * * If cpu_idle_enter returns nonzero, we've found out at the last minute * that we don't actually want to idle. */ boolean_t idle_ok = cpu_idle_enter(cs_type, 0, check_func, (void *)mcpu_mwait) == 0; if (idle_ok) { if (type == ACPI_ADR_SPACE_FIXED_HARDWARE) { if (mcpu_mwait != NULL) { /* * We're on our way to being halted. * To avoid a lost wakeup, arm the monitor * before checking if another cpu wrote to * mcpu_mwait to wake us up. */ i86_monitor(mcpu_mwait, 0, 0); if (*mcpu_mwait == mwait_idle_state) { i86_mwait(cstate->cs_address, 1); } } else { mach_cpu_idle(); } } else if (type == ACPI_ADR_SPACE_SYSTEM_IO) { /* * mcpu_mwait is not directly part of idling or wakeup * in the ACPI System I/O case, but if available it can * hint that we shouldn't actually try to idle because * we're about to be woken up anyway. * * A trip through idle/wakeup can be upwards of a few * microseconds, so avoiding that makes this a helpful * optimization, but consulting mcpu_mwait is still not * necessary for correctness here. */ if (!mcpu_mwait || *mcpu_mwait == mwait_idle_state) { acpi_io_idle(cstate->cs_address); } } /* * We've either idled and woken up, or decided not to idle. * Either way, tell the cpu idle framework that we're not trying * to idle anymore. */ cpu_idle_exit(CPU_IDLE_CB_FLAG_IDLE); } /* * The LAPIC timer may have stopped in deep c-state. * Reprogram this CPU's LAPIC here before enabling interrupts. */ (void) cstate_use_timer(&lapic_expire, CSTATE_USING_LAT); sti(); /* * We're no longer halted */ if (hset_update) { cpup->cpu_disp_flags &= ~CPU_DISP_HALTED; bitset_atomic_del(&cp->cp_haltset, cpu_sid); } } /* * Idle the present CPU, deep c-state is supported */ void cpu_acpi_idle(void) { cpu_t *cp = CPU; cpu_acpi_handle_t handle; cma_c_state_t *cs_data; cpu_acpi_cstate_t *cstates; hrtime_t start, end; int cpu_max_cstates; uint32_t cs_indx; uint16_t cs_type; cpupm_mach_state_t *mach_state = (cpupm_mach_state_t *)cp->cpu_m.mcpu_pm_mach_state; handle = mach_state->ms_acpi_handle; ASSERT(CPU_ACPI_CSTATES(handle) != NULL); cs_data = mach_state->ms_cstate.cma_state.cstate; cstates = (cpu_acpi_cstate_t *)CPU_ACPI_CSTATES(handle); ASSERT(cstates != NULL); cpu_max_cstates = cpu_acpi_get_max_cstates(handle); if (cpu_max_cstates > CPU_MAX_CSTATES) cpu_max_cstates = CPU_MAX_CSTATES; if (cpu_max_cstates == 1) { /* no ACPI c-state data */ (*non_deep_idle_cpu)(); return; } start = gethrtime_unscaled(); cs_indx = cpupm_next_cstate(cs_data, cstates, cpu_max_cstates, start); cs_type = cstates[cs_indx].cs_type; switch (cs_type) { default: /* FALLTHROUGH */ case CPU_ACPI_C1: (*non_deep_idle_cpu)(); break; case CPU_ACPI_C2: acpi_cpu_cstate(&cstates[cs_indx]); break; case CPU_ACPI_C3: /* * All supported Intel processors maintain cache coherency * during C3. Currently when entering C3 processors flush * core caches to higher level shared cache. The shared cache * maintains state and supports probes during C3. * Consequently there is no need to handle cache coherency * and Bus Master activity here with the cache flush, BM_RLD * bit, BM_STS bit, nor PM2_CNT.ARB_DIS mechanisms described * in section 8.1.4 of the ACPI Specification 4.0. */ acpi_cpu_cstate(&cstates[cs_indx]); break; } end = gethrtime_unscaled(); /* * Update statistics */ cpupm_wakeup_cstate_data(cs_data, end); } boolean_t cpu_deep_cstates_supported(void) { extern int idle_cpu_no_deep_c; if (idle_cpu_no_deep_c) return (B_FALSE); if (!cpuid_deep_cstates_supported()) return (B_FALSE); if (cpuid_arat_supported()) { cpu_cstate_arat = B_TRUE; return (B_TRUE); } /* * In theory we can use the HPET as a proxy timer in case we can't rely * on the LAPIC in deep C-states. In practice on AMD it seems something * isn't quite right and we just don't get woken up, so the proxy timer * approach doesn't work. Only set up the HPET as proxy timer on Intel * systems for now. */ if (cpuid_getvendor(CPU) == X86_VENDOR_Intel && (hpet.supported == HPET_FULL_SUPPORT) && hpet.install_proxy()) { cpu_cstate_hpet = B_TRUE; return (B_TRUE); } return (B_FALSE); } /* * Validate that this processor supports deep cstate and if so, * get the c-state data from ACPI and cache it. */ static int cpu_idle_init(cpu_t *cp) { cpupm_mach_state_t *mach_state = (cpupm_mach_state_t *)cp->cpu_m.mcpu_pm_mach_state; cpu_acpi_handle_t handle = mach_state->ms_acpi_handle; cpu_acpi_cstate_t *cstate; char name[KSTAT_STRLEN]; int cpu_max_cstates, i; int ret; /* * Cache the C-state specific ACPI data. */ if ((ret = cpu_acpi_cache_cstate_data(handle)) != 0) { if (ret < 0) cmn_err(CE_NOTE, "!Support for CPU deep idle states is being " "disabled due to errors parsing ACPI C-state " "objects exported by BIOS."); cpu_idle_fini(cp); return (-1); } cstate = (cpu_acpi_cstate_t *)CPU_ACPI_CSTATES(handle); cpu_max_cstates = cpu_acpi_get_max_cstates(handle); for (i = CPU_ACPI_C1; i <= cpu_max_cstates; i++) { (void) snprintf(name, KSTAT_STRLEN - 1, "c%d", cstate->cs_type); /* * Allocate, initialize and install cstate kstat */ cstate->cs_ksp = kstat_create("cstate", cp->cpu_id, name, "misc", KSTAT_TYPE_NAMED, sizeof (cpu_idle_kstat) / sizeof (kstat_named_t), KSTAT_FLAG_VIRTUAL); if (cstate->cs_ksp == NULL) { cmn_err(CE_NOTE, "kstat_create(c_state) fail"); } else { cstate->cs_ksp->ks_data = &cpu_idle_kstat; cstate->cs_ksp->ks_lock = &cpu_idle_mutex; cstate->cs_ksp->ks_update = cpu_idle_kstat_update; cstate->cs_ksp->ks_data_size += MAXNAMELEN; cstate->cs_ksp->ks_private = cstate; kstat_install(cstate->cs_ksp); } cstate++; } cpupm_alloc_domains(cp, CPUPM_C_STATES); cpupm_alloc_ms_cstate(cp); if (cpu_deep_cstates_supported()) { uint32_t value; mutex_enter(&cpu_idle_callb_mutex); if (cpu_deep_idle_callb_id == (callb_id_t)0) cpu_deep_idle_callb_id = callb_add(&cpu_deep_idle_callb, (void *)NULL, CB_CL_CPU_DEEP_IDLE, "cpu_deep_idle"); if (cpu_idle_cpr_callb_id == (callb_id_t)0) cpu_idle_cpr_callb_id = callb_add(&cpu_idle_cpr_callb, (void *)NULL, CB_CL_CPR_PM, "cpu_idle_cpr"); mutex_exit(&cpu_idle_callb_mutex); /* * All supported CPUs (Nehalem and later) will remain in C3 * during Bus Master activity. * All CPUs set ACPI_BITREG_BUS_MASTER_RLD to 0 here if it * is not already 0 before enabling Deeper C-states. */ cpu_acpi_get_register(ACPI_BITREG_BUS_MASTER_RLD, &value); if (value & 1) cpu_acpi_set_register(ACPI_BITREG_BUS_MASTER_RLD, 0); } return (0); } /* * Free resources allocated by cpu_idle_init(). */ static void cpu_idle_fini(cpu_t *cp) { cpupm_mach_state_t *mach_state = (cpupm_mach_state_t *)(cp->cpu_m.mcpu_pm_mach_state); cpu_acpi_handle_t handle = mach_state->ms_acpi_handle; cpu_acpi_cstate_t *cstate; uint_t cpu_max_cstates, i; /* * idle cpu points back to the generic one */ idle_cpu = cp->cpu_m.mcpu_idle_cpu = non_deep_idle_cpu; disp_enq_thread = non_deep_idle_disp_enq_thread; cstate = (cpu_acpi_cstate_t *)CPU_ACPI_CSTATES(handle); if (cstate) { cpu_max_cstates = cpu_acpi_get_max_cstates(handle); for (i = CPU_ACPI_C1; i <= cpu_max_cstates; i++) { if (cstate->cs_ksp != NULL) kstat_delete(cstate->cs_ksp); cstate++; } } cpupm_free_ms_cstate(cp); cpupm_free_domains(&cpupm_cstate_domains); cpu_acpi_free_cstate_data(handle); mutex_enter(&cpu_idle_callb_mutex); if (cpu_deep_idle_callb_id != (callb_id_t)0) { (void) callb_delete(cpu_deep_idle_callb_id); cpu_deep_idle_callb_id = (callb_id_t)0; } if (cpu_idle_cpr_callb_id != (callb_id_t)0) { (void) callb_delete(cpu_idle_cpr_callb_id); cpu_idle_cpr_callb_id = (callb_id_t)0; } mutex_exit(&cpu_idle_callb_mutex); } /* * This function is introduced here to solve a race condition * between the master and the slave to touch c-state data structure. * After the slave calls this idle function to switch to the non * deep idle function, the master can go on to reclaim the resource. */ static void cpu_idle_stop_sync(void) { /* switch to the non deep idle function */ CPU->cpu_m.mcpu_idle_cpu = non_deep_idle_cpu; } static void cpu_idle_stop(cpu_t *cp) { cpupm_mach_state_t *mach_state = (cpupm_mach_state_t *)(cp->cpu_m.mcpu_pm_mach_state); cpu_acpi_handle_t handle = mach_state->ms_acpi_handle; cpu_acpi_cstate_t *cstate; uint_t cpu_max_cstates, i = 0; mutex_enter(&cpu_idle_callb_mutex); if (idle_cpu == cpu_idle_adaptive) { /* * invoke the slave to call synchronous idle function. */ cp->cpu_m.mcpu_idle_cpu = cpu_idle_stop_sync; poke_cpu(cp->cpu_id); /* * wait until the slave switchs to non deep idle function, * so that the master is safe to go on to reclaim the resource. */ while (cp->cpu_m.mcpu_idle_cpu != non_deep_idle_cpu) { drv_usecwait(10); if ((++i % CPU_IDLE_STOP_TIMEOUT) == 0) cmn_err(CE_NOTE, "!cpu_idle_stop: the slave" " idle stop timeout"); } } mutex_exit(&cpu_idle_callb_mutex); cstate = (cpu_acpi_cstate_t *)CPU_ACPI_CSTATES(handle); if (cstate) { cpu_max_cstates = cpu_acpi_get_max_cstates(handle); for (i = CPU_ACPI_C1; i <= cpu_max_cstates; i++) { if (cstate->cs_ksp != NULL) kstat_delete(cstate->cs_ksp); cstate++; } } cpupm_free_ms_cstate(cp); cpupm_remove_domains(cp, CPUPM_C_STATES, &cpupm_cstate_domains); cpu_acpi_free_cstate_data(handle); } /*ARGSUSED*/ static boolean_t cpu_deep_idle_callb(void *arg, int code) { boolean_t rslt = B_TRUE; mutex_enter(&cpu_idle_callb_mutex); switch (code) { case PM_DEFAULT_CPU_DEEP_IDLE: /* * Default policy is same as enable */ /*FALLTHROUGH*/ case PM_ENABLE_CPU_DEEP_IDLE: if ((cpu_idle_cfg_state & CPU_IDLE_DEEP_CFG) == 0) break; if (cstate_timer_callback(PM_ENABLE_CPU_DEEP_IDLE)) { disp_enq_thread = cstate_wakeup; idle_cpu = cpu_idle_adaptive; cpu_idle_cfg_state &= ~CPU_IDLE_DEEP_CFG; } else { rslt = B_FALSE; } break; case PM_DISABLE_CPU_DEEP_IDLE: if (cpu_idle_cfg_state & CPU_IDLE_DEEP_CFG) break; idle_cpu = non_deep_idle_cpu; if (cstate_timer_callback(PM_DISABLE_CPU_DEEP_IDLE)) { disp_enq_thread = non_deep_idle_disp_enq_thread; cpu_idle_cfg_state |= CPU_IDLE_DEEP_CFG; } break; default: cmn_err(CE_NOTE, "!cpu deep_idle_callb: invalid code %d\n", code); break; } mutex_exit(&cpu_idle_callb_mutex); return (rslt); } /*ARGSUSED*/ static boolean_t cpu_idle_cpr_callb(void *arg, int code) { boolean_t rslt = B_TRUE; mutex_enter(&cpu_idle_callb_mutex); switch (code) { case CB_CODE_CPR_RESUME: if (cstate_timer_callback(CB_CODE_CPR_RESUME)) { /* * Do not enable dispatcher hooks if disabled by user. */ if (cpu_idle_cfg_state & CPU_IDLE_DEEP_CFG) break; disp_enq_thread = cstate_wakeup; idle_cpu = cpu_idle_adaptive; } else { rslt = B_FALSE; } break; case CB_CODE_CPR_CHKPT: idle_cpu = non_deep_idle_cpu; disp_enq_thread = non_deep_idle_disp_enq_thread; (void) cstate_timer_callback(CB_CODE_CPR_CHKPT); break; default: cmn_err(CE_NOTE, "!cpudvr cpr_callb: invalid code %d\n", code); break; } mutex_exit(&cpu_idle_callb_mutex); return (rslt); } /* * handle _CST notification */ void cpuidle_cstate_instance(cpu_t *cp) { #ifndef __xpv cpupm_mach_state_t *mach_state = (cpupm_mach_state_t *)cp->cpu_m.mcpu_pm_mach_state; cpu_acpi_handle_t handle; struct machcpu *mcpu; cpuset_t dom_cpu_set; kmutex_t *pm_lock; int result = 0; processorid_t cpu_id; if (mach_state == NULL) { return; } ASSERT(mach_state->ms_cstate.cma_domain != NULL); dom_cpu_set = mach_state->ms_cstate.cma_domain->pm_cpus; pm_lock = &mach_state->ms_cstate.cma_domain->pm_lock; /* * Do for all the CPU's in the domain */ mutex_enter(pm_lock); do { CPUSET_FIND(dom_cpu_set, cpu_id); if (cpu_id == CPUSET_NOTINSET) break; ASSERT(cpu_id >= 0 && cpu_id < NCPU); cp = cpu[cpu_id]; mach_state = (cpupm_mach_state_t *) cp->cpu_m.mcpu_pm_mach_state; if (!(mach_state->ms_caps & CPUPM_C_STATES)) { mutex_exit(pm_lock); return; } handle = mach_state->ms_acpi_handle; ASSERT(handle != NULL); /* * re-evaluate cstate object */ if (cpu_acpi_cache_cstate_data(handle) != 0) { cmn_err(CE_WARN, "Cannot re-evaluate the cpu c-state" " object Instance: %d", cpu_id); } mcpu = &(cp->cpu_m); mcpu->max_cstates = cpu_acpi_get_max_cstates(handle); if (mcpu->max_cstates > CPU_ACPI_C1) { (void) cstate_timer_callback( CST_EVENT_MULTIPLE_CSTATES); disp_enq_thread = cstate_wakeup; cp->cpu_m.mcpu_idle_cpu = cpu_acpi_idle; } else if (mcpu->max_cstates == CPU_ACPI_C1) { disp_enq_thread = non_deep_idle_disp_enq_thread; cp->cpu_m.mcpu_idle_cpu = non_deep_idle_cpu; (void) cstate_timer_callback(CST_EVENT_ONE_CSTATE); } CPUSET_ATOMIC_XDEL(dom_cpu_set, cpu_id, result); } while (result < 0); mutex_exit(pm_lock); #endif } /* * handle the number or the type of available processor power states change */ void cpuidle_manage_cstates(void *ctx) { cpu_t *cp = ctx; cpupm_mach_state_t *mach_state = (cpupm_mach_state_t *)cp->cpu_m.mcpu_pm_mach_state; boolean_t is_ready; if (mach_state == NULL) { return; } /* * We currently refuse to power manage if the CPU is not ready to * take cross calls (cross calls fail silently if CPU is not ready * for it). * * Additionally, for x86 platforms we cannot power manage an instance, * until it has been initialized. */ is_ready = (cp->cpu_flags & CPU_READY) && cpupm_cstate_ready(cp); if (!is_ready) return; cpuidle_cstate_instance(cp); } /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2009 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. * * Copyright 2024 Oxide Computer Company */ /* * AMD-specific CPU power management support. * * And, a brief history of AMD CPU power management. Or, "Why you care about CPU * power management even when you are not worried about a few watts from the * wall." This history is intended to provide lodestones for this domain, but is * not a fully comprehensive AMD power management feature chronology. * * In the early 2000s, AMD shipped a feature called PowerNow! in the K6 era - * K6-2E+ and K6-III+ cores, according to "AMD PowerNow! Technology Dynamically * Manages Power and Performance", publication number 24404A. This feature * allowed operating systems to control power and performance settings in a way * that is very similar to ACPI P-states. That is, selectable core voltage and * frequency levels, with default "power-saver" and "high-performance" modes * that are reflective of Pmin and Pmax on a 2024-era AMD processor. * * With Thuban and Zosma parts later in the K10 era, AMD extended power and * frequency management with the "Turbo Core" feature. They talk about this in * more detail in blogs about the Bulldozer architecture, though many materials * are now dead links. Exactly how Turbo Core is informed and managed is less * discussed, or at least I have been unable to find good technical material on * the topic, but we can draw some inferences from what *is* discussed with * those Bulldozer cores: * * introduces the notion of boosting all cores beyond a "base frequency" * * introduces the notion of boosting further with only half or fewer cores * active * * introduces the notion of power-governed turbo boost * * Somewhere in the K10 era, AMD also introduced C-state support, allowing cores * to be put into low-power idle states when not used. Some articles from * reviewers and system integrators around this time indicate that setting the * "C-state mode to C6" is "required to get the highest Turbo Core frequencies." * * As the AMD 15h BIOS and Kernel Developers Guide (BKDG) is clear to note, AMD * C-states do not directly correspond to ACPI C-states. But when an ACPI * low-power C-state is entered, the CPU's low-power implementation is one of * these AMD C-states, and C6 is the lowest-power of them. * * Further, note that in the Bulldozer era, CPUs were in the range of 4-8 cores, * so "half or fewer cores" means "2-4 active cores." * * At this point and onwards, for some families of AMD parts, best single-core * performance can only be achieved if an operating system parks idle CPU cores * in the lowest-power states - AMD's C6, aka ACPI C3. * * Boosting beyond a base clock, in a AMD-defined and approved manner, * potentially on all cores, has since also been branded as "AMD Core * Performance Boost." This is the name you can find this behavior known as in * Zen and later parts. * * Zen included a more expansive power management approach, "Precision Boost." * "Precision Boost" is where we see start to see power management more * explicitly *managed* - core clocks and voltages are decided by some software * running on the new System Management Unit (SMU). Correspondingly, exactly * what voltage/termperature/power inputs will produce what operational outcomes * from the processor become less and less clearly documented. * * For example, a (Zen 1) Ryzen 7 1700 part is labeled as 3.0GHz base clock, * with up to 3.8GHz boost clock. This 800MHz gamut is the purview of the SMU * implementing "Precision Boost." * * In practice, later AMD marketing material implies that Precision Boost * retained "Turbo Core" behavior that peak boost frequences are only attainable * when one or two cores are actually active. Additionally, even if all cores * are loaded, Precision Boost provides some amount of boost if thermal and * power headroom allows. * * Taking the above Ryzen 7 1700 part as an example, the "base clock" of 3.0 GHz * is relatively unlikely to be an actual operational frequency of the part. * Either a core will be off (as in AMD-defined C1 or C6), on in a low-power * P-state (the processor's minimum operational frequency, probably P1 or * whatever Pmin the part supports), or on in a high-power P-state (P0). In the * high-power P-state, if "boost above base clock" feature is enabled, a core * will probably be some hundreds of MHz above its requested clock speed! * * Further, somewhere around the Zen architecture AMD introduced the "Extended * Frequency Range" (XFR) feature, which allows the processor to clock * 100-150MHz (depending on SKU) higher than "max turbo." This is still * constrained by the silicon, power, and thermal limits indicated by a * combination of fused values set at fabrication time, platform firmware, and * potentially user customization (if firmware allows). Specifics here are * still slim pickings. * * Forum-goers in 2018 would discuss their Ryzen 7 1700s having a clock speed of * 3.1-3.2GHz under all-core load, going up to 3.7GHz under one- or two-core * loads. All frequency selection in this range is up to the SMU, potentially * capped by BIOS or OS-provided parameters. * * As of Zen 5, the latest development here is "Precision Boost 2", which began * shipping with Zen 2. This seems to be an upgrade of the power/frequency * selection regime used by the SMU - instead of "all-core" and "low-core" * turbos, the processor measures its utilization of system-specific paramters * such as package temperature and power draw. Exactly how frequency choices are * made at this point appears to be a black box, other than blanket statements * like "the processor will pick the highest permissible frequency given its * operating environment." * * An interesting detail in the marketing material and slide decks surrounding * the introduction of Precision Boost 2.0 is an implicit confirmation that * Precision Boost did maintain a strict "all-core" and "low-core" pair of * frequencies. This comes from the marketing statement that Precision Boost 2.0 * has done away with those concepts from previous generations, instead * providing a "linear scaling" of frequencies under increasing load levels. * * This brings us to 2024; empirically the above blanket statements are only * correct given the operating system managing CPU cores in a way roughly * commensurate with how AMD would expect an operating system to manage them. * * This is especially dramatic on AMD's server parts - Naples, Rome, Milan, and * onward - where with all cores in high-power C-states, but possibly low-power * P-states, still prevent individual cores from boosting closer to a part's * Fmax. The difference between a peak clock without C-state management, and * peak clock with C-state management, can be as much as 20% of a part's Fmax. * This has also been seen on Threadripper systems. But the impact of C-state * management seems much less dramatic on "desktop" parts; a 7950x without * C-state management can see individual cores clocking to 5.4 GHz or above, * much closer to its rated Fmax of 5.75 GHz. * * From empirical measurement, the difference here appears to be an undocumented * "all-core" turbo that the part limits itself to if all cores are in C0, even * if they are in C0 but in Pmax and stopped in hlt/mwait idle - the actual * power draw differences between these states may be small, but simply being * powered seems to trip some threshold. * * One conclusion from all this is that across the board, C-state management can * have a surprising relationship to performance. Unfortunately, the direct * relationships are undocumented. We are entirely dependent on ACPI-provided * latency information to decide if C-state transitions are profitable given * instantaneous workloads and performance needs. * * Finally, CPPC (Collaborative Processor Performance Control) is a feature * that currently seems to be more oriented towards desktop enthusiast parts, * but stretches the above even further. CPPC includes an abstract "performance * scale" a processor supports, where the operating system requests some factor * along this scale based on workloads it must run. CPPC also introduces the * idea of "Preferred Cores", where at manufacturing time individual cores in a * die are fused with information indicating how highly they can be driven. * This is reportedly reflected as higher peak clocks under load, lower voltage * (and less power) at intermediate clocks. * * It would be nice, in the limit of time, to find if a given processor supports * CPPC, collect its preferred cores, and prefer scheduling tasks on those cores * if they are not already busy. This extends somewhat beyond simply managing * power states of loaded cores. */ #include #include #include #include boolean_t cpupm_amd_init(cpu_t *cp) { cpupm_mach_state_t *mach_state = (cpupm_mach_state_t *)(cp->cpu_m.mcpu_pm_mach_state); /* AMD or Hygon? */ if (x86_vendor != X86_VENDOR_AMD && x86_vendor != X86_VENDOR_HYGON) return (B_FALSE); /* * If we support PowerNow! on this processor, then set the * correct cma_ops for the processor. */ mach_state->ms_pstate.cma_ops = pwrnow_supported() ? &pwrnow_ops : NULL; /* * AMD systems may support C-states, so optimistically set cma_ops to * drive C-states. If the system does not *actually* support C-states, * ACPI tables will not include _CST objects and `cpus_init` will fail. * This, in turn, will cause `cpupm_init` to reset idle handling to not * use C-states including clearing `ms_cstate.cma_ops`. */ mach_state->ms_cstate.cma_ops = &cpu_idle_ops; return (B_TRUE); } /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2009 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ /* * Copyright (c) 2009, Intel Corporation. * All rights reserved. */ /* * Intel specific CPU power management support. */ #include #include #include #include #include #include /* * The Intel Processor Driver Capabilities (_PDC). * See Intel Processor Vendor-Specific ACPI Interface Specification * for details. */ #define CPUPM_INTEL_PDC_REVISION 0x1 #define CPUPM_INTEL_PDC_PS_MSR 0x0001 #define CPUPM_INTEL_PDC_C1_HALT 0x0002 #define CPUPM_INTEL_PDC_TS_MSR 0x0004 #define CPUPM_INTEL_PDC_MP 0x0008 #define CPUPM_INTEL_PDC_C2C3_MP 0x0010 #define CPUPM_INTEL_PDC_SW_PSD 0x0020 #define CPUPM_INTEL_PDC_TSD 0x0080 #define CPUPM_INTEL_PDC_C1_FFH 0x0100 #define CPUPM_INTEL_PDC_HW_PSD 0x0800 static uint32_t cpupm_intel_pdccap = 0; /* * MSR for Intel ENERGY_PERF_BIAS feature. * The default processor power operation policy is max performance. * Power control unit drives to max performance at any energy cost. * This MSR is designed to be a power master control knob, * it provides 4-bit OS input to the HW for the logical CPU, based on * user power-policy preference(scale of 0 to 15). 0 is highest * performance, 15 is minimal energy consumption. * 7 is a good balance between performance and energy consumption. */ #define IA32_ENERGY_PERF_BIAS_MSR 0x1B0 #define EPB_MSR_MASK 0xF #define EPB_MAX_PERF 0 #define EPB_BALANCE 7 #define EPB_MAX_POWER_SAVE 15 /* * The value is used to initialize the user power policy preference * in IA32_ENERGY_PERF_BIAS_MSR. Variable is used here to allow tuning * from the /etc/system file. */ uint64_t cpupm_iepb_policy = EPB_MAX_PERF; static void cpupm_iepb_set_policy(uint64_t power_policy); boolean_t cpupm_intel_init(cpu_t *cp) { cpupm_mach_state_t *mach_state = (cpupm_mach_state_t *)(cp->cpu_m.mcpu_pm_mach_state); uint_t family; uint_t model; if (x86_vendor != X86_VENDOR_Intel) return (B_FALSE); family = cpuid_getfamily(cp); model = cpuid_getmodel(cp); cpupm_intel_pdccap = CPUPM_INTEL_PDC_MP; /* * If we support SpeedStep on this processor, then set the * correct cma_ops for the processor and enable appropriate * _PDC bits. */ if (speedstep_supported(family, model)) { mach_state->ms_pstate.cma_ops = &speedstep_ops; cpupm_intel_pdccap |= CPUPM_INTEL_PDC_PS_MSR | CPUPM_INTEL_PDC_C1_HALT | CPUPM_INTEL_PDC_SW_PSD | CPUPM_INTEL_PDC_HW_PSD; } else { mach_state->ms_pstate.cma_ops = NULL; } /* * Set the correct tstate_ops for the processor and * enable appropriate _PDC bits. */ mach_state->ms_tstate.cma_ops = &cpupm_throttle_ops; cpupm_intel_pdccap |= CPUPM_INTEL_PDC_TS_MSR | CPUPM_INTEL_PDC_TSD; /* * If we support deep cstates on this processor, then set the * correct cstate_ops for the processor and enable appropriate * _PDC bits. */ mach_state->ms_cstate.cma_ops = &cpu_idle_ops; cpupm_intel_pdccap |= CPUPM_INTEL_PDC_C1_HALT | CPUPM_INTEL_PDC_C2C3_MP | CPUPM_INTEL_PDC_C1_FFH; /* * _PDC support is optional and the driver should * function even if the _PDC write fails. */ (void) cpu_acpi_write_pdc(mach_state->ms_acpi_handle, CPUPM_INTEL_PDC_REVISION, 1, &cpupm_intel_pdccap); /* * If Intel ENERGY PERFORMANCE BIAS feature is supported, * provides input to the HW, based on user power-policy. */ if (cpuid_iepb_supported(cp)) { cpupm_iepb_set_policy(cpupm_iepb_policy); } return (B_TRUE); } /* * ENERGY_PERF_BIAS setting, * A hint to HW, based on user power-policy */ static void cpupm_iepb_set_policy(uint64_t iepb_policy) { ulong_t iflag; uint64_t epb_value; epb_value = iepb_policy & EPB_MSR_MASK; iflag = intr_clear(); wrmsr(IA32_ENERGY_PERF_BIAS_MSR, epb_value); intr_restore(iflag); } /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2009 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ /* * Copyright (c) 2009, Intel Corporation. * All rights reserved. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* * This callback is used to build the PPM CPU domains once * a CPU device has been started. The callback is initialized * by the PPM driver to point to a routine that will build the * domains. */ void (*cpupm_ppm_alloc_pstate_domains)(cpu_t *); /* * This callback is used to remove CPU from the PPM CPU domains * when the cpu driver is detached. The callback is initialized * by the PPM driver to point to a routine that will remove CPU * from the domains. */ void (*cpupm_ppm_free_pstate_domains)(cpu_t *); /* * This callback is used to redefine the topspeed for a CPU device. * Since all CPUs in a domain should have identical properties, this * callback is initialized by the PPM driver to point to a routine * that will redefine the topspeed for all devices in a CPU domain. * This callback is exercised whenever an ACPI _PPC change notification * is received by the CPU driver. */ void (*cpupm_redefine_topspeed)(void *); /* * This callback is used by the PPM driver to call into the CPU driver * to find a CPU's current topspeed (i.e., it's current ACPI _PPC value). */ void (*cpupm_set_topspeed_callb)(void *, int); /* * This callback is used by the PPM driver to call into the CPU driver * to set a new topspeed for a CPU. */ int (*cpupm_get_topspeed_callb)(void *); static void cpupm_event_notify_handler(ACPI_HANDLE, UINT32, void *); static void cpupm_free_notify_handlers(cpu_t *); static void cpupm_power_manage_notifications(void *); /* * Until proven otherwise, all power states are manageable. */ static uint32_t cpupm_enabled = CPUPM_ALL_STATES; cpupm_state_domains_t *cpupm_pstate_domains = NULL; cpupm_state_domains_t *cpupm_tstate_domains = NULL; cpupm_state_domains_t *cpupm_cstate_domains = NULL; /* * c-state tunables * * cpupm_cs_sample_interval is the length of time we wait before * recalculating c-state statistics. When a CPU goes idle it checks * to see if it has been longer than cpupm_cs_sample_interval since it last * caculated which C-state to go to. * * cpupm_cs_idle_cost_tunable is the ratio of time CPU spends executing + idle * divided by time spent in the idle state transitions. * A value of 10 means the CPU will not spend more than 1/10 of its time * in idle latency. The worst case performance will be 90% of non Deep C-state * kernel. * * cpupm_cs_idle_save_tunable is how long we must stay in a deeper C-state * before it is worth going there. Expressed as a multiple of latency. */ uint32_t cpupm_cs_sample_interval = 100*1000*1000; /* 100 milliseconds */ uint32_t cpupm_cs_idle_cost_tunable = 10; /* work time / latency cost */ uint32_t cpupm_cs_idle_save_tunable = 2; /* idle power savings */ uint16_t cpupm_C2_idle_pct_tunable = 70; uint16_t cpupm_C3_idle_pct_tunable = 80; #ifndef __xpv extern boolean_t cpupm_intel_init(cpu_t *); extern boolean_t cpupm_amd_init(cpu_t *); typedef struct cpupm_vendor { boolean_t (*cpuv_init)(cpu_t *); } cpupm_vendor_t; /* * Table of supported vendors. */ static cpupm_vendor_t cpupm_vendors[] = { cpupm_intel_init, cpupm_amd_init, NULL }; #endif /* * Initialize the machine. * See if a module exists for managing power for this CPU. */ /*ARGSUSED*/ void cpupm_init(cpu_t *cp) { #ifndef __xpv cpupm_vendor_t *vendors; cpupm_mach_state_t *mach_state; struct machcpu *mcpu = &(cp->cpu_m); static boolean_t first = B_TRUE; int *speeds; uint_t nspeeds; int ret; mach_state = cp->cpu_m.mcpu_pm_mach_state = kmem_zalloc(sizeof (cpupm_mach_state_t), KM_SLEEP); mach_state->ms_caps = CPUPM_NO_STATES; mutex_init(&mach_state->ms_lock, NULL, MUTEX_DRIVER, NULL); mach_state->ms_acpi_handle = cpu_acpi_init(cp); if (mach_state->ms_acpi_handle == NULL) { cpupm_fini(cp); cmn_err(CE_WARN, "!cpupm_init: processor %d: " "unable to get ACPI handle", cp->cpu_id); cmn_err(CE_NOTE, "!CPU power management will not function."); CPUPM_DISABLE(); first = B_FALSE; return; } /* * Loop through the CPU management module table and see if * any of the modules implement CPU power management * for this CPU. */ for (vendors = cpupm_vendors; vendors->cpuv_init != NULL; vendors++) { if (vendors->cpuv_init(cp)) break; } /* * Nope, we can't power manage this CPU. */ if (vendors == NULL) { cpupm_fini(cp); CPUPM_DISABLE(); first = B_FALSE; return; } /* * If P-state support exists for this system, then initialize it. */ if (mach_state->ms_pstate.cma_ops != NULL) { ret = mach_state->ms_pstate.cma_ops->cpus_init(cp); if (ret != 0) { mach_state->ms_pstate.cma_ops = NULL; cpupm_disable(CPUPM_P_STATES); } else { nspeeds = cpupm_get_speeds(cp, &speeds); if (nspeeds == 0) { cmn_err(CE_NOTE, "!cpupm_init: processor %d:" " no speeds to manage", cp->cpu_id); } else { cpupm_set_supp_freqs(cp, speeds, nspeeds); cpupm_free_speeds(speeds, nspeeds); mach_state->ms_caps |= CPUPM_P_STATES; } } } else { cpupm_disable(CPUPM_P_STATES); } if (mach_state->ms_tstate.cma_ops != NULL) { ret = mach_state->ms_tstate.cma_ops->cpus_init(cp); if (ret != 0) { mach_state->ms_tstate.cma_ops = NULL; cpupm_disable(CPUPM_T_STATES); } else { mach_state->ms_caps |= CPUPM_T_STATES; } } else { cpupm_disable(CPUPM_T_STATES); } /* * If C-states support exists for this system, then initialize it. */ if (mach_state->ms_cstate.cma_ops != NULL) { ret = mach_state->ms_cstate.cma_ops->cpus_init(cp); if (ret != 0) { mach_state->ms_cstate.cma_ops = NULL; mcpu->max_cstates = CPU_ACPI_C1; cpupm_disable(CPUPM_C_STATES); idle_cpu = non_deep_idle_cpu; disp_enq_thread = non_deep_idle_disp_enq_thread; } else if (cpu_deep_cstates_supported()) { mcpu->max_cstates = cpu_acpi_get_max_cstates( mach_state->ms_acpi_handle); if (mcpu->max_cstates > CPU_ACPI_C1) { (void) cstate_timer_callback( CST_EVENT_MULTIPLE_CSTATES); cp->cpu_m.mcpu_idle_cpu = cpu_acpi_idle; mcpu->mcpu_idle_type = CPU_ACPI_C1; disp_enq_thread = cstate_wakeup; } else { (void) cstate_timer_callback( CST_EVENT_ONE_CSTATE); } mach_state->ms_caps |= CPUPM_C_STATES; } else { mcpu->max_cstates = CPU_ACPI_C1; idle_cpu = non_deep_idle_cpu; disp_enq_thread = non_deep_idle_disp_enq_thread; } } else { cpupm_disable(CPUPM_C_STATES); } if (mach_state->ms_caps == CPUPM_NO_STATES) { cpupm_fini(cp); CPUPM_DISABLE(); first = B_FALSE; return; } if ((mach_state->ms_caps & CPUPM_T_STATES) || (mach_state->ms_caps & CPUPM_P_STATES) || (mach_state->ms_caps & CPUPM_C_STATES)) { if (first) { acpica_write_cpupm_capabilities( mach_state->ms_caps & CPUPM_P_STATES, mach_state->ms_caps & CPUPM_C_STATES); } if (mach_state->ms_caps & CPUPM_T_STATES) { cpupm_throttle_manage_notification(cp); } if (mach_state->ms_caps & CPUPM_C_STATES) { cpuidle_manage_cstates(cp); } if (mach_state->ms_caps & CPUPM_P_STATES) { cpupm_power_manage_notifications(cp); } cpupm_add_notify_handler(cp, cpupm_event_notify_handler, cp); } first = B_FALSE; #endif } /* * Free any resources allocated during cpupm initialization or cpupm start. */ /*ARGSUSED*/ void cpupm_free(cpu_t *cp, boolean_t cpupm_stop) { #ifndef __xpv cpupm_mach_state_t *mach_state = (cpupm_mach_state_t *)cp->cpu_m.mcpu_pm_mach_state; if (mach_state == NULL) return; if (mach_state->ms_pstate.cma_ops != NULL) { if (cpupm_stop) mach_state->ms_pstate.cma_ops->cpus_stop(cp); else mach_state->ms_pstate.cma_ops->cpus_fini(cp); mach_state->ms_pstate.cma_ops = NULL; } if (mach_state->ms_tstate.cma_ops != NULL) { if (cpupm_stop) mach_state->ms_tstate.cma_ops->cpus_stop(cp); else mach_state->ms_tstate.cma_ops->cpus_fini(cp); mach_state->ms_tstate.cma_ops = NULL; } if (mach_state->ms_cstate.cma_ops != NULL) { if (cpupm_stop) mach_state->ms_cstate.cma_ops->cpus_stop(cp); else mach_state->ms_cstate.cma_ops->cpus_fini(cp); mach_state->ms_cstate.cma_ops = NULL; } cpupm_free_notify_handlers(cp); if (mach_state->ms_acpi_handle != NULL) { cpu_acpi_fini(mach_state->ms_acpi_handle); mach_state->ms_acpi_handle = NULL; } mutex_destroy(&mach_state->ms_lock); kmem_free(mach_state, sizeof (cpupm_mach_state_t)); cp->cpu_m.mcpu_pm_mach_state = NULL; #endif } void cpupm_fini(cpu_t *cp) { /* * call (*cpus_fini)() ops to release the cpupm resource * in the P/C/T-state driver */ cpupm_free(cp, B_FALSE); } void cpupm_start(cpu_t *cp) { cpupm_init(cp); } void cpupm_stop(cpu_t *cp) { /* * call (*cpus_stop)() ops to reclaim the cpupm resource * in the P/C/T-state driver */ cpupm_free(cp, B_TRUE); } /* * If A CPU has started and at least one power state is manageable, * then the CPU is ready for power management. */ boolean_t cpupm_is_ready(cpu_t *cp) { #ifndef __xpv cpupm_mach_state_t *mach_state = (cpupm_mach_state_t *)cp->cpu_m.mcpu_pm_mach_state; uint32_t cpupm_caps = mach_state->ms_caps; if (cpupm_enabled == CPUPM_NO_STATES) return (B_FALSE); if ((cpupm_caps & CPUPM_T_STATES) || (cpupm_caps & CPUPM_P_STATES) || (cpupm_caps & CPUPM_C_STATES)) return (B_TRUE); return (B_FALSE); #else _NOTE(ARGUNUSED(cp)); return (B_FALSE); #endif } boolean_t cpupm_is_enabled(uint32_t state) { return ((cpupm_enabled & state) == state); } /* * By default, all states are enabled. */ void cpupm_disable(uint32_t state) { if (state & CPUPM_P_STATES) { cpupm_free_domains(&cpupm_pstate_domains); } if (state & CPUPM_T_STATES) { cpupm_free_domains(&cpupm_tstate_domains); } if (state & CPUPM_C_STATES) { cpupm_free_domains(&cpupm_cstate_domains); } cpupm_enabled &= ~state; } /* * Allocate power domains for C,P and T States */ void cpupm_alloc_domains(cpu_t *cp, int state) { cpupm_mach_state_t *mach_state = (cpupm_mach_state_t *)(cp->cpu_m.mcpu_pm_mach_state); cpu_acpi_handle_t handle = mach_state->ms_acpi_handle; cpupm_state_domains_t **dom_ptr; cpupm_state_domains_t *dptr; cpupm_state_domains_t **mach_dom_state_ptr; uint32_t domain; uint32_t type; switch (state) { case CPUPM_P_STATES: if (CPU_ACPI_IS_OBJ_CACHED(handle, CPU_ACPI_PSD_CACHED)) { domain = CPU_ACPI_PSD(handle).sd_domain; type = CPU_ACPI_PSD(handle).sd_type; } else { if (MUTEX_HELD(&cpu_lock)) { domain = cpuid_get_chipid(cp); } else { mutex_enter(&cpu_lock); domain = cpuid_get_chipid(cp); mutex_exit(&cpu_lock); } type = CPU_ACPI_HW_ALL; } dom_ptr = &cpupm_pstate_domains; mach_dom_state_ptr = &mach_state->ms_pstate.cma_domain; break; case CPUPM_T_STATES: if (CPU_ACPI_IS_OBJ_CACHED(handle, CPU_ACPI_TSD_CACHED)) { domain = CPU_ACPI_TSD(handle).sd_domain; type = CPU_ACPI_TSD(handle).sd_type; } else { if (MUTEX_HELD(&cpu_lock)) { domain = cpuid_get_chipid(cp); } else { mutex_enter(&cpu_lock); domain = cpuid_get_chipid(cp); mutex_exit(&cpu_lock); } type = CPU_ACPI_HW_ALL; } dom_ptr = &cpupm_tstate_domains; mach_dom_state_ptr = &mach_state->ms_tstate.cma_domain; break; case CPUPM_C_STATES: if (CPU_ACPI_IS_OBJ_CACHED(handle, CPU_ACPI_CSD_CACHED)) { domain = CPU_ACPI_CSD(handle).sd_domain; type = CPU_ACPI_CSD(handle).sd_type; } else { if (MUTEX_HELD(&cpu_lock)) { domain = cpuid_get_coreid(cp); } else { mutex_enter(&cpu_lock); domain = cpuid_get_coreid(cp); mutex_exit(&cpu_lock); } type = CPU_ACPI_HW_ALL; } dom_ptr = &cpupm_cstate_domains; mach_dom_state_ptr = &mach_state->ms_cstate.cma_domain; break; default: return; } for (dptr = *dom_ptr; dptr != NULL; dptr = dptr->pm_next) { if (dptr->pm_domain == domain) break; } /* new domain is created and linked at the head */ if (dptr == NULL) { dptr = kmem_zalloc(sizeof (cpupm_state_domains_t), KM_SLEEP); dptr->pm_domain = domain; dptr->pm_type = type; dptr->pm_next = *dom_ptr; mutex_init(&dptr->pm_lock, NULL, MUTEX_SPIN, (void *)ipltospl(DISP_LEVEL)); CPUSET_ZERO(dptr->pm_cpus); *dom_ptr = dptr; } CPUSET_ADD(dptr->pm_cpus, cp->cpu_id); *mach_dom_state_ptr = dptr; } /* * Free C, P or T state power domains */ void cpupm_free_domains(cpupm_state_domains_t **dom_ptr) { cpupm_state_domains_t *this_domain, *next_domain; this_domain = *dom_ptr; while (this_domain != NULL) { next_domain = this_domain->pm_next; mutex_destroy(&this_domain->pm_lock); kmem_free((void *)this_domain, sizeof (cpupm_state_domains_t)); this_domain = next_domain; } *dom_ptr = NULL; } /* * Remove CPU from C, P or T state power domains */ void cpupm_remove_domains(cpu_t *cp, int state, cpupm_state_domains_t **dom_ptr) { cpupm_mach_state_t *mach_state = (cpupm_mach_state_t *)(cp->cpu_m.mcpu_pm_mach_state); cpupm_state_domains_t *dptr; uint32_t pm_domain; ASSERT(mach_state); switch (state) { case CPUPM_P_STATES: pm_domain = mach_state->ms_pstate.cma_domain->pm_domain; break; case CPUPM_T_STATES: pm_domain = mach_state->ms_tstate.cma_domain->pm_domain; break; case CPUPM_C_STATES: pm_domain = mach_state->ms_cstate.cma_domain->pm_domain; break; default: return; } /* * Find the CPU C, P or T state power domain */ for (dptr = *dom_ptr; dptr != NULL; dptr = dptr->pm_next) { if (dptr->pm_domain == pm_domain) break; } /* * return if no matched domain found */ if (dptr == NULL) return; /* * We found one matched power domain, remove CPU from its cpuset. * pm_lock(spin lock) here to avoid the race conditions between * event change notification and cpu remove. */ mutex_enter(&dptr->pm_lock); if (CPU_IN_SET(dptr->pm_cpus, cp->cpu_id)) CPUSET_DEL(dptr->pm_cpus, cp->cpu_id); mutex_exit(&dptr->pm_lock); } void cpupm_alloc_ms_cstate(cpu_t *cp) { cpupm_mach_state_t *mach_state; cpupm_mach_acpi_state_t *ms_cstate; mach_state = (cpupm_mach_state_t *)(cp->cpu_m.mcpu_pm_mach_state); ms_cstate = &mach_state->ms_cstate; ASSERT(ms_cstate->cma_state.cstate == NULL); ms_cstate->cma_state.cstate = kmem_zalloc(sizeof (cma_c_state_t), KM_SLEEP); ms_cstate->cma_state.cstate->cs_next_cstate = CPU_ACPI_C1; } void cpupm_free_ms_cstate(cpu_t *cp) { cpupm_mach_state_t *mach_state = (cpupm_mach_state_t *)(cp->cpu_m.mcpu_pm_mach_state); cpupm_mach_acpi_state_t *ms_cstate = &mach_state->ms_cstate; if (ms_cstate->cma_state.cstate != NULL) { kmem_free(ms_cstate->cma_state.cstate, sizeof (cma_c_state_t)); ms_cstate->cma_state.cstate = NULL; } } void cpupm_state_change(cpu_t *cp, int level, int state) { cpupm_mach_state_t *mach_state = (cpupm_mach_state_t *)(cp->cpu_m.mcpu_pm_mach_state); cpupm_state_ops_t *state_ops; cpupm_state_domains_t *state_domain; cpuset_t set; DTRACE_PROBE2(cpupm__state__change, cpu_t *, cp, int, level); if (mach_state == NULL) { return; } switch (state) { case CPUPM_P_STATES: state_ops = mach_state->ms_pstate.cma_ops; state_domain = mach_state->ms_pstate.cma_domain; break; case CPUPM_T_STATES: state_ops = mach_state->ms_tstate.cma_ops; state_domain = mach_state->ms_tstate.cma_domain; break; default: return; } switch (state_domain->pm_type) { case CPU_ACPI_SW_ANY: /* * A request on any CPU in the domain transitions the domain */ CPUSET_ONLY(set, cp->cpu_id); state_ops->cpus_change(set, level); break; case CPU_ACPI_SW_ALL: /* * All CPUs in the domain must request the transition */ case CPU_ACPI_HW_ALL: /* * P/T-state transitions are coordinated by the hardware * For now, request the transition on all CPUs in the domain, * but looking ahead we can probably be smarter about this. */ mutex_enter(&state_domain->pm_lock); state_ops->cpus_change(state_domain->pm_cpus, level); mutex_exit(&state_domain->pm_lock); break; default: cmn_err(CE_NOTE, "Unknown domain coordination type: %d", state_domain->pm_type); } } /* * CPU PM interfaces exposed to the CPU power manager */ /*ARGSUSED*/ id_t cpupm_plat_domain_id(cpu_t *cp, cpupm_dtype_t type) { cpupm_mach_state_t *mach_state = (cpupm_mach_state_t *)(cp->cpu_m.mcpu_pm_mach_state); if ((mach_state == NULL) || (!cpupm_is_enabled(CPUPM_P_STATES) && !cpupm_is_enabled(CPUPM_C_STATES))) { return (CPUPM_NO_DOMAIN); } if (type == CPUPM_DTYPE_ACTIVE) { /* * Return P-State domain for the specified CPU */ if (mach_state->ms_pstate.cma_domain) { return (mach_state->ms_pstate.cma_domain->pm_domain); } } else if (type == CPUPM_DTYPE_IDLE) { /* * Return C-State domain for the specified CPU */ if (mach_state->ms_cstate.cma_domain) { return (mach_state->ms_cstate.cma_domain->pm_domain); } } return (CPUPM_NO_DOMAIN); } uint_t cpupm_plat_state_enumerate(cpu_t *cp, cpupm_dtype_t type, cpupm_state_t *states) { int *speeds = NULL; uint_t nspeeds, i; /* * Idle domain support unimplemented */ if (type != CPUPM_DTYPE_ACTIVE) { return (0); } nspeeds = cpupm_get_speeds(cp, &speeds); /* * If the caller passes NULL for states, just return the * number of states. */ if (states != NULL) { for (i = 0; i < nspeeds; i++) { states[i].cps_speed = speeds[i]; states[i].cps_handle = (cpupm_handle_t)i; } } cpupm_free_speeds(speeds, nspeeds); return (nspeeds); } /*ARGSUSED*/ int cpupm_plat_change_state(cpu_t *cp, cpupm_state_t *state) { if (!cpupm_is_ready(cp)) return (-1); cpupm_state_change(cp, (int)state->cps_handle, CPUPM_P_STATES); return (0); } /*ARGSUSED*/ /* * Note: It is the responsibility of the users of * cpupm_get_speeds() to free the memory allocated * for speeds using cpupm_free_speeds() */ uint_t cpupm_get_speeds(cpu_t *cp, int **speeds) { #ifndef __xpv cpupm_mach_state_t *mach_state = (cpupm_mach_state_t *)cp->cpu_m.mcpu_pm_mach_state; return (cpu_acpi_get_speeds(mach_state->ms_acpi_handle, speeds)); #else return (0); #endif } /*ARGSUSED*/ void cpupm_free_speeds(int *speeds, uint_t nspeeds) { #ifndef __xpv cpu_acpi_free_speeds(speeds, nspeeds); #endif } /* * All CPU instances have been initialized successfully. */ boolean_t cpupm_power_ready(cpu_t *cp) { return (cpupm_is_enabled(CPUPM_P_STATES) && cpupm_is_ready(cp)); } /* * All CPU instances have been initialized successfully. */ boolean_t cpupm_throttle_ready(cpu_t *cp) { return (cpupm_is_enabled(CPUPM_T_STATES) && cpupm_is_ready(cp)); } /* * All CPU instances have been initialized successfully. */ boolean_t cpupm_cstate_ready(cpu_t *cp) { return (cpupm_is_enabled(CPUPM_C_STATES) && cpupm_is_ready(cp)); } void cpupm_notify_handler(ACPI_HANDLE obj, UINT32 val, void *ctx) { cpu_t *cp = ctx; cpupm_mach_state_t *mach_state = (cpupm_mach_state_t *)(cp->cpu_m.mcpu_pm_mach_state); cpupm_notification_t *entry; mutex_enter(&mach_state->ms_lock); for (entry = mach_state->ms_handlers; entry != NULL; entry = entry->nq_next) { entry->nq_handler(obj, val, entry->nq_ctx); } mutex_exit(&mach_state->ms_lock); } /*ARGSUSED*/ void cpupm_add_notify_handler(cpu_t *cp, CPUPM_NOTIFY_HANDLER handler, void *ctx) { #ifndef __xpv cpupm_mach_state_t *mach_state = (cpupm_mach_state_t *)cp->cpu_m.mcpu_pm_mach_state; cpupm_notification_t *entry; entry = kmem_zalloc(sizeof (cpupm_notification_t), KM_SLEEP); entry->nq_handler = handler; entry->nq_ctx = ctx; mutex_enter(&mach_state->ms_lock); if (mach_state->ms_handlers == NULL) { entry->nq_next = NULL; mach_state->ms_handlers = entry; cpu_acpi_install_notify_handler(mach_state->ms_acpi_handle, cpupm_notify_handler, cp); } else { entry->nq_next = mach_state->ms_handlers; mach_state->ms_handlers = entry; } mutex_exit(&mach_state->ms_lock); #endif } /*ARGSUSED*/ static void cpupm_free_notify_handlers(cpu_t *cp) { #ifndef __xpv cpupm_mach_state_t *mach_state = (cpupm_mach_state_t *)cp->cpu_m.mcpu_pm_mach_state; cpupm_notification_t *entry; cpupm_notification_t *next; mutex_enter(&mach_state->ms_lock); if (mach_state->ms_handlers == NULL) { mutex_exit(&mach_state->ms_lock); return; } if (mach_state->ms_acpi_handle != NULL) { cpu_acpi_remove_notify_handler(mach_state->ms_acpi_handle, cpupm_notify_handler); } entry = mach_state->ms_handlers; while (entry != NULL) { next = entry->nq_next; kmem_free(entry, sizeof (cpupm_notification_t)); entry = next; } mach_state->ms_handlers = NULL; mutex_exit(&mach_state->ms_lock); #endif } /* * Get the current max speed from the ACPI _PPC object */ /*ARGSUSED*/ int cpupm_get_top_speed(cpu_t *cp) { #ifndef __xpv cpupm_mach_state_t *mach_state; cpu_acpi_handle_t handle; int plat_level; uint_t nspeeds; int max_level; mach_state = (cpupm_mach_state_t *)cp->cpu_m.mcpu_pm_mach_state; handle = mach_state->ms_acpi_handle; cpu_acpi_cache_ppc(handle); plat_level = CPU_ACPI_PPC(handle); nspeeds = CPU_ACPI_PSTATES_COUNT(handle); max_level = nspeeds - 1; if ((plat_level < 0) || (plat_level > max_level)) { cmn_err(CE_NOTE, "!cpupm_get_top_speed: CPU %d: " "_PPC out of range %d", cp->cpu_id, plat_level); plat_level = 0; } return (plat_level); #else return (0); #endif } /* * This notification handler is called whenever the ACPI _PPC * object changes. The _PPC is a sort of governor on power levels. * It sets an upper threshold on which, _PSS defined, power levels * are usuable. The _PPC value is dynamic and may change as properties * (i.e., thermal or AC source) of the system change. */ static void cpupm_power_manage_notifications(void *ctx) { cpu_t *cp = ctx; int top_speed; top_speed = cpupm_get_top_speed(cp); cpupm_redefine_max_activepwr_state(cp, top_speed); } /* ARGSUSED */ static void cpupm_event_notify_handler(ACPI_HANDLE obj, UINT32 val, void *ctx) { #ifndef __xpv cpu_t *cp = ctx; cpupm_mach_state_t *mach_state = (cpupm_mach_state_t *)(cp->cpu_m.mcpu_pm_mach_state); if (mach_state == NULL) return; /* * Currently, we handle _TPC,_CST and _PPC change notifications. */ if (val == CPUPM_TPC_CHANGE_NOTIFICATION && mach_state->ms_caps & CPUPM_T_STATES) { cpupm_throttle_manage_notification(ctx); } else if (val == CPUPM_CST_CHANGE_NOTIFICATION && mach_state->ms_caps & CPUPM_C_STATES) { cpuidle_manage_cstates(ctx); } else if (val == CPUPM_PPC_CHANGE_NOTIFICATION && mach_state->ms_caps & CPUPM_P_STATES) { cpupm_power_manage_notifications(ctx); } #endif } /* * Update cpupm cstate data each time CPU exits idle. */ void cpupm_wakeup_cstate_data(cma_c_state_t *cs_data, hrtime_t end) { cs_data->cs_idle_exit = end; } /* * Determine next cstate based on cpupm data. * Update cpupm cstate data each time CPU goes idle. * Do as much as possible in the idle state bookkeeping function because the * performance impact while idle is minimal compared to in the wakeup function * when there is real work to do. */ uint32_t cpupm_next_cstate(cma_c_state_t *cs_data, cpu_acpi_cstate_t *cstates, uint32_t cs_count, hrtime_t start) { hrtime_t duration; hrtime_t ave_interval; hrtime_t ave_idle_time; uint32_t i, smpl_cnt; duration = cs_data->cs_idle_exit - cs_data->cs_idle_enter; scalehrtime(&duration); cs_data->cs_idle += duration; cs_data->cs_idle_enter = start; smpl_cnt = ++cs_data->cs_cnt; cs_data->cs_smpl_len = start - cs_data->cs_smpl_start; scalehrtime(&cs_data->cs_smpl_len); if (cs_data->cs_smpl_len > cpupm_cs_sample_interval) { cs_data->cs_smpl_idle = cs_data->cs_idle; cs_data->cs_idle = 0; cs_data->cs_smpl_idle_pct = ((100 * cs_data->cs_smpl_idle) / cs_data->cs_smpl_len); cs_data->cs_smpl_start = start; cs_data->cs_cnt = 0; /* * Strand level C-state policy * The cpu_acpi_cstate_t *cstates array is not required to * have an entry for both CPU_ACPI_C2 and CPU_ACPI_C3. * There are cs_count entries in the cstates array. * cs_data->cs_next_cstate contains the index of the next * C-state this CPU should enter. */ ASSERT(cstates[0].cs_type == CPU_ACPI_C1); /* * Will CPU be idle long enough to save power? */ ave_idle_time = (cs_data->cs_smpl_idle / smpl_cnt) / 1000; for (i = 1; i < cs_count; ++i) { if (ave_idle_time < (cstates[i].cs_latency * cpupm_cs_idle_save_tunable)) { cs_count = i; DTRACE_PROBE2(cpupm__next__cstate, cpu_t *, CPU, int, i); } } /* * Wakeup often (even when non-idle time is very short)? * Some producer/consumer type loads fall into this category. */ ave_interval = (cs_data->cs_smpl_len / smpl_cnt) / 1000; for (i = 1; i < cs_count; ++i) { if (ave_interval <= (cstates[i].cs_latency * cpupm_cs_idle_cost_tunable)) { cs_count = i; DTRACE_PROBE2(cpupm__next__cstate, cpu_t *, CPU, int, (CPU_MAX_CSTATES + i)); } } /* * Idle percent */ for (i = 1; i < cs_count; ++i) { switch (cstates[i].cs_type) { case CPU_ACPI_C2: if (cs_data->cs_smpl_idle_pct < cpupm_C2_idle_pct_tunable) { cs_count = i; DTRACE_PROBE2(cpupm__next__cstate, cpu_t *, CPU, int, ((2 * CPU_MAX_CSTATES) + i)); } break; case CPU_ACPI_C3: if (cs_data->cs_smpl_idle_pct < cpupm_C3_idle_pct_tunable) { cs_count = i; DTRACE_PROBE2(cpupm__next__cstate, cpu_t *, CPU, int, ((2 * CPU_MAX_CSTATES) + i)); } break; } } cs_data->cs_next_cstate = cs_count - 1; } return (cs_data->cs_next_cstate); } /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2009 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ #include #include #include #include #include #include #include static int cpupm_throttle_init(cpu_t *); static void cpupm_throttle_fini(cpu_t *); static void cpupm_throttle(cpuset_t, uint32_t); static void cpupm_throttle_stop(cpu_t *); cpupm_state_ops_t cpupm_throttle_ops = { "Generic ACPI T-state Support", cpupm_throttle_init, cpupm_throttle_fini, cpupm_throttle, cpupm_throttle_stop }; /* * Error returns */ #define THROTTLE_RET_SUCCESS 0x00 #define THROTTLE_RET_INCOMPLETE_DATA 0x01 #define THROTTLE_RET_UNSUP_STATE 0x02 #define THROTTLE_RET_TRANS_INCOMPLETE 0x03 #define THROTTLE_LATENCY_WAIT 1 /* * MSR register for clock modulation */ #define IA32_CLOCK_MODULATION_MSR 0x19A /* * Debugging support */ #ifdef DEBUG volatile int cpupm_throttle_debug = 0; #define CTDEBUG(arglist) if (cpupm_throttle_debug) printf arglist; #else #define CTDEBUG(arglist) #endif /* * Write the _PTC ctrl register. How it is written, depends upon the _PTC * APCI object value. */ static int write_ctrl(cpu_acpi_handle_t handle, uint32_t ctrl) { cpu_acpi_ptc_t *ptc_ctrl; uint64_t reg; int ret = 0; ptc_ctrl = CPU_ACPI_PTC_CTRL(handle); switch (ptc_ctrl->cr_addrspace_id) { case ACPI_ADR_SPACE_FIXED_HARDWARE: /* * Read current thermal state because reserved bits must be * preserved, compose new value, and write it.The writable * bits are 4:1 (1 to 4). * Bits 3:1 => On-Demand Clock Modulation Duty Cycle * Bit 4 => On-Demand Clock Modulation Enable * Left shift ctrl by 1 to allign with bits 1-4 of MSR */ reg = rdmsr(IA32_CLOCK_MODULATION_MSR); reg &= ~((uint64_t)0x1E); reg |= ctrl; wrmsr(IA32_CLOCK_MODULATION_MSR, reg); break; case ACPI_ADR_SPACE_SYSTEM_IO: ret = cpu_acpi_write_port(ptc_ctrl->cr_address, ctrl, ptc_ctrl->cr_width); break; default: DTRACE_PROBE1(throttle_ctrl_unsupported_type, uint8_t, ptc_ctrl->cr_addrspace_id); ret = -1; } DTRACE_PROBE1(throttle_ctrl_write, uint32_t, ctrl); DTRACE_PROBE1(throttle_ctrl_write_err, int, ret); return (ret); } static int read_status(cpu_acpi_handle_t handle, uint32_t *stat) { cpu_acpi_ptc_t *ptc_stat; uint64_t reg; int ret = 0; ptc_stat = CPU_ACPI_PTC_STATUS(handle); switch (ptc_stat->cr_addrspace_id) { case ACPI_ADR_SPACE_FIXED_HARDWARE: reg = rdmsr(IA32_CLOCK_MODULATION_MSR); *stat = reg & 0x1E; ret = 0; break; case ACPI_ADR_SPACE_SYSTEM_IO: ret = cpu_acpi_read_port(ptc_stat->cr_address, stat, ptc_stat->cr_width); break; default: DTRACE_PROBE1(throttle_status_unsupported_type, uint8_t, ptc_stat->cr_addrspace_id); return (-1); } DTRACE_PROBE1(throttle_status_read, uint32_t, *stat); DTRACE_PROBE1(throttle_status_read_err, int, ret); return (ret); } /* * Transition the current processor to the requested throttling state. */ static int cpupm_tstate_transition(xc_arg_t arg1, xc_arg_t arg2 __unused, xc_arg_t arg3 __unused) { uint32_t req_state = arg1; cpupm_mach_state_t *mach_state = (cpupm_mach_state_t *)CPU->cpu_m.mcpu_pm_mach_state; cpu_acpi_handle_t handle = mach_state->ms_acpi_handle; cpu_acpi_tstate_t *req_tstate; uint32_t ctrl; uint32_t stat; int i; req_tstate = (cpu_acpi_tstate_t *)CPU_ACPI_TSTATES(handle); req_tstate += req_state; DTRACE_PROBE1(throttle_transition, uint32_t, CPU_ACPI_FREQPER(req_tstate)); /* * Initiate the processor t-state change. */ ctrl = CPU_ACPI_TSTATE_CTRL(req_tstate); if (write_ctrl(handle, ctrl) != 0) { return (0); } /* * If status is zero, then transition is synchronous and * no status value comparison is required. */ if (CPU_ACPI_TSTATE_STAT(req_tstate) == 0) { return (0); } /* Wait until switch is complete, but bound the loop just in case. */ for (i = CPU_ACPI_TSTATE_TRANSLAT(req_tstate) * 2; i >= 0; i -= THROTTLE_LATENCY_WAIT) { if (read_status(handle, &stat) == 0 && CPU_ACPI_TSTATE_STAT(req_tstate) == stat) break; drv_usecwait(THROTTLE_LATENCY_WAIT); } if (CPU_ACPI_TSTATE_STAT(req_tstate) != stat) { DTRACE_PROBE(throttle_transition_incomplete); } return (0); } static void cpupm_throttle(cpuset_t set, uint32_t throtl_lvl) { xc_arg_t xc_arg = (xc_arg_t)throtl_lvl; /* * If thread is already running on target CPU then just * make the transition request. Otherwise, we'll need to * make a cross-call. */ kpreempt_disable(); if (CPU_IN_SET(set, CPU->cpu_id)) { cpupm_tstate_transition(xc_arg, 0, 0); CPUSET_DEL(set, CPU->cpu_id); } if (!CPUSET_ISNULL(set)) { xc_call(xc_arg, 0, 0, CPUSET2BV(set), cpupm_tstate_transition); } kpreempt_enable(); } static int cpupm_throttle_init(cpu_t *cp) { cpupm_mach_state_t *mach_state = (cpupm_mach_state_t *)cp->cpu_m.mcpu_pm_mach_state; cpu_acpi_handle_t handle = mach_state->ms_acpi_handle; cpu_acpi_ptc_t *ptc_stat; int ret; if ((ret = cpu_acpi_cache_tstate_data(handle)) != 0) { if (ret < 0) cmn_err(CE_NOTE, "!Support for CPU throttling is being " "disabled due to errors parsing ACPI T-state " "objects exported by BIOS."); cpupm_throttle_fini(cp); return (THROTTLE_RET_INCOMPLETE_DATA); } /* * Check the address space used for transitions */ ptc_stat = CPU_ACPI_PTC_STATUS(handle); switch (ptc_stat->cr_addrspace_id) { case ACPI_ADR_SPACE_FIXED_HARDWARE: CTDEBUG(("T-State transitions will use fixed hardware\n")); break; case ACPI_ADR_SPACE_SYSTEM_IO: CTDEBUG(("T-State transitions will use System IO\n")); break; default: cmn_err(CE_NOTE, "!_PTC configured for unsupported " "address space type = %d.", ptc_stat->cr_addrspace_id); return (THROTTLE_RET_INCOMPLETE_DATA); } cpupm_alloc_domains(cp, CPUPM_T_STATES); return (THROTTLE_RET_SUCCESS); } static void cpupm_throttle_fini(cpu_t *cp) { cpupm_mach_state_t *mach_state = (cpupm_mach_state_t *)cp->cpu_m.mcpu_pm_mach_state; cpu_acpi_handle_t handle = mach_state->ms_acpi_handle; cpupm_free_domains(&cpupm_tstate_domains); cpu_acpi_free_tstate_data(handle); } static void cpupm_throttle_stop(cpu_t *cp) { cpupm_mach_state_t *mach_state = (cpupm_mach_state_t *)cp->cpu_m.mcpu_pm_mach_state; cpu_acpi_handle_t handle = mach_state->ms_acpi_handle; cpupm_remove_domains(cp, CPUPM_T_STATES, &cpupm_tstate_domains); cpu_acpi_free_tstate_data(handle); } /* * This routine reads the ACPI _TPC object. It's accessed as a callback * by the cpu driver whenever a _TPC change notification is received. */ static int cpupm_throttle_get_max(processorid_t cpu_id) { cpu_t *cp = cpu[cpu_id]; cpupm_mach_state_t *mach_state = (cpupm_mach_state_t *)(cp->cpu_m.mcpu_pm_mach_state); cpu_acpi_handle_t handle; int throtl_level; int max_throttle_lvl; uint_t num_throtl; if (mach_state == NULL) { return (-1); } handle = mach_state->ms_acpi_handle; ASSERT(handle != NULL); cpu_acpi_cache_tpc(handle); throtl_level = CPU_ACPI_TPC(handle); num_throtl = CPU_ACPI_TSTATES_COUNT(handle); max_throttle_lvl = num_throtl - 1; if ((throtl_level < 0) || (throtl_level > max_throttle_lvl)) { cmn_err(CE_NOTE, "!cpupm_throttle_get_max: CPU %d: " "_TPC out of range %d", cp->cpu_id, throtl_level); throtl_level = 0; } return (throtl_level); } /* * Take care of CPU throttling when _TPC notification arrives */ void cpupm_throttle_manage_notification(void *ctx) { cpu_t *cp = ctx; processorid_t cpu_id = cp->cpu_id; cpupm_mach_state_t *mach_state = (cpupm_mach_state_t *)cp->cpu_m.mcpu_pm_mach_state; boolean_t is_ready; int new_level; if (mach_state == NULL) { return; } /* * We currently refuse to power-manage if the CPU is not ready to * take cross calls (cross calls fail silently if CPU is not ready * for it). * * Additionally, for x86 platforms we cannot power-manage an instance, * until it has been initialized. */ is_ready = (cp->cpu_flags & CPU_READY) && cpupm_throttle_ready(cp); if (!is_ready) return; if (!(mach_state->ms_caps & CPUPM_T_STATES)) return; ASSERT(mach_state->ms_tstate.cma_ops != NULL); /* * Get the new T-State support level */ new_level = cpupm_throttle_get_max(cpu_id); cpupm_state_change(cp, new_level, CPUPM_T_STATES); } /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved. * Copyright 2022 Oxide Computer Co. */ #include #include #include #include #include #include #include #include #include #include static int pwrnow_init(cpu_t *); static void pwrnow_fini(cpu_t *); static void pwrnow_power(cpuset_t, uint32_t); static void pwrnow_stop(cpu_t *); static boolean_t pwrnow_cpb_supported(void); /* * Interfaces for modules implementing AMD's PowerNow!. */ cpupm_state_ops_t pwrnow_ops = { "PowerNow! Technology", pwrnow_init, pwrnow_fini, pwrnow_power, pwrnow_stop }; /* * Error returns */ #define PWRNOW_RET_SUCCESS 0x00 #define PWRNOW_RET_NO_PM 0x01 #define PWRNOW_RET_UNSUP_STATE 0x02 #define PWRNOW_RET_TRANS_INCOMPLETE 0x03 #define PWRNOW_LATENCY_WAIT 10 /* * MSR registers for changing and reading processor power state. */ #define PWRNOW_PERF_CTL_MSR 0xC0010062 #define PWRNOW_PERF_STATUS_MSR 0xC0010063 #define AMD_CPUID_PSTATE_HARDWARE (1<<7) #define AMD_CPUID_TSC_CONSTANT (1<<8) #define AMD_CPUID_CPB (1<<9) /* * Debugging support */ #ifdef DEBUG volatile int pwrnow_debug = 0; #define PWRNOW_DEBUG(arglist) if (pwrnow_debug) printf arglist; #else #define PWRNOW_DEBUG(arglist) #endif /* * Write the ctrl register. */ static void write_ctrl(cpu_acpi_handle_t handle, uint32_t ctrl) { cpu_acpi_pct_t *pct_ctrl; uint64_t reg; pct_ctrl = CPU_ACPI_PCT_CTRL(handle); switch (pct_ctrl->cr_addrspace_id) { case ACPI_ADR_SPACE_FIXED_HARDWARE: reg = ctrl; wrmsr(PWRNOW_PERF_CTL_MSR, reg); break; default: DTRACE_PROBE1(pwrnow_ctrl_unsupported_type, uint8_t, pct_ctrl->cr_addrspace_id); return; } DTRACE_PROBE1(pwrnow_ctrl_write, uint32_t, ctrl); } /* * Transition the current processor to the requested state. */ static int pwrnow_pstate_transition(xc_arg_t arg1, xc_arg_t arg2 __unused, xc_arg_t arg3 __unused) { uint32_t req_state = (uint32_t)arg1; cpupm_mach_state_t *mach_state = (cpupm_mach_state_t *)CPU->cpu_m.mcpu_pm_mach_state; cpu_acpi_handle_t handle = mach_state->ms_acpi_handle; cpu_acpi_pstate_t *req_pstate; uint32_t ctrl; req_pstate = (cpu_acpi_pstate_t *)CPU_ACPI_PSTATES(handle); req_pstate += req_state; DTRACE_PROBE1(pwrnow_transition_freq, uint32_t, CPU_ACPI_FREQ(req_pstate)); /* * Initiate the processor p-state change. */ ctrl = CPU_ACPI_PSTATE_CTRL(req_pstate); write_ctrl(handle, ctrl); if (mach_state->ms_turbo != NULL) cpupm_record_turbo_info(mach_state->ms_turbo, mach_state->ms_pstate.cma_state.pstate, req_state); mach_state->ms_pstate.cma_state.pstate = req_state; cpu_set_curr_clock((uint64_t)CPU_ACPI_FREQ(req_pstate) * 1000000); return (0); } static void pwrnow_power(cpuset_t set, uint32_t req_state) { /* * If thread is already running on target CPU then just * make the transition request. Otherwise, we'll need to * make a cross-call. */ kpreempt_disable(); if (CPU_IN_SET(set, CPU->cpu_id)) { (void) pwrnow_pstate_transition(req_state, 0, 0); CPUSET_DEL(set, CPU->cpu_id); } if (!CPUSET_ISNULL(set)) { xc_call((xc_arg_t)req_state, 0, 0, CPUSET2BV(set), pwrnow_pstate_transition); } kpreempt_enable(); } /* * Validate that this processor supports PowerNow! and if so, * get the P-state data from ACPI and cache it. */ static int pwrnow_init(cpu_t *cp) { cpupm_mach_state_t *mach_state = (cpupm_mach_state_t *)cp->cpu_m.mcpu_pm_mach_state; cpu_acpi_handle_t handle = mach_state->ms_acpi_handle; cpu_acpi_pct_t *pct_stat; static int logged = 0; PWRNOW_DEBUG(("pwrnow_init: processor %d\n", cp->cpu_id)); /* * Cache the P-state specific ACPI data. */ if (cpu_acpi_cache_pstate_data(handle) != 0) { if (!logged) { cmn_err(CE_NOTE, "!PowerNow! support is being " "disabled due to errors parsing ACPI P-state " "objects exported by BIOS."); logged = 1; } pwrnow_fini(cp); return (PWRNOW_RET_NO_PM); } pct_stat = CPU_ACPI_PCT_STATUS(handle); switch (pct_stat->cr_addrspace_id) { case ACPI_ADR_SPACE_FIXED_HARDWARE: PWRNOW_DEBUG(("Transitions will use fixed hardware\n")); break; default: cmn_err(CE_WARN, "!_PCT configured for unsupported " "addrspace = %d.", pct_stat->cr_addrspace_id); cmn_err(CE_NOTE, "!CPU power management will not function."); pwrnow_fini(cp); return (PWRNOW_RET_NO_PM); } cpupm_alloc_domains(cp, CPUPM_P_STATES); /* * Check for Core Performance Boost support */ if (pwrnow_cpb_supported()) mach_state->ms_turbo = cpupm_turbo_init(cp); PWRNOW_DEBUG(("Processor %d succeeded.\n", cp->cpu_id)) return (PWRNOW_RET_SUCCESS); } /* * Free resources allocated by pwrnow_init(). */ static void pwrnow_fini(cpu_t *cp) { cpupm_mach_state_t *mach_state = (cpupm_mach_state_t *)(cp->cpu_m.mcpu_pm_mach_state); cpu_acpi_handle_t handle = mach_state->ms_acpi_handle; cpupm_free_domains(&cpupm_pstate_domains); cpu_acpi_free_pstate_data(handle); if (mach_state->ms_turbo != NULL) cpupm_turbo_fini(mach_state->ms_turbo); mach_state->ms_turbo = NULL; } boolean_t pwrnow_supported() { struct cpuid_regs cpu_regs; /* Required features */ ASSERT(is_x86_feature(x86_featureset, X86FSET_CPUID)); if (!is_x86_feature(x86_featureset, X86FSET_MSR)) { PWRNOW_DEBUG(("No CPUID or MSR support.")); return (B_FALSE); } /* * Get the Advanced Power Management Information. */ cpu_regs.cp_eax = 0x80000007; (void) __cpuid_insn(&cpu_regs); /* * We currently only support CPU power management of * processors that are P-state TSC invariant */ if (!(cpu_regs.cp_edx & AMD_CPUID_TSC_CONSTANT)) { PWRNOW_DEBUG(("No support for CPUs that are not P-state " "TSC invariant.\n")); return (B_FALSE); } /* * We only support the "Fire and Forget" style of PowerNow! (i.e., * single MSR write to change speed). */ if (!(cpu_regs.cp_edx & AMD_CPUID_PSTATE_HARDWARE)) { PWRNOW_DEBUG(("Hardware P-State control is not supported.\n")); return (B_FALSE); } return (B_TRUE); } static boolean_t pwrnow_cpb_supported(void) { struct cpuid_regs cpu_regs; /* Required features */ ASSERT(is_x86_feature(x86_featureset, X86FSET_CPUID)); if (!is_x86_feature(x86_featureset, X86FSET_MSR)) { PWRNOW_DEBUG(("No CPUID or MSR support.")); return (B_FALSE); } /* * Get the Advanced Power Management Information. */ cpu_regs.cp_eax = 0x80000007; (void) __cpuid_insn(&cpu_regs); if (!(cpu_regs.cp_edx & AMD_CPUID_CPB)) return (B_FALSE); return (B_TRUE); } static void pwrnow_stop(cpu_t *cp) { cpupm_mach_state_t *mach_state = (cpupm_mach_state_t *)(cp->cpu_m.mcpu_pm_mach_state); cpu_acpi_handle_t handle = mach_state->ms_acpi_handle; cpupm_remove_domains(cp, CPUPM_P_STATES, &cpupm_pstate_domains); cpu_acpi_free_pstate_data(handle); if (mach_state->ms_turbo != NULL) cpupm_turbo_fini(mach_state->ms_turbo); mach_state->ms_turbo = NULL; } /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved. * Copyright 2022 Oxide Computer Co. */ /* * Copyright (c) 2009, Intel Corporation. * All Rights Reserved. */ #include #include #include #include #include #include #include #include #include #include #include static int speedstep_init(cpu_t *); static void speedstep_fini(cpu_t *); static void speedstep_power(cpuset_t, uint32_t); static void speedstep_stop(cpu_t *); static boolean_t speedstep_turbo_supported(void); /* * Interfaces for modules implementing Intel's Enhanced SpeedStep. */ cpupm_state_ops_t speedstep_ops = { "Enhanced SpeedStep Technology", speedstep_init, speedstep_fini, speedstep_power, speedstep_stop }; /* * Error returns */ #define ESS_RET_SUCCESS 0x00 #define ESS_RET_NO_PM 0x01 #define ESS_RET_UNSUP_STATE 0x02 /* * MSR registers for changing and reading processor power state. */ #define IA32_PERF_STAT_MSR 0x198 #define IA32_PERF_CTL_MSR 0x199 #define IA32_CPUID_TSC_CONSTANT 0xF30 #define IA32_MISC_ENABLE_MSR 0x1A0 #define IA32_MISC_ENABLE_EST (1<<16) #define IA32_MISC_ENABLE_CXE (1<<25) #define CPUID_TURBO_SUPPORT (1 << 1) /* * Debugging support */ #ifdef DEBUG volatile int ess_debug = 0; #define ESSDEBUG(arglist) if (ess_debug) printf arglist; #else #define ESSDEBUG(arglist) #endif /* * Write the ctrl register. How it is written, depends upon the _PCT * APCI object value. */ static void write_ctrl(cpu_acpi_handle_t handle, uint32_t ctrl) { cpu_acpi_pct_t *pct_ctrl; uint64_t reg; pct_ctrl = CPU_ACPI_PCT_CTRL(handle); switch (pct_ctrl->cr_addrspace_id) { case ACPI_ADR_SPACE_FIXED_HARDWARE: /* * Read current power state because reserved bits must be * preserved, compose new value, and write it. */ reg = rdmsr(IA32_PERF_CTL_MSR); reg &= ~((uint64_t)0xFFFF); reg |= ctrl; wrmsr(IA32_PERF_CTL_MSR, reg); break; case ACPI_ADR_SPACE_SYSTEM_IO: (void) cpu_acpi_write_port(pct_ctrl->cr_address, ctrl, pct_ctrl->cr_width); break; default: DTRACE_PROBE1(ess_ctrl_unsupported_type, uint8_t, pct_ctrl->cr_addrspace_id); return; } DTRACE_PROBE1(ess_ctrl_write, uint32_t, ctrl); } /* * Transition the current processor to the requested state. */ int speedstep_pstate_transition(xc_arg_t arg1, xc_arg_t arg2 __unused, xc_arg_t arg3 __unused) { uint32_t req_state = (uint32_t)arg1; cpupm_mach_state_t *mach_state = (cpupm_mach_state_t *)CPU->cpu_m.mcpu_pm_mach_state; cpu_acpi_handle_t handle = mach_state->ms_acpi_handle; cpu_acpi_pstate_t *req_pstate; uint32_t ctrl; req_pstate = (cpu_acpi_pstate_t *)CPU_ACPI_PSTATES(handle); req_pstate += req_state; DTRACE_PROBE1(ess_transition, uint32_t, CPU_ACPI_FREQ(req_pstate)); /* * Initiate the processor p-state change. */ ctrl = CPU_ACPI_PSTATE_CTRL(req_pstate); write_ctrl(handle, ctrl); if (mach_state->ms_turbo != NULL) cpupm_record_turbo_info(mach_state->ms_turbo, mach_state->ms_pstate.cma_state.pstate, req_state); mach_state->ms_pstate.cma_state.pstate = req_state; cpu_set_curr_clock(((uint64_t)CPU_ACPI_FREQ(req_pstate) * 1000000)); return (0); } static void speedstep_power(cpuset_t set, uint32_t req_state) { /* * If thread is already running on target CPU then just * make the transition request. Otherwise, we'll need to * make a cross-call. */ kpreempt_disable(); if (CPU_IN_SET(set, CPU->cpu_id)) { (void) speedstep_pstate_transition(req_state, 0, 0); CPUSET_DEL(set, CPU->cpu_id); } if (!CPUSET_ISNULL(set)) { xc_call((xc_arg_t)req_state, 0, 0, CPUSET2BV(set), speedstep_pstate_transition); } kpreempt_enable(); } /* * Validate that this processor supports Speedstep and if so, * get the P-state data from ACPI and cache it. */ static int speedstep_init(cpu_t *cp) { cpupm_mach_state_t *mach_state = (cpupm_mach_state_t *)cp->cpu_m.mcpu_pm_mach_state; cpu_acpi_handle_t handle = mach_state->ms_acpi_handle; cpu_acpi_pct_t *pct_stat; static int logged = 0; ESSDEBUG(("speedstep_init: processor %d\n", cp->cpu_id)); /* * Cache the P-state specific ACPI data. */ if (cpu_acpi_cache_pstate_data(handle) != 0) { if (!logged) { cmn_err(CE_NOTE, "!SpeedStep support is being " "disabled due to errors parsing ACPI P-state " "objects exported by BIOS."); logged = 1; } speedstep_fini(cp); return (ESS_RET_NO_PM); } pct_stat = CPU_ACPI_PCT_STATUS(handle); switch (pct_stat->cr_addrspace_id) { case ACPI_ADR_SPACE_FIXED_HARDWARE: ESSDEBUG(("Transitions will use fixed hardware\n")); break; case ACPI_ADR_SPACE_SYSTEM_IO: ESSDEBUG(("Transitions will use system IO\n")); break; default: cmn_err(CE_WARN, "!_PCT conifgured for unsupported " "addrspace = %d.", pct_stat->cr_addrspace_id); cmn_err(CE_NOTE, "!CPU power management will not function."); speedstep_fini(cp); return (ESS_RET_NO_PM); } cpupm_alloc_domains(cp, CPUPM_P_STATES); if (speedstep_turbo_supported()) mach_state->ms_turbo = cpupm_turbo_init(cp); ESSDEBUG(("Processor %d succeeded.\n", cp->cpu_id)) return (ESS_RET_SUCCESS); } /* * Free resources allocated by speedstep_init(). */ static void speedstep_fini(cpu_t *cp) { cpupm_mach_state_t *mach_state = (cpupm_mach_state_t *)(cp->cpu_m.mcpu_pm_mach_state); cpu_acpi_handle_t handle = mach_state->ms_acpi_handle; cpupm_free_domains(&cpupm_pstate_domains); cpu_acpi_free_pstate_data(handle); if (mach_state->ms_turbo != NULL) cpupm_turbo_fini(mach_state->ms_turbo); mach_state->ms_turbo = NULL; } static void speedstep_stop(cpu_t *cp) { cpupm_mach_state_t *mach_state = (cpupm_mach_state_t *)(cp->cpu_m.mcpu_pm_mach_state); cpu_acpi_handle_t handle = mach_state->ms_acpi_handle; cpupm_remove_domains(cp, CPUPM_P_STATES, &cpupm_pstate_domains); cpu_acpi_free_pstate_data(handle); if (mach_state->ms_turbo != NULL) cpupm_turbo_fini(mach_state->ms_turbo); mach_state->ms_turbo = NULL; } boolean_t speedstep_supported(uint_t family, uint_t model) { struct cpuid_regs cpu_regs; /* Required features */ ASSERT(is_x86_feature(x86_featureset, X86FSET_CPUID)); if (!is_x86_feature(x86_featureset, X86FSET_MSR)) { return (B_FALSE); } /* * We only support family/model combinations which * are P-state TSC invariant. */ if (!((family == 0xf && model >= 0x3) || (family == 0x6 && model >= 0xe))) { return (B_FALSE); } /* * Enhanced SpeedStep supported? */ cpu_regs.cp_eax = 0x1; (void) __cpuid_insn(&cpu_regs); if (!(cpu_regs.cp_ecx & CPUID_INTC_ECX_EST)) { return (B_FALSE); } return (B_TRUE); } boolean_t speedstep_turbo_supported(void) { struct cpuid_regs cpu_regs; /* Required features */ ASSERT(is_x86_feature(x86_featureset, X86FSET_CPUID)); if (!is_x86_feature(x86_featureset, X86FSET_MSR)) { return (B_FALSE); } /* * turbo mode supported? */ cpu_regs.cp_eax = 0x6; (void) __cpuid_insn(&cpu_regs); if (!(cpu_regs.cp_eax & CPUID_TURBO_SUPPORT)) { return (B_FALSE); } return (B_TRUE); } /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved. */ /* * Copyright (c) 2009, Intel Corporation. * All Rights Reserved. */ #include #include #include #include #include #include #include #include #include #include #include typedef struct turbo_kstat_s { struct kstat_named turbo_supported; /* turbo flag */ struct kstat_named t_mcnt; /* IA32_MPERF_MSR */ struct kstat_named t_acnt; /* IA32_APERF_MSR */ } turbo_kstat_t; static int turbo_kstat_update(kstat_t *, int); static void get_turbo_info(cpupm_mach_turbo_info_t *); static void reset_turbo_info(void); static void record_turbo_info(cpupm_mach_turbo_info_t *, uint32_t, uint32_t); static void update_turbo_info(cpupm_mach_turbo_info_t *); static kmutex_t turbo_mutex; turbo_kstat_t turbo_kstat = { { "turbo_supported", KSTAT_DATA_UINT32 }, { "turbo_mcnt", KSTAT_DATA_UINT64 }, { "turbo_acnt", KSTAT_DATA_UINT64 }, }; #define CPU_ACPI_P0 0 #define CPU_IN_TURBO 1 /* * MSR for hardware coordination feedback mechanism * - IA32_MPERF: increments in proportion to a fixed frequency * - IA32_APERF: increments in proportion to actual performance */ #define IA32_MPERF_MSR 0xE7 #define IA32_APERF_MSR 0xE8 /* * kstat update function of the turbo mode info */ static int turbo_kstat_update(kstat_t *ksp, int flag) { cpupm_mach_turbo_info_t *turbo_info = ksp->ks_private; if (flag == KSTAT_WRITE) { return (EACCES); } /* * update the count in case CPU is in the turbo * mode for a long time */ if (turbo_info->in_turbo == CPU_IN_TURBO) update_turbo_info(turbo_info); turbo_kstat.turbo_supported.value.ui32 = turbo_info->turbo_supported; turbo_kstat.t_mcnt.value.ui64 = turbo_info->t_mcnt; turbo_kstat.t_acnt.value.ui64 = turbo_info->t_acnt; return (0); } /* * update the sum of counts and clear MSRs */ static void update_turbo_info(cpupm_mach_turbo_info_t *turbo_info) { ulong_t iflag; uint64_t mcnt, acnt; iflag = intr_clear(); mcnt = rdmsr(IA32_MPERF_MSR); acnt = rdmsr(IA32_APERF_MSR); wrmsr(IA32_MPERF_MSR, 0); wrmsr(IA32_APERF_MSR, 0); turbo_info->t_mcnt += mcnt; turbo_info->t_acnt += acnt; intr_restore(iflag); } /* * Get count of MPERF/APERF MSR */ static void get_turbo_info(cpupm_mach_turbo_info_t *turbo_info) { ulong_t iflag; uint64_t mcnt, acnt; iflag = intr_clear(); mcnt = rdmsr(IA32_MPERF_MSR); acnt = rdmsr(IA32_APERF_MSR); turbo_info->t_mcnt += mcnt; turbo_info->t_acnt += acnt; intr_restore(iflag); } /* * Clear MPERF/APERF MSR */ static void reset_turbo_info(void) { ulong_t iflag; iflag = intr_clear(); wrmsr(IA32_MPERF_MSR, 0); wrmsr(IA32_APERF_MSR, 0); intr_restore(iflag); } /* * sum up the count of one CPU_ACPI_P0 transition */ void cpupm_record_turbo_info(cpupm_mach_turbo_info_t *turbo_info, uint32_t cur_state, uint32_t req_state) { if (!turbo_info->turbo_supported) return; /* * enter P0 state */ if (req_state == CPU_ACPI_P0) { reset_turbo_info(); turbo_info->in_turbo = CPU_IN_TURBO; } /* * Leave P0 state */ else if (cur_state == CPU_ACPI_P0) { turbo_info->in_turbo = 0; get_turbo_info(turbo_info); } } cpupm_mach_turbo_info_t * cpupm_turbo_init(cpu_t *cp) { cpupm_mach_turbo_info_t *turbo_info; turbo_info = kmem_zalloc(sizeof (cpupm_mach_turbo_info_t), KM_SLEEP); turbo_info->turbo_supported = 1; turbo_info->turbo_ksp = kstat_create("turbo", cp->cpu_id, "turbo", "misc", KSTAT_TYPE_NAMED, sizeof (turbo_kstat) / sizeof (kstat_named_t), KSTAT_FLAG_VIRTUAL); if (turbo_info->turbo_ksp == NULL) { cmn_err(CE_NOTE, "kstat_create(turbo) fail"); } else { turbo_info->turbo_ksp->ks_data = &turbo_kstat; turbo_info->turbo_ksp->ks_lock = &turbo_mutex; turbo_info->turbo_ksp->ks_update = turbo_kstat_update; turbo_info->turbo_ksp->ks_data_size += MAXNAMELEN; turbo_info->turbo_ksp->ks_private = turbo_info; kstat_install(turbo_info->turbo_ksp); } return (turbo_info); } void cpupm_turbo_fini(cpupm_mach_turbo_info_t *turbo_info) { if (turbo_info->turbo_ksp != NULL) kstat_delete(turbo_info->turbo_ksp); kmem_free(turbo_info, sizeof (cpupm_mach_turbo_info_t)); } /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved. * Copyright 2012 Garrett D'Amore * Copyright 2014 Pluribus Networks, Inc. * Copyright 2016 Nexenta Systems, Inc. * Copyright 2018 Joyent, Inc. */ /* * PC specific DDI implementation */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #if defined(__xpv) #include #endif #include #include #include #include #include /* * DDI Boot Configuration */ /* * Platform drivers on this platform */ char *platform_module_list[] = { "acpippm", "ppm", (char *)0 }; /* pci bus resource maps */ struct pci_bus_resource *pci_bus_res; size_t dma_max_copybuf_size = 0x101000; /* 1M + 4K */ uint64_t ramdisk_start, ramdisk_end; int pseudo_isa = 0; /* * Forward declarations */ static int getlongprop_buf(); static void get_boot_properties(void); static void impl_bus_initialprobe(void); static void impl_bus_reprobe(void); static int poke_mem(peekpoke_ctlops_t *in_args); static int peek_mem(peekpoke_ctlops_t *in_args); static int kmem_override_cache_attrs(caddr_t, size_t, uint_t); #if !defined(__xpv) extern void immu_init(void); #endif /* * We use an AVL tree to store contiguous address allocations made with the * kalloca() routine, so that we can return the size to free with kfreea(). * Note that in the future it would be vastly faster if we could eliminate * this lookup by insisting that all callers keep track of their own sizes, * just as for kmem_alloc(). */ struct ctgas { avl_node_t ctg_link; void *ctg_addr; size_t ctg_size; }; static avl_tree_t ctgtree; static kmutex_t ctgmutex; #define CTGLOCK() mutex_enter(&ctgmutex) #define CTGUNLOCK() mutex_exit(&ctgmutex) /* * Minimum pfn value of page_t's put on the free list. This is to simplify * support of ddi dma memory requests which specify small, non-zero addr_lo * values. * * The default value of 2, which corresponds to the only known non-zero addr_lo * value used, means a single page will be sacrificed (pfn typically starts * at 1). ddiphysmin can be set to 0 to disable. It cannot be set above 0x100 * otherwise mp startup panics. */ pfn_t ddiphysmin = 2; static void check_driver_disable(void) { int proplen = 128; char *prop_name; char *drv_name, *propval; major_t major; prop_name = kmem_alloc(proplen, KM_SLEEP); for (major = 0; major < devcnt; major++) { drv_name = ddi_major_to_name(major); if (drv_name == NULL) continue; (void) snprintf(prop_name, proplen, "disable-%s", drv_name); if (ddi_prop_lookup_string(DDI_DEV_T_ANY, ddi_root_node(), DDI_PROP_DONTPASS, prop_name, &propval) == DDI_SUCCESS) { if (strcmp(propval, "true") == 0) { devnamesp[major].dn_flags |= DN_DRIVER_REMOVED; cmn_err(CE_NOTE, "driver %s disabled", drv_name); } ddi_prop_free(propval); } } kmem_free(prop_name, proplen); } /* * Configure the hardware on the system. * Called before the rootfs is mounted */ void configure(void) { extern void i_ddi_init_root(); extern int fpu_ignored; /* * Determine if an FPU is attached */ PRM_POINT("fpu_probe()"); fpu_probe(); if (fpu_ignored) { printf("FP hardware will not be used\n"); } else if (!fpu_exists) { printf("No FPU in configuration\n"); } /* * Initialize devices on the machine. * Uses configuration tree built by the PROMs to determine what * is present, and builds a tree of prototype dev_info nodes * corresponding to the hardware which identified itself. */ /* * Initialize root node. */ PRM_POINT("i_ddi_init_root()"); i_ddi_init_root(); /* reprogram devices not set up by firmware (BIOS) */ PRM_POINT("impl_bus_reprobe()"); impl_bus_reprobe(); #if !defined(__xpv) /* * Setup but don't startup the IOMMU * Startup happens later via a direct call * to IOMMU code by boot code. * At this point, all PCI bus renumbering * is done, so safe to init the IMMU * AKA Intel IOMMU. */ PRM_POINT("immu_init()"); immu_init(); #endif /* * attach the isa nexus to get ACPI resource usage * isa is "kind of" a pseudo node */ #if defined(__xpv) if (DOMAIN_IS_INITDOMAIN(xen_info)) { if (pseudo_isa) (void) i_ddi_attach_pseudo_node("isa"); else (void) i_ddi_attach_hw_nodes("isa"); } #else PRM_POINT("isa attach"); if (pseudo_isa) (void) i_ddi_attach_pseudo_node("isa"); else (void) i_ddi_attach_hw_nodes("isa"); #endif PRM_POINT("configure() exit"); } /* * The "status" property indicates the operational status of a device. * If this property is present, the value is a string indicating the * status of the device as follows: * * "okay" operational. * "disabled" not operational, but might become operational. * "fail" not operational because a fault has been detected, * and it is unlikely that the device will become * operational without repair. no additional details * are available. * "fail-xxx" not operational because a fault has been detected, * and it is unlikely that the device will become * operational without repair. "xxx" is additional * human-readable information about the particular * fault condition that was detected. * * The absence of this property means that the operational status is * unknown or okay. * * This routine checks the status property of the specified device node * and returns 0 if the operational status indicates failure, and 1 otherwise. * * The property may exist on plug-in cards the existed before IEEE 1275-1994. * And, in that case, the property may not even be a string. So we carefully * check for the value "fail", in the beginning of the string, noting * the property length. */ int status_okay(int id, char *buf, int buflen) { char status_buf[OBP_MAXPROPNAME]; char *bufp = buf; int len = buflen; int proplen; static const char *status = "status"; static const char *fail = "fail"; int fail_len = (int)strlen(fail); /* * Get the proplen ... if it's smaller than "fail", * or doesn't exist ... then we don't care, since * the value can't begin with the char string "fail". * * NB: proplen, if it's a string, includes the NULL in the * the size of the property, and fail_len does not. */ proplen = prom_getproplen((pnode_t)id, (caddr_t)status); if (proplen <= fail_len) /* nonexistant or uninteresting len */ return (1); /* * if a buffer was provided, use it */ if ((buf == (char *)NULL) || (buflen <= 0)) { bufp = status_buf; len = sizeof (status_buf); } *bufp = (char)0; /* * Get the property into the buffer, to the extent of the buffer, * and in case the buffer is smaller than the property size, * NULL terminate the buffer. (This handles the case where * a buffer was passed in and the caller wants to print the * value, but the buffer was too small). */ (void) prom_bounded_getprop((pnode_t)id, (caddr_t)status, (caddr_t)bufp, len); *(bufp + len - 1) = (char)0; /* * If the value begins with the char string "fail", * then it means the node is failed. We don't care * about any other values. We assume the node is ok * although it might be 'disabled'. */ if (strncmp(bufp, fail, fail_len) == 0) return (0); return (1); } /* * Check the status of the device node passed as an argument. * * if ((status is OKAY) || (status is DISABLED)) * return DDI_SUCCESS * else * print a warning and return DDI_FAILURE */ /*ARGSUSED1*/ int check_status(int id, char *name, dev_info_t *parent) { char status_buf[64]; char devtype_buf[OBP_MAXPROPNAME]; int retval = DDI_FAILURE; /* * is the status okay? */ if (status_okay(id, status_buf, sizeof (status_buf))) return (DDI_SUCCESS); /* * a status property indicating bad memory will be associated * with a node which has a "device_type" property with a value of * "memory-controller". in this situation, return DDI_SUCCESS */ if (getlongprop_buf(id, OBP_DEVICETYPE, devtype_buf, sizeof (devtype_buf)) > 0) { if (strcmp(devtype_buf, "memory-controller") == 0) retval = DDI_SUCCESS; } /* * print the status property information */ cmn_err(CE_WARN, "status '%s' for '%s'", status_buf, name); return (retval); } /*ARGSUSED*/ uint_t softlevel1(caddr_t arg1, caddr_t arg2) { softint(); return (1); } /* * Allow for implementation specific correction of PROM property values. */ /*ARGSUSED*/ void impl_fix_props(dev_info_t *dip, dev_info_t *ch_dip, char *name, int len, caddr_t buffer) { /* * There are no adjustments needed in this implementation. */ } static int getlongprop_buf(int id, char *name, char *buf, int maxlen) { int size; size = prom_getproplen((pnode_t)id, name); if (size <= 0 || (size > maxlen - 1)) return (-1); if (-1 == prom_getprop((pnode_t)id, name, buf)) return (-1); if (strcmp("name", name) == 0) { if (buf[size - 1] != '\0') { buf[size] = '\0'; size += 1; } } return (size); } static int get_prop_int_array(dev_info_t *di, char *pname, int **pval, uint_t *plen) { int ret; if ((ret = ddi_prop_lookup_int_array(DDI_DEV_T_ANY, di, DDI_PROP_DONTPASS, pname, pval, plen)) == DDI_PROP_SUCCESS) { *plen = (*plen) * (sizeof (int)); } return (ret); } /* * Node Configuration */ struct prop_ispec { uint_t pri, vec; }; /* * For the x86, we're prepared to claim that the interrupt string * is in the form of a list of specifications. */ #define VEC_MIN 1 #define VEC_MAX 255 static int impl_xlate_intrs(dev_info_t *child, int *in, struct ddi_parent_private_data *pdptr) { size_t size; int n; struct intrspec *new; caddr_t got_prop; int *inpri; int got_len; extern int ignore_hardware_nodes; /* force flag from ddi_impl.c */ static char bad_intr_fmt[] = "bad interrupt spec from %s%d - ipl %d, irq %d\n"; /* * determine if the driver is expecting the new style "interrupts" * property which just contains the IRQ, or the old style which * contains pairs of . if it is the new style, we always * assign IPL 5 unless an "interrupt-priorities" property exists. * in that case, the "interrupt-priorities" property contains the * IPL values that match, one for one, the IRQ values in the * "interrupts" property. */ inpri = NULL; if ((ddi_getprop(DDI_DEV_T_ANY, child, DDI_PROP_DONTPASS, "ignore-hardware-nodes", -1) != -1) || ignore_hardware_nodes) { /* the old style "interrupts" property... */ /* * The list consists of elements */ if ((n = (*in++ >> 1)) < 1) return (DDI_FAILURE); pdptr->par_nintr = n; size = n * sizeof (struct intrspec); new = pdptr->par_intr = kmem_zalloc(size, KM_SLEEP); while (n--) { int level = *in++; int vec = *in++; if (level < 1 || level > MAXIPL || vec < VEC_MIN || vec > VEC_MAX) { cmn_err(CE_CONT, bad_intr_fmt, DEVI(child)->devi_name, DEVI(child)->devi_instance, level, vec); goto broken; } new->intrspec_pri = level; if (vec != 2) new->intrspec_vec = vec; else /* * irq 2 on the PC bus is tied to irq 9 * on ISA, EISA and MicroChannel */ new->intrspec_vec = 9; new++; } return (DDI_SUCCESS); } else { /* the new style "interrupts" property... */ /* * The list consists of elements */ if ((n = (*in++)) < 1) return (DDI_FAILURE); pdptr->par_nintr = n; size = n * sizeof (struct intrspec); new = pdptr->par_intr = kmem_zalloc(size, KM_SLEEP); /* XXX check for "interrupt-priorities" property... */ if (ddi_getlongprop(DDI_DEV_T_ANY, child, DDI_PROP_DONTPASS, "interrupt-priorities", (caddr_t)&got_prop, &got_len) == DDI_PROP_SUCCESS) { if (n != (got_len / sizeof (int))) { cmn_err(CE_CONT, "bad interrupt-priorities length" " from %s%d: expected %d, got %d\n", DEVI(child)->devi_name, DEVI(child)->devi_instance, n, (int)(got_len / sizeof (int))); goto broken; } inpri = (int *)got_prop; } while (n--) { int level; int vec = *in++; if (inpri == NULL) level = 5; else level = *inpri++; if (level < 1 || level > MAXIPL || vec < VEC_MIN || vec > VEC_MAX) { cmn_err(CE_CONT, bad_intr_fmt, DEVI(child)->devi_name, DEVI(child)->devi_instance, level, vec); goto broken; } new->intrspec_pri = level; if (vec != 2) new->intrspec_vec = vec; else /* * irq 2 on the PC bus is tied to irq 9 * on ISA, EISA and MicroChannel */ new->intrspec_vec = 9; new++; } if (inpri != NULL) kmem_free(got_prop, got_len); return (DDI_SUCCESS); } broken: kmem_free(pdptr->par_intr, size); pdptr->par_intr = NULL; pdptr->par_nintr = 0; if (inpri != NULL) kmem_free(got_prop, got_len); return (DDI_FAILURE); } /* * Create a ddi_parent_private_data structure from the ddi properties of * the dev_info node. * * The "reg" and either an "intr" or "interrupts" properties are required * if the driver wishes to create mappings or field interrupts on behalf * of the device. * * The "reg" property is assumed to be a list of at least one triple * * *1 * * The "intr" property is assumed to be a list of at least one duple * * *1 * * The "interrupts" property is assumed to be a list of at least one * n-tuples that describes the interrupt capabilities of the bus the device * is connected to. For SBus, this looks like * * *1 * * (This property obsoletes the 'intr' property). * * The "ranges" property is optional. */ void make_ddi_ppd(dev_info_t *child, struct ddi_parent_private_data **ppd) { struct ddi_parent_private_data *pdptr; int n; int *reg_prop, *rng_prop, *intr_prop, *irupts_prop; uint_t reg_len, rng_len, intr_len, irupts_len; *ppd = pdptr = kmem_zalloc(sizeof (*pdptr), KM_SLEEP); /* * Handle the 'reg' property. */ if ((get_prop_int_array(child, "reg", ®_prop, ®_len) == DDI_PROP_SUCCESS) && (reg_len != 0)) { pdptr->par_nreg = reg_len / (int)sizeof (struct regspec); pdptr->par_reg = (struct regspec *)reg_prop; } /* * See if I have a range (adding one where needed - this * means to add one for sbus node in sun4c, when romvec > 0, * if no range is already defined in the PROM node. * (Currently no sun4c PROMS define range properties, * but they should and may in the future.) For the SBus * node, the range is defined by the SBus reg property. */ if (get_prop_int_array(child, "ranges", &rng_prop, &rng_len) == DDI_PROP_SUCCESS) { pdptr->par_nrng = rng_len / (int)(sizeof (struct rangespec)); pdptr->par_rng = (struct rangespec *)rng_prop; } /* * Handle the 'intr' and 'interrupts' properties */ /* * For backwards compatibility * we first look for the 'intr' property for the device. */ if (get_prop_int_array(child, "intr", &intr_prop, &intr_len) != DDI_PROP_SUCCESS) { intr_len = 0; } /* * If we're to support bus adapters and future platforms cleanly, * we need to support the generalized 'interrupts' property. */ if (get_prop_int_array(child, "interrupts", &irupts_prop, &irupts_len) != DDI_PROP_SUCCESS) { irupts_len = 0; } else if (intr_len != 0) { /* * If both 'intr' and 'interrupts' are defined, * then 'interrupts' wins and we toss the 'intr' away. */ ddi_prop_free((void *)intr_prop); intr_len = 0; } if (intr_len != 0) { /* * Translate the 'intr' property into an array * an array of struct intrspec's. There's not really * very much to do here except copy what's out there. */ struct intrspec *new; struct prop_ispec *l; n = pdptr->par_nintr = intr_len / sizeof (struct prop_ispec); l = (struct prop_ispec *)intr_prop; pdptr->par_intr = new = kmem_zalloc(n * sizeof (struct intrspec), KM_SLEEP); while (n--) { new->intrspec_pri = l->pri; new->intrspec_vec = l->vec; new++; l++; } ddi_prop_free((void *)intr_prop); } else if ((n = irupts_len) != 0) { size_t size; int *out; /* * Translate the 'interrupts' property into an array * of intrspecs for the rest of the DDI framework to * toy with. Only our ancestors really know how to * do this, so ask 'em. We massage the 'interrupts' * property so that it is pre-pended by a count of * the number of integers in the argument. */ size = sizeof (int) + n; out = kmem_alloc(size, KM_SLEEP); *out = n / sizeof (int); bcopy(irupts_prop, out + 1, (size_t)n); ddi_prop_free((void *)irupts_prop); if (impl_xlate_intrs(child, out, pdptr) != DDI_SUCCESS) { cmn_err(CE_CONT, "Unable to translate 'interrupts' for %s%d\n", DEVI(child)->devi_binding_name, DEVI(child)->devi_instance); } kmem_free(out, size); } } /* * Name a child */ static int impl_sunbus_name_child(dev_info_t *child, char *name, int namelen) { /* * Fill in parent-private data and this function returns to us * an indication if it used "registers" to fill in the data. */ if (ddi_get_parent_data(child) == NULL) { struct ddi_parent_private_data *pdptr; make_ddi_ppd(child, &pdptr); ddi_set_parent_data(child, pdptr); } name[0] = '\0'; if (sparc_pd_getnreg(child) > 0) { (void) snprintf(name, namelen, "%x,%x", (uint_t)sparc_pd_getreg(child, 0)->regspec_bustype, (uint_t)sparc_pd_getreg(child, 0)->regspec_addr); } return (DDI_SUCCESS); } /* * Called from the bus_ctl op of sunbus (sbus, obio, etc) nexus drivers * to implement the DDI_CTLOPS_INITCHILD operation. That is, it names * the children of sun busses based on the reg spec. * * Handles the following properties (in make_ddi_ppd): * Property value * Name type * reg register spec * intr old-form interrupt spec * interrupts new (bus-oriented) interrupt spec * ranges range spec */ int impl_ddi_sunbus_initchild(dev_info_t *child) { char name[MAXNAMELEN]; void impl_ddi_sunbus_removechild(dev_info_t *); /* * Name the child, also makes parent private data */ (void) impl_sunbus_name_child(child, name, MAXNAMELEN); ddi_set_name_addr(child, name); /* * Attempt to merge a .conf node; if successful, remove the * .conf node. */ if ((ndi_dev_is_persistent_node(child) == 0) && (ndi_merge_node(child, impl_sunbus_name_child) == DDI_SUCCESS)) { /* * Return failure to remove node */ impl_ddi_sunbus_removechild(child); return (DDI_FAILURE); } return (DDI_SUCCESS); } void impl_free_ddi_ppd(dev_info_t *dip) { struct ddi_parent_private_data *pdptr; size_t n; if ((pdptr = ddi_get_parent_data(dip)) == NULL) return; if ((n = (size_t)pdptr->par_nintr) != 0) /* * Note that kmem_free is used here (instead of * ddi_prop_free) because the contents of the * property were placed into a separate buffer and * mucked with a bit before being stored in par_intr. * The actual return value from the prop lookup * was freed with ddi_prop_free previously. */ kmem_free(pdptr->par_intr, n * sizeof (struct intrspec)); if ((n = (size_t)pdptr->par_nrng) != 0) ddi_prop_free((void *)pdptr->par_rng); if ((n = pdptr->par_nreg) != 0) ddi_prop_free((void *)pdptr->par_reg); kmem_free(pdptr, sizeof (*pdptr)); ddi_set_parent_data(dip, NULL); } void impl_ddi_sunbus_removechild(dev_info_t *dip) { impl_free_ddi_ppd(dip); ddi_set_name_addr(dip, NULL); /* * Strip the node to properly convert it back to prototype form */ impl_rem_dev_props(dip); } /* * DDI Interrupt */ /* * turn this on to force isa, eisa, and mca device to ignore the new * hardware nodes in the device tree (normally turned on only for * drivers that need it by setting the property "ignore-hardware-nodes" * in their driver.conf file). * * 7/31/96 -- Turned off globally. Leaving variable in for the moment * as safety valve. */ int ignore_hardware_nodes = 0; /* * New DDI interrupt framework */ /* * i_ddi_intr_ops: * * This is the interrupt operator function wrapper for the bus function * bus_intr_op. */ int i_ddi_intr_ops(dev_info_t *dip, dev_info_t *rdip, ddi_intr_op_t op, ddi_intr_handle_impl_t *hdlp, void * result) { dev_info_t *pdip = (dev_info_t *)DEVI(dip)->devi_parent; int ret = DDI_FAILURE; /* request parent to process this interrupt op */ if (NEXUS_HAS_INTR_OP(pdip)) ret = (*(DEVI(pdip)->devi_ops->devo_bus_ops->bus_intr_op))( pdip, rdip, op, hdlp, result); else cmn_err(CE_WARN, "Failed to process interrupt " "for %s%d due to down-rev nexus driver %s%d", ddi_get_name(rdip), ddi_get_instance(rdip), ddi_get_name(pdip), ddi_get_instance(pdip)); return (ret); } /* * i_ddi_add_softint - allocate and add a soft interrupt to the system */ int i_ddi_add_softint(ddi_softint_hdl_impl_t *hdlp) { int ret; /* add soft interrupt handler */ ret = add_avsoftintr((void *)hdlp, hdlp->ih_pri, hdlp->ih_cb_func, DEVI(hdlp->ih_dip)->devi_name, hdlp->ih_cb_arg1, hdlp->ih_cb_arg2); return (ret ? DDI_SUCCESS : DDI_FAILURE); } void i_ddi_remove_softint(ddi_softint_hdl_impl_t *hdlp) { (void) rem_avsoftintr((void *)hdlp, hdlp->ih_pri, hdlp->ih_cb_func); } extern void (*setsoftint)(int, struct av_softinfo *); extern boolean_t av_check_softint_pending(struct av_softinfo *, boolean_t); int i_ddi_trigger_softint(ddi_softint_hdl_impl_t *hdlp, void *arg2) { if (av_check_softint_pending(hdlp->ih_pending, B_FALSE)) return (DDI_EPENDING); update_avsoftintr_args((void *)hdlp, hdlp->ih_pri, arg2); (*setsoftint)(hdlp->ih_pri, hdlp->ih_pending); return (DDI_SUCCESS); } /* * i_ddi_set_softint_pri: * * The way this works is that it first tries to add a softint vector * at the new priority in hdlp. If that succeeds; then it removes the * existing softint vector at the old priority. */ int i_ddi_set_softint_pri(ddi_softint_hdl_impl_t *hdlp, uint_t old_pri) { int ret; /* * If a softint is pending at the old priority then fail the request. */ if (av_check_softint_pending(hdlp->ih_pending, B_TRUE)) return (DDI_FAILURE); ret = av_softint_movepri((void *)hdlp, old_pri); return (ret ? DDI_SUCCESS : DDI_FAILURE); } void i_ddi_alloc_intr_phdl(ddi_intr_handle_impl_t *hdlp) { hdlp->ih_private = (void *)kmem_zalloc(sizeof (ihdl_plat_t), KM_SLEEP); } void i_ddi_free_intr_phdl(ddi_intr_handle_impl_t *hdlp) { kmem_free(hdlp->ih_private, sizeof (ihdl_plat_t)); hdlp->ih_private = NULL; } int i_ddi_get_intx_nintrs(dev_info_t *dip) { struct ddi_parent_private_data *pdp; if ((pdp = ddi_get_parent_data(dip)) == NULL) return (0); return (pdp->par_nintr); } /* * DDI Memory/DMA */ /* * Support for allocating DMAable memory to implement * ddi_dma_mem_alloc(9F) interface. */ #define KA_ALIGN_SHIFT 7 #define KA_ALIGN (1 << KA_ALIGN_SHIFT) #define KA_NCACHE (PAGESHIFT + 1 - KA_ALIGN_SHIFT) /* * Dummy DMA attribute template for kmem_io[].kmem_io_attr. We only * care about addr_lo, addr_hi, and align. addr_hi will be dynamically set. */ static ddi_dma_attr_t kmem_io_attr = { DMA_ATTR_V0, 0x0000000000000000ULL, /* dma_attr_addr_lo */ 0x0000000000000000ULL, /* dma_attr_addr_hi */ 0x00ffffff, 0x1000, /* dma_attr_align */ 1, 1, 0xffffffffULL, 0xffffffffULL, 0x1, 1, 0 }; /* kmem io memory ranges and indices */ enum { IO_4P, IO_64G, IO_4G, IO_2G, IO_1G, IO_512M, IO_256M, IO_128M, IO_64M, IO_32M, IO_16M, MAX_MEM_RANGES }; static struct { vmem_t *kmem_io_arena; kmem_cache_t *kmem_io_cache[KA_NCACHE]; ddi_dma_attr_t kmem_io_attr; } kmem_io[MAX_MEM_RANGES]; static int kmem_io_idx; /* index of first populated kmem_io[] */ static page_t * page_create_io_wrapper(void *addr, size_t len, int vmflag, void *arg) { extern page_t *page_create_io(vnode_t *, u_offset_t, uint_t, uint_t, struct as *, caddr_t, ddi_dma_attr_t *); return (page_create_io(&kvp, (u_offset_t)(uintptr_t)addr, len, PG_EXCL | ((vmflag & VM_NOSLEEP) ? 0 : PG_WAIT), &kas, addr, arg)); } #ifdef __xpv static void segkmem_free_io(vmem_t *vmp, void *ptr, size_t size) { extern void page_destroy_io(page_t *); segkmem_xfree(vmp, ptr, size, &kvp, page_destroy_io); } #endif static void * segkmem_alloc_io_4P(vmem_t *vmp, size_t size, int vmflag) { return (segkmem_xalloc(vmp, NULL, size, vmflag, 0, page_create_io_wrapper, &kmem_io[IO_4P].kmem_io_attr)); } static void * segkmem_alloc_io_64G(vmem_t *vmp, size_t size, int vmflag) { return (segkmem_xalloc(vmp, NULL, size, vmflag, 0, page_create_io_wrapper, &kmem_io[IO_64G].kmem_io_attr)); } static void * segkmem_alloc_io_4G(vmem_t *vmp, size_t size, int vmflag) { return (segkmem_xalloc(vmp, NULL, size, vmflag, 0, page_create_io_wrapper, &kmem_io[IO_4G].kmem_io_attr)); } static void * segkmem_alloc_io_2G(vmem_t *vmp, size_t size, int vmflag) { return (segkmem_xalloc(vmp, NULL, size, vmflag, 0, page_create_io_wrapper, &kmem_io[IO_2G].kmem_io_attr)); } static void * segkmem_alloc_io_1G(vmem_t *vmp, size_t size, int vmflag) { return (segkmem_xalloc(vmp, NULL, size, vmflag, 0, page_create_io_wrapper, &kmem_io[IO_1G].kmem_io_attr)); } static void * segkmem_alloc_io_512M(vmem_t *vmp, size_t size, int vmflag) { return (segkmem_xalloc(vmp, NULL, size, vmflag, 0, page_create_io_wrapper, &kmem_io[IO_512M].kmem_io_attr)); } static void * segkmem_alloc_io_256M(vmem_t *vmp, size_t size, int vmflag) { return (segkmem_xalloc(vmp, NULL, size, vmflag, 0, page_create_io_wrapper, &kmem_io[IO_256M].kmem_io_attr)); } static void * segkmem_alloc_io_128M(vmem_t *vmp, size_t size, int vmflag) { return (segkmem_xalloc(vmp, NULL, size, vmflag, 0, page_create_io_wrapper, &kmem_io[IO_128M].kmem_io_attr)); } static void * segkmem_alloc_io_64M(vmem_t *vmp, size_t size, int vmflag) { return (segkmem_xalloc(vmp, NULL, size, vmflag, 0, page_create_io_wrapper, &kmem_io[IO_64M].kmem_io_attr)); } static void * segkmem_alloc_io_32M(vmem_t *vmp, size_t size, int vmflag) { return (segkmem_xalloc(vmp, NULL, size, vmflag, 0, page_create_io_wrapper, &kmem_io[IO_32M].kmem_io_attr)); } static void * segkmem_alloc_io_16M(vmem_t *vmp, size_t size, int vmflag) { return (segkmem_xalloc(vmp, NULL, size, vmflag, 0, page_create_io_wrapper, &kmem_io[IO_16M].kmem_io_attr)); } struct { uint64_t io_limit; char *io_name; void *(*io_alloc)(vmem_t *, size_t, int); int io_initial; /* kmem_io_init during startup */ } io_arena_params[MAX_MEM_RANGES] = { {0x000fffffffffffffULL, "kmem_io_4P", segkmem_alloc_io_4P, 1}, {0x0000000fffffffffULL, "kmem_io_64G", segkmem_alloc_io_64G, 0}, {0x00000000ffffffffULL, "kmem_io_4G", segkmem_alloc_io_4G, 1}, {0x000000007fffffffULL, "kmem_io_2G", segkmem_alloc_io_2G, 1}, {0x000000003fffffffULL, "kmem_io_1G", segkmem_alloc_io_1G, 0}, {0x000000001fffffffULL, "kmem_io_512M", segkmem_alloc_io_512M, 0}, {0x000000000fffffffULL, "kmem_io_256M", segkmem_alloc_io_256M, 0}, {0x0000000007ffffffULL, "kmem_io_128M", segkmem_alloc_io_128M, 0}, {0x0000000003ffffffULL, "kmem_io_64M", segkmem_alloc_io_64M, 0}, {0x0000000001ffffffULL, "kmem_io_32M", segkmem_alloc_io_32M, 0}, {0x0000000000ffffffULL, "kmem_io_16M", segkmem_alloc_io_16M, 1} }; void kmem_io_init(int a) { int c; char name[40]; kmem_io[a].kmem_io_arena = vmem_create(io_arena_params[a].io_name, NULL, 0, PAGESIZE, io_arena_params[a].io_alloc, #ifdef __xpv segkmem_free_io, #else segkmem_free, #endif heap_arena, 0, VM_SLEEP); for (c = 0; c < KA_NCACHE; c++) { size_t size = KA_ALIGN << c; (void) sprintf(name, "%s_%lu", io_arena_params[a].io_name, size); kmem_io[a].kmem_io_cache[c] = kmem_cache_create(name, size, size, NULL, NULL, NULL, NULL, kmem_io[a].kmem_io_arena, 0); } } /* * Return the index of the highest memory range for addr. */ static int kmem_io_index(uint64_t addr) { int n; for (n = kmem_io_idx; n < MAX_MEM_RANGES; n++) { if (kmem_io[n].kmem_io_attr.dma_attr_addr_hi <= addr) { if (kmem_io[n].kmem_io_arena == NULL) kmem_io_init(n); return (n); } } panic("kmem_io_index: invalid addr - must be at least 16m"); /*NOTREACHED*/ } /* * Return the index of the next kmem_io populated memory range * after curindex. */ static int kmem_io_index_next(int curindex) { int n; for (n = curindex + 1; n < MAX_MEM_RANGES; n++) { if (kmem_io[n].kmem_io_arena) return (n); } return (-1); } /* * allow kmem to be mapped in with different PTE cache attribute settings. * Used by i_ddi_mem_alloc() */ int kmem_override_cache_attrs(caddr_t kva, size_t size, uint_t order) { uint_t hat_flags; caddr_t kva_end; uint_t hat_attr; pfn_t pfn; if (hat_getattr(kas.a_hat, kva, &hat_attr) == -1) { return (-1); } hat_attr &= ~HAT_ORDER_MASK; hat_attr |= order | HAT_NOSYNC; hat_flags = HAT_LOAD_LOCK; kva_end = (caddr_t)(((uintptr_t)kva + size + PAGEOFFSET) & (uintptr_t)PAGEMASK); kva = (caddr_t)((uintptr_t)kva & (uintptr_t)PAGEMASK); while (kva < kva_end) { pfn = hat_getpfnum(kas.a_hat, kva); hat_unload(kas.a_hat, kva, PAGESIZE, HAT_UNLOAD_UNLOCK); hat_devload(kas.a_hat, kva, PAGESIZE, pfn, hat_attr, hat_flags); kva += MMU_PAGESIZE; } return (0); } static int ctgcompare(const void *a1, const void *a2) { /* we just want to compare virtual addresses */ a1 = ((struct ctgas *)a1)->ctg_addr; a2 = ((struct ctgas *)a2)->ctg_addr; return (a1 == a2 ? 0 : (a1 < a2 ? -1 : 1)); } void ka_init(void) { int a; paddr_t maxphysaddr; #if !defined(__xpv) extern pfn_t physmax; maxphysaddr = mmu_ptob((paddr_t)physmax) + MMU_PAGEOFFSET; #else maxphysaddr = mmu_ptob((paddr_t)HYPERVISOR_memory_op( XENMEM_maximum_ram_page, NULL)) + MMU_PAGEOFFSET; #endif ASSERT(maxphysaddr <= io_arena_params[0].io_limit); for (a = 0; a < MAX_MEM_RANGES; a++) { if (maxphysaddr >= io_arena_params[a + 1].io_limit) { if (maxphysaddr > io_arena_params[a + 1].io_limit) io_arena_params[a].io_limit = maxphysaddr; else a++; break; } } kmem_io_idx = a; for (; a < MAX_MEM_RANGES; a++) { kmem_io[a].kmem_io_attr = kmem_io_attr; kmem_io[a].kmem_io_attr.dma_attr_addr_hi = io_arena_params[a].io_limit; /* * initialize kmem_io[] arena/cache corresponding to * maxphysaddr and to the "common" io memory ranges that * have io_initial set to a non-zero value. */ if (io_arena_params[a].io_initial || a == kmem_io_idx) kmem_io_init(a); } /* initialize ctgtree */ avl_create(&ctgtree, ctgcompare, sizeof (struct ctgas), offsetof(struct ctgas, ctg_link)); } /* * put contig address/size */ static void * putctgas(void *addr, size_t size) { struct ctgas *ctgp; if ((ctgp = kmem_zalloc(sizeof (*ctgp), KM_NOSLEEP)) != NULL) { ctgp->ctg_addr = addr; ctgp->ctg_size = size; CTGLOCK(); avl_add(&ctgtree, ctgp); CTGUNLOCK(); } return (ctgp); } /* * get contig size by addr */ static size_t getctgsz(void *addr) { struct ctgas *ctgp; struct ctgas find; size_t sz = 0; find.ctg_addr = addr; CTGLOCK(); if ((ctgp = avl_find(&ctgtree, &find, NULL)) != NULL) { avl_remove(&ctgtree, ctgp); } CTGUNLOCK(); if (ctgp != NULL) { sz = ctgp->ctg_size; kmem_free(ctgp, sizeof (*ctgp)); } return (sz); } /* * contig_alloc: * * allocates contiguous memory to satisfy the 'size' and dma attributes * specified in 'attr'. * * Not all of memory need to be physically contiguous if the * scatter-gather list length is greater than 1. */ /*ARGSUSED*/ void * contig_alloc(size_t size, ddi_dma_attr_t *attr, uintptr_t align, int cansleep) { pgcnt_t pgcnt = btopr(size); size_t asize = pgcnt * PAGESIZE; page_t *ppl; int pflag; void *addr; extern page_t *page_create_io(vnode_t *, u_offset_t, uint_t, uint_t, struct as *, caddr_t, ddi_dma_attr_t *); /* segkmem_xalloc */ if (align <= PAGESIZE) addr = vmem_alloc(heap_arena, asize, (cansleep) ? VM_SLEEP : VM_NOSLEEP); else addr = vmem_xalloc(heap_arena, asize, align, 0, 0, NULL, NULL, (cansleep) ? VM_SLEEP : VM_NOSLEEP); if (addr) { ASSERT(!((uintptr_t)addr & (align - 1))); if (page_resv(pgcnt, (cansleep) ? KM_SLEEP : KM_NOSLEEP) == 0) { vmem_free(heap_arena, addr, asize); return (NULL); } pflag = PG_EXCL; if (cansleep) pflag |= PG_WAIT; /* 4k req gets from freelists rather than pfn search */ if (pgcnt > 1 || align > PAGESIZE) pflag |= PG_PHYSCONTIG; ppl = page_create_io(&kvp, (u_offset_t)(uintptr_t)addr, asize, pflag, &kas, (caddr_t)addr, attr); if (!ppl) { vmem_free(heap_arena, addr, asize); page_unresv(pgcnt); return (NULL); } while (ppl != NULL) { page_t *pp = ppl; page_sub(&ppl, pp); ASSERT(page_iolock_assert(pp)); page_io_unlock(pp); page_downgrade(pp); hat_memload(kas.a_hat, (caddr_t)(uintptr_t)pp->p_offset, pp, (PROT_ALL & ~PROT_USER) | HAT_NOSYNC, HAT_LOAD_LOCK); } } return (addr); } void contig_free(void *addr, size_t size) { pgcnt_t pgcnt = btopr(size); size_t asize = pgcnt * PAGESIZE; caddr_t a, ea; page_t *pp; hat_unload(kas.a_hat, addr, asize, HAT_UNLOAD_UNLOCK); for (a = addr, ea = a + asize; a < ea; a += PAGESIZE) { pp = page_find(&kvp, (u_offset_t)(uintptr_t)a); if (!pp) panic("contig_free: contig pp not found"); if (!page_tryupgrade(pp)) { page_unlock(pp); pp = page_lookup(&kvp, (u_offset_t)(uintptr_t)a, SE_EXCL); if (pp == NULL) panic("contig_free: page freed"); } page_destroy(pp, 0); } page_unresv(pgcnt); vmem_free(heap_arena, addr, asize); } /* * Allocate from the system, aligned on a specific boundary. * The alignment, if non-zero, must be a power of 2. */ static void * kalloca(size_t size, size_t align, int cansleep, int physcontig, ddi_dma_attr_t *attr) { size_t *addr, *raddr, rsize; size_t hdrsize = 4 * sizeof (size_t); /* must be power of 2 */ int a, i, c; vmem_t *vmp = NULL; kmem_cache_t *cp = NULL; if (attr->dma_attr_addr_lo > mmu_ptob((uint64_t)ddiphysmin)) return (NULL); align = MAX(align, hdrsize); ASSERT((align & (align - 1)) == 0); /* * All of our allocators guarantee 16-byte alignment, so we don't * need to reserve additional space for the header. * To simplify picking the correct kmem_io_cache, we round up to * a multiple of KA_ALIGN. */ rsize = P2ROUNDUP_TYPED(size + align, KA_ALIGN, size_t); if (physcontig && rsize > PAGESIZE) { if ((addr = contig_alloc(size, attr, align, cansleep)) != NULL) { if (!putctgas(addr, size)) contig_free(addr, size); else return (addr); } return (NULL); } a = kmem_io_index(attr->dma_attr_addr_hi); if (rsize > PAGESIZE) { vmp = kmem_io[a].kmem_io_arena; raddr = vmem_alloc(vmp, rsize, (cansleep) ? VM_SLEEP : VM_NOSLEEP); } else { c = highbit((rsize >> KA_ALIGN_SHIFT) - 1); cp = kmem_io[a].kmem_io_cache[c]; raddr = kmem_cache_alloc(cp, (cansleep) ? KM_SLEEP : KM_NOSLEEP); } if (raddr == NULL) { int na; ASSERT(cansleep == 0); if (rsize > PAGESIZE) return (NULL); /* * System does not have memory in the requested range. * Try smaller kmem io ranges and larger cache sizes * to see if there might be memory available in * these other caches. */ for (na = kmem_io_index_next(a); na >= 0; na = kmem_io_index_next(na)) { ASSERT(kmem_io[na].kmem_io_arena); cp = kmem_io[na].kmem_io_cache[c]; raddr = kmem_cache_alloc(cp, KM_NOSLEEP); if (raddr) goto kallocdone; } /* now try the larger kmem io cache sizes */ for (na = a; na >= 0; na = kmem_io_index_next(na)) { for (i = c + 1; i < KA_NCACHE; i++) { cp = kmem_io[na].kmem_io_cache[i]; raddr = kmem_cache_alloc(cp, KM_NOSLEEP); if (raddr) goto kallocdone; } } return (NULL); } kallocdone: ASSERT(!P2BOUNDARY((uintptr_t)raddr, rsize, PAGESIZE) || rsize > PAGESIZE); addr = (size_t *)P2ROUNDUP((uintptr_t)raddr + hdrsize, align); ASSERT((uintptr_t)addr + size - (uintptr_t)raddr <= rsize); addr[-4] = (size_t)cp; addr[-3] = (size_t)vmp; addr[-2] = (size_t)raddr; addr[-1] = rsize; return (addr); } static void kfreea(void *addr) { size_t size; if (!((uintptr_t)addr & PAGEOFFSET) && (size = getctgsz(addr))) { contig_free(addr, size); } else { size_t *saddr = addr; if (saddr[-4] == 0) vmem_free((vmem_t *)saddr[-3], (void *)saddr[-2], saddr[-1]); else kmem_cache_free((kmem_cache_t *)saddr[-4], (void *)saddr[-2]); } } /*ARGSUSED*/ void i_ddi_devacc_to_hatacc(const ddi_device_acc_attr_t *devaccp, uint_t *hataccp) { } /* * Check if the specified cache attribute is supported on the platform. * This function must be called before i_ddi_cacheattr_to_hatacc(). */ boolean_t i_ddi_check_cache_attr(uint_t flags) { /* * The cache attributes are mutually exclusive. Any combination of * the attributes leads to a failure. */ uint_t cache_attr = IOMEM_CACHE_ATTR(flags); if ((cache_attr != 0) && !ISP2(cache_attr)) return (B_FALSE); /* All cache attributes are supported on X86/X64 */ if (cache_attr & (IOMEM_DATA_UNCACHED | IOMEM_DATA_CACHED | IOMEM_DATA_UC_WR_COMBINE)) return (B_TRUE); /* undefined attributes */ return (B_FALSE); } /* set HAT cache attributes from the cache attributes */ void i_ddi_cacheattr_to_hatacc(uint_t flags, uint_t *hataccp) { uint_t cache_attr = IOMEM_CACHE_ATTR(flags); static char *fname = "i_ddi_cacheattr_to_hatacc"; /* * If write-combining is not supported, then it falls back * to uncacheable. */ if (cache_attr == IOMEM_DATA_UC_WR_COMBINE && !is_x86_feature(x86_featureset, X86FSET_PAT)) cache_attr = IOMEM_DATA_UNCACHED; /* * set HAT attrs according to the cache attrs. */ switch (cache_attr) { case IOMEM_DATA_UNCACHED: *hataccp &= ~HAT_ORDER_MASK; *hataccp |= (HAT_STRICTORDER | HAT_PLAT_NOCACHE); break; case IOMEM_DATA_UC_WR_COMBINE: *hataccp &= ~HAT_ORDER_MASK; *hataccp |= (HAT_MERGING_OK | HAT_PLAT_NOCACHE); break; case IOMEM_DATA_CACHED: *hataccp &= ~HAT_ORDER_MASK; *hataccp |= HAT_UNORDERED_OK; break; /* * This case must not occur because the cache attribute is scrutinized * before this function is called. */ default: /* * set cacheable to hat attrs. */ *hataccp &= ~HAT_ORDER_MASK; *hataccp |= HAT_UNORDERED_OK; cmn_err(CE_WARN, "%s: cache_attr=0x%x is ignored.", fname, cache_attr); } } /* * This should actually be called i_ddi_dma_mem_alloc. There should * also be an i_ddi_pio_mem_alloc. i_ddi_dma_mem_alloc should call * through the device tree with the DDI_CTLOPS_DMA_ALIGN ctl ops to * get alignment requirements for DMA memory. i_ddi_pio_mem_alloc * should use DDI_CTLOPS_PIO_ALIGN. Since we only have i_ddi_mem_alloc * so far which is used for both, DMA and PIO, we have to use the DMA * ctl ops to make everybody happy. */ /*ARGSUSED*/ int i_ddi_mem_alloc(dev_info_t *dip, ddi_dma_attr_t *attr, size_t length, int cansleep, int flags, const ddi_device_acc_attr_t *accattrp, caddr_t *kaddrp, size_t *real_length, ddi_acc_hdl_t *ap) { caddr_t a; int iomin; ddi_acc_impl_t *iap; int physcontig = 0; pgcnt_t npages; pgcnt_t minctg; uint_t order; int e; /* * Check legality of arguments */ if (length == 0 || kaddrp == NULL || attr == NULL) { return (DDI_FAILURE); } if (attr->dma_attr_minxfer == 0 || attr->dma_attr_align == 0 || !ISP2(attr->dma_attr_align) || !ISP2(attr->dma_attr_minxfer)) { return (DDI_FAILURE); } /* * figure out most restrictive alignment requirement */ iomin = attr->dma_attr_minxfer; iomin = maxbit(iomin, attr->dma_attr_align); if (iomin == 0) return (DDI_FAILURE); ASSERT((iomin & (iomin - 1)) == 0); /* * if we allocate memory with IOMEM_DATA_UNCACHED or * IOMEM_DATA_UC_WR_COMBINE, make sure we allocate a page aligned * memory that ends on a page boundry. * Don't want to have to different cache mappings to the same * physical page. */ if (OVERRIDE_CACHE_ATTR(flags)) { iomin = (iomin + MMU_PAGEOFFSET) & MMU_PAGEMASK; length = (length + MMU_PAGEOFFSET) & (size_t)MMU_PAGEMASK; } /* * Determine if we need to satisfy the request for physically * contiguous memory or alignments larger than pagesize. */ npages = btopr(length + attr->dma_attr_align); minctg = howmany(npages, attr->dma_attr_sgllen); if (minctg > 1) { uint64_t pfnseg = attr->dma_attr_seg >> PAGESHIFT; /* * verify that the minimum contig requirement for the * actual length does not cross segment boundary. */ length = P2ROUNDUP_TYPED(length, attr->dma_attr_minxfer, size_t); npages = btopr(length); minctg = howmany(npages, attr->dma_attr_sgllen); if (minctg > pfnseg + 1) return (DDI_FAILURE); physcontig = 1; } else { length = P2ROUNDUP_TYPED(length, iomin, size_t); } /* * Allocate the requested amount from the system. */ a = kalloca(length, iomin, cansleep, physcontig, attr); if ((*kaddrp = a) == NULL) return (DDI_FAILURE); /* * if we to modify the cache attributes, go back and muck with the * mappings. */ if (OVERRIDE_CACHE_ATTR(flags)) { order = 0; i_ddi_cacheattr_to_hatacc(flags, &order); e = kmem_override_cache_attrs(a, length, order); if (e != 0) { kfreea(a); return (DDI_FAILURE); } } if (real_length) { *real_length = length; } if (ap) { /* * initialize access handle */ iap = (ddi_acc_impl_t *)ap->ah_platform_private; iap->ahi_acc_attr |= DDI_ACCATTR_CPU_VADDR; impl_acc_hdl_init(ap); } return (DDI_SUCCESS); } /* ARGSUSED */ void i_ddi_mem_free(caddr_t kaddr, ddi_acc_hdl_t *ap) { if (ap != NULL) { /* * if we modified the cache attributes on alloc, go back and * fix them since this memory could be returned to the * general pool. */ if (OVERRIDE_CACHE_ATTR(ap->ah_xfermodes)) { uint_t order = 0; int e; i_ddi_cacheattr_to_hatacc(IOMEM_DATA_CACHED, &order); e = kmem_override_cache_attrs(kaddr, ap->ah_len, order); if (e != 0) { cmn_err(CE_WARN, "i_ddi_mem_free() failed to " "override cache attrs, memory leaked\n"); return; } } } kfreea(kaddr); } /* * Access Barriers * */ /*ARGSUSED*/ int i_ddi_ontrap(ddi_acc_handle_t hp) { return (DDI_FAILURE); } /*ARGSUSED*/ void i_ddi_notrap(ddi_acc_handle_t hp) { } /* * Misc Functions */ /* * Implementation instance override functions * * No override on i86pc */ /*ARGSUSED*/ uint_t impl_assign_instance(dev_info_t *dip) { return ((uint_t)-1); } /*ARGSUSED*/ int impl_keep_instance(dev_info_t *dip) { #if defined(__xpv) /* * Do not persist instance numbers assigned to devices in dom0 */ dev_info_t *pdip; if (DOMAIN_IS_INITDOMAIN(xen_info)) { if (((pdip = ddi_get_parent(dip)) != NULL) && (strcmp(ddi_get_name(pdip), "xpvd") == 0)) return (DDI_SUCCESS); } #endif return (DDI_FAILURE); } /*ARGSUSED*/ int impl_free_instance(dev_info_t *dip) { return (DDI_FAILURE); } /*ARGSUSED*/ int impl_check_cpu(dev_info_t *devi) { return (DDI_SUCCESS); } /* * Referenced in common/cpr_driver.c: Power off machine. * Don't know how to power off i86pc. */ void arch_power_down() {} /* * Copy name to property_name, since name * is in the low address range below kernelbase. */ static void copy_boot_str(const char *boot_str, char *kern_str, int len) { int i = 0; while (i < len - 1 && boot_str[i] != '\0') { kern_str[i] = boot_str[i]; i++; } kern_str[i] = 0; /* null terminate */ if (boot_str[i] != '\0') cmn_err(CE_WARN, "boot property string is truncated to %s", kern_str); } static void get_boot_properties(void) { extern char hw_provider[]; dev_info_t *devi; char *name; int length, flags; char property_name[50], property_val[50]; void *bop_staging_area; bop_staging_area = kmem_zalloc(MMU_PAGESIZE, KM_NOSLEEP); /* * Import "root" properties from the boot. * * We do this by invoking BOP_NEXTPROP until the list * is completely copied in. */ devi = ddi_root_node(); for (name = BOP_NEXTPROP(bootops, ""); /* get first */ name; /* NULL => DONE */ name = BOP_NEXTPROP(bootops, name)) { /* get next */ /* copy string to memory above kernelbase */ copy_boot_str(name, property_name, 50); /* * Skip vga properties. They will be picked up later * by get_vga_properties. */ if (strcmp(property_name, "display-edif-block") == 0 || strcmp(property_name, "display-edif-id") == 0) { continue; } length = BOP_GETPROPLEN(bootops, property_name); if (length < 0) continue; if (length > MMU_PAGESIZE) { cmn_err(CE_NOTE, "boot property %s longer than 0x%x, ignored\n", property_name, MMU_PAGESIZE); continue; } BOP_GETPROP(bootops, property_name, bop_staging_area); flags = do_bsys_getproptype(bootops, property_name); /* * special properties: * si-machine, si-hw-provider * goes to kernel data structures. * bios-boot-device and stdout * goes to hardware property list so it may show up * in the prtconf -vp output. This is needed by * Install/Upgrade. Once we fix install upgrade, * this can be taken out. */ if (strcmp(name, "si-machine") == 0) { (void) strncpy(utsname.machine, bop_staging_area, SYS_NMLN); utsname.machine[SYS_NMLN - 1] = '\0'; continue; } if (strcmp(name, "si-hw-provider") == 0) { (void) strncpy(hw_provider, bop_staging_area, SYS_NMLN); hw_provider[SYS_NMLN - 1] = '\0'; continue; } if (strcmp(name, "bios-boot-device") == 0) { copy_boot_str(bop_staging_area, property_val, 50); (void) ndi_prop_update_string(DDI_DEV_T_NONE, devi, property_name, property_val); continue; } if (strcmp(name, "stdout") == 0) { (void) ndi_prop_update_int(DDI_DEV_T_NONE, devi, property_name, *((int *)bop_staging_area)); continue; } /* Boolean property */ if (length == 0) { (void) e_ddi_prop_create(DDI_DEV_T_NONE, devi, DDI_PROP_CANSLEEP, property_name, NULL, 0); continue; } /* Now anything else based on type. */ switch (flags) { case DDI_PROP_TYPE_INT: if (length == sizeof (int)) { (void) e_ddi_prop_update_int(DDI_DEV_T_NONE, devi, property_name, *((int *)bop_staging_area)); } else { (void) e_ddi_prop_update_int_array( DDI_DEV_T_NONE, devi, property_name, bop_staging_area, length / sizeof (int)); } break; case DDI_PROP_TYPE_STRING: (void) e_ddi_prop_update_string(DDI_DEV_T_NONE, devi, property_name, bop_staging_area); break; case DDI_PROP_TYPE_BYTE: (void) e_ddi_prop_update_byte_array(DDI_DEV_T_NONE, devi, property_name, bop_staging_area, length); break; case DDI_PROP_TYPE_INT64: if (length == sizeof (int64_t)) { (void) e_ddi_prop_update_int64(DDI_DEV_T_NONE, devi, property_name, *((int64_t *)bop_staging_area)); } else { (void) e_ddi_prop_update_int64_array( DDI_DEV_T_NONE, devi, property_name, bop_staging_area, length / sizeof (int64_t)); } break; default: /* Property type unknown, use old prop interface */ (void) e_ddi_prop_create(DDI_DEV_T_NONE, devi, DDI_PROP_CANSLEEP, property_name, bop_staging_area, length); } } kmem_free(bop_staging_area, MMU_PAGESIZE); } static void get_vga_properties(void) { dev_info_t *devi; major_t major; char *name; int length; char property_val[50]; void *bop_staging_area; /* * XXXX Hack Allert! * There really needs to be a better way for identifying various * console framebuffers and their related issues. Till then, * check for this one as a replacement to vgatext. */ major = ddi_name_to_major("ragexl"); if (major == (major_t)-1) { major = ddi_name_to_major("vgatext"); if (major == (major_t)-1) return; } devi = devnamesp[major].dn_head; if (devi == NULL) return; bop_staging_area = kmem_zalloc(MMU_PAGESIZE, KM_SLEEP); /* * Import "vga" properties from the boot. */ name = "display-edif-block"; length = BOP_GETPROPLEN(bootops, name); if (length > 0 && length < MMU_PAGESIZE) { BOP_GETPROP(bootops, name, bop_staging_area); (void) ndi_prop_update_byte_array(DDI_DEV_T_NONE, devi, name, bop_staging_area, length); } /* * kdmconfig is also looking for display-type and * video-adapter-type. We default to color and svga. * * Could it be "monochrome", "vga"? * Nah, you've got to come to the 21st century... * And you can set monitor type manually in kdmconfig * if you are really an old junky. */ (void) ndi_prop_update_string(DDI_DEV_T_NONE, devi, "display-type", "color"); (void) ndi_prop_update_string(DDI_DEV_T_NONE, devi, "video-adapter-type", "svga"); name = "display-edif-id"; length = BOP_GETPROPLEN(bootops, name); if (length > 0 && length < MMU_PAGESIZE) { BOP_GETPROP(bootops, name, bop_staging_area); copy_boot_str(bop_staging_area, property_val, length); (void) ndi_prop_update_string(DDI_DEV_T_NONE, devi, name, property_val); } kmem_free(bop_staging_area, MMU_PAGESIZE); } /* * Copy console font to kernel memory. The temporary font setup * to use font module was done in early console setup, using low * memory and data from font module. Now we need to allocate * kernel memory and copy data over, so the low memory can be freed. * We can have at most one entry in font list from early boot. */ static void get_console_font(void) { struct fontlist *fp, *fl; bitmap_data_t *bd; struct font *fd, *tmp; int i; if (STAILQ_EMPTY(&fonts)) return; fl = STAILQ_FIRST(&fonts); STAILQ_REMOVE_HEAD(&fonts, font_next); fp = kmem_zalloc(sizeof (*fp), KM_SLEEP); bd = kmem_zalloc(sizeof (*bd), KM_SLEEP); fd = kmem_zalloc(sizeof (*fd), KM_SLEEP); fp->font_name = NULL; fp->font_flags = FONT_BOOT; fp->font_data = bd; bd->width = fl->font_data->width; bd->height = fl->font_data->height; bd->uncompressed_size = fl->font_data->uncompressed_size; bd->font = fd; tmp = fl->font_data->font; fd->vf_width = tmp->vf_width; fd->vf_height = tmp->vf_height; for (i = 0; i < VFNT_MAPS; i++) { if (tmp->vf_map_count[i] == 0) continue; fd->vf_map_count[i] = tmp->vf_map_count[i]; fd->vf_map[i] = kmem_alloc(fd->vf_map_count[i] * sizeof (*fd->vf_map[i]), KM_SLEEP); bcopy(tmp->vf_map[i], fd->vf_map[i], fd->vf_map_count[i] * sizeof (*fd->vf_map[i])); } fd->vf_bytes = kmem_alloc(bd->uncompressed_size, KM_SLEEP); bcopy(tmp->vf_bytes, fd->vf_bytes, bd->uncompressed_size); STAILQ_INSERT_HEAD(&fonts, fp, font_next); } /* * This is temporary, but absolutely necessary. If we are being * booted with a device tree created by the DevConf project's bootconf * program, then we have device information nodes that reflect * reality. At this point in time in the Solaris release schedule, the * kernel drivers aren't prepared for reality. They still depend on their * own ad-hoc interpretations of the properties created when their .conf * files were interpreted. These drivers use an "ignore-hardware-nodes" * property to prevent them from using the nodes passed up from the bootconf * device tree. * * Trying to assemble root file system drivers as we are booting from * devconf will fail if the kernel driver is basing its name_addr's on the * psuedo-node device info while the bootpath passed up from bootconf is using * reality-based name_addrs. We help the boot along in this case by * looking at the pre-bootconf bootpath and determining if we would have * successfully matched if that had been the bootpath we had chosen. * * Note that we only even perform this extra check if we've booted * using bootconf's 1275 compliant bootpath, this is the boot device, and * we're trying to match the name_addr specified in the 1275 bootpath. */ #define MAXCOMPONENTLEN 32 int x86_old_bootpath_name_addr_match(dev_info_t *cdip, char *caddr, char *naddr) { /* * There are multiple criteria to be met before we can even * consider allowing a name_addr match here. * * 1) We must have been booted such that the bootconf program * created device tree nodes and properties. This can be * determined by examining the 'bootpath' property. This * property will be a non-null string iff bootconf was * involved in the boot. * * 2) The module that we want to match must be the boot device. * * 3) The instance of the module we are thinking of letting be * our match must be ignoring hardware nodes. * * 4) The name_addr we want to match must be the name_addr * specified in the 1275 bootpath. */ static char bootdev_module[MAXCOMPONENTLEN]; static char bootdev_oldmod[MAXCOMPONENTLEN]; static char bootdev_newaddr[MAXCOMPONENTLEN]; static char bootdev_oldaddr[MAXCOMPONENTLEN]; static int quickexit; char *daddr; int dlen; char *lkupname; int rv = DDI_FAILURE; if ((ddi_getlongprop(DDI_DEV_T_ANY, cdip, DDI_PROP_DONTPASS, "devconf-addr", (caddr_t)&daddr, &dlen) == DDI_PROP_SUCCESS) && (ddi_getprop(DDI_DEV_T_ANY, cdip, DDI_PROP_DONTPASS, "ignore-hardware-nodes", -1) != -1)) { if (strcmp(daddr, caddr) == 0) { return (DDI_SUCCESS); } } if (quickexit) return (rv); if (bootdev_module[0] == '\0') { char *addrp, *eoaddrp; char *busp, *modp, *atp; char *bp1275, *bp; int bp1275len, bplen; bp1275 = bp = addrp = eoaddrp = busp = modp = atp = NULL; if (ddi_getlongprop(DDI_DEV_T_ANY, ddi_root_node(), 0, "bootpath", (caddr_t)&bp1275, &bp1275len) != DDI_PROP_SUCCESS || bp1275len <= 1) { /* * We didn't boot from bootconf so we never need to * do any special matches. */ quickexit = 1; if (bp1275) kmem_free(bp1275, bp1275len); return (rv); } if (ddi_getlongprop(DDI_DEV_T_ANY, ddi_root_node(), 0, "boot-path", (caddr_t)&bp, &bplen) != DDI_PROP_SUCCESS || bplen <= 1) { /* * No fallback position for matching. This is * certainly unexpected, but we'll handle it * just in case. */ quickexit = 1; kmem_free(bp1275, bp1275len); if (bp) kmem_free(bp, bplen); return (rv); } /* * Determine boot device module and 1275 name_addr * * bootpath assumed to be of the form /bus/module@name_addr */ if ((busp = strchr(bp1275, '/')) != NULL) { if ((modp = strchr(busp + 1, '/')) != NULL) { if ((atp = strchr(modp + 1, '@')) != NULL) { *atp = '\0'; addrp = atp + 1; if ((eoaddrp = strchr(addrp, '/')) != NULL) *eoaddrp = '\0'; } } } if (modp && addrp) { (void) strncpy(bootdev_module, modp + 1, MAXCOMPONENTLEN); bootdev_module[MAXCOMPONENTLEN - 1] = '\0'; (void) strncpy(bootdev_newaddr, addrp, MAXCOMPONENTLEN); bootdev_newaddr[MAXCOMPONENTLEN - 1] = '\0'; } else { quickexit = 1; kmem_free(bp1275, bp1275len); kmem_free(bp, bplen); return (rv); } /* * Determine fallback name_addr * * 10/3/96 - Also save fallback module name because it * might actually be different than the current module * name. E.G., ISA pnp drivers have new names. * * bootpath assumed to be of the form /bus/module@name_addr */ addrp = NULL; if ((busp = strchr(bp, '/')) != NULL) { if ((modp = strchr(busp + 1, '/')) != NULL) { if ((atp = strchr(modp + 1, '@')) != NULL) { *atp = '\0'; addrp = atp + 1; if ((eoaddrp = strchr(addrp, '/')) != NULL) *eoaddrp = '\0'; } } } if (modp && addrp) { (void) strncpy(bootdev_oldmod, modp + 1, MAXCOMPONENTLEN); bootdev_module[MAXCOMPONENTLEN - 1] = '\0'; (void) strncpy(bootdev_oldaddr, addrp, MAXCOMPONENTLEN); bootdev_oldaddr[MAXCOMPONENTLEN - 1] = '\0'; } /* Free up the bootpath storage now that we're done with it. */ kmem_free(bp1275, bp1275len); kmem_free(bp, bplen); if (bootdev_oldaddr[0] == '\0') { quickexit = 1; return (rv); } } if (((lkupname = ddi_get_name(cdip)) != NULL) && (strcmp(bootdev_module, lkupname) == 0 || strcmp(bootdev_oldmod, lkupname) == 0) && ((ddi_getprop(DDI_DEV_T_ANY, cdip, DDI_PROP_DONTPASS, "ignore-hardware-nodes", -1) != -1) || ignore_hardware_nodes) && strcmp(bootdev_newaddr, caddr) == 0 && strcmp(bootdev_oldaddr, naddr) == 0) { rv = DDI_SUCCESS; } return (rv); } /* * Perform a copy from a memory mapped device (whose devinfo pointer is devi) * separately mapped at devaddr in the kernel to a kernel buffer at kaddr. */ /*ARGSUSED*/ int e_ddi_copyfromdev(dev_info_t *devi, off_t off, const void *devaddr, void *kaddr, size_t len) { bcopy(devaddr, kaddr, len); return (0); } /* * Perform a copy to a memory mapped device (whose devinfo pointer is devi) * separately mapped at devaddr in the kernel from a kernel buffer at kaddr. */ /*ARGSUSED*/ int e_ddi_copytodev(dev_info_t *devi, off_t off, const void *kaddr, void *devaddr, size_t len) { bcopy(kaddr, devaddr, len); return (0); } static int poke_mem(peekpoke_ctlops_t *in_args) { int err; on_trap_data_t otd; /* Set up protected environment. */ if (!on_trap(&otd, OT_DATA_ACCESS)) { err = DDI_SUCCESS; switch (in_args->size) { case sizeof (uint8_t): *(uint8_t *)(in_args->dev_addr) = *(uint8_t *)in_args->host_addr; break; case sizeof (uint16_t): *(uint16_t *)(in_args->dev_addr) = *(uint16_t *)in_args->host_addr; break; case sizeof (uint32_t): *(uint32_t *)(in_args->dev_addr) = *(uint32_t *)in_args->host_addr; break; case sizeof (uint64_t): *(uint64_t *)(in_args->dev_addr) = *(uint64_t *)in_args->host_addr; break; default: err = DDI_FAILURE; break; } } else { err = DDI_FAILURE; } /* Take down protected environment. */ no_trap(); return (err); } static int peek_mem(peekpoke_ctlops_t *in_args) { int err; on_trap_data_t otd; if (!on_trap(&otd, OT_DATA_ACCESS)) { err = DDI_SUCCESS; switch (in_args->size) { case sizeof (uint8_t): *(uint8_t *)in_args->host_addr = *(uint8_t *)in_args->dev_addr; break; case sizeof (uint16_t): *(uint16_t *)in_args->host_addr = *(uint16_t *)in_args->dev_addr; break; case sizeof (uint32_t): *(uint32_t *)in_args->host_addr = *(uint32_t *)in_args->dev_addr; break; case sizeof (uint64_t): *(uint64_t *)in_args->host_addr = *(uint64_t *)in_args->dev_addr; break; default: err = DDI_FAILURE; break; } } else { err = DDI_FAILURE; } no_trap(); return (err); } /* * This is called only to process peek/poke when the DIP is NULL. * Assume that this is for memory, as nexi take care of device safe accesses. */ int peekpoke_mem(ddi_ctl_enum_t cmd, peekpoke_ctlops_t *in_args) { return (cmd == DDI_CTLOPS_PEEK ? peek_mem(in_args) : poke_mem(in_args)); } /* * we've just done a cautious put/get. Check if it was successful by * calling pci_ereport_post() on all puts and for any gets that return -1 */ static int pci_peekpoke_check_fma(dev_info_t *dip, void *arg, ddi_ctl_enum_t ctlop, void (*scan)(dev_info_t *, ddi_fm_error_t *)) { int rval = DDI_SUCCESS; peekpoke_ctlops_t *in_args = (peekpoke_ctlops_t *)arg; ddi_fm_error_t de; ddi_acc_impl_t *hp = (ddi_acc_impl_t *)in_args->handle; ddi_acc_hdl_t *hdlp = (ddi_acc_hdl_t *)in_args->handle; int check_err = 0; int repcount = in_args->repcount; if (ctlop == DDI_CTLOPS_POKE && hdlp->ah_acc.devacc_attr_access != DDI_CAUTIOUS_ACC) return (DDI_SUCCESS); if (ctlop == DDI_CTLOPS_PEEK && hdlp->ah_acc.devacc_attr_access != DDI_CAUTIOUS_ACC) { for (; repcount; repcount--) { switch (in_args->size) { case sizeof (uint8_t): if (*(uint8_t *)in_args->host_addr == 0xff) check_err = 1; break; case sizeof (uint16_t): if (*(uint16_t *)in_args->host_addr == 0xffff) check_err = 1; break; case sizeof (uint32_t): if (*(uint32_t *)in_args->host_addr == 0xffffffff) check_err = 1; break; case sizeof (uint64_t): if (*(uint64_t *)in_args->host_addr == 0xffffffffffffffff) check_err = 1; break; } } if (check_err == 0) return (DDI_SUCCESS); } /* * for a cautious put or get or a non-cautious get that returned -1 call * io framework to see if there really was an error */ bzero(&de, sizeof (ddi_fm_error_t)); de.fme_version = DDI_FME_VERSION; de.fme_ena = fm_ena_generate(0, FM_ENA_FMT1); if (hdlp->ah_acc.devacc_attr_access == DDI_CAUTIOUS_ACC) { de.fme_flag = DDI_FM_ERR_EXPECTED; de.fme_acc_handle = in_args->handle; } else if (hdlp->ah_acc.devacc_attr_access == DDI_DEFAULT_ACC) { /* * We only get here with DDI_DEFAULT_ACC for config space gets. * Non-hardened drivers may be probing the hardware and * expecting -1 returned. So need to treat errors on * DDI_DEFAULT_ACC as DDI_FM_ERR_EXPECTED. */ de.fme_flag = DDI_FM_ERR_EXPECTED; de.fme_acc_handle = in_args->handle; } else { /* * Hardened driver doing protected accesses shouldn't * get errors unless there's a hardware problem. Treat * as nonfatal if there's an error, but set UNEXPECTED * so we raise ereports on any errors and potentially * fault the device */ de.fme_flag = DDI_FM_ERR_UNEXPECTED; } (void) scan(dip, &de); if (hdlp->ah_acc.devacc_attr_access != DDI_DEFAULT_ACC && de.fme_status != DDI_FM_OK) { ndi_err_t *errp = (ndi_err_t *)hp->ahi_err; rval = DDI_FAILURE; errp->err_ena = de.fme_ena; errp->err_expected = de.fme_flag; errp->err_status = DDI_FM_NONFATAL; } return (rval); } /* * pci_peekpoke_check_nofma() is for when an error occurs on a register access * during pci_ereport_post(). We can't call pci_ereport_post() again or we'd * recurse, so assume all puts are OK and gets have failed if they return -1 */ static int pci_peekpoke_check_nofma(void *arg, ddi_ctl_enum_t ctlop) { int rval = DDI_SUCCESS; peekpoke_ctlops_t *in_args = (peekpoke_ctlops_t *)arg; ddi_acc_impl_t *hp = (ddi_acc_impl_t *)in_args->handle; ddi_acc_hdl_t *hdlp = (ddi_acc_hdl_t *)in_args->handle; int repcount = in_args->repcount; if (ctlop == DDI_CTLOPS_POKE) return (rval); for (; repcount; repcount--) { switch (in_args->size) { case sizeof (uint8_t): if (*(uint8_t *)in_args->host_addr == 0xff) rval = DDI_FAILURE; break; case sizeof (uint16_t): if (*(uint16_t *)in_args->host_addr == 0xffff) rval = DDI_FAILURE; break; case sizeof (uint32_t): if (*(uint32_t *)in_args->host_addr == 0xffffffff) rval = DDI_FAILURE; break; case sizeof (uint64_t): if (*(uint64_t *)in_args->host_addr == 0xffffffffffffffff) rval = DDI_FAILURE; break; } } if (hdlp->ah_acc.devacc_attr_access != DDI_DEFAULT_ACC && rval == DDI_FAILURE) { ndi_err_t *errp = (ndi_err_t *)hp->ahi_err; errp->err_ena = fm_ena_generate(0, FM_ENA_FMT1); errp->err_expected = DDI_FM_ERR_UNEXPECTED; errp->err_status = DDI_FM_NONFATAL; } return (rval); } int pci_peekpoke_check(dev_info_t *dip, dev_info_t *rdip, ddi_ctl_enum_t ctlop, void *arg, void *result, int (*handler)(dev_info_t *, dev_info_t *, ddi_ctl_enum_t, void *, void *), kmutex_t *err_mutexp, kmutex_t *peek_poke_mutexp, void (*scan)(dev_info_t *, ddi_fm_error_t *)) { int rval; peekpoke_ctlops_t *in_args = (peekpoke_ctlops_t *)arg; ddi_acc_impl_t *hp = (ddi_acc_impl_t *)in_args->handle; /* * this function only supports cautious accesses, not peeks/pokes * which don't have a handle */ if (hp == NULL) return (DDI_FAILURE); if (hp->ahi_acc_attr & DDI_ACCATTR_CONFIG_SPACE) { if (!mutex_tryenter(err_mutexp)) { /* * As this may be a recursive call from within * pci_ereport_post() we can't wait for the mutexes. * Fortunately we know someone is already calling * pci_ereport_post() which will handle the error bits * for us, and as this is a config space access we can * just do the access and check return value for -1 * using pci_peekpoke_check_nofma(). */ rval = handler(dip, rdip, ctlop, arg, result); if (rval == DDI_SUCCESS) rval = pci_peekpoke_check_nofma(arg, ctlop); return (rval); } /* * This can't be a recursive call. Drop the err_mutex and get * both mutexes in the right order. If an error hasn't already * been detected by the ontrap code, use pci_peekpoke_check_fma * which will call pci_ereport_post() to check error status. */ mutex_exit(err_mutexp); } mutex_enter(peek_poke_mutexp); rval = handler(dip, rdip, ctlop, arg, result); if (rval == DDI_SUCCESS) { mutex_enter(err_mutexp); rval = pci_peekpoke_check_fma(dip, arg, ctlop, scan); mutex_exit(err_mutexp); } mutex_exit(peek_poke_mutexp); return (rval); } void impl_setup_ddi(void) { #if !defined(__xpv) extern void startup_bios_disk(void); extern int post_fastreboot; #endif dev_info_t *xdip, *isa_dip; rd_existing_t rd_mem_prop; int err; ndi_devi_alloc_sleep(ddi_root_node(), "ramdisk", (pnode_t)DEVI_SID_NODEID, &xdip); (void) BOP_GETPROP(bootops, "ramdisk_start", (void *)&ramdisk_start); (void) BOP_GETPROP(bootops, "ramdisk_end", (void *)&ramdisk_end); #ifdef __xpv ramdisk_start -= ONE_GIG; ramdisk_end -= ONE_GIG; #endif rd_mem_prop.phys = ramdisk_start; rd_mem_prop.size = ramdisk_end - ramdisk_start + 1; (void) ndi_prop_update_byte_array(DDI_DEV_T_NONE, xdip, RD_EXISTING_PROP_NAME, (uchar_t *)&rd_mem_prop, sizeof (rd_mem_prop)); err = ndi_devi_bind_driver(xdip, 0); ASSERT(err == 0); /* isa node */ if (pseudo_isa) { ndi_devi_alloc_sleep(ddi_root_node(), "isa", (pnode_t)DEVI_SID_NODEID, &isa_dip); (void) ndi_prop_update_string(DDI_DEV_T_NONE, isa_dip, "device_type", "isa"); (void) ndi_prop_update_string(DDI_DEV_T_NONE, isa_dip, "bus-type", "isa"); (void) ndi_devi_bind_driver(isa_dip, 0); } /* * Read in the properties from the boot. */ get_boot_properties(); /* not framebuffer should be enumerated, if present */ get_vga_properties(); /* Copy console font if provided by boot. */ get_console_font(); /* * Check for administratively disabled drivers. */ check_driver_disable(); #if !defined(__xpv) if (!post_fastreboot && BOP_GETPROPLEN(bootops, "efi-systab") < 0) startup_bios_disk(); #endif /* do bus dependent probes. */ impl_bus_initialprobe(); } dev_t getrootdev(void) { /* * Usually rootfs.bo_name is initialized by the * the bootpath property from bootenv.rc, but * defaults to "/ramdisk:a" otherwise. */ return (ddi_pathname_to_dev_t(rootfs.bo_name)); } static struct bus_probe { struct bus_probe *next; void (*probe)(int); } *bus_probes; void impl_bus_add_probe(void (*func)(int)) { struct bus_probe *probe; struct bus_probe *lastprobe = NULL; probe = kmem_alloc(sizeof (*probe), KM_SLEEP); probe->probe = func; probe->next = NULL; if (!bus_probes) { bus_probes = probe; return; } lastprobe = bus_probes; while (lastprobe->next) lastprobe = lastprobe->next; lastprobe->next = probe; } /*ARGSUSED*/ void impl_bus_delete_probe(void (*func)(int)) { struct bus_probe *prev = NULL; struct bus_probe *probe = bus_probes; while (probe) { if (probe->probe == func) break; prev = probe; probe = probe->next; } if (probe == NULL) return; if (prev) prev->next = probe->next; else bus_probes = probe->next; kmem_free(probe, sizeof (struct bus_probe)); } /* * impl_bus_initialprobe * Modload the prom simulator, then let it probe to verify existence * and type of PCI support. */ static void impl_bus_initialprobe(void) { struct bus_probe *probe; /* load modules to install bus probes */ #if defined(__xpv) if (DOMAIN_IS_INITDOMAIN(xen_info)) { if (modload("misc", "pci_autoconfig") < 0) { panic("failed to load misc/pci_autoconfig"); } if (modload("drv", "isa") < 0) panic("failed to load drv/isa"); } (void) modload("misc", "xpv_autoconfig"); #else if (modload("misc", "pci_autoconfig") < 0) { panic("failed to load misc/pci_autoconfig"); } (void) modload("misc", "acpidev"); if (modload("drv", "isa") < 0) panic("failed to load drv/isa"); #endif probe = bus_probes; while (probe) { /* run the probe functions */ (*probe->probe)(0); probe = probe->next; } } /* * impl_bus_reprobe * Reprogram devices not set up by firmware. */ static void impl_bus_reprobe(void) { struct bus_probe *probe; probe = bus_probes; while (probe) { /* run the probe function */ (*probe->probe)(1); probe = probe->next; } } /* * The following functions ready a cautious request to go up to the nexus * driver. It is up to the nexus driver to decide how to process the request. * It may choose to call i_ddi_do_caut_get/put in this file, or do it * differently. */ static void i_ddi_caut_getput_ctlops(ddi_acc_impl_t *hp, uint64_t host_addr, uint64_t dev_addr, size_t size, size_t repcount, uint_t flags, ddi_ctl_enum_t cmd) { peekpoke_ctlops_t cautacc_ctlops_arg; cautacc_ctlops_arg.size = size; cautacc_ctlops_arg.dev_addr = dev_addr; cautacc_ctlops_arg.host_addr = host_addr; cautacc_ctlops_arg.handle = (ddi_acc_handle_t)hp; cautacc_ctlops_arg.repcount = repcount; cautacc_ctlops_arg.flags = flags; (void) ddi_ctlops(hp->ahi_common.ah_dip, hp->ahi_common.ah_dip, cmd, &cautacc_ctlops_arg, NULL); } uint8_t i_ddi_caut_get8(ddi_acc_impl_t *hp, uint8_t *addr) { uint8_t value; i_ddi_caut_getput_ctlops(hp, (uintptr_t)&value, (uintptr_t)addr, sizeof (uint8_t), 1, 0, DDI_CTLOPS_PEEK); return (value); } uint16_t i_ddi_caut_get16(ddi_acc_impl_t *hp, uint16_t *addr) { uint16_t value; i_ddi_caut_getput_ctlops(hp, (uintptr_t)&value, (uintptr_t)addr, sizeof (uint16_t), 1, 0, DDI_CTLOPS_PEEK); return (value); } uint32_t i_ddi_caut_get32(ddi_acc_impl_t *hp, uint32_t *addr) { uint32_t value; i_ddi_caut_getput_ctlops(hp, (uintptr_t)&value, (uintptr_t)addr, sizeof (uint32_t), 1, 0, DDI_CTLOPS_PEEK); return (value); } uint64_t i_ddi_caut_get64(ddi_acc_impl_t *hp, uint64_t *addr) { uint64_t value; i_ddi_caut_getput_ctlops(hp, (uintptr_t)&value, (uintptr_t)addr, sizeof (uint64_t), 1, 0, DDI_CTLOPS_PEEK); return (value); } void i_ddi_caut_put8(ddi_acc_impl_t *hp, uint8_t *addr, uint8_t value) { i_ddi_caut_getput_ctlops(hp, (uintptr_t)&value, (uintptr_t)addr, sizeof (uint8_t), 1, 0, DDI_CTLOPS_POKE); } void i_ddi_caut_put16(ddi_acc_impl_t *hp, uint16_t *addr, uint16_t value) { i_ddi_caut_getput_ctlops(hp, (uintptr_t)&value, (uintptr_t)addr, sizeof (uint16_t), 1, 0, DDI_CTLOPS_POKE); } void i_ddi_caut_put32(ddi_acc_impl_t *hp, uint32_t *addr, uint32_t value) { i_ddi_caut_getput_ctlops(hp, (uintptr_t)&value, (uintptr_t)addr, sizeof (uint32_t), 1, 0, DDI_CTLOPS_POKE); } void i_ddi_caut_put64(ddi_acc_impl_t *hp, uint64_t *addr, uint64_t value) { i_ddi_caut_getput_ctlops(hp, (uintptr_t)&value, (uintptr_t)addr, sizeof (uint64_t), 1, 0, DDI_CTLOPS_POKE); } void i_ddi_caut_rep_get8(ddi_acc_impl_t *hp, uint8_t *host_addr, uint8_t *dev_addr, size_t repcount, uint_t flags) { i_ddi_caut_getput_ctlops(hp, (uintptr_t)host_addr, (uintptr_t)dev_addr, sizeof (uint8_t), repcount, flags, DDI_CTLOPS_PEEK); } void i_ddi_caut_rep_get16(ddi_acc_impl_t *hp, uint16_t *host_addr, uint16_t *dev_addr, size_t repcount, uint_t flags) { i_ddi_caut_getput_ctlops(hp, (uintptr_t)host_addr, (uintptr_t)dev_addr, sizeof (uint16_t), repcount, flags, DDI_CTLOPS_PEEK); } void i_ddi_caut_rep_get32(ddi_acc_impl_t *hp, uint32_t *host_addr, uint32_t *dev_addr, size_t repcount, uint_t flags) { i_ddi_caut_getput_ctlops(hp, (uintptr_t)host_addr, (uintptr_t)dev_addr, sizeof (uint32_t), repcount, flags, DDI_CTLOPS_PEEK); } void i_ddi_caut_rep_get64(ddi_acc_impl_t *hp, uint64_t *host_addr, uint64_t *dev_addr, size_t repcount, uint_t flags) { i_ddi_caut_getput_ctlops(hp, (uintptr_t)host_addr, (uintptr_t)dev_addr, sizeof (uint64_t), repcount, flags, DDI_CTLOPS_PEEK); } void i_ddi_caut_rep_put8(ddi_acc_impl_t *hp, uint8_t *host_addr, uint8_t *dev_addr, size_t repcount, uint_t flags) { i_ddi_caut_getput_ctlops(hp, (uintptr_t)host_addr, (uintptr_t)dev_addr, sizeof (uint8_t), repcount, flags, DDI_CTLOPS_POKE); } void i_ddi_caut_rep_put16(ddi_acc_impl_t *hp, uint16_t *host_addr, uint16_t *dev_addr, size_t repcount, uint_t flags) { i_ddi_caut_getput_ctlops(hp, (uintptr_t)host_addr, (uintptr_t)dev_addr, sizeof (uint16_t), repcount, flags, DDI_CTLOPS_POKE); } void i_ddi_caut_rep_put32(ddi_acc_impl_t *hp, uint32_t *host_addr, uint32_t *dev_addr, size_t repcount, uint_t flags) { i_ddi_caut_getput_ctlops(hp, (uintptr_t)host_addr, (uintptr_t)dev_addr, sizeof (uint32_t), repcount, flags, DDI_CTLOPS_POKE); } void i_ddi_caut_rep_put64(ddi_acc_impl_t *hp, uint64_t *host_addr, uint64_t *dev_addr, size_t repcount, uint_t flags) { i_ddi_caut_getput_ctlops(hp, (uintptr_t)host_addr, (uintptr_t)dev_addr, sizeof (uint64_t), repcount, flags, DDI_CTLOPS_POKE); } boolean_t i_ddi_copybuf_required(ddi_dma_attr_t *attrp) { uint64_t hi_pa; hi_pa = ((uint64_t)physmax + 1ull) << PAGESHIFT; if (attrp->dma_attr_addr_hi < hi_pa) { return (B_TRUE); } return (B_FALSE); } size_t i_ddi_copybuf_size() { return (dma_max_copybuf_size); } /* * i_ddi_dma_max() * returns the maximum DMA size which can be performed in a single DMA * window taking into account the devices DMA contraints (attrp), the * maximum copy buffer size (if applicable), and the worse case buffer * fragmentation. */ /*ARGSUSED*/ uint32_t i_ddi_dma_max(dev_info_t *dip, ddi_dma_attr_t *attrp) { uint64_t maxxfer; /* * take the min of maxxfer and the the worse case fragementation * (e.g. every cookie <= 1 page) */ maxxfer = MIN(attrp->dma_attr_maxxfer, ((uint64_t)(attrp->dma_attr_sgllen - 1) << PAGESHIFT)); /* * If the DMA engine can't reach all off memory, we also need to take * the max size of the copybuf into consideration. */ if (i_ddi_copybuf_required(attrp)) { maxxfer = MIN(i_ddi_copybuf_size(), maxxfer); } /* * we only return a 32-bit value. Make sure it's not -1. Round to a * page so it won't be mistaken for an error value during debug. */ if (maxxfer >= 0xFFFFFFFF) { maxxfer = 0xFFFFF000; } /* * make sure the value we return is a whole multiple of the * granlarity. */ if (attrp->dma_attr_granular > 1) { maxxfer = maxxfer - (maxxfer % attrp->dma_attr_granular); } return ((uint32_t)maxxfer); } pfn_t i_ddi_paddr_to_pfn(paddr_t paddr) { pfn_t pfn; #ifdef __xpv if (DOMAIN_IS_INITDOMAIN(xen_info)) { pfn = xen_assign_pfn(mmu_btop(paddr)); } else { pfn = mmu_btop(paddr); } #else pfn = mmu_btop(paddr); #endif return (pfn); } /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2009 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ /* * Copyright (c) 2011, Joyent, Inc. All rights reserved. */ #include #include #include #include #include #include #include #include #include typedef struct dtrace_invop_hdlr { int (*dtih_func)(uintptr_t, uintptr_t *, uintptr_t); struct dtrace_invop_hdlr *dtih_next; } dtrace_invop_hdlr_t; dtrace_invop_hdlr_t *dtrace_invop_hdlr; int dtrace_invop(uintptr_t addr, uintptr_t *stack, uintptr_t eax) { dtrace_invop_hdlr_t *hdlr; int rval; for (hdlr = dtrace_invop_hdlr; hdlr != NULL; hdlr = hdlr->dtih_next) { if ((rval = hdlr->dtih_func(addr, stack, eax)) != 0) return (rval); } return (0); } void dtrace_invop_add(int (*func)(uintptr_t, uintptr_t *, uintptr_t)) { dtrace_invop_hdlr_t *hdlr; hdlr = kmem_alloc(sizeof (dtrace_invop_hdlr_t), KM_SLEEP); hdlr->dtih_func = func; hdlr->dtih_next = dtrace_invop_hdlr; dtrace_invop_hdlr = hdlr; } void dtrace_invop_remove(int (*func)(uintptr_t, uintptr_t *, uintptr_t)) { dtrace_invop_hdlr_t *hdlr = dtrace_invop_hdlr, *prev = NULL; for (;;) { if (hdlr == NULL) panic("attempt to remove non-existent invop handler"); if (hdlr->dtih_func == func) break; prev = hdlr; hdlr = hdlr->dtih_next; } if (prev == NULL) { ASSERT(dtrace_invop_hdlr == hdlr); dtrace_invop_hdlr = hdlr->dtih_next; } else { ASSERT(dtrace_invop_hdlr != hdlr); prev->dtih_next = hdlr->dtih_next; } kmem_free(hdlr, sizeof (dtrace_invop_hdlr_t)); } int dtrace_getipl(void) { return (CPU->cpu_pri); } /*ARGSUSED*/ void dtrace_toxic_ranges(void (*func)(uintptr_t base, uintptr_t limit)) { extern uintptr_t toxic_addr; extern size_t toxic_size; (*func)(0, _userlimit); if (hole_end > hole_start) (*func)(hole_start, hole_end); (*func)(toxic_addr, toxic_addr + toxic_size); (*func)(0, _userlimit); } static int dtrace_xcall_func(xc_arg_t arg1, xc_arg_t arg2, xc_arg_t arg3 __unused) { dtrace_xcall_t func = (dtrace_xcall_t)arg1; (*func)((void*)arg2); return (0); } /*ARGSUSED*/ void dtrace_xcall(processorid_t cpu, dtrace_xcall_t func, void *arg) { cpuset_t set; CPUSET_ZERO(set); if (cpu == DTRACE_CPUALL) { CPUSET_ALL(set); } else { CPUSET_ADD(set, cpu); } kpreempt_disable(); xc_sync((xc_arg_t)func, (xc_arg_t)arg, 0, CPUSET2BV(set), dtrace_xcall_func); kpreempt_enable(); } void dtrace_sync_func(void) {} void dtrace_sync(void) { dtrace_xcall(DTRACE_CPUALL, (dtrace_xcall_t)dtrace_sync_func, NULL); } int (*dtrace_pid_probe_ptr)(struct regs *); int (*dtrace_return_probe_ptr)(struct regs *); void dtrace_user_probe(struct regs *rp, caddr_t addr, processorid_t cpuid) { krwlock_t *rwp; proc_t *p = curproc; extern void trap(struct regs *, caddr_t, processorid_t); if (USERMODE(rp->r_cs) || (rp->r_ps & PS_VM)) { if (curthread->t_cred != p->p_cred) { cred_t *oldcred = curthread->t_cred; /* * DTrace accesses t_cred in probe context. t_cred * must always be either NULL, or point to a valid, * allocated cred structure. */ curthread->t_cred = crgetcred(); crfree(oldcred); } } if (rp->r_trapno == T_DTRACE_RET) { uint8_t step = curthread->t_dtrace_step; uint8_t ret = curthread->t_dtrace_ret; uintptr_t npc = curthread->t_dtrace_npc; if (curthread->t_dtrace_ast) { aston(curthread); curthread->t_sig_check = 1; } /* * Clear all user tracing flags. */ curthread->t_dtrace_ft = 0; /* * If we weren't expecting to take a return probe trap, kill * the process as though it had just executed an unassigned * trap instruction. */ if (step == 0) { tsignal(curthread, SIGILL); return; } /* * If we hit this trap unrelated to a return probe, we're * just here to reset the AST flag since we deferred a signal * until after we logically single-stepped the instruction we * copied out. */ if (ret == 0) { rp->r_pc = npc; return; } /* * We need to wait until after we've called the * dtrace_return_probe_ptr function pointer to set %pc. */ rwp = &CPU->cpu_ft_lock; rw_enter(rwp, RW_READER); if (dtrace_return_probe_ptr != NULL) (void) (*dtrace_return_probe_ptr)(rp); rw_exit(rwp); rp->r_pc = npc; } else if (rp->r_trapno == T_BPTFLT) { uint8_t instr, instr2; caddr_t linearpc; rwp = &CPU->cpu_ft_lock; /* * The DTrace fasttrap provider uses the breakpoint trap * (int 3). We let DTrace take the first crack at handling * this trap; if it's not a probe that DTrace knowns about, * we call into the trap() routine to handle it like a * breakpoint placed by a conventional debugger. */ rw_enter(rwp, RW_READER); if (dtrace_pid_probe_ptr != NULL && (*dtrace_pid_probe_ptr)(rp) == 0) { rw_exit(rwp); return; } rw_exit(rwp); if (dtrace_linear_pc(rp, p, &linearpc) != 0) { trap(rp, addr, cpuid); return; } /* * If the instruction that caused the breakpoint trap doesn't * look like an int 3 anymore, it may be that this tracepoint * was removed just after the user thread executed it. In * that case, return to user land to retry the instuction. * Note that we assume the length of the instruction to retry * is 1 byte because that's the length of FASTTRAP_INSTR. * We check for r_pc > 0 and > 2 so that we don't have to * deal with segment wraparound. */ if (rp->r_pc > 0 && fuword8(linearpc - 1, &instr) == 0 && instr != FASTTRAP_INSTR && (instr != 3 || (rp->r_pc >= 2 && (fuword8(linearpc - 2, &instr2) != 0 || instr2 != 0xCD)))) { rp->r_pc--; return; } trap(rp, addr, cpuid); } else { trap(rp, addr, cpuid); } } void dtrace_safe_synchronous_signal(void) { kthread_t *t = curthread; struct regs *rp = lwptoregs(ttolwp(t)); size_t isz = t->t_dtrace_npc - t->t_dtrace_pc; ASSERT(t->t_dtrace_on); /* * If we're not in the range of scratch addresses, we're not actually * tracing user instructions so turn off the flags. If the instruction * we copied out caused a synchonous trap, reset the pc back to its * original value and turn off the flags. */ if (rp->r_pc < t->t_dtrace_scrpc || rp->r_pc > t->t_dtrace_astpc + isz) { t->t_dtrace_ft = 0; } else if (rp->r_pc == t->t_dtrace_scrpc || rp->r_pc == t->t_dtrace_astpc) { rp->r_pc = t->t_dtrace_pc; t->t_dtrace_ft = 0; } } int dtrace_safe_defer_signal(void) { kthread_t *t = curthread; struct regs *rp = lwptoregs(ttolwp(t)); size_t isz = t->t_dtrace_npc - t->t_dtrace_pc; ASSERT(t->t_dtrace_on); /* * If we're not in the range of scratch addresses, we're not actually * tracing user instructions so turn off the flags. */ if (rp->r_pc < t->t_dtrace_scrpc || rp->r_pc > t->t_dtrace_astpc + isz) { t->t_dtrace_ft = 0; return (0); } /* * If we have executed the original instruction, but we have performed * neither the jmp back to t->t_dtrace_npc nor the clean up of any * registers used to emulate %rip-relative instructions in 64-bit mode, * we'll save ourselves some effort by doing that here and taking the * signal right away. We detect this condition by seeing if the program * counter is the range [scrpc + isz, astpc). */ if (rp->r_pc >= t->t_dtrace_scrpc + isz && rp->r_pc < t->t_dtrace_astpc) { /* * If there is a scratch register and we're on the * instruction immediately after the modified instruction, * restore the value of that scratch register. */ if (t->t_dtrace_reg != 0 && rp->r_pc == t->t_dtrace_scrpc + isz) { switch (t->t_dtrace_reg) { case REG_RAX: rp->r_rax = t->t_dtrace_regv; break; case REG_RCX: rp->r_rcx = t->t_dtrace_regv; break; case REG_R8: rp->r_r8 = t->t_dtrace_regv; break; case REG_R9: rp->r_r9 = t->t_dtrace_regv; break; } } rp->r_pc = t->t_dtrace_npc; t->t_dtrace_ft = 0; return (0); } /* * Otherwise, make sure we'll return to the kernel after executing * the copied out instruction and defer the signal. */ if (!t->t_dtrace_step) { ASSERT(rp->r_pc < t->t_dtrace_astpc); rp->r_pc += t->t_dtrace_astpc - t->t_dtrace_scrpc; t->t_dtrace_step = 1; } t->t_dtrace_ast = 1; return (1); } /* * Additional artificial frames for the machine type. For i86pc, we're already * accounted for, so return 0. On the hypervisor, we have an additional frame * (xen_callback_handler). */ int dtrace_mach_aframes(void) { #ifdef __xpv return (1); #else return (0); #endif } /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License, Version 1.0 only * (the "License"). You may not use this file except in compliance * with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright (c) 1990-1998, by Sun Microsystems, Inc. * All rights reserved. */ #include #include #include #include #include #include #include /*ARGSUSED*/ unsigned long dvma_pagesize(dev_info_t *dip) { return (0); } /*ARGSUSED*/ int dvma_reserve(dev_info_t *dip, ddi_dma_lim_t *limp, u_int pages, ddi_dma_handle_t *handlep) { return (DDI_DMA_NORESOURCES); } /*ARGSUSED*/ void dvma_release(ddi_dma_handle_t h) { } /*ARGSUSED*/ void dvma_kaddr_load(ddi_dma_handle_t h, caddr_t a, u_int len, u_int index, ddi_dma_cookie_t *cp) { } /*ARGSUSED*/ void dvma_unload(ddi_dma_handle_t h, u_int objindex, u_int type) { } /*ARGSUSED*/ void dvma_sync(ddi_dma_handle_t h, u_int objindex, u_int type) { } /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2010 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. * * Copyright (c) 2010, Intel Corporation. * All rights reserved. * * Copyright 2020 Joyent, Inc. * Copyright 2024 Oxide Computer Company */ /* * This file contains the functionality that mimics the boot operations * on SPARC systems or the old boot.bin/multiboot programs on x86 systems. * The x86 kernel now does everything on its own. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef __xpv #include #include #endif #include #include #include #include #include #include #include #include #include /* For DDI prop types */ static int have_console = 0; /* set once primitive console is initialized */ static char *boot_args = ""; /* * Debugging macros */ static uint_t kbm_debug = 0; #define DBG_MSG(s) { if (kbm_debug) bop_printf(NULL, "%s", s); } #define DBG(x) { if (kbm_debug) \ bop_printf(NULL, "%s is %" PRIx64 "\n", #x, (uint64_t)(x)); \ } #define PUT_STRING(s) { \ char *cp; \ for (cp = (s); *cp; ++cp) \ bcons_putchar(*cp); \ } /* callback to boot_fb to set shadow frame buffer */ extern void boot_fb_shadow_init(bootops_t *); bootops_t bootop; /* simple bootops we'll pass on to kernel */ struct bsys_mem bm; /* * Boot info from "glue" code in low memory. xbootp is used by: * do_bop_phys_alloc(), do_bsys_alloc() and read_bootenvrc(). */ static struct xboot_info *xbootp; static uintptr_t next_virt; /* next available virtual address */ static paddr_t next_phys; /* next available physical address from dboot */ static paddr_t high_phys = -(paddr_t)1; /* last used physical address */ /* * buffer for vsnprintf for console I/O */ #define BUFFERSIZE 512 static char buffer[BUFFERSIZE]; /* * stuff to store/report/manipulate boot property settings. */ typedef struct bootprop { struct bootprop *bp_next; char *bp_name; int bp_flags; /* DDI prop type */ uint_t bp_vlen; /* 0 for boolean */ char *bp_value; } bootprop_t; static bootprop_t *bprops = NULL; static char *curr_page = NULL; /* ptr to avail bprop memory */ static int curr_space = 0; /* amount of memory at curr_page */ #ifdef __xpv extern start_info_t *xen_info; extern shared_info_t *HYPERVISOR_shared_info; #endif /* * some allocator statistics */ static ulong_t total_bop_alloc_scratch = 0; static ulong_t total_bop_alloc_kernel = 0; static void build_firmware_properties(struct xboot_info *); static int early_allocation = 1; int force_fastreboot = 0; volatile int fastreboot_onpanic = 0; int post_fastreboot = 0; #ifdef __xpv volatile int fastreboot_capable = 0; #else volatile int fastreboot_capable = 1; #endif /* * Information saved from current boot for fast reboot. * If the information size exceeds what we have allocated, fast reboot * will not be supported. */ multiboot_info_t saved_mbi; mb_memory_map_t saved_mmap[FASTBOOT_SAVED_MMAP_COUNT]; uint8_t saved_drives[FASTBOOT_SAVED_DRIVES_SIZE]; char saved_cmdline[FASTBOOT_SAVED_CMDLINE_LEN]; int saved_cmdline_len = 0; size_t saved_file_size[FASTBOOT_MAX_FILES_MAP]; /* * Turn off fastreboot_onpanic to avoid panic loop. */ char fastreboot_onpanic_cmdline[FASTBOOT_SAVED_CMDLINE_LEN]; static const char fastreboot_onpanic_args[] = " -B fastreboot_onpanic=0"; /* * Pointers to where System Resource Affinity Table (SRAT), System Locality * Information Table (SLIT) and Maximum System Capability Table (MSCT) * are mapped into virtual memory */ ACPI_TABLE_SRAT *srat_ptr = NULL; ACPI_TABLE_SLIT *slit_ptr = NULL; ACPI_TABLE_MSCT *msct_ptr = NULL; /* * Arbitrary limit on number of localities we handle; if * this limit is raised to more than UINT16_MAX, make sure * process_slit() knows how to handle it. */ #define SLIT_LOCALITIES_MAX (4096) #define SLIT_NUM_PROPNAME "acpi-slit-localities" #define SLIT_PROPNAME "acpi-slit" /* * Allocate aligned physical memory at boot time. This allocator allocates * from the highest possible addresses. This avoids exhausting memory that * would be useful for DMA buffers. */ paddr_t do_bop_phys_alloc(uint64_t size, uint64_t align) { paddr_t pa = 0; paddr_t start; paddr_t end; struct memlist *ml = (struct memlist *)xbootp->bi_phys_install; /* * Be careful if high memory usage is limited in startup.c * Since there are holes in the low part of the physical address * space we can treat physmem as a pfn (not just a pgcnt) and * get a conservative upper limit. */ if (physmem != 0 && high_phys > pfn_to_pa(physmem)) high_phys = pfn_to_pa(physmem); /* * find the highest available memory in physinstalled */ size = P2ROUNDUP(size, align); for (; ml; ml = ml->ml_next) { start = P2ROUNDUP(ml->ml_address, align); end = P2ALIGN(ml->ml_address + ml->ml_size, align); if (start < next_phys) start = P2ROUNDUP(next_phys, align); if (end > high_phys) end = P2ALIGN(high_phys, align); if (end <= start) continue; if (end - start < size) continue; /* * Early allocations need to use low memory, since * physmem might be further limited by bootenv.rc */ if (early_allocation) { if (pa == 0 || start < pa) pa = start; } else { if (end - size > pa) pa = end - size; } } if (pa != 0) { if (early_allocation) next_phys = pa + size; else high_phys = pa; return (pa); } bop_panic("do_bop_phys_alloc(0x%" PRIx64 ", 0x%" PRIx64 ") Out of memory\n", size, align); /*NOTREACHED*/ } uintptr_t alloc_vaddr(size_t size, paddr_t align) { uintptr_t rv; next_virt = P2ROUNDUP(next_virt, (uintptr_t)align); rv = (uintptr_t)next_virt; next_virt += size; return (rv); } /* * Allocate virtual memory. The size is always rounded up to a multiple * of base pagesize. */ /*ARGSUSED*/ static caddr_t do_bsys_alloc(bootops_t *bop, caddr_t virthint, size_t size, int align) { paddr_t a = align; /* same type as pa for masking */ uint_t pgsize; paddr_t pa; uintptr_t va; ssize_t s; /* the aligned size */ uint_t level; uint_t is_kernel = (virthint != 0); if (a < MMU_PAGESIZE) a = MMU_PAGESIZE; else if (!ISP2(a)) prom_panic("do_bsys_alloc() incorrect alignment"); size = P2ROUNDUP(size, MMU_PAGESIZE); /* * Use the next aligned virtual address if we weren't given one. */ if (virthint == NULL) { virthint = (caddr_t)alloc_vaddr(size, a); total_bop_alloc_scratch += size; } else { total_bop_alloc_kernel += size; } /* * allocate the physical memory */ pa = do_bop_phys_alloc(size, a); /* * Add the mappings to the page tables, try large pages first. */ va = (uintptr_t)virthint; s = size; level = 1; pgsize = xbootp->bi_use_pae ? TWO_MEG : FOUR_MEG; if (xbootp->bi_use_largepage && a == pgsize) { while (IS_P2ALIGNED(pa, pgsize) && IS_P2ALIGNED(va, pgsize) && s >= pgsize) { kbm_map(va, pa, level, is_kernel); va += pgsize; pa += pgsize; s -= pgsize; } } /* * Map remaining pages use small mappings */ level = 0; pgsize = MMU_PAGESIZE; while (s > 0) { kbm_map(va, pa, level, is_kernel); va += pgsize; pa += pgsize; s -= pgsize; } return (virthint); } /* * Free virtual memory - we'll just ignore these. */ /*ARGSUSED*/ static void do_bsys_free(bootops_t *bop, caddr_t virt, size_t size) { bop_printf(NULL, "do_bsys_free(virt=0x%p, size=0x%lx) ignored\n", (void *)virt, size); } /* * Old interface */ /*ARGSUSED*/ static caddr_t do_bsys_ealloc(bootops_t *bop, caddr_t virthint, size_t size, int align, int flags) { prom_panic("unsupported call to BOP_EALLOC()\n"); return (0); } static void bsetprop(int flags, char *name, int nlen, void *value, int vlen) { uint_t size; uint_t need_size; bootprop_t *b; /* * align the size to 16 byte boundary */ size = sizeof (bootprop_t) + nlen + 1 + vlen; size = (size + 0xf) & ~0xf; if (size > curr_space) { need_size = (size + (MMU_PAGEOFFSET)) & MMU_PAGEMASK; curr_page = do_bsys_alloc(NULL, 0, need_size, MMU_PAGESIZE); curr_space = need_size; } /* * use a bootprop_t at curr_page and link into list */ b = (bootprop_t *)curr_page; curr_page += sizeof (bootprop_t); curr_space -= sizeof (bootprop_t); b->bp_next = bprops; bprops = b; /* * follow by name and ending zero byte */ b->bp_name = curr_page; bcopy(name, curr_page, nlen); curr_page += nlen; *curr_page++ = 0; curr_space -= nlen + 1; /* * set the property type */ b->bp_flags = flags & DDI_PROP_TYPE_MASK; /* * copy in value, but no ending zero byte */ b->bp_value = curr_page; b->bp_vlen = vlen; if (vlen > 0) { bcopy(value, curr_page, vlen); curr_page += vlen; curr_space -= vlen; } /* * align new values of curr_page, curr_space */ while (curr_space & 0xf) { ++curr_page; --curr_space; } } static void bsetprops(char *name, char *value) { bsetprop(DDI_PROP_TYPE_STRING, name, strlen(name), value, strlen(value) + 1); } static void bsetprop32(char *name, uint32_t value) { bsetprop(DDI_PROP_TYPE_INT, name, strlen(name), (void *)&value, sizeof (value)); } static void bsetprop64(char *name, uint64_t value) { bsetprop(DDI_PROP_TYPE_INT64, name, strlen(name), (void *)&value, sizeof (value)); } static void bsetpropsi(char *name, int value) { char prop_val[32]; (void) snprintf(prop_val, sizeof (prop_val), "%d", value); bsetprops(name, prop_val); } /* * to find the type of the value associated with this name */ /*ARGSUSED*/ int do_bsys_getproptype(bootops_t *bop, const char *name) { bootprop_t *b; for (b = bprops; b != NULL; b = b->bp_next) { if (strcmp(name, b->bp_name) != 0) continue; return (b->bp_flags); } return (-1); } /* * to find the size of the buffer to allocate */ /*ARGSUSED*/ int do_bsys_getproplen(bootops_t *bop, const char *name) { bootprop_t *b; for (b = bprops; b; b = b->bp_next) { if (strcmp(name, b->bp_name) != 0) continue; return (b->bp_vlen); } return (-1); } /* * get the value associated with this name */ /*ARGSUSED*/ int do_bsys_getprop(bootops_t *bop, const char *name, void *value) { bootprop_t *b; for (b = bprops; b; b = b->bp_next) { if (strcmp(name, b->bp_name) != 0) continue; bcopy(b->bp_value, value, b->bp_vlen); return (0); } return (-1); } /* * get the name of the next property in succession from the standalone */ /*ARGSUSED*/ static char * do_bsys_nextprop(bootops_t *bop, char *name) { bootprop_t *b; /* * A null name is a special signal for the 1st boot property */ if (name == NULL || strlen(name) == 0) { if (bprops == NULL) return (NULL); return (bprops->bp_name); } for (b = bprops; b; b = b->bp_next) { if (name != b->bp_name) continue; b = b->bp_next; if (b == NULL) return (NULL); return (b->bp_name); } return (NULL); } /* * Parse numeric value from a string. Understands decimal, hex, octal, - and ~ */ static int parse_value(char *p, uint64_t *retval) { int adjust = 0; uint64_t tmp = 0; int digit; int radix = 10; *retval = 0; if (*p == '-' || *p == '~') adjust = *p++; if (*p == '0') { ++p; if (*p == 0) return (0); if (*p == 'x' || *p == 'X') { radix = 16; ++p; } else { radix = 8; ++p; } } while (*p) { if ('0' <= *p && *p <= '9') digit = *p - '0'; else if ('a' <= *p && *p <= 'f') digit = 10 + *p - 'a'; else if ('A' <= *p && *p <= 'F') digit = 10 + *p - 'A'; else return (-1); if (digit >= radix) return (-1); tmp = tmp * radix + digit; ++p; } if (adjust == '-') tmp = -tmp; else if (adjust == '~') tmp = ~tmp; *retval = tmp; return (0); } static boolean_t unprintable(char *value, int size) { int i; if (size <= 0 || value[0] == '\0') return (B_TRUE); for (i = 0; i < size; i++) { if (value[i] == '\0') return (i != (size - 1)); if (!isprint(value[i])) return (B_TRUE); } return (B_FALSE); } /* * Print out information about all boot properties. * buffer is pointer to pre-allocated space to be used as temporary * space for property values. */ static void boot_prop_display(char *buffer) { char *name = ""; int i, len, flags, *buf32; int64_t *buf64; bop_printf(NULL, "\nBoot properties:\n"); while ((name = do_bsys_nextprop(NULL, name)) != NULL) { bop_printf(NULL, "\t0x%p %s = ", (void *)name, name); (void) do_bsys_getprop(NULL, name, buffer); len = do_bsys_getproplen(NULL, name); flags = do_bsys_getproptype(NULL, name); bop_printf(NULL, "len=%d ", len); switch (flags) { case DDI_PROP_TYPE_INT: len = len / sizeof (int); buf32 = (int *)buffer; for (i = 0; i < len; i++) { bop_printf(NULL, "%08x", buf32[i]); if (i < len - 1) bop_printf(NULL, "."); } break; case DDI_PROP_TYPE_STRING: bop_printf(NULL, "%s", buffer); break; case DDI_PROP_TYPE_INT64: len = len / sizeof (int64_t); buf64 = (int64_t *)buffer; for (i = 0; i < len; i++) { bop_printf(NULL, "%016" PRIx64, buf64[i]); if (i < len - 1) bop_printf(NULL, "."); } break; default: if (!unprintable(buffer, len)) { buffer[len] = 0; bop_printf(NULL, "%s", buffer); break; } for (i = 0; i < len; i++) { bop_printf(NULL, "%02x", buffer[i] & 0xff); if (i < len - 1) bop_printf(NULL, "."); } break; } bop_printf(NULL, "\n"); } } /* * 2nd part of building the table of boot properties. This includes: * - values from /boot/solaris/bootenv.rc (ie. eeprom(8) values) * * lines look like one of: * ^$ * ^# comment till end of line * setprop name 'value' * setprop name value * setprop name "value" * * we do single character I/O since this is really just looking at memory */ void read_bootenvrc(void) { int fd; char *line; int c; int bytes_read; char *name; int n_len; char *value; int v_len; char *inputdev; /* these override the command line if serial ports */ char *outputdev; char *consoledev; uint64_t lvalue; int use_xencons = 0; extern int bootrd_debug; #ifdef __xpv if (!DOMAIN_IS_INITDOMAIN(xen_info)) use_xencons = 1; #endif /* __xpv */ DBG_MSG("Opening /boot/solaris/bootenv.rc\n"); fd = BRD_OPEN(bfs_ops, "/boot/solaris/bootenv.rc", 0); DBG(fd); line = do_bsys_alloc(NULL, NULL, MMU_PAGESIZE, MMU_PAGESIZE); while (fd >= 0) { /* * get a line */ for (c = 0; ; ++c) { bytes_read = BRD_READ(bfs_ops, fd, line + c, 1); if (bytes_read == 0) { if (c == 0) goto done; break; } if (line[c] == '\n') break; } line[c] = 0; /* * ignore comment lines */ c = 0; while (ISSPACE(line[c])) ++c; if (line[c] == '#' || line[c] == 0) continue; /* * must have "setprop " or "setprop\t" */ if (strncmp(line + c, "setprop ", 8) != 0 && strncmp(line + c, "setprop\t", 8) != 0) continue; c += 8; while (ISSPACE(line[c])) ++c; if (line[c] == 0) continue; /* * gather up the property name */ name = line + c; n_len = 0; while (line[c] && !ISSPACE(line[c])) ++n_len, ++c; /* * gather up the value, if any */ value = ""; v_len = 0; while (ISSPACE(line[c])) ++c; if (line[c] != 0) { value = line + c; while (line[c] && !ISSPACE(line[c])) ++v_len, ++c; } if (v_len >= 2 && value[0] == value[v_len - 1] && (value[0] == '\'' || value[0] == '"')) { ++value; v_len -= 2; } name[n_len] = 0; if (v_len > 0) value[v_len] = 0; else continue; /* * ignore "boot-file" property, it's now meaningless */ if (strcmp(name, "boot-file") == 0) continue; if (strcmp(name, "boot-args") == 0 && strlen(boot_args) > 0) continue; /* * If a property was explicitly set on the command line * it will override a setting in bootenv.rc. We make an * exception for a property from the bootloader such as: * * console="text,ttya,ttyb,ttyc,ttyd" * * In such a case, picking the first value here (as * lookup_console_devices() does) is at best a guess; if * bootenv.rc has a value, it's probably better. */ if (strcmp(name, "console") == 0) { char propval[BP_MAX_STRLEN] = ""; if (do_bsys_getprop(NULL, name, propval) == -1 || strchr(propval, ',') != NULL) bsetprops(name, value); continue; } if (do_bsys_getproplen(NULL, name) == -1) bsetprops(name, value); } done: if (fd >= 0) (void) BRD_CLOSE(bfs_ops, fd); /* * Check if we have to limit the boot time allocator */ if (do_bsys_getproplen(NULL, "physmem") != -1 && do_bsys_getprop(NULL, "physmem", line) >= 0 && parse_value(line, &lvalue) != -1) { if (0 < lvalue && (lvalue < physmem || physmem == 0)) { physmem = (pgcnt_t)lvalue; DBG(physmem); } } early_allocation = 0; /* * Check for bootrd_debug. */ if (find_boot_prop("bootrd_debug")) bootrd_debug = 1; /* * check to see if we have to override the default value of the console */ if (!use_xencons) { inputdev = line; v_len = do_bsys_getproplen(NULL, "input-device"); if (v_len > 0) (void) do_bsys_getprop(NULL, "input-device", inputdev); else v_len = 0; inputdev[v_len] = 0; outputdev = inputdev + v_len + 1; v_len = do_bsys_getproplen(NULL, "output-device"); if (v_len > 0) (void) do_bsys_getprop(NULL, "output-device", outputdev); else v_len = 0; outputdev[v_len] = 0; consoledev = outputdev + v_len + 1; v_len = do_bsys_getproplen(NULL, "console"); if (v_len > 0) { (void) do_bsys_getprop(NULL, "console", consoledev); if (post_fastreboot && strcmp(consoledev, "graphics") == 0) { bsetprops("console", "text"); v_len = strlen("text"); bcopy("text", consoledev, v_len); } } else { v_len = 0; } consoledev[v_len] = 0; bcons_post_bootenvrc(inputdev, outputdev, consoledev); } else { /* * Ensure console property exists * If not create it as "hypervisor" */ v_len = do_bsys_getproplen(NULL, "console"); if (v_len < 0) bsetprops("console", "hypervisor"); inputdev = outputdev = consoledev = "hypervisor"; bcons_post_bootenvrc(inputdev, outputdev, consoledev); } if (find_boot_prop("prom_debug") || kbm_debug) boot_prop_display(line); } /* * print formatted output */ /*ARGSUSED*/ void vbop_printf(void *ptr, const char *fmt, va_list ap) { if (have_console == 0) return; (void) vsnprintf(buffer, BUFFERSIZE, fmt, ap); PUT_STRING(buffer); } /*PRINTFLIKE2*/ void bop_printf(void *bop, const char *fmt, ...) { va_list ap; va_start(ap, fmt); vbop_printf(bop, fmt, ap); va_end(ap); } /* * Another panic() variant; this one can be used even earlier during boot than * prom_panic(). */ /*PRINTFLIKE1*/ void bop_panic(const char *fmt, ...) { va_list ap; va_start(ap, fmt); vbop_printf(NULL, fmt, ap); va_end(ap); bop_printf(NULL, "\nPress any key to reboot.\n"); (void) bcons_getchar(); bop_printf(NULL, "Resetting...\n"); pc_reset(); } /* * Do a real mode interrupt BIOS call */ typedef struct bios_regs { unsigned short ax, bx, cx, dx, si, di, bp, es, ds; } bios_regs_t; typedef int (*bios_func_t)(int, bios_regs_t *); /*ARGSUSED*/ static void do_bsys_doint(bootops_t *bop, int intnum, struct bop_regs *rp) { #if defined(__xpv) prom_panic("unsupported call to BOP_DOINT()\n"); #else /* __xpv */ static int firsttime = 1; bios_func_t bios_func = (bios_func_t)(void *)(uintptr_t)0x5000; bios_regs_t br; /* * We're about to disable paging; we shouldn't be PCID enabled. */ if (getcr4() & CR4_PCIDE) prom_panic("do_bsys_doint() with PCID enabled\n"); /* * The first time we do this, we have to copy the pre-packaged * low memory bios call code image into place. */ if (firsttime) { extern char bios_image[]; extern uint32_t bios_size; bcopy(bios_image, (void *)bios_func, bios_size); firsttime = 0; } br.ax = rp->eax.word.ax; br.bx = rp->ebx.word.bx; br.cx = rp->ecx.word.cx; br.dx = rp->edx.word.dx; br.bp = rp->ebp.word.bp; br.si = rp->esi.word.si; br.di = rp->edi.word.di; br.ds = rp->ds; br.es = rp->es; DBG_MSG("Doing BIOS call...\n"); DBG(br.ax); DBG(br.bx); DBG(br.dx); rp->eflags = bios_func(intnum, &br); DBG_MSG("done\n"); DBG(rp->eflags); DBG(br.ax); DBG(br.bx); DBG(br.dx); rp->eax.word.ax = br.ax; rp->ebx.word.bx = br.bx; rp->ecx.word.cx = br.cx; rp->edx.word.dx = br.dx; rp->ebp.word.bp = br.bp; rp->esi.word.si = br.si; rp->edi.word.di = br.di; rp->ds = br.ds; rp->es = br.es; #endif /* __xpv */ } static struct boot_syscalls bop_sysp = { bcons_getchar, bcons_putchar, bcons_ischar, }; static char *whoami; #define BUFLEN 64 #if defined(__xpv) static char namebuf[32]; static void xen_parse_props(char *s, char *prop_map[], int n_prop) { char **prop_name = prop_map; char *cp = s, *scp; do { scp = cp; while ((*cp != '\0') && (*cp != ':')) cp++; if ((scp != cp) && (*prop_name != NULL)) { *cp = '\0'; bsetprops(*prop_name, scp); } cp++; prop_name++; n_prop--; } while (n_prop > 0); } #define VBDPATHLEN 64 /* * parse the 'xpv-root' property to create properties used by * ufs_mountroot. */ static void xen_vbdroot_props(char *s) { char vbdpath[VBDPATHLEN] = "/xpvd/xdf@"; const char lnamefix[] = "/dev/dsk/c0d"; char *pnp; char *prop_p; char mi; short minor; long addr = 0; mi = '\0'; pnp = vbdpath + strlen(vbdpath); prop_p = s + strlen(lnamefix); while ((*prop_p != '\0') && (*prop_p != 's') && (*prop_p != 'p')) addr = addr * 10 + *prop_p++ - '0'; (void) snprintf(pnp, VBDPATHLEN, "%lx", addr); pnp = vbdpath + strlen(vbdpath); if (*prop_p == 's') mi = 'a'; else if (*prop_p == 'p') mi = 'q'; else ASSERT(0); /* shouldn't be here */ prop_p++; ASSERT(*prop_p != '\0'); if (ISDIGIT(*prop_p)) { minor = *prop_p - '0'; prop_p++; if (ISDIGIT(*prop_p)) { minor = minor * 10 + *prop_p - '0'; } } else { /* malformed root path, use 0 as default */ minor = 0; } ASSERT(minor < 16); /* at most 16 partitions */ mi += minor; *pnp++ = ':'; *pnp++ = mi; *pnp++ = '\0'; bsetprops("fstype", "ufs"); bsetprops("bootpath", vbdpath); DBG_MSG("VBD bootpath set to "); DBG_MSG(vbdpath); DBG_MSG("\n"); } /* * parse the xpv-nfsroot property to create properties used by * nfs_mountroot. */ static void xen_nfsroot_props(char *s) { char *prop_map[] = { BP_SERVER_IP, /* server IP address */ BP_SERVER_NAME, /* server hostname */ BP_SERVER_PATH, /* root path */ }; int n_prop = sizeof (prop_map) / sizeof (prop_map[0]); bsetprops("fstype", "nfs"); xen_parse_props(s, prop_map, n_prop); /* * If a server name wasn't specified, use a default. */ if (do_bsys_getproplen(NULL, BP_SERVER_NAME) == -1) bsetprops(BP_SERVER_NAME, "unknown"); } /* * Extract our IP address, etc. from the "xpv-ip" property. */ static void xen_ip_props(char *s) { char *prop_map[] = { BP_HOST_IP, /* IP address */ NULL, /* NFS server IP address (ignored in */ /* favour of xpv-nfsroot) */ BP_ROUTER_IP, /* IP gateway */ BP_SUBNET_MASK, /* IP subnet mask */ "xpv-hostname", /* hostname (ignored) */ BP_NETWORK_INTERFACE, /* interface name */ "xpv-hcp", /* host configuration protocol */ }; int n_prop = sizeof (prop_map) / sizeof (prop_map[0]); char ifname[IFNAMSIZ]; xen_parse_props(s, prop_map, n_prop); /* * A Linux dom0 administrator expects all interfaces to be * called "ethX", which is not the case here. * * If the interface name specified is "eth0", presume that * this is really intended to be "xnf0" (the first domU -> * dom0 interface for this domain). */ if ((do_bsys_getprop(NULL, BP_NETWORK_INTERFACE, ifname) == 0) && (strcmp("eth0", ifname) == 0)) { bsetprops(BP_NETWORK_INTERFACE, "xnf0"); bop_printf(NULL, "network interface name 'eth0' replaced with 'xnf0'\n"); } } #else /* __xpv */ static void setup_rarp_props(struct sol_netinfo *sip) { char buf[BUFLEN]; /* to hold ip/mac addrs */ uint8_t *val; val = (uint8_t *)&sip->sn_ciaddr; (void) snprintf(buf, BUFLEN, "%d.%d.%d.%d", val[0], val[1], val[2], val[3]); bsetprops(BP_HOST_IP, buf); val = (uint8_t *)&sip->sn_siaddr; (void) snprintf(buf, BUFLEN, "%d.%d.%d.%d", val[0], val[1], val[2], val[3]); bsetprops(BP_SERVER_IP, buf); if (sip->sn_giaddr != 0) { val = (uint8_t *)&sip->sn_giaddr; (void) snprintf(buf, BUFLEN, "%d.%d.%d.%d", val[0], val[1], val[2], val[3]); bsetprops(BP_ROUTER_IP, buf); } if (sip->sn_netmask != 0) { val = (uint8_t *)&sip->sn_netmask; (void) snprintf(buf, BUFLEN, "%d.%d.%d.%d", val[0], val[1], val[2], val[3]); bsetprops(BP_SUBNET_MASK, buf); } if (sip->sn_mactype != 4 || sip->sn_maclen != 6) { bop_printf(NULL, "unsupported mac type %d, mac len %d\n", sip->sn_mactype, sip->sn_maclen); } else { val = sip->sn_macaddr; (void) snprintf(buf, BUFLEN, "%x:%x:%x:%x:%x:%x", val[0], val[1], val[2], val[3], val[4], val[5]); bsetprops(BP_BOOT_MAC, buf); } } #endif /* __xpv */ static void build_panic_cmdline(const char *cmd, int cmdlen) { int proplen; size_t arglen; arglen = sizeof (fastreboot_onpanic_args); /* * If we allready have fastreboot-onpanic set to zero, * don't add them again. */ if ((proplen = do_bsys_getproplen(NULL, FASTREBOOT_ONPANIC)) > 0 && proplen <= sizeof (fastreboot_onpanic_cmdline)) { (void) do_bsys_getprop(NULL, FASTREBOOT_ONPANIC, fastreboot_onpanic_cmdline); if (FASTREBOOT_ONPANIC_NOTSET(fastreboot_onpanic_cmdline)) arglen = 1; } /* * construct fastreboot_onpanic_cmdline */ if (cmdlen + arglen > sizeof (fastreboot_onpanic_cmdline)) { DBG_MSG("Command line too long: clearing " FASTREBOOT_ONPANIC "\n"); fastreboot_onpanic = 0; } else { bcopy(cmd, fastreboot_onpanic_cmdline, cmdlen); if (arglen != 1) bcopy(fastreboot_onpanic_args, fastreboot_onpanic_cmdline + cmdlen, arglen); else fastreboot_onpanic_cmdline[cmdlen] = 0; } } #ifndef __xpv /* * Construct boot command line for Fast Reboot. The saved_cmdline * is also reported by "eeprom bootcmd". */ static void build_fastboot_cmdline(struct xboot_info *xbp) { saved_cmdline_len = strlen(xbp->bi_cmdline) + 1; if (saved_cmdline_len > FASTBOOT_SAVED_CMDLINE_LEN) { DBG(saved_cmdline_len); DBG_MSG("Command line too long: clearing fastreboot_capable\n"); fastreboot_capable = 0; } else { bcopy((void *)(xbp->bi_cmdline), (void *)saved_cmdline, saved_cmdline_len); saved_cmdline[saved_cmdline_len - 1] = '\0'; build_panic_cmdline(saved_cmdline, saved_cmdline_len - 1); } } /* * Save memory layout, disk drive information, unix and boot archive sizes for * Fast Reboot. */ static void save_boot_info(struct xboot_info *xbi) { multiboot_info_t *mbi = xbi->bi_mb_info; struct boot_modules *modp; int i; bcopy(mbi, &saved_mbi, sizeof (multiboot_info_t)); if (mbi->mmap_length > sizeof (saved_mmap)) { DBG_MSG("mbi->mmap_length too big: clearing " "fastreboot_capable\n"); fastreboot_capable = 0; } else { bcopy((void *)(uintptr_t)mbi->mmap_addr, (void *)saved_mmap, mbi->mmap_length); } if ((mbi->flags & MB_INFO_DRIVE_INFO) != 0) { if (mbi->drives_length > sizeof (saved_drives)) { DBG(mbi->drives_length); DBG_MSG("mbi->drives_length too big: clearing " "fastreboot_capable\n"); fastreboot_capable = 0; } else { bcopy((void *)(uintptr_t)mbi->drives_addr, (void *)saved_drives, mbi->drives_length); } } else { saved_mbi.drives_length = 0; saved_mbi.drives_addr = 0; } /* * Current file sizes. Used by fastboot.c to figure out how much * memory to reserve for panic reboot. * Use the module list from the dboot-constructed xboot_info * instead of the list referenced by the multiboot structure * because that structure may not be addressable now. */ saved_file_size[FASTBOOT_NAME_UNIX] = FOUR_MEG - PAGESIZE; for (i = 0, modp = (struct boot_modules *)(uintptr_t)xbi->bi_modules; i < xbi->bi_module_cnt; i++, modp++) { saved_file_size[FASTBOOT_NAME_BOOTARCHIVE] += modp->bm_size; } } #endif /* __xpv */ /* * Import boot environment module variables as properties, applying * blacklist filter for variables we know we will not use. * * Since the environment can be relatively large, containing many variables * used only for boot loader purposes, we will use a blacklist based filter. * To keep the blacklist from growing too large, we use prefix based filtering. * This is possible because in many cases, the loader variable names are * using a structured layout. * * We will not overwrite already set properties. * * Note that the menu items in particular can contain characters not * well-handled as bootparams, such as spaces, brackets, and the like, so that's * another reason. */ static struct bop_blacklist { const char *bl_name; int bl_name_len; } bop_prop_blacklist[] = { { "ISADIR", sizeof ("ISADIR") }, { "acpi.", sizeof ("acpi.") }, { "autoboot_delay", sizeof ("autoboot_delay") }, { "beansi_", sizeof ("beansi_") }, { "beastie", sizeof ("beastie") }, { "bemenu", sizeof ("bemenu") }, { "boot.", sizeof ("boot.") }, { "bootenv", sizeof ("bootenv") }, { "currdev", sizeof ("currdev") }, { "dhcp.", sizeof ("dhcp.") }, { "interpret", sizeof ("interpret") }, { "kernel", sizeof ("kernel") }, { "loaddev", sizeof ("loaddev") }, { "loader_", sizeof ("loader_") }, { "mainansi_", sizeof ("mainansi_") }, { "mainmenu_", sizeof ("mainmenu_") }, { "maintoggled_", sizeof ("maintoggled_") }, { "menu_timeout_command", sizeof ("menu_timeout_command") }, { "menuset_", sizeof ("menuset_") }, { "module_path", sizeof ("module_path") }, { "nfs.", sizeof ("nfs.") }, { "optionsansi_", sizeof ("optionsansi_") }, { "optionsmenu_", sizeof ("optionsmenu_") }, { "optionstoggled_", sizeof ("optionstoggled_") }, { "pcibios", sizeof ("pcibios") }, { "prompt", sizeof ("prompt") }, { "smbios.", sizeof ("smbios.") }, { "tem", sizeof ("tem") }, { "twiddle_divisor", sizeof ("twiddle_divisor") }, { "zfs_be", sizeof ("zfs_be") }, }; /* * Match the name against prefixes in above blacklist. If the match was * found, this name is blacklisted. */ static boolean_t name_is_blacklisted(const char *name) { int i, n; n = sizeof (bop_prop_blacklist) / sizeof (bop_prop_blacklist[0]); for (i = 0; i < n; i++) { if (strncmp(bop_prop_blacklist[i].bl_name, name, bop_prop_blacklist[i].bl_name_len - 1) == 0) { return (B_TRUE); } } return (B_FALSE); } static void process_boot_environment(struct boot_modules *benv) { char *env, *ptr, *name, *value; uint32_t size, name_len, value_len; if (benv == NULL || benv->bm_type != BMT_ENV) return; ptr = env = benv->bm_addr; size = benv->bm_size; do { name = ptr; /* find '=' */ while (*ptr != '=') { ptr++; if (ptr > env + size) /* Something is very wrong. */ return; } name_len = ptr - name; if (sizeof (buffer) <= name_len) continue; (void) strncpy(buffer, name, sizeof (buffer)); buffer[name_len] = '\0'; name = buffer; value_len = 0; value = ++ptr; while ((uintptr_t)ptr - (uintptr_t)env < size) { if (*ptr == '\0') { ptr++; value_len = (uintptr_t)ptr - (uintptr_t)env; break; } ptr++; } /* Did we reach the end of the module? */ if (value_len == 0) return; if (*value == '\0') continue; /* Is this property already set? */ if (do_bsys_getproplen(NULL, name) >= 0) continue; /* Translate netboot variables */ if (strcmp(name, "boot.netif.gateway") == 0) { bsetprops(BP_ROUTER_IP, value); continue; } if (strcmp(name, "boot.netif.hwaddr") == 0) { bsetprops(BP_BOOT_MAC, value); continue; } if (strcmp(name, "boot.netif.ip") == 0) { bsetprops(BP_HOST_IP, value); continue; } if (strcmp(name, "boot.netif.netmask") == 0) { bsetprops(BP_SUBNET_MASK, value); continue; } if (strcmp(name, "boot.netif.server") == 0) { bsetprops(BP_SERVER_IP, value); continue; } if (strcmp(name, "boot.netif.server") == 0) { if (do_bsys_getproplen(NULL, BP_SERVER_IP) < 0) bsetprops(BP_SERVER_IP, value); continue; } if (strcmp(name, "boot.nfsroot.server") == 0) { if (do_bsys_getproplen(NULL, BP_SERVER_IP) < 0) bsetprops(BP_SERVER_IP, value); continue; } if (strcmp(name, "boot.nfsroot.path") == 0) { bsetprops(BP_SERVER_PATH, value); continue; } if (strcmp(name, "acpi-root-tab") == 0) { uint64_t i64; if (parse_value(value, &i64) == 0) bsetprop64(name, i64); continue; } if (strcmp(name, "smbios-address") == 0) { uint64_t i64; if (parse_value(value, &i64) == 0) bsetprop64(name, i64); continue; } if (strcmp(name, "efi-systab") == 0) { uint64_t i64; if (parse_value(value, &i64) == 0) bsetprop64(name, i64); continue; } /* * The loader allows multiple console devices to be specified * as a comma-separated list, but the kernel does not yet * support multiple console devices. If a list is provided, * ignore all but the first entry: */ if (strcmp(name, "console") == 0) { char propval[BP_MAX_STRLEN]; for (uint32_t i = 0; i < BP_MAX_STRLEN; i++) { propval[i] = value[i]; if (value[i] == ' ' || value[i] == ',' || value[i] == '\0') { propval[i] = '\0'; break; } if (i + 1 == BP_MAX_STRLEN) propval[i] = '\0'; } bsetprops(name, propval); continue; } if (name_is_blacklisted(name) == B_TRUE) continue; /* Create new property. */ bsetprops(name, value); /* Avoid reading past the module end. */ if (size <= (uintptr_t)ptr - (uintptr_t)env) return; } while (*ptr != '\0'); } /* * 1st pass at building the table of boot properties. This includes: * - values set on the command line: -B a=x,b=y,c=z .... * - known values we just compute (ie. from xbp) * - values from /boot/solaris/bootenv.rc (ie. eeprom(8) values) * * the grub command line looked like: * kernel boot-file [-B prop=value[,prop=value]...] [boot-args] * * whoami is the same as boot-file */ static void build_boot_properties(struct xboot_info *xbp) { char *name; int name_len; char *value; int value_len; struct boot_modules *bm, *rdbm, *benv = NULL; char *propbuf; int quoted = 0; int boot_arg_len; uint_t i, midx; char modid[32]; #ifndef __xpv static int stdout_val = 0; uchar_t boot_device; char str[3]; #endif /* * These have to be done first, so that kobj_mount_root() works */ DBG_MSG("Building boot properties\n"); propbuf = do_bsys_alloc(NULL, NULL, MMU_PAGESIZE, 0); DBG((uintptr_t)propbuf); if (xbp->bi_module_cnt > 0) { bm = xbp->bi_modules; rdbm = NULL; for (midx = i = 0; i < xbp->bi_module_cnt; i++) { if (bm[i].bm_type == BMT_ROOTFS) { rdbm = &bm[i]; continue; } if (bm[i].bm_type == BMT_HASH || bm[i].bm_type == BMT_FONT || bm[i].bm_name == NULL) continue; if (bm[i].bm_type == BMT_ENV) { if (benv == NULL) benv = &bm[i]; else continue; } (void) snprintf(modid, sizeof (modid), "module-name-%u", midx); bsetprops(modid, (char *)bm[i].bm_name); (void) snprintf(modid, sizeof (modid), "module-addr-%u", midx); bsetprop64(modid, (uint64_t)(uintptr_t)bm[i].bm_addr); (void) snprintf(modid, sizeof (modid), "module-size-%u", midx); bsetprop64(modid, (uint64_t)bm[i].bm_size); ++midx; } if (rdbm != NULL) { bsetprop64("ramdisk_start", (uint64_t)(uintptr_t)rdbm->bm_addr); bsetprop64("ramdisk_end", (uint64_t)(uintptr_t)rdbm->bm_addr + rdbm->bm_size); } } /* * If there are any boot time modules or hashes present, then disable * fast reboot. */ if (xbp->bi_module_cnt > 1) { fastreboot_disable(FBNS_BOOTMOD); } #ifndef __xpv /* * Disable fast reboot if we're using the Multiboot 2 boot protocol, * since we don't currently support MB2 info and module relocation. * Note that fast reboot will have already been disabled if multiple * modules are present, since the current implementation assumes that * we only have a single module, the boot_archive. */ if (xbp->bi_mb_version != 1) { fastreboot_disable(FBNS_MULTIBOOT2); } #endif DBG_MSG("Parsing command line for boot properties\n"); value = xbp->bi_cmdline; /* * allocate memory to collect boot_args into */ boot_arg_len = strlen(xbp->bi_cmdline) + 1; boot_args = do_bsys_alloc(NULL, NULL, boot_arg_len, MMU_PAGESIZE); boot_args[0] = 0; boot_arg_len = 0; #ifdef __xpv /* * Xen puts a lot of device information in front of the kernel name * let's grab them and make them boot properties. The first * string w/o an "=" in it will be the boot-file property. */ (void) strcpy(namebuf, "xpv-"); for (;;) { /* * get to next property */ while (ISSPACE(*value)) ++value; name = value; /* * look for an "=" */ while (*value && !ISSPACE(*value) && *value != '=') { value++; } if (*value != '=') { /* no "=" in the property */ value = name; break; } name_len = value - name; value_len = 0; /* * skip over the "=" */ value++; while (value[value_len] && !ISSPACE(value[value_len])) { ++value_len; } /* * build property name with "xpv-" prefix */ if (name_len + 4 > 32) { /* skip if name too long */ value += value_len; continue; } bcopy(name, &namebuf[4], name_len); name_len += 4; namebuf[name_len] = 0; bcopy(value, propbuf, value_len); propbuf[value_len] = 0; bsetprops(namebuf, propbuf); /* * xpv-root is set to the logical disk name of the xen * VBD when booting from a disk-based filesystem. */ if (strcmp(namebuf, "xpv-root") == 0) xen_vbdroot_props(propbuf); /* * While we're here, if we have a "xpv-nfsroot" property * then we need to set "fstype" to "nfs" so we mount * our root from the nfs server. Also parse the xpv-nfsroot * property to create the properties that nfs_mountroot will * need to find the root and mount it. */ if (strcmp(namebuf, "xpv-nfsroot") == 0) xen_nfsroot_props(propbuf); if (strcmp(namebuf, "xpv-ip") == 0) xen_ip_props(propbuf); value += value_len; } #endif while (ISSPACE(*value)) ++value; /* * value now points at the boot-file */ value_len = 0; while (value[value_len] && !ISSPACE(value[value_len])) ++value_len; if (value_len > 0) { whoami = propbuf; bcopy(value, whoami, value_len); whoami[value_len] = 0; bsetprops("boot-file", whoami); /* * strip leading path stuff from whoami, so running from * PXE/miniroot makes sense. */ if (strstr(whoami, "/platform/") != NULL) whoami = strstr(whoami, "/platform/"); bsetprops("whoami", whoami); } /* * Values forcibly set boot properties on the command line via -B. * Allow use of quotes in values. Other stuff goes on kernel * command line. */ name = value + value_len; while (*name != 0) { /* * anything not " -B" is copied to the command line */ if (!ISSPACE(name[0]) || name[1] != '-' || name[2] != 'B') { boot_args[boot_arg_len++] = *name; boot_args[boot_arg_len] = 0; ++name; continue; } /* * skip the " -B" and following white space */ name += 3; while (ISSPACE(*name)) ++name; while (*name && !ISSPACE(*name)) { value = strstr(name, "="); if (value == NULL) break; name_len = value - name; ++value; value_len = 0; quoted = 0; for (; ; ++value_len) { if (!value[value_len]) break; /* * is this value quoted? */ if (value_len == 0 && (value[0] == '\'' || value[0] == '"')) { quoted = value[0]; ++value_len; } /* * In the quote accept any character, * but look for ending quote. */ if (quoted) { if (value[value_len] == quoted) quoted = 0; continue; } /* * a comma or white space ends the value */ if (value[value_len] == ',' || ISSPACE(value[value_len])) break; } if (value_len == 0) { bsetprop(DDI_PROP_TYPE_ANY, name, name_len, NULL, 0); } else { char *v = value; int l = value_len; if (v[0] == v[l - 1] && (v[0] == '\'' || v[0] == '"')) { ++v; l -= 2; } bcopy(v, propbuf, l); propbuf[l] = '\0'; bsetprop(DDI_PROP_TYPE_STRING, name, name_len, propbuf, l + 1); } name = value + value_len; while (*name == ',') ++name; } } /* * set boot-args property * 1275 name is bootargs, so set * that too */ bsetprops("boot-args", boot_args); bsetprops("bootargs", boot_args); process_boot_environment(benv); #ifndef __xpv /* * Build boot command line for Fast Reboot */ build_fastboot_cmdline(xbp); if (xbp->bi_mb_version == 1) { multiboot_info_t *mbi = xbp->bi_mb_info; int netboot; struct sol_netinfo *sip; /* * set the BIOS boot device from GRUB */ netboot = 0; /* * Save various boot information for Fast Reboot */ save_boot_info(xbp); if (mbi != NULL && mbi->flags & MB_INFO_BOOTDEV) { boot_device = mbi->boot_device >> 24; if (boot_device == 0x20) netboot++; str[0] = (boot_device >> 4) + '0'; str[1] = (boot_device & 0xf) + '0'; str[2] = 0; bsetprops("bios-boot-device", str); } else { netboot = 1; } /* * In the netboot case, drives_info is overloaded with the * dhcp ack. This is not multiboot compliant and requires * special pxegrub! */ if (netboot && mbi->drives_length != 0) { sip = (struct sol_netinfo *)(uintptr_t)mbi->drives_addr; if (sip->sn_infotype == SN_TYPE_BOOTP) bsetprop(DDI_PROP_TYPE_BYTE, "bootp-response", sizeof ("bootp-response"), (void *)(uintptr_t)mbi->drives_addr, mbi->drives_length); else if (sip->sn_infotype == SN_TYPE_RARP) setup_rarp_props(sip); } } else { multiboot2_info_header_t *mbi = xbp->bi_mb_info; multiboot_tag_bootdev_t *bootdev = NULL; multiboot_tag_network_t *netdev = NULL; if (mbi != NULL) { bootdev = dboot_multiboot2_find_tag(mbi, MULTIBOOT_TAG_TYPE_BOOTDEV); netdev = dboot_multiboot2_find_tag(mbi, MULTIBOOT_TAG_TYPE_NETWORK); } if (bootdev != NULL) { DBG(bootdev->mb_biosdev); boot_device = bootdev->mb_biosdev; str[0] = (boot_device >> 4) + '0'; str[1] = (boot_device & 0xf) + '0'; str[2] = 0; bsetprops("bios-boot-device", str); } if (netdev != NULL) { bsetprop(DDI_PROP_TYPE_BYTE, "bootp-response", sizeof ("bootp-response"), (void *)(uintptr_t)netdev->mb_dhcpack, netdev->mb_size - sizeof (multiboot_tag_network_t)); } } bsetprop32("stdout", stdout_val); #endif /* __xpv */ /* * more conjured up values for made up things.... */ #if defined(__xpv) bsetprops("mfg-name", "i86xpv"); bsetprops("impl-arch-name", "i86xpv"); #else bsetprops("mfg-name", "amd64"); bsetprops("impl-arch-name", "amd64"); #endif /* * Build firmware-provided system properties */ build_firmware_properties(xbp); /* * XXPV * * Find out what these are: * - cpuid_feature_ecx_include * - cpuid_feature_ecx_exclude * - cpuid_feature_edx_include * - cpuid_feature_edx_exclude * * Find out what these are in multiboot: * - netdev-path * - fstype */ } #ifdef __xpv /* * Under the Hypervisor, memory usable for DMA may be scarce. One * very likely large pool of DMA friendly memory is occupied by * the boot_archive, as it was loaded by grub into low MFNs. * * Here we free up that memory by copying the boot archive to what are * likely higher MFN pages and then swapping the mfn/pfn mappings. */ #define PFN_2GIG 0x80000 static void relocate_boot_archive(struct xboot_info *xbp) { mfn_t max_mfn = HYPERVISOR_memory_op(XENMEM_maximum_ram_page, NULL); struct boot_modules *bm = xbp->bi_modules; uintptr_t va; pfn_t va_pfn; mfn_t va_mfn; caddr_t copy; pfn_t copy_pfn; mfn_t copy_mfn; size_t len; int slop; int total = 0; int relocated = 0; int mmu_update_return; mmu_update_t t[2]; x86pte_t pte; /* * If all MFN's are below 2Gig, don't bother doing this. */ if (max_mfn < PFN_2GIG) return; if (xbp->bi_module_cnt < 1) { DBG_MSG("no boot_archive!"); return; } DBG_MSG("moving boot_archive to high MFN memory\n"); va = (uintptr_t)bm->bm_addr; len = bm->bm_size; slop = va & MMU_PAGEOFFSET; if (slop) { va += MMU_PAGESIZE - slop; len -= MMU_PAGESIZE - slop; } len = P2ALIGN(len, MMU_PAGESIZE); /* * Go through all boot_archive pages, swapping any low MFN pages * with memory at next_phys. */ while (len != 0) { ++total; va_pfn = mmu_btop(va - ONE_GIG); va_mfn = mfn_list[va_pfn]; if (mfn_list[va_pfn] < PFN_2GIG) { copy = kbm_remap_window(next_phys, 1); bcopy((void *)va, copy, MMU_PAGESIZE); copy_pfn = mmu_btop(next_phys); copy_mfn = mfn_list[copy_pfn]; pte = mfn_to_ma(copy_mfn) | PT_NOCONSIST | PT_VALID; if (HYPERVISOR_update_va_mapping(va, pte, UVMF_INVLPG | UVMF_LOCAL)) bop_panic("relocate_boot_archive(): " "HYPERVISOR_update_va_mapping() failed"); mfn_list[va_pfn] = copy_mfn; mfn_list[copy_pfn] = va_mfn; t[0].ptr = mfn_to_ma(copy_mfn) | MMU_MACHPHYS_UPDATE; t[0].val = va_pfn; t[1].ptr = mfn_to_ma(va_mfn) | MMU_MACHPHYS_UPDATE; t[1].val = copy_pfn; if (HYPERVISOR_mmu_update(t, 2, &mmu_update_return, DOMID_SELF) != 0 || mmu_update_return != 2) bop_panic("relocate_boot_archive(): " "HYPERVISOR_mmu_update() failed"); next_phys += MMU_PAGESIZE; ++relocated; } len -= MMU_PAGESIZE; va += MMU_PAGESIZE; } DBG_MSG("Relocated pages:\n"); DBG(relocated); DBG_MSG("Out of total pages:\n"); DBG(total); } #endif /* __xpv */ #if !defined(__xpv) /* * simple description of a stack frame (args are 32 bit only currently) */ typedef struct bop_frame { struct bop_frame *old_frame; pc_t retaddr; long arg[1]; } bop_frame_t; void bop_traceback(bop_frame_t *frame) { pc_t pc; int cnt; char *ksym; ulong_t off; bop_printf(NULL, "Stack traceback:\n"); for (cnt = 0; cnt < 30; ++cnt) { /* up to 30 frames */ pc = frame->retaddr; if (pc == 0) break; ksym = kobj_getsymname(pc, &off); if (ksym) bop_printf(NULL, " %s+%lx", ksym, off); else bop_printf(NULL, " 0x%lx", pc); frame = frame->old_frame; if (frame == 0) { bop_printf(NULL, "\n"); break; } bop_printf(NULL, "\n"); } } struct trapframe { ulong_t error_code; /* optional */ ulong_t inst_ptr; ulong_t code_seg; ulong_t flags_reg; ulong_t stk_ptr; ulong_t stk_seg; }; void bop_trap(ulong_t *tfp) { struct trapframe *tf = (struct trapframe *)tfp; bop_frame_t fakeframe; static int depth = 0; /* * Check for an infinite loop of traps. */ if (++depth > 2) bop_panic("Nested trap"); bop_printf(NULL, "Unexpected trap\n"); /* * adjust the tf for optional error_code by detecting the code selector */ if (tf->code_seg != B64CODE_SEL) tf = (struct trapframe *)(tfp - 1); else bop_printf(NULL, "error code 0x%lx\n", tf->error_code & 0xffffffff); bop_printf(NULL, "instruction pointer 0x%lx\n", tf->inst_ptr); bop_printf(NULL, "code segment 0x%lx\n", tf->code_seg & 0xffff); bop_printf(NULL, "flags register 0x%lx\n", tf->flags_reg); bop_printf(NULL, "return %%rsp 0x%lx\n", tf->stk_ptr); bop_printf(NULL, "return %%ss 0x%lx\n", tf->stk_seg & 0xffff); bop_printf(NULL, "%%cr2 0x%lx\n", getcr2()); /* grab %[er]bp pushed by our code from the stack */ fakeframe.old_frame = (bop_frame_t *)*(tfp - 3); fakeframe.retaddr = (pc_t)tf->inst_ptr; bop_printf(NULL, "Attempting stack backtrace:\n"); bop_traceback(&fakeframe); bop_panic("unexpected trap in early boot"); } extern void bop_trap_handler(void); static gate_desc_t *bop_idt; static desctbr_t bop_idt_info; /* * Install a temporary IDT that lets us catch errors in the boot time code. * We shouldn't get any faults at all while this is installed, so we'll * just generate a traceback and exit. */ static void bop_idt_init(void) { int t; bop_idt = (gate_desc_t *) do_bsys_alloc(NULL, NULL, MMU_PAGESIZE, MMU_PAGESIZE); bzero(bop_idt, MMU_PAGESIZE); for (t = 0; t < NIDT; ++t) { /* * Note that since boot runs without a TSS, the * double fault handler cannot use an alternate stack (64-bit). */ set_gatesegd(&bop_idt[t], &bop_trap_handler, B64CODE_SEL, SDT_SYSIGT, TRP_KPL, 0); } bop_idt_info.dtr_limit = (NIDT * sizeof (gate_desc_t)) - 1; bop_idt_info.dtr_base = (uintptr_t)bop_idt; wr_idtr(&bop_idt_info); } #endif /* !defined(__xpv) */ /* * This is where we enter the kernel. It dummies up the boot_ops and * boot_syscalls vectors and jumps off to _kobj_boot() */ void _start(struct xboot_info *xbp) { bootops_t *bops = &bootop; extern void _kobj_boot(); /* * 1st off - initialize the console for any error messages */ xbootp = xbp; #ifdef __xpv HYPERVISOR_shared_info = (void *)xbp->bi_shared_info; xen_info = xbp->bi_xen_start_info; #endif #ifndef __xpv if (*((uint32_t *)(FASTBOOT_SWTCH_PA + FASTBOOT_STACK_OFFSET)) == FASTBOOT_MAGIC) { post_fastreboot = 1; *((uint32_t *)(FASTBOOT_SWTCH_PA + FASTBOOT_STACK_OFFSET)) = 0; } #endif bcons_init(xbp); have_console = 1; /* * enable debugging */ if (find_boot_prop("kbm_debug") != NULL) kbm_debug = 1; DBG_MSG("\n\n*** Entered illumos in _start() cmdline is: "); DBG_MSG((char *)xbp->bi_cmdline); DBG_MSG("\n\n\n"); /* * physavail is no longer used by startup */ bm.physinstalled = xbp->bi_phys_install; bm.pcimem = xbp->bi_pcimem; bm.rsvdmem = xbp->bi_rsvdmem; bm.physavail = NULL; /* * initialize the boot time allocator */ next_phys = xbp->bi_next_paddr; DBG(next_phys); next_virt = (uintptr_t)xbp->bi_next_vaddr; DBG(next_virt); DBG_MSG("Initializing boot time memory management..."); #ifdef __xpv { xen_platform_parameters_t p; /* This call shouldn't fail, dboot already did it once. */ (void) HYPERVISOR_xen_version(XENVER_platform_parameters, &p); mfn_to_pfn_mapping = (pfn_t *)(xen_virt_start = p.virt_start); DBG(xen_virt_start); } #endif kbm_init(xbp); DBG_MSG("done\n"); /* * Fill in the bootops vector */ bops->bsys_version = BO_VERSION; bops->boot_mem = &bm; bops->bsys_alloc = do_bsys_alloc; bops->bsys_free = do_bsys_free; bops->bsys_getproplen = do_bsys_getproplen; bops->bsys_getprop = do_bsys_getprop; bops->bsys_nextprop = do_bsys_nextprop; bops->bsys_printf = bop_printf; bops->bsys_doint = do_bsys_doint; /* * BOP_EALLOC() is no longer needed */ bops->bsys_ealloc = do_bsys_ealloc; #ifdef __xpv /* * On domain 0 we need to free up some physical memory that is * usable for DMA. Since GRUB loaded the boot_archive, it is * sitting in low MFN memory. We'll relocated the boot archive * pages to high PFN memory. */ if (DOMAIN_IS_INITDOMAIN(xen_info)) relocate_boot_archive(xbp); #endif #ifndef __xpv /* * Install an IDT to catch early pagefaults (shouldn't have any). * Also needed for kmdb. */ bop_idt_init(); #endif /* Set up the shadow fb for framebuffer console */ boot_fb_shadow_init(bops); /* * Start building the boot properties from the command line */ DBG_MSG("Initializing boot properties:\n"); build_boot_properties(xbp); if (find_boot_prop("prom_debug") || kbm_debug) { char *value; value = do_bsys_alloc(NULL, NULL, MMU_PAGESIZE, MMU_PAGESIZE); boot_prop_display(value); } /* * jump into krtld... */ _kobj_boot(&bop_sysp, NULL, bops, NULL); } /*ARGSUSED*/ static caddr_t no_more_alloc(bootops_t *bop, caddr_t virthint, size_t size, int align) { panic("Attempt to bsys_alloc() too late\n"); return (NULL); } /*ARGSUSED*/ static void no_more_free(bootops_t *bop, caddr_t virt, size_t size) { panic("Attempt to bsys_free() too late\n"); } void bop_no_more_mem(void) { DBG(total_bop_alloc_scratch); DBG(total_bop_alloc_kernel); bootops->bsys_alloc = no_more_alloc; bootops->bsys_free = no_more_free; } /* * Set ACPI firmware properties */ static caddr_t vmap_phys(size_t length, paddr_t pa) { paddr_t start, end; caddr_t va; size_t len, page; #ifdef __xpv pa = pfn_to_pa(xen_assign_pfn(mmu_btop(pa))) | (pa & MMU_PAGEOFFSET); #endif start = P2ALIGN(pa, MMU_PAGESIZE); end = P2ROUNDUP(pa + length, MMU_PAGESIZE); len = end - start; va = (caddr_t)alloc_vaddr(len, MMU_PAGESIZE); for (page = 0; page < len; page += MMU_PAGESIZE) kbm_map((uintptr_t)va + page, start + page, 0, 0); return (va + (pa & MMU_PAGEOFFSET)); } static uint8_t checksum_table(uint8_t *tp, size_t len) { uint8_t sum = 0; while (len-- > 0) sum += *tp++; return (sum); } static int valid_rsdp(ACPI_TABLE_RSDP *rp) { /* validate the V1.x checksum */ if (checksum_table((uint8_t *)rp, ACPI_RSDP_CHECKSUM_LENGTH) != 0) return (0); /* If pre-ACPI 2.0, this is a valid RSDP */ if (rp->Revision < 2) return (1); /* validate the V2.x checksum */ if (checksum_table((uint8_t *)rp, ACPI_RSDP_XCHECKSUM_LENGTH) != 0) return (0); return (1); } /* * Scan memory range for an RSDP; * see ACPI 3.0 Spec, 5.2.5.1 */ static ACPI_TABLE_RSDP * scan_rsdp(paddr_t *paddrp, size_t len) { paddr_t paddr = *paddrp; caddr_t ptr; ptr = vmap_phys(len, paddr); while (len > 0) { if (strncmp(ptr, ACPI_SIG_RSDP, strlen(ACPI_SIG_RSDP)) == 0 && valid_rsdp((ACPI_TABLE_RSDP *)ptr)) { *paddrp = paddr; return ((ACPI_TABLE_RSDP *)ptr); } ptr += ACPI_RSDP_SCAN_STEP; paddr += ACPI_RSDP_SCAN_STEP; len -= ACPI_RSDP_SCAN_STEP; } return (NULL); } /* * Locate the ACPI RSDP. We search in a particular order: * * - If the bootloader told us the location of the RSDP (via the EFI system * table), try that first. * - Otherwise, look in the EBDA and BIOS memory as per ACPI 5.2.5.1 (legacy * case). * - Finally, our bootloader may have a copy of the RSDP in its info: this might * get freed after boot, so we always prefer to find the original RSDP first. * * Once found, we set acpi-root-tab property (a physical address) for the * benefit of acpica, acpidump etc. */ static ACPI_TABLE_RSDP * find_rsdp(struct xboot_info *xbp) { ACPI_TABLE_RSDP *rsdp = NULL; paddr_t paddr = 0; if (do_bsys_getproplen(NULL, "acpi-root-tab") == sizeof (uint64_t)) { (void) do_bsys_getprop(NULL, "acpi-root-tab", &paddr); rsdp = scan_rsdp(&paddr, sizeof (*rsdp)); } #ifndef __xpv if (rsdp == NULL && xbp->bi_acpi_rsdp != NULL) { paddr = (uintptr_t)xbp->bi_acpi_rsdp; rsdp = scan_rsdp(&paddr, sizeof (*rsdp)); } #endif if (rsdp == NULL) { uint16_t *ebda_seg = (uint16_t *)vmap_phys(sizeof (uint16_t), ACPI_EBDA_PTR_LOCATION); paddr = *ebda_seg << 4; rsdp = scan_rsdp(&paddr, ACPI_EBDA_WINDOW_SIZE); } if (rsdp == NULL) { paddr = ACPI_HI_RSDP_WINDOW_BASE; rsdp = scan_rsdp(&paddr, ACPI_HI_RSDP_WINDOW_SIZE); } #ifndef __xpv if (rsdp == NULL && xbp->bi_acpi_rsdp_copy != NULL) { paddr = (uintptr_t)xbp->bi_acpi_rsdp_copy; rsdp = scan_rsdp(&paddr, sizeof (*rsdp)); } #endif if (rsdp == NULL) { bop_printf(NULL, "no RSDP found!\n"); return (NULL); } if (kbm_debug) bop_printf(NULL, "RSDP found at physical 0x%lx\n", paddr); if (do_bsys_getproplen(NULL, "acpi-root-tab") != sizeof (uint64_t)) bsetprop64("acpi-root-tab", paddr); return (rsdp); } static ACPI_TABLE_HEADER * map_fw_table(paddr_t table_addr) { ACPI_TABLE_HEADER *tp; size_t len = MAX(sizeof (*tp), MMU_PAGESIZE); /* * Map at least a page; if the table is larger than this, remap it */ tp = (ACPI_TABLE_HEADER *)vmap_phys(len, table_addr); if (tp->Length > len) tp = (ACPI_TABLE_HEADER *)vmap_phys(tp->Length, table_addr); return (tp); } static ACPI_TABLE_HEADER * find_fw_table(ACPI_TABLE_RSDP *rsdp, char *signature) { static int revision = 0; static ACPI_TABLE_XSDT *xsdt; static int len; paddr_t xsdt_addr; ACPI_TABLE_HEADER *tp; paddr_t table_addr; int n; if (strlen(signature) != ACPI_NAME_SIZE) return (NULL); /* * Reading the ACPI 3.0 Spec, section 5.2.5.3 will help * understand this code. If we haven't already found the RSDT/XSDT, * revision will be 0. Find the RSDP and check the revision * to find out whether to use the RSDT or XSDT. If revision is * 0 or 1, use the RSDT and set internal revision to 1; if it is 2, * use the XSDT. If the XSDT address is 0, though, fall back to * revision 1 and use the RSDT. */ xsdt_addr = 0; if (revision == 0) { if (rsdp == NULL) return (NULL); revision = rsdp->Revision; /* * ACPI 6.0 states that current revision is 2 * from acpi_table_rsdp definition: * Must be (0) for ACPI 1.0 or (2) for ACPI 2.0+ */ if (revision > 2) revision = 2; switch (revision) { case 2: /* * Use the XSDT unless BIOS is buggy and * claims to be rev 2 but has a null XSDT * address */ xsdt_addr = rsdp->XsdtPhysicalAddress; if (xsdt_addr != 0) break; /* FALLTHROUGH */ case 0: /* treat RSDP rev 0 as revision 1 internally */ revision = 1; /* FALLTHROUGH */ case 1: /* use the RSDT for rev 0/1 */ xsdt_addr = rsdp->RsdtPhysicalAddress; break; default: /* unknown revision */ revision = 0; break; } if (revision == 0) return (NULL); /* cache the XSDT info */ xsdt = (ACPI_TABLE_XSDT *)map_fw_table(xsdt_addr); len = (xsdt->Header.Length - sizeof (xsdt->Header)) / ((revision == 1) ? sizeof (uint32_t) : sizeof (uint64_t)); } /* * Scan the table headers looking for a signature match */ for (n = 0; n < len; n++) { ACPI_TABLE_RSDT *rsdt = (ACPI_TABLE_RSDT *)xsdt; table_addr = (revision == 1) ? rsdt->TableOffsetEntry[n] : xsdt->TableOffsetEntry[n]; if (table_addr == 0) continue; tp = map_fw_table(table_addr); if (strncmp(tp->Signature, signature, ACPI_NAME_SIZE) == 0) { return (tp); } } return (NULL); } static void process_mcfg(ACPI_TABLE_MCFG *tp) { ACPI_MCFG_ALLOCATION *cfg_baap; char *cfg_baa_endp; int64_t ecfginfo[4]; cfg_baap = (ACPI_MCFG_ALLOCATION *)((uintptr_t)tp + sizeof (*tp)); cfg_baa_endp = ((char *)tp) + tp->Header.Length; while ((char *)cfg_baap < cfg_baa_endp) { if (cfg_baap->Address != 0 && cfg_baap->PciSegment == 0) { ecfginfo[0] = cfg_baap->Address; ecfginfo[1] = cfg_baap->PciSegment; ecfginfo[2] = cfg_baap->StartBusNumber; ecfginfo[3] = cfg_baap->EndBusNumber; bsetprop(DDI_PROP_TYPE_INT64, MCFG_PROPNAME, strlen(MCFG_PROPNAME), ecfginfo, sizeof (ecfginfo)); break; } cfg_baap++; } } #ifndef __xpv static void process_madt_entries(ACPI_TABLE_MADT *tp, uint32_t *cpu_countp, uint32_t *cpu_possible_countp, uint32_t *cpu_apicid_array) { ACPI_SUBTABLE_HEADER *item, *end; uint32_t cpu_count = 0; uint32_t cpu_possible_count = 0; /* * Determine number of CPUs and keep track of "final" APIC ID * for each CPU by walking through ACPI MADT processor list */ end = (ACPI_SUBTABLE_HEADER *)(tp->Header.Length + (uintptr_t)tp); item = (ACPI_SUBTABLE_HEADER *)((uintptr_t)tp + sizeof (*tp)); while (item < end) { switch (item->Type) { case ACPI_MADT_TYPE_LOCAL_APIC: { ACPI_MADT_LOCAL_APIC *cpu = (ACPI_MADT_LOCAL_APIC *) item; if (cpu->LapicFlags & ACPI_MADT_ENABLED) { if (cpu_apicid_array != NULL) cpu_apicid_array[cpu_count] = cpu->Id; cpu_count++; } cpu_possible_count++; break; } case ACPI_MADT_TYPE_LOCAL_X2APIC: { ACPI_MADT_LOCAL_X2APIC *cpu = (ACPI_MADT_LOCAL_X2APIC *) item; if (cpu->LapicFlags & ACPI_MADT_ENABLED) { if (cpu_apicid_array != NULL) cpu_apicid_array[cpu_count] = cpu->LocalApicId; cpu_count++; } cpu_possible_count++; break; } default: if (kbm_debug) bop_printf(NULL, "MADT type %d\n", item->Type); break; } item = (ACPI_SUBTABLE_HEADER *)((uintptr_t)item + item->Length); } if (cpu_countp) *cpu_countp = cpu_count; if (cpu_possible_countp) *cpu_possible_countp = cpu_possible_count; } static void process_madt(ACPI_TABLE_MADT *tp) { uint32_t cpu_count = 0; uint32_t cpu_possible_count = 0; uint32_t *cpu_apicid_array; /* x2APIC ID is 32bit! */ if (tp != NULL) { /* count cpu's */ process_madt_entries(tp, &cpu_count, &cpu_possible_count, NULL); cpu_apicid_array = (uint32_t *)do_bsys_alloc(NULL, NULL, cpu_count * sizeof (*cpu_apicid_array), MMU_PAGESIZE); if (cpu_apicid_array == NULL) bop_panic("Not enough memory for APIC ID array"); /* copy IDs */ process_madt_entries(tp, NULL, NULL, cpu_apicid_array); /* * Make boot property for array of "final" APIC IDs for each * CPU */ bsetprop(DDI_PROP_TYPE_INT, BP_CPU_APICID_ARRAY, strlen(BP_CPU_APICID_ARRAY), cpu_apicid_array, cpu_count * sizeof (*cpu_apicid_array)); } /* * Check whether property plat-max-ncpus is already set. */ if (do_bsys_getproplen(NULL, PLAT_MAX_NCPUS_NAME) < 0) { /* * Set plat-max-ncpus to number of maximum possible CPUs given * in MADT if it hasn't been set. * There's no formal way to detect max possible CPUs supported * by platform according to ACPI spec3.0b. So current CPU * hotplug implementation expects that all possible CPUs will * have an entry in MADT table and set plat-max-ncpus to number * of entries in MADT. * With introducing of ACPI4.0, Maximum System Capability Table * (MSCT) provides maximum number of CPUs supported by platform. * If MSCT is unavailable, fall back to old way. */ if (tp != NULL) bsetpropsi(PLAT_MAX_NCPUS_NAME, cpu_possible_count); } /* * Set boot property boot-max-ncpus to number of CPUs existing at * boot time. boot-max-ncpus is mainly used for optimization. */ if (tp != NULL) bsetpropsi(BOOT_MAX_NCPUS_NAME, cpu_count); /* * User-set boot-ncpus overrides firmware count */ if (do_bsys_getproplen(NULL, BOOT_NCPUS_NAME) >= 0) return; /* * Set boot property boot-ncpus to number of active CPUs given in MADT * if it hasn't been set yet. */ if (tp != NULL) bsetpropsi(BOOT_NCPUS_NAME, cpu_count); } static void process_srat(ACPI_TABLE_SRAT *tp) { ACPI_SUBTABLE_HEADER *item, *end; int i; int proc_num, mem_num; #pragma pack(1) struct { uint32_t domain; uint32_t apic_id; uint32_t sapic_id; } processor; struct { uint32_t domain; uint32_t x2apic_id; } x2apic; struct { uint32_t domain; uint64_t addr; uint64_t length; uint32_t flags; } memory; #pragma pack() char prop_name[30]; uint64_t maxmem = 0; if (tp == NULL) return; proc_num = mem_num = 0; end = (ACPI_SUBTABLE_HEADER *)(tp->Header.Length + (uintptr_t)tp); item = (ACPI_SUBTABLE_HEADER *)((uintptr_t)tp + sizeof (*tp)); while (item < end) { switch (item->Type) { case ACPI_SRAT_TYPE_CPU_AFFINITY: { ACPI_SRAT_CPU_AFFINITY *cpu = (ACPI_SRAT_CPU_AFFINITY *) item; if (!(cpu->Flags & ACPI_SRAT_CPU_ENABLED)) break; processor.domain = cpu->ProximityDomainLo; for (i = 0; i < 3; i++) processor.domain += cpu->ProximityDomainHi[i] << ((i + 1) * 8); processor.apic_id = cpu->ApicId; processor.sapic_id = cpu->LocalSapicEid; (void) snprintf(prop_name, 30, "acpi-srat-processor-%d", proc_num); bsetprop(DDI_PROP_TYPE_INT, prop_name, strlen(prop_name), &processor, sizeof (processor)); proc_num++; break; } case ACPI_SRAT_TYPE_MEMORY_AFFINITY: { ACPI_SRAT_MEM_AFFINITY *mem = (ACPI_SRAT_MEM_AFFINITY *)item; if (!(mem->Flags & ACPI_SRAT_MEM_ENABLED)) break; memory.domain = mem->ProximityDomain; memory.addr = mem->BaseAddress; memory.length = mem->Length; memory.flags = mem->Flags; (void) snprintf(prop_name, 30, "acpi-srat-memory-%d", mem_num); bsetprop(DDI_PROP_TYPE_INT, prop_name, strlen(prop_name), &memory, sizeof (memory)); if ((mem->Flags & ACPI_SRAT_MEM_HOT_PLUGGABLE) && (memory.addr + memory.length > maxmem)) { maxmem = memory.addr + memory.length; } mem_num++; break; } case ACPI_SRAT_TYPE_X2APIC_CPU_AFFINITY: { ACPI_SRAT_X2APIC_CPU_AFFINITY *x2cpu = (ACPI_SRAT_X2APIC_CPU_AFFINITY *) item; if (!(x2cpu->Flags & ACPI_SRAT_CPU_ENABLED)) break; x2apic.domain = x2cpu->ProximityDomain; x2apic.x2apic_id = x2cpu->ApicId; (void) snprintf(prop_name, 30, "acpi-srat-processor-%d", proc_num); bsetprop(DDI_PROP_TYPE_INT, prop_name, strlen(prop_name), &x2apic, sizeof (x2apic)); proc_num++; break; } default: if (kbm_debug) bop_printf(NULL, "SRAT type %d\n", item->Type); break; } item = (ACPI_SUBTABLE_HEADER *) (item->Length + (uintptr_t)item); } /* * The maximum physical address calculated from the SRAT table is more * accurate than that calculated from the MSCT table. */ if (maxmem != 0) { plat_dr_physmax = btop(maxmem); } } static void process_slit(ACPI_TABLE_SLIT *tp) { /* * Check the number of localities; if it's too huge, we just * return and locality enumeration code will handle this later, * if possible. * * Note that the size of the table is the square of the * number of localities; if the number of localities exceeds * UINT16_MAX, the table size may overflow an int when being * passed to bsetprop() below. */ if (tp->LocalityCount >= SLIT_LOCALITIES_MAX) return; bsetprop64(SLIT_NUM_PROPNAME, tp->LocalityCount); bsetprop(DDI_PROP_TYPE_BYTE, SLIT_PROPNAME, strlen(SLIT_PROPNAME), &tp->Entry, tp->LocalityCount * tp->LocalityCount); } static ACPI_TABLE_MSCT * process_msct(ACPI_TABLE_MSCT *tp) { int last_seen = 0; int proc_num = 0; ACPI_MSCT_PROXIMITY *item, *end; extern uint64_t plat_dr_options; ASSERT(tp != NULL); end = (ACPI_MSCT_PROXIMITY *)(tp->Header.Length + (uintptr_t)tp); for (item = (void *)((uintptr_t)tp + tp->ProximityOffset); item < end; item = (void *)(item->Length + (uintptr_t)item)) { /* * Sanity check according to section 5.2.19.1 of ACPI 4.0. * Revision 1 * Length 22 */ if (item->Revision != 1 || item->Length != 22) { cmn_err(CE_CONT, "?boot: unknown proximity domain structure in MSCT " "with Revision(%d), Length(%d).\n", (int)item->Revision, (int)item->Length); return (NULL); } else if (item->RangeStart > item->RangeEnd) { cmn_err(CE_CONT, "?boot: invalid proximity domain structure in MSCT " "with RangeStart(%u), RangeEnd(%u).\n", item->RangeStart, item->RangeEnd); return (NULL); } else if (item->RangeStart != last_seen) { /* * Items must be organized in ascending order of the * proximity domain enumerations. */ cmn_err(CE_CONT, "?boot: invalid proximity domain structure in MSCT," " items are not orginized in ascending order.\n"); return (NULL); } /* * If ProcessorCapacity is 0 then there would be no CPUs in this * domain. */ if (item->ProcessorCapacity != 0) { proc_num += (item->RangeEnd - item->RangeStart + 1) * item->ProcessorCapacity; } last_seen = item->RangeEnd - item->RangeStart + 1; /* * Break out if all proximity domains have been processed. * Some BIOSes may have unused items at the end of MSCT table. */ if (last_seen > tp->MaxProximityDomains) { break; } } if (last_seen != tp->MaxProximityDomains + 1) { cmn_err(CE_CONT, "?boot: invalid proximity domain structure in MSCT, " "proximity domain count doesn't match.\n"); return (NULL); } /* * Set plat-max-ncpus property if it hasn't been set yet. */ if (do_bsys_getproplen(NULL, PLAT_MAX_NCPUS_NAME) < 0) { if (proc_num != 0) { bsetpropsi(PLAT_MAX_NCPUS_NAME, proc_num); } } /* * Use Maximum Physical Address from the MSCT table as upper limit for * memory hot-adding by default. It may be overridden by value from * the SRAT table or the "plat-dr-physmax" boot option. */ plat_dr_physmax = btop(tp->MaxAddress + 1); /* * Existence of MSCT implies CPU/memory hotplug-capability for the * platform. */ plat_dr_options |= PLAT_DR_FEATURE_CPU; plat_dr_options |= PLAT_DR_FEATURE_MEMORY; return (tp); } #else /* __xpv */ static void enumerate_xen_cpus() { processorid_t id, max_id; /* * User-set boot-ncpus overrides enumeration */ if (do_bsys_getproplen(NULL, BOOT_NCPUS_NAME) >= 0) return; /* * Probe every possible virtual CPU id and remember the * highest id present; the count of CPUs is one greater * than this. This tacitly assumes at least cpu 0 is present. */ max_id = 0; for (id = 0; id < MAX_VIRT_CPUS; id++) if (HYPERVISOR_vcpu_op(VCPUOP_is_up, id, NULL) == 0) max_id = id; bsetpropsi(BOOT_NCPUS_NAME, max_id+1); } #endif /* __xpv */ /*ARGSUSED*/ static void build_firmware_properties(struct xboot_info *xbp) { ACPI_TABLE_HEADER *tp = NULL; ACPI_TABLE_RSDP *rsdp; #ifndef __xpv if (xbp->bi_uefi_arch == XBI_UEFI_ARCH_64) { if (do_bsys_getproplen(NULL, "efi-systype") == -1) bsetprops("efi-systype", "64"); if (do_bsys_getproplen(NULL, "efi-systab") == -1) bsetprop64("efi-systab", (uint64_t)(uintptr_t)xbp->bi_uefi_systab); if (kbm_debug) bop_printf(NULL, "64-bit UEFI detected.\n"); } else if (xbp->bi_uefi_arch == XBI_UEFI_ARCH_32) { if (do_bsys_getproplen(NULL, "efi-systype") == -1) bsetprops("efi-systype", "32"); if (do_bsys_getproplen(NULL, "efi-systab") == -1) bsetprop64("efi-systab", (uint64_t)(uintptr_t)xbp->bi_uefi_systab); if (kbm_debug) bop_printf(NULL, "32-bit UEFI detected.\n"); } if (xbp->bi_smbios != NULL && do_bsys_getproplen(NULL, "smbios-address") == -1) { bsetprop64("smbios-address", (uint64_t)(uintptr_t)xbp->bi_smbios); } rsdp = find_rsdp(xbp); if ((tp = find_fw_table(rsdp, ACPI_SIG_MSCT)) != NULL) msct_ptr = process_msct((ACPI_TABLE_MSCT *)tp); else msct_ptr = NULL; if ((tp = find_fw_table(rsdp, ACPI_SIG_MADT)) != NULL) process_madt((ACPI_TABLE_MADT *)tp); if ((srat_ptr = (ACPI_TABLE_SRAT *) find_fw_table(rsdp, ACPI_SIG_SRAT)) != NULL) process_srat(srat_ptr); if ((slit_ptr = (ACPI_TABLE_SLIT *)find_fw_table(rsdp, ACPI_SIG_SLIT)) != NULL) process_slit(slit_ptr); tp = find_fw_table(rsdp, ACPI_SIG_MCFG); #else /* __xpv */ enumerate_xen_cpus(); if (DOMAIN_IS_INITDOMAIN(xen_info)) { rsdp = find_rsdp(xbp); tp = find_fw_table(rsdp, ACPI_SIG_MCFG); } #endif /* __xpv */ if (tp != NULL) process_mcfg((ACPI_TABLE_MCFG *)tp); /* * Map the first HPET table (if it exists) and save the address. * If the HPET is required to calibrate the TSC, we require the * HPET table prior to being able to load modules, so we cannot use * the acpica module (and thus AcpiGetTable()) to locate it. */ if ((tp = find_fw_table(rsdp, ACPI_SIG_HPET)) != NULL) bsetprop64("hpet-table", (uint64_t)(uintptr_t)tp); } /* * fake up a boot property for deferred early console output * this is used by both graphical boot and the (developer only) * USB serial console */ void * defcons_init(size_t size) { static char *p = NULL; p = do_bsys_alloc(NULL, NULL, size, MMU_PAGESIZE); *p = 0; bsetprop32("deferred-console-buf", (uint32_t)((uintptr_t)&p)); return (p); } /*ARGSUSED*/ int boot_compinfo(int fd, struct compinfo *cbp) { cbp->iscmp = 0; cbp->blksize = MAXBSIZE; return (0); } /* * Get an integer value for given boot property */ int bootprop_getval(const char *prop_name, u_longlong_t *prop_value) { int boot_prop_len; char str[BP_MAX_STRLEN]; u_longlong_t value; boot_prop_len = BOP_GETPROPLEN(bootops, prop_name); if (boot_prop_len < 0 || boot_prop_len >= sizeof (str) || BOP_GETPROP(bootops, prop_name, str) < 0 || kobj_getvalue(str, &value) == -1) return (-1); if (prop_value) *prop_value = value; return (0); } int bootprop_getstr(const char *prop_name, char *buf, size_t buflen) { int boot_prop_len = BOP_GETPROPLEN(bootops, prop_name); if (boot_prop_len < 0 || boot_prop_len >= buflen || BOP_GETPROP(bootops, prop_name, buf) < 0) return (-1); return (0); } /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved. */ /* * This file contains the functions for performing Fast Reboot -- a * reboot which bypasses the firmware and bootloader, considerably * reducing downtime. * * fastboot_load_kernel(): This function is invoked by mdpreboot() in the * reboot path. It loads the new kernel and boot archive into memory, builds * the data structure containing sufficient information about the new * kernel and boot archive to be passed to the fast reboot switcher * (see fb_swtch_src.S for details). When invoked the switcher relocates * the new kernel and boot archive to physically contiguous low memory, * similar to where the boot loader would have loaded them, and jumps to * the new kernel. * * If fastreboot_onpanic is enabled, fastboot_load_kernel() is called * by fastreboot_post_startup() to load the back up kernel in case of * panic. * * The physical addresses of the memory allocated for the new kernel, boot * archive and their page tables must be above where the boot archive ends * after it has been relocated by the switcher, otherwise the new files * and their page tables could be overridden during relocation. * * fast_reboot(): This function is invoked by mdboot() once it's determined * that the system is capable of fast reboot. It jumps to the fast reboot * switcher with the data structure built by fastboot_load_kernel() as the * argument. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* * Macro to determine how many pages are needed for PTEs to map a particular * file. Allocate one extra page table entry for terminating the list. */ #define FASTBOOT_PTE_LIST_SIZE(fsize) \ P2ROUNDUP((((fsize) >> PAGESHIFT) + 1) * sizeof (x86pte_t), PAGESIZE) /* * Data structure containing necessary information for the fast reboot * switcher to jump to the new kernel. */ fastboot_info_t newkernel = { 0 }; char fastboot_args[OBP_MAXPATHLEN]; static char fastboot_filename[2][OBP_MAXPATHLEN] = { { 0 }, { 0 }}; static x86pte_t ptp_bits = PT_VALID | PT_REF | PT_USER | PT_WRITABLE; static x86pte_t pte_bits = PT_VALID | PT_REF | PT_MOD | PT_NOCONSIST | PT_WRITABLE; static uint_t fastboot_shift_amt_pae[] = {12, 21, 30, 39}; /* Index into Fast Reboot not supported message array */ static uint32_t fastreboot_nosup_id = FBNS_DEFAULT; /* Fast Reboot not supported message array */ static const char * const fastreboot_nosup_desc[FBNS_END] = { #define fastboot_nosup_msg(id, str) str, #include }; int fastboot_debug = 0; int fastboot_contig = 0; /* * Fake starting va for new kernel and boot archive. */ static uintptr_t fake_va = FASTBOOT_FAKE_VA; /* * Reserve memory below PA 1G in preparation of fast reboot. * * This variable is only checked when fastreboot_capable is set, but * fastreboot_onpanic is not set. The amount of memory reserved * is negligible, but just in case we are really short of low memory, * this variable will give us a backdoor to not consume memory at all. */ int reserve_mem_enabled = 1; /* * Mutex to protect fastreboot_onpanic. */ kmutex_t fastreboot_config_mutex; /* * Amount of memory below PA 1G to reserve for constructing the multiboot * data structure and the page tables as we tend to run out of those * when more drivers are loaded. */ static size_t fastboot_mbi_size = 0x2000; /* 8K */ static size_t fastboot_pagetable_size = 0x5000; /* 20K */ /* * Minimum system uptime in clock_t before Fast Reboot should be used * on panic. Will be initialized in fastboot_post_startup(). */ clock_t fastreboot_onpanic_uptime = LONG_MAX; /* * lbolt value when the system booted. This value will be used if the system * panics to calculate how long the system has been up. If the uptime is less * than fastreboot_onpanic_uptime, a reboot through BIOS will be performed to * avoid a potential panic/reboot loop. */ clock_t lbolt_at_boot = LONG_MAX; /* * Use below 1G for page tables as * 1. we are only doing 1:1 mapping of the bottom 1G of physical memory. * 2. we are using 2G as the fake virtual address for the new kernel and * boot archive. */ static ddi_dma_attr_t fastboot_below_1G_dma_attr = { DMA_ATTR_V0, 0x0000000008000000ULL, /* dma_attr_addr_lo: 128MB */ 0x000000003FFFFFFFULL, /* dma_attr_addr_hi: 1G */ 0x00000000FFFFFFFFULL, /* dma_attr_count_max */ 0x0000000000001000ULL, /* dma_attr_align: 4KB */ 1, /* dma_attr_burstsize */ 1, /* dma_attr_minxfer */ 0x00000000FFFFFFFFULL, /* dma_attr_maxxfer */ 0x00000000FFFFFFFFULL, /* dma_attr_seg */ 1, /* dma_attr_sgllen */ 0x1000ULL, /* dma_attr_granular */ 0, /* dma_attr_flags */ }; static ddi_dma_attr_t fastboot_dma_attr = { DMA_ATTR_V0, 0x0000000008000000ULL, /* dma_attr_addr_lo: 128MB */ 0xFFFFFFFFFFFFFFFFULL, /* dma_attr_addr_hi: 2^64B */ 0x00000000FFFFFFFFULL, /* dma_attr_count_max */ 0x0000000000001000ULL, /* dma_attr_align: 4KB */ 1, /* dma_attr_burstsize */ 1, /* dma_attr_minxfer */ 0x00000000FFFFFFFFULL, /* dma_attr_maxxfer */ 0x00000000FFFFFFFFULL, /* dma_attr_seg */ 1, /* dma_attr_sgllen */ 0x1000ULL, /* dma_attr_granular */ 0, /* dma_attr_flags */ }; /* * Various information saved from the previous boot to reconstruct * multiboot_info. */ extern multiboot_info_t saved_mbi; extern mb_memory_map_t saved_mmap[FASTBOOT_SAVED_MMAP_COUNT]; extern uint8_t saved_drives[FASTBOOT_SAVED_DRIVES_SIZE]; extern char saved_cmdline[FASTBOOT_SAVED_CMDLINE_LEN]; extern int saved_cmdline_len; extern size_t saved_file_size[]; extern void* contig_alloc(size_t size, ddi_dma_attr_t *attr, uintptr_t align, int cansleep); extern void contig_free(void *addr, size_t size); /* PRINTLIKE */ extern void vprintf(const char *, va_list); /* * Need to be able to get boot_archives from other places */ #define BOOTARCHIVE64 "/boot/boot_archive" #define BOOTARCHIVE32 BOOTARCHIVE64 #define BOOTARCHIVE64_FAILSAFE "/boot/x86.miniroot-safe" #define BOOTARCHIVE32_FAILSAFE BOOTARCHIVE64_FAILSAFE #define FAILSAFE_BOOTFILE64 "/boot/kernel/unix" #define FAILSAFE_BOOTFILE32 FAILSAFE_BOOTFILE64 static uint_t fastboot_vatoindex(fastboot_info_t *, uintptr_t, int); static void fastboot_map_with_size(fastboot_info_t *, uintptr_t, paddr_t, size_t, int); static void fastboot_build_pagetables(fastboot_info_t *); static int fastboot_build_mbi(char *, fastboot_info_t *); static void fastboot_free_file(fastboot_file_t *); static const char fastboot_enomem_msg[] = "!Fastboot: Couldn't allocate 0x%" PRIx64" bytes below %s to do fast reboot"; static void dprintf(char *fmt, ...) { va_list adx; if (!fastboot_debug) return; va_start(adx, fmt); vprintf(fmt, adx); va_end(adx); } /* * Return the index corresponding to a virt address at a given page table level. */ static uint_t fastboot_vatoindex(fastboot_info_t *nk, uintptr_t va, int level) { return ((va >> nk->fi_shift_amt[level]) & (nk->fi_ptes_per_table - 1)); } /* * Add mapping from vstart to pstart for the specified size. * vstart, pstart and size should all have been aligned at 2M boundaries. */ static void fastboot_map_with_size(fastboot_info_t *nk, uintptr_t vstart, paddr_t pstart, size_t size, int level) { x86pte_t pteval, *table; uintptr_t vaddr; paddr_t paddr; int index, l; table = (x86pte_t *)(nk->fi_pagetable_va); for (l = nk->fi_top_level; l >= level; l--) { index = fastboot_vatoindex(nk, vstart, l); if (l == level) { /* * Last level. Program the page table entries. */ for (vaddr = vstart, paddr = pstart; vaddr < vstart + size; vaddr += (1ULL << nk->fi_shift_amt[l]), paddr += (1ULL << nk->fi_shift_amt[l])) { uint_t index = fastboot_vatoindex(nk, vaddr, l); if (l > 0) pteval = paddr | pte_bits | PT_PAGESIZE; else pteval = paddr | pte_bits; table[index] = pteval; } } else if (table[index] & PT_VALID) { table = (x86pte_t *) ((uintptr_t)(((paddr_t)table[index] & MMU_PAGEMASK) - nk->fi_pagetable_pa) + nk->fi_pagetable_va); } else { /* * Intermediate levels. * Program with either valid bit or PTP bits. */ if (l == nk->fi_top_level) { ASSERT(nk->fi_top_level == 3); table[index] = nk->fi_next_table_pa | ptp_bits; } else { table[index] = nk->fi_next_table_pa | ptp_bits; } table = (x86pte_t *)(nk->fi_next_table_va); nk->fi_next_table_va += MMU_PAGESIZE; nk->fi_next_table_pa += MMU_PAGESIZE; } } } /* * Build page tables for the lower 1G of physical memory using 2M * pages, and prepare page tables for mapping new kernel and boot * archive pages using 4K pages. */ static void fastboot_build_pagetables(fastboot_info_t *nk) { /* * Map lower 1G physical memory. Use large pages. */ fastboot_map_with_size(nk, 0, 0, ONE_GIG, 1); /* * Map one 4K page to get the middle page tables set up. */ fake_va = P2ALIGN_TYPED(fake_va, nk->fi_lpagesize, uintptr_t); fastboot_map_with_size(nk, fake_va, nk->fi_files[0].fb_pte_list_va[0] & MMU_PAGEMASK, PAGESIZE, 0); } /* * Sanity check. Look for dboot offset. */ static int fastboot_elf64_find_dboot_load_offset(void *img, off_t imgsz, uint32_t *offp) { Elf64_Ehdr *ehdr = (Elf64_Ehdr *)img; Elf64_Phdr *phdr; uint8_t *phdrbase; int i; if ((ehdr->e_phoff + ehdr->e_phnum * ehdr->e_phentsize) >= imgsz) return (-1); phdrbase = (uint8_t *)img + ehdr->e_phoff; for (i = 0; i < ehdr->e_phnum; i++) { phdr = (Elf64_Phdr *)(phdrbase + ehdr->e_phentsize * i); if (phdr->p_type == PT_LOAD) { if (phdr->p_vaddr == phdr->p_paddr && phdr->p_vaddr == DBOOT_ENTRY_ADDRESS) { ASSERT(phdr->p_offset <= UINT32_MAX); *offp = (uint32_t)phdr->p_offset; return (0); } } } return (-1); } /* * Initialize text and data section information for 32-bit kernel. * sectcntp - is both input/output parameter. * On entry, *sectcntp contains maximum allowable number of sections; * on return, it contains the actual number of sections filled. */ static int fastboot_elf32_find_loadables(void *img, off_t imgsz, fastboot_section_t *sectp, int *sectcntp, uint32_t *offp) { Elf32_Ehdr *ehdr = (Elf32_Ehdr *)img; Elf32_Phdr *phdr; uint8_t *phdrbase; int i; int used_sections = 0; const int max_sectcnt = *sectcntp; if ((ehdr->e_phoff + ehdr->e_phnum * ehdr->e_phentsize) >= imgsz) return (-1); phdrbase = (uint8_t *)img + ehdr->e_phoff; for (i = 0; i < ehdr->e_phnum; i++) { phdr = (Elf32_Phdr *)(phdrbase + ehdr->e_phentsize * i); if (phdr->p_type == PT_INTERP) return (-1); if (phdr->p_type != PT_LOAD) continue; if (phdr->p_vaddr == phdr->p_paddr && phdr->p_paddr == DBOOT_ENTRY_ADDRESS) { *offp = (uint32_t)phdr->p_offset; } else { if (max_sectcnt <= used_sections) return (-1); sectp[used_sections].fb_sec_offset = phdr->p_offset; sectp[used_sections].fb_sec_paddr = phdr->p_paddr; sectp[used_sections].fb_sec_size = phdr->p_filesz; sectp[used_sections].fb_sec_bss_size = (phdr->p_filesz < phdr->p_memsz) ? (phdr->p_memsz - phdr->p_filesz) : 0; /* Extra sanity check for the input object file */ if (sectp[used_sections].fb_sec_paddr + sectp[used_sections].fb_sec_size + sectp[used_sections].fb_sec_bss_size >= DBOOT_ENTRY_ADDRESS) return (-1); used_sections++; } } *sectcntp = used_sections; return (0); } /* * Create multiboot info structure (mbi) base on the saved mbi. * Recalculate values of the pointer type fields in the data * structure based on the new starting physical address of the * data structure. */ static int fastboot_build_mbi(char *mdep, fastboot_info_t *nk) { mb_module_t *mbp; multiboot_info_t *mbi; /* pointer to multiboot structure */ uintptr_t start_addr_va; /* starting VA of mbi */ uintptr_t start_addr_pa; /* starting PA of mbi */ size_t offs = 0; /* offset from the starting address */ size_t arglen; /* length of the command line arg */ size_t size; /* size of the memory reserved for mbi */ size_t mdnsz; /* length of the boot archive name */ /* * If mdep is not NULL or empty, use the length of mdep + 1 * (for NULL terminating) as the length of the new command * line; else use the saved command line length as the * length for the new command line. */ if (mdep != NULL && strlen(mdep) != 0) { arglen = strlen(mdep) + 1; } else { arglen = saved_cmdline_len; } /* * Allocate memory for the new multiboot info structure (mbi). * If we have reserved memory for mbi but it's not enough, * free it and reallocate. */ size = PAGESIZE + P2ROUNDUP(arglen, PAGESIZE); if (nk->fi_mbi_size && nk->fi_mbi_size < size) { contig_free((void *)nk->fi_new_mbi_va, nk->fi_mbi_size); nk->fi_mbi_size = 0; } if (nk->fi_mbi_size == 0) { if ((nk->fi_new_mbi_va = (uintptr_t)contig_alloc(size, &fastboot_below_1G_dma_attr, PAGESIZE, 0)) == 0) { cmn_err(CE_NOTE, fastboot_enomem_msg, (uint64_t)size, "1G"); return (-1); } /* * fi_mbi_size must be set after the allocation succeeds * as it's used to determine how much memory to free. */ nk->fi_mbi_size = size; } /* * Initalize memory */ bzero((void *)nk->fi_new_mbi_va, nk->fi_mbi_size); /* * Get PA for the new mbi */ start_addr_va = nk->fi_new_mbi_va; start_addr_pa = mmu_ptob((uint64_t)hat_getpfnum(kas.a_hat, (caddr_t)start_addr_va)); nk->fi_new_mbi_pa = (paddr_t)start_addr_pa; /* * Populate the rest of the fields in the data structure */ /* * Copy from the saved mbi to preserve all non-pointer type fields. */ mbi = (multiboot_info_t *)start_addr_va; bcopy(&saved_mbi, mbi, sizeof (*mbi)); /* * Recalculate mods_addr. Set mod_start and mod_end based on * the physical address of the new boot archive. Set mod_name * to the name of the new boto archive. */ offs += sizeof (multiboot_info_t); mbi->mods_addr = start_addr_pa + offs; mbp = (mb_module_t *)(start_addr_va + offs); mbp->mod_start = nk->fi_files[FASTBOOT_BOOTARCHIVE].fb_dest_pa; mbp->mod_end = nk->fi_files[FASTBOOT_BOOTARCHIVE].fb_next_pa; offs += sizeof (mb_module_t); mdnsz = strlen(fastboot_filename[FASTBOOT_NAME_BOOTARCHIVE]) + 1; bcopy(fastboot_filename[FASTBOOT_NAME_BOOTARCHIVE], (void *)(start_addr_va + offs), mdnsz); mbp->mod_name = start_addr_pa + offs; mbp->reserved = 0; /* * Make sure the offset is 16-byte aligned to avoid unaligned access. */ offs += mdnsz; offs = P2ROUNDUP_TYPED(offs, 16, size_t); /* * Recalculate mmap_addr */ mbi->mmap_addr = start_addr_pa + offs; bcopy((void *)(uintptr_t)saved_mmap, (void *)(start_addr_va + offs), saved_mbi.mmap_length); offs += saved_mbi.mmap_length; /* * Recalculate drives_addr */ mbi->drives_addr = start_addr_pa + offs; bcopy((void *)(uintptr_t)saved_drives, (void *)(start_addr_va + offs), saved_mbi.drives_length); offs += saved_mbi.drives_length; /* * Recalculate the address of cmdline. Set cmdline to contain the * new boot argument. */ mbi->cmdline = start_addr_pa + offs; if (mdep != NULL && strlen(mdep) != 0) { bcopy(mdep, (void *)(start_addr_va + offs), arglen); } else { bcopy((void *)saved_cmdline, (void *)(start_addr_va + offs), arglen); } /* clear fields and flags that are not copied */ bzero(&mbi->config_table, sizeof (*mbi) - offsetof(multiboot_info_t, config_table)); mbi->flags &= ~(MB_INFO_CONFIG_TABLE | MB_INFO_BOOT_LOADER_NAME | MB_INFO_APM_TABLE | MB_INFO_VIDEO_INFO); return (0); } /* * Initialize HAT related fields */ static void fastboot_init_fields(fastboot_info_t *nk) { if (is_x86_feature(x86_featureset, X86FSET_PAE)) { nk->fi_has_pae = 1; nk->fi_shift_amt = fastboot_shift_amt_pae; nk->fi_ptes_per_table = 512; nk->fi_lpagesize = (2 << 20); /* 2M */ nk->fi_top_level = 3; } } /* * Process boot argument */ static void fastboot_parse_mdep(char *mdep, char *kern_bootpath, int *bootpath_len, char *bootargs) { int i; /* * If mdep is not NULL, it comes in the format of * mountpoint unix args */ if (mdep != NULL && strlen(mdep) != 0) { if (mdep[0] != '-') { /* First get the root argument */ i = 0; while (mdep[i] != '\0' && mdep[i] != ' ') { i++; } if (i < 4 || strncmp(&mdep[i-4], "unix", 4) != 0) { /* mount point */ bcopy(mdep, kern_bootpath, i); kern_bootpath[i] = '\0'; *bootpath_len = i; /* * Get the next argument. It should be unix as * we have validated in in halt.c. */ if (strlen(mdep) > i) { mdep += (i + 1); i = 0; while (mdep[i] != '\0' && mdep[i] != ' ') { i++; } } } bcopy(mdep, kern_bootfile, i); kern_bootfile[i] = '\0'; bcopy(mdep, bootargs, strlen(mdep)); } else { int off = strlen(kern_bootfile); bcopy(kern_bootfile, bootargs, off); bcopy(" ", &bootargs[off++], 1); bcopy(mdep, &bootargs[off], strlen(mdep)); off += strlen(mdep); bootargs[off] = '\0'; } } } /* * Reserve memory under PA 1G for mapping the new kernel and boot archive. * This function is only called if fastreboot_onpanic is *not* set. */ static void fastboot_reserve_mem(fastboot_info_t *nk) { int i; /* * A valid kernel is in place. No need to reserve any memory. */ if (nk->fi_valid) return; /* * Reserve memory under PA 1G for PTE lists. */ for (i = 0; i < FASTBOOT_MAX_FILES_MAP; i++) { fastboot_file_t *fb = &nk->fi_files[i]; size_t fsize_roundup, size; fsize_roundup = P2ROUNDUP_TYPED(saved_file_size[i], PAGESIZE, size_t); size = FASTBOOT_PTE_LIST_SIZE(fsize_roundup); if ((fb->fb_pte_list_va = contig_alloc(size, &fastboot_below_1G_dma_attr, PAGESIZE, 0)) == NULL) { return; } fb->fb_pte_list_size = size; } /* * Reserve memory under PA 1G for page tables. */ if ((nk->fi_pagetable_va = (uintptr_t)contig_alloc(fastboot_pagetable_size, &fastboot_below_1G_dma_attr, PAGESIZE, 0)) == 0) { return; } nk->fi_pagetable_size = fastboot_pagetable_size; /* * Reserve memory under PA 1G for multiboot structure. */ if ((nk->fi_new_mbi_va = (uintptr_t)contig_alloc(fastboot_mbi_size, &fastboot_below_1G_dma_attr, PAGESIZE, 0)) == 0) { return; } nk->fi_mbi_size = fastboot_mbi_size; } /* * Calculate MD5 digest for the given fastboot_file. * Assumes that the file is allready loaded properly. */ static void fastboot_cksum_file(fastboot_file_t *fb, uchar_t *md5_hash) { MD5_CTX md5_ctx; MD5Init(&md5_ctx); MD5Update(&md5_ctx, (void *)fb->fb_va, fb->fb_size); MD5Final(md5_hash, &md5_ctx); } /* * Free up the memory we have allocated for a file */ static void fastboot_free_file(fastboot_file_t *fb) { size_t fsize_roundup; fsize_roundup = P2ROUNDUP_TYPED(fb->fb_size, PAGESIZE, size_t); if (fsize_roundup) { contig_free((void *)fb->fb_va, fsize_roundup); fb->fb_va = 0; fb->fb_size = 0; } } /* * Free up memory used by the PTEs for a file. */ static void fastboot_free_file_pte(fastboot_file_t *fb, uint64_t endaddr) { if (fb->fb_pte_list_size && fb->fb_pte_list_pa < endaddr) { contig_free((void *)fb->fb_pte_list_va, fb->fb_pte_list_size); fb->fb_pte_list_va = 0; fb->fb_pte_list_pa = 0; fb->fb_pte_list_size = 0; } } /* * Free up all the memory used for representing a kernel with * fastboot_info_t. */ static void fastboot_free_mem(fastboot_info_t *nk, uint64_t endaddr) { int i; for (i = 0; i < FASTBOOT_MAX_FILES_MAP; i++) { fastboot_free_file(nk->fi_files + i); fastboot_free_file_pte(nk->fi_files + i, endaddr); } if (nk->fi_pagetable_size && nk->fi_pagetable_pa < endaddr) { contig_free((void *)nk->fi_pagetable_va, nk->fi_pagetable_size); nk->fi_pagetable_va = 0; nk->fi_pagetable_pa = 0; nk->fi_pagetable_size = 0; } if (nk->fi_mbi_size && nk->fi_new_mbi_pa < endaddr) { contig_free((void *)nk->fi_new_mbi_va, nk->fi_mbi_size); nk->fi_new_mbi_va = 0; nk->fi_new_mbi_pa = 0; nk->fi_mbi_size = 0; } } /* * Only free up the memory allocated for the kernel and boot archive, * but not for the page tables. */ void fastboot_free_newkernel(fastboot_info_t *nk) { int i; nk->fi_valid = 0; /* * Free the memory we have allocated */ for (i = 0; i < FASTBOOT_MAX_FILES_MAP; i++) { fastboot_free_file(&(nk->fi_files[i])); } } static void fastboot_cksum_cdata(fastboot_info_t *nk, uchar_t *md5_hash) { int i; MD5_CTX md5_ctx; MD5Init(&md5_ctx); for (i = 0; i < FASTBOOT_MAX_FILES_MAP; i++) { MD5Update(&md5_ctx, nk->fi_files[i].fb_pte_list_va, nk->fi_files[i].fb_pte_list_size); } MD5Update(&md5_ctx, (void *)nk->fi_pagetable_va, nk->fi_pagetable_size); MD5Update(&md5_ctx, (void *)nk->fi_new_mbi_va, nk->fi_mbi_size); MD5Final(md5_hash, &md5_ctx); } /* * Generate MD5 checksum of the given kernel. */ static void fastboot_cksum_generate(fastboot_info_t *nk) { int i; for (i = 0; i < FASTBOOT_MAX_FILES_MAP; i++) { fastboot_cksum_file(nk->fi_files + i, nk->fi_md5_hash[i]); } fastboot_cksum_cdata(nk, nk->fi_md5_hash[i]); } /* * Calculate MD5 checksum of the given kernel and verify that * it matches with what was calculated before. */ int fastboot_cksum_verify(fastboot_info_t *nk) { int i; uchar_t md5_hash[MD5_DIGEST_LENGTH]; for (i = 0; i < FASTBOOT_MAX_FILES_MAP; i++) { fastboot_cksum_file(nk->fi_files + i, md5_hash); if (bcmp(nk->fi_md5_hash[i], md5_hash, sizeof (nk->fi_md5_hash[i])) != 0) return (i + 1); } fastboot_cksum_cdata(nk, md5_hash); if (bcmp(nk->fi_md5_hash[i], md5_hash, sizeof (nk->fi_md5_hash[i])) != 0) return (i + 1); return (0); } /* * This function performs the following tasks: * - Read the sizes of the new kernel and boot archive. * - Allocate memory for the new kernel and boot archive. * - Allocate memory for page tables necessary for mapping the memory * allocated for the files. * - Read the new kernel and boot archive into memory. * - Map in the fast reboot switcher. * - Load the fast reboot switcher to FASTBOOT_SWTCH_PA. * - Build the new multiboot_info structure * - Build page tables for the low 1G of physical memory. * - Mark the data structure as valid if all steps have succeeded. */ void fastboot_load_kernel(char *mdep) { void *buf = NULL; int i; fastboot_file_t *fb; uint32_t dboot_start_offset; char kern_bootpath[OBP_MAXPATHLEN]; extern uintptr_t postbootkernelbase; uintptr_t saved_kernelbase; int bootpath_len = 0; int is_failsafe = 0; int is_retry = 0; uint64_t end_addr; if (!fastreboot_capable) return; if (newkernel.fi_valid) fastboot_free_newkernel(&newkernel); saved_kernelbase = postbootkernelbase; postbootkernelbase = 0; /* * Initialize various HAT related fields in the data structure */ fastboot_init_fields(&newkernel); bzero(kern_bootpath, OBP_MAXPATHLEN); /* * Process the boot argument */ bzero(fastboot_args, OBP_MAXPATHLEN); fastboot_parse_mdep(mdep, kern_bootpath, &bootpath_len, fastboot_args); /* * Make sure we get the null character */ bcopy(kern_bootpath, fastboot_filename[FASTBOOT_NAME_UNIX], bootpath_len); bcopy(kern_bootfile, &fastboot_filename[FASTBOOT_NAME_UNIX][bootpath_len], strlen(kern_bootfile) + 1); bcopy(kern_bootpath, fastboot_filename[FASTBOOT_NAME_BOOTARCHIVE], bootpath_len); if (bcmp(kern_bootfile, FAILSAFE_BOOTFILE32, (sizeof (FAILSAFE_BOOTFILE32) - 1)) == 0 || bcmp(kern_bootfile, FAILSAFE_BOOTFILE64, (sizeof (FAILSAFE_BOOTFILE64) - 1)) == 0) { is_failsafe = 1; } load_kernel_retry: /* * Read in unix and boot_archive */ end_addr = DBOOT_ENTRY_ADDRESS; for (i = 0; i < FASTBOOT_MAX_FILES_MAP; i++) { struct _buf *file; uintptr_t va; uint64_t fsize; size_t fsize_roundup, pt_size; int page_index; uintptr_t offset; ddi_dma_attr_t dma_attr = fastboot_dma_attr; dprintf("fastboot_filename[%d] = %s\n", i, fastboot_filename[i]); if ((file = kobj_open_file(fastboot_filename[i])) == (struct _buf *)-1) { cmn_err(CE_NOTE, "!Fastboot: Couldn't open %s", fastboot_filename[i]); goto err_out; } if (kobj_get_filesize(file, &fsize) != 0) { cmn_err(CE_NOTE, "!Fastboot: Couldn't get filesize for %s", fastboot_filename[i]); goto err_out; } fsize_roundup = P2ROUNDUP_TYPED(fsize, PAGESIZE, size_t); /* * Where the files end in physical memory after being * relocated by the fast boot switcher. */ end_addr += fsize_roundup; if (end_addr > fastboot_below_1G_dma_attr.dma_attr_addr_hi) { cmn_err(CE_NOTE, "!Fastboot: boot archive is too big"); goto err_out; } /* * Adjust dma_attr_addr_lo so that the new kernel and boot * archive will not be overridden during relocation. */ if (end_addr > fastboot_dma_attr.dma_attr_addr_lo || end_addr > fastboot_below_1G_dma_attr.dma_attr_addr_lo) { if (is_retry) { /* * If we have already tried and didn't succeed, * just give up. */ cmn_err(CE_NOTE, "!Fastboot: boot archive is too big"); goto err_out; } else { /* Set the flag so we don't keep retrying */ is_retry++; /* Adjust dma_attr_addr_lo */ fastboot_dma_attr.dma_attr_addr_lo = end_addr; fastboot_below_1G_dma_attr.dma_attr_addr_lo = end_addr; /* * Free the memory we have already allocated * whose physical addresses might not fit * the new lo and hi constraints. */ fastboot_free_mem(&newkernel, end_addr); goto load_kernel_retry; } } if (!fastboot_contig) dma_attr.dma_attr_sgllen = (fsize / PAGESIZE) + (((fsize % PAGESIZE) == 0) ? 0 : 1); if ((buf = contig_alloc(fsize, &dma_attr, PAGESIZE, 0)) == NULL) { cmn_err(CE_NOTE, fastboot_enomem_msg, fsize, "64G"); goto err_out; } va = P2ROUNDUP_TYPED((uintptr_t)buf, PAGESIZE, uintptr_t); if (kobj_read_file(file, (char *)va, fsize, 0) < 0) { cmn_err(CE_NOTE, "!Fastboot: Couldn't read %s", fastboot_filename[i]); goto err_out; } fb = &newkernel.fi_files[i]; fb->fb_va = va; fb->fb_size = fsize; fb->fb_sectcnt = 0; pt_size = FASTBOOT_PTE_LIST_SIZE(fsize_roundup); /* * If we have reserved memory but it not enough, free it. */ if (fb->fb_pte_list_size && fb->fb_pte_list_size < pt_size) { contig_free((void *)fb->fb_pte_list_va, fb->fb_pte_list_size); fb->fb_pte_list_size = 0; } if (fb->fb_pte_list_size == 0) { if ((fb->fb_pte_list_va = (x86pte_t *)contig_alloc(pt_size, &fastboot_below_1G_dma_attr, PAGESIZE, 0)) == NULL) { cmn_err(CE_NOTE, fastboot_enomem_msg, (uint64_t)pt_size, "1G"); goto err_out; } /* * fb_pte_list_size must be set after the allocation * succeeds as it's used to determine how much memory to * free. */ fb->fb_pte_list_size = pt_size; } bzero((void *)(fb->fb_pte_list_va), fb->fb_pte_list_size); fb->fb_pte_list_pa = mmu_ptob((uint64_t)hat_getpfnum(kas.a_hat, (caddr_t)fb->fb_pte_list_va)); for (page_index = 0, offset = 0; offset < fb->fb_size; offset += PAGESIZE) { uint64_t paddr; paddr = mmu_ptob((uint64_t)hat_getpfnum(kas.a_hat, (caddr_t)fb->fb_va + offset)); ASSERT(paddr >= fastboot_dma_attr.dma_attr_addr_lo); /* * Include the pte_bits so we don't have to make * it in assembly. */ fb->fb_pte_list_va[page_index++] = (x86pte_t) (paddr | pte_bits); } fb->fb_pte_list_va[page_index] = FASTBOOT_TERMINATE; if (i == FASTBOOT_UNIX) { Ehdr *ehdr = (Ehdr *)va; int j; /* * Sanity checks: */ for (j = 0; j < SELFMAG; j++) { if (ehdr->e_ident[j] != ELFMAG[j]) { cmn_err(CE_NOTE, "!Fastboot: Bad ELF " "signature"); goto err_out; } } if (ehdr->e_ident[EI_CLASS] == ELFCLASS32 && ehdr->e_ident[EI_DATA] == ELFDATA2LSB && ehdr->e_machine == EM_386) { fb->fb_sectcnt = sizeof (fb->fb_sections) / sizeof (fb->fb_sections[0]); if (fastboot_elf32_find_loadables((void *)va, fsize, &fb->fb_sections[0], &fb->fb_sectcnt, &dboot_start_offset) < 0) { cmn_err(CE_NOTE, "!Fastboot: ELF32 " "program section failure"); goto err_out; } if (fb->fb_sectcnt == 0) { cmn_err(CE_NOTE, "!Fastboot: No ELF32 " "program sections found"); goto err_out; } if (is_failsafe) { /* Failsafe boot_archive */ bcopy(BOOTARCHIVE32_FAILSAFE, &fastboot_filename [FASTBOOT_NAME_BOOTARCHIVE] [bootpath_len], sizeof (BOOTARCHIVE32_FAILSAFE)); } else { bcopy(BOOTARCHIVE32, &fastboot_filename [FASTBOOT_NAME_BOOTARCHIVE] [bootpath_len], sizeof (BOOTARCHIVE32)); } } else if (ehdr->e_ident[EI_CLASS] == ELFCLASS64 && ehdr->e_ident[EI_DATA] == ELFDATA2LSB && ehdr->e_machine == EM_AMD64) { if (fastboot_elf64_find_dboot_load_offset( (void *)va, fsize, &dboot_start_offset) != 0) { cmn_err(CE_NOTE, "!Fastboot: Couldn't " "find ELF64 dboot entry offset"); goto err_out; } if (!is_x86_feature(x86_featureset, X86FSET_64) || !is_x86_feature(x86_featureset, X86FSET_PAE)) { cmn_err(CE_NOTE, "Fastboot: Cannot " "reboot to %s: " "not a 64-bit capable system", kern_bootfile); goto err_out; } if (is_failsafe) { /* Failsafe boot_archive */ bcopy(BOOTARCHIVE64_FAILSAFE, &fastboot_filename [FASTBOOT_NAME_BOOTARCHIVE] [bootpath_len], sizeof (BOOTARCHIVE64_FAILSAFE)); } else { bcopy(BOOTARCHIVE64, &fastboot_filename [FASTBOOT_NAME_BOOTARCHIVE] [bootpath_len], sizeof (BOOTARCHIVE64)); } } else { cmn_err(CE_NOTE, "!Fastboot: Unknown ELF type"); goto err_out; } fb->fb_dest_pa = DBOOT_ENTRY_ADDRESS - dboot_start_offset; fb->fb_next_pa = DBOOT_ENTRY_ADDRESS + fsize_roundup; } else { fb->fb_dest_pa = newkernel.fi_files[i - 1].fb_next_pa; fb->fb_next_pa = fb->fb_dest_pa + fsize_roundup; } kobj_close_file(file); } /* * Add the function that will switch us to 32-bit protected mode */ fb = &newkernel.fi_files[FASTBOOT_SWTCH]; fb->fb_va = fb->fb_dest_pa = FASTBOOT_SWTCH_PA; fb->fb_size = MMU_PAGESIZE; hat_devload(kas.a_hat, (caddr_t)fb->fb_va, MMU_PAGESIZE, mmu_btop(fb->fb_dest_pa), PROT_READ | PROT_WRITE | PROT_EXEC, HAT_LOAD_NOCONSIST | HAT_LOAD_LOCK); /* * Build the new multiboot_info structure */ if (fastboot_build_mbi(fastboot_args, &newkernel) != 0) { goto err_out; } /* * Build page table for low 1G physical memory. Use big pages. * Allocate 4 (5 for amd64) pages for the page tables. * 1 page for PML4 (amd64) * 1 page for Page-Directory-Pointer Table * 2 pages for Page Directory * 1 page for Page Table. * The page table entry will be rewritten to map the physical * address as we do the copying. */ if (newkernel.fi_has_pae) { size_t size = MMU_PAGESIZE * 5; if (newkernel.fi_pagetable_size && newkernel.fi_pagetable_size < size) { contig_free((void *)newkernel.fi_pagetable_va, newkernel.fi_pagetable_size); newkernel.fi_pagetable_size = 0; } if (newkernel.fi_pagetable_size == 0) { if ((newkernel.fi_pagetable_va = (uintptr_t) contig_alloc(size, &fastboot_below_1G_dma_attr, MMU_PAGESIZE, 0)) == 0) { cmn_err(CE_NOTE, fastboot_enomem_msg, (uint64_t)size, "1G"); goto err_out; } /* * fi_pagetable_size must be set after the allocation * succeeds as it's used to determine how much memory to * free. */ newkernel.fi_pagetable_size = size; } bzero((void *)(newkernel.fi_pagetable_va), size); newkernel.fi_pagetable_pa = mmu_ptob((uint64_t)hat_getpfnum(kas.a_hat, (caddr_t)newkernel.fi_pagetable_va)); newkernel.fi_last_table_pa = newkernel.fi_pagetable_pa + size - MMU_PAGESIZE; newkernel.fi_next_table_va = newkernel.fi_pagetable_va + MMU_PAGESIZE; newkernel.fi_next_table_pa = newkernel.fi_pagetable_pa + MMU_PAGESIZE; fastboot_build_pagetables(&newkernel); } /* Generate MD5 checksums */ fastboot_cksum_generate(&newkernel); /* Mark it as valid */ newkernel.fi_valid = 1; newkernel.fi_magic = FASTBOOT_MAGIC; postbootkernelbase = saved_kernelbase; return; err_out: postbootkernelbase = saved_kernelbase; newkernel.fi_valid = 0; fastboot_free_newkernel(&newkernel); } /* ARGSUSED */ static int fastboot_xc_func(xc_arg_t arg1, xc_arg_t arg2 __unused, xc_arg_t arg3 __unused) { fastboot_info_t *nk = (fastboot_info_t *)arg1; void (*fastboot_func)(fastboot_info_t *); fastboot_file_t *fb = &nk->fi_files[FASTBOOT_SWTCH]; fastboot_func = (void (*)())(fb->fb_va); kthread_t *t_intr = curthread->t_intr; if (&kas != curproc->p_as) { hat_devload(curproc->p_as->a_hat, (caddr_t)fb->fb_va, MMU_PAGESIZE, mmu_btop(fb->fb_dest_pa), PROT_READ | PROT_WRITE | PROT_EXEC, HAT_LOAD_NOCONSIST | HAT_LOAD_LOCK); } /* * If we have pinned a thread, make sure the address is mapped * in the address space of the pinned thread. */ if (t_intr && t_intr->t_procp->p_as->a_hat != curproc->p_as->a_hat && t_intr->t_procp->p_as != &kas) hat_devload(t_intr->t_procp->p_as->a_hat, (caddr_t)fb->fb_va, MMU_PAGESIZE, mmu_btop(fb->fb_dest_pa), PROT_READ | PROT_WRITE | PROT_EXEC, HAT_LOAD_NOCONSIST | HAT_LOAD_LOCK); (*psm_shutdownf)(A_SHUTDOWN, AD_FASTREBOOT); (*fastboot_func)(nk); /*NOTREACHED*/ return (0); } /* * Jump to the fast reboot switcher. This function never returns. */ void fast_reboot() { processorid_t bootcpuid = 0; extern uintptr_t postbootkernelbase; extern char fb_swtch_image[]; fastboot_file_t *fb; int i; postbootkernelbase = 0; fb = &newkernel.fi_files[FASTBOOT_SWTCH]; /* * Map the address into both the current proc's address * space and the kernel's address space in case the panic * is forced by kmdb. */ if (&kas != curproc->p_as) { hat_devload(curproc->p_as->a_hat, (caddr_t)fb->fb_va, MMU_PAGESIZE, mmu_btop(fb->fb_dest_pa), PROT_READ | PROT_WRITE | PROT_EXEC, HAT_LOAD_NOCONSIST | HAT_LOAD_LOCK); } bcopy((void *)fb_swtch_image, (void *)fb->fb_va, fb->fb_size); /* * Set fb_va to fake_va */ for (i = 0; i < FASTBOOT_MAX_FILES_MAP; i++) { newkernel.fi_files[i].fb_va = fake_va; } if (panicstr && CPU->cpu_id != bootcpuid && CPU_ACTIVE(cpu_get(bootcpuid))) { extern void panic_idle(void); cpuset_t cpuset; CPUSET_ZERO(cpuset); CPUSET_ADD(cpuset, bootcpuid); xc_priority((xc_arg_t)&newkernel, 0, 0, CPUSET2BV(cpuset), fastboot_xc_func); panic_idle(); } else (void) fastboot_xc_func((xc_arg_t)&newkernel, 0, 0); } /* * Get boot property value for fastreboot_onpanic. * * NOTE: If fastreboot_onpanic is set to non-zero in /etc/system, * new setting passed in via "-B fastreboot_onpanic" is ignored. * This order of precedence is to enable developers debugging panics * that occur early in boot to utilize Fast Reboot on panic. */ static void fastboot_get_bootprop(void) { int val = 0xaa, len, ret; dev_info_t *devi; char *propstr = NULL; devi = ddi_root_node(); ret = ddi_prop_lookup_string(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS, FASTREBOOT_ONPANIC, &propstr); if (ret == DDI_PROP_SUCCESS) { if (FASTREBOOT_ONPANIC_NOTSET(propstr)) val = 0; else if (FASTREBOOT_ONPANIC_ISSET(propstr)) val = UA_FASTREBOOT_ONPANIC; /* * Only set fastreboot_onpanic to the value passed in * if it's not already set to non-zero, and the value * has indeed been passed in via command line. */ if (!fastreboot_onpanic && val != 0xaa) fastreboot_onpanic = val; ddi_prop_free(propstr); } else if (ret != DDI_PROP_NOT_FOUND && ret != DDI_PROP_UNDEFINED) { cmn_err(CE_NOTE, "!%s value is invalid, will be ignored", FASTREBOOT_ONPANIC); } len = sizeof (fastreboot_onpanic_cmdline); ret = ddi_getlongprop_buf(DDI_DEV_T_ANY, devi, DDI_PROP_DONTPASS, FASTREBOOT_ONPANIC_CMDLINE, fastreboot_onpanic_cmdline, &len); if (ret == DDI_PROP_BUF_TOO_SMALL) cmn_err(CE_NOTE, "!%s value is too long, will be ignored", FASTREBOOT_ONPANIC_CMDLINE); } /* * This function is called by main() to either load the backup kernel for panic * fast reboot, or to reserve low physical memory for fast reboot. */ void fastboot_post_startup() { lbolt_at_boot = ddi_get_lbolt(); /* Default to 10 minutes */ if (fastreboot_onpanic_uptime == LONG_MAX) fastreboot_onpanic_uptime = SEC_TO_TICK(10 * 60); if (!fastreboot_capable) return; mutex_enter(&fastreboot_config_mutex); fastboot_get_bootprop(); if (fastreboot_onpanic) fastboot_load_kernel(fastreboot_onpanic_cmdline); else if (reserve_mem_enabled) fastboot_reserve_mem(&newkernel); mutex_exit(&fastreboot_config_mutex); } /* * Update boot configuration settings. * If the new fastreboot_onpanic setting is false, and a kernel has * been preloaded, free the memory; * if the new fastreboot_onpanic setting is true and newkernel is * not valid, load the new kernel. */ void fastboot_update_config(const char *mdep) { uint8_t boot_config = (uint8_t)*mdep; int cur_fastreboot_onpanic; if (!fastreboot_capable) return; mutex_enter(&fastreboot_config_mutex); cur_fastreboot_onpanic = fastreboot_onpanic; fastreboot_onpanic = boot_config & UA_FASTREBOOT_ONPANIC; if (fastreboot_onpanic && (!cur_fastreboot_onpanic || !newkernel.fi_valid)) fastboot_load_kernel(fastreboot_onpanic_cmdline); if (cur_fastreboot_onpanic && !fastreboot_onpanic) fastboot_free_newkernel(&newkernel); mutex_exit(&fastreboot_config_mutex); } /* * This is an internal interface to disable Fast Reboot on Panic. * It frees up memory allocated for the backup kernel and sets * fastreboot_onpanic to zero. */ static void fastreboot_onpanic_disable(void) { uint8_t boot_config = (uint8_t)(~UA_FASTREBOOT_ONPANIC); fastboot_update_config((const char *)&boot_config); } /* * This is the interface to be called by fm_panic() in case FMA has diagnosed * a terminal machine check exception. It does not free up memory allocated * for the backup kernel. General disabling fastreboot_onpanic in a * non-panicking situation must go through fastboot_onpanic_disable(). */ void fastreboot_disable_highpil(void) { fastreboot_onpanic = 0; } /* * This is an internal interface to disable Fast Reboot by Default. * It does not free up memory allocated for the backup kernel. */ static void fastreboot_capable_disable(uint32_t msgid) { if (fastreboot_capable != 0) { fastreboot_capable = 0; if (msgid < sizeof (fastreboot_nosup_desc) / sizeof (fastreboot_nosup_desc[0])) fastreboot_nosup_id = msgid; else fastreboot_nosup_id = FBNS_DEFAULT; } } /* * This is the kernel interface for disabling * Fast Reboot by Default and Fast Reboot on Panic. * Frees up memory allocated for the backup kernel. * General disabling of the Fast Reboot by Default feature should be done * via the userland interface scf_fastreboot_default_set_transient(). */ void fastreboot_disable(uint32_t msgid) { fastreboot_capable_disable(msgid); fastreboot_onpanic_disable(); } /* * Returns Fast Reboot not support message for fastreboot_nosup_id. * If fastreboot_nosup_id contains invalid index, default * Fast Reboot not support message is returned. */ const char * fastreboot_nosup_message(void) { uint32_t msgid; msgid = fastreboot_nosup_id; if (msgid >= sizeof (fastreboot_nosup_desc) / sizeof (fastreboot_nosup_desc[0])) msgid = FBNS_DEFAULT; return (fastreboot_nosup_desc[msgid]); } /* * A simplified interface for uadmin to call to update the configuration * setting and load a new kernel if necessary. */ void fastboot_update_and_load(int fcn, char *mdep) { if (fcn != AD_FASTREBOOT) { /* * If user has explicitly requested reboot to prom, * or uadmin(8) was invoked with other functions, * don't try to fast reboot after dumping. */ fastreboot_onpanic_disable(); } mutex_enter(&fastreboot_config_mutex); if (fastreboot_onpanic) fastboot_load_kernel(mdep); mutex_exit(&fastreboot_config_mutex); } /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2009 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ #include #include #include #include #include #include #include #include #include #include #define VIDEOMEM 0xa0000 extern void outb(int, uchar_t); static int graphics_mode; static int cursor_y = 309; static int cursor_x = 136; #define BAR_STEPS 46 static uchar_t bar[BAR_STEPS]; static kthread_t *progressbar_tid; static kmutex_t pbar_lock; static kcondvar_t pbar_cv; static char *videomem = (caddr_t)VIDEOMEM; static int videomem_size; /* select the plane(s) to draw to */ static void mapmask(int plane) { outb(0x3c4, 2); outb(0x3c5, plane); } static void bitmask(int value) { outb(0x3ce, 8); outb(0x3cf, value); } static void progressbar_show(void) { int j, k, offset; uchar_t *mem, *ptr; offset = cursor_y * 80 + cursor_x / 8; mem = (uchar_t *)videomem + offset; bitmask(0xff); mapmask(0xff); /* write to all planes at once? */ for (j = 0; j < 4; j++) { /* bar height: 4 pixels */ ptr = mem + j * 80; for (k = 0; k < BAR_STEPS; k++, ptr++) *ptr = bar[k]; } bitmask(0x00); } /* * Initialize a rectangle area for progress bar * * Multiboot has initialized graphics mode to 640x480 * with 16 colors. */ void progressbar_init() { int i; char cons[10]; /* see if we are in graphics mode */ if (BOP_GETPROPLEN(bootops, "console") != sizeof ("graphics")) return; (void) BOP_GETPROP(bootops, "console", cons); if (strncmp(cons, "graphics", strlen("graphics")) != 0) return; if (BOP_GETPROPLEN(bootops, "efi-systab") > 0) return; graphics_mode = 1; for (i = 0; i < BAR_STEPS; i++) { bar[i] = 0x00; } progressbar_show(); } static void progressbar_step() { static int limit = 0; bar[limit] = 0xff; if (limit > 3) bar[limit - 4] = 0x00; else bar[limit + BAR_STEPS - 4] = 0x00; limit++; if (limit == BAR_STEPS) limit = 0; progressbar_show(); } /*ARGSUSED*/ static void progressbar_thread(void *arg) { clock_t end = drv_usectohz(150000); mutex_enter(&pbar_lock); while (graphics_mode) { progressbar_step(); (void) cv_reltimedwait(&pbar_cv, &pbar_lock, end, TR_CLOCK_TICK); } mutex_exit(&pbar_lock); } void progressbar_start(void) { #if !defined(__xpv) extern pri_t minclsyspri; if (graphics_mode == 0) return; /* map video memory to kernel heap */ videomem_size = ptob(btopr(38400)); /* 640 x 480 / 8 bytes */ videomem = vmem_alloc(heap_arena, videomem_size, VM_SLEEP); if (videomem == NULL) { cmn_err(CE_NOTE, "!failed to start progress bar"); graphics_mode = 0; return; } hat_devload(kas.a_hat, videomem, videomem_size, btop(VIDEOMEM), (PROT_READ | PROT_WRITE), HAT_LOAD_NOCONSIST | HAT_LOAD_LOCK); progressbar_tid = thread_create(NULL, 0, progressbar_thread, NULL, 0, &p0, TS_RUN, minclsyspri); #endif } void progressbar_stop(void) { #if !defined(__xpv) if (graphics_mode == 0) return; graphics_mode = 0; mutex_enter(&pbar_lock); cv_signal(&pbar_cv); mutex_exit(&pbar_lock); if (progressbar_tid != NULL) thread_join(progressbar_tid->t_did); /* unmap video memory */ hat_unload(kas.a_hat, videomem, videomem_size, HAT_UNLOAD_UNLOCK); vmem_free(heap_arena, videomem, videomem_size); #endif } /*ARGSUSED*/ void progressbar_key_abort(ldi_ident_t li) { #if !defined(__xpv) char *fbpath; int ret; ldi_handle_t hdl; extern char *consconfig_get_plat_fbpath(void); if (graphics_mode == 0) return; fbpath = consconfig_get_plat_fbpath(); if (ldi_open_by_name(fbpath, FWRITE, kcred, &hdl, li) != 0) { cmn_err(CE_NOTE, "!ldi_open_by_name failed"); } else { if (ldi_ioctl(hdl, KDSETMODE, KD_RESETTEXT, FKIOCTL, kcred, &ret) != 0) cmn_err(CE_NOTE, "!ldi_ioctl for KD_RESETTEXT failed"); (void) ldi_close(hdl, 0, kcred); } #endif } /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2009 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ #include #if defined(__xpv) #include #endif int plat_hold_page(pfn_t pfn, int lock, page_t **pp_ret) { page_t *pp = page_numtopp_nolock(pfn); if (pp == NULL) return (PLAT_HOLD_FAIL); #if !defined(__xpv) /* * Pages are locked SE_SHARED because some hypervisors * like xVM ESX reclaim Guest OS memory by locking * it SE_EXCL so we want to leave these pages alone. */ if (lock == PLAT_HOLD_LOCK) { ASSERT(pp_ret != NULL); if (page_trylock(pp, SE_SHARED) == 0) return (PLAT_HOLD_FAIL); } #else /* __xpv */ if (lock == PLAT_HOLD_LOCK) { ASSERT(pp_ret != NULL); if (page_trylock(pp, SE_EXCL) == 0) return (PLAT_HOLD_FAIL); } if (mfn_list[pfn] == MFN_INVALID) { /* We failed - release the lock if we grabbed it earlier */ if (lock == PLAT_HOLD_LOCK) { page_unlock(pp); } return (PLAT_HOLD_FAIL); } #endif /* __xpv */ if (lock == PLAT_HOLD_LOCK) *pp_ret = pp; return (PLAT_HOLD_OK); } void plat_release_page(page_t *pp) { ASSERT((pp != NULL) && PAGE_LOCKED(pp)); page_unlock(pp); } /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2009 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ /* * This is the place to implement ld_ib_props() * For x86 it is to load iBFT and costruct the global ib props */ #include #include #include #include #include #include #include #include #include #include typedef enum ibft_structure_type { Reserved = 0, Control = 1, Initiator = 2, Nic = 3, Target = 4, Extensions = 5, Type_End }ibft_struct_type; typedef enum _chap_type { NO_CHAP = 0, CHAP = 1, Mutual_CHAP = 2, TYPE_UNKNOWN }chap_type; typedef struct ibft_entry { int af; int e_port; char target_name[224]; char target_addr[INET6_ADDRSTRLEN]; }ibft_entry_t; typedef struct iSCSI_ibft_tbl_hdr { char Signature[4]; int Length; char Revision; char Checksum; char oem_id[6]; char oem_table_id[8]; char Reserved[24]; }iscsi_ibft_tbl_hdr_t; typedef struct iSCSI_ibft_hdr { char Structure_id; char Version; ushort_t Length; char Index; char Flags; }iscsi_ibft_hdr_t; typedef struct iSCSI_ibft_control { iscsi_ibft_hdr_t header; ushort_t Extensions; ushort_t Initiator_offset; ushort_t Nic0_offset; ushort_t Target0_offset; ushort_t Nic1_offset; ushort_t Target1_offset; }iscsi_ibft_ctl_t; typedef struct iSCSI_ibft_initiator { iscsi_ibft_hdr_t header; uchar_t iSNS_Server[16]; uchar_t SLP_Server[16]; uchar_t Pri_Radius_Server[16]; uchar_t Sec_Radius_Server[16]; ushort_t ini_name_len; ushort_t ini_name_offset; }iscsi_ibft_initiator_t; typedef struct iSCSI_ibft_nic { iscsi_ibft_hdr_t header; uchar_t ip_addr[16]; char Subnet_Mask_Prefix; char Origin; uchar_t Gateway[16]; uchar_t Primary_dns[16]; uchar_t Secondary_dns[16]; uchar_t dhcp[16]; ushort_t vlan; char mac[6]; ushort_t pci_BDF; ushort_t Hostname_len; ushort_t Hostname_offset; }iscsi_ibft_nic_t; typedef struct iSCSI_ibft_target { iscsi_ibft_hdr_t header; uchar_t ip_addr[16]; ushort_t port; uchar_t boot_lun[8]; uchar_t chap_type; uchar_t nic_association; ushort_t target_name_len; ushort_t target_name_offset; ushort_t chap_name_len; ushort_t chap_name_offset; ushort_t chap_secret_len; ushort_t chap_secret_offset; ushort_t rev_chap_name_len; ushort_t rev_chap_name_offset; ushort_t rev_chap_secret_len; ushort_t rev_chap_secret_offset; }iscsi_ibft_tgt_t; #define ISCSI_IBFT_LOWER_ADDR 0x80000 /* 512K */ #define ISCSI_IBFT_HIGHER_ADDR 0x100000 /* 1024K */ #define ISCSI_IBFT_SIGNATRUE "iBFT" #define ISCSI_IBFT_SIGNATURE_LEN 4 #define ISCSI_IBFT_TBL_BUF_LEN 1024 #define ISCSI_IBFT_ALIGNED 16 #define ISCSI_IBFT_CTL_OFFSET 48 #define IBFT_BLOCK_VALID_YES 0x01 /* bit 0 */ #define IBFT_FIRMWARE_BOOT_SELECTED 0x02 /* bit 1 */ #define IBFT_USE_RADIUS_CHAP 0x04 /* bit 2 */ #define IBFT_USE_GLOBLE 0x04 /* NIC structure */ #define IBFT_USE_RADIUS_RHCAP 0x08 /* bit 3 */ /* * Currently, we only support initiator offset, NIC0 offset, Target0 offset, * NIC1 offset and Target1 offset. So the length is 5. If we want to support * extensions, we should change this number. */ #define IBFT_OFFSET_BUF_LEN 5 #define IPV4_OFFSET 12 #define IBFT_INVALID_MSG "Invalid iBFT table 0x%x" #define IBFT_NOPROBE_MSG "iSCSI boot is disabled" typedef enum ibft_status { IBFT_STATUS_OK = 0, /* General error */ IBFT_STATUS_ERR, /* Bad header */ IBFT_STATUS_BADHDR, /* Bad control ID */ IBFT_STATUS_BADCID, /* Bad ip addr */ IBFT_STATUS_BADIP, /* Bad af */ IBFT_STATUS_BADAF, /* Bad chap name */ IBFT_STATUS_BADCHAPNAME, /* Bad chap secret */ IBFT_STATUS_BADCHAPSEC, /* Bad checksum */ IBFT_STATUS_BADCHECKSUM, /* Low memory */ IBFT_STATUS_LOWMEM, /* No table */ IBFT_STATUS_NOTABLE } ibft_status_t; extern void *memset(void *s, int c, size_t n); extern int memcmp(const void *s1, const void *s2, size_t n); extern void bcopy(const void *s1, void *s2, size_t n); extern void iscsi_print_boot_property(); int ibft_noprobe = 0; ib_boot_prop_t boot_property; /* static allocated */ extern ib_boot_prop_t *iscsiboot_prop; /* to be filled */ static ibft_status_t iscsi_parse_ibft_control(iscsi_ibft_ctl_t *ctl_hdr, ushort_t *iscsi_offset_buf); static ibft_status_t iscsi_parse_ibft_initiator(char *begin_of_ibft, iscsi_ibft_initiator_t *initiator); static ibft_status_t iscsi_parse_ibft_NIC(iscsi_ibft_nic_t *nicp); static ibft_status_t iscsi_parse_ibft_target(char *begin_of_ibft, iscsi_ibft_tgt_t *tgtp); /* * Return value: * Success: IBFT_STATUS_OK * Fail: IBFT_STATUS_BADCHECKSUM */ static ibft_status_t iscsi_ibft_hdr_checksum(iscsi_ibft_tbl_hdr_t *tbl_hdr) { uchar_t checksum = 0; uchar_t *start = NULL; int length = 0; int i = 0; if (tbl_hdr == NULL) { return (IBFT_STATUS_BADHDR); } length = tbl_hdr->Length; start = (uchar_t *)tbl_hdr; for (i = 0; i < length; i++) { checksum = checksum + start[i]; } if (!checksum) return (IBFT_STATUS_OK); else return (IBFT_STATUS_BADCHECKSUM); } /* * Now we only support one control structure in the IBFT. * So there is no Control ID here. */ static ibft_status_t iscsi_parse_ibft_structure(char *begin_of_ibft, char *buf) { iscsi_ibft_hdr_t *hdr = NULL; ibft_status_t ret = IBFT_STATUS_OK; if (buf == NULL) { return (IBFT_STATUS_ERR); } hdr = (iscsi_ibft_hdr_t *)buf; switch (hdr->Structure_id) { case Initiator: ret = iscsi_parse_ibft_initiator( begin_of_ibft, (iscsi_ibft_initiator_t *)buf); break; case Nic: ret = iscsi_parse_ibft_NIC( (iscsi_ibft_nic_t *)buf); break; case Target: ret = iscsi_parse_ibft_target( begin_of_ibft, (iscsi_ibft_tgt_t *)buf); break; default: ret = IBFT_STATUS_BADHDR; break; } return (ret); } /* * Parse the iBFT table * return IBFT_STATUS_OK upon sucess */ static ibft_status_t iscsi_parse_ibft_tbl(iscsi_ibft_tbl_hdr_t *tbl_hdr) { char *outbuf = NULL; int i = 0; ibft_status_t ret = IBFT_STATUS_OK; ushort_t iscsi_offset_buf[IBFT_OFFSET_BUF_LEN] = {0}; if (tbl_hdr == NULL) { return (IBFT_STATUS_ERR); } if (iscsi_ibft_hdr_checksum(tbl_hdr) != IBFT_STATUS_OK) { return (IBFT_STATUS_BADCHECKSUM); } outbuf = (char *)tbl_hdr; ret = iscsi_parse_ibft_control( (iscsi_ibft_ctl_t *)&outbuf[ISCSI_IBFT_CTL_OFFSET], iscsi_offset_buf); if (ret == IBFT_STATUS_OK) { ret = IBFT_STATUS_ERR; for (i = 0; i < IBFT_OFFSET_BUF_LEN; i++) { if (iscsi_offset_buf[i] != 0) { ret = iscsi_parse_ibft_structure( (char *)tbl_hdr, (char *)tbl_hdr + iscsi_offset_buf[i]); if (ret != IBFT_STATUS_OK) { return (ret); } } } } return (ret); } static ibft_status_t iscsi_parse_ibft_control(iscsi_ibft_ctl_t *ctl_hdr, ushort_t *iscsi_offset_buf) { int i = 0; ushort_t *offsetp = NULL; if (ctl_hdr == NULL) { return (IBFT_STATUS_BADHDR); } if (ctl_hdr->header.Structure_id != Control) { return (IBFT_STATUS_BADCID); } /* * Copy the offsets to offset buffer. */ for (offsetp = &(ctl_hdr->Initiator_offset); i < IBFT_OFFSET_BUF_LEN; offsetp++) { iscsi_offset_buf[i++] = *offsetp; } return (IBFT_STATUS_OK); } /* * We only copy the "Firmare Boot Selseted" and valid initiator * to the boot property. */ static ibft_status_t iscsi_parse_ibft_initiator(char *begin_of_ibft, iscsi_ibft_initiator_t *initiator) { if (initiator == NULL) { return (IBFT_STATUS_ERR); } if (initiator->header.Structure_id != Initiator) { return (IBFT_STATUS_BADHDR); } if ((initiator->header.Flags & IBFT_FIRMWARE_BOOT_SELECTED) && (initiator->header.Flags & IBFT_BLOCK_VALID_YES)) { /* * If the initiator name exists, we will copy it to our own * property structure */ if (initiator->ini_name_len != 0) { boot_property.boot_init.ini_name = (uchar_t *)kmem_zalloc( initiator->ini_name_len + 1, KM_SLEEP); boot_property.boot_init.ini_name_len = initiator->ini_name_len + 1; (void) snprintf( (char *)boot_property.boot_init.ini_name, initiator->ini_name_len + 1, "%s", begin_of_ibft + initiator->ini_name_offset); } } return (IBFT_STATUS_OK); } static ibft_status_t iscsi_parse_ipaddr(uchar_t *source, char *dest, int *af) { int i = 0; if (source == NULL) { return (IBFT_STATUS_ERR); } if (source[0] == 0x00 && source[1] == 0x00 && source[2] == 0x00 && source[3] == 0x00 && source[4] == 0x00 && source[5] == 0x00 && source[6] == 0x00 && source[7] == 0x00 && source[8] == 0x00 && source[9] == 0x00 && (source[10] == 0xff) && (source[11] == 0xff)) { /* * IPv4 address */ if (dest != NULL) { (void) sprintf(dest, "%d.%d.%d.%d", source[12], source[13], source[14], source[15]); } if (af != NULL) { *af = AF_INET; } } else { if (dest != NULL) { for (i = 0; i < 14; i = i + 2) { (void) sprintf(dest, "%02x%02x:", source[i], source[i+1]); dest = dest + 5; } (void) sprintf(dest, "%02x%02x", source[i], source[i+1]); } if (af != NULL) { *af = AF_INET6; } } return (IBFT_STATUS_OK); } /* * Copy the ip address from ibft. If IPv4 is used, we should copy * the address from 12th byte. */ static ibft_status_t iscsi_copy_ibft_ipaddr(uchar_t *source, void *dest, int *af) { ibft_status_t ret = IBFT_STATUS_OK; int sin_family = 0; if (source == NULL || dest == NULL) { return (IBFT_STATUS_ERR); } ret = iscsi_parse_ipaddr(source, NULL, &sin_family); if (ret != 0) { return (IBFT_STATUS_BADIP); } if (sin_family == AF_INET) { bcopy(source+IPV4_OFFSET, dest, sizeof (struct in_addr)); } else if (sin_family == AF_INET6) { bcopy(source, dest, sizeof (struct in6_addr)); } else { return (IBFT_STATUS_BADAF); } if (af != NULL) { *af = sin_family; } return (IBFT_STATUS_OK); } /* * Maybe there are multiply NICs are available. We only copy the * "Firmare Boot Selseted" and valid one to the boot property. */ static ibft_status_t iscsi_parse_ibft_NIC(iscsi_ibft_nic_t *nicp) { ibft_status_t ret = IBFT_STATUS_OK; int af = 0; if (nicp == NULL) { return (IBFT_STATUS_ERR); } if (nicp->header.Structure_id != Nic) { return (IBFT_STATUS_ERR); } if ((nicp->header.Flags & IBFT_FIRMWARE_BOOT_SELECTED) && (nicp->header.Flags & IBFT_BLOCK_VALID_YES)) { ret = iscsi_copy_ibft_ipaddr(nicp->ip_addr, &boot_property.boot_nic.nic_ip_u, &af); if (ret != IBFT_STATUS_OK) { return (ret); } boot_property.boot_nic.sin_family = af; ret = iscsi_copy_ibft_ipaddr(nicp->Gateway, &boot_property.boot_nic.nic_gw_u, NULL); if (ret != IBFT_STATUS_OK) { return (ret); } ret = iscsi_copy_ibft_ipaddr(nicp->dhcp, &boot_property.boot_nic.nic_dhcp_u, NULL); if (ret != IBFT_STATUS_OK) { return (ret); } bcopy(nicp->mac, boot_property.boot_nic.nic_mac, 6); boot_property.boot_nic.sub_mask_prefix = nicp->Subnet_Mask_Prefix; } return (IBFT_STATUS_OK); } /* * Maybe there are multiply targets are available. We only copy the * "Firmare Boot Selseted" and valid one to the boot property. */ static ibft_status_t iscsi_parse_ibft_target(char *begin_of_ibft, iscsi_ibft_tgt_t *tgtp) { char *tmp = NULL; int af = 0; ibft_status_t ret = IBFT_STATUS_OK; if (tgtp == NULL) { return (IBFT_STATUS_ERR); } if (tgtp->header.Structure_id != Target) { return (IBFT_STATUS_BADHDR); } if ((tgtp->header.Flags & IBFT_FIRMWARE_BOOT_SELECTED) && (tgtp->header.Flags & IBFT_BLOCK_VALID_YES)) { /* * Get Target Address */ ret = iscsi_copy_ibft_ipaddr(tgtp->ip_addr, &boot_property.boot_tgt.tgt_ip_u, &af); if (ret != IBFT_STATUS_OK) { return (ret); } boot_property.boot_tgt.sin_family = af; /* * Get Target Name */ if (tgtp->target_name_len != 0) { boot_property.boot_tgt.tgt_name = (uchar_t *)kmem_zalloc(tgtp->target_name_len + 1, KM_SLEEP); boot_property.boot_tgt.tgt_name_len = tgtp->target_name_len + 1; (void) snprintf( (char *)boot_property.boot_tgt.tgt_name, tgtp->target_name_len + 1, "%s", begin_of_ibft + tgtp->target_name_offset); } else { boot_property.boot_tgt.tgt_name = NULL; } /* Get Dest Port */ boot_property.boot_tgt.tgt_port = tgtp->port; boot_property.boot_tgt.lun_online = 0; /* * Get CHAP secret and name. */ if (tgtp->chap_type != NO_CHAP) { if (tgtp->chap_name_len != 0) { boot_property.boot_init.ini_chap_name = (uchar_t *)kmem_zalloc( tgtp->chap_name_len + 1, KM_SLEEP); boot_property.boot_init.ini_chap_name_len = tgtp->chap_name_len + 1; tmp = (char *) boot_property.boot_init.ini_chap_name; (void) snprintf( tmp, tgtp->chap_name_len + 1, "%s", begin_of_ibft + tgtp->chap_name_offset); } else { /* * Just set NULL, initiator is able to deal * with this */ boot_property.boot_init.ini_chap_name = NULL; } if (tgtp->chap_secret_len != 0) { boot_property.boot_init.ini_chap_sec = (uchar_t *)kmem_zalloc( tgtp->chap_secret_len + 1, KM_SLEEP); boot_property.boot_init.ini_chap_sec_len = tgtp->chap_secret_len + 1; bcopy(begin_of_ibft + tgtp->chap_secret_offset, boot_property.boot_init.ini_chap_sec, tgtp->chap_secret_len); } else { boot_property.boot_init.ini_chap_sec = NULL; return (IBFT_STATUS_ERR); } if (tgtp->chap_type == Mutual_CHAP) { if (tgtp->rev_chap_name_len != 0) { boot_property.boot_tgt.tgt_chap_name = (uchar_t *)kmem_zalloc( tgtp->rev_chap_name_len + 1, KM_SLEEP); boot_property.boot_tgt.tgt_chap_name_len = tgtp->rev_chap_name_len + 1; #define TGT_CHAP_NAME boot_property.boot_tgt.tgt_chap_name tmp = (char *)TGT_CHAP_NAME; #undef TGT_CHAP_NAME (void) snprintf( tmp, tgtp->rev_chap_name_len + 1, "%s", begin_of_ibft + tgtp->rev_chap_name_offset); } else { /* * Just set NULL, initiator is able * to deal with this */ boot_property.boot_tgt.tgt_chap_name = NULL; } if (tgtp->rev_chap_secret_len != 0) { boot_property.boot_tgt.tgt_chap_sec = (uchar_t *)kmem_zalloc( tgtp->rev_chap_secret_len + 1, KM_SLEEP); boot_property.boot_tgt.tgt_chap_sec_len = tgtp->rev_chap_secret_len + 1; tmp = (char *) boot_property.boot_tgt.tgt_chap_sec; (void) snprintf( tmp, tgtp->rev_chap_secret_len + 1, "%s", begin_of_ibft + tgtp->chap_secret_offset); } else { boot_property.boot_tgt.tgt_chap_sec = NULL; return (IBFT_STATUS_BADCHAPSEC); } } } else { boot_property.boot_init.ini_chap_name = NULL; boot_property.boot_init.ini_chap_sec = NULL; } /* * Get Boot LUN */ (void) bcopy(tgtp->boot_lun, boot_property.boot_tgt.tgt_boot_lun, 8); } return (IBFT_STATUS_OK); } /* * This function is used for scanning iBFT from the physical memory. * Return Value: * IBFT_STATUS_OK * IBFT_STATUS_ERR */ static ibft_status_t iscsi_scan_ibft_tbl(char *ibft_tbl_buf) { int start; void *va = NULL; int *len = NULL; ibft_status_t ret = IBFT_STATUS_NOTABLE; for (start = ISCSI_IBFT_LOWER_ADDR; start < ISCSI_IBFT_HIGHER_ADDR; start = start + ISCSI_IBFT_ALIGNED) { va = (void *)psm_map((paddr_t)(start&0xffffffff), ISCSI_IBFT_SIGNATURE_LEN, PROT_READ); if (va == NULL) { continue; } if (memcmp(va, ISCSI_IBFT_SIGNATRUE, ISCSI_IBFT_SIGNATURE_LEN) == 0) { ret = IBFT_STATUS_ERR; /* Acquire table length */ len = (int *)psm_map( (paddr_t)((start+\ ISCSI_IBFT_SIGNATURE_LEN)&0xffffffff), ISCSI_IBFT_SIGNATURE_LEN, PROT_READ); if (len == NULL) { psm_unmap((caddr_t)va, ISCSI_IBFT_SIGNATURE_LEN); continue; } if (ISCSI_IBFT_LOWER_ADDR + *len < ISCSI_IBFT_HIGHER_ADDR - 1) { psm_unmap(va, ISCSI_IBFT_SIGNATURE_LEN); va = psm_map((paddr_t)(start&0xffffffff), *len, PROT_READ); if (va != NULL) { /* * Copy data to our own buffer */ bcopy(va, ibft_tbl_buf, *len); ret = IBFT_STATUS_OK; } psm_unmap((caddr_t)va, *len); psm_unmap((caddr_t)len, ISCSI_IBFT_SIGNATURE_LEN); break; } else { psm_unmap((caddr_t)va, ISCSI_IBFT_SIGNATURE_LEN); psm_unmap((caddr_t)len, ISCSI_IBFT_SIGNATURE_LEN); } } else { psm_unmap((caddr_t)va, ISCSI_IBFT_SIGNATURE_LEN); } } return (ret); } /* * Scan the ibft table and store the iSCSI boot properties * If there is a valid table then set the iscsiboot_prop * iBF should be off if the host is not intended * to be booted from iSCSI disk */ void ld_ib_prop() { ibft_status_t ret = IBFT_STATUS_OK; char *ibft_tbl_buf; if (do_bsys_getproplen(NULL, "ibft-noprobe") > 0) ibft_noprobe = 1; if (ibft_noprobe != 0) { /* * Scanning for iBFT may conflict with devices which use memory * in 640-1024KB of physical address space. The iBFT * specification suggests use of low RAM method - scanning * physical memory 512-1024 KB for iBFT table. However, the * Upper Memory Area (UMA) 640-1024 KB may contain device * memory or memory mapped I/O. Although reading from I/O area * is usually fine, the actual behavior depends on device * implementation. In some cases, the user may want to disable * low RAM method and prevent reading from device I/O area. * * To disable low RAM method: * 1) pass "-B ibft-noprobe=1" on kernel command line * 2) add line "set ibft_noprobe=1" in /etc/system */ cmn_err(CE_NOTE, IBFT_NOPROBE_MSG); return; } ibft_tbl_buf = (char *)kmem_zalloc(ISCSI_IBFT_TBL_BUF_LEN, KM_SLEEP); if (!ibft_tbl_buf) { /* Unlikely to happen */ cmn_err(CE_NOTE, IBFT_INVALID_MSG, IBFT_STATUS_LOWMEM); return; } (void) memset(&boot_property, 0, sizeof (boot_property)); if ((ret = iscsi_scan_ibft_tbl(ibft_tbl_buf)) == IBFT_STATUS_OK) { ret = iscsi_parse_ibft_tbl( (iscsi_ibft_tbl_hdr_t *)ibft_tbl_buf); if (ret == IBFT_STATUS_OK) { iscsiboot_prop = &boot_property; iscsi_print_boot_property(); } else { cmn_err(CE_NOTE, IBFT_INVALID_MSG, ret); } } else if (ret != IBFT_STATUS_NOTABLE) { cmn_err(CE_NOTE, IBFT_INVALID_MSG, ret); } kmem_free(ibft_tbl_buf, ISCSI_IBFT_TBL_BUF_LEN); } /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License, Version 1.0 only * (the "License"). You may not use this file except in compliance * with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2005 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ /* Copyright (c) 1988 AT&T */ /* All Rights Reserved */ #include #include #include #include #include #include #include #include /* * This subsystem (with the minor exception of the instr_size() function) is * is called from DTrace probe context. This imposes several requirements on * the implementation: * * 1. External subsystems and functions may not be referenced. The one current * exception is for cmn_err, but only to signal the detection of table * errors. Assuming the tables are correct, no combination of input is to * trigger a cmn_err call. * * 2. These functions can't be allowed to be traced. To prevent this, * all functions in the probe path (everything except instr_size()) must * have names that begin with "dtrace_". */ typedef enum dis_isize { DIS_ISIZE_INSTR, DIS_ISIZE_OPERAND } dis_isize_t; /* * get a byte from instruction stream */ static int dtrace_dis_get_byte(void *p) { int ret; uchar_t **instr = p; ret = **instr; *instr += 1; return (ret); } /* * Returns either the size of a given instruction, in bytes, or the size of that * instruction's memory access (if any), depending on the value of `which'. * If a programming error in the tables is detected, the system will panic to * ease diagnosis. Invalid instructions will not be flagged. They will appear * to have an instruction size between 1 and the actual size, and will be * reported as having no memory impact. */ /* ARGSUSED2 */ static int dtrace_dis_isize(uchar_t *instr, dis_isize_t which, model_t model, int *rmindex) { int sz; dis86_t x; uint_t mode = SIZE32; mode = (model == DATAMODEL_LP64) ? SIZE64 : SIZE32; x.d86_data = (void **)&instr; x.d86_get_byte = dtrace_dis_get_byte; x.d86_check_func = NULL; if (dtrace_disx86(&x, mode) != 0) return (-1); if (which == DIS_ISIZE_INSTR) sz = x.d86_len; /* length of the instruction */ else sz = x.d86_memsize; /* length of memory operand */ if (rmindex != NULL) *rmindex = x.d86_rmindex; return (sz); } int dtrace_instr_size_isa(uchar_t *instr, model_t model, int *rmindex) { return (dtrace_dis_isize(instr, DIS_ISIZE_INSTR, model, rmindex)); } int dtrace_instr_size(uchar_t *instr) { return (dtrace_dis_isize(instr, DIS_ISIZE_INSTR, DATAMODEL_NATIVE, NULL)); } /*ARGSUSED*/ int instr_size(struct regs *rp, caddr_t *addrp, enum seg_rw rw) { uchar_t instr[16]; /* maximum size instruction */ caddr_t pc = (caddr_t)rp->r_pc; (void) copyin_nowatch(pc, (caddr_t)instr, sizeof (instr)); return (dtrace_dis_isize(instr, rw == S_EXEC ? DIS_ISIZE_INSTR : DIS_ISIZE_OPERAND, curproc->p_model, NULL)); } /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved. * Copyright 2019 Joyent, Inc. */ /* * To understand the present state of interrupt handling on i86pc, we must * first consider the history of interrupt controllers and our way of handling * interrupts. * * History of Interrupt Controllers on i86pc * ----------------------------------------- * * Intel 8259 and 8259A * * The first interrupt controller that attained widespread use on i86pc was * the Intel 8259(A) Programmable Interrupt Controller that first saw use with * the 8086. It took up to 8 interrupt sources and combined them into one * output wire. Up to 8 8259s could be slaved together providing up to 64 IRQs. * With the switch to the 8259A, level mode interrupts became possible. For a * long time on i86pc the 8259A was the only way to handle interrupts and it * had its own set of quirks. The 8259A and its corresponding interval timer * the 8254 are programmed using outb and inb instructions. * * Intel Advanced Programmable Interrupt Controller (APIC) * * Starting around the time of the introduction of the P6 family * microarchitecture (i686) Intel introduced a new interrupt controller. * Instead of having the series of slaved 8259A devices, Intel opted to outfit * each processor with a Local APIC (lapic) and to outfit the system with at * least one, but potentially more, I/O APICs (ioapic). The lapics and ioapics * initially communicated over a dedicated bus, but this has since been * replaced. Each physical core and even hyperthread currently contains its * own local apic, which is not shared. There are a few exceptions for * hyperthreads, but that does not usually concern us. * * Instead of talking directly to 8259 for status, sending End Of Interrupt * (EOI), etc. a microprocessor now communicates directly to the lapic. This * also allows for each microprocessor to be able to have independent controls. * The programming method is different from the 8259. Consumers map the lapic * registers into uncacheable memory to read and manipulate the state. * * The number of addressable interrupt vectors was increased to 256. However * vectors 0-31 are reserved for the processor exception handling, leaving the * remaining vectors for general use. In addition to hardware generated * interrupts, the lapic provides a way for generating inter-processor * interrupts (IPI) which are the basis for CPU cross calls and CPU pokes. * * AMD ended up implementing the Intel APIC architecture in lieu of their work * with Cyrix. * * Intel x2apic * * The x2apic is an extension to the lapic which started showing up around the * same time as the Sandy Bridge chipsets. It provides a new programming mode * as well as new features. The goal of the x2apic is to solve a few problems * with the previous generation of lapic and the x2apic is backwards compatible * with the previous programming and model. The only downsides to using the * backwards compatibility is that you are not able to take advantage of the new * x2apic features. * * o The APIC ID is increased from an 8-bit value to a 32-bit value. This * increases the maximum number of addressable physical processors beyond * 256. This new ID is assembled in a similar manner as the information that * is obtainable by the extended cpuid topology leaves. * * o A new means of generating IPIs was introduced. * * o Instead of memory mapping the registers, the x2apic only allows for * programming it through a series of wrmsrs. This has important semantic * side effects. Recall that the registers were previously all mapped to * uncachable memory which meant that all operations to the local apic were * serializing instructions. With the switch to using wrmsrs this has been * relaxed and these operations can no longer be assumed to be serializing * instructions. * * Note for the rest of this we are only going to concern ourselves with the * apic and x2apic which practically all of i86pc has been using now for * quite some time. * * Interrupt Priority Levels * ------------------------- * * On i86pc systems there are a total of fifteen interrupt priority levels * (ipls) which range from 1-15. Level 0 is for normal processing and * non-interrupt processing. To manipulate these values the family of spl * functions (which date back to UNIX on the PDP-11) are used. Specifically, * splr() to raise the priority level and splx() to lower it. One should not * generally call setspl() directly. * * Both i86pc and the supported SPARC platforms honor the same conventions for * the meaning behind these IPLs. The most important IPL is the platform's * LOCK_LEVEL (0xa on i86pc). If a thread is above LOCK_LEVEL it _must_ not * sleep on any synchronization object. The only allowed synchronization * primitive is a mutex that has been specifically initialized to be a spin * lock (see mutex_init(9F)). Another important level is DISP_LEVEL (0xb on * i86pc). You must be at DISP_LEVEL if you want to control the dispatcher. * The XC_HI_PIL is the highest level (0xf) and is used during cross-calls. * * Each interrupt that is registered in the system fires at a specific IPL. * Generally most interrupts fire below LOCK_LEVEL. * * PSM Drivers * ----------- * * We currently have three sets of PSM (platform specific module) drivers * available. uppc, pcplusmp, and apix. uppc (uni-processor PC) is the original * driver that interacts with the 8259A and 8254. In general, it is not used * anymore given the prevalence of the apic. * * The system prefers to use the apix driver over the pcplusmp driver. The apix * driver requires HW support for an x2apic. If there is no x2apic HW, apix * will not be used. In general we prefer using the apix driver over the * pcplusmp driver because it gives us much more flexibility with respect to * interrupts. In the apix driver each local apic has its own independent set * of interrupts, whereas the pcplusmp driver only has a single global set of * interrupts. This is why pcplusmp only supports a finite number of interrupts * per IPL -- generally 16, often less. The apix driver supports using either * the x2apic or the local apic programing modes. The programming mode does not * change the number of interrupts available, just the number of processors * that we can address. For the apix driver, the x2apic mode is enabled if the * system supports interrupt re-mapping, otherwise the module manages the * x2apic in local mode. * * When there is no x2apic present, we default back to the pcplusmp PSM driver. * In general, this is not problematic unless you have more than 256 * processors in the machine or you do not have enough interrupts available. * * Controlling Interrupt Generation on i86pc * ----------------------------------------- * * There are two different ways to manipulate which interrupts will be * generated on i86pc. Each offers different degrees of control. * * The first is through the flags register (eflags and rflags on i386 and amd64 * respectively). The IF bit determines whether or not interrupts are enabled * or disabled. This is manipulated in one of several ways. The most common way * is through the cli and sti instructions. These clear the IF flag and set it, * respectively, for the current processor. The other common way is through the * use of the intr_clear and intr_restore functions. * * Assuming interrupts are not blocked by the IF flag, then the second form is * through the Processor-Priority Register (PPR). The PPR is used to determine * whether or not a pending interrupt should be delivered. If the ipl of the * new interrupt is higher than the current value in the PPR, then the lapic * will either deliver it immediately (if interrupts are not in progress) or it * will deliver it once the current interrupt processing has issued an EOI. The * highest unmasked interrupt will be the one delivered. * * The PPR register is based upon the max of the following two registers in the * lapic, the TPR register (also known as CR8 on amd64) that can be used to * mask interrupt levels, and the current vector. Because the pcplusmp module * always sets TPR appropriately early in the do_interrupt path, we can usually * just think that the PPR is the TPR. The pcplusmp module also issues an EOI * once it has set the TPR, so higher priority interrupts can come in while * we're servicing a lower priority interrupt. * * Handling Interrupts * ------------------- * * Interrupts can be broken down into three categories based on priority and * source: * * o High level interrupts * o Low level hardware interrupts * o Low level software interrupts * * High Level Interrupts * * High level interrupts encompasses both hardware-sourced and software-sourced * interrupts. Examples of high level hardware interrupts include the serial * console. High level software-sourced interrupts are still delivered through * the local apic through IPIs. This is primarily cross calls. * * When a high level interrupt comes in, we will raise the SPL and then pin the * current lwp to the processor. We will use its lwp, but our own interrupt * stack and process the high level interrupt in-situ. These handlers are * designed to be very short in nature and cannot go to sleep, only block on a * spin lock. If the interrupt has a lot of work to do, it must generate a * low-priority software interrupt that will be processed later. * * Low level hardware interrupts * * Low level hardware interrupts start off like their high-level cousins. The * current CPU contains a number of kernel threads (kthread_t) that can be used * to process low level interrupts. These are shared between both low level * hardware and software interrupts. Note that while we run with our * kthread_t, we borrow the pinned threads lwp_t until such a time as we hit a * synchronization object. If we hit one and need to sleep, then the scheduler * will instead create the rest of what we need. * * Low level software interrupts * * Low level software interrupts are handled in a similar way as hardware * interrupts, but the notification vector is different. Each CPU has a bitmask * of pending software interrupts. We can notify a CPU to process software * interrupts through a specific trap vector as well as through several * checks that are performed throughout the code. These checks will look at * processing software interrupts as we lower our spl. * * We attempt to process the highest pending software interrupt that we can * which is greater than our current IPL. If none currently exist, then we move * on. We process a software interrupt in a similar fashion to a hardware * interrupt. * * Traditional Interrupt Flow * -------------------------- * * The following diagram tracks the flow of the traditional uppc and pcplusmp * interrupt handlers. The apix driver has its own version of do_interrupt(). * We come into the interrupt handler with all interrupts masked by the IF * flag. This is because we set up the handler using an interrupt-gate, which * is defined architecturally to have cleared the IF flag for us. * * +--------------+ +----------------+ +-----------+ * | _interrupt() |--->| do_interrupt() |--->| *setlvl() | * +--------------+ +----------------+ +-----------+ * | | | * | | | * low-level| | | softint * HW int | | +---------------------------------------+ * +--------------+ | | | * | intr_thread_ |<-----+ | hi-level int | * | prolog() | | +----------+ | * +--------------+ +--->| hilevel_ | Not on intr stack | * | | intr_ |-----------------+ | * | | prolog() | | | * +------------+ +----------+ | | * | switch_sp_ | | On intr v | * | and_call() | | Stack +------------+ | * +------------+ | | switch_sp_ | | * | v | and_call() | | * v +-----------+ +------------+ | * +-----------+ | dispatch_ | | | * | dispatch_ | +-------------------| hilevel() |<------------+ | * | hardint() | | +-----------+ | * +-----------+ | | * | v | * | +-----+ +----------------------+ +-----+ hi-level | * +---->| sti |->| av_dispatch_autovect |->| cli |---------+ | * +-----+ +----------------------+ +-----+ | | * | | | | * v | | | * +----------+ | | | * | for each | | | | * | handler | | | | * | *intr() | | v | * +--------------+ +----------+ | +----------------+ | * | intr_thread_ | low-level | | hilevel_intr_ | | * | epilog() |<-------------------------------+ | epilog() | | * +--------------+ +----------------+ | * | | | | * | +----------------------v v---------------------+ | * | +------------+ | * | +---------------------->| *setlvlx() | | * | | +------------+ | * | | | | * | | v | * | | +--------+ +------------------+ +-------------+ | * | | | return |<----| softint pending? |----->| dosoftint() |<-----+ * | | +--------+ no +------------------+ yes +-------------+ * | | ^ | | * | | | softint pil too low | | * | | +--------------------------------------+ | * | | v * | | +-----------+ +------------+ +-----------+ * | | | dispatch_ |<-----| switch_sp_ |<---------| *setspl() | * | | | softint() | | and_call() | +-----------+ * | | +-----------+ +------------+ * | | | * | | v * | | +-----+ +----------------------+ +-----+ +------------+ * | | | sti |->| av_dispatch_autovect |->| cli |->| dosoftint_ | * | | +-----+ +----------------------+ +-----+ | epilog() | * | | +------------+ * | | | | * | +----------------------------------------------------+ | * v | * +-----------+ | * | interrupt | | * | thread |<---------------------------------------------------+ * | blocked | * +-----------+ * | * v * +----------------+ +------------+ +-----------+ +-------+ +---------+ * | set_base_spl() |->| *setlvlx() |->| splhigh() |->| sti() |->| swtch() | * +----------------+ +------------+ +-----------+ +-------+ +---------+ * * Calls made on Interrupt Stacks and Epilogue routines * * We use the switch_sp_and_call() assembly routine to switch our sp to the * interrupt stacks and then call the appropriate dispatch function. In the * case of interrupts which may block, softints and hardints, we always ensure * that we are still on the interrupt thread when we call the epilog routine. * This is not just important, it's necessary. If the interrupt thread blocked, * we won't return from our switch_sp_and_call() function and instead we'll go * through and set ourselves up to swtch() directly. * * New Interrupt Flow * ------------------ * * The apix module has its own interrupt path. This is done for various * reasons. The first is that rather than having global interrupt vectors, we * now have per-cpu vectors. * * The other substantial change is that the apix design does not use the TPR to * mask interrupts below the current level. In fact, except for one special * case, it does not use the TPR at all. Instead, it only uses the IF flag * (cli/sti) to either block all interrupts or allow any interrupts to come in. * The design is such that when interrupts are allowed to come in, if we are * currently servicing a higher priority interupt, the new interrupt is treated * as pending and serviced later. Specifically, in the pcplusmp module's * apic_intr_enter() the code masks interrupts at or below the current * IPL using the TPR before sending EOI, whereas the apix module's * apix_intr_enter() simply sends EOI. * * The one special case where the apix code uses the TPR is when it calls * through the apic_reg_ops function pointer apic_write_task_reg in * apix_init_intr() to initially mask all levels and then finally to enable all * levels. * * Recall that we come into the interrupt handler with all interrupts masked * by the IF flag. This is because we set up the handler using an * interrupt-gate which is defined architecturally to have cleared the IF flag * for us. * * +--------------+ +---------------------+ * | _interrupt() |--->| apix_do_interrupt() | * +--------------+ +---------------------+ * | * hard int? +----+--------+ softint? * | | (but no low-level looping) * +-----------+ | * | *setlvl() | | * +---------+ +-----------+ +----------------------------------+ * |apix_add_| check IPL | | * |pending_ |<-------------+------+----------------------+ | * |hardint()| low-level int| hi-level int| | * +---------+ v v | * | check IPL +-----------------+ +---------------+ | * +--+-----+ | apix_intr_ | | apix_hilevel_ | | * | | | thread_prolog() | | intr_prolog() | | * | return +-----------------+ +---------------+ | * | | | On intr | * | +------------+ | stack? +------------+ | * | | switch_sp_ | +---------| switch_sp_ | | * | | and_call() | | | and_call() | | * | +------------+ | +------------+ | * | | | | | * | +----------------+ +----------------+ | * | | apix_dispatch_ | | apix_dispatch_ | | * | | lowlevel() | | hilevel() | | * | +----------------+ +----------------+ | * | | | | * | v v | * | +-------------------------+ | * | |apix_dispatch_by_vector()|----+ | * | +-------------------------+ | | * | !XC_HI_PIL| | | | | * | +---+ +-------+ +---+ | | * | |sti| |*intr()| |cli| | | * | +---+ +-------+ +---+ | hi-level? | * | +---------------------------+----+ | * | v low-level? v | * | +----------------+ +----------------+ | * | | apix_intr_ | | apix_hilevel_ | | * | | thread_epilog()| | intr_epilog() | | * | +----------------+ +----------------+ | * | | | | * | v-----------------+--------------------------------+ | * | +------------+ | * | | *setlvlx() | +----------------------------------------------------+ * | +------------+ | * | | | +--------------------------------+ low * v v v------+ v | level * +------------------+ +------------------+ +-----------+ | pending? * | apix_do_pending_ |----->| apix_do_pending_ |----->| apix_do_ |--+ * | hilevel() | | hardint() | | softint() | | * +------------------+ +------------------+ +-----------+ return * | | | * | while pending | while pending | while pending * | hi-level | low-level | softint * | | | * +---------------+ +-----------------+ +-----------------+ * | apix_hilevel_ | | apix_intr_ | | apix_do_ | * | intr_prolog() | | thread_prolog() | | softint_prolog()| * +---------------+ +-----------------+ +-----------------+ * | On intr | | * | stack? +------------+ +------------+ +------------+ * +--------| switch_sp_ | | switch_sp_ | | switch_sp_ | * | | and_call() | | and_call() | | and_call() | * | +------------+ +------------+ +------------+ * | | | | * +------------------+ +------------------+ +------------------------+ * | apix_dispatch_ | | apix_dispatch_ | | apix_dispatch_softint()| * | pending_hilevel()| | pending_hardint()| +------------------------+ * +------------------+ +------------------+ | | | | * | | | | | | | | * | +----------------+ | +----------------+ | | | | * | | apix_hilevel_ | | | apix_intr_ | | | | | * | | intr_epilog() | | | thread_epilog()| | | | | * | +----------------+ | +----------------+ | | | | * | | | | | | | | * | +------------+ | +----------+ +------+ | | | * | | *setlvlx() | | |*setlvlx()| | | | | * | +------------+ | +----------+ | +----------+ | +---------+ * | | +---+ |av_ | +---+ |apix_do_ | * +---------------------------------+ |sti| |dispatch_ | |cli| |softint_ | * | apix_dispatch_pending_autovect()| +---+ |softvect()| +---+ |epilog() | * +---------------------------------+ +----------+ +---------+ * |!XC_HI_PIL | | | | * +---+ +-------+ +---+ +----------+ +-------+ * |sti| |*intr()| |cli| |apix_post_| |*intr()| * +---+ +-------+ +---+ |hardint() | +-------+ * +----------+ */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #if defined(__xpv) #include #endif /* If these fail, then the padding numbers in machcpuvar.h are wrong. */ #if !defined(__xpv) #define MCOFF(member) \ (offsetof(cpu_t, cpu_m) + offsetof(struct machcpu, member)) CTASSERT(MCOFF(mcpu_pad) == MACHCPU_SIZE); CTASSERT(MCOFF(mcpu_pad2) == MMU_PAGESIZE); CTASSERT((MCOFF(mcpu_kpti) & 0xF) == 0); CTASSERT(((sizeof (struct kpti_frame)) & 0xF) == 0); CTASSERT((offsetof(struct kpti_frame, kf_tr_rsp) & 0xF) == 0); CTASSERT(MCOFF(mcpu_pad3) < 2 * MMU_PAGESIZE); #endif #if defined(__xpv) && defined(DEBUG) /* * This panic message is intended as an aid to interrupt debugging. * * The associated assertion tests the condition of enabling * events when events are already enabled. The implication * being that whatever code the programmer thought was * protected by having events disabled until the second * enable happened really wasn't protected at all .. */ int stistipanic = 1; /* controls the debug panic check */ const char *stistimsg = "stisti"; ulong_t laststi[NCPU]; /* * This variable tracks the last place events were disabled on each cpu * it assists in debugging when asserts that interrupts are enabled trip. */ ulong_t lastcli[NCPU]; #endif void do_interrupt(struct regs *rp, trap_trace_rec_t *ttp); void (*do_interrupt_common)(struct regs *, trap_trace_rec_t *) = do_interrupt; uintptr_t (*get_intr_handler)(int, short) = NULL; /* * Set cpu's base SPL level to the highest active interrupt level */ void set_base_spl(void) { struct cpu *cpu = CPU; uint16_t active = (uint16_t)cpu->cpu_intr_actv; cpu->cpu_base_spl = active == 0 ? 0 : bsrw_insn(active); } /* * Do all the work necessary to set up the cpu and thread structures * to dispatch a high-level interrupt. * * Returns 0 if we're -not- already on the high-level interrupt stack, * (and *must* switch to it), non-zero if we are already on that stack. * * Called with interrupts masked. * The 'pil' is already set to the appropriate level for rp->r_trapno. */ static int hilevel_intr_prolog(struct cpu *cpu, uint_t pil, uint_t oldpil, struct regs *rp) { struct machcpu *mcpu = &cpu->cpu_m; uint_t mask; hrtime_t intrtime; hrtime_t now = tsc_read(); ASSERT(pil > LOCK_LEVEL); if (pil == CBE_HIGH_PIL) { cpu->cpu_profile_pil = oldpil; if (USERMODE(rp->r_cs)) { cpu->cpu_profile_pc = 0; cpu->cpu_profile_upc = rp->r_pc; cpu->cpu_cpcprofile_pc = 0; cpu->cpu_cpcprofile_upc = rp->r_pc; } else { cpu->cpu_profile_pc = rp->r_pc; cpu->cpu_profile_upc = 0; cpu->cpu_cpcprofile_pc = rp->r_pc; cpu->cpu_cpcprofile_upc = 0; } } mask = cpu->cpu_intr_actv & CPU_INTR_ACTV_HIGH_LEVEL_MASK; if (mask != 0) { int nestpil; /* * We have interrupted another high-level interrupt. * Load starting timestamp, compute interval, update * cumulative counter. */ nestpil = bsrw_insn((uint16_t)mask); ASSERT(nestpil < pil); intrtime = now - mcpu->pil_high_start[nestpil - (LOCK_LEVEL + 1)]; mcpu->intrstat[nestpil][0] += intrtime; cpu->cpu_intracct[cpu->cpu_mstate] += intrtime; /* * Another high-level interrupt is active below this one, so * there is no need to check for an interrupt thread. That * will be done by the lowest priority high-level interrupt * active. */ } else { kthread_t *t = cpu->cpu_thread; /* * See if we are interrupting a low-level interrupt thread. * If so, account for its time slice only if its time stamp * is non-zero. */ if ((t->t_flag & T_INTR_THREAD) != 0 && t->t_intr_start != 0) { intrtime = now - t->t_intr_start; mcpu->intrstat[t->t_pil][0] += intrtime; cpu->cpu_intracct[cpu->cpu_mstate] += intrtime; t->t_intr_start = 0; } } smt_begin_intr(pil); /* * Store starting timestamp in CPU structure for this PIL. */ mcpu->pil_high_start[pil - (LOCK_LEVEL + 1)] = now; ASSERT((cpu->cpu_intr_actv & (1 << pil)) == 0); if (pil == 15) { /* * To support reentrant level 15 interrupts, we maintain a * recursion count in the top half of cpu_intr_actv. Only * when this count hits zero do we clear the PIL 15 bit from * the lower half of cpu_intr_actv. */ uint16_t *refcntp = (uint16_t *)&cpu->cpu_intr_actv + 1; (*refcntp)++; } mask = cpu->cpu_intr_actv; cpu->cpu_intr_actv |= (1 << pil); return (mask & CPU_INTR_ACTV_HIGH_LEVEL_MASK); } /* * Does most of the work of returning from a high level interrupt. * * Returns 0 if there are no more high level interrupts (in which * case we must switch back to the interrupted thread stack) or * non-zero if there are more (in which case we should stay on it). * * Called with interrupts masked */ static int hilevel_intr_epilog(struct cpu *cpu, uint_t pil, uint_t oldpil, uint_t vecnum) { struct machcpu *mcpu = &cpu->cpu_m; uint_t mask; hrtime_t intrtime; hrtime_t now = tsc_read(); ASSERT(mcpu->mcpu_pri == pil); cpu->cpu_stats.sys.intr[pil - 1]++; ASSERT(cpu->cpu_intr_actv & (1 << pil)); if (pil == 15) { /* * To support reentrant level 15 interrupts, we maintain a * recursion count in the top half of cpu_intr_actv. Only * when this count hits zero do we clear the PIL 15 bit from * the lower half of cpu_intr_actv. */ uint16_t *refcntp = (uint16_t *)&cpu->cpu_intr_actv + 1; ASSERT(*refcntp > 0); if (--(*refcntp) == 0) cpu->cpu_intr_actv &= ~(1 << pil); } else { cpu->cpu_intr_actv &= ~(1 << pil); } ASSERT(mcpu->pil_high_start[pil - (LOCK_LEVEL + 1)] != 0); intrtime = now - mcpu->pil_high_start[pil - (LOCK_LEVEL + 1)]; mcpu->intrstat[pil][0] += intrtime; cpu->cpu_intracct[cpu->cpu_mstate] += intrtime; /* * Check for lower-pil nested high-level interrupt beneath * current one. If so, place a starting timestamp in its * pil_high_start entry. */ mask = cpu->cpu_intr_actv & CPU_INTR_ACTV_HIGH_LEVEL_MASK; if (mask != 0) { int nestpil; /* * find PIL of nested interrupt */ nestpil = bsrw_insn((uint16_t)mask); ASSERT(nestpil < pil); mcpu->pil_high_start[nestpil - (LOCK_LEVEL + 1)] = now; /* * (Another high-level interrupt is active below this one, * so there is no need to check for an interrupt * thread. That will be done by the lowest priority * high-level interrupt active.) */ } else { /* * Check to see if there is a low-level interrupt active. * If so, place a starting timestamp in the thread * structure. */ kthread_t *t = cpu->cpu_thread; if (t->t_flag & T_INTR_THREAD) t->t_intr_start = now; } smt_end_intr(); mcpu->mcpu_pri = oldpil; (void) (*setlvlx)(oldpil, vecnum); return (cpu->cpu_intr_actv & CPU_INTR_ACTV_HIGH_LEVEL_MASK); } /* * Set up the cpu, thread and interrupt thread structures for * executing an interrupt thread. The new stack pointer of the * interrupt thread (which *must* be switched to) is returned. */ static caddr_t intr_thread_prolog(struct cpu *cpu, caddr_t stackptr, uint_t pil) { struct machcpu *mcpu = &cpu->cpu_m; kthread_t *t, *volatile it; hrtime_t now = tsc_read(); ASSERT(pil > 0); ASSERT((cpu->cpu_intr_actv & (1 << pil)) == 0); cpu->cpu_intr_actv |= (1 << pil); /* * Get set to run an interrupt thread. * There should always be an interrupt thread, since we * allocate one for each level on each CPU. * * t_intr_start could be zero due to cpu_intr_swtch_enter. */ t = cpu->cpu_thread; if ((t->t_flag & T_INTR_THREAD) && t->t_intr_start != 0) { hrtime_t intrtime = now - t->t_intr_start; mcpu->intrstat[t->t_pil][0] += intrtime; cpu->cpu_intracct[cpu->cpu_mstate] += intrtime; t->t_intr_start = 0; } ASSERT(SA((uintptr_t)stackptr) == (uintptr_t)stackptr); t->t_sp = (uintptr_t)stackptr; /* mark stack in curthread for resume */ /* * unlink the interrupt thread off the cpu * * Note that the code in kcpc_overflow_intr -relies- on the * ordering of events here - in particular that t->t_lwp of * the interrupt thread is set to the pinned thread *before* * curthread is changed. */ it = cpu->cpu_intr_thread; cpu->cpu_intr_thread = it->t_link; it->t_intr = t; it->t_lwp = t->t_lwp; /* * (threads on the interrupt thread free list could have state * preset to TS_ONPROC, but it helps in debugging if * they're TS_FREE.) */ it->t_state = TS_ONPROC; cpu->cpu_thread = it; /* new curthread on this cpu */ smt_begin_intr(pil); it->t_pil = (uchar_t)pil; it->t_pri = intr_pri + (pri_t)pil; it->t_intr_start = now; return (it->t_stk); } #ifdef DEBUG int intr_thread_cnt; #endif /* * Called with interrupts disabled */ static void intr_thread_epilog(struct cpu *cpu, uint_t vec, uint_t oldpil) { struct machcpu *mcpu = &cpu->cpu_m; kthread_t *t; kthread_t *it = cpu->cpu_thread; /* curthread */ uint_t pil, basespl; hrtime_t intrtime; hrtime_t now = tsc_read(); pil = it->t_pil; cpu->cpu_stats.sys.intr[pil - 1]++; ASSERT(it->t_intr_start != 0); intrtime = now - it->t_intr_start; mcpu->intrstat[pil][0] += intrtime; cpu->cpu_intracct[cpu->cpu_mstate] += intrtime; ASSERT(cpu->cpu_intr_actv & (1 << pil)); cpu->cpu_intr_actv &= ~(1 << pil); /* * If there is still an interrupted thread underneath this one * then the interrupt was never blocked and the return is * fairly simple. Otherwise it isn't. */ if ((t = it->t_intr) == NULL) { /* * The interrupted thread is no longer pinned underneath * the interrupt thread. This means the interrupt must * have blocked, and the interrupted thread has been * unpinned, and has probably been running around the * system for a while. * * Since there is no longer a thread under this one, put * this interrupt thread back on the CPU's free list and * resume the idle thread which will dispatch the next * thread to run. */ #ifdef DEBUG intr_thread_cnt++; #endif cpu->cpu_stats.sys.intrblk++; /* * Set CPU's base SPL based on active interrupts bitmask */ set_base_spl(); basespl = cpu->cpu_base_spl; mcpu->mcpu_pri = basespl; (*setlvlx)(basespl, vec); (void) splhigh(); sti(); it->t_state = TS_FREE; /* * Return interrupt thread to pool */ it->t_link = cpu->cpu_intr_thread; cpu->cpu_intr_thread = it; swtch(); panic("intr_thread_epilog: swtch returned"); /*NOTREACHED*/ } /* * Return interrupt thread to the pool */ it->t_link = cpu->cpu_intr_thread; cpu->cpu_intr_thread = it; it->t_state = TS_FREE; basespl = cpu->cpu_base_spl; pil = MAX(oldpil, basespl); mcpu->mcpu_pri = pil; (*setlvlx)(pil, vec); t->t_intr_start = now; smt_end_intr(); cpu->cpu_thread = t; } /* * intr_get_time() is a resource for interrupt handlers to determine how * much time has been spent handling the current interrupt. Such a function * is needed because higher level interrupts can arrive during the * processing of an interrupt. intr_get_time() only returns time spent in the * current interrupt handler. * * The caller must be calling from an interrupt handler running at a pil * below or at lock level. Timings are not provided for high-level * interrupts. * * The first time intr_get_time() is called while handling an interrupt, * it returns the time since the interrupt handler was invoked. Subsequent * calls will return the time since the prior call to intr_get_time(). Time * is returned as ticks. Use scalehrtimef() to convert ticks to nsec. * * Theory Of Intrstat[][]: * * uint64_t intrstat[pil][0..1] is an array indexed by pil level, with two * uint64_ts per pil. * * intrstat[pil][0] is a cumulative count of the number of ticks spent * handling all interrupts at the specified pil on this CPU. It is * exported via kstats to the user. * * intrstat[pil][1] is always a count of ticks less than or equal to the * value in [0]. The difference between [1] and [0] is the value returned * by a call to intr_get_time(). At the start of interrupt processing, * [0] and [1] will be equal (or nearly so). As the interrupt consumes * time, [0] will increase, but [1] will remain the same. A call to * intr_get_time() will return the difference, then update [1] to be the * same as [0]. Future calls will return the time since the last call. * Finally, when the interrupt completes, [1] is updated to the same as [0]. * * Implementation: * * intr_get_time() works much like a higher level interrupt arriving. It * "checkpoints" the timing information by incrementing intrstat[pil][0] * to include elapsed running time, and by setting t_intr_start to rdtsc. * It then sets the return value to intrstat[pil][0] - intrstat[pil][1], * and updates intrstat[pil][1] to be the same as the new value of * intrstat[pil][0]. * * In the normal handling of interrupts, after an interrupt handler returns * and the code in intr_thread() updates intrstat[pil][0], it then sets * intrstat[pil][1] to the new value of intrstat[pil][0]. When [0] == [1], * the timings are reset, i.e. intr_get_time() will return [0] - [1] which * is 0. * * Whenever interrupts arrive on a CPU which is handling a lower pil * interrupt, they update the lower pil's [0] to show time spent in the * handler that they've interrupted. This results in a growing discrepancy * between [0] and [1], which is returned the next time intr_get_time() is * called. Time spent in the higher-pil interrupt will not be returned in * the next intr_get_time() call from the original interrupt, because * the higher-pil interrupt's time is accumulated in intrstat[higherpil][]. */ uint64_t intr_get_time(void) { struct cpu *cpu; struct machcpu *mcpu; kthread_t *t; uint64_t time, delta, ret; uint_t pil; cli(); cpu = CPU; mcpu = &cpu->cpu_m; t = cpu->cpu_thread; pil = t->t_pil; ASSERT((cpu->cpu_intr_actv & CPU_INTR_ACTV_HIGH_LEVEL_MASK) == 0); ASSERT(t->t_flag & T_INTR_THREAD); ASSERT(pil != 0); ASSERT(t->t_intr_start != 0); time = tsc_read(); delta = time - t->t_intr_start; t->t_intr_start = time; time = mcpu->intrstat[pil][0] + delta; ret = time - mcpu->intrstat[pil][1]; mcpu->intrstat[pil][0] = time; mcpu->intrstat[pil][1] = time; cpu->cpu_intracct[cpu->cpu_mstate] += delta; sti(); return (ret); } static caddr_t dosoftint_prolog( struct cpu *cpu, caddr_t stackptr, uint32_t st_pending, uint_t oldpil) { kthread_t *t, *volatile it; struct machcpu *mcpu = &cpu->cpu_m; uint_t pil; hrtime_t now; top: ASSERT(st_pending == mcpu->mcpu_softinfo.st_pending); pil = bsrw_insn((uint16_t)st_pending); if (pil <= oldpil || pil <= cpu->cpu_base_spl) return (0); /* * XX64 Sigh. * * This is a transliteration of the i386 assembler code for * soft interrupts. One question is "why does this need * to be atomic?" One possible race is -other- processors * posting soft interrupts to us in set_pending() i.e. the * CPU might get preempted just after the address computation, * but just before the atomic transaction, so another CPU would * actually set the original CPU's st_pending bit. However, * it looks like it would be simpler to disable preemption there. * Are there other races for which preemption control doesn't work? * * The i386 assembler version -also- checks to see if the bit * being cleared was actually set; if it wasn't, it rechecks * for more. This seems a bit strange, as the only code that * ever clears the bit is -this- code running with interrupts * disabled on -this- CPU. This code would probably be cheaper: * * atomic_and_32((uint32_t *)&mcpu->mcpu_softinfo.st_pending, * ~(1 << pil)); * * and t->t_preempt--/++ around set_pending() even cheaper, * but at this point, correctness is critical, so we slavishly * emulate the i386 port. */ if (atomic_btr32((uint32_t *) &mcpu->mcpu_softinfo.st_pending, pil) == 0) { st_pending = mcpu->mcpu_softinfo.st_pending; goto top; } mcpu->mcpu_pri = pil; (*setspl)(pil); now = tsc_read(); /* * Get set to run interrupt thread. * There should always be an interrupt thread since we * allocate one for each level on the CPU. */ it = cpu->cpu_intr_thread; cpu->cpu_intr_thread = it->t_link; /* t_intr_start could be zero due to cpu_intr_swtch_enter. */ t = cpu->cpu_thread; if ((t->t_flag & T_INTR_THREAD) && t->t_intr_start != 0) { hrtime_t intrtime = now - t->t_intr_start; mcpu->intrstat[pil][0] += intrtime; cpu->cpu_intracct[cpu->cpu_mstate] += intrtime; t->t_intr_start = 0; } /* * Note that the code in kcpc_overflow_intr -relies- on the * ordering of events here - in particular that t->t_lwp of * the interrupt thread is set to the pinned thread *before* * curthread is changed. */ it->t_lwp = t->t_lwp; it->t_state = TS_ONPROC; /* * Push interrupted thread onto list from new thread. * Set the new thread as the current one. * Set interrupted thread's T_SP because if it is the idle thread, * resume() may use that stack between threads. */ ASSERT(SA((uintptr_t)stackptr) == (uintptr_t)stackptr); t->t_sp = (uintptr_t)stackptr; it->t_intr = t; cpu->cpu_thread = it; smt_begin_intr(pil); /* * Set bit for this pil in CPU's interrupt active bitmask. */ ASSERT((cpu->cpu_intr_actv & (1 << pil)) == 0); cpu->cpu_intr_actv |= (1 << pil); /* * Initialize thread priority level from intr_pri */ it->t_pil = (uchar_t)pil; it->t_pri = (pri_t)pil + intr_pri; it->t_intr_start = now; return (it->t_stk); } static void dosoftint_epilog(struct cpu *cpu, uint_t oldpil) { struct machcpu *mcpu = &cpu->cpu_m; kthread_t *t, *it; uint_t pil, basespl; hrtime_t intrtime; hrtime_t now = tsc_read(); it = cpu->cpu_thread; pil = it->t_pil; cpu->cpu_stats.sys.intr[pil - 1]++; ASSERT(cpu->cpu_intr_actv & (1 << pil)); cpu->cpu_intr_actv &= ~(1 << pil); intrtime = now - it->t_intr_start; mcpu->intrstat[pil][0] += intrtime; cpu->cpu_intracct[cpu->cpu_mstate] += intrtime; /* * If there is still an interrupted thread underneath this one * then the interrupt was never blocked and the return is * fairly simple. Otherwise it isn't. */ if ((t = it->t_intr) == NULL) { /* * Put thread back on the interrupt thread list. * This was an interrupt thread, so set CPU's base SPL. */ set_base_spl(); it->t_state = TS_FREE; it->t_link = cpu->cpu_intr_thread; cpu->cpu_intr_thread = it; (void) splhigh(); sti(); swtch(); /*NOTREACHED*/ panic("dosoftint_epilog: swtch returned"); } it->t_link = cpu->cpu_intr_thread; cpu->cpu_intr_thread = it; it->t_state = TS_FREE; smt_end_intr(); cpu->cpu_thread = t; if (t->t_flag & T_INTR_THREAD) t->t_intr_start = now; basespl = cpu->cpu_base_spl; pil = MAX(oldpil, basespl); mcpu->mcpu_pri = pil; (*setspl)(pil); } /* * Make the interrupted thread 'to' be runnable. * * Since t->t_sp has already been saved, t->t_pc is all * that needs to be set in this function. * * Returns the interrupt level of the interrupt thread. */ int intr_passivate( kthread_t *it, /* interrupt thread */ kthread_t *t) /* interrupted thread */ { extern void _sys_rtt(); ASSERT(it->t_flag & T_INTR_THREAD); ASSERT(SA(t->t_sp) == t->t_sp); t->t_pc = (uintptr_t)_sys_rtt; return (it->t_pil); } /* * Create interrupt kstats for this CPU. */ void cpu_create_intrstat(cpu_t *cp) { int i; kstat_t *intr_ksp; kstat_named_t *knp; char name[KSTAT_STRLEN]; zoneid_t zoneid; ASSERT(MUTEX_HELD(&cpu_lock)); if (pool_pset_enabled()) zoneid = GLOBAL_ZONEID; else zoneid = ALL_ZONES; intr_ksp = kstat_create_zone("cpu", cp->cpu_id, "intrstat", "misc", KSTAT_TYPE_NAMED, PIL_MAX * 2, 0, zoneid); /* * Initialize each PIL's named kstat */ if (intr_ksp != NULL) { intr_ksp->ks_update = cpu_kstat_intrstat_update; knp = (kstat_named_t *)intr_ksp->ks_data; intr_ksp->ks_private = cp; for (i = 0; i < PIL_MAX; i++) { (void) snprintf(name, KSTAT_STRLEN, "level-%d-time", i + 1); kstat_named_init(&knp[i * 2], name, KSTAT_DATA_UINT64); (void) snprintf(name, KSTAT_STRLEN, "level-%d-count", i + 1); kstat_named_init(&knp[(i * 2) + 1], name, KSTAT_DATA_UINT64); } kstat_install(intr_ksp); } } /* * Delete interrupt kstats for this CPU. */ void cpu_delete_intrstat(cpu_t *cp) { kstat_delete_byname_zone("cpu", cp->cpu_id, "intrstat", ALL_ZONES); } /* * Convert interrupt statistics from CPU ticks to nanoseconds and * update kstat. */ int cpu_kstat_intrstat_update(kstat_t *ksp, int rw) { kstat_named_t *knp = ksp->ks_data; cpu_t *cpup = (cpu_t *)ksp->ks_private; int i; hrtime_t hrt; if (rw == KSTAT_WRITE) return (EACCES); for (i = 0; i < PIL_MAX; i++) { hrt = (hrtime_t)cpup->cpu_m.intrstat[i + 1][0]; scalehrtimef(&hrt); knp[i * 2].value.ui64 = (uint64_t)hrt; knp[(i * 2) + 1].value.ui64 = cpup->cpu_stats.sys.intr[i]; } return (0); } /* * An interrupt thread is ending a time slice, so compute the interval it * ran for and update the statistic for its PIL. */ void cpu_intr_swtch_enter(kthread_id_t t) { uint64_t interval; uint64_t start; cpu_t *cpu; ASSERT((t->t_flag & T_INTR_THREAD) != 0); ASSERT(t->t_pil > 0 && t->t_pil <= LOCK_LEVEL); /* * We could be here with a zero timestamp. This could happen if: * an interrupt thread which no longer has a pinned thread underneath * it (i.e. it blocked at some point in its past) has finished running * its handler. intr_thread() updated the interrupt statistic for its * PIL and zeroed its timestamp. Since there was no pinned thread to * return to, swtch() gets called and we end up here. * * Note that we use atomic ops below (atomic_cas_64 and * atomic_add_64), which we don't use in the functions above, * because we're not called with interrupts blocked, but the * epilog/prolog functions are. */ if (t->t_intr_start) { do { start = t->t_intr_start; interval = tsc_read() - start; } while (atomic_cas_64(&t->t_intr_start, start, 0) != start); cpu = CPU; cpu->cpu_m.intrstat[t->t_pil][0] += interval; atomic_add_64((uint64_t *)&cpu->cpu_intracct[cpu->cpu_mstate], interval); } else ASSERT(t->t_intr == NULL); } /* * An interrupt thread is returning from swtch(). Place a starting timestamp * in its thread structure. */ void cpu_intr_swtch_exit(kthread_id_t t) { uint64_t ts; ASSERT((t->t_flag & T_INTR_THREAD) != 0); ASSERT(t->t_pil > 0 && t->t_pil <= LOCK_LEVEL); do { ts = t->t_intr_start; } while (atomic_cas_64(&t->t_intr_start, ts, tsc_read()) != ts); } /* * Dispatch a hilevel interrupt (one above LOCK_LEVEL) */ /*ARGSUSED*/ static void dispatch_hilevel(uint_t vector, uint_t arg2) { sti(); av_dispatch_autovect(vector); cli(); } /* * Dispatch a soft interrupt */ /*ARGSUSED*/ static void dispatch_softint(uint_t oldpil, uint_t arg2) { struct cpu *cpu = CPU; sti(); av_dispatch_softvect((int)cpu->cpu_thread->t_pil); cli(); /* * Must run softint_epilog() on the interrupt thread stack, since * there may not be a return from it if the interrupt thread blocked. */ dosoftint_epilog(cpu, oldpil); } /* * Dispatch a normal interrupt */ static void dispatch_hardint(uint_t vector, uint_t oldipl) { struct cpu *cpu = CPU; sti(); av_dispatch_autovect(vector); cli(); /* * Must run intr_thread_epilog() on the interrupt thread stack, since * there may not be a return from it if the interrupt thread blocked. */ intr_thread_epilog(cpu, vector, oldipl); } /* * Deliver any softints the current interrupt priority allows. * Called with interrupts disabled. */ void dosoftint(struct regs *regs) { struct cpu *cpu = CPU; int oldipl; caddr_t newsp; while (cpu->cpu_softinfo.st_pending) { oldipl = cpu->cpu_pri; newsp = dosoftint_prolog(cpu, (caddr_t)regs, cpu->cpu_softinfo.st_pending, oldipl); /* * If returned stack pointer is NULL, priority is too high * to run any of the pending softints now. * Break out and they will be run later. */ if (newsp == NULL) break; switch_sp_and_call(newsp, dispatch_softint, oldipl, 0); } } /* * Interrupt service routine, called with interrupts disabled. */ /*ARGSUSED*/ void do_interrupt(struct regs *rp, trap_trace_rec_t *ttp) { struct cpu *cpu = CPU; int newipl, oldipl = cpu->cpu_pri; uint_t vector; caddr_t newsp; #ifdef TRAPTRACE ttp->ttr_marker = TT_INTERRUPT; ttp->ttr_ipl = 0xff; ttp->ttr_pri = oldipl; ttp->ttr_spl = cpu->cpu_base_spl; ttp->ttr_vector = 0xff; #endif /* TRAPTRACE */ cpu_idle_exit(CPU_IDLE_CB_FLAG_INTR); ++*(uint16_t *)&cpu->cpu_m.mcpu_istamp; /* * If it's a softint go do it now. */ if (rp->r_trapno == T_SOFTINT) { dosoftint(rp); ASSERT(!interrupts_enabled()); return; } /* * Raise the interrupt priority. */ newipl = (*setlvl)(oldipl, (int *)&rp->r_trapno); #ifdef TRAPTRACE ttp->ttr_ipl = newipl; #endif /* TRAPTRACE */ /* * Bail if it is a spurious interrupt */ if (newipl == -1) return; cpu->cpu_pri = newipl; vector = rp->r_trapno; #ifdef TRAPTRACE ttp->ttr_vector = vector; #endif /* TRAPTRACE */ if (newipl > LOCK_LEVEL) { /* * High priority interrupts run on this cpu's interrupt stack. */ if (hilevel_intr_prolog(cpu, newipl, oldipl, rp) == 0) { newsp = cpu->cpu_intr_stack; switch_sp_and_call(newsp, dispatch_hilevel, vector, 0); } else { /* already on the interrupt stack */ dispatch_hilevel(vector, 0); } (void) hilevel_intr_epilog(cpu, newipl, oldipl, vector); } else { /* * Run this interrupt in a separate thread. */ newsp = intr_thread_prolog(cpu, (caddr_t)rp, newipl); switch_sp_and_call(newsp, dispatch_hardint, vector, oldipl); } #if !defined(__xpv) /* * Deliver any pending soft interrupts. */ if (cpu->cpu_softinfo.st_pending) dosoftint(rp); #endif /* !__xpv */ } /* * Common tasks always done by _sys_rtt, called with interrupts disabled. * Returns 1 if returning to userland, 0 if returning to system mode. */ int sys_rtt_common(struct regs *rp) { kthread_t *tp; extern void mutex_exit_critical_start(); extern long mutex_exit_critical_size; extern void mutex_owner_running_critical_start(); extern long mutex_owner_running_critical_size; loop: /* * Check if returning to user */ tp = CPU->cpu_thread; if (USERMODE(rp->r_cs)) { pcb_t *pcb; /* * Check if AST pending. */ if (tp->t_astflag) { /* * Let trap() handle the AST */ sti(); rp->r_trapno = T_AST; trap(rp, (caddr_t)0, CPU->cpu_id); cli(); goto loop; } pcb = &tp->t_lwp->lwp_pcb; /* * Check to see if we need to initialize the FPU for this * thread. This should be an uncommon occurrence, but may happen * in the case where the system creates an lwp through an * abnormal path such as the agent lwp. Make sure that we still * happen to have the FPU in a good state. */ if ((pcb->pcb_fpu.fpu_flags & FPU_EN) == 0) { kpreempt_disable(); fp_seed(); kpreempt_enable(); PCB_SET_UPDATE_FPU(pcb); } /* * We are done if segment registers do not need updating. */ if (!PCB_NEED_UPDATE(pcb)) return (1); if (PCB_NEED_UPDATE_SEGS(pcb) && update_sregs(rp, tp->t_lwp)) { /* * 1 or more of the selectors is bad. * Deliver a SIGSEGV. */ proc_t *p = ttoproc(tp); sti(); mutex_enter(&p->p_lock); tp->t_lwp->lwp_cursig = SIGSEGV; mutex_exit(&p->p_lock); psig(); tp->t_sig_check = 1; cli(); } PCB_CLEAR_UPDATE_SEGS(pcb); if (PCB_NEED_UPDATE_FPU(pcb)) { fprestore_ctxt(&pcb->pcb_fpu); } PCB_CLEAR_UPDATE_FPU(pcb); ASSERT0(PCB_NEED_UPDATE(pcb)); return (1); } #if !defined(__xpv) /* * Assert that we're not trying to return into the syscall return * trampolines. Things will go baaaaad if we try to do that. * * Note that none of these run with interrupts on, so this should * never happen (even in the sysexit case the STI doesn't take effect * until after sysexit finishes). */ extern void tr_sysc_ret_start(); extern void tr_sysc_ret_end(); ASSERT(!(rp->r_pc >= (uintptr_t)tr_sysc_ret_start && rp->r_pc <= (uintptr_t)tr_sysc_ret_end)); #endif /* * Here if we are returning to supervisor mode. * Check for a kernel preemption request. */ if (CPU->cpu_kprunrun && (rp->r_ps & PS_IE)) { /* * Do nothing if already in kpreempt */ if (!tp->t_preempt_lk) { tp->t_preempt_lk = 1; sti(); kpreempt(1); /* asynchronous kpreempt call */ cli(); tp->t_preempt_lk = 0; } } /* * If we interrupted the mutex_exit() critical region we must * reset the PC back to the beginning to prevent missed wakeups * See the comments in mutex_exit() for details. */ if ((uintptr_t)rp->r_pc - (uintptr_t)mutex_exit_critical_start < mutex_exit_critical_size) { rp->r_pc = (greg_t)mutex_exit_critical_start; } /* * If we interrupted the mutex_owner_running() critical region we * must reset the PC back to the beginning to prevent dereferencing * of a freed thread pointer. See the comments in mutex_owner_running * for details. */ if ((uintptr_t)rp->r_pc - (uintptr_t)mutex_owner_running_critical_start < mutex_owner_running_critical_size) { rp->r_pc = (greg_t)mutex_owner_running_critical_start; } return (0); } void send_dirint(int cpuid, int int_level) { (*send_dirintf)(cpuid, int_level); } #define IS_FAKE_SOFTINT(flag, newpri) \ (((flag) & PS_IE) && \ (((*get_pending_spl)() > (newpri)) || \ bsrw_insn((uint16_t)cpu->cpu_softinfo.st_pending) > (newpri))) /* * do_splx routine, takes new ipl to set * returns the old ipl. * We are careful not to set priority lower than CPU->cpu_base_pri, * even though it seems we're raising the priority, it could be set * higher at any time by an interrupt routine, so we must block interrupts * and look at CPU->cpu_base_pri */ int do_splx(int newpri) { ulong_t flag; cpu_t *cpu; int curpri, basepri; flag = intr_clear(); cpu = CPU; /* ints are disabled, now safe to cache cpu ptr */ curpri = cpu->cpu_m.mcpu_pri; basepri = cpu->cpu_base_spl; if (newpri < basepri) newpri = basepri; cpu->cpu_m.mcpu_pri = newpri; (*setspl)(newpri); /* * If we are going to reenable interrupts see if new priority level * allows pending softint delivery. */ if (IS_FAKE_SOFTINT(flag, newpri)) fakesoftint(); ASSERT(!interrupts_enabled()); intr_restore(flag); return (curpri); } /* * Common spl raise routine, takes new ipl to set * returns the old ipl, will not lower ipl. */ int splr(int newpri) { ulong_t flag; cpu_t *cpu; int curpri, basepri; flag = intr_clear(); cpu = CPU; /* ints are disabled, now safe to cache cpu ptr */ curpri = cpu->cpu_m.mcpu_pri; /* * Only do something if new priority is larger */ if (newpri > curpri) { basepri = cpu->cpu_base_spl; if (newpri < basepri) newpri = basepri; cpu->cpu_m.mcpu_pri = newpri; (*setspl)(newpri); /* * See if new priority level allows pending softint delivery */ if (IS_FAKE_SOFTINT(flag, newpri)) fakesoftint(); } intr_restore(flag); return (curpri); } int getpil(void) { return (CPU->cpu_m.mcpu_pri); } int spl_xcall(void) { return (splr(ipltospl(XCALL_PIL))); } int interrupts_enabled(void) { ulong_t flag; flag = getflags(); return ((flag & PS_IE) == PS_IE); } #ifdef DEBUG void assert_ints_enabled(void) { ASSERT(!interrupts_unleashed || interrupts_enabled()); } #endif /* DEBUG */ /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved. */ /* * Copyright (c) 2010, Intel Corporation. * All rights reserved. * Copyright 2024 Oxide Computer Company */ /* * LOCALITY GROUP (LGROUP) PLATFORM SUPPORT FOR X86/AMD64 PLATFORMS * ================================================================ * Multiprocessor AMD and Intel systems may have Non Uniform Memory Access * (NUMA). A NUMA machine consists of one or more "nodes" that each consist of * one or more CPUs and some local memory. The CPUs in each node can access * the memory in the other nodes but at a higher latency than accessing their * local memory. Typically, a system with only one node has Uniform Memory * Access (UMA), but it may be possible to have a one node system that has * some global memory outside of the node which is higher latency. * * Module Description * ------------------ * This module provides a platform interface for determining which CPUs and * which memory (and how much) are in a NUMA node and how far each node is from * each other. The interface is used by the Virtual Memory (VM) system and the * common lgroup framework. The VM system uses the plat_*() routines to fill * in its memory node (memnode) array with the physical address range spanned * by each NUMA node to know which memory belongs to which node, so it can * build and manage a physical page free list for each NUMA node and allocate * local memory from each node as needed. The common lgroup framework uses the * exported lgrp_plat_*() routines to figure out which CPUs and memory belong * to each node (leaf lgroup) and how far each node is from each other, so it * can build the latency (lgroup) topology for the machine in order to optimize * for locality. Also, an lgroup platform handle instead of lgroups are used * in the interface with this module, so this module shouldn't need to know * anything about lgroups. Instead, it just needs to know which CPUs, memory, * etc. are in each NUMA node, how far each node is from each other, and to use * a unique lgroup platform handle to refer to each node through the interface. * * Determining NUMA Configuration * ------------------------------ * By default, this module will try to determine the NUMA configuration of the * machine by reading the ACPI System Resource Affinity Table (SRAT) and System * Locality Information Table (SLIT). The SRAT contains info to tell which * CPUs and memory are local to a given proximity domain (NUMA node). The SLIT * is a matrix that gives the distance between each system locality (which is * a NUMA node and should correspond to proximity domains in the SRAT). For * more details on the SRAT and SLIT, please refer to an ACPI 3.0 or newer * specification. * * If the SRAT doesn't exist on a system with AMD Opteron processors, we * examine registers in PCI configuration space to determine how many nodes are * in the system and which CPUs and memory are in each node. * do while booting the kernel. * * NOTE: Using these PCI configuration space registers to determine this * locality info is not guaranteed to work or be compatible across all * Opteron processor families. * * If the SLIT does not exist or look right, the kernel will probe to determine * the distance between nodes as long as the NUMA CPU and memory configuration * has been determined (see lgrp_plat_probe() for details). * * Data Structures * --------------- * The main data structures used by this code are the following: * * - lgrp_plat_cpu_node[] CPU to node ID mapping table indexed by * CPU ID (only used for SRAT) * * - lgrp_plat_lat_stats.latencies[][] Table of latencies between same and * different nodes indexed by node ID * * - lgrp_plat_node_cnt Number of NUMA nodes in system for * non-DR-capable systems, * maximum possible number of NUMA nodes * in system for DR capable systems. * * - lgrp_plat_node_domain[] Node ID to proximity domain ID mapping * table indexed by node ID (only used * for SRAT) * * - lgrp_plat_memnode_info[] Table with physical address range for * each memory node indexed by memory node * ID * * The code is implemented to make the following always be true: * * lgroup platform handle == node ID == memnode ID * * Moreover, it allows for the proximity domain ID to be equal to all of the * above as long as the proximity domains IDs are numbered from 0 to . This is done by hashing each proximity domain ID into the range * from 0 to . Then proximity ID N will hash into node ID * N and proximity domain ID N will be entered into lgrp_plat_node_domain[N] * and be assigned node ID N. If the proximity domain IDs aren't numbered * from 0 to , then hashing the proximity domain IDs into * lgrp_plat_node_domain[] will still work for assigning proximity domain IDs * to node IDs. However, the proximity domain IDs may not map to the * equivalent node ID since we want to keep the node IDs numbered from 0 to * to minimize cost of searching and potentially space. * * With the introduction of support of memory DR operations on x86 platforms, * things get a little complicated. The addresses of hot-added memory may not * be continuous with other memory connected to the same lgrp node. In other * words, memory addresses may get interleaved among lgrp nodes after memory * DR operations. To work around this limitation, we have extended the * relationship between lgrp node and memory node from 1:1 map to 1:N map, * that means there may be multiple memory nodes associated with a lgrp node * after memory DR operations. * * To minimize the code changes to support memory DR operations, the * following policies have been adopted. * 1) On non-DR-capable systems, the relationship among lgroup platform handle, * node ID and memnode ID is still kept as: * lgroup platform handle == node ID == memnode ID * 2) For memory present at boot time on DR capable platforms, the relationship * is still kept as is. * lgroup platform handle == node ID == memnode ID * 3) For hot-added memory, the relationship between lgrp ID and memnode ID have * been changed from 1:1 map to 1:N map. Memnode IDs [0 - lgrp_plat_node_cnt) * are reserved for memory present at boot time, and memnode IDs * [lgrp_plat_node_cnt, max_mem_nodes) are used to dynamically allocate * memnode ID for hot-added memory. * 4) All boot code having the assumption "node ID == memnode ID" can live as * is, that's because node ID is always equal to memnode ID at boot time. * 5) The lgrp_plat_memnode_info_update(), plat_pfn_to_mem_node() and * lgrp_plat_mem_size() related logics have been enhanced to deal with * the 1:N map relationship. * 6) The latency probing related logics, which have the assumption * "node ID == memnode ID" and may be called at run time, is disabled if * memory DR operation is enabled. */ #include /* for {in,out}{b,w,l}() */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* for prom_printf() */ #include #include #include #include #include #include #include #include #include #include #include /* for SRAT, SLIT and MSCT */ /* from fakebop.c */ extern ACPI_TABLE_SRAT *srat_ptr; extern ACPI_TABLE_SLIT *slit_ptr; extern ACPI_TABLE_MSCT *msct_ptr; #define MAX_NODES 8 #define NLGRP (MAX_NODES * (MAX_NODES - 1) + 1) /* * Constants for configuring probing */ #define LGRP_PLAT_PROBE_NROUNDS 64 /* default laps for probing */ #define LGRP_PLAT_PROBE_NSAMPLES 1 /* default samples to take */ #define LGRP_PLAT_PROBE_NREADS 256 /* number of vendor ID reads */ /* * Flags for probing */ #define LGRP_PLAT_PROBE_ENABLE 0x1 /* enable probing */ #define LGRP_PLAT_PROBE_PGCPY 0x2 /* probe using page copy */ #define LGRP_PLAT_PROBE_VENDOR 0x4 /* probe vendor ID register */ /* * Hash proximity domain ID into node to domain mapping table "mod" number of * nodes to minimize span of entries used and try to have lowest numbered * proximity domain be node 0 */ #define NODE_DOMAIN_HASH(domain, node_cnt) \ ((lgrp_plat_prox_domain_min == UINT32_MAX) ? (domain) % node_cnt : \ ((domain) - lgrp_plat_prox_domain_min) % node_cnt) /* * CPU to node ID mapping structure (only used with SRAT) */ typedef struct cpu_node_map { int exists; uint_t node; uint32_t apicid; uint32_t prox_domain; } cpu_node_map_t; /* * Latency statistics */ typedef struct lgrp_plat_latency_stats { hrtime_t latencies[MAX_NODES][MAX_NODES]; hrtime_t latency_max; hrtime_t latency_min; } lgrp_plat_latency_stats_t; /* * Memory configuration for probing */ typedef struct lgrp_plat_probe_mem_config { size_t probe_memsize; /* how much memory to probe per node */ caddr_t probe_va[MAX_NODES]; /* where memory mapped for probing */ pfn_t probe_pfn[MAX_NODES]; /* physical pages to map for probing */ } lgrp_plat_probe_mem_config_t; /* * Statistics kept for probing */ typedef struct lgrp_plat_probe_stats { hrtime_t flush_cost; hrtime_t probe_cost; hrtime_t probe_cost_total; hrtime_t probe_error_code; hrtime_t probe_errors[MAX_NODES][MAX_NODES]; int probe_suspect[MAX_NODES][MAX_NODES]; hrtime_t probe_max[MAX_NODES][MAX_NODES]; hrtime_t probe_min[MAX_NODES][MAX_NODES]; } lgrp_plat_probe_stats_t; /* * Node to proximity domain ID mapping structure (only used with SRAT) */ typedef struct node_domain_map { int exists; uint32_t prox_domain; } node_domain_map_t; /* * Node ID and starting and ending page for physical memory in memory node */ typedef struct memnode_phys_addr_map { pfn_t start; pfn_t end; int exists; uint32_t prox_domain; uint32_t device_id; uint_t lgrphand; } memnode_phys_addr_map_t; /* * Number of CPUs for which we got APIC IDs */ static int lgrp_plat_apic_ncpus = 0; /* * CPU to node ID mapping table (only used for SRAT) and its max number of * entries */ static cpu_node_map_t *lgrp_plat_cpu_node = NULL; static uint_t lgrp_plat_cpu_node_nentries = 0; /* * Latency statistics */ lgrp_plat_latency_stats_t lgrp_plat_lat_stats; /* * Whether memory is interleaved across nodes causing MPO to be disabled */ static int lgrp_plat_mem_intrlv = 0; /* * Node ID to proximity domain ID mapping table (only used for SRAT) */ static node_domain_map_t lgrp_plat_node_domain[MAX_NODES]; /* * Physical address range for memory in each node */ static memnode_phys_addr_map_t lgrp_plat_memnode_info[MAX_MEM_NODES]; /* * Statistics gotten from probing */ static lgrp_plat_probe_stats_t lgrp_plat_probe_stats; /* * Memory configuration for probing */ static lgrp_plat_probe_mem_config_t lgrp_plat_probe_mem_config; /* * Lowest proximity domain ID seen in ACPI SRAT */ static uint32_t lgrp_plat_prox_domain_min = UINT32_MAX; /* * Error code from processing ACPI SRAT */ static int lgrp_plat_srat_error = 0; /* * Error code from processing ACPI SLIT */ static int lgrp_plat_slit_error = 0; /* * Whether lgrp topology has been flattened to 2 levels. */ static int lgrp_plat_topo_flatten = 0; /* * Maximum memory node ID in use. */ static uint_t lgrp_plat_max_mem_node; /* * Allocate lgroup array statically */ static lgrp_t lgrp_space[NLGRP]; static int nlgrps_alloc; /* * Enable finding and using minimum proximity domain ID when hashing */ int lgrp_plat_domain_min_enable = 1; /* * Maximum possible number of nodes in system */ uint_t lgrp_plat_node_cnt = 1; /* * Enable sorting nodes in ascending order by starting physical address */ int lgrp_plat_node_sort_enable = 1; /* * Configuration Parameters for Probing * - lgrp_plat_probe_flags Flags to specify enabling probing, probe * operation, etc. * - lgrp_plat_probe_nrounds How many rounds of probing to do * - lgrp_plat_probe_nsamples Number of samples to take when probing each * node * - lgrp_plat_probe_nreads Number of times to read vendor ID from * Northbridge for each probe */ uint_t lgrp_plat_probe_flags = 0; int lgrp_plat_probe_nrounds = LGRP_PLAT_PROBE_NROUNDS; int lgrp_plat_probe_nsamples = LGRP_PLAT_PROBE_NSAMPLES; int lgrp_plat_probe_nreads = LGRP_PLAT_PROBE_NREADS; /* * Enable use of ACPI System Resource Affinity Table (SRAT), System * Locality Information Table (SLIT) and Maximum System Capability Table (MSCT) */ int lgrp_plat_srat_enable = 1; int lgrp_plat_slit_enable = 1; int lgrp_plat_msct_enable = 1; /* * mnode_xwa: set to non-zero value to initiate workaround if large pages are * found to be crossing memory node boundaries. The workaround will eliminate * a base size page at the end of each memory node boundary to ensure that * a large page with constituent pages that span more than 1 memory node * can never be formed. * */ int mnode_xwa = 1; /* * Static array to hold lgroup statistics */ struct lgrp_stats lgrp_stats[NLGRP]; /* * Forward declarations of platform interface routines */ void plat_build_mem_nodes(struct memlist *list); int plat_mnode_xcheck(pfn_t pfncnt); lgrp_handle_t plat_mem_node_to_lgrphand(int mnode); int plat_pfn_to_mem_node(pfn_t pfn); /* * Forward declarations of lgroup platform interface routines */ lgrp_t *lgrp_plat_alloc(lgrp_id_t lgrpid); void lgrp_plat_config(lgrp_config_flag_t flag, uintptr_t arg); lgrp_handle_t lgrp_plat_cpu_to_hand(processorid_t id); void lgrp_plat_init(lgrp_init_stages_t stage); int lgrp_plat_latency(lgrp_handle_t from, lgrp_handle_t to); int lgrp_plat_max_lgrps(void); pgcnt_t lgrp_plat_mem_size(lgrp_handle_t plathand, lgrp_mem_query_t query); lgrp_handle_t lgrp_plat_pfn_to_hand(pfn_t pfn); void lgrp_plat_probe(void); lgrp_handle_t lgrp_plat_root_hand(void); /* * Forward declarations of local routines */ static int is_opteron(void); static int lgrp_plat_cpu_node_update(node_domain_map_t *node_domain, int node_cnt, cpu_node_map_t *cpu_node, int nentries, uint32_t apicid, uint32_t domain); static int lgrp_plat_cpu_to_node(cpu_t *cp, cpu_node_map_t *cpu_node, int cpu_node_nentries); static int lgrp_plat_domain_to_node(node_domain_map_t *node_domain, int node_cnt, uint32_t domain); static void lgrp_plat_get_numa_config(void); static void lgrp_plat_latency_adjust(memnode_phys_addr_map_t *memnode_info, lgrp_plat_latency_stats_t *lat_stats, lgrp_plat_probe_stats_t *probe_stats); static int lgrp_plat_latency_verify(memnode_phys_addr_map_t *memnode_info, lgrp_plat_latency_stats_t *lat_stats); static void lgrp_plat_main_init(void); static pgcnt_t lgrp_plat_mem_size_default(lgrp_handle_t, lgrp_mem_query_t); static int lgrp_plat_node_domain_update(node_domain_map_t *node_domain, int node_cnt, uint32_t domain); static int lgrp_plat_memnode_info_update(node_domain_map_t *node_domain, int node_cnt, memnode_phys_addr_map_t *memnode_info, int memnode_cnt, uint64_t start, uint64_t end, uint32_t domain, uint32_t device_id); static void lgrp_plat_node_sort(node_domain_map_t *node_domain, int node_cnt, cpu_node_map_t *cpu_node, int cpu_count, memnode_phys_addr_map_t *memnode_info); static hrtime_t lgrp_plat_probe_time(int to, cpu_node_map_t *cpu_node, int cpu_node_nentries, lgrp_plat_probe_mem_config_t *probe_mem_config, lgrp_plat_latency_stats_t *lat_stats, lgrp_plat_probe_stats_t *probe_stats); static int lgrp_plat_process_cpu_apicids(cpu_node_map_t *cpu_node); static int lgrp_plat_process_slit(ACPI_TABLE_SLIT *tp, node_domain_map_t *node_domain, uint_t node_cnt, memnode_phys_addr_map_t *memnode_info, lgrp_plat_latency_stats_t *lat_stats); static int lgrp_plat_process_sli(uint32_t domain, uchar_t *sli_info, uint32_t sli_cnt, node_domain_map_t *node_domain, uint_t node_cnt, lgrp_plat_latency_stats_t *lat_stats); static int lgrp_plat_process_srat(ACPI_TABLE_SRAT *tp, ACPI_TABLE_MSCT *mp, uint32_t *prox_domain_min, node_domain_map_t *node_domain, cpu_node_map_t *cpu_node, int cpu_count, memnode_phys_addr_map_t *memnode_info); static void lgrp_plat_release_bootstrap(void); static int lgrp_plat_srat_domains(ACPI_TABLE_SRAT *tp, uint32_t *prox_domain_min); static int lgrp_plat_msct_domains(ACPI_TABLE_MSCT *tp, uint32_t *prox_domain_min); static void lgrp_plat_2level_setup(lgrp_plat_latency_stats_t *lat_stats); static void opt_get_numa_config(uint_t *node_cnt, int *mem_intrlv, memnode_phys_addr_map_t *memnode_info); static hrtime_t opt_probe_vendor(int dest_node, int nreads); /* * PLATFORM INTERFACE ROUTINES */ /* * Configure memory nodes for machines with more than one node (ie NUMA) */ void plat_build_mem_nodes(struct memlist *list) { pfn_t cur_start; /* start addr of subrange */ pfn_t cur_end; /* end addr of subrange */ pfn_t start; /* start addr of whole range */ pfn_t end; /* end addr of whole range */ pgcnt_t endcnt; /* pages to sacrifice */ /* * Boot install lists are arranged , ... */ while (list) { int node; start = list->ml_address >> PAGESHIFT; end = (list->ml_address + list->ml_size - 1) >> PAGESHIFT; if (start > physmax) { list = list->ml_next; continue; } if (end > physmax) end = physmax; /* * When there is only one memnode, just add memory to memnode */ if (max_mem_nodes == 1) { mem_node_add_slice(start, end); list = list->ml_next; continue; } /* * mem_node_add_slice() expects to get a memory range that * is within one memnode, so need to split any memory range * that spans multiple memnodes into subranges that are each * contained within one memnode when feeding them to * mem_node_add_slice() */ cur_start = start; do { node = plat_pfn_to_mem_node(cur_start); /* * Panic if DRAM address map registers or SRAT say * memory in node doesn't exist or address from * boot installed memory list entry isn't in this node. * This shouldn't happen and rest of code can't deal * with this if it does. */ if (node < 0 || node >= lgrp_plat_max_mem_node || !lgrp_plat_memnode_info[node].exists || cur_start < lgrp_plat_memnode_info[node].start || cur_start > lgrp_plat_memnode_info[node].end) { cmn_err(CE_PANIC, "Don't know which memnode " "to add installed memory address 0x%lx\n", cur_start); } /* * End of current subrange should not span memnodes */ cur_end = end; endcnt = 0; if (lgrp_plat_memnode_info[node].exists && cur_end > lgrp_plat_memnode_info[node].end) { cur_end = lgrp_plat_memnode_info[node].end; if (mnode_xwa > 1) { /* * sacrifice the last page in each * node to eliminate large pages * that span more than 1 memory node. */ endcnt = 1; physinstalled--; } } mem_node_add_slice(cur_start, cur_end - endcnt); /* * Next subrange starts after end of current one */ cur_start = cur_end + 1; } while (cur_end < end); list = list->ml_next; } mem_node_physalign = 0; mem_node_pfn_shift = 0; } /* * plat_mnode_xcheck: checks the node memory ranges to see if there is a pfncnt * range of pages aligned on pfncnt that crosses an node boundary. Returns 1 if * a crossing is found and returns 0 otherwise. */ int plat_mnode_xcheck(pfn_t pfncnt) { int node, prevnode = -1, basenode; pfn_t ea, sa; for (node = 0; node < lgrp_plat_max_mem_node; node++) { if (lgrp_plat_memnode_info[node].exists == 0) continue; if (prevnode == -1) { prevnode = node; basenode = node; continue; } /* assume x86 node pfn ranges are in increasing order */ ASSERT(lgrp_plat_memnode_info[node].start > lgrp_plat_memnode_info[prevnode].end); /* * continue if the starting address of node is not contiguous * with the previous node. */ if (lgrp_plat_memnode_info[node].start != (lgrp_plat_memnode_info[prevnode].end + 1)) { basenode = node; prevnode = node; continue; } /* check if the starting address of node is pfncnt aligned */ if ((lgrp_plat_memnode_info[node].start & (pfncnt - 1)) != 0) { /* * at this point, node starts at an unaligned boundary * and is contiguous with the previous node(s) to * basenode. Check if there is an aligned contiguous * range of length pfncnt that crosses this boundary. */ sa = P2ALIGN(lgrp_plat_memnode_info[prevnode].end, pfncnt); ea = P2ROUNDUP((lgrp_plat_memnode_info[node].start), pfncnt); ASSERT((ea - sa) == pfncnt); if (sa >= lgrp_plat_memnode_info[basenode].start && ea <= (lgrp_plat_memnode_info[node].end + 1)) { /* * large page found to cross mnode boundary. * Return Failure if workaround not enabled. */ if (mnode_xwa == 0) return (1); mnode_xwa++; } } prevnode = node; } return (0); } lgrp_handle_t plat_mem_node_to_lgrphand(int mnode) { if (max_mem_nodes == 1) return (LGRP_DEFAULT_HANDLE); ASSERT(0 <= mnode && mnode < lgrp_plat_max_mem_node); return ((lgrp_handle_t)(lgrp_plat_memnode_info[mnode].lgrphand)); } int plat_pfn_to_mem_node(pfn_t pfn) { int node; if (max_mem_nodes == 1) return (0); for (node = 0; node < lgrp_plat_max_mem_node; node++) { /* * Skip nodes with no memory */ if (!lgrp_plat_memnode_info[node].exists) continue; membar_consumer(); if (pfn >= lgrp_plat_memnode_info[node].start && pfn <= lgrp_plat_memnode_info[node].end) return (node); } /* * Didn't find memnode where this PFN lives which should never happen */ ASSERT(node < lgrp_plat_max_mem_node); return (-1); } /* * LGROUP PLATFORM INTERFACE ROUTINES */ /* * Allocate additional space for an lgroup. */ lgrp_t * lgrp_plat_alloc(lgrp_id_t lgrpid) { lgrp_t *lgrp; lgrp = &lgrp_space[nlgrps_alloc++]; if (lgrpid >= NLGRP || nlgrps_alloc > NLGRP) return (NULL); return (lgrp); } /* * Platform handling for (re)configuration changes * * Mechanism to protect lgrp_plat_cpu_node[] at CPU hotplug: * 1) Use cpu_lock to synchronize between lgrp_plat_config() and * lgrp_plat_cpu_to_hand(). * 2) Disable latency probing logic by making sure that the flag * LGRP_PLAT_PROBE_ENABLE is cleared. * * Mechanism to protect lgrp_plat_memnode_info[] at memory hotplug: * 1) Only inserts into lgrp_plat_memnode_info at memory hotplug, no removal. * 2) Only expansion to existing entries, no shrinking. * 3) On writing side, DR framework ensures that lgrp_plat_config() is called * in single-threaded context. And membar_producer() is used to ensure that * all changes are visible to other CPUs before setting the "exists" flag. * 4) On reading side, membar_consumer() after checking the "exists" flag * ensures that right values are retrieved. * * Mechanism to protect lgrp_plat_node_domain[] at hotplug: * 1) Only insertion into lgrp_plat_node_domain at hotplug, no removal. * 2) On writing side, it's single-threaded and membar_producer() is used to * ensure all changes are visible to other CPUs before setting the "exists" * flag. * 3) On reading side, membar_consumer() after checking the "exists" flag * ensures that right values are retrieved. */ void lgrp_plat_config(lgrp_config_flag_t flag, uintptr_t arg) { #ifdef __xpv _NOTE(ARGUNUSED(flag, arg)); #else int rc, node; cpu_t *cp; void *hdl = NULL; uchar_t *sliptr = NULL; uint32_t domain, apicid, slicnt = 0; update_membounds_t *mp; extern int acpidev_dr_get_cpu_numa_info(cpu_t *, void **, uint32_t *, uint32_t *, uint32_t *, uchar_t **); extern void acpidev_dr_free_cpu_numa_info(void *); /* * This interface is used to support CPU/memory DR operations. * Don't bother here if it's still during boot or only one lgrp node * is supported. */ if (!lgrp_topo_initialized || lgrp_plat_node_cnt == 1) return; switch (flag) { case LGRP_CONFIG_CPU_ADD: cp = (cpu_t *)arg; ASSERT(cp != NULL); ASSERT(MUTEX_HELD(&cpu_lock)); /* Check whether CPU already exists. */ ASSERT(!lgrp_plat_cpu_node[cp->cpu_id].exists); if (lgrp_plat_cpu_node[cp->cpu_id].exists) { cmn_err(CE_WARN, "!lgrp: CPU(%d) already exists in cpu_node map.", cp->cpu_id); break; } /* Query CPU lgrp information. */ rc = acpidev_dr_get_cpu_numa_info(cp, &hdl, &apicid, &domain, &slicnt, &sliptr); ASSERT(rc == 0); if (rc != 0) { cmn_err(CE_WARN, "!lgrp: failed to query lgrp info for CPU(%d).", cp->cpu_id); break; } /* Update node to proximity domain mapping */ node = lgrp_plat_domain_to_node(lgrp_plat_node_domain, lgrp_plat_node_cnt, domain); if (node == -1) { node = lgrp_plat_node_domain_update( lgrp_plat_node_domain, lgrp_plat_node_cnt, domain); ASSERT(node != -1); if (node == -1) { acpidev_dr_free_cpu_numa_info(hdl); cmn_err(CE_WARN, "!lgrp: failed to update " "node_domain map for domain(%u).", domain); break; } } /* Update latency information among lgrps. */ if (slicnt != 0 && sliptr != NULL) { if (lgrp_plat_process_sli(domain, sliptr, slicnt, lgrp_plat_node_domain, lgrp_plat_node_cnt, &lgrp_plat_lat_stats) != 0) { cmn_err(CE_WARN, "!lgrp: failed to update " "latency information for domain (%u).", domain); } } /* Update CPU to node mapping. */ lgrp_plat_cpu_node[cp->cpu_id].prox_domain = domain; lgrp_plat_cpu_node[cp->cpu_id].node = node; lgrp_plat_cpu_node[cp->cpu_id].apicid = apicid; lgrp_plat_cpu_node[cp->cpu_id].exists = 1; lgrp_plat_apic_ncpus++; acpidev_dr_free_cpu_numa_info(hdl); break; case LGRP_CONFIG_CPU_DEL: cp = (cpu_t *)arg; ASSERT(cp != NULL); ASSERT(MUTEX_HELD(&cpu_lock)); /* Check whether CPU exists. */ ASSERT(lgrp_plat_cpu_node[cp->cpu_id].exists); if (!lgrp_plat_cpu_node[cp->cpu_id].exists) { cmn_err(CE_WARN, "!lgrp: CPU(%d) doesn't exist in cpu_node map.", cp->cpu_id); break; } /* Query CPU lgrp information. */ rc = acpidev_dr_get_cpu_numa_info(cp, &hdl, &apicid, &domain, NULL, NULL); ASSERT(rc == 0); if (rc != 0) { cmn_err(CE_WARN, "!lgrp: failed to query lgrp info for CPU(%d).", cp->cpu_id); break; } /* Update map. */ ASSERT(lgrp_plat_cpu_node[cp->cpu_id].apicid == apicid); ASSERT(lgrp_plat_cpu_node[cp->cpu_id].prox_domain == domain); lgrp_plat_cpu_node[cp->cpu_id].exists = 0; lgrp_plat_cpu_node[cp->cpu_id].apicid = UINT32_MAX; lgrp_plat_cpu_node[cp->cpu_id].prox_domain = UINT32_MAX; lgrp_plat_cpu_node[cp->cpu_id].node = UINT_MAX; lgrp_plat_apic_ncpus--; acpidev_dr_free_cpu_numa_info(hdl); break; case LGRP_CONFIG_MEM_ADD: mp = (update_membounds_t *)arg; ASSERT(mp != NULL); /* Update latency information among lgrps. */ if (mp->u_sli_cnt != 0 && mp->u_sli_ptr != NULL) { if (lgrp_plat_process_sli(mp->u_domain, mp->u_sli_ptr, mp->u_sli_cnt, lgrp_plat_node_domain, lgrp_plat_node_cnt, &lgrp_plat_lat_stats) != 0) { cmn_err(CE_WARN, "!lgrp: failed to update " "latency information for domain (%u).", domain); } } if (lgrp_plat_memnode_info_update(lgrp_plat_node_domain, lgrp_plat_node_cnt, lgrp_plat_memnode_info, max_mem_nodes, mp->u_base, mp->u_base + mp->u_length, mp->u_domain, mp->u_device_id) < 0) { cmn_err(CE_WARN, "!lgrp: failed to update latency information for " "memory (0x%" PRIx64 " - 0x%" PRIx64 ").", mp->u_base, mp->u_base + mp->u_length); } break; default: break; } #endif /* __xpv */ } /* * Return the platform handle for the lgroup containing the given CPU */ lgrp_handle_t lgrp_plat_cpu_to_hand(processorid_t id) { lgrp_handle_t hand; ASSERT(!lgrp_initialized || MUTEX_HELD(&cpu_lock)); if (lgrp_plat_node_cnt == 1) return (LGRP_DEFAULT_HANDLE); hand = (lgrp_handle_t)lgrp_plat_cpu_to_node(cpu[id], lgrp_plat_cpu_node, lgrp_plat_cpu_node_nentries); if (hand == (lgrp_handle_t)-1) return (LGRP_NULL_HANDLE); return (hand); } /* * Platform-specific initialization of lgroups */ void lgrp_plat_init(lgrp_init_stages_t stage) { #if defined(__xpv) #else /* __xpv */ u_longlong_t value; #endif /* __xpv */ switch (stage) { case LGRP_INIT_STAGE1: #if defined(__xpv) /* * XXPV For now, the hypervisor treats all memory equally. */ lgrp_plat_node_cnt = max_mem_nodes = 1; #else /* __xpv */ /* * Get boot property for lgroup topology height limit */ if (bootprop_getval(BP_LGRP_TOPO_LEVELS, &value) == 0) (void) lgrp_topo_ht_limit_set((int)value); /* * Get boot property for enabling/disabling SRAT */ if (bootprop_getval(BP_LGRP_SRAT_ENABLE, &value) == 0) lgrp_plat_srat_enable = (int)value; /* * Get boot property for enabling/disabling SLIT */ if (bootprop_getval(BP_LGRP_SLIT_ENABLE, &value) == 0) lgrp_plat_slit_enable = (int)value; /* * Get boot property for enabling/disabling MSCT */ if (bootprop_getval(BP_LGRP_MSCT_ENABLE, &value) == 0) lgrp_plat_msct_enable = (int)value; /* * Initialize as a UMA machine */ if (lgrp_topo_ht_limit() == 1) { lgrp_plat_node_cnt = max_mem_nodes = 1; lgrp_plat_max_mem_node = 1; return; } lgrp_plat_get_numa_config(); /* * Each lgrp node needs MAX_MEM_NODES_PER_LGROUP memnodes * to support memory DR operations if memory DR is enabled. */ lgrp_plat_max_mem_node = lgrp_plat_node_cnt; if (plat_dr_support_memory() && lgrp_plat_node_cnt != 1) { max_mem_nodes = MAX_MEM_NODES_PER_LGROUP * lgrp_plat_node_cnt; ASSERT(max_mem_nodes <= MAX_MEM_NODES); } #endif /* __xpv */ break; case LGRP_INIT_STAGE3: lgrp_plat_probe(); lgrp_plat_release_bootstrap(); break; case LGRP_INIT_STAGE4: lgrp_plat_main_init(); break; default: break; } } /* * Return latency between "from" and "to" lgroups * * This latency number can only be used for relative comparison * between lgroups on the running system, cannot be used across platforms, * and may not reflect the actual latency. It is platform and implementation * specific, so platform gets to decide its value. It would be nice if the * number was at least proportional to make comparisons more meaningful though. */ int lgrp_plat_latency(lgrp_handle_t from, lgrp_handle_t to) { lgrp_handle_t src, dest; int node; if (max_mem_nodes == 1) return (0); /* * Return max latency for root lgroup */ if (from == LGRP_DEFAULT_HANDLE || to == LGRP_DEFAULT_HANDLE) return (lgrp_plat_lat_stats.latency_max); src = from; dest = to; /* * Return 0 for nodes (lgroup platform handles) out of range */ if (src >= MAX_NODES || dest >= MAX_NODES) return (0); /* * Probe from current CPU if its lgroup latencies haven't been set yet * and we are trying to get latency from current CPU to some node. * Avoid probing if CPU/memory DR is enabled. */ if (lgrp_plat_lat_stats.latencies[src][src] == 0) { /* * Latency information should be updated by lgrp_plat_config() * for DR operations. Something is wrong if reaches here. * For safety, flatten lgrp topology to two levels. */ if (plat_dr_support_cpu() || plat_dr_support_memory()) { ASSERT(lgrp_plat_lat_stats.latencies[src][src]); cmn_err(CE_WARN, "lgrp: failed to get latency information, " "fall back to two-level topology."); lgrp_plat_2level_setup(&lgrp_plat_lat_stats); } else { node = lgrp_plat_cpu_to_node(CPU, lgrp_plat_cpu_node, lgrp_plat_cpu_node_nentries); ASSERT3U(node, <, lgrp_plat_node_cnt); if (node == (lgrp_handle_t)-1) return (0); if (node == src) lgrp_plat_probe(); } } return (lgrp_plat_lat_stats.latencies[src][dest]); } /* * Return the maximum number of lgrps supported by the platform. * Before lgrp topology is known it returns an estimate based on the number of * nodes. Once topology is known it returns: * 1) the actual maximim number of lgrps created if CPU/memory DR operations * are not suppported. * 2) the maximum possible number of lgrps if CPU/memory DR operations are * supported. */ int lgrp_plat_max_lgrps(void) { if (!lgrp_topo_initialized || plat_dr_support_cpu() || plat_dr_support_memory()) { return (lgrp_plat_node_cnt * (lgrp_plat_node_cnt - 1) + 1); } else { return (lgrp_alloc_max + 1); } } /* * Count number of memory pages (_t) based on mnode id (_n) and query type (_t). */ #define _LGRP_PLAT_MEM_SIZE(_n, _q, _t) \ if (mem_node_config[_n].exists) { \ switch (_q) { \ case LGRP_MEM_SIZE_FREE: \ _t += MNODE_PGCNT(_n); \ break; \ case LGRP_MEM_SIZE_AVAIL: \ _t += mem_node_memlist_pages(_n, phys_avail); \ break; \ case LGRP_MEM_SIZE_INSTALL: \ _t += mem_node_memlist_pages(_n, phys_install); \ break; \ default: \ break; \ } \ } /* * Return the number of free pages in an lgroup. * * For query of LGRP_MEM_SIZE_FREE, return the number of base pagesize * pages on freelists. For query of LGRP_MEM_SIZE_AVAIL, return the * number of allocatable base pagesize pages corresponding to the * lgroup (e.g. do not include page_t's, BOP_ALLOC()'ed memory, ..) * For query of LGRP_MEM_SIZE_INSTALL, return the amount of physical * memory installed, regardless of whether or not it's usable. */ pgcnt_t lgrp_plat_mem_size(lgrp_handle_t plathand, lgrp_mem_query_t query) { int mnode; pgcnt_t npgs = (pgcnt_t)0; extern struct memlist *phys_avail; extern struct memlist *phys_install; if (plathand == LGRP_DEFAULT_HANDLE) return (lgrp_plat_mem_size_default(plathand, query)); if (plathand != LGRP_NULL_HANDLE) { /* Count memory node present at boot. */ mnode = (int)plathand; ASSERT(mnode < lgrp_plat_node_cnt); _LGRP_PLAT_MEM_SIZE(mnode, query, npgs); /* Count possible hot-added memory nodes. */ for (mnode = lgrp_plat_node_cnt; mnode < lgrp_plat_max_mem_node; mnode++) { if (lgrp_plat_memnode_info[mnode].lgrphand == plathand) _LGRP_PLAT_MEM_SIZE(mnode, query, npgs); } } return (npgs); } /* * Return the platform handle of the lgroup that contains the physical memory * corresponding to the given page frame number */ lgrp_handle_t lgrp_plat_pfn_to_hand(pfn_t pfn) { int mnode; if (max_mem_nodes == 1) return (LGRP_DEFAULT_HANDLE); if (pfn > physmax) return (LGRP_NULL_HANDLE); mnode = plat_pfn_to_mem_node(pfn); if (mnode < 0) return (LGRP_NULL_HANDLE); return (MEM_NODE_2_LGRPHAND(mnode)); } /* * Probe memory in each node from current CPU to determine latency topology * * The probing code will probe the vendor ID register on the Northbridge of * Opteron processors and probe memory for other processors by default. * * Since probing is inherently error prone, the code takes laps across all the * nodes probing from each node to each of the other nodes some number of * times. Furthermore, each node is probed some number of times before moving * onto the next one during each lap. The minimum latency gotten between nodes * is kept as the latency between the nodes. * * After all that, the probe times are adjusted by normalizing values that are * close to each other and local latencies are made the same. Lastly, the * latencies are verified to make sure that certain conditions are met (eg. * local < remote, latency(a, b) == latency(b, a), etc.). * * If any of the conditions aren't met, the code will export a NUMA * configuration with the local CPUs and memory given by the SRAT or PCI config * space registers and one remote memory latency since it can't tell exactly * how far each node is from each other. */ void lgrp_plat_probe(void) { int from; int i; lgrp_plat_latency_stats_t *lat_stats; boolean_t probed; hrtime_t probe_time; int to; if (!(lgrp_plat_probe_flags & LGRP_PLAT_PROBE_ENABLE) || max_mem_nodes == 1 || lgrp_topo_ht_limit() <= 2) return; /* SRAT and SLIT should be enabled if DR operations are enabled. */ if (plat_dr_support_cpu() || plat_dr_support_memory()) return; /* * Determine ID of node containing current CPU */ from = lgrp_plat_cpu_to_node(CPU, lgrp_plat_cpu_node, lgrp_plat_cpu_node_nentries); ASSERT3U(from, <, lgrp_plat_node_cnt); if (from == (lgrp_handle_t)-1) return; if (srat_ptr && lgrp_plat_srat_enable && !lgrp_plat_srat_error) ASSERT(lgrp_plat_node_domain[from].exists); /* * Don't need to probe if got times already */ lat_stats = &lgrp_plat_lat_stats; if (lat_stats->latencies[from][from] != 0) return; /* * Read vendor ID in Northbridge or read and write page(s) * in each node from current CPU and remember how long it takes, * so we can build latency topology of machine later. * This should approximate the memory latency between each node. */ probed = B_FALSE; for (i = 0; i < lgrp_plat_probe_nrounds; i++) { for (to = 0; to < lgrp_plat_node_cnt; to++) { /* * Get probe time and skip over any nodes that can't be * probed yet or don't have memory */ probe_time = lgrp_plat_probe_time(to, lgrp_plat_cpu_node, lgrp_plat_cpu_node_nentries, &lgrp_plat_probe_mem_config, &lgrp_plat_lat_stats, &lgrp_plat_probe_stats); if (probe_time == 0) continue; probed = B_TRUE; /* * Keep lowest probe time as latency between nodes */ if (lat_stats->latencies[from][to] == 0 || probe_time < lat_stats->latencies[from][to]) lat_stats->latencies[from][to] = probe_time; /* * Update overall minimum and maximum probe times * across all nodes */ if (probe_time < lat_stats->latency_min || lat_stats->latency_min == -1) lat_stats->latency_min = probe_time; if (probe_time > lat_stats->latency_max) lat_stats->latency_max = probe_time; } } /* * Bail out if weren't able to probe any nodes from current CPU */ if (probed == B_FALSE) return; /* * - Fix up latencies such that local latencies are same, * latency(i, j) == latency(j, i), etc. (if possible) * * - Verify that latencies look ok * * - Fallback to just optimizing for local and remote if * latencies didn't look right */ lgrp_plat_latency_adjust(lgrp_plat_memnode_info, &lgrp_plat_lat_stats, &lgrp_plat_probe_stats); lgrp_plat_probe_stats.probe_error_code = lgrp_plat_latency_verify(lgrp_plat_memnode_info, &lgrp_plat_lat_stats); if (lgrp_plat_probe_stats.probe_error_code) lgrp_plat_2level_setup(&lgrp_plat_lat_stats); } /* * Return platform handle for root lgroup */ lgrp_handle_t lgrp_plat_root_hand(void) { return (LGRP_DEFAULT_HANDLE); } /* * INTERNAL ROUTINES */ /* * Update CPU to node mapping for given CPU and proximity domain. * Return values: * - zero for success * - positive numbers for warnings * - negative numbers for errors */ static int lgrp_plat_cpu_node_update(node_domain_map_t *node_domain, int node_cnt, cpu_node_map_t *cpu_node, int nentries, uint32_t apicid, uint32_t domain) { uint_t i; int node; /* * Get node number for proximity domain */ node = lgrp_plat_domain_to_node(node_domain, node_cnt, domain); if (node == -1) { node = lgrp_plat_node_domain_update(node_domain, node_cnt, domain); if (node == -1) return (-1); } /* * Search for entry with given APIC ID and fill in its node and * proximity domain IDs (if they haven't been set already) */ for (i = 0; i < nentries; i++) { /* * Skip nonexistent entries and ones without matching APIC ID */ if (!cpu_node[i].exists || cpu_node[i].apicid != apicid) continue; /* * Just return if entry completely and correctly filled in * already */ if (cpu_node[i].prox_domain == domain && cpu_node[i].node == node) return (1); /* * It's invalid to have more than one entry with the same * local APIC ID in SRAT table. */ if (cpu_node[i].node != UINT_MAX) return (-2); /* * Fill in node and proximity domain IDs */ cpu_node[i].prox_domain = domain; cpu_node[i].node = node; return (0); } /* * It's possible that an apicid doesn't exist in the cpu_node map due * to user limits number of CPUs powered on at boot by specifying the * boot_ncpus kernel option. */ return (2); } /* * Get node ID for given CPU */ static int lgrp_plat_cpu_to_node(cpu_t *cp, cpu_node_map_t *cpu_node, int cpu_node_nentries) { processorid_t cpuid; if (cp == NULL) return (-1); cpuid = cp->cpu_id; if (cpuid < 0 || cpuid >= max_ncpus) return (-1); /* * SRAT doesn't exist, isn't enabled, or there was an error processing * it, so return node ID for Opteron and -1 otherwise. */ if (srat_ptr == NULL || !lgrp_plat_srat_enable || lgrp_plat_srat_error) { if (is_opteron()) return (pg_plat_hw_instance_id(cp, PGHW_PROCNODE)); return (-1); } /* * Return -1 when CPU to node ID mapping entry doesn't exist for given * CPU */ if (cpuid >= cpu_node_nentries || !cpu_node[cpuid].exists) return (-1); return (cpu_node[cpuid].node); } /* * Return node number for given proximity domain/system locality */ static int lgrp_plat_domain_to_node(node_domain_map_t *node_domain, int node_cnt, uint32_t domain) { uint_t node; uint_t start; /* * Hash proximity domain ID into node to domain mapping table (array), * search for entry with matching proximity domain ID, and return index * of matching entry as node ID. */ node = start = NODE_DOMAIN_HASH(domain, node_cnt); do { if (node_domain[node].exists) { membar_consumer(); if (node_domain[node].prox_domain == domain) return (node); } node = (node + 1) % node_cnt; } while (node != start); return (-1); } /* * Get NUMA configuration of machine */ static void lgrp_plat_get_numa_config(void) { uint_t probe_op; /* * Read boot property with CPU to APIC ID mapping table/array to * determine number of CPUs */ lgrp_plat_apic_ncpus = lgrp_plat_process_cpu_apicids(NULL); /* * Determine which CPUs and memory are local to each other and number * of NUMA nodes by reading ACPI System Resource Affinity Table (SRAT) */ if (lgrp_plat_apic_ncpus > 0) { int retval; /* Reserve enough resources if CPU DR is enabled. */ if (plat_dr_support_cpu() && max_ncpus > lgrp_plat_apic_ncpus) lgrp_plat_cpu_node_nentries = max_ncpus; else lgrp_plat_cpu_node_nentries = lgrp_plat_apic_ncpus; /* * Temporarily allocate boot memory to use for CPU to node * mapping since kernel memory allocator isn't alive yet */ lgrp_plat_cpu_node = (cpu_node_map_t *)BOP_ALLOC(bootops, NULL, lgrp_plat_cpu_node_nentries * sizeof (cpu_node_map_t), sizeof (int)); ASSERT(lgrp_plat_cpu_node != NULL); if (lgrp_plat_cpu_node) { bzero(lgrp_plat_cpu_node, lgrp_plat_cpu_node_nentries * sizeof (cpu_node_map_t)); } else { lgrp_plat_cpu_node_nentries = 0; } /* * Fill in CPU to node ID mapping table with APIC ID for each * CPU */ (void) lgrp_plat_process_cpu_apicids(lgrp_plat_cpu_node); retval = lgrp_plat_process_srat(srat_ptr, msct_ptr, &lgrp_plat_prox_domain_min, lgrp_plat_node_domain, lgrp_plat_cpu_node, lgrp_plat_apic_ncpus, lgrp_plat_memnode_info); if (retval <= 0) { lgrp_plat_srat_error = retval; lgrp_plat_node_cnt = 1; } else { lgrp_plat_srat_error = 0; lgrp_plat_node_cnt = retval; } } /* * Try to use PCI config space registers on Opteron if there's an error * processing CPU to APIC ID mapping or SRAT */ if ((lgrp_plat_apic_ncpus <= 0 || lgrp_plat_srat_error != 0) && is_opteron()) opt_get_numa_config(&lgrp_plat_node_cnt, &lgrp_plat_mem_intrlv, lgrp_plat_memnode_info); /* * Don't bother to setup system for multiple lgroups and only use one * memory node when memory is interleaved between any nodes or there is * only one NUMA node */ if (lgrp_plat_mem_intrlv || lgrp_plat_node_cnt == 1) { lgrp_plat_node_cnt = max_mem_nodes = 1; (void) lgrp_topo_ht_limit_set(1); return; } /* * Leaf lgroups on x86/x64 architectures contain one physical * processor chip. Tune lgrp_expand_proc_thresh and * lgrp_expand_proc_diff so that lgrp_choose() will spread * things out aggressively. */ lgrp_expand_proc_thresh = LGRP_LOADAVG_THREAD_MAX / 2; lgrp_expand_proc_diff = 0; /* * There should be one memnode (physical page free list(s)) for * each node if memory DR is disabled. */ max_mem_nodes = lgrp_plat_node_cnt; /* * Initialize min and max latency before reading SLIT or probing */ lgrp_plat_lat_stats.latency_min = -1; lgrp_plat_lat_stats.latency_max = 0; /* * Determine how far each NUMA node is from each other by * reading ACPI System Locality Information Table (SLIT) if it * exists */ lgrp_plat_slit_error = lgrp_plat_process_slit(slit_ptr, lgrp_plat_node_domain, lgrp_plat_node_cnt, lgrp_plat_memnode_info, &lgrp_plat_lat_stats); /* * Disable support of CPU/memory DR operations if multiple locality * domains exist in system and either of following is true. * 1) Failed to process SLIT table. * 2) Latency probing is enabled by user. */ if (lgrp_plat_node_cnt > 1 && (plat_dr_support_cpu() || plat_dr_support_memory())) { if (!lgrp_plat_slit_enable || lgrp_plat_slit_error != 0 || !lgrp_plat_srat_enable || lgrp_plat_srat_error != 0 || lgrp_plat_apic_ncpus <= 0) { cmn_err(CE_CONT, "?lgrp: failed to process ACPI SRAT/SLIT table, " "disable support of CPU/memory DR operations."); plat_dr_disable_cpu(); plat_dr_disable_memory(); } else if (lgrp_plat_probe_flags & LGRP_PLAT_PROBE_ENABLE) { cmn_err(CE_CONT, "?lgrp: latency probing enabled by user, " "disable support of CPU/memory DR operations."); plat_dr_disable_cpu(); plat_dr_disable_memory(); } } /* Done if succeeded to process SLIT table. */ if (lgrp_plat_slit_error == 0) return; /* * Probe to determine latency between NUMA nodes when SLIT * doesn't exist or make sense */ lgrp_plat_probe_flags |= LGRP_PLAT_PROBE_ENABLE; /* * Specify whether to probe using vendor ID register or page copy * if hasn't been specified already or is overspecified */ probe_op = lgrp_plat_probe_flags & (LGRP_PLAT_PROBE_PGCPY|LGRP_PLAT_PROBE_VENDOR); if (probe_op == 0 || probe_op == (LGRP_PLAT_PROBE_PGCPY|LGRP_PLAT_PROBE_VENDOR)) { lgrp_plat_probe_flags &= ~(LGRP_PLAT_PROBE_PGCPY|LGRP_PLAT_PROBE_VENDOR); if (is_opteron()) lgrp_plat_probe_flags |= LGRP_PLAT_PROBE_VENDOR; else lgrp_plat_probe_flags |= LGRP_PLAT_PROBE_PGCPY; } /* * Probing errors can mess up the lgroup topology and * force us fall back to a 2 level lgroup topology. * Here we bound how tall the lgroup topology can grow * in hopes of avoiding any anamolies in probing from * messing up the lgroup topology by limiting the * accuracy of the latency topology. * * Assume that nodes will at least be configured in a * ring, so limit height of lgroup topology to be less * than number of nodes on a system with 4 or more * nodes */ if (lgrp_plat_node_cnt >= 4 && lgrp_topo_ht_limit() == lgrp_topo_ht_limit_default()) (void) lgrp_topo_ht_limit_set(lgrp_plat_node_cnt - 1); } /* * Latencies must be within 1/(2**LGRP_LAT_TOLERANCE_SHIFT) of each other to * be considered same */ #define LGRP_LAT_TOLERANCE_SHIFT 4 int lgrp_plat_probe_lt_shift = LGRP_LAT_TOLERANCE_SHIFT; /* * Adjust latencies between nodes to be symmetric, normalize latencies between * any nodes that are within some tolerance to be same, and make local * latencies be same */ static void lgrp_plat_latency_adjust(memnode_phys_addr_map_t *memnode_info, lgrp_plat_latency_stats_t *lat_stats, lgrp_plat_probe_stats_t *probe_stats) { int i; int j; int k; int l; u_longlong_t max; u_longlong_t min; u_longlong_t t; u_longlong_t t1; u_longlong_t t2; const lgrp_config_flag_t cflag = LGRP_CONFIG_LAT_CHANGE_ALL; int lat_corrected[MAX_NODES][MAX_NODES]; t = 0; /* * Nothing to do when this is an UMA machine or don't have args needed */ if (max_mem_nodes == 1) return; ASSERT(memnode_info != NULL && lat_stats != NULL && probe_stats != NULL); /* * Make sure that latencies are symmetric between any two nodes * (ie. latency(node0, node1) == latency(node1, node0)) */ for (i = 0; i < lgrp_plat_node_cnt; i++) { if (!memnode_info[i].exists) continue; for (j = 0; j < lgrp_plat_node_cnt; j++) { if (!memnode_info[j].exists) continue; t1 = lat_stats->latencies[i][j]; t2 = lat_stats->latencies[j][i]; if (t1 == 0 || t2 == 0 || t1 == t2) continue; /* * Latencies should be same * - Use minimum of two latencies which should be same * - Track suspect probe times not within tolerance of * min value * - Remember how much values are corrected by */ if (t1 > t2) { t = t2; probe_stats->probe_errors[i][j] += t1 - t2; if (t1 - t2 > t2 >> lgrp_plat_probe_lt_shift) { probe_stats->probe_suspect[i][j]++; probe_stats->probe_suspect[j][i]++; } } else if (t2 > t1) { t = t1; probe_stats->probe_errors[j][i] += t2 - t1; if (t2 - t1 > t1 >> lgrp_plat_probe_lt_shift) { probe_stats->probe_suspect[i][j]++; probe_stats->probe_suspect[j][i]++; } } lat_stats->latencies[i][j] = lat_stats->latencies[j][i] = t; lgrp_config(cflag, t1, t); lgrp_config(cflag, t2, t); } } /* * Keep track of which latencies get corrected */ for (i = 0; i < MAX_NODES; i++) for (j = 0; j < MAX_NODES; j++) lat_corrected[i][j] = 0; /* * For every two nodes, see whether there is another pair of nodes which * are about the same distance apart and make the latencies be the same * if they are close enough together */ for (i = 0; i < lgrp_plat_node_cnt; i++) { for (j = 0; j < lgrp_plat_node_cnt; j++) { if (!memnode_info[j].exists) continue; /* * Pick one pair of nodes (i, j) * and get latency between them */ t1 = lat_stats->latencies[i][j]; /* * Skip this pair of nodes if there isn't a latency * for it yet */ if (t1 == 0) continue; for (k = 0; k < lgrp_plat_node_cnt; k++) { for (l = 0; l < lgrp_plat_node_cnt; l++) { if (!memnode_info[l].exists) continue; /* * Pick another pair of nodes (k, l) * not same as (i, j) and get latency * between them */ if (k == i && l == j) continue; t2 = lat_stats->latencies[k][l]; /* * Skip this pair of nodes if there * isn't a latency for it yet */ if (t2 == 0) continue; /* * Skip nodes (k, l) if they already * have same latency as (i, j) or * their latency isn't close enough to * be considered/made the same */ if (t1 == t2 || (t1 > t2 && t1 - t2 > t1 >> lgrp_plat_probe_lt_shift) || (t2 > t1 && t2 - t1 > t2 >> lgrp_plat_probe_lt_shift)) continue; /* * Make latency(i, j) same as * latency(k, l), try to use latency * that has been adjusted already to get * more consistency (if possible), and * remember which latencies were * adjusted for next time */ if (lat_corrected[i][j]) { t = t1; lgrp_config(cflag, t2, t); t2 = t; } else if (lat_corrected[k][l]) { t = t2; lgrp_config(cflag, t1, t); t1 = t; } else { if (t1 > t2) t = t2; else t = t1; lgrp_config(cflag, t1, t); lgrp_config(cflag, t2, t); t1 = t2 = t; } lat_stats->latencies[i][j] = lat_stats->latencies[k][l] = t; lat_corrected[i][j] = lat_corrected[k][l] = 1; } } } } /* * Local latencies should be same * - Find min and max local latencies * - Make all local latencies be minimum */ min = -1; max = 0; for (i = 0; i < lgrp_plat_node_cnt; i++) { if (!memnode_info[i].exists) continue; t = lat_stats->latencies[i][i]; if (t == 0) continue; if (min == -1 || t < min) min = t; if (t > max) max = t; } if (min != max) { for (i = 0; i < lgrp_plat_node_cnt; i++) { int local; if (!memnode_info[i].exists) continue; local = lat_stats->latencies[i][i]; if (local == 0) continue; /* * Track suspect probe times that aren't within * tolerance of minimum local latency and how much * probe times are corrected by */ if (local - min > min >> lgrp_plat_probe_lt_shift) probe_stats->probe_suspect[i][i]++; probe_stats->probe_errors[i][i] += local - min; /* * Make local latencies be minimum */ lgrp_config(LGRP_CONFIG_LAT_CHANGE, i, min); lat_stats->latencies[i][i] = min; } } /* * Determine max probe time again since just adjusted latencies */ lat_stats->latency_max = 0; for (i = 0; i < lgrp_plat_node_cnt; i++) { for (j = 0; j < lgrp_plat_node_cnt; j++) { if (!memnode_info[j].exists) continue; t = lat_stats->latencies[i][j]; if (t > lat_stats->latency_max) lat_stats->latency_max = t; } } } /* * Verify following about latencies between nodes: * * - Latencies should be symmetric (ie. latency(a, b) == latency(b, a)) * - Local latencies same * - Local < remote * - Number of latencies seen is reasonable * - Number of occurrences of a given latency should be more than 1 * * Returns: * 0 Success * -1 Not symmetric * -2 Local latencies not same * -3 Local >= remote */ static int lgrp_plat_latency_verify(memnode_phys_addr_map_t *memnode_info, lgrp_plat_latency_stats_t *lat_stats) { int i; int j; u_longlong_t t1; u_longlong_t t2; ASSERT(memnode_info != NULL && lat_stats != NULL); /* * Nothing to do when this is an UMA machine, lgroup topology is * limited to 2 levels, or there aren't any probe times yet */ if (max_mem_nodes == 1 || lgrp_topo_levels < 2 || lat_stats->latencies[0][0] == 0) return (0); /* * Make sure that latencies are symmetric between any two nodes * (ie. latency(node0, node1) == latency(node1, node0)) */ for (i = 0; i < lgrp_plat_node_cnt; i++) { if (!memnode_info[i].exists) continue; for (j = 0; j < lgrp_plat_node_cnt; j++) { if (!memnode_info[j].exists) continue; t1 = lat_stats->latencies[i][j]; t2 = lat_stats->latencies[j][i]; if (t1 == 0 || t2 == 0 || t1 == t2) continue; return (-1); } } /* * Local latencies should be same */ t1 = lat_stats->latencies[0][0]; for (i = 1; i < lgrp_plat_node_cnt; i++) { if (!memnode_info[i].exists) continue; t2 = lat_stats->latencies[i][i]; if (t2 == 0) continue; if (t1 == 0) { t1 = t2; continue; } if (t1 != t2) return (-2); } /* * Local latencies should be less than remote */ if (t1) { for (i = 0; i < lgrp_plat_node_cnt; i++) { for (j = 0; j < lgrp_plat_node_cnt; j++) { if (!memnode_info[j].exists) continue; t2 = lat_stats->latencies[i][j]; if (i == j || t2 == 0) continue; if (t1 >= t2) return (-3); } } } return (0); } /* * Platform-specific initialization */ static void lgrp_plat_main_init(void) { int curnode; int ht_limit; int i; /* * Print a notice that MPO is disabled when memory is interleaved * across nodes....Would do this when it is discovered, but can't * because it happens way too early during boot.... */ if (lgrp_plat_mem_intrlv) cmn_err(CE_NOTE, "MPO disabled because memory is interleaved\n"); /* * Don't bother to do any probing if it is disabled, there is only one * node, or the height of the lgroup topology less than or equal to 2 */ ht_limit = lgrp_topo_ht_limit(); if (!(lgrp_plat_probe_flags & LGRP_PLAT_PROBE_ENABLE) || max_mem_nodes == 1 || ht_limit <= 2) { /* * Setup lgroup latencies for 2 level lgroup topology * (ie. local and remote only) if they haven't been set yet */ if (ht_limit == 2 && lgrp_plat_lat_stats.latency_min == -1 && lgrp_plat_lat_stats.latency_max == 0) lgrp_plat_2level_setup(&lgrp_plat_lat_stats); return; } if (lgrp_plat_probe_flags & LGRP_PLAT_PROBE_VENDOR) { /* * Should have been able to probe from CPU 0 when it was added * to lgroup hierarchy, but may not have been able to then * because it happens so early in boot that gethrtime() hasn't * been initialized. (:-( */ curnode = lgrp_plat_cpu_to_node(CPU, lgrp_plat_cpu_node, lgrp_plat_cpu_node_nentries); ASSERT3U(curnode, <, lgrp_plat_node_cnt); if (curnode == (lgrp_handle_t)-1) return; if (lgrp_plat_lat_stats.latencies[curnode][curnode] == 0) lgrp_plat_probe(); return; } /* * When probing memory, use one page for every sample to determine * lgroup topology and taking multiple samples */ if (lgrp_plat_probe_mem_config.probe_memsize == 0) lgrp_plat_probe_mem_config.probe_memsize = PAGESIZE * lgrp_plat_probe_nsamples; /* * Map memory in each node needed for probing to determine latency * topology */ for (i = 0; i < lgrp_plat_node_cnt; i++) { int mnode; /* * Skip this node and leave its probe page NULL * if it doesn't have any memory */ mnode = i; if (!mem_node_config[mnode].exists) { lgrp_plat_probe_mem_config.probe_va[i] = NULL; continue; } /* * Allocate one kernel virtual page */ lgrp_plat_probe_mem_config.probe_va[i] = vmem_alloc(heap_arena, lgrp_plat_probe_mem_config.probe_memsize, VM_NOSLEEP); if (lgrp_plat_probe_mem_config.probe_va[i] == NULL) { cmn_err(CE_WARN, "lgrp_plat_main_init: couldn't allocate memory"); return; } /* * Get PFN for first page in each node */ lgrp_plat_probe_mem_config.probe_pfn[i] = mem_node_config[mnode].physbase; /* * Map virtual page to first page in node */ hat_devload(kas.a_hat, lgrp_plat_probe_mem_config.probe_va[i], lgrp_plat_probe_mem_config.probe_memsize, lgrp_plat_probe_mem_config.probe_pfn[i], PROT_READ | PROT_WRITE | HAT_PLAT_NOCACHE, HAT_LOAD_NOCONSIST); } /* * Probe from current CPU */ lgrp_plat_probe(); } /* * Return the number of free, allocatable, or installed * pages in an lgroup * This is a copy of the MAX_MEM_NODES == 1 version of the routine * used when MPO is disabled (i.e. single lgroup) or this is the root lgroup */ static pgcnt_t lgrp_plat_mem_size_default(lgrp_handle_t lgrphand, lgrp_mem_query_t query) { _NOTE(ARGUNUSED(lgrphand)); struct memlist *mlist; pgcnt_t npgs = 0; extern struct memlist *phys_avail; extern struct memlist *phys_install; switch (query) { case LGRP_MEM_SIZE_FREE: return ((pgcnt_t)freemem); case LGRP_MEM_SIZE_AVAIL: memlist_read_lock(); for (mlist = phys_avail; mlist; mlist = mlist->ml_next) npgs += btop(mlist->ml_size); memlist_read_unlock(); return (npgs); case LGRP_MEM_SIZE_INSTALL: memlist_read_lock(); for (mlist = phys_install; mlist; mlist = mlist->ml_next) npgs += btop(mlist->ml_size); memlist_read_unlock(); return (npgs); default: return ((pgcnt_t)0); } } /* * Update node to proximity domain mappings for given domain and return node ID */ static int lgrp_plat_node_domain_update(node_domain_map_t *node_domain, int node_cnt, uint32_t domain) { uint_t node; uint_t start; /* * Hash proximity domain ID into node to domain mapping table (array) * and add entry for it into first non-existent or matching entry found */ node = start = NODE_DOMAIN_HASH(domain, node_cnt); do { /* * Entry doesn't exist yet, so create one for this proximity * domain and return node ID which is index into mapping table. */ if (!node_domain[node].exists) { node_domain[node].prox_domain = domain; membar_producer(); node_domain[node].exists = 1; return (node); } /* * Entry exists for this proximity domain already, so just * return node ID (index into table). */ if (node_domain[node].prox_domain == domain) return (node); node = NODE_DOMAIN_HASH(node + 1, node_cnt); } while (node != start); /* * Ran out of supported number of entries which shouldn't happen.... */ ASSERT(node != start); return (-1); } /* * Update node memory information for given proximity domain with specified * starting and ending physical address range (and return positive numbers for * success and negative ones for errors) */ static int lgrp_plat_memnode_info_update(node_domain_map_t *node_domain, int node_cnt, memnode_phys_addr_map_t *memnode_info, int memnode_cnt, uint64_t start, uint64_t end, uint32_t domain, uint32_t device_id) { int node, mnode; /* * Get node number for proximity domain */ node = lgrp_plat_domain_to_node(node_domain, node_cnt, domain); if (node == -1) { node = lgrp_plat_node_domain_update(node_domain, node_cnt, domain); if (node == -1) return (-1); } /* * This function is called during boot if device_id is * ACPI_MEMNODE_DEVID_BOOT, otherwise it's called at runtime for * memory DR operations. */ if (device_id != ACPI_MEMNODE_DEVID_BOOT) { ASSERT(lgrp_plat_max_mem_node <= memnode_cnt); for (mnode = lgrp_plat_node_cnt; mnode < lgrp_plat_max_mem_node; mnode++) { if (memnode_info[mnode].exists && memnode_info[mnode].prox_domain == domain && memnode_info[mnode].device_id == device_id) { if (btop(start) < memnode_info[mnode].start) memnode_info[mnode].start = btop(start); if (btop(end) > memnode_info[mnode].end) memnode_info[mnode].end = btop(end); return (1); } } if (lgrp_plat_max_mem_node >= memnode_cnt) { return (-3); } else { lgrp_plat_max_mem_node++; memnode_info[mnode].start = btop(start); memnode_info[mnode].end = btop(end); memnode_info[mnode].prox_domain = domain; memnode_info[mnode].device_id = device_id; memnode_info[mnode].lgrphand = node; membar_producer(); memnode_info[mnode].exists = 1; return (0); } } /* * Create entry in table for node if it doesn't exist */ ASSERT(node < memnode_cnt); if (!memnode_info[node].exists) { memnode_info[node].start = btop(start); memnode_info[node].end = btop(end); memnode_info[node].prox_domain = domain; memnode_info[node].device_id = device_id; memnode_info[node].lgrphand = node; membar_producer(); memnode_info[node].exists = 1; return (0); } /* * Entry already exists for this proximity domain * * There may be more than one SRAT memory entry for a domain, so we may * need to update existing start or end address for the node. */ if (memnode_info[node].prox_domain == domain) { if (btop(start) < memnode_info[node].start) memnode_info[node].start = btop(start); if (btop(end) > memnode_info[node].end) memnode_info[node].end = btop(end); return (1); } return (-2); } /* * Have to sort nodes by starting physical address because plat_mnode_xcheck() * assumes and expects memnodes to be sorted in ascending order by physical * address. */ static void lgrp_plat_node_sort(node_domain_map_t *node_domain, int node_cnt, cpu_node_map_t *cpu_node, int cpu_count, memnode_phys_addr_map_t *memnode_info) { boolean_t found; int i; int j; int n; boolean_t sorted; boolean_t swapped; if (!lgrp_plat_node_sort_enable || node_cnt <= 1 || node_domain == NULL || memnode_info == NULL) return; /* * Sorted already? */ sorted = B_TRUE; for (i = 0; i < node_cnt - 1; i++) { /* * Skip entries that don't exist */ if (!memnode_info[i].exists) continue; /* * Try to find next existing entry to compare against */ found = B_FALSE; for (j = i + 1; j < node_cnt; j++) { if (memnode_info[j].exists) { found = B_TRUE; break; } } /* * Done if no more existing entries to compare against */ if (found == B_FALSE) break; /* * Not sorted if starting address of current entry is bigger * than starting address of next existing entry */ if (memnode_info[i].start > memnode_info[j].start) { sorted = B_FALSE; break; } } /* * Don't need to sort if sorted already */ if (sorted == B_TRUE) return; /* * Just use bubble sort since number of nodes is small */ n = node_cnt; do { swapped = B_FALSE; n--; for (i = 0; i < n; i++) { /* * Skip entries that don't exist */ if (!memnode_info[i].exists) continue; /* * Try to find next existing entry to compare against */ found = B_FALSE; for (j = i + 1; j <= n; j++) { if (memnode_info[j].exists) { found = B_TRUE; break; } } /* * Done if no more existing entries to compare against */ if (found == B_FALSE) break; if (memnode_info[i].start > memnode_info[j].start) { memnode_phys_addr_map_t save_addr; node_domain_map_t save_node; /* * Swap node to proxmity domain ID assignments */ bcopy(&node_domain[i], &save_node, sizeof (node_domain_map_t)); bcopy(&node_domain[j], &node_domain[i], sizeof (node_domain_map_t)); bcopy(&save_node, &node_domain[j], sizeof (node_domain_map_t)); /* * Swap node to physical memory assignments */ bcopy(&memnode_info[i], &save_addr, sizeof (memnode_phys_addr_map_t)); bcopy(&memnode_info[j], &memnode_info[i], sizeof (memnode_phys_addr_map_t)); bcopy(&save_addr, &memnode_info[j], sizeof (memnode_phys_addr_map_t)); swapped = B_TRUE; } } } while (swapped == B_TRUE); /* * Check to make sure that CPUs assigned to correct node IDs now since * node to proximity domain ID assignments may have been changed above */ if (n == node_cnt - 1 || cpu_node == NULL || cpu_count < 1) return; for (i = 0; i < cpu_count; i++) { int node; node = lgrp_plat_domain_to_node(node_domain, node_cnt, cpu_node[i].prox_domain); if (cpu_node[i].node != node) cpu_node[i].node = node; } } /* * Return time needed to probe from current CPU to memory in given node */ static hrtime_t lgrp_plat_probe_time(int to, cpu_node_map_t *cpu_node, int cpu_node_nentries, lgrp_plat_probe_mem_config_t *probe_mem_config, lgrp_plat_latency_stats_t *lat_stats, lgrp_plat_probe_stats_t *probe_stats) { caddr_t buf; hrtime_t elapsed; hrtime_t end; int from; int i; int ipl; hrtime_t max; hrtime_t min; hrtime_t start; extern int use_sse_pagecopy; /* * Determine ID of node containing current CPU */ from = lgrp_plat_cpu_to_node(CPU, cpu_node, cpu_node_nentries); ASSERT3U(from, <, lgrp_plat_node_cnt); if (from == (lgrp_handle_t)-1) return (0); /* * Do common work for probing main memory */ if (lgrp_plat_probe_flags & LGRP_PLAT_PROBE_PGCPY) { /* * Skip probing any nodes without memory and * set probe time to 0 */ if (probe_mem_config->probe_va[to] == NULL) { lat_stats->latencies[from][to] = 0; return (0); } /* * Invalidate caches once instead of once every sample * which should cut cost of probing by a lot */ probe_stats->flush_cost = gethrtime(); invalidate_cache(); probe_stats->flush_cost = gethrtime() - probe_stats->flush_cost; probe_stats->probe_cost_total += probe_stats->flush_cost; } /* * Probe from current CPU to given memory using specified operation * and take specified number of samples */ max = 0; min = -1; for (i = 0; i < lgrp_plat_probe_nsamples; i++) { probe_stats->probe_cost = gethrtime(); /* * Can't measure probe time if gethrtime() isn't working yet */ if (probe_stats->probe_cost == 0 && gethrtime() == 0) return (0); if (lgrp_plat_probe_flags & LGRP_PLAT_PROBE_VENDOR) { /* * Measure how long it takes to read vendor ID from * Northbridge */ elapsed = opt_probe_vendor(to, lgrp_plat_probe_nreads); } else { /* * Measure how long it takes to copy page * on top of itself */ buf = probe_mem_config->probe_va[to] + (i * PAGESIZE); kpreempt_disable(); ipl = splhigh(); start = gethrtime(); if (use_sse_pagecopy) hwblkpagecopy(buf, buf); else bcopy(buf, buf, PAGESIZE); end = gethrtime(); elapsed = end - start; splx(ipl); kpreempt_enable(); } probe_stats->probe_cost = gethrtime() - probe_stats->probe_cost; probe_stats->probe_cost_total += probe_stats->probe_cost; if (min == -1 || elapsed < min) min = elapsed; if (elapsed > max) max = elapsed; } /* * Update minimum and maximum probe times between * these two nodes */ if (min < probe_stats->probe_min[from][to] || probe_stats->probe_min[from][to] == 0) probe_stats->probe_min[from][to] = min; if (max > probe_stats->probe_max[from][to]) probe_stats->probe_max[from][to] = max; return (min); } /* * Read boot property with CPU to APIC ID array, fill in CPU to node ID * mapping table with APIC ID for each CPU (if pointer to table isn't NULL), * and return number of CPU APIC IDs. * * NOTE: This code assumes that CPU IDs are assigned in order that they appear * in in cpu_apicid_array boot property which is based on and follows * same ordering as processor list in ACPI MADT. If the code in * usr/src/uts/i86pc/io/pcplusmp/apic.c that reads MADT and assigns * CPU IDs ever changes, then this code will need to change too.... */ static int lgrp_plat_process_cpu_apicids(cpu_node_map_t *cpu_node) { int boot_prop_len; char *boot_prop_name = BP_CPU_APICID_ARRAY; uint32_t *cpu_apicid_array; int i; int n; /* * Check length of property value */ boot_prop_len = BOP_GETPROPLEN(bootops, boot_prop_name); if (boot_prop_len <= 0) return (-1); /* * Calculate number of entries in array and return when the system is * not very interesting for NUMA. It's not interesting for NUMA if * system has only one CPU and doesn't support CPU hotplug. */ n = boot_prop_len / sizeof (*cpu_apicid_array); if (n == 1 && !plat_dr_support_cpu()) return (-2); cpu_apicid_array = (uint32_t *)BOP_ALLOC(bootops, NULL, boot_prop_len, sizeof (*cpu_apicid_array)); /* * Get CPU to APIC ID property value */ if (cpu_apicid_array == NULL || BOP_GETPROP(bootops, boot_prop_name, cpu_apicid_array) < 0) return (-3); /* * Just return number of CPU APIC IDs if CPU to node mapping table is * NULL */ if (cpu_node == NULL) { if (plat_dr_support_cpu() && n >= boot_ncpus) { return (boot_ncpus); } else { return (n); } } /* * Fill in CPU to node ID mapping table with APIC ID for each CPU */ for (i = 0; i < n; i++) { /* Only add boot CPUs into the map if CPU DR is enabled. */ if (plat_dr_support_cpu() && i >= boot_ncpus) break; cpu_node[i].exists = 1; cpu_node[i].apicid = cpu_apicid_array[i]; cpu_node[i].prox_domain = UINT32_MAX; cpu_node[i].node = UINT_MAX; } /* * Return number of CPUs based on number of APIC IDs */ return (i); } /* * Read ACPI System Locality Information Table (SLIT) to determine how far each * NUMA node is from each other */ static int lgrp_plat_process_slit(ACPI_TABLE_SLIT *tp, node_domain_map_t *node_domain, uint_t node_cnt, memnode_phys_addr_map_t *memnode_info, lgrp_plat_latency_stats_t *lat_stats) { int i; int j; int src; int dst; int localities; hrtime_t max; hrtime_t min; int retval; uint8_t *slit_entries; if (tp == NULL || !lgrp_plat_slit_enable) return (1); if (lat_stats == NULL) return (2); localities = tp->LocalityCount; min = lat_stats->latency_min; max = lat_stats->latency_max; /* * Fill in latency matrix based on SLIT entries */ slit_entries = tp->Entry; for (i = 0; i < localities; i++) { src = lgrp_plat_domain_to_node(node_domain, node_cnt, i); if (src == -1) continue; for (j = 0; j < localities; j++) { uint8_t latency; dst = lgrp_plat_domain_to_node(node_domain, node_cnt, j); if (dst == -1) continue; latency = slit_entries[(i * localities) + j]; lat_stats->latencies[src][dst] = latency; if (latency < min || min == -1) min = latency; if (latency > max) max = latency; } } /* * Verify that latencies/distances given in SLIT look reasonable */ retval = lgrp_plat_latency_verify(memnode_info, lat_stats); if (retval) { /* * Reinitialize (zero) latency table since SLIT doesn't look * right */ for (i = 0; i < localities; i++) { for (j = 0; j < localities; j++) lat_stats->latencies[i][j] = 0; } } else { /* * Update min and max latencies seen since SLIT looks valid */ lat_stats->latency_min = min; lat_stats->latency_max = max; } return (retval); } /* * Update lgrp latencies according to information returned by ACPI _SLI method. */ static int lgrp_plat_process_sli(uint32_t domain_id, uchar_t *sli_info, uint32_t sli_cnt, node_domain_map_t *node_domain, uint_t node_cnt, lgrp_plat_latency_stats_t *lat_stats) { int i; int src, dst; uint8_t latency; hrtime_t max, min; if (lat_stats == NULL || sli_info == NULL || sli_cnt == 0 || domain_id >= sli_cnt) return (-1); src = lgrp_plat_domain_to_node(node_domain, node_cnt, domain_id); if (src == -1) { src = lgrp_plat_node_domain_update(node_domain, node_cnt, domain_id); if (src == -1) return (-1); } /* * Don't update latency info if topology has been flattened to 2 levels. */ if (lgrp_plat_topo_flatten != 0) { return (0); } /* * Latency information for proximity domain is ready. * TODO: support adjusting latency information at runtime. */ if (lat_stats->latencies[src][src] != 0) { return (0); } /* Validate latency information. */ for (i = 0; i < sli_cnt; i++) { if (i == domain_id) { if (sli_info[i] != ACPI_SLIT_SELF_LATENCY || sli_info[sli_cnt + i] != ACPI_SLIT_SELF_LATENCY) { return (-1); } } else { if (sli_info[i] <= ACPI_SLIT_SELF_LATENCY || sli_info[sli_cnt + i] <= ACPI_SLIT_SELF_LATENCY || sli_info[i] != sli_info[sli_cnt + i]) { return (-1); } } } min = lat_stats->latency_min; max = lat_stats->latency_max; for (i = 0; i < sli_cnt; i++) { dst = lgrp_plat_domain_to_node(node_domain, node_cnt, i); if (dst == -1) continue; ASSERT(sli_info[i] == sli_info[sli_cnt + i]); /* Update row in latencies matrix. */ latency = sli_info[i]; lat_stats->latencies[src][dst] = latency; if (latency < min || min == -1) min = latency; if (latency > max) max = latency; /* Update column in latencies matrix. */ latency = sli_info[sli_cnt + i]; lat_stats->latencies[dst][src] = latency; if (latency < min || min == -1) min = latency; if (latency > max) max = latency; } lat_stats->latency_min = min; lat_stats->latency_max = max; return (0); } /* * Read ACPI System Resource Affinity Table (SRAT) to determine which CPUs * and memory are local to each other in the same NUMA node and return number * of nodes */ static int lgrp_plat_process_srat(ACPI_TABLE_SRAT *tp, ACPI_TABLE_MSCT *mp, uint32_t *prox_domain_min, node_domain_map_t *node_domain, cpu_node_map_t *cpu_node, int cpu_count, memnode_phys_addr_map_t *memnode_info) { ACPI_SUBTABLE_HEADER *item, *srat_end; int i; int node_cnt; int proc_entry_count; int rc; /* * Nothing to do when no SRAT or disabled */ if (tp == NULL || !lgrp_plat_srat_enable) return (-1); /* * Try to get domain information from MSCT table. * ACPI4.0: OSPM will use information provided by the MSCT only * when the System Resource Affinity Table (SRAT) exists. */ node_cnt = lgrp_plat_msct_domains(mp, prox_domain_min); if (node_cnt <= 0) { /* * Determine number of nodes by counting number of proximity * domains in SRAT. */ node_cnt = lgrp_plat_srat_domains(tp, prox_domain_min); } /* * Return if number of nodes is 1 or less since don't need to read SRAT. */ if (node_cnt == 1) return (1); else if (node_cnt <= 0) return (-2); /* * Walk through SRAT, examining each CPU and memory entry to determine * which CPUs and memory belong to which node. */ item = (ACPI_SUBTABLE_HEADER *)((uintptr_t)tp + sizeof (*tp)); srat_end = (ACPI_SUBTABLE_HEADER *)(tp->Header.Length + (uintptr_t)tp); proc_entry_count = 0; while (item < srat_end) { uint32_t apic_id; uint32_t domain; uint64_t end; uint64_t length; uint64_t start; switch (item->Type) { case ACPI_SRAT_TYPE_CPU_AFFINITY: { /* CPU entry */ ACPI_SRAT_CPU_AFFINITY *cpu = (ACPI_SRAT_CPU_AFFINITY *) item; if (!(cpu->Flags & ACPI_SRAT_CPU_ENABLED) || cpu_node == NULL) break; /* * Calculate domain (node) ID and fill in APIC ID to * domain/node mapping table */ domain = cpu->ProximityDomainLo; for (i = 0; i < 3; i++) { domain += cpu->ProximityDomainHi[i] << ((i + 1) * 8); } apic_id = cpu->ApicId; rc = lgrp_plat_cpu_node_update(node_domain, node_cnt, cpu_node, cpu_count, apic_id, domain); if (rc < 0) return (-3); else if (rc == 0) proc_entry_count++; break; } case ACPI_SRAT_TYPE_MEMORY_AFFINITY: { /* memory entry */ ACPI_SRAT_MEM_AFFINITY *mem = (ACPI_SRAT_MEM_AFFINITY *)item; if (!(mem->Flags & ACPI_SRAT_MEM_ENABLED) || memnode_info == NULL) break; /* * Get domain (node) ID and fill in domain/node * to memory mapping table */ domain = mem->ProximityDomain; start = mem->BaseAddress; length = mem->Length; end = start + length - 1; /* * According to ACPI 4.0, both ENABLE and HOTPLUG flags * may be set for memory address range entries in SRAT * table which are reserved for memory hot plug. * We intersect memory address ranges in SRAT table * with memory ranges in physinstalled to filter out * memory address ranges reserved for hot plug. */ if (mem->Flags & ACPI_SRAT_MEM_HOT_PLUGGABLE) { uint64_t rstart = UINT64_MAX; uint64_t rend = 0; struct memlist *ml; extern struct bootops *bootops; memlist_read_lock(); for (ml = bootops->boot_mem->physinstalled; ml; ml = ml->ml_next) { uint64_t tstart = ml->ml_address; uint64_t tend; tend = ml->ml_address + ml->ml_size; if (tstart > end || tend < start) continue; if (start > tstart) tstart = start; if (rstart > tstart) rstart = tstart; if (end < tend) tend = end; if (rend < tend) rend = tend; } memlist_read_unlock(); start = rstart; end = rend; /* Skip this entry if no memory installed. */ if (start > end) break; } if (lgrp_plat_memnode_info_update(node_domain, node_cnt, memnode_info, node_cnt, start, end, domain, ACPI_MEMNODE_DEVID_BOOT) < 0) return (-4); break; } case ACPI_SRAT_TYPE_X2APIC_CPU_AFFINITY: { /* x2apic CPU */ ACPI_SRAT_X2APIC_CPU_AFFINITY *x2cpu = (ACPI_SRAT_X2APIC_CPU_AFFINITY *) item; if (!(x2cpu->Flags & ACPI_SRAT_CPU_ENABLED) || cpu_node == NULL) break; /* * Calculate domain (node) ID and fill in APIC ID to * domain/node mapping table */ domain = x2cpu->ProximityDomain; apic_id = x2cpu->ApicId; rc = lgrp_plat_cpu_node_update(node_domain, node_cnt, cpu_node, cpu_count, apic_id, domain); if (rc < 0) return (-3); else if (rc == 0) proc_entry_count++; break; } default: break; } item = (ACPI_SUBTABLE_HEADER *)((uintptr_t)item + item->Length); } /* * Should have seen at least as many SRAT processor entries as CPUs */ if (proc_entry_count < cpu_count) return (-5); /* * Need to sort nodes by starting physical address since VM system * assumes and expects memnodes to be sorted in ascending order by * physical address */ lgrp_plat_node_sort(node_domain, node_cnt, cpu_node, cpu_count, memnode_info); return (node_cnt); } /* * Allocate permanent memory for any temporary memory that we needed to * allocate using BOP_ALLOC() before kmem_alloc() and VM system were * initialized and copy everything from temporary to permanent memory since * temporary boot memory will eventually be released during boot */ static void lgrp_plat_release_bootstrap(void) { void *buf; size_t size; if (lgrp_plat_cpu_node_nentries > 0) { size = lgrp_plat_cpu_node_nentries * sizeof (cpu_node_map_t); buf = kmem_alloc(size, KM_SLEEP); bcopy(lgrp_plat_cpu_node, buf, size); lgrp_plat_cpu_node = buf; } } /* * Return number of proximity domains given in ACPI SRAT */ static int lgrp_plat_srat_domains(ACPI_TABLE_SRAT *tp, uint32_t *prox_domain_min) { int domain_cnt; uint32_t domain_min; ACPI_SUBTABLE_HEADER *item, *end; int i; node_domain_map_t node_domain[MAX_NODES]; if (tp == NULL || !lgrp_plat_srat_enable) return (1); /* * Walk through SRAT to find minimum proximity domain ID */ domain_min = UINT32_MAX; item = (ACPI_SUBTABLE_HEADER *)((uintptr_t)tp + sizeof (*tp)); end = (ACPI_SUBTABLE_HEADER *)(tp->Header.Length + (uintptr_t)tp); while (item < end) { uint32_t domain; switch (item->Type) { case ACPI_SRAT_TYPE_CPU_AFFINITY: { /* CPU entry */ ACPI_SRAT_CPU_AFFINITY *cpu = (ACPI_SRAT_CPU_AFFINITY *) item; if (!(cpu->Flags & ACPI_SRAT_CPU_ENABLED)) { item = (ACPI_SUBTABLE_HEADER *) ((uintptr_t)item + item->Length); continue; } domain = cpu->ProximityDomainLo; for (i = 0; i < 3; i++) { domain += cpu->ProximityDomainHi[i] << ((i + 1) * 8); } break; } case ACPI_SRAT_TYPE_MEMORY_AFFINITY: { /* memory entry */ ACPI_SRAT_MEM_AFFINITY *mem = (ACPI_SRAT_MEM_AFFINITY *)item; if (!(mem->Flags & ACPI_SRAT_MEM_ENABLED)) { item = (ACPI_SUBTABLE_HEADER *) ((uintptr_t)item + item->Length); continue; } domain = mem->ProximityDomain; break; } case ACPI_SRAT_TYPE_X2APIC_CPU_AFFINITY: { /* x2apic CPU */ ACPI_SRAT_X2APIC_CPU_AFFINITY *x2cpu = (ACPI_SRAT_X2APIC_CPU_AFFINITY *) item; if (!(x2cpu->Flags & ACPI_SRAT_CPU_ENABLED)) { item = (ACPI_SUBTABLE_HEADER *) ((uintptr_t)item + item->Length); continue; } domain = x2cpu->ProximityDomain; break; } default: item = (ACPI_SUBTABLE_HEADER *)((uintptr_t)item + item->Length); continue; } /* * Keep track of minimum proximity domain ID */ if (domain < domain_min) domain_min = domain; item = (ACPI_SUBTABLE_HEADER *)((uintptr_t)item + item->Length); } if (lgrp_plat_domain_min_enable && prox_domain_min != NULL) *prox_domain_min = domain_min; /* * Walk through SRAT, examining each CPU and memory entry to determine * proximity domain ID for each. */ domain_cnt = 0; item = (ACPI_SUBTABLE_HEADER *)((uintptr_t)tp + sizeof (*tp)); end = (ACPI_SUBTABLE_HEADER *)(tp->Header.Length + (uintptr_t)tp); bzero(node_domain, MAX_NODES * sizeof (node_domain_map_t)); while (item < end) { uint32_t domain; boolean_t overflow; uint_t start; switch (item->Type) { case ACPI_SRAT_TYPE_CPU_AFFINITY: { /* CPU entry */ ACPI_SRAT_CPU_AFFINITY *cpu = (ACPI_SRAT_CPU_AFFINITY *) item; if (!(cpu->Flags & ACPI_SRAT_CPU_ENABLED)) { item = (ACPI_SUBTABLE_HEADER *) ((uintptr_t)item + item->Length); continue; } domain = cpu->ProximityDomainLo; for (i = 0; i < 3; i++) { domain += cpu->ProximityDomainHi[i] << ((i + 1) * 8); } break; } case ACPI_SRAT_TYPE_MEMORY_AFFINITY: { /* memory entry */ ACPI_SRAT_MEM_AFFINITY *mem = (ACPI_SRAT_MEM_AFFINITY *)item; if (!(mem->Flags & ACPI_SRAT_MEM_ENABLED)) { item = (ACPI_SUBTABLE_HEADER *) ((uintptr_t)item + item->Length); continue; } domain = mem->ProximityDomain; break; } case ACPI_SRAT_TYPE_X2APIC_CPU_AFFINITY: { /* x2apic CPU */ ACPI_SRAT_X2APIC_CPU_AFFINITY *x2cpu = (ACPI_SRAT_X2APIC_CPU_AFFINITY *) item; if (!(x2cpu->Flags & ACPI_SRAT_CPU_ENABLED)) { item = (ACPI_SUBTABLE_HEADER *) ((uintptr_t)item + item->Length); continue; } domain = x2cpu->ProximityDomain; break; } default: item = (ACPI_SUBTABLE_HEADER *)((uintptr_t)item + item->Length); continue; } /* * Count and keep track of which proximity domain IDs seen */ start = i = domain % MAX_NODES; overflow = B_TRUE; do { /* * Create entry for proximity domain and increment * count when no entry exists where proximity domain * hashed */ if (!node_domain[i].exists) { node_domain[i].exists = 1; node_domain[i].prox_domain = domain; domain_cnt++; overflow = B_FALSE; break; } /* * Nothing to do when proximity domain seen already * and its entry exists */ if (node_domain[i].prox_domain == domain) { overflow = B_FALSE; break; } /* * Entry exists where proximity domain hashed, but for * different proximity domain so keep search for empty * slot to put it or matching entry whichever comes * first. */ i = (i + 1) % MAX_NODES; } while (i != start); /* * Didn't find empty or matching entry which means have more * proximity domains than supported nodes (:-( */ ASSERT(overflow != B_TRUE); if (overflow == B_TRUE) return (-1); item = (ACPI_SUBTABLE_HEADER *)((uintptr_t)item + item->Length); } return (domain_cnt); } /* * Parse domain information in ACPI Maximum System Capability Table (MSCT). * MSCT table has been verified in function process_msct() in fakebop.c. */ static int lgrp_plat_msct_domains(ACPI_TABLE_MSCT *tp, uint32_t *prox_domain_min) { int last_seen = 0; uint32_t proxmin = UINT32_MAX; ACPI_MSCT_PROXIMITY *item, *end; if (tp == NULL || lgrp_plat_msct_enable == 0) return (-1); if (tp->MaxProximityDomains >= MAX_NODES) { cmn_err(CE_CONT, "?lgrp: too many proximity domains (%d), max %d supported, " "disable support of CPU/memory DR operations.", tp->MaxProximityDomains + 1, MAX_NODES); plat_dr_disable_cpu(); plat_dr_disable_memory(); return (-1); } if (prox_domain_min != NULL) { end = (void *)(tp->Header.Length + (uintptr_t)tp); for (item = (void *)((uintptr_t)tp + tp->ProximityOffset); item < end; item = (void *)(item->Length + (uintptr_t)item)) { if (item->RangeStart < proxmin) { proxmin = item->RangeStart; } last_seen = item->RangeEnd - item->RangeStart + 1; /* * Break out if all proximity domains have been * processed. Some BIOSes may have unused items * at the end of MSCT table. */ if (last_seen > tp->MaxProximityDomains) { break; } } *prox_domain_min = proxmin; } return (tp->MaxProximityDomains + 1); } /* * Set lgroup latencies for 2 level lgroup topology */ static void lgrp_plat_2level_setup(lgrp_plat_latency_stats_t *lat_stats) { int i, j; ASSERT(lat_stats != NULL); if (lgrp_plat_node_cnt >= 4) cmn_err(CE_NOTE, "MPO only optimizing for local and remote\n"); for (i = 0; i < lgrp_plat_node_cnt; i++) { for (j = 0; j < lgrp_plat_node_cnt; j++) { if (i == j) lat_stats->latencies[i][j] = 2; else lat_stats->latencies[i][j] = 3; } } lat_stats->latency_min = 2; lat_stats->latency_max = 3; /* TODO: check it. */ lgrp_config(LGRP_CONFIG_FLATTEN, 2, 0); lgrp_plat_topo_flatten = 1; } /* * The following Opteron specific constants, macros, types, and routines define * PCI configuration space registers and how to read them to determine the NUMA * configuration of *supported* Opteron processors. They provide the same * information that may be gotten from the ACPI System Resource Affinity Table * (SRAT) if it exists on the machine of interest. * * The AMD BIOS and Kernel Developer's Guide (BKDG) for the processor family * of interest describes all of these registers and their contents. The main * registers used by this code to determine the NUMA configuration of the * machine are the node ID register for the number of NUMA nodes and the DRAM * address map registers for the physical address range of each node. * * NOTE: The format and how to determine the NUMA configuration using PCI * config space registers may change or may not be supported in future * Opteron processor families. */ /* * How many bits to shift Opteron DRAM Address Map base and limit registers * to get actual value */ #define OPT_DRAMADDR_HI_LSHIFT_ADDR 40 /* shift left for address */ #define OPT_DRAMADDR_LO_LSHIFT_ADDR 8 /* shift left for address */ #define OPT_DRAMADDR_HI_MASK_ADDR 0x000000FF /* address bits 47-40 */ #define OPT_DRAMADDR_LO_MASK_ADDR 0xFFFF0000 /* address bits 39-24 */ #define OPT_DRAMADDR_LO_MASK_OFF 0xFFFFFF /* offset for address */ /* * Macros to derive addresses from Opteron DRAM Address Map registers */ #define OPT_DRAMADDR_HI(reg) \ (((u_longlong_t)reg & OPT_DRAMADDR_HI_MASK_ADDR) << \ OPT_DRAMADDR_HI_LSHIFT_ADDR) #define OPT_DRAMADDR_LO(reg) \ (((u_longlong_t)reg & OPT_DRAMADDR_LO_MASK_ADDR) << \ OPT_DRAMADDR_LO_LSHIFT_ADDR) #define OPT_DRAMADDR(high, low) \ (OPT_DRAMADDR_HI(high) | OPT_DRAMADDR_LO(low)) /* * Bit masks defining what's in Opteron DRAM Address Map base register */ #define OPT_DRAMBASE_LO_MASK_RE 0x1 /* read enable */ #define OPT_DRAMBASE_LO_MASK_WE 0x2 /* write enable */ #define OPT_DRAMBASE_LO_MASK_INTRLVEN 0x700 /* interleave */ /* * Bit masks defining what's in Opteron DRAM Address Map limit register */ #define OPT_DRAMLIMIT_LO_MASK_DSTNODE 0x7 /* destination node */ #define OPT_DRAMLIMIT_LO_MASK_INTRLVSEL 0x700 /* interleave select */ /* * Opteron Node ID register in PCI configuration space contains * number of nodes in system, etc. for Opteron K8. The following * constants and macros define its contents, structure, and access. */ /* * Bit masks defining what's in Opteron Node ID register */ #define OPT_NODE_MASK_ID 0x7 /* node ID */ #define OPT_NODE_MASK_CNT 0x70 /* node count */ #define OPT_NODE_MASK_IONODE 0x700 /* Hypertransport I/O hub node ID */ #define OPT_NODE_MASK_LCKNODE 0x7000 /* lock controller node ID */ #define OPT_NODE_MASK_CPUCNT 0xF0000 /* CPUs in system (0 means 1 CPU) */ /* * How many bits in Opteron Node ID register to shift right to get actual value */ #define OPT_NODE_RSHIFT_CNT 0x4 /* shift right for node count value */ /* * Macros to get values from Opteron Node ID register */ #define OPT_NODE_CNT(reg) \ ((reg & OPT_NODE_MASK_CNT) >> OPT_NODE_RSHIFT_CNT) /* * Macro to setup PCI Extended Configuration Space (ECS) address to give to * "in/out" instructions * * NOTE: Should only be used in lgrp_plat_init() before MMIO setup because any * other uses should just do MMIO to access PCI ECS. * Must enable special bit in Northbridge Configuration Register on * Greyhound for extended CF8 space access to be able to access PCI ECS * using "in/out" instructions and restore special bit after done * accessing PCI ECS. */ #define OPT_PCI_ECS_ADDR(bus, device, function, reg) \ (PCI_CONE | (((bus) & 0xff) << 16) | (((device & 0x1f)) << 11) | \ (((function) & 0x7) << 8) | ((reg) & 0xfc) | \ ((((reg) >> 8) & 0xf) << 24)) /* * PCI configuration space registers accessed by specifying * a bus, device, function, and offset. The following constants * define the values needed to access Opteron K8 configuration * info to determine its node topology */ #define OPT_PCS_BUS_CONFIG 0 /* Hypertransport config space bus */ /* * Opteron PCI configuration space register function values */ #define OPT_PCS_FUNC_HT 0 /* Hypertransport configuration */ #define OPT_PCS_FUNC_ADDRMAP 1 /* Address map configuration */ #define OPT_PCS_FUNC_DRAM 2 /* DRAM configuration */ #define OPT_PCS_FUNC_MISC 3 /* Miscellaneous configuration */ /* * PCI Configuration Space register offsets */ #define OPT_PCS_OFF_VENDOR 0x0 /* device/vendor ID register */ #define OPT_PCS_OFF_DRAMBASE_HI 0x140 /* DRAM Base register (node 0) */ #define OPT_PCS_OFF_DRAMBASE_LO 0x40 /* DRAM Base register (node 0) */ #define OPT_PCS_OFF_NODEID 0x60 /* Node ID register */ /* * Opteron PCI Configuration Space device IDs for nodes */ #define OPT_PCS_DEV_NODE0 24 /* device number for node 0 */ /* * Opteron DRAM address map gives base and limit for physical memory in a node */ typedef struct opt_dram_addr_map { uint32_t base_hi; uint32_t base_lo; uint32_t limit_hi; uint32_t limit_lo; } opt_dram_addr_map_t; /* * Supported AMD processor families */ #define AMD_FAMILY_HAMMER 15 #define AMD_FAMILY_GREYHOUND 16 /* * Whether to have is_opteron() return 1 even when processor isn't supported */ uint_t is_opteron_override = 0; /* * AMD processor family for current CPU */ uint_t opt_family = 0; /* * Determine whether we're running on a supported AMD Opteron since reading * node count and DRAM address map registers may have different format or * may not be supported across processor families */ static int is_opteron(void) { if (x86_vendor != X86_VENDOR_AMD) return (0); opt_family = cpuid_getfamily(CPU); if (opt_family == AMD_FAMILY_HAMMER || opt_family == AMD_FAMILY_GREYHOUND || is_opteron_override) return (1); else return (0); } /* * Determine NUMA configuration for Opteron from registers that live in PCI * configuration space */ static void opt_get_numa_config(uint_t *node_cnt, int *mem_intrlv, memnode_phys_addr_map_t *memnode_info) { uint_t bus; uint_t dev; struct opt_dram_addr_map dram_map[MAX_NODES]; uint_t node; uint_t node_info[MAX_NODES]; uint_t off_hi; uint_t off_lo; uint64_t nb_cfg_reg; /* * Read configuration registers from PCI configuration space to * determine node information, which memory is in each node, etc. * * Write to PCI configuration space address register to specify * which configuration register to read and read/write PCI * configuration space data register to get/set contents */ bus = OPT_PCS_BUS_CONFIG; dev = OPT_PCS_DEV_NODE0; off_hi = OPT_PCS_OFF_DRAMBASE_HI; off_lo = OPT_PCS_OFF_DRAMBASE_LO; /* * Read node ID register for node 0 to get node count */ node_info[0] = pci_getl_func(bus, dev, OPT_PCS_FUNC_HT, OPT_PCS_OFF_NODEID); *node_cnt = OPT_NODE_CNT(node_info[0]) + 1; /* * If number of nodes is more than maximum supported, then set node * count to 1 and treat system as UMA instead of NUMA. */ if (*node_cnt > MAX_NODES) { *node_cnt = 1; return; } /* * For Greyhound, PCI Extended Configuration Space must be enabled to * read high DRAM address map base and limit registers */ nb_cfg_reg = 0; if (opt_family == AMD_FAMILY_GREYHOUND) { nb_cfg_reg = rdmsr(MSR_AMD_NB_CFG); if ((nb_cfg_reg & AMD_GH_NB_CFG_EN_ECS) == 0) wrmsr(MSR_AMD_NB_CFG, nb_cfg_reg | AMD_GH_NB_CFG_EN_ECS); } for (node = 0; node < *node_cnt; node++) { uint32_t base_hi; uint32_t base_lo; uint32_t limit_hi; uint32_t limit_lo; /* * Read node ID register (except for node 0 which we just read) */ if (node > 0) { node_info[node] = pci_getl_func(bus, dev, OPT_PCS_FUNC_HT, OPT_PCS_OFF_NODEID); } /* * Read DRAM base and limit registers which specify * physical memory range of each node */ if (opt_family != AMD_FAMILY_GREYHOUND) base_hi = 0; else { outl(PCI_CONFADD, OPT_PCI_ECS_ADDR(bus, dev, OPT_PCS_FUNC_ADDRMAP, off_hi)); base_hi = dram_map[node].base_hi = inl(PCI_CONFDATA); } base_lo = dram_map[node].base_lo = pci_getl_func(bus, dev, OPT_PCS_FUNC_ADDRMAP, off_lo); if ((dram_map[node].base_lo & OPT_DRAMBASE_LO_MASK_INTRLVEN) && mem_intrlv) *mem_intrlv = *mem_intrlv + 1; off_hi += 4; /* high limit register offset */ if (opt_family != AMD_FAMILY_GREYHOUND) limit_hi = 0; else { outl(PCI_CONFADD, OPT_PCI_ECS_ADDR(bus, dev, OPT_PCS_FUNC_ADDRMAP, off_hi)); limit_hi = dram_map[node].limit_hi = inl(PCI_CONFDATA); } off_lo += 4; /* low limit register offset */ limit_lo = dram_map[node].limit_lo = pci_getl_func(bus, dev, OPT_PCS_FUNC_ADDRMAP, off_lo); /* * Increment device number to next node and register offsets * for DRAM base register of next node */ off_hi += 4; off_lo += 4; dev++; /* * Both read and write enable bits must be enabled in DRAM * address map base register for physical memory to exist in * node */ if ((base_lo & OPT_DRAMBASE_LO_MASK_RE) == 0 || (base_lo & OPT_DRAMBASE_LO_MASK_WE) == 0) { /* * Mark node memory as non-existent and set start and * end addresses to be same in memnode_info[] */ memnode_info[node].exists = 0; memnode_info[node].start = memnode_info[node].end = (pfn_t)-1; continue; } /* * Mark node memory as existing and remember physical address * range of each node for use later */ memnode_info[node].exists = 1; memnode_info[node].start = btop(OPT_DRAMADDR(base_hi, base_lo)); memnode_info[node].end = btop(OPT_DRAMADDR(limit_hi, limit_lo) | OPT_DRAMADDR_LO_MASK_OFF); } /* * Restore PCI Extended Configuration Space enable bit */ if (opt_family == AMD_FAMILY_GREYHOUND) { if ((nb_cfg_reg & AMD_GH_NB_CFG_EN_ECS) == 0) wrmsr(MSR_AMD_NB_CFG, nb_cfg_reg); } } /* * Return average amount of time to read vendor ID register on Northbridge * N times on specified destination node from current CPU */ static hrtime_t opt_probe_vendor(int dest_node, int nreads) { int cnt; uint_t dev; /* LINTED: set but not used in function */ volatile uint_t dev_vendor __unused; hrtime_t elapsed; hrtime_t end; int ipl; hrtime_t start; dev = OPT_PCS_DEV_NODE0 + dest_node; kpreempt_disable(); ipl = spl8(); outl(PCI_CONFADD, PCI_CADDR1(0, dev, OPT_PCS_FUNC_DRAM, OPT_PCS_OFF_VENDOR)); start = gethrtime(); for (cnt = 0; cnt < nreads; cnt++) dev_vendor = inl(PCI_CONFDATA); end = gethrtime(); elapsed = (end - start) / nreads; splx(ipl); kpreempt_enable(); return (elapsed); } /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2007 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. * * Copyright 2018 Joyent, Inc. */ /* * Kernel/Debugger Interface (KDI) routines. Called during debugger under * various system states (boot, while running, while the debugger has control). * Functions intended for use while the debugger has control may not grab any * locks or perform any functions that assume the availability of other system * services. */ #include #include #include #include #include #include #include #include #include #include #include #include void kdi_idt_write(gate_desc_t *gate, uint_t vec) { gate_desc_t *idt = CPU->cpu_m.mcpu_idt; /* * See kdi_idtr_set(). */ if (idt == NULL) { desctbr_t idtr; rd_idtr(&idtr); idt = (gate_desc_t *)idtr.dtr_base; } idt[vec] = *gate; } ulong_t kdi_dreg_get(int reg) { switch (reg) { case 0: return (kdi_getdr0()); case 1: return (kdi_getdr1()); case 2: return (kdi_getdr2()); case 3: return (kdi_getdr3()); case 6: return (kdi_getdr6()); case 7: return (kdi_getdr7()); default: panic("invalid debug register dr%d", reg); /*NOTREACHED*/ } } void kdi_dreg_set(int reg, ulong_t value) { switch (reg) { case 0: kdi_setdr0(value); break; case 1: kdi_setdr1(value); break; case 2: kdi_setdr2(value); break; case 3: kdi_setdr3(value); break; case 6: kdi_setdr6(value); break; case 7: kdi_setdr7(value); break; default: panic("invalid debug register dr%d", reg); /*NOTREACHED*/ } } extern void kdi_slave_entry(void); void kdi_stop_slaves(int cpu, int doxc) { if (doxc) kdi_xc_others(cpu, kdi_slave_entry); } /* * On i86pc, slaves busy-loop, so we don't need to do anything here. */ void kdi_start_slaves(void) { } void kdi_slave_wait(void) { } /* * Caution. * These routines are called -extremely- early, during kmdb initialization. * * Many common kernel functions assume that %gs has been initialized, * and fail horribly if it hasn't. At this point, the boot code has * reserved a descriptor for us (KMDBGS_SEL) in it's GDT; arrange for it * to point at a dummy cpu_t, temporarily at least. * * Note that kmdb entry relies on the fake cpu_t having zero cpu_idt/cpu_id. */ void * boot_kdi_tmpinit(void) { cpu_t *cpu = kobj_zalloc(sizeof (*cpu), KM_TMP); uintptr_t old; cpu->cpu_self = cpu; old = (uintptr_t)rdmsr(MSR_AMD_GSBASE); wrmsr(MSR_AMD_GSBASE, (uint64_t)cpu); return ((void *)old); } void boot_kdi_tmpfini(void *old) { wrmsr(MSR_AMD_GSBASE, (uint64_t)old); } /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License, Version 1.0 only * (the "License"). You may not use this file except in compliance * with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */ /* All Rights Reserved */ #include #include #include #include #include /*ARGSUSED*/ int mach_sysconfig(int which) { return (set_errno(EINVAL)); } /* * Indicate to the platforms that the utsname.nodename was set. */ void nodename_set(void) { } /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved. * Copyright 2020 Joyent, Inc. * Copyright 2025 Oxide Computer Company */ /* * Copyright (c) 2010, Intel Corporation. * All rights reserved. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #if defined(__xpv) #include #include #endif #include #include #include #include #ifdef TRAPTRACE #include #endif /* TRAPTRACE */ #include #include extern void audit_enterprom(int); extern void audit_exitprom(int); /* * Tunable to enable apix PSM; if set to 0, pcplusmp PSM will be used. */ int apix_enable = 1; int apic_nvidia_io_max = 0; /* no. of NVIDIA i/o apics */ /* * Occassionally the kernel knows better whether to power-off or reboot. */ int force_shutdown_method = AD_UNKNOWN; /* * The panicbuf array is used to record messages and state: */ char panicbuf[PANICBUFSIZE]; /* * Flags to control Dynamic Reconfiguration features. */ uint64_t plat_dr_options; /* * Maximum physical address for memory DR operations. */ uint64_t plat_dr_physmax; /* * maxphys - used during physio * klustsize - used for klustering by swapfs and specfs */ int maxphys = 56 * 1024; /* XXX See vm_subr.c - max b_count in physio */ int klustsize = 56 * 1024; caddr_t p0_va; /* Virtual address for accessing physical page 0 */ /* * defined here, though unused on x86, * to make kstat_fr.c happy. */ int vac; void debug_enter(char *); extern void pm_cfb_check_and_powerup(void); extern void pm_cfb_rele(void); extern fastboot_info_t newkernel; /* * Instructions to enable or disable SMAP, respectively. */ static const uint8_t clac_instr[3] = { 0x0f, 0x01, 0xca }; static const uint8_t stac_instr[3] = { 0x0f, 0x01, 0xcb }; /* * Machine dependent code to reboot. * "mdep" is interpreted as a character pointer; if non-null, it is a pointer * to a string to be used as the argument string when rebooting. * * "invoke_cb" is a boolean. It is set to true when mdboot() can safely * invoke CB_CL_MDBOOT callbacks before shutting the system down, i.e. when * we are in a normal shutdown sequence (interrupts are not blocked, the * system is not panic'ing or being suspended). */ /*ARGSUSED*/ void mdboot(int cmd, int fcn, char *mdep, boolean_t invoke_cb) { processorid_t bootcpuid = 0; static int is_first_quiesce = 1; static int is_first_reset = 1; int reset_status = 0; static char fallback_str[] = "Falling back to regular reboot.\n"; if (fcn == AD_FASTREBOOT && !newkernel.fi_valid) fcn = AD_BOOT; if (!panicstr) { kpreempt_disable(); if (fcn == AD_FASTREBOOT) { mutex_enter(&cpu_lock); if (CPU_ACTIVE(cpu_get(bootcpuid))) { affinity_set(bootcpuid); } mutex_exit(&cpu_lock); } else { affinity_set(CPU_CURRENT); } } if (force_shutdown_method != AD_UNKNOWN) fcn = force_shutdown_method; /* * XXX - rconsvp is set to NULL to ensure that output messages * are sent to the underlying "hardware" device using the * monitor's printf routine since we are in the process of * either rebooting or halting the machine. */ rconsvp = NULL; /* * Print the reboot message now, before pausing other cpus. * There is a race condition in the printing support that * can deadlock multiprocessor machines. */ if (!(fcn == AD_HALT || fcn == AD_POWEROFF)) prom_printf("rebooting...\n"); if (IN_XPV_PANIC()) reset(); /* * We can't bring up the console from above lock level, so do it now */ pm_cfb_check_and_powerup(); /* make sure there are no more changes to the device tree */ devtree_freeze(); if (invoke_cb) (void) callb_execute_class(CB_CL_MDBOOT, 0); /* * Clear any unresolved UEs from memory. */ page_retire_mdboot(); #if defined(__xpv) /* * XXPV Should probably think some more about how we deal * with panicing before it's really safe to panic. * On hypervisors, we reboot very quickly.. Perhaps panic * should only attempt to recover by rebooting if, * say, we were able to mount the root filesystem, * or if we successfully launched init(8). */ if (panicstr && proc_init == NULL) (void) HYPERVISOR_shutdown(SHUTDOWN_poweroff); #endif /* * stop other cpus and raise our priority. since there is only * one active cpu after this, and our priority will be too high * for us to be preempted, we're essentially single threaded * from here on out. */ (void) spl6(); if (!panicstr) { mutex_enter(&cpu_lock); pause_cpus(NULL, NULL); mutex_exit(&cpu_lock); } /* * If the system is panicking, the preloaded kernel is valid, and * fastreboot_onpanic has been set, and the system has been up for * longer than fastreboot_onpanic_uptime (default to 10 minutes), * choose Fast Reboot. */ if (fcn == AD_BOOT && panicstr && newkernel.fi_valid && fastreboot_onpanic && (panic_lbolt - lbolt_at_boot) > fastreboot_onpanic_uptime) { fcn = AD_FASTREBOOT; } /* * Try to quiesce devices. */ if (is_first_quiesce) { /* * Clear is_first_quiesce before calling quiesce_devices() * so that if quiesce_devices() causes panics, it will not * be invoked again. */ is_first_quiesce = 0; quiesce_active = 1; quiesce_devices(ddi_root_node(), &reset_status); if (reset_status == -1) { if (fcn == AD_FASTREBOOT && !force_fastreboot) { prom_printf("Driver(s) not capable of fast " "reboot.\n"); prom_printf(fallback_str); fastreboot_capable = 0; fcn = AD_BOOT; } else if (fcn != AD_FASTREBOOT) fastreboot_capable = 0; } quiesce_active = 0; } /* * Try to reset devices. reset_leaves() should only be called * a) when there are no other threads that could be accessing devices, * and * b) on a system that's not capable of fast reboot (fastreboot_capable * being 0), or on a system where quiesce_devices() failed to * complete (quiesce_active being 1). */ if (is_first_reset && (!fastreboot_capable || quiesce_active)) { /* * Clear is_first_reset before calling reset_devices() * so that if reset_devices() causes panics, it will not * be invoked again. */ is_first_reset = 0; reset_leaves(); } /* Verify newkernel checksum */ if (fastreboot_capable && fcn == AD_FASTREBOOT && fastboot_cksum_verify(&newkernel) != 0) { fastreboot_capable = 0; prom_printf("Fast reboot: checksum failed for the new " "kernel.\n"); prom_printf(fallback_str); } (void) spl8(); if (fastreboot_capable && fcn == AD_FASTREBOOT) { /* * psm_shutdown is called within fast_reboot() */ fast_reboot(); } else { (*psm_shutdownf)(cmd, fcn); if (fcn == AD_HALT || fcn == AD_POWEROFF) halt((char *)NULL); else prom_reboot(""); } /*NOTREACHED*/ } /* mdpreboot - may be called prior to mdboot while root fs still mounted */ /*ARGSUSED*/ void mdpreboot(int cmd, int fcn, char *mdep) { if (fcn == AD_FASTREBOOT && !fastreboot_capable) { fcn = AD_BOOT; #ifdef __xpv cmn_err(CE_WARN, "Fast reboot is not supported on xVM"); #else cmn_err(CE_WARN, "Fast reboot is not supported on this platform%s", fastreboot_nosup_message()); #endif } if (fcn == AD_FASTREBOOT) { fastboot_load_kernel(mdep); if (!newkernel.fi_valid) fcn = AD_BOOT; } (*psm_preshutdownf)(cmd, fcn); } static void stop_other_cpus(void) { ulong_t s = clear_int_flag(); /* fast way to keep CPU from changing */ cpuset_t xcset; CPUSET_ALL_BUT(xcset, CPU->cpu_id); xc_priority(0, 0, 0, CPUSET2BV(xcset), mach_cpu_halt); restore_int_flag(s); } /* * Machine dependent abort sequence handling */ void abort_sequence_enter(char *msg) { if (abort_enable == 0) { if (AU_ZONE_AUDITING(GET_KCTX_GZ)) audit_enterprom(0); return; } if (AU_ZONE_AUDITING(GET_KCTX_GZ)) audit_enterprom(1); debug_enter(msg); if (AU_ZONE_AUDITING(GET_KCTX_GZ)) audit_exitprom(1); } /* * Enter debugger. Called when the user types ctrl-alt-d or whenever * code wants to enter the debugger and possibly resume later. * * msg: message to print, possibly NULL. */ void debug_enter(char *msg) { if (dtrace_debugger_init != NULL) (*dtrace_debugger_init)(); if (msg != NULL || (boothowto & RB_DEBUG)) prom_printf("\n"); if (msg != NULL) prom_printf("%s\n", msg); if (boothowto & RB_DEBUG) kmdb_enter(); if (dtrace_debugger_fini != NULL) (*dtrace_debugger_fini)(); } void reset(void) { extern void acpi_reset_system(); #if !defined(__xpv) ushort_t *bios_memchk; /* * Can't use psm_map_phys or acpi_reset_system before the hat is * initialized. */ if (khat_running) { bios_memchk = (ushort_t *)psm_map_phys(0x472, sizeof (ushort_t), PROT_READ | PROT_WRITE); if (bios_memchk) *bios_memchk = 0x1234; /* bios memory check disable */ if (options_dip != NULL && ddi_prop_exists(DDI_DEV_T_ANY, ddi_root_node(), 0, "efi-systab")) { if (bootops == NULL) acpi_reset_system(); efi_reset(); } /* * The problem with using stubs is that we can call * acpi_reset_system only after the kernel is up and running. * * We should create a global state to keep track of how far * up the kernel is but for the time being we will depend on * bootops. bootops cleared in startup_end(). */ if (bootops == NULL) acpi_reset_system(); } pc_reset(); #else if (IN_XPV_PANIC()) { if (khat_running && bootops == NULL) { acpi_reset_system(); } pc_reset(); } (void) HYPERVISOR_shutdown(SHUTDOWN_reboot); panic("HYPERVISOR_shutdown() failed"); #endif /*NOTREACHED*/ } /* * Halt the machine and return to the monitor */ void halt(char *s) { stop_other_cpus(); /* send stop signal to other CPUs */ if (s) prom_printf("(%s) \n", s); prom_exit_to_mon(); /*NOTREACHED*/ } /* * Initiate interrupt redistribution. */ void i_ddi_intr_redist_all_cpus() { } /* * XXX These probably ought to live somewhere else * XXX They are called from mem.c */ /* * Convert page frame number to an OBMEM page frame number * (i.e. put in the type bits -- zero for this implementation) */ pfn_t impl_obmem_pfnum(pfn_t pf) { return (pf); } #ifdef NM_DEBUG int nmi_test = 0; /* checked in intentry.s during clock int */ int nmtest = -1; nmfunc1(int arg, struct regs *rp) { printf("nmi called with arg = %x, regs = %x\n", arg, rp); nmtest += 50; if (arg == nmtest) { printf("ip = %x\n", rp->r_pc); return (1); } return (0); } #endif #include /* Hacked up initialization for initial kernel check out is HERE. */ /* The basic steps are: */ /* kernel bootfuncs definition/initialization for KADB */ /* kadb bootfuncs pointer initialization */ /* putchar/getchar (interrupts disabled) */ /* kadb bootfuncs pointer initialization */ int sysp_getchar() { int i; ulong_t s; if (cons_polledio == NULL) { /* Uh oh */ prom_printf("getchar called with no console\n"); for (;;) /* LOOP FOREVER */; } s = clear_int_flag(); i = cons_polledio->cons_polledio_getchar( cons_polledio->cons_polledio_argument); restore_int_flag(s); return (i); } void sysp_putchar(int c) { ulong_t s; /* * We have no alternative but to drop the output on the floor. */ if (cons_polledio == NULL || cons_polledio->cons_polledio_putchar == NULL) return; s = clear_int_flag(); cons_polledio->cons_polledio_putchar( cons_polledio->cons_polledio_argument, c); restore_int_flag(s); } int sysp_ischar() { int i; ulong_t s; if (cons_polledio == NULL || cons_polledio->cons_polledio_ischar == NULL) return (0); s = clear_int_flag(); i = cons_polledio->cons_polledio_ischar( cons_polledio->cons_polledio_argument); restore_int_flag(s); return (i); } int goany(void) { prom_printf("Type any key to continue "); (void) prom_getchar(); prom_printf("\n"); return (1); } static struct boot_syscalls kern_sysp = { sysp_getchar, /* unchar (*getchar)(); 7 */ sysp_putchar, /* int (*putchar)(); 8 */ sysp_ischar, /* int (*ischar)(); 9 */ }; #if defined(__xpv) int using_kern_polledio; #endif /* * Switch the prom_* layer to using kernel routines for I/O after the system * is sufficiently booted */ void prom_io_use_kernel() { sysp = &kern_sysp; #if defined(__xpv) using_kern_polledio = 1; #endif } /* * the interface to the outside world */ /* * poll_port -- wait for a register to achieve a * specific state. Arguments are a mask of bits we care about, * and two sub-masks. To return normally, all the bits in the * first sub-mask must be ON, all the bits in the second sub- * mask must be OFF. If about seconds pass without the register * achieving the desired bit configuration, we return 1, else * 0. */ int poll_port(ushort_t port, ushort_t mask, ushort_t onbits, ushort_t offbits) { int i; ushort_t maskval; for (i = 500000; i; i--) { maskval = inb(port) & mask; if (((maskval & onbits) == onbits) && ((maskval & offbits) == 0)) return (0); drv_usecwait(10); } return (1); } /* * set_idle_cpu is called from idle() when a CPU becomes idle. */ /*LINTED: static unused */ static uint_t last_idle_cpu; /*ARGSUSED*/ void set_idle_cpu(int cpun) { last_idle_cpu = cpun; (*psm_set_idle_cpuf)(cpun); } /* * unset_idle_cpu is called from idle() when a CPU is no longer idle. */ /*ARGSUSED*/ void unset_idle_cpu(int cpun) { (*psm_unset_idle_cpuf)(cpun); } /* * This routine is almost correct now, but not quite. It still needs the * equivalent concept of "hres_last_tick", just like on the sparc side. * The idea is to take a snapshot of the hi-res timer while doing the * hrestime_adj updates under hres_lock in locore, so that the small * interval between interrupt assertion and interrupt processing is * accounted for correctly. Once we have this, the code below should * be modified to subtract off hres_last_tick rather than hrtime_base. * * I'd have done this myself, but I don't have source to all of the * vendor-specific hi-res timer routines (grrr...). The generic hook I * need is something like "gethrtime_unlocked()", which would be just like * gethrtime() but would assume that you're already holding CLOCK_LOCK(). * This is what the GET_HRTIME() macro is for on sparc (although it also * serves the function of making time available without a function call * so you don't take a register window overflow while traps are disabled). */ void pc_gethrestime(timestruc_t *tp) { int lock_prev; timestruc_t now; int nslt; /* nsec since last tick */ int adj; /* amount of adjustment to apply */ loop: lock_prev = hres_lock; now = hrestime; nslt = (int)(gethrtime() - hres_last_tick); if (nslt < 0) { /* * nslt < 0 means a tick came between sampling * gethrtime() and hres_last_tick; restart the loop */ goto loop; } now.tv_nsec += nslt; if (hrestime_adj != 0) { if (hrestime_adj > 0) { adj = (nslt >> ADJ_SHIFT); if (adj > hrestime_adj) adj = (int)hrestime_adj; } else { adj = -(nslt >> ADJ_SHIFT); if (adj < hrestime_adj) adj = (int)hrestime_adj; } now.tv_nsec += adj; } while ((unsigned long)now.tv_nsec >= NANOSEC) { /* * We might have a large adjustment or have been in the * debugger for a long time; take care of (at most) four * of those missed seconds (tv_nsec is 32 bits, so * anything >4s will be wrapping around). However, * anything more than 2 seconds out of sync will trigger * timedelta from clock() to go correct the time anyway, * so do what we can, and let the big crowbar do the * rest. A similar correction while loop exists inside * hres_tick(); in all cases we'd like tv_nsec to * satisfy 0 <= tv_nsec < NANOSEC to avoid confusing * user processes, but if tv_sec's a little behind for a * little while, that's OK; time still monotonically * increases. */ now.tv_nsec -= NANOSEC; now.tv_sec++; } if ((hres_lock & ~1) != lock_prev) goto loop; *tp = now; } void gethrestime_lasttick(timespec_t *tp) { int s; s = hr_clock_lock(); *tp = hrestime; hr_clock_unlock(s); } time_t gethrestime_sec(void) { timestruc_t now; gethrestime(&now); return (now.tv_sec); } /* * Initialize a kernel thread's stack */ caddr_t thread_stk_init(caddr_t stk) { ASSERT(((uintptr_t)stk & (STACK_ALIGN - 1)) == 0); return (stk - SA(MINFRAME)); } /* * Initialize lwp's kernel stack. */ #ifdef TRAPTRACE /* * There's a tricky interdependency here between use of sysenter and * TRAPTRACE which needs recording to avoid future confusion (this is * about the third time I've re-figured this out ..) * * Here's how debugging lcall works with TRAPTRACE. * * 1 We're in userland with a breakpoint on the lcall instruction. * 2 We execute the instruction - the instruction pushes the userland * %ss, %esp, %efl, %cs, %eip on the stack and zips into the kernel * via the call gate. * 3 The hardware raises a debug trap in kernel mode, the hardware * pushes %efl, %cs, %eip and gets to dbgtrap via the idt. * 4 dbgtrap pushes the error code and trapno and calls cmntrap * 5 cmntrap finishes building a trap frame * 6 The TRACE_REGS macros in cmntrap copy a REGSIZE worth chunk * off the stack into the traptrace buffer. * * This means that the traptrace buffer contains the wrong values in * %esp and %ss, but everything else in there is correct. * * Here's how debugging sysenter works with TRAPTRACE. * * a We're in userland with a breakpoint on the sysenter instruction. * b We execute the instruction - the instruction pushes -nothing- * on the stack, but sets %cs, %eip, %ss, %esp to prearranged * values to take us to sys_sysenter, at the top of the lwp's * stack. * c goto 3 * * At this point, because we got into the kernel without the requisite * five pushes on the stack, if we didn't make extra room, we'd * end up with the TRACE_REGS macro fetching the saved %ss and %esp * values from negative (unmapped) stack addresses -- which really bites. * That's why we do the '-= 8' below. * * XXX Note that reading "up" lwp0's stack works because t0 is declared * right next to t0stack in locore.s */ #endif caddr_t lwp_stk_init(klwp_t *lwp, caddr_t stk) { caddr_t oldstk; struct pcb *pcb = &lwp->lwp_pcb; oldstk = stk; stk -= SA(sizeof (struct regs) + SA(MINFRAME)); #ifdef TRAPTRACE stk -= 2 * sizeof (greg_t); /* space for phony %ss:%sp (see above) */ #endif stk = (caddr_t)((uintptr_t)stk & ~(STACK_ALIGN - 1ul)); bzero(stk, oldstk - stk); lwp->lwp_regs = (void *)(stk + SA(MINFRAME)); /* * Arrange that the virtualized %fs and %gs GDT descriptors * have a well-defined initial state (present, ring 3 * and of type data). */ if (lwp_getdatamodel(lwp) == DATAMODEL_NATIVE) pcb->pcb_fsdesc = pcb->pcb_gsdesc = zero_udesc; else pcb->pcb_fsdesc = pcb->pcb_gsdesc = zero_u32desc; lwp_installctx(lwp); return (stk); } /* * Use this opportunity to free any dynamically allocated fp storage. */ void lwp_stk_fini(klwp_t *lwp) { fp_lwp_cleanup(lwp); } void lwp_fp_init(klwp_t *lwp) { fp_lwp_init(lwp); } /* * If we're not the panic CPU, we wait in panic_idle for reboot. */ void panic_idle(void) { splx(ipltospl(CLOCK_LEVEL)); (void) setjmp(&curthread->t_pcb); dumpsys_helper(); #ifndef __xpv for (;;) i86_halt(); #else for (;;) ; #endif } /* * Stop the other CPUs by cross-calling them and forcing them to enter * the panic_idle() loop above. */ /*ARGSUSED*/ void panic_stopcpus(cpu_t *cp, kthread_t *t, int spl) { processorid_t i; cpuset_t xcset; /* * In the case of a Xen panic, the hypervisor has already stopped * all of the CPUs. */ if (!IN_XPV_PANIC()) { (void) splzs(); CPUSET_ALL_BUT(xcset, cp->cpu_id); xc_priority(0, 0, 0, CPUSET2BV(xcset), (xc_func_t)panic_idle); } for (i = 0; i < NCPU; i++) { if (i != cp->cpu_id && cpu[i] != NULL && (cpu[i]->cpu_flags & CPU_EXISTS)) cpu[i]->cpu_flags |= CPU_QUIESCED; } } /* * Platform callback following each entry to panicsys(). */ /*ARGSUSED*/ void panic_enter_hw(int spl) { /* Nothing to do here */ } /* * Platform-specific code to execute after panicstr is set: we invoke * the PSM entry point to indicate that a panic has occurred. */ /*ARGSUSED*/ void panic_quiesce_hw(panic_data_t *pdp) { psm_notifyf(PSM_PANIC_ENTER); cmi_panic_callback(); #ifdef TRAPTRACE /* * Turn off TRAPTRACE */ TRAPTRACE_FREEZE; #endif /* TRAPTRACE */ } /* * Platform callback prior to writing crash dump. */ /*ARGSUSED*/ void panic_dump_hw(int spl) { /* Nothing to do here */ } void * plat_traceback(void *fpreg) { #ifdef __xpv if (IN_XPV_PANIC()) return (xpv_traceback(fpreg)); #endif return (fpreg); } /*ARGSUSED*/ void plat_tod_fault(enum tod_fault_type tod_bad) {} /*ARGSUSED*/ int blacklist(int cmd, const char *scheme, nvlist_t *fmri, const char *class) { return (ENOTSUP); } /* * The underlying console output routines are protected by raising IPL in case * we are still calling into the early boot services. Once we start calling * the kernel console emulator, it will disable interrupts completely during * character rendering (see sysp_putchar, for example). Refer to the comments * and code in common/os/console.c for more information on these callbacks. */ /*ARGSUSED*/ int console_enter(int busy) { return (splzs()); } /*ARGSUSED*/ void console_exit(int busy, int spl) { splx(spl); } /* * Allocate a region of virtual address space, unmapped. * Stubbed out except on sparc, at least for now. */ /*ARGSUSED*/ void * boot_virt_alloc(void *addr, size_t size) { return (addr); } void tenmicrosec(void) { extern int gethrtime_hires; if (gethrtime_hires) { hrtime_t start, end; start = end = gethrtime(); while ((end - start) < (10 * (NANOSEC / MICROSEC))) { SMT_PAUSE(); end = gethrtime(); } } else { #if defined(__xpv) hrtime_t newtime; newtime = xpv_gethrtime() + 10000; /* now + 10 us */ while (xpv_gethrtime() < newtime) SMT_PAUSE(); #else /* __xpv */ panic("TSC was not calibrated!"); #endif /* __xpv */ } } /* * get_cpu_mstate() is passed an array of timestamps, NCMSTATES * long, and it fills in the array with the time spent on cpu in * each of the mstates, where time is returned in nsec. * * No guarantee is made that the returned values in times[] will * monotonically increase on sequential calls, although this will * be true in the long run. Any such guarantee must be handled by * the caller, if needed. This can happen if we fail to account * for elapsed time due to a generation counter conflict, yet we * did account for it on a prior call (see below). * * The complication is that the cpu in question may be updating * its microstate at the same time that we are reading it. * Because the microstate is only updated when the CPU's state * changes, the values in cpu_intracct[] can be indefinitely out * of date. To determine true current values, it is necessary to * compare the current time with cpu_mstate_start, and add the * difference to times[cpu_mstate]. * * This can be a problem if those values are changing out from * under us. Because the code path in new_cpu_mstate() is * performance critical, we have not added a lock to it. Instead, * we have added a generation counter. Before beginning * modifications, the counter is set to 0. After modifications, * it is set to the old value plus one. * * get_cpu_mstate() will not consider the values of cpu_mstate * and cpu_mstate_start to be usable unless the value of * cpu_mstate_gen is both non-zero and unchanged, both before and * after reading the mstate information. Note that we must * protect against out-of-order loads around accesses to the * generation counter. Also, this is a best effort approach in * that we do not retry should the counter be found to have * changed. * * cpu_intracct[] is used to identify time spent in each CPU * mstate while handling interrupts. Such time should be reported * against system time, and so is subtracted out from its * corresponding cpu_acct[] time and added to * cpu_acct[CMS_SYSTEM]. */ void get_cpu_mstate(cpu_t *cpu, hrtime_t *times) { int i; hrtime_t now, start; uint16_t gen; uint16_t state; hrtime_t intracct[NCMSTATES]; /* * Load all volatile state under the protection of membar. * cpu_acct[cpu_mstate] must be loaded to avoid double counting * of (now - cpu_mstate_start) by a change in CPU mstate that * arrives after we make our last check of cpu_mstate_gen. */ now = gethrtime_unscaled(); gen = cpu->cpu_mstate_gen; membar_consumer(); /* guarantee load ordering */ start = cpu->cpu_mstate_start; state = cpu->cpu_mstate; for (i = 0; i < NCMSTATES; i++) { intracct[i] = cpu->cpu_intracct[i]; times[i] = cpu->cpu_acct[i]; } membar_consumer(); /* guarantee load ordering */ if (gen != 0 && gen == cpu->cpu_mstate_gen && now > start) times[state] += now - start; for (i = 0; i < NCMSTATES; i++) { if (i == CMS_SYSTEM) continue; times[i] -= intracct[i]; if (times[i] < 0) { intracct[i] += times[i]; times[i] = 0; } times[CMS_SYSTEM] += intracct[i]; scalehrtime(×[i]); } scalehrtime(×[CMS_SYSTEM]); } /* * This is a version of the rdmsr instruction that allows * an error code to be returned in the case of failure. */ int checked_rdmsr(uint_t msr, uint64_t *value) { if (!is_x86_feature(x86_featureset, X86FSET_MSR)) return (ENOTSUP); *value = rdmsr(msr); return (0); } /* * This is a version of the wrmsr instruction that allows * an error code to be returned in the case of failure. */ int checked_wrmsr(uint_t msr, uint64_t value) { if (!is_x86_feature(x86_featureset, X86FSET_MSR)) return (ENOTSUP); wrmsr(msr, value); return (0); } void wrmsr_and_test(uint_t msr, const uint64_t v) { wrmsr(msr, v); #ifdef DEBUG uint64_t rv = rdmsr(msr); if (rv != v) { cmn_err(CE_PANIC, "MSR 0x%x written with value 0x%lx " "has value 0x%lx\n", msr, v, rv); } #endif } /* * The mem driver's usual method of using hat_devload() to establish a * temporary mapping will not work for foreign pages mapped into this * domain or for the special hypervisor-provided pages. For the foreign * pages, we often don't know which domain owns them, so we can't ask the * hypervisor to set up a new mapping. For the other pages, we don't have * a pfn, so we can't create a new PTE. For these special cases, we do a * direct uiomove() from the existing kernel virtual address. */ /*ARGSUSED*/ int plat_mem_do_mmio(struct uio *uio, enum uio_rw rw) { #if defined(__xpv) void *va = (void *)(uintptr_t)uio->uio_loffset; off_t pageoff = uio->uio_loffset & PAGEOFFSET; size_t nbytes = MIN((size_t)(PAGESIZE - pageoff), (size_t)uio->uio_iov->iov_len); if ((rw == UIO_READ && (va == HYPERVISOR_shared_info || va == xen_info)) || (pfn_is_foreign(hat_getpfnum(kas.a_hat, va)))) return (uiomove(va, nbytes, rw, uio)); #endif return (ENOTSUP); } pgcnt_t num_phys_pages() { pgcnt_t npages = 0; struct memlist *mp; #if defined(__xpv) if (DOMAIN_IS_INITDOMAIN(xen_info)) return (xpv_nr_phys_pages()); #endif /* __xpv */ for (mp = phys_install; mp != NULL; mp = mp->ml_next) npages += mp->ml_size >> PAGESHIFT; return (npages); } /* cpu threshold for compressed dumps */ #ifdef _LP64 uint_t dump_plat_mincpu_default = DUMP_PLAT_X86_64_MINCPU; #else uint_t dump_plat_mincpu_default = DUMP_PLAT_X86_32_MINCPU; #endif int dump_plat_addr() { #ifdef __xpv pfn_t pfn = mmu_btop(xen_info->shared_info) | PFN_IS_FOREIGN_MFN; mem_vtop_t mem_vtop; int cnt; /* * On the hypervisor, we want to dump the page with shared_info on it. */ if (!IN_XPV_PANIC()) { mem_vtop.m_as = &kas; mem_vtop.m_va = HYPERVISOR_shared_info; mem_vtop.m_pfn = pfn; dumpvp_write(&mem_vtop, sizeof (mem_vtop_t)); cnt = 1; } else { cnt = dump_xpv_addr(); } return (cnt); #else return (0); #endif } void dump_plat_pfn() { #ifdef __xpv pfn_t pfn = mmu_btop(xen_info->shared_info) | PFN_IS_FOREIGN_MFN; if (!IN_XPV_PANIC()) dumpvp_write(&pfn, sizeof (pfn)); else dump_xpv_pfn(); #endif } /*ARGSUSED*/ int dump_plat_data(void *dump_cbuf) { #ifdef __xpv uint32_t csize; int cnt; if (!IN_XPV_PANIC()) { csize = (uint32_t)compress(HYPERVISOR_shared_info, dump_cbuf, PAGESIZE); dumpvp_write(&csize, sizeof (uint32_t)); dumpvp_write(dump_cbuf, csize); cnt = 1; } else { cnt = dump_xpv_data(dump_cbuf); } return (cnt); #else return (0); #endif } /* * Calculates a linear address, given the CS selector and PC values * by looking up the %cs selector process's LDT or the CPU's GDT. * proc->p_ldtlock must be held across this call. */ int linear_pc(struct regs *rp, proc_t *p, caddr_t *linearp) { user_desc_t *descrp; caddr_t baseaddr; uint16_t idx = SELTOIDX(rp->r_cs); ASSERT(rp->r_cs <= 0xFFFF); ASSERT(MUTEX_HELD(&p->p_ldtlock)); if (SELISLDT(rp->r_cs)) { /* * Currently 64 bit processes cannot have private LDTs. */ ASSERT(p->p_model != DATAMODEL_LP64); if (p->p_ldt == NULL) return (-1); descrp = &p->p_ldt[idx]; baseaddr = (caddr_t)(uintptr_t)USEGD_GETBASE(descrp); /* * Calculate the linear address (wraparound is not only ok, * it's expected behavior). The cast to uint32_t is because * LDT selectors are only allowed in 32-bit processes. */ *linearp = (caddr_t)(uintptr_t)(uint32_t)((uintptr_t)baseaddr + rp->r_pc); } else { #ifdef DEBUG descrp = &CPU->cpu_gdt[idx]; baseaddr = (caddr_t)(uintptr_t)USEGD_GETBASE(descrp); /* GDT-based descriptors' base addresses should always be 0 */ ASSERT(baseaddr == 0); #endif *linearp = (caddr_t)(uintptr_t)rp->r_pc; } return (0); } /* * The implementation of dtrace_linear_pc is similar to the that of * linear_pc, above, but here we acquire p_ldtlock before accessing * p_ldt. This implementation is used by the pid provider; we prefix * it with "dtrace_" to avoid inducing spurious tracing events. */ int dtrace_linear_pc(struct regs *rp, proc_t *p, caddr_t *linearp) { user_desc_t *descrp; caddr_t baseaddr; uint16_t idx = SELTOIDX(rp->r_cs); ASSERT(rp->r_cs <= 0xFFFF); if (SELISLDT(rp->r_cs)) { /* * Currently 64 bit processes cannot have private LDTs. */ ASSERT(p->p_model != DATAMODEL_LP64); mutex_enter(&p->p_ldtlock); if (p->p_ldt == NULL) { mutex_exit(&p->p_ldtlock); return (-1); } descrp = &p->p_ldt[idx]; baseaddr = (caddr_t)(uintptr_t)USEGD_GETBASE(descrp); mutex_exit(&p->p_ldtlock); /* * Calculate the linear address (wraparound is not only ok, * it's expected behavior). The cast to uint32_t is because * LDT selectors are only allowed in 32-bit processes. */ *linearp = (caddr_t)(uintptr_t)(uint32_t)((uintptr_t)baseaddr + rp->r_pc); } else { #ifdef DEBUG descrp = &CPU->cpu_gdt[idx]; baseaddr = (caddr_t)(uintptr_t)USEGD_GETBASE(descrp); /* GDT-based descriptors' base addresses should always be 0 */ ASSERT(baseaddr == 0); #endif *linearp = (caddr_t)(uintptr_t)rp->r_pc; } return (0); } /* * We need to post a soft interrupt to reprogram the lbolt cyclic when * switching from event to cyclic driven lbolt. The following code adds * and posts the softint for x86. */ static ddi_softint_hdl_impl_t lbolt_softint_hdl = {0, 0, NULL, NULL, 0, NULL, NULL, NULL}; void lbolt_softint_add(void) { (void) add_avsoftintr((void *)&lbolt_softint_hdl, LOCK_LEVEL, (avfunc)lbolt_ev_to_cyclic, "lbolt_ev_to_cyclic", NULL, NULL); } void lbolt_softint_post(void) { (*setsoftint)(CBE_LOCK_PIL, lbolt_softint_hdl.ih_pending); } boolean_t plat_dr_check_capability(uint64_t features) { return ((plat_dr_options & features) == features); } boolean_t plat_dr_support_cpu(void) { return (plat_dr_options & PLAT_DR_FEATURE_CPU); } boolean_t plat_dr_support_memory(void) { return (plat_dr_options & PLAT_DR_FEATURE_MEMORY); } void plat_dr_enable_capability(uint64_t features) { atomic_or_64(&plat_dr_options, features); } void plat_dr_disable_capability(uint64_t features) { atomic_and_64(&plat_dr_options, ~features); } /* * If SMAP is supported, look through hi_calls and inline * calls to smap_enable() to clac and smap_disable() to stac. */ void hotinline_smap(hotinline_desc_t *hid) { if (is_x86_feature(x86_featureset, X86FSET_SMAP) == B_FALSE) return; if (strcmp(hid->hid_symname, "smap_enable") == 0) { bcopy(clac_instr, (void *)hid->hid_instr_offset, sizeof (clac_instr)); } else if (strcmp(hid->hid_symname, "smap_disable") == 0) { bcopy(stac_instr, (void *)hid->hid_instr_offset, sizeof (stac_instr)); } } /* * Loop through hi_calls and hand off the inlining to * the appropriate calls. */ void do_hotinlines(struct module *mp) { for (hotinline_desc_t *hid = mp->hi_calls; hid != NULL; hid = hid->hid_next) { #if !defined(__xpv) hotinline_smap(hid); #endif /* __xpv */ } } /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright (c) 1997-1998 by Sun Microsystems, Inc. * All rights reserved. */ /* * Copyright (c) 2010, Intel Corporation. * All rights reserved. */ #include #include #include #include #include /*ARGSUSED*/ int arch_kphysm_del_span_ok(pfn_t base, pgcnt_t npgs) { ASSERT(npgs != 0); return (0); } /*ARGSUSED*/ int arch_kphysm_relocate(pfn_t base, pgcnt_t npgs) { ASSERT(npgs != 0); return (ENOTSUP); } int arch_kphysm_del_supported(void) { return (0); } /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2010 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ #include #include #include #include #include #include #include #include #include int max_mem_nodes = 1; struct mem_node_conf mem_node_config[MAX_MEM_NODES]; int mem_node_pfn_shift; /* * num_memnodes should be updated atomically and always >= * the number of bits in memnodes_mask or the algorithm may fail. */ uint16_t num_memnodes; mnodeset_t memnodes_mask; /* assumes 8*(sizeof(mnodeset_t)) >= MAX_MEM_NODES */ /* * If set, mem_node_physalign should be a power of two, and * should reflect the minimum address alignment of each node. */ uint64_t mem_node_physalign; /* * Platform hooks we will need. */ #pragma weak plat_build_mem_nodes #pragma weak plat_slice_add #pragma weak plat_slice_del /* * Adjust the memnode config after a DR operation. * * It is rather tricky to do these updates since we can't * protect the memnode structures with locks, so we must * be mindful of the order in which updates and reads to * these values can occur. */ void mem_node_add_slice(pfn_t start, pfn_t end) { int mnode; mnodeset_t newmask, oldmask; /* * DR will pass us the first pfn that is allocatable. * We need to round down to get the real start of * the slice. */ if (mem_node_physalign) { start &= ~(btop(mem_node_physalign) - 1); end = roundup(end, btop(mem_node_physalign)) - 1; } mnode = PFN_2_MEM_NODE(start); ASSERT(mnode >= 0 && mnode < max_mem_nodes); if (atomic_cas_32((uint32_t *)&mem_node_config[mnode].exists, 0, 1)) { /* * Add slice to existing node. */ if (start < mem_node_config[mnode].physbase) mem_node_config[mnode].physbase = start; if (end > mem_node_config[mnode].physmax) mem_node_config[mnode].physmax = end; } else { mem_node_config[mnode].physbase = start; mem_node_config[mnode].physmax = end; atomic_inc_16(&num_memnodes); do { oldmask = memnodes_mask; newmask = memnodes_mask | (1ull << mnode); } while (atomic_cas_64(&memnodes_mask, oldmask, newmask) != oldmask); } /* * Inform the common lgrp framework about the new memory */ lgrp_config(LGRP_CONFIG_MEM_ADD, mnode, MEM_NODE_2_LGRPHAND(mnode)); } /* * Remove a PFN range from a memnode. On some platforms, * the memnode will be created with physbase at the first * allocatable PFN, but later deleted with the MC slice * base address converted to a PFN, in which case we need * to assume physbase and up. */ void mem_node_del_slice(pfn_t start, pfn_t end) { int mnode; pgcnt_t delta_pgcnt, node_size; mnodeset_t omask, nmask; if (mem_node_physalign) { start &= ~(btop(mem_node_physalign) - 1); end = roundup(end, btop(mem_node_physalign)) - 1; } mnode = PFN_2_MEM_NODE(start); ASSERT(mnode >= 0 && mnode < max_mem_nodes); ASSERT(mem_node_config[mnode].exists == 1); delta_pgcnt = end - start; node_size = mem_node_config[mnode].physmax - mem_node_config[mnode].physbase; if (node_size > delta_pgcnt) { /* * Subtract the slice from the memnode. */ if (start <= mem_node_config[mnode].physbase) mem_node_config[mnode].physbase = end + 1; ASSERT(end <= mem_node_config[mnode].physmax); if (end == mem_node_config[mnode].physmax) mem_node_config[mnode].physmax = start - 1; } else { /* * Let the common lgrp framework know this mnode is * leaving */ lgrp_config(LGRP_CONFIG_MEM_DEL, mnode, MEM_NODE_2_LGRPHAND(mnode)); /* * Delete the whole node. */ ASSERT(MNODE_PGCNT(mnode) == 0); do { omask = memnodes_mask; nmask = omask & ~(1ull << mnode); } while (atomic_cas_64(&memnodes_mask, omask, nmask) != omask); atomic_dec_16(&num_memnodes); mem_node_config[mnode].exists = 0; } } void mem_node_add_range(pfn_t start, pfn_t end) { if (&plat_slice_add) plat_slice_add(start, end); else mem_node_add_slice(start, end); } void mem_node_del_range(pfn_t start, pfn_t end) { if (&plat_slice_del) plat_slice_del(start, end); else mem_node_del_slice(start, end); } void startup_build_mem_nodes(struct memlist *list) { pfn_t start, end; /* LINTED: ASSERT will always true or false */ ASSERT(NBBY * sizeof (mnodeset_t) >= max_mem_nodes); if (&plat_build_mem_nodes) { plat_build_mem_nodes(list); } else { /* * Boot install lists are arranged , ... */ while (list) { start = list->ml_address >> PAGESHIFT; if (start > physmax) continue; end = (list->ml_address + list->ml_size - 1) >> PAGESHIFT; if (end > physmax) end = physmax; mem_node_add_range(start, end); list = list->ml_next; } mem_node_physalign = 0; mem_node_pfn_shift = 0; } } /* * Allocate an unassigned memnode. */ int mem_node_alloc() { int mnode; mnodeset_t newmask, oldmask; /* * Find an unused memnode. Update it atomically to prevent * a first time memnode creation race. */ for (mnode = 0; mnode < max_mem_nodes; mnode++) if (atomic_cas_32((uint32_t *)&mem_node_config[mnode].exists, 0, 1) == 0) break; if (mnode >= max_mem_nodes) panic("Out of free memnodes\n"); mem_node_config[mnode].physbase = (pfn_t)-1l; mem_node_config[mnode].physmax = 0; atomic_inc_16(&num_memnodes); do { oldmask = memnodes_mask; newmask = memnodes_mask | (1ull << mnode); } while (atomic_cas_64(&memnodes_mask, oldmask, newmask) != oldmask); return (mnode); } /* * Find the intersection between a memnode and a memlist * and returns the number of pages that overlap. * * Assumes the list is protected from DR operations by * the memlist lock. */ pgcnt_t mem_node_memlist_pages(int mnode, struct memlist *mlist) { pfn_t base, end; pfn_t cur_base, cur_end; pgcnt_t npgs; struct memlist *pmem; base = mem_node_config[mnode].physbase; end = mem_node_config[mnode].physmax; npgs = 0; memlist_read_lock(); for (pmem = mlist; pmem; pmem = pmem->ml_next) { cur_base = btop(pmem->ml_address); cur_end = cur_base + btop(pmem->ml_size) - 1; if (end < cur_base || base > cur_end) continue; npgs = npgs + (MIN(cur_end, end) - MAX(cur_base, base)) + 1; } memlist_read_unlock(); return (npgs); } /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2010 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ /* * i86pc Memory Scrubbing * * On detection of a correctable memory ECC error, the i86pc hardware * returns the corrected data to the requester and may re-write it * to memory (DRAM or NVRAM). Machines which do not re-write this to * memory should add an NMI handler to correct and rewrite. * * Scrubbing thus reduces the likelyhood that multiple transient errors * will occur in the same memory word, making uncorrectable errors due * to transients less likely. * * Thus is born the desire that every memory location be periodically * accessed. * * This file implements a memory scrubbing thread. This scrubber * guarantees that all of physical memory is accessed periodically * (memscrub_period_sec -- 12 hours). * * It attempts to do this as unobtrusively as possible. The thread * schedules itself to wake up at an interval such that if it reads * memscrub_span_pages (4MB) on each wakeup, it will read all of physical * memory in in memscrub_period_sec (12 hours). * * The scrubber uses the REP LODS so it reads 4MB in 0.15 secs (on P5-200). * When it completes a span, if all the CPUs are idle, it reads another span. * Typically it soaks up idle time this way to reach its deadline early * -- and sleeps until the next period begins. * * Maximal Cost Estimate: 8GB @ xxMB/s = xxx seconds spent in 640 wakeups * that run for 0.15 seconds at intervals of 67 seconds. * * In practice, the scrubber finds enough idle time to finish in a few * minutes, and sleeps until its 12 hour deadline. * * The scrubber maintains a private copy of the phys_install memory list * to keep track of what memory should be scrubbed. * * The following parameters can be set via /etc/system * * memscrub_span_pages = MEMSCRUB_DFL_SPAN_PAGES (4MB) * memscrub_period_sec = MEMSCRUB_DFL_PERIOD_SEC (12 hours) * memscrub_thread_pri = MEMSCRUB_DFL_THREAD_PRI (0) * memscrub_delay_start_sec = (10 seconds) * disable_memscrub = (0) * * the scrubber will exit (or never be started) if it finds the variable * "disable_memscrub" set. * * MEMSCRUB_DFL_SPAN_PAGES is based on the guess that 0.15 sec * is a "good" amount of minimum time for the thread to run at a time. * * MEMSCRUB_DFL_PERIOD_SEC (12 hours) is nearly a total guess -- * twice the frequency the hardware folk estimated would be necessary. * * MEMSCRUB_DFL_THREAD_PRI (0) is based on the assumption that nearly * any other use of the system should be higher priority than scrubbing. */ #include #include /* timeout, types, t_lock */ #include #include /* MIN */ #include /* memlist */ #include /* KMEM_NOSLEEP */ #include /* ncpus_online */ #include /* ASSERTs */ #include #include #include #include #include #include /* CPR callback */ static caddr_t memscrub_window; static hat_mempte_t memscrub_pte; /* * Global Data: */ /* * scan all of physical memory at least once every MEMSCRUB_PERIOD_SEC */ #define MEMSCRUB_DFL_PERIOD_SEC (12 * 60 * 60) /* 12 hours */ /* * start only if at least MEMSCRUB_MIN_PAGES in system */ #define MEMSCRUB_MIN_PAGES ((32 * 1024 * 1024) / PAGESIZE) /* * scan at least MEMSCRUB_DFL_SPAN_PAGES each iteration */ #define MEMSCRUB_DFL_SPAN_PAGES ((4 * 1024 * 1024) / PAGESIZE) /* * almost anything is higher priority than scrubbing */ #define MEMSCRUB_DFL_THREAD_PRI 0 /* * we can patch these defaults in /etc/system if necessary */ uint_t disable_memscrub = 0; static uint_t disable_memscrub_quietly = 0; pgcnt_t memscrub_min_pages = MEMSCRUB_MIN_PAGES; pgcnt_t memscrub_span_pages = MEMSCRUB_DFL_SPAN_PAGES; time_t memscrub_period_sec = MEMSCRUB_DFL_PERIOD_SEC; uint_t memscrub_thread_pri = MEMSCRUB_DFL_THREAD_PRI; time_t memscrub_delay_start_sec = 10; /* * Static Routines */ static void memscrubber(void); static int system_is_idle(void); static int memscrub_add_span(uint64_t, uint64_t); /* * Static Data */ static struct memlist *memscrub_memlist; static uint_t memscrub_phys_pages; static kcondvar_t memscrub_cv; static kmutex_t memscrub_lock; /* * memscrub_lock protects memscrub_memlist */ uint_t memscrub_scans_done; uint_t memscrub_done_early; uint_t memscrub_early_sec; uint_t memscrub_done_late; time_t memscrub_late_sec; /* * create memscrub_memlist from phys_install list * initialize locks, set memscrub_phys_pages. */ void memscrub_init() { struct memlist *src; if (physmem < memscrub_min_pages) return; if (!kpm_enable) { memscrub_window = vmem_alloc(heap_arena, PAGESIZE, VM_SLEEP); memscrub_pte = hat_mempte_setup(memscrub_window); } /* * copy phys_install to memscrub_memlist */ for (src = phys_install; src; src = src->ml_next) { if (memscrub_add_span(src->ml_address, src->ml_size)) { cmn_err(CE_WARN, "Software memory scrubber failed to initialize\n"); return; } } mutex_init(&memscrub_lock, NULL, MUTEX_DRIVER, NULL); cv_init(&memscrub_cv, NULL, CV_DRIVER, NULL); /* * create memscrubber thread */ (void) thread_create(NULL, 0, (void (*)())memscrubber, NULL, 0, &p0, TS_RUN, memscrub_thread_pri); } /* * Function to cause the software memscrubber to exit quietly if the * platform support has located a hardware scrubber and enabled it. */ void memscrub_disable(void) { disable_memscrub_quietly = 1; } #ifdef MEMSCRUB_DEBUG static void memscrub_printmemlist(char *title, struct memlist *listp) { struct memlist *list; cmn_err(CE_CONT, "%s:\n", title); for (list = listp; list; list = list->next) { cmn_err(CE_CONT, "addr = 0x%llx, size = 0x%llx\n", list->address, list->size); } } #endif /* MEMSCRUB_DEBUG */ /* ARGSUSED */ static void memscrub_wakeup(void *c) { /* * grab mutex to guarantee that our wakeup call * arrives after we go to sleep -- so we can't sleep forever. */ mutex_enter(&memscrub_lock); cv_signal(&memscrub_cv); mutex_exit(&memscrub_lock); } /* * this calculation doesn't account for the time that the actual scan * consumes -- so we'd fall slightly behind schedule with this * interval_sec. but the idle loop optimization below usually makes us * come in way ahead of schedule. */ static int compute_interval_sec() { if (memscrub_phys_pages <= memscrub_span_pages) return (memscrub_period_sec); else return (memscrub_period_sec/ (memscrub_phys_pages/memscrub_span_pages)); } static void memscrubber() { time_t deadline; uint64_t mlp_last_addr; uint64_t mlp_next_addr; int reached_end = 1; time_t interval_sec = 0; struct memlist *mlp; extern void scan_memory(caddr_t, size_t); callb_cpr_t cprinfo; /* * notify CPR of our existence */ CALLB_CPR_INIT(&cprinfo, &memscrub_lock, callb_generic_cpr, "memscrub"); if (memscrub_memlist == NULL) { cmn_err(CE_WARN, "memscrub_memlist not initialized."); goto memscrub_exit; } mlp = memscrub_memlist; mlp_next_addr = mlp->ml_address; mlp_last_addr = mlp->ml_address + mlp->ml_size; deadline = gethrestime_sec() + memscrub_delay_start_sec; for (;;) { if (disable_memscrub || disable_memscrub_quietly) break; mutex_enter(&memscrub_lock); /* * did we just reach the end of memory? */ if (reached_end) { time_t now = gethrestime_sec(); if (now >= deadline) { memscrub_done_late++; memscrub_late_sec += (now - deadline); /* * past deadline, start right away */ interval_sec = 0; deadline = now + memscrub_period_sec; } else { /* * we finished ahead of schedule. * wait till previous dealine before re-start. */ interval_sec = deadline - now; memscrub_done_early++; memscrub_early_sec += interval_sec; deadline += memscrub_period_sec; } } else { interval_sec = compute_interval_sec(); } /* * it is safe from our standpoint for CPR to * suspend the system */ CALLB_CPR_SAFE_BEGIN(&cprinfo); /* * hit the snooze bar */ (void) timeout(memscrub_wakeup, NULL, interval_sec * hz); /* * go to sleep */ cv_wait(&memscrub_cv, &memscrub_lock); /* we need to goto work */ CALLB_CPR_SAFE_END(&cprinfo, &memscrub_lock); mutex_exit(&memscrub_lock); do { pgcnt_t pages = memscrub_span_pages; uint64_t address = mlp_next_addr; if (disable_memscrub || disable_memscrub_quietly) break; mutex_enter(&memscrub_lock); /* * Make sure we don't try to scan beyond the end of * the current memlist. If we would, then resize * our scan target for this iteration, and prepare * to read the next memlist entry on the next * iteration. */ reached_end = 0; if (address + mmu_ptob(pages) >= mlp_last_addr) { pages = mmu_btop(mlp_last_addr - address); mlp = mlp->ml_next; if (mlp == NULL) { reached_end = 1; mlp = memscrub_memlist; } mlp_next_addr = mlp->ml_address; mlp_last_addr = mlp->ml_address + mlp->ml_size; } else { mlp_next_addr += mmu_ptob(pages); } mutex_exit(&memscrub_lock); while (pages--) { pfn_t pfn = btop(address); /* * Without segkpm, the memscrubber cannot * be allowed to migrate across CPUs, as * the CPU-specific mapping of * memscrub_window would be incorrect. * With segkpm, switching CPUs is legal, but * inefficient. We don't use * kpreempt_disable as it might hold a * higher priority thread (eg, RT) too long * off CPU. */ thread_affinity_set(curthread, CPU_CURRENT); if (kpm_enable) memscrub_window = hat_kpm_pfn2va(pfn); else hat_mempte_remap(pfn, memscrub_window, memscrub_pte, PROT_READ, HAT_LOAD_NOCONSIST); scan_memory(memscrub_window, PAGESIZE); thread_affinity_clear(curthread); address += MMU_PAGESIZE; } memscrub_scans_done++; } while (!reached_end && system_is_idle()); } memscrub_exit: if (!disable_memscrub_quietly) cmn_err(CE_NOTE, "Software memory scrubber exiting."); /* * We are about to bail, but don't have the memscrub_lock, * and it is needed for CALLB_CPR_EXIT. */ mutex_enter(&memscrub_lock); CALLB_CPR_EXIT(&cprinfo); cv_destroy(&memscrub_cv); thread_exit(); } /* * return 1 if we're MP and all the other CPUs are idle */ static int system_is_idle() { int cpu_id; int found = 0; if (1 == ncpus_online) return (0); for (cpu_id = 0; cpu_id < NCPU; ++cpu_id) { if (!cpu[cpu_id]) continue; found++; if (cpu[cpu_id]->cpu_thread != cpu[cpu_id]->cpu_idle_thread) { if (CPU->cpu_id == cpu_id && CPU->cpu_disp->disp_nrunnable == 0) continue; return (0); } if (found == ncpus) break; } return (1); } /* * add a span to the memscrub list */ static int memscrub_add_span(uint64_t start, uint64_t bytes) { struct memlist *dst; struct memlist *prev, *next; uint64_t end = start + bytes - 1; int retval = 0; mutex_enter(&memscrub_lock); #ifdef MEMSCRUB_DEBUG memscrub_printmemlist("memscrub_memlist before", memscrub_memlist); cmn_err(CE_CONT, "memscrub_phys_pages: 0x%x\n", memscrub_phys_pages); cmn_err(CE_CONT, "memscrub_add_span: address: 0x%llx" " size: 0x%llx\n", start, bytes); #endif /* MEMSCRUB_DEBUG */ /* * Scan through the list to find the proper place to install it. */ prev = NULL; next = memscrub_memlist; while (next) { uint64_t ns = next->ml_address; uint64_t ne = next->ml_address + next->ml_size - 1; /* * If this span overlaps with an existing span, then * something has gone horribly wrong with the phys_install * list. In fact, I'm surprised we made it this far. */ if ((start >= ns && start <= ne) || (end >= ns && end <= ne) || (start < ns && end > ne)) panic("memscrub found overlapping memory ranges " "(0x%p-0x%p) and (0x%p-0x%p)", (void *)(uintptr_t)start, (void *)(uintptr_t)end, (void *)(uintptr_t)ns, (void *)(uintptr_t)ne); /* * New span can be appended to an existing one. */ if (start == ne + 1) { next->ml_size += bytes; goto add_done; } /* * New span can be prepended to an existing one. */ if (end + 1 == ns) { next->ml_size += bytes; next->ml_address = start; goto add_done; } /* * If the next span has a higher start address than the new * one, then we have found the right spot for our * insertion. */ if (ns > start) break; prev = next; next = next->ml_next; } /* * allocate a new struct memlist */ dst = kmem_alloc(sizeof (struct memlist), KM_NOSLEEP); if (dst == NULL) { retval = -1; goto add_done; } dst->ml_address = start; dst->ml_size = bytes; dst->ml_prev = prev; dst->ml_next = next; if (prev) prev->ml_next = dst; else memscrub_memlist = dst; if (next) next->ml_prev = dst; add_done: if (retval != -1) memscrub_phys_pages += mmu_btop(bytes); #ifdef MEMSCRUB_DEBUG memscrub_printmemlist("memscrub_memlist after", memscrub_memlist); cmn_err(CE_CONT, "memscrub_phys_pages: 0x%x\n", memscrub_phys_pages); #endif /* MEMSCRUB_DEBUG */ mutex_exit(&memscrub_lock); return (retval); } /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright (c) 2012 Gary Mills * * Copyright (c) 1993, 2010, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2011 by Delphix. All rights reserved. * Copyright 2019 Joyent, Inc. * Copyright 2024 Oxide Computer Company */ /* * Copyright (c) 2010, Intel Corporation. * All rights reserved. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef __xpv #include #else #include #endif /* * some globals for patching the result of cpuid * to solve problems w/ creative cpu vendors */ extern uint32_t cpuid_feature_ecx_include; extern uint32_t cpuid_feature_ecx_exclude; extern uint32_t cpuid_feature_edx_include; extern uint32_t cpuid_feature_edx_exclude; nmi_action_t nmi_action = NMI_ACTION_UNSET; /* * Set console mode */ static void set_console_mode(uint8_t val) { struct bop_regs rp = {0}; rp.eax.byte.ah = 0x0; rp.eax.byte.al = val; rp.ebx.word.bx = 0x0; BOP_DOINT(bootops, 0x10, &rp); } /* * Setup routine called right before main(). Interposing this function * before main() allows us to call it in a machine-independent fashion. */ void mlsetup(struct regs *rp) { u_longlong_t prop_value; char prop_str[BP_MAX_STRLEN]; extern struct classfuncs sys_classfuncs; extern disp_t cpu0_disp; extern char t0stack[]; extern int post_fastreboot; extern uint64_t plat_dr_options; ASSERT_STACK_ALIGNED(); /* * initialize cpu_self */ cpu[0]->cpu_self = cpu[0]; #if defined(__xpv) /* * Point at the hypervisor's virtual cpu structure */ cpu[0]->cpu_m.mcpu_vcpu_info = &HYPERVISOR_shared_info->vcpu_info[0]; #endif /* * check if we've got special bits to clear or set * when checking cpu features */ if (bootprop_getval("cpuid_feature_ecx_include", &prop_value) != 0) cpuid_feature_ecx_include = 0; else cpuid_feature_ecx_include = (uint32_t)prop_value; if (bootprop_getval("cpuid_feature_ecx_exclude", &prop_value) != 0) cpuid_feature_ecx_exclude = 0; else cpuid_feature_ecx_exclude = (uint32_t)prop_value; if (bootprop_getval("cpuid_feature_edx_include", &prop_value) != 0) cpuid_feature_edx_include = 0; else cpuid_feature_edx_include = (uint32_t)prop_value; if (bootprop_getval("cpuid_feature_edx_exclude", &prop_value) != 0) cpuid_feature_edx_exclude = 0; else cpuid_feature_edx_exclude = (uint32_t)prop_value; #if !defined(__xpv) if (bootprop_getstr("nmi", prop_str, sizeof (prop_str)) == 0) { if (strcmp(prop_str, "ignore") == 0) { nmi_action = NMI_ACTION_IGNORE; } else if (strcmp(prop_str, "panic") == 0) { nmi_action = NMI_ACTION_PANIC; } else if (strcmp(prop_str, "kmdb") == 0) { nmi_action = NMI_ACTION_KMDB; } else { prom_printf("unix: ignoring unknown nmi=%s\n", prop_str); } } /* * Check to see if KPTI has been explicitly enabled or disabled. * We have to check this before init_desctbls(). */ if (bootprop_getval("kpti", &prop_value) == 0) { kpti_enable = (uint64_t)(prop_value == 1); prom_printf("unix: forcing kpti to %s due to boot argument\n", (kpti_enable == 1) ? "ON" : "OFF"); } else { kpti_enable = 1; } if (bootprop_getval("pcid", &prop_value) == 0 && prop_value == 0) { prom_printf("unix: forcing pcid to OFF due to boot argument\n"); x86_use_pcid = 0; } else if (kpti_enable != 1) { x86_use_pcid = 0; } /* * While we don't need to check this until later, we might as well do it * here. */ if (bootprop_getstr("smt_enabled", prop_str, sizeof (prop_str)) == 0) { if (strcasecmp(prop_str, "false") == 0 || strcmp(prop_str, "0") == 0) smt_boot_disable = 1; } #endif /* * Initialize idt0, gdt0, ldt0_default, ktss0 and dftss. */ init_desctbls(); /* * initialize t0 */ t0.t_stk = (caddr_t)rp - MINFRAME; t0.t_stkbase = t0stack; t0.t_pri = maxclsyspri - 3; t0.t_schedflag = TS_LOAD | TS_DONT_SWAP; t0.t_procp = &p0; t0.t_plockp = &p0lock.pl_lock; t0.t_lwp = &lwp0; t0.t_forw = &t0; t0.t_back = &t0; t0.t_next = &t0; t0.t_prev = &t0; t0.t_cpu = cpu[0]; t0.t_disp_queue = &cpu0_disp; t0.t_bind_cpu = PBIND_NONE; t0.t_bind_pset = PS_NONE; t0.t_bindflag = (uchar_t)default_binding_mode; t0.t_cpupart = &cp_default; t0.t_clfuncs = &sys_classfuncs.thread; t0.t_copyops = NULL; THREAD_ONPROC(&t0, CPU); lwp0.lwp_thread = &t0; lwp0.lwp_regs = (void *)rp; lwp0.lwp_procp = &p0; t0.t_tid = p0.p_lwpcnt = p0.p_lwprcnt = p0.p_lwpid = 1; p0.p_exec = NULL; p0.p_stat = SRUN; p0.p_flag = SSYS; p0.p_tlist = &t0; p0.p_stksize = 2*PAGESIZE; p0.p_stkpageszc = 0; p0.p_as = &kas; p0.p_lockp = &p0lock; p0.p_brkpageszc = 0; p0.p_t1_lgrpid = LGRP_NONE; p0.p_tr_lgrpid = LGRP_NONE; psecflags_default(&p0.p_secflags); sigorset(&p0.p_ignore, &ignoredefault); CPU->cpu_thread = &t0; bzero(&cpu0_disp, sizeof (disp_t)); CPU->cpu_disp = &cpu0_disp; CPU->cpu_disp->disp_cpu = CPU; CPU->cpu_dispthread = &t0; CPU->cpu_idle_thread = &t0; CPU->cpu_flags = CPU_READY | CPU_RUNNING | CPU_EXISTS | CPU_ENABLE; CPU->cpu_dispatch_pri = t0.t_pri; CPU->cpu_id = 0; CPU->cpu_pri = 12; /* initial PIL for the boot CPU */ /* * Ensure that we have set the necessary feature bits before setting up * PCI config space access. */ cpuid_execpass(cpu[0], CPUID_PASS_PRELUDE, x86_featureset); /* * lgrp_init() and possibly cpuid_pass1() need PCI config * space access */ #if defined(__xpv) if (DOMAIN_IS_INITDOMAIN(xen_info)) pci_cfgspace_init(); #else pci_cfgspace_init(); /* * Initialize the platform type from CPU 0 to ensure that * determine_platform() is only ever called once. */ determine_platform(); #endif /* * While the BIOS may have already applied some microcode updates, we * may have more recent updates available that we'd like to apply. The * application of said microcode may end up resulting in architecturally * visible changes (e.g., changed MSR or CPUID bits) which we'd like to * have in place before we start querying the CPU for its capabilities. * So we first run the IDENT pass to determine the specific CPU vendor, * model, rev etc., fill out cpu_ucode_info and update the microcode, if * necessary. */ cpuid_execpass(cpu[0], CPUID_PASS_IDENT, NULL); ucode_init(); ucode_check_boot(); /* * Now we're ready to run the BASIC cpuid pass. * * The x86_featureset is initialized here based on the capabilities of * the boot CPU. Note that if we choose to support CPUs that have * different feature sets (at which point we would almost certainly want * to set the feature bits to correspond to the feature minimum) this * value may be altered. */ cpuid_execpass(cpu[0], CPUID_PASS_BASIC, x86_featureset); #if !defined(__xpv) if ((get_hwenv() & HW_XEN_HVM) != 0) xen_hvm_init(); /* * Before we do anything with the TSCs, we need to work around * Intel erratum BT81. On some CPUs, warm reset does not * clear the TSC. If we are on such a CPU, we will clear TSC ourselves * here. Other CPUs will clear it when we boot them later, and the * resulting skew will be handled by tsc_sync_master()/_slave(); * note that such skew already exists and has to be handled anyway. * * We do this only on metal. This same problem can occur with a * hypervisor that does not happen to virtualise a TSC that starts from * zero, regardless of CPU type; however, we do not expect hypervisors * that do not virtualise TSC that way to handle writes to TSC * correctly, either. */ if (get_hwenv() == HW_NATIVE && cpuid_getvendor(CPU) == X86_VENDOR_Intel && cpuid_getfamily(CPU) == 6 && (cpuid_getmodel(CPU) == 0x2d || cpuid_getmodel(CPU) == 0x3e) && is_x86_feature(x86_featureset, X86FSET_TSC)) { (void) wrmsr(REG_TSC, 0UL); } /* * Patch the tsc_read routine with appropriate set of instructions, * depending on the processor family and architecure, to read the * time-stamp counter while ensuring no out-of-order execution. * Patch it while the kernel text is still writable. * * The Xen hypervisor does not correctly report whether rdtscp is * supported or not, so we must assume that it is not. */ if ((get_hwenv() & HW_XEN_HVM) == 0 && is_x86_feature(x86_featureset, X86FSET_TSCP)) { patch_tsc_read(TSC_TSCP); } else if (is_x86_feature(x86_featureset, X86FSET_LFENCE_SER)) { ASSERT(is_x86_feature(x86_featureset, X86FSET_SSE2)); patch_tsc_read(TSC_RDTSC_LFENCE); } #endif /* !__xpv */ #if !defined(__xpv) patch_memops(cpuid_getvendor(CPU)); #endif /* !__xpv */ #if !defined(__xpv) /* XXPV what, if anything, should be dorked with here under xen? */ /* * While we're thinking about the TSC, let's set up %cr4 so that * userland can issue rdtsc, and initialize the TSC_AUX value * (the cpuid) for the rdtscp instruction on appropriately * capable hardware. */ if (is_x86_feature(x86_featureset, X86FSET_TSC)) setcr4(getcr4() & ~CR4_TSD); if (is_x86_feature(x86_featureset, X86FSET_TSCP)) (void) wrmsr(MSR_AMD_TSCAUX, 0); /* * Let's get the other %cr4 stuff while we're here. Note, we defer * enabling CR4_SMAP until startup_end(); however, that's importantly * before we start other CPUs. That ensures that it will be synced out * to other CPUs. */ if (is_x86_feature(x86_featureset, X86FSET_DE)) setcr4(getcr4() | CR4_DE); if (is_x86_feature(x86_featureset, X86FSET_SMEP)) setcr4(getcr4() | CR4_SMEP); #endif /* __xpv */ /* * Initialize thread/cpu microstate accounting */ init_mstate(&t0, LMS_SYSTEM); init_cpu_mstate(CPU, CMS_SYSTEM); /* * Initialize lists of available and active CPUs. */ cpu_list_init(CPU); pg_cpu_bootstrap(CPU); /* * Now that we have taken over the GDT, IDT and have initialized * active CPU list it's time to inform kmdb if present. */ if (boothowto & RB_DEBUG) kdi_idt_sync(); if (BOP_GETPROPLEN(bootops, "efi-systab") < 0) { /* * In BIOS system, explicitly set console to text mode (0x3) * if this is a boot post Fast Reboot, and the console is set * to CONS_SCREEN_TEXT. */ if (post_fastreboot && boot_console_type(NULL) == CONS_SCREEN_TEXT) { set_console_mode(0x3); } } /* * If requested (boot -d) drop into kmdb. * * This must be done after cpu_list_init() on the 64-bit kernel * since taking a trap requires that we re-compute gsbase based * on the cpu list. */ if (boothowto & RB_DEBUGENTER) kmdb_enter(); cpu_vm_data_init(CPU); rp->r_fp = 0; /* terminate kernel stack traces! */ prom_init("kernel", (void *)NULL); /* User-set option overrides firmware value. */ if (bootprop_getval(PLAT_DR_OPTIONS_NAME, &prop_value) == 0) { plat_dr_options = (uint64_t)prop_value; } #if defined(__xpv) /* No support of DR operations on xpv */ plat_dr_options = 0; #else /* __xpv */ /* Flag PLAT_DR_FEATURE_ENABLED should only be set by DR driver. */ plat_dr_options &= ~PLAT_DR_FEATURE_ENABLED; #endif /* __xpv */ /* * Get value of "plat_dr_physmax" boot option. * It overrides values calculated from MSCT or SRAT table. */ if (bootprop_getval(PLAT_DR_PHYSMAX_NAME, &prop_value) == 0) { plat_dr_physmax = ((uint64_t)prop_value) >> PAGESHIFT; } /* Get value of boot_ncpus. */ if (bootprop_getval(BOOT_NCPUS_NAME, &prop_value) != 0) { boot_ncpus = NCPU; } else { boot_ncpus = (int)prop_value; if (boot_ncpus <= 0 || boot_ncpus > NCPU) boot_ncpus = NCPU; } /* * Set max_ncpus and boot_max_ncpus to boot_ncpus if platform doesn't * support CPU DR operations. */ if (plat_dr_support_cpu() == 0) { max_ncpus = boot_max_ncpus = boot_ncpus; } else { if (bootprop_getval(PLAT_MAX_NCPUS_NAME, &prop_value) != 0) { max_ncpus = NCPU; } else { max_ncpus = (int)prop_value; if (max_ncpus <= 0 || max_ncpus > NCPU) { max_ncpus = NCPU; } if (boot_ncpus > max_ncpus) { boot_ncpus = max_ncpus; } } if (bootprop_getval(BOOT_MAX_NCPUS_NAME, &prop_value) != 0) { boot_max_ncpus = boot_ncpus; } else { boot_max_ncpus = (int)prop_value; if (boot_max_ncpus <= 0 || boot_max_ncpus > NCPU) { boot_max_ncpus = boot_ncpus; } else if (boot_max_ncpus > max_ncpus) { boot_max_ncpus = max_ncpus; } } } /* * Initialize the lgrp framework */ lgrp_init(LGRP_INIT_STAGE1); if (boothowto & RB_HALT) { prom_printf("unix: kernel halted by -h flag\n"); prom_enter_mon(); } ASSERT_STACK_ALIGNED(); if (workaround_errata(CPU) != 0) panic("critical workaround(s) missing for boot cpu"); } void mach_modpath(char *path, const char *filename) { /* * Hammerhead: Unified kernel module path. * * All modules live in /kernel (no /platform/ prefix, no /usr/kernel/). * The boot archive search path is prepended so early boot can find * modules before the root filesystem is mounted. * * getmodpath() in kobj.c appends MOD_DEFPATH ("/kernel") after this. */ int len; len = strlen(SYSTEM_BOOT_PATH "/kernel"); (void) strcpy(path, SYSTEM_BOOT_PATH "/kernel"); } /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2009 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ #include #include #include #include #include #include #include #include #include #include /* * Interrupt another CPU. * This is useful to make the other CPU go through a trap so that * it recognizes an address space trap (AST) for preempting a thread. * * It is possible to be preempted here and be resumed on the CPU * being poked, so it isn't an error to poke the current CPU. * We could check this and still get preempted after the check, so * we don't bother. */ void poke_cpu(int cpun) { if (panicstr) return; /* * We don't need to receive an ACK from the CPU being poked, * so just send out a directed interrupt. */ send_dirint(cpun, XC_CPUPOKE_PIL); } /* * Call a function on a target CPU */ void cpu_call(cpu_t *cp, cpu_call_func_t func, uintptr_t arg1, uintptr_t arg2) { cpuset_t set; if (panicstr) return; /* * Prevent CPU from going off-line */ kpreempt_disable(); /* * If we are on the target CPU, call the function directly, but raise * the PIL to XC_PIL. * This guarantees that functions called via cpu_call() can not ever * interrupt each other. */ if (CPU == cp) { int save_spl = splr(ipltospl(XC_HI_PIL)); (*func)(arg1, arg2); splx(save_spl); } else { CPUSET_ONLY(set, cp->cpu_id); xc_call((xc_arg_t)arg1, (xc_arg_t)arg2, 0, CPUSET2BV(set), (xc_func_t)(uintptr_t)func); } kpreempt_enable(); } /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2008 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. * Copyright 2020 Oxide Computer Company */ #define PSMI_1_7 #include #include #include #include #include #include #include #include #include #if defined(__xpv) #include #include #include #endif /* * External reference functions */ extern void *get_next_mach(void *, char *); extern void close_mach_list(void); extern void open_mach_list(void); /* * from startup.c - kernel VA range allocator for device mappings */ extern void *device_arena_alloc(size_t size, int vm_flag); extern void device_arena_free(void * vaddr, size_t size); void psm_modloadonly(void); void psm_install(void); /* * Local Function Prototypes */ static struct modlinkage *psm_modlinkage_alloc(struct psm_info *infop); static void psm_modlinkage_free(struct modlinkage *mlinkp); static char *psm_get_impl_module(int first); static int mod_installpsm(struct modlpsm *modl, struct modlinkage *modlp); static int mod_removepsm(struct modlpsm *modl, struct modlinkage *modlp); static int mod_infopsm(struct modlpsm *modl, struct modlinkage *modlp, int *p0); struct mod_ops mod_psmops = { mod_installpsm, mod_removepsm, mod_infopsm }; static struct psm_sw psm_swtab = { &psm_swtab, &psm_swtab, NULL, 0 }; kmutex_t psmsw_lock; /* lock accesses to psmsw */ struct psm_sw *psmsw = &psm_swtab; /* start of all psm_sw */ static struct modlinkage * psm_modlinkage_alloc(struct psm_info *infop) { int memsz; struct modlinkage *mlinkp; struct modlpsm *mlpsmp; struct psm_sw *swp; memsz = sizeof (struct modlinkage) + sizeof (struct modlpsm) + sizeof (struct psm_sw); mlinkp = (struct modlinkage *)kmem_zalloc(memsz, KM_NOSLEEP); if (!mlinkp) { cmn_err(CE_WARN, "!psm_mod_init: Cannot install %s", infop->p_mach_idstring); return (NULL); } mlpsmp = (struct modlpsm *)(mlinkp + 1); swp = (struct psm_sw *)(mlpsmp + 1); mlinkp->ml_rev = MODREV_1; mlinkp->ml_linkage[0] = (void *)mlpsmp; mlinkp->ml_linkage[1] = (void *)NULL; mlpsmp->psm_modops = &mod_psmops; mlpsmp->psm_linkinfo = infop->p_mach_desc; mlpsmp->psm_swp = swp; swp->psw_infop = infop; return (mlinkp); } static void psm_modlinkage_free(struct modlinkage *mlinkp) { if (!mlinkp) return; (void) kmem_free(mlinkp, (sizeof (struct modlinkage) + sizeof (struct modlpsm) + sizeof (struct psm_sw))); } int psm_mod_init(void **handlepp, struct psm_info *infop) { struct modlinkage **modlpp = (struct modlinkage **)handlepp; int status; struct modlinkage *mlinkp; if (!*modlpp) { mlinkp = psm_modlinkage_alloc(infop); if (!mlinkp) return (ENOSPC); } else mlinkp = *modlpp; status = mod_install(mlinkp); if (status) { psm_modlinkage_free(mlinkp); *modlpp = NULL; } else *modlpp = mlinkp; return (status); } /*ARGSUSED1*/ int psm_mod_fini(void **handlepp, struct psm_info *infop) { struct modlinkage **modlpp = (struct modlinkage **)handlepp; int status; status = mod_remove(*modlpp); if (status == 0) { psm_modlinkage_free(*modlpp); *modlpp = NULL; } return (status); } int psm_mod_info(void **handlepp, struct psm_info *infop, struct modinfo *modinfop) { struct modlinkage **modlpp = (struct modlinkage **)handlepp; int status; struct modlinkage *mlinkp; if (!*modlpp) { mlinkp = psm_modlinkage_alloc(infop); if (!mlinkp) return (0); } else mlinkp = *modlpp; status = mod_info(mlinkp, modinfop); if (!status) { psm_modlinkage_free(mlinkp); *modlpp = NULL; } else *modlpp = mlinkp; return (status); } int psm_add_intr(int lvl, avfunc xxintr, char *name, int vect, caddr_t arg) { return (add_avintr((void *)NULL, lvl, xxintr, name, vect, arg, NULL, NULL, NULL)); } int psm_add_nmintr(int lvl, avfunc xxintr, char *name, caddr_t arg) { return (add_nmintr(lvl, xxintr, name, arg)); } processorid_t psm_get_cpu_id(void) { return (CPU->cpu_id); } caddr_t psm_map_phys_new(paddr_t addr, size_t len, int prot) { uint_t pgoffset; paddr_t base; pgcnt_t npages; caddr_t cvaddr; if (len == 0) return (0); pgoffset = addr & MMU_PAGEOFFSET; #ifdef __xpv /* * If we're dom0, we're starting from a MA. translate that to a PA * XXPV - what about driver domains??? */ if (DOMAIN_IS_INITDOMAIN(xen_info)) { base = pfn_to_pa(xen_assign_pfn(mmu_btop(addr))) | (addr & MMU_PAGEOFFSET); } else { base = addr; } #else base = addr; #endif npages = mmu_btopr(len + pgoffset); cvaddr = device_arena_alloc(ptob(npages), VM_NOSLEEP); if (cvaddr == NULL) return (0); hat_devload(kas.a_hat, cvaddr, mmu_ptob(npages), mmu_btop(base), prot, HAT_LOAD_LOCK); return (cvaddr + pgoffset); } void psm_unmap_phys(caddr_t addr, size_t len) { uint_t pgoffset; caddr_t base; pgcnt_t npages; if (len == 0) return; pgoffset = (uintptr_t)addr & MMU_PAGEOFFSET; base = addr - pgoffset; npages = mmu_btopr(len + pgoffset); hat_unload(kas.a_hat, base, ptob(npages), HAT_UNLOAD_UNLOCK); device_arena_free(base, ptob(npages)); } caddr_t psm_map_new(paddr_t addr, size_t len, int prot) { int phys_prot = PROT_READ; ASSERT(prot == (prot & (PSM_PROT_WRITE | PSM_PROT_READ))); if (prot & PSM_PROT_WRITE) phys_prot |= PROT_WRITE; return (psm_map_phys(addr, len, phys_prot)); } #undef psm_map_phys #undef psm_map caddr_t psm_map_phys(uint32_t addr, size_t len, int prot) { return (psm_map_phys_new((paddr_t)(addr & 0xffffffff), len, prot)); } caddr_t psm_map(uint32_t addr, size_t len, int prot) { return (psm_map_new((paddr_t)(addr & 0xffffffff), len, prot)); } void psm_unmap(caddr_t addr, size_t len) { uint_t pgoffset; caddr_t base; pgcnt_t npages; if (len == 0) return; pgoffset = (uintptr_t)addr & MMU_PAGEOFFSET; base = addr - pgoffset; npages = mmu_btopr(len + pgoffset); hat_unload(kas.a_hat, base, ptob(npages), HAT_UNLOAD_UNLOCK); device_arena_free(base, ptob(npages)); } /*ARGSUSED1*/ static int mod_installpsm(struct modlpsm *modl, struct modlinkage *modlp) { struct psm_sw *swp; swp = modl->psm_swp; mutex_enter(&psmsw_lock); psmsw->psw_back->psw_forw = swp; swp->psw_back = psmsw->psw_back; swp->psw_forw = psmsw; psmsw->psw_back = swp; swp->psw_flag |= PSM_MOD_INSTALL; mutex_exit(&psmsw_lock); return (0); } /*ARGSUSED1*/ static int mod_removepsm(struct modlpsm *modl, struct modlinkage *modlp) { struct psm_sw *swp; swp = modl->psm_swp; mutex_enter(&psmsw_lock); if (swp->psw_flag & PSM_MOD_IDENTIFY) { mutex_exit(&psmsw_lock); return (EBUSY); } if (!(swp->psw_flag & PSM_MOD_INSTALL)) { mutex_exit(&psmsw_lock); return (0); } swp->psw_back->psw_forw = swp->psw_forw; swp->psw_forw->psw_back = swp->psw_back; mutex_exit(&psmsw_lock); return (0); } /*ARGSUSED1*/ static int mod_infopsm(struct modlpsm *modl, struct modlinkage *modlp, int *p0) { *p0 = (int)modl->psm_swp->psw_infop->p_owner; return (0); } #if defined(__xpv) #define DEFAULT_PSM_MODULE "xpv_uppc" #else #define DEFAULT_PSM_MODULE "uppc" #endif static char * psm_get_impl_module(int first) { static char **pnamep; static char *psm_impl_module_list[] = { DEFAULT_PSM_MODULE, (char *)0 }; static void *mhdl = NULL; static char machname[MAXNAMELEN]; if (first) pnamep = psm_impl_module_list; if (*pnamep != (char *)0) return (*pnamep++); mhdl = get_next_mach(mhdl, machname); if (mhdl) return (machname); return ((char *)0); } void psm_modload(void) { char *this; mutex_init(&psmsw_lock, NULL, MUTEX_DEFAULT, NULL); open_mach_list(); for (this = psm_get_impl_module(1); this != (char *)NULL; this = psm_get_impl_module(0)) { if (prom_debug) prom_printf("%s:%d: psm load %s\n", __FILE__, __LINE__, this); if (modload("mach", this) == -1) cmn_err(CE_CONT, "!Skipping psm: %s\n", this); } close_mach_list(); } void psm_install(void) { struct psm_sw *swp, *cswp; struct psm_ops *opsp; char machstring[15]; int err, psmcnt = 0; mutex_enter(&psmsw_lock); for (swp = psmsw->psw_forw; swp != psmsw; ) { PRM_DEBUGS(swp->psw_infop->p_mach_idstring); opsp = swp->psw_infop->p_ops; if (opsp->psm_probe) { PRM_POINT("psm_probe()"); if ((*opsp->psm_probe)() == PSM_SUCCESS) { PRM_POINT("psm_probe() PSM_SUCCESS"); psmcnt++; swp->psw_flag |= PSM_MOD_IDENTIFY; swp = swp->psw_forw; continue; } PRM_POINT("psm_probe() FAILURE"); } /* remove the unsuccessful psm modules */ cswp = swp; swp = swp->psw_forw; mutex_exit(&psmsw_lock); (void) strcpy(&machstring[0], cswp->psw_infop->p_mach_idstring); err = mod_remove_by_name(cswp->psw_infop->p_mach_idstring); if (err) cmn_err(CE_WARN, "!%s: mod_remove_by_name failed %d", &machstring[0], err); mutex_enter(&psmsw_lock); } mutex_exit(&psmsw_lock); if (psmcnt == 0) halt("the operating system does not yet support this hardware"); PRM_POINT("psminitf()"); (*psminitf)(); PRM_POINT("psminitf() done"); } /* * Return 1 if kernel debugger is present, and 0 if not. */ int psm_debugger(void) { return ((boothowto & RB_DEBUG) != 0); } /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved. */ /* * Copyright (c) 2009-2010, Intel Corporation. * All rights reserved. * Copyright 2020 Joyent, Inc. * Copyright 2023 Oxide Computer Company */ #define PSMI_1_7 #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #if defined(__xpv) #include #endif #include #include #include #include #include #include #include #include #include #include #define OFFSETOF(s, m) (size_t)(&(((s *)0)->m)) /* * Local function prototypes */ static int mp_disable_intr(processorid_t cpun); static void mp_enable_intr(processorid_t cpun); static void mach_init(); static void mach_picinit(); static int machhztomhz(uint64_t cpu_freq_hz); static uint64_t mach_getcpufreq(void); static void mach_fixcpufreq(void); static int mach_clkinit(int, int *); static void mach_smpinit(void); static int mach_softlvl_to_vect(int ipl); static void mach_get_platform(int owner); static void mach_construct_info(); static int mach_translate_irq(dev_info_t *dip, int irqno); static int mach_intr_ops(dev_info_t *, ddi_intr_handle_impl_t *, psm_intr_op_t, int *); static void mach_notify_error(int level, char *errmsg); static hrtime_t dummy_hrtime(void); static void dummy_scalehrtime(hrtime_t *); static uint64_t dummy_unscalehrtime(hrtime_t); void cpu_idle(void); static void cpu_wakeup(cpu_t *, int); #ifndef __xpv void cpu_idle_mwait(void); static void cpu_wakeup_mwait(cpu_t *, int); #endif static int mach_cpu_create_devinfo(cpu_t *cp, dev_info_t **dipp); /* * External reference functions */ extern void return_instr(); extern void pc_gethrestime(timestruc_t *); extern int cpuid_get_coreid(cpu_t *); extern int cpuid_get_chipid(cpu_t *); /* * PSM functions initialization */ void (*psm_shutdownf)(int, int) = (void (*)(int, int))return_instr; void (*psm_preshutdownf)(int, int) = (void (*)(int, int))return_instr; void (*psm_notifyf)(int) = (void (*)(int))return_instr; void (*psm_set_idle_cpuf)(int) = (void (*)(int))return_instr; void (*psm_unset_idle_cpuf)(int) = (void (*)(int))return_instr; void (*psminitf)() = mach_init; void (*picinitf)() = return_instr; int (*clkinitf)(int, int *) = (int (*)(int, int *))return_instr; int (*ap_mlsetup)() = (int (*)(void))return_instr; void (*send_dirintf)() = return_instr; void (*setspl)(int) = (void (*)(int))return_instr; int (*addspl)(int, int, int, int) = (int (*)(int, int, int, int))return_instr; int (*delspl)(int, int, int, int) = (int (*)(int, int, int, int))return_instr; int (*get_pending_spl)(void) = (int (*)(void))return_instr; int (*addintr)(void *, int, avfunc, char *, int, caddr_t, caddr_t, uint64_t *, dev_info_t *) = NULL; void (*remintr)(void *, int, avfunc, int) = NULL; void (*kdisetsoftint)(int, struct av_softinfo *)= (void (*)(int, struct av_softinfo *))return_instr; void (*setsoftint)(int, struct av_softinfo *)= (void (*)(int, struct av_softinfo *))return_instr; int (*slvltovect)(int) = (int (*)(int))return_instr; int (*setlvl)(int, int *) = (int (*)(int, int *))return_instr; void (*setlvlx)(int, int) = (void (*)(int, int))return_instr; int (*psm_disable_intr)(int) = mp_disable_intr; void (*psm_enable_intr)(int) = mp_enable_intr; hrtime_t (*gethrtimef)(void) = dummy_hrtime; hrtime_t (*gethrtimeunscaledf)(void) = dummy_hrtime; void (*scalehrtimef)(hrtime_t *) = dummy_scalehrtime; uint64_t (*unscalehrtimef)(hrtime_t) = dummy_unscalehrtime; int (*psm_translate_irq)(dev_info_t *, int) = mach_translate_irq; void (*gethrestimef)(timestruc_t *) = pc_gethrestime; void (*psm_notify_error)(int, char *) = (void (*)(int, char *))NULL; int (*psm_get_clockirq)(int) = NULL; int (*psm_get_ipivect)(int, int) = NULL; uchar_t (*psm_get_ioapicid)(uchar_t) = NULL; uint32_t (*psm_get_localapicid)(uint32_t) = NULL; uchar_t (*psm_xlate_vector_by_irq)(uchar_t) = NULL; int (*psm_get_pir_ipivect)(void) = NULL; void (*psm_send_pir_ipi)(processorid_t) = NULL; void (*psm_cmci_setup)(processorid_t, boolean_t) = NULL; int (*psm_clkinit)(int) = NULL; void (*psm_timer_reprogram)(hrtime_t) = NULL; void (*psm_timer_enable)(void) = NULL; void (*psm_timer_disable)(void) = NULL; void (*psm_post_cyclic_setup)(void *arg) = NULL; int (*psm_intr_ops)(dev_info_t *, ddi_intr_handle_impl_t *, psm_intr_op_t, int *) = mach_intr_ops; int (*psm_state)(psm_state_request_t *) = (int (*)(psm_state_request_t *)) return_instr; void (*notify_error)(int, char *) = (void (*)(int, char *))return_instr; void (*hrtime_tick)(void) = return_instr; int (*psm_cpu_create_devinfo)(cpu_t *, dev_info_t **) = mach_cpu_create_devinfo; int (*psm_cpu_get_devinfo)(cpu_t *, dev_info_t **) = NULL; /* global IRM pool for APIX (PSM) module */ ddi_irm_pool_t *apix_irm_pool_p = NULL; /* * True if the generic TSC code is our source of hrtime, rather than whatever * the PSM can provide. */ #ifdef __xpv int tsc_gethrtime_enable = 0; #else int tsc_gethrtime_enable = 1; #endif int tsc_gethrtime_initted = 0; /* * True if the hrtime implementation is "hires"; namely, better than microdata. */ int gethrtime_hires = 0; /* * Local Static Data */ static struct psm_ops mach_ops; static struct psm_ops *mach_set[4] = {&mach_ops, NULL, NULL, NULL}; static ushort_t mach_ver[4] = {0, 0, 0, 0}; /* * virtualization support for psm */ void *psm_vt_ops = NULL; /* * If non-zero, idle cpus will become "halted" when there's * no work to do. */ int idle_cpu_use_hlt = 1; #ifndef __xpv /* * If non-zero, idle cpus will use mwait if available to halt instead of hlt. */ int idle_cpu_prefer_mwait = 1; /* * Set to 0 to avoid MONITOR+CLFLUSH assertion. */ int idle_cpu_assert_cflush_monitor = 1; /* * If non-zero, idle cpus will not use power saving Deep C-States idle loop. */ int idle_cpu_no_deep_c = 0; /* * Non-power saving idle loop and wakeup pointers. * Allows user to toggle Deep Idle power saving feature on/off. */ void (*non_deep_idle_cpu)() = cpu_idle; void (*non_deep_idle_disp_enq_thread)(cpu_t *, int); /* * Object for the kernel to access the HPET. */ hpet_t hpet; #endif /* ifndef __xpv */ uint_t cp_haltset_fanout = 0; /*ARGSUSED*/ int pg_plat_hw_shared(cpu_t *cp, pghw_type_t hw) { switch (hw) { case PGHW_IPIPE: if (is_x86_feature(x86_featureset, X86FSET_HTT)) { /* * Hyper-threading is SMT */ return (1); } else { return (0); } case PGHW_FPU: if (cpuid_get_cores_per_compunit(cp) > 1) return (1); else return (0); case PGHW_PROCNODE: if (cpuid_get_procnodes_per_pkg(cp) > 1) return (1); else return (0); case PGHW_CHIP: if (is_x86_feature(x86_featureset, X86FSET_CMP) || is_x86_feature(x86_featureset, X86FSET_HTT)) return (1); else return (0); case PGHW_CACHE: if (cpuid_get_ncpu_sharing_last_cache(cp) > 1) return (1); else return (0); case PGHW_POW_ACTIVE: if (cpupm_domain_id(cp, CPUPM_DTYPE_ACTIVE) != (id_t)-1) return (1); else return (0); case PGHW_POW_IDLE: if (cpupm_domain_id(cp, CPUPM_DTYPE_IDLE) != (id_t)-1) return (1); else return (0); default: return (0); } } /* * Compare two CPUs and see if they have a pghw_type_t sharing relationship * If pghw_type_t is an unsupported hardware type, then return -1 */ int pg_plat_cpus_share(cpu_t *cpu_a, cpu_t *cpu_b, pghw_type_t hw) { id_t pgp_a, pgp_b; pgp_a = pg_plat_hw_instance_id(cpu_a, hw); pgp_b = pg_plat_hw_instance_id(cpu_b, hw); if (pgp_a == -1 || pgp_b == -1) return (-1); return (pgp_a == pgp_b); } /* * Return a physical instance identifier for known hardware sharing * relationships */ id_t pg_plat_hw_instance_id(cpu_t *cpu, pghw_type_t hw) { switch (hw) { case PGHW_IPIPE: return (cpuid_get_coreid(cpu)); case PGHW_CACHE: return (cpuid_get_last_lvl_cacheid(cpu)); case PGHW_FPU: return (cpuid_get_compunitid(cpu)); case PGHW_PROCNODE: return (cpuid_get_procnodeid(cpu)); case PGHW_CHIP: return (cpuid_get_chipid(cpu)); case PGHW_POW_ACTIVE: return (cpupm_domain_id(cpu, CPUPM_DTYPE_ACTIVE)); case PGHW_POW_IDLE: return (cpupm_domain_id(cpu, CPUPM_DTYPE_IDLE)); default: return (-1); } } /* * Express preference for optimizing for sharing relationship * hw1 vs hw2 */ pghw_type_t pg_plat_hw_rank(pghw_type_t hw1, pghw_type_t hw2) { int i, rank1, rank2; static pghw_type_t hw_hier[] = { PGHW_IPIPE, PGHW_CACHE, PGHW_FPU, PGHW_PROCNODE, PGHW_CHIP, PGHW_POW_IDLE, PGHW_POW_ACTIVE, PGHW_NUM_COMPONENTS }; rank1 = 0; rank2 = 0; for (i = 0; hw_hier[i] != PGHW_NUM_COMPONENTS; i++) { if (hw_hier[i] == hw1) rank1 = i; if (hw_hier[i] == hw2) rank2 = i; } if (rank1 > rank2) return (hw1); else return (hw2); } /* * Override the default CMT dispatcher policy for the specified * hardware sharing relationship */ pg_cmt_policy_t pg_plat_cmt_policy(pghw_type_t hw) { /* * For shared caches, also load balance across them to * maximize aggregate cache capacity * * On AMD family 0x15 CPUs, cores come in pairs called * compute units, sharing the FPU and the I$ and L2 * caches. Use balancing and cache affinity. */ switch (hw) { case PGHW_FPU: case PGHW_CACHE: return (CMT_BALANCE|CMT_AFFINITY); default: return (CMT_NO_POLICY); } } id_t pg_plat_get_core_id(cpu_t *cpu) { return ((id_t)cpuid_get_coreid(cpu)); } void cmp_set_nosteal_interval(void) { /* Set the nosteal interval (used by disp_getbest()) to 100us */ nosteal_nsec = 100000UL; } /* * Routine to ensure initial callers to hrtime gets 0 as return */ static hrtime_t dummy_hrtime(void) { return (0); } /* ARGSUSED */ static void dummy_scalehrtime(hrtime_t *ticks) {} static uint64_t dummy_unscalehrtime(hrtime_t nsecs) { return ((uint64_t)nsecs); } /* * Supports Deep C-State power saving idle loop. */ void cpu_idle_adaptive(void) { (*CPU->cpu_m.mcpu_idle_cpu)(); } /* * Function called by CPU idle notification framework to check whether CPU * has been awakened. It will be called with interrupt disabled. * If CPU has been awakened, call cpu_idle_exit() to notify CPU idle * notification framework. */ /*ARGSUSED*/ static void cpu_idle_check_wakeup(void *arg) { /* * Toggle interrupt flag to detect pending interrupts. * If interrupt happened, do_interrupt() will notify CPU idle * notification framework so no need to call cpu_idle_exit() here. */ sti(); SMT_PAUSE(); cli(); } /* * Idle the present CPU until wakened via an interrupt */ void cpu_idle(void) { cpu_t *cpup = CPU; processorid_t cpu_sid = cpup->cpu_seqid; cpupart_t *cp = cpup->cpu_part; int hset_update = 1; /* * If this CPU is online, and there's multiple CPUs * in the system, then we should notate our halting * by adding ourselves to the partition's halted CPU * bitmap. This allows other CPUs to find/awaken us when * work becomes available. */ if (cpup->cpu_flags & CPU_OFFLINE || ncpus == 1) hset_update = 0; /* * Add ourselves to the partition's halted CPUs bitmap * and set our HALTED flag, if necessary. * * When a thread becomes runnable, it is placed on the queue * and then the halted CPU bitmap is checked to determine who * (if anyone) should be awakened. We therefore need to first * add ourselves to the bitmap, and and then check if there * is any work available. The order is important to prevent a race * that can lead to work languishing on a run queue somewhere while * this CPU remains halted. * * Either the producing CPU will see we're halted and will awaken us, * or this CPU will see the work available in disp_anywork(). * * Note that memory barriers after updating the HALTED flag * are not necessary since an atomic operation (updating the bitset) * immediately follows. On x86 the atomic operation acts as a * memory barrier for the update of cpu_disp_flags. */ if (hset_update) { cpup->cpu_disp_flags |= CPU_DISP_HALTED; bitset_atomic_add(&cp->cp_haltset, cpu_sid); } /* * Check to make sure there's really nothing to do. * Work destined for this CPU may become available after * this check. We'll be notified through the clearing of our * bit in the halted CPU bitmap, and a poke. */ if (disp_anywork()) { if (hset_update) { cpup->cpu_disp_flags &= ~CPU_DISP_HALTED; bitset_atomic_del(&cp->cp_haltset, cpu_sid); } return; } /* * We're on our way to being halted. * * Disable interrupts now, so that we'll awaken immediately * after halting if someone tries to poke us between now and * the time we actually halt. * * We check for the presence of our bit after disabling interrupts. * If it's cleared, we'll return. If the bit is cleared after * we check then the poke will pop us out of the halted state. * * This means that the ordering of the poke and the clearing * of the bit by cpu_wakeup is important. * cpu_wakeup() must clear, then poke. * cpu_idle() must disable interrupts, then check for the bit. */ cli(); if (hset_update && bitset_in_set(&cp->cp_haltset, cpu_sid) == 0) { cpup->cpu_disp_flags &= ~CPU_DISP_HALTED; sti(); return; } /* * The check for anything locally runnable is here for performance * and isn't needed for correctness. disp_nrunnable ought to be * in our cache still, so it's inexpensive to check, and if there * is anything runnable we won't have to wait for the poke. */ if (cpup->cpu_disp->disp_nrunnable != 0) { if (hset_update) { cpup->cpu_disp_flags &= ~CPU_DISP_HALTED; bitset_atomic_del(&cp->cp_haltset, cpu_sid); } sti(); return; } if (cpu_idle_enter(IDLE_STATE_C1, 0, cpu_idle_check_wakeup, NULL) == 0) { mach_cpu_idle(); cpu_idle_exit(CPU_IDLE_CB_FLAG_IDLE); } /* * We're no longer halted */ if (hset_update) { cpup->cpu_disp_flags &= ~CPU_DISP_HALTED; bitset_atomic_del(&cp->cp_haltset, cpu_sid); } } /* * If "cpu" is halted, then wake it up clearing its halted bit in advance. * Otherwise, see if other CPUs in the cpu partition are halted and need to * be woken up so that they can steal the thread we placed on this CPU. * This function is only used on MP systems. */ static void cpu_wakeup(cpu_t *cpu, int bound) { uint_t cpu_found; processorid_t cpu_sid; cpupart_t *cp; cp = cpu->cpu_part; cpu_sid = cpu->cpu_seqid; if (bitset_in_set(&cp->cp_haltset, cpu_sid)) { /* * Clear the halted bit for that CPU since it will be * poked in a moment. */ bitset_atomic_del(&cp->cp_haltset, cpu_sid); /* * We may find the current CPU present in the halted cpuset * if we're in the context of an interrupt that occurred * before we had a chance to clear our bit in cpu_idle(). * Poking ourself is obviously unnecessary, since if * we're here, we're not halted. */ if (cpu != CPU) poke_cpu(cpu->cpu_id); return; } else { /* * This cpu isn't halted, but it's idle or undergoing a * context switch. No need to awaken anyone else. */ if (cpu->cpu_thread == cpu->cpu_idle_thread || cpu->cpu_disp_flags & CPU_DISP_DONTSTEAL) return; } /* * No need to wake up other CPUs if this is for a bound thread. */ if (bound) return; /* * The CPU specified for wakeup isn't currently halted, so check * to see if there are any other halted CPUs in the partition, * and if there are then awaken one. */ do { cpu_found = bitset_find(&cp->cp_haltset); if (cpu_found == (uint_t)-1) return; } while (bitset_atomic_test_and_del(&cp->cp_haltset, cpu_found) < 0); if (cpu_found != CPU->cpu_seqid) { poke_cpu(cpu_seq[cpu_found]->cpu_id); } } #ifndef __xpv /* * Function called by CPU idle notification framework to check whether CPU * has been awakened. It will be called with interrupt disabled. * If CPU has been awakened, call cpu_idle_exit() to notify CPU idle * notification framework. */ static void cpu_idle_mwait_check_wakeup(void *arg) { volatile uint32_t *mcpu_mwait = (volatile uint32_t *)arg; ASSERT(arg != NULL); if (*mcpu_mwait != MWAIT_HALTED) { /* * CPU has been awakened, notify CPU idle notification system. */ cpu_idle_exit(CPU_IDLE_CB_FLAG_IDLE); } else { /* * Toggle interrupt flag to detect pending interrupts. * If interrupt happened, do_interrupt() will notify CPU idle * notification framework so no need to call cpu_idle_exit() * here. */ sti(); SMT_PAUSE(); cli(); } } /* * Idle the present CPU until awakened via touching its monitored line */ void cpu_idle_mwait(void) { volatile uint32_t *mcpu_mwait = CPU->cpu_m.mcpu_mwait; cpu_t *cpup = CPU; processorid_t cpu_sid = cpup->cpu_seqid; cpupart_t *cp = cpup->cpu_part; int hset_update = 1; /* * Set our mcpu_mwait here, so we can tell if anyone tries to * wake us between now and when we call mwait. No other cpu will * attempt to set our mcpu_mwait until we add ourself to the halted * CPU bitmap. */ *mcpu_mwait = MWAIT_HALTED; /* * If this CPU is online, and there's multiple CPUs * in the system, then we should note our halting * by adding ourselves to the partition's halted CPU * bitmap. This allows other CPUs to find/awaken us when * work becomes available. */ if (cpup->cpu_flags & CPU_OFFLINE || ncpus == 1) hset_update = 0; /* * Add ourselves to the partition's halted CPUs bitmap * and set our HALTED flag, if necessary. * * When a thread becomes runnable, it is placed on the queue * and then the halted CPU bitmap is checked to determine who * (if anyone) should be awakened. We therefore need to first * add ourselves to the bitmap, and and then check if there * is any work available. * * Note that memory barriers after updating the HALTED flag * are not necessary since an atomic operation (updating the bitmap) * immediately follows. On x86 the atomic operation acts as a * memory barrier for the update of cpu_disp_flags. */ if (hset_update) { cpup->cpu_disp_flags |= CPU_DISP_HALTED; bitset_atomic_add(&cp->cp_haltset, cpu_sid); } /* * Check to make sure there's really nothing to do. * Work destined for this CPU may become available after * this check. We'll be notified through the clearing of our * bit in the halted CPU bitmap, and a write to our mcpu_mwait. * * disp_anywork() checks disp_nrunnable, so we do not have to later. */ if (disp_anywork()) { if (hset_update) { cpup->cpu_disp_flags &= ~CPU_DISP_HALTED; bitset_atomic_del(&cp->cp_haltset, cpu_sid); } return; } /* * We're on our way to being halted. * To avoid a lost wakeup, arm the monitor before checking if another * cpu wrote to mcpu_mwait to wake us up. */ i86_monitor(mcpu_mwait, 0, 0); if (*mcpu_mwait == MWAIT_HALTED) { if (cpu_idle_enter(IDLE_STATE_C1, 0, cpu_idle_mwait_check_wakeup, (void *)mcpu_mwait) == 0) { if (*mcpu_mwait == MWAIT_HALTED) { i86_mwait(0, 0); } cpu_idle_exit(CPU_IDLE_CB_FLAG_IDLE); } } /* * We're no longer halted */ if (hset_update) { cpup->cpu_disp_flags &= ~CPU_DISP_HALTED; bitset_atomic_del(&cp->cp_haltset, cpu_sid); } } /* * If "cpu" is halted in mwait, then wake it up clearing its halted bit in * advance. Otherwise, see if other CPUs in the cpu partition are halted and * need to be woken up so that they can steal the thread we placed on this CPU. * This function is only used on MP systems. */ static void cpu_wakeup_mwait(cpu_t *cp, int bound) { cpupart_t *cpu_part; uint_t cpu_found; processorid_t cpu_sid; cpu_part = cp->cpu_part; cpu_sid = cp->cpu_seqid; /* * Clear the halted bit for that CPU since it will be woken up * in a moment. */ if (bitset_in_set(&cpu_part->cp_haltset, cpu_sid)) { /* * Clear the halted bit for that CPU since it will be * poked in a moment. */ bitset_atomic_del(&cpu_part->cp_haltset, cpu_sid); /* * We may find the current CPU present in the halted cpuset * if we're in the context of an interrupt that occurred * before we had a chance to clear our bit in cpu_idle(). * Waking ourself is obviously unnecessary, since if * we're here, we're not halted. * * monitor/mwait wakeup via writing to our cache line is * harmless and less expensive than always checking if we * are waking ourself which is an uncommon case. */ MWAIT_WAKEUP(cp); /* write to monitored line */ return; } else { /* * This cpu isn't halted, but it's idle or undergoing a * context switch. No need to awaken anyone else. */ if (cp->cpu_thread == cp->cpu_idle_thread || cp->cpu_disp_flags & CPU_DISP_DONTSTEAL) return; } /* * No need to wake up other CPUs if the thread we just enqueued * is bound. */ if (bound || ncpus == 1) return; /* * See if there's any other halted CPUs. If there are, then * select one, and awaken it. * It's possible that after we find a CPU, somebody else * will awaken it before we get the chance. * In that case, look again. */ do { cpu_found = bitset_find(&cpu_part->cp_haltset); if (cpu_found == (uint_t)-1) return; } while (bitset_atomic_test_and_del(&cpu_part->cp_haltset, cpu_found) < 0); /* * Do not check if cpu_found is ourself as monitor/mwait * wakeup is cheap. */ MWAIT_WAKEUP(cpu_seq[cpu_found]); /* write to monitored line */ } #endif void (*cpu_pause_handler)(volatile char *) = NULL; static int mp_disable_intr(int cpun) { /* * switch to the offline cpu */ affinity_set(cpun); /* * raise ipl to just below cross call */ splx(XC_SYS_PIL - 1); /* * set base spl to prevent the next swtch to idle from * lowering back to ipl 0 */ CPU->cpu_intr_actv |= (1 << (XC_SYS_PIL - 1)); set_base_spl(); affinity_clear(); return (DDI_SUCCESS); } static void mp_enable_intr(int cpun) { /* * switch to the online cpu */ affinity_set(cpun); /* * clear the interrupt active mask */ CPU->cpu_intr_actv &= ~(1 << (XC_SYS_PIL - 1)); set_base_spl(); (void) spl0(); affinity_clear(); } static void mach_get_platform(int owner) { void **srv_opsp; void **clt_opsp; int i; int total_ops; /* fix up psm ops */ srv_opsp = (void **)mach_set[0]; clt_opsp = (void **)mach_set[owner]; if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01) total_ops = sizeof (struct psm_ops_ver01) / sizeof (void (*)(void)); else if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01_1) /* no psm_notify_func */ total_ops = OFFSETOF(struct psm_ops, psm_notify_func) / sizeof (void (*)(void)); else if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01_2) /* no psm_timer funcs */ total_ops = OFFSETOF(struct psm_ops, psm_timer_reprogram) / sizeof (void (*)(void)); else if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01_3) /* no psm_preshutdown function */ total_ops = OFFSETOF(struct psm_ops, psm_preshutdown) / sizeof (void (*)(void)); else if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01_4) /* no psm_intr_ops function */ total_ops = OFFSETOF(struct psm_ops, psm_intr_ops) / sizeof (void (*)(void)); else if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01_5) /* no psm_state function */ total_ops = OFFSETOF(struct psm_ops, psm_state) / sizeof (void (*)(void)); else if (mach_ver[owner] == (ushort_t)PSM_INFO_VER01_6) /* no psm_cpu_ops function */ total_ops = OFFSETOF(struct psm_ops, psm_cpu_ops) / sizeof (void (*)(void)); else total_ops = sizeof (struct psm_ops) / sizeof (void (*)(void)); /* * Save the version of the PSM module, in case we need to * behave differently based on version. */ mach_ver[0] = mach_ver[owner]; for (i = 0; i < total_ops; i++) if (clt_opsp[i] != NULL) srv_opsp[i] = clt_opsp[i]; } static void mach_construct_info() { struct psm_sw *swp; int mach_cnt[PSM_OWN_OVERRIDE+1] = {0}; int conflict_owner = 0; if (psmsw->psw_forw == psmsw) panic("No valid PSM modules found"); mutex_enter(&psmsw_lock); for (swp = psmsw->psw_forw; swp != psmsw; swp = swp->psw_forw) { if (!(swp->psw_flag & PSM_MOD_IDENTIFY)) continue; mach_set[swp->psw_infop->p_owner] = swp->psw_infop->p_ops; mach_ver[swp->psw_infop->p_owner] = swp->psw_infop->p_version; mach_cnt[swp->psw_infop->p_owner]++; } mutex_exit(&psmsw_lock); mach_get_platform(PSM_OWN_SYS_DEFAULT); /* check to see are there any conflicts */ if (mach_cnt[PSM_OWN_EXCLUSIVE] > 1) conflict_owner = PSM_OWN_EXCLUSIVE; if (mach_cnt[PSM_OWN_OVERRIDE] > 1) conflict_owner = PSM_OWN_OVERRIDE; if (conflict_owner) { /* remove all psm modules except uppc */ cmn_err(CE_WARN, "Conflicts detected on the following PSM modules:"); mutex_enter(&psmsw_lock); for (swp = psmsw->psw_forw; swp != psmsw; swp = swp->psw_forw) { if (swp->psw_infop->p_owner == conflict_owner) cmn_err(CE_WARN, "%s ", swp->psw_infop->p_mach_idstring); } mutex_exit(&psmsw_lock); cmn_err(CE_WARN, "Setting the system back to SINGLE processor mode!"); cmn_err(CE_WARN, "Please edit /etc/mach to remove the invalid PSM module."); return; } if (mach_set[PSM_OWN_EXCLUSIVE]) mach_get_platform(PSM_OWN_EXCLUSIVE); if (mach_set[PSM_OWN_OVERRIDE]) mach_get_platform(PSM_OWN_OVERRIDE); } static void mach_init() { struct psm_ops *pops; PRM_POINT("mach_construct_info()"); mach_construct_info(); pops = mach_set[0]; /* register the interrupt and clock initialization rotuines */ picinitf = mach_picinit; clkinitf = mach_clkinit; psm_get_clockirq = pops->psm_get_clockirq; /* register the interrupt setup code */ slvltovect = mach_softlvl_to_vect; addspl = pops->psm_addspl; delspl = pops->psm_delspl; if (pops->psm_translate_irq) psm_translate_irq = pops->psm_translate_irq; if (pops->psm_intr_ops) psm_intr_ops = pops->psm_intr_ops; #if defined(PSMI_1_2) || defined(PSMI_1_3) || defined(PSMI_1_4) /* * Time-of-day functionality now handled in TOD modules. * (Warn about PSM modules that think that we're going to use * their ops vectors.) */ if (pops->psm_tod_get) cmn_err(CE_WARN, "obsolete psm_tod_get op %p", (void *)pops->psm_tod_get); if (pops->psm_tod_set) cmn_err(CE_WARN, "obsolete psm_tod_set op %p", (void *)pops->psm_tod_set); #endif if (pops->psm_notify_error) { psm_notify_error = mach_notify_error; notify_error = pops->psm_notify_error; } PRM_POINT("psm_softinit()"); (*pops->psm_softinit)(); /* * Initialize the dispatcher's function hooks to enable CPU halting * when idle. Set both the deep-idle and non-deep-idle hooks. * * Assume we can use power saving deep-idle loop cpu_idle_adaptive. * Platform deep-idle driver will reset our idle loop to * non_deep_idle_cpu if power saving deep-idle feature is not available. * * Do not use monitor/mwait if idle_cpu_use_hlt is not set(spin idle) * or idle_cpu_prefer_mwait is not set. * Allocate monitor/mwait buffer for cpu0. */ #ifndef __xpv non_deep_idle_disp_enq_thread = disp_enq_thread; #endif PRM_DEBUG(idle_cpu_use_hlt); if (idle_cpu_use_hlt) { idle_cpu = cpu_idle_adaptive; CPU->cpu_m.mcpu_idle_cpu = cpu_idle; #ifndef __xpv if (is_x86_feature(x86_featureset, X86FSET_MWAIT) && idle_cpu_prefer_mwait) { CPU->cpu_m.mcpu_mwait = cpuid_mwait_alloc(CPU); /* * Protect ourself from insane mwait size. */ if (CPU->cpu_m.mcpu_mwait == NULL) { #ifdef DEBUG cmn_err(CE_NOTE, "Using hlt idle. Cannot " "handle cpu 0 mwait size."); #endif idle_cpu_prefer_mwait = 0; CPU->cpu_m.mcpu_idle_cpu = cpu_idle; } else { CPU->cpu_m.mcpu_idle_cpu = cpu_idle_mwait; } } else { CPU->cpu_m.mcpu_idle_cpu = cpu_idle; } non_deep_idle_cpu = CPU->cpu_m.mcpu_idle_cpu; /* * Disable power saving deep idle loop? */ if (idle_cpu_no_deep_c) { idle_cpu = non_deep_idle_cpu; } #endif } PRM_POINT("mach_smpinit()"); mach_smpinit(); } static void mach_smpinit(void) { struct psm_ops *pops; processorid_t cpu_id; int cnt; cpuset_t cpumask; pops = mach_set[0]; CPUSET_ZERO(cpumask); cpu_id = -1; cpu_id = (*pops->psm_get_next_processorid)(cpu_id); /* * Only add boot_ncpus CPUs to mp_cpus. Other CPUs will be handled * by CPU DR driver at runtime. */ for (cnt = 0; cpu_id != -1 && cnt < boot_ncpus; cnt++) { CPUSET_ADD(cpumask, cpu_id); cpu_id = (*pops->psm_get_next_processorid)(cpu_id); } mp_cpus = cpumask; /* MP related routines */ ap_mlsetup = pops->psm_post_cpu_start; send_dirintf = pops->psm_send_ipi; /* optional MP related routines */ if (pops->psm_shutdown) psm_shutdownf = pops->psm_shutdown; if (pops->psm_preshutdown) psm_preshutdownf = pops->psm_preshutdown; if (pops->psm_notify_func) psm_notifyf = pops->psm_notify_func; if (pops->psm_set_idlecpu) psm_set_idle_cpuf = pops->psm_set_idlecpu; if (pops->psm_unset_idlecpu) psm_unset_idle_cpuf = pops->psm_unset_idlecpu; psm_clkinit = pops->psm_clkinit; if (pops->psm_timer_reprogram) psm_timer_reprogram = pops->psm_timer_reprogram; if (pops->psm_timer_enable) psm_timer_enable = pops->psm_timer_enable; if (pops->psm_timer_disable) psm_timer_disable = pops->psm_timer_disable; if (pops->psm_post_cyclic_setup) psm_post_cyclic_setup = pops->psm_post_cyclic_setup; if (pops->psm_state) psm_state = pops->psm_state; /* * Set these vectors here so they can be used by Suspend/Resume * on UP machines. */ if (pops->psm_disable_intr) psm_disable_intr = pops->psm_disable_intr; if (pops->psm_enable_intr) psm_enable_intr = pops->psm_enable_intr; /* * Set this vector so it can be used by vmbus (for Hyper-V) * Need this even for single-CPU systems. This works for * "pcplusmp" and "apix" platforms, but not "uppc" (because * "Uni-processor PC" does not provide a _get_ipivect). */ psm_get_ipivect = pops->psm_get_ipivect; /* check for multiple CPUs */ if (cnt < 2 && plat_dr_support_cpu() == B_FALSE) return; /* check for MP platforms */ if (pops->psm_cpu_start == NULL) return; /* * Set the dispatcher hook to enable cpu "wake up" * when a thread becomes runnable. */ if (idle_cpu_use_hlt) { disp_enq_thread = cpu_wakeup; #ifndef __xpv if (is_x86_feature(x86_featureset, X86FSET_MWAIT) && idle_cpu_prefer_mwait) disp_enq_thread = cpu_wakeup_mwait; non_deep_idle_disp_enq_thread = disp_enq_thread; #endif } psm_get_pir_ipivect = pops->psm_get_pir_ipivect; psm_send_pir_ipi = pops->psm_send_pir_ipi; psm_cmci_setup = pops->psm_cmci_setup; (void) add_avintr((void *)NULL, XC_HI_PIL, xc_serv, "xc_intr", (*pops->psm_get_ipivect)(XC_HI_PIL, PSM_INTR_IPI_HI), NULL, NULL, NULL, NULL); (void) (*pops->psm_get_ipivect)(XC_CPUPOKE_PIL, PSM_INTR_POKE); } static void mach_picinit() { struct psm_ops *pops; pops = mach_set[0]; /* register the interrupt handlers */ setlvl = pops->psm_intr_enter; setlvlx = pops->psm_intr_exit; /* initialize the interrupt hardware */ (*pops->psm_picinit)(); /* set interrupt mask for current ipl */ setspl = pops->psm_setspl; cli(); setspl(CPU->cpu_pri); } uint_t cpu_freq; /* MHz */ uint64_t cpu_freq_hz; /* measured (in hertz) */ #define MEGA_HZ 1000000 #ifdef __xpv int xpv_cpufreq_workaround = 1; int xpv_cpufreq_verbose = 0; #endif /* __xpv */ static uint64_t mach_getcpufreq(void) { #ifndef __xpv return (tsc_get_freq()); #else vcpu_time_info_t *vti = &CPU->cpu_m.mcpu_vcpu_info->time; uint64_t cpu_hz; /* * During dom0 bringup, it was noted that on at least one older * Intel HT machine, the hypervisor initially gives a tsc_to_system_mul * value that is quite wrong (the 3.06GHz clock was reported * as 4.77GHz) * * The curious thing is, that if you stop the kernel at entry, * breakpoint here and inspect the value with kmdb, the value * is correct - but if you don't stop and simply enable the * printf statement (below), you can see the bad value printed * here. Almost as if something kmdb did caused the hypervisor to * figure it out correctly. And, note that the hypervisor * eventually -does- figure it out correctly ... if you look at * the field later in the life of dom0, it is correct. * * For now, on dom0, we employ a slightly cheesy workaround of * using the DOM0_PHYSINFO hypercall. */ if (DOMAIN_IS_INITDOMAIN(xen_info) && xpv_cpufreq_workaround) { cpu_hz = 1000 * xpv_cpu_khz(); } else { cpu_hz = (UINT64_C(1000000000) << 32) / vti->tsc_to_system_mul; if (vti->tsc_shift < 0) cpu_hz <<= -vti->tsc_shift; else cpu_hz >>= vti->tsc_shift; } if (xpv_cpufreq_verbose) printf("mach_getcpufreq: system_mul 0x%x, shift %d, " "cpu_hz %" PRId64 "Hz\n", vti->tsc_to_system_mul, vti->tsc_shift, cpu_hz); return (cpu_hz); #endif /* __xpv */ } /* * If the clock speed of a cpu is found to be reported incorrectly, do not add * to this array, instead improve the accuracy of the algorithm that determines * the clock speed of the processor or extend the implementation to support the * vendor as appropriate. This is here only to support adjusting the speed on * older slower processors that mach_fixcpufreq() would not be able to account * for otherwise. */ static int x86_cpu_freq[] = { 60, 75, 80, 90, 120, 160, 166, 175, 180, 233 }; /* * On fast processors the clock frequency that is measured may be off by * a few MHz from the value printed on the part. This is a combination of * the factors that for such fast parts being off by this much is within * the tolerances for manufacture and because of the difficulties in the * measurement that can lead to small error. This function uses some * heuristics in order to tweak the value that was measured to match what * is most likely printed on the part. * * Some examples: * AMD Athlon 1000 mhz measured as 998 mhz * Intel Pentium III Xeon 733 mhz measured as 731 mhz * Intel Pentium IV 1500 mhz measured as 1495mhz * * If in the future this function is no longer sufficient to correct * for the error in the measurement, then the algorithm used to perform * the measurement will have to be improved in order to increase accuracy * rather than adding horrible and questionable kludges here. * * This is called after the cyclics subsystem because of the potential * that the heuristics within may give a worse estimate of the clock * frequency than the value that was measured. */ static void mach_fixcpufreq(void) { uint32_t freq, mul, near66, delta66, near50, delta50, fixed, delta, i; freq = (uint32_t)cpu_freq; /* * Find the nearest integer multiple of 200/3 (about 66) MHz to the * measured speed taking into account that the 667 MHz parts were * the first to round-up. */ mul = (uint32_t)((3 * (uint64_t)freq + 100) / 200); near66 = (uint32_t)((200 * (uint64_t)mul + ((mul >= 10) ? 1 : 0)) / 3); delta66 = (near66 > freq) ? (near66 - freq) : (freq - near66); /* Find the nearest integer multiple of 50 MHz to the measured speed */ mul = (freq + 25) / 50; near50 = mul * 50; delta50 = (near50 > freq) ? (near50 - freq) : (freq - near50); /* Find the closer of the two */ if (delta66 < delta50) { fixed = near66; delta = delta66; } else { fixed = near50; delta = delta50; } if (fixed > INT_MAX) return; /* * Some older parts have a core clock frequency that is not an * integral multiple of 50 or 66 MHz. Check if one of the old * clock frequencies is closer to the measured value than any * of the integral multiples of 50 an 66, and if so set fixed * and delta appropriately to represent the closest value. */ i = sizeof (x86_cpu_freq) / sizeof (int); while (i > 0) { i--; if (x86_cpu_freq[i] <= freq) { mul = freq - x86_cpu_freq[i]; if (mul < delta) { fixed = x86_cpu_freq[i]; delta = mul; } break; } mul = x86_cpu_freq[i] - freq; if (mul < delta) { fixed = x86_cpu_freq[i]; delta = mul; } } /* * Set a reasonable maximum for how much to correct the measured * result by. This check is here to prevent the adjustment made * by this function from being more harm than good. It is entirely * possible that in the future parts will be made that are not * integral multiples of 66 or 50 in clock frequency or that * someone may overclock a part to some odd frequency. If the * measured value is farther from the corrected value than * allowed, then assume the corrected value is in error and use * the measured value. */ if (6 < delta) return; cpu_freq = (int)fixed; } static int machhztomhz(uint64_t cpu_freq_hz) { uint64_t cpu_mhz; /* Round to nearest MHZ */ cpu_mhz = (cpu_freq_hz + (MEGA_HZ / 2)) / MEGA_HZ; if (cpu_mhz > INT_MAX) return (0); return ((int)cpu_mhz); } static int mach_clkinit(int preferred_mode, int *set_mode) { struct psm_ops *pops; int resolution; pops = mach_set[0]; cpu_freq_hz = mach_getcpufreq(); cpu_freq = machhztomhz(cpu_freq_hz); /* * For most systems, we retain the default TSC-based gethrtime() * implementation that was initialized early in the boot process. */ #ifdef __xpv if (pops->psm_hrtimeinit) (*pops->psm_hrtimeinit)(); gethrtimef = pops->psm_gethrtime; gethrtimeunscaledf = gethrtimef; /* scalehrtimef will remain dummy */ #endif /* __xpv */ mach_fixcpufreq(); if (mach_ver[0] >= PSM_INFO_VER01_3) { if (preferred_mode == TIMER_ONESHOT) { resolution = (*pops->psm_clkinit)(0); if (resolution != 0) { *set_mode = TIMER_ONESHOT; return (resolution); } } /* * either periodic mode was requested or could not set to * one-shot mode */ resolution = (*pops->psm_clkinit)(hz); /* * psm should be able to do periodic, so we do not check * for return value of psm_clkinit here. */ *set_mode = TIMER_PERIODIC; return (resolution); } else { /* * PSMI interface prior to PSMI_3 does not define a return * value for psm_clkinit, so the return value is ignored. */ (void) (*pops->psm_clkinit)(hz); *set_mode = TIMER_PERIODIC; return (nsec_per_tick); } } /*ARGSUSED*/ static int mach_softlvl_to_vect(int ipl) { setsoftint = av_set_softint_pending; kdisetsoftint = kdi_av_set_softint_pending; return (PSM_SV_SOFTWARE); } #ifdef DEBUG /* * This is here to allow us to simulate cpus that refuse to start. */ cpuset_t cpufailset; #endif int mach_cpu_start(struct cpu *cp, void *ctx) { struct psm_ops *pops = mach_set[0]; processorid_t id = cp->cpu_id; #ifdef DEBUG if (CPU_IN_SET(cpufailset, id)) return (0); #endif return ((*pops->psm_cpu_start)(id, ctx)); } int mach_cpuid_start(processorid_t id, void *ctx) { struct psm_ops *pops = mach_set[0]; #ifdef DEBUG if (CPU_IN_SET(cpufailset, id)) return (0); #endif return ((*pops->psm_cpu_start)(id, ctx)); } int mach_cpu_stop(cpu_t *cp, void *ctx) { struct psm_ops *pops = mach_set[0]; psm_cpu_request_t request; if (pops->psm_cpu_ops == NULL) { return (ENOTSUP); } ASSERT(cp->cpu_id != -1); request.pcr_cmd = PSM_CPU_STOP; request.req.cpu_stop.cpuid = cp->cpu_id; request.req.cpu_stop.ctx = ctx; return ((*pops->psm_cpu_ops)(&request)); } int mach_cpu_add(mach_cpu_add_arg_t *argp, processorid_t *cpuidp) { int rc; struct psm_ops *pops = mach_set[0]; psm_cpu_request_t request; if (pops->psm_cpu_ops == NULL) { return (ENOTSUP); } request.pcr_cmd = PSM_CPU_ADD; request.req.cpu_add.argp = argp; request.req.cpu_add.cpuid = -1; rc = (*pops->psm_cpu_ops)(&request); if (rc == 0) { ASSERT(request.req.cpu_add.cpuid != -1); *cpuidp = request.req.cpu_add.cpuid; } return (rc); } int mach_cpu_remove(processorid_t cpuid) { struct psm_ops *pops = mach_set[0]; psm_cpu_request_t request; if (pops->psm_cpu_ops == NULL) { return (ENOTSUP); } request.pcr_cmd = PSM_CPU_REMOVE; request.req.cpu_remove.cpuid = cpuid; return ((*pops->psm_cpu_ops)(&request)); } /* * Default handler to create device node for CPU. * One reference count will be held on created device node. */ static int mach_cpu_create_devinfo(cpu_t *cp, dev_info_t **dipp) { int rv; dev_info_t *dip; static kmutex_t cpu_node_lock; static dev_info_t *cpu_nex_devi = NULL; ASSERT(cp != NULL); ASSERT(dipp != NULL); *dipp = NULL; if (cpu_nex_devi == NULL) { mutex_enter(&cpu_node_lock); /* First check whether cpus exists. */ cpu_nex_devi = ddi_find_devinfo("cpus", -1, 0); /* Create cpus if it doesn't exist. */ if (cpu_nex_devi == NULL) { ndi_devi_enter(ddi_root_node()); rv = ndi_devi_alloc(ddi_root_node(), "cpus", (pnode_t)DEVI_SID_NODEID, &dip); if (rv != NDI_SUCCESS) { mutex_exit(&cpu_node_lock); cmn_err(CE_CONT, "?failed to create cpu nexus device.\n"); return (PSM_FAILURE); } ASSERT(dip != NULL); (void) ndi_devi_online(dip, 0); ndi_devi_exit(ddi_root_node()); cpu_nex_devi = dip; } mutex_exit(&cpu_node_lock); } /* * create a child node for cpu identified as 'cpu_id' */ ndi_devi_enter(cpu_nex_devi); dip = ddi_add_child(cpu_nex_devi, "cpu", DEVI_SID_NODEID, -1); if (dip == NULL) { cmn_err(CE_CONT, "?failed to create device node for cpu%d.\n", cp->cpu_id); rv = PSM_FAILURE; } else { *dipp = dip; (void) ndi_hold_devi(dip); rv = PSM_SUCCESS; } ndi_devi_exit(cpu_nex_devi); return (rv); } /* * Create cpu device node in device tree and online it. * Return created dip with reference count held if requested. */ int mach_cpu_create_device_node(struct cpu *cp, dev_info_t **dipp) { int rv; dev_info_t *dip = NULL; ASSERT(psm_cpu_create_devinfo != NULL); rv = psm_cpu_create_devinfo(cp, &dip); if (rv == PSM_SUCCESS) { cpuid_set_cpu_properties(dip, cp->cpu_id, cp->cpu_m.mcpu_cpi); /* Recursively attach driver for parent nexus device. */ if (i_ddi_attach_node_hierarchy(ddi_get_parent(dip)) == DDI_SUCCESS) { /* Configure cpu itself and descendants. */ (void) ndi_devi_online(dip, NDI_ONLINE_ATTACH | NDI_CONFIG); } if (dipp != NULL) { *dipp = dip; } else { (void) ndi_rele_devi(dip); } } return (rv); } /* * The dipp contains one of following values on return: * - NULL if no device node found * - pointer to device node if found */ int mach_cpu_get_device_node(struct cpu *cp, dev_info_t **dipp) { *dipp = NULL; if (psm_cpu_get_devinfo != NULL) { if (psm_cpu_get_devinfo(cp, dipp) == PSM_SUCCESS) { return (PSM_SUCCESS); } } return (PSM_FAILURE); } /*ARGSUSED*/ static int mach_translate_irq(dev_info_t *dip, int irqno) { return (irqno); /* default to NO translation */ } static void mach_notify_error(int level, char *errmsg) { /* * SL_FATAL is pass in once panicstr is set, deliver it * as CE_PANIC. Also, translate SL_ codes back to CE_ * codes for the psmi handler */ if (level & SL_FATAL) (*notify_error)(CE_PANIC, errmsg); else if (level & SL_WARN) (*notify_error)(CE_WARN, errmsg); else if (level & SL_NOTE) (*notify_error)(CE_NOTE, errmsg); else if (level & SL_CONSOLE) (*notify_error)(CE_CONT, errmsg); } /* * It provides the default basic intr_ops interface for the new DDI * interrupt framework if the PSM doesn't have one. * * Input: * dip - pointer to the dev_info structure of the requested device * hdlp - pointer to the internal interrupt handle structure for the * requested interrupt * intr_op - opcode for this call * result - pointer to the integer that will hold the result to be * passed back if return value is PSM_SUCCESS * * Output: * return value is either PSM_SUCCESS or PSM_FAILURE */ static int mach_intr_ops(dev_info_t *dip, ddi_intr_handle_impl_t *hdlp, psm_intr_op_t intr_op, int *result) { struct intrspec *ispec; switch (intr_op) { case PSM_INTR_OP_CHECK_MSI: *result = hdlp->ih_type & ~(DDI_INTR_TYPE_MSI | DDI_INTR_TYPE_MSIX); break; case PSM_INTR_OP_ALLOC_VECTORS: if (hdlp->ih_type == DDI_INTR_TYPE_FIXED) *result = 1; else *result = 0; break; case PSM_INTR_OP_FREE_VECTORS: break; case PSM_INTR_OP_NAVAIL_VECTORS: if (hdlp->ih_type == DDI_INTR_TYPE_FIXED) *result = 1; else *result = 0; break; case PSM_INTR_OP_XLATE_VECTOR: ispec = ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp; *result = psm_translate_irq(dip, ispec->intrspec_vec); break; case PSM_INTR_OP_GET_CAP: *result = 0; break; case PSM_INTR_OP_GET_PENDING: case PSM_INTR_OP_CLEAR_MASK: case PSM_INTR_OP_SET_MASK: case PSM_INTR_OP_GET_SHARED: case PSM_INTR_OP_SET_PRI: case PSM_INTR_OP_SET_CAP: case PSM_INTR_OP_SET_CPU: case PSM_INTR_OP_GET_INTR: default: return (PSM_FAILURE); } return (PSM_SUCCESS); } /* * Return 1 if CMT load balancing policies should be * implemented across instances of the specified hardware * sharing relationship. */ int pg_cmt_load_bal_hw(pghw_type_t hw) { if (hw == PGHW_IPIPE || hw == PGHW_FPU || hw == PGHW_PROCNODE || hw == PGHW_CHIP) return (1); else return (0); } /* * Return 1 if thread affinity polices should be implemented * for instances of the specifed hardware sharing relationship. */ int pg_cmt_affinity_hw(pghw_type_t hw) { if (hw == PGHW_CACHE) return (1); else return (0); } /* * Return number of counter events requested to measure hardware capacity and * utilization and setup CPC requests for specified CPU as needed * * May return 0 when platform or processor specific code knows that no CPC * events should be programmed on this CPU or -1 when platform or processor * specific code doesn't know which counter events are best to use and common * code should decide for itself */ int /* LINTED E_FUNC_ARG_UNUSED */ cu_plat_cpc_init(cpu_t *cp, kcpc_request_list_t *reqs, int nreqs) { const char *impl_name; /* * Return error if pcbe_ops not set */ if (pcbe_ops == NULL) return (-1); /* * Return that no CPC events should be programmed on hyperthreaded * Pentium 4 and return error for all other x86 processors to tell * common code to decide what counter events to program on those CPUs * for measuring hardware capacity and utilization */ impl_name = pcbe_ops->pcbe_impl_name(); if (impl_name != NULL && strcmp(impl_name, PCBE_IMPL_NAME_P4HT) == 0) return (0); else return (-1); } /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved. */ /* * Copyright (c) 2010, Intel Corporation. * All rights reserved. */ /* * Copyright 2019 Joyent, Inc. */ /* * Welcome to the world of the "real mode platter". * See also startup.c, mpcore.s and apic.c for related routines. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include extern cpuset_t cpu_ready_set; extern int mp_start_cpu_common(cpu_t *cp, boolean_t boot); extern void real_mode_start_cpu(void); extern void real_mode_start_cpu_end(void); extern void real_mode_stop_cpu_stage1(void); extern void real_mode_stop_cpu_stage1_end(void); extern void real_mode_stop_cpu_stage2(void); extern void real_mode_stop_cpu_stage2_end(void); void rmp_gdt_init(rm_platter_t *); /* * Fill up the real mode platter to make it easy for real mode code to * kick it off. This area should really be one passed by boot to kernel * and guaranteed to be below 1MB and aligned to 16 bytes. Should also * have identical physical and virtual address in paged mode. */ static ushort_t *warm_reset_vector = NULL; int mach_cpucontext_init(void) { ushort_t *vec; ulong_t addr; struct rm_platter *rm = (struct rm_platter *)rm_platter_va; if (!(vec = (ushort_t *)psm_map_phys(WARM_RESET_VECTOR, sizeof (vec), PROT_READ | PROT_WRITE))) return (-1); /* * setup secondary cpu bios boot up vector * Write page offset to 0x467 and page frame number to 0x469. */ addr = (ulong_t)((caddr_t)rm->rm_code - (caddr_t)rm) + rm_platter_pa; vec[0] = (ushort_t)(addr & PAGEOFFSET); vec[1] = (ushort_t)((addr & (0xfffff & PAGEMASK)) >> 4); warm_reset_vector = vec; /* Map real mode platter into kas so kernel can access it. */ hat_devload(kas.a_hat, (caddr_t)(uintptr_t)rm_platter_pa, MMU_PAGESIZE, btop(rm_platter_pa), PROT_READ | PROT_WRITE | PROT_EXEC, HAT_LOAD_NOCONSIST); /* Copy CPU startup code to rm_platter if it's still during boot. */ if (!plat_dr_enabled()) { ASSERT((size_t)real_mode_start_cpu_end - (size_t)real_mode_start_cpu <= RM_PLATTER_CODE_SIZE); bcopy((caddr_t)real_mode_start_cpu, (caddr_t)rm->rm_code, (size_t)real_mode_start_cpu_end - (size_t)real_mode_start_cpu); } return (0); } void mach_cpucontext_fini(void) { if (warm_reset_vector) psm_unmap_phys((caddr_t)warm_reset_vector, sizeof (warm_reset_vector)); hat_unload(kas.a_hat, (caddr_t)(uintptr_t)rm_platter_pa, MMU_PAGESIZE, HAT_UNLOAD); } extern void *long_mode_64(void); /*ARGSUSED*/ void rmp_gdt_init(rm_platter_t *rm) { /* Use the kas address space for the CPU startup thread. */ if (mmu_ptob(kas.a_hat->hat_htable->ht_pfn) > 0xffffffffUL) { panic("Cannot initialize CPUs; kernel's 64-bit page tables\n" "located above 4G in physical memory (@ 0x%lx)", mmu_ptob(kas.a_hat->hat_htable->ht_pfn)); } /* * Setup pseudo-descriptors for temporary GDT and IDT for use ONLY * by code in real_mode_start_cpu(): * * GDT[0]: NULL selector * GDT[1]: 64-bit CS: Long = 1, Present = 1, bits 12, 11 = 1 * * Clear the IDT as interrupts will be off and a limit of 0 will cause * the CPU to triple fault and reset on an NMI, seemingly as reasonable * a course of action as any other, though it may cause the entire * platform to reset in some cases... */ rm->rm_temp_gdt[0] = 0; rm->rm_temp_gdt[TEMPGDT_KCODE64] = 0x20980000000000ULL; rm->rm_temp_gdt_lim = (ushort_t)(sizeof (rm->rm_temp_gdt) - 1); rm->rm_temp_gdt_base = rm_platter_pa + (uint32_t)offsetof(rm_platter_t, rm_temp_gdt); rm->rm_temp_idt_lim = 0; rm->rm_temp_idt_base = 0; /* * Since the CPU needs to jump to protected mode using an identity * mapped address, we need to calculate it here. */ rm->rm_longmode64_addr = rm_platter_pa + (uint32_t)((uintptr_t)long_mode_64 - (uintptr_t)real_mode_start_cpu); } static void * mach_cpucontext_alloc_tables(struct cpu *cp) { tss_t *ntss; struct cpu_tables *ct; size_t ctsize; /* * Allocate space for stack, tss, gdt and idt. We round the size * allotted for cpu_tables up, so that the TSS is on a unique page. * This is more efficient when running in virtual machines. */ ctsize = P2ROUNDUP(sizeof (*ct), PAGESIZE); ct = kmem_zalloc(ctsize, KM_SLEEP); if ((uintptr_t)ct & PAGEOFFSET) panic("mach_cpucontext_alloc_tables: cpu%d misaligned tables", cp->cpu_id); ntss = cp->cpu_tss = &ct->ct_tss; uintptr_t va; size_t len; /* * #DF (double fault). */ ntss->tss_ist1 = (uintptr_t)&ct->ct_stack1[sizeof (ct->ct_stack1)]; /* * #NM (non-maskable interrupt) */ ntss->tss_ist2 = (uintptr_t)&ct->ct_stack2[sizeof (ct->ct_stack2)]; /* * #MC (machine check exception / hardware error) */ ntss->tss_ist3 = (uintptr_t)&ct->ct_stack3[sizeof (ct->ct_stack3)]; /* * #DB, #BP debug interrupts and KDI/kmdb */ ntss->tss_ist4 = (uintptr_t)&cp->cpu_m.mcpu_kpti_dbg.kf_tr_rsp; if (kpti_enable == 1) { /* * #GP, #PF, #SS fault interrupts */ ntss->tss_ist5 = (uintptr_t)&cp->cpu_m.mcpu_kpti_flt.kf_tr_rsp; /* * Used by all other interrupts */ ntss->tss_ist6 = (uint64_t)&cp->cpu_m.mcpu_kpti.kf_tr_rsp; /* * On AMD64 we need to make sure that all of the pages of the * struct cpu_tables are punched through onto the user CPU for * kpti. * * The final page will always be the TSS, so treat that * separately. */ for (va = (uintptr_t)ct, len = ctsize - MMU_PAGESIZE; len >= MMU_PAGESIZE; len -= MMU_PAGESIZE, va += MMU_PAGESIZE) { /* The doublefault stack must be RW */ hati_cpu_punchin(cp, va, PROT_READ | PROT_WRITE); } ASSERT3U((uintptr_t)ntss, ==, va); hati_cpu_punchin(cp, (uintptr_t)ntss, PROT_READ); } /* * Set I/O bit map offset equal to size of TSS segment limit * for no I/O permission map. This will cause all user I/O * instructions to generate #gp fault. */ ntss->tss_bitmapbase = sizeof (*ntss); /* * Setup kernel tss. */ set_syssegd((system_desc_t *)&cp->cpu_gdt[GDT_KTSS], cp->cpu_tss, sizeof (*cp->cpu_tss) - 1, SDT_SYSTSS, SEL_KPL); return (ct); } void * mach_cpucontext_xalloc(struct cpu *cp, int optype) { size_t len; struct cpu_tables *ct; rm_platter_t *rm = (rm_platter_t *)rm_platter_va; static int cpu_halt_code_ready; if (optype == MACH_CPUCONTEXT_OP_STOP) { ASSERT(plat_dr_enabled()); /* * The WARM_RESET_VECTOR has a limitation that the physical * address written to it must be page-aligned. To work around * this limitation, the CPU stop code has been splitted into * two stages. * The stage 2 code, which implements the real logic to halt * CPUs, is copied to the rm_cpu_halt_code field in the real * mode platter. The stage 1 code, which simply jumps to the * stage 2 code in the rm_cpu_halt_code field, is copied to * rm_code field in the real mode platter and it may be * overwritten after the CPU has been stopped. */ if (!cpu_halt_code_ready) { /* * The rm_cpu_halt_code field in the real mode platter * is used by the CPU stop code only. So only copy the * CPU stop stage 2 code into the rm_cpu_halt_code * field on the first call. */ len = (size_t)real_mode_stop_cpu_stage2_end - (size_t)real_mode_stop_cpu_stage2; ASSERT(len <= RM_PLATTER_CPU_HALT_CODE_SIZE); bcopy((caddr_t)real_mode_stop_cpu_stage2, (caddr_t)rm->rm_cpu_halt_code, len); cpu_halt_code_ready = 1; } /* * The rm_code field in the real mode platter is shared by * the CPU start, CPU stop, CPR and fast reboot code. So copy * the CPU stop stage 1 code into the rm_code field every time. */ len = (size_t)real_mode_stop_cpu_stage1_end - (size_t)real_mode_stop_cpu_stage1; ASSERT(len <= RM_PLATTER_CODE_SIZE); bcopy((caddr_t)real_mode_stop_cpu_stage1, (caddr_t)rm->rm_code, len); rm->rm_cpu_halted = 0; return (cp->cpu_m.mcpu_mach_ctx_ptr); } else if (optype != MACH_CPUCONTEXT_OP_START) { return (NULL); } /* * Only need to allocate tables when starting CPU. * Tables allocated when starting CPU will be reused when stopping CPU. */ ct = mach_cpucontext_alloc_tables(cp); if (ct == NULL) { return (NULL); } /* Copy CPU startup code to rm_platter for CPU hot-add operations. */ if (plat_dr_enabled()) { bcopy((caddr_t)real_mode_start_cpu, (caddr_t)rm->rm_code, (size_t)real_mode_start_cpu_end - (size_t)real_mode_start_cpu); } /* * Now copy all that we've set up onto the real mode platter * for the real mode code to digest as part of starting the cpu. */ rm->rm_idt_base = cp->cpu_idt; rm->rm_idt_lim = sizeof (*cp->cpu_idt) * NIDT - 1; rm->rm_gdt_base = cp->cpu_gdt; rm->rm_gdt_lim = sizeof (*cp->cpu_gdt) * NGDT - 1; /* * CPU needs to access kernel address space after powering on. */ rm->rm_pdbr = MAKECR3(kas.a_hat->hat_htable->ht_pfn, PCID_NONE); rm->rm_cpu = cp->cpu_id; /* * We need to mask off any bits set on our boot CPU that can't apply * while the subject CPU is initializing. If appropriate, they are * enabled later on. */ rm->rm_cr4 = getcr4(); rm->rm_cr4 &= ~(CR4_MCE | CR4_PCE | CR4_PCIDE); rmp_gdt_init(rm); return (ct); } void mach_cpucontext_xfree(struct cpu *cp, void *arg, int err, int optype) { struct cpu_tables *ct = arg; ASSERT(&ct->ct_tss == cp->cpu_tss); if (optype == MACH_CPUCONTEXT_OP_START) { switch (err) { case 0: /* * Save pointer for reuse when stopping CPU. */ cp->cpu_m.mcpu_mach_ctx_ptr = arg; break; case ETIMEDOUT: /* * The processor was poked, but failed to start before * we gave up waiting for it. In case it starts later, * don't free anything. */ cp->cpu_m.mcpu_mach_ctx_ptr = arg; break; default: /* * Some other, passive, error occurred. */ kmem_free(ct, P2ROUNDUP(sizeof (*ct), PAGESIZE)); cp->cpu_tss = NULL; break; } } else if (optype == MACH_CPUCONTEXT_OP_STOP) { switch (err) { case 0: /* * Free resources allocated when starting CPU. */ kmem_free(ct, P2ROUNDUP(sizeof (*ct), PAGESIZE)); cp->cpu_tss = NULL; cp->cpu_m.mcpu_mach_ctx_ptr = NULL; break; default: /* * Don't touch table pointer in case of failure. */ break; } } else { ASSERT(0); } } void * mach_cpucontext_alloc(struct cpu *cp) { return (mach_cpucontext_xalloc(cp, MACH_CPUCONTEXT_OP_START)); } void mach_cpucontext_free(struct cpu *cp, void *arg, int err) { mach_cpucontext_xfree(cp, arg, err, MACH_CPUCONTEXT_OP_START); } /* * "Enter monitor." Called via cross-call from stop_other_cpus(). */ int mach_cpu_halt(xc_arg_t arg1, xc_arg_t arg2 __unused, xc_arg_t arg3 __unused) { char *msg = (char *)arg1; if (msg) prom_printf("%s\n", msg); /*CONSTANTCONDITION*/ while (1) ; return (0); } void mach_cpu_idle(void) { x86_md_clear(); i86_halt(); } void mach_cpu_pause(volatile char *safe) { /* * This cpu is now safe. */ *safe = PAUSE_WAIT; membar_enter(); /* make sure stores are flushed */ /* * Now we wait. When we are allowed to continue, safe * will be set to PAUSE_IDLE. */ while (*safe != PAUSE_IDLE) SMT_PAUSE(); } /* * Power on the target CPU. */ int mp_cpu_poweron(struct cpu *cp) { int error; cpuset_t tempset; processorid_t cpuid; ASSERT(cp != NULL); cpuid = cp->cpu_id; if (use_mp == 0 || plat_dr_support_cpu() == 0) { return (ENOTSUP); } else if (cpuid < 0 || cpuid >= max_ncpus) { return (EINVAL); } /* * The currrent x86 implementaiton of mp_cpu_configure() and * mp_cpu_poweron() have a limitation that mp_cpu_poweron() could only * be called once after calling mp_cpu_configure() for a specific CPU. * It's because mp_cpu_poweron() will destroy data structure created * by mp_cpu_configure(). So reject the request if the CPU has already * been powered on once after calling mp_cpu_configure(). * This limitaiton only affects the p_online syscall and the DR driver * won't be affected because the DR driver always invoke public CPU * management interfaces in the predefined order: * cpu_configure()->cpu_poweron()...->cpu_poweroff()->cpu_unconfigure() */ if (cpuid_checkpass(cp, 4) || cp->cpu_thread == cp->cpu_idle_thread) { return (ENOTSUP); } /* * Check if there's at least a Mbyte of kmem available * before attempting to start the cpu. */ if (kmem_avail() < 1024 * 1024) { /* * Kick off a reap in case that helps us with * later attempts .. */ kmem_reap(); return (ENOMEM); } affinity_set(CPU->cpu_id); /* * Start the target CPU. No need to call mach_cpucontext_fini() * if mach_cpucontext_init() fails. */ if ((error = mach_cpucontext_init()) == 0) { error = mp_start_cpu_common(cp, B_FALSE); mach_cpucontext_fini(); } if (error != 0) { affinity_clear(); return (error); } /* Wait for the target cpu to reach READY state. */ tempset = cpu_ready_set; while (!CPU_IN_SET(tempset, cpuid)) { delay(1); tempset = *((volatile cpuset_t *)&cpu_ready_set); } /* Mark the target CPU as available for mp operation. */ CPUSET_ATOMIC_ADD(mp_cpus, cpuid); /* Free the space allocated to hold the microcode file */ ucode_cleanup(); affinity_clear(); return (0); } #define MP_CPU_DETACH_MAX_TRIES 5 #define MP_CPU_DETACH_DELAY 100 static int mp_cpu_detach_driver(dev_info_t *dip) { int i; int rv = EBUSY; dev_info_t *pdip; pdip = ddi_get_parent(dip); ASSERT(pdip != NULL); /* * Check if caller holds pdip busy - can cause deadlocks in * e_ddi_branch_unconfigure(), which calls devfs_clean(). */ if (DEVI_BUSY_OWNED(pdip)) { return (EDEADLOCK); } for (i = 0; i < MP_CPU_DETACH_MAX_TRIES; i++) { if (e_ddi_branch_unconfigure(dip, NULL, 0) == 0) { rv = 0; break; } DELAY(MP_CPU_DETACH_DELAY); } return (rv); } /* * Power off the target CPU. * Note: cpu_lock will be released and then reacquired. */ int mp_cpu_poweroff(struct cpu *cp) { int rv = 0; void *ctx; dev_info_t *dip = NULL; rm_platter_t *rm = (rm_platter_t *)rm_platter_va; extern void cpupm_start(cpu_t *); extern void cpupm_stop(cpu_t *); ASSERT(cp != NULL); ASSERT((cp->cpu_flags & CPU_OFFLINE) != 0); ASSERT((cp->cpu_flags & CPU_QUIESCED) != 0); if (use_mp == 0 || plat_dr_support_cpu() == 0) { return (ENOTSUP); } /* * There is no support for powering off cpu0 yet. * There are many pieces of code which have a hard dependency on cpu0. */ if (cp->cpu_id == 0) { return (ENOTSUP); }; if (mach_cpu_get_device_node(cp, &dip) != PSM_SUCCESS) { return (ENXIO); } ASSERT(dip != NULL); if (mp_cpu_detach_driver(dip) != 0) { rv = EBUSY; goto out_online; } /* Allocate CPU context for stopping */ if (mach_cpucontext_init() != 0) { rv = ENXIO; goto out_online; } ctx = mach_cpucontext_xalloc(cp, MACH_CPUCONTEXT_OP_STOP); if (ctx == NULL) { rv = ENXIO; goto out_context_fini; } cpupm_stop(cp); cpu_event_fini_cpu(cp); if (cp->cpu_m.mcpu_cmi_hdl != NULL) { cmi_fini(cp->cpu_m.mcpu_cmi_hdl); cp->cpu_m.mcpu_cmi_hdl = NULL; } rv = mach_cpu_stop(cp, ctx); if (rv != 0) { goto out_enable_cmi; } /* Wait until the target CPU has been halted. */ while (*(volatile ushort_t *)&(rm->rm_cpu_halted) != 0xdead) { delay(1); } rm->rm_cpu_halted = 0xffff; /* CPU_READY has been cleared by mach_cpu_stop. */ ASSERT((cp->cpu_flags & CPU_READY) == 0); ASSERT((cp->cpu_flags & CPU_RUNNING) == 0); cp->cpu_flags = CPU_OFFLINE | CPU_QUIESCED | CPU_POWEROFF; CPUSET_ATOMIC_DEL(mp_cpus, cp->cpu_id); mach_cpucontext_xfree(cp, ctx, 0, MACH_CPUCONTEXT_OP_STOP); mach_cpucontext_fini(); return (0); out_enable_cmi: { cmi_hdl_t hdl; if ((hdl = cmi_init(CMI_HDL_NATIVE, cmi_ntv_hwchipid(cp), cmi_ntv_hwcoreid(cp), cmi_ntv_hwstrandid(cp))) != NULL) { if (is_x86_feature(x86_featureset, X86FSET_MCA)) cmi_mca_init(hdl); cp->cpu_m.mcpu_cmi_hdl = hdl; } } cpu_event_init_cpu(cp); cpupm_start(cp); mach_cpucontext_xfree(cp, ctx, rv, MACH_CPUCONTEXT_OP_STOP); out_context_fini: mach_cpucontext_fini(); out_online: (void) e_ddi_branch_configure(dip, NULL, 0); if (rv != EAGAIN && rv != ETIME) { rv = ENXIO; } return (rv); } /* * Return vcpu state, since this could be a virtual environment that we * are unaware of, return "unknown". */ /* ARGSUSED */ int vcpu_on_pcpu(processorid_t cpu) { return (VCPU_STATE_UNKNOWN); } /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved. */ /* * Copyright (c) 2010, Intel Corporation. * All rights reserved. */ /* * Copyright 2020 Joyent, Inc. * Copyright 2013 Nexenta Systems, Inc. All rights reserved. * Copyright 2018 OmniOS Community Edition (OmniOSce) Association. * Copyright 2024 Oxide Computer Company */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #if defined(__xpv) #include #else #include #endif #include #include struct cpu cpus[1] __aligned(MMU_PAGESIZE); struct cpu *cpu[NCPU] = {&cpus[0]}; struct cpu *cpu_free_list; cpu_core_t cpu_core[NCPU]; #define cpu_next_free cpu_prev /* * Useful for disabling MP bring-up on a MP capable system. */ int use_mp = 1; /* * to be set by a PSM to indicate what cpus * are sitting around on the system. */ cpuset_t mp_cpus; /* * This variable is used by the hat layer to decide whether or not * critical sections are needed to prevent race conditions. For sun4m, * this variable is set once enough MP initialization has been done in * order to allow cross calls. */ int flushes_require_xcalls; cpuset_t cpu_ready_set; /* initialized in startup() */ static void mp_startup_boot(void); static void mp_startup_hotplug(void); static void cpu_sep_enable(void); static void cpu_sep_disable(void); static void cpu_asysc_enable(void); static void cpu_asysc_disable(void); /* * Init CPU info - get CPU type info for processor_info system call. */ void init_cpu_info(struct cpu *cp) { processor_info_t *pi = &cp->cpu_type_info; /* * Get clock-frequency property for the CPU. */ pi->pi_clock = cpu_freq; /* * Current frequency in Hz. */ cp->cpu_curr_clock = cpu_freq_hz; /* * Supported frequencies. */ if (cp->cpu_supp_freqs == NULL) { cpu_set_supp_freqs(cp, NULL); } (void) strcpy(pi->pi_processor_type, "amd64"); if (fpu_exists) (void) strcpy(pi->pi_fputypes, "i387 compatible"); cp->cpu_idstr = kmem_zalloc(CPU_IDSTRLEN, KM_SLEEP); cp->cpu_brandstr = kmem_zalloc(CPU_IDSTRLEN, KM_SLEEP); /* * If called for the BSP, cp is equal to current CPU. * For non-BSPs, cpuid info of cp is not ready yet, so use cpuid info * of current CPU as default values for cpu_idstr and cpu_brandstr. * They will be corrected in mp_startup_common() after * CPUID_PASS_DYNAMIC has been invoked on target CPU. */ (void) cpuid_getidstr(CPU, cp->cpu_idstr, CPU_IDSTRLEN); (void) cpuid_getbrandstr(CPU, cp->cpu_brandstr, CPU_IDSTRLEN); } /* * Configure syscall support on this CPU. */ /*ARGSUSED*/ void init_cpu_syscall(struct cpu *cp) { kpreempt_disable(); if (is_x86_feature(x86_featureset, X86FSET_MSR) && is_x86_feature(x86_featureset, X86FSET_ASYSC)) { uint64_t flags; #if !defined(__xpv) /* * The syscall instruction imposes a certain ordering on * segment selectors, so we double-check that ordering * here. */ CTASSERT(KDS_SEL == KCS_SEL + 8); CTASSERT(UDS_SEL == U32CS_SEL + 8); CTASSERT(UCS_SEL == U32CS_SEL + 16); #endif /* * Turn syscall/sysret extensions on. */ cpu_asysc_enable(); /* * Program the magic registers .. */ wrmsr(MSR_AMD_STAR, ((uint64_t)(U32CS_SEL << 16 | KCS_SEL)) << 32); if (kpti_enable == 1) { wrmsr(MSR_AMD_LSTAR, (uint64_t)(uintptr_t)tr_sys_syscall); wrmsr(MSR_AMD_CSTAR, (uint64_t)(uintptr_t)tr_sys_syscall32); } else { wrmsr(MSR_AMD_LSTAR, (uint64_t)(uintptr_t)sys_syscall); wrmsr(MSR_AMD_CSTAR, (uint64_t)(uintptr_t)sys_syscall32); } /* * This list of flags is masked off the incoming * %rfl when we enter the kernel. */ flags = PS_IE | PS_T; if (is_x86_feature(x86_featureset, X86FSET_SMAP) == B_TRUE) flags |= PS_ACHK; wrmsr(MSR_AMD_SFMASK, flags); } /* * On 64-bit kernels on Nocona machines, the 32-bit syscall * variant isn't available to 32-bit applications, but sysenter is. */ if (is_x86_feature(x86_featureset, X86FSET_MSR) && is_x86_feature(x86_featureset, X86FSET_SEP)) { #if !defined(__xpv) /* * The sysenter instruction imposes a certain ordering on * segment selectors, so we double-check that ordering * here. See "sysenter" in Intel document 245471-012, "IA-32 * Intel Architecture Software Developer's Manual Volume 2: * Instruction Set Reference" */ CTASSERT(KDS_SEL == KCS_SEL + 8); CTASSERT(U32CS_SEL == ((KCS_SEL + 16) | 3)); CTASSERT(UDS_SEL == U32CS_SEL + 8); #endif cpu_sep_enable(); /* * resume() sets this value to the base of the threads stack * via a context handler. */ wrmsr(MSR_INTC_SEP_ESP, 0); if (kpti_enable == 1) { wrmsr(MSR_INTC_SEP_EIP, (uint64_t)(uintptr_t)tr_sys_sysenter); } else { wrmsr(MSR_INTC_SEP_EIP, (uint64_t)(uintptr_t)sys_sysenter); } } kpreempt_enable(); } #if !defined(__xpv) /* * Configure per-cpu ID GDT */ static void init_cpu_id_gdt(struct cpu *cp) { /* Write cpu_id into limit field of GDT for usermode retrieval */ set_usegd(&cp->cpu_gdt[GDT_CPUID], SDP_SHORT, NULL, cp->cpu_id, SDT_MEMRODA, SEL_UPL, SDP_BYTES, SDP_OP32); } #endif /* !defined(__xpv) */ /* * Multiprocessor initialization. * * Allocate and initialize the cpu structure, TRAPTRACE buffer, and the * startup and idle threads for the specified CPU. * Parameter boot is true for boot time operations and is false for CPU * DR operations. */ static struct cpu * mp_cpu_configure_common(int cpun, boolean_t boot) { struct cpu *cp; kthread_id_t tp; caddr_t sp; proc_t *procp; #if !defined(__xpv) extern int idle_cpu_prefer_mwait; extern void cpu_idle_mwait(); #endif extern void idle(); extern void cpu_idle(); #ifdef TRAPTRACE trap_trace_ctl_t *ttc = &trap_trace_ctl[cpun]; #endif ASSERT(MUTEX_HELD(&cpu_lock)); ASSERT(cpun < NCPU && cpu[cpun] == NULL); if (cpu_free_list == NULL) { cp = kmem_zalloc(sizeof (*cp), KM_SLEEP); } else { cp = cpu_free_list; cpu_free_list = cp->cpu_next_free; } cp->cpu_m.mcpu_istamp = cpun << 16; /* Create per CPU specific threads in the process p0. */ procp = &p0; /* * Initialize the dispatcher first. */ disp_cpu_init(cp); cpu_vm_data_init(cp); /* * Allocate and initialize the startup thread for this CPU. * Interrupt and process switch stacks get allocated later * when the CPU starts running. */ tp = thread_create(NULL, 0, NULL, NULL, 0, procp, TS_STOPPED, maxclsyspri); /* * Set state to TS_ONPROC since this thread will start running * as soon as the CPU comes online. * * All the other fields of the thread structure are setup by * thread_create(). */ THREAD_ONPROC(tp, cp); tp->t_preempt = 1; tp->t_bound_cpu = cp; tp->t_affinitycnt = 1; tp->t_cpu = cp; tp->t_disp_queue = cp->cpu_disp; /* * Setup thread to start in mp_startup_common. */ sp = tp->t_stk; tp->t_sp = (uintptr_t)(sp - MINFRAME); tp->t_sp -= STACK_ENTRY_ALIGN; /* fake a call */ /* * Setup thread start entry point for boot or hotplug. */ if (boot) { tp->t_pc = (uintptr_t)mp_startup_boot; } else { tp->t_pc = (uintptr_t)mp_startup_hotplug; } cp->cpu_id = cpun; cp->cpu_self = cp; cp->cpu_thread = tp; cp->cpu_lwp = NULL; cp->cpu_dispthread = tp; cp->cpu_dispatch_pri = DISP_PRIO(tp); /* * cpu_base_spl must be set explicitly here to prevent any blocking * operations in mp_startup_common from causing the spl of the cpu * to drop to 0 (allowing device interrupts before we're ready) in * resume(). * cpu_base_spl MUST remain at LOCK_LEVEL until the cpu is CPU_READY. * As an extra bit of security on DEBUG kernels, this is enforced with * an assertion in mp_startup_common() -- before cpu_base_spl is set * to its proper value. */ cp->cpu_base_spl = ipltospl(LOCK_LEVEL); /* * Now, initialize per-CPU idle thread for this CPU. */ tp = thread_create(NULL, PAGESIZE, idle, NULL, 0, procp, TS_ONPROC, -1); cp->cpu_idle_thread = tp; tp->t_preempt = 1; tp->t_bound_cpu = cp; tp->t_affinitycnt = 1; tp->t_cpu = cp; tp->t_disp_queue = cp->cpu_disp; /* * Bootstrap the CPU's PG data */ pg_cpu_bootstrap(cp); /* * Perform CPC initialization on the new CPU. */ kcpc_hw_init(cp); /* * Allocate virtual addresses for cpu_caddr1 and cpu_caddr2 * for each CPU. */ setup_vaddr_for_ppcopy(cp); /* * Allocate page for new GDT and initialize from current GDT. */ #if !defined(__lint) ASSERT((sizeof (*cp->cpu_gdt) * NGDT) <= PAGESIZE); #endif cp->cpu_gdt = kmem_zalloc(PAGESIZE, KM_SLEEP); bcopy(CPU->cpu_gdt, cp->cpu_gdt, (sizeof (*cp->cpu_gdt) * NGDT)); /* * Allocate pages for the CPU LDT. */ cp->cpu_m.mcpu_ldt = kmem_zalloc(LDT_CPU_SIZE, KM_SLEEP); cp->cpu_m.mcpu_ldt_len = 0; /* * Allocate a per-CPU IDT and initialize the new IDT to the currently * runing CPU. */ #if !defined(__lint) ASSERT((sizeof (*CPU->cpu_idt) * NIDT) <= PAGESIZE); #endif cp->cpu_idt = kmem_alloc(PAGESIZE, KM_SLEEP); bcopy(CPU->cpu_idt, cp->cpu_idt, PAGESIZE); /* * alloc space for cpuid info */ cpuid_alloc_space(cp); #if !defined(__xpv) if (is_x86_feature(x86_featureset, X86FSET_MWAIT) && idle_cpu_prefer_mwait) { cp->cpu_m.mcpu_mwait = cpuid_mwait_alloc(cp); cp->cpu_m.mcpu_idle_cpu = cpu_idle_mwait; } else #endif cp->cpu_m.mcpu_idle_cpu = cpu_idle; init_cpu_info(cp); #if !defined(__xpv) init_cpu_id_gdt(cp); #endif /* * alloc space for ucode_info */ ucode_alloc_space(cp); xc_init_cpu(cp); hat_cpu_online(cp); #ifdef TRAPTRACE /* * If this is a TRAPTRACE kernel, allocate TRAPTRACE buffers */ ttc->ttc_first = (uintptr_t)kmem_zalloc(trap_trace_bufsize, KM_SLEEP); ttc->ttc_next = ttc->ttc_first; ttc->ttc_limit = ttc->ttc_first + trap_trace_bufsize; #endif /* * Record that we have another CPU. */ /* * Initialize the interrupt threads for this CPU */ cpu_intr_alloc(cp, NINTR_THREADS); cp->cpu_flags = CPU_OFFLINE | CPU_QUIESCED | CPU_POWEROFF; cpu_set_state(cp); /* * Add CPU to list of available CPUs. It'll be on the active list * after mp_startup_common(). */ cpu_add_unit(cp); return (cp); } /* * Undo what was done in mp_cpu_configure_common */ static void mp_cpu_unconfigure_common(struct cpu *cp, int error) { ASSERT(MUTEX_HELD(&cpu_lock)); /* * Remove the CPU from the list of available CPUs. */ cpu_del_unit(cp->cpu_id); if (error == ETIMEDOUT) { /* * The cpu was started, but never *seemed* to run any * code in the kernel; it's probably off spinning in its * own private world, though with potential references to * our kmem-allocated IDTs and GDTs (for example). * * Worse still, it may actually wake up some time later, * so rather than guess what it might or might not do, we * leave the fundamental data structures intact. */ cp->cpu_flags = 0; return; } /* * At this point, the only threads bound to this CPU should * special per-cpu threads: it's idle thread, it's pause threads, * and it's interrupt threads. Clean these up. */ cpu_destroy_bound_threads(cp); cp->cpu_idle_thread = NULL; /* * Free the interrupt stack. */ segkp_release(segkp, cp->cpu_intr_stack - (INTR_STACK_SIZE - SA(MINFRAME))); cp->cpu_intr_stack = NULL; #ifdef TRAPTRACE /* * Discard the trap trace buffer */ { trap_trace_ctl_t *ttc = &trap_trace_ctl[cp->cpu_id]; kmem_free((void *)ttc->ttc_first, trap_trace_bufsize); ttc->ttc_first = (uintptr_t)NULL; } #endif hat_cpu_offline(cp); ucode_free_space(cp); /* Free CPU ID string and brand string. */ if (cp->cpu_idstr) { kmem_free(cp->cpu_idstr, CPU_IDSTRLEN); cp->cpu_idstr = NULL; } if (cp->cpu_brandstr) { kmem_free(cp->cpu_brandstr, CPU_IDSTRLEN); cp->cpu_brandstr = NULL; } #if !defined(__xpv) if (cp->cpu_m.mcpu_mwait != NULL) { cpuid_mwait_free(cp); cp->cpu_m.mcpu_mwait = NULL; } #endif cpuid_free_space(cp); if (cp->cpu_idt != CPU->cpu_idt) kmem_free(cp->cpu_idt, PAGESIZE); cp->cpu_idt = NULL; kmem_free(cp->cpu_m.mcpu_ldt, LDT_CPU_SIZE); cp->cpu_m.mcpu_ldt = NULL; cp->cpu_m.mcpu_ldt_len = 0; kmem_free(cp->cpu_gdt, PAGESIZE); cp->cpu_gdt = NULL; if (cp->cpu_supp_freqs != NULL) { size_t len = strlen(cp->cpu_supp_freqs) + 1; kmem_free(cp->cpu_supp_freqs, len); cp->cpu_supp_freqs = NULL; } teardown_vaddr_for_ppcopy(cp); kcpc_hw_fini(cp); cp->cpu_dispthread = NULL; cp->cpu_thread = NULL; /* discarded by cpu_destroy_bound_threads() */ cpu_vm_data_destroy(cp); xc_fini_cpu(cp); disp_cpu_fini(cp); ASSERT(cp != CPU0); bzero(cp, sizeof (*cp)); cp->cpu_next_free = cpu_free_list; cpu_free_list = cp; } /* * Apply workarounds for known errata, and warn about those that are absent. * * System vendors occasionally create configurations which contain different * revisions of the CPUs that are almost but not exactly the same. At the * time of writing, this meant that their clock rates were the same, their * feature sets were the same, but the required workaround were -not- * necessarily the same. So, this routine is invoked on -every- CPU soon * after starting to make sure that the resulting system contains the most * pessimal set of workarounds needed to cope with *any* of the CPUs in the * system. * * workaround_errata is invoked early in mlsetup() for CPU 0, and in * mp_startup_common() for all slave CPUs. Slaves process workaround_errata * prior to acknowledging their readiness to the master, so this routine will * never be executed by multiple CPUs in parallel, thus making updates to * global data safe. * * These workarounds are based on Rev 3.57 of the Revision Guide for * AMD Athlon(tm) 64 and AMD Opteron(tm) Processors, August 2005. */ #if defined(OPTERON_ERRATUM_88) int opteron_erratum_88; /* if non-zero -> at least one cpu has it */ #endif #if defined(OPTERON_ERRATUM_91) int opteron_erratum_91; /* if non-zero -> at least one cpu has it */ #endif #if defined(OPTERON_ERRATUM_93) int opteron_erratum_93; /* if non-zero -> at least one cpu has it */ #endif #if defined(OPTERON_ERRATUM_95) int opteron_erratum_95; /* if non-zero -> at least one cpu has it */ #endif #if defined(OPTERON_ERRATUM_100) int opteron_erratum_100; /* if non-zero -> at least one cpu has it */ #endif #if defined(OPTERON_ERRATUM_108) int opteron_erratum_108; /* if non-zero -> at least one cpu has it */ #endif #if defined(OPTERON_ERRATUM_109) int opteron_erratum_109; /* if non-zero -> at least one cpu has it */ #endif #if defined(OPTERON_ERRATUM_121) int opteron_erratum_121; /* if non-zero -> at least one cpu has it */ #endif #if defined(OPTERON_ERRATUM_122) int opteron_erratum_122; /* if non-zero -> at least one cpu has it */ #endif #if defined(OPTERON_ERRATUM_123) int opteron_erratum_123; /* if non-zero -> at least one cpu has it */ #endif #if defined(OPTERON_ERRATUM_131) int opteron_erratum_131; /* if non-zero -> at least one cpu has it */ #endif #if defined(OPTERON_WORKAROUND_6336786) int opteron_workaround_6336786; /* non-zero -> WA relevant and applied */ int opteron_workaround_6336786_UP = 0; /* Not needed for UP */ #endif #if defined(OPTERON_ERRATUM_147) int opteron_erratum_147; /* if non-zero -> at least one cpu has it */ #endif #if defined(OPTERON_ERRATUM_298) int opteron_erratum_298; #endif #if defined(OPTERON_ERRATUM_721) int opteron_erratum_721; #endif static void workaround_warning(cpu_t *cp, uint_t erratum) { cmn_err(CE_WARN, "cpu%d: no workaround for erratum %u", cp->cpu_id, erratum); } static void workaround_applied(uint_t erratum) { if (erratum > 1000000) cmn_err(CE_CONT, "?workaround applied for cpu issue #%d\n", erratum); else cmn_err(CE_CONT, "?workaround applied for cpu erratum #%d\n", erratum); } static void msr_warning(cpu_t *cp, const char *rw, uint_t msr, int error) { cmn_err(CE_WARN, "cpu%d: couldn't %smsr 0x%x, error %d", cp->cpu_id, rw, msr, error); } /* * Determine the number of nodes in a Hammer / Greyhound / Griffin family * system. */ static uint_t opteron_get_nnodes(void) { static uint_t nnodes = 0; if (nnodes == 0) { #ifdef DEBUG uint_t family; /* * This routine uses a PCI config space based mechanism * for retrieving the number of nodes in the system. * Device 24, function 0, offset 0x60 as used here is not * AMD processor architectural, and may not work on processor * families other than those listed below. * * Callers of this routine must ensure that we're running on * a processor which supports this mechanism. * The assertion below is meant to catch calls on unsupported * processors. */ family = cpuid_getfamily(CPU); ASSERT(family == 0xf || family == 0x10 || family == 0x11); #endif /* DEBUG */ /* * Obtain the number of nodes in the system from * bits [6:4] of the Node ID register on node 0. * * The actual node count is NodeID[6:4] + 1 * * The Node ID register is accessed via function 0, * offset 0x60. Node 0 is device 24. */ nnodes = ((pci_getl_func(0, 24, 0, 0x60) & 0x70) >> 4) + 1; } return (nnodes); } uint_t do_erratum_298(struct cpu *cpu) { static int osvwrc = -3; extern int osvw_opteron_erratum(cpu_t *, uint_t); /* * L2 Eviction May Occur During Processor Operation To Set * Accessed or Dirty Bit. */ if (osvwrc == -3) { osvwrc = osvw_opteron_erratum(cpu, 298); } else { /* osvw return codes should be consistent for all cpus */ ASSERT(osvwrc == osvw_opteron_erratum(cpu, 298)); } switch (osvwrc) { case 0: /* erratum is not present: do nothing */ break; case 1: /* erratum is present: BIOS workaround applied */ /* * check if workaround is actually in place and issue warning * if not. */ if (((rdmsr(MSR_AMD_HWCR) & AMD_HWCR_TLBCACHEDIS) == 0) || ((rdmsr(MSR_AMD_BU_CFG) & AMD_BU_CFG_E298) == 0)) { #if defined(OPTERON_ERRATUM_298) opteron_erratum_298++; #else workaround_warning(cpu, 298); return (1); #endif } break; case -1: /* cannot determine via osvw: check cpuid */ if ((cpuid_opteron_erratum(cpu, 298) > 0) && (((rdmsr(MSR_AMD_HWCR) & AMD_HWCR_TLBCACHEDIS) == 0) || ((rdmsr(MSR_AMD_BU_CFG) & AMD_BU_CFG_E298) == 0))) { #if defined(OPTERON_ERRATUM_298) opteron_erratum_298++; #else workaround_warning(cpu, 298); return (1); #endif } break; } return (0); } uint_t workaround_errata(struct cpu *cpu) { volatile uint_t missing = 0; ASSERT(cpu == CPU); /*LINTED*/ if (cpuid_opteron_erratum(cpu, 88) > 0) { /* * SWAPGS May Fail To Read Correct GS Base */ #if defined(OPTERON_ERRATUM_88) /* * The workaround is an mfence in the relevant assembler code */ opteron_erratum_88++; #else workaround_warning(cpu, 88); missing++; #endif } if (cpuid_opteron_erratum(cpu, 91) > 0) { /* * Software Prefetches May Report A Page Fault */ #if defined(OPTERON_ERRATUM_91) /* * fix is in trap.c */ opteron_erratum_91++; #else workaround_warning(cpu, 91); missing++; #endif } if (cpuid_opteron_erratum(cpu, 93) > 0) { /* * RSM Auto-Halt Restart Returns to Incorrect RIP */ #if defined(OPTERON_ERRATUM_93) /* * fix is in trap.c */ opteron_erratum_93++; #else workaround_warning(cpu, 93); missing++; #endif } /*LINTED*/ if (cpuid_opteron_erratum(cpu, 95) > 0) { /* * RET Instruction May Return to Incorrect EIP */ #if defined(OPTERON_ERRATUM_95) #if defined(_LP64) /* * Workaround this by ensuring that 32-bit user code and * 64-bit kernel code never occupy the same address * range mod 4G. */ if (_userlimit32 > 0xc0000000ul) *(uintptr_t *)&_userlimit32 = 0xc0000000ul; /*LINTED*/ ASSERT((uint32_t)COREHEAP_BASE == 0xc0000000u); opteron_erratum_95++; #endif /* _LP64 */ #else workaround_warning(cpu, 95); missing++; #endif } if (cpuid_opteron_erratum(cpu, 100) > 0) { /* * Compatibility Mode Branches Transfer to Illegal Address */ #if defined(OPTERON_ERRATUM_100) /* * fix is in trap.c */ opteron_erratum_100++; #else workaround_warning(cpu, 100); missing++; #endif } /*LINTED*/ if (cpuid_opteron_erratum(cpu, 108) > 0) { /* * CPUID Instruction May Return Incorrect Model Number In * Some Processors */ #if defined(OPTERON_ERRATUM_108) /* * (Our cpuid-handling code corrects the model number on * those processors) */ #else workaround_warning(cpu, 108); missing++; #endif } /*LINTED*/ if (cpuid_opteron_erratum(cpu, 109) > 0) do { /* * Certain Reverse REP MOVS May Produce Unpredictable Behavior */ #if defined(OPTERON_ERRATUM_109) /* * The "workaround" is to print a warning to upgrade the BIOS */ uint64_t value; const uint_t msr = MSR_AMD_PATCHLEVEL; int err; if ((err = checked_rdmsr(msr, &value)) != 0) { msr_warning(cpu, "rd", msr, err); workaround_warning(cpu, 109); missing++; } if (value == 0) opteron_erratum_109++; #else workaround_warning(cpu, 109); missing++; #endif /*CONSTANTCONDITION*/ } while (0); /*LINTED*/ if (cpuid_opteron_erratum(cpu, 121) > 0) { /* * Sequential Execution Across Non_Canonical Boundary Caused * Processor Hang */ #if defined(OPTERON_ERRATUM_121) #if defined(_LP64) /* * Erratum 121 is only present in long (64 bit) mode. * Workaround is to include the page immediately before the * va hole to eliminate the possibility of system hangs due to * sequential execution across the va hole boundary. */ if (opteron_erratum_121) opteron_erratum_121++; else { if (hole_start) { hole_start -= PAGESIZE; } else { /* * hole_start not yet initialized by * mmu_init. Initialize hole_start * with value to be subtracted. */ hole_start = PAGESIZE; } opteron_erratum_121++; } #endif /* _LP64 */ #else workaround_warning(cpu, 121); missing++; #endif } /*LINTED*/ if (cpuid_opteron_erratum(cpu, 122) > 0) do { /* * TLB Flush Filter May Cause Coherency Problem in * Multiprocessor Systems */ #if defined(OPTERON_ERRATUM_122) uint64_t value; const uint_t msr = MSR_AMD_HWCR; int error; /* * Erratum 122 is only present in MP configurations (multi-core * or multi-processor). */ #if defined(__xpv) if (!DOMAIN_IS_INITDOMAIN(xen_info)) break; if (!opteron_erratum_122 && xpv_nr_phys_cpus() == 1) break; #else if (!opteron_erratum_122 && opteron_get_nnodes() == 1 && cpuid_get_ncpu_per_chip(cpu) == 1) break; #endif /* disable TLB Flush Filter */ if ((error = checked_rdmsr(msr, &value)) != 0) { msr_warning(cpu, "rd", msr, error); workaround_warning(cpu, 122); missing++; } else { value |= (uint64_t)AMD_HWCR_FFDIS; if ((error = checked_wrmsr(msr, value)) != 0) { msr_warning(cpu, "wr", msr, error); workaround_warning(cpu, 122); missing++; } } opteron_erratum_122++; #else workaround_warning(cpu, 122); missing++; #endif /*CONSTANTCONDITION*/ } while (0); /*LINTED*/ if (cpuid_opteron_erratum(cpu, 123) > 0) do { /* * Bypassed Reads May Cause Data Corruption of System Hang in * Dual Core Processors */ #if defined(OPTERON_ERRATUM_123) uint64_t value; const uint_t msr = MSR_AMD_PATCHLEVEL; int err; /* * Erratum 123 applies only to multi-core cpus. */ if (cpuid_get_ncpu_per_chip(cpu) < 2) break; #if defined(__xpv) if (!DOMAIN_IS_INITDOMAIN(xen_info)) break; #endif /* * The "workaround" is to print a warning to upgrade the BIOS */ if ((err = checked_rdmsr(msr, &value)) != 0) { msr_warning(cpu, "rd", msr, err); workaround_warning(cpu, 123); missing++; } if (value == 0) opteron_erratum_123++; #else workaround_warning(cpu, 123); missing++; #endif /*CONSTANTCONDITION*/ } while (0); /*LINTED*/ if (cpuid_opteron_erratum(cpu, 131) > 0) do { /* * Multiprocessor Systems with Four or More Cores May Deadlock * Waiting for a Probe Response */ #if defined(OPTERON_ERRATUM_131) uint64_t nbcfg; const uint_t msr = MSR_AMD_NB_CFG; const uint64_t wabits = AMD_NB_CFG_SRQ_HEARTBEAT | AMD_NB_CFG_SRQ_SPR; int error; /* * Erratum 131 applies to any system with four or more cores. */ if (opteron_erratum_131) break; #if defined(__xpv) if (!DOMAIN_IS_INITDOMAIN(xen_info)) break; if (xpv_nr_phys_cpus() < 4) break; #else if (opteron_get_nnodes() * cpuid_get_ncpu_per_chip(cpu) < 4) break; #endif /* * Print a warning if neither of the workarounds for * erratum 131 is present. */ if ((error = checked_rdmsr(msr, &nbcfg)) != 0) { msr_warning(cpu, "rd", msr, error); workaround_warning(cpu, 131); missing++; } else if ((nbcfg & wabits) == 0) { opteron_erratum_131++; } else { /* cannot have both workarounds set */ ASSERT((nbcfg & wabits) != wabits); } #else workaround_warning(cpu, 131); missing++; #endif /*CONSTANTCONDITION*/ } while (0); /* * This isn't really an erratum, but for convenience the * detection/workaround code lives here and in cpuid_opteron_erratum. * Note, the technique only is valid on families before 12h and * certainly doesn't work when we're virtualized. This is checked for in * the erratum workaround. */ if (cpuid_opteron_erratum(cpu, 6336786) > 0) { #if defined(OPTERON_WORKAROUND_6336786) /* * Disable C1-Clock ramping on multi-core/multi-processor * K8 platforms to guard against TSC drift. */ if (opteron_workaround_6336786) { opteron_workaround_6336786++; #if defined(__xpv) } else if ((DOMAIN_IS_INITDOMAIN(xen_info) && xpv_nr_phys_cpus() > 1) || opteron_workaround_6336786_UP) { /* * XXPV Hmm. We can't walk the Northbridges on * the hypervisor; so just complain and drive * on. This probably needs to be fixed in * the hypervisor itself. */ opteron_workaround_6336786++; workaround_warning(cpu, 6336786); #else /* __xpv */ } else if ((opteron_get_nnodes() * cpuid_get_ncpu_per_chip(cpu) > 1) || opteron_workaround_6336786_UP) { uint_t node, nnodes; uint8_t data; nnodes = opteron_get_nnodes(); for (node = 0; node < nnodes; node++) { /* * Clear PMM7[1:0] (function 3, offset 0x87) * Northbridge device is the node id + 24. */ data = pci_getb_func(0, node + 24, 3, 0x87); data &= 0xFC; pci_putb_func(0, node + 24, 3, 0x87, data); } opteron_workaround_6336786++; #endif /* __xpv */ } #else workaround_warning(cpu, 6336786); missing++; #endif } /*LINTED*/ /* * Mutex primitives don't work as expected. This is erratum #147 from * 'Revision Guide for AMD Athlon 64 and AMD Opteron Processors' * document 25759. */ if (cpuid_opteron_erratum(cpu, 147) > 0) { #if defined(OPTERON_ERRATUM_147) /* * This problem only occurs with 2 or more cores. If bit in * MSR_AMD_BU_CFG set, then not applicable. The workaround * is to patch the semaphone routines with the lfence * instruction to provide necessary load memory barrier with * possible subsequent read-modify-write ops. * * It is too early in boot to call the patch routine so * set erratum variable to be done in startup_end(). */ if (opteron_erratum_147) { opteron_erratum_147++; #if defined(__xpv) } else if (is_x86_feature(x86_featureset, X86FSET_SSE2)) { if (DOMAIN_IS_INITDOMAIN(xen_info)) { /* * XXPV Use dom0_msr here when extended * operations are supported? */ if (xpv_nr_phys_cpus() > 1) opteron_erratum_147++; } else { /* * We have no way to tell how many physical * cpus there are, or even if this processor * has the problem, so enable the workaround * unconditionally (at some performance cost). */ opteron_erratum_147++; } #else /* __xpv */ } else if (is_x86_feature(x86_featureset, X86FSET_SSE2) && ((opteron_get_nnodes() * cpuid_get_ncpu_per_chip(cpu)) > 1)) { if ((xrdmsr(MSR_AMD_BU_CFG) & (UINT64_C(1) << 33)) == 0) opteron_erratum_147++; #endif /* __xpv */ } #else workaround_warning(cpu, 147); missing++; #endif } missing += do_erratum_298(cpu); if (cpuid_opteron_erratum(cpu, 721) > 0) { #if defined(OPTERON_ERRATUM_721) on_trap_data_t otd; if (!on_trap(&otd, OT_DATA_ACCESS)) wrmsr(MSR_AMD_DE_CFG, rdmsr(MSR_AMD_DE_CFG) | AMD_DE_CFG_E721); no_trap(); opteron_erratum_721++; #else workaround_warning(cpu, 721); missing++; #endif } #ifdef __xpv return (0); #else return (missing); #endif } void workaround_errata_end() { #if defined(OPTERON_ERRATUM_88) if (opteron_erratum_88) workaround_applied(88); #endif #if defined(OPTERON_ERRATUM_91) if (opteron_erratum_91) workaround_applied(91); #endif #if defined(OPTERON_ERRATUM_93) if (opteron_erratum_93) workaround_applied(93); #endif #if defined(OPTERON_ERRATUM_95) if (opteron_erratum_95) workaround_applied(95); #endif #if defined(OPTERON_ERRATUM_100) if (opteron_erratum_100) workaround_applied(100); #endif #if defined(OPTERON_ERRATUM_108) if (opteron_erratum_108) workaround_applied(108); #endif #if defined(OPTERON_ERRATUM_109) if (opteron_erratum_109) { cmn_err(CE_WARN, "BIOS microcode patch for AMD Athlon(tm) 64/Opteron(tm)" " processor\nerratum 109 was not detected; updating your" " system's BIOS to a version\ncontaining this" " microcode patch is HIGHLY recommended or erroneous" " system\noperation may occur.\n"); } #endif #if defined(OPTERON_ERRATUM_121) if (opteron_erratum_121) workaround_applied(121); #endif #if defined(OPTERON_ERRATUM_122) if (opteron_erratum_122) workaround_applied(122); #endif #if defined(OPTERON_ERRATUM_123) if (opteron_erratum_123) { cmn_err(CE_WARN, "BIOS microcode patch for AMD Athlon(tm) 64/Opteron(tm)" " processor\nerratum 123 was not detected; updating your" " system's BIOS to a version\ncontaining this" " microcode patch is HIGHLY recommended or erroneous" " system\noperation may occur.\n"); } #endif #if defined(OPTERON_ERRATUM_131) if (opteron_erratum_131) { cmn_err(CE_WARN, "BIOS microcode patch for AMD Athlon(tm) 64/Opteron(tm)" " processor\nerratum 131 was not detected; updating your" " system's BIOS to a version\ncontaining this" " microcode patch is HIGHLY recommended or erroneous" " system\noperation may occur.\n"); } #endif #if defined(OPTERON_WORKAROUND_6336786) if (opteron_workaround_6336786) workaround_applied(6336786); #endif #if defined(OPTERON_ERRATUM_147) if (opteron_erratum_147) workaround_applied(147); #endif #if defined(OPTERON_ERRATUM_298) if (opteron_erratum_298) { cmn_err(CE_WARN, "BIOS microcode patch for AMD 64/Opteron(tm)" " processor\nerratum 298 was not detected; updating your" " system's BIOS to a version\ncontaining this" " microcode patch is HIGHLY recommended or erroneous" " system\noperation may occur.\n"); } #endif #if defined(OPTERON_ERRATUM_721) if (opteron_erratum_721) workaround_applied(721); #endif } /* * The procset_slave and procset_master are used to synchronize * between the control CPU and the target CPU when starting CPUs. */ static cpuset_t procset_slave, procset_master; static void mp_startup_wait(cpuset_t *sp, processorid_t cpuid) { cpuset_t tempset; for (tempset = *sp; !CPU_IN_SET(tempset, cpuid); tempset = *(volatile cpuset_t *)sp) { SMT_PAUSE(); } CPUSET_ATOMIC_DEL(*(cpuset_t *)sp, cpuid); } static void mp_startup_signal(cpuset_t *sp, processorid_t cpuid) { cpuset_t tempset; CPUSET_ATOMIC_ADD(*(cpuset_t *)sp, cpuid); for (tempset = *sp; CPU_IN_SET(tempset, cpuid); tempset = *(volatile cpuset_t *)sp) { SMT_PAUSE(); } } int mp_start_cpu_common(cpu_t *cp, boolean_t boot) { _NOTE(ARGUNUSED(boot)); void *ctx; int delays; int error = 0; cpuset_t tempset; processorid_t cpuid; #ifndef __xpv extern void cpupm_init(cpu_t *); #endif ASSERT(cp != NULL); cpuid = cp->cpu_id; ctx = mach_cpucontext_alloc(cp); if (ctx == NULL) { cmn_err(CE_WARN, "cpu%d: failed to allocate context", cp->cpu_id); return (EAGAIN); } error = mach_cpu_start(cp, ctx); if (error != 0) { cmn_err(CE_WARN, "cpu%d: failed to start, error %d", cp->cpu_id, error); mach_cpucontext_free(cp, ctx, error); return (error); } for (delays = 0, tempset = procset_slave; !CPU_IN_SET(tempset, cpuid); delays++) { if (delays == 500) { /* * After five seconds, things are probably looking * a bit bleak - explain the hang. */ cmn_err(CE_NOTE, "cpu%d: started, " "but not running in the kernel yet", cpuid); } else if (delays > 2000) { /* * We waited at least 20 seconds, bail .. */ error = ETIMEDOUT; cmn_err(CE_WARN, "cpu%d: timed out", cpuid); mach_cpucontext_free(cp, ctx, error); return (error); } /* * wait at least 10ms, then check again.. */ delay(USEC_TO_TICK_ROUNDUP(10000)); tempset = *((volatile cpuset_t *)&procset_slave); } CPUSET_ATOMIC_DEL(procset_slave, cpuid); mach_cpucontext_free(cp, ctx, 0); #ifndef __xpv if (tsc_gethrtime_enable) tsc_sync_master(cpuid); #endif /* * At this point, the CPU in question is past the IDENT cpuid phase and * grabbed the current microcode revision so we can now look for any * relevant microcode updates it should load. We'll fill out * cpu_ucode_info for it along with the microcode to load, if any, * before signaling back to the CPU to continue startup. */ mp_startup_wait(&procset_slave, cpuid); ucode_locate(cp); mp_startup_signal(&procset_master, cpuid); if (dtrace_cpu_init != NULL) { (*dtrace_cpu_init)(cpuid); } /* * During CPU DR operations, the cpu_lock is held by current * (the control) thread. We can't release the cpu_lock here * because that will break the CPU DR logic. * On the other hand, CPUPM and processor group initialization * routines need to access the cpu_lock. So we invoke those * routines here on behalf of mp_startup_common(). * * CPUPM and processor group initialization routines depend * on the cpuid probing results. Wait for mp_startup_common() * to signal that cpuid probing is done. */ mp_startup_wait(&procset_slave, cpuid); #ifndef __xpv cpupm_init(cp); #endif (void) pg_cpu_init(cp, B_FALSE); cpu_set_state(cp); mp_startup_signal(&procset_master, cpuid); return (0); } /* * Start a single cpu, assuming that the kernel context is available * to successfully start another cpu. * * (For example, real mode code is mapped into the right place * in memory and is ready to be run.) */ int start_cpu(processorid_t who) { cpu_t *cp; int error = 0; cpuset_t tempset; ASSERT(who != 0); /* * Check if there's at least a Mbyte of kmem available * before attempting to start the cpu. */ if (kmem_avail() < 1024 * 1024) { /* * Kick off a reap in case that helps us with * later attempts .. */ kmem_reap(); return (ENOMEM); } /* * First configure cpu. */ cp = mp_cpu_configure_common(who, B_TRUE); ASSERT(cp != NULL); /* * Then start cpu. */ error = mp_start_cpu_common(cp, B_TRUE); if (error != 0) { mp_cpu_unconfigure_common(cp, error); return (error); } mutex_exit(&cpu_lock); tempset = cpu_ready_set; while (!CPU_IN_SET(tempset, who)) { drv_usecwait(1); tempset = *((volatile cpuset_t *)&cpu_ready_set); } mutex_enter(&cpu_lock); return (0); } void start_other_cpus(int cprboot) { _NOTE(ARGUNUSED(cprboot)); uint_t who; uint_t bootcpuid = 0; /* * Initialize our own cpu_info. */ init_cpu_info(CPU); #if !defined(__xpv) init_cpu_id_gdt(CPU); #endif cmn_err(CE_CONT, "?cpu%d: %s\n", CPU->cpu_id, CPU->cpu_idstr); cmn_err(CE_CONT, "?cpu%d: %s\n", CPU->cpu_id, CPU->cpu_brandstr); /* * KPTI initialisation happens very early in boot, before logging is * set up. Output a status message now as the boot CPU comes online. */ cmn_err(CE_CONT, "?KPTI %s (PCID %s, INVPCID %s)\n", kpti_enable ? "enabled" : "disabled", x86_use_pcid == 1 ? "in use" : (is_x86_feature(x86_featureset, X86FSET_PCID) ? "disabled" : "not supported"), x86_use_pcid == 1 && x86_use_invpcid == 1 ? "in use" : (is_x86_feature(x86_featureset, X86FSET_INVPCID) ? "disabled" : "not supported")); /* * Initialize our syscall handlers */ init_cpu_syscall(CPU); /* * Take the boot cpu out of the mp_cpus set because we know * it's already running. Add it to the cpu_ready_set for * precisely the same reason. */ CPUSET_DEL(mp_cpus, bootcpuid); CPUSET_ADD(cpu_ready_set, bootcpuid); /* * skip the rest of this if * . only 1 cpu dectected and system isn't hotplug-capable * . not using MP */ if ((CPUSET_ISNULL(mp_cpus) && plat_dr_support_cpu() == 0) || use_mp == 0) { if (use_mp == 0) cmn_err(CE_CONT, "?***** Not in MP mode\n"); goto done; } /* * perform such initialization as is needed * to be able to take CPUs on- and off-line. */ cpu_pause_init(); xc_init_cpu(CPU); /* initialize processor crosscalls */ if (mach_cpucontext_init() != 0) goto done; flushes_require_xcalls = 1; /* * We lock our affinity to the master CPU to ensure that all slave CPUs * do their TSC syncs with the same CPU. */ affinity_set(CPU_CURRENT); for (who = 0; who < NCPU; who++) { if (!CPU_IN_SET(mp_cpus, who)) continue; ASSERT(who != bootcpuid); mutex_enter(&cpu_lock); if (start_cpu(who) != 0) CPUSET_DEL(mp_cpus, who); cpu_state_change_notify(who, CPU_SETUP); mutex_exit(&cpu_lock); } /* Free the space allocated to hold the microcode file */ ucode_cleanup(); affinity_clear(); mach_cpucontext_fini(); done: if (get_hwenv() == HW_NATIVE) workaround_errata_end(); cmi_post_mpstartup(); #if !defined(__xpv) /* * Once other CPUs have completed startup procedures, perform * initialization of hypervisor resources for HMA. */ hma_init(); #endif if (use_mp && ncpus != boot_max_ncpus) { cmn_err(CE_NOTE, "System detected %d cpus, but " "only %d cpu(s) were enabled during boot.", boot_max_ncpus, ncpus); cmn_err(CE_NOTE, "Use \"boot-ncpus\" parameter to enable more CPU(s). " "See eeprom(8)."); } } int mp_cpu_configure(int cpuid) { cpu_t *cp; if (use_mp == 0 || plat_dr_support_cpu() == 0) { return (ENOTSUP); } cp = cpu_get(cpuid); if (cp != NULL) { return (EALREADY); } /* * Check if there's at least a Mbyte of kmem available * before attempting to start the cpu. */ if (kmem_avail() < 1024 * 1024) { /* * Kick off a reap in case that helps us with * later attempts .. */ kmem_reap(); return (ENOMEM); } cp = mp_cpu_configure_common(cpuid, B_FALSE); ASSERT(cp != NULL && cpu_get(cpuid) == cp); return (cp != NULL ? 0 : EAGAIN); } int mp_cpu_unconfigure(int cpuid) { cpu_t *cp; if (use_mp == 0 || plat_dr_support_cpu() == 0) { return (ENOTSUP); } else if (cpuid < 0 || cpuid >= max_ncpus) { return (EINVAL); } cp = cpu_get(cpuid); if (cp == NULL) { return (ENODEV); } mp_cpu_unconfigure_common(cp, 0); return (0); } /* * Startup function for 'other' CPUs (besides boot cpu). * Called from real_mode_start. * * WARNING: until CPU_READY is set, mp_startup_common and routines called by * mp_startup_common should not call routines (e.g. kmem_free) that could call * hat_unload which requires CPU_READY to be set. */ static void mp_startup_common(boolean_t boot) { cpu_t *cp = CPU; uchar_t new_x86_featureset[BT_SIZEOFMAP(NUM_X86_FEATURES)]; extern void cpu_event_init_cpu(cpu_t *); /* * We need to get TSC on this proc synced (i.e., any delta * from cpu0 accounted for) as soon as we can, because many * many things use gethrtime/pc_gethrestime, including * interrupts, cmn_err, etc. Before we can do that, we want to * clear TSC if we're on a buggy Sandy/Ivy Bridge CPU, so do that * right away. Note that the TSC sync procedure run by * tsc_sync_{master,slave} will not yield reliable results if caching is * disabled on either CPU. We rely on code in mpcore.S to guarantee * that it is enabled before this function is called. Caching has * already been enabled on the BSP long before APs are started. */ bzero(new_x86_featureset, BT_SIZEOFMAP(NUM_X86_FEATURES)); cpuid_execpass(cp, CPUID_PASS_PRELUDE, new_x86_featureset); cpuid_execpass(cp, CPUID_PASS_IDENT, NULL); /* * We want to apply any microcode updates before the BASIC cpuid pass, * but as per the above comment, we want to make sure TSC is synced * ASAP. Thus we check for TSC support in the boot CPU's feature set * instead -- this should be fine as we'd expect TSC support to be * consistent across all CPUs (and certainly for the buggy CPUs we're * concerned about here). */ if (boot && get_hwenv() == HW_NATIVE && cpuid_getvendor(CPU) == X86_VENDOR_Intel && cpuid_getfamily(CPU) == 6 && (cpuid_getmodel(CPU) == 0x2d || cpuid_getmodel(CPU) == 0x3e) && is_x86_feature(x86_featureset, X86FSET_TSC)) { (void) wrmsr(REG_TSC, 0UL); } /* Let the control CPU continue into tsc_sync_master() */ mp_startup_signal(&procset_slave, cp->cpu_id); #ifndef __xpv if (tsc_gethrtime_enable) tsc_sync_slave(); #endif /* * As with the boot CPU, we may have a more recent update compared to * whatever the BIOS may have already applied. If so, we want to apply * it here before the BASIC cpuid pass so that any architecturally * visible changes (e.g., changed MSR or CPUID bits) happen before we * start querying the CPU for its capabilities. * * Since we're still in the early stages of bringing up this CPU, we're * limited in what we can do (e.g., no kmem_alloc/free), so after * reading the current microcode revision we have the control CPU do the * work of locating the microcode file and setting up the cpu_ucode_info * structure via ucode_locate(). With that done, we can apply the * microcode to this CPU (if any) and proceed with the BASIC cpuid pass. */ ucode_read_rev(cp); mp_startup_signal(&procset_slave, cp->cpu_id); mp_startup_wait(&procset_master, cp->cpu_id); ucode_apply(cp); cpuid_execpass(cp, CPUID_PASS_BASIC, new_x86_featureset); /* * Once this was done from assembly, but it's safer here; if * it blocks, we need to be able to swtch() to and from, and * since we get here by calling t_pc, we need to do that call * before swtch() overwrites it. */ (void) (*ap_mlsetup)(); #ifndef __xpv /* * Program this cpu's PAT */ pat_sync(); #endif /* * Set up TSC_AUX to contain the cpuid for this processor * for the rdtscp instruction. */ if (is_x86_feature(new_x86_featureset, X86FSET_TSCP)) (void) wrmsr(MSR_AMD_TSCAUX, cp->cpu_id); /* * Initialize this CPU's syscall handlers */ init_cpu_syscall(cp); /* * Enable interrupts with spl set to LOCK_LEVEL. LOCK_LEVEL is the * highest level at which a routine is permitted to block on * an adaptive mutex (allows for cpu poke interrupt in case * the cpu is blocked on a mutex and halts). Setting LOCK_LEVEL blocks * device interrupts that may end up in the hat layer issuing cross * calls before CPU_READY is set. */ splx(ipltospl(LOCK_LEVEL)); sti(); /* * There exists a small subset of systems which expose differing * MWAIT/MONITOR support between CPUs. If MWAIT support is absent from * the boot CPU, but is found on a later CPU, the system continues to * operate as if no MWAIT support is available. * * The reverse case, where MWAIT is available on the boot CPU but not * on a subsequently initialized CPU, is not presently allowed and will * result in a panic. */ if (is_x86_feature(x86_featureset, X86FSET_MWAIT) != is_x86_feature(new_x86_featureset, X86FSET_MWAIT)) { if (!is_x86_feature(x86_featureset, X86FSET_MWAIT)) { remove_x86_feature(new_x86_featureset, X86FSET_MWAIT); } else { panic("unsupported mixed cpu mwait support detected"); } } /* * We could be more sophisticated here, and just mark the CPU * as "faulted" but at this point we'll opt for the easier * answer of dying horribly. Provided the boot cpu is ok, * the system can be recovered by booting with use_mp set to zero. */ if (workaround_errata(cp) != 0) panic("critical workaround(s) missing for cpu%d", cp->cpu_id); /* * We can touch cpu_flags here without acquiring the cpu_lock here * because the cpu_lock is held by the control CPU which is running * mp_start_cpu_common(). * Need to clear CPU_QUIESCED flag before calling any function which * may cause thread context switching, such as kmem_alloc() etc. * The idle thread checks for CPU_QUIESCED flag and loops for ever if * it's set. So the startup thread may have no chance to switch back * again if it's switched away with CPU_QUIESCED set. */ cp->cpu_flags &= ~(CPU_POWEROFF | CPU_QUIESCED); enable_pcid(); /* * Setup this processor for XSAVE. */ if (fp_save_mech == FP_XSAVE) { xsave_setup_msr(cp); } cpuid_execpass(cp, CPUID_PASS_EXTENDED, NULL); cpuid_execpass(cp, CPUID_PASS_DYNAMIC, NULL); cpuid_execpass(cp, CPUID_PASS_RESOLVE, NULL); /* * Correct cpu_idstr and cpu_brandstr on target CPU after * CPUID_PASS_DYNAMIC is done. */ (void) cpuid_getidstr(cp, cp->cpu_idstr, CPU_IDSTRLEN); (void) cpuid_getbrandstr(cp, cp->cpu_brandstr, CPU_IDSTRLEN); cp->cpu_flags |= CPU_RUNNING | CPU_READY | CPU_EXISTS; post_startup_cpu_fixups(); cpu_event_init_cpu(cp); /* * Enable preemption here so that contention for any locks acquired * later in mp_startup_common may be preempted if the thread owning * those locks is continuously executing on other CPUs (for example, * this CPU must be preemptible to allow other CPUs to pause it during * their startup phases). It's safe to enable preemption here because * the CPU state is pretty-much fully constructed. */ curthread->t_preempt = 0; /* The base spl should still be at LOCK LEVEL here */ ASSERT(cp->cpu_base_spl == ipltospl(LOCK_LEVEL)); set_base_spl(); /* Restore the spl to its proper value */ pghw_physid_create(cp); /* * Delegate initialization tasks, which need to access the cpu_lock, * to mp_start_cpu_common() because we can't acquire the cpu_lock here * during CPU DR operations. */ mp_startup_signal(&procset_slave, cp->cpu_id); mp_startup_wait(&procset_master, cp->cpu_id); pg_cmt_cpu_startup(cp); if (boot) { mutex_enter(&cpu_lock); cp->cpu_flags &= ~CPU_OFFLINE; cpu_enable_intr(cp); cpu_add_active(cp); mutex_exit(&cpu_lock); } /* Enable interrupts */ (void) spl0(); /* * Clear the microcode update buffer allocated via ucode_locate(), if * any, for this CPU. */ ucode_finish(cp); /* * Do a sanity check to make sure this new CPU is a sane thing * to add to the collection of processors running this system. * * XXX Clearly this needs to get more sophisticated, if x86 * systems start to get built out of heterogenous CPUs; as is * likely to happen once the number of processors in a configuration * gets large enough. */ if (compare_x86_featureset(x86_featureset, new_x86_featureset) == B_FALSE) { cmn_err(CE_CONT, "cpu%d: featureset\n", cp->cpu_id); print_x86_featureset(new_x86_featureset); cmn_err(CE_WARN, "cpu%d feature mismatch", cp->cpu_id); } #ifndef __xpv { /* * Set up the CPU module for this CPU. This can't be done * before this CPU is made CPU_READY, because we may (in * heterogeneous systems) need to go load another CPU module. * The act of attempting to load a module may trigger a * cross-call, which will ASSERT unless this cpu is CPU_READY. */ cmi_hdl_t hdl; if ((hdl = cmi_init(CMI_HDL_NATIVE, cmi_ntv_hwchipid(CPU), cmi_ntv_hwcoreid(CPU), cmi_ntv_hwstrandid(CPU))) != NULL) { if (is_x86_feature(x86_featureset, X86FSET_MCA)) cmi_mca_init(hdl); cp->cpu_m.mcpu_cmi_hdl = hdl; } } #endif /* __xpv */ if (boothowto & RB_DEBUG) kdi_cpu_init(); (void) mach_cpu_create_device_node(cp, NULL); /* * Setting the bit in cpu_ready_set must be the last operation in * processor initialization; the boot CPU will continue to boot once * it sees this bit set for all active CPUs. */ CPUSET_ATOMIC_ADD(cpu_ready_set, cp->cpu_id); cmn_err(CE_CONT, "?cpu%d: %s\n", cp->cpu_id, cp->cpu_idstr); cmn_err(CE_CONT, "?cpu%d: %s\n", cp->cpu_id, cp->cpu_brandstr); cmn_err(CE_CONT, "?cpu%d initialization complete - online\n", cp->cpu_id); /* * Now we are done with the startup thread, so free it up. */ thread_exit(); /*NOTREACHED*/ } /* * Startup function for 'other' CPUs at boot time (besides boot cpu). */ static void mp_startup_boot(void) { mp_startup_common(B_TRUE); } /* * Startup function for hotplug CPUs at runtime. */ void mp_startup_hotplug(void) { mp_startup_common(B_FALSE); } /* * Start CPU on user request. */ /* ARGSUSED */ int mp_cpu_start(struct cpu *cp) { ASSERT(MUTEX_HELD(&cpu_lock)); return (0); } /* * Stop CPU on user request. */ int mp_cpu_stop(struct cpu *cp) { extern int cbe_psm_timer_mode; ASSERT(MUTEX_HELD(&cpu_lock)); #ifdef __xpv /* * We can't offline vcpu0. */ if (cp->cpu_id == 0) return (EBUSY); #endif /* * If TIMER_PERIODIC mode is used, CPU0 is the one running it; * can't stop it. (This is true only for machines with no TSC.) */ if ((cbe_psm_timer_mode == TIMER_PERIODIC) && (cp->cpu_id == 0)) return (EBUSY); return (0); } /* * Take the specified CPU out of participation in interrupts. * * Usually, we hold cpu_lock. But we cannot assert as such due to the * exception - i_cpr_save_context() - where we have mutual exclusion via a * separate mechanism. */ int cpu_disable_intr(struct cpu *cp) { if (psm_disable_intr(cp->cpu_id) != DDI_SUCCESS) return (EBUSY); cp->cpu_flags &= ~CPU_ENABLE; ncpus_intr_enabled--; return (0); } /* * Allow the specified CPU to participate in interrupts. */ void cpu_enable_intr(struct cpu *cp) { ASSERT(MUTEX_HELD(&cpu_lock)); cp->cpu_flags |= CPU_ENABLE; ncpus_intr_enabled++; psm_enable_intr(cp->cpu_id); } void mp_cpu_faulted_enter(struct cpu *cp) { #ifdef __xpv _NOTE(ARGUNUSED(cp)); #else cmi_hdl_t hdl = cp->cpu_m.mcpu_cmi_hdl; if (hdl != NULL) { cmi_hdl_hold(hdl); } else { hdl = cmi_hdl_lookup(CMI_HDL_NATIVE, cmi_ntv_hwchipid(cp), cmi_ntv_hwcoreid(cp), cmi_ntv_hwstrandid(cp)); } if (hdl != NULL) { cmi_faulted_enter(hdl); cmi_hdl_rele(hdl); } #endif } void mp_cpu_faulted_exit(struct cpu *cp) { #ifdef __xpv _NOTE(ARGUNUSED(cp)); #else cmi_hdl_t hdl = cp->cpu_m.mcpu_cmi_hdl; if (hdl != NULL) { cmi_hdl_hold(hdl); } else { hdl = cmi_hdl_lookup(CMI_HDL_NATIVE, cmi_ntv_hwchipid(cp), cmi_ntv_hwcoreid(cp), cmi_ntv_hwstrandid(cp)); } if (hdl != NULL) { cmi_faulted_exit(hdl); cmi_hdl_rele(hdl); } #endif } /* * The following two routines are used as context operators on threads belonging * to processes with a private LDT (see sysi86). Due to the rarity of such * processes, these routines are currently written for best code readability and * organization rather than speed. We could avoid checking x86_featureset at * every context switch by installing different context ops, depending on * x86_featureset, at LDT creation time -- one for each combination of fast * syscall features. */ void cpu_fast_syscall_disable(void) { if (is_x86_feature(x86_featureset, X86FSET_MSR) && is_x86_feature(x86_featureset, X86FSET_SEP)) cpu_sep_disable(); if (is_x86_feature(x86_featureset, X86FSET_MSR) && is_x86_feature(x86_featureset, X86FSET_ASYSC)) cpu_asysc_disable(); } void cpu_fast_syscall_enable(void) { if (is_x86_feature(x86_featureset, X86FSET_MSR) && is_x86_feature(x86_featureset, X86FSET_SEP)) cpu_sep_enable(); if (is_x86_feature(x86_featureset, X86FSET_MSR) && is_x86_feature(x86_featureset, X86FSET_ASYSC)) cpu_asysc_enable(); } static void cpu_sep_enable(void) { ASSERT(is_x86_feature(x86_featureset, X86FSET_SEP)); ASSERT(curthread->t_preempt || getpil() >= LOCK_LEVEL); wrmsr(MSR_INTC_SEP_CS, (uint64_t)(uintptr_t)KCS_SEL); CPU->cpu_m.mcpu_fast_syscall_state |= FSS_SEP_ENABLED; } static void cpu_sep_disable(void) { ASSERT(is_x86_feature(x86_featureset, X86FSET_SEP)); ASSERT(curthread->t_preempt || getpil() >= LOCK_LEVEL); /* * Setting the SYSENTER_CS_MSR register to 0 causes software executing * the sysenter or sysexit instruction to trigger a #gp fault. */ wrmsr(MSR_INTC_SEP_CS, 0); CPU->cpu_m.mcpu_fast_syscall_state &= ~FSS_SEP_ENABLED; } static void cpu_asysc_enable(void) { ASSERT(is_x86_feature(x86_featureset, X86FSET_ASYSC)); ASSERT(curthread->t_preempt || getpil() >= LOCK_LEVEL); wrmsr(MSR_AMD_EFER, rdmsr(MSR_AMD_EFER) | (uint64_t)(uintptr_t)AMD_EFER_SCE); CPU->cpu_m.mcpu_fast_syscall_state |= FSS_ASYSC_ENABLED; } static void cpu_asysc_disable(void) { ASSERT(is_x86_feature(x86_featureset, X86FSET_ASYSC)); ASSERT(curthread->t_preempt || getpil() >= LOCK_LEVEL); /* * Turn off the SCE (syscall enable) bit in the EFER register. Software * executing syscall or sysret with this bit off will incur a #ud trap. */ wrmsr(MSR_AMD_EFER, rdmsr(MSR_AMD_EFER) & ~((uint64_t)(uintptr_t)AMD_EFER_SCE)); CPU->cpu_m.mcpu_fast_syscall_state &= ~FSS_ASYSC_ENABLED; } /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright (c) 2006, 2010, Oracle and/or its affiliates. All rights reserved. * Copyright 2020 OmniOS Community Edition (OmniOSce) Association. * Copyright 2022 Oxide Computer Company */ #include #include #include #include #include #include #include #include #include #include /* * pci irq routing information table */ int pci_irq_nroutes; static pci_irq_route_t *pci_irq_routes; static int pci_bios_get_irq_routing(pci_irq_route_t *, int, int *); static void pci_get_irq_routing_table(void); /* * Retrieve information from the bios needed for system * configuration early during startup. */ void startup_pci_bios(void) { pci_get_irq_routing_table(); } /* * Issue the bios get irq routing information table interrupt * * Despite the name, the information in the table is only * used to derive slot names for some named pci hot-plug slots. * * Returns the number of irq routing table entries returned * by the bios, or 0 and optionally, the number of entries required. */ static int pci_bios_get_irq_routing(pci_irq_route_t *routes, int nroutes, int *nneededp) { struct bop_regs regs; uchar_t *hdrp; uchar_t *bufp; int i, n; int rval = 0; if (nneededp) *nneededp = 0; /* in UEFI system, there is no BIOS data */ if (BOP_GETPROPLEN(bootops, "efi-systab") > 0) return (0); /* * Set up irq routing header with the size and address * of some useable low-memory data addresses. Initalize * data area to zero, avoiding memcpy/bzero. */ hdrp = (uchar_t *)BIOS_IRQ_ROUTING_HDR; bufp = (uchar_t *)BIOS_IRQ_ROUTING_DATA; n = nroutes * sizeof (pci_irq_route_t); for (i = 0; i < n; i++) bufp[i] = 0; ((pci_irq_route_hdr_t *)hdrp)->pir_size = n; ((pci_irq_route_hdr_t *)hdrp)->pir_addr = (uint32_t)(uintptr_t)bufp; bzero(®s, sizeof (regs)); regs.eax.word.ax = (PCI_FUNCTION_ID << 8) | PCI_GET_IRQ_ROUTING; regs.ds = 0xf000; regs.es = FP_SEG((uint_t)(uintptr_t)hdrp); regs.edi.word.di = FP_OFF((uint_t)(uintptr_t)hdrp); BOP_DOINT(bootops, 0x1a, ®s); n = (int)(((pci_irq_route_hdr_t *)hdrp)->pir_size / sizeof (pci_irq_route_t)); if ((regs.eflags & PS_C) != 0) { if (nneededp) *nneededp = n; } else { /* * Copy resulting irq routing data from low memory up to * the kernel address space, avoiding memcpy as usual. */ if (n <= nroutes) { for (i = 0; i < n * sizeof (pci_irq_route_t); i++) ((uchar_t *)routes)[i] = bufp[i]; rval = n; } } return (rval); } static void pci_get_irq_routing_table(void) { pci_irq_route_t *routes; int n = N_PCI_IRQ_ROUTES; int nneeded = 0; int nroutes; /* * Get irq routing table information. * Allocate a buffer for an initial default number of entries. * If the bios indicates it needs a larger buffer, try it again. * Drive on if it still won't cooperate and play nice after that. */ routes = kmem_zalloc(n * sizeof (pci_irq_route_t), KM_SLEEP); nroutes = pci_bios_get_irq_routing(routes, n, &nneeded); if (nroutes == 0 && nneeded > n) { kmem_free(routes, n * sizeof (pci_irq_route_t)); if (nneeded > N_PCI_IRQ_ROUTES_MAX) { cmn_err(CE_CONT, "pci: unable to get IRQ routing information, " "required buffer space of %d entries exceeds max\n", nneeded); return; } n = nneeded; routes = kmem_zalloc(n * sizeof (pci_irq_route_t), KM_SLEEP); nroutes = pci_bios_get_irq_routing(routes, n, NULL); if (nroutes == 0) { cmn_err(CE_CONT, "pci: unable to get IRQ routing information, " "required buffer space for %d entries\n", n); } } if (nroutes > 0) { pci_irq_routes = routes; pci_irq_nroutes = nroutes; } else { kmem_free(routes, n * sizeof (pci_irq_route_t)); } } /* * Use the results of the PCI BIOS call that returned the routing tables * to build the 1275 slot-names property for the indicated bus. * Results are returned in buf. Length is return value, -1 is returned on * overflow and zero is returned if no data exists to build a property. */ int pci_slot_names_prop(int bus, char *buf, int len) { uchar_t dev; uchar_t slot[N_PCI_IRQ_ROUTES_MAX+1]; uint32_t mask; int i, nnames, plen; ASSERT(pci_irq_nroutes <= N_PCI_IRQ_ROUTES_MAX); if (pci_irq_nroutes == 0) return (0); nnames = 0; mask = 0; for (i = 0; i < pci_irq_nroutes; i++) slot[i] = 0xff; for (i = 0; i < pci_irq_nroutes; i++) { if (pci_irq_routes[i].pir_bus != bus) continue; if (pci_irq_routes[i].pir_slot != 0) { dev = (pci_irq_routes[i].pir_dev & 0xf8) >> 3; slot[dev] = pci_irq_routes[i].pir_slot; mask |= (1 << dev); nnames++; } } if (nnames == 0) return (0); if (len < (4 + nnames * 8)) return (-1); *(uint32_t *)buf = mask; plen = 4; for (i = 0; i < pci_irq_nroutes; i++) { if (slot[i] == 0xff) continue; (void) sprintf(buf + plen, "Slot%d", slot[i]); plen += strlen(buf+plen) + 1; *(buf + plen) = 0; } for (; plen % 4; plen++) *(buf + plen) = 0; return (plen); } /* * This is used to discover additional PCI buses that may exist in the system in * addition to the ACPI _BBN method. Historically these were discovered by * asking if there was a valid slot property, e.g. pci_slot_names_prop() * returned valid data. In this case we return any entry that has a bus number * and a non-zero slot value. We rely on the core PCI code to do dedup for us. */ void pci_bios_bus_iter(pci_prd_root_complex_f cbfunc, void *arg) { int i; for (i = 0; i < pci_irq_nroutes; i++) { if (pci_irq_routes[i].pir_slot != 0) { if (!cbfunc(pci_irq_routes[i].pir_bus, arg)) { return; } } } } /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved. */ #include #include #include #include #include #include #include #include #include #include #include #include #define PCIE_CFG_SPACE_SIZE (PCI_CONF_HDR_SIZE << 4) #define PCI_BDF_BUS(bdf) ((((uint16_t)bdf) & 0xff00) >> 8) #define PCI_BDF_DEV(bdf) ((((uint16_t)bdf) & 0xf8) >> 3) #define PCI_BDF_FUNC(bdf) (((uint16_t)bdf) & 0x7) /* patchable variables */ volatile boolean_t pci_cfgacc_force_io = B_FALSE; extern uintptr_t alloc_vaddr(size_t, paddr_t); void pci_cfgacc_acc(pci_cfgacc_req_t *); boolean_t pci_cfgacc_find_workaround(uint16_t); /* * IS_P2ALIGNED() is used to make sure offset is 'size'-aligned, so * it's guaranteed that the access will not cross 4k page boundary. * Thus only 1 page is allocated for all config space access, and the * virtual address of that page is cached in pci_cfgacc_virt_base. */ static caddr_t pci_cfgacc_virt_base = NULL; static caddr_t pci_cfgacc_map(paddr_t phys_addr) { #ifdef __xpv phys_addr = pfn_to_pa(xen_assign_pfn(mmu_btop(phys_addr))) | (phys_addr & MMU_PAGEOFFSET); #endif if (khat_running) { pfn_t pfn = mmu_btop(phys_addr); /* * pci_cfgacc_virt_base may hold address left from early * boot, which points to low mem. Realloc virtual address * in kernel space since it's already late in boot now. * Note: no need to unmap first, clear_boot_mappings() will * do that for us. */ if (pci_cfgacc_virt_base < (caddr_t)kernelbase) pci_cfgacc_virt_base = vmem_alloc(heap_arena, MMU_PAGESIZE, VM_SLEEP); hat_devload(kas.a_hat, pci_cfgacc_virt_base, MMU_PAGESIZE, pfn, PROT_READ | PROT_WRITE | HAT_STRICTORDER, HAT_LOAD_LOCK); } else { paddr_t pa_base = P2ALIGN(phys_addr, MMU_PAGESIZE); if (pci_cfgacc_virt_base == NULL) pci_cfgacc_virt_base = (caddr_t)alloc_vaddr(MMU_PAGESIZE, MMU_PAGESIZE); kbm_map((uintptr_t)pci_cfgacc_virt_base, pa_base, 0, 0); } return (pci_cfgacc_virt_base + (phys_addr & MMU_PAGEOFFSET)); } static void pci_cfgacc_unmap() { if (khat_running) hat_unload(kas.a_hat, pci_cfgacc_virt_base, MMU_PAGESIZE, HAT_UNLOAD_UNLOCK); } static void pci_cfgacc_io(pci_cfgacc_req_t *req) { uint8_t bus, dev, func; uint16_t ioacc_offset; /* 4K config access with IO ECS */ bus = PCI_BDF_BUS(req->bdf); dev = PCI_BDF_DEV(req->bdf); func = PCI_BDF_FUNC(req->bdf); ioacc_offset = req->offset; switch (req->size) { case 1: if (req->write) (*pci_putb_func)(bus, dev, func, ioacc_offset, VAL8(req)); else VAL8(req) = (*pci_getb_func)(bus, dev, func, ioacc_offset); break; case 2: if (req->write) (*pci_putw_func)(bus, dev, func, ioacc_offset, VAL16(req)); else VAL16(req) = (*pci_getw_func)(bus, dev, func, ioacc_offset); break; case 4: if (req->write) (*pci_putl_func)(bus, dev, func, ioacc_offset, VAL32(req)); else VAL32(req) = (*pci_getl_func)(bus, dev, func, ioacc_offset); break; case 8: if (req->write) { (*pci_putl_func)(bus, dev, func, ioacc_offset, VAL64(req) & 0xffffffff); (*pci_putl_func)(bus, dev, func, ioacc_offset + 4, VAL64(req) >> 32); } else { VAL64(req) = (*pci_getl_func)(bus, dev, func, ioacc_offset); VAL64(req) |= (uint64_t)(*pci_getl_func)(bus, dev, func, ioacc_offset + 4) << 32; } break; } } static void pci_cfgacc_mmio(pci_cfgacc_req_t *req) { caddr_t vaddr; paddr_t paddr; paddr = (paddr_t)req->bdf << 12; paddr += mcfg_mem_base + req->offset; mutex_enter(&pcicfg_mmio_mutex); vaddr = pci_cfgacc_map(paddr); switch (req->size) { case 1: if (req->write) *((uint8_t *)vaddr) = VAL8(req); else VAL8(req) = *((uint8_t *)vaddr); break; case 2: if (req->write) *((uint16_t *)vaddr) = VAL16(req); else VAL16(req) = *((uint16_t *)vaddr); break; case 4: if (req->write) *((uint32_t *)vaddr) = VAL32(req); else VAL32(req) = *((uint32_t *)vaddr); break; case 8: if (req->write) *((uint64_t *)vaddr) = VAL64(req); else VAL64(req) = *((uint64_t *)vaddr); break; } pci_cfgacc_unmap(); mutex_exit(&pcicfg_mmio_mutex); } static boolean_t pci_cfgacc_valid(pci_cfgacc_req_t *req, uint16_t cfgspc_size) { int sz = req->size; if (IS_P2ALIGNED(req->offset, sz) && (req->offset + sz - 1 < cfgspc_size) && ((sz & 0xf) && ISP2(sz))) return (B_TRUE); cmn_err(CE_WARN, "illegal PCI request: offset = %x, size = %d", req->offset, sz); return (B_FALSE); } void pci_cfgacc_check_io(pci_cfgacc_req_t *req) { uint8_t bus; bus = PCI_BDF_BUS(req->bdf); if (pci_cfgacc_force_io || (mcfg_mem_base == 0) || (bus < mcfg_bus_start) || (bus > mcfg_bus_end) || pci_cfgacc_find_workaround(req->bdf)) req->ioacc = B_TRUE; } void pci_cfgacc_acc(pci_cfgacc_req_t *req) { if (!req->write) VAL64(req) = (uint64_t)-1; pci_cfgacc_check_io(req); if (req->ioacc) { if (pci_cfgacc_valid(req, pci_iocfg_max_offset + 1)) pci_cfgacc_io(req); } else { if (pci_cfgacc_valid(req, PCIE_CFG_SPACE_SIZE)) pci_cfgacc_mmio(req); } } typedef struct cfgacc_bus_range { struct cfgacc_bus_range *next; uint16_t bdf; uchar_t secbus; uchar_t subbus; } cfgacc_bus_range_t; cfgacc_bus_range_t *pci_cfgacc_bus_head = NULL; #define BUS_INSERT(prev, el) \ el->next = *prev; \ *prev = el; #define BUS_REMOVE(prev, el) \ *prev = el->next; /* * This function is only supposed to be called in device tree setup time, * thus no lock is needed. */ void pci_cfgacc_add_workaround(uint16_t bdf, uchar_t secbus, uchar_t subbus) { cfgacc_bus_range_t *entry; entry = kmem_zalloc(sizeof (cfgacc_bus_range_t), KM_SLEEP); entry->bdf = bdf; entry->secbus = secbus; entry->subbus = subbus; BUS_INSERT(&pci_cfgacc_bus_head, entry); } boolean_t pci_cfgacc_find_workaround(uint16_t bdf) { cfgacc_bus_range_t *entry; uchar_t bus; for (entry = pci_cfgacc_bus_head; entry != NULL; entry = entry->next) { if (bdf == entry->bdf) { /* found a device which is known to be broken */ return (B_TRUE); } bus = PCI_BDF_BUS(bdf); if ((bus != 0) && (bus >= entry->secbus) && (bus <= entry->subbus)) { /* * found a device whose parent/grandparent is * known to be broken. */ return (B_TRUE); } } return (B_FALSE); } /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. * Copyright 2019 Joyent, Inc. * Copyright 2024 Oxide Computer Company */ /* * PCI configuration space access routines */ #include #include #include #include #include #include #include #include #if defined(__xpv) #include #endif #if defined(__xpv) int pci_max_nbus = 0xFE; #else int pci_max_nbus = 0xFF; #endif int pci_bios_cfg_type = PCI_MECHANISM_UNKNOWN; int pci_bios_maxbus; int pci_bios_mech; int pci_bios_vers; /* * These two variables can be used to force a configuration mechanism or * to force which function is used to probe for the presence of the PCI bus. */ int PCI_CFG_TYPE = 0; int PCI_PROBE_TYPE = 0; /* * No valid mcfg_mem_base by default, and accessing pci config space * in mem-mapped way is disabled. */ uint64_t mcfg_mem_base = 0; uint8_t mcfg_bus_start = 0; uint8_t mcfg_bus_end = 0xff; /* * Maximum offset in config space when not using MMIO */ uint_t pci_iocfg_max_offset = 0xff; /* * These function pointers lead to the actual implementation routines * for configuration space access. Normally they lead to either the * pci_mech1_* or pci_mech2_* routines, but they can also lead to * routines that work around chipset bugs. * These functions are accessing pci config space via I/O way. * Pci_cfgacc_get/put functions shoul be used as more common interfaces, * which also provide accessing pci config space via mem-mapped way. */ uint8_t (*pci_getb_func)(int bus, int dev, int func, int reg); uint16_t (*pci_getw_func)(int bus, int dev, int func, int reg); uint32_t (*pci_getl_func)(int bus, int dev, int func, int reg); void (*pci_putb_func)(int bus, int dev, int func, int reg, uint8_t val); void (*pci_putw_func)(int bus, int dev, int func, int reg, uint16_t val); void (*pci_putl_func)(int bus, int dev, int func, int reg, uint32_t val); extern void (*pci_cfgacc_acc_p)(pci_cfgacc_req_t *req); /* * Internal routines */ static int pci_check(void); #if !defined(__xpv) static int pci_check_bios(void); static int pci_get_cfg_type(void); #endif /* for legacy io-based config space access */ kmutex_t pcicfg_mutex; /* for mmio-based config space access */ kmutex_t pcicfg_mmio_mutex; /* ..except Orion and Neptune, which have to have their own */ kmutex_t pcicfg_chipset_mutex; void pci_cfgspace_init(void) { mutex_init(&pcicfg_mutex, NULL, MUTEX_SPIN, (ddi_iblock_cookie_t)ipltospl(15)); mutex_init(&pcicfg_mmio_mutex, NULL, MUTEX_SPIN, (ddi_iblock_cookie_t)ipltospl(DISP_LEVEL)); mutex_init(&pcicfg_chipset_mutex, NULL, MUTEX_SPIN, (ddi_iblock_cookie_t)ipltospl(15)); if (!pci_check()) { mutex_destroy(&pcicfg_mutex); mutex_destroy(&pcicfg_mmio_mutex); mutex_destroy(&pcicfg_chipset_mutex); } } /* * This code determines if this system supports PCI/PCIE and which * type of configuration access method is used */ static int pci_check(void) { uint64_t ecfginfo[4]; /* * Only do this once. NB: If this is not a PCI system, and we * get called twice, we can't detect it and will probably die * horribly when we try to ask the BIOS whether PCI is present. * This code is safe *ONLY* during system startup when the * BIOS is still available. */ if (pci_bios_cfg_type != PCI_MECHANISM_UNKNOWN) return (TRUE); #if defined(__xpv) /* * only support PCI config mechanism 1 in i86xpv. This should be fine * since the other ones are workarounds for old broken H/W which won't * be supported in i86xpv anyway. */ if (DOMAIN_IS_INITDOMAIN(xen_info)) { pci_bios_cfg_type = PCI_MECHANISM_1; pci_getb_func = pci_mech1_getb; pci_getw_func = pci_mech1_getw; pci_getl_func = pci_mech1_getl; pci_putb_func = pci_mech1_putb; pci_putw_func = pci_mech1_putw; pci_putl_func = pci_mech1_putl; /* * Since we can't get the BIOS info in i86xpv, we will do an * exhaustive search of all PCI buses. We have to do this until * we start using the PCI information in ACPI. */ pci_bios_maxbus = pci_max_nbus; } #else /* !__xpv */ pci_bios_cfg_type = pci_check_bios(); if (pci_bios_cfg_type == PCI_MECHANISM_NONE) { /* * Default to mechanism 1, and scan all PCI buses */ pci_bios_cfg_type = PCI_MECHANISM_1; pci_bios_maxbus = pci_max_nbus; } switch (pci_get_cfg_type()) { case PCI_MECHANISM_1: if (pci_is_broken_orion()) { pci_getb_func = pci_orion_getb; pci_getw_func = pci_orion_getw; pci_getl_func = pci_orion_getl; pci_putb_func = pci_orion_putb; pci_putw_func = pci_orion_putw; pci_putl_func = pci_orion_putl; } else if (pci_check_amd_ioecs()) { pci_getb_func = pci_mech1_amd_getb; pci_getw_func = pci_mech1_amd_getw; pci_getl_func = pci_mech1_amd_getl; pci_putb_func = pci_mech1_amd_putb; pci_putw_func = pci_mech1_amd_putw; pci_putl_func = pci_mech1_amd_putl; pci_iocfg_max_offset = 0xfff; } else { pci_getb_func = pci_mech1_getb; pci_getw_func = pci_mech1_getw; pci_getl_func = pci_mech1_getl; pci_putb_func = pci_mech1_putb; pci_putw_func = pci_mech1_putw; pci_putl_func = pci_mech1_putl; } break; case PCI_MECHANISM_2: if (pci_check_neptune()) { /* * The BIOS for some systems with the Intel * Neptune chipset seem to default to #2 even * though the chipset can do #1. Override * the BIOS so that MP systems will work * correctly. */ pci_getb_func = pci_neptune_getb; pci_getw_func = pci_neptune_getw; pci_getl_func = pci_neptune_getl; pci_putb_func = pci_neptune_putb; pci_putw_func = pci_neptune_putw; pci_putl_func = pci_neptune_putl; } else { pci_getb_func = pci_mech2_getb; pci_getw_func = pci_mech2_getw; pci_getl_func = pci_mech2_getl; pci_putb_func = pci_mech2_putb; pci_putw_func = pci_mech2_putw; pci_putl_func = pci_mech2_putl; } break; default: return (FALSE); } #endif /* __xpv */ /* * Try to get a valid mcfg_mem_base in early boot * If failed, leave mem-mapped pci config space accessing disabled * until pci boot code (pci_autoconfig) makes sure this is a PCIE * platform. */ if (do_bsys_getprop(NULL, MCFG_PROPNAME, ecfginfo) != -1) { mcfg_mem_base = ecfginfo[0]; mcfg_bus_start = ecfginfo[2]; mcfg_bus_end = ecfginfo[3]; } /* See pci_cfgacc.c */ pci_cfgacc_acc_p = pci_cfgacc_acc; return (TRUE); } #if !defined(__xpv) static int pci_check_bios(void) { struct bop_regs regs; uint32_t carryflag; uint16_t ax, dx; /* * This mechanism uses a legacy BIOS call to detect PCI configuration, * but such calls are not available on systems with UEFI firmware. * For UEFI systems we must assume some reasonable defaults and scan * all possible buses. */ if (BOP_GETPROPLEN(bootops, "efi-systab") > 0) { pci_bios_mech = 1; pci_bios_vers = 0; pci_bios_maxbus = pci_max_nbus; return (PCI_MECHANISM_1); } bzero(®s, sizeof (regs)); regs.eax.word.ax = (PCI_FUNCTION_ID << 8) | PCI_BIOS_PRESENT; BOP_DOINT(bootops, 0x1a, ®s); carryflag = regs.eflags & PS_C; ax = regs.eax.word.ax; dx = regs.edx.word.dx; /* the carry flag must not be set */ if (carryflag != 0) return (PCI_MECHANISM_NONE); if (dx != ('P' | 'C'<<8)) return (PCI_MECHANISM_NONE); /* ah (the high byte of ax) must be zero */ if ((ax & 0xff00) != 0) return (PCI_MECHANISM_NONE); pci_bios_mech = (ax & 0x3); pci_bios_vers = regs.ebx.word.bx; /* * Several BIOS implementations have known problems where they don't end * up correctly telling us to scan all PCI buses in the system. In * particular, many on-die CPU PCI devices are on a last bus that is * sometimes not enumerated. As such, do not trust the BIOS. */ pci_bios_maxbus = pci_max_nbus; switch (pci_bios_mech) { default: /* ?!? */ case 0: /* supports neither? */ return (PCI_MECHANISM_NONE); case 1: case 3: /* supports both */ return (PCI_MECHANISM_1); case 2: return (PCI_MECHANISM_2); } } static int pci_get_cfg_type(void) { /* Check to see if the config mechanism has been set in /etc/system */ switch (PCI_CFG_TYPE) { default: case 0: break; case 1: return (PCI_MECHANISM_1); case 2: return (PCI_MECHANISM_2); case -1: return (PCI_MECHANISM_NONE); } /* call one of the PCI detection algorithms */ switch (PCI_PROBE_TYPE) { default: case 0: /* From pci_check() and pci_check_bios() */ return (pci_bios_cfg_type); case -1: return (PCI_MECHANISM_NONE); } } #endif /* __xpv */ /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License, Version 1.0 only * (the "License"). You may not use this file except in compliance * with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2005 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ /* * Copyright 2021 Oxide Computer Company */ /* * PCI Mechanism 1 low-level routines */ #include #include #include #include #include /* * Per PCI 2.1 section 3.7.4.1 and PCI-PCI Bridge Architecture 1.0 section * 5.3.1.2: dev=31 func=7 reg=0 means a special cycle. We don't want to * trigger that by accident, so we pretend that dev 31, func 7 doesn't * exist. If we ever want special cycle support, we'll add explicit * special cycle support. */ uint8_t pci_mech1_getb(int bus, int device, int function, int reg) { uint8_t val; if (device == PCI_MECH1_SPEC_CYCLE_DEV && function == PCI_MECH1_SPEC_CYCLE_FUNC) { return (PCI_EINVAL8); } if (reg > pci_iocfg_max_offset) { return (PCI_EINVAL8); } mutex_enter(&pcicfg_mutex); outl(PCI_CONFADD, PCI_CADDR1(bus, device, function, reg)); val = inb(PCI_CONFDATA | (reg & 0x3)); mutex_exit(&pcicfg_mutex); return (val); } uint16_t pci_mech1_getw(int bus, int device, int function, int reg) { uint16_t val; if (device == PCI_MECH1_SPEC_CYCLE_DEV && function == PCI_MECH1_SPEC_CYCLE_FUNC) { return (PCI_EINVAL16); } if (reg > pci_iocfg_max_offset) { return (PCI_EINVAL16); } mutex_enter(&pcicfg_mutex); outl(PCI_CONFADD, PCI_CADDR1(bus, device, function, reg)); val = inw(PCI_CONFDATA | (reg & 0x2)); mutex_exit(&pcicfg_mutex); return (val); } uint32_t pci_mech1_getl(int bus, int device, int function, int reg) { uint32_t val; if (device == PCI_MECH1_SPEC_CYCLE_DEV && function == PCI_MECH1_SPEC_CYCLE_FUNC) { return (PCI_EINVAL32); } if (reg > pci_iocfg_max_offset) { return (PCI_EINVAL32); } mutex_enter(&pcicfg_mutex); outl(PCI_CONFADD, PCI_CADDR1(bus, device, function, reg)); val = inl(PCI_CONFDATA); mutex_exit(&pcicfg_mutex); return (val); } void pci_mech1_putb(int bus, int device, int function, int reg, uint8_t val) { if (device == PCI_MECH1_SPEC_CYCLE_DEV && function == PCI_MECH1_SPEC_CYCLE_FUNC) { return; } if (reg > pci_iocfg_max_offset) { return; } mutex_enter(&pcicfg_mutex); outl(PCI_CONFADD, PCI_CADDR1(bus, device, function, reg)); outb(PCI_CONFDATA | (reg & 0x3), val); mutex_exit(&pcicfg_mutex); } void pci_mech1_putw(int bus, int device, int function, int reg, uint16_t val) { if (device == PCI_MECH1_SPEC_CYCLE_DEV && function == PCI_MECH1_SPEC_CYCLE_FUNC) { return; } if (reg > pci_iocfg_max_offset) { return; } mutex_enter(&pcicfg_mutex); outl(PCI_CONFADD, PCI_CADDR1(bus, device, function, reg)); outw(PCI_CONFDATA | (reg & 0x2), val); mutex_exit(&pcicfg_mutex); } void pci_mech1_putl(int bus, int device, int function, int reg, uint32_t val) { if (device == PCI_MECH1_SPEC_CYCLE_DEV && function == PCI_MECH1_SPEC_CYCLE_FUNC) { return; } if (reg > pci_iocfg_max_offset) { return; } mutex_enter(&pcicfg_mutex); outl(PCI_CONFADD, PCI_CADDR1(bus, device, function, reg)); outl(PCI_CONFDATA, val); mutex_exit(&pcicfg_mutex); } /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2010 Advanced Micro Devices, Inc. * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved. * Copyright 2022 Oxide Computer Company */ /* * PCI Mechanism 1 low-level routines with ECS support for AMD family >= 0x10 */ #include #include #include #include #include #include #include #include boolean_t pci_check_amd_ioecs(void) { struct cpuid_regs cp; int family; ASSERT(is_x86_feature(x86_featureset, X86FSET_CPUID)); /* * Get the CPU vendor string from CPUID. * This PCI mechanism only applies to AMD CPUs. */ cp.cp_eax = 0; (void) __cpuid_insn(&cp); if ((cp.cp_ebx != 0x68747541) || /* Auth */ (cp.cp_edx != 0x69746e65) || /* enti */ (cp.cp_ecx != 0x444d4163)) /* cAMD */ return (B_FALSE); /* * Get the CPU family from CPUID. * This PCI mechanism is only available on family 0x10 or higher. */ cp.cp_eax = 1; (void) __cpuid_insn(&cp); family = ((cp.cp_eax >> 8) & 0xf) + ((cp.cp_eax >> 20) & 0xff); if (family < 0x10) return (B_FALSE); /* * Set the EnableCf8ExtCfg bit in the Northbridge Configuration Register * to enable accessing PCI ECS using in/out instructions. */ wrmsr(MSR_AMD_NB_CFG, rdmsr(MSR_AMD_NB_CFG) | AMD_GH_NB_CFG_EN_ECS); return (B_TRUE); } /* * Macro to setup PCI Extended Configuration Space (ECS) address to give to * "in/out" instructions */ #define PCI_CADDR1_ECS(b, d, f, r) \ (PCI_CADDR1((b), (d), (f), (r)) | ((((r) >> 8) & 0xf) << 24)) /* * Per PCI 2.1 section 3.7.4.1 and PCI-PCI Bridge Architecture 1.0 section * 5.3.1.2: dev=31 func=7 reg=0 means a special cycle. We don't want to * trigger that by accident, so we pretend that dev 31, func 7 doesn't * exist. If we ever want special cycle support, we'll add explicit * special cycle support. */ uint8_t pci_mech1_amd_getb(int bus, int device, int function, int reg) { uint8_t val; if (device == PCI_MECH1_SPEC_CYCLE_DEV && function == PCI_MECH1_SPEC_CYCLE_FUNC) { return (PCI_EINVAL8); } if (reg > pci_iocfg_max_offset) { return (PCI_EINVAL8); } mutex_enter(&pcicfg_mutex); outl(PCI_CONFADD, PCI_CADDR1_ECS(bus, device, function, reg)); val = inb(PCI_CONFDATA | (reg & 0x3)); mutex_exit(&pcicfg_mutex); return (val); } uint16_t pci_mech1_amd_getw(int bus, int device, int function, int reg) { uint16_t val; if (device == PCI_MECH1_SPEC_CYCLE_DEV && function == PCI_MECH1_SPEC_CYCLE_FUNC) { return (PCI_EINVAL16); } if (reg > pci_iocfg_max_offset) { return (PCI_EINVAL16); } mutex_enter(&pcicfg_mutex); outl(PCI_CONFADD, PCI_CADDR1_ECS(bus, device, function, reg)); val = inw(PCI_CONFDATA | (reg & 0x2)); mutex_exit(&pcicfg_mutex); return (val); } uint32_t pci_mech1_amd_getl(int bus, int device, int function, int reg) { uint32_t val; if (device == PCI_MECH1_SPEC_CYCLE_DEV && function == PCI_MECH1_SPEC_CYCLE_FUNC) { return (PCI_EINVAL32); } if (reg > pci_iocfg_max_offset) { return (PCI_EINVAL32); } mutex_enter(&pcicfg_mutex); outl(PCI_CONFADD, PCI_CADDR1_ECS(bus, device, function, reg)); val = inl(PCI_CONFDATA); mutex_exit(&pcicfg_mutex); return (val); } void pci_mech1_amd_putb(int bus, int device, int function, int reg, uint8_t val) { if (device == PCI_MECH1_SPEC_CYCLE_DEV && function == PCI_MECH1_SPEC_CYCLE_FUNC) { return; } mutex_enter(&pcicfg_mutex); outl(PCI_CONFADD, PCI_CADDR1_ECS(bus, device, function, reg)); outb(PCI_CONFDATA | (reg & 0x3), val); mutex_exit(&pcicfg_mutex); } void pci_mech1_amd_putw(int bus, int device, int function, int reg, uint16_t val) { if (device == PCI_MECH1_SPEC_CYCLE_DEV && function == PCI_MECH1_SPEC_CYCLE_FUNC) { return; } mutex_enter(&pcicfg_mutex); outl(PCI_CONFADD, PCI_CADDR1_ECS(bus, device, function, reg)); outw(PCI_CONFDATA | (reg & 0x2), val); mutex_exit(&pcicfg_mutex); } void pci_mech1_amd_putl(int bus, int device, int function, int reg, uint32_t val) { if (device == PCI_MECH1_SPEC_CYCLE_DEV && function == PCI_MECH1_SPEC_CYCLE_FUNC) { return; } mutex_enter(&pcicfg_mutex); outl(PCI_CONFADD, PCI_CADDR1_ECS(bus, device, function, reg)); outl(PCI_CONFDATA, val); mutex_exit(&pcicfg_mutex); } /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License, Version 1.0 only * (the "License"). You may not use this file except in compliance * with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2005 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ /* * Copyright 2021 Oxide Computer Company */ /* * PCI Mechanism 2 primitives */ #include #include #include #include #include /* * The "mechanism 2" interface only has 4 bits for device number. To * hide this implementation detail, we return all ones for accesses to * devices 16..31. */ #define PCI_MAX_DEVS_2 16 /* * the PCI LOCAL BUS SPECIFICATION 2.0 does not say that you need to * save the value of the register and restore them. The Intel chip * set documentation indicates that you should. */ static uint8_t pci_mech2_config_enable(uchar_t bus, uchar_t function) { uint8_t old; mutex_enter(&pcicfg_mutex); old = inb(PCI_CSE_PORT); outb(PCI_CSE_PORT, PCI_MECH2_CONFIG_ENABLE | ((function & PCI_FUNC_MASK) << 1)); outb(PCI_FORW_PORT, bus); return (old); } static void pci_mech2_config_restore(uint8_t oldstatus) { outb(PCI_CSE_PORT, oldstatus); mutex_exit(&pcicfg_mutex); } uint8_t pci_mech2_getb(int bus, int device, int function, int reg) { uint8_t tmp; uint8_t val; if (device >= PCI_MAX_DEVS_2 || reg > pci_iocfg_max_offset) return (PCI_EINVAL8); tmp = pci_mech2_config_enable(bus, function); val = inb(PCI_CADDR2(device, reg)); pci_mech2_config_restore(tmp); return (val); } uint16_t pci_mech2_getw(int bus, int device, int function, int reg) { uint8_t tmp; uint16_t val; if (device >= PCI_MAX_DEVS_2 || reg > pci_iocfg_max_offset) return (PCI_EINVAL16); tmp = pci_mech2_config_enable(bus, function); val = inw(PCI_CADDR2(device, reg)); pci_mech2_config_restore(tmp); return (val); } uint32_t pci_mech2_getl(int bus, int device, int function, int reg) { uint8_t tmp; uint32_t val; if (device >= PCI_MAX_DEVS_2 || reg > pci_iocfg_max_offset) return (PCI_EINVAL32); tmp = pci_mech2_config_enable(bus, function); val = inl(PCI_CADDR2(device, reg)); pci_mech2_config_restore(tmp); return (val); } void pci_mech2_putb(int bus, int device, int function, int reg, uint8_t val) { uint8_t tmp; if (device >= PCI_MAX_DEVS_2 || reg > pci_iocfg_max_offset) return; tmp = pci_mech2_config_enable(bus, function); outb(PCI_CADDR2(device, reg), val); pci_mech2_config_restore(tmp); } void pci_mech2_putw(int bus, int device, int function, int reg, uint16_t val) { uint8_t tmp; if (device >= PCI_MAX_DEVS_2 || reg > pci_iocfg_max_offset) return; tmp = pci_mech2_config_enable(bus, function); outw(PCI_CADDR2(device, reg), val); pci_mech2_config_restore(tmp); } void pci_mech2_putl(int bus, int device, int function, int reg, uint32_t val) { uint8_t tmp; if (device >= PCI_MAX_DEVS_2 || reg > pci_iocfg_max_offset) return; tmp = pci_mech2_config_enable(bus, function); outl(PCI_CADDR2(device, reg), val); pci_mech2_config_restore(tmp); } /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License, Version 1.0 only * (the "License"). You may not use this file except in compliance * with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2005 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ /* * Support for Intel "Neptune" PCI chip set */ #include #include #include #include #include /* * This variable is a place holder for the initial value in PCI_PMC register * of neptune chipset. */ static unsigned char neptune_BIOS_cfg_method = 0; /* * Special hack for Intel's Neptune chipset, 82433NX and 82434NX. * * The motherboards I've seen still use a version of the BIOS * that operates using Configuration Mechanism #2 like the older * Mercury BIOS and chipset (the 82433LX and 82434LX). * */ boolean_t pci_check_neptune(void) { uint8_t oldstatus; uint32_t tmp; /* enable the config address space, bus=0 function=0 */ oldstatus = inb(PCI_CSE_PORT); outb(PCI_CSE_PORT, PCI_MECH2_CONFIG_ENABLE); outb(PCI_FORW_PORT, 0); /* * First check the vendor and device ids of the Host to * PCI bridge. But it isn't sufficient just to do this check * because the same device ID can refer to either * the Neptune or Mercury chipset. */ /* check the vendor id, the device id, and the revision id */ /* the Neptune revision ID == 0x11, allow 0x1? */ if ((inl(PCI_CADDR2(0, PCI_CONF_VENID)) != 0x04a38086) || (inb(PCI_CADDR2(0, PCI_CONF_REVID)) & 0xf0) != 0x10) { /* disable mechanism #2 config address space */ outb(PCI_CSE_PORT, oldstatus); return (B_FALSE); } /* disable mechanism #2 config address space */ outb(PCI_CSE_PORT, oldstatus); /* * Now I know that the bridge *might* be a Neptune (it could be * a Mercury chip.) Try enabling mechanism #1 to differentiate * between the two chipsets. */ /* * save the old value in case it's not Neptune (the Mercury * chip has the deturbo and reset bits in the 0xcf9 register * and the forward register at 0xcfa) */ tmp = inl(PCI_CONFADD); /* * The Intel Neptune chipset defines this extra register * to enable Config Mechanism #1. */ neptune_BIOS_cfg_method = inb(PCI_PMC); outb(PCI_PMC, neptune_BIOS_cfg_method | 1); /* make certain mechanism #1 works correctly */ /* check the vendor and device id's of the Host to PCI bridge */ outl(PCI_CONFADD, PCI_CADDR1(0, 0, 0, PCI_CONF_VENID)); if (inl(PCI_CONFDATA) != ((0x04a3 << 16) | 0x8086)) { outb(PCI_PMC, neptune_BIOS_cfg_method); outl(PCI_CONFADD, tmp); return (B_FALSE); } outb(PCI_PMC, neptune_BIOS_cfg_method); return (B_TRUE); } static void pci_neptune_enable() { /* * Switch the chipset to use Mechanism 1. */ mutex_enter(&pcicfg_chipset_mutex); outb(PCI_PMC, neptune_BIOS_cfg_method | 1); } static void pci_neptune_disable() { /* * The Neptune chipset has a bug that if you write the PMC, * it erroneously looks at some of the bits in the latches for * adjacent registers... like, say, the "reset" bit. We zero * out the config address register to work around this bug. */ outl(PCI_CONFADD, PCI_CADDR1(0, 0, 0, 0)); outb(PCI_PMC, neptune_BIOS_cfg_method); mutex_exit(&pcicfg_chipset_mutex); } uint8_t pci_neptune_getb(int bus, int device, int function, int reg) { uint8_t val; pci_neptune_enable(); val = pci_mech1_getb(bus, device, function, reg); pci_neptune_disable(); return (val); } uint16_t pci_neptune_getw(int bus, int device, int function, int reg) { uint16_t val; pci_neptune_enable(); val = pci_mech1_getw(bus, device, function, reg); pci_neptune_disable(); return (val); } uint32_t pci_neptune_getl(int bus, int device, int function, int reg) { uint32_t val; pci_neptune_enable(); val = pci_mech1_getl(bus, device, function, reg); pci_neptune_disable(); return (val); } void pci_neptune_putb(int bus, int device, int function, int reg, uint8_t val) { pci_neptune_enable(); pci_mech1_putb(bus, device, function, reg, val); pci_neptune_disable(); } void pci_neptune_putw(int bus, int device, int function, int reg, uint16_t val) { pci_neptune_enable(); pci_mech1_putw(bus, device, function, reg, val); pci_neptune_disable(); } void pci_neptune_putl(int bus, int device, int function, int reg, uint32_t val) { pci_neptune_enable(); pci_mech1_putl(bus, device, function, reg, val); pci_neptune_disable(); } /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License, Version 1.0 only * (the "License"). You may not use this file except in compliance * with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2005 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. * * Derived from pseudocode supplied by Intel. */ /* * Workaround for Intel Orion chipset bug * * It is intended that this code implements exactly the workaround * described in the errata. There is one exception, described below. */ #include #include #include #include #define PCI_82454_RW_CONTROL 0x54 static int ncDevNo; boolean_t pci_is_broken_orion() { int Num82454 = 0; boolean_t A2B0Found = B_FALSE; boolean_t c82454PostingEnabled = B_FALSE; uint8_t PciReg; uint16_t VendorID; uint16_t DeviceID; boolean_t A2B0WorkAroundReqd; int BusNo = 0; int FunctionNo = 0; int DeviceNo; uint8_t RevisionID; for (DeviceNo = 0; DeviceNo < PCI_MAX_DEVS; DeviceNo++) { VendorID = pci_mech1_getw(BusNo, DeviceNo, FunctionNo, PCI_CONF_VENID); DeviceID = pci_mech1_getw(BusNo, DeviceNo, FunctionNo, PCI_CONF_DEVID); RevisionID = pci_mech1_getb(BusNo, DeviceNo, FunctionNo, PCI_CONF_REVID); if (VendorID == 0x8086 && DeviceID == 0x84c4) { /* Found 82454 PCI Bridge */ Num82454++; if (RevisionID <= 4) { A2B0Found = B_TRUE; } if (DeviceNo == (0xc8 >> 3)) { /* * c82454 Found - determine the status of * inbound posting. */ PciReg = pci_mech1_getb(BusNo, DeviceNo, FunctionNo, PCI_82454_RW_CONTROL); if (PciReg & 0x01) { c82454PostingEnabled = B_TRUE; } } else { /* nc82454 Found - store device no. */ ncDevNo = DeviceNo; } } } /* DeviceNo */ /* * Determine if nc82454 posting is to be enabled * and need of workaround. * * [[ This is a deviation from the pseudocode in the errata. * The errata has mismatched braces, leading to uncertainty * as to whether this code is inside the test for 8086/84c4. * The errata has this code clearly inside the DeviceNo loop. * This code is obviously pointless until you've at least found * the second 82454, and there's no need to execute it more * than once, so I'm moving it outside that loop to execute * once on completion of the scan. ]] */ if (Num82454 >= 2 && A2B0Found && c82454PostingEnabled) { A2B0WorkAroundReqd = B_TRUE; /* Enable inbound posting on nc82454 */ PciReg = pci_mech1_getb(0, ncDevNo, 0, PCI_82454_RW_CONTROL); PciReg |= 0x01; pci_mech1_putb(0, ncDevNo, 0, PCI_82454_RW_CONTROL, PciReg); } else { A2B0WorkAroundReqd = B_FALSE; } return (A2B0WorkAroundReqd); } /* * When I first read this code in the errata document, I asked "why doesn't * the initial read of CFC (possibly) lead to the 'two responses' problem?" * * After thinking about it for a while, the answer is that we're trying to * talk to the nc82454 itself. The c82454 doesn't have the problem, so it * will recognize that this request is *not* for it, and won't respond. * The nc82454 will either respond or not, depending on whether it "saw" * the CF8 write, and if it responds it might or might not return the * right data. That's all pretty much OK, if we're willing to assume * that the only way that 84C48086 will come back is from the vendor ID/ * device ID registers on the nc82454. This is probabilistic, of course, * because the nc82454 *could* be pointing at a register on some device * that just *happened* to have that value, but that seems unlikely. */ static void FuncDisableInboundPostingnc82454() { uint32_t test; uint8_t PciReg; mutex_enter(&pcicfg_chipset_mutex); do { test = pci_mech1_getl(0, ncDevNo, 0, PCI_CONF_VENID); } while (test != 0x84c48086UL); /* * At this point we are guaranteed to be pointing to the nc82454 PCI * bridge Vendor ID register. */ do { /* * Impact of the erratum is that the configuration read will * return the value which was last read. * Hence read register 0x54 until the previous read value * (VendorId/DeviceId) is not read anymore. */ test = pci_mech1_getl(0, ncDevNo, 0, PCI_82454_RW_CONTROL); } while (test == 0x84c48086UL); /* * At this point we are guaranteed to be pointing to the PCI * Read/Write Control Register in the nc82454 PCI Bridge. */ PciReg = pci_mech1_getb(0, ncDevNo, 0, PCI_82454_RW_CONTROL); PciReg &= ~0x01; pci_mech1_putb(0, ncDevNo, 0, PCI_82454_RW_CONTROL, PciReg); } static void FuncEnableInboundPostingnc82454() { uint8_t PciReg; PciReg = pci_mech1_getb(0, ncDevNo, 0, PCI_82454_RW_CONTROL); PciReg |= 0x01; pci_mech1_putb(0, ncDevNo, 0, PCI_82454_RW_CONTROL, PciReg); mutex_exit(&pcicfg_chipset_mutex); } uint8_t pci_orion_getb(int bus, int device, int function, int reg) { uint8_t val; FuncDisableInboundPostingnc82454(); val = pci_mech1_getb(bus, device, function, reg); FuncEnableInboundPostingnc82454(); return (val); } uint16_t pci_orion_getw(int bus, int device, int function, int reg) { uint16_t val; FuncDisableInboundPostingnc82454(); val = pci_mech1_getw(bus, device, function, reg); FuncEnableInboundPostingnc82454(); return (val); } uint32_t pci_orion_getl(int bus, int device, int function, int reg) { uint32_t val; FuncDisableInboundPostingnc82454(); val = pci_mech1_getl(bus, device, function, reg); FuncEnableInboundPostingnc82454(); return (val); } void pci_orion_putb(int bus, int device, int function, int reg, uint8_t val) { FuncDisableInboundPostingnc82454(); pci_mech1_putb(bus, device, function, reg, val); FuncEnableInboundPostingnc82454(); } void pci_orion_putw(int bus, int device, int function, int reg, uint16_t val) { FuncDisableInboundPostingnc82454(); pci_mech1_putw(bus, device, function, reg, val); FuncEnableInboundPostingnc82454(); } void pci_orion_putl(int bus, int device, int function, int reg, uint32_t val) { FuncDisableInboundPostingnc82454(); pci_mech1_putl(bus, device, function, reg, val); FuncEnableInboundPostingnc82454(); } /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2007 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ /* * PMEM - Direct mapping physical memory pages to userland process * * Provide functions used for directly (w/o occupying kernel virtual address * space) allocating and exporting physical memory pages to userland. */ #include #include #include #include #include #include #include #include #include #include #include /* * The routines in this file allocate memory which will be accessed through * the AGP GART hardware. The GART is programmed with the PFNs for this * memory, and the only mechanism for removing these entries is by an * explicit process operation (ioctl/close of the driver, or process exit). * As such, the pages need to remain locked to ensure that they won't be * relocated or paged out. * * To prevent these locked pages from getting in the way of page * coalescing, we try to allocate large pages from the system, and carve * them up to satisfy pmem allocation requests. This will keep the locked * pages within a constrained area of physical memory, limiting the number * of large pages that would be pinned by our locked pages. This is, of * course, another take on the infamous kernel cage, and it has many of the * downsides of the original cage. It also interferes with system-wide * resource management decisions, as it maintains its own pool of unused * pages which can't be easily reclaimed and used during low-memory * situations. * * The right solution is for pmem to register a callback that the VM system * could call, which would temporarily remove any GART entries for pages * that were being relocated. This would let us leave the pages unlocked, * which would remove the need for using large pages, which would simplify * this code a great deal. Unfortunately, the support for these callbacks * only exists on some SPARC platforms right now. * * Note that this is the *only* reason that large pages are used here. The * GART can't perform large-page translations, and the code appropriately * falls back to using small pages if page_create_va_large() fails. */ #define HOLD_DHP_LOCK(dhp) if (dhp->dh_flags & DEVMAP_ALLOW_REMAP) \ { mutex_enter(&dhp->dh_lock); } #define RELE_DHP_LOCK(dhp) if (dhp->dh_flags & DEVMAP_ALLOW_REMAP) \ { mutex_exit(&dhp->dh_lock); } #define FROM_LPG(pp) (pp->p_szc != 0) #define PFIND(pp) (page_pptonum(pp) & (pmem_pgcnt - 1)) /* * Structs and static variables used for pmem only. */ typedef struct pmem_lpg { page_t *pl_pp; /* start pp */ ulong_t *pl_bitmap; /* allocation status for each page */ ushort_t pl_pfree; /* this large page might be fully freed */ struct pmem_lpg *pl_next; struct pmem_lpg *pl_prev; } pmem_lpg_t; static size_t pmem_lpgsize; /* the size of one large page */ static pgcnt_t pmem_pgcnt; /* the number of small pages in a large page */ static uint_t pmem_lszc; /* page size code of the large page */ /* The segment to be associated with all the allocated pages. */ static struct seg pmem_seg; /* Fully occupied large pages allocated for pmem. */ static pmem_lpg_t *pmem_occ_lpgs; /* Memory pool to store residual small pages from large pages. */ static page_t *pmem_mpool = NULL; /* Number of small pages reside in pmem_mpool currently. */ static pgcnt_t pmem_nmpages = 0; /* To protect pmem_nmpages, pmem_mpool and pmem_occ_lpgs. */ kmutex_t pmem_mutex; static int lpg_isfree(pmem_lpg_t *); static void pmem_lpg_sub(pmem_lpg_t **, pmem_lpg_t *); static void pmem_lpg_concat(pmem_lpg_t **, pmem_lpg_t **); static pmem_lpg_t *pmem_lpg_get(pmem_lpg_t *, page_t *, pmem_lpg_t **); static pmem_lpg_t *pmem_lpg_alloc(uint_t); static void pmem_lpg_free(pmem_lpg_t **, pmem_lpg_t *); static void lpg_free(page_t *spp); static pgcnt_t mpool_break(page_t **, pgcnt_t); static void mpool_append(page_t **, pgcnt_t); static void lpp_break(page_t **, pgcnt_t, pgcnt_t, pmem_lpg_t *); static void lpp_free(page_t *, pgcnt_t, pmem_lpg_t **); static int lpp_create(page_t **, pgcnt_t, pgcnt_t *, pmem_lpg_t **, vnode_t *, u_offset_t *, uint_t); static void tlist_in(page_t *, pgcnt_t, vnode_t *, u_offset_t *); static void tlist_out(page_t *, pgcnt_t); static int pmem_cookie_alloc(struct devmap_pmem_cookie **, pgcnt_t, uint_t); static int pmem_lock(pgcnt_t, proc_t *p); /* * Called by driver devmap routine to pass physical memory mapping info to * seg_dev framework, used only for physical memory allocated from * devmap_pmem_alloc(). */ /* ARGSUSED */ int devmap_pmem_setup(devmap_cookie_t dhc, dev_info_t *dip, struct devmap_callback_ctl *callbackops, devmap_pmem_cookie_t cookie, offset_t off, size_t len, uint_t maxprot, uint_t flags, const ddi_device_acc_attr_t *accattrp) { devmap_handle_t *dhp = (devmap_handle_t *)dhc; struct devmap_pmem_cookie *pcp = (struct devmap_pmem_cookie *)cookie; uint_t cache_attr = IOMEM_CACHE_ATTR(flags); if (pcp == NULL || (off + len) > ptob(pcp->dp_npages)) return (DDI_FAILURE); /* * First to check if this function has been called for this dhp. */ if (dhp->dh_flags & DEVMAP_SETUP_DONE) return (DDI_FAILURE); if ((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) != dhp->dh_prot) return (DDI_FAILURE); /* * Check if the cache attributes are supported. Need to pay * attention that only uncachable or write-combining is * permitted for pmem. */ if (i_ddi_check_cache_attr(flags) == B_FALSE || (cache_attr & (IOMEM_DATA_UNCACHED|IOMEM_DATA_UC_WR_COMBINE)) == 0) return (DDI_FAILURE); if (flags & DEVMAP_MAPPING_INVALID) { /* * If DEVMAP_MAPPING_INVALID is specified, we have to grant * remap permission. */ if (!(flags & DEVMAP_ALLOW_REMAP)) return (DDI_FAILURE); } else { dhp->dh_pcookie = (devmap_pmem_cookie_t)pcp; /* dh_roff is the offset inside the dh_pcookie. */ dhp->dh_roff = ptob(btop(off)); /* Set the cache attributes correctly */ i_ddi_cacheattr_to_hatacc(cache_attr, &dhp->dh_hat_attr); } dhp->dh_cookie = DEVMAP_PMEM_COOKIE; dhp->dh_flags |= (flags & DEVMAP_SETUP_FLAGS); dhp->dh_len = ptob(btopr(len)); dhp->dh_maxprot = maxprot & dhp->dh_orig_maxprot; ASSERT((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) == dhp->dh_prot); if (callbackops != NULL) { bcopy(callbackops, &dhp->dh_callbackops, sizeof (struct devmap_callback_ctl)); } /* * Initialize dh_lock if we want to do remap. */ if (dhp->dh_flags & DEVMAP_ALLOW_REMAP) { mutex_init(&dhp->dh_lock, NULL, MUTEX_DEFAULT, NULL); dhp->dh_flags |= DEVMAP_LOCK_INITED; } dhp->dh_flags |= DEVMAP_SETUP_DONE; return (DDI_SUCCESS); } /* * Replace existing mapping using a new cookie, mainly gets called when doing * fork(). Should be called in associated devmap_dup(9E). */ /* ARGSUSED */ int devmap_pmem_remap(devmap_cookie_t dhc, dev_info_t *dip, devmap_pmem_cookie_t cookie, offset_t off, size_t len, uint_t maxprot, uint_t flags, const ddi_device_acc_attr_t *accattrp) { devmap_handle_t *dhp = (devmap_handle_t *)dhc; struct devmap_pmem_cookie *pcp = (struct devmap_pmem_cookie *)cookie; uint_t cache_attr = IOMEM_CACHE_ATTR(flags); /* * Reture failure if setup has not been done or no remap permission * has been granted during the setup. */ if ((dhp->dh_flags & DEVMAP_SETUP_DONE) == 0 || (dhp->dh_flags & DEVMAP_ALLOW_REMAP) == 0) return (DDI_FAILURE); /* No flags supported for remap yet. */ if (flags != 0) return (DDI_FAILURE); if ((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) != dhp->dh_prot) return (DDI_FAILURE); if (pcp == NULL || (off + len) > ptob(pcp->dp_npages)) return (DDI_FAILURE); /* * Check if the cache attributes are supported. Need to pay * attention that only uncachable or write-combining is * permitted for pmem. */ if (i_ddi_check_cache_attr(flags) == B_FALSE || (cache_attr & (IOMEM_DATA_UNCACHED|IOMEM_DATA_UC_WR_COMBINE)) == 0) return (DDI_FAILURE); HOLD_DHP_LOCK(dhp); /* * Unload the old mapping of pages reloated with this dhp, so next * fault will setup the new mappings. It is in segdev_faultpage that * calls hat_devload to establish the mapping. Do this while holding * the dhp lock so other faults dont reestablish the mappings. */ hat_unload(dhp->dh_seg->s_as->a_hat, dhp->dh_uvaddr, dhp->dh_len, HAT_UNLOAD|HAT_UNLOAD_OTHER); /* Set the cache attributes correctly */ i_ddi_cacheattr_to_hatacc(cache_attr, &dhp->dh_hat_attr); dhp->dh_pcookie = cookie; dhp->dh_roff = ptob(btop(off)); dhp->dh_len = ptob(btopr(len)); /* Clear the large page size flag. */ dhp->dh_flags &= ~DEVMAP_FLAG_LARGE; dhp->dh_maxprot = maxprot & dhp->dh_orig_maxprot; ASSERT((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) == dhp->dh_prot); RELE_DHP_LOCK(dhp); return (DDI_SUCCESS); } /* * Directly (i.e., without occupying kernel virtual address space) allocate * 'npages' physical memory pages for exporting to user land. The allocated * page_t pointer will be recorded in cookie. */ int devmap_pmem_alloc(size_t size, uint_t flags, devmap_pmem_cookie_t *cookiep) { u_offset_t pmem_off = 0; page_t *pp = NULL; page_t *lpp = NULL; page_t *tlist = NULL; pgcnt_t i = 0; pgcnt_t rpages = 0; pgcnt_t lpages = 0; pgcnt_t tpages = 0; pgcnt_t npages = btopr(size); pmem_lpg_t *plp = NULL; struct devmap_pmem_cookie *pcp; uint_t reserved = 0; uint_t locked = 0; uint_t pflags, kflags; *cookiep = NULL; /* * Number larger than this will cause page_create_va() to loop * infinitely. */ if (npages == 0 || npages >= total_pages / 2) return (DDI_FAILURE); if ((flags & (PMEM_SLEEP | PMEM_NOSLEEP)) == 0) return (DDI_FAILURE); pflags = flags & PMEM_NOSLEEP ? PG_EXCL : PG_WAIT; kflags = flags & PMEM_NOSLEEP ? KM_NOSLEEP : KM_SLEEP; /* Allocate pmem cookie. */ if (pmem_cookie_alloc(&pcp, npages, kflags) == DDI_FAILURE) return (DDI_FAILURE); pcp->dp_npages = npages; /* * See if the requested memory can be locked. */ pcp->dp_proc = curproc; if (pmem_lock(npages, curproc) == DDI_FAILURE) goto alloc_fail; locked = 1; /* * First, grab as many as possible from pmem_mpool. If pages in * pmem_mpool are enough for this request, we are done. */ mutex_enter(&pmem_mutex); tpages = mpool_break(&tlist, npages); /* IOlock and hashin them into the new offset. */ if (tpages) tlist_in(tlist, tpages, pcp->dp_vnp, &pmem_off); mutex_exit(&pmem_mutex); if (tpages == npages) goto done; rpages = npages - tpages; /* Quit now if memory cannot be reserved. */ if (!page_resv(rpages, kflags)) goto alloc_fail; reserved = 1; /* If we have large pages */ if (pmem_lpgsize > PAGESIZE) { /* Try to alloc large pages first to decrease fragmentation. */ i = (rpages + (pmem_pgcnt - 1)) / pmem_pgcnt; if (lpp_create(&lpp, i, &lpages, &plp, pcp->dp_vnp, &pmem_off, kflags) == DDI_FAILURE) goto alloc_fail; ASSERT(lpages == 0 ? lpp == NULL : 1); } /* * Pages in large pages is more than the request, put the residual * pages into pmem_mpool. */ if (lpages >= rpages) { lpp_break(&lpp, lpages, lpages - rpages, plp); goto done; } /* Allocate small pages if lpp+tlist cannot satisfy the request. */ i = rpages - lpages; if ((pp = page_create_va(pcp->dp_vnp, pmem_off, ptob(i), pflags, &pmem_seg, (caddr_t)(uintptr_t)pmem_off)) == NULL) goto alloc_fail; done: page_list_concat(&tlist, &lpp); page_list_concat(&tlist, &pp); /* Set those small pages from large pages as allocated. */ mutex_enter(&pmem_mutex); pmem_lpg_concat(&pmem_occ_lpgs, &plp); mutex_exit(&pmem_mutex); /* * Now tlist holds all the pages for this cookie. Record these pages in * pmem cookie. */ for (pp = tlist, i = 0; i < npages; i++) { pcp->dp_pparray[i] = pp; page_io_unlock(pp); pp = pp->p_next; page_sub(&tlist, pp->p_prev); } ASSERT(tlist == NULL); *cookiep = (devmap_pmem_cookie_t)pcp; return (DDI_SUCCESS); alloc_fail: DTRACE_PROBE(pmem__alloc__fail); /* Free large pages and the associated allocation records. */ if (lpp) lpp_free(lpp, lpages / pmem_pgcnt, &plp); if (reserved == 1) page_unresv(rpages); /* Put those pages in tlist back into pmem_mpool. */ if (tpages != 0) { mutex_enter(&pmem_mutex); /* IOunlock, hashout and update the allocation records. */ tlist_out(tlist, tpages); mpool_append(&tlist, tpages); mutex_exit(&pmem_mutex); } if (locked == 1) i_ddi_decr_locked_memory(pcp->dp_proc, ptob(pcp->dp_npages)); /* Freeing pmem_cookie. */ kmem_free(pcp->dp_vnp, sizeof (vnode_t)); kmem_free(pcp->dp_pparray, npages * sizeof (page_t *)); kmem_free(pcp, sizeof (struct devmap_pmem_cookie)); return (DDI_FAILURE); } /* * Free all small pages inside cookie, and return pages from large pages into * mpool, if all the pages from one large page is in mpool, free it as a whole. */ void devmap_pmem_free(devmap_pmem_cookie_t cookie) { struct devmap_pmem_cookie *pcp = (struct devmap_pmem_cookie *)cookie; pgcnt_t i; pgcnt_t tpages = 0; page_t *pp; pmem_lpg_t *pl1, *plp; pmem_lpg_t *pf_lpgs = NULL; uint_t npls = 0; pmem_lpg_t *last_pl = NULL; pmem_lpg_t *plast_pl = NULL; ASSERT(pcp); mutex_enter(&pmem_mutex); /* Free small pages and return them to memory pool. */ for (i = pcp->dp_npages; i > 0; i--) { pp = pcp->dp_pparray[i - 1]; page_hashout(pp, NULL); /* * Remove the mapping of this single page, this mapping is * created using hat_devload() in segdev_faultpage(). */ (void) hat_pageunload(pp, HAT_FORCE_PGUNLOAD); if (!FROM_LPG(pp)) { /* Normal small page. */ page_free(pp, 1); page_unresv(1); } else { /* Small page from large pages. */ plp = pmem_lpg_get(pmem_occ_lpgs, pp, &last_pl); if (plp && !(plp->pl_pfree)) { /* * Move this record to pf_lpgs list, this large * page may be able to be freed as a whole. */ pmem_lpg_sub(&pmem_occ_lpgs, plp); pmem_lpg_concat(&pf_lpgs, &plp); plp->pl_pfree = 1; npls++; last_pl = NULL; } else { /* Search in pf_lpgs list. */ plp = pmem_lpg_get(pf_lpgs, pp, &plast_pl); } ASSERT(plp); /* Mark this page as free. */ BT_SET(plp->pl_bitmap, PFIND(pp)); /* Record this page in pmem_mpool. */ mpool_append(&pp, 1); } } /* * Find out the large pages whose pages have been freed, remove them * from plp list, free them and the associated pmem_lpg struct. */ for (plp = pf_lpgs; npls != 0; npls--) { pl1 = plp; plp = plp->pl_next; if (lpg_isfree(pl1)) { /* * Get one free large page. Find all pages in this * large page and remove them from pmem_mpool. */ lpg_free(pl1->pl_pp); /* Remove associated allocation records. */ pmem_lpg_sub(&pf_lpgs, pl1); pmem_lpg_free(&pf_lpgs, pl1); tpages -= pmem_pgcnt; } else pl1->pl_pfree = 0; } /* Update allocation records accordingly. */ pmem_lpg_concat(&pmem_occ_lpgs, &pf_lpgs); mutex_exit(&pmem_mutex); if (curproc == pcp->dp_proc) i_ddi_decr_locked_memory(curproc, ptob(pcp->dp_npages)); kmem_free(pcp->dp_vnp, sizeof (vnode_t)); kmem_free(pcp->dp_pparray, pcp->dp_npages * sizeof (page_t *)); kmem_free(pcp, sizeof (struct devmap_pmem_cookie)); } /* * To extract page frame number from specified range in a cookie. */ int devmap_pmem_getpfns(devmap_pmem_cookie_t cookie, uint_t start, pgcnt_t npages, pfn_t *pfnarray) { struct devmap_pmem_cookie *pcp = (struct devmap_pmem_cookie *)cookie; pgcnt_t i; if (pcp == NULL || start + npages > pcp->dp_npages) return (DDI_FAILURE); for (i = start; i < start + npages; i++) pfnarray[i - start] = pfn_to_mfn(pcp->dp_pparray[i]->p_pagenum); return (DDI_SUCCESS); } void pmem_init() { mutex_init(&pmem_mutex, NULL, MUTEX_DEFAULT, NULL); pmem_lszc = MIN(1, page_num_pagesizes() - 1); pmem_lpgsize = page_get_pagesize(pmem_lszc); pmem_pgcnt = pmem_lpgsize >> PAGESHIFT; bzero(&pmem_seg, sizeof (struct seg)); pmem_seg.s_as = &kas; } /* Allocate kernel memory for one pmem cookie with n pages. */ static int pmem_cookie_alloc(struct devmap_pmem_cookie **pcpp, pgcnt_t n, uint_t kflags) { struct devmap_pmem_cookie *pcp; if ((*pcpp = kmem_zalloc(sizeof (struct devmap_pmem_cookie), kflags)) == NULL) return (DDI_FAILURE); pcp = *pcpp; if ((pcp->dp_vnp = kmem_zalloc(sizeof (vnode_t), kflags)) == NULL) { kmem_free(pcp, sizeof (struct devmap_pmem_cookie)); return (DDI_FAILURE); } if ((pcp->dp_pparray = kmem_zalloc(n * sizeof (page_t *), kflags)) == NULL) { kmem_free(pcp->dp_vnp, sizeof (vnode_t)); kmem_free(pcp, sizeof (struct devmap_pmem_cookie)); return (DDI_FAILURE); } return (DDI_SUCCESS); } /* Try to lock down n pages resource */ static int pmem_lock(pgcnt_t n, proc_t *p) { if (i_ddi_incr_locked_memory(p, ptob(n)) != 0) { return (DDI_FAILURE); } return (DDI_SUCCESS); } /* To check if all the pages in a large page are freed. */ static int lpg_isfree(pmem_lpg_t *plp) { uint_t i; for (i = 0; i < BT_BITOUL(pmem_pgcnt); i++) if (plp->pl_bitmap[i] != BT_ULMAXMASK) return (0); /* All 1 means all pages are freed. */ return (1); } /* * Using pp to get the associated large page allocation record, searching in * the splp linked list with *last as the heuristic pointer. Return NULL if * not found. */ static pmem_lpg_t * pmem_lpg_get(pmem_lpg_t *splp, page_t *pp, pmem_lpg_t **last) { pmem_lpg_t *plp; pgcnt_t root_pfn; ASSERT(pp); if (splp == NULL) return (NULL); root_pfn = page_pptonum(pp) & ~(pmem_pgcnt - 1); /* Try last winner first. */ if (*last && root_pfn == page_pptonum((*last)->pl_pp)) goto pl_found; /* Else search the whole pmem_lpg list. */ for (plp = splp; root_pfn != page_pptonum(plp->pl_pp); ) { plp = plp->pl_next; if (plp == splp) { plp = NULL; break; } ASSERT(plp->pl_pp); } *last = plp; pl_found: return (*last); } /* * Remove one pmem_lpg plp from the oplpp list. */ static void pmem_lpg_sub(pmem_lpg_t **oplpp, pmem_lpg_t *plp) { if (*oplpp == plp) *oplpp = plp->pl_next; /* go to next pmem_lpg */ if (*oplpp == plp) *oplpp = NULL; /* pmem_lpg list is gone */ else { plp->pl_prev->pl_next = plp->pl_next; plp->pl_next->pl_prev = plp->pl_prev; } plp->pl_prev = plp->pl_next = plp; /* make plp a list of one */ } /* * Concatenate page list nplpp onto the end of list plpp. */ static void pmem_lpg_concat(pmem_lpg_t **plpp, pmem_lpg_t **nplpp) { pmem_lpg_t *s1p, *s2p, *e1p, *e2p; if (*nplpp == NULL) { return; } if (*plpp == NULL) { *plpp = *nplpp; return; } s1p = *plpp; e1p = s1p->pl_prev; s2p = *nplpp; e2p = s2p->pl_prev; s1p->pl_prev = e2p; e2p->pl_next = s1p; e1p->pl_next = s2p; s2p->pl_prev = e1p; } /* * Allocate and initialize the allocation record of one large page, the init * value is 'allocated'. */ static pmem_lpg_t * pmem_lpg_alloc(uint_t kflags) { pmem_lpg_t *plp; ASSERT(pmem_pgcnt % BT_NBIPUL == 0); plp = kmem_zalloc(sizeof (pmem_lpg_t), kflags); if (plp == NULL) return (NULL); plp->pl_bitmap = kmem_zalloc(BT_SIZEOFMAP(pmem_pgcnt), kflags); if (plp->pl_bitmap == NULL) { kmem_free(plp, sizeof (*plp)); return (NULL); } plp->pl_next = plp->pl_prev = plp; return (plp); } /* Free one allocation record pointed by oplp. */ static void pmem_lpg_free(pmem_lpg_t **headp, pmem_lpg_t *plp) { if (*headp == plp) *headp = plp->pl_next; /* go to next pmem_lpg_t */ if (*headp == plp) *headp = NULL; /* this list is gone */ else { plp->pl_prev->pl_next = plp->pl_next; plp->pl_next->pl_prev = plp->pl_prev; } kmem_free(plp->pl_bitmap, BT_SIZEOFMAP(pmem_pgcnt)); kmem_free(plp, sizeof (*plp)); } /* Free one large page headed by spp from pmem_mpool. */ static void lpg_free(page_t *spp) { page_t *pp1 = spp; uint_t i; ASSERT(MUTEX_HELD(&pmem_mutex)); for (i = 0; i < pmem_pgcnt; i++) { /* Break pp1 from pmem_mpool. */ page_sub(&pmem_mpool, pp1); pp1++; } /* Free pages in this large page. */ page_free_pages(spp); page_unresv(pmem_pgcnt); pmem_nmpages -= pmem_pgcnt; ASSERT((pmem_nmpages && pmem_mpool) || (!pmem_nmpages && !pmem_mpool)); } /* Put n pages in *ppp list back into pmem_mpool. */ static void mpool_append(page_t **ppp, pgcnt_t n) { ASSERT(MUTEX_HELD(&pmem_mutex)); /* Put back pages. */ page_list_concat(&pmem_mpool, ppp); pmem_nmpages += n; ASSERT((pmem_nmpages && pmem_mpool) || (!pmem_nmpages && !pmem_mpool)); } /* * Try to grab MIN(pmem_nmpages, n) pages from pmem_mpool, put them into *ppp * list, and return the number of grabbed pages. */ static pgcnt_t mpool_break(page_t **ppp, pgcnt_t n) { pgcnt_t i; ASSERT(MUTEX_HELD(&pmem_mutex)); /* Grab the pages. */ i = MIN(pmem_nmpages, n); *ppp = pmem_mpool; page_list_break(ppp, &pmem_mpool, i); pmem_nmpages -= i; ASSERT((pmem_nmpages && pmem_mpool) || (!pmem_nmpages && !pmem_mpool)); return (i); } /* * Create n large pages, lpages and plpp contains the number of small pages and * allocation records list respectively. */ static int lpp_create(page_t **lppp, pgcnt_t n, pgcnt_t *lpages, pmem_lpg_t **plpp, vnode_t *vnp, u_offset_t *offp, uint_t kflags) { pgcnt_t i; pmem_lpg_t *plp; page_t *pp; for (i = 0, *lpages = 0; i < n; i++) { /* Allocte one large page each time. */ pp = page_create_va_large(vnp, *offp, pmem_lpgsize, PG_EXCL, &pmem_seg, (caddr_t)(uintptr_t)*offp, NULL); if (pp == NULL) break; *offp += pmem_lpgsize; page_list_concat(lppp, &pp); *lpages += pmem_pgcnt; /* Add one allocation record for this large page. */ if ((plp = pmem_lpg_alloc(kflags)) == NULL) return (DDI_FAILURE); plp->pl_pp = pp; pmem_lpg_concat(plpp, &plp); } return (DDI_SUCCESS); } /* * Break the last r small pages from the large page list *lppp (with totally n * small pages) and put them into pmem_mpool. */ static void lpp_break(page_t **lppp, pgcnt_t n, pgcnt_t r, pmem_lpg_t *oplp) { page_t *pp, *pp1; pgcnt_t i; pmem_lpg_t *plp; if (r == 0) return; ASSERT(*lppp != NULL && r < pmem_pgcnt); page_list_break(lppp, &pp, n - r); /* The residual should reside in the last large page. */ plp = oplp->pl_prev; /* IOunlock and hashout the residual pages. */ for (pp1 = pp, i = 0; i < r; i++) { page_io_unlock(pp1); page_hashout(pp1, NULL); /* Mark this page as free. */ BT_SET(plp->pl_bitmap, PFIND(pp1)); pp1 = pp1->p_next; } ASSERT(pp1 == pp); /* Put these residual pages into memory pool. */ mutex_enter(&pmem_mutex); mpool_append(&pp, r); mutex_exit(&pmem_mutex); } /* Freeing large pages in lpp and the associated allocation records in plp. */ static void lpp_free(page_t *lpp, pgcnt_t lpgs, pmem_lpg_t **plpp) { pgcnt_t i, j; page_t *pp = lpp, *pp1; pmem_lpg_t *plp1, *plp2; for (i = 0; i < lpgs; i++) { for (j = 0; j < pmem_pgcnt; j++) { /* IO unlock and hashout this small page. */ page_io_unlock(pp); page_hashout(pp, NULL); pp1 = pp->p_next; pp->p_prev = pp->p_next = pp; pp = pp1; } /* Free one large page at one time. */ page_free_pages(lpp); lpp = pp; } /* Free associate pmem large page allocation records. */ for (plp1 = *plpp; *plpp; plp1 = plp2) { plp2 = plp1->pl_next; pmem_lpg_free(plpp, plp1); } } /* * IOlock and hashin all pages in tlist, associate them with vnode *pvnp * and offset starting with *poffp. Update allocation records accordingly at * the same time. */ static void tlist_in(page_t *tlist, pgcnt_t tpages, vnode_t *pvnp, u_offset_t *poffp) { page_t *pp; pgcnt_t i = 0; pmem_lpg_t *plp, *last_pl = NULL; ASSERT(MUTEX_HELD(&pmem_mutex)); for (pp = tlist; i < tpages; i++) { ASSERT(FROM_LPG(pp)); page_io_lock(pp); (void) page_hashin(pp, pvnp, *poffp, NULL); plp = pmem_lpg_get(pmem_occ_lpgs, pp, &last_pl); /* Mark this page as allocated. */ BT_CLEAR(plp->pl_bitmap, PFIND(pp)); *poffp += PAGESIZE; pp = pp->p_next; } ASSERT(pp == tlist); } /* * IOunlock and hashout all pages in tlist, update allocation records * accordingly at the same time. */ static void tlist_out(page_t *tlist, pgcnt_t tpages) { page_t *pp; pgcnt_t i = 0; pmem_lpg_t *plp, *last_pl = NULL; ASSERT(MUTEX_HELD(&pmem_mutex)); for (pp = tlist; i < tpages; i++) { ASSERT(FROM_LPG(pp)); page_io_unlock(pp); page_hashout(pp, NULL); plp = pmem_lpg_get(pmem_occ_lpgs, pp, &last_pl); /* Mark this page as free. */ BT_SET(plp->pl_bitmap, PFIND(pp)); pp = pp->p_next; } ASSERT(pp == tlist); } /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License, Version 1.0 only * (the "License"). You may not use this file except in compliance * with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2004 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* * ppcopy() and pagezero() have moved to i86/vm/vm_machdep.c */ /* * Architecures with a vac use this routine to quickly make * a vac-aligned mapping. We don't have a vac, so we don't * care about that - just make this simple. */ /* ARGSUSED2 */ caddr_t ppmapin(page_t *pp, uint_t vprot, caddr_t avoid) { caddr_t va; va = vmem_alloc(heap_arena, PAGESIZE, VM_SLEEP); hat_memload(kas.a_hat, va, pp, vprot | HAT_NOSYNC, HAT_LOAD_LOCK); return (va); } void ppmapout(caddr_t va) { hat_unload(kas.a_hat, va, PAGESIZE, HAT_UNLOAD_UNLOCK); vmem_free(heap_arena, va, PAGESIZE); } /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License, Version 1.0 only * (the "License"). You may not use this file except in compliance * with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2006 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. * Copyright (c) 2018, Joyent, Inc. * Copyright 2026 Edgecast Cloud LLC. */ /* * Platform-Specific SMBIOS Subroutines * * The routines in this file form part of and combine with * the usr/src/common/smbios code to form an in-kernel SMBIOS decoding service. * The SMBIOS entry point is locating by scanning a range of physical memory * assigned to BIOS as described in Section 2 of the DMTF SMBIOS specification. */ #include #include #include #include #include smbios_hdl_t *ksmbios; int ksmbios_flags; smbios_hdl_t * smb_open_error(smbios_hdl_t *shp, int *errp, int err) { if (shp != NULL) smbios_close(shp); if (errp != NULL) *errp = err; if (ksmbios == NULL) cmn_err(CE_CONT, "?SMBIOS not loaded (%s)", smbios_errmsg(err)); return (NULL); } smbios_hdl_t * smbios_open(const char *file, int version, int flags, int *errp) { smbios_hdl_t *shp = NULL; smbios_entry_t *ep; caddr_t stbuf, bios, p, q; caddr_t smb2, smb3; uint64_t startaddr, startoff = 0; size_t bioslen; uint_t smbe_stlen; smbios_entry_point_t ep_type; uint8_t smbe_major, smbe_minor; int err; if (file != NULL || (flags & ~SMB_O_MASK)) return (smb_open_error(shp, errp, ESMB_INVAL)); if ((startaddr = ddi_prop_get_int64(DDI_DEV_T_ANY, ddi_root_node(), DDI_PROP_DONTPASS, "smbios-address", 0)) == 0) { startaddr = SMB_RANGE_START; bioslen = SMB_RANGE_LIMIT - SMB_RANGE_START + 1; } else { /* * We have smbios address from boot loader, map a page or two. */ bioslen = MMU_PAGESIZE; startoff = startaddr & MMU_PAGEOFFSET; startaddr &= MMU_PAGEMASK; if (bioslen - startoff <= startoff) bioslen += MMU_PAGESIZE; } bios = psm_map_phys(startaddr, bioslen, PSM_PROT_READ); if (bios == NULL) return (smb_open_error(shp, errp, ESMB_MAPDEV)); /* * In case we did map one page, make sure we will not cross * the end of the page. */ p = bios + startoff; q = bios + bioslen - startoff; smb2 = smb3 = NULL; while (p < q) { if (smb2 != NULL && smb3 != NULL) break; if (smb3 == NULL && strncmp(p, SMB3_ENTRY_EANCHOR, SMB3_ENTRY_EANCHORLEN) == 0) { smb3 = p; } else if (smb2 == NULL && strncmp(p, SMB_ENTRY_EANCHOR, SMB_ENTRY_EANCHORLEN) == 0) { smb2 = p; } p += SMB_SCAN_STEP; } if (smb2 == NULL && smb3 == NULL) { psm_unmap_phys(bios, bioslen); return (smb_open_error(shp, errp, ESMB_NOTFOUND)); } /* * While they're not supposed to (as per the SMBIOS 3.2 spec), some * vendors end up having a newer version in one of the two entry points * than the other. If we found multiple tables then we will prefer the * one with the newer version. If they're equivalent, we prefer the * 32-bit version. If only one is present, then we use that. */ ep = smb_alloc(SMB_ENTRY_MAXLEN); if (smb2 != NULL && smb3 != NULL) { uint8_t smb2maj, smb2min, smb3maj, smb3min; bcopy(smb2, ep, sizeof (smbios_entry_t)); smb2maj = ep->ep21.smbe_major; smb2min = ep->ep21.smbe_minor; bcopy(smb3, ep, sizeof (smbios_entry_t)); smb3maj = ep->ep30.smbe_major; smb3min = ep->ep30.smbe_minor; if (smb3maj > smb2maj || (smb3maj == smb2maj && smb3min > smb2min)) { ep_type = SMBIOS_ENTRY_POINT_30; p = smb3; } else { ep_type = SMBIOS_ENTRY_POINT_21; p = smb2; } } else if (smb3 != NULL) { ep_type = SMBIOS_ENTRY_POINT_30; p = smb3; } else { ep_type = SMBIOS_ENTRY_POINT_21; p = smb2; } bcopy(p, ep, sizeof (smbios_entry_t)); if (ep_type == SMBIOS_ENTRY_POINT_21) { ep->ep21.smbe_elen = MIN(ep->ep21.smbe_elen, SMB_ENTRY_MAXLEN); bcopy(p, ep, ep->ep21.smbe_elen); } else if (ep_type == SMBIOS_ENTRY_POINT_30) { ep->ep30.smbe_elen = MIN(ep->ep30.smbe_elen, SMB_ENTRY_MAXLEN); bcopy(p, ep, ep->ep30.smbe_elen); } psm_unmap_phys(bios, bioslen); switch (ep_type) { case SMBIOS_ENTRY_POINT_21: smbe_major = ep->ep21.smbe_major; smbe_minor = ep->ep21.smbe_minor; smbe_stlen = ep->ep21.smbe_stlen; bios = psm_map_phys(ep->ep21.smbe_staddr, smbe_stlen, PSM_PROT_READ); break; case SMBIOS_ENTRY_POINT_30: smbe_major = ep->ep30.smbe_major; smbe_minor = ep->ep30.smbe_minor; smbe_stlen = ep->ep30.smbe_stlen; bios = psm_map_phys_new(ep->ep30.smbe_staddr, smbe_stlen, PSM_PROT_READ); break; default: smb_free(ep, SMB_ENTRY_MAXLEN); return (smb_open_error(shp, errp, ESMB_VERSION)); } if (bios == NULL) { smb_free(ep, SMB_ENTRY_MAXLEN); return (smb_open_error(shp, errp, ESMB_MAPDEV)); } stbuf = smb_alloc(smbe_stlen); bcopy(bios, stbuf, smbe_stlen); psm_unmap_phys(bios, smbe_stlen); shp = smbios_bufopen(ep, stbuf, smbe_stlen, version, flags, &err); if (shp == NULL) { smb_free(stbuf, smbe_stlen); smb_free(ep, SMB_ENTRY_MAXLEN); return (smb_open_error(shp, errp, err)); } if (ksmbios == NULL) { cmn_err(CE_CONT, "?SMBIOS v%u.%u loaded (%u bytes)", smbe_major, smbe_minor, smbe_stlen); if (shp->sh_flags & SMB_FL_TRUNC) cmn_err(CE_CONT, "?SMBIOS table is truncated"); } shp->sh_flags |= SMB_FL_BUFALLOC; smb_free(ep, SMB_ENTRY_MAXLEN); return (shp); } /*ARGSUSED*/ smbios_hdl_t * smbios_fdopen(int fd, int version, int flags, int *errp) { return (smb_open_error(NULL, errp, ENOTSUP)); } /*ARGSUSED*/ int smbios_write(smbios_hdl_t *shp, int fd) { return (smb_set_errno(shp, ENOTSUP)); } /* * This file and its contents are supplied under the terms of the * Common Development and Distribution License ("CDDL"), version 1.0. * You may only use this file in accordance with the terms of version * 1.0 of the CDDL. * * A full copy of the text of the CDDL should have accompanied this * source. A copy of the CDDL is also available via the Internet at * http://www.illumos.org/license/CDDL. */ /* * Copyright 2015 Alex Wilson, the University of Queensland * Use is subject to license terms. */ /* * Support functions for stack smashing protection (-fstack-protector * and family) * * The principle behind SSP is to place a "canary" value on the stack * just below the arguments to a given function (which are in turn * below the previous %rbp and return pointer). We write it onto the * stack at the start of a function, and then at the end just before * we execute "leave" and "ret", we check that the value is still there. * * If the check fails, we jump immediately to a handler (which typically * just executes panic() straight away). * * Since an attacker will not know the value of the "canary", they will * not be able to repair it correctly when overwriting the stack (and in * almost all cases they must overwrite the canary to get to the return * pointer), and the check will fail (and safely panic) instead of * letting them gain control over %rip in a kernel thread. * * To debugging tools the canary just looks like another local variable * (since it's placed below the normal argument space), and so there * should be minimal/no impact on things that try to parse the * function preamble. * * Of course, adding these guards to every single function does not come * without a price in performance, so normally only a subset of functions * in a given program are guarded. Selecting which subset, and adding the * guards is all handled automatically by the compiler. * * There are 3 (or 4) major relevant compiler options in GCC: * * -fstack-protector * * -fstack-protector-strong (only in GCC >= 4.9) * * -fstack-protector-all * * -fno-stack-protector * * The only differences between -fstack-protector, -strong and -all is in * which functions are selected for adding guards. * * -fstack-protector adds guards to functions that make use of a stack- * allocated char array (or aggregate containing one) of at least 8 bytes * in length. * * -fstack-protector-strong adds guards everywhere -fstack-protector * does, and also adds guards to all functions that take or pass an address * to a stack-allocated array of any type (eg arr, &arr[1] etc), as well as * functions containing certain kinds of pointer arithmetic. * * -fstack-protector-all (as the name suggests) adds guards to every single * function. * * There is also another variant, in the ProPolice patches which are used * by some members of the BSD family (eg OpenBSD), which also guards any * functions that store function pointers on the stack, as well as a few * other heuristics (like re-ordering variables so arrays are as close as * possible to the canary) */ #include #include #include #include /* * The symbol __stack_chk_guard contains the magic guard value used * to check stack integrity before returning from selected functions. * * Its value is set at startup to a "random" number -- this does not have * to be cryptographically secure, but it does have to be done before * calling any C functions that the stack guards may have been generated * for. * * For this reason, the uts/i86pc/os directory is always built *without* * stack protection enabled so that we can bootstrap. */ uintptr_t __stack_chk_guard = 0; /* * The function __stack_chk_fail is called whenever a guard check fails. */ void __stack_chk_fail(void) { /* * Currently we just panic, but some more debug info could be useful. * Note that we absolutely cannot trust any part of our stack at this * point (we already know there's an attack in progress). */ panic("Stack smashing detected"); } static void salsa_hash(unsigned int *); #ifdef __sparc extern uint64_t ultra_gettick(void); #define SSP_GET_TICK ultra_gettick #else extern hrtime_t tsc_read(void); #define SSP_GET_TICK tsc_read #endif /* __sparc */ /* called from os/startup.c */ void ssp_init(void) { int i; if (__stack_chk_guard == 0) { union { unsigned int state[16]; hrtime_t ts[8]; uintptr_t g; } s; for (i = 0; i < 8; ++i) s.ts[i] = SSP_GET_TICK(); salsa_hash(s.state); __stack_chk_guard = s.g; } } /* * Stealing the chacha/salsa hash function. It's simple, fast and * public domain. We don't need/want the full cipher (which would * belong in crypto) and we can't use the fully fledged PRNG * framework either, since ssp_init has to be called extremely * early in startup. * * Since we don't have to be cryptographically secure, just using * this to hash some high res timer values should be good enough. */ #define QR(a, b, c, d) do { \ a += b; d ^= a; d <<= 16; \ c += d; b ^= c; b <<= 12; \ a += b; d ^= a; d <<= 8; \ c += d; b ^= c; b <<= 7; \ _NOTE(CONSTANTCONDITION) \ } while (0) static inline void salsa_dr(unsigned int *state) { QR(state[0], state[4], state[ 8], state[12]); QR(state[1], state[5], state[ 9], state[13]); QR(state[2], state[6], state[10], state[14]); QR(state[3], state[7], state[11], state[15]); QR(state[0], state[5], state[10], state[15]); QR(state[1], state[6], state[11], state[12]); QR(state[2], state[7], state[ 8], state[13]); QR(state[3], state[4], state[ 9], state[14]); } static void salsa_hash(unsigned int *state) { /* 10x applications of salsa doubleround */ salsa_dr(state); salsa_dr(state); salsa_dr(state); salsa_dr(state); salsa_dr(state); salsa_dr(state); salsa_dr(state); salsa_dr(state); salsa_dr(state); salsa_dr(state); } /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright (c) 1993, 2010, Oracle and/or its affiliates. All rights reserved. * Copyright 2012 DEY Storage Systems, Inc. All rights reserved. * Copyright 2017 Nexenta Systems, Inc. * Copyright 2020 Joyent, Inc. * Copyright (c) 2015 by Delphix. All rights reserved. * Copyright (c) 2020 Carlos Neira * Copyright 2025 Oxide Computer Company * Copyright 2025 Edgecast Cloud LLC. */ /* * Copyright (c) 2010, Intel Corporation. * All rights reserved. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef __xpv #include #include #include #include #include #include #include extern void xen_late_startup(void); struct xen_evt_data cpu0_evt_data; #else /* __xpv */ #include extern void mem_config_init(void); #endif /* __xpv */ extern void progressbar_init(void); extern void brand_init(void); extern void pcf_init(void); extern void pg_init(void); extern void ssp_init(void); extern int size_pse_array(pgcnt_t, int); #if defined(_SOFT_HOSTID) static int32_t set_soft_hostid(void); static char hostid_file[] = "/etc/hostid"; #endif void *gfx_devinfo_list; #if !defined(__xpv) extern void immu_startup(void); #endif /* * XXX make declaration below "static" when drivers no longer use this * interface. */ extern caddr_t p0_va; /* Virtual address for accessing physical page 0 */ /* * segkp */ extern int segkp_fromheap; static void kvm_init(void); static void startup_init(void); static void startup_memlist(void); static void startup_kmem(void); static void startup_modules(void); static void startup_vm(void); #ifndef __xpv static void startup_tsc(void); #endif static void startup_end(void); static void layout_kernel_va(void); static void setx86isalist(void); /* * Declare these as initialized data so we can patch them. */ /* * For now we can handle memory with physical addresses up to about * 64 Terabytes. This keeps the kernel above the VA hole, leaving roughly * half the VA space for seg_kpm. When systems get bigger than 64TB this * code will need revisiting. There is an implicit assumption that there * are no *huge* holes in the physical address space too. */ #define TERABYTE (1ul << 40) #define PHYSMEM_MAX64 mmu_btop(64 * TERABYTE) #define PHYSMEM PHYSMEM_MAX64 #define AMD64_VA_HOLE_END 0xFFFF800000000000ul pgcnt_t physmem = PHYSMEM; pgcnt_t obp_pages; /* Memory used by PROM for its text and data */ extern char *kobj_file_buf; extern int kobj_file_bufsize; /* set in /etc/system */ /* Global variables for MP support. Used in mp_startup */ caddr_t rm_platter_va = 0; uint32_t rm_platter_pa; int auto_lpg_disable = 1; /* * Some CPUs have holes in the middle of the 64-bit virtual address range. */ uintptr_t hole_start, hole_end; /* * kpm mapping window */ caddr_t kpm_vbase; size_t kpm_size; static int kpm_desired; static uintptr_t segkpm_base = (uintptr_t)SEGKPM_BASE; /* * Configuration parameters set at boot time. */ caddr_t econtig; /* end of first block of contiguous kernel */ struct bootops *bootops = 0; /* passed in from boot */ struct bootops **bootopsp; struct boot_syscalls *sysp; /* passed in from boot */ char bootblock_fstype[16]; char kern_bootargs[OBP_MAXPATHLEN]; char kern_bootfile[OBP_MAXPATHLEN]; /* * ZFS zio segment. This allows us to exclude large portions of ZFS data that * gets cached in kmem caches on the heap. If this is set to zero, we allocate * zio buffers from their own segment, otherwise they are allocated from the * heap. The optimization of allocating zio buffers from their own segment is * only valid on 64-bit kernels. */ int segzio_fromheap = 0; /* * Give folks an escape hatch for disabling SMAP via kmdb. Doesn't work * post-boot. */ int disable_smap = 0; /* * new memory fragmentations are possible in startup() due to BOP_ALLOCs. this * depends on number of BOP_ALLOC calls made and requested size, memory size * combination and whether boot.bin memory needs to be freed. */ #define POSS_NEW_FRAGMENTS 12 /* * VM data structures */ long page_hashsz; /* Size of page hash table (power of two) */ unsigned int page_hashsz_shift; /* log2(page_hashsz) */ struct page *pp_base; /* Base of initial system page struct array */ struct page **page_hash; /* Page hash table */ pad_mutex_t *pse_mutex; /* Locks protecting pp->p_selock */ size_t pse_table_size; /* Number of mutexes in pse_mutex[] */ int pse_shift; /* log2(pse_table_size) */ struct seg ktextseg; /* Segment used for kernel executable image */ struct seg kvalloc; /* Segment used for "valloc" mapping */ struct seg kpseg; /* Segment used for pageable kernel virt mem */ struct seg kmapseg; /* Segment used for generic kernel mappings */ struct seg kdebugseg; /* Segment used for the kernel debugger */ struct seg *segkmap = &kmapseg; /* Kernel generic mapping segment */ static struct seg *segmap = &kmapseg; /* easier to use name for in here */ struct seg *segkp = &kpseg; /* Pageable kernel virtual memory segment */ extern struct seg kvseg_core; /* Segment used for the core heap */ struct seg kpmseg; /* Segment used for physical mapping */ struct seg *segkpm = &kpmseg; /* 64bit kernel physical mapping segment */ caddr_t segkp_base; /* Base address of segkp */ caddr_t segzio_base; /* Base address of segzio */ pgcnt_t segkpsize; /* size of segkp segment in pages */ caddr_t segkvmm_base; pgcnt_t segkvmmsize; pgcnt_t segziosize; /* * A static DR page_t VA map is reserved that can map the page structures * for a domain's entire RA space. The pages that back this space are * dynamically allocated and need not be physically contiguous. The DR * map size is derived from KPM size. * This mechanism isn't used by x86 yet, so just stubs here. */ int ppvm_enable = 0; /* Static virtual map for page structs */ page_t *ppvm_base = NULL; /* Base of page struct map */ pgcnt_t ppvm_size = 0; /* Size of page struct map */ /* * VA range available to the debugger */ const caddr_t kdi_segdebugbase = (const caddr_t)SEGDEBUGBASE; const size_t kdi_segdebugsize = SEGDEBUGSIZE; struct memseg *memseg_base; struct vnode unused_pages_vp; #define FOURGB 0x100000000LL struct memlist *memlist; caddr_t s_text; /* start of kernel text segment */ caddr_t e_text; /* end of kernel text segment */ caddr_t s_data; /* start of kernel data segment */ caddr_t e_data; /* end of kernel data segment */ caddr_t modtext; /* start of loadable module text reserved */ caddr_t e_modtext; /* end of loadable module text reserved */ caddr_t moddata; /* start of loadable module data reserved */ caddr_t e_moddata; /* end of loadable module data reserved */ struct memlist *phys_install; /* Total installed physical memory */ struct memlist *phys_avail; /* Total available physical memory */ struct memlist *bios_rsvd; /* Bios reserved memory */ /* * kphysm_init returns the number of pages that were processed */ static pgcnt_t kphysm_init(page_t *, pgcnt_t); #define IO_PROP_SIZE 64 /* device property size */ /* * a couple useful roundup macros */ #define ROUND_UP_PAGE(x) \ ((uintptr_t)P2ROUNDUP((uintptr_t)(x), (uintptr_t)MMU_PAGESIZE)) #define ROUND_UP_LPAGE(x) \ ((uintptr_t)P2ROUNDUP((uintptr_t)(x), mmu.level_size[1])) #define ROUND_UP_4MEG(x) \ ((uintptr_t)P2ROUNDUP((uintptr_t)(x), (uintptr_t)FOUR_MEG)) #define ROUND_UP_TOPLEVEL(x) \ ((uintptr_t)P2ROUNDUP((uintptr_t)(x), mmu.level_size[mmu.max_level])) /* * 64-bit Kernel's Virtual memory layout. (assuming 64 bit app) * +-----------------------+ * | debugger (?) | * 0xFFFFFFFF.FF800000 |-----------------------|- SEGDEBUGBASE * | unused | * +-----------------------+ * | Kernel Data | * 0xFFFFFFFF.FBC00000 |-----------------------| * | Kernel Text | * 0xFFFFFFFF.FB800000 |-----------------------|- KERNEL_TEXT * |--- debug info ---|- debug info (DEBUG_INFO_VA) * |--- GDT ---|- GDT page (GDT_VA) * |--- IDT ---|- IDT page (IDT_VA) * |--- LDT ---|- LDT pages (LDT_VA) * | | * | Core heap | (used for loadable modules) * 0xFFFFFFFF.C0000000 |-----------------------|- core_base / ekernelheap * | Kernel | * | heap | * | | * | | * 0xFFFFFXXX.XXX00000 |-----------------------|- kernelheap (floating) * | segmap | * 0xFFFFFXXX.XXX00000 |-----------------------|- segmap_start (floating) * | device mappings | * 0xFFFFFXXX.XXX00000 |-----------------------|- toxic_addr (floating) * | segzio | * 0xFFFFFXXX.XXX00000 |-----------------------|- segzio_base (floating) * | segkvmm | * | | * | | * | | * 0xFFFFFXXX.XXX00000 |-----------------------|- segkvmm_base (floating) * | segkp | * |-----------------------|- segkp_base (floating) * | page_t structures | valloc_base + valloc_sz * | memsegs, memlists, | * | page hash, etc. | * 0xFFFFFE00.00000000 |-----------------------|- valloc_base (lower if >256GB) * | segkpm | * | | * 0xFFFFFD00.00000000 |-----------------------|- SEGKPM_BASE (lower if >256GB) * | Red Zone | * 0xFFFFFC80.00000000 |-----------------------|- KERNELBASE (lower if >256GB) * 0xFFFFFC7F.FFE00000 |-----------------------|- USERLIMIT (lower if >256GB) * | User stack |- User space memory * | | * | shared objects, etc | (grows downwards) * : : * | | * 0xFFFF8000.00000000 |-----------------------| * | | * | VA Hole / unused | * | | * 0x00008000.00000000 |-----------------------| * | | * | | * : : * | user heap | (grows upwards) * | | * | user data | * |-----------------------| * | user text | * 0x00000000.04000000 |-----------------------| * | invalid | * 0x00000000.00000000 +-----------------------+ * * A 32 bit app on the 64 bit kernel sees the same layout as on the 32 bit * kernel, except that userlimit is raised to 0xfe000000 * * Floating values: * * valloc_base: start of the kernel's memory management/tracking data * structures. This region contains page_t structures for * physical memory, memsegs, memlists, and the page hash. * * core_base: start of the kernel's "core" heap area on 64-bit systems. * This area is intended to be used for global data as well as for module * text/data that does not fit into the nucleus pages. The core heap is * restricted to a 2GB range, allowing every address within it to be * accessed using rip-relative addressing * * ekernelheap: end of kernelheap and start of segmap. * * kernelheap: start of kernel heap. On 32-bit systems, this starts right * above a red zone that separates the user's address space from the * kernel's. On 64-bit systems, it sits above segkp and segkpm. * * segmap_start: start of segmap. The length of segmap can be modified * through eeprom. The default length is 16MB on 32-bit systems and 64MB * on 64-bit systems. * * kernelbase: On a 32-bit kernel the default value of 0xd4000000 will be * decreased by 2X the size required for page_t. This allows the kernel * heap to grow in size with physical memory. With sizeof(page_t) == 80 * bytes, the following shows the values of kernelbase and kernel heap * sizes for different memory configurations (assuming default segmap and * segkp sizes). * * mem size for kernelbase kernel heap * size page_t's size * ---- --------- ---------- ----------- * 1gb 0x01400000 0xd1800000 684MB * 2gb 0x02800000 0xcf000000 704MB * 4gb 0x05000000 0xca000000 744MB * 6gb 0x07800000 0xc5000000 784MB * 8gb 0x0a000000 0xc0000000 824MB * 16gb 0x14000000 0xac000000 984MB * 32gb 0x28000000 0x84000000 1304MB * 64gb 0x50000000 0x34000000 1944MB (*) * * kernelbase is less than the abi minimum of 0xc0000000 for memory * configurations above 8gb. * * (*) support for memory configurations above 32gb will require manual tuning * of kernelbase to balance out the need of user applications. */ /* real-time-clock initialization parameters */ extern time_t process_rtc_config_file(void); uintptr_t kernelbase; uintptr_t postbootkernelbase; /* not set till boot loader is gone */ uintptr_t eprom_kernelbase; size_t segmapsize; uintptr_t segmap_start; int segmapfreelists; pgcnt_t npages; pgcnt_t orig_npages; size_t core_size; /* size of "core" heap */ uintptr_t core_base; /* base address of "core" heap */ /* * List of bootstrap pages. We mark these as allocated in startup. * release_bootstrap() will free them when we're completely done with * the bootstrap. */ static page_t *bootpages; /* * boot time pages that have a vnode from the ramdisk will keep that forever. */ static page_t *rd_pages; /* * Lower 64K */ static page_t *lower_pages = NULL; static int lower_pages_count = 0; struct system_hardware system_hardware; /* * Enable some debugging messages concerning memory usage... */ static void print_memlist(char *title, struct memlist *mp) { prom_printf("MEMLIST: %s:\n", title); while (mp != NULL) { prom_printf("\tAddress 0x%" PRIx64 ", size 0x%" PRIx64 "\n", mp->ml_address, mp->ml_size); mp = mp->ml_next; } } /* * XX64 need a comment here.. are these just default values, surely * we read the "cpuid" type information to figure this out. */ int l2cache_sz = 0x80000; int l2cache_linesz = 0x40; int l2cache_assoc = 1; static size_t textrepl_min_gb = 10; /* * on 64 bit we use a predifined VA range for mapping devices in the kernel * on 32 bit the mappings are intermixed in the heap, so we use a bit map */ vmem_t *device_arena; uintptr_t toxic_addr = (uintptr_t)NULL; size_t toxic_size = 1024 * 1024 * 1024; /* Sparc uses 1 gig too */ int prom_debug; /* * This structure is used to keep track of the intial allocations * done in startup_memlist(). The value of NUM_ALLOCATIONS needs to * be >= the number of ADD_TO_ALLOCATIONS() executed in the code. */ #define NUM_ALLOCATIONS 8 int num_allocations = 0; struct { void **al_ptr; size_t al_size; } allocations[NUM_ALLOCATIONS]; size_t valloc_sz = 0; uintptr_t valloc_base; #define ADD_TO_ALLOCATIONS(ptr, size) { \ size = ROUND_UP_PAGE(size); \ if (num_allocations == NUM_ALLOCATIONS) \ panic("too many ADD_TO_ALLOCATIONS()"); \ allocations[num_allocations].al_ptr = (void**)&ptr; \ allocations[num_allocations].al_size = size; \ valloc_sz += size; \ ++num_allocations; \ } /* * Allocate all the initial memory needed by the page allocator. */ static void perform_allocations(void) { caddr_t mem; int i; int valloc_align; PRM_DEBUG(valloc_base); PRM_DEBUG(valloc_sz); valloc_align = mmu.level_size[mmu.max_page_level > 0]; mem = BOP_ALLOC(bootops, (caddr_t)valloc_base, valloc_sz, valloc_align); if (mem != (caddr_t)valloc_base) panic("BOP_ALLOC() failed"); bzero(mem, valloc_sz); for (i = 0; i < num_allocations; ++i) { *allocations[i].al_ptr = (void *)mem; mem += allocations[i].al_size; } } /* * Set up and enable SMAP now before we start other CPUs, but after the kernel's * VM has been set up so we can use hot_patch_kernel_text(). * * We can only patch 1, 2, or 4 bytes, but not three bytes. So instead, we * replace the four byte word at the patch point. See uts/intel/ml/copy.s * for more information on what's going on here. */ static void startup_smap(void) { int i; uint32_t inst; uint8_t *instp; char sym[128]; struct modctl *modp; extern int _smap_enable_patch_count; extern int _smap_disable_patch_count; if (disable_smap != 0) remove_x86_feature(x86_featureset, X86FSET_SMAP); if (is_x86_feature(x86_featureset, X86FSET_SMAP) == B_FALSE) return; for (i = 0; i < _smap_enable_patch_count; i++) { int sizep; VERIFY3U(i, <, _smap_enable_patch_count); VERIFY(snprintf(sym, sizeof (sym), "_smap_enable_patch_%d", i) < sizeof (sym)); instp = (uint8_t *)(void *)kobj_getelfsym(sym, NULL, &sizep); VERIFY(instp != 0); inst = (instp[3] << 24) | (SMAP_CLAC_INSTR & 0x00ffffff); hot_patch_kernel_text((caddr_t)instp, inst, 4); } for (i = 0; i < _smap_disable_patch_count; i++) { int sizep; VERIFY(snprintf(sym, sizeof (sym), "_smap_disable_patch_%d", i) < sizeof (sym)); instp = (uint8_t *)(void *)kobj_getelfsym(sym, NULL, &sizep); VERIFY(instp != 0); inst = (instp[3] << 24) | (SMAP_STAC_INSTR & 0x00ffffff); hot_patch_kernel_text((caddr_t)instp, inst, 4); } /* * Hotinline calls to smap_enable and smap_disable within * unix module. Hotinlines in other modules are done on * mod_load(). */ modp = mod_hold_by_name("unix"); do_hotinlines(modp->mod_mp); mod_release_mod(modp); setcr4(getcr4() | CR4_SMAP); smap_enable(); } /* * Our world looks like this at startup time. * * In a 32-bit OS, boot loads the kernel text at 0xfe800000 and kernel data * at 0xfec00000. On a 64-bit OS, kernel text and data are loaded at * 0xffffffff.fe800000 and 0xffffffff.fec00000 respectively. Those * addresses are fixed in the binary at link time. * * On the text page: * unix/genunix/krtld/module text loads. * * On the data page: * unix/genunix/krtld/module data loads. * * Machine-dependent startup code */ void startup(void) { #if !defined(__xpv) extern void startup_pci_bios(void); #endif extern cpuset_t cpu_ready_set; /* * Make sure that nobody tries to use sekpm until we have * initialized it properly. */ kpm_desired = 1; kpm_enable = 0; CPUSET_ONLY(cpu_ready_set, 0); /* cpu 0 is boot cpu */ #if defined(__xpv) /* XXPV fix me! */ { extern int segvn_use_regions; segvn_use_regions = 0; } #endif ssp_init(); progressbar_init(); startup_init(); #if defined(__xpv) startup_xen_version(); #endif startup_memlist(); startup_kmem(); startup_vm(); #if !defined(__xpv) /* * Up until this point, we cannot use any time delay functions * (e.g. tenmicrosec()). Once the TSC is setup, we can. This is * purposely done after the VM system as been setup to allow * calibration sources which might require mapping for access * (e.g. the HPET), but still early enough to allow the rest of * the startup code to make use of the TSC (via tenmicrosec() or * the default TSC-based gethrtime()) as required. */ startup_tsc(); /* * Note we need to do this even on fast reboot in order to access * the irq routing table (used for pci labels). */ startup_pci_bios(); startup_smap(); #endif #if defined(__xpv) startup_xen_mca(); #endif startup_modules(); PRM_POINT(">> startup_end"); startup_end(); } static void startup_init() { PRM_POINT("startup_init() starting..."); /* * Complete the extraction of cpuid data */ cpuid_execpass(CPU, CPUID_PASS_EXTENDED, NULL); (void) check_boot_version(BOP_GETVERSION(bootops)); /* * Check for prom_debug in boot environment */ if (BOP_GETPROPLEN(bootops, "prom_debug") >= 0) { ++prom_debug; PRM_POINT("prom_debug found in boot enviroment"); } /* * Collect node, cpu and memory configuration information. */ get_system_configuration(); /* * Halt if this is an unsupported processor. */ if (x86_type == X86_TYPE_486 || x86_type == X86_TYPE_CYRIX_486) { printf("\n486 processor (\"%s\") detected.\n", CPU->cpu_brandstr); halt("This processor is not supported by this release " "of Solaris."); } PRM_POINT("startup_init() done"); } /* * Callback for copy_memlist_filter() to filter nucleus, kadb/kmdb, (ie. * everything mapped above KERNEL_TEXT) pages from phys_avail. Note it * also filters out physical page zero. There is some reliance on the * boot loader allocating only a few contiguous physical memory chunks. */ static void avail_filter(uint64_t *addr, uint64_t *size) { uintptr_t va; uintptr_t next_va; pfn_t pfn; uint64_t pfn_addr; uint64_t pfn_eaddr; uint_t prot; size_t len; uint_t change; if (prom_debug) prom_printf("\tFilter: in: a=%" PRIx64 ", s=%" PRIx64 "\n", *addr, *size); /* * page zero is required for BIOS.. never make it available */ if (*addr == 0) { *addr += MMU_PAGESIZE; *size -= MMU_PAGESIZE; } /* * First we trim from the front of the range. Since kbm_probe() * walks ranges in virtual order, but addr/size are physical, we need * to the list until no changes are seen. This deals with the case * where page "p" is mapped at v, page "p + PAGESIZE" is mapped at w * but w < v. */ do { change = 0; for (va = KERNEL_TEXT; *size > 0 && kbm_probe(&va, &len, &pfn, &prot) != 0; va = next_va) { next_va = va + len; pfn_addr = pfn_to_pa(pfn); pfn_eaddr = pfn_addr + len; if (pfn_addr <= *addr && pfn_eaddr > *addr) { change = 1; while (*size > 0 && len > 0) { *addr += MMU_PAGESIZE; *size -= MMU_PAGESIZE; len -= MMU_PAGESIZE; } } } if (change && prom_debug) prom_printf("\t\ttrim: a=%" PRIx64 ", s=%" PRIx64 "\n", *addr, *size); } while (change); /* * Trim pages from the end of the range. */ for (va = KERNEL_TEXT; *size > 0 && kbm_probe(&va, &len, &pfn, &prot) != 0; va = next_va) { next_va = va + len; pfn_addr = pfn_to_pa(pfn); if (pfn_addr >= *addr && pfn_addr < *addr + *size) *size = pfn_addr - *addr; } if (prom_debug) prom_printf("\tFilter out: a=%" PRIx64 ", s=%" PRIx64 "\n", *addr, *size); } static void kpm_init() { struct segkpm_crargs b; /* * These variables were all designed for sfmmu in which segkpm is * mapped using a single pagesize - either 8KB or 4MB. On x86, we * might use 2+ page sizes on a single machine, so none of these * variables have a single correct value. They are set up as if we * always use a 4KB pagesize, which should do no harm. In the long * run, we should get rid of KPM's assumption that only a single * pagesize is used. */ kpm_pgshft = MMU_PAGESHIFT; kpm_pgsz = MMU_PAGESIZE; kpm_pgoff = MMU_PAGEOFFSET; kpmp2pshft = 0; kpmpnpgs = 1; ASSERT(((uintptr_t)kpm_vbase & (kpm_pgsz - 1)) == 0); PRM_POINT("about to create segkpm"); rw_enter(&kas.a_lock, RW_WRITER); if (seg_attach(&kas, kpm_vbase, kpm_size, segkpm) < 0) panic("cannot attach segkpm"); b.prot = PROT_READ | PROT_WRITE; b.nvcolors = 1; if (segkpm_create(segkpm, (caddr_t)&b) != 0) panic("segkpm_create segkpm"); rw_exit(&kas.a_lock); kpm_enable = 1; /* * As the KPM was disabled while setting up the system, go back and fix * CPU zero's access to its user page table. This is a bit gross, but * we have a chicken and egg problem otherwise. */ ASSERT(CPU->cpu_hat_info->hci_user_l3ptes == NULL); CPU->cpu_hat_info->hci_user_l3ptes = (x86pte_t *)hat_kpm_mapin_pfn(CPU->cpu_hat_info->hci_user_l3pfn); } /* * The debug info page provides enough information to allow external * inspectors (e.g. when running under a hypervisor) to bootstrap * themselves into allowing full-blown kernel debugging. */ static void init_debug_info(void) { caddr_t mem; debug_info_t *di; #ifndef __lint ASSERT(sizeof (debug_info_t) < MMU_PAGESIZE); #endif mem = BOP_ALLOC(bootops, (caddr_t)DEBUG_INFO_VA, MMU_PAGESIZE, MMU_PAGESIZE); if (mem != (caddr_t)DEBUG_INFO_VA) panic("BOP_ALLOC() failed"); bzero(mem, MMU_PAGESIZE); di = (debug_info_t *)mem; di->di_magic = DEBUG_INFO_MAGIC; di->di_version = DEBUG_INFO_VERSION; di->di_modules = (uintptr_t)&modules; di->di_s_text = (uintptr_t)s_text; di->di_e_text = (uintptr_t)e_text; di->di_s_data = (uintptr_t)s_data; di->di_e_data = (uintptr_t)e_data; di->di_hat_htable_off = offsetof(hat_t, hat_htable); di->di_ht_pfn_off = offsetof(htable_t, ht_pfn); } /* * Build the memlists and other kernel essential memory system data structures. * This is everything at valloc_base. */ static void startup_memlist(void) { size_t memlist_sz; size_t memseg_sz; size_t pagehash_sz; size_t pp_sz; uintptr_t va; size_t len; uint_t prot; pfn_t pfn; int memblocks; pfn_t rsvd_high_pfn; pgcnt_t rsvd_pgcnt; size_t rsvdmemlist_sz; int rsvdmemblocks; caddr_t pagecolor_mem; size_t pagecolor_memsz; caddr_t page_ctrs_mem; size_t page_ctrs_size; size_t pse_table_alloc_size; struct memlist *current; PRM_POINT("startup_memlist() starting..."); /* * Use leftover large page nucleus text/data space for loadable modules. * Use at most MODTEXT/MODDATA. */ len = kbm_nucleus_size; ASSERT(len > MMU_PAGESIZE); moddata = (caddr_t)ROUND_UP_PAGE(e_data); e_moddata = (caddr_t)P2ROUNDUP((uintptr_t)e_data, (uintptr_t)len); if (e_moddata - moddata > MODDATA) e_moddata = moddata + MODDATA; modtext = (caddr_t)ROUND_UP_PAGE(e_text); e_modtext = (caddr_t)P2ROUNDUP((uintptr_t)e_text, (uintptr_t)len); if (e_modtext - modtext > MODTEXT) e_modtext = modtext + MODTEXT; econtig = e_moddata; PRM_DEBUG(modtext); PRM_DEBUG(e_modtext); PRM_DEBUG(moddata); PRM_DEBUG(e_moddata); PRM_DEBUG(econtig); /* * Examine the boot loader physical memory map to find out: * - total memory in system - physinstalled * - the max physical address - physmax * - the number of discontiguous segments of memory. */ if (prom_debug) print_memlist("boot physinstalled", bootops->boot_mem->physinstalled); installed_top_size_ex(bootops->boot_mem->physinstalled, &physmax, &physinstalled, &memblocks); PRM_DEBUG(physmax); PRM_DEBUG(physinstalled); PRM_DEBUG(memblocks); /* * We no longer support any form of memory DR. */ plat_dr_physmax = 0; /* * Examine the bios reserved memory to find out: * - the number of discontiguous segments of memory. */ if (prom_debug) print_memlist("boot reserved mem", bootops->boot_mem->rsvdmem); installed_top_size_ex(bootops->boot_mem->rsvdmem, &rsvd_high_pfn, &rsvd_pgcnt, &rsvdmemblocks); PRM_DEBUG(rsvd_high_pfn); PRM_DEBUG(rsvd_pgcnt); PRM_DEBUG(rsvdmemblocks); /* * Initialize hat's mmu parameters. * Check for enforce-prot-exec in boot environment. It's used to * enable/disable support for the page table entry NX bit. * The default is to enforce PROT_EXEC on processors that support NX. * Boot seems to round up the "len", but 8 seems to be big enough. */ mmu_init(); startup_build_mem_nodes(bootops->boot_mem->physinstalled); if (BOP_GETPROPLEN(bootops, "enforce-prot-exec") >= 0) { int len = BOP_GETPROPLEN(bootops, "enforce-prot-exec"); char value[8]; if (len < 8) (void) BOP_GETPROP(bootops, "enforce-prot-exec", value); else (void) strcpy(value, ""); if (strcmp(value, "off") == 0) mmu.pt_nx = 0; } PRM_DEBUG(mmu.pt_nx); /* * We will need page_t's for every page in the system, except for * memory mapped at or above above the start of the kernel text segment. * * pages above e_modtext are attributed to kernel debugger (obp_pages) */ npages = physinstalled - 1; /* avail_filter() skips page 0, so "- 1" */ obp_pages = 0; va = KERNEL_TEXT; while (kbm_probe(&va, &len, &pfn, &prot) != 0) { npages -= len >> MMU_PAGESHIFT; if (va >= (uintptr_t)e_moddata) obp_pages += len >> MMU_PAGESHIFT; va += len; } PRM_DEBUG(npages); PRM_DEBUG(obp_pages); /* * If physmem is patched to be non-zero, use it instead of the computed * value unless it is larger than the actual amount of memory on hand. */ if (physmem == 0 || physmem > npages) { physmem = npages; } else if (physmem < npages) { orig_npages = npages; npages = physmem; } PRM_DEBUG(physmem); /* * We now compute the sizes of all the initial allocations for * structures the kernel needs in order do kmem_alloc(). These * include: * memsegs * memlists * page hash table * page_t's * page coloring data structs */ memseg_sz = sizeof (struct memseg) * (memblocks + POSS_NEW_FRAGMENTS); ADD_TO_ALLOCATIONS(memseg_base, memseg_sz); PRM_DEBUG(memseg_sz); /* * Reserve space for memlists. There's no real good way to know exactly * how much room we'll need, but this should be a good upper bound. */ memlist_sz = ROUND_UP_PAGE(2 * sizeof (struct memlist) * (memblocks + POSS_NEW_FRAGMENTS)); ADD_TO_ALLOCATIONS(memlist, memlist_sz); PRM_DEBUG(memlist_sz); /* * Reserve space for bios reserved memlists. */ rsvdmemlist_sz = ROUND_UP_PAGE(2 * sizeof (struct memlist) * (rsvdmemblocks + POSS_NEW_FRAGMENTS)); ADD_TO_ALLOCATIONS(bios_rsvd, rsvdmemlist_sz); PRM_DEBUG(rsvdmemlist_sz); /* LINTED */ ASSERT(P2SAMEHIGHBIT((1 << PP_SHIFT), sizeof (struct page))); /* * The page structure hash table size is a power of 2 * such that the average hash chain length is PAGE_HASHAVELEN. */ page_hashsz = npages / PAGE_HASHAVELEN; page_hashsz_shift = highbit(page_hashsz); page_hashsz = 1 << page_hashsz_shift; pagehash_sz = sizeof (struct page *) * page_hashsz; ADD_TO_ALLOCATIONS(page_hash, pagehash_sz); PRM_DEBUG(pagehash_sz); /* * Set aside room for the page structures themselves. */ PRM_DEBUG(npages); pp_sz = sizeof (struct page) * npages; ADD_TO_ALLOCATIONS(pp_base, pp_sz); PRM_DEBUG(pp_sz); /* * determine l2 cache info and memory size for page coloring */ (void) getl2cacheinfo(CPU, &l2cache_sz, &l2cache_linesz, &l2cache_assoc); pagecolor_memsz = page_coloring_init(l2cache_sz, l2cache_linesz, l2cache_assoc); ADD_TO_ALLOCATIONS(pagecolor_mem, pagecolor_memsz); PRM_DEBUG(pagecolor_memsz); page_ctrs_size = page_ctrs_sz(); ADD_TO_ALLOCATIONS(page_ctrs_mem, page_ctrs_size); PRM_DEBUG(page_ctrs_size); /* * Allocate the array that protects pp->p_selock. */ pse_shift = size_pse_array(physmem, max_ncpus); pse_table_size = 1 << pse_shift; pse_table_alloc_size = pse_table_size * sizeof (pad_mutex_t); ADD_TO_ALLOCATIONS(pse_mutex, pse_table_alloc_size); valloc_sz = ROUND_UP_LPAGE(valloc_sz); valloc_base = VALLOC_BASE; /* * The signicant memory-sized regions are roughly sized as follows in * the default layout with max physmem: * segkpm: 1x physmem allocated (but 1Tb room, below VALLOC_BASE) * segzio: 1.5x physmem * segkvmm: 4x physmem * heap: whatever's left up to COREHEAP_BASE, at least 1.5x physmem * * The idea is that we leave enough room to avoid fragmentation issues, * so we would like the VA arenas to have some extra. * * Ignoring the loose change of segkp, valloc, and such, this means that * as COREHEAP_BASE-VALLOC_BASE=2Tb, we can accommodate a physmem up to * about (2Tb / 7.0), rounded down to 256Gb in the check below. * * Note that KPM lives below VALLOC_BASE, but we want to include it in * adjustments, hence the 8 below. * * Beyond 256Gb, we push segkpm_base (and hence kernelbase and * _userlimit) down to accommodate the VA requirements above. */ if (physmax + 1 > mmu_btop(TERABYTE / 4)) { uint64_t physmem_bytes = mmu_ptob(physmax + 1); uint64_t adjustment = 8 * (physmem_bytes - (TERABYTE / 4)); PRM_DEBUG(adjustment); /* * segkpm_base is always aligned on a L3 PTE boundary. */ segkpm_base -= P2ROUNDUP(adjustment, KERNEL_REDZONE_SIZE); /* * But make sure we leave some space for user apps above hole. */ segkpm_base = MAX(segkpm_base, AMD64_VA_HOLE_END + TERABYTE); ASSERT(segkpm_base <= SEGKPM_BASE); valloc_base = segkpm_base + P2ROUNDUP(physmem_bytes, ONE_GIG); if (valloc_base < segkpm_base) panic("not enough kernel VA to support memory size"); } PRM_DEBUG(segkpm_base); PRM_DEBUG(valloc_base); /* * do all the initial allocations */ perform_allocations(); /* * Build phys_install and phys_avail in kernel memspace. * - phys_install should be all memory in the system. * - phys_avail is phys_install minus any memory mapped before this * point above KERNEL_TEXT. */ current = phys_install = memlist; copy_memlist_filter(bootops->boot_mem->physinstalled, ¤t, NULL); if ((caddr_t)current > (caddr_t)memlist + memlist_sz) panic("physinstalled was too big!"); if (prom_debug) print_memlist("phys_install", phys_install); phys_avail = current; PRM_POINT("Building phys_avail:\n"); copy_memlist_filter(bootops->boot_mem->physinstalled, ¤t, avail_filter); if ((caddr_t)current > (caddr_t)memlist + memlist_sz) panic("physavail was too big!"); if (prom_debug) print_memlist("phys_avail", phys_avail); #ifndef __xpv /* * Free unused memlist items, which may be used by memory DR driver * at runtime. */ if ((caddr_t)current < (caddr_t)memlist + memlist_sz) { memlist_free_block((caddr_t)current, (caddr_t)memlist + memlist_sz - (caddr_t)current); } #endif /* * Build bios reserved memspace */ current = bios_rsvd; copy_memlist_filter(bootops->boot_mem->rsvdmem, ¤t, NULL); if ((caddr_t)current > (caddr_t)bios_rsvd + rsvdmemlist_sz) panic("bios_rsvd was too big!"); if (prom_debug) print_memlist("bios_rsvd", bios_rsvd); #ifndef __xpv /* * Free unused memlist items, which may be used by memory DR driver * at runtime. */ if ((caddr_t)current < (caddr_t)bios_rsvd + rsvdmemlist_sz) { memlist_free_block((caddr_t)current, (caddr_t)bios_rsvd + rsvdmemlist_sz - (caddr_t)current); } #endif /* * setup page coloring */ page_coloring_setup(pagecolor_mem); page_lock_init(); /* currently a no-op */ /* * free page list counters */ (void) page_ctrs_alloc(page_ctrs_mem); /* * Size the pcf array based on the number of cpus in the box at * boot time. */ pcf_init(); /* * Initialize the page structures from the memory lists. */ availrmem_initial = availrmem = freemem = 0; PRM_POINT("Calling kphysm_init()..."); npages = kphysm_init(pp_base, npages); PRM_POINT("kphysm_init() done"); PRM_DEBUG(npages); init_debug_info(); /* * Now that page_t's have been initialized, remove all the * initial allocation pages from the kernel free page lists. */ boot_mapin((caddr_t)valloc_base, valloc_sz); boot_mapin((caddr_t)MISC_VA_BASE, MISC_VA_SIZE); PRM_POINT("startup_memlist() done"); PRM_DEBUG(valloc_sz); if ((availrmem >> (30 - MMU_PAGESHIFT)) >= textrepl_min_gb && l2cache_sz <= 2 << 20) { extern size_t textrepl_size_thresh; textrepl_size_thresh = (16 << 20) - 1; } } /* * Layout the kernel's part of address space and initialize kmem allocator. */ static void startup_kmem(void) { #if !defined(__xpv) extern uint64_t kpti_kbase; #endif PRM_POINT("startup_kmem() starting..."); if (eprom_kernelbase && eprom_kernelbase != KERNELBASE) cmn_err(CE_NOTE, "!kernelbase cannot be changed on 64-bit " "systems."); kernelbase = segkpm_base - KERNEL_REDZONE_SIZE; core_base = (uintptr_t)COREHEAP_BASE; core_size = (size_t)MISC_VA_BASE - COREHEAP_BASE; PRM_DEBUG(core_base); PRM_DEBUG(core_size); PRM_DEBUG(kernelbase); ekernelheap = (char *)core_base; PRM_DEBUG(ekernelheap); /* * Now that we know the real value of kernelbase, * update variables that were initialized with a value of * KERNELBASE (in common/conf/param.c). * * XXX The problem with this sort of hackery is that the * compiler just may feel like putting the const declarations * (in param.c) into the .text section. Perhaps they should * just be declared as variables there? */ *(uintptr_t *)&_kernelbase = kernelbase; *(uintptr_t *)&_userlimit = kernelbase; *(uintptr_t *)&_userlimit -= KERNELBASE - USERLIMIT; #if !defined(__xpv) kpti_kbase = kernelbase; #endif PRM_DEBUG(_kernelbase); PRM_DEBUG(_userlimit); PRM_DEBUG(_userlimit32); /* We have to re-do this now that we've modified _userlimit. */ mmu_calc_user_slots(); layout_kernel_va(); /* * Initialize the kernel heap. Note 3rd argument must be > 1st. */ kernelheap_init(kernelheap, ekernelheap, kernelheap + MMU_PAGESIZE, (void *)core_base, (void *)(core_base + core_size)); #if defined(__xpv) /* * Link pending events struct into cpu struct */ CPU->cpu_m.mcpu_evt_pend = &cpu0_evt_data; #endif /* * Initialize kernel memory allocator. */ kmem_init(); /* * Factor in colorequiv to check additional 'equivalent' bins */ page_set_colorequiv_arr(); /* * print this out early so that we know what's going on */ print_x86_featureset(x86_featureset); /* * Initialize bp_mapin(). */ bp_init(MMU_PAGESIZE, HAT_STORECACHING_OK); /* * orig_npages is non-zero if physmem has been configured for less * than the available memory. */ if (orig_npages) { cmn_err(CE_WARN, "!%slimiting physmem to 0x%lx of 0x%lx pages", (npages == PHYSMEM ? "Due to virtual address space " : ""), npages, orig_npages); } #ifdef KERNELBASE_ABI_MIN if (kernelbase < (uintptr_t)KERNELBASE_ABI_MIN) { cmn_err(CE_NOTE, "!kernelbase set to 0x%lx, system is not " "i386 ABI compliant.", (uintptr_t)kernelbase); } #endif #ifndef __xpv if (plat_dr_support_memory()) { mem_config_init(); } #else /* __xpv */ /* * Some of the xen start information has to be relocated up * into the kernel's permanent address space. */ PRM_POINT("calling xen_relocate_start_info()"); xen_relocate_start_info(); PRM_POINT("xen_relocate_start_info() done"); /* * (Update the vcpu pointer in our cpu structure to point into * the relocated shared info.) */ CPU->cpu_m.mcpu_vcpu_info = &HYPERVISOR_shared_info->vcpu_info[CPU->cpu_id]; #endif /* __xpv */ PRM_POINT("startup_kmem() done"); } /* Hammerhead: Xen HVM module path support removed — all modules in /kernel/ */ static void startup_modules(void) { int cnt; extern void prom_setup(void); int32_t v, h; char d[11]; char *cp; cmi_hdl_t hdl; PRM_POINT("startup_modules() starting..."); /* Hammerhead: Xen HVM update_default_path() removed */ /* * Read the GMT lag from /etc/rtc_config. */ sgmtl(process_rtc_config_file()); /* * Calculate default settings of system parameters based upon * maxusers, yet allow to be overridden via the /etc/system file. */ param_calc(0); mod_setup(); /* * Initialize system parameters. */ param_init(); /* * Initialize the default brands */ brand_init(); /* * maxmem is the amount of physical memory we're playing with. */ maxmem = physmem; /* * Initialize segment management stuff. */ seg_init(); if (modload("fs", "specfs") == -1) halt("Can't load specfs"); if (modload("fs", "devfs") == -1) halt("Can't load devfs"); if (modload("fs", "dev") == -1) halt("Can't load dev"); if (modload("fs", "procfs") == -1) halt("Can't load procfs"); (void) modloadonly("sys", "lbl_edition"); dispinit(); /* Read cluster configuration data. */ clconf_init(); #if defined(__xpv) (void) ec_init(); gnttab_init(); (void) xs_early_init(); #endif /* __xpv */ /* * Create a kernel device tree. First, create rootnex and * then invoke bus specific code to probe devices. */ setup_ddi(); #ifdef __xpv if (DOMAIN_IS_INITDOMAIN(xen_info)) #endif { id_t smid; smbios_system_t smsys; smbios_info_t sminfo; char *mfg; /* * Load the System Management BIOS into the global ksmbios * handle, if an SMBIOS is present on this system. * Also set "si-hw-provider" property, if not already set. */ ksmbios = smbios_open(NULL, SMB_VERSION, ksmbios_flags, NULL); if (ksmbios != NULL && ((smid = smbios_info_system(ksmbios, &smsys)) != SMB_ERR) && (smbios_info_common(ksmbios, smid, &sminfo)) != SMB_ERR) { mfg = (char *)sminfo.smbi_manufacturer; if (BOP_GETPROPLEN(bootops, "si-hw-provider") < 0) { extern char hw_provider[]; int i; for (i = 0; i < SYS_NMLN; i++) { if (isprint(mfg[i])) hw_provider[i] = mfg[i]; else { hw_provider[i] = '\0'; break; } } hw_provider[SYS_NMLN - 1] = '\0'; } } } /* * Originally clconf_init() apparently needed the hostid. But * this no longer appears to be true - it uses its own nodeid. * By placing the hostid logic here, we are able to make use of * the SMBIOS UUID. */ if ((h = set_soft_hostid()) == HW_INVALID_HOSTID) { cmn_err(CE_WARN, "Unable to set hostid"); } else { for (v = h, cnt = 0; cnt < 10; cnt++) { d[cnt] = (char)(v % 10); v /= 10; if (v == 0) break; } for (cp = hw_serial; cnt >= 0; cnt--) *cp++ = d[cnt] + '0'; *cp = 0; } /* * Set up the CPU module subsystem for the boot cpu in the native * case, and all physical cpu resource in the xpv dom0 case. * Modifies the device tree, so this must be done after * setup_ddi(). */ #ifdef __xpv /* * If paravirtualized and on dom0 then we initialize all physical * cpu handles now; if paravirtualized on a domU then do not * initialize. */ if (DOMAIN_IS_INITDOMAIN(xen_info)) { xen_mc_lcpu_cookie_t cpi; for (cpi = xen_physcpu_next(NULL); cpi != NULL; cpi = xen_physcpu_next(cpi)) { if ((hdl = cmi_init(CMI_HDL_SOLARIS_xVM_MCA, xen_physcpu_chipid(cpi), xen_physcpu_coreid(cpi), xen_physcpu_strandid(cpi))) != NULL && is_x86_feature(x86_featureset, X86FSET_MCA)) cmi_mca_init(hdl); } } #else /* * Initialize a handle for the boot cpu - others will initialize * as they startup. */ if ((hdl = cmi_init(CMI_HDL_NATIVE, cmi_ntv_hwchipid(CPU), cmi_ntv_hwcoreid(CPU), cmi_ntv_hwstrandid(CPU))) != NULL) { if (is_x86_feature(x86_featureset, X86FSET_MCA)) cmi_mca_init(hdl); CPU->cpu_m.mcpu_cmi_hdl = hdl; } #endif /* __xpv */ /* * Fake a prom tree such that /dev/openprom continues to work */ PRM_POINT("startup_modules: calling prom_setup..."); prom_setup(); PRM_POINT("startup_modules: done"); /* * Load all platform specific modules */ PRM_POINT("startup_modules: calling psm_modload..."); psm_modload(); PRM_POINT("startup_modules() done"); } /* * claim a "setaside" boot page for use in the kernel */ page_t * boot_claim_page(pfn_t pfn) { page_t *pp; pp = page_numtopp_nolock(pfn); ASSERT(pp != NULL); if (PP_ISBOOTPAGES(pp)) { if (pp->p_next != NULL) pp->p_next->p_prev = pp->p_prev; if (pp->p_prev == NULL) bootpages = pp->p_next; else pp->p_prev->p_next = pp->p_next; } else { /* * htable_attach() expects a base pagesize page */ if (pp->p_szc != 0) page_boot_demote(pp); pp = page_numtopp(pfn, SE_EXCL); } return (pp); } /* * Walk through the pagetables looking for pages mapped in by boot. If the * setaside flag is set the pages are expected to be returned to the * kernel later in boot, so we add them to the bootpages list. */ static void protect_boot_range(uintptr_t low, uintptr_t high, int setaside) { uintptr_t va = low; size_t len; uint_t prot; pfn_t pfn; page_t *pp; pgcnt_t boot_protect_cnt = 0; while (kbm_probe(&va, &len, &pfn, &prot) != 0 && va < high) { if (va + len >= high) panic("0x%lx byte mapping at 0x%p exceeds boot's " "legal range.", len, (void *)va); while (len > 0) { pp = page_numtopp_alloc(pfn); if (pp != NULL) { if (setaside == 0) panic("Unexpected mapping by boot. " "addr=%p pfn=%lx\n", (void *)va, pfn); pp->p_next = bootpages; pp->p_prev = NULL; PP_SETBOOTPAGES(pp); if (bootpages != NULL) { bootpages->p_prev = pp; } bootpages = pp; ++boot_protect_cnt; } ++pfn; len -= MMU_PAGESIZE; va += MMU_PAGESIZE; } } PRM_DEBUG(boot_protect_cnt); } /* * Establish the final size of the kernel's heap, size of segmap, segkp, etc. */ static void layout_kernel_va(void) { const size_t physmem_size = mmu_ptob(physmem); size_t size; PRM_POINT("layout_kernel_va() starting..."); kpm_vbase = (caddr_t)segkpm_base; kpm_size = ROUND_UP_LPAGE(mmu_ptob(physmax + 1)); if ((uintptr_t)kpm_vbase + kpm_size > (uintptr_t)valloc_base) panic("not enough room for kpm!"); PRM_DEBUG(kpm_size); PRM_DEBUG(kpm_vbase); segkp_base = (caddr_t)valloc_base + valloc_sz; if (!segkp_fromheap) { size = mmu_ptob(segkpsize); /* * Determine size of segkp * Users can change segkpsize through eeprom. */ if (size < SEGKPMINSIZE || size > SEGKPMAXSIZE) { size = SEGKPDEFSIZE; cmn_err(CE_WARN, "!Illegal value for segkpsize. " "segkpsize has been reset to %ld pages", mmu_btop(size)); } size = MIN(size, MAX(SEGKPMINSIZE, physmem_size)); segkpsize = mmu_btop(ROUND_UP_LPAGE(size)); } PRM_DEBUG(segkp_base); PRM_DEBUG(segkpsize); /* * segkvmm: backing for vmm guest memory. Like segzio, we have a * separate segment for two reasons: it makes it easy to skip our pages * on kernel crash dumps, and it helps avoid fragmentation. With this * segment, we're expecting significantly-sized allocations only; we'll * default to 4x the size of physmem. */ segkvmm_base = segkp_base + mmu_ptob(segkpsize); size = segkvmmsize != 0 ? mmu_ptob(segkvmmsize) : (physmem_size * 4); size = MAX(size, SEGVMMMINSIZE); segkvmmsize = mmu_btop(ROUND_UP_LPAGE(size)); PRM_DEBUG(segkvmmsize); PRM_DEBUG(segkvmm_base); /* * segzio is used for ZFS cached data. For segzio, we use 1.5x physmem. */ segzio_base = segkvmm_base + mmu_ptob(segkvmmsize); if (segzio_fromheap) { segziosize = 0; } else { size = (segziosize != 0) ? mmu_ptob(segziosize) : (physmem_size * 3) / 2; size = MAX(size, SEGZIOMINSIZE); segziosize = mmu_btop(ROUND_UP_LPAGE(size)); } PRM_DEBUG(segziosize); PRM_DEBUG(segzio_base); /* * Put the range of VA for device mappings next, kmdb knows to not * grep in this range of addresses. */ toxic_addr = ROUND_UP_LPAGE((uintptr_t)segzio_base + mmu_ptob(segziosize)); PRM_DEBUG(toxic_addr); segmap_start = ROUND_UP_LPAGE(toxic_addr + toxic_size); /* * Users can change segmapsize through eeprom. If the variable * is tuned through eeprom, there is no upper bound on the * size of segmap. */ segmapsize = MAX(ROUND_UP_LPAGE(segmapsize), SEGMAPDEFAULT); PRM_DEBUG(segmap_start); PRM_DEBUG(segmapsize); kernelheap = (caddr_t)ROUND_UP_LPAGE(segmap_start + segmapsize); PRM_DEBUG(kernelheap); PRM_POINT("layout_kernel_va() done..."); } /* * Finish initializing the VM system, now that we are no longer * relying on the boot time memory allocators. */ static void startup_vm(void) { struct segmap_crargs a; extern int use_brk_lpg, use_stk_lpg; PRM_POINT("startup_vm() starting..."); /* * Initialize the hat layer. */ hat_init(); /* * Do final allocations of HAT data structures that need to * be allocated before quiescing the boot loader. */ PRM_POINT("Calling hat_kern_alloc()..."); hat_kern_alloc((caddr_t)segmap_start, segmapsize, ekernelheap); PRM_POINT("hat_kern_alloc() done"); #ifndef __xpv /* * Setup Page Attribute Table */ pat_sync(); #endif /* * The next two loops are done in distinct steps in order * to be sure that any page that is doubly mapped (both above * KERNEL_TEXT and below kernelbase) is dealt with correctly. * Note this may never happen, but it might someday. */ bootpages = NULL; PRM_POINT("Protecting boot pages"); /* * Protect any pages mapped above KERNEL_TEXT that somehow have * page_t's. This can only happen if something weird allocated * in this range (like kadb/kmdb). */ protect_boot_range(KERNEL_TEXT, (uintptr_t)-1, 0); /* * Before we can take over memory allocation/mapping from the boot * loader we must remove from our free page lists any boot allocated * pages that stay mapped until release_bootstrap(). */ protect_boot_range(0, kernelbase, 1); /* * Switch to running on regular HAT (not boot_mmu) */ PRM_POINT("Calling hat_kern_setup()..."); hat_kern_setup(); /* * It is no longer safe to call BOP_ALLOC(), so make sure we don't. */ bop_no_more_mem(); PRM_POINT("hat_kern_setup() done"); hat_cpu_online(CPU); /* * Initialize VM system */ PRM_POINT("Calling kvm_init()..."); kvm_init(); PRM_POINT("kvm_init() done"); /* * Tell kmdb that the VM system is now working */ if (boothowto & RB_DEBUG) kdi_dvec_vmready(); #if defined(__xpv) /* * Populate the I/O pool on domain 0 */ if (DOMAIN_IS_INITDOMAIN(xen_info)) { extern long populate_io_pool(void); long init_io_pool_cnt; PRM_POINT("Populating reserve I/O page pool"); init_io_pool_cnt = populate_io_pool(); PRM_DEBUG(init_io_pool_cnt); } #endif /* * Mangle the brand string etc. */ cpuid_execpass(CPU, CPUID_PASS_DYNAMIC, NULL); /* * Create the device arena for toxic (to dtrace/kmdb) mappings. */ device_arena = vmem_create("device", (void *)toxic_addr, toxic_size, MMU_PAGESIZE, NULL, NULL, NULL, 0, VM_SLEEP); /* * Now that we've got more VA, as well as the ability to allocate from * it, tell the debugger. */ if (boothowto & RB_DEBUG) kdi_dvec_memavail(); #if !defined(__xpv) /* * Map page pfn=0 for drivers, such as kd, that need to pick up * parameters left there by controllers/BIOS. */ PRM_POINT("setup up p0_va"); p0_va = i86devmap(0, 1, PROT_READ); PRM_DEBUG(p0_va); #endif cmn_err(CE_CONT, "?mem = %luK (0x%lx)\n", physinstalled << (MMU_PAGESHIFT - 10), ptob(physinstalled)); /* * disable automatic large pages for small memory systems or * when the disable flag is set. * * Do not yet consider page sizes larger than 2m/4m. */ if (!auto_lpg_disable && mmu.max_page_level > 0) { max_uheap_lpsize = LEVEL_SIZE(1); max_ustack_lpsize = LEVEL_SIZE(1); max_privmap_lpsize = LEVEL_SIZE(1); max_uidata_lpsize = LEVEL_SIZE(1); max_utext_lpsize = LEVEL_SIZE(1); max_shm_lpsize = LEVEL_SIZE(1); } if (physmem < privm_lpg_min_physmem || mmu.max_page_level == 0 || auto_lpg_disable) { use_brk_lpg = 0; use_stk_lpg = 0; } mcntl0_lpsize = LEVEL_SIZE(mmu.umax_page_level); PRM_POINT("Calling hat_init_finish()..."); hat_init_finish(); PRM_POINT("hat_init_finish() done"); /* * Initialize the segkp segment type. */ rw_enter(&kas.a_lock, RW_WRITER); PRM_POINT("Attaching segkp"); if (segkp_fromheap) { segkp->s_as = &kas; } else if (seg_attach(&kas, (caddr_t)segkp_base, mmu_ptob(segkpsize), segkp) < 0) { panic("startup: cannot attach segkp"); /*NOTREACHED*/ } PRM_POINT("Doing segkp_create()"); if (segkp_create(segkp) != 0) { panic("startup: segkp_create failed"); /*NOTREACHED*/ } PRM_DEBUG(segkp); rw_exit(&kas.a_lock); /* * kpm segment */ segmap_kpm = 0; if (kpm_desired) kpm_init(); /* * Now create segmap segment. */ rw_enter(&kas.a_lock, RW_WRITER); if (seg_attach(&kas, (caddr_t)segmap_start, segmapsize, segmap) < 0) { panic("cannot attach segmap"); /*NOTREACHED*/ } PRM_DEBUG(segmap); a.prot = PROT_READ | PROT_WRITE; a.shmsize = 0; a.nfreelist = segmapfreelists; if (segmap_create(segmap, (caddr_t)&a) != 0) panic("segmap_create segmap"); rw_exit(&kas.a_lock); setup_vaddr_for_ppcopy(CPU); segdev_init(); #if defined(__xpv) if (DOMAIN_IS_INITDOMAIN(xen_info)) #endif pmem_init(); PRM_POINT("startup_vm() done"); } /* * Load a tod module for the non-standard tod part found on this system. */ static void load_tod_module(char *todmod) { if (modload("tod", todmod) == -1) halt("Can't load TOD module"); } #ifndef __xpv static void startup_tsc(void) { uint64_t tsc_freq; PRM_POINT("startup_tsc() starting..."); tsc_freq = tsc_calibrate(); PRM_DEBUG(tsc_freq); tsc_hrtimeinit(tsc_freq); } #endif static void startup_end(void) { int i; extern void cpu_event_init(void); PRM_POINT("startup_end() starting..."); /* * Perform tasks that get done after most of the VM * initialization has been done but before the clock * and other devices get started. */ PRM_POINT("kern_setup1"); kern_setup1(); PRM_POINT("kern_setup1 done"); /* * Perform CPC initialization for this CPU. */ kcpc_hw_init(CPU); /* * Initialize cpu event framework. */ cpu_event_init(); #if defined(OPTERON_ERRATUM_147) if (opteron_erratum_147) patch_erratum_147(); #endif /* * If needed, load TOD module now so that ddi_get_time(9F) etc. work * (For now, "needed" is defined as set tod_module_name in /etc/system) */ if (tod_module_name != NULL) { PRM_POINT("load_tod_module()"); load_tod_module(tod_module_name); } #if defined(__xpv) /* * Forceload interposing TOD module for the hypervisor. */ PRM_POINT("load_tod_module()"); load_tod_module("xpvtod"); #endif /* * Configure the system. */ PRM_POINT("Calling configure()..."); configure(); /* set up devices */ PRM_POINT("configure() done"); /* * configure() will have called fpu_probe() so we can now finish off * the last pieces. */ if (fp_save_mech == FP_XSAVE) { PRM_POINT("xsave_setup_msr()"); xsave_setup_msr(CPU); } /* * Set up the kmem caches for FP saving AFTER we have determined the * final set of FPU features that we have enabled and programmed into * the CPU in xsave_setup_msr(). */ fpu_save_cache_init(); /* * Set up the FPU save area for LWP0. */ lwp_fp_init(&lwp0); /* * Set the isa_list string to the defined instruction sets we * support. */ setx86isalist(); PRM_POINT("cpu_intr_alloc()"); cpu_intr_alloc(CPU, NINTR_THREADS); PRM_POINT("psm_install()"); psm_install(); PRM_POINT("psm_install() done"); /* * We're done with bootops. We don't unmap the bootstrap yet because * we're still using bootsvcs. */ PRM_POINT("NULLing out bootops"); *bootopsp = (struct bootops *)NULL; bootops = (struct bootops *)NULL; #if defined(__xpv) ec_init_debug_irq(); xs_domu_init(); #endif #if !defined(__xpv) /* * Intel IOMMU has been setup/initialized in ddi_impl.c * Start it up now. */ immu_startup(); /* * Now that we're no longer going to drop into real mode for a BIOS call * via bootops, we can enable PCID (which requires CR0.PG). */ enable_pcid(); #endif PRM_POINT("Enabling interrupts: picinitf"); (*picinitf)(); PRM_POINT("Enabling interrupts: sti"); sti(); PRM_POINT("Enabling interrupts: on"); #if defined(__xpv) ASSERT(CPU->cpu_m.mcpu_vcpu_info->evtchn_upcall_mask == 0); xen_late_startup(); #endif (void) add_avsoftintr((void *)&softlevel1_hdl, 1, softlevel1, "softlevel1", NULL, NULL); /* XXX to be moved later */ /* * Register software interrupt handlers for ddi_periodic_add(9F). * Software interrupts up to the level 10 are supported. */ for (i = DDI_IPL_1; i <= DDI_IPL_10; i++) { (void) add_avsoftintr((void *)&softlevel_hdl[i-1], i, (avfunc)(uintptr_t)ddi_periodic_softintr, "ddi_periodic", (caddr_t)(uintptr_t)i, NULL); } #if !defined(__xpv) if (modload("drv", "amd_iommu") < 0) { PRM_POINT("No AMD IOMMU present\n"); } else if (ddi_hold_installed_driver(ddi_name_to_major( "amd_iommu")) == NULL) { PRM_POINT("AMD IOMMU failed to attach\n"); } #endif post_startup_cpu_fixups(); PRM_POINT("startup_end() done"); } /* * Don't remove the following 2 variables. They are necessary * for reading the hostid from the legacy file (/kernel/misc/sysinit). */ char *_hs1107 = hw_serial; ulong_t _bdhs34; void post_startup(void) { extern void cpupm_init(cpu_t *); extern void cpu_event_init_cpu(cpu_t *); /* * Set the system wide, processor-specific flags to be passed * to userland via the aux vector for performance hints and * instruction set extensions. */ bind_hwcap(); #ifdef __xpv if (DOMAIN_IS_INITDOMAIN(xen_info)) #endif { #if defined(__xpv) xpv_panic_init(); #else /* * Startup the memory scrubber. * XXPV This should be running somewhere .. */ if ((get_hwenv() & HW_VIRTUAL) == 0) memscrub_init(); #endif } /* * Complete CPU module initialization */ cmi_post_startup(); /* * Perform forceloading tasks for /etc/system. */ (void) mod_sysctl(SYS_FORCELOAD, NULL); /* * ON4.0: Force /proc module in until clock interrupt handle fixed * ON4.0: This must be fixed or restated in /etc/systems. */ (void) modload("fs", "procfs"); (void) i_ddi_attach_hw_nodes("pit_beep"); maxmem = freemem; cpu_event_init_cpu(CPU); cpupm_init(CPU); (void) mach_cpu_create_device_node(CPU, NULL); pg_init(); } static int pp_in_range(page_t *pp, uint64_t low_addr, uint64_t high_addr) { return ((pp->p_pagenum >= btop(low_addr)) && (pp->p_pagenum < btopr(high_addr))); } static int pp_in_module(page_t *pp, const rd_existing_t *modranges) { uint_t i; for (i = 0; modranges[i].phys != 0; i++) { if (pp_in_range(pp, modranges[i].phys, modranges[i].phys + modranges[i].size)) return (1); } return (0); } void release_bootstrap(void) { int root_is_ramdisk; page_t *pp; extern void kobj_boot_unmountroot(void); extern dev_t rootdev; uint_t i; char propname[32]; rd_existing_t *modranges; #if !defined(__xpv) pfn_t pfn; #endif /* * Save the bootfs module ranges so that we can reserve them below * for the real bootfs. */ modranges = kmem_alloc(sizeof (rd_existing_t) * MAX_BOOT_MODULES, KM_SLEEP); for (i = 0; ; i++) { uint64_t start, size; modranges[i].phys = 0; (void) snprintf(propname, sizeof (propname), "module-addr-%u", i); if (do_bsys_getproplen(NULL, propname) <= 0) break; (void) do_bsys_getprop(NULL, propname, &start); (void) snprintf(propname, sizeof (propname), "module-size-%u", i); if (do_bsys_getproplen(NULL, propname) <= 0) break; (void) do_bsys_getprop(NULL, propname, &size); modranges[i].phys = start; modranges[i].size = size; } /* unmount boot ramdisk and release kmem usage */ kobj_boot_unmountroot(); /* * We're finished using the boot loader so free its pages. */ PRM_POINT("Unmapping lower boot pages"); clear_boot_mappings(0, _userlimit); postbootkernelbase = kernelbase; /* * If root isn't on ramdisk, destroy the hardcoded * ramdisk node now and release the memory. Else, * ramdisk memory is kept in rd_pages. */ root_is_ramdisk = (getmajor(rootdev) == ddi_name_to_major("ramdisk")); if (!root_is_ramdisk) { dev_info_t *dip = ddi_find_devinfo("ramdisk", -1, 0); ASSERT(dip && ddi_get_parent(dip) == ddi_root_node()); ndi_rele_devi(dip); /* held from ddi_find_devinfo */ (void) ddi_remove_child(dip, 0); } PRM_POINT("Releasing boot pages"); while (bootpages) { extern uint64_t ramdisk_start, ramdisk_end; pp = bootpages; bootpages = pp->p_next; /* Keep pages for the lower 64K */ if (pp_in_range(pp, 0, 0x40000)) { pp->p_next = lower_pages; lower_pages = pp; lower_pages_count++; continue; } if ((root_is_ramdisk && pp_in_range(pp, ramdisk_start, ramdisk_end)) || pp_in_module(pp, modranges)) { pp->p_next = rd_pages; rd_pages = pp; continue; } pp->p_next = (struct page *)0; pp->p_prev = (struct page *)0; PP_CLRBOOTPAGES(pp); page_free(pp, 1); } PRM_POINT("Boot pages released"); kmem_free(modranges, sizeof (rd_existing_t) * 99); #if !defined(__xpv) /* XXPV -- note this following bunch of code needs to be revisited in Xen 3.0 */ /* * Find 1 page below 1 MB so that other processors can boot up or * so that any processor can resume. * Make sure it has a kernel VA as well as a 1:1 mapping. * We should have just free'd one up. */ /* * 0x10 pages is 64K. Leave the bottom 64K alone * for BIOS. */ for (pfn = 0x10; pfn < btop(1*1024*1024); pfn++) { if (page_numtopp_alloc(pfn) == NULL) continue; rm_platter_va = i86devmap(pfn, 1, PROT_READ | PROT_WRITE | PROT_EXEC); rm_platter_pa = ptob(pfn); break; } if (pfn == btop(1*1024*1024) && use_mp) panic("No page below 1M available for starting " "other processors or for resuming from system-suspend"); #endif /* !__xpv */ } /* * Initialize the platform-specific parts of a page_t. */ void add_physmem_cb(page_t *pp, pfn_t pnum) { pp->p_pagenum = pnum; pp->p_mapping = NULL; pp->p_embed = 0; pp->p_share = 0; pp->p_mlentry = 0; } /* * kphysm_init() initializes physical memory. */ static pgcnt_t kphysm_init(page_t *pp, pgcnt_t npages) { struct memlist *pmem; struct memseg *cur_memseg; pfn_t base_pfn; pfn_t end_pfn; pgcnt_t num; pgcnt_t pages_done = 0; uint64_t addr; uint64_t size; extern pfn_t ddiphysmin; extern int mnode_xwa; int ms = 0, me = 0; ASSERT(page_hash != NULL && page_hashsz != 0); cur_memseg = memseg_base; for (pmem = phys_avail; pmem && npages; pmem = pmem->ml_next) { /* * In a 32 bit kernel can't use higher memory if we're * not booting in PAE mode. This check takes care of that. */ addr = pmem->ml_address; size = pmem->ml_size; if (btop(addr) > physmax) continue; /* * align addr and size - they may not be at page boundaries */ if ((addr & MMU_PAGEOFFSET) != 0) { addr += MMU_PAGEOFFSET; addr &= ~(uint64_t)MMU_PAGEOFFSET; size -= addr - pmem->ml_address; } /* only process pages below or equal to physmax */ if ((btop(addr + size) - 1) > physmax) size = ptob(physmax - btop(addr) + 1); num = btop(size); if (num == 0) continue; if (num > npages) num = npages; npages -= num; pages_done += num; base_pfn = btop(addr); if (prom_debug) prom_printf("MEMSEG addr=0x%" PRIx64 " pgs=0x%lx pfn 0x%lx-0x%lx\n", addr, num, base_pfn, base_pfn + num); /* * Ignore pages below ddiphysmin to simplify ddi memory * allocation with non-zero addr_lo requests. */ if (base_pfn < ddiphysmin) { if (base_pfn + num <= ddiphysmin) continue; pp += (ddiphysmin - base_pfn); num -= (ddiphysmin - base_pfn); base_pfn = ddiphysmin; } /* * mnode_xwa is greater than 1 when large pages regions can * cross memory node boundaries. To prevent the formation * of these large pages, configure the memsegs based on the * memory node ranges which had been made non-contiguous. */ end_pfn = base_pfn + num - 1; if (mnode_xwa > 1) { ms = PFN_2_MEM_NODE(base_pfn); me = PFN_2_MEM_NODE(end_pfn); if (ms != me) { /* * current range spans more than 1 memory node. * Set num to only the pfn range in the start * memory node. */ num = mem_node_config[ms].physmax - base_pfn + 1; ASSERT(end_pfn > mem_node_config[ms].physmax); } } for (;;) { /* * Build the memsegs entry */ cur_memseg->pages = pp; cur_memseg->epages = pp + num; cur_memseg->pages_base = base_pfn; cur_memseg->pages_end = base_pfn + num; /* * Insert into memseg list in decreasing pfn range * order. Low memory is typically more fragmented such * that this ordering keeps the larger ranges at the * front of the list for code that searches memseg. * This ASSERTS that the memsegs coming in from boot * are in increasing physical address order and not * contiguous. */ if (memsegs != NULL) { ASSERT(cur_memseg->pages_base >= memsegs->pages_end); cur_memseg->next = memsegs; } memsegs = cur_memseg; /* * add_physmem() initializes the PSM part of the page * struct by calling the PSM back with add_physmem_cb(). * In addition it coalesces pages into larger pages as * it initializes them. */ add_physmem(pp, num, base_pfn); cur_memseg++; availrmem_initial += num; availrmem += num; pp += num; if (ms >= me) break; /* process next memory node range */ ms++; base_pfn = mem_node_config[ms].physbase; if (mnode_xwa > 1) { num = MIN(mem_node_config[ms].physmax, end_pfn) - base_pfn + 1; } else { num = mem_node_config[ms].physmax - base_pfn + 1; } } } PRM_DEBUG(availrmem_initial); PRM_DEBUG(availrmem); PRM_DEBUG(freemem); build_pfn_hash(); return (pages_done); } /* * Kernel VM initialization. */ static void kvm_init(void) { ASSERT((((uintptr_t)s_text) & MMU_PAGEOFFSET) == 0); /* * Put the kernel segments in kernel address space. */ rw_enter(&kas.a_lock, RW_WRITER); as_avlinit(&kas); (void) seg_attach(&kas, s_text, e_moddata - s_text, &ktextseg); (void) segkmem_create(&ktextseg); (void) seg_attach(&kas, (caddr_t)valloc_base, valloc_sz, &kvalloc); (void) segkmem_create(&kvalloc); (void) seg_attach(&kas, kernelheap, ekernelheap - kernelheap, &kvseg); (void) segkmem_create(&kvseg); if (core_size > 0) { PRM_POINT("attaching kvseg_core"); (void) seg_attach(&kas, (caddr_t)core_base, core_size, &kvseg_core); (void) segkmem_create(&kvseg_core); } PRM_POINT("attaching segkvmm"); (void) seg_attach(&kas, segkvmm_base, mmu_ptob(segkvmmsize), &kvmmseg); (void) segkmem_create(&kvmmseg); segkmem_kvmm_init(segkvmm_base, mmu_ptob(segkvmmsize)); if (segziosize > 0) { PRM_POINT("attaching segzio"); (void) seg_attach(&kas, segzio_base, mmu_ptob(segziosize), &kzioseg); (void) segkmem_create(&kzioseg); /* create zio area covering new segment */ segkmem_zio_init(segzio_base, mmu_ptob(segziosize)); } (void) seg_attach(&kas, kdi_segdebugbase, kdi_segdebugsize, &kdebugseg); (void) segkmem_create(&kdebugseg); rw_exit(&kas.a_lock); /* * Ensure that the red zone at kernelbase is never accessible. */ PRM_POINT("protecting redzone"); (void) as_setprot(&kas, (caddr_t)kernelbase, KERNEL_REDZONE_SIZE, 0); /* * Make the text writable so that it can be hot patched by DTrace. */ (void) as_setprot(&kas, s_text, e_modtext - s_text, PROT_READ | PROT_WRITE | PROT_EXEC); /* * Make data writable until end. */ (void) as_setprot(&kas, s_data, e_moddata - s_data, PROT_READ | PROT_WRITE | PROT_EXEC); } #ifndef __xpv /* * Solaris adds an entry for Write Combining caching to the PAT */ static uint64_t pat_attr_reg = PAT_DEFAULT_ATTRIBUTE; void pat_sync(void) { ulong_t cr0, cr0_orig, cr4; if (!is_x86_feature(x86_featureset, X86FSET_PAT)) return; cr0_orig = cr0 = getcr0(); cr4 = getcr4(); /* disable caching and flush all caches and TLBs */ cr0 |= CR0_CD; cr0 &= ~CR0_NW; setcr0(cr0); invalidate_cache(); if (cr4 & CR4_PGE) { setcr4(cr4 & ~(ulong_t)CR4_PGE); setcr4(cr4); } else { reload_cr3(); } /* add our entry to the PAT */ wrmsr(REG_PAT, pat_attr_reg); /* flush TLBs and cache again, then reenable cr0 caching */ if (cr4 & CR4_PGE) { setcr4(cr4 & ~(ulong_t)CR4_PGE); setcr4(cr4); } else { reload_cr3(); } invalidate_cache(); setcr0(cr0_orig); } #endif /* !__xpv */ #if defined(_SOFT_HOSTID) /* * On platforms that do not have a hardware serial number, attempt * to set one based on the contents of /etc/hostid. If this file does * not exist, assume that we are to generate a new hostid and set * it in the kernel, for subsequent saving by a userland process * once the system is up and the root filesystem is mounted r/w. * * In order to gracefully support upgrade on OpenSolaris, if * /etc/hostid does not exist, we will attempt to get a serial number * using the legacy method (/kernel/misc/sysinit). * * If that isn't present, we attempt to use an SMBIOS UUID, which is * a hardware serial number. Note that we don't automatically trust * all SMBIOS UUIDs (some older platforms are defective and ship duplicate * UUIDs in violation of the standard), we check against a blacklist. * * In an attempt to make the hostid less prone to abuse * (for license circumvention, etc), we store it in /etc/hostid * in rot47 format. */ static int atoi(char *); /* * Set this to non-zero in /etc/system if you think your SMBIOS returns a * UUID that is not unique. (Also report it so that the smbios_uuid_blacklist * array can be updated.) */ int smbios_broken_uuid = 0; /* * List of known bad UUIDs. This is just the lower 32-bit values, since * that's what we use for the host id. If your hostid falls here, you need * to contact your hardware OEM for a fix for your BIOS. */ static unsigned char smbios_uuid_blacklist[][16] = { { /* Reported bad UUID (Google search) */ 0x00, 0x02, 0x00, 0x03, 0x00, 0x04, 0x00, 0x05, 0x00, 0x06, 0x00, 0x07, 0x00, 0x08, 0x00, 0x09, }, { /* Known bad DELL UUID */ 0x4C, 0x4C, 0x45, 0x44, 0x00, 0x00, 0x20, 0x10, 0x80, 0x20, 0x80, 0xC0, 0x4F, 0x20, 0x20, 0x20, }, { /* Uninitialized flash */ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }, { /* All zeros */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, }; static int32_t uuid_to_hostid(const uint8_t *uuid) { /* * Although the UUIDs are 128-bits, they may not distribute entropy * evenly. We would like to use SHA or MD5, but those are located * in loadable modules and not available this early in boot. As we * don't need the values to be cryptographically strong, we just * generate 32-bit vaue by xor'ing the various sequences together, * which ensures that the entire UUID contributes to the hostid. */ uint32_t id = 0; /* first check against the blacklist */ for (int i = 0; i < (sizeof (smbios_uuid_blacklist) / 16); i++) { if (bcmp(smbios_uuid_blacklist[0], uuid, 16) == 0) { cmn_err(CE_CONT, "?Broken SMBIOS UUID. " "Contact BIOS manufacturer for repair.\n"); return ((int32_t)HW_INVALID_HOSTID); } } for (int i = 0; i < 16; i++) id ^= ((uuid[i]) << (8 * (i % sizeof (id)))); /* Make sure return value is positive */ return (id & 0x7fffffff); } static int32_t set_soft_hostid(void) { struct _buf *file; char tokbuf[MAXNAMELEN]; token_t token; int done = 0; u_longlong_t tmp; int i; int32_t hostid = (int32_t)HW_INVALID_HOSTID; unsigned char *c; smbios_system_t smsys; /* * If /etc/hostid file not found, we'd like to get a pseudo * random number to use at the hostid. A nice way to do this * is to read the real time clock. To remain xen-compatible, * we can't poke the real hardware, so we use tsc_read() to * read the real time clock. */ if ((file = kobj_open_file(hostid_file)) == (struct _buf *)-1) { /* * hostid file not found - try to load sysinit module * and see if it has a nonzero hostid value...use that * instead of generating a new hostid here if so. */ if ((i = modload("misc", "sysinit")) != -1) { if (strlen(hw_serial) > 0) hostid = (int32_t)atoi(hw_serial); (void) modunload(i); } /* * We try to use the SMBIOS UUID. But not if it is blacklisted * in /etc/system. */ if ((hostid == HW_INVALID_HOSTID) && (smbios_broken_uuid == 0) && (ksmbios != NULL) && (smbios_info_system(ksmbios, &smsys) != SMB_ERR) && (smsys.smbs_uuidlen >= 16)) { hostid = uuid_to_hostid(smsys.smbs_uuid); } /* * Generate a "random" hostid using the clock. These * hostids will change on each boot if the value is not * saved to a persistent /etc/hostid file. */ if (hostid == HW_INVALID_HOSTID) { hostid = tsc_read() & 0x0CFFFFF; } } else { /* hostid file found */ while (!done) { token = kobj_lex(file, tokbuf, sizeof (tokbuf)); switch (token) { case POUND: /* * skip comments */ kobj_find_eol(file); break; case STRING: /* * un-rot47 - obviously this * nonsense is ascii-specific */ for (c = (unsigned char *)tokbuf; *c != '\0'; c++) { *c += 47; if (*c > '~') *c -= 94; else if (*c < '!') *c += 94; } /* * now we should have a real number */ if (kobj_getvalue(tokbuf, &tmp) != 0) kobj_file_err(CE_WARN, file, "Bad value %s for hostid", tokbuf); else hostid = (int32_t)tmp; break; case EOF: done = 1; /* FALLTHROUGH */ case NEWLINE: kobj_newline(file); break; default: break; } } if (hostid == HW_INVALID_HOSTID) /* didn't find a hostid */ kobj_file_err(CE_WARN, file, "hostid missing or corrupt"); kobj_close_file(file); } /* * hostid is now the value read from /etc/hostid, or the * new hostid we generated in this routine or HW_INVALID_HOSTID if not * set. */ return (hostid); } static int atoi(char *p) { int i = 0; while (*p != '\0') i = 10 * i + (*p++ - '0'); return (i); } #endif /* _SOFT_HOSTID */ void get_system_configuration(void) { char prop[32]; u_longlong_t nodes_ll, cpus_pernode_ll, lvalue; if (BOP_GETPROPLEN(bootops, "nodes") > sizeof (prop) || BOP_GETPROP(bootops, "nodes", prop) < 0 || kobj_getvalue(prop, &nodes_ll) == -1 || nodes_ll > MAXNODES || BOP_GETPROPLEN(bootops, "cpus_pernode") > sizeof (prop) || BOP_GETPROP(bootops, "cpus_pernode", prop) < 0 || kobj_getvalue(prop, &cpus_pernode_ll) == -1) { system_hardware.hd_nodes = 1; system_hardware.hd_cpus_per_node = 0; } else { system_hardware.hd_nodes = (int)nodes_ll; system_hardware.hd_cpus_per_node = (int)cpus_pernode_ll; } if (BOP_GETPROPLEN(bootops, "kernelbase") > sizeof (prop) || BOP_GETPROP(bootops, "kernelbase", prop) < 0 || kobj_getvalue(prop, &lvalue) == -1) eprom_kernelbase = 0; else eprom_kernelbase = (uintptr_t)lvalue; if (BOP_GETPROPLEN(bootops, "segmapsize") > sizeof (prop) || BOP_GETPROP(bootops, "segmapsize", prop) < 0 || kobj_getvalue(prop, &lvalue) == -1) segmapsize = SEGMAPDEFAULT; else segmapsize = (uintptr_t)lvalue; if (BOP_GETPROPLEN(bootops, "segmapfreelists") > sizeof (prop) || BOP_GETPROP(bootops, "segmapfreelists", prop) < 0 || kobj_getvalue(prop, &lvalue) == -1) segmapfreelists = 0; /* use segmap driver default */ else segmapfreelists = (int)lvalue; if (BOP_GETPROPLEN(bootops, "segkpsize") > sizeof (prop) || BOP_GETPROP(bootops, "segkpsize", prop) < 0 || kobj_getvalue(prop, &lvalue) == -1) segkpsize = mmu_btop(SEGKPDEFSIZE); else segkpsize = mmu_btop((size_t)lvalue); /* physmem used to be here, but moved much earlier to fakebop.c */ } /* * Add to a memory list. * start = start of new memory segment * len = length of new memory segment in bytes * new = pointer to a new struct memlist * memlistp = memory list to which to add segment. */ void memlist_add( uint64_t start, uint64_t len, struct memlist *new, struct memlist **memlistp) { struct memlist *cur; uint64_t end = start + len; new->ml_address = start; new->ml_size = len; cur = *memlistp; while (cur) { if (cur->ml_address >= end) { new->ml_next = cur; *memlistp = new; new->ml_prev = cur->ml_prev; cur->ml_prev = new; return; } ASSERT(cur->ml_address + cur->ml_size <= start); if (cur->ml_next == NULL) { cur->ml_next = new; new->ml_prev = cur; new->ml_next = NULL; return; } memlistp = &cur->ml_next; cur = cur->ml_next; } } void kobj_vmem_init(vmem_t **text_arena, vmem_t **data_arena) { size_t tsize = e_modtext - modtext; size_t dsize = e_moddata - moddata; *text_arena = vmem_create("module_text", tsize ? modtext : NULL, tsize, 1, segkmem_alloc, segkmem_free, heaptext_arena, 0, VM_SLEEP); *data_arena = vmem_create("module_data", dsize ? moddata : NULL, dsize, 1, segkmem_alloc, segkmem_free, heap32_arena, 0, VM_SLEEP); } caddr_t kobj_text_alloc(vmem_t *arena, size_t size) { return (vmem_alloc(arena, size, VM_SLEEP | VM_BESTFIT)); } /*ARGSUSED*/ caddr_t kobj_texthole_alloc(caddr_t addr, size_t size) { panic("unexpected call to kobj_texthole_alloc()"); /*NOTREACHED*/ return (0); } /*ARGSUSED*/ void kobj_texthole_free(caddr_t addr, size_t size) { panic("unexpected call to kobj_texthole_free()"); } /* * This is called just after configure() in startup(). * * The ISALIST concept is a bit hopeless on Intel, because * there's no guarantee of an ever-more-capable processor * given that various parts of the instruction set may appear * and disappear between different implementations. * * While it would be possible to correct it and even enhance * it somewhat, the explicit hardware capability bitmask allows * more flexibility. * * So, we just leave this alone. */ static void setx86isalist(void) { char *tp; size_t len; extern char *isa_list; #define TBUFSIZE 1024 tp = kmem_alloc(TBUFSIZE, KM_SLEEP); *tp = '\0'; (void) strcpy(tp, "amd64 "); switch (x86_vendor) { case X86_VENDOR_Intel: case X86_VENDOR_AMD: case X86_VENDOR_HYGON: case X86_VENDOR_TM: if (is_x86_feature(x86_featureset, X86FSET_CMOV)) { /* * Pentium Pro or later */ (void) strcat(tp, "pentium_pro"); (void) strcat(tp, is_x86_feature(x86_featureset, X86FSET_MMX) ? "+mmx pentium_pro " : " "); } /*FALLTHROUGH*/ case X86_VENDOR_Cyrix: ASSERT(is_x86_feature(x86_featureset, X86FSET_CPUID)); (void) strcat(tp, "pentium"); (void) strcat(tp, is_x86_feature(x86_featureset, X86FSET_MMX) ? "+mmx pentium " : " "); break; default: break; } (void) strcat(tp, "i486 i386 i86"); len = strlen(tp) + 1; /* account for NULL at end of string */ isa_list = strcpy(kmem_alloc(len, KM_SLEEP), tp); kmem_free(tp, TBUFSIZE); #undef TBUFSIZE } void * device_arena_alloc(size_t size, int vm_flag) { return (vmem_alloc(device_arena, size, vm_flag)); } void device_arena_free(void *vaddr, size_t size) { vmem_free(device_arena, vaddr, size); } /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2009 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. * * Copyright 2012 Nexenta Systems, Inc. All rights reserved. * Copyright (c) 2014, 2016 by Delphix. All rights reserved. * Copyright 2020 Joyent, Inc. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* * Using the Pentium's TSC register for gethrtime() * ------------------------------------------------ * * The Pentium family, like many chip architectures, has a high-resolution * timestamp counter ("TSC") which increments once per CPU cycle. The contents * of the timestamp counter are read with the RDTSC instruction. * * As with its UltraSPARC equivalent (the %tick register), TSC's cycle count * must be translated into nanoseconds in order to implement gethrtime(). * We avoid inducing floating point operations in this conversion by * implementing the same nsec_scale algorithm as that found in the sun4u * platform code. The sun4u NATIVE_TIME_TO_NSEC_SCALE block comment contains * a detailed description of the algorithm; the comment is not reproduced * here. This implementation differs only in its value for NSEC_SHIFT: * we implement an NSEC_SHIFT of 5 (instead of sun4u's 4) to allow for * 60 MHz Pentiums. * * While TSC and %tick are both cycle counting registers, TSC's functionality * falls short in several critical ways: * * (a) TSCs on different CPUs are not guaranteed to be in sync. While in * practice they often _are_ in sync, this isn't guaranteed by the * architecture. * * (b) The TSC cannot be reliably set to an arbitrary value. The architecture * only supports writing the low 32-bits of TSC, making it impractical * to rewrite. * * (c) The architecture doesn't have the capacity to interrupt based on * arbitrary values of TSC; there is no TICK_CMPR equivalent. * * Together, (a) and (b) imply that software must track the skew between * TSCs and account for it (it is assumed that while there may exist skew, * there does not exist drift). To determine the skew between CPUs, we * have newly onlined CPUs call tsc_sync_slave(), while the CPU performing * the online operation calls tsc_sync_master(). * * In the absence of time-of-day clock adjustments, gethrtime() must stay in * sync with gettimeofday(). This is problematic; given (c), the software * cannot drive its time-of-day source from TSC, and yet they must somehow be * kept in sync. We implement this by having a routine, tsc_tick(), which * is called once per second from the interrupt which drives time-of-day. * * Note that the hrtime base for gethrtime, tsc_hrtime_base, is modified * atomically with nsec_scale under CLOCK_LOCK. This assures that time * monotonically increases. */ #define NSEC_SHIFT 5 static uint_t nsec_unscale; /* * These two variables used to be grouped together inside of a structure that * lived on a single cache line. A regression (bug ID 4623398) caused the * compiler to emit code that "optimized" away the while-loops below. The * result was that no synchronization between the onlining and onlined CPUs * took place. */ static volatile int tsc_ready; static volatile int tsc_sync_go; /* * Used as indices into the tsc_sync_snaps[] array. */ #define TSC_MASTER 0 #define TSC_SLAVE 1 /* * Used in the tsc_master_sync()/tsc_slave_sync() rendezvous. */ #define TSC_SYNC_STOP 1 #define TSC_SYNC_GO 2 #define TSC_SYNC_DONE 3 #define SYNC_ITERATIONS 10 #define TSC_CONVERT_AND_ADD(tsc, hrt, scale) { \ unsigned int *_l = (unsigned int *)&(tsc); \ (hrt) += mul32(_l[1], scale) << NSEC_SHIFT; \ (hrt) += mul32(_l[0], scale) >> (32 - NSEC_SHIFT); \ } #define TSC_CONVERT(tsc, hrt, scale) { \ unsigned int *_l = (unsigned int *)&(tsc); \ (hrt) = mul32(_l[1], scale) << NSEC_SHIFT; \ (hrt) += mul32(_l[0], scale) >> (32 - NSEC_SHIFT); \ } int tsc_master_slave_sync_needed = 1; typedef struct tsc_sync { volatile hrtime_t master_tsc, slave_tsc; } tsc_sync_t; static tsc_sync_t *tscp; static hrtime_t tsc_last_jumped = 0; static int tsc_jumped = 0; static uint32_t tsc_wayback = 0; /* * The cap of 1 second was chosen since it is the frequency at which the * tsc_tick() function runs which means that when gethrtime() is called it * should never be more than 1 second since tsc_last was updated. */ static hrtime_t tsc_resume_cap_ns = NANOSEC; /* 1s */ static hrtime_t shadow_tsc_hrtime_base; static hrtime_t shadow_tsc_last; static uint_t shadow_nsec_scale; static uint32_t shadow_hres_lock; int get_tsc_ready(); /* * Allow an operator specify an explicit TSC calibration source * via /etc/system e.g. `set tsc_calibration="pit"` */ char *tsc_calibration; /* * The source that was used to calibrate the TSC. This is currently just * for diagnostic purposes. */ static tsc_calibrate_t *tsc_calibration_source; /* The TSC frequency after calibration */ static uint64_t tsc_freq; static inline hrtime_t tsc_protect(hrtime_t a) { if (a > tsc_resume_cap) { atomic_inc_32(&tsc_wayback); DTRACE_PROBE3(tsc__wayback, htrime_t, a, hrtime_t, tsc_last, uint32_t, tsc_wayback); return (tsc_resume_cap); } return (a); } hrtime_t tsc_gethrtime(void) { uint32_t old_hres_lock; hrtime_t tsc, hrt; do { old_hres_lock = hres_lock; if ((tsc = tsc_read()) >= tsc_last) { /* * It would seem to be obvious that this is true * (that is, the past is less than the present), * but it isn't true in the presence of suspend/resume * cycles. If we manage to call gethrtime() * after a resume, but before the first call to * tsc_tick(), we will see the jump. In this case, * we will simply use the value in TSC as the delta. */ tsc -= tsc_last; } else if (tsc >= tsc_last - 2*tsc_max_delta) { /* * There is a chance that tsc_tick() has just run on * another CPU, and we have drifted just enough so that * we appear behind tsc_last. In this case, force the * delta to be zero. */ tsc = 0; } else { /* * If we reach this else clause we assume that we have * gone through a suspend/resume cycle and use the * current tsc value as the delta. * * In rare cases we can reach this else clause due to * a lack of monotonicity in the TSC value. In such * cases using the current TSC value as the delta would * cause us to return a value ~2x of what it should * be. To protect against these cases we cap the * suspend/resume delta at tsc_resume_cap. */ tsc = tsc_protect(tsc); } hrt = tsc_hrtime_base; TSC_CONVERT_AND_ADD(tsc, hrt, nsec_scale); } while ((old_hres_lock & ~1) != hres_lock); return (hrt); } hrtime_t tsc_gethrtime_delta(void) { uint32_t old_hres_lock; hrtime_t tsc, hrt; ulong_t flags; do { old_hres_lock = hres_lock; /* * We need to disable interrupts here to assure that we * don't migrate between the call to tsc_read() and * adding the CPU's TSC tick delta. Note that disabling * and reenabling preemption is forbidden here because * we may be in the middle of a fast trap. In the amd64 * kernel we cannot tolerate preemption during a fast * trap. See _update_sregs(). */ flags = clear_int_flag(); tsc = tsc_read() + tsc_sync_tick_delta[CPU->cpu_id]; restore_int_flag(flags); /* See comments in tsc_gethrtime() above */ if (tsc >= tsc_last) { tsc -= tsc_last; } else if (tsc >= tsc_last - 2 * tsc_max_delta) { tsc = 0; } else { tsc = tsc_protect(tsc); } hrt = tsc_hrtime_base; TSC_CONVERT_AND_ADD(tsc, hrt, nsec_scale); } while ((old_hres_lock & ~1) != hres_lock); return (hrt); } hrtime_t tsc_gethrtime_tick_delta(void) { hrtime_t hrt; ulong_t flags; flags = clear_int_flag(); hrt = tsc_sync_tick_delta[CPU->cpu_id]; restore_int_flag(flags); return (hrt); } /* Calculate the hrtime while exposing the parameters of that calculation. */ hrtime_t tsc_gethrtime_params(uint64_t *tscp, uint32_t *scalep, uint8_t *shiftp) { uint32_t old_hres_lock, scale; hrtime_t tsc, last, base; do { old_hres_lock = hres_lock; if (gethrtimef == tsc_gethrtime_delta) { ulong_t flags; flags = clear_int_flag(); tsc = tsc_read() + tsc_sync_tick_delta[CPU->cpu_id]; restore_int_flag(flags); } else { tsc = tsc_read(); } last = tsc_last; base = tsc_hrtime_base; scale = nsec_scale; } while ((old_hres_lock & ~1) != hres_lock); /* See comments in tsc_gethrtime() above */ if (tsc >= last) { tsc -= last; } else if (tsc >= last - 2 * tsc_max_delta) { tsc = 0; } else { tsc = tsc_protect(tsc); } TSC_CONVERT_AND_ADD(tsc, base, nsec_scale); if (tscp != NULL) { /* * Do not simply communicate the delta applied to the hrtime * base, but rather the effective TSC measurement. */ *tscp = tsc + last; } if (scalep != NULL) { *scalep = scale; } if (shiftp != NULL) { *shiftp = NSEC_SHIFT; } return (base); } /* * This is similar to tsc_gethrtime_delta, but it cannot actually spin on * hres_lock. As a result, it caches all of the variables it needs; if the * variables don't change, it's done. */ hrtime_t dtrace_gethrtime(void) { uint32_t old_hres_lock; hrtime_t tsc, hrt; ulong_t flags; do { old_hres_lock = hres_lock; /* * Interrupts are disabled to ensure that the thread isn't * migrated between the tsc_read() and adding the CPU's * TSC tick delta. */ flags = clear_int_flag(); tsc = tsc_read(); if (gethrtimef == tsc_gethrtime_delta) tsc += tsc_sync_tick_delta[CPU->cpu_id]; restore_int_flag(flags); /* * See the comments in tsc_gethrtime(), above. */ if (tsc >= tsc_last) tsc -= tsc_last; else if (tsc >= tsc_last - 2*tsc_max_delta) tsc = 0; else tsc = tsc_protect(tsc); hrt = tsc_hrtime_base; TSC_CONVERT_AND_ADD(tsc, hrt, nsec_scale); if ((old_hres_lock & ~1) == hres_lock) break; /* * If we're here, the clock lock is locked -- or it has been * unlocked and locked since we looked. This may be due to * tsc_tick() running on another CPU -- or it may be because * some code path has ended up in dtrace_probe() with * CLOCK_LOCK held. We'll try to determine that we're in * the former case by taking another lap if the lock has * changed since when we first looked at it. */ if (old_hres_lock != hres_lock) continue; /* * So the lock was and is locked. We'll use the old data * instead. */ old_hres_lock = shadow_hres_lock; /* * Again, disable interrupts to ensure that the thread * isn't migrated between the tsc_read() and adding * the CPU's TSC tick delta. */ flags = clear_int_flag(); tsc = tsc_read(); if (gethrtimef == tsc_gethrtime_delta) tsc += tsc_sync_tick_delta[CPU->cpu_id]; restore_int_flag(flags); /* * See the comments in tsc_gethrtime(), above. */ if (tsc >= shadow_tsc_last) tsc -= shadow_tsc_last; else if (tsc >= shadow_tsc_last - 2 * tsc_max_delta) tsc = 0; else tsc = tsc_protect(tsc); hrt = shadow_tsc_hrtime_base; TSC_CONVERT_AND_ADD(tsc, hrt, shadow_nsec_scale); } while ((old_hres_lock & ~1) != shadow_hres_lock); return (hrt); } hrtime_t tsc_gethrtimeunscaled(void) { uint32_t old_hres_lock; hrtime_t tsc; do { old_hres_lock = hres_lock; /* See tsc_tick(). */ tsc = tsc_read() + tsc_last_jumped; } while ((old_hres_lock & ~1) != hres_lock); return (tsc); } /* * Convert a nanosecond based timestamp to tsc */ uint64_t tsc_unscalehrtime(hrtime_t nsec) { hrtime_t tsc; if (tsc_gethrtime_enable) { TSC_CONVERT(nsec, tsc, nsec_unscale); return (tsc); } return ((uint64_t)nsec); } /* Convert a tsc timestamp to nanoseconds */ void tsc_scalehrtime(hrtime_t *tsc) { hrtime_t hrt; hrtime_t mytsc; if (tsc == NULL) return; mytsc = *tsc; TSC_CONVERT(mytsc, hrt, nsec_scale); *tsc = hrt; } hrtime_t tsc_gethrtimeunscaled_delta(void) { hrtime_t hrt; ulong_t flags; /* * Similarly to tsc_gethrtime_delta, we need to disable preemption * to prevent migration between the call to tsc_gethrtimeunscaled * and adding the CPU's hrtime delta. Note that disabling and * reenabling preemption is forbidden here because we may be in the * middle of a fast trap. In the amd64 kernel we cannot tolerate * preemption during a fast trap. See _update_sregs(). */ flags = clear_int_flag(); hrt = tsc_gethrtimeunscaled() + tsc_sync_tick_delta[CPU->cpu_id]; restore_int_flag(flags); return (hrt); } /* * TSC Sync Master * * Typically called on the boot CPU, this attempts to quantify TSC skew between * different CPUs. If an appreciable difference is found, gethrtimef will be * changed to point to tsc_gethrtime_delta(). * * Calculating skews is precise only when the master and slave TSCs are read * simultaneously; however, there is no algorithm that can read both CPUs in * perfect simultaneity. The proposed algorithm is an approximate method based * on the behaviour of cache management. The slave CPU continuously polls the * TSC while reading a global variable updated by the master CPU. The latest * TSC reading is saved when the master's update (forced via mfence) reaches * visibility on the slave. The master will also take a TSC reading * immediately following the mfence. * * While the delay between cache line invalidation on the slave and mfence * completion on the master is not repeatable, the error is heuristically * assumed to be 1/4th of the write time recorded by the master. Multiple * samples are taken to control for the variance caused by external factors * such as bus contention. Each sample set is independent per-CPU to control * for differing memory latency on NUMA systems. * * TSC sync is disabled in the context of virtualization because the CPUs * assigned to the guest are virtual CPUs which means the real CPUs on which * guest runs keep changing during life time of guest OS. So we would end up * calculating TSC skews for a set of CPUs during boot whereas the guest * might migrate to a different set of physical CPUs at a later point of * time. */ void tsc_sync_master(processorid_t slave) { ulong_t flags, source, min_write_time = ~0UL; hrtime_t write_time, mtsc_after, last_delta = 0; tsc_sync_t *tsc = tscp; int cnt; int hwtype; hwtype = get_hwenv(); if (!tsc_master_slave_sync_needed || (hwtype & HW_VIRTUAL) != 0) return; flags = clear_int_flag(); source = CPU->cpu_id; for (cnt = 0; cnt < SYNC_ITERATIONS; cnt++) { while (tsc_sync_go != TSC_SYNC_GO) SMT_PAUSE(); tsc->master_tsc = tsc_read(); membar_enter(); mtsc_after = tsc_read(); while (tsc_sync_go != TSC_SYNC_DONE) SMT_PAUSE(); write_time = mtsc_after - tsc->master_tsc; if (write_time <= min_write_time) { hrtime_t tdelta; tdelta = tsc->slave_tsc - mtsc_after; if (tdelta < 0) tdelta = -tdelta; /* * If the margin exists, subtract 1/4th of the measured * write time from the master's TSC value. This is an * estimate of how late the mfence completion came * after the slave noticed the cache line change. */ if (tdelta > (write_time/4)) { tdelta = tsc->slave_tsc - (mtsc_after - (write_time/4)); } else { tdelta = tsc->slave_tsc - mtsc_after; } last_delta = tsc_sync_tick_delta[source] - tdelta; tsc_sync_tick_delta[slave] = last_delta; min_write_time = write_time; } tsc->master_tsc = tsc->slave_tsc = write_time = 0; membar_enter(); tsc_sync_go = TSC_SYNC_STOP; } /* * Only enable the delta variants of the TSC functions if the measured * skew is greater than the fastest write time. */ last_delta = (last_delta < 0) ? -last_delta : last_delta; if (last_delta > min_write_time) { gethrtimef = tsc_gethrtime_delta; gethrtimeunscaledf = tsc_gethrtimeunscaled_delta; tsc_ncpu = NCPU; } restore_int_flag(flags); } /* * TSC Sync Slave * * Called by a CPU which has just been onlined. It is expected that the CPU * performing the online operation will call tsc_sync_master(). * * Like tsc_sync_master, this logic is skipped on virtualized platforms. */ void tsc_sync_slave(void) { ulong_t flags; hrtime_t s1; tsc_sync_t *tsc = tscp; int cnt; int hwtype; hwtype = get_hwenv(); if (!tsc_master_slave_sync_needed || (hwtype & HW_VIRTUAL) != 0) return; flags = clear_int_flag(); for (cnt = 0; cnt < SYNC_ITERATIONS; cnt++) { /* Re-fill the cache line */ s1 = tsc->master_tsc; membar_enter(); tsc_sync_go = TSC_SYNC_GO; do { /* * Do not put an SMT_PAUSE here. If the master and * slave are the same hyper-threaded CPU, we want the * master to yield as quickly as possible to the slave. */ s1 = tsc_read(); } while (tsc->master_tsc == 0); tsc->slave_tsc = s1; membar_enter(); tsc_sync_go = TSC_SYNC_DONE; while (tsc_sync_go != TSC_SYNC_STOP) SMT_PAUSE(); } restore_int_flag(flags); } /* * Called once per second on a CPU from the cyclic subsystem's * CY_HIGH_LEVEL interrupt. (No longer just cpu0-only) */ void tsc_tick(void) { hrtime_t now, delta; ushort_t spl; /* * Before we set the new variables, we set the shadow values. This * allows for lock free operation in dtrace_gethrtime(). */ lock_set_spl((lock_t *)&shadow_hres_lock + HRES_LOCK_OFFSET, ipltospl(CBE_HIGH_PIL), &spl); shadow_tsc_hrtime_base = tsc_hrtime_base; shadow_tsc_last = tsc_last; shadow_nsec_scale = nsec_scale; shadow_hres_lock++; splx(spl); CLOCK_LOCK(&spl); now = tsc_read(); if (gethrtimef == tsc_gethrtime_delta) now += tsc_sync_tick_delta[CPU->cpu_id]; if (now < tsc_last) { /* * The TSC has just jumped into the past. We assume that * this is due to a suspend/resume cycle, and we're going * to use the _current_ value of TSC as the delta. This * will keep tsc_hrtime_base correct. We're also going to * assume that rate of tsc does not change after a suspend * resume (i.e nsec_scale remains the same). */ delta = now; delta = tsc_protect(delta); tsc_last_jumped += tsc_last; tsc_jumped = 1; } else { /* * Determine the number of TSC ticks since the last clock * tick, and add that to the hrtime base. */ delta = now - tsc_last; } TSC_CONVERT_AND_ADD(delta, tsc_hrtime_base, nsec_scale); tsc_last = now; CLOCK_UNLOCK(spl); } void tsc_hrtimeinit(uint64_t cpu_freq_hz) { extern int gethrtime_hires; longlong_t tsc; ulong_t flags; /* * cpu_freq_hz is the measured cpu frequency in hertz */ /* * We can't accommodate CPUs slower than 31.25 MHz. */ ASSERT(cpu_freq_hz > NANOSEC / (1 << NSEC_SHIFT)); nsec_scale = (uint_t)(((uint64_t)NANOSEC << (32 - NSEC_SHIFT)) / cpu_freq_hz); nsec_unscale = (uint_t)(((uint64_t)cpu_freq_hz << (32 - NSEC_SHIFT)) / NANOSEC); flags = clear_int_flag(); tsc = tsc_read(); (void) tsc_gethrtime(); tsc_max_delta = tsc_read() - tsc; restore_int_flag(flags); gethrtimef = tsc_gethrtime; gethrtimeunscaledf = tsc_gethrtimeunscaled; scalehrtimef = tsc_scalehrtime; unscalehrtimef = tsc_unscalehrtime; hrtime_tick = tsc_tick; gethrtime_hires = 1; /* * Being part of the comm page, tsc_ncpu communicates the published * length of the tsc_sync_tick_delta array. This is kept zeroed to * ignore the absent delta data while the TSCs are synced. */ tsc_ncpu = 0; /* * Allocate memory for the structure used in the tsc sync logic. * This structure should be aligned on a multiple of cache line size. */ tscp = kmem_zalloc(PAGESIZE, KM_SLEEP); /* * Convert the TSC resume cap ns value into its unscaled TSC value. * See tsc_gethrtime(). */ if (tsc_resume_cap == 0) TSC_CONVERT(tsc_resume_cap_ns, tsc_resume_cap, nsec_unscale); } int get_tsc_ready() { return (tsc_ready); } /* * Adjust all the deltas by adding the passed value to the array and activate * the "delta" versions of the gethrtime functions. It is possible that the * adjustment could be negative. Such may occur if the SunOS instance was * moved by a virtual manager to a machine with a higher value of TSC. */ void tsc_adjust_delta(hrtime_t tdelta) { int i; for (i = 0; i < NCPU; i++) { tsc_sync_tick_delta[i] += tdelta; } gethrtimef = tsc_gethrtime_delta; gethrtimeunscaledf = tsc_gethrtimeunscaled_delta; tsc_ncpu = NCPU; } /* * Functions to manage TSC and high-res time on suspend and resume. */ /* tod_ops from "uts/i86pc/io/todpc_subr.c" */ extern tod_ops_t *tod_ops; static uint64_t tsc_saved_tsc = 0; /* 1 in 2^64 chance this'll screw up! */ static timestruc_t tsc_saved_ts; static int tsc_needs_resume = 0; /* We only want to do this once. */ int tsc_delta_onsuspend = 0; int tsc_adjust_seconds = 1; int tsc_suspend_count = 0; int tsc_resume_in_cyclic = 0; /* * Take snapshots of the current time and do any other pre-suspend work. */ void tsc_suspend(void) { /* * We need to collect the time at which we suspended here so we know * now much should be added during the resume. This is called by each * CPU, so reentry must be properly handled. */ if (tsc_gethrtime_enable) { /* * Perform the tsc_read after acquiring the lock to make it as * accurate as possible in the face of contention. */ mutex_enter(&tod_lock); tsc_saved_tsc = tsc_read(); tsc_saved_ts = TODOP_GET(tod_ops); mutex_exit(&tod_lock); /* We only want to do this once. */ if (tsc_needs_resume == 0) { if (tsc_delta_onsuspend) { tsc_adjust_delta(tsc_saved_tsc); } else { tsc_adjust_delta(nsec_scale); } tsc_suspend_count++; } } invalidate_cache(); tsc_needs_resume = 1; } /* * Restore all timestamp state based on the snapshots taken at suspend time. */ void tsc_resume(void) { /* * We only need to (and want to) do this once. So let the first * caller handle this (we are locked by the cpu lock), as it * is preferential that we get the earliest sync. */ if (tsc_needs_resume) { /* * If using the TSC, adjust the delta based on how long * we were sleeping (or away). We also adjust for * migration and a grown TSC. */ if (tsc_saved_tsc != 0) { timestruc_t ts; hrtime_t now, sleep_tsc = 0; int sleep_sec; extern void tsc_tick(void); extern uint64_t cpu_freq_hz; /* tsc_read() MUST be before TODOP_GET() */ mutex_enter(&tod_lock); now = tsc_read(); ts = TODOP_GET(tod_ops); mutex_exit(&tod_lock); /* Compute seconds of sleep time */ sleep_sec = ts.tv_sec - tsc_saved_ts.tv_sec; /* * If the saved sec is less that or equal to * the current ts, then there is likely a * problem with the clock. Assume at least * one second has passed, so that time goes forward. */ if (sleep_sec <= 0) { sleep_sec = 1; } /* How many TSC's should have occured while sleeping */ if (tsc_adjust_seconds) sleep_tsc = sleep_sec * cpu_freq_hz; /* * We also want to subtract from the "sleep_tsc" * the current value of tsc_read(), so that our * adjustment accounts for the amount of time we * have been resumed _or_ an adjustment based on * the fact that we didn't actually power off the * CPU (migration is another issue, but _should_ * also comply with this calculation). If the CPU * never powered off, then: * 'now == sleep_tsc + saved_tsc' * and the delta will effectively be "0". */ sleep_tsc -= now; if (tsc_delta_onsuspend) { tsc_adjust_delta(sleep_tsc); } else { tsc_adjust_delta(tsc_saved_tsc + sleep_tsc); } tsc_saved_tsc = 0; tsc_tick(); } tsc_needs_resume = 0; } } static int tsc_calibrate_cmp(const void *a, const void *b) { const tsc_calibrate_t * const *a1 = a; const tsc_calibrate_t * const *b1 = b; const tsc_calibrate_t *l = *a1; const tsc_calibrate_t *r = *b1; /* Sort from highest preference to lowest preference */ if (l->tscc_preference > r->tscc_preference) return (-1); if (l->tscc_preference < r->tscc_preference) return (1); /* For equal preference sources, sort alphabetically */ int c = strcmp(l->tscc_source, r->tscc_source); if (c < 0) return (-1); if (c > 0) return (1); return (0); } SET_DECLARE(tsc_calibration_set, tsc_calibrate_t); static tsc_calibrate_t * tsc_calibrate_get_force(const char *source) { tsc_calibrate_t **tsccpp; VERIFY3P(source, !=, NULL); SET_FOREACH(tsccpp, tsc_calibration_set) { tsc_calibrate_t *tsccp = *tsccpp; if (strcasecmp(source, tsccp->tscc_source) == 0) return (tsccp); } /* * If an operator explicitly gave a TSC value and we didn't find it, * we should let them know. */ cmn_err(CE_NOTE, "Explicit TSC calibration source '%s' not found; using default", source); return (NULL); } /* * As described in tscc_pit.c, as an intertim measure as we transition to * alternate calibration sources besides the PIT, we still want to gather * what the values would have been had we used the PIT. Therefore, if we're * using a source other than the PIT, we explicitly run the PIT calibration * which will store the TSC frequency as measured by the PIT for the * benefit of the APIC code (as well as any potential diagnostics). */ static void tsc_pit_also(void) { tsc_calibrate_t *pit = tsc_calibrate_get_force("PIT"); uint64_t dummy; /* We should always have the PIT as a possible calibration source */ VERIFY3P(pit, !=, NULL); /* If we used the PIT to calibrate, we don't need to run again */ if (tsc_calibration_source == pit) return; /* * Since we're not using the PIT as the actual TSC calibration source, * we don't care about the results or saving the result -- tscc_pit.c * saves the frequency in a global for the benefit of the APIC code. */ (void) pit->tscc_calibrate(&dummy); } uint64_t tsc_calibrate(void) { tsc_calibrate_t **tsccpp, *force; size_t tsc_set_size; int tsc_name_len; /* * Every x86 system since the Pentium has TSC support. Since we * only support 64-bit x86 systems, there should always be a TSC * present, and something's horribly wrong if it's missing. */ if (!is_x86_feature(x86_featureset, X86FSET_TSC)) panic("System does not have TSC support"); /* * If we already successfully calibrated the TSC, no need to do * it again. */ if (tsc_freq > 0) return (tsc_freq); PRM_POINT("Calibrating the TSC..."); /* * Allow an operator to explicitly specify a calibration source via * `set tsc_calibration=foo` in the bootloader or * `set tsc_calibration="foo"` in /etc/system (preferring a bootloader * supplied value over /etc/system). * * If no source is given, or the specified source is not found, we * fallback to trying all of the known sources in order by preference * (high preference value to low preference value) until one succeeds. */ tsc_name_len = BOP_GETPROPLEN(bootops, "tsc_calibration"); if (tsc_name_len > 0) { /* Overwrite any /etc/system supplied value */ if (tsc_calibration != NULL) { size_t len = strlen(tsc_calibration) + 1; kobj_free_string(tsc_calibration, len); } tsc_calibration = kmem_zalloc(tsc_name_len + 1, KM_SLEEP); BOP_GETPROP(bootops, "tsc_calibration", tsc_calibration); } if (tsc_calibration != NULL && (force = tsc_calibrate_get_force(tsc_calibration)) != NULL) { if (tsc_name_len > 0) { PRM_POINT("Forcing bootloader specified TSC calibration" " source"); } else { PRM_POINT("Forcing /etc/system specified TSC " "calibration source"); } PRM_DEBUGS(force->tscc_source); if (!force->tscc_calibrate(&tsc_freq)) panic("Failed to calibrate the TSC"); tsc_calibration_source = force; /* * We've saved the tsc_calibration_t that matched the value * of tsc_calibration at this point, so we can release the * memory for the value now. */ if (tsc_name_len > 0) { kmem_free(tsc_calibration, tsc_name_len + 1); } else if (tsc_calibration != NULL) { size_t len = strlen(tsc_calibration) + 1; kobj_free_string(tsc_calibration, len); } tsc_calibration = NULL; tsc_pit_also(); return (tsc_freq); } /* * While we could sort the set contents in place, we'll make a copy * of the set and avoid modifying the original set. */ tsc_set_size = SET_COUNT(tsc_calibration_set) * sizeof (tsc_calibrate_t **); tsccpp = kmem_zalloc(tsc_set_size, KM_SLEEP); bcopy(SET_BEGIN(tsc_calibration_set), tsccpp, tsc_set_size); /* * Sort by preference, highest to lowest */ qsort(tsccpp, SET_COUNT(tsc_calibration_set), sizeof (tsc_calibrate_t **), tsc_calibrate_cmp); for (uint_t i = 0; i < SET_COUNT(tsc_calibration_set); i++) { PRM_DEBUGS(tsccpp[i]->tscc_source); if (tsccpp[i]->tscc_calibrate(&tsc_freq)) { VERIFY3U(tsc_freq, >, 0); cmn_err(CE_CONT, "?TSC calibrated using %s; freq is %lu MHz\n", tsccpp[i]->tscc_source, tsc_freq / 1000000); /* * Note that tsccpp is just a (sorted) array of * pointers to the tsc_calibration_t's (from the * linker set). The actual tsc_calibration_t's aren't * kmem_alloc()ed (being part of the linker set), so * it's safe to keep a pointer to the one that was * used for calibration (intended for diagnostic * purposes). */ tsc_calibration_source = tsccpp[i]; kmem_free(tsccpp, tsc_set_size); tsc_pit_also(); return (tsc_freq); } } /* * In case it's useful, we don't free tsccpp -- we're about to panic * anyway. */ panic("Failed to calibrate TSC"); } uint64_t tsc_get_freq(void) { VERIFY(tsc_freq > 0); return (tsc_freq); } /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved. */ /* Copyright (c) 1990, 1991 UNIX System Laboratories, Inc. */ /* Copyright (c) 1984, 1986, 1987, 1988, 1989, 1990 AT&T */ /* All Rights Reserved */ /* */ /* Copyright (c) 1987, 1988 Microsoft Corporation */ /* All Rights Reserved */ /* */ /* * Copyright 2018 Joyent, Inc. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #if defined(__xpv) #include #endif #include #define USER 0x10000 /* user-mode flag added to trap type */ static const char *trap_type_mnemonic[] = { "de", "db", "2", "bp", "of", "br", "ud", "nm", "df", "9", "ts", "np", "ss", "gp", "pf", "15", "mf", "ac", "mc", "xf" }; static const char *trap_type[] = { "Divide error", /* trap id 0 */ "Debug", /* trap id 1 */ "NMI interrupt", /* trap id 2 */ "Breakpoint", /* trap id 3 */ "Overflow", /* trap id 4 */ "BOUND range exceeded", /* trap id 5 */ "Invalid opcode", /* trap id 6 */ "Device not available", /* trap id 7 */ "Double fault", /* trap id 8 */ "Coprocessor segment overrun", /* trap id 9 */ "Invalid TSS", /* trap id 10 */ "Segment not present", /* trap id 11 */ "Stack segment fault", /* trap id 12 */ "General protection", /* trap id 13 */ "Page fault", /* trap id 14 */ "Reserved", /* trap id 15 */ "x87 floating point error", /* trap id 16 */ "Alignment check", /* trap id 17 */ "Machine check", /* trap id 18 */ "SIMD floating point exception", /* trap id 19 */ }; #define TRAP_TYPES (sizeof (trap_type) / sizeof (trap_type[0])) #define SLOW_SCALL_SIZE 2 #define FAST_SCALL_SIZE 2 int tudebug = 0; int tudebugbpt = 0; int tudebugfpe = 0; int tudebugsse = 0; #if defined(TRAPDEBUG) || defined(lint) int tdebug = 0; int lodebug = 0; int faultdebug = 0; #else #define tdebug 0 #define lodebug 0 #define faultdebug 0 #endif /* defined(TRAPDEBUG) || defined(lint) */ #if defined(TRAPTRACE) /* * trap trace record for cpu0 is allocated here. * trap trace records for non-boot cpus are allocated in mp_startup_init(). */ static trap_trace_rec_t trap_tr0[TRAPTR_NENT]; trap_trace_ctl_t trap_trace_ctl[NCPU] = { { (uintptr_t)trap_tr0, /* next record */ (uintptr_t)trap_tr0, /* first record */ (uintptr_t)(trap_tr0 + TRAPTR_NENT), /* limit */ (uintptr_t)0 /* current */ }, }; /* * default trap buffer size */ size_t trap_trace_bufsize = TRAPTR_NENT * sizeof (trap_trace_rec_t); int trap_trace_freeze = 0; int trap_trace_off = 0; /* * A dummy TRAPTRACE entry to use after death. */ trap_trace_rec_t trap_trace_postmort; static void dump_ttrace(void); #endif /* TRAPTRACE */ static void dumpregs(struct regs *); static void showregs(uint_t, struct regs *, caddr_t); static int kern_gpfault(struct regs *); /*ARGSUSED*/ static int die(uint_t type, struct regs *rp, caddr_t addr, processorid_t cpuid) { struct panic_trap_info ti; const char *trap_name, *trap_mnemonic; if (type < TRAP_TYPES) { trap_name = trap_type[type]; trap_mnemonic = trap_type_mnemonic[type]; } else { trap_name = "trap"; trap_mnemonic = "-"; } #ifdef TRAPTRACE TRAPTRACE_FREEZE; #endif ti.trap_regs = rp; ti.trap_type = type & ~USER; ti.trap_addr = addr; curthread->t_panic_trap = &ti; if (type == T_PGFLT && addr < (caddr_t)kernelbase) { panic("BAD TRAP: type=%x (#%s %s) rp=%p addr=%p " "occurred in module \"%s\" due to %s", type, trap_mnemonic, trap_name, (void *)rp, (void *)addr, mod_containing_pc((caddr_t)rp->r_pc), addr < (caddr_t)PAGESIZE ? "a NULL pointer dereference" : "an illegal access to a user address"); } else panic("BAD TRAP: type=%x (#%s %s) rp=%p addr=%p", type, trap_mnemonic, trap_name, (void *)rp, (void *)addr); return (0); } /* * Rewrite the instruction at pc to be an int $T_SYSCALLINT instruction. * * int is two bytes: 0xCD */ static int rewrite_syscall(caddr_t pc) { uchar_t instr[SLOW_SCALL_SIZE] = { 0xCD, T_SYSCALLINT }; if (uwrite(curthread->t_procp, instr, SLOW_SCALL_SIZE, (uintptr_t)pc) != 0) return (1); return (0); } /* * Test to see if the instruction at pc is sysenter or syscall. The second * argument should be the x86 feature flag corresponding to the expected * instruction. * * sysenter is two bytes: 0x0F 0x34 * syscall is two bytes: 0x0F 0x05 * int $T_SYSCALLINT is two bytes: 0xCD 0x91 */ static int instr_is_other_syscall(caddr_t pc, int which) { uchar_t instr[FAST_SCALL_SIZE]; ASSERT(which == X86FSET_SEP || which == X86FSET_ASYSC || which == 0xCD); if (copyin_nowatch(pc, (caddr_t)instr, FAST_SCALL_SIZE) != 0) return (0); switch (which) { case X86FSET_SEP: if (instr[0] == 0x0F && instr[1] == 0x34) return (1); break; case X86FSET_ASYSC: if (instr[0] == 0x0F && instr[1] == 0x05) return (1); break; case 0xCD: if (instr[0] == 0xCD && instr[1] == T_SYSCALLINT) return (1); break; } return (0); } #ifdef DEBUG static const char * syscall_insn_string(int syscall_insn) { switch (syscall_insn) { case X86FSET_SEP: return ("sysenter"); case X86FSET_ASYSC: return ("syscall"); case 0xCD: return ("int"); default: return ("Unknown"); } } #endif /* DEBUG */ static int ldt_rewrite_syscall(struct regs *rp, proc_t *p, int syscall_insn) { caddr_t linearpc; int return_code = 0; mutex_enter(&p->p_ldtlock); /* Must be held across linear_pc() */ if (linear_pc(rp, p, &linearpc) == 0) { /* * If another thread beat us here, it already changed * this site to the slower (int) syscall instruction. */ if (instr_is_other_syscall(linearpc, 0xCD)) { return_code = 1; } else if (instr_is_other_syscall(linearpc, syscall_insn)) { if (rewrite_syscall(linearpc) == 0) { return_code = 1; } #ifdef DEBUG else cmn_err(CE_WARN, "failed to rewrite %s " "instruction in process %d", syscall_insn_string(syscall_insn), p->p_pid); #endif /* DEBUG */ } } mutex_exit(&p->p_ldtlock); /* Must be held across linear_pc() */ return (return_code); } /* * Test to see if the instruction at pc is a system call instruction. * * The bytes of an lcall instruction used for the syscall trap. * static uchar_t lcall[7] = { 0x9a, 0, 0, 0, 0, 0x7, 0 }; * static uchar_t lcallalt[7] = { 0x9a, 0, 0, 0, 0, 0x27, 0 }; */ #define LCALLSIZE 7 static int instr_is_lcall_syscall(caddr_t pc) { uchar_t instr[LCALLSIZE]; if (copyin_nowatch(pc, (caddr_t)instr, LCALLSIZE) == 0 && instr[0] == 0x9a && instr[1] == 0 && instr[2] == 0 && instr[3] == 0 && instr[4] == 0 && (instr[5] == 0x7 || instr[5] == 0x27) && instr[6] == 0) return (1); return (0); } /* * In the first revisions of amd64 CPUs produced by AMD, the LAHF and * SAHF instructions were not implemented in 64-bit mode. Later revisions * did implement these instructions. An extension to the cpuid instruction * was added to check for the capability of executing these instructions * in 64-bit mode. * * Intel originally did not implement these instructions in EM64T either, * but added them in later revisions. * * So, there are different chip revisions by both vendors out there that * may or may not implement these instructions. The easy solution is to * just always emulate these instructions on demand. * * SAHF == store %ah in the lower 8 bits of %rflags (opcode 0x9e) * LAHF == load the lower 8 bits of %rflags into %ah (opcode 0x9f) */ #define LSAHFSIZE 1 static int instr_is_lsahf(caddr_t pc, uchar_t *instr) { if (copyin_nowatch(pc, (caddr_t)instr, LSAHFSIZE) == 0 && (*instr == 0x9e || *instr == 0x9f)) return (1); return (0); } /* * Emulate the LAHF and SAHF instructions. The reference manuals define * these instructions to always load/store bit 1 as a 1, and bits 3 and 5 * as a 0. The other, defined, bits are copied (the PS_ICC bits and PS_P). * * Note that %ah is bits 8-15 of %rax. */ static void emulate_lsahf(struct regs *rp, uchar_t instr) { if (instr == 0x9e) { /* sahf. Copy bits from %ah to flags. */ rp->r_ps = (rp->r_ps & ~0xff) | ((rp->r_rax >> 8) & PSL_LSAHFMASK) | PS_MB1; } else { /* lahf. Copy bits from flags to %ah. */ rp->r_rax = (rp->r_rax & ~0xff00) | (((rp->r_ps & PSL_LSAHFMASK) | PS_MB1) << 8); } rp->r_pc += LSAHFSIZE; } #ifdef OPTERON_ERRATUM_91 /* * Test to see if the instruction at pc is a prefetch instruction. * * The first byte of prefetch instructions is always 0x0F. * The second byte is 0x18 for regular prefetch or 0x0D for AMD 3dnow prefetch. * The third byte (ModRM) contains the register field bits (bits 3-5). * These bits must be between 0 and 3 inclusive for regular prefetch and * 0 and 1 inclusive for AMD 3dnow prefetch. * * In 64-bit mode, there may be a one-byte REX prefex (0x40-0x4F). */ static int cmp_to_prefetch(uchar_t *p) { #ifdef _LP64 if ((p[0] & 0xF0) == 0x40) /* 64-bit REX prefix */ p++; #endif return ((p[0] == 0x0F && p[1] == 0x18 && ((p[2] >> 3) & 7) <= 3) || (p[0] == 0x0F && p[1] == 0x0D && ((p[2] >> 3) & 7) <= 1)); } static int instr_is_prefetch(caddr_t pc) { uchar_t instr[4]; /* optional REX prefix plus 3-byte opcode */ return (copyin_nowatch(pc, instr, sizeof (instr)) == 0 && cmp_to_prefetch(instr)); } #endif /* OPTERON_ERRATUM_91 */ /* * Called from the trap handler when a processor trap occurs. * * Note: All user-level traps that might call stop() must exit * trap() by 'goto out' or by falling through. * Note Also: trap() is usually called with interrupts enabled, (PS_IE == 1) * however, there are paths that arrive here with PS_IE == 0 so special care * must be taken in those cases. */ void trap(struct regs *rp, caddr_t addr, processorid_t cpuid) { kthread_t *ct = curthread; enum seg_rw rw; unsigned type; proc_t *p = ttoproc(ct); klwp_t *lwp = ttolwp(ct); uintptr_t lofault; label_t *onfault; faultcode_t pagefault(), res, errcode; enum fault_type fault_type; k_siginfo_t siginfo; uint_t fault = 0; int mstate; int sicode = 0; int watchcode; int watchpage; caddr_t vaddr; size_t sz; int ta; uchar_t instr; ASSERT_STACK_ALIGNED(); errcode = 0; mstate = 0; rw = S_OTHER; type = rp->r_trapno; CPU_STATS_ADDQ(CPU, sys, trap, 1); ASSERT(ct->t_schedflag & TS_DONT_SWAP); if (type == T_PGFLT) { errcode = rp->r_err; if (errcode & PF_ERR_WRITE) { rw = S_WRITE; } else if ((caddr_t)rp->r_pc == addr || (mmu.pt_nx != 0 && (errcode & PF_ERR_EXEC))) { rw = S_EXEC; } else { rw = S_READ; } } else if (type == T_SGLSTP && lwp != NULL) { lwp->lwp_pcb.pcb_drstat = (uintptr_t)addr; } if (tdebug) showregs(type, rp, addr); if (USERMODE(rp->r_cs)) { /* * Set up the current cred to use during this trap. u_cred * no longer exists. t_cred is used instead. * The current process credential applies to the thread for * the entire trap. If trapping from the kernel, this * should already be set up. */ if (ct->t_cred != p->p_cred) { cred_t *oldcred = ct->t_cred; /* * DTrace accesses t_cred in probe context. t_cred * must always be either NULL, or point to a valid, * allocated cred structure. */ ct->t_cred = crgetcred(); crfree(oldcred); } ASSERT(lwp != NULL); type |= USER; ASSERT(lwptoregs(lwp) == rp); lwp->lwp_state = LWP_SYS; switch (type) { case T_PGFLT + USER: if ((caddr_t)rp->r_pc == addr) mstate = LMS_TFAULT; else mstate = LMS_DFAULT; break; default: mstate = LMS_TRAP; break; } mstate = new_mstate(ct, mstate); bzero(&siginfo, sizeof (siginfo)); } switch (type) { case T_PGFLT + USER: case T_SGLSTP: case T_SGLSTP + USER: case T_BPTFLT + USER: break; default: FTRACE_2("trap(): type=0x%lx, regs=0x%lx", (ulong_t)type, (ulong_t)rp); break; } switch (type) { case T_SIMDFPE: /* Make sure we enable interrupts before die()ing */ sti(); /* The SIMD exception comes in via cmninttrap */ /*FALLTHROUGH*/ default: if (type & USER) { if (tudebug) showregs(type, rp, (caddr_t)0); printf("trap: Unknown trap type %d in user mode\n", type & ~USER); siginfo.si_signo = SIGILL; siginfo.si_code = ILL_ILLTRP; siginfo.si_addr = (caddr_t)rp->r_pc; siginfo.si_trapno = type & ~USER; fault = FLTILL; } else { (void) die(type, rp, addr, cpuid); /*NOTREACHED*/ } break; case T_PGFLT: /* system page fault */ /* * If we're under on_trap() protection (see ), * set ot_trap and bounce back to the on_trap() call site * via the installed trampoline. */ if ((ct->t_ontrap != NULL) && (ct->t_ontrap->ot_prot & OT_DATA_ACCESS)) { ct->t_ontrap->ot_trap |= OT_DATA_ACCESS; rp->r_pc = ct->t_ontrap->ot_trampoline; goto cleanup; } /* * If we have an Instruction fault in kernel mode, then that * means we've tried to execute a user page (SMEP) or both of * PAE and NXE are enabled. In either case, given that it's a * kernel fault, we should panic immediately and not try to make * any more forward progress. This indicates a bug in the * kernel, which if execution continued, could be exploited to * wreak havoc on the system. */ if (errcode & PF_ERR_EXEC) { (void) die(type, rp, addr, cpuid); } /* * We need to check if SMAP is in play. If SMAP is in play, then * any access to a user page will show up as a protection * violation. To see if SMAP is enabled we first check if it's a * user address and whether we have the feature flag set. If we * do and the interrupted registers do not allow for user * accesses (PS_ACHK is not enabled), then we need to die * immediately. */ if (addr < (caddr_t)kernelbase && is_x86_feature(x86_featureset, X86FSET_SMAP) == B_TRUE && (rp->r_ps & PS_ACHK) == 0) { (void) die(type, rp, addr, cpuid); } /* * See if we can handle as pagefault. Save lofault and onfault * across this. Here we assume that an address less than * KERNELBASE is a user fault. We can do this as copy.s * routines verify that the starting address is less than * KERNELBASE before starting and because we know that we * always have KERNELBASE mapped as invalid to serve as a * "barrier". */ lofault = ct->t_lofault; onfault = ct->t_onfault; ct->t_lofault = 0; mstate = new_mstate(ct, LMS_KFAULT); if (addr < (caddr_t)kernelbase) { res = pagefault(addr, (errcode & PF_ERR_PROT)? F_PROT: F_INVAL, rw, 0); if (res == FC_NOMAP && addr < p->p_usrstack && grow(addr)) res = 0; } else { res = pagefault(addr, (errcode & PF_ERR_PROT)? F_PROT: F_INVAL, rw, 1); } (void) new_mstate(ct, mstate); /* * Restore lofault and onfault. If we resolved the fault, exit. * If we didn't and lofault wasn't set, die. */ ct->t_lofault = lofault; ct->t_onfault = onfault; if (res == 0) goto cleanup; #if defined(OPTERON_ERRATUM_93) && defined(_LP64) if (lofault == 0 && opteron_erratum_93) { /* * Workaround for Opteron Erratum 93. On return from * a System Managment Interrupt at a HLT instruction * the %rip might be truncated to a 32 bit value. * BIOS is supposed to fix this, but some don't. * If this occurs we simply restore the high order bits. * The HLT instruction is 1 byte of 0xf4. */ uintptr_t rip = rp->r_pc; if ((rip & 0xfffffffful) == rip) { rip |= 0xfffffffful << 32; if (hat_getpfnum(kas.a_hat, (caddr_t)rip) != PFN_INVALID && (*(uchar_t *)rip == 0xf4 || *(uchar_t *)(rip - 1) == 0xf4)) { rp->r_pc = rip; goto cleanup; } } } #endif /* OPTERON_ERRATUM_93 && _LP64 */ #ifdef OPTERON_ERRATUM_91 if (lofault == 0 && opteron_erratum_91) { /* * Workaround for Opteron Erratum 91. Prefetches may * generate a page fault (they're not supposed to do * that!). If this occurs we simply return back to the * instruction. */ caddr_t pc = (caddr_t)rp->r_pc; /* * If the faulting PC is not mapped, this is a * legitimate kernel page fault that must result in a * panic. If the faulting PC is mapped, it could contain * a prefetch instruction. Check for that here. */ if (hat_getpfnum(kas.a_hat, pc) != PFN_INVALID) { if (cmp_to_prefetch((uchar_t *)pc)) { #ifdef DEBUG cmn_err(CE_WARN, "Opteron erratum 91 " "occurred: kernel prefetch" " at %p generated a page fault!", (void *)rp->r_pc); #endif /* DEBUG */ goto cleanup; } } (void) die(type, rp, addr, cpuid); } #endif /* OPTERON_ERRATUM_91 */ if (lofault == 0) (void) die(type, rp, addr, cpuid); /* * Cannot resolve fault. Return to lofault. */ if (lodebug) { showregs(type, rp, addr); traceregs(rp); } if (FC_CODE(res) == FC_OBJERR) res = FC_ERRNO(res); else res = EFAULT; rp->r_r0 = res; rp->r_pc = ct->t_lofault; goto cleanup; case T_PGFLT + USER: /* user page fault */ if (faultdebug) { char *fault_str; switch (rw) { case S_READ: fault_str = "read"; break; case S_WRITE: fault_str = "write"; break; case S_EXEC: fault_str = "exec"; break; default: fault_str = ""; break; } printf("user %s fault: addr=0x%lx errcode=0x%x\n", fault_str, (uintptr_t)addr, errcode); } #if defined(OPTERON_ERRATUM_100) && defined(_LP64) /* * Workaround for AMD erratum 100 * * A 32-bit process may receive a page fault on a non * 32-bit address by mistake. The range of the faulting * address will be * * 0xffffffff80000000 .. 0xffffffffffffffff or * 0x0000000100000000 .. 0x000000017fffffff * * The fault is always due to an instruction fetch, however * the value of r_pc should be correct (in 32 bit range), * so we ignore the page fault on the bogus address. */ if (p->p_model == DATAMODEL_ILP32 && (0xffffffff80000000 <= (uintptr_t)addr || (0x100000000 <= (uintptr_t)addr && (uintptr_t)addr <= 0x17fffffff))) { if (!opteron_erratum_100) panic("unexpected erratum #100"); if (rp->r_pc <= 0xffffffff) goto out; } #endif /* OPTERON_ERRATUM_100 && _LP64 */ ASSERT(!(curthread->t_flag & T_WATCHPT)); watchpage = (pr_watch_active(p) && pr_is_watchpage(addr, rw)); vaddr = addr; if (!watchpage || (sz = instr_size(rp, &vaddr, rw)) <= 0) fault_type = (errcode & PF_ERR_PROT)? F_PROT: F_INVAL; else if ((watchcode = pr_is_watchpoint(&vaddr, &ta, sz, NULL, rw)) != 0) { if (ta) { do_watch_step(vaddr, sz, rw, watchcode, rp->r_pc); fault_type = F_INVAL; } else { bzero(&siginfo, sizeof (siginfo)); siginfo.si_signo = SIGTRAP; siginfo.si_code = watchcode; siginfo.si_addr = vaddr; siginfo.si_trapafter = 0; siginfo.si_pc = (caddr_t)rp->r_pc; fault = FLTWATCH; break; } } else { /* XXX pr_watch_emul() never succeeds (for now) */ if (rw != S_EXEC && pr_watch_emul(rp, vaddr, rw)) goto out; do_watch_step(vaddr, sz, rw, 0, 0); fault_type = F_INVAL; } res = pagefault(addr, fault_type, rw, 0); /* * If pagefault() succeeded, ok. * Otherwise attempt to grow the stack. */ if (res == 0 || (res == FC_NOMAP && addr < p->p_usrstack && grow(addr))) { lwp->lwp_lastfault = FLTPAGE; lwp->lwp_lastfaddr = addr; if (prismember(&p->p_fltmask, FLTPAGE)) { bzero(&siginfo, sizeof (siginfo)); siginfo.si_addr = addr; (void) stop_on_fault(FLTPAGE, &siginfo); } goto out; } else if (res == FC_PROT && addr < p->p_usrstack && (mmu.pt_nx != 0 && (errcode & PF_ERR_EXEC))) { report_stack_exec(p, addr); } #ifdef OPTERON_ERRATUM_91 /* * Workaround for Opteron Erratum 91. Prefetches may generate a * page fault (they're not supposed to do that!). If this * occurs we simply return back to the instruction. * * We rely on copyin to properly fault in the page with r_pc. */ if (opteron_erratum_91 && addr != (caddr_t)rp->r_pc && instr_is_prefetch((caddr_t)rp->r_pc)) { #ifdef DEBUG cmn_err(CE_WARN, "Opteron erratum 91 occurred: " "prefetch at %p in pid %d generated a trap!", (void *)rp->r_pc, p->p_pid); #endif /* DEBUG */ goto out; } #endif /* OPTERON_ERRATUM_91 */ if (tudebug) showregs(type, rp, addr); /* * In the case where both pagefault and grow fail, * set the code to the value provided by pagefault. * We map all errors returned from pagefault() to SIGSEGV. */ bzero(&siginfo, sizeof (siginfo)); siginfo.si_addr = addr; switch (FC_CODE(res)) { case FC_HWERR: case FC_NOSUPPORT: siginfo.si_signo = SIGBUS; siginfo.si_code = BUS_ADRERR; fault = FLTACCESS; break; case FC_ALIGN: siginfo.si_signo = SIGBUS; siginfo.si_code = BUS_ADRALN; fault = FLTACCESS; break; case FC_OBJERR: if ((siginfo.si_errno = FC_ERRNO(res)) != EINTR) { siginfo.si_signo = SIGBUS; siginfo.si_code = BUS_OBJERR; fault = FLTACCESS; } break; default: /* FC_NOMAP or FC_PROT */ siginfo.si_signo = SIGSEGV; siginfo.si_code = (res == FC_NOMAP)? SEGV_MAPERR : SEGV_ACCERR; fault = FLTBOUNDS; break; } break; case T_ILLINST + USER: /* invalid opcode fault */ /* * If the syscall instruction is disabled due to LDT usage, a * user program that attempts to execute it will trigger a #ud * trap. Check for that case here. If this occurs on a CPU which * doesn't even support syscall, the result of all of this will * be to emulate that particular instruction. */ if (p->p_ldt != NULL && ldt_rewrite_syscall(rp, p, X86FSET_ASYSC)) goto out; /* * Emulate the LAHF and SAHF instructions if needed. * See the instr_is_lsahf function for details. */ if (p->p_model == DATAMODEL_LP64 && instr_is_lsahf((caddr_t)rp->r_pc, &instr)) { emulate_lsahf(rp, instr); goto out; } /*FALLTHROUGH*/ if (tudebug) showregs(type, rp, (caddr_t)0); siginfo.si_signo = SIGILL; siginfo.si_code = ILL_ILLOPC; siginfo.si_addr = (caddr_t)rp->r_pc; fault = FLTILL; break; case T_ZERODIV + USER: /* integer divide by zero */ if (tudebug && tudebugfpe) showregs(type, rp, (caddr_t)0); siginfo.si_signo = SIGFPE; siginfo.si_code = FPE_INTDIV; siginfo.si_addr = (caddr_t)rp->r_pc; fault = FLTIZDIV; break; case T_OVFLW + USER: /* integer overflow */ if (tudebug && tudebugfpe) showregs(type, rp, (caddr_t)0); siginfo.si_signo = SIGFPE; siginfo.si_code = FPE_INTOVF; siginfo.si_addr = (caddr_t)rp->r_pc; fault = FLTIOVF; break; /* * When using an eager FPU on x86, the #NM trap is no longer meaningful. * Userland should not be able to trigger it. Anything that does * represents a fatal error in the kernel and likely in the register * state of the system. User FPU state should always be valid. */ case T_NOEXTFLT + USER: /* math coprocessor not available */ case T_NOEXTFLT: (void) die(type, rp, addr, cpuid); break; /* * Kernel threads leveraging floating point need to mask the exceptions * or ensure that they cannot happen. There is no recovery from this. */ case T_EXTERRFLT: /* x87 floating point exception pending */ sti(); /* T_EXTERRFLT comes in via cmninttrap */ (void) die(type, rp, addr, cpuid); break; case T_EXTERRFLT + USER: /* x87 floating point exception pending */ if (tudebug && tudebugfpe) showregs(type, rp, addr); if ((sicode = fpexterrflt(rp)) != 0) { siginfo.si_signo = SIGFPE; siginfo.si_code = sicode; siginfo.si_addr = (caddr_t)rp->r_pc; fault = FLTFPE; } break; case T_SIMDFPE + USER: /* SSE and SSE2 exceptions */ if (tudebug && tudebugsse) showregs(type, rp, addr); if (!is_x86_feature(x86_featureset, X86FSET_SSE) && !is_x86_feature(x86_featureset, X86FSET_SSE2)) { /* * There are rumours that some user instructions * on older CPUs can cause this trap to occur; in * which case send a SIGILL instead of a SIGFPE. */ siginfo.si_signo = SIGILL; siginfo.si_code = ILL_ILLTRP; siginfo.si_addr = (caddr_t)rp->r_pc; siginfo.si_trapno = type & ~USER; fault = FLTILL; } else if ((sicode = fpsimderrflt(rp)) != 0) { siginfo.si_signo = SIGFPE; siginfo.si_code = sicode; siginfo.si_addr = (caddr_t)rp->r_pc; fault = FLTFPE; } sti(); /* The SIMD exception comes in via cmninttrap */ break; case T_BPTFLT: /* breakpoint trap */ /* * Kernel breakpoint traps should only happen when kmdb is * active, and even then, it'll have interposed on the IDT, so * control won't get here. If it does, we've hit a breakpoint * without the debugger, which is very strange, and very * fatal. */ if (tudebug && tudebugbpt) showregs(type, rp, (caddr_t)0); (void) die(type, rp, addr, cpuid); break; case T_SGLSTP: /* single step/hw breakpoint exception */ #if !defined(__xpv) /* * We'd never normally get here, as kmdb handles its own single * step traps. There is one nasty exception though, as * described in more detail in sys_sysenter(). Note that * checking for all four locations covers both the KPTI and the * non-KPTI cases correctly: the former will never be found at * (brand_)sys_sysenter, and vice versa. */ if (lwp != NULL && (lwp->lwp_pcb.pcb_drstat & DR_SINGLESTEP)) { if (rp->r_pc == (greg_t)brand_sys_sysenter || rp->r_pc == (greg_t)sys_sysenter || rp->r_pc == (greg_t)tr_brand_sys_sysenter || rp->r_pc == (greg_t)tr_sys_sysenter) { rp->r_pc += 0x3; /* sizeof (swapgs) */ rp->r_ps &= ~PS_T; /* turn off trace */ lwp->lwp_pcb.pcb_flags |= DEBUG_PENDING; ct->t_post_sys = 1; aston(curthread); goto cleanup; } else { if (tudebug && tudebugbpt) showregs(type, rp, (caddr_t)0); } } #endif /* !__xpv */ if (boothowto & RB_DEBUG) debug_enter((char *)NULL); else (void) die(type, rp, addr, cpuid); break; case T_NMIFLT: /* NMI interrupt */ printf("Unexpected NMI in system mode\n"); goto cleanup; case T_NMIFLT + USER: /* NMI interrupt */ printf("Unexpected NMI in user mode\n"); break; case T_GPFLT: /* general protection violation */ /* * Any #GP that occurs during an on_trap .. no_trap bracket * with OT_DATA_ACCESS or OT_SEGMENT_ACCESS protection, * or in a on_fault .. no_fault bracket, is forgiven * and we trampoline. This protection is given regardless * of whether we are 32/64 bit etc - if a distinction is * required then define new on_trap protection types. * * On amd64, we can get a #gp from referencing addresses * in the virtual address hole e.g. from a copyin or in * update_sregs while updating user segment registers. * * On the 32-bit hypervisor we could also generate one in * mfn_to_pfn by reaching around or into where the hypervisor * lives which is protected by segmentation. */ /* * If we're under on_trap() protection (see ), * set ot_trap and trampoline back to the on_trap() call site * for OT_DATA_ACCESS or OT_SEGMENT_ACCESS. */ if (ct->t_ontrap != NULL) { int ttype = ct->t_ontrap->ot_prot & (OT_DATA_ACCESS | OT_SEGMENT_ACCESS); if (ttype != 0) { ct->t_ontrap->ot_trap |= ttype; if (tudebug) showregs(type, rp, (caddr_t)0); rp->r_pc = ct->t_ontrap->ot_trampoline; goto cleanup; } } /* * If we're under lofault protection (copyin etc.), * longjmp back to lofault with an EFAULT. */ if (ct->t_lofault) { /* * Fault is not resolvable, so just return to lofault */ if (lodebug) { showregs(type, rp, addr); traceregs(rp); } rp->r_r0 = EFAULT; rp->r_pc = ct->t_lofault; goto cleanup; } /* * We fall through to the next case, which repeats * the OT_SEGMENT_ACCESS check which we've already * done, so we'll always fall through to the * T_STKFLT case. */ /*FALLTHROUGH*/ case T_SEGFLT: /* segment not present fault */ /* * One example of this is #NP in update_sregs while * attempting to update a user segment register * that points to a descriptor that is marked not * present. */ if (ct->t_ontrap != NULL && ct->t_ontrap->ot_prot & OT_SEGMENT_ACCESS) { ct->t_ontrap->ot_trap |= OT_SEGMENT_ACCESS; if (tudebug) showregs(type, rp, (caddr_t)0); rp->r_pc = ct->t_ontrap->ot_trampoline; goto cleanup; } /*FALLTHROUGH*/ case T_STKFLT: /* stack fault */ case T_TSSFLT: /* invalid TSS fault */ if (tudebug) showregs(type, rp, (caddr_t)0); if (kern_gpfault(rp)) (void) die(type, rp, addr, cpuid); goto cleanup; /* * ONLY 32-bit PROCESSES can USE a PRIVATE LDT! 64-bit apps * should have no need for them, so we put a stop to it here. * * So: not-present fault is ONLY valid for 32-bit processes with * a private LDT trying to do a system call. Emulate it. * * #gp fault is ONLY valid for 32-bit processes also, which DO NOT * have a private LDT, and are trying to do a system call. Emulate it. */ case T_SEGFLT + USER: /* segment not present fault */ case T_GPFLT + USER: /* general protection violation */ #ifdef _SYSCALL32_IMPL if (p->p_model != DATAMODEL_NATIVE) { #endif /* _SYSCALL32_IMPL */ if (instr_is_lcall_syscall((caddr_t)rp->r_pc)) { if (type == T_SEGFLT + USER) ASSERT(p->p_ldt != NULL); if ((p->p_ldt == NULL && type == T_GPFLT + USER) || type == T_SEGFLT + USER) { /* * The user attempted a system call via the obsolete * call gate mechanism. Because the process doesn't have * an LDT (i.e. the ldtr contains 0), a #gp results. * Emulate the syscall here, just as we do above for a * #np trap. */ /* * Since this is a not-present trap, rp->r_pc points to * the trapping lcall instruction. We need to bump it * to the next insn so the app can continue on. */ rp->r_pc += LCALLSIZE; lwp->lwp_regs = rp; /* * Normally the microstate of the LWP is forced back to * LMS_USER by the syscall handlers. Emulate that * behavior here. */ mstate = LMS_USER; dosyscall(); goto out; } } #ifdef _SYSCALL32_IMPL } #endif /* _SYSCALL32_IMPL */ /* * If the current process is using a private LDT and the * trapping instruction is sysenter, the sysenter instruction * has been disabled on the CPU because it destroys segment * registers. If this is the case, rewrite the instruction to * be a safe system call and retry it. If this occurs on a CPU * which doesn't even support sysenter, the result of all of * this will be to emulate that particular instruction. */ if (p->p_ldt != NULL && ldt_rewrite_syscall(rp, p, X86FSET_SEP)) goto out; /*FALLTHROUGH*/ case T_BOUNDFLT + USER: /* bound fault */ case T_STKFLT + USER: /* stack fault */ case T_TSSFLT + USER: /* invalid TSS fault */ if (tudebug) showregs(type, rp, (caddr_t)0); siginfo.si_signo = SIGSEGV; siginfo.si_code = SEGV_MAPERR; siginfo.si_addr = (caddr_t)rp->r_pc; fault = FLTBOUNDS; break; case T_ALIGNMENT + USER: /* user alignment error (486) */ if (tudebug) showregs(type, rp, (caddr_t)0); bzero(&siginfo, sizeof (siginfo)); siginfo.si_signo = SIGBUS; siginfo.si_code = BUS_ADRALN; siginfo.si_addr = (caddr_t)rp->r_pc; fault = FLTACCESS; break; case T_SGLSTP + USER: /* single step/hw breakpoint exception */ if (tudebug && tudebugbpt) showregs(type, rp, (caddr_t)0); /* Was it single-stepping? */ if (lwp->lwp_pcb.pcb_drstat & DR_SINGLESTEP) { pcb_t *pcb = &lwp->lwp_pcb; rp->r_ps &= ~PS_T; /* * If both NORMAL_STEP and WATCH_STEP are in effect, * give precedence to WATCH_STEP. If neither is set, * user must have set the PS_T bit in %efl; treat this * as NORMAL_STEP. */ if ((fault = undo_watch_step(&siginfo)) == 0 && ((pcb->pcb_flags & NORMAL_STEP) || !(pcb->pcb_flags & WATCH_STEP))) { siginfo.si_signo = SIGTRAP; siginfo.si_code = TRAP_TRACE; siginfo.si_addr = (caddr_t)rp->r_pc; fault = FLTTRACE; } pcb->pcb_flags &= ~(NORMAL_STEP|WATCH_STEP); } break; case T_BPTFLT + USER: /* breakpoint trap */ if (tudebug && tudebugbpt) showregs(type, rp, (caddr_t)0); /* * int 3 (the breakpoint instruction) leaves the pc referring * to the address one byte after the breakpointed address. * If the P_PR_BPTADJ flag has been set via /proc, We adjust * it back so it refers to the breakpointed address. */ if (p->p_proc_flag & P_PR_BPTADJ) rp->r_pc--; siginfo.si_signo = SIGTRAP; siginfo.si_code = TRAP_BRKPT; siginfo.si_addr = (caddr_t)rp->r_pc; fault = FLTBPT; break; case T_AST: /* * This occurs only after the cs register has been made to * look like a kernel selector, either through debugging or * possibly by functions like setcontext(). The thread is * about to cause a general protection fault at common_iret() * in locore. We let that happen immediately instead of * doing the T_AST processing. */ goto cleanup; case T_AST + USER: /* profiling, resched, h/w error pseudo trap */ if (lwp->lwp_pcb.pcb_flags & ASYNC_HWERR) { proc_t *p = ttoproc(curthread); extern void print_msg_hwerr(ctid_t ct_id, proc_t *p); lwp->lwp_pcb.pcb_flags &= ~ASYNC_HWERR; print_msg_hwerr(p->p_ct_process->conp_contract.ct_id, p); contract_process_hwerr(p->p_ct_process, p); siginfo.si_signo = SIGKILL; siginfo.si_code = SI_NOINFO; } else if (lwp->lwp_pcb.pcb_flags & CPC_OVERFLOW) { lwp->lwp_pcb.pcb_flags &= ~CPC_OVERFLOW; if (kcpc_overflow_ast()) { /* * Signal performance counter overflow */ if (tudebug) showregs(type, rp, (caddr_t)0); bzero(&siginfo, sizeof (siginfo)); siginfo.si_signo = SIGEMT; siginfo.si_code = EMT_CPCOVF; siginfo.si_addr = (caddr_t)rp->r_pc; fault = FLTCPCOVF; } } break; } /* * We can't get here from a system trap */ ASSERT(type & USER); if (fault) { /* We took a fault so abort single step. */ lwp->lwp_pcb.pcb_flags &= ~(NORMAL_STEP|WATCH_STEP); /* * Remember the fault and fault adddress * for real-time (SIGPROF) profiling. */ lwp->lwp_lastfault = fault; lwp->lwp_lastfaddr = siginfo.si_addr; DTRACE_PROC2(fault, int, fault, ksiginfo_t *, &siginfo); /* * If a debugger has declared this fault to be an * event of interest, stop the lwp. Otherwise just * deliver the associated signal. */ if (siginfo.si_signo != SIGKILL && prismember(&p->p_fltmask, fault) && stop_on_fault(fault, &siginfo) == 0) siginfo.si_signo = 0; } if (siginfo.si_signo) trapsig(&siginfo, (fault != FLTFPE && fault != FLTCPCOVF)); if (lwp->lwp_oweupc) profil_tick(rp->r_pc); if (ct->t_astflag | ct->t_sig_check) { /* * Turn off the AST flag before checking all the conditions that * may have caused an AST. This flag is on whenever a signal or * unusual condition should be handled after the next trap or * syscall. */ astoff(ct); /* * If a single-step trap occurred on a syscall (see above) * recognize it now. Do this before checking for signals * because deferred_singlestep_trap() may generate a SIGTRAP to * the LWP or may otherwise mark the LWP to call issig(FORREAL). */ if (lwp->lwp_pcb.pcb_flags & DEBUG_PENDING) deferred_singlestep_trap((caddr_t)rp->r_pc); ct->t_sig_check = 0; /* * As in other code paths that check against TP_CHANGEBIND, * we perform the check first without p_lock held -- only * acquiring p_lock in the unlikely event that it is indeed * set. This is safe because we are doing this after the * astoff(); if we are racing another thread setting * TP_CHANGEBIND on us, we will pick it up on a subsequent * lap through. */ if (curthread->t_proc_flag & TP_CHANGEBIND) { mutex_enter(&p->p_lock); if (curthread->t_proc_flag & TP_CHANGEBIND) { timer_lwpbind(); curthread->t_proc_flag &= ~TP_CHANGEBIND; } mutex_exit(&p->p_lock); } /* * for kaio requests that are on the per-process poll queue, * aiop->aio_pollq, they're AIO_POLL bit is set, the kernel * should copyout their result_t to user memory. by copying * out the result_t, the user can poll on memory waiting * for the kaio request to complete. */ if (p->p_aio) aio_cleanup(0); /* * If this LWP was asked to hold, call holdlwp(), which will * stop. holdlwps() sets this up and calls pokelwps() which * sets the AST flag. * * Also check TP_EXITLWP, since this is used by fresh new LWPs * through lwp_rtt(). That flag is set if the lwp_create(2) * syscall failed after creating the LWP. */ if (ISHOLD(p)) holdlwp(); /* * All code that sets signals and makes ISSIG evaluate true must * set t_astflag afterwards. */ if (ISSIG_PENDING(ct, lwp, p)) { if (issig(FORREAL)) psig(); ct->t_sig_check = 1; } if (ct->t_rprof != NULL) { realsigprof(0, 0, 0); ct->t_sig_check = 1; } /* * /proc can't enable/disable the trace bit itself * because that could race with the call gate used by * system calls via "lcall". If that happened, an * invalid EFLAGS would result. prstep()/prnostep() * therefore schedule an AST for the purpose. */ if (lwp->lwp_pcb.pcb_flags & REQUEST_STEP) { lwp->lwp_pcb.pcb_flags &= ~REQUEST_STEP; rp->r_ps |= PS_T; } if (lwp->lwp_pcb.pcb_flags & REQUEST_NOSTEP) { lwp->lwp_pcb.pcb_flags &= ~REQUEST_NOSTEP; rp->r_ps &= ~PS_T; } } out: /* We can't get here from a system trap */ ASSERT(type & USER); if (ISHOLD(p)) holdlwp(); /* * Set state to LWP_USER here so preempt won't give us a kernel * priority if it occurs after this point. Call CL_TRAPRET() to * restore the user-level priority. * * It is important that no locks (other than spinlocks) be entered * after this point before returning to user mode (unless lwp_state * is set back to LWP_SYS). */ lwp->lwp_state = LWP_USER; if (ct->t_trapret) { ct->t_trapret = 0; thread_lock(ct); CL_TRAPRET(ct); thread_unlock(ct); } if (CPU->cpu_runrun || curthread->t_schedflag & TS_ANYWAITQ) preempt(); prunstop(); (void) new_mstate(ct, mstate); return; cleanup: /* system traps end up here */ ASSERT(!(type & USER)); } /* * Patch non-zero to disable preemption of threads in the kernel. */ int IGNORE_KERNEL_PREEMPTION = 0; /* XXX - delete this someday */ struct kpreempt_cnts { /* kernel preemption statistics */ int kpc_idle; /* executing idle thread */ int kpc_intr; /* executing interrupt thread */ int kpc_clock; /* executing clock thread */ int kpc_blocked; /* thread has blocked preemption (t_preempt) */ int kpc_notonproc; /* thread is surrendering processor */ int kpc_inswtch; /* thread has ratified scheduling decision */ int kpc_prilevel; /* processor interrupt level is too high */ int kpc_apreempt; /* asynchronous preemption */ int kpc_spreempt; /* synchronous preemption */ } kpreempt_cnts; /* * kernel preemption: forced rescheduling, preempt the running kernel thread. * the argument is old PIL for an interrupt, * or the distingished value KPREEMPT_SYNC. */ void kpreempt(int asyncspl) { kthread_t *ct = curthread; if (IGNORE_KERNEL_PREEMPTION) { aston(CPU->cpu_dispthread); return; } /* * Check that conditions are right for kernel preemption */ do { if (ct->t_preempt) { /* * either a privileged thread (idle, panic, interrupt) * or will check when t_preempt is lowered * We need to specifically handle the case where * the thread is in the middle of swtch (resume has * been called) and has its t_preempt set * [idle thread and a thread which is in kpreempt * already] and then a high priority thread is * available in the local dispatch queue. * In this case the resumed thread needs to take a * trap so that it can call kpreempt. We achieve * this by using siron(). * How do we detect this condition: * idle thread is running and is in the midst of * resume: curthread->t_pri == -1 && CPU->dispthread * != CPU->thread * Need to ensure that this happens only at high pil * resume is called at high pil * Only in resume_from_idle is the pil changed. */ if (ct->t_pri < 0) { kpreempt_cnts.kpc_idle++; if (CPU->cpu_dispthread != CPU->cpu_thread) siron(); } else if (ct->t_flag & T_INTR_THREAD) { kpreempt_cnts.kpc_intr++; if (ct->t_pil == CLOCK_LEVEL) kpreempt_cnts.kpc_clock++; } else { kpreempt_cnts.kpc_blocked++; if (CPU->cpu_dispthread != CPU->cpu_thread) siron(); } aston(CPU->cpu_dispthread); return; } if (ct->t_state != TS_ONPROC || ct->t_disp_queue != CPU->cpu_disp) { /* this thread will be calling swtch() shortly */ kpreempt_cnts.kpc_notonproc++; if (CPU->cpu_thread != CPU->cpu_dispthread) { /* already in swtch(), force another */ kpreempt_cnts.kpc_inswtch++; siron(); } return; } if (getpil() >= DISP_LEVEL) { /* * We can't preempt this thread if it is at * a PIL >= DISP_LEVEL since it may be holding * a spin lock (like sched_lock). */ siron(); /* check back later */ kpreempt_cnts.kpc_prilevel++; return; } if (!interrupts_enabled()) { /* * Can't preempt while running with ints disabled */ kpreempt_cnts.kpc_prilevel++; return; } if (asyncspl != KPREEMPT_SYNC) kpreempt_cnts.kpc_apreempt++; else kpreempt_cnts.kpc_spreempt++; ct->t_preempt++; preempt(); ct->t_preempt--; } while (CPU->cpu_kprunrun); } /* * Print out debugging info. */ static void showregs(uint_t type, struct regs *rp, caddr_t addr) { int s; s = spl7(); type &= ~USER; if (PTOU(curproc)->u_comm[0]) printf("%s: ", PTOU(curproc)->u_comm); if (type < TRAP_TYPES) printf("#%s %s\n", trap_type_mnemonic[type], trap_type[type]); else switch (type) { case T_SYSCALL: printf("Syscall Trap:\n"); break; case T_AST: printf("AST\n"); break; default: printf("Bad Trap = %d\n", type); break; } if (type == T_PGFLT) { printf("Bad %s fault at addr=0x%lx\n", USERMODE(rp->r_cs) ? "user": "kernel", (uintptr_t)addr); } else if (addr) { printf("addr=0x%lx\n", (uintptr_t)addr); } printf("pid=%d, pc=0x%lx, sp=0x%lx, eflags=0x%lx\n", (ttoproc(curthread) && ttoproc(curthread)->p_pidp) ? ttoproc(curthread)->p_pid : 0, rp->r_pc, rp->r_sp, rp->r_ps); #if defined(__lint) /* * this clause can be deleted when lint bug 4870403 is fixed * (lint thinks that bit 32 is illegal in a %b format string) */ printf("cr0: %x cr4: %b\n", (uint_t)getcr0(), (uint_t)getcr4(), FMT_CR4); #else printf("cr0: %b cr4: %b\n", (uint_t)getcr0(), FMT_CR0, (uint_t)getcr4(), FMT_CR4); #endif /* __lint */ printf("cr2: %lx ", getcr2()); #if !defined(__xpv) printf("cr3: %lx ", getcr3()); printf("cr8: %lx\n", getcr8()); #endif printf("\n"); dumpregs(rp); splx(s); } static void dumpregs(struct regs *rp) { const char fmt[] = "\t%3s: %16lx %3s: %16lx %3s: %16lx\n"; printf(fmt, "rdi", rp->r_rdi, "rsi", rp->r_rsi, "rdx", rp->r_rdx); printf(fmt, "rcx", rp->r_rcx, " r8", rp->r_r8, " r9", rp->r_r9); printf(fmt, "rax", rp->r_rax, "rbx", rp->r_rbx, "rbp", rp->r_rbp); printf(fmt, "r10", rp->r_r10, "r11", rp->r_r11, "r12", rp->r_r12); printf(fmt, "r13", rp->r_r13, "r14", rp->r_r14, "r15", rp->r_r15); printf(fmt, "fsb", rdmsr(MSR_AMD_FSBASE), "gsb", rdmsr(MSR_AMD_GSBASE), " ds", rp->r_ds); printf(fmt, " es", rp->r_es, " fs", rp->r_fs, " gs", rp->r_gs); printf(fmt, "trp", rp->r_trapno, "err", rp->r_err, "rip", rp->r_rip); printf(fmt, " cs", rp->r_cs, "rfl", rp->r_rfl, "rsp", rp->r_rsp); printf("\t%3s: %16lx\n", " ss", rp->r_ss); } /* * Test to see if the instruction is iret on i386 or iretq on amd64. * * On the hypervisor we can only test for nopop_sys_rtt_syscall. If true * then we are in the context of hypervisor's failsafe handler because it * tried to iret and failed due to a bad selector. See xen_failsafe_callback. */ static int instr_is_iret(caddr_t pc) { #if defined(__xpv) extern void nopop_sys_rtt_syscall(void); return ((pc == (caddr_t)nopop_sys_rtt_syscall) ? 1 : 0); #else static const uint8_t iret_insn[2] = { 0x48, 0xcf }; /* iretq */ return (bcmp(pc, iret_insn, sizeof (iret_insn)) == 0); #endif /* __xpv */ } /* * Test to see if the instruction is part of _sys_rtt (or the KPTI trampolines * which are used by _sys_rtt). * * Again on the hypervisor if we try to IRET to user land with a bad code * or stack selector we will get vectored through xen_failsafe_callback. * In which case we assume we got here via _sys_rtt since we only allow * IRET to user land to take place in _sys_rtt. */ static int instr_is_sys_rtt(caddr_t pc) { extern void _sys_rtt(), _sys_rtt_end(); #if !defined(__xpv) extern void tr_sysc_ret_start(), tr_sysc_ret_end(); extern void tr_intr_ret_start(), tr_intr_ret_end(); if ((uintptr_t)pc >= (uintptr_t)tr_sysc_ret_start && (uintptr_t)pc <= (uintptr_t)tr_sysc_ret_end) return (1); if ((uintptr_t)pc >= (uintptr_t)tr_intr_ret_start && (uintptr_t)pc <= (uintptr_t)tr_intr_ret_end) return (1); #endif if ((uintptr_t)pc < (uintptr_t)_sys_rtt || (uintptr_t)pc > (uintptr_t)_sys_rtt_end) return (0); return (1); } /* * Handle #gp faults in kernel mode. * * One legitimate way this can happen is if we attempt to update segment * registers to naughty values on the way out of the kernel. * * This can happen in a couple of ways: someone - either accidentally or * on purpose - creates (setcontext(2), lwp_create(2)) or modifies * (signal(3C)) a ucontext that contains silly segment register values. * Or someone - either accidentally or on purpose - modifies the prgregset_t * of a subject process via /proc to contain silly segment register values. * * (The unfortunate part is that we can end up discovering the bad segment * register value in the middle of an 'iret' after we've popped most of the * stack. So it becomes quite difficult to associate an accurate ucontext * with the lwp, because the act of taking the #gp trap overwrites most of * what we were going to send the lwp.) * * OTOH if it turns out that's -not- the problem, and we're -not- an lwp * trying to return to user mode and we get a #gp fault, then we need * to die() -- which will happen if we return non-zero from this routine. */ static int kern_gpfault(struct regs *rp) { kthread_t *t = curthread; proc_t *p = ttoproc(t); klwp_t *lwp = ttolwp(t); struct regs tmpregs, *trp = NULL; caddr_t pc = (caddr_t)rp->r_pc; int v; uint32_t auditing = AU_AUDITING(); /* * if we're not an lwp, or in the case of running native the * pc range is outside _sys_rtt, then we should immediately * be die()ing horribly. */ if (lwp == NULL || !instr_is_sys_rtt(pc)) return (1); /* * So at least we're in the right part of the kernel. * * Disassemble the instruction at the faulting pc. * Once we know what it is, we carefully reconstruct the stack * based on the order in which the stack is deconstructed in * _sys_rtt. Ew. */ if (instr_is_iret(pc)) { /* * We took the #gp while trying to perform the IRET. * This means that either %cs or %ss are bad. * All we know for sure is that most of the general * registers have been restored, including the * segment registers, and all we have left on the * topmost part of the lwp's stack are the * registers that the iretq was unable to consume. * * All the rest of the state was crushed by the #gp * which pushed -its- registers atop our old save area * (because we had to decrement the stack pointer, sigh) so * all that we can try and do is to reconstruct the * crushed frame from the #gp trap frame itself. */ trp = &tmpregs; trp->r_ss = lwptoregs(lwp)->r_ss; trp->r_sp = lwptoregs(lwp)->r_sp; trp->r_ps = lwptoregs(lwp)->r_ps; trp->r_cs = lwptoregs(lwp)->r_cs; trp->r_pc = lwptoregs(lwp)->r_pc; bcopy(rp, trp, offsetof(struct regs, r_pc)); /* * Validate simple math */ ASSERT(trp->r_pc == lwptoregs(lwp)->r_pc); ASSERT(trp->r_err == rp->r_err); } if (trp == NULL && PCB_NEED_UPDATE_SEGS(&lwp->lwp_pcb)) { /* * This is the common case -- we're trying to load * a bad segment register value in the only section * of kernel code that ever loads segment registers. * * We don't need to do anything at this point because * the pcb contains all the pending segment register * state, and the regs are still intact because we * didn't adjust the stack pointer yet. Given the fidelity * of all this, we could conceivably send a signal * to the lwp, rather than core-ing. */ trp = lwptoregs(lwp); ASSERT((caddr_t)trp == (caddr_t)rp->r_sp); } if (trp == NULL) return (1); /* * If we get to here, we're reasonably confident that we've * correctly decoded what happened on the way out of the kernel. * Rewrite the lwp's registers so that we can create a core dump * the (at least vaguely) represents the mcontext we were * being asked to restore when things went so terribly wrong. */ /* * Make sure that we have a meaningful %trapno and %err. */ trp->r_trapno = rp->r_trapno; trp->r_err = rp->r_err; if ((caddr_t)trp != (caddr_t)lwptoregs(lwp)) bcopy(trp, lwptoregs(lwp), sizeof (*trp)); mutex_enter(&p->p_lock); lwp->lwp_cursig = SIGSEGV; mutex_exit(&p->p_lock); /* * Terminate all LWPs but don't discard them. If another lwp beat * us to the punch by calling exit(), evaporate now. */ proc_is_exiting(p); if (exitlwps(1) != 0) { mutex_enter(&p->p_lock); lwp_exit(); } if (auditing) /* audit core dump */ audit_core_start(SIGSEGV); v = core(SIGSEGV, B_FALSE); if (auditing) /* audit core dump */ audit_core_finish(v ? CLD_KILLED : CLD_DUMPED); exit(v ? CLD_KILLED : CLD_DUMPED, SIGSEGV); return (0); } /* * dump_tss() - Display the TSS structure */ #if !defined(__xpv) static void dump_tss(void) { const char tss_fmt[] = "tss.%s:\t0x%p\n"; /* Format string */ tss_t *tss = CPU->cpu_tss; printf(tss_fmt, "tss_rsp0", (void *)tss->tss_rsp0); printf(tss_fmt, "tss_rsp1", (void *)tss->tss_rsp1); printf(tss_fmt, "tss_rsp2", (void *)tss->tss_rsp2); printf(tss_fmt, "tss_ist1", (void *)tss->tss_ist1); printf(tss_fmt, "tss_ist2", (void *)tss->tss_ist2); printf(tss_fmt, "tss_ist3", (void *)tss->tss_ist3); printf(tss_fmt, "tss_ist4", (void *)tss->tss_ist4); printf(tss_fmt, "tss_ist5", (void *)tss->tss_ist5); printf(tss_fmt, "tss_ist6", (void *)tss->tss_ist6); printf(tss_fmt, "tss_ist7", (void *)tss->tss_ist7); } #endif /* !__xpv */ #if defined(TRAPTRACE) int ttrace_nrec = 10; /* number of records to dump out */ int ttrace_dump_nregs = 0; /* dump out this many records with regs too */ /* * Dump out the last ttrace_nrec traptrace records on each CPU */ static void dump_ttrace(void) { trap_trace_ctl_t *ttc; trap_trace_rec_t *rec; uintptr_t current; int i, j; int n = NCPU; const char banner[] = "CPU ADDRESS TIMESTAMP TYPE VC HANDLER PC\n"; /* Define format for the CPU, ADDRESS, and TIMESTAMP fields */ const char fmt1[] = "%3d %016lx %12llx"; char data1[34]; /* length of string formatted by fmt1 + 1 */ /* Define format for the TYPE and VC fields */ const char fmt2[] = "%4s %3x"; const char fmt2s[] = "%4s %3s"; char data2[9]; /* length of string formatted by fmt2 + 1 */ /* * Define format for the HANDLER field. Width is arbitrary, but should * be enough for common handler's names, and leave enough space for * the PC field, especially when we are in kmdb. */ const char fmt3h[] = "#%-15s"; const char fmt3p[] = "%-16p"; const char fmt3s[] = "%-16s"; char data3[17]; /* length of string formatted by fmt3* + 1 */ if (ttrace_nrec == 0) return; printf("\n"); printf(banner); for (i = 0; i < n; i++) { ttc = &trap_trace_ctl[i]; if (ttc->ttc_first == (uintptr_t)NULL) continue; current = ttc->ttc_next - sizeof (trap_trace_rec_t); for (j = 0; j < ttrace_nrec; j++) { struct sysent *sys; struct autovec *vec; extern struct av_head autovect[]; int type; ulong_t off; char *sym, *stype; if (current < ttc->ttc_first) current = ttc->ttc_limit - sizeof (trap_trace_rec_t); if (current == (uintptr_t)NULL) continue; rec = (trap_trace_rec_t *)current; if (rec->ttr_stamp == 0) break; (void) snprintf(data1, sizeof (data1), fmt1, i, (uintptr_t)rec, rec->ttr_stamp); switch (rec->ttr_marker) { case TT_SYSCALL: case TT_SYSENTER: case TT_SYSC: case TT_SYSC64: sys = &sysent32[rec->ttr_sysnum]; switch (rec->ttr_marker) { case TT_SYSC64: sys = &sysent[rec->ttr_sysnum]; /* FALLTHROUGH */ case TT_SYSC: stype = "sysc"; /* syscall */ break; case TT_SYSCALL: stype = "lcal"; /* lcall */ break; case TT_SYSENTER: stype = "syse"; /* sysenter */ break; default: stype = ""; break; } (void) snprintf(data2, sizeof (data2), fmt2, stype, rec->ttr_sysnum); if (sys != NULL) { sym = kobj_getsymname( (uintptr_t)sys->sy_callc, &off); if (sym != NULL) { (void) snprintf(data3, sizeof (data3), fmt3s, sym); } else { (void) snprintf(data3, sizeof (data3), fmt3p, sys->sy_callc); } } else { (void) snprintf(data3, sizeof (data3), fmt3s, "unknown"); } break; case TT_INTERRUPT: if (rec->ttr_regs.r_trapno == T_SOFTINT) { (void) snprintf(data2, sizeof (data2), fmt2s, "intr", "-"); (void) snprintf(data3, sizeof (data3), fmt3s, "(fakesoftint)"); break; } (void) snprintf(data2, sizeof (data2), fmt2, "intr", rec->ttr_vector); if (get_intr_handler != NULL) vec = (struct autovec *) (*get_intr_handler) (rec->ttr_cpuid, rec->ttr_vector); else vec = autovect[rec->ttr_vector].avh_link; if (vec != NULL) { sym = kobj_getsymname( (uintptr_t)vec->av_vector, &off); if (sym != NULL) { (void) snprintf(data3, sizeof (data3), fmt3s, sym); } else { (void) snprintf(data3, sizeof (data3), fmt3p, vec->av_vector); } } else { (void) snprintf(data3, sizeof (data3), fmt3s, "unknown"); } break; case TT_TRAP: case TT_EVENT: type = rec->ttr_regs.r_trapno; (void) snprintf(data2, sizeof (data2), fmt2, "trap", type); if (type < TRAP_TYPES) { (void) snprintf(data3, sizeof (data3), fmt3h, trap_type_mnemonic[type]); } else { switch (type) { case T_AST: (void) snprintf(data3, sizeof (data3), fmt3s, "ast"); break; default: (void) snprintf(data3, sizeof (data3), fmt3s, ""); break; } } break; default: break; } sym = kobj_getsymname(rec->ttr_regs.r_pc, &off); if (sym != NULL) { printf("%s %s %s %s+%lx\n", data1, data2, data3, sym, off); } else { printf("%s %s %s %lx\n", data1, data2, data3, rec->ttr_regs.r_pc); } if (ttrace_dump_nregs-- > 0) { int s; if (rec->ttr_marker == TT_INTERRUPT) printf( "\t\tipl %x spl %x pri %x\n", rec->ttr_ipl, rec->ttr_spl, rec->ttr_pri); dumpregs(&rec->ttr_regs); printf("\t%3s: %p\n\n", " ct", (void *)rec->ttr_curthread); /* * print out the pc stack that we recorded * at trap time (if any) */ for (s = 0; s < rec->ttr_sdepth; s++) { uintptr_t fullpc; if (s >= TTR_STACK_DEPTH) { printf("ttr_sdepth corrupt\n"); break; } fullpc = (uintptr_t)rec->ttr_stack[s]; sym = kobj_getsymname(fullpc, &off); if (sym != NULL) printf("-> %s+0x%lx()\n", sym, off); else printf("-> 0x%lx()\n", fullpc); } printf("\n"); } current -= sizeof (trap_trace_rec_t); } } } #endif /* TRAPTRACE */ void panic_showtrap(struct panic_trap_info *tip) { showregs(tip->trap_type, tip->trap_regs, tip->trap_addr); #if defined(TRAPTRACE) dump_ttrace(); #endif #if !defined(__xpv) if (tip->trap_type == T_DBLFLT) dump_tss(); #endif } void panic_savetrap(panic_data_t *pdp, struct panic_trap_info *tip) { panic_saveregs(pdp, tip->trap_regs); } /* * This file and its contents are supplied under the terms of the * Common Development and Distribution License ("CDDL"), version 1.0. * You may only use this file in accordance with the terms of version * 1.0 of the CDDL. * * A full copy of the text of the CDDL should have accompanied this * source. A copy of the CDDL is also available via the Internet at * http://www.illumos.org/license/CDDL. */ /* * Copyright 2020 Joyent, Inc. */ #include #include #include #include /* * The amount of time (in microseconds) between tsc samples. This is * somewhat arbitrary, but seems reasonable. A frequency of 1GHz is * 1,000,000,000 ticks / sec (10^9). 100us is 10^(-6) * 10^2 => 10^(-4), so * 100us would represent 10^5 (100,000) ticks. */ #define HPET_SAMPLE_INTERVAL_US (100) /* * The same as above, but in nanoseconds (for ease in converting to HPET * ticks) */ #define HPET_SAMPLE_INTERVAL_NS (USEC2NSEC(HPET_SAMPLE_INTERVAL_US)) /* The amount of HPET sample ticks to wait */ #define HPET_SAMPLE_TICKS (HRTIME_TO_HPET_TICKS(HPET_SAMPLE_INTERVAL_NS)) #define TSC_NUM_SAMPLES 10 static boolean_t tsc_calibrate_hpet(uint64_t *freqp) { uint64_t hpet_sum = 0; uint64_t tsc_sum = 0; uint_t i; PRM_POINT("Attempting to use HPET for TSC calibration..."); if (hpet_early_init() != DDI_SUCCESS) return (B_FALSE); /* * The expansion of HPET_SAMPLE_TICKS (specifically * HRTIME_TO_HPET_TICKS) uses the HPET period to calculate the number * of HPET ticks for the given time period. Therefore, we cannot * set hpet_num_ticks until after the early HPET initialization has * been performed by hpet_early_init() (and the HPET period is known). */ const uint64_t hpet_num_ticks = HPET_SAMPLE_TICKS; for (i = 0; i < TSC_NUM_SAMPLES; i++) { uint64_t hpet_now, hpet_end; uint64_t tsc_start, tsc_end; hpet_now = hpet_read_timer(); hpet_end = hpet_now + hpet_num_ticks; tsc_start = tsc_read(); while (hpet_now < hpet_end) hpet_now = hpet_read_timer(); tsc_end = tsc_read(); /* * If our TSC isn't advancing after 100us, we're pretty much * hosed. */ VERIFY3P(tsc_end, >, tsc_start); tsc_sum += tsc_end - tsc_start; /* * We likely did not end exactly HPET_SAMPLE_TICKS after * we started, so save the actual amount. */ hpet_sum += hpet_num_ticks + hpet_now - hpet_end; } uint64_t hpet_avg = hpet_sum / TSC_NUM_SAMPLES; uint64_t tsc_avg = tsc_sum / TSC_NUM_SAMPLES; uint64_t hpet_ns = hpet_avg * hpet_info.period / HPET_FEMTO_TO_NANO; PRM_POINT("HPET calibration complete"); *freqp = tsc_avg * NANOSEC / hpet_ns; PRM_DEBUG(*freqp); return (B_TRUE); } /* * Reports from the field suggest that HPET calibration is currently producing * a substantially greater error than PIT calibration on a wide variety of * systems. We are placing it last in the preference order until that can be * resolved. HPET calibration cannot be disabled completely, as some systems * no longer emulate the PIT at all. */ static tsc_calibrate_t tsc_calibration_hpet = { .tscc_source = "HPET", .tscc_preference = 1, .tscc_calibrate = tsc_calibrate_hpet, }; TSC_CALIBRATION_SOURCE(tsc_calibration_hpet); /* * This file and its contents are supplied under the terms of the * Common Development and Distribution License ("CDDL"), version 1.0. * You may only use this file in accordance with the terms of version * 1.0 of the CDDL. * * A full copy of the text of the CDDL should have accompanied this * source. A copy of the CDDL is also available via the Internet at * http://www.illumos.org/license/CDDL. */ /* * Copyright 2020 Joyent, Inc. */ #include #include #include #include extern uint64_t freq_tsc_pit(uint32_t *); /* * Traditionally, the PIT has been used to calibrate both the TSC and the * APIC. As we transition to supporting alternate TSC calibration sources * and using the TSC to calibrate the APIC, we may still want (for diagnostic * purposes) to know what would have happened if we had used the PIT * instead. As a result, if we are using an alternate calibration source * we will still measure the frequency using the PIT and save the result in * pit_tsc_hz for use by the APIC (to similarly save the timings using the * PIT). * * A wrinkle in this is that some systems no longer have a functioning PIT. * In these instances, we simply have no way to provide the 'what if the PIT * was used' values. When we try to use the PIT, we first perform a small * test to see if it appears to be working (i.e. will it count down). If * it does not, we set pit_is_broken to let the APIC calibration code that * it shouldn't attempt to get PIC timings. * * While the systems without a functioning PIT don't seem to experience * any undesirable behavior when attempting to use the non-functional/not * present PIT (i.e. they don't lock up or otherwise act funny -- the counter * values that are read just never change), we still allow pit_is_broken to be * set in /etc/system to inform the system to avoid attempting to use the PIT * at all. * * In the future, we could remove these transitional bits once we have more * history built up using the alternative calibration sources. */ uint64_t pit_tsc_hz; int pit_is_broken; /* * On all of the systems seen so far without functioning PITs, it appears * that they always just return the values written to the PITCTR0_PORT (or * more specifically when they've been programmed to start counting down from * 0xFFFF, they always return 0xFFFF no matter how little/much time has * elapsed). * * Since we have no better way to know if the PIT is broken, we use this * behavior to sanity check the PIT. We program the PIT to count down from * 0xFFFF and wait an amount of time and re-read the result. While we cannot * rely on the TSC frequency being known at this point, we do know that * we are almost certainly never going to see a TSC frequency below 1GHz * on any supported system. * * As such, we (somewhat) arbitrarily pick 400,000 TSC ticks as the amount * of time we wait before re-reading the PIT counter. On a 1GHz machine, * 1 PIT tick would correspond to approximately 838 TSC ticks, therefore * waiting 400,000 TSC ticks should correspond to approx 477 PIT ticks. * On a (currently) theoritical 100GHz machine, 400,000 TSC ticks would still * correspond to approx 4-5 PIT ticks, so this seems a reasonably safe value. */ #define TSC_MIN_TICKS 400000ULL static boolean_t pit_sanity_check(void) { uint64_t tsc_now, tsc_end; ulong_t flags; uint16_t pit_count; flags = clear_int_flag(); tsc_now = tsc_read(); tsc_end = tsc_now + TSC_MIN_TICKS; /* * Put the PIT in mode 0, "Interrupt On Terminal Count": */ outb(PITCTL_PORT, PIT_C0 | PIT_LOADMODE | PIT_ENDSIGMODE); outb(PITCTR0_PORT, 0xFF); outb(PITCTR0_PORT, 0xFF); while (tsc_now < tsc_end) tsc_now = tsc_read(); /* * Latch the counter value and status for counter 0 with the * readback command. */ outb(PITCTL_PORT, PIT_READBACK | PIT_READBACKC0); /* * In readback mode, reading from the counter port produces a * status byte, the low counter byte, and finally the high counter byte. * * We ignore the status byte -- as noted above, we've delayed for an * amount of time that should allow the counter to count off at least * 4-5 ticks (and more realistically at least a hundred), so we just * want to see if the count has changed at all. */ (void) inb(PITCTR0_PORT); pit_count = inb(PITCTR0_PORT); pit_count |= inb(PITCTR0_PORT) << 8; restore_int_flag(flags); if (pit_count == 0xFFFF) { pit_is_broken = 1; return (B_FALSE); } return (B_TRUE); } static boolean_t tsc_calibrate_pit(uint64_t *freqp) { uint64_t processor_clks; ulong_t flags; uint32_t pit_counter; if (pit_is_broken) return (B_FALSE); if (!pit_sanity_check()) return (B_FALSE); /* * freq_tsc_pit() is a hand-rolled assembly function that returns * the number of TSC ticks and sets pit_counter to the number * of corresponding PIT ticks in the same time period. */ flags = clear_int_flag(); processor_clks = freq_tsc_pit(&pit_counter); restore_int_flag(flags); if (pit_counter == 0 || processor_clks == 0 || processor_clks > (((uint64_t)-1) / PIT_HZ)) { return (B_FALSE); } *freqp = pit_tsc_hz = ((uint64_t)PIT_HZ * processor_clks) / pit_counter; return (B_TRUE); } /* * Typically a calibration source that allows the hardware or the hypervisor to * simply declare a specific frequency, rather than requiring calibration at * runtime, is going to provide better results than the using PIT. */ static tsc_calibrate_t tsc_calibration_pit = { .tscc_source = "PIT", .tscc_preference = 10, .tscc_calibrate = tsc_calibrate_pit, }; TSC_CALIBRATION_SOURCE(tsc_calibration_pit); /* * This file and its contents are supplied under the terms of the * Common Development and Distribution License ("CDDL"), version 1.0. * You may only use this file in accordance with the terms of version * 1.0 of the CDDL. * * A full copy of the text of the CDDL should have accompanied this * source. A copy of the CDDL is also available via the Internet at * http://www.illumos.org/license/CDDL. */ /* * Copyright 2021 Jason King * Copyright 2024 Oxide Computer Company */ #include #include /* For VMs, this leaf will contain the largest VM leaf supported in EAX */ #define CPUID_VM_LEAF_MAX 0x40000000 /* * From https://lwn.net/Articles/301888/: * CPUID leaf 0x40000010 (when present on supported VMMs) will contain the TSC * frequency in kHz. */ #define CPUID_VM_LEAF_FREQ 0x40000010 /* * These get_hwenv() types correspond to the platforms which are known to have * support for exposing the TSC frequency via the aforementioned leaf. */ #define HW_SUPPORTS_FREQ (HW_VMWARE | HW_KVM | HW_VIRTUALBOX | HW_ACRN) /* * Allow bypassing the platform identification step when trying to determine if * the host has support for the VM frequency leaf. This allows use of the leaf * on hypervisors which are otherwise unknown. */ int tscc_vmware_match_any = 0; static boolean_t tsc_calibrate_vmware(uint64_t *freqp) { struct cpuid_regs regs = { 0 }; /* * Are we on a platform with support? (or has the administrator * expressed their intent to bypass this check via the config option.) */ if ((get_hwenv() & HW_SUPPORTS_FREQ) == 0 && tscc_vmware_match_any == 0) { return (B_FALSE); } /* ... And does it expose up through the required leaf? */ regs.cp_eax = CPUID_VM_LEAF_MAX; __cpuid_insn(®s); if (regs.cp_eax < CPUID_VM_LEAF_FREQ) { return (B_FALSE); } regs.cp_eax = CPUID_VM_LEAF_FREQ; __cpuid_insn(®s); /* * While not observed in the wild, as a precautionary measure, * we treat a value of 0 as a failure out of an excess of caution. */ if (regs.cp_eax == 0) { return (B_FALSE); } /* Convert from kHz to Hz */ *freqp = (uint64_t)regs.cp_eax * 1000; return (B_TRUE); } static tsc_calibrate_t tsc_calibration_vmware = { .tscc_source = "VMware", .tscc_preference = 100, .tscc_calibrate = tsc_calibrate_vmware, }; TSC_CALIBRATION_SOURCE(tsc_calibration_vmware); /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2009 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ /* * Copyright (c) 2010, Intel Corporation. * All rights reserved. * Copyright 2018 Joyent, Inc. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* * Implementation for cross-processor calls via interprocessor interrupts * * This implementation uses a message passing architecture to allow multiple * concurrent cross calls to be in flight at any given time. We use the cmpxchg * instruction, aka atomic_cas_ptr(), to implement simple efficient work * queues for message passing between CPUs with almost no need for regular * locking. See xc_extract() and xc_insert() below. * * The general idea is that initiating a cross call means putting a message * on a target(s) CPU's work queue. Any synchronization is handled by passing * the message back and forth between initiator and target(s). * * Every CPU has xc_work_cnt, which indicates it has messages to process. * This value is incremented as message traffic is initiated and decremented * with every message that finishes all processing. * * The code needs no mfence or other membar_*() calls. The uses of * atomic_cas_ptr(), atomic_cas_32() and atomic_dec_32() for the message * passing are implemented with LOCK prefix instructions which are * equivalent to mfence. * * One interesting aspect of this implmentation is that it allows 2 or more * CPUs to initiate cross calls to intersecting sets of CPUs at the same time. * The cross call processing by the CPUs will happen in any order with only * a guarantee, for xc_call() and xc_sync(), that an initiator won't return * from cross calls before all slaves have invoked the function. * * The reason for this asynchronous approach is to allow for fast global * TLB shootdowns. If all CPUs, say N, tried to do a global TLB invalidation * on a different Virtual Address at the same time. The old code required * N squared IPIs. With this method, depending on timing, it could happen * with just N IPIs. * * Here are the normal transitions for XC_MSG_* values in ->xc_command. A * transition of "->" happens in the slave cpu and "=>" happens in the master * cpu as the messages are passed back and forth. * * FREE => ASYNC -> DONE => FREE * FREE => CALL -> DONE => FREE * FREE => SYNC -> WAITING => RELEASED -> DONE => FREE * * The interesting one above is ASYNC. You might ask, why not go directly * to FREE, instead of DONE? If it did that, it might be possible to exhaust * the master's xc_free list if a master can generate ASYNC messages faster * then the slave can process them. That could be handled with more complicated * handling. However since nothing important uses ASYNC, I've not bothered. */ /* * The default is to not enable collecting counts of IPI information, since * the updating of shared cachelines could cause excess bus traffic. */ uint_t xc_collect_enable = 0; uint64_t xc_total_cnt = 0; /* total #IPIs sent for cross calls */ uint64_t xc_multi_cnt = 0; /* # times we piggy backed on another IPI */ /* * We allow for one high priority message at a time to happen in the system. * This is used for panic, kmdb, etc., so no locking is done. */ static volatile cpuset_t xc_priority_set_store; static volatile ulong_t *xc_priority_set = CPUSET2BV(xc_priority_set_store); static xc_data_t xc_priority_data; /* * Decrement a CPU's work count */ static void xc_decrement(struct machcpu *mcpu) { atomic_dec_32(&mcpu->xc_work_cnt); } /* * Increment a CPU's work count and return the old value */ static int xc_increment(struct machcpu *mcpu) { int old; do { old = mcpu->xc_work_cnt; } while (atomic_cas_32(&mcpu->xc_work_cnt, old, old + 1) != old); return (old); } /* * Put a message into a queue. The insertion is atomic no matter * how many different inserts/extracts to the same queue happen. */ static void xc_insert(void *queue, xc_msg_t *msg) { xc_msg_t *old_head; /* * FREE messages should only ever be getting inserted into * the xc_master CPUs xc_free queue. */ ASSERT(msg->xc_command != XC_MSG_FREE || cpu[msg->xc_master] == NULL || /* possible only during init */ queue == &cpu[msg->xc_master]->cpu_m.xc_free); do { old_head = (xc_msg_t *)*(volatile xc_msg_t **)queue; msg->xc_next = old_head; } while (atomic_cas_ptr(queue, old_head, msg) != old_head); } /* * Extract a message from a queue. The extraction is atomic only * when just one thread does extractions from the queue. * If the queue is empty, NULL is returned. */ static xc_msg_t * xc_extract(xc_msg_t **queue) { xc_msg_t *old_head; do { old_head = (xc_msg_t *)*(volatile xc_msg_t **)queue; if (old_head == NULL) return (old_head); } while (atomic_cas_ptr(queue, old_head, old_head->xc_next) != old_head); old_head->xc_next = NULL; return (old_head); } /* * Extract the next message from the CPU's queue, and place the message in * .xc_curmsg. The latter is solely to make debugging (and ::xcall) more * useful. */ static xc_msg_t * xc_get(void) { struct machcpu *mcpup = &CPU->cpu_m; xc_msg_t *msg = xc_extract(&mcpup->xc_msgbox); mcpup->xc_curmsg = msg; return (msg); } /* * Initialize the machcpu fields used for cross calls */ static uint_t xc_initialized = 0; void xc_init_cpu(struct cpu *cpup) { xc_msg_t *msg; int c; /* * Allocate message buffers for the new CPU. */ for (c = 0; c < max_ncpus; ++c) { if (plat_dr_support_cpu()) { /* * Allocate a message buffer for every CPU possible * in system, including our own, and add them to our xc * message queue. */ msg = kmem_zalloc(sizeof (*msg), KM_SLEEP); msg->xc_command = XC_MSG_FREE; msg->xc_master = cpup->cpu_id; xc_insert(&cpup->cpu_m.xc_free, msg); } else if (cpu[c] != NULL && cpu[c] != cpup) { /* * Add a new message buffer to each existing CPU's free * list, as well as one for my list for each of them. * Note: cpu0 is statically inserted into cpu[] array, * so need to check cpu[c] isn't cpup itself to avoid * allocating extra message buffers for cpu0. */ msg = kmem_zalloc(sizeof (*msg), KM_SLEEP); msg->xc_command = XC_MSG_FREE; msg->xc_master = c; xc_insert(&cpu[c]->cpu_m.xc_free, msg); msg = kmem_zalloc(sizeof (*msg), KM_SLEEP); msg->xc_command = XC_MSG_FREE; msg->xc_master = cpup->cpu_id; xc_insert(&cpup->cpu_m.xc_free, msg); } } if (!plat_dr_support_cpu()) { /* * Add one for self messages if CPU hotplug is disabled. */ msg = kmem_zalloc(sizeof (*msg), KM_SLEEP); msg->xc_command = XC_MSG_FREE; msg->xc_master = cpup->cpu_id; xc_insert(&cpup->cpu_m.xc_free, msg); } if (!xc_initialized) xc_initialized = 1; } void xc_fini_cpu(struct cpu *cpup) { xc_msg_t *msg; ASSERT((cpup->cpu_flags & CPU_READY) == 0); ASSERT(cpup->cpu_m.xc_msgbox == NULL); ASSERT(cpup->cpu_m.xc_work_cnt == 0); while ((msg = xc_extract(&cpup->cpu_m.xc_free)) != NULL) { kmem_free(msg, sizeof (*msg)); } } #define XC_FLUSH_MAX_WAITS 1000 /* Flush inflight message buffers. */ int xc_flush_cpu(struct cpu *cpup) { int i; ASSERT((cpup->cpu_flags & CPU_READY) == 0); /* * Pause all working CPUs, which ensures that there's no CPU in * function xc_common(). * This is used to work around a race condition window in xc_common() * between checking CPU_READY flag and increasing working item count. */ pause_cpus(cpup, NULL); start_cpus(); for (i = 0; i < XC_FLUSH_MAX_WAITS; i++) { if (cpup->cpu_m.xc_work_cnt == 0) { break; } DELAY(1); } for (; i < XC_FLUSH_MAX_WAITS; i++) { if (!BT_TEST(xc_priority_set, cpup->cpu_id)) { break; } DELAY(1); } return (i >= XC_FLUSH_MAX_WAITS ? ETIME : 0); } /* * X-call message processing routine. Note that this is used by both * senders and recipients of messages. * * We're protected against changing CPUs by either being in a high-priority * interrupt, having preemption disabled or by having a raised SPL. */ /*ARGSUSED*/ uint_t xc_serv(caddr_t arg1, caddr_t arg2) { struct machcpu *mcpup = &(CPU->cpu_m); xc_msg_t *msg; xc_data_t *data; xc_msg_t *xc_waiters = NULL; uint32_t num_waiting = 0; xc_func_t func; xc_arg_t a1; xc_arg_t a2; xc_arg_t a3; uint_t rc = DDI_INTR_UNCLAIMED; while (mcpup->xc_work_cnt != 0) { rc = DDI_INTR_CLAIMED; /* * We may have to wait for a message to arrive. */ for (msg = NULL; msg == NULL; msg = xc_get()) { /* * Alway check for and handle a priority message. */ if (BT_TEST(xc_priority_set, CPU->cpu_id)) { func = xc_priority_data.xc_func; a1 = xc_priority_data.xc_a1; a2 = xc_priority_data.xc_a2; a3 = xc_priority_data.xc_a3; BT_ATOMIC_CLEAR(xc_priority_set, CPU->cpu_id); xc_decrement(mcpup); func(a1, a2, a3); if (mcpup->xc_work_cnt == 0) return (rc); } /* * wait for a message to arrive */ SMT_PAUSE(); } /* * process the message */ switch (msg->xc_command) { /* * ASYNC gives back the message immediately, then we do the * function and return with no more waiting. */ case XC_MSG_ASYNC: data = &cpu[msg->xc_master]->cpu_m.xc_data; func = data->xc_func; a1 = data->xc_a1; a2 = data->xc_a2; a3 = data->xc_a3; msg->xc_command = XC_MSG_DONE; xc_insert(&cpu[msg->xc_master]->cpu_m.xc_msgbox, msg); if (func != NULL) (void) (*func)(a1, a2, a3); xc_decrement(mcpup); break; /* * SYNC messages do the call, then send it back to the master * in WAITING mode */ case XC_MSG_SYNC: data = &cpu[msg->xc_master]->cpu_m.xc_data; if (data->xc_func != NULL) (void) (*data->xc_func)(data->xc_a1, data->xc_a2, data->xc_a3); msg->xc_command = XC_MSG_WAITING; xc_insert(&cpu[msg->xc_master]->cpu_m.xc_msgbox, msg); break; /* * WAITING messsages are collected by the master until all * have arrived. Once all arrive, we release them back to * the slaves */ case XC_MSG_WAITING: xc_insert(&xc_waiters, msg); if (++num_waiting < mcpup->xc_wait_cnt) break; while ((msg = xc_extract(&xc_waiters)) != NULL) { msg->xc_command = XC_MSG_RELEASED; xc_insert(&cpu[msg->xc_slave]->cpu_m.xc_msgbox, msg); --num_waiting; } if (num_waiting != 0) panic("wrong number waiting"); mcpup->xc_wait_cnt = 0; break; /* * CALL messages do the function and then, like RELEASE, * send the message is back to master as DONE. */ case XC_MSG_CALL: data = &cpu[msg->xc_master]->cpu_m.xc_data; if (data->xc_func != NULL) (void) (*data->xc_func)(data->xc_a1, data->xc_a2, data->xc_a3); /*FALLTHROUGH*/ case XC_MSG_RELEASED: msg->xc_command = XC_MSG_DONE; xc_insert(&cpu[msg->xc_master]->cpu_m.xc_msgbox, msg); xc_decrement(mcpup); break; /* * DONE means a slave has completely finished up. * Once we collect all the DONE messages, we'll exit * processing too. */ case XC_MSG_DONE: msg->xc_command = XC_MSG_FREE; xc_insert(&mcpup->xc_free, msg); xc_decrement(mcpup); break; case XC_MSG_FREE: panic("free message 0x%p in msgbox", (void *)msg); break; default: panic("bad message 0x%p in msgbox", (void *)msg); break; } CPU->cpu_m.xc_curmsg = NULL; } return (rc); } /* * Initiate cross call processing. */ static void xc_common( xc_func_t func, xc_arg_t arg1, xc_arg_t arg2, xc_arg_t arg3, ulong_t *set, uint_t command) { int c; struct cpu *cpup; xc_msg_t *msg; xc_data_t *data; int cnt; int save_spl; if (!xc_initialized) { if (BT_TEST(set, CPU->cpu_id) && (CPU->cpu_flags & CPU_READY) && func != NULL) (void) (*func)(arg1, arg2, arg3); return; } save_spl = splr(ipltospl(XC_HI_PIL)); /* * fill in cross call data */ data = &CPU->cpu_m.xc_data; data->xc_func = func; data->xc_a1 = arg1; data->xc_a2 = arg2; data->xc_a3 = arg3; /* * Post messages to all CPUs involved that are CPU_READY */ CPU->cpu_m.xc_wait_cnt = 0; for (c = 0; c < max_ncpus; ++c) { if (!BT_TEST(set, c)) continue; cpup = cpu[c]; if (cpup == NULL || !(cpup->cpu_flags & CPU_READY)) continue; /* * Fill out a new message. */ msg = xc_extract(&CPU->cpu_m.xc_free); if (msg == NULL) panic("Ran out of free xc_msg_t's"); msg->xc_command = command; if (msg->xc_master != CPU->cpu_id) panic("msg %p has wrong xc_master", (void *)msg); msg->xc_slave = c; /* * Increment my work count for all messages that I'll * transition from DONE to FREE. * Also remember how many XC_MSG_WAITINGs to look for */ (void) xc_increment(&CPU->cpu_m); if (command == XC_MSG_SYNC) ++CPU->cpu_m.xc_wait_cnt; /* * Increment the target CPU work count then insert the message * in the target msgbox. If I post the first bit of work * for the target to do, send an IPI to the target CPU. */ cnt = xc_increment(&cpup->cpu_m); xc_insert(&cpup->cpu_m.xc_msgbox, msg); if (cpup != CPU) { if (cnt == 0) { CPU_STATS_ADDQ(CPU, sys, xcalls, 1); send_dirint(c, XC_HI_PIL); if (xc_collect_enable) ++xc_total_cnt; } else if (xc_collect_enable) { ++xc_multi_cnt; } } } /* * Now drop into the message handler until all work is done */ (void) xc_serv(NULL, NULL); splx(save_spl); } /* * Push out a priority cross call. */ static void xc_priority_common( xc_func_t func, xc_arg_t arg1, xc_arg_t arg2, xc_arg_t arg3, ulong_t *set) { int i; int c; struct cpu *cpup; /* * Wait briefly for any previous xc_priority to have finished. */ for (c = 0; c < max_ncpus; ++c) { cpup = cpu[c]; if (cpup == NULL || !(cpup->cpu_flags & CPU_READY)) continue; /* * The value of 40000 here is from old kernel code. It * really should be changed to some time based value, since * under a hypervisor, there's no guarantee a remote CPU * is even scheduled. */ for (i = 0; BT_TEST(xc_priority_set, c) && i < 40000; ++i) SMT_PAUSE(); /* * Some CPU did not respond to a previous priority request. It's * probably deadlocked with interrupts blocked or some such * problem. We'll just erase the previous request - which was * most likely a kmdb_enter that has already expired - and plow * ahead. */ if (BT_TEST(xc_priority_set, c)) { BT_ATOMIC_CLEAR(xc_priority_set, c); if (cpup->cpu_m.xc_work_cnt > 0) xc_decrement(&cpup->cpu_m); } } /* * fill in cross call data */ xc_priority_data.xc_func = func; xc_priority_data.xc_a1 = arg1; xc_priority_data.xc_a2 = arg2; xc_priority_data.xc_a3 = arg3; /* * Post messages to all CPUs involved that are CPU_READY * We'll always IPI, plus bang on the xc_msgbox for i86_mwait() */ for (c = 0; c < max_ncpus; ++c) { if (!BT_TEST(set, c)) continue; cpup = cpu[c]; if (cpup == NULL || !(cpup->cpu_flags & CPU_READY) || cpup == CPU) continue; (void) xc_increment(&cpup->cpu_m); BT_ATOMIC_SET(xc_priority_set, c); send_dirint(c, XC_HI_PIL); for (i = 0; i < 10; ++i) { (void) atomic_cas_ptr(&cpup->cpu_m.xc_msgbox, cpup->cpu_m.xc_msgbox, cpup->cpu_m.xc_msgbox); } } } /* * Do cross call to all other CPUs with absolutely no waiting or handshaking. * This should only be used for extraordinary operations, like panic(), which * need to work, in some fashion, in a not completely functional system. * All other uses that want minimal waiting should use xc_call_nowait(). */ void xc_priority( xc_arg_t arg1, xc_arg_t arg2, xc_arg_t arg3, ulong_t *set, xc_func_t func) { extern int IGNORE_KERNEL_PREEMPTION; int save_spl = splr(ipltospl(XC_HI_PIL)); int save_kernel_preemption = IGNORE_KERNEL_PREEMPTION; IGNORE_KERNEL_PREEMPTION = 1; xc_priority_common((xc_func_t)func, arg1, arg2, arg3, set); IGNORE_KERNEL_PREEMPTION = save_kernel_preemption; splx(save_spl); } /* * Wrapper for kmdb to capture other CPUs, causing them to enter the debugger. */ void kdi_xc_others(int this_cpu, void (*func)(void)) { extern int IGNORE_KERNEL_PREEMPTION; int save_kernel_preemption; cpuset_t set; if (!xc_initialized) return; save_kernel_preemption = IGNORE_KERNEL_PREEMPTION; IGNORE_KERNEL_PREEMPTION = 1; CPUSET_ALL_BUT(set, this_cpu); xc_priority_common((xc_func_t)func, 0, 0, 0, CPUSET2BV(set)); IGNORE_KERNEL_PREEMPTION = save_kernel_preemption; } /* * Invoke function on specified processors. Remotes may continue after * service with no waiting. xc_call_nowait() may return immediately too. */ void xc_call_nowait( xc_arg_t arg1, xc_arg_t arg2, xc_arg_t arg3, ulong_t *set, xc_func_t func) { xc_common(func, arg1, arg2, arg3, set, XC_MSG_ASYNC); } /* * Invoke function on specified processors. Remotes may continue after * service with no waiting. xc_call() returns only after remotes have finished. */ void xc_call( xc_arg_t arg1, xc_arg_t arg2, xc_arg_t arg3, ulong_t *set, xc_func_t func) { xc_common(func, arg1, arg2, arg3, set, XC_MSG_CALL); } /* * Invoke function on specified processors. Remotes wait until all have * finished. xc_sync() also waits until all remotes have finished. */ void xc_sync( xc_arg_t arg1, xc_arg_t arg2, xc_arg_t arg3, ulong_t *set, xc_func_t func) { xc_common(func, arg1, arg2, arg3, set, XC_MSG_SYNC); } /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include static int xen_hvm_inited; /* * This structure is ordinarily constructed by Xen. In the HVM world, we * manually fill in the few fields the PV drivers need. */ static start_info_t __xen_info; start_info_t *xen_info = NULL; static int xen_bits = -1; static int xen_major = -1, xen_minor = -1; /* * Feature bits; more bits will be added, like direct I/O, etc. */ #define XEN_HVM_HYPERCALLS 0x0001 #define XEN_HVM_TLBFLUSH 0x0002 static uint64_t xen_hvm_features; /* Metadata page shared between domain and Xen */ shared_info_t *HYPERVISOR_shared_info = NULL; pfn_t xen_shared_info_frame; /* Page containing code to issue hypercalls. */ extern caddr_t hypercall_page; extern caddr_t hypercall_shared_info_page; static int hvm_get_param(int param_id, uint64_t *val) { struct xen_hvm_param xhp; xhp.domid = DOMID_SELF; xhp.index = param_id; if ((HYPERVISOR_hvm_op(HVMOP_get_param, &xhp) < 0)) return (-1); *val = xhp.value; return (0); } void xen_hvm_init(void) { struct cpuid_regs cp; uint32_t xen_signature[4], base; char *xen_str; struct xen_add_to_physmap xatp; xen_capabilities_info_t caps; pfn_t pfn; uint64_t msrval, val; if (xen_hvm_inited != 0) return; xen_hvm_inited = 1; /* * Xen's pseudo-cpuid function returns a string representing * the Xen signature in %ebx, %ecx, and %edx. * Loop over the base values, since it may be different if * the hypervisor has hyper-v emulation switched on. * * %eax contains the maximum supported cpuid function. */ for (base = 0x40000000; base < 0x40010000; base += 0x100) { cp.cp_eax = base; (void) __cpuid_insn(&cp); xen_signature[0] = cp.cp_ebx; xen_signature[1] = cp.cp_ecx; xen_signature[2] = cp.cp_edx; xen_signature[3] = 0; xen_str = (char *)xen_signature; if (strcmp("XenVMMXenVMM", xen_str) == 0 && cp.cp_eax >= (base + 2)) break; } if (base >= 0x40010000) return; /* * cpuid function at base + 1 returns the Xen version in %eax. The * top 16 bits are the major version, the bottom 16 are the minor * version. */ cp.cp_eax = base + 1; (void) __cpuid_insn(&cp); xen_major = cp.cp_eax >> 16; xen_minor = cp.cp_eax & 0xffff; /* * Below version 3.1 we can't do anything special as a HVM domain; * the PV drivers don't work, many hypercalls are not available, * etc. */ if (xen_major < 3 || (xen_major == 3 && xen_minor < 1)) return; /* * cpuid function at base + 2 returns information about the * hypercall page. %eax nominally contains the number of pages * with hypercall code, but according to the Xen guys, "I'll * guarantee that remains one forever more, so you can just * allocate a single page and get quite upset if you ever see CPUID * return more than one page." %ebx contains an MSR we use to ask * Xen to remap each page at a specific pfn. */ cp.cp_eax = base + 2; (void) __cpuid_insn(&cp); /* * Let Xen know where we want the hypercall page mapped. We * already have a page allocated in the .text section to simplify * the wrapper code. */ pfn = va_to_pfn(&hypercall_page); msrval = mmu_ptob(pfn); wrmsr(cp.cp_ebx, msrval); /* Fill in the xen_info data */ xen_info = &__xen_info; (void) sprintf(xen_info->magic, "xen-%d.%d", xen_major, xen_minor); if (hvm_get_param(HVM_PARAM_STORE_PFN, &val) < 0) return; /* * The first hypercall worked, so mark hypercalls as working. */ xen_hvm_features |= XEN_HVM_HYPERCALLS; xen_info->store_mfn = (mfn_t)val; if (hvm_get_param(HVM_PARAM_STORE_EVTCHN, &val) < 0) return; xen_info->store_evtchn = (mfn_t)val; /* Figure out whether the hypervisor is 32-bit or 64-bit. */ if ((HYPERVISOR_xen_version(XENVER_capabilities, &caps) == 0)) { ((char *)(caps))[sizeof (caps) - 1] = '\0'; if (strstr(caps, "x86_64") != NULL) xen_bits = 64; else if (strstr(caps, "x86_32") != NULL) xen_bits = 32; } if (xen_bits < 0) return; ASSERT(xen_bits == 64); /* * Allocate space for the shared_info page and tell Xen where it * is. */ xen_shared_info_frame = va_to_pfn(&hypercall_shared_info_page); xatp.domid = DOMID_SELF; xatp.idx = 0; xatp.space = XENMAPSPACE_shared_info; xatp.gpfn = xen_shared_info_frame; if (HYPERVISOR_memory_op(XENMEM_add_to_physmap, &xatp) != 0) return; HYPERVISOR_shared_info = (void *)&hypercall_shared_info_page; /* * A working HVM tlb flush hypercall was introduced in Xen 3.3. */ if (xen_major > 3 || (xen_major == 3 && xen_minor >= 3)) xen_hvm_features |= XEN_HVM_TLBFLUSH; } /* * Returns: * -1 if a feature is not available * 1 if a boolean feature is available * > 0 if numeric feature is available */ int xpv_feature(int which) { switch (which) { case XPVF_BITS: return (xen_bits); case XPVF_VERSION_MAJOR: return (xen_major); case XPVF_VERSION_MINOR: return (xen_minor); case XPVF_HYPERCALLS: if (xen_hvm_features & XEN_HVM_HYPERCALLS) return (1); break; case XPVF_SHARED_INFO: if (HYPERVISOR_shared_info != NULL) return (1); break; case XPVF_TLB_FLUSH: if (xen_hvm_features & XEN_HVM_TLBFLUSH) return (1); break; default: break; } return (-1); }