# # CDDL HEADER START # # The contents of this file are subject to the terms of the # Common Development and Distribution License (the "License"). # You may not use this file except in compliance with the License. # # You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE # or http://www.opensolaris.org/os/licensing. # See the License for the specific language governing permissions # and limitations under the License. # # When distributing Covered Code, include this CDDL HEADER in each # file and include the License file at usr/src/OPENSOLARIS.LICENSE. # If applicable, add the following below this CDDL HEADER, with the # fields enclosed by brackets "[]" replaced with your own identifying # information: Portions Copyright [yyyy] [name of copyright owner] # # CDDL HEADER END # # # Copyright 2008 Sun Microsystems, Inc. All rights reserved. # Use is subject to license terms. # # Copyright 2022 Garrett D'Amore # MODULE = cpumem-retire CLASS = common SRCS = \ cma_main.c \ cma_cpu.c \ cma_cpu_arch.c \ cma_page.c \ cma_page_arch.c include ../../Makefile.plugin CMA_VERSION = "1.1" INCDIRS = . CPPFLAGS += $(INCDIRS:%=-I%) -DCMA_VERSION='$(CMA_VERSION)' LDFLAGS += -R/usr/lib/fm LDLIBS += -L$(ROOTLIB)/fm -lfmd_agent /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2008 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ #ifndef _CMA_H #define _CMA_H #include #include #ifdef __cplusplus extern "C" { #endif #define CMA_RA_SUCCESS 0 #define CMA_RA_FAILURE 1 #ifdef opl #define FM_FMRI_HC_CPUIDS "hc-xscf-cpuids" #endif #ifdef i386 extern boolean_t cma_is_native; #endif typedef struct cma_page { struct cma_page *pg_next; /* List of page retirements for retry */ nvlist_t *pg_rsrc; /* Resource for this page */ nvlist_t *pg_asru; /* ASRU for this page */ uint64_t pg_addr; /* Address of this page */ char *pg_uuid; /* UUID for this page's case */ uint_t pg_nretries; /* Number of retries so far for page */ } cma_page_t; #ifdef sun4v typedef struct cma_cpu { struct cma_cpu *cpu_next; /* List of cpus */ nvlist_t *cpu_fmri; /* FMRI for this cpu entry */ int cpuid; /* physical id of this cpu */ char *cpu_uuid; /* UUID for this cpu's case */ uint_t cpu_nretries; /* Number of retries so far for cpu */ } cma_cpu_t; #endif /* sun4v */ typedef struct cma { struct timespec cma_cpu_delay; /* CPU offline retry interval */ uint_t cma_cpu_tries; /* Number of CPU offline retries */ uint_t cma_cpu_dooffline; /* Whether to offline CPUs */ uint_t cma_cpu_forcedoffline; /* Whether to do forced CPU offline */ uint_t cma_cpu_doonline; /* Whether to online CPUs */ uint_t cma_cpu_doblacklist; /* Whether to blacklist CPUs */ uint_t cma_cpu_dounblacklist; /* Whether to unblacklist CPUs */ cma_page_t *cma_pages; /* List of page retirements for retry */ hrtime_t cma_page_curdelay; /* Current retry sleep interval */ hrtime_t cma_page_mindelay; /* Minimum retry sleep interval */ hrtime_t cma_page_maxdelay; /* Maximum retry sleep interval */ id_t cma_page_timerid; /* fmd timer ID for retry sleep */ uint_t cma_page_doretire; /* Whether to retire pages */ uint_t cma_page_dounretire; /* Whether to unretire pages */ #ifdef sun4v cma_cpu_t *cma_cpus; /* List of cpus */ hrtime_t cma_cpu_curdelay; /* Current retry sleep interval */ hrtime_t cma_cpu_mindelay; /* Minimum retry sleep interval */ hrtime_t cma_cpu_maxdelay; /* Maximum retry sleep interval */ id_t cma_cpu_timerid; /* LDOM cpu timer */ #endif /* sun4v */ } cma_t; typedef struct cma_stats { fmd_stat_t cpu_flts; /* Successful offlines */ fmd_stat_t cpu_repairs; /* Successful onlines */ fmd_stat_t cpu_fails; /* Failed offlines/onlines */ fmd_stat_t cpu_blfails; /* Failed blacklists */ fmd_stat_t cpu_supp; /* Suppressed offlines/onlines */ fmd_stat_t cpu_blsupp; /* Suppressed blacklists */ fmd_stat_t page_flts; /* Successful page retires */ fmd_stat_t page_repairs; /* Successful page unretires */ fmd_stat_t page_fails; /* Failed page retires/unretires */ fmd_stat_t page_supp; /* Suppressed retires/unretires */ fmd_stat_t page_nonent; /* Retires for non-present pages */ fmd_stat_t bad_flts; /* Malformed faults */ fmd_stat_t nop_flts; /* Inapplicable faults */ fmd_stat_t auto_flts; /* Auto-close faults */ } cma_stats_t; extern cma_stats_t cma_stats; extern cma_t cma; extern int cma_cpu_cpu_retire(fmd_hdl_t *, nvlist_t *, nvlist_t *, const char *, boolean_t); extern int cma_cpu_hc_retire(fmd_hdl_t *, nvlist_t *, nvlist_t *, const char *, boolean_t); extern int cma_page_retire(fmd_hdl_t *, nvlist_t *, nvlist_t *, const char *, boolean_t); extern void cma_page_retry(fmd_hdl_t *); extern void cma_page_fini(fmd_hdl_t *); extern int cma_set_errno(int); extern int cma_cache_way_retire(fmd_hdl_t *, nvlist_t *, nvlist_t *, const char *, boolean_t); /* * Platforms may have their own implementations of these functions */ extern int cma_cpu_blacklist(fmd_hdl_t *, nvlist_t *, nvlist_t *, boolean_t); extern int cma_cpu_statechange(fmd_hdl_t *, nvlist_t *, const char *, int, boolean_t); extern int cma_fmri_page_service_state(fmd_hdl_t *, nvlist_t *); extern int cma_fmri_page_retire(fmd_hdl_t *, nvlist_t *); extern int cma_fmri_page_unretire(fmd_hdl_t *, nvlist_t *); #ifdef sun4v extern void cma_cpu_start_retry(fmd_hdl_t *, nvlist_t *, const char *, boolean_t); extern void cma_cpu_fini(fmd_hdl_t *); extern void cma_cpu_retry(fmd_hdl_t *); #endif /* sun4v */ extern const char *p_online_state_fmt(int); #ifdef __cplusplus } #endif #endif /* _CMA_H */ /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2008 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. * * Copyright 2019 Joyent, Inc. */ #include #include #include #include #include #include #include #include #include #include #include static int cpu_statechange(fmd_hdl_t *, nvlist_t *, nvlist_t *, const char *, uint32_t, boolean_t); #ifndef opl /* * Perform retire/unretire by invoking the topo methods registered in the * hc-scheme resource. * * If the fault is found to be diagnosed under the old topology, the resource * will not exist in the current topology, then we fall back to legacy retire * (using the "cpu" scheme ASRU). */ static boolean_t old_topo_fault(nvlist_t *nvl) { nvlist_t *rsrc; #ifdef i386 nvlist_t **hcl; uint_t nhcl = 0; char *name; #endif if (nvlist_lookup_nvlist(nvl, FM_FAULT_RESOURCE, &rsrc) != 0) return (B_TRUE); #ifdef i386 /* * x86 has moved from "motherboard/chip/cpu" topo to * "motherboard/chip/core/strand" */ if (nvlist_lookup_nvlist_array(rsrc, FM_FMRI_HC_LIST, &hcl, &nhcl) == 0 && nhcl == 3 && nvlist_lookup_string(hcl[0], FM_FMRI_HC_NAME, &name) == 0 && strcmp(name, "motherboard") == 0 && nvlist_lookup_string(hcl[1], FM_FMRI_HC_NAME, &name) == 0 && strcmp(name, "chip") == 0 && nvlist_lookup_string(hcl[2], FM_FMRI_HC_NAME, &name) == 0 && strcmp(name, "cpu") == 0) return (B_TRUE); #endif return (B_FALSE); } /* ARGSUSED */ int cma_cpu_hc_retire(fmd_hdl_t *hdl, nvlist_t *nvl, nvlist_t *asru, const char *uuid, boolean_t repair) { int i, err; int rc = CMA_RA_SUCCESS; nvlist_t *rsrc; /* * For the cached faults which were diagnosed under the old * topology, we fall back to retire by using cpu-scheme ASRUs. * Under xVM Dom0, since logic cpuid in "cpu" scheme ASRU makes no * sense, the fault should be ignored. */ if (old_topo_fault(nvl)) { #ifdef i386 if (! cma_is_native) return (CMA_RA_FAILURE); #endif return (cma_cpu_cpu_retire(hdl, nvl, asru, uuid, repair)); } /* * Lookup the resource and call its topo methods to do retire/unretire */ if ((! repair && ! cma.cma_cpu_dooffline) || (repair && ! cma.cma_cpu_doonline)) { fmd_hdl_debug(hdl, "suppressed %s of CPU\n", repair ? "unretire" : "retire"); cma_stats.cpu_supp.fmds_value.ui64++; } else { err = FMD_AGENT_RETIRE_FAIL; if (nvlist_lookup_nvlist(nvl, FM_FAULT_RESOURCE, &rsrc) == 0) { if (repair) { err = fmd_nvl_fmri_unretire(hdl, rsrc); } else { for (i = 0; i < cma.cma_cpu_tries; i++) { err = fmd_nvl_fmri_retire(hdl, rsrc); if (err == FMD_AGENT_RETIRE_DONE) break; (void) nanosleep(&cma.cma_cpu_delay, NULL); } } } if (err == FMD_AGENT_RETIRE_DONE) { if (repair) cma_stats.cpu_repairs.fmds_value.ui64++; else cma_stats.cpu_flts.fmds_value.ui64++; } else { rc = CMA_RA_FAILURE; cma_stats.bad_flts.fmds_value.ui64++; #ifdef sun4v /* libldom requests are processed asynchronously */ cma_cpu_start_retry(hdl, nvl, uuid, repair); #endif } } if ((! repair && ! cma.cma_cpu_doblacklist) || (repair && ! cma.cma_cpu_dounblacklist)) { fmd_hdl_debug(hdl, "suppressed %s of CPU\n", repair ? "unblacklist" : "blacklist"); cma_stats.cpu_blsupp.fmds_value.ui64++; } else { if (cma_cpu_blacklist(hdl, nvl, asru, repair) < 0) cma_stats.cpu_blfails.fmds_value.ui64++; } return (rc); } #else /* opl */ /* ARGSUSED 4 */ int cma_cpu_hc_retire(fmd_hdl_t *hdl, nvlist_t *nvl, nvlist_t *asru, const char *uuid, boolean_t repair) { uint_t cpuid; uint_t i, nprs; nvlist_t **hc_prs = NULL, *hc_spec_nvl; /* OPL has ASRU in "hc" scheme */ if (nvlist_lookup_nvlist(asru, FM_FMRI_HC_SPECIFIC, &hc_spec_nvl) != 0) { cma_stats.bad_flts.fmds_value.ui64++; fmd_hdl_debug(hdl, "cma_cpu_hc_retire lookup hc_spec_nvl failed\n"); return (CMA_RA_FAILURE); } if (nvlist_lookup_nvlist_array(hc_spec_nvl, FM_FMRI_HC_CPUIDS, &hc_prs, &nprs) != 0) { cma_stats.bad_flts.fmds_value.ui64++; fmd_hdl_debug(hdl, "cma_cpu_hc_retire lookup cpuid array failed\n"); return (CMA_RA_FAILURE); } for (i = 0; i < nprs; i++) { if (nvlist_lookup_uint32(hc_prs[i], FM_FMRI_CPU_ID, &cpuid) != 0) { cma_stats.bad_flts.fmds_value.ui64++; return (CMA_RA_FAILURE); } if (cpu_statechange(hdl, nvl, hc_prs[i], uuid, cpuid, repair) != CMA_RA_SUCCESS) { cma_stats.bad_flts.fmds_value.ui64++; return (CMA_RA_FAILURE); } } return (CMA_RA_SUCCESS); } #endif /* opl */ /* * The rest of this file uses ASRUs to do retire, this is now not the * preferable way, but it's still needed for some circumstances when * retire via topo methods can't work, ie. * * 1) There are legacy platforms which don't have full topology. * 2) The resources in the FMD cached faults may not be set or exist in the * up-to-dated topology. */ /* ARGSUSED */ static int cpu_online(fmd_hdl_t *hdl, nvlist_t *nvl, nvlist_t *asru, const char *uuid, uint32_t cpuid) { int err = CMA_RA_SUCCESS; if (cma.cma_cpu_doonline) { err = cma_cpu_statechange(hdl, asru, uuid, P_ONLINE, B_TRUE); } else { fmd_hdl_debug(hdl, "suppressed online of CPU %u\n", cpuid); cma_stats.cpu_supp.fmds_value.ui64++; } /* OPL performs the blacklist in the service processor */ #ifndef opl if (cma.cma_cpu_dounblacklist) { if (cma_cpu_blacklist(hdl, nvl, asru, B_TRUE) < 0) cma_stats.cpu_blfails.fmds_value.ui64++; } else { fmd_hdl_debug(hdl, "suppressed unblacklist of CPU %u\n", cpuid); cma_stats.cpu_blsupp.fmds_value.ui64++; } #endif /* opl */ return (err); } /* ARGSUSED */ static int cpu_offline(fmd_hdl_t *hdl, nvlist_t *nvl, nvlist_t *asru, const char *uuid, uint32_t cpuid) { int err = CMA_RA_FAILURE; if (cma.cma_cpu_dooffline) { int cpustate = P_FAULTED; if (cma.cma_cpu_forcedoffline) cpustate |= P_FORCED; err = cma_cpu_statechange(hdl, asru, uuid, cpustate, B_FALSE); } else { fmd_hdl_debug(hdl, "suppressed offline of CPU %u\n", cpuid); cma_stats.cpu_supp.fmds_value.ui64++; } /* OPL performs the blacklist in the service processor */ #ifndef opl if (cma.cma_cpu_doblacklist) { if (cma_cpu_blacklist(hdl, nvl, asru, B_FALSE) < 0) cma_stats.cpu_blfails.fmds_value.ui64++; } else { fmd_hdl_debug(hdl, "suppressed blacklist of CPU %u\n", cpuid); cma_stats.cpu_blsupp.fmds_value.ui64++; } #endif /* opl */ return (err); } static int cpu_statechange(fmd_hdl_t *hdl, nvlist_t *nvl, nvlist_t *asru, const char *uuid, uint32_t cpuid, boolean_t repair) { if (repair) return (cpu_online(hdl, nvl, asru, uuid, cpuid)); else return (cpu_offline(hdl, nvl, asru, uuid, cpuid)); } const char * p_online_state_fmt(int state) { state &= ~P_FORCED; switch (state) { case P_OFFLINE: return (PS_OFFLINE); case P_ONLINE: return (PS_ONLINE); case P_FAULTED: return (PS_FAULTED); case P_POWEROFF: return (PS_POWEROFF); case P_NOINTR: return (PS_NOINTR); case P_SPARE: return (PS_SPARE); case P_DISABLED: return (PS_DISABLED); default: return ("unknown"); } } int cma_cpu_cpu_retire(fmd_hdl_t *hdl, nvlist_t *nvl, nvlist_t *asru, const char *uuid, boolean_t repair) { uint_t cpuid; if (nvlist_lookup_uint32(asru, FM_FMRI_CPU_ID, &cpuid) != 0) { fmd_hdl_debug(hdl, "cpu fault missing '%s'\n", FM_FMRI_CPU_ID); cma_stats.bad_flts.fmds_value.ui64++; return (CMA_RA_FAILURE); } return (cpu_statechange(hdl, nvl, asru, uuid, cpuid, repair)); } /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2008 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ #include #include #include #include #include #include #include #include #include #include int cma_cpu_blacklist(fmd_hdl_t *hdl, nvlist_t *nvl, nvlist_t *asru, boolean_t repair) { bl_req_t blr; nvlist_t *fmri; char *fmribuf; size_t fmrisz; int fd, rc, err; char *class; /* * Some platforms have special unums for the E$ DIMMs. If we're dealing * with a platform that has these unums, one will have been added to the * fault as the resource. We'll use that for the blacklisting. If we * can't find a resource, we'll fall back to the ASRU. */ if (nvlist_lookup_nvlist(nvl, FM_FAULT_RESOURCE, &fmri) != 0) fmri = asru; if ((nvlist_lookup_string(nvl, FM_CLASS, &class) != 0) || (class == NULL) || (*class == '\0')) { fmd_hdl_debug(hdl, "failed to get the fault class name\n"); errno = EINVAL; return (-1); } if ((fd = open("/dev/bl", O_RDONLY)) < 0) return (-1); /* errno is set for us */ if ((errno = nvlist_size(fmri, &fmrisz, NV_ENCODE_NATIVE)) != 0 || (fmribuf = fmd_hdl_alloc(hdl, fmrisz, FMD_SLEEP)) == NULL) { (void) close(fd); return (-1); /* errno is set for us */ } if ((errno = nvlist_pack(fmri, &fmribuf, &fmrisz, NV_ENCODE_NATIVE, 0)) != 0) { fmd_hdl_free(hdl, fmribuf, fmrisz); (void) close(fd); return (-1); /* errno is set for us */ } blr.bl_fmri = fmribuf; blr.bl_fmrisz = fmrisz; blr.bl_class = class; rc = ioctl(fd, repair ? BLIOC_DELETE : BLIOC_INSERT, &blr); err = errno; fmd_hdl_free(hdl, fmribuf, fmrisz); (void) close(fd); if (rc < 0 && err != ENOTSUP) { errno = err; return (-1); } return (0); } /* ARGSUSED */ int cma_cpu_statechange(fmd_hdl_t *hdl, nvlist_t *asru, const char *uuid, int cpustate, boolean_t repair) { int i; uint_t cpuid; if (nvlist_lookup_uint32(asru, FM_FMRI_CPU_ID, &cpuid) != 0) { fmd_hdl_debug(hdl, "missing '%s'\n", FM_FMRI_CPU_ID); cma_stats.bad_flts.fmds_value.ui64++; return (CMA_RA_FAILURE); } for (i = 0; i < cma.cma_cpu_tries; i++, (void) nanosleep(&cma.cma_cpu_delay, NULL)) { int oldstate; if ((oldstate = p_online(cpuid, cpustate)) != -1) { fmd_hdl_debug(hdl, "changed cpu %u state from \"%s\" " "to \"%s\"\n", cpuid, p_online_state_fmt(oldstate), p_online_state_fmt(cpustate)); if (repair) cma_stats.cpu_repairs.fmds_value.ui64++; else cma_stats.cpu_flts.fmds_value.ui64++; return (CMA_RA_SUCCESS); } } fmd_hdl_debug(hdl, "failed to changed cpu %u state to \"%s\": %s\n", cpuid, p_online_state_fmt(cpustate), strerror(errno)); cma_stats.cpu_fails.fmds_value.ui64++; return (CMA_RA_FAILURE); } /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved. */ #include #include #include #include #include #include #include #include #include #include #ifdef sun4v #include static fmd_hdl_t *init_hdl; ldom_hdl_t *cma_lhp; #endif #ifdef i386 boolean_t cma_is_native; #endif extern const char *fmd_fmri_get_platform(); cma_t cma; cma_stats_t cma_stats = { { "cpu_flts", FMD_TYPE_UINT64, "cpu faults resolved" }, { "cpu_repairs", FMD_TYPE_UINT64, "cpu faults repaired" }, { "cpu_fails", FMD_TYPE_UINT64, "cpu faults unresolveable" }, { "cpu_blfails", FMD_TYPE_UINT64, "failed cpu blacklists" }, { "cpu_supp", FMD_TYPE_UINT64, "cpu offlines suppressed" }, { "cpu_blsupp", FMD_TYPE_UINT64, "cpu blacklists suppressed" }, { "page_flts", FMD_TYPE_UINT64, "page faults resolved" }, { "page_repairs", FMD_TYPE_UINT64, "page faults repaired" }, { "page_fails", FMD_TYPE_UINT64, "page faults unresolveable" }, { "page_supp", FMD_TYPE_UINT64, "page retires suppressed" }, { "page_nonent", FMD_TYPE_UINT64, "retires for non-existent fmris" }, { "bad_flts", FMD_TYPE_UINT64, "invalid fault events received" }, { "nop_flts", FMD_TYPE_UINT64, "inapplicable fault events received" }, { "auto_flts", FMD_TYPE_UINT64, "auto-close faults received" } }; typedef struct cma_subscriber { const char *subr_class; const char *subr_sname; uint_t subr_svers; int (*subr_func)(fmd_hdl_t *, nvlist_t *, nvlist_t *, const char *, boolean_t); } cma_subscriber_t; static const cma_subscriber_t cma_subrs[] = { #if defined(i386) /* * On x86, the ASRUs are expected to be in hc scheme. When * cpumem-retire wants to retire a cpu or mem page, it calls the * methods registered in the topo node to do that. The topo * enumerator, which necessarily knows all the config info that * we'd ever need in deciding what/how to retire etc. This takes * away much of that complexity from the agent into the entity * that knows all config/topo information. */ { "fault.memory.page", FM_FMRI_SCHEME_HC, FM_HC_SCHEME_VERSION, cma_page_retire }, { "fault.memory.page_sb", FM_FMRI_SCHEME_HC, FM_HC_SCHEME_VERSION, cma_page_retire }, { "fault.memory.page_ck", FM_FMRI_SCHEME_HC, FM_HC_SCHEME_VERSION, cma_page_retire }, { "fault.memory.page_ue", FM_FMRI_SCHEME_HC, FM_HC_SCHEME_VERSION, cma_page_retire }, { "fault.memory.generic-x86.page_ce", FM_FMRI_SCHEME_HC, FM_HC_SCHEME_VERSION, cma_page_retire }, { "fault.memory.generic-x86.page_ue", FM_FMRI_SCHEME_HC, FM_HC_SCHEME_VERSION, cma_page_retire }, { "fault.memory.intel.page_ce", FM_FMRI_SCHEME_HC, FM_HC_SCHEME_VERSION, cma_page_retire }, { "fault.memory.intel.page_ue", FM_FMRI_SCHEME_HC, FM_HC_SCHEME_VERSION, cma_page_retire }, { "fault.memory.dimm", FM_FMRI_SCHEME_HC, FM_HC_SCHEME_VERSION, NULL }, { "fault.memory.dimm_sb", FM_FMRI_SCHEME_HC, FM_HC_SCHEME_VERSION, NULL }, { "fault.memory.dimm_ck", FM_FMRI_SCHEME_HC, FM_HC_SCHEME_VERSION, NULL }, { "fault.memory.dimm_ue", FM_FMRI_SCHEME_HC, FM_HC_SCHEME_VERSION, NULL }, { "fault.memory.generic-x86.dimm_ce", FM_FMRI_SCHEME_HC, FM_HC_SCHEME_VERSION, NULL }, { "fault.memory.generic-x86.dimm_ue", FM_FMRI_SCHEME_HC, FM_HC_SCHEME_VERSION, NULL }, { "fault.memory.intel.dimm_ce", FM_FMRI_SCHEME_HC, FM_HC_SCHEME_VERSION, NULL }, { "fault.memory.intel.dimm_ue", FM_FMRI_SCHEME_HC, FM_HC_SCHEME_VERSION, NULL }, { "fault.memory.intel.fbd.*", FM_FMRI_SCHEME_HC, FM_HC_SCHEME_VERSION, NULL }, { "fault.memory.dimm_testfail", FM_FMRI_SCHEME_HC, FM_HC_SCHEME_VERSION, NULL }, { "fault.memory.bank", FM_FMRI_SCHEME_HC, FM_HC_SCHEME_VERSION, NULL }, { "fault.memory.datapath", FM_FMRI_SCHEME_HC, FM_HC_SCHEME_VERSION, NULL }, { "fault.cpu.intel.quickpath.mem_scrubbing", FM_FMRI_SCHEME_HC, FM_HC_SCHEME_VERSION, cma_page_retire }, { "fault.cpu.intel.quickpath.*", FM_FMRI_SCHEME_HC, FM_HC_SCHEME_VERSION, NULL }, { "fault.cpu.generic-x86.mc", FM_FMRI_SCHEME_HC, FM_HC_SCHEME_VERSION, NULL }, { "fault.cpu.intel.dma", FM_FMRI_SCHEME_HC, FM_HC_SCHEME_VERSION, NULL }, { "fault.cpu.intel.dma", FM_FMRI_SCHEME_CPU, FM_CPU_SCHEME_VERSION, NULL }, /* * The ASRU for cpu faults are in cpu scheme on native and in hc * scheme on xpv. So each cpu fault class needs to be listed twice. */ /* * The following faults do NOT retire a cpu thread, * and therefore must be intercepted before * the default "fault.cpu.*" dispatch to cma_cpu_hc_retire. */ { "fault.cpu.amd.dramchannel", FM_FMRI_SCHEME_HC, FM_HC_SCHEME_VERSION, NULL }, { "fault.cpu.amd.dramchannel", FM_FMRI_SCHEME_CPU, FM_CPU_SCHEME_VERSION, NULL }, { "fault.cpu.generic-x86.bus_interconnect_memory", FM_FMRI_SCHEME_HC, FM_HC_SCHEME_VERSION, NULL }, { "fault.cpu.generic-x86.bus_interconnect_memory", FM_FMRI_SCHEME_CPU, FM_CPU_SCHEME_VERSION, NULL }, { "fault.cpu.generic-x86.bus_interconnect_io", FM_FMRI_SCHEME_HC, FM_HC_SCHEME_VERSION, NULL }, { "fault.cpu.generic-x86.bus_interconnect_io", FM_FMRI_SCHEME_CPU, FM_CPU_SCHEME_VERSION, NULL }, { "fault.cpu.generic-x86.bus_interconnect", FM_FMRI_SCHEME_HC, FM_HC_SCHEME_VERSION, NULL }, { "fault.cpu.generic-x86.bus_interconnect", FM_FMRI_SCHEME_CPU, FM_CPU_SCHEME_VERSION, NULL }, { "fault.cpu.intel.bus_interconnect_memory", FM_FMRI_SCHEME_HC, FM_HC_SCHEME_VERSION, NULL }, { "fault.cpu.intel.bus_interconnect_memory", FM_FMRI_SCHEME_CPU, FM_CPU_SCHEME_VERSION, NULL }, { "fault.cpu.intel.bus_interconnect_io", FM_FMRI_SCHEME_HC, FM_HC_SCHEME_VERSION, NULL }, { "fault.cpu.intel.bus_interconnect_io", FM_FMRI_SCHEME_CPU, FM_CPU_SCHEME_VERSION, NULL }, { "fault.cpu.intel.bus_interconnect", FM_FMRI_SCHEME_HC, FM_HC_SCHEME_VERSION, NULL }, { "fault.cpu.intel.bus_interconnect", FM_FMRI_SCHEME_CPU, FM_CPU_SCHEME_VERSION, NULL }, { "fault.cpu.intel.nb.*", FM_FMRI_SCHEME_HC, FM_HC_SCHEME_VERSION, NULL }, { "fault.cpu.intel.nb.*", FM_FMRI_SCHEME_CPU, FM_CPU_SCHEME_VERSION, NULL }, { "fault.cpu.intel.dma", FM_FMRI_SCHEME_HC, FM_HC_SCHEME_VERSION, NULL }, { "fault.cpu.intel.dma", FM_FMRI_SCHEME_CPU, FM_CPU_SCHEME_VERSION, NULL }, { "fault.cpu.*", FM_FMRI_SCHEME_HC, FM_HC_SCHEME_VERSION, cma_cpu_hc_retire }, { "fault.cpu.*", FM_FMRI_SCHEME_CPU, FM_CPU_SCHEME_VERSION, cma_cpu_hc_retire }, #elif defined(sun4v) /* * The following are PI sun4v faults */ { "fault.memory.memlink", FM_FMRI_SCHEME_HC, FM_HC_SCHEME_VERSION, NULL }, { "fault.memory.memlink-uc", FM_FMRI_SCHEME_HC, FM_HC_SCHEME_VERSION, NULL }, { "fault.memory.memlink-failover", FM_FMRI_SCHEME_HC, FM_HC_SCHEME_VERSION, NULL }, { "fault.memory.dimm-ue-imminent", FM_FMRI_SCHEME_HC, FM_HC_SCHEME_VERSION, NULL }, { "fault.memory.dram-ue-imminent", FM_FMRI_SCHEME_HC, FM_HC_SCHEME_VERSION, NULL }, { "fault.memory.dimm-page-retires-excessive", FM_FMRI_SCHEME_HC, FM_HC_SCHEME_VERSION, NULL }, { "fault.memory.page", FM_FMRI_SCHEME_MEM, FM_MEM_SCHEME_VERSION, cma_page_retire }, { "fault.memory.dimm", FM_FMRI_SCHEME_MEM, FM_MEM_SCHEME_VERSION, NULL }, { "fault.memory.dimm_sb", FM_FMRI_SCHEME_MEM, FM_MEM_SCHEME_VERSION, NULL }, { "fault.memory.dimm_ck", FM_FMRI_SCHEME_MEM, FM_MEM_SCHEME_VERSION, NULL }, { "fault.memory.dimm_ue", FM_FMRI_SCHEME_MEM, FM_MEM_SCHEME_VERSION, NULL }, { "fault.memory.dimm-page-retires-excessive", FM_FMRI_SCHEME_MEM, FM_MEM_SCHEME_VERSION, NULL }, { "fault.memory.dimm-ue-imminent", FM_FMRI_SCHEME_MEM, FM_MEM_SCHEME_VERSION, NULL }, { "fault.memory.dram-ue-imminent", FM_FMRI_SCHEME_MEM, FM_MEM_SCHEME_VERSION, NULL }, { "fault.memory.bank", FM_FMRI_SCHEME_MEM, FM_MEM_SCHEME_VERSION, NULL }, { "fault.memory.datapath", FM_FMRI_SCHEME_MEM, FM_MEM_SCHEME_VERSION, NULL }, { "fault.memory.datapath", FM_FMRI_SCHEME_HC, FM_HC_SCHEME_VERSION, NULL }, { "fault.memory.link-c", FM_FMRI_SCHEME_MEM, FM_MEM_SCHEME_VERSION, NULL }, { "fault.memory.link-u", FM_FMRI_SCHEME_MEM, FM_MEM_SCHEME_VERSION, NULL }, { "fault.memory.link-f", FM_FMRI_SCHEME_MEM, FM_MEM_SCHEME_VERSION, NULL }, { "fault.memory.link-c", FM_FMRI_SCHEME_HC, FM_HC_SCHEME_VERSION, NULL }, { "fault.memory.link-u", FM_FMRI_SCHEME_HC, FM_HC_SCHEME_VERSION, NULL }, { "fault.memory.link-f", FM_FMRI_SCHEME_HC, FM_HC_SCHEME_VERSION, NULL }, /* * The following ultraSPARC-T1/T2 faults do NOT retire a cpu thread, * and therefore must be intercepted before * the default "fault.cpu.*" dispatch to cma_cpu_hc_retire. */ { "fault.cpu.*.l2cachedata", FM_FMRI_SCHEME_CPU, FM_CPU_SCHEME_VERSION, NULL }, { "fault.cpu.*.l2cachetag", FM_FMRI_SCHEME_CPU, FM_CPU_SCHEME_VERSION, NULL }, { "fault.cpu.*.l2cachectl", FM_FMRI_SCHEME_CPU, FM_CPU_SCHEME_VERSION, NULL }, { "fault.cpu.*.l2data-c", FM_FMRI_SCHEME_CPU, FM_CPU_SCHEME_VERSION, NULL }, { "fault.cpu.*.l2data-u", FM_FMRI_SCHEME_CPU, FM_CPU_SCHEME_VERSION, NULL }, { "fault.cpu.*.mau", FM_FMRI_SCHEME_CPU, FM_CPU_SCHEME_VERSION, NULL }, { "fault.cpu.*.lfu-u", FM_FMRI_SCHEME_CPU, FM_CPU_SCHEME_VERSION, NULL }, { "fault.cpu.*.lfu-f", FM_FMRI_SCHEME_CPU, FM_CPU_SCHEME_VERSION, NULL }, { "fault.cpu.*.lfu-p", FM_FMRI_SCHEME_CPU, FM_CPU_SCHEME_VERSION, NULL }, { "fault.cpu.ultraSPARC-T1.freg", FM_FMRI_SCHEME_CPU, FM_CPU_SCHEME_VERSION, NULL }, { "fault.cpu.ultraSPARC-T1.l2cachedata", FM_FMRI_SCHEME_CPU, FM_CPU_SCHEME_VERSION, NULL }, { "fault.cpu.ultraSPARC-T1.l2cachetag", FM_FMRI_SCHEME_CPU, FM_CPU_SCHEME_VERSION, NULL }, { "fault.cpu.ultraSPARC-T1.l2cachectl", FM_FMRI_SCHEME_CPU, FM_CPU_SCHEME_VERSION, NULL }, { "fault.cpu.ultraSPARC-T1.mau", FM_FMRI_SCHEME_CPU, FM_CPU_SCHEME_VERSION, NULL }, { "fault.cpu.ultraSPARC-T2plus.chip", FM_FMRI_SCHEME_HC, FM_HC_SCHEME_VERSION, NULL }, { "fault.cpu.*", FM_FMRI_SCHEME_HC, FM_HC_SCHEME_VERSION, cma_cpu_hc_retire }, { "fault.cpu.*", FM_FMRI_SCHEME_CPU, FM_CPU_SCHEME_VERSION, cma_cpu_hc_retire }, #elif defined(opl) { "fault.memory.page", FM_FMRI_SCHEME_MEM, FM_MEM_SCHEME_VERSION, cma_page_retire }, { "fault.memory.dimm", FM_FMRI_SCHEME_MEM, FM_MEM_SCHEME_VERSION, NULL }, { "fault.memory.dimm-page-retires-excessive", FM_FMRI_SCHEME_MEM, FM_MEM_SCHEME_VERSION, NULL }, { "fault.memory.dimm-ue-imminent", FM_FMRI_SCHEME_MEM, FM_MEM_SCHEME_VERSION, NULL }, { "fault.memory.dram-ue-imminent", FM_FMRI_SCHEME_MEM, FM_MEM_SCHEME_VERSION, NULL }, { "fault.memory.bank", FM_FMRI_SCHEME_MEM, FM_MEM_SCHEME_VERSION, NULL }, { "fault.cpu.SPARC64-VI.*", FM_FMRI_SCHEME_CPU, FM_CPU_SCHEME_VERSION, cma_cpu_cpu_retire }, { "fault.cpu.SPARC64-VII.*", FM_FMRI_SCHEME_CPU, FM_CPU_SCHEME_VERSION, cma_cpu_cpu_retire }, { "fault.chassis.SPARC-Enterprise.cpu.SPARC64-VI.core.se", FM_FMRI_SCHEME_HC, FM_HC_SCHEME_VERSION, cma_cpu_hc_retire }, { "fault.chassis.SPARC-Enterprise.cpu.SPARC64-VI.core.se-offlinereq", FM_FMRI_SCHEME_HC, FM_HC_SCHEME_VERSION, cma_cpu_hc_retire }, { "fault.chassis.SPARC-Enterprise.cpu.SPARC64-VI.core.ce", FM_FMRI_SCHEME_HC, FM_HC_SCHEME_VERSION, cma_cpu_hc_retire }, { "fault.chassis.SPARC-Enterprise.cpu.SPARC64-VI.core.ce-offlinereq", FM_FMRI_SCHEME_HC, FM_HC_SCHEME_VERSION, cma_cpu_hc_retire }, { "fault.chassis.SPARC-Enterprise.cpu.SPARC64-VII.core.se", FM_FMRI_SCHEME_HC, FM_HC_SCHEME_VERSION, cma_cpu_hc_retire }, { "fault.chassis.SPARC-Enterprise.cpu.SPARC64-VII.core.se-offlinereq", FM_FMRI_SCHEME_HC, FM_HC_SCHEME_VERSION, cma_cpu_hc_retire }, { "fault.chassis.SPARC-Enterprise.cpu.SPARC64-VII.core.ce", FM_FMRI_SCHEME_HC, FM_HC_SCHEME_VERSION, cma_cpu_hc_retire }, { "fault.chassis.SPARC-Enterprise.cpu.SPARC64-VII.core.ce-offlinereq", FM_FMRI_SCHEME_HC, FM_HC_SCHEME_VERSION, cma_cpu_hc_retire }, #else /* * For platforms excluding i386, sun4v and opl. */ { "fault.memory.page", FM_FMRI_SCHEME_MEM, FM_MEM_SCHEME_VERSION, cma_page_retire }, { "fault.memory.page_sb", FM_FMRI_SCHEME_MEM, FM_MEM_SCHEME_VERSION, cma_page_retire }, { "fault.memory.page_ck", FM_FMRI_SCHEME_MEM, FM_MEM_SCHEME_VERSION, cma_page_retire }, { "fault.memory.page_ue", FM_FMRI_SCHEME_MEM, FM_MEM_SCHEME_VERSION, cma_page_retire }, { "fault.memory.dimm", FM_FMRI_SCHEME_MEM, FM_MEM_SCHEME_VERSION, NULL }, { "fault.memory.dimm_sb", FM_FMRI_SCHEME_MEM, FM_MEM_SCHEME_VERSION, NULL }, { "fault.memory.dimm_ck", FM_FMRI_SCHEME_MEM, FM_MEM_SCHEME_VERSION, NULL }, { "fault.memory.dimm_ue", FM_FMRI_SCHEME_MEM, FM_MEM_SCHEME_VERSION, NULL }, { "fault.memory.dimm-page-retires-excessive", FM_FMRI_SCHEME_MEM, FM_MEM_SCHEME_VERSION, NULL }, { "fault.memory.dimm-ue-imminent", FM_FMRI_SCHEME_MEM, FM_MEM_SCHEME_VERSION, NULL }, { "fault.memory.dram-ue-imminent", FM_FMRI_SCHEME_MEM, FM_MEM_SCHEME_VERSION, NULL }, { "fault.memory.dimm_testfail", FM_FMRI_SCHEME_MEM, FM_MEM_SCHEME_VERSION, NULL }, { "fault.memory.bank", FM_FMRI_SCHEME_MEM, FM_MEM_SCHEME_VERSION, NULL }, { "fault.memory.datapath", FM_FMRI_SCHEME_MEM, FM_MEM_SCHEME_VERSION, NULL }, { "fault.memory.datapath", FM_FMRI_SCHEME_HC, FM_HC_SCHEME_VERSION, NULL }, { "fault.memory.datapath", FM_FMRI_SCHEME_CPU, FM_CPU_SCHEME_VERSION, NULL }, /* * The following faults do NOT retire a cpu thread, * and therefore must be intercepted before * the default "fault.cpu.*" dispatch to cma_cpu_cpu_retire. */ { "fault.cpu.ultraSPARC-IVplus.l2cachedata-line", FM_FMRI_SCHEME_CPU, FM_CPU_SCHEME_VERSION, cma_cache_way_retire }, { "fault.cpu.ultraSPARC-IVplus.l3cachedata-line", FM_FMRI_SCHEME_CPU, FM_CPU_SCHEME_VERSION, cma_cache_way_retire }, { "fault.cpu.ultraSPARC-IVplus.l2cachetag-line", FM_FMRI_SCHEME_CPU, FM_CPU_SCHEME_VERSION, cma_cache_way_retire }, { "fault.cpu.ultraSPARC-IVplus.l3cachetag-line", FM_FMRI_SCHEME_CPU, FM_CPU_SCHEME_VERSION, cma_cache_way_retire }, /* * Default "fault.cpu.*" for "cpu" scheme ASRU dispatch. */ { "fault.cpu.*", FM_FMRI_SCHEME_CPU, FM_CPU_SCHEME_VERSION, cma_cpu_cpu_retire }, #endif { NULL, NULL, 0, NULL } }; static const cma_subscriber_t * nvl2subr(fmd_hdl_t *hdl, nvlist_t *nvl, nvlist_t **asrup) { const cma_subscriber_t *sp; nvlist_t *asru; char *scheme; uint8_t version; boolean_t retire; if (nvlist_lookup_boolean_value(nvl, FM_SUSPECT_RETIRE, &retire) == 0 && retire == 0) { fmd_hdl_debug(hdl, "cma_recv: retire suppressed"); return (NULL); } if (nvlist_lookup_nvlist(nvl, FM_FAULT_ASRU, &asru) != 0 || nvlist_lookup_string(asru, FM_FMRI_SCHEME, &scheme) != 0 || nvlist_lookup_uint8(asru, FM_VERSION, &version) != 0) { cma_stats.bad_flts.fmds_value.ui64++; return (NULL); } for (sp = cma_subrs; sp->subr_class != NULL; sp++) { if (fmd_nvl_class_match(hdl, nvl, sp->subr_class) && strcmp(scheme, sp->subr_sname) == 0 && version <= sp->subr_svers) { *asrup = asru; return (sp); } } cma_stats.nop_flts.fmds_value.ui64++; return (NULL); } static void cma_recv_list(fmd_hdl_t *hdl, nvlist_t *nvl, const char *class) { char *uuid = NULL; nvlist_t **nva, **save_nva; uint_t nvc = 0, save_nvc; uint_t keepopen; int err = 0; nvlist_t *asru = NULL; uint32_t index; err |= nvlist_lookup_string(nvl, FM_SUSPECT_UUID, &uuid); err |= nvlist_lookup_nvlist_array(nvl, FM_SUSPECT_FAULT_LIST, &nva, &nvc); if (err != 0) { cma_stats.bad_flts.fmds_value.ui64++; return; } save_nvc = keepopen = nvc; save_nva = nva; while (nvc-- != 0 && (strcmp(class, FM_LIST_SUSPECT_CLASS) != 0 || !fmd_case_uuclosed(hdl, uuid))) { nvlist_t *nvl = *nva++; const cma_subscriber_t *subr; int has_fault; if ((subr = nvl2subr(hdl, nvl, &asru)) == NULL) continue; /* * A handler returns CMA_RA_SUCCESS to indicate that * from this suspects point-of-view the case may be * closed, CMA_RA_FAILURE otherwise. * A handler must not close the case itself. */ if (subr->subr_func != NULL) { has_fault = fmd_nvl_fmri_has_fault(hdl, asru, FMD_HAS_FAULT_ASRU, NULL); if (strcmp(class, FM_LIST_SUSPECT_CLASS) == 0) { if (has_fault == 1) err = subr->subr_func(hdl, nvl, asru, uuid, 0); } else { if (has_fault == 0) err = subr->subr_func(hdl, nvl, asru, uuid, 1); } if (err == CMA_RA_SUCCESS) keepopen--; } } /* * Run though again to catch any new faults in list.updated. */ while (save_nvc-- != 0 && (strcmp(class, FM_LIST_UPDATED_CLASS) == 0)) { nvlist_t *nvl = *save_nva++; const cma_subscriber_t *subr; int has_fault; if ((subr = nvl2subr(hdl, nvl, &asru)) == NULL) continue; if (subr->subr_func != NULL) { has_fault = fmd_nvl_fmri_has_fault(hdl, asru, FMD_HAS_FAULT_ASRU, NULL); if (has_fault == 1) err = subr->subr_func(hdl, nvl, asru, uuid, 0); } } /* * Do not close the case if we are handling cache faults. */ if (asru != NULL) { if (nvlist_lookup_uint32(asru, FM_FMRI_CPU_CACHE_INDEX, &index) != 0) { if (!keepopen && strcmp(class, FM_LIST_SUSPECT_CLASS) == 0) { fmd_case_uuclose(hdl, uuid); } } } if (!keepopen && strcmp(class, FM_LIST_REPAIRED_CLASS) == 0) fmd_case_uuresolved(hdl, uuid); } static void cma_recv_one(fmd_hdl_t *hdl, nvlist_t *nvl) { const cma_subscriber_t *subr; nvlist_t *asru; if ((subr = nvl2subr(hdl, nvl, &asru)) == NULL) return; if (subr->subr_func != NULL) { if (fmd_nvl_fmri_has_fault(hdl, asru, FMD_HAS_FAULT_ASRU, NULL) == 1) (void) subr->subr_func(hdl, nvl, asru, NULL, 0); } } /*ARGSUSED*/ static void cma_recv(fmd_hdl_t *hdl, fmd_event_t *ep, nvlist_t *nvl, const char *class) { fmd_hdl_debug(hdl, "received %s\n", class); if (strcmp(class, FM_LIST_RESOLVED_CLASS) == 0) return; if (strcmp(class, FM_LIST_SUSPECT_CLASS) == 0 || strcmp(class, FM_LIST_REPAIRED_CLASS) == 0 || strcmp(class, FM_LIST_UPDATED_CLASS) == 0) cma_recv_list(hdl, nvl, class); else cma_recv_one(hdl, nvl); } /*ARGSUSED*/ static void cma_timeout(fmd_hdl_t *hdl, id_t id, void *arg) { if (id == cma.cma_page_timerid) cma_page_retry(hdl); #ifdef sun4v /* * cpu offline/online needs to be retried on sun4v because * ldom request can be asynchronous. */ else if (id == cma.cma_cpu_timerid) cma_cpu_retry(hdl); #endif } #ifdef sun4v static void * cma_init_alloc(size_t size) { return (fmd_hdl_alloc(init_hdl, size, FMD_SLEEP)); } static void cma_init_free(void *addr, size_t size) { fmd_hdl_free(init_hdl, addr, size); } #endif static const fmd_hdl_ops_t fmd_ops = { cma_recv, /* fmdo_recv */ cma_timeout, /* fmdo_timeout */ NULL, /* fmdo_close */ NULL, /* fmdo_stats */ NULL, /* fmdo_gc */ }; static const fmd_prop_t fmd_props[] = { { "cpu_tries", FMD_TYPE_UINT32, "10" }, { "cpu_delay", FMD_TYPE_TIME, "1sec" }, #ifdef sun4v { "cpu_ret_mindelay", FMD_TYPE_TIME, "5sec" }, { "cpu_ret_maxdelay", FMD_TYPE_TIME, "5min" }, #endif /* sun4v */ { "cpu_offline_enable", FMD_TYPE_BOOL, "true" }, { "cpu_online_enable", FMD_TYPE_BOOL, "true" }, { "cpu_forced_offline", FMD_TYPE_BOOL, "true" }, #ifdef opl { "cpu_blacklist_enable", FMD_TYPE_BOOL, "false" }, { "cpu_unblacklist_enable", FMD_TYPE_BOOL, "false" }, #else { "cpu_blacklist_enable", FMD_TYPE_BOOL, "true" }, { "cpu_unblacklist_enable", FMD_TYPE_BOOL, "true" }, #endif /* opl */ { "page_ret_mindelay", FMD_TYPE_TIME, "1sec" }, { "page_ret_maxdelay", FMD_TYPE_TIME, "5min" }, { "page_retire_enable", FMD_TYPE_BOOL, "true" }, { "page_unretire_enable", FMD_TYPE_BOOL, "true" }, { NULL, 0, NULL } }; static const fmd_hdl_info_t fmd_info = { "CPU/Memory Retire Agent", CMA_VERSION, &fmd_ops, fmd_props }; void _fmd_init(fmd_hdl_t *hdl) { hrtime_t nsec; #ifdef i386 char buf[BUFSIZ]; /* * Abort the cpumem-retire module if not running on amd64/i86pc. */ if (sysinfo(SI_PLATFORM, buf, sizeof (buf)) == -1) return; if (strncmp(buf, "amd64", sizeof (buf)) == 0 || strncmp(buf, "i86pc", sizeof (buf)) == 0) { cma_is_native = B_TRUE; } else { return; } #endif /* i386 */ if (fmd_hdl_register(hdl, FMD_API_VERSION, &fmd_info) != 0) return; /* invalid data in configuration file */ fmd_hdl_subscribe(hdl, "fault.cpu.*"); fmd_hdl_subscribe(hdl, "fault.memory.*"); #ifdef opl fmd_hdl_subscribe(hdl, "fault.chassis.SPARC-Enterprise.cpu.*"); #endif (void) fmd_stat_create(hdl, FMD_STAT_NOALLOC, sizeof (cma_stats) / sizeof (fmd_stat_t), (fmd_stat_t *)&cma_stats); cma.cma_cpu_tries = fmd_prop_get_int32(hdl, "cpu_tries"); nsec = fmd_prop_get_int64(hdl, "cpu_delay"); cma.cma_cpu_delay.tv_sec = nsec / NANOSEC; cma.cma_cpu_delay.tv_nsec = nsec % NANOSEC; cma.cma_page_mindelay = fmd_prop_get_int64(hdl, "page_ret_mindelay"); cma.cma_page_maxdelay = fmd_prop_get_int64(hdl, "page_ret_maxdelay"); #ifdef sun4v cma.cma_cpu_mindelay = fmd_prop_get_int64(hdl, "cpu_ret_mindelay"); cma.cma_cpu_maxdelay = fmd_prop_get_int64(hdl, "cpu_ret_maxdelay"); #endif cma.cma_cpu_dooffline = fmd_prop_get_int32(hdl, "cpu_offline_enable"); cma.cma_cpu_forcedoffline = fmd_prop_get_int32(hdl, "cpu_forced_offline"); cma.cma_cpu_doonline = fmd_prop_get_int32(hdl, "cpu_online_enable"); cma.cma_cpu_doblacklist = fmd_prop_get_int32(hdl, "cpu_blacklist_enable"); cma.cma_cpu_dounblacklist = fmd_prop_get_int32(hdl, "cpu_unblacklist_enable"); cma.cma_page_doretire = fmd_prop_get_int32(hdl, "page_retire_enable"); cma.cma_page_dounretire = fmd_prop_get_int32(hdl, "page_unretire_enable"); if (cma.cma_page_maxdelay < cma.cma_page_mindelay) fmd_hdl_abort(hdl, "page retirement delays conflict\n"); #ifdef sun4v init_hdl = hdl; cma_lhp = ldom_init(cma_init_alloc, cma_init_free); #endif } void _fmd_fini(fmd_hdl_t *hdl) { #ifdef sun4v ldom_fini(cma_lhp); cma_cpu_fini(hdl); #endif cma_page_fini(hdl); } /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2008 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ /* * Page retirement can be an extended process due to the fact that a retirement * may not be possible when the original request is made. The kernel will * repeatedly attempt to retire a given page, but will not let us know when the * page has been retired. We therefore have to poll to see if the retirement * has been completed. This poll is implemented with a bounded exponential * backoff to reduce the burden which we impose upon the system. * * To reduce the burden on fmd in the face of retirement storms, we schedule * all retries as a group. In the simplest case, we attempt to retire a single * page. When forced to retry, we initially schedule a retry at a configurable * interval t. If the retry fails, we schedule another at 2 * t, and so on, * until t reaches the maximum interval (also configurable). Future retries * for that page will occur with t equal to the maximum interval value. We * will never give up on a retirement. * * With multiple retirements, the situation gets slightly more complicated. As * indicated above, we schedule retries as a group. We don't want to deny new * pages their short retry intervals, so we'll (re)set the retry interval to the * value appropriate for the newest page. */ #include #include #include #include #include #include #include #include #include #include static void cma_page_free(fmd_hdl_t *hdl, cma_page_t *page) { nvlist_free(page->pg_asru); nvlist_free(page->pg_rsrc); fmd_hdl_free(hdl, page, sizeof (cma_page_t)); } /* * Retire the specified ASRU, referring to a memory page by PA or by DIMM * offset (i.e. the encoded coordinates internal bank, row, and column). * In the initial FMA implementation, fault.memory.page exported an ASRU * with an explicit physical address, which is valid at the initial time of * diagnosis but may not be later following DR, DIMM removal, or interleave * changes. On SPARC, this issue was solved by exporting the DIMM offset * and pushing the entire FMRI to the platform memory controller through * /dev/fm so it can derive the current PA from the DIMM and offset. * On x86, we also encode DIMM and offset in hc-specific, which is then used * by the x64 memory controller driver. * At some point these three approaches need to be rationalized: all platforms * should use the same scheme, either with decoding in the kernel or decoding * in userland (i.e. with a libtopo method to compute and update the PA). */ /*ARGSUSED*/ int cma_page_retire(fmd_hdl_t *hdl, nvlist_t *nvl, nvlist_t *asru, const char *uuid, boolean_t repair) { cma_page_t *page; uint64_t pageaddr; const char *action = repair ? "unretire" : "retire"; int rc; nvlist_t *rsrc = NULL, *asrucp = NULL, *hcsp; (void) nvlist_lookup_nvlist(nvl, FM_FAULT_RESOURCE, &rsrc); if (nvlist_dup(asru, &asrucp, 0) != 0) { fmd_hdl_debug(hdl, "page retire nvlist dup failed\n"); return (CMA_RA_FAILURE); } /* It should already be expanded, but we'll do it again anyway */ if (fmd_nvl_fmri_expand(hdl, asrucp) < 0) { fmd_hdl_debug(hdl, "failed to expand page asru\n"); cma_stats.bad_flts.fmds_value.ui64++; nvlist_free(asrucp); return (CMA_RA_FAILURE); } if (!repair && !fmd_nvl_fmri_present(hdl, asrucp)) { fmd_hdl_debug(hdl, "page retire overtaken by events\n"); cma_stats.page_nonent.fmds_value.ui64++; nvlist_free(asrucp); return (CMA_RA_SUCCESS); } /* Figure out physaddr from resource or asru */ if (rsrc == NULL || nvlist_lookup_nvlist(rsrc, FM_FMRI_HC_SPECIFIC, &hcsp) != 0 || (nvlist_lookup_uint64(hcsp, "asru-" FM_FMRI_HC_SPECIFIC_PHYSADDR, &pageaddr) != 0 && nvlist_lookup_uint64(hcsp, FM_FMRI_HC_SPECIFIC_PHYSADDR, &pageaddr) != 0)) { if (nvlist_lookup_uint64(asrucp, FM_FMRI_MEM_PHYSADDR, &pageaddr) != 0) { fmd_hdl_debug(hdl, "mem fault missing 'physaddr'\n"); cma_stats.bad_flts.fmds_value.ui64++; nvlist_free(asrucp); return (CMA_RA_FAILURE); } } if (repair) { if (!cma.cma_page_dounretire) { fmd_hdl_debug(hdl, "suppressed unretire of page %llx\n", (u_longlong_t)pageaddr); cma_stats.page_supp.fmds_value.ui64++; nvlist_free(asrucp); return (CMA_RA_SUCCESS); } /* If unretire via topo fails, we fall back to legacy way */ if (rsrc == NULL || (rc = fmd_nvl_fmri_unretire(hdl, rsrc)) < 0) rc = cma_fmri_page_unretire(hdl, asrucp); } else { if (!cma.cma_page_doretire) { fmd_hdl_debug(hdl, "suppressed retire of page %llx\n", (u_longlong_t)pageaddr); cma_stats.page_supp.fmds_value.ui64++; nvlist_free(asrucp); return (CMA_RA_FAILURE); } /* If retire via topo fails, we fall back to legacy way */ if (rsrc == NULL || (rc = fmd_nvl_fmri_retire(hdl, rsrc)) < 0) rc = cma_fmri_page_retire(hdl, asrucp); } if (rc == FMD_AGENT_RETIRE_DONE) { fmd_hdl_debug(hdl, "%sd page 0x%llx\n", action, (u_longlong_t)pageaddr); if (repair) cma_stats.page_repairs.fmds_value.ui64++; else cma_stats.page_flts.fmds_value.ui64++; nvlist_free(asrucp); return (CMA_RA_SUCCESS); } else if (repair || rc != FMD_AGENT_RETIRE_ASYNC) { fmd_hdl_debug(hdl, "%s of page 0x%llx failed, will not " "retry: %s\n", action, (u_longlong_t)pageaddr, strerror(errno)); cma_stats.page_fails.fmds_value.ui64++; nvlist_free(asrucp); return (CMA_RA_FAILURE); } /* * The page didn't immediately retire. We'll need to periodically * check to see if it has been retired. */ fmd_hdl_debug(hdl, "page didn't retire - sleeping\n"); page = fmd_hdl_zalloc(hdl, sizeof (cma_page_t), FMD_SLEEP); page->pg_addr = pageaddr; if (rsrc != NULL) (void) nvlist_dup(rsrc, &page->pg_rsrc, 0); page->pg_asru = asrucp; if (uuid != NULL) page->pg_uuid = fmd_hdl_strdup(hdl, uuid, FMD_SLEEP); page->pg_next = cma.cma_pages; cma.cma_pages = page; if (cma.cma_page_timerid != 0) fmd_timer_remove(hdl, cma.cma_page_timerid); cma.cma_page_curdelay = cma.cma_page_mindelay; cma.cma_page_timerid = fmd_timer_install(hdl, NULL, NULL, cma.cma_page_curdelay); /* Don't free asrucp here. This FMRI will be needed for retry. */ return (CMA_RA_FAILURE); } static int page_retry(fmd_hdl_t *hdl, cma_page_t *page) { int rc; if (page->pg_asru != NULL && !fmd_nvl_fmri_present(hdl, page->pg_asru)) { fmd_hdl_debug(hdl, "page retire overtaken by events"); cma_stats.page_nonent.fmds_value.ui64++; if (page->pg_uuid != NULL) fmd_case_uuclose(hdl, page->pg_uuid); return (1); /* no longer a page to retire */ } if (page->pg_rsrc == NULL || (rc = fmd_nvl_fmri_service_state(hdl, page->pg_rsrc)) < 0) rc = cma_fmri_page_service_state(hdl, page->pg_asru); if (rc == FMD_SERVICE_STATE_UNUSABLE) { fmd_hdl_debug(hdl, "retired page 0x%llx on retry %u\n", page->pg_addr, page->pg_nretries); cma_stats.page_flts.fmds_value.ui64++; if (page->pg_uuid != NULL) fmd_case_uuclose(hdl, page->pg_uuid); return (1); /* page retired */ } if (rc == FMD_SERVICE_STATE_ISOLATE_PENDING) { fmd_hdl_debug(hdl, "scheduling another retry for 0x%llx\n", page->pg_addr); return (0); /* schedule another retry */ } else { fmd_hdl_debug(hdl, "failed to retry page 0x%llx " "retirement: %s\n", page->pg_addr, strerror(errno)); cma_stats.page_fails.fmds_value.ui64++; return (1); /* give up */ } } void cma_page_retry(fmd_hdl_t *hdl) { cma_page_t **pagep; cma.cma_page_timerid = 0; fmd_hdl_debug(hdl, "page_retry: timer fired\n"); pagep = &cma.cma_pages; while (*pagep != NULL) { cma_page_t *page = *pagep; if (page_retry(hdl, page)) { /* * Successful retry or we're giving up - remove from * the list */ *pagep = page->pg_next; if (page->pg_uuid != NULL) fmd_hdl_strfree(hdl, page->pg_uuid); cma_page_free(hdl, page); } else { page->pg_nretries++; pagep = &page->pg_next; } } if (cma.cma_pages == NULL) return; /* no more retirements */ /* * We still have retirements that haven't completed. Back the delay * off, and schedule a retry. */ cma.cma_page_curdelay = MIN(cma.cma_page_curdelay * 2, cma.cma_page_maxdelay); fmd_hdl_debug(hdl, "scheduled page retirement retry for %llu secs\n", (u_longlong_t)(cma.cma_page_curdelay / NANOSEC)); cma.cma_page_timerid = fmd_timer_install(hdl, NULL, NULL, cma.cma_page_curdelay); } void cma_page_fini(fmd_hdl_t *hdl) { cma_page_t *page; while ((page = cma.cma_pages) != NULL) { cma.cma_pages = page->pg_next; if (page->pg_uuid != NULL) fmd_hdl_strfree(hdl, page->pg_uuid); cma_page_free(hdl, page); } } /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2008 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ #include #include #include /* ARGSUSED */ int cma_fmri_page_service_state(fmd_hdl_t *hdl, nvlist_t *nvl) { fmd_agent_hdl_t *fa_hdl; int rc; if ((fa_hdl = fmd_agent_open(FMD_AGENT_VERSION)) != NULL) { rc = fmd_agent_page_isretired(fa_hdl, nvl); if (rc == FMD_AGENT_RETIRE_DONE) rc = FMD_SERVICE_STATE_UNUSABLE; else if (rc == FMD_AGENT_RETIRE_FAIL) rc = FMD_SERVICE_STATE_OK; else if (rc == FMD_AGENT_RETIRE_ASYNC) rc = FMD_SERVICE_STATE_ISOLATE_PENDING; fmd_agent_close(fa_hdl); return (rc); } return (FMD_SERVICE_STATE_UNKNOWN); } /* ARGSUSED */ int cma_fmri_page_retire(fmd_hdl_t *hdl, nvlist_t *nvl) { fmd_agent_hdl_t *fa_hdl; int rc; if ((fa_hdl = fmd_agent_open(FMD_AGENT_VERSION)) != NULL) { rc = fmd_agent_page_retire(fa_hdl, nvl); fmd_agent_close(fa_hdl); return (rc); } return (FMD_AGENT_RETIRE_FAIL); } /* ARGSUSED */ int cma_fmri_page_unretire(fmd_hdl_t *hdl, nvlist_t *nvl) { fmd_agent_hdl_t *fa_hdl; int rc; if ((fa_hdl = fmd_agent_open(FMD_AGENT_VERSION)) != NULL) { rc = fmd_agent_page_unretire(fa_hdl, nvl); fmd_agent_close(fa_hdl); return (rc); } return (FMD_AGENT_RETIRE_FAIL); } # # CDDL HEADER START # # The contents of this file are subject to the terms of the # Common Development and Distribution License, Version 1.0 only # (the "License"). You may not use this file except in compliance # with the License. # # You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE # or http://www.opensolaris.org/os/licensing. # See the License for the specific language governing permissions # and limitations under the License. # # When distributing Covered Code, include this CDDL HEADER in each # file and include the License file at usr/src/OPENSOLARIS.LICENSE. # If applicable, add the following below this CDDL HEADER, with the # fields enclosed by brackets "[]" replaced with your own identifying # information: Portions Copyright [yyyy] [name of copyright owner] # # CDDL HEADER END # # # Copyright 2004 Sun Microsystems, Inc. All rights reserved. # Use is subject to license terms. # #ident "%Z%%M% %I% %E% SMI" # # cpumem-retire # setprop cpu_tries 10 setprop cpu_delay 1sec # setprop page_ret_mindelay 1sec setprop page_ret_maxdelay 5min # setprop cpu_offline_enable true setprop cpu_forced_offline true setprop cpu_blacklist_enable true setprop page_retire_enable true