cpumem-retire - zygaena/hammerhead

root / base / usr / src / cmd / fm / modules / common / cpumem-retire
cpumem-retire Plain Text 1755 lines 53.2 KB
#
# CDDL HEADER START
#
# The contents of this file are subject to the terms of the
# Common Development and Distribution License (the "License").
# You may not use this file except in compliance with the License.
#
# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
# or http://www.opensolaris.org/os/licensing.
# See the License for the specific language governing permissions
# and limitations under the License.
#
# When distributing Covered Code, include this CDDL HEADER in each
# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
# If applicable, add the following below this CDDL HEADER, with the
# fields enclosed by brackets "[]" replaced with your own identifying
# information: Portions Copyright [yyyy] [name of copyright owner]
#
# CDDL HEADER END
#
#
# Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
# Use is subject to license terms.
#
# Copyright 2022 Garrett D'Amore
#

MODULE = cpumem-retire
CLASS = common
SRCS = \
	cma_main.c \
	cma_cpu.c \
	cma_cpu_arch.c \
	cma_page.c \
	cma_page_arch.c

include ../../Makefile.plugin

CMA_VERSION = "1.1"

INCDIRS = .

CPPFLAGS += $(INCDIRS:%=-I%) -DCMA_VERSION='$(CMA_VERSION)'

LDFLAGS += -R/usr/lib/fm
LDLIBS += -L$(ROOTLIB)/fm -lfmd_agent
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#ifndef _CMA_H
#define	_CMA_H

#include <fm/fmd_api.h>
#include <sys/types.h>

#ifdef __cplusplus
extern "C" {
#endif

#define	CMA_RA_SUCCESS	0
#define	CMA_RA_FAILURE	1

#ifdef opl
#define	FM_FMRI_HC_CPUIDS	"hc-xscf-cpuids"
#endif

#ifdef i386
extern boolean_t cma_is_native;
#endif

typedef struct cma_page {
	struct cma_page *pg_next;	/* List of page retirements for retry */
	nvlist_t *pg_rsrc;		/* Resource for this page */
	nvlist_t *pg_asru;		/* ASRU for this page */
	uint64_t pg_addr;		/* Address of this page */
	char *pg_uuid;			/* UUID for this page's case */
	uint_t pg_nretries;		/* Number of retries so far for page */
} cma_page_t;

#ifdef sun4v
typedef struct cma_cpu {
	struct cma_cpu *cpu_next;	/* List of cpus */
	nvlist_t *cpu_fmri;		/* FMRI for this cpu entry  */
	int cpuid;			/* physical id of this cpu */
	char *cpu_uuid;			/* UUID for this cpu's case */
	uint_t cpu_nretries;		/* Number of retries so far for cpu */
} cma_cpu_t;
#endif /* sun4v */

typedef struct cma {
	struct timespec cma_cpu_delay;	/* CPU offline retry interval */
	uint_t cma_cpu_tries;		/* Number of CPU offline retries */
	uint_t cma_cpu_dooffline;	/* Whether to offline CPUs */
	uint_t cma_cpu_forcedoffline;	/* Whether to do forced CPU offline */
	uint_t cma_cpu_doonline;	/* Whether to online CPUs */
	uint_t cma_cpu_doblacklist;	/* Whether to blacklist CPUs */
	uint_t cma_cpu_dounblacklist;	/* Whether to unblacklist CPUs */
	cma_page_t *cma_pages;		/* List of page retirements for retry */
	hrtime_t cma_page_curdelay;	/* Current retry sleep interval */
	hrtime_t cma_page_mindelay;	/* Minimum retry sleep interval */
	hrtime_t cma_page_maxdelay;	/* Maximum retry sleep interval */
	id_t cma_page_timerid;		/* fmd timer ID for retry sleep */
	uint_t cma_page_doretire;	/* Whether to retire pages */
	uint_t cma_page_dounretire;	/* Whether to unretire pages */
#ifdef sun4v
	cma_cpu_t *cma_cpus;		/* List of cpus */
	hrtime_t cma_cpu_curdelay;	/* Current retry sleep interval */
	hrtime_t cma_cpu_mindelay;	/* Minimum retry sleep interval */
	hrtime_t cma_cpu_maxdelay;	/* Maximum retry sleep interval */
	id_t cma_cpu_timerid;		/* LDOM cpu timer */
#endif /* sun4v */
} cma_t;

typedef struct cma_stats {
	fmd_stat_t cpu_flts;		/* Successful offlines */
	fmd_stat_t cpu_repairs;		/* Successful onlines */
	fmd_stat_t cpu_fails;		/* Failed offlines/onlines */
	fmd_stat_t cpu_blfails;		/* Failed blacklists */
	fmd_stat_t cpu_supp;		/* Suppressed offlines/onlines */
	fmd_stat_t cpu_blsupp;		/* Suppressed blacklists */
	fmd_stat_t page_flts;		/* Successful page retires */
	fmd_stat_t page_repairs;	/* Successful page unretires */
	fmd_stat_t page_fails;		/* Failed page retires/unretires */
	fmd_stat_t page_supp;		/* Suppressed retires/unretires */
	fmd_stat_t page_nonent;		/* Retires for non-present pages */
	fmd_stat_t bad_flts;		/* Malformed faults */
	fmd_stat_t nop_flts;		/* Inapplicable faults */
	fmd_stat_t auto_flts;		/* Auto-close faults */
} cma_stats_t;

extern cma_stats_t cma_stats;
extern cma_t cma;

extern int cma_cpu_cpu_retire(fmd_hdl_t *, nvlist_t *, nvlist_t *,
    const char *, boolean_t);
extern int cma_cpu_hc_retire(fmd_hdl_t *, nvlist_t *, nvlist_t *,
    const char *, boolean_t);
extern int cma_page_retire(fmd_hdl_t *, nvlist_t *, nvlist_t *,
    const char *, boolean_t);
extern void cma_page_retry(fmd_hdl_t *);
extern void cma_page_fini(fmd_hdl_t *);
extern int cma_set_errno(int);

extern int cma_cache_way_retire(fmd_hdl_t *, nvlist_t *, nvlist_t *,
    const char *, boolean_t);
/*
 * Platforms may have their own implementations of these functions
 */
extern int cma_cpu_blacklist(fmd_hdl_t *, nvlist_t *, nvlist_t *, boolean_t);
extern int cma_cpu_statechange(fmd_hdl_t *, nvlist_t *, const char *, int,
    boolean_t);
extern int cma_fmri_page_service_state(fmd_hdl_t *, nvlist_t *);
extern int cma_fmri_page_retire(fmd_hdl_t *, nvlist_t *);
extern int cma_fmri_page_unretire(fmd_hdl_t *, nvlist_t *);

#ifdef sun4v
extern void cma_cpu_start_retry(fmd_hdl_t *, nvlist_t *, const char *,
    boolean_t);
extern void cma_cpu_fini(fmd_hdl_t *);
extern void cma_cpu_retry(fmd_hdl_t *);
#endif /* sun4v */

extern const char *p_online_state_fmt(int);

#ifdef __cplusplus
}
#endif

#endif /* _CMA_H */
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 *
 * Copyright 2019 Joyent, Inc.
 */

#include <cma.h>

#include <fcntl.h>
#include <unistd.h>
#include <strings.h>
#include <errno.h>
#include <time.h>
#include <fm/fmd_api.h>
#include <fm/fmd_agent.h>
#include <sys/fm/protocol.h>
#include <sys/bl.h>
#include <sys/processor.h>

static int cpu_statechange(fmd_hdl_t *, nvlist_t *, nvlist_t *, const char *,
    uint32_t, boolean_t);

#ifndef opl
/*
 * Perform retire/unretire by invoking the topo methods registered in the
 * hc-scheme resource.
 *
 * If the fault is found to be diagnosed under the old topology, the resource
 * will not exist in the current topology, then we fall back to legacy retire
 * (using the "cpu" scheme ASRU).
 */

static boolean_t
old_topo_fault(nvlist_t *nvl)
{
	nvlist_t *rsrc;
#ifdef i386
	nvlist_t **hcl;
	uint_t nhcl = 0;
	char *name;
#endif

	if (nvlist_lookup_nvlist(nvl, FM_FAULT_RESOURCE, &rsrc) != 0)
		return (B_TRUE);
#ifdef i386
	/*
	 * x86 has moved from "motherboard/chip/cpu" topo to
	 * "motherboard/chip/core/strand"
	 */
	if (nvlist_lookup_nvlist_array(rsrc, FM_FMRI_HC_LIST, &hcl, &nhcl)
	    == 0 && nhcl == 3 &&
	    nvlist_lookup_string(hcl[0], FM_FMRI_HC_NAME, &name) == 0 &&
	    strcmp(name, "motherboard") == 0 &&
	    nvlist_lookup_string(hcl[1], FM_FMRI_HC_NAME, &name) == 0 &&
	    strcmp(name, "chip") == 0 &&
	    nvlist_lookup_string(hcl[2], FM_FMRI_HC_NAME, &name) == 0 &&
	    strcmp(name, "cpu") == 0)
		return (B_TRUE);
#endif

	return (B_FALSE);
}

/* ARGSUSED */
int
cma_cpu_hc_retire(fmd_hdl_t *hdl, nvlist_t *nvl, nvlist_t *asru,
    const char *uuid, boolean_t repair)
{
	int i, err;
	int rc = CMA_RA_SUCCESS;
	nvlist_t *rsrc;

	/*
	 * For the cached faults which were diagnosed under the old
	 * topology,  we fall back to retire by using cpu-scheme ASRUs.
	 * Under xVM Dom0, since logic cpuid in "cpu" scheme ASRU makes no
	 * sense, the fault should be ignored.
	 */
	if (old_topo_fault(nvl)) {
#ifdef i386
		if (! cma_is_native)
			return (CMA_RA_FAILURE);
#endif
		return (cma_cpu_cpu_retire(hdl, nvl, asru, uuid, repair));
	}

	/*
	 * Lookup the resource and call its topo methods to do retire/unretire
	 */
	if ((! repair && ! cma.cma_cpu_dooffline) ||
	    (repair && ! cma.cma_cpu_doonline)) {
		fmd_hdl_debug(hdl, "suppressed %s of CPU\n",
		    repair ? "unretire" : "retire");
		cma_stats.cpu_supp.fmds_value.ui64++;
	} else {
		err = FMD_AGENT_RETIRE_FAIL;
		if (nvlist_lookup_nvlist(nvl, FM_FAULT_RESOURCE, &rsrc) == 0) {
			if (repair) {
				err = fmd_nvl_fmri_unretire(hdl, rsrc);
			} else {
				for (i = 0; i < cma.cma_cpu_tries; i++) {
					err = fmd_nvl_fmri_retire(hdl, rsrc);
					if (err == FMD_AGENT_RETIRE_DONE)
						break;
					(void) nanosleep(&cma.cma_cpu_delay,
					    NULL);
				}
			}
		}
		if (err == FMD_AGENT_RETIRE_DONE) {
			if (repair)
				cma_stats.cpu_repairs.fmds_value.ui64++;
			else
				cma_stats.cpu_flts.fmds_value.ui64++;
		} else {
			rc = CMA_RA_FAILURE;
			cma_stats.bad_flts.fmds_value.ui64++;
#ifdef sun4v
			/* libldom requests are processed asynchronously */
			cma_cpu_start_retry(hdl, nvl, uuid, repair);
#endif
		}
	}

	if ((! repair && ! cma.cma_cpu_doblacklist) ||
	    (repair && ! cma.cma_cpu_dounblacklist)) {
		fmd_hdl_debug(hdl, "suppressed %s of CPU\n",
		    repair ? "unblacklist" : "blacklist");
		cma_stats.cpu_blsupp.fmds_value.ui64++;
	} else {
		if (cma_cpu_blacklist(hdl, nvl, asru, repair) < 0)
			cma_stats.cpu_blfails.fmds_value.ui64++;
	}

	return (rc);
}

#else /* opl */

/* ARGSUSED 4 */
int
cma_cpu_hc_retire(fmd_hdl_t *hdl, nvlist_t *nvl, nvlist_t *asru,
    const char *uuid, boolean_t repair)
{
	uint_t cpuid;
	uint_t i, nprs;
	nvlist_t **hc_prs = NULL, *hc_spec_nvl;

	/* OPL has ASRU in "hc" scheme */
	if (nvlist_lookup_nvlist(asru, FM_FMRI_HC_SPECIFIC,
	    &hc_spec_nvl) != 0) {
		cma_stats.bad_flts.fmds_value.ui64++;
		fmd_hdl_debug(hdl,
		    "cma_cpu_hc_retire lookup hc_spec_nvl failed\n");
		return (CMA_RA_FAILURE);
	}

	if (nvlist_lookup_nvlist_array(hc_spec_nvl, FM_FMRI_HC_CPUIDS,
	    &hc_prs, &nprs) != 0) {
		cma_stats.bad_flts.fmds_value.ui64++;
		fmd_hdl_debug(hdl,
		    "cma_cpu_hc_retire lookup cpuid array failed\n");
		return (CMA_RA_FAILURE);
	}

	for (i = 0; i < nprs; i++) {
		if (nvlist_lookup_uint32(hc_prs[i],
		    FM_FMRI_CPU_ID, &cpuid) != 0) {
			cma_stats.bad_flts.fmds_value.ui64++;
			return (CMA_RA_FAILURE);
		}

		if (cpu_statechange(hdl, nvl, hc_prs[i], uuid, cpuid, repair)
		    != CMA_RA_SUCCESS) {
			cma_stats.bad_flts.fmds_value.ui64++;
			return (CMA_RA_FAILURE);
		}
	}

	return (CMA_RA_SUCCESS);
}
#endif /* opl */

/*
 * The rest of this file uses ASRUs to do retire, this is now not the
 * preferable way, but it's still needed for some circumstances when
 * retire via topo methods can't work, ie.
 *
 * 1) There are legacy platforms which don't have full topology.
 * 2) The resources in the FMD cached faults may not be set or exist in the
 *    up-to-dated topology.
 */

/* ARGSUSED */
static int
cpu_online(fmd_hdl_t *hdl, nvlist_t *nvl, nvlist_t *asru, const char *uuid,
    uint32_t cpuid)
{
	int err = CMA_RA_SUCCESS;

	if (cma.cma_cpu_doonline) {
		err = cma_cpu_statechange(hdl, asru, uuid, P_ONLINE,
		    B_TRUE);
	} else {
		fmd_hdl_debug(hdl, "suppressed online of CPU %u\n",
		    cpuid);
		cma_stats.cpu_supp.fmds_value.ui64++;
	}

	/* OPL performs the blacklist in the service processor */
#ifndef opl
	if (cma.cma_cpu_dounblacklist) {
		if (cma_cpu_blacklist(hdl, nvl, asru, B_TRUE) < 0)
			cma_stats.cpu_blfails.fmds_value.ui64++;
	} else {
		fmd_hdl_debug(hdl, "suppressed unblacklist of CPU %u\n", cpuid);
		cma_stats.cpu_blsupp.fmds_value.ui64++;
	}
#endif /* opl */

	return (err);
}

/* ARGSUSED */
static int
cpu_offline(fmd_hdl_t *hdl, nvlist_t *nvl, nvlist_t *asru, const char *uuid,
    uint32_t cpuid)
{
	int err = CMA_RA_FAILURE;

	if (cma.cma_cpu_dooffline) {
		int cpustate = P_FAULTED;

		if (cma.cma_cpu_forcedoffline)
			cpustate |= P_FORCED;
		err = cma_cpu_statechange(hdl, asru, uuid, cpustate,
		    B_FALSE);
	} else {
		fmd_hdl_debug(hdl, "suppressed offline of CPU %u\n",
		    cpuid);
		cma_stats.cpu_supp.fmds_value.ui64++;
	}

	/* OPL performs the blacklist in the service processor */
#ifndef opl
	if (cma.cma_cpu_doblacklist) {
		if (cma_cpu_blacklist(hdl, nvl, asru, B_FALSE) < 0)
			cma_stats.cpu_blfails.fmds_value.ui64++;
	} else {
		fmd_hdl_debug(hdl, "suppressed blacklist of CPU %u\n",
		    cpuid);
		cma_stats.cpu_blsupp.fmds_value.ui64++;
	}
#endif /* opl */

	return (err);
}

static int
cpu_statechange(fmd_hdl_t *hdl, nvlist_t *nvl, nvlist_t *asru, const char *uuid,
    uint32_t cpuid, boolean_t repair)
{
	if (repair)
		return (cpu_online(hdl, nvl, asru, uuid, cpuid));
	else
		return (cpu_offline(hdl, nvl, asru, uuid, cpuid));
}

const char *
p_online_state_fmt(int state)
{
	state &= ~P_FORCED;
	switch (state) {
	case P_OFFLINE:
		return (PS_OFFLINE);
	case P_ONLINE:
		return (PS_ONLINE);
	case P_FAULTED:
		return (PS_FAULTED);
	case P_POWEROFF:
		return (PS_POWEROFF);
	case P_NOINTR:
		return (PS_NOINTR);
	case P_SPARE:
		return (PS_SPARE);
	case P_DISABLED:
		return (PS_DISABLED);
	default:
		return ("unknown");
	}
}

int
cma_cpu_cpu_retire(fmd_hdl_t *hdl, nvlist_t *nvl, nvlist_t *asru,
    const char *uuid, boolean_t repair)
{
	uint_t cpuid;

	if (nvlist_lookup_uint32(asru, FM_FMRI_CPU_ID, &cpuid) != 0) {
		fmd_hdl_debug(hdl, "cpu fault missing '%s'\n", FM_FMRI_CPU_ID);
		cma_stats.bad_flts.fmds_value.ui64++;
		return (CMA_RA_FAILURE);
	}

	return (cpu_statechange(hdl, nvl, asru, uuid, cpuid, repair));
}
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#include <cma.h>

#include <fcntl.h>
#include <unistd.h>
#include <strings.h>
#include <errno.h>
#include <time.h>
#include <fm/fmd_api.h>
#include <sys/fm/protocol.h>
#include <sys/bl.h>
#include <sys/processor.h>

int
cma_cpu_blacklist(fmd_hdl_t *hdl, nvlist_t *nvl, nvlist_t *asru,
    boolean_t repair)
{
	bl_req_t blr;
	nvlist_t *fmri;
	char *fmribuf;
	size_t fmrisz;
	int fd, rc, err;
	char *class;

	/*
	 * Some platforms have special unums for the E$ DIMMs.  If we're dealing
	 * with a platform that has these unums, one will have been added to the
	 * fault as the resource.  We'll use that for the blacklisting.  If we
	 * can't find a resource, we'll fall back to the ASRU.
	 */
	if (nvlist_lookup_nvlist(nvl, FM_FAULT_RESOURCE, &fmri) != 0)
		fmri = asru;

	if ((nvlist_lookup_string(nvl, FM_CLASS, &class) != 0) ||
	    (class == NULL) || (*class == '\0')) {
		fmd_hdl_debug(hdl, "failed to get the fault class name\n");
		errno = EINVAL;
		return (-1);
	}

	if ((fd = open("/dev/bl", O_RDONLY)) < 0)
		return (-1); /* errno is set for us */

	if ((errno = nvlist_size(fmri, &fmrisz, NV_ENCODE_NATIVE)) != 0 ||
	    (fmribuf = fmd_hdl_alloc(hdl, fmrisz, FMD_SLEEP)) == NULL) {
		(void) close(fd);
		return (-1); /* errno is set for us */
	}

	if ((errno = nvlist_pack(fmri, &fmribuf, &fmrisz,
	    NV_ENCODE_NATIVE, 0)) != 0) {
		fmd_hdl_free(hdl, fmribuf, fmrisz);
		(void) close(fd);
		return (-1); /* errno is set for us */
	}

	blr.bl_fmri = fmribuf;
	blr.bl_fmrisz = fmrisz;
	blr.bl_class = class;

	rc = ioctl(fd, repair ? BLIOC_DELETE : BLIOC_INSERT, &blr);
	err = errno;

	fmd_hdl_free(hdl, fmribuf, fmrisz);
	(void) close(fd);

	if (rc < 0 && err != ENOTSUP) {
		errno = err;
		return (-1);
	}

	return (0);
}

/* ARGSUSED */
int
cma_cpu_statechange(fmd_hdl_t *hdl, nvlist_t *asru, const char *uuid,
    int cpustate, boolean_t repair)
{
	int i;
	uint_t cpuid;

	if (nvlist_lookup_uint32(asru, FM_FMRI_CPU_ID, &cpuid) != 0) {
		fmd_hdl_debug(hdl, "missing '%s'\n", FM_FMRI_CPU_ID);
		cma_stats.bad_flts.fmds_value.ui64++;
		return (CMA_RA_FAILURE);
	}

	for (i = 0; i < cma.cma_cpu_tries;
	    i++, (void) nanosleep(&cma.cma_cpu_delay, NULL)) {
		int oldstate;
		if ((oldstate = p_online(cpuid, cpustate)) != -1) {
			fmd_hdl_debug(hdl, "changed cpu %u state from \"%s\" "
			    "to \"%s\"\n", cpuid, p_online_state_fmt(oldstate),
			    p_online_state_fmt(cpustate));
			if (repair)
				cma_stats.cpu_repairs.fmds_value.ui64++;
			else
				cma_stats.cpu_flts.fmds_value.ui64++;
			return (CMA_RA_SUCCESS);
		}
	}

	fmd_hdl_debug(hdl, "failed to changed cpu %u state to \"%s\": %s\n",
	    cpuid, p_online_state_fmt(cpustate), strerror(errno));
	cma_stats.cpu_fails.fmds_value.ui64++;
	return (CMA_RA_FAILURE);
}
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
 */

#include <cma.h>

#include <unistd.h>
#include <fcntl.h>
#include <strings.h>
#include <errno.h>
#include <time.h>
#include <fm/fmd_api.h>
#include <sys/fm/protocol.h>
#include <sys/systeminfo.h>
#include <sys/utsname.h>

#ifdef sun4v
#include <sys/fm/ldom.h>

static fmd_hdl_t *init_hdl;
ldom_hdl_t *cma_lhp;
#endif

#ifdef i386
boolean_t cma_is_native;
#endif

extern const char *fmd_fmri_get_platform();

cma_t cma;

cma_stats_t cma_stats = {
	{ "cpu_flts", FMD_TYPE_UINT64, "cpu faults resolved" },
	{ "cpu_repairs", FMD_TYPE_UINT64, "cpu faults repaired" },
	{ "cpu_fails", FMD_TYPE_UINT64, "cpu faults unresolveable" },
	{ "cpu_blfails", FMD_TYPE_UINT64, "failed cpu blacklists" },
	{ "cpu_supp", FMD_TYPE_UINT64, "cpu offlines suppressed" },
	{ "cpu_blsupp", FMD_TYPE_UINT64, "cpu blacklists suppressed" },
	{ "page_flts", FMD_TYPE_UINT64, "page faults resolved" },
	{ "page_repairs", FMD_TYPE_UINT64, "page faults repaired" },
	{ "page_fails", FMD_TYPE_UINT64, "page faults unresolveable" },
	{ "page_supp", FMD_TYPE_UINT64, "page retires suppressed" },
	{ "page_nonent", FMD_TYPE_UINT64, "retires for non-existent fmris" },
	{ "bad_flts", FMD_TYPE_UINT64, "invalid fault events received" },
	{ "nop_flts", FMD_TYPE_UINT64, "inapplicable fault events received" },
	{ "auto_flts", FMD_TYPE_UINT64, "auto-close faults received" }
};

typedef struct cma_subscriber {
	const char *subr_class;
	const char *subr_sname;
	uint_t subr_svers;
	int (*subr_func)(fmd_hdl_t *, nvlist_t *, nvlist_t *, const char *,
	    boolean_t);
} cma_subscriber_t;

static const cma_subscriber_t cma_subrs[] = {
#if defined(i386)
	/*
	 * On x86, the ASRUs are expected to be in hc scheme.  When
	 * cpumem-retire wants to retire a cpu or mem page, it calls the
	 * methods registered in the topo node to do that.  The topo
	 * enumerator, which necessarily knows all the config info that
	 * we'd ever need in deciding what/how to retire etc.  This takes
	 * away much of that complexity from the agent into the entity
	 * that knows all config/topo information.
	 */
	{ "fault.memory.page", FM_FMRI_SCHEME_HC, FM_HC_SCHEME_VERSION,
	    cma_page_retire },
	{ "fault.memory.page_sb", FM_FMRI_SCHEME_HC, FM_HC_SCHEME_VERSION,
	    cma_page_retire },
	{ "fault.memory.page_ck", FM_FMRI_SCHEME_HC, FM_HC_SCHEME_VERSION,
	    cma_page_retire },
	{ "fault.memory.page_ue", FM_FMRI_SCHEME_HC, FM_HC_SCHEME_VERSION,
	    cma_page_retire },
	{ "fault.memory.generic-x86.page_ce", FM_FMRI_SCHEME_HC,
	    FM_HC_SCHEME_VERSION, cma_page_retire },
	{ "fault.memory.generic-x86.page_ue", FM_FMRI_SCHEME_HC,
	    FM_HC_SCHEME_VERSION, cma_page_retire },
	{ "fault.memory.intel.page_ce", FM_FMRI_SCHEME_HC,
	    FM_HC_SCHEME_VERSION, cma_page_retire },
	{ "fault.memory.intel.page_ue", FM_FMRI_SCHEME_HC,
	    FM_HC_SCHEME_VERSION, cma_page_retire },
	{ "fault.memory.dimm", FM_FMRI_SCHEME_HC, FM_HC_SCHEME_VERSION,
	    NULL },
	{ "fault.memory.dimm_sb", FM_FMRI_SCHEME_HC, FM_HC_SCHEME_VERSION,
	    NULL },
	{ "fault.memory.dimm_ck", FM_FMRI_SCHEME_HC, FM_HC_SCHEME_VERSION,
	    NULL },
	{ "fault.memory.dimm_ue", FM_FMRI_SCHEME_HC, FM_HC_SCHEME_VERSION,
	    NULL },
	{ "fault.memory.generic-x86.dimm_ce", FM_FMRI_SCHEME_HC,
	    FM_HC_SCHEME_VERSION, NULL },
	{ "fault.memory.generic-x86.dimm_ue", FM_FMRI_SCHEME_HC,
	    FM_HC_SCHEME_VERSION, NULL },
	{ "fault.memory.intel.dimm_ce", FM_FMRI_SCHEME_HC,
	    FM_HC_SCHEME_VERSION, NULL },
	{ "fault.memory.intel.dimm_ue", FM_FMRI_SCHEME_HC,
	    FM_HC_SCHEME_VERSION, NULL },
	{ "fault.memory.intel.fbd.*", FM_FMRI_SCHEME_HC,
	    FM_HC_SCHEME_VERSION, NULL },
	{ "fault.memory.dimm_testfail", FM_FMRI_SCHEME_HC,
	    FM_HC_SCHEME_VERSION, NULL },
	{ "fault.memory.bank", FM_FMRI_SCHEME_HC, FM_HC_SCHEME_VERSION,
	    NULL },
	{ "fault.memory.datapath", FM_FMRI_SCHEME_HC, FM_HC_SCHEME_VERSION,
	    NULL },
	{ "fault.cpu.intel.quickpath.mem_scrubbing", FM_FMRI_SCHEME_HC,
	    FM_HC_SCHEME_VERSION, cma_page_retire },
	{ "fault.cpu.intel.quickpath.*", FM_FMRI_SCHEME_HC,
	    FM_HC_SCHEME_VERSION, NULL },
	{ "fault.cpu.generic-x86.mc", FM_FMRI_SCHEME_HC,
	    FM_HC_SCHEME_VERSION, NULL },
	{ "fault.cpu.intel.dma", FM_FMRI_SCHEME_HC,
	    FM_HC_SCHEME_VERSION, NULL },
	{ "fault.cpu.intel.dma", FM_FMRI_SCHEME_CPU,
	    FM_CPU_SCHEME_VERSION, NULL },

	/*
	 * The ASRU for cpu faults are in cpu scheme on native and in hc
	 * scheme on xpv.  So each cpu fault class needs to be listed twice.
	 */

	/*
	 * The following faults do NOT retire a cpu thread,
	 * and therefore must be intercepted before
	 * the default "fault.cpu.*" dispatch to cma_cpu_hc_retire.
	 */
	{ "fault.cpu.amd.dramchannel", FM_FMRI_SCHEME_HC, FM_HC_SCHEME_VERSION,
	    NULL },
	{ "fault.cpu.amd.dramchannel", FM_FMRI_SCHEME_CPU,
	    FM_CPU_SCHEME_VERSION, NULL },
	{ "fault.cpu.generic-x86.bus_interconnect_memory", FM_FMRI_SCHEME_HC,
	    FM_HC_SCHEME_VERSION, NULL },
	{ "fault.cpu.generic-x86.bus_interconnect_memory", FM_FMRI_SCHEME_CPU,
	    FM_CPU_SCHEME_VERSION, NULL },
	{ "fault.cpu.generic-x86.bus_interconnect_io", FM_FMRI_SCHEME_HC,
	    FM_HC_SCHEME_VERSION, NULL },
	{ "fault.cpu.generic-x86.bus_interconnect_io", FM_FMRI_SCHEME_CPU,
	    FM_CPU_SCHEME_VERSION, NULL },
	{ "fault.cpu.generic-x86.bus_interconnect", FM_FMRI_SCHEME_HC,
	    FM_HC_SCHEME_VERSION, NULL },
	{ "fault.cpu.generic-x86.bus_interconnect", FM_FMRI_SCHEME_CPU,
	    FM_CPU_SCHEME_VERSION, NULL },
	{ "fault.cpu.intel.bus_interconnect_memory", FM_FMRI_SCHEME_HC,
	    FM_HC_SCHEME_VERSION, NULL },
	{ "fault.cpu.intel.bus_interconnect_memory", FM_FMRI_SCHEME_CPU,
	    FM_CPU_SCHEME_VERSION, NULL },
	{ "fault.cpu.intel.bus_interconnect_io", FM_FMRI_SCHEME_HC,
	    FM_HC_SCHEME_VERSION, NULL },
	{ "fault.cpu.intel.bus_interconnect_io", FM_FMRI_SCHEME_CPU,
	    FM_CPU_SCHEME_VERSION, NULL },
	{ "fault.cpu.intel.bus_interconnect", FM_FMRI_SCHEME_HC,
	    FM_HC_SCHEME_VERSION, NULL },
	{ "fault.cpu.intel.bus_interconnect", FM_FMRI_SCHEME_CPU,
	    FM_CPU_SCHEME_VERSION, NULL },
	{ "fault.cpu.intel.nb.*", FM_FMRI_SCHEME_HC, FM_HC_SCHEME_VERSION,
	    NULL },
	{ "fault.cpu.intel.nb.*", FM_FMRI_SCHEME_CPU, FM_CPU_SCHEME_VERSION,
	    NULL },
	{ "fault.cpu.intel.dma", FM_FMRI_SCHEME_HC, FM_HC_SCHEME_VERSION,
	    NULL },
	{ "fault.cpu.intel.dma", FM_FMRI_SCHEME_CPU, FM_CPU_SCHEME_VERSION,
	    NULL },
	{ "fault.cpu.*", FM_FMRI_SCHEME_HC, FM_HC_SCHEME_VERSION,
	    cma_cpu_hc_retire },
	{ "fault.cpu.*", FM_FMRI_SCHEME_CPU, FM_CPU_SCHEME_VERSION,
	    cma_cpu_hc_retire },
#elif defined(sun4v)
	/*
	 * The following are PI sun4v faults
	 */
	{ "fault.memory.memlink", FM_FMRI_SCHEME_HC,
	    FM_HC_SCHEME_VERSION, NULL },
	{ "fault.memory.memlink-uc", FM_FMRI_SCHEME_HC,
	    FM_HC_SCHEME_VERSION, NULL },
	{ "fault.memory.memlink-failover", FM_FMRI_SCHEME_HC,
	    FM_HC_SCHEME_VERSION, NULL },
	{ "fault.memory.dimm-ue-imminent", FM_FMRI_SCHEME_HC,
	    FM_HC_SCHEME_VERSION, NULL },
	{ "fault.memory.dram-ue-imminent", FM_FMRI_SCHEME_HC,
	    FM_HC_SCHEME_VERSION, NULL },
	{ "fault.memory.dimm-page-retires-excessive", FM_FMRI_SCHEME_HC,
	    FM_HC_SCHEME_VERSION, NULL },
	{ "fault.memory.page", FM_FMRI_SCHEME_MEM, FM_MEM_SCHEME_VERSION,
	    cma_page_retire },
	{ "fault.memory.dimm", FM_FMRI_SCHEME_MEM, FM_MEM_SCHEME_VERSION,
	    NULL },
	{ "fault.memory.dimm_sb", FM_FMRI_SCHEME_MEM, FM_MEM_SCHEME_VERSION,
	    NULL },
	{ "fault.memory.dimm_ck", FM_FMRI_SCHEME_MEM, FM_MEM_SCHEME_VERSION,
	    NULL },
	{ "fault.memory.dimm_ue", FM_FMRI_SCHEME_MEM, FM_MEM_SCHEME_VERSION,
	    NULL },
	{ "fault.memory.dimm-page-retires-excessive", FM_FMRI_SCHEME_MEM,
	    FM_MEM_SCHEME_VERSION, NULL },
	{ "fault.memory.dimm-ue-imminent", FM_FMRI_SCHEME_MEM,
	    FM_MEM_SCHEME_VERSION, NULL },
	{ "fault.memory.dram-ue-imminent", FM_FMRI_SCHEME_MEM,
	    FM_MEM_SCHEME_VERSION, NULL },
	{ "fault.memory.bank", FM_FMRI_SCHEME_MEM, FM_MEM_SCHEME_VERSION,
	    NULL },
	{ "fault.memory.datapath", FM_FMRI_SCHEME_MEM, FM_MEM_SCHEME_VERSION,
	    NULL },
	{ "fault.memory.datapath", FM_FMRI_SCHEME_HC, FM_HC_SCHEME_VERSION,
	    NULL },
	{ "fault.memory.link-c", FM_FMRI_SCHEME_MEM, FM_MEM_SCHEME_VERSION,
	    NULL },
	{ "fault.memory.link-u", FM_FMRI_SCHEME_MEM, FM_MEM_SCHEME_VERSION,
	    NULL },
	{ "fault.memory.link-f", FM_FMRI_SCHEME_MEM, FM_MEM_SCHEME_VERSION,
	    NULL },
	{ "fault.memory.link-c", FM_FMRI_SCHEME_HC, FM_HC_SCHEME_VERSION,
	    NULL },
	{ "fault.memory.link-u", FM_FMRI_SCHEME_HC, FM_HC_SCHEME_VERSION,
	    NULL },
	{ "fault.memory.link-f", FM_FMRI_SCHEME_HC, FM_HC_SCHEME_VERSION,
	    NULL },

	/*
	 * The following ultraSPARC-T1/T2 faults do NOT retire a cpu thread,
	 * and therefore must be intercepted before
	 * the default "fault.cpu.*" dispatch to cma_cpu_hc_retire.
	 */
	{ "fault.cpu.*.l2cachedata", FM_FMRI_SCHEME_CPU,
	    FM_CPU_SCHEME_VERSION, NULL },
	{ "fault.cpu.*.l2cachetag", FM_FMRI_SCHEME_CPU,
	    FM_CPU_SCHEME_VERSION, NULL },
	{ "fault.cpu.*.l2cachectl", FM_FMRI_SCHEME_CPU,
	    FM_CPU_SCHEME_VERSION, NULL },
	{ "fault.cpu.*.l2data-c", FM_FMRI_SCHEME_CPU,
	    FM_CPU_SCHEME_VERSION, NULL },
	{ "fault.cpu.*.l2data-u", FM_FMRI_SCHEME_CPU,
	    FM_CPU_SCHEME_VERSION, NULL },
	{ "fault.cpu.*.mau", FM_FMRI_SCHEME_CPU,
	    FM_CPU_SCHEME_VERSION, NULL },
	{ "fault.cpu.*.lfu-u", FM_FMRI_SCHEME_CPU,
	    FM_CPU_SCHEME_VERSION, NULL },
	{ "fault.cpu.*.lfu-f", FM_FMRI_SCHEME_CPU,
	    FM_CPU_SCHEME_VERSION, NULL },
	{ "fault.cpu.*.lfu-p", FM_FMRI_SCHEME_CPU,
	    FM_CPU_SCHEME_VERSION, NULL },
	{ "fault.cpu.ultraSPARC-T1.freg", FM_FMRI_SCHEME_CPU,
	    FM_CPU_SCHEME_VERSION, NULL },
	{ "fault.cpu.ultraSPARC-T1.l2cachedata", FM_FMRI_SCHEME_CPU,
	    FM_CPU_SCHEME_VERSION, NULL },
	{ "fault.cpu.ultraSPARC-T1.l2cachetag", FM_FMRI_SCHEME_CPU,
	    FM_CPU_SCHEME_VERSION, NULL },
	{ "fault.cpu.ultraSPARC-T1.l2cachectl", FM_FMRI_SCHEME_CPU,
	    FM_CPU_SCHEME_VERSION, NULL },
	{ "fault.cpu.ultraSPARC-T1.mau", FM_FMRI_SCHEME_CPU,
	    FM_CPU_SCHEME_VERSION, NULL },
	{ "fault.cpu.ultraSPARC-T2plus.chip", FM_FMRI_SCHEME_HC,
	    FM_HC_SCHEME_VERSION, NULL },
	{ "fault.cpu.*", FM_FMRI_SCHEME_HC, FM_HC_SCHEME_VERSION,
	    cma_cpu_hc_retire },
	{ "fault.cpu.*", FM_FMRI_SCHEME_CPU, FM_CPU_SCHEME_VERSION,
	    cma_cpu_hc_retire },
#elif defined(opl)
	{ "fault.memory.page", FM_FMRI_SCHEME_MEM, FM_MEM_SCHEME_VERSION,
	    cma_page_retire },
	{ "fault.memory.dimm", FM_FMRI_SCHEME_MEM, FM_MEM_SCHEME_VERSION,
	    NULL },
	{ "fault.memory.dimm-page-retires-excessive", FM_FMRI_SCHEME_MEM,
	    FM_MEM_SCHEME_VERSION, NULL },
	{ "fault.memory.dimm-ue-imminent", FM_FMRI_SCHEME_MEM,
	    FM_MEM_SCHEME_VERSION, NULL },
	{ "fault.memory.dram-ue-imminent", FM_FMRI_SCHEME_MEM,
	    FM_MEM_SCHEME_VERSION, NULL },
	{ "fault.memory.bank", FM_FMRI_SCHEME_MEM, FM_MEM_SCHEME_VERSION,
	    NULL },
	{ "fault.cpu.SPARC64-VI.*", FM_FMRI_SCHEME_CPU, FM_CPU_SCHEME_VERSION,
	    cma_cpu_cpu_retire },
	{ "fault.cpu.SPARC64-VII.*", FM_FMRI_SCHEME_CPU, FM_CPU_SCHEME_VERSION,
	    cma_cpu_cpu_retire },
	{ "fault.chassis.SPARC-Enterprise.cpu.SPARC64-VI.core.se",
		FM_FMRI_SCHEME_HC, FM_HC_SCHEME_VERSION, cma_cpu_hc_retire },
	{ "fault.chassis.SPARC-Enterprise.cpu.SPARC64-VI.core.se-offlinereq",
		FM_FMRI_SCHEME_HC, FM_HC_SCHEME_VERSION, cma_cpu_hc_retire },
	{ "fault.chassis.SPARC-Enterprise.cpu.SPARC64-VI.core.ce",
		FM_FMRI_SCHEME_HC, FM_HC_SCHEME_VERSION, cma_cpu_hc_retire },
	{ "fault.chassis.SPARC-Enterprise.cpu.SPARC64-VI.core.ce-offlinereq",
		FM_FMRI_SCHEME_HC, FM_HC_SCHEME_VERSION, cma_cpu_hc_retire },
	{ "fault.chassis.SPARC-Enterprise.cpu.SPARC64-VII.core.se",
		FM_FMRI_SCHEME_HC, FM_HC_SCHEME_VERSION, cma_cpu_hc_retire },
	{ "fault.chassis.SPARC-Enterprise.cpu.SPARC64-VII.core.se-offlinereq",
		FM_FMRI_SCHEME_HC, FM_HC_SCHEME_VERSION, cma_cpu_hc_retire },
	{ "fault.chassis.SPARC-Enterprise.cpu.SPARC64-VII.core.ce",
		FM_FMRI_SCHEME_HC, FM_HC_SCHEME_VERSION, cma_cpu_hc_retire },
	{ "fault.chassis.SPARC-Enterprise.cpu.SPARC64-VII.core.ce-offlinereq",
		FM_FMRI_SCHEME_HC, FM_HC_SCHEME_VERSION, cma_cpu_hc_retire },
#else
	/*
	 * For platforms excluding i386, sun4v and opl.
	 */
	{ "fault.memory.page", FM_FMRI_SCHEME_MEM, FM_MEM_SCHEME_VERSION,
	    cma_page_retire },
	{ "fault.memory.page_sb", FM_FMRI_SCHEME_MEM, FM_MEM_SCHEME_VERSION,
	    cma_page_retire },
	{ "fault.memory.page_ck", FM_FMRI_SCHEME_MEM, FM_MEM_SCHEME_VERSION,
	    cma_page_retire },
	{ "fault.memory.page_ue", FM_FMRI_SCHEME_MEM, FM_MEM_SCHEME_VERSION,
	    cma_page_retire },
	{ "fault.memory.dimm", FM_FMRI_SCHEME_MEM, FM_MEM_SCHEME_VERSION,
	    NULL },
	{ "fault.memory.dimm_sb", FM_FMRI_SCHEME_MEM, FM_MEM_SCHEME_VERSION,
	    NULL },
	{ "fault.memory.dimm_ck", FM_FMRI_SCHEME_MEM, FM_MEM_SCHEME_VERSION,
	    NULL },
	{ "fault.memory.dimm_ue", FM_FMRI_SCHEME_MEM, FM_MEM_SCHEME_VERSION,
	    NULL },
	{ "fault.memory.dimm-page-retires-excessive", FM_FMRI_SCHEME_MEM,
	    FM_MEM_SCHEME_VERSION, NULL },
	{ "fault.memory.dimm-ue-imminent", FM_FMRI_SCHEME_MEM,
	    FM_MEM_SCHEME_VERSION, NULL },
	{ "fault.memory.dram-ue-imminent", FM_FMRI_SCHEME_MEM,
	    FM_MEM_SCHEME_VERSION, NULL },
	{ "fault.memory.dimm_testfail", FM_FMRI_SCHEME_MEM,
	    FM_MEM_SCHEME_VERSION, NULL },
	{ "fault.memory.bank", FM_FMRI_SCHEME_MEM, FM_MEM_SCHEME_VERSION,
	    NULL },
	{ "fault.memory.datapath", FM_FMRI_SCHEME_MEM, FM_MEM_SCHEME_VERSION,
	    NULL },
	{ "fault.memory.datapath", FM_FMRI_SCHEME_HC, FM_HC_SCHEME_VERSION,
	    NULL },
	{ "fault.memory.datapath", FM_FMRI_SCHEME_CPU, FM_CPU_SCHEME_VERSION,
	    NULL },

	/*
	 * The following faults do NOT retire a cpu thread,
	 * and therefore must be intercepted before
	 * the default "fault.cpu.*" dispatch to cma_cpu_cpu_retire.
	 */
	{ "fault.cpu.ultraSPARC-IVplus.l2cachedata-line",
	    FM_FMRI_SCHEME_CPU, FM_CPU_SCHEME_VERSION,
	    cma_cache_way_retire },
	{ "fault.cpu.ultraSPARC-IVplus.l3cachedata-line",
	    FM_FMRI_SCHEME_CPU, FM_CPU_SCHEME_VERSION,
	    cma_cache_way_retire },
	{ "fault.cpu.ultraSPARC-IVplus.l2cachetag-line",
	    FM_FMRI_SCHEME_CPU, FM_CPU_SCHEME_VERSION,
	    cma_cache_way_retire },
	{ "fault.cpu.ultraSPARC-IVplus.l3cachetag-line",
	    FM_FMRI_SCHEME_CPU, FM_CPU_SCHEME_VERSION,
	    cma_cache_way_retire },

	/*
	 * Default "fault.cpu.*" for "cpu" scheme ASRU dispatch.
	 */
	{ "fault.cpu.*", FM_FMRI_SCHEME_CPU, FM_CPU_SCHEME_VERSION,
	    cma_cpu_cpu_retire },
#endif
	{ NULL, NULL, 0, NULL }
};

static const cma_subscriber_t *
nvl2subr(fmd_hdl_t *hdl, nvlist_t *nvl, nvlist_t **asrup)
{
	const cma_subscriber_t *sp;
	nvlist_t *asru;
	char *scheme;
	uint8_t version;
	boolean_t retire;

	if (nvlist_lookup_boolean_value(nvl, FM_SUSPECT_RETIRE, &retire) == 0 &&
	    retire == 0) {
		fmd_hdl_debug(hdl, "cma_recv: retire suppressed");
		return (NULL);
	}

	if (nvlist_lookup_nvlist(nvl, FM_FAULT_ASRU, &asru) != 0 ||
	    nvlist_lookup_string(asru, FM_FMRI_SCHEME, &scheme) != 0 ||
	    nvlist_lookup_uint8(asru, FM_VERSION, &version) != 0) {
		cma_stats.bad_flts.fmds_value.ui64++;
		return (NULL);
	}

	for (sp = cma_subrs; sp->subr_class != NULL; sp++) {
		if (fmd_nvl_class_match(hdl, nvl, sp->subr_class) &&
		    strcmp(scheme, sp->subr_sname) == 0 &&
		    version <= sp->subr_svers) {
			*asrup = asru;
			return (sp);
		}
	}

	cma_stats.nop_flts.fmds_value.ui64++;
	return (NULL);
}

static void
cma_recv_list(fmd_hdl_t *hdl, nvlist_t *nvl, const char *class)
{
	char *uuid = NULL;
	nvlist_t **nva, **save_nva;
	uint_t nvc = 0, save_nvc;
	uint_t keepopen;
	int err = 0;
	nvlist_t *asru = NULL;
	uint32_t index;

	err |= nvlist_lookup_string(nvl, FM_SUSPECT_UUID, &uuid);
	err |= nvlist_lookup_nvlist_array(nvl, FM_SUSPECT_FAULT_LIST,
	    &nva, &nvc);
	if (err != 0) {
		cma_stats.bad_flts.fmds_value.ui64++;
		return;
	}

	save_nvc = keepopen = nvc;
	save_nva = nva;
	while (nvc-- != 0 && (strcmp(class, FM_LIST_SUSPECT_CLASS) != 0 ||
	    !fmd_case_uuclosed(hdl, uuid))) {
		nvlist_t *nvl = *nva++;
		const cma_subscriber_t *subr;
		int has_fault;

		if ((subr = nvl2subr(hdl, nvl, &asru)) == NULL)
			continue;

		/*
		 * A handler returns CMA_RA_SUCCESS to indicate that
		 * from this suspects  point-of-view the case may be
		 * closed, CMA_RA_FAILURE otherwise.
		 * A handler must not close the case itself.
		 */
		if (subr->subr_func != NULL) {
			has_fault = fmd_nvl_fmri_has_fault(hdl, asru,
			    FMD_HAS_FAULT_ASRU, NULL);
			if (strcmp(class, FM_LIST_SUSPECT_CLASS) == 0) {
				if (has_fault == 1)
					err = subr->subr_func(hdl, nvl, asru,
					    uuid, 0);
			} else {
				if (has_fault == 0)
					err = subr->subr_func(hdl, nvl, asru,
					    uuid, 1);
			}
			if (err == CMA_RA_SUCCESS)
				keepopen--;
		}
	}

	/*
	 * Run though again to catch any new faults in list.updated.
	 */
	while (save_nvc-- != 0 && (strcmp(class, FM_LIST_UPDATED_CLASS) == 0)) {
		nvlist_t *nvl = *save_nva++;
		const cma_subscriber_t *subr;
		int has_fault;

		if ((subr = nvl2subr(hdl, nvl, &asru)) == NULL)
			continue;
		if (subr->subr_func != NULL) {
			has_fault = fmd_nvl_fmri_has_fault(hdl, asru,
			    FMD_HAS_FAULT_ASRU, NULL);
			if (has_fault == 1)
				err = subr->subr_func(hdl, nvl, asru, uuid, 0);
		}
	}

	/*
	 * Do not close the case if we are handling cache faults.
	 */
	if (asru != NULL) {
		if (nvlist_lookup_uint32(asru, FM_FMRI_CPU_CACHE_INDEX,
		    &index) != 0) {
			if (!keepopen && strcmp(class,
			    FM_LIST_SUSPECT_CLASS) == 0) {
				fmd_case_uuclose(hdl, uuid);
			}
		}
	}

	if (!keepopen && strcmp(class, FM_LIST_REPAIRED_CLASS) == 0)
		fmd_case_uuresolved(hdl, uuid);
}

static void
cma_recv_one(fmd_hdl_t *hdl, nvlist_t *nvl)
{
	const cma_subscriber_t *subr;
	nvlist_t *asru;

	if ((subr = nvl2subr(hdl, nvl, &asru)) == NULL)
		return;

	if (subr->subr_func != NULL) {
		if (fmd_nvl_fmri_has_fault(hdl, asru,
		    FMD_HAS_FAULT_ASRU, NULL) == 1)
			(void) subr->subr_func(hdl, nvl, asru, NULL, 0);
	}
}

/*ARGSUSED*/
static void
cma_recv(fmd_hdl_t *hdl, fmd_event_t *ep, nvlist_t *nvl, const char *class)
{
	fmd_hdl_debug(hdl, "received %s\n", class);

	if (strcmp(class, FM_LIST_RESOLVED_CLASS) == 0)
		return;

	if (strcmp(class, FM_LIST_SUSPECT_CLASS) == 0 ||
	    strcmp(class, FM_LIST_REPAIRED_CLASS) == 0 ||
	    strcmp(class, FM_LIST_UPDATED_CLASS) == 0)
		cma_recv_list(hdl, nvl, class);
	else
		cma_recv_one(hdl, nvl);
}

/*ARGSUSED*/
static void
cma_timeout(fmd_hdl_t *hdl, id_t id, void *arg)
{
	if (id == cma.cma_page_timerid)
		cma_page_retry(hdl);
#ifdef sun4v
	/*
	 * cpu offline/online needs to be retried on sun4v because
	 * ldom request can be asynchronous.
	 */
	else if (id == cma.cma_cpu_timerid)
		cma_cpu_retry(hdl);
#endif
}

#ifdef sun4v
static void *
cma_init_alloc(size_t size)
{
	return (fmd_hdl_alloc(init_hdl, size, FMD_SLEEP));
}

static void
cma_init_free(void *addr, size_t size)
{
	fmd_hdl_free(init_hdl, addr, size);
}
#endif

static const fmd_hdl_ops_t fmd_ops = {
	cma_recv,	/* fmdo_recv */
	cma_timeout,	/* fmdo_timeout */
	NULL,		/* fmdo_close */
	NULL,		/* fmdo_stats */
	NULL,		/* fmdo_gc */
};

static const fmd_prop_t fmd_props[] = {
	{ "cpu_tries", FMD_TYPE_UINT32, "10" },
	{ "cpu_delay", FMD_TYPE_TIME, "1sec" },
#ifdef sun4v
	{ "cpu_ret_mindelay", FMD_TYPE_TIME, "5sec" },
	{ "cpu_ret_maxdelay", FMD_TYPE_TIME, "5min" },
#endif /* sun4v */
	{ "cpu_offline_enable", FMD_TYPE_BOOL, "true" },
	{ "cpu_online_enable", FMD_TYPE_BOOL, "true" },
	{ "cpu_forced_offline", FMD_TYPE_BOOL, "true" },
#ifdef opl
	{ "cpu_blacklist_enable", FMD_TYPE_BOOL, "false" },
	{ "cpu_unblacklist_enable", FMD_TYPE_BOOL, "false" },
#else
	{ "cpu_blacklist_enable", FMD_TYPE_BOOL, "true" },
	{ "cpu_unblacklist_enable", FMD_TYPE_BOOL, "true" },
#endif /* opl */
	{ "page_ret_mindelay", FMD_TYPE_TIME, "1sec" },
	{ "page_ret_maxdelay", FMD_TYPE_TIME, "5min" },
	{ "page_retire_enable", FMD_TYPE_BOOL, "true" },
	{ "page_unretire_enable", FMD_TYPE_BOOL, "true" },
	{ NULL, 0, NULL }
};

static const fmd_hdl_info_t fmd_info = {
	"CPU/Memory Retire Agent", CMA_VERSION, &fmd_ops, fmd_props
};

void
_fmd_init(fmd_hdl_t *hdl)
{
	hrtime_t nsec;
#ifdef i386
	char buf[BUFSIZ];

	/*
	 * Abort the cpumem-retire module if not running on amd64/i86pc.
	 */
	if (sysinfo(SI_PLATFORM, buf, sizeof (buf)) == -1)
		return;

	if (strncmp(buf, "amd64", sizeof (buf)) == 0 ||
	    strncmp(buf, "i86pc", sizeof (buf)) == 0) {
		cma_is_native = B_TRUE;
	} else {
		return;
	}
#endif /* i386 */

	if (fmd_hdl_register(hdl, FMD_API_VERSION, &fmd_info) != 0)
		return; /* invalid data in configuration file */

	fmd_hdl_subscribe(hdl, "fault.cpu.*");
	fmd_hdl_subscribe(hdl, "fault.memory.*");
#ifdef opl
	fmd_hdl_subscribe(hdl, "fault.chassis.SPARC-Enterprise.cpu.*");
#endif

	(void) fmd_stat_create(hdl, FMD_STAT_NOALLOC, sizeof (cma_stats) /
	    sizeof (fmd_stat_t), (fmd_stat_t *)&cma_stats);

	cma.cma_cpu_tries = fmd_prop_get_int32(hdl, "cpu_tries");

	nsec = fmd_prop_get_int64(hdl, "cpu_delay");
	cma.cma_cpu_delay.tv_sec = nsec / NANOSEC;
	cma.cma_cpu_delay.tv_nsec = nsec % NANOSEC;

	cma.cma_page_mindelay = fmd_prop_get_int64(hdl, "page_ret_mindelay");
	cma.cma_page_maxdelay = fmd_prop_get_int64(hdl, "page_ret_maxdelay");

#ifdef sun4v
	cma.cma_cpu_mindelay = fmd_prop_get_int64(hdl, "cpu_ret_mindelay");
	cma.cma_cpu_maxdelay = fmd_prop_get_int64(hdl, "cpu_ret_maxdelay");
#endif

	cma.cma_cpu_dooffline = fmd_prop_get_int32(hdl, "cpu_offline_enable");
	cma.cma_cpu_forcedoffline = fmd_prop_get_int32(hdl,
	    "cpu_forced_offline");
	cma.cma_cpu_doonline = fmd_prop_get_int32(hdl, "cpu_online_enable");
	cma.cma_cpu_doblacklist = fmd_prop_get_int32(hdl,
	    "cpu_blacklist_enable");
	cma.cma_cpu_dounblacklist = fmd_prop_get_int32(hdl,
	    "cpu_unblacklist_enable");
	cma.cma_page_doretire = fmd_prop_get_int32(hdl, "page_retire_enable");
	cma.cma_page_dounretire = fmd_prop_get_int32(hdl,
	    "page_unretire_enable");

	if (cma.cma_page_maxdelay < cma.cma_page_mindelay)
		fmd_hdl_abort(hdl, "page retirement delays conflict\n");

#ifdef sun4v
	init_hdl = hdl;
	cma_lhp = ldom_init(cma_init_alloc, cma_init_free);
#endif
}

void
_fmd_fini(fmd_hdl_t *hdl)
{
#ifdef sun4v
	ldom_fini(cma_lhp);
	cma_cpu_fini(hdl);
#endif
	cma_page_fini(hdl);
}
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

/*
 * Page retirement can be an extended process due to the fact that a retirement
 * may not be possible when the original request is made.  The kernel will
 * repeatedly attempt to retire a given page, but will not let us know when the
 * page has been retired.  We therefore have to poll to see if the retirement
 * has been completed.  This poll is implemented with a bounded exponential
 * backoff to reduce the burden which we impose upon the system.
 *
 * To reduce the burden on fmd in the face of retirement storms, we schedule
 * all retries as a group.  In the simplest case, we attempt to retire a single
 * page.  When forced to retry, we initially schedule a retry at a configurable
 * interval t.  If the retry fails, we schedule another at 2 * t, and so on,
 * until t reaches the maximum interval (also configurable).  Future retries
 * for that page will occur with t equal to the maximum interval value.  We
 * will never give up on a retirement.
 *
 * With multiple retirements, the situation gets slightly more complicated.  As
 * indicated above, we schedule retries as a group.  We don't want to deny new
 * pages their short retry intervals, so we'll (re)set the retry interval to the
 * value appropriate for the newest page.
 */

#include <cma.h>

#include <time.h>
#include <errno.h>
#include <unistd.h>
#include <strings.h>
#include <fm/fmd_api.h>
#include <fm/libtopo.h>
#include <fm/fmd_fmri.h>
#include <fm/fmd_agent.h>
#include <sys/fm/protocol.h>

static void
cma_page_free(fmd_hdl_t *hdl, cma_page_t *page)
{
	nvlist_free(page->pg_asru);
	nvlist_free(page->pg_rsrc);
	fmd_hdl_free(hdl, page, sizeof (cma_page_t));
}

/*
 * Retire the specified ASRU, referring to a memory page by PA or by DIMM
 * offset (i.e. the encoded coordinates internal bank, row, and column).
 * In the initial FMA implementation, fault.memory.page exported an ASRU
 * with an explicit physical address, which is valid at the initial time of
 * diagnosis but may not be later following DR, DIMM removal, or interleave
 * changes.  On SPARC, this issue was solved by exporting the DIMM offset
 * and pushing the entire FMRI to the platform memory controller through
 * /dev/fm so it can derive the current PA from the DIMM and offset.
 * On x86, we also encode DIMM and offset in hc-specific, which is then used
 * by the x64 memory controller driver.
 * At some point these three approaches need to be rationalized: all platforms
 * should use the same scheme, either with decoding in the kernel or decoding
 * in userland (i.e. with a libtopo method to compute and update the PA).
 */
/*ARGSUSED*/
int
cma_page_retire(fmd_hdl_t *hdl, nvlist_t *nvl, nvlist_t *asru,
    const char *uuid, boolean_t repair)
{
	cma_page_t *page;
	uint64_t pageaddr;
	const char *action = repair ? "unretire" : "retire";
	int rc;
	nvlist_t *rsrc = NULL, *asrucp = NULL, *hcsp;

	(void) nvlist_lookup_nvlist(nvl, FM_FAULT_RESOURCE, &rsrc);

	if (nvlist_dup(asru, &asrucp, 0) != 0) {
		fmd_hdl_debug(hdl, "page retire nvlist dup failed\n");
		return (CMA_RA_FAILURE);
	}

	/* It should already be expanded, but we'll do it again anyway */
	if (fmd_nvl_fmri_expand(hdl, asrucp) < 0) {
		fmd_hdl_debug(hdl, "failed to expand page asru\n");
		cma_stats.bad_flts.fmds_value.ui64++;
		nvlist_free(asrucp);
		return (CMA_RA_FAILURE);
	}

	if (!repair && !fmd_nvl_fmri_present(hdl, asrucp)) {
		fmd_hdl_debug(hdl, "page retire overtaken by events\n");
		cma_stats.page_nonent.fmds_value.ui64++;
		nvlist_free(asrucp);
		return (CMA_RA_SUCCESS);
	}

	/* Figure out physaddr from resource or asru */
	if (rsrc == NULL ||
	    nvlist_lookup_nvlist(rsrc, FM_FMRI_HC_SPECIFIC, &hcsp) != 0 ||
	    (nvlist_lookup_uint64(hcsp, "asru-" FM_FMRI_HC_SPECIFIC_PHYSADDR,
	    &pageaddr) != 0 && nvlist_lookup_uint64(hcsp,
	    FM_FMRI_HC_SPECIFIC_PHYSADDR, &pageaddr) != 0)) {
		if (nvlist_lookup_uint64(asrucp, FM_FMRI_MEM_PHYSADDR,
		    &pageaddr) != 0) {
			fmd_hdl_debug(hdl, "mem fault missing 'physaddr'\n");
			cma_stats.bad_flts.fmds_value.ui64++;
			nvlist_free(asrucp);
			return (CMA_RA_FAILURE);
		}
	}

	if (repair) {
		if (!cma.cma_page_dounretire) {
			fmd_hdl_debug(hdl, "suppressed unretire of page %llx\n",
			    (u_longlong_t)pageaddr);
			cma_stats.page_supp.fmds_value.ui64++;
			nvlist_free(asrucp);
			return (CMA_RA_SUCCESS);
		}
		/* If unretire via topo fails, we fall back to legacy way */
		if (rsrc == NULL || (rc = fmd_nvl_fmri_unretire(hdl, rsrc)) < 0)
			rc = cma_fmri_page_unretire(hdl, asrucp);
	} else {
		if (!cma.cma_page_doretire) {
			fmd_hdl_debug(hdl, "suppressed retire of page %llx\n",
			    (u_longlong_t)pageaddr);
			cma_stats.page_supp.fmds_value.ui64++;
			nvlist_free(asrucp);
			return (CMA_RA_FAILURE);
		}
		/* If retire via topo fails, we fall back to legacy way */
		if (rsrc == NULL || (rc = fmd_nvl_fmri_retire(hdl, rsrc)) < 0)
			rc = cma_fmri_page_retire(hdl, asrucp);
	}

	if (rc == FMD_AGENT_RETIRE_DONE) {
		fmd_hdl_debug(hdl, "%sd page 0x%llx\n",
		    action, (u_longlong_t)pageaddr);
		if (repair)
			cma_stats.page_repairs.fmds_value.ui64++;
		else
			cma_stats.page_flts.fmds_value.ui64++;
		nvlist_free(asrucp);
		return (CMA_RA_SUCCESS);
	} else if (repair || rc != FMD_AGENT_RETIRE_ASYNC) {
		fmd_hdl_debug(hdl, "%s of page 0x%llx failed, will not "
		    "retry: %s\n", action, (u_longlong_t)pageaddr,
		    strerror(errno));

		cma_stats.page_fails.fmds_value.ui64++;
		nvlist_free(asrucp);
		return (CMA_RA_FAILURE);
	}

	/*
	 * The page didn't immediately retire.  We'll need to periodically
	 * check to see if it has been retired.
	 */
	fmd_hdl_debug(hdl, "page didn't retire - sleeping\n");

	page = fmd_hdl_zalloc(hdl, sizeof (cma_page_t), FMD_SLEEP);
	page->pg_addr = pageaddr;
	if (rsrc != NULL)
		(void) nvlist_dup(rsrc, &page->pg_rsrc, 0);
	page->pg_asru = asrucp;
	if (uuid != NULL)
		page->pg_uuid = fmd_hdl_strdup(hdl, uuid, FMD_SLEEP);

	page->pg_next = cma.cma_pages;
	cma.cma_pages = page;

	if (cma.cma_page_timerid != 0)
		fmd_timer_remove(hdl, cma.cma_page_timerid);

	cma.cma_page_curdelay = cma.cma_page_mindelay;

	cma.cma_page_timerid =
	    fmd_timer_install(hdl, NULL, NULL, cma.cma_page_curdelay);

	/* Don't free asrucp here.  This FMRI will be needed for retry. */
	return (CMA_RA_FAILURE);
}

static int
page_retry(fmd_hdl_t *hdl, cma_page_t *page)
{
	int rc;

	if (page->pg_asru != NULL &&
	    !fmd_nvl_fmri_present(hdl, page->pg_asru)) {
		fmd_hdl_debug(hdl, "page retire overtaken by events");
		cma_stats.page_nonent.fmds_value.ui64++;

		if (page->pg_uuid != NULL)
			fmd_case_uuclose(hdl, page->pg_uuid);
		return (1); /* no longer a page to retire */
	}

	if (page->pg_rsrc == NULL ||
	    (rc = fmd_nvl_fmri_service_state(hdl, page->pg_rsrc)) < 0)
		rc = cma_fmri_page_service_state(hdl, page->pg_asru);

	if (rc == FMD_SERVICE_STATE_UNUSABLE) {
		fmd_hdl_debug(hdl, "retired page 0x%llx on retry %u\n",
		    page->pg_addr, page->pg_nretries);
		cma_stats.page_flts.fmds_value.ui64++;

		if (page->pg_uuid != NULL)
			fmd_case_uuclose(hdl, page->pg_uuid);
		return (1); /* page retired */
	}

	if (rc == FMD_SERVICE_STATE_ISOLATE_PENDING) {
		fmd_hdl_debug(hdl, "scheduling another retry for 0x%llx\n",
		    page->pg_addr);
		return (0); /* schedule another retry */
	} else {
		fmd_hdl_debug(hdl, "failed to retry page 0x%llx "
		    "retirement: %s\n", page->pg_addr,
		    strerror(errno));

		cma_stats.page_fails.fmds_value.ui64++;
		return (1); /* give up */
	}
}

void
cma_page_retry(fmd_hdl_t *hdl)
{
	cma_page_t **pagep;

	cma.cma_page_timerid = 0;

	fmd_hdl_debug(hdl, "page_retry: timer fired\n");

	pagep = &cma.cma_pages;
	while (*pagep != NULL) {
		cma_page_t *page = *pagep;

		if (page_retry(hdl, page)) {
			/*
			 * Successful retry or we're giving up - remove from
			 * the list
			 */
			*pagep = page->pg_next;

			if (page->pg_uuid != NULL)
				fmd_hdl_strfree(hdl, page->pg_uuid);

			cma_page_free(hdl, page);
		} else {
			page->pg_nretries++;
			pagep = &page->pg_next;
		}
	}

	if (cma.cma_pages == NULL)
		return; /* no more retirements */

	/*
	 * We still have retirements that haven't completed.  Back the delay
	 * off, and schedule a retry.
	 */
	cma.cma_page_curdelay = MIN(cma.cma_page_curdelay * 2,
	    cma.cma_page_maxdelay);

	fmd_hdl_debug(hdl, "scheduled page retirement retry for %llu secs\n",
	    (u_longlong_t)(cma.cma_page_curdelay / NANOSEC));

	cma.cma_page_timerid =
	    fmd_timer_install(hdl, NULL, NULL, cma.cma_page_curdelay);
}

void
cma_page_fini(fmd_hdl_t *hdl)
{
	cma_page_t *page;

	while ((page = cma.cma_pages) != NULL) {
		cma.cma_pages = page->pg_next;
		if (page->pg_uuid != NULL)
			fmd_hdl_strfree(hdl, page->pg_uuid);
		cma_page_free(hdl, page);
	}
}
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#include <fm/fmd_api.h>
#include <fm/fmd_agent.h>
#include <fm/fmd_fmri.h>

/* ARGSUSED */
int
cma_fmri_page_service_state(fmd_hdl_t *hdl, nvlist_t *nvl)
{
	fmd_agent_hdl_t *fa_hdl;
	int rc;

	if ((fa_hdl = fmd_agent_open(FMD_AGENT_VERSION)) != NULL) {
		rc = fmd_agent_page_isretired(fa_hdl, nvl);
		if (rc == FMD_AGENT_RETIRE_DONE)
			rc = FMD_SERVICE_STATE_UNUSABLE;
		else if (rc == FMD_AGENT_RETIRE_FAIL)
			rc = FMD_SERVICE_STATE_OK;
		else if (rc == FMD_AGENT_RETIRE_ASYNC)
			rc = FMD_SERVICE_STATE_ISOLATE_PENDING;
		fmd_agent_close(fa_hdl);
		return (rc);
	}

	return (FMD_SERVICE_STATE_UNKNOWN);
}

/* ARGSUSED */
int
cma_fmri_page_retire(fmd_hdl_t *hdl, nvlist_t *nvl)
{
	fmd_agent_hdl_t *fa_hdl;
	int rc;

	if ((fa_hdl = fmd_agent_open(FMD_AGENT_VERSION)) != NULL) {
		rc = fmd_agent_page_retire(fa_hdl, nvl);
		fmd_agent_close(fa_hdl);
		return (rc);
	}

	return (FMD_AGENT_RETIRE_FAIL);
}

/* ARGSUSED */
int
cma_fmri_page_unretire(fmd_hdl_t *hdl, nvlist_t *nvl)
{
	fmd_agent_hdl_t *fa_hdl;
	int rc;

	if ((fa_hdl = fmd_agent_open(FMD_AGENT_VERSION)) != NULL) {
		rc = fmd_agent_page_unretire(fa_hdl, nvl);
		fmd_agent_close(fa_hdl);
		return (rc);
	}

	return (FMD_AGENT_RETIRE_FAIL);
}
#
# CDDL HEADER START
#
# The contents of this file are subject to the terms of the
# Common Development and Distribution License, Version 1.0 only
# (the "License").  You may not use this file except in compliance
# with the License.
#
# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
# or http://www.opensolaris.org/os/licensing.
# See the License for the specific language governing permissions
# and limitations under the License.
#
# When distributing Covered Code, include this CDDL HEADER in each
# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
# If applicable, add the following below this CDDL HEADER, with the
# fields enclosed by brackets "[]" replaced with your own identifying
# information: Portions Copyright [yyyy] [name of copyright owner]
#
# CDDL HEADER END
#
#
# Copyright 2004 Sun Microsystems, Inc.  All rights reserved.
# Use is subject to license terms.
#
#ident	"%Z%%M%	%I%	%E% SMI"

#
# cpumem-retire
#
setprop cpu_tries 10
setprop cpu_delay 1sec
#
setprop page_ret_mindelay 1sec
setprop page_ret_maxdelay 5min
#
setprop cpu_offline_enable true
setprop cpu_forced_offline true
setprop cpu_blacklist_enable true
setprop page_retire_enable true