pcbe - zygaena/hammerhead

root / base / usr / src / uts / intel / pcbe
pcbe Plain Text 3902 lines 112.5 KB
                              Copyright (c) 2005,6
                           Innovative Computing Labs
                          Computer Science Department, 
                            University of Tennessee, 
                                 Knoxville, TN. 
                              All Rights Reserved. 


Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:

    * Redistributions of source code must retain the above copyright notice, 
      this list of conditions and the following disclaimer.
    * Redistributions in binary form must reproduce the above copyright notice, 
      this list of conditions and the following disclaimer in the documentation 
      and/or other materials provided with the distribution.
    * Neither the name of the University of Tennessee nor the names of its 
      contributors may be used to endorse or promote products derived from this
      software without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 
ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.


This open source software license conforms to the BSD License template.
PAPI PRESET EVENT NAMES
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
 * Copyright 2019 Joyent, Inc.
 */

/*
 * This file contains preset event names from the Performance Application
 * Programming Interface v3.5 which included the following notice:
 *
 *                             Copyright (c) 2005,6
 *                           Innovative Computing Labs
 *                         Computer Science Department,
 *                            University of Tennessee,
 *                                 Knoxville, TN.
 *                              All Rights Reserved.
 *
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 *    * Redistributions of source code must retain the above copyright notice,
 *      this list of conditions and the following disclaimer.
 *    * Redistributions in binary form must reproduce the above copyright
 *      notice, this list of conditions and the following disclaimer in the
 *      documentation and/or other materials provided with the distribution.
 *    * Neither the name of the University of Tennessee nor the names of its
 *      contributors may be used to endorse or promote products derived from
 *      this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 *
 *
 * This open source software license conforms to the BSD License template.
 */


/*
 * Performance Counter Back-End for Intel processors supporting Architectural
 * Performance Monitoring.
 */

#include <sys/cpuvar.h>
#include <sys/param.h>
#include <sys/cpc_impl.h>
#include <sys/cpc_pcbe.h>
#include <sys/modctl.h>
#include <sys/inttypes.h>
#include <sys/systm.h>
#include <sys/cmn_err.h>
#include <sys/x86_archext.h>
#include <sys/sdt.h>
#include <sys/archsystm.h>
#include <sys/privregs.h>
#include <sys/ddi.h>
#include <sys/sunddi.h>
#include <sys/cred.h>
#include <sys/policy.h>

#include "core_pcbe_table.h"
#include <core_pcbe_cpcgen.h>

static int core_pcbe_init(void);
static uint_t core_pcbe_ncounters(void);
static const char *core_pcbe_impl_name(void);
static const char *core_pcbe_cpuref(void);
static char *core_pcbe_list_events(uint_t picnum);
static char *core_pcbe_list_attrs(void);
static uint64_t core_pcbe_event_coverage(char *event);
static uint64_t core_pcbe_overflow_bitmap(void);
static int core_pcbe_configure(uint_t picnum, char *event, uint64_t preset,
    uint32_t flags, uint_t nattrs, kcpc_attr_t *attrs, void **data,
    void *token);
static void core_pcbe_program(void *token);
static void core_pcbe_allstop(void);
static void core_pcbe_sample(void *token);
static void core_pcbe_free(void *config);

#define	FALSE	0
#define	TRUE	1

/* Counter Type */
#define	CORE_GPC	0	/* General-Purpose Counter (GPC) */
#define	CORE_FFC	1	/* Fixed-Function Counter (FFC) */

/* MSR Addresses */
#define	GPC_BASE_PMC		0x00c1	/* First GPC */
#define	GPC_BASE_PES		0x0186	/* First GPC Event Select register */
#define	FFC_BASE_PMC		0x0309	/* First FFC */
#define	PERF_FIXED_CTR_CTRL	0x038d	/* Used to enable/disable FFCs */
#define	PERF_GLOBAL_STATUS	0x038e	/* Overflow status register */
#define	PERF_GLOBAL_CTRL	0x038f	/* Used to enable/disable counting */
#define	PERF_GLOBAL_OVF_CTRL	0x0390	/* Used to clear overflow status */

/*
 * Processor Event Select register fields
 */
#define	CORE_USR	(1ULL << 16)	/* Count while not in ring 0 */
#define	CORE_OS		(1ULL << 17)	/* Count while in ring 0 */
#define	CORE_EDGE	(1ULL << 18)	/* Enable edge detection */
#define	CORE_PC		(1ULL << 19)	/* Enable pin control */
#define	CORE_INT	(1ULL << 20)	/* Enable interrupt on overflow */
#define	CORE_EN		(1ULL << 22)	/* Enable counting */
#define	CORE_INV	(1ULL << 23)	/* Invert the CMASK */
#define	CORE_ANYTHR	(1ULL << 21)	/* Count event for any thread on core */

#define	CORE_UMASK_SHIFT	8
#define	CORE_UMASK_MASK		0xffu
#define	CORE_CMASK_SHIFT	24
#define	CORE_CMASK_MASK		0xffu

/*
 * Fixed-function counter attributes
 */
#define	CORE_FFC_OS_EN	(1ULL << 0)	/* Count while not in ring 0 */
#define	CORE_FFC_USR_EN	(1ULL << 1)	/* Count while in ring 1 */
#define	CORE_FFC_ANYTHR	(1ULL << 2)	/* Count event for any thread on core */
#define	CORE_FFC_PMI	(1ULL << 3)	/* Enable interrupt on overflow */

/*
 * Number of bits for specifying each FFC's attributes in the control register
 */
#define	CORE_FFC_ATTR_SIZE	4

/*
 * CondChgd and OvfBuffer fields of global status and overflow control registers
 */
#define	CONDCHGD	(1ULL << 63)
#define	OVFBUFFER	(1ULL << 62)
#define	MASK_CONDCHGD_OVFBUFFER	(CONDCHGD | OVFBUFFER)

#define	ALL_STOPPED	0ULL

#define	BITMASK_XBITS(x)	((1ull << (x)) - 1ull)

/*
 * Only the lower 32-bits can be written to in the general-purpose
 * counters.  The higher bits are extended from bit 31; all ones if
 * bit 31 is one and all zeros otherwise.
 *
 * The fixed-function counters do not have this restriction.
 */
#define	BITS_EXTENDED_FROM_31	(BITMASK_XBITS(width_gpc) & ~BITMASK_XBITS(31))

#define	WRMSR(msr, value)						\
	wrmsr((msr), (value));						\
	DTRACE_PROBE2(wrmsr, uint64_t, (msr), uint64_t, (value));

#define	RDMSR(msr, value)						\
	(value) = rdmsr((msr));						\
	DTRACE_PROBE2(rdmsr, uint64_t, (msr), uint64_t, (value));

typedef struct core_pcbe_config {
	uint64_t	core_rawpic;
	uint64_t	core_ctl;	/* Event Select bits */
	uint64_t	core_pmc;	/* Counter register address */
	uint64_t	core_pes;	/* Event Select register address */
	uint_t		core_picno;
	uint8_t		core_pictype;	/* CORE_GPC or CORE_FFC */
} core_pcbe_config_t;

pcbe_ops_t core_pcbe_ops = {
	PCBE_VER_1,			/* pcbe_ver */
	CPC_CAP_OVERFLOW_INTERRUPT | CPC_CAP_OVERFLOW_PRECISE,	/* pcbe_caps */
	core_pcbe_ncounters,		/* pcbe_ncounters */
	core_pcbe_impl_name,		/* pcbe_impl_name */
	core_pcbe_cpuref,		/* pcbe_cpuref */
	core_pcbe_list_events,		/* pcbe_list_events */
	core_pcbe_list_attrs,		/* pcbe_list_attrs */
	core_pcbe_event_coverage,	/* pcbe_event_coverage */
	core_pcbe_overflow_bitmap,	/* pcbe_overflow_bitmap */
	core_pcbe_configure,		/* pcbe_configure */
	core_pcbe_program,		/* pcbe_program */
	core_pcbe_allstop,		/* pcbe_allstop */
	core_pcbe_sample,		/* pcbe_sample */
	core_pcbe_free			/* pcbe_free */
};

struct nametable_core_uarch {
	const char	*name;
	uint64_t	restricted_bits;
	uint8_t		event_num;
};

/*
 * Counting an event for all cores or all bus agents requires cpc_cpu privileges
 */
#define	ALL_CORES	(1ULL << 15)
#define	ALL_AGENTS	(1ULL << 13)

struct generic_events {
	const char	*name;
	uint8_t		event_num;
	uint8_t		umask;
};

static const struct generic_events cmn_generic_events[] = {
	{ "PAPI_tot_cyc", 0x3c, 0x00 }, /* cpu_clk_unhalted.thread_p/core */
	{ "PAPI_tot_ins", 0xc0, 0x00 }, /* inst_retired.any_p		  */
	{ "PAPI_br_ins",  0xc4, 0x0c }, /* br_inst_retired.taken	  */
	{ "PAPI_br_msp",  0xc5, 0x00 }, /* br_inst_retired.mispred	  */
	{ "PAPI_br_ntk",  0xc4, 0x03 },
				/* br_inst_retired.pred_not_taken|pred_taken */
	{ "PAPI_br_prc",  0xc4, 0x05 },
				/* br_inst_retired.pred_not_taken|pred_taken */
	{ "PAPI_hw_int",  0xc8, 0x00 }, /* hw_int_rvc			  */
	{ "PAPI_tot_iis", 0xaa, 0x01 }, /* macro_insts.decoded		  */
	{ "PAPI_l1_dca",  0x43, 0x01 }, /* l1d_all_ref			  */
	{ "PAPI_l1_icm",  0x81, 0x00 }, /* l1i_misses			  */
	{ "PAPI_l1_icr",  0x80, 0x00 }, /* l1i_reads			  */
	{ "PAPI_l1_tcw",  0x41, 0x0f }, /* l1d_cache_st.mesi		  */
	{ "PAPI_l2_stm",  0x2a, 0x41 }, /* l2_st.self.i_state		  */
	{ "PAPI_l2_tca",  0x2e, 0x4f }, /* l2_rqsts.self.demand.mesi	  */
	{ "PAPI_l2_tch",  0x2e, 0x4e }, /* l2_rqsts.mes			  */
	{ "PAPI_l2_tcm",  0x2e, 0x41 }, /* l2_rqsts.self.demand.i_state   */
	{ "PAPI_l2_tcw",  0x2a, 0x4f }, /* l2_st.self.mesi		  */
	{ "PAPI_ld_ins",  0xc0, 0x01 }, /* inst_retired.loads		  */
	{ "PAPI_lst_ins", 0xc0, 0x03 }, /* inst_retired.loads|stores	  */
	{ "PAPI_sr_ins",  0xc0, 0x02 }, /* inst_retired.stores		  */
	{ "PAPI_tlb_dm",  0x08, 0x01 }, /* dtlb_misses.any		  */
	{ "PAPI_tlb_im",  0x82, 0x12 }, /* itlb.small_miss|large_miss	  */
	{ "PAPI_tlb_tl",  0x0c, 0x03 }, /* page_walks			  */
	{ "",		  NT_END, 0  }
};

static const struct generic_events generic_events_pic0[] = {
	{ "PAPI_l1_dcm",  0xcb, 0x01 }, /* mem_load_retired.l1d_miss */
	{ "",		  NT_END, 0  }
};

/*
 * The events listed in the following table can be counted on all
 * general-purpose counters on processors that are of Penryn and Merom Family
 */
static const struct nametable_core_uarch cmn_gpc_events_core_uarch[] = {
	/* Alphabetical order of event name */

	{ "baclears",			0x0,	0xe6 },
	{ "bogus_br",			0x0,	0xe4 },
	{ "br_bac_missp_exec",		0x0,	0x8a },

	{ "br_call_exec",		0x0,	0x92 },
	{ "br_call_missp_exec",		0x0,	0x93 },
	{ "br_cnd_exec",		0x0,	0x8b },

	{ "br_cnd_missp_exec",		0x0,	0x8c },
	{ "br_ind_call_exec",		0x0,	0x94 },
	{ "br_ind_exec",		0x0,	0x8d },

	{ "br_ind_missp_exec",		0x0,	0x8e },
	{ "br_inst_decoded",		0x0,	0xe0 },
	{ "br_inst_exec",		0x0,	0x88 },

	{ "br_inst_retired",		0x0,	0xc4 },
	{ "br_inst_retired_mispred",	0x0,	0xc5 },
	{ "br_missp_exec",		0x0,	0x89 },

	{ "br_ret_bac_missp_exec",	0x0,	0x91 },
	{ "br_ret_exec",		0x0,	0x8f },
	{ "br_ret_missp_exec",		0x0,	0x90 },

	{ "br_tkn_bubble_1",		0x0,	0x97 },
	{ "br_tkn_bubble_2",		0x0,	0x98 },
	{ "bus_bnr_drv",		ALL_AGENTS,	0x61 },

	{ "bus_data_rcv",		ALL_CORES,	0x64 },
	{ "bus_drdy_clocks",		ALL_AGENTS,	0x62 },
	{ "bus_hit_drv",		ALL_AGENTS,	0x7a },

	{ "bus_hitm_drv",		ALL_AGENTS,	0x7b },
	{ "bus_io_wait",		ALL_CORES,	0x7f },
	{ "bus_lock_clocks",		ALL_CORES | ALL_AGENTS,	0x63 },

	{ "bus_request_outstanding",	ALL_CORES | ALL_AGENTS,	0x60 },
	{ "bus_trans_any",		ALL_CORES | ALL_AGENTS,	0x70 },
	{ "bus_trans_brd",		ALL_CORES | ALL_AGENTS,	0x65 },

	{ "bus_trans_burst",		ALL_CORES | ALL_AGENTS,	0x6e },
	{ "bus_trans_def",		ALL_CORES | ALL_AGENTS,	0x6d },
	{ "bus_trans_ifetch",		ALL_CORES | ALL_AGENTS,	0x68 },

	{ "bus_trans_inval",		ALL_CORES | ALL_AGENTS,	0x69 },
	{ "bus_trans_io",		ALL_CORES | ALL_AGENTS,	0x6c },
	{ "bus_trans_mem",		ALL_CORES | ALL_AGENTS,	0x6f },

	{ "bus_trans_p",		ALL_CORES | ALL_AGENTS,	0x6b },
	{ "bus_trans_pwr",		ALL_CORES | ALL_AGENTS,	0x6a },
	{ "bus_trans_rfo",		ALL_CORES | ALL_AGENTS,	0x66 },

	{ "bus_trans_wb",		ALL_CORES | ALL_AGENTS,	0x67 },
	{ "busq_empty",			ALL_CORES,	0x7d },
	{ "cmp_snoop",			ALL_CORES,	0x78 },

	{ "cpu_clk_unhalted",		0x0,	0x3c },
	{ "cycles_int",			0x0,	0xc6 },
	{ "cycles_l1i_mem_stalled",	0x0,	0x86 },

	{ "dtlb_misses",		0x0,	0x08 },
	{ "eist_trans",			0x0,	0x3a },
	{ "esp",			0x0,	0xab },

	{ "ext_snoop",			ALL_AGENTS,	0x77 },
	{ "fp_mmx_trans",		0x0,	0xcc },
	{ "hw_int_rcv",			0x0,	0xc8 },

	{ "ild_stall",			0x0,	0x87 },
	{ "inst_queue",			0x0,	0x83 },
	{ "inst_retired",		0x0,	0xc0 },

	{ "itlb",			0x0,	0x82 },
	{ "itlb_miss_retired",		0x0,	0xc9 },
	{ "l1d_all_ref",		0x0,	0x43 },

	{ "l1d_cache_ld",		0x0,	0x40 },
	{ "l1d_cache_lock",		0x0,	0x42 },
	{ "l1d_cache_st",		0x0,	0x41 },

	{ "l1d_m_evict",		0x0,	0x47 },
	{ "l1d_m_repl",			0x0,	0x46 },
	{ "l1d_pend_miss",		0x0,	0x48 },

	{ "l1d_prefetch",		0x0,	0x4e },
	{ "l1d_repl",			0x0,	0x45 },
	{ "l1d_split",			0x0,	0x49 },

	{ "l1i_misses",			0x0,	0x81 },
	{ "l1i_reads",			0x0,	0x80 },
	{ "l2_ads",			ALL_CORES,	0x21 },

	{ "l2_dbus_busy_rd",		ALL_CORES,	0x23 },
	{ "l2_ifetch",			ALL_CORES,	0x28 },
	{ "l2_ld",			ALL_CORES,	0x29 },

	{ "l2_lines_in",		ALL_CORES,	0x24 },
	{ "l2_lines_out",		ALL_CORES,	0x26 },
	{ "l2_lock",			ALL_CORES,	0x2b },

	{ "l2_m_lines_in",		ALL_CORES,	0x25 },
	{ "l2_m_lines_out",		ALL_CORES,	0x27 },
	{ "l2_no_req",			ALL_CORES,	0x32 },

	{ "l2_reject_busq",		ALL_CORES,	0x30 },
	{ "l2_rqsts",			ALL_CORES,	0x2e },
	{ "l2_st",			ALL_CORES,	0x2a },

	{ "load_block",			0x0,	0x03 },
	{ "load_hit_pre",		0x0,	0x4c },
	{ "machine_nukes",		0x0,	0xc3 },

	{ "macro_insts",		0x0,	0xaa },
	{ "memory_disambiguation",	0x0,	0x09 },
	{ "misalign_mem_ref",		0x0,	0x05 },
	{ "page_walks",			0x0,	0x0c },

	{ "pref_rqsts_dn",		0x0,	0xf8 },
	{ "pref_rqsts_up",		0x0,	0xf0 },
	{ "rat_stalls",			0x0,	0xd2 },

	{ "resource_stalls",		0x0,	0xdc },
	{ "rs_uops_dispatched",		0x0,	0xa0 },
	{ "seg_reg_renames",		0x0,	0xd5 },

	{ "seg_rename_stalls",		0x0,	0xd4 },
	{ "segment_reg_loads",		0x0,	0x06 },
	{ "simd_assist",		0x0,	0xcd },

	{ "simd_comp_inst_retired",	0x0,	0xca },
	{ "simd_inst_retired",		0x0,	0xc7 },
	{ "simd_instr_retired",		0x0,	0xce },

	{ "simd_sat_instr_retired",	0x0,	0xcf },
	{ "simd_sat_uop_exec",		0x0,	0xb1 },
	{ "simd_uop_type_exec",		0x0,	0xb3 },

	{ "simd_uops_exec",		0x0,	0xb0 },
	{ "snoop_stall_drv",		ALL_CORES | ALL_AGENTS,	0x7e },
	{ "sse_pre_exec",		0x0,	0x07 },

	{ "sse_pre_miss",		0x0,	0x4b },
	{ "store_block",		0x0,	0x04 },
	{ "thermal_trip",		0x0,	0x3b },

	{ "uops_retired",		0x0,	0xc2 },
	{ "x87_ops_retired",		0x0,	0xc1 },
	{ "",				0x0,	NT_END }
};

/*
 * If any of the pic specific events require privileges, make sure to add a
 * check in configure_gpc() to find whether an event hard-coded as a number by
 * the user has any privilege requirements
 */
static const struct nametable_core_uarch pic0_events[] = {
	/* Alphabetical order of event name */

	{ "cycles_div_busy",		0x0,	0x14 },
	{ "fp_comp_ops_exe",		0x0,	0x10 },
	{ "idle_during_div",		0x0,	0x18 },

	{ "mem_load_retired",		0x0,	0xcb },
	{ "rs_uops_dispatched_port",	0x0,	0xa1 },
	{ "",				0x0,	NT_END }
};

static const struct nametable_core_uarch pic1_events[] = {
	/* Alphabetical order of event name */

	{ "delayed_bypass",	0x0,	0x19 },
	{ "div",		0x0,	0x13 },
	{ "fp_assist",		0x0,	0x11 },

	{ "mul",		0x0,	0x12 },
	{ "",			0x0,	NT_END }
};

/* FFC entries must be in order */
static char *ffc_names_non_htt[] = {
	"instr_retired.any",
	"cpu_clk_unhalted.core",
	"cpu_clk_unhalted.ref",
	NULL
};

static char *ffc_names_htt[] = {
	"instr_retired.any",
	"cpu_clk_unhalted.thread",
	"cpu_clk_unhalted.ref",
	NULL
};

static char *ffc_genericnames[] = {
	"PAPI_tot_ins",
	"PAPI_tot_cyc",
	"",
	NULL
};

static char	**ffc_names = NULL;
static char	**ffc_allnames = NULL;
static char	**gpc_names = NULL;
static uint32_t	versionid;
static uint64_t	num_gpc;
static uint64_t	width_gpc;
static uint64_t	mask_gpc;
static uint64_t	num_ffc;
static uint64_t	width_ffc;
static uint64_t	mask_ffc;
static uint_t	total_pmc;
static uint64_t	control_ffc;
static uint64_t	control_gpc;
static uint64_t	control_mask;
static uint32_t	arch_events_vector;

#define	IMPL_NAME_LEN 100
static char core_impl_name[IMPL_NAME_LEN];

static const char *core_cpuref =
	"See https://download.01.org/perfmon/index/ or Chapers 18 and 19 " \
	"of the \"Intel 64 and IA-32 Architectures Software Developer's " \
	"Manual Volume 3: System Programming Guide\" Order Number: " \
	"325384-062US, March 2017.";


/* Architectural events */
#define	ARCH_EVENTS_COMMON					\
	{ 0xc0, 0x00, C_ALL, "inst_retired.any_p" },		\
	{ 0x3c, 0x01, C_ALL, "cpu_clk_unhalted.ref_p" },	\
	{ 0x2e, 0x4f, C_ALL, "longest_lat_cache.reference" },	\
	{ 0x2e, 0x41, C_ALL, "longest_lat_cache.miss" },	\
	{ 0xc4, 0x00, C_ALL, "br_inst_retired.all_branches" },	\
	{ 0xc5, 0x00, C_ALL, "br_misp_retired.all_branches" }

static const struct events_table_t arch_events_table_non_htt[] = {
	{ 0x3c, 0x00, C_ALL, "cpu_clk_unhalted.core" },
	ARCH_EVENTS_COMMON
};

static const struct events_table_t arch_events_table_htt[] = {
	{ 0x3c, 0x00, C_ALL, "cpu_clk_unhalted.thread_p" },
	ARCH_EVENTS_COMMON
};

static char *arch_genevents_table[] = {
	"PAPI_tot_cyc", /* cpu_clk_unhalted.thread_p/core */
	"PAPI_tot_ins", /* inst_retired.any_p		  */
	"",		/* cpu_clk_unhalted.ref_p	  */
	"",		/* longest_lat_cache.reference	  */
	"",		/* longest_lat_cache.miss	  */
	"",		/* br_inst_retired.all_branches	  */
	"",		/* br_misp_retired.all_branches	  */
};

static const struct events_table_t *arch_events_table = NULL;
static uint64_t known_arch_events;
static uint64_t known_ffc_num;
static const struct events_table_t *events_table = NULL;

/*
 * Initialize string containing list of supported general-purpose counter
 * events for processors of Penryn and Merom Family
 */
static void
pcbe_init_core_uarch()
{
	const struct nametable_core_uarch	*n;
	const struct generic_events		*k;
	const struct nametable_core_uarch	*picspecific_events;
	const struct generic_events		*picspecific_genericevents;
	size_t			common_size;
	size_t			size;
	uint64_t		i;

	gpc_names = kmem_alloc(num_gpc * sizeof (char *), KM_SLEEP);

	/* Calculate space needed to save all the common event names */
	common_size = 0;
	for (n = cmn_gpc_events_core_uarch; n->event_num != NT_END; n++) {
		common_size += strlen(n->name) + 1;
	}

	for (k = cmn_generic_events; k->event_num != NT_END; k++) {
		common_size += strlen(k->name) + 1;
	}

	for (i = 0; i < num_gpc; i++) {
		size = 0;
		picspecific_genericevents = NULL;

		switch (i) {
			case 0:
				picspecific_events = pic0_events;
				picspecific_genericevents = generic_events_pic0;
				break;
			case 1:
				picspecific_events = pic1_events;
				break;
			default:
				picspecific_events = NULL;
				break;
		}
		if (picspecific_events != NULL) {
			for (n = picspecific_events;
			    n->event_num != NT_END;
			    n++) {
				size += strlen(n->name) + 1;
			}
		}
		if (picspecific_genericevents != NULL) {
			for (k = picspecific_genericevents;
			    k->event_num != NT_END; k++) {
				size += strlen(k->name) + 1;
			}
		}

		gpc_names[i] =
		    kmem_alloc(size + common_size + 1, KM_SLEEP);

		gpc_names[i][0] = '\0';
		if (picspecific_events != NULL) {
			for (n = picspecific_events;
			    n->event_num != NT_END; n++) {
				(void) strcat(gpc_names[i], n->name);
				(void) strcat(gpc_names[i], ",");
			}
		}
		if (picspecific_genericevents != NULL) {
			for (k = picspecific_genericevents;
			    k->event_num != NT_END; k++) {
				(void) strcat(gpc_names[i], k->name);
				(void) strcat(gpc_names[i], ",");
			}
		}
		for (n = cmn_gpc_events_core_uarch; n->event_num != NT_END;
		    n++) {
			(void) strcat(gpc_names[i], n->name);
			(void) strcat(gpc_names[i], ",");
		}
		for (k = cmn_generic_events; k->event_num != NT_END; k++) {
			(void) strcat(gpc_names[i], k->name);
			(void) strcat(gpc_names[i], ",");
		}

		/*
		 * Remove trailing comma.
		 */
		gpc_names[i][common_size + size - 1] = '\0';
	}
}

static int
core_pcbe_init(void)
{
	struct cpuid_regs	cp;
	size_t			size;
	uint64_t		i;
	uint64_t		j;
	uint64_t		arch_events_vector_length;
	size_t			arch_events_string_length;
	uint_t			model, stepping;

	if (cpuid_getvendor(CPU) != X86_VENDOR_Intel)
		return (-1);

	/* Obtain Basic CPUID information */
	cp.cp_eax = 0x0;
	(void) __cpuid_insn(&cp);

	/* No Architectural Performance Monitoring Leaf returned by CPUID */
	if (cp.cp_eax < 0xa) {
		return (-1);
	}

	/* Obtain the Architectural Performance Monitoring Leaf */
	cp.cp_eax = 0xa;
	(void) __cpuid_insn(&cp);

	versionid = cp.cp_eax & 0xFF;

	/*
	 * Fixed-Function Counters (FFC)
	 *
	 * All Family 6 Model 15 and Model 23 processors have fixed-function
	 * counters.  These counters were made Architectural with
	 * Family 6 Model 15 Stepping 9.
	 */
	switch (versionid) {

		case 0:
			return (-1);

		case 2:
			num_ffc = cp.cp_edx & 0x1F;
			width_ffc = (cp.cp_edx >> 5) & 0xFF;

			/*
			 * Some processors have an errata (AW34) where
			 * versionid is reported as 2 when actually 1.
			 * In this case, fixed-function counters are
			 * model-specific as in Version 1.
			 */
			if (num_ffc != 0) {
				break;
			}
			/* FALLTHROUGH */
		case 1:
			num_ffc = 3;
			width_ffc = 40;
			versionid = 1;
			break;

		default:
			num_ffc = cp.cp_edx & 0x1F;
			width_ffc = (cp.cp_edx >> 5) & 0xFF;
			break;
	}


	if (num_ffc >= 64)
		return (-1);

	/* Set HTT-specific names of architectural & FFC events */
	if (is_x86_feature(x86_featureset, X86FSET_HTT)) {
		ffc_names = ffc_names_htt;
		arch_events_table = arch_events_table_htt;
		known_arch_events =
		    sizeof (arch_events_table_htt) /
		    sizeof (struct events_table_t);
		known_ffc_num =
		    sizeof (ffc_names_htt) / sizeof (char *);
	} else {
		ffc_names = ffc_names_non_htt;
		arch_events_table = arch_events_table_non_htt;
		known_arch_events =
		    sizeof (arch_events_table_non_htt) /
		    sizeof (struct events_table_t);
		known_ffc_num =
		    sizeof (ffc_names_non_htt) / sizeof (char *);
	}

	if (num_ffc >= known_ffc_num) {
		/*
		 * The system seems to have more fixed-function counters than
		 * what this PCBE is able to handle correctly.  Default to the
		 * maximum number of fixed-function counters that this driver
		 * is aware of.
		 */
		num_ffc = known_ffc_num - 1;
	}

	mask_ffc = BITMASK_XBITS(width_ffc);
	control_ffc = BITMASK_XBITS(num_ffc);

	/*
	 * General Purpose Counters (GPC)
	 */
	num_gpc = (cp.cp_eax >> 8) & 0xFF;
	width_gpc = (cp.cp_eax >> 16) & 0xFF;

	if (num_gpc >= 64)
		return (-1);

	mask_gpc = BITMASK_XBITS(width_gpc);

	control_gpc = BITMASK_XBITS(num_gpc);

	control_mask = (control_ffc << 32) | control_gpc;

	total_pmc = num_gpc + num_ffc;
	if (total_pmc > 64) {
		/* Too wide for the overflow bitmap */
		return (-1);
	}

	/* FFC names */
	ffc_allnames = kmem_alloc(num_ffc * sizeof (char *), KM_SLEEP);
	for (i = 0; i < num_ffc; i++) {
		ffc_allnames[i] = kmem_alloc(
		    strlen(ffc_names[i]) + strlen(ffc_genericnames[i]) + 2,
		    KM_SLEEP);

		ffc_allnames[i][0] = '\0';
		(void) strcat(ffc_allnames[i], ffc_names[i]);

		/* Check if this ffc has a generic name */
		if (strcmp(ffc_genericnames[i], "") != 0) {
			(void) strcat(ffc_allnames[i], ",");
			(void) strcat(ffc_allnames[i], ffc_genericnames[i]);
		}
	}

	/* GPC events for Family 6 Models 15, 23 and 29 only */
	if ((cpuid_getfamily(CPU) == 6) &&
	    ((cpuid_getmodel(CPU) == 15) || (cpuid_getmodel(CPU) == 23) ||
	    (cpuid_getmodel(CPU) == 29))) {
		(void) snprintf(core_impl_name, IMPL_NAME_LEN,
		    "Core Microarchitecture");
		pcbe_init_core_uarch();
		return (0);
	}

	(void) snprintf(core_impl_name, IMPL_NAME_LEN,
	    "Intel Arch PerfMon v%d on Family %d Model %d",
	    versionid, cpuid_getfamily(CPU), cpuid_getmodel(CPU));

	/*
	 * Architectural events
	 */
	arch_events_vector_length = (cp.cp_eax >> 24) & 0xFF;

	ASSERT(known_arch_events == arch_events_vector_length);

	/*
	 * To handle the case where a new performance monitoring setup is run
	 * on a non-debug kernel
	 */
	if (known_arch_events > arch_events_vector_length) {
		known_arch_events = arch_events_vector_length;
	} else {
		arch_events_vector_length = known_arch_events;
	}

	arch_events_vector = cp.cp_ebx &
	    BITMASK_XBITS(arch_events_vector_length);

	/*
	 * Process architectural and non-architectural events using GPC
	 */
	if (num_gpc > 0) {

		gpc_names = kmem_alloc(num_gpc * sizeof (char *), KM_SLEEP);

		/* Calculate space required for the architectural gpc events */
		arch_events_string_length = 0;
		for (i = 0; i < known_arch_events; i++) {
			if (((1U << i) & arch_events_vector) == 0) {
				arch_events_string_length +=
				    strlen(arch_events_table[i].name) + 1;
				if (strcmp(arch_genevents_table[i], "") != 0) {
					arch_events_string_length +=
					    strlen(arch_genevents_table[i]) + 1;
				}
			}
		}

		/* Non-architectural events list */
		model = cpuid_getmodel(CPU);
		stepping = cpuid_getstep(CPU);
		events_table = core_cpcgen_table(model, stepping);

		for (i = 0; i < num_gpc; i++) {

			/*
			 * Determine length of all supported event names
			 * (architectural + non-architectural)
			 */
			size = arch_events_string_length;
			for (j = 0; events_table != NULL &&
			    events_table[j].eventselect != NT_END;
			    j++) {
				if (C(i) & events_table[j].supported_counters) {
					size += strlen(events_table[j].name) +
					    1;
				}
			}

			/* Allocate memory for this pics list */
			gpc_names[i] = kmem_alloc(size + 1, KM_SLEEP);
			gpc_names[i][0] = '\0';
			if (size == 0) {
				continue;
			}

			/*
			 * Create the list of all supported events
			 * (architectural + non-architectural)
			 */
			for (j = 0; j < known_arch_events; j++) {
				if (((1U << j) & arch_events_vector) == 0) {
					(void) strcat(gpc_names[i],
					    arch_events_table[j].name);
					(void) strcat(gpc_names[i], ",");
					if (strcmp(
					    arch_genevents_table[j], "")
					    != 0) {
						(void) strcat(gpc_names[i],
						    arch_genevents_table[j]);
						(void) strcat(gpc_names[i],
						    ",");
					}
				}
			}

			for (j = 0; events_table != NULL &&
			    events_table[j].eventselect != NT_END;
			    j++) {
				if (C(i) & events_table[j].supported_counters) {
					(void) strcat(gpc_names[i],
					    events_table[j].name);
					(void) strcat(gpc_names[i], ",");
				}
			}

			/* Remove trailing comma */
			gpc_names[i][size - 1] = '\0';
		}
	}

	return (0);
}

static uint_t core_pcbe_ncounters()
{
	return (total_pmc);
}

static const char *core_pcbe_impl_name(void)
{
	return (core_impl_name);
}

static const char *core_pcbe_cpuref(void)
{
	return (core_cpuref);
}

static char *core_pcbe_list_events(uint_t picnum)
{
	ASSERT(picnum < cpc_ncounters);

	if (picnum < num_gpc) {
		return (gpc_names[picnum]);
	} else {
		return (ffc_allnames[picnum - num_gpc]);
	}
}

static char *core_pcbe_list_attrs(void)
{
	if (versionid >= 3) {
		return ("edge,inv,umask,cmask,anythr");
	} else {
		return ("edge,pc,inv,umask,cmask");
	}
}

static const struct nametable_core_uarch *
find_gpcevent_core_uarch(char *name,
    const struct nametable_core_uarch *nametable)
{
	const struct nametable_core_uarch *n;
	int compare_result = -1;

	for (n = nametable; n->event_num != NT_END; n++) {
		compare_result = strcmp(name, n->name);
		if (compare_result <= 0) {
			break;
		}
	}

	if (compare_result == 0) {
		return (n);
	}

	return (NULL);
}

static const struct generic_events *
find_generic_events(char *name, const struct generic_events *table)
{
	const struct generic_events *n;

	for (n = table; n->event_num != NT_END; n++) {
		if (strcmp(name, n->name) == 0) {
			return (n);
		};
	}

	return (NULL);
}

static const struct events_table_t *
find_gpcevent(char *name)
{
	int i;

	/* Search architectural events */
	for (i = 0; i < known_arch_events; i++) {
		if (strcmp(name, arch_events_table[i].name) == 0 ||
		    strcmp(name, arch_genevents_table[i]) == 0) {
			if (((1U << i) & arch_events_vector) == 0) {
				return (&arch_events_table[i]);
			}
		}
	}

	/* Search non-architectural events */
	if (events_table != NULL) {
		for (i = 0; events_table[i].eventselect != NT_END; i++) {
			if (strcmp(name, events_table[i].name) == 0) {
				return (&events_table[i]);
			}
		}
	}

	return (NULL);
}

static uint64_t
core_pcbe_event_coverage(char *event)
{
	uint64_t bitmap;
	uint64_t bitmask;
	const struct events_table_t *n;
	int i;

	bitmap = 0;

	/* Is it an event that a GPC can track? */
	if (versionid >= 3) {
		n = find_gpcevent(event);
		if (n != NULL) {
			bitmap |= (n->supported_counters &
			    BITMASK_XBITS(num_gpc));
		}
	} else {
		if (find_generic_events(event, cmn_generic_events) != NULL) {
			bitmap |= BITMASK_XBITS(num_gpc);
		} else if (find_generic_events(event,
		    generic_events_pic0) != NULL) {
			bitmap |= 1ULL;
		} else if (find_gpcevent_core_uarch(event,
		    cmn_gpc_events_core_uarch) != NULL) {
			bitmap |= BITMASK_XBITS(num_gpc);
		} else if (find_gpcevent_core_uarch(event, pic0_events) !=
		    NULL) {
			bitmap |= 1ULL;
		} else if (find_gpcevent_core_uarch(event, pic1_events) !=
		    NULL) {
			bitmap |= 1ULL << 1;
		}
	}

	/* Check if the event can be counted in the fixed-function counters */
	if (num_ffc > 0) {
		bitmask = 1ULL << num_gpc;
		for (i = 0; i < num_ffc; i++) {
			if (strcmp(event, ffc_names[i]) == 0) {
				bitmap |= bitmask;
			} else if (strcmp(event, ffc_genericnames[i]) == 0) {
				bitmap |= bitmask;
			}
			bitmask = bitmask << 1;
		}
	}

	return (bitmap);
}

static uint64_t
core_pcbe_overflow_bitmap(void)
{
	uint64_t interrupt_status;
	uint64_t intrbits_ffc;
	uint64_t intrbits_gpc;
	extern int kcpc_hw_overflow_intr_installed;
	uint64_t overflow_bitmap;

	RDMSR(PERF_GLOBAL_STATUS, interrupt_status);
	WRMSR(PERF_GLOBAL_OVF_CTRL, interrupt_status);

	interrupt_status = interrupt_status & control_mask;
	intrbits_ffc = (interrupt_status >> 32) & control_ffc;
	intrbits_gpc = interrupt_status & control_gpc;
	overflow_bitmap = (intrbits_ffc << num_gpc) | intrbits_gpc;

	ASSERT(kcpc_hw_overflow_intr_installed);
	(*kcpc_hw_enable_cpc_intr)();

	return (overflow_bitmap);
}

static int
check_cpc_securitypolicy(core_pcbe_config_t *conf,
    const struct nametable_core_uarch *n)
{
	if (conf->core_ctl & n->restricted_bits) {
		if (secpolicy_cpc_cpu(crgetcred()) != 0) {
			return (CPC_ATTR_REQUIRES_PRIVILEGE);
		}
	}
	return (0);
}

static int
configure_gpc(uint_t picnum, char *event, uint64_t preset, uint32_t flags,
    uint_t nattrs, kcpc_attr_t *attrs, void **data)
{
	core_pcbe_config_t	conf;
	const struct nametable_core_uarch	*n;
	const struct generic_events *k = NULL;
	const struct nametable_core_uarch	*m;
	const struct nametable_core_uarch	*picspecific_events;
	struct nametable_core_uarch	nt_raw = { "", 0x0, 0x0 };
	uint_t			i;
	long			event_num;
	const struct events_table_t *eventcode;

	if (((preset & BITS_EXTENDED_FROM_31) != 0) &&
	    ((preset & BITS_EXTENDED_FROM_31) !=
	    BITS_EXTENDED_FROM_31)) {

		/*
		 * Bits beyond bit-31 in the general-purpose counters can only
		 * be written to by extension of bit 31.  We cannot preset
		 * these bits to any value other than all 1s or all 0s.
		 */
		return (CPC_ATTRIBUTE_OUT_OF_RANGE);
	}

	if (versionid >= 3) {
		eventcode = find_gpcevent(event);
		if (eventcode != NULL) {
			if ((C(picnum) & eventcode->supported_counters) == 0) {
				return (CPC_PIC_NOT_CAPABLE);
			}
			if (nattrs > 0 &&
			    (strncmp("PAPI_", event, 5) == 0)) {
				return (CPC_ATTRIBUTE_OUT_OF_RANGE);
			}
			conf.core_ctl = eventcode->eventselect;
			conf.core_ctl |= eventcode->unitmask <<
			    CORE_UMASK_SHIFT;
		} else {
			/* Event specified as raw event code */
			if (ddi_strtol(event, NULL, 0, &event_num) != 0) {
				return (CPC_INVALID_EVENT);
			}
			conf.core_ctl = event_num & 0xFF;
		}
	} else {
		if ((k = find_generic_events(event, cmn_generic_events)) !=
		    NULL ||
		    (picnum == 0 &&
		    (k = find_generic_events(event, generic_events_pic0)) !=
		    NULL)) {
			if (nattrs > 0) {
				return (CPC_ATTRIBUTE_OUT_OF_RANGE);
			}
			conf.core_ctl = k->event_num;
			conf.core_ctl |= k->umask << CORE_UMASK_SHIFT;
		} else {
			/* Not a generic event */

			n = find_gpcevent_core_uarch(event,
			    cmn_gpc_events_core_uarch);
			if (n == NULL) {
				switch (picnum) {
					case 0:
						picspecific_events =
						    pic0_events;
						break;
					case 1:
						picspecific_events =
						    pic1_events;
						break;
					default:
						picspecific_events = NULL;
						break;
				}
				if (picspecific_events != NULL) {
					n = find_gpcevent_core_uarch(event,
					    picspecific_events);
				}
			}
			if (n == NULL) {

				/*
				 * Check if this is a case where the event was
				 * specified directly by its event number
				 * instead of its name string.
				 */
				if (ddi_strtol(event, NULL, 0, &event_num) !=
				    0) {
					return (CPC_INVALID_EVENT);
				}

				event_num = event_num & 0xFF;

				/*
				 * Search the event table to find out if the
				 * event specified has an privilege
				 * requirements.  Currently none of the
				 * pic-specific counters have any privilege
				 * requirements.  Hence only the table
				 * cmn_gpc_events_core_uarch is searched.
				 */
				for (m = cmn_gpc_events_core_uarch;
				    m->event_num != NT_END;
				    m++) {
					if (event_num == m->event_num) {
						break;
					}
				}
				if (m->event_num == NT_END) {
					nt_raw.event_num = (uint8_t)event_num;
					n = &nt_raw;
				} else {
					n = m;
				}
			}
			conf.core_ctl = n->event_num; /* Event Select */
		}
	}


	conf.core_picno = picnum;
	conf.core_pictype = CORE_GPC;
	conf.core_rawpic = preset & mask_gpc;

	conf.core_pes = GPC_BASE_PES + picnum;
	conf.core_pmc = GPC_BASE_PMC + picnum;

	for (i = 0; i < nattrs; i++) {
		if (strncmp(attrs[i].ka_name, "umask", 6) == 0) {
			if ((attrs[i].ka_val | CORE_UMASK_MASK) !=
			    CORE_UMASK_MASK) {
				return (CPC_ATTRIBUTE_OUT_OF_RANGE);
			}
			/* Clear out the default umask */
			conf.core_ctl &= ~ (CORE_UMASK_MASK <<
			    CORE_UMASK_SHIFT);
			/* Use the user provided umask */
			conf.core_ctl |= attrs[i].ka_val <<
			    CORE_UMASK_SHIFT;
		} else  if (strncmp(attrs[i].ka_name, "edge", 6) == 0) {
			if (attrs[i].ka_val != 0)
				conf.core_ctl |= CORE_EDGE;
		} else if (strncmp(attrs[i].ka_name, "inv", 4) == 0) {
			if (attrs[i].ka_val != 0)
				conf.core_ctl |= CORE_INV;
		} else if (strncmp(attrs[i].ka_name, "cmask", 6) == 0) {
			if ((attrs[i].ka_val | CORE_CMASK_MASK) !=
			    CORE_CMASK_MASK) {
				return (CPC_ATTRIBUTE_OUT_OF_RANGE);
			}
			conf.core_ctl |= attrs[i].ka_val <<
			    CORE_CMASK_SHIFT;
		} else if (strncmp(attrs[i].ka_name, "anythr", 7) ==
		    0) {
			if (versionid < 3)
				return (CPC_INVALID_ATTRIBUTE);
			if (secpolicy_cpc_cpu(crgetcred()) != 0) {
				return (CPC_ATTR_REQUIRES_PRIVILEGE);
			}
			if (attrs[i].ka_val != 0)
				conf.core_ctl |= CORE_ANYTHR;
		} else {
			return (CPC_INVALID_ATTRIBUTE);
		}
	}

	if (flags & CPC_COUNT_USER)
		conf.core_ctl |= CORE_USR;
	if (flags & CPC_COUNT_SYSTEM)
		conf.core_ctl |= CORE_OS;
	if (flags & CPC_OVF_NOTIFY_EMT)
		conf.core_ctl |= CORE_INT;
	conf.core_ctl |= CORE_EN;

	if (versionid < 3 && k == NULL) {
		if (check_cpc_securitypolicy(&conf, n) != 0) {
			return (CPC_ATTR_REQUIRES_PRIVILEGE);
		}
	}

	*data = kmem_alloc(sizeof (core_pcbe_config_t), KM_SLEEP);
	*((core_pcbe_config_t *)*data) = conf;

	return (0);
}

static int
configure_ffc(uint_t picnum, char *event, uint64_t preset, uint32_t flags,
    uint_t nattrs, kcpc_attr_t *attrs, void **data)
{
	core_pcbe_config_t	*conf;
	uint_t			i;

	if (picnum - num_gpc >= num_ffc) {
		return (CPC_INVALID_PICNUM);
	}

	if ((strcmp(ffc_names[picnum-num_gpc], event) != 0) &&
	    (strcmp(ffc_genericnames[picnum-num_gpc], event) != 0)) {
		return (CPC_INVALID_EVENT);
	}

	if ((versionid < 3) && (nattrs != 0)) {
		return (CPC_INVALID_ATTRIBUTE);
	}

	conf = kmem_alloc(sizeof (core_pcbe_config_t), KM_SLEEP);
	conf->core_ctl = 0;

	for (i = 0; i < nattrs; i++) {
		if (strncmp(attrs[i].ka_name, "anythr", 7) == 0) {
			if (secpolicy_cpc_cpu(crgetcred()) != 0) {
				kmem_free(conf, sizeof (core_pcbe_config_t));
				return (CPC_ATTR_REQUIRES_PRIVILEGE);
			}
			if (attrs[i].ka_val != 0) {
				conf->core_ctl |= CORE_FFC_ANYTHR;
			}
		} else {
			kmem_free(conf, sizeof (core_pcbe_config_t));
			return (CPC_INVALID_ATTRIBUTE);
		}
	}

	conf->core_picno = picnum;
	conf->core_pictype = CORE_FFC;
	conf->core_rawpic = preset & mask_ffc;
	conf->core_pmc = FFC_BASE_PMC + (picnum - num_gpc);

	/* All fixed-function counters have the same control register */
	conf->core_pes = PERF_FIXED_CTR_CTRL;

	if (flags & CPC_COUNT_USER)
		conf->core_ctl |= CORE_FFC_USR_EN;
	if (flags & CPC_COUNT_SYSTEM)
		conf->core_ctl |= CORE_FFC_OS_EN;
	if (flags & CPC_OVF_NOTIFY_EMT)
		conf->core_ctl |= CORE_FFC_PMI;

	*data = conf;
	return (0);
}

/*ARGSUSED*/
static int
core_pcbe_configure(uint_t picnum, char *event, uint64_t preset,
    uint32_t flags, uint_t nattrs, kcpc_attr_t *attrs, void **data,
    void *token)
{
	int			ret;
	core_pcbe_config_t	*conf;

	/*
	 * If we've been handed an existing configuration, we need only preset
	 * the counter value.
	 */
	if (*data != NULL) {
		conf = *data;
		ASSERT(conf->core_pictype == CORE_GPC ||
		    conf->core_pictype == CORE_FFC);
		if (conf->core_pictype == CORE_GPC)
			conf->core_rawpic = preset & mask_gpc;
		else /* CORE_FFC */
			conf->core_rawpic = preset & mask_ffc;
		return (0);
	}

	if (picnum >= total_pmc) {
		return (CPC_INVALID_PICNUM);
	}

	if (picnum < num_gpc) {
		ret = configure_gpc(picnum, event, preset, flags,
		    nattrs, attrs, data);
	} else {
		ret = configure_ffc(picnum, event, preset, flags,
		    nattrs, attrs, data);
	}
	return (ret);
}

static void
core_pcbe_program(void *token)
{
	core_pcbe_config_t	*cfg;
	uint64_t		perf_global_ctrl;
	uint64_t		perf_fixed_ctr_ctrl;
	uint64_t		curcr4;

	core_pcbe_allstop();

	curcr4 = getcr4();
	if (kcpc_allow_nonpriv(token))
		/* Allow RDPMC at any ring level */
		setcr4(curcr4 | CR4_PCE);
	else
		/* Allow RDPMC only at ring 0 */
		setcr4(curcr4 & ~CR4_PCE);

	/* Clear any overflow indicators before programming the counters */
	WRMSR(PERF_GLOBAL_OVF_CTRL, MASK_CONDCHGD_OVFBUFFER | control_mask);

	cfg = NULL;
	perf_global_ctrl = 0;
	perf_fixed_ctr_ctrl = 0;
	cfg = (core_pcbe_config_t *)kcpc_next_config(token, cfg, NULL);
	while (cfg != NULL) {
		ASSERT(cfg->core_pictype == CORE_GPC ||
		    cfg->core_pictype == CORE_FFC);

		if (cfg->core_pictype == CORE_GPC) {
			/*
			 * General-purpose counter registers have write
			 * restrictions where only the lower 32-bits can be
			 * written to.  The rest of the relevant bits are
			 * written to by extension from bit 31 (all ZEROS if
			 * bit-31 is ZERO and all ONE if bit-31 is ONE).  This
			 * makes it possible to write to the counter register
			 * only values that have all ONEs or all ZEROs in the
			 * higher bits.
			 */
			if (((cfg->core_rawpic & BITS_EXTENDED_FROM_31) == 0) ||
			    ((cfg->core_rawpic & BITS_EXTENDED_FROM_31) ==
			    BITS_EXTENDED_FROM_31)) {
				/*
				 * Straighforward case where the higher bits
				 * are all ZEROs or all ONEs.
				 */
				WRMSR(cfg->core_pmc,
				    (cfg->core_rawpic & mask_gpc));
			} else {
				/*
				 * The high order bits are not all the same.
				 * We save what is currently in the registers
				 * and do not write to it.  When we want to do
				 * a read from this register later (in
				 * core_pcbe_sample()), we subtract the value
				 * we save here to get the actual event count.
				 *
				 * NOTE: As a result, we will not get overflow
				 * interrupts as expected.
				 */
				RDMSR(cfg->core_pmc, cfg->core_rawpic);
				cfg->core_rawpic = cfg->core_rawpic & mask_gpc;
			}
			WRMSR(cfg->core_pes, cfg->core_ctl);
			perf_global_ctrl |= 1ull << cfg->core_picno;
		} else {
			/*
			 * Unlike the general-purpose counters, all relevant
			 * bits of fixed-function counters can be written to.
			 */
			WRMSR(cfg->core_pmc, cfg->core_rawpic & mask_ffc);

			/*
			 * Collect the control bits for all the
			 * fixed-function counters and write it at one shot
			 * later in this function
			 */
			perf_fixed_ctr_ctrl |= cfg->core_ctl <<
			    ((cfg->core_picno - num_gpc) * CORE_FFC_ATTR_SIZE);
			perf_global_ctrl |=
			    1ull << (cfg->core_picno - num_gpc + 32);
		}

		cfg = (core_pcbe_config_t *)
		    kcpc_next_config(token, cfg, NULL);
	}

	/* Enable all the counters */
	WRMSR(PERF_FIXED_CTR_CTRL, perf_fixed_ctr_ctrl);
	WRMSR(PERF_GLOBAL_CTRL, perf_global_ctrl);
}

static void
core_pcbe_allstop(void)
{
	/* Disable all the counters together */
	WRMSR(PERF_GLOBAL_CTRL, ALL_STOPPED);

	setcr4(getcr4() & ~CR4_PCE);
}

static void
core_pcbe_sample(void *token)
{
	uint64_t		*daddr;
	uint64_t		curpic;
	core_pcbe_config_t	*cfg;
	uint64_t			counter_mask;

	cfg = (core_pcbe_config_t *)kcpc_next_config(token, NULL, &daddr);
	while (cfg != NULL) {
		ASSERT(cfg->core_pictype == CORE_GPC ||
		    cfg->core_pictype == CORE_FFC);

		curpic = rdmsr(cfg->core_pmc);

		DTRACE_PROBE4(core__pcbe__sample,
		    uint64_t, cfg->core_pmc,
		    uint64_t, curpic,
		    uint64_t, cfg->core_rawpic,
		    uint64_t, *daddr);

		if (cfg->core_pictype == CORE_GPC) {
			counter_mask = mask_gpc;
		} else {
			counter_mask = mask_ffc;
		}
		curpic = curpic & counter_mask;
		if (curpic >= cfg->core_rawpic) {
			*daddr += curpic - cfg->core_rawpic;
		} else {
			/* Counter overflowed since our last sample */
			*daddr += counter_mask - (cfg->core_rawpic - curpic) +
			    1;
		}
		cfg->core_rawpic = *daddr & counter_mask;

		cfg =
		    (core_pcbe_config_t *)kcpc_next_config(token, cfg, &daddr);
	}
}

static void
core_pcbe_free(void *config)
{
	kmem_free(config, sizeof (core_pcbe_config_t));
}

static struct modlpcbe core_modlpcbe = {
	&mod_pcbeops,
	"Core Performance Counters",
	&core_pcbe_ops
};

static struct modlinkage core_modl = {
	MODREV_1,
	&core_modlpcbe,
};

int
_init(void)
{
	if (core_pcbe_init() != 0) {
		return (ENOTSUP);
	}
	return (mod_install(&core_modl));
}

int
_fini(void)
{
	return (mod_remove(&core_modl));
}

int
_info(struct modinfo *mi)
{
	return (mod_info(&core_modl, mi));
}
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
 * Copyright 2018 Joyent, Inc.
 */

/*
 * This file contains preset event names from the Performance Application
 * Programming Interface v3.5 which included the following notice:
 *
 *                             Copyright (c) 2005,6
 *                           Innovative Computing Labs
 *                         Computer Science Department,
 *                            University of Tennessee,
 *                                 Knoxville, TN.
 *                              All Rights Reserved.
 *
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 *    * Redistributions of source code must retain the above copyright notice,
 *      this list of conditions and the following disclaimer.
 *    * Redistributions in binary form must reproduce the above copyright
 *      notice, this list of conditions and the following disclaimer in the
 *      documentation and/or other materials provided with the distribution.
 *    * Neither the name of the University of Tennessee nor the names of its
 *      contributors may be used to endorse or promote products derived from
 *      this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 *
 *
 * This open source software license conforms to the BSD License template.
 */


#ifndef _CORE_PCBE_TABLE_H
#define	_CORE_PCBE_TABLE_H

/*
 * Structure definition for PCBE events.
 */

#ifdef __cplusplus
extern "C" {
#endif

#include <sys/types.h>

struct events_table_t {
	uint8_t		eventselect;
	uint8_t		unitmask;
	uint64_t	supported_counters;
	const char	*name;
};

/* Used to describe which counters support an event */
#define	C(x) (1 << (x))
#define	C0 C(0)
#define	C1 C(1)
#define	C2 C(2)
#define	C3 C(3)
#define	C_ALL 0xFFFFFFFFFFFFFFFF

#define	NT_END	0xFF

#ifdef __cplusplus
}
#endif

#endif /* _CORE_PCBE_TABLE_H */
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

/*
 * This file contains preset event names from the Performance Application
 * Programming Interface v3.5 which included the following notice:
 *
 *                             Copyright (c) 2005,6
 *                           Innovative Computing Labs
 *                         Computer Science Department,
 *                            University of Tennessee,
 *                                 Knoxville, TN.
 *                              All Rights Reserved.
 *
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 *    * Redistributions of source code must retain the above copyright notice,
 *      this list of conditions and the following disclaimer.
 *    * Redistributions in binary form must reproduce the above copyright
 *	notice, this list of conditions and the following disclaimer in the
 *	documentation and/or other materials provided with the distribution.
 *    * Neither the name of the University of Tennessee nor the names of its
 *      contributors may be used to endorse or promote products derived from
 *	this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 *
 *
 * This open source software license conforms to the BSD License template.
 */

/*
 * Portions Copyright 2009 Advanced Micro Devices, Inc.
 * Copyright 2019 Joyent, Inc.
 * Copyright 2024 Oxide Computer Company
 */

/*
 * Performance Counter Back-End for AMD Opteron, AMD Athlon 64, and Zen
 * era processors.
 */

#include <sys/cpuvar.h>
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/cpc_pcbe.h>
#include <sys/kmem.h>
#include <sys/sdt.h>
#include <sys/modctl.h>
#include <sys/errno.h>
#include <sys/debug.h>
#include <sys/archsystm.h>
#include <sys/x86_archext.h>
#include <sys/privregs.h>
#include <sys/ddi.h>
#include <sys/sunddi.h>

#include "opteron_pcbe_table.h"
#include <opteron_pcbe_cpcgen.h>

static int opt_pcbe_init(void);
static uint_t opt_pcbe_ncounters(void);
static const char *opt_pcbe_impl_name(void);
static const char *opt_pcbe_cpuref(void);
static char *opt_pcbe_list_events(uint_t picnum);
static char *opt_pcbe_list_attrs(void);
static uint64_t opt_pcbe_event_coverage(char *event);
static uint64_t opt_pcbe_overflow_bitmap(void);
static int opt_pcbe_configure(uint_t picnum, char *event, uint64_t preset,
    uint32_t flags, uint_t nattrs, kcpc_attr_t *attrs, void **data,
    void *token);
static void opt_pcbe_program(void *token);
static void opt_pcbe_allstop(void);
static void opt_pcbe_sample(void *token);
static void opt_pcbe_free(void *config);

static pcbe_ops_t opt_pcbe_ops = {
	PCBE_VER_1,
	CPC_CAP_OVERFLOW_INTERRUPT,
	opt_pcbe_ncounters,
	opt_pcbe_impl_name,
	opt_pcbe_cpuref,
	opt_pcbe_list_events,
	opt_pcbe_list_attrs,
	opt_pcbe_event_coverage,
	opt_pcbe_overflow_bitmap,
	opt_pcbe_configure,
	opt_pcbe_program,
	opt_pcbe_allstop,
	opt_pcbe_sample,
	opt_pcbe_free
};

/*
 * Base MSR addresses for the PerfEvtSel registers and the counters themselves.
 * Add counter number to base address to get corresponding MSR address.
 */
#define	PES_BASE_ADDR	0xC0010000
#define	PIC_BASE_ADDR	0xC0010004

/*
 * Base MSR addresses for the PerfEvtSel registers and counters. The counter and
 * event select registers are interleaved, so one needs to multiply the counter
 * number by two to determine what they should be set to.
 */
#define	PES_EXT_BASE_ADDR	0xC0010200
#define	PIC_EXT_BASE_ADDR	0xC0010201

/*
 * The number of counters present depends on which CPU features are present.
 */
#define	OPT_PCBE_DEF_NCOUNTERS	4
#define	OPT_PCBE_EXT_NCOUNTERS	6

/*
 * Define offsets and masks for the fields in the Performance
 * Event-Select (PES) registers.
 */
#define	OPT_PES_HOST_SHIFT	41
#define	OPT_PES_GUEST_SHIFT	40
#define	OPT_PES_EVSELHI_SHIFT	32
#define	OPT_PES_CMASK_SHIFT	24
#define	OPT_PES_CMASK_MASK	0xFF
#define	OPT_PES_INV_SHIFT	23
#define	OPT_PES_ENABLE_SHIFT	22
#define	OPT_PES_INT_SHIFT	20
#define	OPT_PES_PC_SHIFT	19
#define	OPT_PES_EDGE_SHIFT	18
#define	OPT_PES_OS_SHIFT	17
#define	OPT_PES_USR_SHIFT	16
#define	OPT_PES_UMASK_SHIFT	8
#define	OPT_PES_UMASK_MASK	0xFF

#define	OPT_PES_INV		(1ULL << OPT_PES_INV_SHIFT)
#define	OPT_PES_ENABLE		(1ULL << OPT_PES_ENABLE_SHIFT)
#define	OPT_PES_INT		(1ULL << OPT_PES_INT_SHIFT)
#define	OPT_PES_PC		(1ULL << OPT_PES_PC_SHIFT)
#define	OPT_PES_EDGE		(1ULL << OPT_PES_EDGE_SHIFT)
#define	OPT_PES_OS		(1ULL << OPT_PES_OS_SHIFT)
#define	OPT_PES_USR		(1ULL << OPT_PES_USR_SHIFT)
#define	OPT_PES_HOST		(1ULL << OPT_PES_HOST_SHIFT)
#define	OPT_PES_GUEST		(1ULL << OPT_PES_GUEST_SHIFT)

typedef struct _opt_pcbe_config {
	uint8_t		opt_picno;	/* Counter number: 0, 1, 2, or 3 */
	uint64_t	opt_evsel;	/* Event Selection register */
	uint64_t	opt_rawpic;	/* Raw counter value */
} opt_pcbe_config_t;

opt_pcbe_config_t nullcfgs[OPT_PCBE_EXT_NCOUNTERS] = {
	{ 0, 0, 0 },
	{ 1, 0, 0 },
	{ 2, 0, 0 },
	{ 3, 0, 0 },
	{ 4, 0, 0 },
	{ 5, 0, 0 },
};

typedef uint64_t (*opt_pcbe_addr_f)(uint_t);

typedef struct opt_pcbe_data {
	uint_t		opd_ncounters;
	uint_t		opd_cmask;
	opt_pcbe_addr_f	opd_pesf;
	opt_pcbe_addr_f	opd_picf;
} opt_pcbe_data_t;

opt_pcbe_data_t opd;

#define	MASK48		0xFFFFFFFFFFFF

#define	EV_END {NULL, 0}
#define	GEN_EV_END {NULL, NULL, 0 }

/*
 * The following Macros are used to define tables of events that are used by
 * various families and some generic classes of events.
 *
 * When programming a performance counter there are two different values that we
 * need to set:
 *
 *   o Event - Determines the general class of event that is being used.
 *   o Unit  - A further breakdown that gives more specific value.
 *
 * Prior to the introduction of family 17h support, all family specific events
 * were programmed based on their event. The generic events, which tried to
 * provide PAPI mappings to events specified an additional unit mask.
 *
 * Starting with Family 17h, CPU performance counters default to using both the
 * unit mask and the event select. Generic events are always aliases to a
 * specific event/unit pair, hence why the units for them are always zero. In
 * addition, the naming of events in family 17h has been changed to reflect
 * AMD's guide. While this is a departure from what people are used to, it is
 * believed that matching the more detailed literature that folks are told to
 * reference is more valuable.
 */

#define	AMD_cmn_events						\
	{ "FP_dispatched_fpu_ops",			0x0 },	\
	{ "FP_cycles_no_fpu_ops_retired",		0x1 },	\
	{ "FP_dispatched_fpu_ops_ff",			0x2 },	\
	{ "LS_seg_reg_load",				0x20 },	\
	{ "LS_uarch_resync_self_modify",		0x21 },	\
	{ "LS_uarch_resync_snoop",			0x22 },	\
	{ "LS_buffer_2_full",				0x23 },	\
	{ "LS_locked_operation",			0x24 },	\
	{ "LS_retired_cflush",				0x26 },	\
	{ "LS_retired_cpuid",				0x27 },	\
	{ "DC_access",					0x40 },	\
	{ "DC_miss",					0x41 },	\
	{ "DC_refill_from_L2",				0x42 },	\
	{ "DC_refill_from_system",			0x43 },	\
	{ "DC_copyback",				0x44 },	\
	{ "DC_dtlb_L1_miss_L2_hit",			0x45 },	\
	{ "DC_dtlb_L1_miss_L2_miss",			0x46 },	\
	{ "DC_misaligned_data_ref",			0x47 },	\
	{ "DC_uarch_late_cancel_access",		0x48 },	\
	{ "DC_uarch_early_cancel_access",		0x49 },	\
	{ "DC_1bit_ecc_error_found",			0x4A },	\
	{ "DC_dispatched_prefetch_instr",		0x4B },	\
	{ "DC_dcache_accesses_by_locks",		0x4C },	\
	{ "BU_memory_requests",				0x65 },	\
	{ "BU_data_prefetch",				0x67 },	\
	{ "BU_system_read_responses",			0x6C },	\
	{ "BU_cpu_clk_unhalted",			0x76 },	\
	{ "BU_internal_L2_req",				0x7D },	\
	{ "BU_fill_req_missed_L2",			0x7E },	\
	{ "BU_fill_into_L2",				0x7F },	\
	{ "IC_fetch",					0x80 },	\
	{ "IC_miss",					0x81 },	\
	{ "IC_refill_from_L2",				0x82 },	\
	{ "IC_refill_from_system",			0x83 },	\
	{ "IC_itlb_L1_miss_L2_hit",			0x84 },	\
	{ "IC_itlb_L1_miss_L2_miss",			0x85 },	\
	{ "IC_uarch_resync_snoop",			0x86 },	\
	{ "IC_instr_fetch_stall",			0x87 },	\
	{ "IC_return_stack_hit",			0x88 },	\
	{ "IC_return_stack_overflow",			0x89 },	\
	{ "FR_retired_x86_instr_w_excp_intr",		0xC0 },	\
	{ "FR_retired_uops",				0xC1 },	\
	{ "FR_retired_branches_w_excp_intr",		0xC2 },	\
	{ "FR_retired_branches_mispred",		0xC3 },	\
	{ "FR_retired_taken_branches",			0xC4 },	\
	{ "FR_retired_taken_branches_mispred",		0xC5 },	\
	{ "FR_retired_far_ctl_transfer",		0xC6 },	\
	{ "FR_retired_resyncs",				0xC7 },	\
	{ "FR_retired_near_rets",			0xC8 },	\
	{ "FR_retired_near_rets_mispred",		0xC9 },	\
	{ "FR_retired_taken_branches_mispred_addr_miscomp",	0xCA },\
	{ "FR_retired_fastpath_double_op_instr",	0xCC },	\
	{ "FR_intr_masked_cycles",			0xCD },	\
	{ "FR_intr_masked_while_pending_cycles",	0xCE },	\
	{ "FR_taken_hardware_intrs",			0xCF },	\
	{ "FR_nothing_to_dispatch",			0xD0 },	\
	{ "FR_dispatch_stalls",				0xD1 },	\
	{ "FR_dispatch_stall_branch_abort_to_retire",	0xD2 },	\
	{ "FR_dispatch_stall_serialization",		0xD3 },	\
	{ "FR_dispatch_stall_segment_load",		0xD4 },	\
	{ "FR_dispatch_stall_reorder_buffer_full",	0xD5 },	\
	{ "FR_dispatch_stall_resv_stations_full",	0xD6 },	\
	{ "FR_dispatch_stall_fpu_full",			0xD7 },	\
	{ "FR_dispatch_stall_ls_full",			0xD8 },	\
	{ "FR_dispatch_stall_waiting_all_quiet",	0xD9 },	\
	{ "FR_dispatch_stall_far_ctl_trsfr_resync_branch_pend",	0xDA },\
	{ "FR_fpu_exception",				0xDB },	\
	{ "FR_num_brkpts_dr0",				0xDC },	\
	{ "FR_num_brkpts_dr1",				0xDD },	\
	{ "FR_num_brkpts_dr2",				0xDE },	\
	{ "FR_num_brkpts_dr3",				0xDF },	\
	{ "NB_mem_ctrlr_page_access",			0xE0 },	\
	{ "NB_mem_ctrlr_turnaround",			0xE3 },	\
	{ "NB_mem_ctrlr_bypass_counter_saturation",	0xE4 },	\
	{ "NB_cpu_io_to_mem_io",			0xE9 },	\
	{ "NB_cache_block_commands",			0xEA },	\
	{ "NB_sized_commands",				0xEB },	\
	{ "NB_ht_bus0_bandwidth",			0xF6 }

#define	AMD_FAMILY_f_events					\
	{ "BU_quadwords_written_to_system",		0x6D },	\
	{ "FR_retired_fpu_instr",			0xCB },	\
	{ "NB_mem_ctrlr_page_table_overflow",		0xE1 },	\
	{ "NB_sized_blocks",				0xE5 },	\
	{ "NB_ECC_errors",				0xE8 },	\
	{ "NB_probe_result",				0xEC },	\
	{ "NB_gart_events",				0xEE },	\
	{ "NB_ht_bus1_bandwidth",			0xF7 },	\
	{ "NB_ht_bus2_bandwidth",			0xF8 }

#define	AMD_FAMILY_10h_events					\
	{ "FP_retired_sse_ops",				0x3 },	\
	{ "FP_retired_move_ops",			0x4 },	\
	{ "FP_retired_serialize_ops",			0x5 },	\
	{ "FP_serialize_ops_cycles",			0x6 },	\
	{ "LS_cancelled_store_to_load_fwd_ops",		0x2A },	\
	{ "LS_smi_received",				0x2B },	\
	{ "DC_dtlb_L1_hit",				0x4D },	\
	{ "LS_ineffective_prefetch",			0x52 },	\
	{ "LS_global_tlb_flush",			0x54 },	\
	{ "BU_octwords_written_to_system",		0x6D },	\
	{ "Page_size_mismatches",			0x165 },	\
	{ "IC_eviction",				0x8B },	\
	{ "IC_cache_lines_invalidate",			0x8C },	\
	{ "IC_itlb_reload",				0x99 },	\
	{ "IC_itlb_reload_aborted",			0x9A },	\
	{ "FR_retired_mmx_sse_fp_instr",		0xCB },	\
	{ "Retired_x87_fp_ops",				0x1C0 },	\
	{ "IBS_ops_tagged",				0x1CF },	\
	{ "LFENCE_inst_retired",			0x1D3 },	\
	{ "SFENCE_inst_retired",			0x1D4 },	\
	{ "MFENCE_inst_retired",			0x1D5 },	\
	{ "NB_mem_ctrlr_page_table_overflow",		0xE1 },	\
	{ "NB_mem_ctrlr_dram_cmd_slots_missed",		0xE2 },	\
	{ "NB_thermal_status",				0xE8 },	\
	{ "NB_probe_results_upstream_req",		0xEC },	\
	{ "NB_gart_events",				0xEE },	\
	{ "NB_mem_ctrlr_req",				0x1F0 },	\
	{ "CB_cpu_to_dram_req_to_target",		0x1E0 },	\
	{ "CB_io_to_dram_req_to_target",		0x1E1 },	\
	{ "CB_cpu_read_cmd_latency_to_target_0_to_3",	0x1E2 },	\
	{ "CB_cpu_read_cmd_req_to_target_0_to_3",	0x1E3 },	\
	{ "CB_cpu_read_cmd_latency_to_target_4_to_7",	0x1E4 },	\
	{ "CB_cpu_read_cmd_req_to_target_4_to_7",	0x1E5 },	\
	{ "CB_cpu_cmd_latency_to_target_0_to_7",	0x1E6 },	\
	{ "CB_cpu_req_to_target_0_to_7",		0x1E7 },	\
	{ "NB_ht_bus1_bandwidth",			0xF7 },	\
	{ "NB_ht_bus2_bandwidth",			0xF8 },	\
	{ "NB_ht_bus3_bandwidth",			0x1F9 },	\
	{ "L3_read_req",				0x4E0 },	\
	{ "L3_miss",					0x4E1 },	\
	{ "L3_l2_eviction_l3_fill",			0x4E2 },	\
	{ "L3_eviction",				0x4E3 }

#define	AMD_FAMILY_11h_events					\
	{ "BU_quadwords_written_to_system",		0x6D },	\
	{ "FR_retired_mmx_fp_instr",			0xCB },	\
	{ "NB_mem_ctrlr_page_table_events",		0xE1 },	\
	{ "NB_thermal_status",				0xE8 },	\
	{ "NB_probe_results_upstream_req",		0xEC },	\
	{ "NB_dev_events",				0xEE },	\
	{ "NB_mem_ctrlr_req",				0x1F0 }

#define	AMD_cmn_generic_events						\
	{ "PAPI_br_ins",	"FR_retired_branches_w_excp_intr", 0x0 },\
	{ "PAPI_br_msp",	"FR_retired_branches_mispred",	0x0 },	\
	{ "PAPI_br_tkn",	"FR_retired_taken_branches",	0x0 },	\
	{ "PAPI_fp_ops",	"FP_dispatched_fpu_ops",	0x3 },	\
	{ "PAPI_fad_ins",	"FP_dispatched_fpu_ops",	0x1 },	\
	{ "PAPI_fml_ins",	"FP_dispatched_fpu_ops",	0x2 },	\
	{ "PAPI_fpu_idl",	"FP_cycles_no_fpu_ops_retired",	0x0 },	\
	{ "PAPI_tot_cyc",	"BU_cpu_clk_unhalted",		0x0 },	\
	{ "PAPI_tot_ins",	"FR_retired_x86_instr_w_excp_intr", 0x0 }, \
	{ "PAPI_l1_dca",	"DC_access",			0x0 },	\
	{ "PAPI_l1_dcm",	"DC_miss",			0x0 },	\
	{ "PAPI_l1_ldm",	"DC_refill_from_L2",		0xe },	\
	{ "PAPI_l1_stm",	"DC_refill_from_L2",		0x10 },	\
	{ "PAPI_l1_ica",	"IC_fetch",			0x0 },	\
	{ "PAPI_l1_icm",	"IC_miss",			0x0 },	\
	{ "PAPI_l1_icr",	"IC_fetch",			0x0 },	\
	{ "PAPI_l2_dch",	"DC_refill_from_L2",		0x1e },	\
	{ "PAPI_l2_dcm",	"DC_refill_from_system",	0x1e },	\
	{ "PAPI_l2_dcr",	"DC_refill_from_L2",		0xe },	\
	{ "PAPI_l2_dcw",	"DC_refill_from_L2",		0x10 },	\
	{ "PAPI_l2_ich",	"IC_refill_from_L2",		0x0 },	\
	{ "PAPI_l2_icm",	"IC_refill_from_system",	0x0 },	\
	{ "PAPI_l2_ldm",	"DC_refill_from_system",	0xe },	\
	{ "PAPI_l2_stm",	"DC_refill_from_system",	0x10 },	\
	{ "PAPI_res_stl",	"FR_dispatch_stalls",		0x0 },	\
	{ "PAPI_stl_icy",	"FR_nothing_to_dispatch",	0x0 },	\
	{ "PAPI_hw_int",	"FR_taken_hardware_intrs",	0x0 }

#define	OPT_cmn_generic_events						\
	{ "PAPI_tlb_dm",	"DC_dtlb_L1_miss_L2_miss",	0x0 },	\
	{ "PAPI_tlb_im",	"IC_itlb_L1_miss_L2_miss",	0x0 },	\
	{ "PAPI_fp_ins",	"FR_retired_fpu_instr",		0xd },	\
	{ "PAPI_vec_ins",	"FR_retired_fpu_instr",		0x4 }

#define	AMD_FAMILY_10h_generic_events					\
	{ "PAPI_tlb_dm",	"DC_dtlb_L1_miss_L2_miss",	0x7 },	\
	{ "PAPI_tlb_im",	"IC_itlb_L1_miss_L2_miss",	0x3 },	\
	{ "PAPI_l3_dcr",	"L3_read_req",			0xf1 }, \
	{ "PAPI_l3_icr",	"L3_read_req",			0xf2 }, \
	{ "PAPI_l3_tcr",	"L3_read_req",			0xf7 }, \
	{ "PAPI_l3_stm",	"L3_miss",			0xf4 }, \
	{ "PAPI_l3_ldm",	"L3_miss",			0xf3 }, \
	{ "PAPI_l3_tcm",	"L3_miss",			0xf7 }

static const amd_event_t family_f_events[] = {
	AMD_cmn_events,
	AMD_FAMILY_f_events,
	EV_END
};

static const amd_event_t family_10h_events[] = {
	AMD_cmn_events,
	AMD_FAMILY_10h_events,
	EV_END
};

static const amd_event_t family_11h_events[] = {
	AMD_cmn_events,
	AMD_FAMILY_11h_events,
	EV_END
};

static const amd_generic_event_t opt_generic_events[] = {
	AMD_cmn_generic_events,
	OPT_cmn_generic_events,
	GEN_EV_END
};

static const amd_generic_event_t family_10h_generic_events[] = {
	AMD_cmn_generic_events,
	AMD_FAMILY_10h_generic_events,
	GEN_EV_END
};

/*
 * For Family 17h and Family 19h, the cpcgen utility generates all of our events
 * including ones that need specific unit codes, therefore we leave all unit
 * codes out of these. Zen 1, Zen 2, and Zen 3 have different event sets that
 * they support.
 */
static const amd_generic_event_t family_17h_zen1_papi_events[] = {
	{ "PAPI_br_cn",		"ExRetCond" },
	{ "PAPI_br_ins",	"ExRetBrn" },
	{ "PAPI_fpu_idl",	"FpSchedEmpty" },
	{ "PAPI_tot_cyc",	"LsNotHaltedCyc" },
	{ "PAPI_tot_ins",	"ExRetInstr" },
	{ "PAPI_tlb_dm",	"LsL1DTlbMiss" },
	{ "PAPI_tlb_im",	"BpL1TlbMissL2Miss" },
	GEN_EV_END
};

static const amd_generic_event_t family_17h_zen2_papi_events[] = {
	{ "PAPI_br_cn",		"ExRetCond" },
	{ "PAPI_br_ins",	"ExRetBrn" },
	{ "PAPI_tot_cyc",	"LsNotHaltedCyc" },
	{ "PAPI_tot_ins",	"ExRetInstr" },
	{ "PAPI_tlb_dm",	"LsL1DTlbMiss" },
	{ "PAPI_tlb_im",	"BpL1TlbMissL2Miss" },
	GEN_EV_END
};

static const amd_generic_event_t family_19h_zen3_papi_events[] = {
	{ "PAPI_br_cn",		"ExRetCond" },
	{ "PAPI_br_ins",	"ExRetBrn" },
	{ "PAPI_tot_cyc",	"LsNotHaltedCyc" },
	{ "PAPI_tot_ins",	"ExRetInstr" },
	{ "PAPI_tlb_dm",	"LsL1DTlbMiss" },
	{ "PAPI_tlb_im",	"BpL1TlbMissL2TlbMiss" },
	GEN_EV_END
};

static const amd_generic_event_t family_19h_zen4_papi_events[] = {
	{ "PAPI_br_cn",		"ExRetCond" },
	{ "PAPI_br_ins",	"ExRetBrn" },
	{ "PAPI_tot_cyc",	"LsNotHaltedCyc" },
	{ "PAPI_tot_ins",	"ExRetInstr" },
	{ "PAPI_tlb_dm",	"LsL1DTlbMiss" },
	{ "PAPI_tlb_im",	"BpL1TlbMissL2TlbMiss" },
	GEN_EV_END
};

static const amd_generic_event_t family_1ah_zen5_papi_events[] = {
	{ "PAPI_br_cn",		"Retired_Conditional_Branch_Instructions" },
	{ "PAPI_br_ins",	"Retired_Branch_Instructions" },
	{ "PAPI_br_msp",
		"Retired_Conditional_Branch_Instructions_Mispredicted" },
	{ "PAPI_br_ucn",	"Retired_Unconditional_Branch_Instructions" },
	{ "PAPI_tot_cyc",	"Cycles_Not_in_Halt" },
	{ "PAPI_tot_ins",	"Retired_Instructions" },
	{ "PAPI_hw_int",	"Interrupts_Taken" },
	{ "PAPI_tlb_sd",	"TLB_Flush_Events" },
	GEN_EV_END
};

static char	*evlist;
static size_t	evlist_sz;
static const amd_event_t *amd_events = NULL;
static uint_t amd_family, amd_model;
static const amd_generic_event_t *amd_generic_events = NULL;

static char amd_fam_f_rev_ae_bkdg[] = "See \"BIOS and Kernel Developer's "
"Guide for AMD Athlon 64 and AMD Opteron Processors\" (AMD publication 26094)";
static char amd_fam_f_NPT_bkdg[] = "See \"BIOS and Kernel Developer's Guide "
"for AMD NPT Family 0Fh Processors\" (AMD publication 32559)";
static char amd_fam_10h_bkdg[] = "See \"BIOS and Kernel Developer's Guide "
"(BKDG) For AMD Family 10h Processors\" (AMD publication 31116)";
static char amd_fam_11h_bkdg[] = "See \"BIOS and Kernel Developer's Guide "
"(BKDG) For AMD Family 11h Processors\" (AMD publication 41256)";
static char amd_fam_17h_zen1_reg[] = "See \"Open-Source Register Reference For "
"AMD Family 17h Processors Models 00h-2Fh\" (AMD publication 56255) and "
"amd_f17h_zen1_events(3CPC)";
static char amd_fam_17h_zen2_reg[] = "See \"Preliminary Processor Programming "
"Reference (PPR) for AMD Family 17h Model 31h, Revision B0 Processors\" "
"(AMD publication 55803), \"Processor Programming Reference (PPR) for AMD "
"Family 17h Model 71h, Revision B0 Processors\" (AMD publication 56176), and "
"amd_f17h_zen2_events(3CPC)";
static char amd_fam_19h_zen3_reg[] = "See \"Preliminary Processor Programming "
"Reference (PPR) for AMD Family 19h Model 01h, Revision B1 Processors Volume "
"1 of 2\" (AMD publication 55898), \"Processor Programming Reference (PPR) "
"for AMD Family 19h Model 21h, Revision B0 Processors\" (AMD publication "
"56214), and amd_f19h_zen3_events(3CPC)";
static char amd_fam_19h_zen4_reg[] = "See \"Processor Programming Reference "
"(PPR) for AMD Family 19h Model 11h, Revision B1 Processors Volume 1 of 6\" "
"(AMD publication 55901), \"Processor Programming Reference (PPR) for AMD "
"Family 19h Model 61h, Revision B1 Processors\" (AMD publication 56713), "
"\"Processor Programming Reference (PPR) for AMD Family 19h Model 70h, "
"Revision A0 Processors\" (AMD publication 57019), and "
"amd_f19h_zen4_events(3CPC)";
static char amd_fam_1ah_zen5_reg[] = "See \"Performance Monitor Counters "
"for AMD Family 1Ah Model 00h-Fh Processors\" (AMD publication 58550) and "
"amd_f1ah_zen5_events(3CPC)";

static char amd_pcbe_impl_name[64];
static char *amd_pcbe_cpuref;


#define	BITS(v, u, l)   \
	(((v) >> (l)) & ((1 << (1 + (u) - (l))) - 1))

static uint64_t
opt_pcbe_pes_addr(uint_t counter)
{
	ASSERT3U(counter, <, opd.opd_ncounters);
	return (PES_BASE_ADDR + counter);
}

static uint64_t
opt_pcbe_pes_ext_addr(uint_t counter)
{
	ASSERT3U(counter, <, opd.opd_ncounters);
	return (PES_EXT_BASE_ADDR + 2 * counter);
}

static uint64_t
opt_pcbe_pic_addr(uint_t counter)
{
	ASSERT3U(counter, <, opd.opd_ncounters);
	return (PIC_BASE_ADDR + counter);
}

static uint64_t
opt_pcbe_pic_ext_addr(uint_t counter)
{
	ASSERT3U(counter, <, opd.opd_ncounters);
	return (PIC_EXT_BASE_ADDR + 2 * counter);
}

static int
opt_pcbe_init(void)
{
	const amd_event_t		*evp;
	const amd_generic_event_t	*gevp;
	x86_uarchrev_t			uarchrev;

	amd_family = cpuid_getfamily(CPU);
	amd_model = cpuid_getmodel(CPU);
	uarchrev = cpuid_getuarchrev(CPU);

	/*
	 * Make sure this really _is_ an Opteron or Athlon 64 system. The kernel
	 * loads this module based on its name in the module directory, but it
	 * could have been renamed.
	 */
	if ((cpuid_getvendor(CPU) != X86_VENDOR_AMD || amd_family < 0xf) &&
	    cpuid_getvendor(CPU) != X86_VENDOR_HYGON)
		return (-1);

	if (amd_family == 0xf) {
		/* Some tools expect this string for family 0fh */
		(void) snprintf(amd_pcbe_impl_name, sizeof (amd_pcbe_impl_name),
		    "AMD Opteron & Athlon64");
	} else {
		(void) snprintf(amd_pcbe_impl_name, sizeof (amd_pcbe_impl_name),
		    "%s Family %02xh",
		    cpuid_getvendor(CPU) == X86_VENDOR_HYGON ? "Hygon" : "AMD",
		    amd_family);
	}

	/*
	 * Determine whether or not the extended counter set is supported on
	 * this processor.
	 *
	 * If access to counters beyond the 6 defined for OPT_PCBE_EXT_NCOUNTERS
	 * are added here, the logic in HMA for saving/restoring host CPC state
	 * will also need to be updated.  See: os/hma.c
	 */
	if (is_x86_feature(x86_featureset, X86FSET_AMD_PCEC)) {
		opd.opd_ncounters = OPT_PCBE_EXT_NCOUNTERS;
		opd.opd_pesf = opt_pcbe_pes_ext_addr;
		opd.opd_picf = opt_pcbe_pic_ext_addr;
	} else {
		opd.opd_ncounters = OPT_PCBE_DEF_NCOUNTERS;
		opd.opd_pesf = opt_pcbe_pes_addr;
		opd.opd_picf = opt_pcbe_pic_addr;
	}
	opd.opd_cmask = (1 << opd.opd_ncounters) - 1;

	/*
	 * Figure out processor revision here and assign appropriate
	 * event configuration.
	 */
	switch (uarchrev_uarch(uarchrev)) {
	case X86_UARCH_AMD_LEGACY:
		switch (amd_family) {
		case 0xf: {
			x86_chiprev_t rev;

			rev = cpuid_getchiprev(CPU);

			if (chiprev_at_least(rev,
			    X86_CHIPREV_AMD_LEGACY_F_REV_F)) {
				amd_pcbe_cpuref = amd_fam_f_NPT_bkdg;
			} else {
				amd_pcbe_cpuref = amd_fam_f_rev_ae_bkdg;
			}
			amd_events = family_f_events;
			amd_generic_events = opt_generic_events;
			break;
		}
		case 0x10:
			amd_pcbe_cpuref = amd_fam_10h_bkdg;
			amd_events = family_10h_events;
			amd_generic_events = family_10h_generic_events;
			break;
		case 0x11:
			amd_pcbe_cpuref = amd_fam_11h_bkdg;
			amd_events = family_11h_events;
			amd_generic_events = opt_generic_events;
			break;
		default:
			return (-1);
		}
		break;
	case X86_UARCH_AMD_ZEN1:
	case X86_UARCH_AMD_ZENPLUS:
		amd_pcbe_cpuref = amd_fam_17h_zen1_reg;
		amd_events = opteron_pcbe_f17h_zen1_events;
		amd_generic_events = family_17h_zen1_papi_events;
		break;
	case X86_UARCH_AMD_ZEN2:
		amd_pcbe_cpuref = amd_fam_17h_zen2_reg;
		amd_events = opteron_pcbe_f17h_zen2_events;
		amd_generic_events = family_17h_zen2_papi_events;
		break;
	case X86_UARCH_AMD_ZEN3:
		amd_pcbe_cpuref = amd_fam_19h_zen3_reg;
		amd_events = opteron_pcbe_f19h_zen3_events;
		amd_generic_events = family_19h_zen3_papi_events;
		break;
	case X86_UARCH_AMD_ZEN4:
		amd_pcbe_cpuref = amd_fam_19h_zen4_reg;
		amd_events = opteron_pcbe_f19h_zen4_events;
		amd_generic_events = family_19h_zen4_papi_events;
		break;
	case X86_UARCH_AMD_ZEN5:
		amd_pcbe_cpuref = amd_fam_1ah_zen5_reg;
		amd_events = opteron_pcbe_f1ah_zen5_events;
		amd_generic_events = family_1ah_zen5_papi_events;
		break;
	default:
		/*
		 * Different families have different meanings on events and even
		 * worse (like family 15h), different constraints around
		 * programming these values.
		 */
		return (-1);
	}

	/*
	 * Construct event list.
	 *
	 * First pass:  Calculate size needed. We'll need an additional byte
	 *		for the NULL pointer during the last strcat.
	 *
	 * Second pass: Copy strings.
	 */
	for (evp = amd_events; evp->name != NULL; evp++)
		evlist_sz += strlen(evp->name) + 1;

	for (gevp = amd_generic_events; gevp->name != NULL; gevp++)
		evlist_sz += strlen(gevp->name) + 1;

	evlist = kmem_alloc(evlist_sz + 1, KM_SLEEP);
	evlist[0] = '\0';

	for (evp = amd_events; evp->name != NULL; evp++) {
		(void) strcat(evlist, evp->name);
		(void) strcat(evlist, ",");
	}

	for (gevp = amd_generic_events; gevp->name != NULL; gevp++) {
		(void) strcat(evlist, gevp->name);
		(void) strcat(evlist, ",");
	}

	/*
	 * Remove trailing comma.
	 */
	evlist[evlist_sz - 1] = '\0';

	return (0);
}

static uint_t
opt_pcbe_ncounters(void)
{
	return (opd.opd_ncounters);
}

static const char *
opt_pcbe_impl_name(void)
{
	return (amd_pcbe_impl_name);
}

static const char *
opt_pcbe_cpuref(void)
{

	return (amd_pcbe_cpuref);
}

/*ARGSUSED*/
static char *
opt_pcbe_list_events(uint_t picnum)
{
	return (evlist);
}

static char *
opt_pcbe_list_attrs(void)
{
	return ("edge,pc,inv,cmask,umask");
}

static const amd_generic_event_t *
find_generic_event(char *name)
{
	const amd_generic_event_t	*gevp;

	for (gevp = amd_generic_events; gevp->name != NULL; gevp++)
		if (strcmp(name, gevp->name) == 0)
			return (gevp);

	return (NULL);
}

static const amd_event_t *
find_event(char *name)
{
	const amd_event_t	*evp;

	for (evp = amd_events; evp->name != NULL; evp++)
		if (strcmp(name, evp->name) == 0)
			return (evp);

	return (NULL);
}

/*ARGSUSED*/
static uint64_t
opt_pcbe_event_coverage(char *event)
{
	/*
	 * Check whether counter event is supported
	 */
	if (find_event(event) == NULL && find_generic_event(event) == NULL)
		return (0);

	/*
	 * Fortunately, all counters can count all events.
	 */
	return (opd.opd_cmask);
}

static uint64_t
opt_pcbe_overflow_bitmap(void)
{
	/*
	 * Unfortunately, this chip cannot detect which counter overflowed, so
	 * we must act as if they all did.
	 */
	return (opd.opd_cmask);
}

/*ARGSUSED*/
static int
opt_pcbe_configure(uint_t picnum, char *event, uint64_t preset, uint32_t flags,
    uint_t nattrs, kcpc_attr_t *attrs, void **data, void *token)
{
	opt_pcbe_config_t		*cfg;
	const amd_event_t		*evp;
	amd_event_t			ev_raw = { "raw", 0};
	const amd_generic_event_t	*gevp;
	int				i;
	uint64_t			evsel = 0, evsel_tmp = 0;

	/*
	 * If we've been handed an existing configuration, we need only preset
	 * the counter value.
	 */
	if (*data != NULL) {
		cfg = *data;
		cfg->opt_rawpic = preset & MASK48;
		return (0);
	}

	if (picnum >= opd.opd_ncounters)
		return (CPC_INVALID_PICNUM);

	if ((evp = find_event(event)) == NULL) {
		if ((gevp = find_generic_event(event)) != NULL) {
			evp = find_event(gevp->event);
			ASSERT(evp != NULL);

			if (nattrs > 0)
				return (CPC_ATTRIBUTE_OUT_OF_RANGE);

			evsel |= gevp->umask << OPT_PES_UMASK_SHIFT;
		} else {
			long tmp;

			/*
			 * If ddi_strtol() likes this event, use it as a raw
			 * event code.
			 */
			if (ddi_strtol(event, NULL, 0, &tmp) != 0)
				return (CPC_INVALID_EVENT);

			ev_raw.emask = tmp;
			evp = &ev_raw;
		}
	}

	/*
	 * Configuration of EventSelect register. While on some families
	 * certain bits might not be supported (e.g. Guest/Host on family
	 * 11h), setting these bits is harmless
	 */

	/* Set GuestOnly bit to 0 and HostOnly bit to 1 */
	evsel &= ~OPT_PES_HOST;
	evsel &= ~OPT_PES_GUEST;

	/* Set bits [35:32] for extended part of Event Select field */
	evsel_tmp = evp->emask & 0x0f00;
	evsel |= evsel_tmp << OPT_PES_EVSELHI_SHIFT;

	evsel |= evp->emask & 0x00ff;
	evsel |= evp->unit << OPT_PES_UMASK_SHIFT;

	if (flags & CPC_COUNT_USER)
		evsel |= OPT_PES_USR;
	if (flags & CPC_COUNT_SYSTEM)
		evsel |= OPT_PES_OS;
	if (flags & CPC_OVF_NOTIFY_EMT)
		evsel |= OPT_PES_INT;

	for (i = 0; i < nattrs; i++) {
		if (strcmp(attrs[i].ka_name, "edge") == 0) {
			if (attrs[i].ka_val != 0)
				evsel |= OPT_PES_EDGE;
		} else if (strcmp(attrs[i].ka_name, "pc") == 0) {
			if (attrs[i].ka_val != 0)
				evsel |= OPT_PES_PC;
		} else if (strcmp(attrs[i].ka_name, "inv") == 0) {
			if (attrs[i].ka_val != 0)
				evsel |= OPT_PES_INV;
		} else if (strcmp(attrs[i].ka_name, "cmask") == 0) {
			if ((attrs[i].ka_val | OPT_PES_CMASK_MASK) !=
			    OPT_PES_CMASK_MASK)
				return (CPC_ATTRIBUTE_OUT_OF_RANGE);
			evsel |= attrs[i].ka_val << OPT_PES_CMASK_SHIFT;
		} else if (strcmp(attrs[i].ka_name, "umask") == 0) {
			if ((attrs[i].ka_val | OPT_PES_UMASK_MASK) !=
			    OPT_PES_UMASK_MASK)
				return (CPC_ATTRIBUTE_OUT_OF_RANGE);
			evsel |= attrs[i].ka_val << OPT_PES_UMASK_SHIFT;
		} else
			return (CPC_INVALID_ATTRIBUTE);
	}

	cfg = kmem_alloc(sizeof (*cfg), KM_SLEEP);

	cfg->opt_picno = picnum;
	cfg->opt_evsel = evsel;
	cfg->opt_rawpic = preset & MASK48;

	*data = cfg;
	return (0);
}

static void
opt_pcbe_program(void *token)
{
	opt_pcbe_config_t	*cfgs[OPT_PCBE_EXT_NCOUNTERS] = { &nullcfgs[0],
						&nullcfgs[1], &nullcfgs[2],
						&nullcfgs[3], &nullcfgs[4],
						&nullcfgs[5] };
	opt_pcbe_config_t	*pcfg = NULL;
	int			i;
	ulong_t			curcr4 = getcr4();

	/*
	 * Allow nonprivileged code to read the performance counters if desired.
	 */
	if (kcpc_allow_nonpriv(token))
		setcr4(curcr4 | CR4_PCE);
	else
		setcr4(curcr4 & ~CR4_PCE);

	/*
	 * Query kernel for all configs which will be co-programmed.
	 */
	do {
		pcfg = (opt_pcbe_config_t *)kcpc_next_config(token, pcfg, NULL);

		if (pcfg != NULL) {
			ASSERT(pcfg->opt_picno < opd.opd_ncounters);
			cfgs[pcfg->opt_picno] = pcfg;
		}
	} while (pcfg != NULL);

	/*
	 * Program in two loops. The first configures and presets the counter,
	 * and the second loop enables the counters. This ensures that the
	 * counters are all enabled as closely together in time as possible.
	 */

	for (i = 0; i < opd.opd_ncounters; i++) {
		wrmsr(opd.opd_pesf(i), cfgs[i]->opt_evsel);
		wrmsr(opd.opd_picf(i), cfgs[i]->opt_rawpic);
	}

	for (i = 0; i < opd.opd_ncounters; i++) {
		wrmsr(opd.opd_pesf(i), cfgs[i]->opt_evsel |
		    (uint64_t)(uintptr_t)OPT_PES_ENABLE);
	}
}

static void
opt_pcbe_allstop(void)
{
	int		i;

	for (i = 0; i < opd.opd_ncounters; i++)
		wrmsr(opd.opd_pesf(i), 0ULL);

	/*
	 * Disable non-privileged access to the counter registers.
	 */
	setcr4(getcr4() & ~CR4_PCE);
}

static void
opt_pcbe_sample(void *token)
{
	opt_pcbe_config_t	*cfgs[OPT_PCBE_EXT_NCOUNTERS] = { NULL, NULL,
						NULL, NULL, NULL, NULL };
	opt_pcbe_config_t	*pcfg = NULL;
	int			i;
	uint64_t		curpic[OPT_PCBE_EXT_NCOUNTERS];
	uint64_t		*addrs[OPT_PCBE_EXT_NCOUNTERS];
	uint64_t		*tmp;
	int64_t			diff;

	for (i = 0; i < opd.opd_ncounters; i++)
		curpic[i] = rdmsr(opd.opd_picf(i));

	/*
	 * Query kernel for all configs which are co-programmed.
	 */
	do {
		pcfg = (opt_pcbe_config_t *)kcpc_next_config(token, pcfg, &tmp);

		if (pcfg != NULL) {
			ASSERT3U(pcfg->opt_picno, <, opd.opd_ncounters);
			cfgs[pcfg->opt_picno] = pcfg;
			addrs[pcfg->opt_picno] = tmp;
		}
	} while (pcfg != NULL);

	for (i = 0; i < opd.opd_ncounters; i++) {
		if (cfgs[i] == NULL)
			continue;

		diff = (curpic[i] - cfgs[i]->opt_rawpic) & MASK48;
		*addrs[i] += diff;
		DTRACE_PROBE4(opt__pcbe__sample, int, i, uint64_t, *addrs[i],
		    uint64_t, curpic[i], uint64_t, cfgs[i]->opt_rawpic);
		cfgs[i]->opt_rawpic = *addrs[i] & MASK48;
	}
}

static void
opt_pcbe_free(void *config)
{
	kmem_free(config, sizeof (opt_pcbe_config_t));
}


static struct modlpcbe modlpcbe = {
	&mod_pcbeops,
	"AMD Performance Counters",
	&opt_pcbe_ops
};

static struct modlinkage modl = {
	MODREV_1,
	&modlpcbe,
};

int
_init(void)
{
	int ret;

	if (opt_pcbe_init() != 0)
		return (ENOTSUP);

	if ((ret = mod_install(&modl)) != 0)
		kmem_free(evlist, evlist_sz + 1);

	return (ret);
}

int
_fini(void)
{
	int ret;

	if ((ret = mod_remove(&modl)) == 0)
		kmem_free(evlist, evlist_sz + 1);
	return (ret);
}

int
_info(struct modinfo *mi)
{
	return (mod_info(&modl, mi));
}
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

/*
 * This file contains preset event names from the Performance Application
 * Programming Interface v3.5 which included the following notice:
 *
 *                             Copyright (c) 2005,6
 *                           Innovative Computing Labs
 *                         Computer Science Department,
 *                            University of Tennessee,
 *                                 Knoxville, TN.
 *                              All Rights Reserved.
 *
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 *    * Redistributions of source code must retain the above copyright notice,
 *      this list of conditions and the following disclaimer.
 *    * Redistributions in binary form must reproduce the above copyright
 *	notice, this list of conditions and the following disclaimer in the
 *	documentation and/or other materials provided with the distribution.
 *    * Neither the name of the University of Tennessee nor the names of its
 *      contributors may be used to endorse or promote products derived from
 *	this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 *
 *
 * This open source software license conforms to the BSD License template.
 */

/*
 * Portions Copyright 2009 Advanced Micro Devices, Inc.
 * Copyright 2019 Joyent, Inc.
 */

/*
 * Structure definition for AMD PCBE events.
 */

#ifndef _OPTERON_PCBE_TABLE_H
#define	_OPTERON_PCBE_TABLE_H

/*
 * Structure definition for PCBE events.
 */

#ifdef __cplusplus
extern "C" {
#endif

#include <sys/types.h>


typedef struct _amd_event {
	char		*name;
	uint16_t	emask;		/* Event mask setting */
	uint8_t		unit;
} amd_event_t;

typedef struct _amd_generic_event {
	char *name;
	char *event;
	uint8_t umask;
} amd_generic_event_t;

#ifdef __cplusplus
}
#endif

#endif /* _OPTERON_PCBE_TABLE_H */
/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
 */

/*
 * This file contains preset event names from the Performance Application
 * Programming Interface v3.5 which included the following notice:
 *
 *                             Copyright (c) 2005,6
 *                           Innovative Computing Labs
 *                         Computer Science Department,
 *                            University of Tennessee,
 *                                 Knoxville, TN.
 *                              All Rights Reserved.
 *
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 *    * Redistributions of source code must retain the above copyright notice,
 *      this list of conditions and the following disclaimer.
 *    * Redistributions in binary form must reproduce the above copyright
 *      notice, this list of conditions and the following disclaimer in the
 *      documentation and/or other materials provided with the distribution.
 *    * Neither the name of the University of Tennessee nor the names of its
 *      contributors may be used to endorse or promote products derived from
 *      this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 *
 *
 * This open source software license conforms to the BSD License template.
 */

/*
 * Performance Counter Back-End for Pentium 4.
 */

#include <sys/cpuvar.h>
#include <sys/param.h>
#include <sys/cpc_impl.h>
#include <sys/cpc_pcbe.h>
#include <sys/inttypes.h>
#include <sys/errno.h>
#include <sys/systm.h>
#include <sys/archsystm.h>
#include <sys/x86_archext.h>
#include <sys/modctl.h>
#include <sys/sdt.h>
#include <sys/cred.h>
#include <sys/policy.h>
#include <sys/privregs.h>

static int p4_pcbe_init(void);
static uint_t p4_pcbe_ncounters(void);
static const char *p4_pcbe_impl_name(void);
static const char *p4_pcbe_cpuref(void);
static char *p4_pcbe_list_events(uint_t picnum);
static char *p4_pcbe_list_attrs(void);
static uint64_t p4_pcbe_event_coverage(char *event);
static uint64_t p4_pcbe_overflow_bitmap(void);
static int p4_pcbe_configure(uint_t picnum, char *event, uint64_t preset,
    uint32_t flags, uint_t nattrs, kcpc_attr_t *attrs, void **data,
    void *token);
static void p4_pcbe_program(void *token);
static void p4_pcbe_allstop(void);
static void p4_pcbe_sample(void *token);
static void p4_pcbe_free(void *config);

extern int cpuid_get_clogid(cpu_t *);

static pcbe_ops_t p4_pcbe_ops = {
	PCBE_VER_1,
	CPC_CAP_OVERFLOW_INTERRUPT | CPC_CAP_OVERFLOW_PRECISE,
	p4_pcbe_ncounters,
	p4_pcbe_impl_name,
	p4_pcbe_cpuref,
	p4_pcbe_list_events,
	p4_pcbe_list_attrs,
	p4_pcbe_event_coverage,
	p4_pcbe_overflow_bitmap,
	p4_pcbe_configure,
	p4_pcbe_program,
	p4_pcbe_allstop,
	p4_pcbe_sample,
	p4_pcbe_free
};

/*
 * P4 Configuration Flags.
 */
#define	P4_THIS_USR	0x1 /* HTT: Measure usr events on this logical CPU */
#define	P4_THIS_SYS	0x2 /* HTT: Measure os events on this logical CPU */
#define	P4_SIBLING_USR	0x4 /* HTT: Measure os events on other logical CPU */
#define	P4_SIBLING_SYS	0x8 /* HTT: Measure usr events on other logical CPU */
#define	P4_PMI		0x10 /* HTT: Set PMI bit for local logical CPU */

typedef struct _p4_pcbe_config {
	uint8_t		p4_flags;
	uint8_t		p4_picno;	/* From 0 to 18 */
	uint8_t		p4_escr_ndx;	/* Which ESCR to use */
	uint32_t	p4_escr;	/* Value to program in selected ESCR */
	uint32_t	p4_cccr;	/* Value to program in counter's CCCR */
	uint64_t	p4_rawpic;
} p4_pcbe_config_t;

typedef uint32_t cntr_map_t;

typedef struct _p4_escr {
	int		pe_num;
	uint32_t	pe_addr;
	uint32_t	pe_map; /* bitmap of counters; bit 1 means ctr 0 */
} p4_escr_t;

#define	MASK40			UINT64_C(0xffffffffff)

/*
 * CCCR field definitions.
 *
 * Note that the Intel Developer's Manual states that the reserved field at
 * bit location 16 and 17 must be set to 11. (??)
 */
#define	CCCR_ENABLE_SHIFT	12
#define	CCCR_ESCR_SEL_SHIFT	13
#define	CCCR_ACTV_THR_SHIFT	16
#define	CCCR_COMPARE_SHIFT	18
#define	CCCR_COMPLEMENT_SHIFT	19
#define	CCCR_THRESHOLD_SHIFT	20
#define	CCCR_EDGE_SHIFT		24
#define	CCCR_OVF_PMI_SHIFT	26
#define	CCCR_OVF_PMI_T0_SHIFT	26
#define	CCCR_OVF_PMI_T1_SHIFT	27
#define	CCCR_OVF_SHIFT		31
#define	CCCR_ACTV_THR_MASK	0x3
#define	CCCR_THRESHOLD_MAX	0xF
#define	CCCR_ENABLE		(1U << CCCR_ENABLE_SHIFT)
#define	CCCR_COMPARE		(1U << CCCR_COMPARE_SHIFT)
#define	CCCR_COMPLEMENT		(1U << CCCR_COMPLEMENT_SHIFT)
#define	CCCR_EDGE		(1U << CCCR_EDGE_SHIFT)
#define	CCCR_OVF_PMI		(1U << CCCR_OVF_PMI_SHIFT)
#define	CCCR_OVF_PMI_T0		(1U << CCCR_OVF_PMI_T0_SHIFT)
#define	CCCR_OVF_PMI_T1		(1U << CCCR_OVF_PMI_T1_SHIFT)
#define	CCCR_INIT		CCCR_ENABLE
#define	CCCR_OVF		(1U << CCCR_OVF_SHIFT)

#define	ESCR_EVSEL_SHIFT	25
#define	ESCR_EVMASK_SHIFT	9
#define	ESCR_TAG_VALUE_SHIFT	5
#define	ESCR_TAG_VALUE_MAX	0xF
#define	ESCR_TAG_ENABLE_SHIFT	4
#define	ESCR_USR_SHIFT		2
#define	ESCR_OS_SHIFT		3
#define	ESCR_USR		(1U << ESCR_USR_SHIFT)
#define	ESCR_OS			(1U << ESCR_OS_SHIFT)
#define	ESCR_TAG_ENABLE		(1U << ESCR_TAG_ENABLE_SHIFT)

/*
 * HyperThreaded ESCR fields.
 */
#define	ESCR_T0_OS_SHIFT	3
#define	ESCR_T0_USR_SHIFT	2
#define	ESCR_T1_OS_SHIFT	1
#define	ESCR_T1_USR_SHIFT	0
#define	ESCR_T0_OS		(1U << ESCR_T0_OS_SHIFT)
#define	ESCR_T0_USR		(1U << ESCR_T0_USR_SHIFT)
#define	ESCR_T1_OS		(1U << ESCR_T1_OS_SHIFT)
#define	ESCR_T1_USR		(1U << ESCR_T1_USR_SHIFT)

/*
 * ESCRs are grouped by counter; each group of ESCRs is associated with a
 * distinct group of counters. Use these macros to fill in the table below.
 */
#define	BPU0_map	(0x1 | 0x2)		/* Counters 0 and 1 */
#define	BPU2_map	(0x4 | 0x8)		/* Counters 2 and 3 */
#define	MS0_map		(0x10 | 0x20)		/* Counters 4 and 5 */
#define	MS2_map		(0x40 | 0x80)		/* Counters 6 and 7 */
#define	FLAME0_map	(0x100 | 0x200)		/* Counters 8 and 9 */
#define	FLAME2_map	(0x400 | 0x800)		/* Counters 10 and 11 */
#define	IQ0_map		(0x1000 | 0x2000 | 0x10000) /* Counters 12, 13, 16 */
#define	IQ2_map		(0x4000 | 0x8000 | 0x20000) /* Counters 14, 15, 17 */

/*
 * Table describing the 45 Event Selection and Control Registers (ESCRs).
 */
const p4_escr_t p4_escrs[] = {
#define	BPU0 (1)
	{ 0, 0x3B2, BPU0_map },		/* 0 */
#define	IS0 (1ULL << 1)
	{ 1, 0x3B4, BPU0_map },		/* 1 */
#define	MOB0 (1ULL << 2)
	{ 2, 0x3AA, BPU0_map },		/* 2 */
#define	ITLB0 (1ULL << 3)
	{ 3, 0x3B6, BPU0_map },		/* 3 */
#define	PMH0 (1ULL << 4)
	{ 4, 0x3AC, BPU0_map },		/* 4 */
#define	IX0 (1ULL << 5)
	{ 5, 0x3C8, BPU0_map },		/* 5 */
#define	FSB0 (1ULL << 6)
	{ 6, 0x3A2, BPU0_map },		/* 6 */
#define	BSU0 (1ULL << 7)
	{ 7, 0x3A0, BPU0_map },		/* 7 */
#define	BPU1 (1ULL << 8)
	{ 0, 0x3B3, BPU2_map },		/* 8 */
#define	IS1 (1ULL << 9)
	{ 1, 0x3B5, BPU2_map },		/* 9 */
#define	MOB1 (1ULL << 10)
	{ 2, 0x3AB, BPU2_map },		/* 10 */
#define	ITLB1 (1ULL << 11)
	{ 3, 0x3B7, BPU2_map },		/* 11 */
#define	PMH1 (1ULL << 12)
	{ 4, 0x3AD, BPU2_map },		/* 12 */
#define	IX1 (1ULL << 13)
	{ 5, 0x3C9, BPU2_map },		/* 13 */
#define	FSB1 (1ULL << 14)
	{ 6, 0x3A3, BPU2_map },		/* 14 */
#define	BSU1 (1ULL << 15)
	{ 7, 0x3A1, BPU2_map },		/* 15 */
#define	MS0 (1ULL << 16)
	{ 0, 0x3C0, MS0_map },		/* 16 */
#define	TC0 (1ULL << 17)
	{ 1, 0x3C4, MS0_map },		/* 17 */
#define	TBPU0 (1ULL << 18)
	{ 2, 0x3C2, MS0_map },		/* 18 */
#define	MS1 (1ULL << 19)
	{ 0, 0x3C1, MS2_map },		/* 19 */
#define	TC1 (1ULL << 20)
	{ 1, 0x3C5, MS2_map },		/* 20 */
#define	TBPU1 (1ULL << 21)
	{ 2, 0x3C3, MS2_map },		/* 21 */
#define	FLAME0 (1ULL << 22)
	{ 0, 0x3A6, FLAME0_map },	/* 22 */
#define	FIRM0 (1ULL << 23)
	{ 1, 0x3A4, FLAME0_map },	/* 23 */
#define	SAAT0 (1ULL << 24)
	{ 2, 0x3AE, FLAME0_map },	/* 24 */
#define	U2L0 (1ULL << 25)
	{ 3, 0x3B0, FLAME0_map },	/* 25 */
#define	DAC0 (1ULL << 26)
	{ 5, 0x3A8, FLAME0_map },	/* 26 */
#define	FLAME1 (1ULL << 27)
	{ 0, 0x3A7, FLAME2_map },	/* 27 */
#define	FIRM1 (1ULL << 28)
	{ 1, 0x3A5, FLAME2_map },	/* 28 */
#define	SAAT1 (1ULL << 29)
	{ 2, 0x3AF, FLAME2_map },	/* 29 */
#define	U2L1 (1ULL << 30)
	{ 3, 0x3B1, FLAME2_map },	/* 30 */
#define	DAC1 (1ULL << 31)
	{ 5, 0x3A9, FLAME2_map },	/* 31 */
#define	IQ0 (1ULL << 32)
	{ 0, 0x3BA, IQ0_map },		/* 32 */
#define	ALF0 (1ULL << 33)
	{ 1, 0x3CA, IQ0_map },		/* 33 */
#define	RAT0 (1ULL << 34)
	{ 2, 0x3BC, IQ0_map },		/* 34 */
#define	SSU0 (1ULL << 35)
	{ 3, 0x3BE, IQ0_map },		/* 35 */
#define	CRU0 (1ULL << 36)
	{ 4, 0x3B8, IQ0_map },		/* 36 */
#define	CRU2 (1ULL << 37)
	{ 5, 0x3CC, IQ0_map },		/* 37 */
#define	CRU4 (1ULL << 38)
	{ 6, 0x3E0, IQ0_map },		/* 38 */
#define	IQ1 (1ULL << 39)
	{ 0, 0x3BB, IQ2_map },		/* 39 */
#define	ALF1 (1ULL << 40)
	{ 1, 0x3CB, IQ2_map },		/* 40 */
#define	RAT1 (1ULL << 41)
	{ 2, 0x3BD, IQ2_map },		/* 41 */
#define	CRU1 (1ULL << 42)
	{ 4, 0x3B9, IQ2_map },		/* 42 */
#define	CRU3 (1ULL << 43)
	{ 5, 0x3CD, IQ2_map },		/* 43 */
#define	CRU5 (1ULL << 44)
	{ 6, 0x3E1, IQ2_map }		/* 44 */
};

#define	ESCR_MAX_INDEX 44

typedef struct _p4_ctr {
	uint32_t	pc_caddr;	/* counter MSR address */
	uint32_t	pc_ctladdr;	/* counter's CCCR MSR address */
	uint64_t	pc_map;		/* bitmap of ESCRs controlling ctr */
} p4_ctr_t;

const p4_ctr_t p4_ctrs[18] = {
{ /* BPU_COUNTER0 */ 0x300, 0x360, BSU0|FSB0|MOB0|PMH0|BPU0|IS0|ITLB0|IX0},
{ /* BPU_COUNTER1 */ 0x301, 0x361, BSU0|FSB0|MOB0|PMH0|BPU0|IS0|ITLB0|IX0},
{ /* BPU_COUNTER2 */ 0x302, 0x362, BSU1|FSB1|MOB1|PMH1|BPU1|IS1|ITLB1|IX1},
{ /* BPU_COUNTER3 */ 0x303, 0x363, BSU1|FSB1|MOB1|PMH1|BPU1|IS1|ITLB1|IX1},
{ /* MS_COUNTER0 */  0x304, 0x364, MS0|TBPU0|TC0 },
{ /* MS_COUNTER1 */  0x305, 0x365, MS0|TBPU0|TC0 },
{ /* MS_COUNTER2 */  0x306, 0x366, MS1|TBPU1|TC1 },
{ /* MS_COUNTER3 */  0x307, 0x367, MS1|TBPU1|TC1 },
{ /* FLAME_COUNTER0 */ 0x308, 0x368, FIRM0|FLAME0|DAC0|SAAT0|U2L0 },
{ /* FLAME_COUNTER1 */ 0x309, 0x369, FIRM0|FLAME0|DAC0|SAAT0|U2L0 },
{ /* FLAME_COUNTER2 */ 0x30A, 0x36A, FIRM1|FLAME1|DAC1|SAAT1|U2L1 },
{ /* FLAME_COUNTER3 */ 0x30B, 0x36B, FIRM1|FLAME1|DAC1|SAAT1|U2L1 },
{ /* IQ_COUNTER0 */  0x30C, 0x36C, CRU0|CRU2|CRU4|IQ0|RAT0|SSU0|ALF0 },
{ /* IQ_COUNTER1 */  0x30D, 0x36D, CRU0|CRU2|CRU4|IQ0|RAT0|SSU0|ALF0 },
{ /* IQ_COUNTER2 */  0x30E, 0x36E, CRU1|CRU3|CRU5|IQ1|RAT1|ALF1 },
{ /* IQ_COUNTER3 */  0x30F, 0x36F, CRU1|CRU3|CRU5|IQ1|RAT1|ALF1 },
{ /* IQ_COUNTER4 */  0x310, 0x370, CRU0|CRU2|CRU4|IQ0|RAT0|SSU0|ALF0 },
{ /* IQ_COUNTER5 */  0x311, 0x371, CRU1|CRU3|CRU5|IQ1|RAT1|ALF1 }
};

typedef struct _p4_event {
	char		*pe_name;	/* Name of event according to docs */
	uint64_t	pe_escr_map;	/* Bitmap of ESCRs capable of event */
	uint32_t	pe_escr_mask;	/* permissible ESCR event mask */
	uint8_t		pe_ev;		/* ESCR event select value */
	uint16_t	pe_cccr;	/* CCCR select value */
	uint32_t	pe_ctr_mask;	/* Bitmap of capable counters */
} p4_event_t;

typedef struct _p4_generic_event {
	char		*name;
	char		*event;
	uint16_t	emask;
	uint32_t	ctr_mask;
} p4_generic_event_t;

#define	C(n) (1 << n)
#define	GEN_EVT_END { NULL, NULL, 0x0, 0x0 }

p4_event_t p4_events[] = {
{ "branch_retired", CRU2|CRU3, 0xF, 0x6, 0x5, C(12)|C(13)|C(14)|C(15)|C(16) },
{ "mispred_branch_retired", CRU0|CRU1, 0x1, 0x3, 0x4,
	C(12)|C(13)|C(14)|C(15)|C(16) },
{ "TC_deliver_mode", TC0|TC1, 0xFF, 0x1, 0x1, C(4)|C(5)|C(6)|C(7) },
{ "BPU_fetch_request", BPU0|BPU1, 0x1, 0x3, 0x0, C(0)|C(1)|C(2)|C(3) },
{ "ITLB_reference", ITLB0|ITLB1, 0x7, 0x18, 0x3, C(0)|C(1)|C(2)|C(3) },
{ "memory_cancel", DAC0|DAC1, 0x6, 0x2, 0x5, C(8)|C(9)|C(10)|C(11) },
{ "memory_complete", SAAT0|SAAT1, 0x3, 0x8, 0x2, C(8)|C(9)|C(10)|C(11) },
{ "load_port_replay", SAAT0|SAAT1, 0x1, 0x4, 0x2, C(8)|C(9)|C(10)|C(11) },
{ "store_port_replay", SAAT0|SAAT1, 0x1, 0x5, 0x2, C(8)|C(9)|C(10)|C(11) },
{ "MOB_load_replay", MOB0|MOB1, 0x35, 0x3, 0x2, C(0)|C(1)|C(2)|C(3) },
{ "page_walk_type", PMH0|PMH1, 0x3, 0x1, 0x4, C(0)|C(1)|C(2)|C(3) },
{ "BSQ_cache_reference", BSU0|BSU1, 0x73F, 0xC, 0x7, C(0)|C(1)|C(2)|C(3) },
{ "IOQ_allocation", FSB0, 0xEFFF, 0x3, 0x6, C(0)|C(1) },
{ "IOQ_active_entries", FSB1, 0xEFFF, 0x1A, 0x6, C(2)|C(3) },
{ "FSB_data_activity", FSB0|FSB1, 0x3F, 0x17, 0x6, C(0)|C(1)|C(2)|C(3) },
{ "BSQ_allocation", BSU0, 0x3FEF, 0x5, 0x7, C(0)|C(1) },
{ "bsq_active_entries", BSU1, 0x3FEF, 0x6, 0x7, C(2)|C(3) },
{ "x87_assist", CRU2|CRU3, 0x1F, 0x3, 0x5, C(12)|C(13)|C(14)|C(15)|C(16)|C(17)},
{ "SSE_input_assist", FIRM0|FIRM1, 0x8000, 0x34, 0x1, C(8)|C(9)|C(10)|C(11) },
{ "packed_SP_uop", FIRM0|FIRM1, 0x8000, 0x8, 0x1, C(8)|C(9)|C(10)|C(11) },
{ "packed_DP_uop", FIRM0|FIRM1, 0x8000, 0xC, 0x1, C(8)|C(9)|C(10)|C(11) },
{ "scalar_SP_uop", FIRM0|FIRM1, 0x8000, 0xA, 0x1, C(8)|C(9)|C(10)|C(11) },
{ "scalar_DP_uop", FIRM0|FIRM1, 0x8000, 0xE, 0x1, C(8)|C(9)|C(10)|C(11) },
{ "64bit_MMX_uop", FIRM0|FIRM1, 0x8000, 0x2, 0x1, C(8)|C(9)|C(10)|C(11) },
{ "128bit_MMX_uop", FIRM0|FIRM1, 0x8000, 0x1A, 0x1, C(8)|C(9)|C(10)|C(11) },
{ "x87_FP_uop", FIRM0|FIRM1, 0x8000, 0x4, 0x1, C(8)|C(9)|C(10)|C(11) },
{ "x87_SIMD_moves_uop", FIRM0|FIRM1, 0x18, 0x2E, 0x1, C(8)|C(9)|C(10)|C(11) },
{ "machine_clear", CRU2|CRU3, 0xD, 0x2, 0x5,
	C(12)|C(13)|C(14)|C(15)|C(16)|C(17)},
{ "global_power_events", FSB0|FSB1, 0x1, 0x13, 0x6, C(0)|C(1)|C(2)|C(3) },
{ "tc_ms_xfer", MS0|MS1, 0x1, 0x5, 0x0, C(4)|C(5)|C(6)|C(7) },
{ "uop_queue_writes", MS0|MS1, 0x7, 0x9, 0x0, C(4)|C(5)|C(6)|C(7) },
{ "front_end_event", CRU2|CRU3, 0x3, 0x8, 0x5,
	C(12)|C(13)|C(14)|C(15)|C(16)|C(17)},
{ "execution_event", CRU2|CRU3, 0xFF, 0xC, 0x5,
	C(12)|C(13)|C(14)|C(15)|C(16)|C(17)},
{ "replay_event", CRU2|CRU3, 0x3, 0x9, 0x5,
	C(12)|C(13)|C(14)|C(15)|C(16)|C(17)},
{ "instr_retired", CRU0|CRU1, 0xF, 0x2, 0x4,
	C(12)|C(13)|C(14)|C(15)|C(16)|C(17)},
{ "uops_retired", CRU0|CRU1, 0x3, 0x1, 0x4,
	C(12)|C(13)|C(14)|C(15)|C(16)|C(17)},
{ "uop_type", RAT0|RAT1, 0x3, 0x2, 0x2, C(12)|C(13)|C(14)|C(15)|C(16)|C(17)},
{ "retired_mispred_branch_type", TBPU0|TBPU1, 0x1F, 0x5, 0x2,
	C(4)|C(5)|C(6)|C(7)},
{ "retired_branch_type", TBPU0|TBPU1, 0x1F, 0x4, 0x2, C(4)|C(5)|C(6)|C(7) },
{ NULL, 0, 0, 0, 0 }
};

static p4_generic_event_t p4_generic_events[] = {
{ "PAPI_br_msp", "branch_retired", 0xa, C(12)|C(13)|C(14)|C(15)|C(16) },
{ "PAPI_br_ins", "branch_retired", 0xf, C(12)|C(13)|C(14)|C(15)|C(16) },
{ "PAPI_br_tkn", "branch_retired", 0xc, C(12)|C(13)|C(14)|C(15)|C(16) },
{ "PAPI_br_ntk", "branch_retired", 0x3, C(12)|C(13)|C(14)|C(15)|C(16) },
{ "PAPI_br_prc", "branch_retired", 0x5, C(12)|C(13)|C(14)|C(15)|C(16) },
{ "PAPI_tot_ins", "instr_retired", 0x3, C(12)|C(13)|C(14)|C(15)|C(16)|C(17) },
{ "PAPI_tot_cyc", "global_power_events", 0x1, C(0)|C(1)|C(2)|C(3) },
{ "PAPI_tlb_dm", "page_walk_type", 0x1, C(0)|C(1)|C(2)|C(3) },
{ "PAPI_tlb_im", "page_walk_type", 0x2, C(0)|C(1)|C(2)|C(3) },
{ "PAPI_tlb_tm", "page_walk_type", 0x3, C(0)|C(1)|C(2)|C(3) },
{ "PAPI_l1_icm", "BPU_fetch_request", 0x1, C(0)|C(1)|C(2)|C(3) },
{ "PAPI_l2_ldm", "BSQ_cache_reference", 0x100, C(0)|C(1)|C(2)|C(3) },
{ "PAPI_l2_stm", "BSQ_cache_reference", 0x400, C(0)|C(1)|C(2)|C(3) },
{ "PAPI_l2_tcm", "BSQ_cache_reference", 0x500, C(0)|C(1)|C(2)|C(3) },
GEN_EVT_END
};

/*
 * Indicates whether the "rdpmc" instruction is available on this processor.
 */
static int p4_rdpmc_avail = 0;
static char *p4_eventlist[18];

/*
 * If set, this processor has HyperThreading.
 */
static int p4_htt = 0;

#define	P4_FAMILY	0xF

static int
p4_pcbe_init(void)
{
	int			i;
	size_t			size;
	p4_event_t		*ev;
	p4_generic_event_t	*gevp;

	/*
	 * If we're not running on a P4, refuse to load.
	 */
	if (cpuid_getvendor(CPU) != X86_VENDOR_Intel ||
	    cpuid_getfamily(CPU) != P4_FAMILY)
		return (-1);

	/*
	 * Set up the event lists for each counter.
	 *
	 * First pass calculates the size of the event list, and the second
	 * pass copies each event name into the event list.
	 */
	for (i = 0; i < 18; i++) {
		size = 0;

		for (ev = p4_events; ev->pe_name != NULL; ev++) {
			if (ev->pe_ctr_mask & C(i))
				size += strlen(ev->pe_name) + 1;
		}

		for (gevp = p4_generic_events; gevp->name != NULL; gevp++) {
			if (gevp->ctr_mask & C(i))
				size += strlen(gevp->name) + 1;
		}

		/*
		 * We use 'size + 1' here to ensure room for the final
		 * strcat when it terminates the string.
		 */
		p4_eventlist[i] = (char *)kmem_alloc(size + 1, KM_SLEEP);
		*p4_eventlist[i] = '\0';

		for (ev = p4_events; ev->pe_name != NULL; ev++) {
			if (ev->pe_ctr_mask & C(i)) {
				(void) strcat(p4_eventlist[i], ev->pe_name);
				(void) strcat(p4_eventlist[i], ",");
			}
		}

		for (gevp = p4_generic_events; gevp->name != NULL; gevp++) {
			if (gevp->ctr_mask & C(i)) {
				(void) strcat(p4_eventlist[i], gevp->name);
				(void) strcat(p4_eventlist[i], ",");
			}
		}

		/*
		 * Remove trailing ','
		 */
		p4_eventlist[i][size - 1] = '\0';
	}

	if (is_x86_feature(x86_featureset, X86FSET_MMX))
		p4_rdpmc_avail = 1;
	/*
	 * The X86_HTT flag may disappear soon, so we'll isolate the impact of
	 * its demise to the following if().
	 */
	if (is_x86_feature(x86_featureset, X86FSET_HTT))
		p4_htt = 1;

	return (0);
}

static uint_t
p4_pcbe_ncounters(void)
{
	return (18);
}

static const char *
p4_pcbe_impl_name(void)
{
	if (p4_htt)
		return (PCBE_IMPL_NAME_P4HT);
	return ("Pentium 4");
}

static const char *
p4_pcbe_cpuref(void)
{
	return ("See Appendix A.1 of the \"IA-32 Intel Architecture Software " \
	    "Developer's Manual Volume 3: System Programming Guide,\" "	       \
	    "Order # 245472-012, 2003");
}

static char *
p4_pcbe_list_events(uint_t picnum)
{
	ASSERT(picnum >= 0 && picnum < 18);

	return (p4_eventlist[picnum]);
}

#define	P4_ATTRS "emask,tag,compare,complement,threshold,edge"

static char *
p4_pcbe_list_attrs(void)
{
	if (p4_htt)
		return (P4_ATTRS ",active_thread,count_sibling_usr,"
		    "count_sibling_sys");
	return (P4_ATTRS);
}

static p4_generic_event_t *
find_generic_event(char *name)
{
	p4_generic_event_t	*gevp;

	for (gevp = p4_generic_events; gevp->name != NULL; gevp++)
		if (strcmp(name, gevp->name) == 0)
			return (gevp);

	return (NULL);
}

static p4_event_t *
find_event(char *name)
{
	p4_event_t		*evp;

	for (evp = p4_events; evp->pe_name != NULL; evp++)
		if (strcmp(name, evp->pe_name) == 0)
			return (evp);

	return (NULL);
}

static uint64_t
p4_pcbe_event_coverage(char *event)
{
	p4_event_t		*ev;
	p4_generic_event_t	*gevp;

	if ((ev = find_event(event)) == NULL) {
		if ((gevp = find_generic_event(event)) != NULL)
			return (gevp->ctr_mask);
		else
			return (0);
	}

	return (ev->pe_ctr_mask);
}

static uint64_t
p4_pcbe_overflow_bitmap(void)
{
	extern int	kcpc_hw_overflow_intr_installed;
	uint64_t	ret = 0;
	int		i;

	/*
	 * The CCCR's OVF bit indicates that the corresponding counter has
	 * overflowed. It must be explicitly cleared by software, so it is
	 * safe to read the CCCR values here.
	 */
	for (i = 0; i < 18; i++) {
		if (rdmsr(p4_ctrs[i].pc_ctladdr) & CCCR_OVF)
			ret |= (1 << i);
	}

	/*
	 * Pentium 4 and Xeon turn off the CPC interrupt mask bit in the LVT at
	 * every overflow. Turn it back on here.
	 */
	ASSERT(kcpc_hw_overflow_intr_installed);
	(*kcpc_hw_enable_cpc_intr)();

	return (ret);
}

static int
p4_escr_inuse(p4_pcbe_config_t **cfgs, int escr_ndx)
{
	int i;

	for (i = 0; i < 18; i++) {
		if (cfgs[i] == NULL)
			continue;
		if (cfgs[i]->p4_escr_ndx == escr_ndx)
			return (1);
	}

	return (0);
}

static void
build_cfgs(p4_pcbe_config_t *cfgs[18], uint64_t *data[18], void *token)
{
	p4_pcbe_config_t	*cfg = NULL;
	uint64_t		*daddr;

	bzero(cfgs, 18 * sizeof (p4_pcbe_config_t *));

	do {
		cfg = (p4_pcbe_config_t *)kcpc_next_config(token, cfg, &daddr);

		if (cfg != NULL) {
			ASSERT(cfg->p4_picno < 18);
			cfgs[cfg->p4_picno] = cfg;
			if (data != NULL) {
				ASSERT(daddr != NULL);
				data[cfg->p4_picno] = daddr;
			}
		}
	} while (cfg != NULL);
}

/*
 * Programming a counter:
 *
 * Select event.
 * Choose an ESCR capable of counting that event.
 * Set up the ESCR with the desired parameters (usr, sys, tag).
 * Set up the CCCR to point to the selected ESCR.
 * Set the CCCR parameters (overflow, cascade, edge, etc).
 */
static int
p4_pcbe_configure(uint_t picnum, char *eventname, uint64_t preset,
    uint32_t flags, uint_t nattrs, kcpc_attr_t *attrs, void **data,
    void *token)
{
	p4_pcbe_config_t	*cfgs[18];
	p4_pcbe_config_t	*cfg;
	p4_event_t		*ev;
	p4_generic_event_t	*gevp;
	int			escr_ndx;
	int			i;
	uint16_t		emask = 0;
	uint8_t			tag;
	int			use_tag = 0;
	int			active_thread = 0x3; /* default is "any" */
	int			compare = 0;
	int			complement = 0;
	int			threshold = 0;
	int			edge = 0;
	int			sibling_usr = 0; /* count usr on other cpu */
	int			sibling_sys = 0; /* count sys on other cpu */
	int			invalid_attr = 0;

	/*
	 * If we've been handed an existing configuration, we need only preset
	 * the counter value.
	 */
	if (*data != NULL) {
		cfg = *data;
		cfg->p4_rawpic = preset & MASK40;
		return (0);
	}

	if (picnum < 0 || picnum >= 18)
		return (CPC_INVALID_PICNUM);

	if ((ev	= find_event(eventname)) == NULL) {
		if ((gevp = find_generic_event(eventname)) != NULL) {
			ev = find_event(gevp->event);
			ASSERT(ev != NULL);

			/*
			 * For generic events a HTT processor is only allowed
			 * to specify the 'active_thread', 'count_sibling_usr'
			 * and 'count_sibling_sys' attributes.
			 */
			if (p4_htt)
				for (i = 0; i < nattrs; i++)
					if (strstr(P4_ATTRS,
					    attrs[i].ka_name) != NULL)
						invalid_attr = 1;

			if ((p4_htt && invalid_attr) ||
			    (!p4_htt && nattrs > 0))
				return (CPC_ATTRIBUTE_OUT_OF_RANGE);

			emask = gevp->emask;
		} else {
			return (CPC_INVALID_EVENT);
		}
	}

	build_cfgs(cfgs, NULL, token);

	/*
	 * Find an ESCR capable of counting this event.
	 */
	for (escr_ndx = 0; escr_ndx < ESCR_MAX_INDEX; escr_ndx++) {
		if ((ev->pe_escr_map & (1ULL << escr_ndx)) &&
		    p4_escr_inuse(cfgs, escr_ndx) == 0)
			break;
	}

	/*
	 * All ESCRs capable of counting this event are already being
	 * used.
	 */
	if (escr_ndx == ESCR_MAX_INDEX)
		return (CPC_RESOURCE_UNAVAIL);

	/*
	 * At this point, ev points to the desired event and escr is the index
	 * of a capable and available ESCR.
	 *
	 * Now process and verify the attributes.
	 */
	for (i = 0; i < nattrs; i++) {
		if (strcmp("emask", attrs[i].ka_name) == 0) {
			if ((attrs[i].ka_val | ev->pe_escr_mask)
			    != ev->pe_escr_mask)
				return (CPC_ATTRIBUTE_OUT_OF_RANGE);
			emask = attrs[i].ka_val;
			continue;
		} else if (strcmp("tag", attrs[i].ka_name) == 0) {
			if (attrs[i].ka_val > ESCR_TAG_VALUE_MAX)
				return (CPC_ATTRIBUTE_OUT_OF_RANGE);
			tag = attrs[i].ka_val;
			use_tag = 1;
			continue;
		} else if (strcmp("compare", attrs[i].ka_name) == 0) {
			if (attrs[i].ka_val != 0)
				compare = 1;
			continue;
		} else if (strcmp("complement", attrs[i].ka_name) == 0) {
			if (attrs[i].ka_val != 0)
				complement = 1;
			continue;
		} else if (strcmp("threshold", attrs[i].ka_name) == 0) {
			if (attrs[i].ka_val > CCCR_THRESHOLD_MAX)
				return (CPC_ATTRIBUTE_OUT_OF_RANGE);
			threshold = attrs[i].ka_val;
			continue;
		} else if (strcmp("edge", attrs[i].ka_name) == 0) {
			if (attrs[i].ka_val != 0)
				edge = 1;
			continue;
		}

		/*
		 * The remaining attributes are valid only on HyperThreaded P4s
		 * for processes with the "cpc_cpu" privilege.
		 */
		if (p4_htt == 0)
			return (CPC_INVALID_ATTRIBUTE);

		if (secpolicy_cpc_cpu(crgetcred()) != 0)
			return (CPC_ATTR_REQUIRES_PRIVILEGE);

		if (strcmp("active_thread", attrs[i].ka_name) == 0) {
			if ((attrs[i].ka_val | CCCR_ACTV_THR_MASK) !=
			    CCCR_ACTV_THR_MASK)
				return (CPC_ATTRIBUTE_OUT_OF_RANGE);
			active_thread = (int)attrs[i].ka_val;
		} else if (strcmp("count_sibling_usr", attrs[i].ka_name) == 0) {
			if (attrs[i].ka_val != 0)
				sibling_usr = 1;
		} else if (strcmp("count_sibling_sys", attrs[i].ka_name) == 0) {
			if (attrs[i].ka_val != 0)
				sibling_sys = 1;
		} else
			return (CPC_INVALID_ATTRIBUTE);
	}

	/*
	 * Make sure the counter can count this event
	 */
	if ((ev->pe_ctr_mask & C(picnum)) == 0)
		return (CPC_PIC_NOT_CAPABLE);

	/*
	 * Find an ESCR that lines up with the event _and_ the counter.
	 */
	for (escr_ndx = 0; escr_ndx < ESCR_MAX_INDEX; escr_ndx++) {
		if ((ev->pe_escr_map & (1ULL << escr_ndx)) &&
		    (p4_escrs[escr_ndx].pe_map & (1 << picnum)) &&
		    p4_escr_inuse(cfgs, escr_ndx) == 0)
			break;
	}
	if (escr_ndx == ESCR_MAX_INDEX)
		return (CPC_RESOURCE_UNAVAIL);

	cfg = (p4_pcbe_config_t *)kmem_alloc(sizeof (p4_pcbe_config_t),
	    KM_SLEEP);

	cfg->p4_flags = 0;
	cfg->p4_picno = picnum;
	cfg->p4_escr_ndx = escr_ndx;
	cfg->p4_escr = (ev->pe_ev << ESCR_EVSEL_SHIFT) |
	    (emask << ESCR_EVMASK_SHIFT);

	if (use_tag == 1) {
		cfg->p4_escr |= tag << ESCR_TAG_VALUE_SHIFT;
		cfg->p4_escr |= ESCR_TAG_ENABLE;
	}

	if (p4_htt) {
		/*
		 * This is a HyperThreaded P4.  Since we don't know which
		 * logical CPU this configuration will eventually be programmed
		 * on, we can't yet decide which fields of the ESCR to select.
		 *
		 * Record the necessary information in the flags for later.
		 */
		if (flags & CPC_COUNT_USER)
			cfg->p4_flags |= P4_THIS_USR;
		if (flags & CPC_COUNT_SYSTEM)
			cfg->p4_flags |= P4_THIS_SYS;
		if (p4_htt && sibling_usr)
			cfg->p4_flags |= P4_SIBLING_USR;
		if (p4_htt && sibling_sys)
			cfg->p4_flags |= P4_SIBLING_SYS;
	} else {
		/*
		 * This is not HyperThreaded, so we can determine the exact
		 * ESCR value necessary now.
		 */
		if (flags & CPC_COUNT_USER)
			cfg->p4_escr |= ESCR_USR;
		if (flags & CPC_COUNT_SYSTEM)
			cfg->p4_escr |= ESCR_OS;
	}

	cfg->p4_rawpic = preset & MASK40;

	/*
	 * Even on non-HT P4s, Intel states the active_thread field (marked as
	 * "reserved" for the non-HT chips) must be set to all 1s.
	 */
	cfg->p4_cccr = CCCR_INIT | (active_thread << CCCR_ACTV_THR_SHIFT);
	if (compare)
		cfg->p4_cccr |= CCCR_COMPARE;
	if (complement)
		cfg->p4_cccr |= CCCR_COMPLEMENT;
	cfg->p4_cccr |= threshold << CCCR_THRESHOLD_SHIFT;
	if (edge)
		cfg->p4_cccr |= CCCR_EDGE;
	cfg->p4_cccr |= p4_escrs[cfg->p4_escr_ndx].pe_num
	    << CCCR_ESCR_SEL_SHIFT;
	if (flags & CPC_OVF_NOTIFY_EMT) {
		if (p4_htt)
			cfg->p4_flags |= P4_PMI;
		else {
			/*
			 * If the user has asked for notification of overflows,
			 * we automatically program the hardware to generate an
			 * interrupt on overflow.
			 *
			 * This can only be programmed now if this P4 doesn't
			 * have HyperThreading. If it does, we must wait until
			 * we know which logical CPU we'll be programming.
			 */
			cfg->p4_cccr |= CCCR_OVF_PMI;
		}
	}

	*data = cfg;

	return (0);
}

static void
p4_pcbe_program(void *token)
{
	int			i;
	uint64_t		cccr;
	p4_pcbe_config_t	*cfgs[18];

	p4_pcbe_allstop();

	build_cfgs(cfgs, NULL, token);

	if (p4_rdpmc_avail) {
		ulong_t curcr4 = getcr4();
		if (kcpc_allow_nonpriv(token))
			setcr4(curcr4 | CR4_PCE);
		else
			setcr4(curcr4 & ~CR4_PCE);
	}

	/*
	 * Ideally we would start all counters with a single operation, but in
	 * P4 each counter is enabled individually via its CCCR. To minimize the
	 * probe effect of enabling the counters, we do it in two passes: the
	 * first programs the counter and ESCR, and the second programs the
	 * CCCR (and thus enables the counter).
	 */
	if (p4_htt) {
		int	lid = cpuid_get_clogid(CPU); /* Logical ID of CPU */

		for (i = 0; i < 18; i++) {
			uint64_t escr;

			if (cfgs[i] == NULL)
				continue;
			escr = (uint64_t)cfgs[i]->p4_escr;

			if (cfgs[i]->p4_flags & P4_THIS_USR)
				escr |= (lid == 0) ? ESCR_T0_USR : ESCR_T1_USR;
			if (cfgs[i]->p4_flags & P4_THIS_SYS)
				escr |= (lid == 0) ? ESCR_T0_OS : ESCR_T1_OS;
			if (cfgs[i]->p4_flags & P4_SIBLING_USR)
				escr |= (lid == 0) ? ESCR_T1_USR : ESCR_T0_USR;
			if (cfgs[i]->p4_flags & P4_SIBLING_SYS)
				escr |= (lid == 0) ? ESCR_T1_OS : ESCR_T0_OS;

			wrmsr(p4_ctrs[i].pc_caddr, cfgs[i]->p4_rawpic);
			wrmsr(p4_escrs[cfgs[i]->p4_escr_ndx].pe_addr, escr);
		}

		for (i = 0; i < 18; i++) {
			if (cfgs[i] == NULL)
				continue;
			cccr = (uint64_t)cfgs[i]->p4_cccr;
			/*
			 * We always target the overflow interrupt at the
			 * logical CPU which is doing the counting.
			 */
			if (cfgs[i]->p4_flags & P4_PMI)
				cccr |= (lid == 0) ?
				    CCCR_OVF_PMI_T0 : CCCR_OVF_PMI_T1;
			wrmsr(p4_ctrs[i].pc_ctladdr, cccr);
		}
	} else {
		for (i = 0; i < 18; i++) {
			if (cfgs[i] == NULL)
				continue;
			wrmsr(p4_ctrs[i].pc_caddr, cfgs[i]->p4_rawpic);
			wrmsr(p4_escrs[cfgs[i]->p4_escr_ndx].pe_addr,
			    (uint64_t)cfgs[i]->p4_escr);
		}

		for (i = 0; i < 18; i++) {
			if (cfgs[i] == NULL)
				continue;
			wrmsr(p4_ctrs[i].pc_ctladdr,
			    (uint64_t)cfgs[i]->p4_cccr);
		}
	}
}

static void
p4_pcbe_allstop(void)
{
	int		i;

	for (i = 0; i < 18; i++)
		wrmsr(p4_ctrs[i].pc_ctladdr, 0ULL);

	setcr4(getcr4() & ~CR4_PCE);
}


static void
p4_pcbe_sample(void *token)
{
	p4_pcbe_config_t	*cfgs[18];
	uint64_t		*addrs[18];
	uint64_t		curpic[18];
	int64_t			diff;
	int			i;

	for (i = 0; i < 18; i++)
		curpic[i] = rdmsr(p4_ctrs[i].pc_caddr);

	build_cfgs(cfgs, addrs, token);

	for (i = 0; i < 18; i++) {
		if (cfgs[i] == NULL)
			continue;
		diff = curpic[i] - cfgs[i]->p4_rawpic;
		if (diff < 0)
			diff += (1ll << 40);
		*addrs[i] += diff;
		DTRACE_PROBE4(p4__pcbe__sample, int, i, uint64_t, *addrs[i],
		    uint64_t, curpic[i], uint64_t, cfgs[i]->p4_rawpic);
		cfgs[i]->p4_rawpic = *addrs[i] & MASK40;
	}
}

static void
p4_pcbe_free(void *config)
{
	kmem_free(config, sizeof (p4_pcbe_config_t));
}

static struct modlpcbe modlpcbe = {
	&mod_pcbeops,
	"Pentium 4 Performance Counters",
	&p4_pcbe_ops
};

static struct modlinkage modl = {
	MODREV_1,
	&modlpcbe,
};

int
_init(void)
{
	if (p4_pcbe_init() != 0)
		return (ENOTSUP);
	return (mod_install(&modl));
}

int
_fini(void)
{
	return (mod_remove(&modl));
}

int
_info(struct modinfo *mi)
{
	return (mod_info(&modl, mi));
}