whiterose

linux unikernel
Log | Files | Refs | README | LICENSE | git clone https://git.ne02ptzero.me/git/whiterose

perf_event.h (28701B)


      1 /*
      2  * Performance events x86 architecture header
      3  *
      4  *  Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
      5  *  Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar
      6  *  Copyright (C) 2009 Jaswinder Singh Rajput
      7  *  Copyright (C) 2009 Advanced Micro Devices, Inc., Robert Richter
      8  *  Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra
      9  *  Copyright (C) 2009 Intel Corporation, <markus.t.metzger@intel.com>
     10  *  Copyright (C) 2009 Google, Inc., Stephane Eranian
     11  *
     12  *  For licencing details see kernel-base/COPYING
     13  */
     14 
     15 #include <linux/perf_event.h>
     16 
     17 #include <asm/intel_ds.h>
     18 
     19 /* To enable MSR tracing please use the generic trace points. */
     20 
     21 /*
     22  *          |   NHM/WSM    |      SNB     |
     23  * register -------------------------------
     24  *          |  HT  | no HT |  HT  | no HT |
     25  *-----------------------------------------
     26  * offcore  | core | core  | cpu  | core  |
     27  * lbr_sel  | core | core  | cpu  | core  |
     28  * ld_lat   | cpu  | core  | cpu  | core  |
     29  *-----------------------------------------
     30  *
     31  * Given that there is a small number of shared regs,
     32  * we can pre-allocate their slot in the per-cpu
     33  * per-core reg tables.
     34  */
     35 enum extra_reg_type {
     36 	EXTRA_REG_NONE  = -1,	/* not used */
     37 
     38 	EXTRA_REG_RSP_0 = 0,	/* offcore_response_0 */
     39 	EXTRA_REG_RSP_1 = 1,	/* offcore_response_1 */
     40 	EXTRA_REG_LBR   = 2,	/* lbr_select */
     41 	EXTRA_REG_LDLAT = 3,	/* ld_lat_threshold */
     42 	EXTRA_REG_FE    = 4,    /* fe_* */
     43 
     44 	EXTRA_REG_MAX		/* number of entries needed */
     45 };
     46 
     47 struct event_constraint {
     48 	union {
     49 		unsigned long	idxmsk[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
     50 		u64		idxmsk64;
     51 	};
     52 	u64	code;
     53 	u64	cmask;
     54 	int	weight;
     55 	int	overlap;
     56 	int	flags;
     57 };
     58 /*
     59  * struct hw_perf_event.flags flags
     60  */
     61 #define PERF_X86_EVENT_PEBS_LDLAT	0x0001 /* ld+ldlat data address sampling */
     62 #define PERF_X86_EVENT_PEBS_ST		0x0002 /* st data address sampling */
     63 #define PERF_X86_EVENT_PEBS_ST_HSW	0x0004 /* haswell style datala, store */
     64 #define PERF_X86_EVENT_COMMITTED	0x0008 /* event passed commit_txn */
     65 #define PERF_X86_EVENT_PEBS_LD_HSW	0x0010 /* haswell style datala, load */
     66 #define PERF_X86_EVENT_PEBS_NA_HSW	0x0020 /* haswell style datala, unknown */
     67 #define PERF_X86_EVENT_EXCL		0x0040 /* HT exclusivity on counter */
     68 #define PERF_X86_EVENT_DYNAMIC		0x0080 /* dynamic alloc'd constraint */
     69 #define PERF_X86_EVENT_RDPMC_ALLOWED	0x0100 /* grant rdpmc permission */
     70 #define PERF_X86_EVENT_EXCL_ACCT	0x0200 /* accounted EXCL event */
     71 #define PERF_X86_EVENT_AUTO_RELOAD	0x0400 /* use PEBS auto-reload */
     72 #define PERF_X86_EVENT_LARGE_PEBS	0x0800 /* use large PEBS */
     73 
     74 
     75 struct amd_nb {
     76 	int nb_id;  /* NorthBridge id */
     77 	int refcnt; /* reference count */
     78 	struct perf_event *owners[X86_PMC_IDX_MAX];
     79 	struct event_constraint event_constraints[X86_PMC_IDX_MAX];
     80 };
     81 
     82 #define PEBS_COUNTER_MASK	((1ULL << MAX_PEBS_EVENTS) - 1)
     83 
     84 /*
     85  * Flags PEBS can handle without an PMI.
     86  *
     87  * TID can only be handled by flushing at context switch.
     88  * REGS_USER can be handled for events limited to ring 3.
     89  *
     90  */
     91 #define LARGE_PEBS_FLAGS \
     92 	(PERF_SAMPLE_IP | PERF_SAMPLE_TID | PERF_SAMPLE_ADDR | \
     93 	PERF_SAMPLE_ID | PERF_SAMPLE_CPU | PERF_SAMPLE_STREAM_ID | \
     94 	PERF_SAMPLE_DATA_SRC | PERF_SAMPLE_IDENTIFIER | \
     95 	PERF_SAMPLE_TRANSACTION | PERF_SAMPLE_PHYS_ADDR | \
     96 	PERF_SAMPLE_REGS_INTR | PERF_SAMPLE_REGS_USER | \
     97 	PERF_SAMPLE_PERIOD)
     98 
     99 #define PEBS_GP_REGS			\
    100 	((1ULL << PERF_REG_X86_AX)    | \
    101 	 (1ULL << PERF_REG_X86_BX)    | \
    102 	 (1ULL << PERF_REG_X86_CX)    | \
    103 	 (1ULL << PERF_REG_X86_DX)    | \
    104 	 (1ULL << PERF_REG_X86_DI)    | \
    105 	 (1ULL << PERF_REG_X86_SI)    | \
    106 	 (1ULL << PERF_REG_X86_SP)    | \
    107 	 (1ULL << PERF_REG_X86_BP)    | \
    108 	 (1ULL << PERF_REG_X86_IP)    | \
    109 	 (1ULL << PERF_REG_X86_FLAGS) | \
    110 	 (1ULL << PERF_REG_X86_R8)    | \
    111 	 (1ULL << PERF_REG_X86_R9)    | \
    112 	 (1ULL << PERF_REG_X86_R10)   | \
    113 	 (1ULL << PERF_REG_X86_R11)   | \
    114 	 (1ULL << PERF_REG_X86_R12)   | \
    115 	 (1ULL << PERF_REG_X86_R13)   | \
    116 	 (1ULL << PERF_REG_X86_R14)   | \
    117 	 (1ULL << PERF_REG_X86_R15))
    118 
    119 /*
    120  * Per register state.
    121  */
    122 struct er_account {
    123 	raw_spinlock_t      lock;	/* per-core: protect structure */
    124 	u64                 config;	/* extra MSR config */
    125 	u64                 reg;	/* extra MSR number */
    126 	atomic_t            ref;	/* reference count */
    127 };
    128 
    129 /*
    130  * Per core/cpu state
    131  *
    132  * Used to coordinate shared registers between HT threads or
    133  * among events on a single PMU.
    134  */
    135 struct intel_shared_regs {
    136 	struct er_account       regs[EXTRA_REG_MAX];
    137 	int                     refcnt;		/* per-core: #HT threads */
    138 	unsigned                core_id;	/* per-core: core id */
    139 };
    140 
    141 enum intel_excl_state_type {
    142 	INTEL_EXCL_UNUSED    = 0, /* counter is unused */
    143 	INTEL_EXCL_SHARED    = 1, /* counter can be used by both threads */
    144 	INTEL_EXCL_EXCLUSIVE = 2, /* counter can be used by one thread only */
    145 };
    146 
    147 struct intel_excl_states {
    148 	enum intel_excl_state_type state[X86_PMC_IDX_MAX];
    149 	bool sched_started; /* true if scheduling has started */
    150 };
    151 
    152 struct intel_excl_cntrs {
    153 	raw_spinlock_t	lock;
    154 
    155 	struct intel_excl_states states[2];
    156 
    157 	union {
    158 		u16	has_exclusive[2];
    159 		u32	exclusive_present;
    160 	};
    161 
    162 	int		refcnt;		/* per-core: #HT threads */
    163 	unsigned	core_id;	/* per-core: core id */
    164 };
    165 
    166 struct x86_perf_task_context;
    167 #define MAX_LBR_ENTRIES		32
    168 
    169 enum {
    170 	X86_PERF_KFREE_SHARED = 0,
    171 	X86_PERF_KFREE_EXCL   = 1,
    172 	X86_PERF_KFREE_MAX
    173 };
    174 
    175 struct cpu_hw_events {
    176 	/*
    177 	 * Generic x86 PMC bits
    178 	 */
    179 	struct perf_event	*events[X86_PMC_IDX_MAX]; /* in counter order */
    180 	unsigned long		active_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
    181 	unsigned long		running[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
    182 	int			enabled;
    183 
    184 	int			n_events; /* the # of events in the below arrays */
    185 	int			n_added;  /* the # last events in the below arrays;
    186 					     they've never been enabled yet */
    187 	int			n_txn;    /* the # last events in the below arrays;
    188 					     added in the current transaction */
    189 	int			assign[X86_PMC_IDX_MAX]; /* event to counter assignment */
    190 	u64			tags[X86_PMC_IDX_MAX];
    191 
    192 	struct perf_event	*event_list[X86_PMC_IDX_MAX]; /* in enabled order */
    193 	struct event_constraint	*event_constraint[X86_PMC_IDX_MAX];
    194 
    195 	int			n_excl; /* the number of exclusive events */
    196 
    197 	unsigned int		txn_flags;
    198 	int			is_fake;
    199 
    200 	/*
    201 	 * Intel DebugStore bits
    202 	 */
    203 	struct debug_store	*ds;
    204 	void			*ds_pebs_vaddr;
    205 	void			*ds_bts_vaddr;
    206 	u64			pebs_enabled;
    207 	int			n_pebs;
    208 	int			n_large_pebs;
    209 
    210 	/*
    211 	 * Intel LBR bits
    212 	 */
    213 	int				lbr_users;
    214 	struct perf_branch_stack	lbr_stack;
    215 	struct perf_branch_entry	lbr_entries[MAX_LBR_ENTRIES];
    216 	struct er_account		*lbr_sel;
    217 	u64				br_sel;
    218 	struct x86_perf_task_context	*last_task_ctx;
    219 	int				last_log_id;
    220 
    221 	/*
    222 	 * Intel host/guest exclude bits
    223 	 */
    224 	u64				intel_ctrl_guest_mask;
    225 	u64				intel_ctrl_host_mask;
    226 	struct perf_guest_switch_msr	guest_switch_msrs[X86_PMC_IDX_MAX];
    227 
    228 	/*
    229 	 * Intel checkpoint mask
    230 	 */
    231 	u64				intel_cp_status;
    232 
    233 	/*
    234 	 * manage shared (per-core, per-cpu) registers
    235 	 * used on Intel NHM/WSM/SNB
    236 	 */
    237 	struct intel_shared_regs	*shared_regs;
    238 	/*
    239 	 * manage exclusive counter access between hyperthread
    240 	 */
    241 	struct event_constraint *constraint_list; /* in enable order */
    242 	struct intel_excl_cntrs		*excl_cntrs;
    243 	int excl_thread_id; /* 0 or 1 */
    244 
    245 	/*
    246 	 * SKL TSX_FORCE_ABORT shadow
    247 	 */
    248 	u64				tfa_shadow;
    249 
    250 	/*
    251 	 * AMD specific bits
    252 	 */
    253 	struct amd_nb			*amd_nb;
    254 	/* Inverted mask of bits to clear in the perf_ctr ctrl registers */
    255 	u64				perf_ctr_virt_mask;
    256 
    257 	void				*kfree_on_online[X86_PERF_KFREE_MAX];
    258 };
    259 
    260 #define __EVENT_CONSTRAINT(c, n, m, w, o, f) {\
    261 	{ .idxmsk64 = (n) },		\
    262 	.code = (c),			\
    263 	.cmask = (m),			\
    264 	.weight = (w),			\
    265 	.overlap = (o),			\
    266 	.flags = f,			\
    267 }
    268 
    269 #define EVENT_CONSTRAINT(c, n, m)	\
    270 	__EVENT_CONSTRAINT(c, n, m, HWEIGHT(n), 0, 0)
    271 
    272 #define INTEL_EXCLEVT_CONSTRAINT(c, n)	\
    273 	__EVENT_CONSTRAINT(c, n, ARCH_PERFMON_EVENTSEL_EVENT, HWEIGHT(n),\
    274 			   0, PERF_X86_EVENT_EXCL)
    275 
    276 /*
    277  * The overlap flag marks event constraints with overlapping counter
    278  * masks. This is the case if the counter mask of such an event is not
    279  * a subset of any other counter mask of a constraint with an equal or
    280  * higher weight, e.g.:
    281  *
    282  *  c_overlaps = EVENT_CONSTRAINT_OVERLAP(0, 0x09, 0);
    283  *  c_another1 = EVENT_CONSTRAINT(0, 0x07, 0);
    284  *  c_another2 = EVENT_CONSTRAINT(0, 0x38, 0);
    285  *
    286  * The event scheduler may not select the correct counter in the first
    287  * cycle because it needs to know which subsequent events will be
    288  * scheduled. It may fail to schedule the events then. So we set the
    289  * overlap flag for such constraints to give the scheduler a hint which
    290  * events to select for counter rescheduling.
    291  *
    292  * Care must be taken as the rescheduling algorithm is O(n!) which
    293  * will increase scheduling cycles for an over-committed system
    294  * dramatically.  The number of such EVENT_CONSTRAINT_OVERLAP() macros
    295  * and its counter masks must be kept at a minimum.
    296  */
    297 #define EVENT_CONSTRAINT_OVERLAP(c, n, m)	\
    298 	__EVENT_CONSTRAINT(c, n, m, HWEIGHT(n), 1, 0)
    299 
    300 /*
    301  * Constraint on the Event code.
    302  */
    303 #define INTEL_EVENT_CONSTRAINT(c, n)	\
    304 	EVENT_CONSTRAINT(c, n, ARCH_PERFMON_EVENTSEL_EVENT)
    305 
    306 /*
    307  * Constraint on the Event code + UMask + fixed-mask
    308  *
    309  * filter mask to validate fixed counter events.
    310  * the following filters disqualify for fixed counters:
    311  *  - inv
    312  *  - edge
    313  *  - cnt-mask
    314  *  - in_tx
    315  *  - in_tx_checkpointed
    316  *  The other filters are supported by fixed counters.
    317  *  The any-thread option is supported starting with v3.
    318  */
    319 #define FIXED_EVENT_FLAGS (X86_RAW_EVENT_MASK|HSW_IN_TX|HSW_IN_TX_CHECKPOINTED)
    320 #define FIXED_EVENT_CONSTRAINT(c, n)	\
    321 	EVENT_CONSTRAINT(c, (1ULL << (32+n)), FIXED_EVENT_FLAGS)
    322 
    323 /*
    324  * Constraint on the Event code + UMask
    325  */
    326 #define INTEL_UEVENT_CONSTRAINT(c, n)	\
    327 	EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK)
    328 
    329 /* Constraint on specific umask bit only + event */
    330 #define INTEL_UBIT_EVENT_CONSTRAINT(c, n)	\
    331 	EVENT_CONSTRAINT(c, n, ARCH_PERFMON_EVENTSEL_EVENT|(c))
    332 
    333 /* Like UEVENT_CONSTRAINT, but match flags too */
    334 #define INTEL_FLAGS_UEVENT_CONSTRAINT(c, n)	\
    335 	EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS)
    336 
    337 #define INTEL_EXCLUEVT_CONSTRAINT(c, n)	\
    338 	__EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK, \
    339 			   HWEIGHT(n), 0, PERF_X86_EVENT_EXCL)
    340 
    341 #define INTEL_PLD_CONSTRAINT(c, n)	\
    342 	__EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \
    343 			   HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_LDLAT)
    344 
    345 #define INTEL_PST_CONSTRAINT(c, n)	\
    346 	__EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \
    347 			  HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_ST)
    348 
    349 /* Event constraint, but match on all event flags too. */
    350 #define INTEL_FLAGS_EVENT_CONSTRAINT(c, n) \
    351 	EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS)
    352 
    353 /* Check only flags, but allow all event/umask */
    354 #define INTEL_ALL_EVENT_CONSTRAINT(code, n)	\
    355 	EVENT_CONSTRAINT(code, n, X86_ALL_EVENT_FLAGS)
    356 
    357 /* Check flags and event code, and set the HSW store flag */
    358 #define INTEL_FLAGS_EVENT_CONSTRAINT_DATALA_ST(code, n) \
    359 	__EVENT_CONSTRAINT(code, n, 			\
    360 			  ARCH_PERFMON_EVENTSEL_EVENT|X86_ALL_EVENT_FLAGS, \
    361 			  HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_ST_HSW)
    362 
    363 /* Check flags and event code, and set the HSW load flag */
    364 #define INTEL_FLAGS_EVENT_CONSTRAINT_DATALA_LD(code, n) \
    365 	__EVENT_CONSTRAINT(code, n,			\
    366 			  ARCH_PERFMON_EVENTSEL_EVENT|X86_ALL_EVENT_FLAGS, \
    367 			  HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_LD_HSW)
    368 
    369 #define INTEL_FLAGS_EVENT_CONSTRAINT_DATALA_XLD(code, n) \
    370 	__EVENT_CONSTRAINT(code, n,			\
    371 			  ARCH_PERFMON_EVENTSEL_EVENT|X86_ALL_EVENT_FLAGS, \
    372 			  HWEIGHT(n), 0, \
    373 			  PERF_X86_EVENT_PEBS_LD_HSW|PERF_X86_EVENT_EXCL)
    374 
    375 /* Check flags and event code/umask, and set the HSW store flag */
    376 #define INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_ST(code, n) \
    377 	__EVENT_CONSTRAINT(code, n, 			\
    378 			  INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \
    379 			  HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_ST_HSW)
    380 
    381 #define INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_XST(code, n) \
    382 	__EVENT_CONSTRAINT(code, n,			\
    383 			  INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \
    384 			  HWEIGHT(n), 0, \
    385 			  PERF_X86_EVENT_PEBS_ST_HSW|PERF_X86_EVENT_EXCL)
    386 
    387 /* Check flags and event code/umask, and set the HSW load flag */
    388 #define INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(code, n) \
    389 	__EVENT_CONSTRAINT(code, n, 			\
    390 			  INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \
    391 			  HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_LD_HSW)
    392 
    393 #define INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_XLD(code, n) \
    394 	__EVENT_CONSTRAINT(code, n,			\
    395 			  INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \
    396 			  HWEIGHT(n), 0, \
    397 			  PERF_X86_EVENT_PEBS_LD_HSW|PERF_X86_EVENT_EXCL)
    398 
    399 /* Check flags and event code/umask, and set the HSW N/A flag */
    400 #define INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_NA(code, n) \
    401 	__EVENT_CONSTRAINT(code, n, 			\
    402 			  INTEL_ARCH_EVENT_MASK|X86_ALL_EVENT_FLAGS, \
    403 			  HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_NA_HSW)
    404 
    405 
    406 /*
    407  * We define the end marker as having a weight of -1
    408  * to enable blacklisting of events using a counter bitmask
    409  * of zero and thus a weight of zero.
    410  * The end marker has a weight that cannot possibly be
    411  * obtained from counting the bits in the bitmask.
    412  */
    413 #define EVENT_CONSTRAINT_END { .weight = -1 }
    414 
    415 /*
    416  * Check for end marker with weight == -1
    417  */
    418 #define for_each_event_constraint(e, c)	\
    419 	for ((e) = (c); (e)->weight != -1; (e)++)
    420 
    421 /*
    422  * Extra registers for specific events.
    423  *
    424  * Some events need large masks and require external MSRs.
    425  * Those extra MSRs end up being shared for all events on
    426  * a PMU and sometimes between PMU of sibling HT threads.
    427  * In either case, the kernel needs to handle conflicting
    428  * accesses to those extra, shared, regs. The data structure
    429  * to manage those registers is stored in cpu_hw_event.
    430  */
    431 struct extra_reg {
    432 	unsigned int		event;
    433 	unsigned int		msr;
    434 	u64			config_mask;
    435 	u64			valid_mask;
    436 	int			idx;  /* per_xxx->regs[] reg index */
    437 	bool			extra_msr_access;
    438 };
    439 
    440 #define EVENT_EXTRA_REG(e, ms, m, vm, i) {	\
    441 	.event = (e),			\
    442 	.msr = (ms),			\
    443 	.config_mask = (m),		\
    444 	.valid_mask = (vm),		\
    445 	.idx = EXTRA_REG_##i,		\
    446 	.extra_msr_access = true,	\
    447 	}
    448 
    449 #define INTEL_EVENT_EXTRA_REG(event, msr, vm, idx)	\
    450 	EVENT_EXTRA_REG(event, msr, ARCH_PERFMON_EVENTSEL_EVENT, vm, idx)
    451 
    452 #define INTEL_UEVENT_EXTRA_REG(event, msr, vm, idx) \
    453 	EVENT_EXTRA_REG(event, msr, ARCH_PERFMON_EVENTSEL_EVENT | \
    454 			ARCH_PERFMON_EVENTSEL_UMASK, vm, idx)
    455 
    456 #define INTEL_UEVENT_PEBS_LDLAT_EXTRA_REG(c) \
    457 	INTEL_UEVENT_EXTRA_REG(c, \
    458 			       MSR_PEBS_LD_LAT_THRESHOLD, \
    459 			       0xffff, \
    460 			       LDLAT)
    461 
    462 #define EVENT_EXTRA_END EVENT_EXTRA_REG(0, 0, 0, 0, RSP_0)
    463 
    464 union perf_capabilities {
    465 	struct {
    466 		u64	lbr_format:6;
    467 		u64	pebs_trap:1;
    468 		u64	pebs_arch_reg:1;
    469 		u64	pebs_format:4;
    470 		u64	smm_freeze:1;
    471 		/*
    472 		 * PMU supports separate counter range for writing
    473 		 * values > 32bit.
    474 		 */
    475 		u64	full_width_write:1;
    476 	};
    477 	u64	capabilities;
    478 };
    479 
    480 struct x86_pmu_quirk {
    481 	struct x86_pmu_quirk *next;
    482 	void (*func)(void);
    483 };
    484 
    485 union x86_pmu_config {
    486 	struct {
    487 		u64 event:8,
    488 		    umask:8,
    489 		    usr:1,
    490 		    os:1,
    491 		    edge:1,
    492 		    pc:1,
    493 		    interrupt:1,
    494 		    __reserved1:1,
    495 		    en:1,
    496 		    inv:1,
    497 		    cmask:8,
    498 		    event2:4,
    499 		    __reserved2:4,
    500 		    go:1,
    501 		    ho:1;
    502 	} bits;
    503 	u64 value;
    504 };
    505 
    506 #define X86_CONFIG(args...) ((union x86_pmu_config){.bits = {args}}).value
    507 
    508 enum {
    509 	x86_lbr_exclusive_lbr,
    510 	x86_lbr_exclusive_bts,
    511 	x86_lbr_exclusive_pt,
    512 	x86_lbr_exclusive_max,
    513 };
    514 
    515 /*
    516  * struct x86_pmu - generic x86 pmu
    517  */
    518 struct x86_pmu {
    519 	/*
    520 	 * Generic x86 PMC bits
    521 	 */
    522 	const char	*name;
    523 	int		version;
    524 	int		(*handle_irq)(struct pt_regs *);
    525 	void		(*disable_all)(void);
    526 	void		(*enable_all)(int added);
    527 	void		(*enable)(struct perf_event *);
    528 	void		(*disable)(struct perf_event *);
    529 	void		(*add)(struct perf_event *);
    530 	void		(*del)(struct perf_event *);
    531 	void		(*read)(struct perf_event *event);
    532 	int		(*hw_config)(struct perf_event *event);
    533 	int		(*schedule_events)(struct cpu_hw_events *cpuc, int n, int *assign);
    534 	unsigned	eventsel;
    535 	unsigned	perfctr;
    536 	int		(*addr_offset)(int index, bool eventsel);
    537 	int		(*rdpmc_index)(int index);
    538 	u64		(*event_map)(int);
    539 	int		max_events;
    540 	int		num_counters;
    541 	int		num_counters_fixed;
    542 	int		cntval_bits;
    543 	u64		cntval_mask;
    544 	union {
    545 			unsigned long events_maskl;
    546 			unsigned long events_mask[BITS_TO_LONGS(ARCH_PERFMON_EVENTS_COUNT)];
    547 	};
    548 	int		events_mask_len;
    549 	int		apic;
    550 	u64		max_period;
    551 	struct event_constraint *
    552 			(*get_event_constraints)(struct cpu_hw_events *cpuc,
    553 						 int idx,
    554 						 struct perf_event *event);
    555 
    556 	void		(*put_event_constraints)(struct cpu_hw_events *cpuc,
    557 						 struct perf_event *event);
    558 
    559 	void		(*start_scheduling)(struct cpu_hw_events *cpuc);
    560 
    561 	void		(*commit_scheduling)(struct cpu_hw_events *cpuc, int idx, int cntr);
    562 
    563 	void		(*stop_scheduling)(struct cpu_hw_events *cpuc);
    564 
    565 	struct event_constraint *event_constraints;
    566 	struct x86_pmu_quirk *quirks;
    567 	int		perfctr_second_write;
    568 	u64		(*limit_period)(struct perf_event *event, u64 l);
    569 
    570 	/* PMI handler bits */
    571 	unsigned int	late_ack		:1,
    572 			counter_freezing	:1;
    573 	/*
    574 	 * sysfs attrs
    575 	 */
    576 	int		attr_rdpmc_broken;
    577 	int		attr_rdpmc;
    578 	struct attribute **format_attrs;
    579 	struct attribute **event_attrs;
    580 	struct attribute **caps_attrs;
    581 
    582 	ssize_t		(*events_sysfs_show)(char *page, u64 config);
    583 	struct attribute **cpu_events;
    584 
    585 	unsigned long	attr_freeze_on_smi;
    586 	struct attribute **attrs;
    587 
    588 	/*
    589 	 * CPU Hotplug hooks
    590 	 */
    591 	int		(*cpu_prepare)(int cpu);
    592 	void		(*cpu_starting)(int cpu);
    593 	void		(*cpu_dying)(int cpu);
    594 	void		(*cpu_dead)(int cpu);
    595 
    596 	void		(*check_microcode)(void);
    597 	void		(*sched_task)(struct perf_event_context *ctx,
    598 				      bool sched_in);
    599 
    600 	/*
    601 	 * Intel Arch Perfmon v2+
    602 	 */
    603 	u64			intel_ctrl;
    604 	union perf_capabilities intel_cap;
    605 
    606 	/*
    607 	 * Intel DebugStore bits
    608 	 */
    609 	unsigned int	bts			:1,
    610 			bts_active		:1,
    611 			pebs			:1,
    612 			pebs_active		:1,
    613 			pebs_broken		:1,
    614 			pebs_prec_dist		:1,
    615 			pebs_no_tlb		:1,
    616 			pebs_no_isolation	:1;
    617 	int		pebs_record_size;
    618 	int		pebs_buffer_size;
    619 	void		(*drain_pebs)(struct pt_regs *regs);
    620 	struct event_constraint *pebs_constraints;
    621 	void		(*pebs_aliases)(struct perf_event *event);
    622 	int 		max_pebs_events;
    623 	unsigned long	large_pebs_flags;
    624 
    625 	/*
    626 	 * Intel LBR
    627 	 */
    628 	unsigned long	lbr_tos, lbr_from, lbr_to; /* MSR base regs       */
    629 	int		lbr_nr;			   /* hardware stack size */
    630 	u64		lbr_sel_mask;		   /* LBR_SELECT valid bits */
    631 	const int	*lbr_sel_map;		   /* lbr_select mappings */
    632 	bool		lbr_double_abort;	   /* duplicated lbr aborts */
    633 	bool		lbr_pt_coexist;		   /* (LBR|BTS) may coexist with PT */
    634 
    635 	/*
    636 	 * Intel PT/LBR/BTS are exclusive
    637 	 */
    638 	atomic_t	lbr_exclusive[x86_lbr_exclusive_max];
    639 
    640 	/*
    641 	 * AMD bits
    642 	 */
    643 	unsigned int	amd_nb_constraints : 1;
    644 
    645 	/*
    646 	 * Extra registers for events
    647 	 */
    648 	struct extra_reg *extra_regs;
    649 	unsigned int flags;
    650 
    651 	/*
    652 	 * Intel host/guest support (KVM)
    653 	 */
    654 	struct perf_guest_switch_msr *(*guest_get_msrs)(int *nr);
    655 
    656 	/*
    657 	 * Check period value for PERF_EVENT_IOC_PERIOD ioctl.
    658 	 */
    659 	int (*check_period) (struct perf_event *event, u64 period);
    660 };
    661 
    662 struct x86_perf_task_context {
    663 	u64 lbr_from[MAX_LBR_ENTRIES];
    664 	u64 lbr_to[MAX_LBR_ENTRIES];
    665 	u64 lbr_info[MAX_LBR_ENTRIES];
    666 	int tos;
    667 	int valid_lbrs;
    668 	int lbr_callstack_users;
    669 	int lbr_stack_state;
    670 	int log_id;
    671 };
    672 
    673 #define x86_add_quirk(func_)						\
    674 do {									\
    675 	static struct x86_pmu_quirk __quirk __initdata = {		\
    676 		.func = func_,						\
    677 	};								\
    678 	__quirk.next = x86_pmu.quirks;					\
    679 	x86_pmu.quirks = &__quirk;					\
    680 } while (0)
    681 
    682 /*
    683  * x86_pmu flags
    684  */
    685 #define PMU_FL_NO_HT_SHARING	0x1 /* no hyper-threading resource sharing */
    686 #define PMU_FL_HAS_RSP_1	0x2 /* has 2 equivalent offcore_rsp regs   */
    687 #define PMU_FL_EXCL_CNTRS	0x4 /* has exclusive counter requirements  */
    688 #define PMU_FL_EXCL_ENABLED	0x8 /* exclusive counter active */
    689 #define PMU_FL_PEBS_ALL		0x10 /* all events are valid PEBS events */
    690 #define PMU_FL_TFA		0x20 /* deal with TSX force abort */
    691 
    692 #define EVENT_VAR(_id)  event_attr_##_id
    693 #define EVENT_PTR(_id) &event_attr_##_id.attr.attr
    694 
    695 #define EVENT_ATTR(_name, _id)						\
    696 static struct perf_pmu_events_attr EVENT_VAR(_id) = {			\
    697 	.attr		= __ATTR(_name, 0444, events_sysfs_show, NULL),	\
    698 	.id		= PERF_COUNT_HW_##_id,				\
    699 	.event_str	= NULL,						\
    700 };
    701 
    702 #define EVENT_ATTR_STR(_name, v, str)					\
    703 static struct perf_pmu_events_attr event_attr_##v = {			\
    704 	.attr		= __ATTR(_name, 0444, events_sysfs_show, NULL),	\
    705 	.id		= 0,						\
    706 	.event_str	= str,						\
    707 };
    708 
    709 #define EVENT_ATTR_STR_HT(_name, v, noht, ht)				\
    710 static struct perf_pmu_events_ht_attr event_attr_##v = {		\
    711 	.attr		= __ATTR(_name, 0444, events_ht_sysfs_show, NULL),\
    712 	.id		= 0,						\
    713 	.event_str_noht	= noht,						\
    714 	.event_str_ht	= ht,						\
    715 }
    716 
    717 extern struct x86_pmu x86_pmu __read_mostly;
    718 
    719 static inline bool x86_pmu_has_lbr_callstack(void)
    720 {
    721 	return  x86_pmu.lbr_sel_map &&
    722 		x86_pmu.lbr_sel_map[PERF_SAMPLE_BRANCH_CALL_STACK_SHIFT] > 0;
    723 }
    724 
    725 DECLARE_PER_CPU(struct cpu_hw_events, cpu_hw_events);
    726 
    727 int x86_perf_event_set_period(struct perf_event *event);
    728 
    729 /*
    730  * Generalized hw caching related hw_event table, filled
    731  * in on a per model basis. A value of 0 means
    732  * 'not supported', -1 means 'hw_event makes no sense on
    733  * this CPU', any other value means the raw hw_event
    734  * ID.
    735  */
    736 
    737 #define C(x) PERF_COUNT_HW_CACHE_##x
    738 
    739 extern u64 __read_mostly hw_cache_event_ids
    740 				[PERF_COUNT_HW_CACHE_MAX]
    741 				[PERF_COUNT_HW_CACHE_OP_MAX]
    742 				[PERF_COUNT_HW_CACHE_RESULT_MAX];
    743 extern u64 __read_mostly hw_cache_extra_regs
    744 				[PERF_COUNT_HW_CACHE_MAX]
    745 				[PERF_COUNT_HW_CACHE_OP_MAX]
    746 				[PERF_COUNT_HW_CACHE_RESULT_MAX];
    747 
    748 u64 x86_perf_event_update(struct perf_event *event);
    749 
    750 static inline unsigned int x86_pmu_config_addr(int index)
    751 {
    752 	return x86_pmu.eventsel + (x86_pmu.addr_offset ?
    753 				   x86_pmu.addr_offset(index, true) : index);
    754 }
    755 
    756 static inline unsigned int x86_pmu_event_addr(int index)
    757 {
    758 	return x86_pmu.perfctr + (x86_pmu.addr_offset ?
    759 				  x86_pmu.addr_offset(index, false) : index);
    760 }
    761 
    762 static inline int x86_pmu_rdpmc_index(int index)
    763 {
    764 	return x86_pmu.rdpmc_index ? x86_pmu.rdpmc_index(index) : index;
    765 }
    766 
    767 int x86_add_exclusive(unsigned int what);
    768 
    769 void x86_del_exclusive(unsigned int what);
    770 
    771 int x86_reserve_hardware(void);
    772 
    773 void x86_release_hardware(void);
    774 
    775 int x86_pmu_max_precise(void);
    776 
    777 void hw_perf_lbr_event_destroy(struct perf_event *event);
    778 
    779 int x86_setup_perfctr(struct perf_event *event);
    780 
    781 int x86_pmu_hw_config(struct perf_event *event);
    782 
    783 void x86_pmu_disable_all(void);
    784 
    785 static inline void __x86_pmu_enable_event(struct hw_perf_event *hwc,
    786 					  u64 enable_mask)
    787 {
    788 	u64 disable_mask = __this_cpu_read(cpu_hw_events.perf_ctr_virt_mask);
    789 
    790 	if (hwc->extra_reg.reg)
    791 		wrmsrl(hwc->extra_reg.reg, hwc->extra_reg.config);
    792 	wrmsrl(hwc->config_base, (hwc->config | enable_mask) & ~disable_mask);
    793 }
    794 
    795 void x86_pmu_enable_all(int added);
    796 
    797 int perf_assign_events(struct event_constraint **constraints, int n,
    798 			int wmin, int wmax, int gpmax, int *assign);
    799 int x86_schedule_events(struct cpu_hw_events *cpuc, int n, int *assign);
    800 
    801 void x86_pmu_stop(struct perf_event *event, int flags);
    802 
    803 static inline void x86_pmu_disable_event(struct perf_event *event)
    804 {
    805 	struct hw_perf_event *hwc = &event->hw;
    806 
    807 	wrmsrl(hwc->config_base, hwc->config);
    808 }
    809 
    810 void x86_pmu_enable_event(struct perf_event *event);
    811 
    812 int x86_pmu_handle_irq(struct pt_regs *regs);
    813 
    814 extern struct event_constraint emptyconstraint;
    815 
    816 extern struct event_constraint unconstrained;
    817 
    818 static inline bool kernel_ip(unsigned long ip)
    819 {
    820 #ifdef CONFIG_X86_32
    821 	return ip > PAGE_OFFSET;
    822 #else
    823 	return (long)ip < 0;
    824 #endif
    825 }
    826 
    827 /*
    828  * Not all PMUs provide the right context information to place the reported IP
    829  * into full context. Specifically segment registers are typically not
    830  * supplied.
    831  *
    832  * Assuming the address is a linear address (it is for IBS), we fake the CS and
    833  * vm86 mode using the known zero-based code segment and 'fix up' the registers
    834  * to reflect this.
    835  *
    836  * Intel PEBS/LBR appear to typically provide the effective address, nothing
    837  * much we can do about that but pray and treat it like a linear address.
    838  */
    839 static inline void set_linear_ip(struct pt_regs *regs, unsigned long ip)
    840 {
    841 	regs->cs = kernel_ip(ip) ? __KERNEL_CS : __USER_CS;
    842 	if (regs->flags & X86_VM_MASK)
    843 		regs->flags ^= (PERF_EFLAGS_VM | X86_VM_MASK);
    844 	regs->ip = ip;
    845 }
    846 
    847 ssize_t x86_event_sysfs_show(char *page, u64 config, u64 event);
    848 ssize_t intel_event_sysfs_show(char *page, u64 config);
    849 
    850 struct attribute **merge_attr(struct attribute **a, struct attribute **b);
    851 
    852 ssize_t events_sysfs_show(struct device *dev, struct device_attribute *attr,
    853 			  char *page);
    854 ssize_t events_ht_sysfs_show(struct device *dev, struct device_attribute *attr,
    855 			  char *page);
    856 
    857 #ifdef CONFIG_CPU_SUP_AMD
    858 
    859 int amd_pmu_init(void);
    860 
    861 #else /* CONFIG_CPU_SUP_AMD */
    862 
    863 static inline int amd_pmu_init(void)
    864 {
    865 	return 0;
    866 }
    867 
    868 #endif /* CONFIG_CPU_SUP_AMD */
    869 
    870 #ifdef CONFIG_CPU_SUP_INTEL
    871 
    872 static inline bool intel_pmu_has_bts_period(struct perf_event *event, u64 period)
    873 {
    874 	struct hw_perf_event *hwc = &event->hw;
    875 	unsigned int hw_event, bts_event;
    876 
    877 	if (event->attr.freq)
    878 		return false;
    879 
    880 	hw_event = hwc->config & INTEL_ARCH_EVENT_MASK;
    881 	bts_event = x86_pmu.event_map(PERF_COUNT_HW_BRANCH_INSTRUCTIONS);
    882 
    883 	return hw_event == bts_event && period == 1;
    884 }
    885 
    886 static inline bool intel_pmu_has_bts(struct perf_event *event)
    887 {
    888 	struct hw_perf_event *hwc = &event->hw;
    889 
    890 	return intel_pmu_has_bts_period(event, hwc->sample_period);
    891 }
    892 
    893 int intel_pmu_save_and_restart(struct perf_event *event);
    894 
    895 struct event_constraint *
    896 x86_get_event_constraints(struct cpu_hw_events *cpuc, int idx,
    897 			  struct perf_event *event);
    898 
    899 extern int intel_cpuc_prepare(struct cpu_hw_events *cpuc, int cpu);
    900 extern void intel_cpuc_finish(struct cpu_hw_events *cpuc);
    901 
    902 int intel_pmu_init(void);
    903 
    904 void init_debug_store_on_cpu(int cpu);
    905 
    906 void fini_debug_store_on_cpu(int cpu);
    907 
    908 void release_ds_buffers(void);
    909 
    910 void reserve_ds_buffers(void);
    911 
    912 extern struct event_constraint bts_constraint;
    913 
    914 void intel_pmu_enable_bts(u64 config);
    915 
    916 void intel_pmu_disable_bts(void);
    917 
    918 int intel_pmu_drain_bts_buffer(void);
    919 
    920 extern struct event_constraint intel_core2_pebs_event_constraints[];
    921 
    922 extern struct event_constraint intel_atom_pebs_event_constraints[];
    923 
    924 extern struct event_constraint intel_slm_pebs_event_constraints[];
    925 
    926 extern struct event_constraint intel_glm_pebs_event_constraints[];
    927 
    928 extern struct event_constraint intel_glp_pebs_event_constraints[];
    929 
    930 extern struct event_constraint intel_nehalem_pebs_event_constraints[];
    931 
    932 extern struct event_constraint intel_westmere_pebs_event_constraints[];
    933 
    934 extern struct event_constraint intel_snb_pebs_event_constraints[];
    935 
    936 extern struct event_constraint intel_ivb_pebs_event_constraints[];
    937 
    938 extern struct event_constraint intel_hsw_pebs_event_constraints[];
    939 
    940 extern struct event_constraint intel_bdw_pebs_event_constraints[];
    941 
    942 extern struct event_constraint intel_skl_pebs_event_constraints[];
    943 
    944 struct event_constraint *intel_pebs_constraints(struct perf_event *event);
    945 
    946 void intel_pmu_pebs_add(struct perf_event *event);
    947 
    948 void intel_pmu_pebs_del(struct perf_event *event);
    949 
    950 void intel_pmu_pebs_enable(struct perf_event *event);
    951 
    952 void intel_pmu_pebs_disable(struct perf_event *event);
    953 
    954 void intel_pmu_pebs_enable_all(void);
    955 
    956 void intel_pmu_pebs_disable_all(void);
    957 
    958 void intel_pmu_pebs_sched_task(struct perf_event_context *ctx, bool sched_in);
    959 
    960 void intel_pmu_auto_reload_read(struct perf_event *event);
    961 
    962 void intel_ds_init(void);
    963 
    964 void intel_pmu_lbr_sched_task(struct perf_event_context *ctx, bool sched_in);
    965 
    966 u64 lbr_from_signext_quirk_wr(u64 val);
    967 
    968 void intel_pmu_lbr_reset(void);
    969 
    970 void intel_pmu_lbr_add(struct perf_event *event);
    971 
    972 void intel_pmu_lbr_del(struct perf_event *event);
    973 
    974 void intel_pmu_lbr_enable_all(bool pmi);
    975 
    976 void intel_pmu_lbr_disable_all(void);
    977 
    978 void intel_pmu_lbr_read(void);
    979 
    980 void intel_pmu_lbr_init_core(void);
    981 
    982 void intel_pmu_lbr_init_nhm(void);
    983 
    984 void intel_pmu_lbr_init_atom(void);
    985 
    986 void intel_pmu_lbr_init_slm(void);
    987 
    988 void intel_pmu_lbr_init_snb(void);
    989 
    990 void intel_pmu_lbr_init_hsw(void);
    991 
    992 void intel_pmu_lbr_init_skl(void);
    993 
    994 void intel_pmu_lbr_init_knl(void);
    995 
    996 void intel_pmu_pebs_data_source_nhm(void);
    997 
    998 void intel_pmu_pebs_data_source_skl(bool pmem);
    999 
   1000 int intel_pmu_setup_lbr_filter(struct perf_event *event);
   1001 
   1002 void intel_pt_interrupt(void);
   1003 
   1004 int intel_bts_interrupt(void);
   1005 
   1006 void intel_bts_enable_local(void);
   1007 
   1008 void intel_bts_disable_local(void);
   1009 
   1010 int p4_pmu_init(void);
   1011 
   1012 int p6_pmu_init(void);
   1013 
   1014 int knc_pmu_init(void);
   1015 
   1016 static inline int is_ht_workaround_enabled(void)
   1017 {
   1018 	return !!(x86_pmu.flags & PMU_FL_EXCL_ENABLED);
   1019 }
   1020 
   1021 #else /* CONFIG_CPU_SUP_INTEL */
   1022 
   1023 static inline void reserve_ds_buffers(void)
   1024 {
   1025 }
   1026 
   1027 static inline void release_ds_buffers(void)
   1028 {
   1029 }
   1030 
   1031 static inline int intel_pmu_init(void)
   1032 {
   1033 	return 0;
   1034 }
   1035 
   1036 static inline int intel_cpuc_prepare(struct cpu_hw_events *cpuc, int cpu)
   1037 {
   1038 	return 0;
   1039 }
   1040 
   1041 static inline void intel_cpuc_finish(struct cpu_hw_events *cpuc)
   1042 {
   1043 }
   1044 
   1045 static inline int is_ht_workaround_enabled(void)
   1046 {
   1047 	return 0;
   1048 }
   1049 #endif /* CONFIG_CPU_SUP_INTEL */