whiterose

linux unikernel
Log | Files | Refs | README | LICENSE | git clone https://git.ne02ptzero.me/git/whiterose

core.c (61909B)


      1 /*
      2  * Performance events x86 architecture code
      3  *
      4  *  Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
      5  *  Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar
      6  *  Copyright (C) 2009 Jaswinder Singh Rajput
      7  *  Copyright (C) 2009 Advanced Micro Devices, Inc., Robert Richter
      8  *  Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra
      9  *  Copyright (C) 2009 Intel Corporation, <markus.t.metzger@intel.com>
     10  *  Copyright (C) 2009 Google, Inc., Stephane Eranian
     11  *
     12  *  For licencing details see kernel-base/COPYING
     13  */
     14 
     15 #include <linux/perf_event.h>
     16 #include <linux/capability.h>
     17 #include <linux/notifier.h>
     18 #include <linux/hardirq.h>
     19 #include <linux/kprobes.h>
     20 #include <linux/export.h>
     21 #include <linux/init.h>
     22 #include <linux/kdebug.h>
     23 #include <linux/sched/mm.h>
     24 #include <linux/sched/clock.h>
     25 #include <linux/uaccess.h>
     26 #include <linux/slab.h>
     27 #include <linux/cpu.h>
     28 #include <linux/bitops.h>
     29 #include <linux/device.h>
     30 #include <linux/nospec.h>
     31 
     32 #include <asm/apic.h>
     33 #include <asm/stacktrace.h>
     34 #include <asm/nmi.h>
     35 #include <asm/smp.h>
     36 #include <asm/alternative.h>
     37 #include <asm/mmu_context.h>
     38 #include <asm/tlbflush.h>
     39 #include <asm/timer.h>
     40 #include <asm/desc.h>
     41 #include <asm/ldt.h>
     42 #include <asm/unwind.h>
     43 
     44 #include "perf_event.h"
     45 
     46 struct x86_pmu x86_pmu __read_mostly;
     47 
     48 DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events) = {
     49 	.enabled = 1,
     50 };
     51 
     52 DEFINE_STATIC_KEY_FALSE(rdpmc_always_available_key);
     53 
     54 u64 __read_mostly hw_cache_event_ids
     55 				[PERF_COUNT_HW_CACHE_MAX]
     56 				[PERF_COUNT_HW_CACHE_OP_MAX]
     57 				[PERF_COUNT_HW_CACHE_RESULT_MAX];
     58 u64 __read_mostly hw_cache_extra_regs
     59 				[PERF_COUNT_HW_CACHE_MAX]
     60 				[PERF_COUNT_HW_CACHE_OP_MAX]
     61 				[PERF_COUNT_HW_CACHE_RESULT_MAX];
     62 
     63 /*
     64  * Propagate event elapsed time into the generic event.
     65  * Can only be executed on the CPU where the event is active.
     66  * Returns the delta events processed.
     67  */
     68 u64 x86_perf_event_update(struct perf_event *event)
     69 {
     70 	struct hw_perf_event *hwc = &event->hw;
     71 	int shift = 64 - x86_pmu.cntval_bits;
     72 	u64 prev_raw_count, new_raw_count;
     73 	int idx = hwc->idx;
     74 	u64 delta;
     75 
     76 	if (idx == INTEL_PMC_IDX_FIXED_BTS)
     77 		return 0;
     78 
     79 	/*
     80 	 * Careful: an NMI might modify the previous event value.
     81 	 *
     82 	 * Our tactic to handle this is to first atomically read and
     83 	 * exchange a new raw count - then add that new-prev delta
     84 	 * count to the generic event atomically:
     85 	 */
     86 again:
     87 	prev_raw_count = local64_read(&hwc->prev_count);
     88 	rdpmcl(hwc->event_base_rdpmc, new_raw_count);
     89 
     90 	if (local64_cmpxchg(&hwc->prev_count, prev_raw_count,
     91 					new_raw_count) != prev_raw_count)
     92 		goto again;
     93 
     94 	/*
     95 	 * Now we have the new raw value and have updated the prev
     96 	 * timestamp already. We can now calculate the elapsed delta
     97 	 * (event-)time and add that to the generic event.
     98 	 *
     99 	 * Careful, not all hw sign-extends above the physical width
    100 	 * of the count.
    101 	 */
    102 	delta = (new_raw_count << shift) - (prev_raw_count << shift);
    103 	delta >>= shift;
    104 
    105 	local64_add(delta, &event->count);
    106 	local64_sub(delta, &hwc->period_left);
    107 
    108 	return new_raw_count;
    109 }
    110 
    111 /*
    112  * Find and validate any extra registers to set up.
    113  */
    114 static int x86_pmu_extra_regs(u64 config, struct perf_event *event)
    115 {
    116 	struct hw_perf_event_extra *reg;
    117 	struct extra_reg *er;
    118 
    119 	reg = &event->hw.extra_reg;
    120 
    121 	if (!x86_pmu.extra_regs)
    122 		return 0;
    123 
    124 	for (er = x86_pmu.extra_regs; er->msr; er++) {
    125 		if (er->event != (config & er->config_mask))
    126 			continue;
    127 		if (event->attr.config1 & ~er->valid_mask)
    128 			return -EINVAL;
    129 		/* Check if the extra msrs can be safely accessed*/
    130 		if (!er->extra_msr_access)
    131 			return -ENXIO;
    132 
    133 		reg->idx = er->idx;
    134 		reg->config = event->attr.config1;
    135 		reg->reg = er->msr;
    136 		break;
    137 	}
    138 	return 0;
    139 }
    140 
    141 static atomic_t active_events;
    142 static atomic_t pmc_refcount;
    143 static DEFINE_MUTEX(pmc_reserve_mutex);
    144 
    145 #ifdef CONFIG_X86_LOCAL_APIC
    146 
    147 static bool reserve_pmc_hardware(void)
    148 {
    149 	int i;
    150 
    151 	for (i = 0; i < x86_pmu.num_counters; i++) {
    152 		if (!reserve_perfctr_nmi(x86_pmu_event_addr(i)))
    153 			goto perfctr_fail;
    154 	}
    155 
    156 	for (i = 0; i < x86_pmu.num_counters; i++) {
    157 		if (!reserve_evntsel_nmi(x86_pmu_config_addr(i)))
    158 			goto eventsel_fail;
    159 	}
    160 
    161 	return true;
    162 
    163 eventsel_fail:
    164 	for (i--; i >= 0; i--)
    165 		release_evntsel_nmi(x86_pmu_config_addr(i));
    166 
    167 	i = x86_pmu.num_counters;
    168 
    169 perfctr_fail:
    170 	for (i--; i >= 0; i--)
    171 		release_perfctr_nmi(x86_pmu_event_addr(i));
    172 
    173 	return false;
    174 }
    175 
    176 static void release_pmc_hardware(void)
    177 {
    178 	int i;
    179 
    180 	for (i = 0; i < x86_pmu.num_counters; i++) {
    181 		release_perfctr_nmi(x86_pmu_event_addr(i));
    182 		release_evntsel_nmi(x86_pmu_config_addr(i));
    183 	}
    184 }
    185 
    186 #else
    187 
    188 static bool reserve_pmc_hardware(void) { return true; }
    189 static void release_pmc_hardware(void) {}
    190 
    191 #endif
    192 
    193 static bool check_hw_exists(void)
    194 {
    195 	u64 val, val_fail = -1, val_new= ~0;
    196 	int i, reg, reg_fail = -1, ret = 0;
    197 	int bios_fail = 0;
    198 	int reg_safe = -1;
    199 
    200 	/*
    201 	 * Check to see if the BIOS enabled any of the counters, if so
    202 	 * complain and bail.
    203 	 */
    204 	for (i = 0; i < x86_pmu.num_counters; i++) {
    205 		reg = x86_pmu_config_addr(i);
    206 		ret = rdmsrl_safe(reg, &val);
    207 		if (ret)
    208 			goto msr_fail;
    209 		if (val & ARCH_PERFMON_EVENTSEL_ENABLE) {
    210 			bios_fail = 1;
    211 			val_fail = val;
    212 			reg_fail = reg;
    213 		} else {
    214 			reg_safe = i;
    215 		}
    216 	}
    217 
    218 	if (x86_pmu.num_counters_fixed) {
    219 		reg = MSR_ARCH_PERFMON_FIXED_CTR_CTRL;
    220 		ret = rdmsrl_safe(reg, &val);
    221 		if (ret)
    222 			goto msr_fail;
    223 		for (i = 0; i < x86_pmu.num_counters_fixed; i++) {
    224 			if (val & (0x03 << i*4)) {
    225 				bios_fail = 1;
    226 				val_fail = val;
    227 				reg_fail = reg;
    228 			}
    229 		}
    230 	}
    231 
    232 	/*
    233 	 * If all the counters are enabled, the below test will always
    234 	 * fail.  The tools will also become useless in this scenario.
    235 	 * Just fail and disable the hardware counters.
    236 	 */
    237 
    238 	if (reg_safe == -1) {
    239 		reg = reg_safe;
    240 		goto msr_fail;
    241 	}
    242 
    243 	/*
    244 	 * Read the current value, change it and read it back to see if it
    245 	 * matches, this is needed to detect certain hardware emulators
    246 	 * (qemu/kvm) that don't trap on the MSR access and always return 0s.
    247 	 */
    248 	reg = x86_pmu_event_addr(reg_safe);
    249 	if (rdmsrl_safe(reg, &val))
    250 		goto msr_fail;
    251 	val ^= 0xffffUL;
    252 	ret = wrmsrl_safe(reg, val);
    253 	ret |= rdmsrl_safe(reg, &val_new);
    254 	if (ret || val != val_new)
    255 		goto msr_fail;
    256 
    257 	/*
    258 	 * We still allow the PMU driver to operate:
    259 	 */
    260 	if (bios_fail) {
    261 		pr_cont("Broken BIOS detected, complain to your hardware vendor.\n");
    262 		pr_err(FW_BUG "the BIOS has corrupted hw-PMU resources (MSR %x is %Lx)\n",
    263 			      reg_fail, val_fail);
    264 	}
    265 
    266 	return true;
    267 
    268 msr_fail:
    269 	if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) {
    270 		pr_cont("PMU not available due to virtualization, using software events only.\n");
    271 	} else {
    272 		pr_cont("Broken PMU hardware detected, using software events only.\n");
    273 		pr_err("Failed to access perfctr msr (MSR %x is %Lx)\n",
    274 		       reg, val_new);
    275 	}
    276 
    277 	return false;
    278 }
    279 
    280 static void hw_perf_event_destroy(struct perf_event *event)
    281 {
    282 	x86_release_hardware();
    283 	atomic_dec(&active_events);
    284 }
    285 
    286 void hw_perf_lbr_event_destroy(struct perf_event *event)
    287 {
    288 	hw_perf_event_destroy(event);
    289 
    290 	/* undo the lbr/bts event accounting */
    291 	x86_del_exclusive(x86_lbr_exclusive_lbr);
    292 }
    293 
    294 static inline int x86_pmu_initialized(void)
    295 {
    296 	return x86_pmu.handle_irq != NULL;
    297 }
    298 
    299 static inline int
    300 set_ext_hw_attr(struct hw_perf_event *hwc, struct perf_event *event)
    301 {
    302 	struct perf_event_attr *attr = &event->attr;
    303 	unsigned int cache_type, cache_op, cache_result;
    304 	u64 config, val;
    305 
    306 	config = attr->config;
    307 
    308 	cache_type = (config >> 0) & 0xff;
    309 	if (cache_type >= PERF_COUNT_HW_CACHE_MAX)
    310 		return -EINVAL;
    311 	cache_type = array_index_nospec(cache_type, PERF_COUNT_HW_CACHE_MAX);
    312 
    313 	cache_op = (config >>  8) & 0xff;
    314 	if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX)
    315 		return -EINVAL;
    316 	cache_op = array_index_nospec(cache_op, PERF_COUNT_HW_CACHE_OP_MAX);
    317 
    318 	cache_result = (config >> 16) & 0xff;
    319 	if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
    320 		return -EINVAL;
    321 	cache_result = array_index_nospec(cache_result, PERF_COUNT_HW_CACHE_RESULT_MAX);
    322 
    323 	val = hw_cache_event_ids[cache_type][cache_op][cache_result];
    324 
    325 	if (val == 0)
    326 		return -ENOENT;
    327 
    328 	if (val == -1)
    329 		return -EINVAL;
    330 
    331 	hwc->config |= val;
    332 	attr->config1 = hw_cache_extra_regs[cache_type][cache_op][cache_result];
    333 	return x86_pmu_extra_regs(val, event);
    334 }
    335 
    336 int x86_reserve_hardware(void)
    337 {
    338 	int err = 0;
    339 
    340 	if (!atomic_inc_not_zero(&pmc_refcount)) {
    341 		mutex_lock(&pmc_reserve_mutex);
    342 		if (atomic_read(&pmc_refcount) == 0) {
    343 			if (!reserve_pmc_hardware())
    344 				err = -EBUSY;
    345 			else
    346 				reserve_ds_buffers();
    347 		}
    348 		if (!err)
    349 			atomic_inc(&pmc_refcount);
    350 		mutex_unlock(&pmc_reserve_mutex);
    351 	}
    352 
    353 	return err;
    354 }
    355 
    356 void x86_release_hardware(void)
    357 {
    358 	if (atomic_dec_and_mutex_lock(&pmc_refcount, &pmc_reserve_mutex)) {
    359 		release_pmc_hardware();
    360 		release_ds_buffers();
    361 		mutex_unlock(&pmc_reserve_mutex);
    362 	}
    363 }
    364 
    365 /*
    366  * Check if we can create event of a certain type (that no conflicting events
    367  * are present).
    368  */
    369 int x86_add_exclusive(unsigned int what)
    370 {
    371 	int i;
    372 
    373 	/*
    374 	 * When lbr_pt_coexist we allow PT to coexist with either LBR or BTS.
    375 	 * LBR and BTS are still mutually exclusive.
    376 	 */
    377 	if (x86_pmu.lbr_pt_coexist && what == x86_lbr_exclusive_pt)
    378 		return 0;
    379 
    380 	if (!atomic_inc_not_zero(&x86_pmu.lbr_exclusive[what])) {
    381 		mutex_lock(&pmc_reserve_mutex);
    382 		for (i = 0; i < ARRAY_SIZE(x86_pmu.lbr_exclusive); i++) {
    383 			if (i != what && atomic_read(&x86_pmu.lbr_exclusive[i]))
    384 				goto fail_unlock;
    385 		}
    386 		atomic_inc(&x86_pmu.lbr_exclusive[what]);
    387 		mutex_unlock(&pmc_reserve_mutex);
    388 	}
    389 
    390 	atomic_inc(&active_events);
    391 	return 0;
    392 
    393 fail_unlock:
    394 	mutex_unlock(&pmc_reserve_mutex);
    395 	return -EBUSY;
    396 }
    397 
    398 void x86_del_exclusive(unsigned int what)
    399 {
    400 	if (x86_pmu.lbr_pt_coexist && what == x86_lbr_exclusive_pt)
    401 		return;
    402 
    403 	atomic_dec(&x86_pmu.lbr_exclusive[what]);
    404 	atomic_dec(&active_events);
    405 }
    406 
    407 int x86_setup_perfctr(struct perf_event *event)
    408 {
    409 	struct perf_event_attr *attr = &event->attr;
    410 	struct hw_perf_event *hwc = &event->hw;
    411 	u64 config;
    412 
    413 	if (!is_sampling_event(event)) {
    414 		hwc->sample_period = x86_pmu.max_period;
    415 		hwc->last_period = hwc->sample_period;
    416 		local64_set(&hwc->period_left, hwc->sample_period);
    417 	}
    418 
    419 	if (attr->type == PERF_TYPE_RAW)
    420 		return x86_pmu_extra_regs(event->attr.config, event);
    421 
    422 	if (attr->type == PERF_TYPE_HW_CACHE)
    423 		return set_ext_hw_attr(hwc, event);
    424 
    425 	if (attr->config >= x86_pmu.max_events)
    426 		return -EINVAL;
    427 
    428 	attr->config = array_index_nospec((unsigned long)attr->config, x86_pmu.max_events);
    429 
    430 	/*
    431 	 * The generic map:
    432 	 */
    433 	config = x86_pmu.event_map(attr->config);
    434 
    435 	if (config == 0)
    436 		return -ENOENT;
    437 
    438 	if (config == -1LL)
    439 		return -EINVAL;
    440 
    441 	hwc->config |= config;
    442 
    443 	return 0;
    444 }
    445 
    446 /*
    447  * check that branch_sample_type is compatible with
    448  * settings needed for precise_ip > 1 which implies
    449  * using the LBR to capture ALL taken branches at the
    450  * priv levels of the measurement
    451  */
    452 static inline int precise_br_compat(struct perf_event *event)
    453 {
    454 	u64 m = event->attr.branch_sample_type;
    455 	u64 b = 0;
    456 
    457 	/* must capture all branches */
    458 	if (!(m & PERF_SAMPLE_BRANCH_ANY))
    459 		return 0;
    460 
    461 	m &= PERF_SAMPLE_BRANCH_KERNEL | PERF_SAMPLE_BRANCH_USER;
    462 
    463 	if (!event->attr.exclude_user)
    464 		b |= PERF_SAMPLE_BRANCH_USER;
    465 
    466 	if (!event->attr.exclude_kernel)
    467 		b |= PERF_SAMPLE_BRANCH_KERNEL;
    468 
    469 	/*
    470 	 * ignore PERF_SAMPLE_BRANCH_HV, not supported on x86
    471 	 */
    472 
    473 	return m == b;
    474 }
    475 
    476 int x86_pmu_max_precise(void)
    477 {
    478 	int precise = 0;
    479 
    480 	/* Support for constant skid */
    481 	if (x86_pmu.pebs_active && !x86_pmu.pebs_broken) {
    482 		precise++;
    483 
    484 		/* Support for IP fixup */
    485 		if (x86_pmu.lbr_nr || x86_pmu.intel_cap.pebs_format >= 2)
    486 			precise++;
    487 
    488 		if (x86_pmu.pebs_prec_dist)
    489 			precise++;
    490 	}
    491 	return precise;
    492 }
    493 
    494 int x86_pmu_hw_config(struct perf_event *event)
    495 {
    496 	if (event->attr.precise_ip) {
    497 		int precise = x86_pmu_max_precise();
    498 
    499 		if (event->attr.precise_ip > precise)
    500 			return -EOPNOTSUPP;
    501 
    502 		/* There's no sense in having PEBS for non sampling events: */
    503 		if (!is_sampling_event(event))
    504 			return -EINVAL;
    505 	}
    506 	/*
    507 	 * check that PEBS LBR correction does not conflict with
    508 	 * whatever the user is asking with attr->branch_sample_type
    509 	 */
    510 	if (event->attr.precise_ip > 1 && x86_pmu.intel_cap.pebs_format < 2) {
    511 		u64 *br_type = &event->attr.branch_sample_type;
    512 
    513 		if (has_branch_stack(event)) {
    514 			if (!precise_br_compat(event))
    515 				return -EOPNOTSUPP;
    516 
    517 			/* branch_sample_type is compatible */
    518 
    519 		} else {
    520 			/*
    521 			 * user did not specify  branch_sample_type
    522 			 *
    523 			 * For PEBS fixups, we capture all
    524 			 * the branches at the priv level of the
    525 			 * event.
    526 			 */
    527 			*br_type = PERF_SAMPLE_BRANCH_ANY;
    528 
    529 			if (!event->attr.exclude_user)
    530 				*br_type |= PERF_SAMPLE_BRANCH_USER;
    531 
    532 			if (!event->attr.exclude_kernel)
    533 				*br_type |= PERF_SAMPLE_BRANCH_KERNEL;
    534 		}
    535 	}
    536 
    537 	if (event->attr.branch_sample_type & PERF_SAMPLE_BRANCH_CALL_STACK)
    538 		event->attach_state |= PERF_ATTACH_TASK_DATA;
    539 
    540 	/*
    541 	 * Generate PMC IRQs:
    542 	 * (keep 'enabled' bit clear for now)
    543 	 */
    544 	event->hw.config = ARCH_PERFMON_EVENTSEL_INT;
    545 
    546 	/*
    547 	 * Count user and OS events unless requested not to
    548 	 */
    549 	if (!event->attr.exclude_user)
    550 		event->hw.config |= ARCH_PERFMON_EVENTSEL_USR;
    551 	if (!event->attr.exclude_kernel)
    552 		event->hw.config |= ARCH_PERFMON_EVENTSEL_OS;
    553 
    554 	if (event->attr.type == PERF_TYPE_RAW)
    555 		event->hw.config |= event->attr.config & X86_RAW_EVENT_MASK;
    556 
    557 	if (event->attr.sample_period && x86_pmu.limit_period) {
    558 		if (x86_pmu.limit_period(event, event->attr.sample_period) >
    559 				event->attr.sample_period)
    560 			return -EINVAL;
    561 	}
    562 
    563 	return x86_setup_perfctr(event);
    564 }
    565 
    566 /*
    567  * Setup the hardware configuration for a given attr_type
    568  */
    569 static int __x86_pmu_event_init(struct perf_event *event)
    570 {
    571 	int err;
    572 
    573 	if (!x86_pmu_initialized())
    574 		return -ENODEV;
    575 
    576 	err = x86_reserve_hardware();
    577 	if (err)
    578 		return err;
    579 
    580 	atomic_inc(&active_events);
    581 	event->destroy = hw_perf_event_destroy;
    582 
    583 	event->hw.idx = -1;
    584 	event->hw.last_cpu = -1;
    585 	event->hw.last_tag = ~0ULL;
    586 
    587 	/* mark unused */
    588 	event->hw.extra_reg.idx = EXTRA_REG_NONE;
    589 	event->hw.branch_reg.idx = EXTRA_REG_NONE;
    590 
    591 	return x86_pmu.hw_config(event);
    592 }
    593 
    594 void x86_pmu_disable_all(void)
    595 {
    596 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
    597 	int idx;
    598 
    599 	for (idx = 0; idx < x86_pmu.num_counters; idx++) {
    600 		u64 val;
    601 
    602 		if (!test_bit(idx, cpuc->active_mask))
    603 			continue;
    604 		rdmsrl(x86_pmu_config_addr(idx), val);
    605 		if (!(val & ARCH_PERFMON_EVENTSEL_ENABLE))
    606 			continue;
    607 		val &= ~ARCH_PERFMON_EVENTSEL_ENABLE;
    608 		wrmsrl(x86_pmu_config_addr(idx), val);
    609 	}
    610 }
    611 
    612 /*
    613  * There may be PMI landing after enabled=0. The PMI hitting could be before or
    614  * after disable_all.
    615  *
    616  * If PMI hits before disable_all, the PMU will be disabled in the NMI handler.
    617  * It will not be re-enabled in the NMI handler again, because enabled=0. After
    618  * handling the NMI, disable_all will be called, which will not change the
    619  * state either. If PMI hits after disable_all, the PMU is already disabled
    620  * before entering NMI handler. The NMI handler will not change the state
    621  * either.
    622  *
    623  * So either situation is harmless.
    624  */
    625 static void x86_pmu_disable(struct pmu *pmu)
    626 {
    627 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
    628 
    629 	if (!x86_pmu_initialized())
    630 		return;
    631 
    632 	if (!cpuc->enabled)
    633 		return;
    634 
    635 	cpuc->n_added = 0;
    636 	cpuc->enabled = 0;
    637 	barrier();
    638 
    639 	x86_pmu.disable_all();
    640 }
    641 
    642 void x86_pmu_enable_all(int added)
    643 {
    644 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
    645 	int idx;
    646 
    647 	for (idx = 0; idx < x86_pmu.num_counters; idx++) {
    648 		struct hw_perf_event *hwc = &cpuc->events[idx]->hw;
    649 
    650 		if (!test_bit(idx, cpuc->active_mask))
    651 			continue;
    652 
    653 		__x86_pmu_enable_event(hwc, ARCH_PERFMON_EVENTSEL_ENABLE);
    654 	}
    655 }
    656 
    657 static struct pmu pmu;
    658 
    659 static inline int is_x86_event(struct perf_event *event)
    660 {
    661 	return event->pmu == &pmu;
    662 }
    663 
    664 /*
    665  * Event scheduler state:
    666  *
    667  * Assign events iterating over all events and counters, beginning
    668  * with events with least weights first. Keep the current iterator
    669  * state in struct sched_state.
    670  */
    671 struct sched_state {
    672 	int	weight;
    673 	int	event;		/* event index */
    674 	int	counter;	/* counter index */
    675 	int	unassigned;	/* number of events to be assigned left */
    676 	int	nr_gp;		/* number of GP counters used */
    677 	unsigned long used[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
    678 };
    679 
    680 /* Total max is X86_PMC_IDX_MAX, but we are O(n!) limited */
    681 #define	SCHED_STATES_MAX	2
    682 
    683 struct perf_sched {
    684 	int			max_weight;
    685 	int			max_events;
    686 	int			max_gp;
    687 	int			saved_states;
    688 	struct event_constraint	**constraints;
    689 	struct sched_state	state;
    690 	struct sched_state	saved[SCHED_STATES_MAX];
    691 };
    692 
    693 /*
    694  * Initialize interator that runs through all events and counters.
    695  */
    696 static void perf_sched_init(struct perf_sched *sched, struct event_constraint **constraints,
    697 			    int num, int wmin, int wmax, int gpmax)
    698 {
    699 	int idx;
    700 
    701 	memset(sched, 0, sizeof(*sched));
    702 	sched->max_events	= num;
    703 	sched->max_weight	= wmax;
    704 	sched->max_gp		= gpmax;
    705 	sched->constraints	= constraints;
    706 
    707 	for (idx = 0; idx < num; idx++) {
    708 		if (constraints[idx]->weight == wmin)
    709 			break;
    710 	}
    711 
    712 	sched->state.event	= idx;		/* start with min weight */
    713 	sched->state.weight	= wmin;
    714 	sched->state.unassigned	= num;
    715 }
    716 
    717 static void perf_sched_save_state(struct perf_sched *sched)
    718 {
    719 	if (WARN_ON_ONCE(sched->saved_states >= SCHED_STATES_MAX))
    720 		return;
    721 
    722 	sched->saved[sched->saved_states] = sched->state;
    723 	sched->saved_states++;
    724 }
    725 
    726 static bool perf_sched_restore_state(struct perf_sched *sched)
    727 {
    728 	if (!sched->saved_states)
    729 		return false;
    730 
    731 	sched->saved_states--;
    732 	sched->state = sched->saved[sched->saved_states];
    733 
    734 	/* continue with next counter: */
    735 	clear_bit(sched->state.counter++, sched->state.used);
    736 
    737 	return true;
    738 }
    739 
    740 /*
    741  * Select a counter for the current event to schedule. Return true on
    742  * success.
    743  */
    744 static bool __perf_sched_find_counter(struct perf_sched *sched)
    745 {
    746 	struct event_constraint *c;
    747 	int idx;
    748 
    749 	if (!sched->state.unassigned)
    750 		return false;
    751 
    752 	if (sched->state.event >= sched->max_events)
    753 		return false;
    754 
    755 	c = sched->constraints[sched->state.event];
    756 	/* Prefer fixed purpose counters */
    757 	if (c->idxmsk64 & (~0ULL << INTEL_PMC_IDX_FIXED)) {
    758 		idx = INTEL_PMC_IDX_FIXED;
    759 		for_each_set_bit_from(idx, c->idxmsk, X86_PMC_IDX_MAX) {
    760 			if (!__test_and_set_bit(idx, sched->state.used))
    761 				goto done;
    762 		}
    763 	}
    764 
    765 	/* Grab the first unused counter starting with idx */
    766 	idx = sched->state.counter;
    767 	for_each_set_bit_from(idx, c->idxmsk, INTEL_PMC_IDX_FIXED) {
    768 		if (!__test_and_set_bit(idx, sched->state.used)) {
    769 			if (sched->state.nr_gp++ >= sched->max_gp)
    770 				return false;
    771 
    772 			goto done;
    773 		}
    774 	}
    775 
    776 	return false;
    777 
    778 done:
    779 	sched->state.counter = idx;
    780 
    781 	if (c->overlap)
    782 		perf_sched_save_state(sched);
    783 
    784 	return true;
    785 }
    786 
    787 static bool perf_sched_find_counter(struct perf_sched *sched)
    788 {
    789 	while (!__perf_sched_find_counter(sched)) {
    790 		if (!perf_sched_restore_state(sched))
    791 			return false;
    792 	}
    793 
    794 	return true;
    795 }
    796 
    797 /*
    798  * Go through all unassigned events and find the next one to schedule.
    799  * Take events with the least weight first. Return true on success.
    800  */
    801 static bool perf_sched_next_event(struct perf_sched *sched)
    802 {
    803 	struct event_constraint *c;
    804 
    805 	if (!sched->state.unassigned || !--sched->state.unassigned)
    806 		return false;
    807 
    808 	do {
    809 		/* next event */
    810 		sched->state.event++;
    811 		if (sched->state.event >= sched->max_events) {
    812 			/* next weight */
    813 			sched->state.event = 0;
    814 			sched->state.weight++;
    815 			if (sched->state.weight > sched->max_weight)
    816 				return false;
    817 		}
    818 		c = sched->constraints[sched->state.event];
    819 	} while (c->weight != sched->state.weight);
    820 
    821 	sched->state.counter = 0;	/* start with first counter */
    822 
    823 	return true;
    824 }
    825 
    826 /*
    827  * Assign a counter for each event.
    828  */
    829 int perf_assign_events(struct event_constraint **constraints, int n,
    830 			int wmin, int wmax, int gpmax, int *assign)
    831 {
    832 	struct perf_sched sched;
    833 
    834 	perf_sched_init(&sched, constraints, n, wmin, wmax, gpmax);
    835 
    836 	do {
    837 		if (!perf_sched_find_counter(&sched))
    838 			break;	/* failed */
    839 		if (assign)
    840 			assign[sched.state.event] = sched.state.counter;
    841 	} while (perf_sched_next_event(&sched));
    842 
    843 	return sched.state.unassigned;
    844 }
    845 EXPORT_SYMBOL_GPL(perf_assign_events);
    846 
    847 int x86_schedule_events(struct cpu_hw_events *cpuc, int n, int *assign)
    848 {
    849 	struct event_constraint *c;
    850 	unsigned long used_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
    851 	struct perf_event *e;
    852 	int i, wmin, wmax, unsched = 0;
    853 	struct hw_perf_event *hwc;
    854 
    855 	bitmap_zero(used_mask, X86_PMC_IDX_MAX);
    856 
    857 	if (x86_pmu.start_scheduling)
    858 		x86_pmu.start_scheduling(cpuc);
    859 
    860 	for (i = 0, wmin = X86_PMC_IDX_MAX, wmax = 0; i < n; i++) {
    861 		cpuc->event_constraint[i] = NULL;
    862 		c = x86_pmu.get_event_constraints(cpuc, i, cpuc->event_list[i]);
    863 		cpuc->event_constraint[i] = c;
    864 
    865 		wmin = min(wmin, c->weight);
    866 		wmax = max(wmax, c->weight);
    867 	}
    868 
    869 	/*
    870 	 * fastpath, try to reuse previous register
    871 	 */
    872 	for (i = 0; i < n; i++) {
    873 		hwc = &cpuc->event_list[i]->hw;
    874 		c = cpuc->event_constraint[i];
    875 
    876 		/* never assigned */
    877 		if (hwc->idx == -1)
    878 			break;
    879 
    880 		/* constraint still honored */
    881 		if (!test_bit(hwc->idx, c->idxmsk))
    882 			break;
    883 
    884 		/* not already used */
    885 		if (test_bit(hwc->idx, used_mask))
    886 			break;
    887 
    888 		__set_bit(hwc->idx, used_mask);
    889 		if (assign)
    890 			assign[i] = hwc->idx;
    891 	}
    892 
    893 	/* slow path */
    894 	if (i != n) {
    895 		int gpmax = x86_pmu.num_counters;
    896 
    897 		/*
    898 		 * Do not allow scheduling of more than half the available
    899 		 * generic counters.
    900 		 *
    901 		 * This helps avoid counter starvation of sibling thread by
    902 		 * ensuring at most half the counters cannot be in exclusive
    903 		 * mode. There is no designated counters for the limits. Any
    904 		 * N/2 counters can be used. This helps with events with
    905 		 * specific counter constraints.
    906 		 */
    907 		if (is_ht_workaround_enabled() && !cpuc->is_fake &&
    908 		    READ_ONCE(cpuc->excl_cntrs->exclusive_present))
    909 			gpmax /= 2;
    910 
    911 		unsched = perf_assign_events(cpuc->event_constraint, n, wmin,
    912 					     wmax, gpmax, assign);
    913 	}
    914 
    915 	/*
    916 	 * In case of success (unsched = 0), mark events as committed,
    917 	 * so we do not put_constraint() in case new events are added
    918 	 * and fail to be scheduled
    919 	 *
    920 	 * We invoke the lower level commit callback to lock the resource
    921 	 *
    922 	 * We do not need to do all of this in case we are called to
    923 	 * validate an event group (assign == NULL)
    924 	 */
    925 	if (!unsched && assign) {
    926 		for (i = 0; i < n; i++) {
    927 			e = cpuc->event_list[i];
    928 			e->hw.flags |= PERF_X86_EVENT_COMMITTED;
    929 			if (x86_pmu.commit_scheduling)
    930 				x86_pmu.commit_scheduling(cpuc, i, assign[i]);
    931 		}
    932 	} else {
    933 		for (i = 0; i < n; i++) {
    934 			e = cpuc->event_list[i];
    935 			/*
    936 			 * do not put_constraint() on comitted events,
    937 			 * because they are good to go
    938 			 */
    939 			if ((e->hw.flags & PERF_X86_EVENT_COMMITTED))
    940 				continue;
    941 
    942 			/*
    943 			 * release events that failed scheduling
    944 			 */
    945 			if (x86_pmu.put_event_constraints)
    946 				x86_pmu.put_event_constraints(cpuc, e);
    947 		}
    948 	}
    949 
    950 	if (x86_pmu.stop_scheduling)
    951 		x86_pmu.stop_scheduling(cpuc);
    952 
    953 	return unsched ? -EINVAL : 0;
    954 }
    955 
    956 /*
    957  * dogrp: true if must collect siblings events (group)
    958  * returns total number of events and error code
    959  */
    960 static int collect_events(struct cpu_hw_events *cpuc, struct perf_event *leader, bool dogrp)
    961 {
    962 	struct perf_event *event;
    963 	int n, max_count;
    964 
    965 	max_count = x86_pmu.num_counters + x86_pmu.num_counters_fixed;
    966 
    967 	/* current number of events already accepted */
    968 	n = cpuc->n_events;
    969 
    970 	if (is_x86_event(leader)) {
    971 		if (n >= max_count)
    972 			return -EINVAL;
    973 		cpuc->event_list[n] = leader;
    974 		n++;
    975 	}
    976 	if (!dogrp)
    977 		return n;
    978 
    979 	for_each_sibling_event(event, leader) {
    980 		if (!is_x86_event(event) ||
    981 		    event->state <= PERF_EVENT_STATE_OFF)
    982 			continue;
    983 
    984 		if (n >= max_count)
    985 			return -EINVAL;
    986 
    987 		cpuc->event_list[n] = event;
    988 		n++;
    989 	}
    990 	return n;
    991 }
    992 
    993 static inline void x86_assign_hw_event(struct perf_event *event,
    994 				struct cpu_hw_events *cpuc, int i)
    995 {
    996 	struct hw_perf_event *hwc = &event->hw;
    997 
    998 	hwc->idx = cpuc->assign[i];
    999 	hwc->last_cpu = smp_processor_id();
   1000 	hwc->last_tag = ++cpuc->tags[i];
   1001 
   1002 	if (hwc->idx == INTEL_PMC_IDX_FIXED_BTS) {
   1003 		hwc->config_base = 0;
   1004 		hwc->event_base	= 0;
   1005 	} else if (hwc->idx >= INTEL_PMC_IDX_FIXED) {
   1006 		hwc->config_base = MSR_ARCH_PERFMON_FIXED_CTR_CTRL;
   1007 		hwc->event_base = MSR_ARCH_PERFMON_FIXED_CTR0 + (hwc->idx - INTEL_PMC_IDX_FIXED);
   1008 		hwc->event_base_rdpmc = (hwc->idx - INTEL_PMC_IDX_FIXED) | 1<<30;
   1009 	} else {
   1010 		hwc->config_base = x86_pmu_config_addr(hwc->idx);
   1011 		hwc->event_base  = x86_pmu_event_addr(hwc->idx);
   1012 		hwc->event_base_rdpmc = x86_pmu_rdpmc_index(hwc->idx);
   1013 	}
   1014 }
   1015 
   1016 /**
   1017  * x86_perf_rdpmc_index - Return PMC counter used for event
   1018  * @event: the perf_event to which the PMC counter was assigned
   1019  *
   1020  * The counter assigned to this performance event may change if interrupts
   1021  * are enabled. This counter should thus never be used while interrupts are
   1022  * enabled. Before this function is used to obtain the assigned counter the
   1023  * event should be checked for validity using, for example,
   1024  * perf_event_read_local(), within the same interrupt disabled section in
   1025  * which this counter is planned to be used.
   1026  *
   1027  * Return: The index of the performance monitoring counter assigned to
   1028  * @perf_event.
   1029  */
   1030 int x86_perf_rdpmc_index(struct perf_event *event)
   1031 {
   1032 	lockdep_assert_irqs_disabled();
   1033 
   1034 	return event->hw.event_base_rdpmc;
   1035 }
   1036 
   1037 static inline int match_prev_assignment(struct hw_perf_event *hwc,
   1038 					struct cpu_hw_events *cpuc,
   1039 					int i)
   1040 {
   1041 	return hwc->idx == cpuc->assign[i] &&
   1042 		hwc->last_cpu == smp_processor_id() &&
   1043 		hwc->last_tag == cpuc->tags[i];
   1044 }
   1045 
   1046 static void x86_pmu_start(struct perf_event *event, int flags);
   1047 
   1048 static void x86_pmu_enable(struct pmu *pmu)
   1049 {
   1050 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
   1051 	struct perf_event *event;
   1052 	struct hw_perf_event *hwc;
   1053 	int i, added = cpuc->n_added;
   1054 
   1055 	if (!x86_pmu_initialized())
   1056 		return;
   1057 
   1058 	if (cpuc->enabled)
   1059 		return;
   1060 
   1061 	if (cpuc->n_added) {
   1062 		int n_running = cpuc->n_events - cpuc->n_added;
   1063 		/*
   1064 		 * apply assignment obtained either from
   1065 		 * hw_perf_group_sched_in() or x86_pmu_enable()
   1066 		 *
   1067 		 * step1: save events moving to new counters
   1068 		 */
   1069 		for (i = 0; i < n_running; i++) {
   1070 			event = cpuc->event_list[i];
   1071 			hwc = &event->hw;
   1072 
   1073 			/*
   1074 			 * we can avoid reprogramming counter if:
   1075 			 * - assigned same counter as last time
   1076 			 * - running on same CPU as last time
   1077 			 * - no other event has used the counter since
   1078 			 */
   1079 			if (hwc->idx == -1 ||
   1080 			    match_prev_assignment(hwc, cpuc, i))
   1081 				continue;
   1082 
   1083 			/*
   1084 			 * Ensure we don't accidentally enable a stopped
   1085 			 * counter simply because we rescheduled.
   1086 			 */
   1087 			if (hwc->state & PERF_HES_STOPPED)
   1088 				hwc->state |= PERF_HES_ARCH;
   1089 
   1090 			x86_pmu_stop(event, PERF_EF_UPDATE);
   1091 		}
   1092 
   1093 		/*
   1094 		 * step2: reprogram moved events into new counters
   1095 		 */
   1096 		for (i = 0; i < cpuc->n_events; i++) {
   1097 			event = cpuc->event_list[i];
   1098 			hwc = &event->hw;
   1099 
   1100 			if (!match_prev_assignment(hwc, cpuc, i))
   1101 				x86_assign_hw_event(event, cpuc, i);
   1102 			else if (i < n_running)
   1103 				continue;
   1104 
   1105 			if (hwc->state & PERF_HES_ARCH)
   1106 				continue;
   1107 
   1108 			x86_pmu_start(event, PERF_EF_RELOAD);
   1109 		}
   1110 		cpuc->n_added = 0;
   1111 		perf_events_lapic_init();
   1112 	}
   1113 
   1114 	cpuc->enabled = 1;
   1115 	barrier();
   1116 
   1117 	x86_pmu.enable_all(added);
   1118 }
   1119 
   1120 static DEFINE_PER_CPU(u64 [X86_PMC_IDX_MAX], pmc_prev_left);
   1121 
   1122 /*
   1123  * Set the next IRQ period, based on the hwc->period_left value.
   1124  * To be called with the event disabled in hw:
   1125  */
   1126 int x86_perf_event_set_period(struct perf_event *event)
   1127 {
   1128 	struct hw_perf_event *hwc = &event->hw;
   1129 	s64 left = local64_read(&hwc->period_left);
   1130 	s64 period = hwc->sample_period;
   1131 	int ret = 0, idx = hwc->idx;
   1132 
   1133 	if (idx == INTEL_PMC_IDX_FIXED_BTS)
   1134 		return 0;
   1135 
   1136 	/*
   1137 	 * If we are way outside a reasonable range then just skip forward:
   1138 	 */
   1139 	if (unlikely(left <= -period)) {
   1140 		left = period;
   1141 		local64_set(&hwc->period_left, left);
   1142 		hwc->last_period = period;
   1143 		ret = 1;
   1144 	}
   1145 
   1146 	if (unlikely(left <= 0)) {
   1147 		left += period;
   1148 		local64_set(&hwc->period_left, left);
   1149 		hwc->last_period = period;
   1150 		ret = 1;
   1151 	}
   1152 	/*
   1153 	 * Quirk: certain CPUs dont like it if just 1 hw_event is left:
   1154 	 */
   1155 	if (unlikely(left < 2))
   1156 		left = 2;
   1157 
   1158 	if (left > x86_pmu.max_period)
   1159 		left = x86_pmu.max_period;
   1160 
   1161 	if (x86_pmu.limit_period)
   1162 		left = x86_pmu.limit_period(event, left);
   1163 
   1164 	per_cpu(pmc_prev_left[idx], smp_processor_id()) = left;
   1165 
   1166 	/*
   1167 	 * The hw event starts counting from this event offset,
   1168 	 * mark it to be able to extra future deltas:
   1169 	 */
   1170 	local64_set(&hwc->prev_count, (u64)-left);
   1171 
   1172 	wrmsrl(hwc->event_base, (u64)(-left) & x86_pmu.cntval_mask);
   1173 
   1174 	/*
   1175 	 * Due to erratum on certan cpu we need
   1176 	 * a second write to be sure the register
   1177 	 * is updated properly
   1178 	 */
   1179 	if (x86_pmu.perfctr_second_write) {
   1180 		wrmsrl(hwc->event_base,
   1181 			(u64)(-left) & x86_pmu.cntval_mask);
   1182 	}
   1183 
   1184 	perf_event_update_userpage(event);
   1185 
   1186 	return ret;
   1187 }
   1188 
   1189 void x86_pmu_enable_event(struct perf_event *event)
   1190 {
   1191 	if (__this_cpu_read(cpu_hw_events.enabled))
   1192 		__x86_pmu_enable_event(&event->hw,
   1193 				       ARCH_PERFMON_EVENTSEL_ENABLE);
   1194 }
   1195 
   1196 /*
   1197  * Add a single event to the PMU.
   1198  *
   1199  * The event is added to the group of enabled events
   1200  * but only if it can be scehduled with existing events.
   1201  */
   1202 static int x86_pmu_add(struct perf_event *event, int flags)
   1203 {
   1204 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
   1205 	struct hw_perf_event *hwc;
   1206 	int assign[X86_PMC_IDX_MAX];
   1207 	int n, n0, ret;
   1208 
   1209 	hwc = &event->hw;
   1210 
   1211 	n0 = cpuc->n_events;
   1212 	ret = n = collect_events(cpuc, event, false);
   1213 	if (ret < 0)
   1214 		goto out;
   1215 
   1216 	hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED;
   1217 	if (!(flags & PERF_EF_START))
   1218 		hwc->state |= PERF_HES_ARCH;
   1219 
   1220 	/*
   1221 	 * If group events scheduling transaction was started,
   1222 	 * skip the schedulability test here, it will be performed
   1223 	 * at commit time (->commit_txn) as a whole.
   1224 	 *
   1225 	 * If commit fails, we'll call ->del() on all events
   1226 	 * for which ->add() was called.
   1227 	 */
   1228 	if (cpuc->txn_flags & PERF_PMU_TXN_ADD)
   1229 		goto done_collect;
   1230 
   1231 	ret = x86_pmu.schedule_events(cpuc, n, assign);
   1232 	if (ret)
   1233 		goto out;
   1234 	/*
   1235 	 * copy new assignment, now we know it is possible
   1236 	 * will be used by hw_perf_enable()
   1237 	 */
   1238 	memcpy(cpuc->assign, assign, n*sizeof(int));
   1239 
   1240 done_collect:
   1241 	/*
   1242 	 * Commit the collect_events() state. See x86_pmu_del() and
   1243 	 * x86_pmu_*_txn().
   1244 	 */
   1245 	cpuc->n_events = n;
   1246 	cpuc->n_added += n - n0;
   1247 	cpuc->n_txn += n - n0;
   1248 
   1249 	if (x86_pmu.add) {
   1250 		/*
   1251 		 * This is before x86_pmu_enable() will call x86_pmu_start(),
   1252 		 * so we enable LBRs before an event needs them etc..
   1253 		 */
   1254 		x86_pmu.add(event);
   1255 	}
   1256 
   1257 	ret = 0;
   1258 out:
   1259 	return ret;
   1260 }
   1261 
   1262 static void x86_pmu_start(struct perf_event *event, int flags)
   1263 {
   1264 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
   1265 	int idx = event->hw.idx;
   1266 
   1267 	if (WARN_ON_ONCE(!(event->hw.state & PERF_HES_STOPPED)))
   1268 		return;
   1269 
   1270 	if (WARN_ON_ONCE(idx == -1))
   1271 		return;
   1272 
   1273 	if (flags & PERF_EF_RELOAD) {
   1274 		WARN_ON_ONCE(!(event->hw.state & PERF_HES_UPTODATE));
   1275 		x86_perf_event_set_period(event);
   1276 	}
   1277 
   1278 	event->hw.state = 0;
   1279 
   1280 	cpuc->events[idx] = event;
   1281 	__set_bit(idx, cpuc->active_mask);
   1282 	__set_bit(idx, cpuc->running);
   1283 	x86_pmu.enable(event);
   1284 	perf_event_update_userpage(event);
   1285 }
   1286 
   1287 void perf_event_print_debug(void)
   1288 {
   1289 	u64 ctrl, status, overflow, pmc_ctrl, pmc_count, prev_left, fixed;
   1290 	u64 pebs, debugctl;
   1291 	struct cpu_hw_events *cpuc;
   1292 	unsigned long flags;
   1293 	int cpu, idx;
   1294 
   1295 	if (!x86_pmu.num_counters)
   1296 		return;
   1297 
   1298 	local_irq_save(flags);
   1299 
   1300 	cpu = smp_processor_id();
   1301 	cpuc = &per_cpu(cpu_hw_events, cpu);
   1302 
   1303 	if (x86_pmu.version >= 2) {
   1304 		rdmsrl(MSR_CORE_PERF_GLOBAL_CTRL, ctrl);
   1305 		rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status);
   1306 		rdmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, overflow);
   1307 		rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR_CTRL, fixed);
   1308 
   1309 		pr_info("\n");
   1310 		pr_info("CPU#%d: ctrl:       %016llx\n", cpu, ctrl);
   1311 		pr_info("CPU#%d: status:     %016llx\n", cpu, status);
   1312 		pr_info("CPU#%d: overflow:   %016llx\n", cpu, overflow);
   1313 		pr_info("CPU#%d: fixed:      %016llx\n", cpu, fixed);
   1314 		if (x86_pmu.pebs_constraints) {
   1315 			rdmsrl(MSR_IA32_PEBS_ENABLE, pebs);
   1316 			pr_info("CPU#%d: pebs:       %016llx\n", cpu, pebs);
   1317 		}
   1318 		if (x86_pmu.lbr_nr) {
   1319 			rdmsrl(MSR_IA32_DEBUGCTLMSR, debugctl);
   1320 			pr_info("CPU#%d: debugctl:   %016llx\n", cpu, debugctl);
   1321 		}
   1322 	}
   1323 	pr_info("CPU#%d: active:     %016llx\n", cpu, *(u64 *)cpuc->active_mask);
   1324 
   1325 	for (idx = 0; idx < x86_pmu.num_counters; idx++) {
   1326 		rdmsrl(x86_pmu_config_addr(idx), pmc_ctrl);
   1327 		rdmsrl(x86_pmu_event_addr(idx), pmc_count);
   1328 
   1329 		prev_left = per_cpu(pmc_prev_left[idx], cpu);
   1330 
   1331 		pr_info("CPU#%d:   gen-PMC%d ctrl:  %016llx\n",
   1332 			cpu, idx, pmc_ctrl);
   1333 		pr_info("CPU#%d:   gen-PMC%d count: %016llx\n",
   1334 			cpu, idx, pmc_count);
   1335 		pr_info("CPU#%d:   gen-PMC%d left:  %016llx\n",
   1336 			cpu, idx, prev_left);
   1337 	}
   1338 	for (idx = 0; idx < x86_pmu.num_counters_fixed; idx++) {
   1339 		rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR0 + idx, pmc_count);
   1340 
   1341 		pr_info("CPU#%d: fixed-PMC%d count: %016llx\n",
   1342 			cpu, idx, pmc_count);
   1343 	}
   1344 	local_irq_restore(flags);
   1345 }
   1346 
   1347 void x86_pmu_stop(struct perf_event *event, int flags)
   1348 {
   1349 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
   1350 	struct hw_perf_event *hwc = &event->hw;
   1351 
   1352 	if (test_bit(hwc->idx, cpuc->active_mask)) {
   1353 		x86_pmu.disable(event);
   1354 		__clear_bit(hwc->idx, cpuc->active_mask);
   1355 		cpuc->events[hwc->idx] = NULL;
   1356 		WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED);
   1357 		hwc->state |= PERF_HES_STOPPED;
   1358 	}
   1359 
   1360 	if ((flags & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE)) {
   1361 		/*
   1362 		 * Drain the remaining delta count out of a event
   1363 		 * that we are disabling:
   1364 		 */
   1365 		x86_perf_event_update(event);
   1366 		hwc->state |= PERF_HES_UPTODATE;
   1367 	}
   1368 }
   1369 
   1370 static void x86_pmu_del(struct perf_event *event, int flags)
   1371 {
   1372 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
   1373 	int i;
   1374 
   1375 	/*
   1376 	 * event is descheduled
   1377 	 */
   1378 	event->hw.flags &= ~PERF_X86_EVENT_COMMITTED;
   1379 
   1380 	/*
   1381 	 * If we're called during a txn, we only need to undo x86_pmu.add.
   1382 	 * The events never got scheduled and ->cancel_txn will truncate
   1383 	 * the event_list.
   1384 	 *
   1385 	 * XXX assumes any ->del() called during a TXN will only be on
   1386 	 * an event added during that same TXN.
   1387 	 */
   1388 	if (cpuc->txn_flags & PERF_PMU_TXN_ADD)
   1389 		goto do_del;
   1390 
   1391 	/*
   1392 	 * Not a TXN, therefore cleanup properly.
   1393 	 */
   1394 	x86_pmu_stop(event, PERF_EF_UPDATE);
   1395 
   1396 	for (i = 0; i < cpuc->n_events; i++) {
   1397 		if (event == cpuc->event_list[i])
   1398 			break;
   1399 	}
   1400 
   1401 	if (WARN_ON_ONCE(i == cpuc->n_events)) /* called ->del() without ->add() ? */
   1402 		return;
   1403 
   1404 	/* If we have a newly added event; make sure to decrease n_added. */
   1405 	if (i >= cpuc->n_events - cpuc->n_added)
   1406 		--cpuc->n_added;
   1407 
   1408 	if (x86_pmu.put_event_constraints)
   1409 		x86_pmu.put_event_constraints(cpuc, event);
   1410 
   1411 	/* Delete the array entry. */
   1412 	while (++i < cpuc->n_events) {
   1413 		cpuc->event_list[i-1] = cpuc->event_list[i];
   1414 		cpuc->event_constraint[i-1] = cpuc->event_constraint[i];
   1415 	}
   1416 	--cpuc->n_events;
   1417 
   1418 	perf_event_update_userpage(event);
   1419 
   1420 do_del:
   1421 	if (x86_pmu.del) {
   1422 		/*
   1423 		 * This is after x86_pmu_stop(); so we disable LBRs after any
   1424 		 * event can need them etc..
   1425 		 */
   1426 		x86_pmu.del(event);
   1427 	}
   1428 }
   1429 
   1430 int x86_pmu_handle_irq(struct pt_regs *regs)
   1431 {
   1432 	struct perf_sample_data data;
   1433 	struct cpu_hw_events *cpuc;
   1434 	struct perf_event *event;
   1435 	int idx, handled = 0;
   1436 	u64 val;
   1437 
   1438 	cpuc = this_cpu_ptr(&cpu_hw_events);
   1439 
   1440 	/*
   1441 	 * Some chipsets need to unmask the LVTPC in a particular spot
   1442 	 * inside the nmi handler.  As a result, the unmasking was pushed
   1443 	 * into all the nmi handlers.
   1444 	 *
   1445 	 * This generic handler doesn't seem to have any issues where the
   1446 	 * unmasking occurs so it was left at the top.
   1447 	 */
   1448 	apic_write(APIC_LVTPC, APIC_DM_NMI);
   1449 
   1450 	for (idx = 0; idx < x86_pmu.num_counters; idx++) {
   1451 		if (!test_bit(idx, cpuc->active_mask))
   1452 			continue;
   1453 
   1454 		event = cpuc->events[idx];
   1455 
   1456 		val = x86_perf_event_update(event);
   1457 		if (val & (1ULL << (x86_pmu.cntval_bits - 1)))
   1458 			continue;
   1459 
   1460 		/*
   1461 		 * event overflow
   1462 		 */
   1463 		handled++;
   1464 		perf_sample_data_init(&data, 0, event->hw.last_period);
   1465 
   1466 		if (!x86_perf_event_set_period(event))
   1467 			continue;
   1468 
   1469 		if (perf_event_overflow(event, &data, regs))
   1470 			x86_pmu_stop(event, 0);
   1471 	}
   1472 
   1473 	if (handled)
   1474 		inc_irq_stat(apic_perf_irqs);
   1475 
   1476 	return handled;
   1477 }
   1478 
   1479 void perf_events_lapic_init(void)
   1480 {
   1481 	if (!x86_pmu.apic || !x86_pmu_initialized())
   1482 		return;
   1483 
   1484 	/*
   1485 	 * Always use NMI for PMU
   1486 	 */
   1487 	apic_write(APIC_LVTPC, APIC_DM_NMI);
   1488 }
   1489 
   1490 static int
   1491 perf_event_nmi_handler(unsigned int cmd, struct pt_regs *regs)
   1492 {
   1493 	u64 start_clock;
   1494 	u64 finish_clock;
   1495 	int ret;
   1496 
   1497 	/*
   1498 	 * All PMUs/events that share this PMI handler should make sure to
   1499 	 * increment active_events for their events.
   1500 	 */
   1501 	if (!atomic_read(&active_events))
   1502 		return NMI_DONE;
   1503 
   1504 	start_clock = sched_clock();
   1505 	ret = x86_pmu.handle_irq(regs);
   1506 	finish_clock = sched_clock();
   1507 
   1508 	perf_sample_event_took(finish_clock - start_clock);
   1509 
   1510 	return ret;
   1511 }
   1512 NOKPROBE_SYMBOL(perf_event_nmi_handler);
   1513 
   1514 struct event_constraint emptyconstraint;
   1515 struct event_constraint unconstrained;
   1516 
   1517 static int x86_pmu_prepare_cpu(unsigned int cpu)
   1518 {
   1519 	struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu);
   1520 	int i;
   1521 
   1522 	for (i = 0 ; i < X86_PERF_KFREE_MAX; i++)
   1523 		cpuc->kfree_on_online[i] = NULL;
   1524 	if (x86_pmu.cpu_prepare)
   1525 		return x86_pmu.cpu_prepare(cpu);
   1526 	return 0;
   1527 }
   1528 
   1529 static int x86_pmu_dead_cpu(unsigned int cpu)
   1530 {
   1531 	if (x86_pmu.cpu_dead)
   1532 		x86_pmu.cpu_dead(cpu);
   1533 	return 0;
   1534 }
   1535 
   1536 static int x86_pmu_online_cpu(unsigned int cpu)
   1537 {
   1538 	struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu);
   1539 	int i;
   1540 
   1541 	for (i = 0 ; i < X86_PERF_KFREE_MAX; i++) {
   1542 		kfree(cpuc->kfree_on_online[i]);
   1543 		cpuc->kfree_on_online[i] = NULL;
   1544 	}
   1545 	return 0;
   1546 }
   1547 
   1548 static int x86_pmu_starting_cpu(unsigned int cpu)
   1549 {
   1550 	if (x86_pmu.cpu_starting)
   1551 		x86_pmu.cpu_starting(cpu);
   1552 	return 0;
   1553 }
   1554 
   1555 static int x86_pmu_dying_cpu(unsigned int cpu)
   1556 {
   1557 	if (x86_pmu.cpu_dying)
   1558 		x86_pmu.cpu_dying(cpu);
   1559 	return 0;
   1560 }
   1561 
   1562 static void __init pmu_check_apic(void)
   1563 {
   1564 	if (boot_cpu_has(X86_FEATURE_APIC))
   1565 		return;
   1566 
   1567 	x86_pmu.apic = 0;
   1568 	pr_info("no APIC, boot with the \"lapic\" boot parameter to force-enable it.\n");
   1569 	pr_info("no hardware sampling interrupt available.\n");
   1570 
   1571 	/*
   1572 	 * If we have a PMU initialized but no APIC
   1573 	 * interrupts, we cannot sample hardware
   1574 	 * events (user-space has to fall back and
   1575 	 * sample via a hrtimer based software event):
   1576 	 */
   1577 	pmu.capabilities |= PERF_PMU_CAP_NO_INTERRUPT;
   1578 
   1579 }
   1580 
   1581 static struct attribute_group x86_pmu_format_group __ro_after_init = {
   1582 	.name = "format",
   1583 	.attrs = NULL,
   1584 };
   1585 
   1586 /*
   1587  * Remove all undefined events (x86_pmu.event_map(id) == 0)
   1588  * out of events_attr attributes.
   1589  */
   1590 static void __init filter_events(struct attribute **attrs)
   1591 {
   1592 	struct device_attribute *d;
   1593 	struct perf_pmu_events_attr *pmu_attr;
   1594 	int offset = 0;
   1595 	int i, j;
   1596 
   1597 	for (i = 0; attrs[i]; i++) {
   1598 		d = (struct device_attribute *)attrs[i];
   1599 		pmu_attr = container_of(d, struct perf_pmu_events_attr, attr);
   1600 		/* str trumps id */
   1601 		if (pmu_attr->event_str)
   1602 			continue;
   1603 		if (x86_pmu.event_map(i + offset))
   1604 			continue;
   1605 
   1606 		for (j = i; attrs[j]; j++)
   1607 			attrs[j] = attrs[j + 1];
   1608 
   1609 		/* Check the shifted attr. */
   1610 		i--;
   1611 
   1612 		/*
   1613 		 * event_map() is index based, the attrs array is organized
   1614 		 * by increasing event index. If we shift the events, then
   1615 		 * we need to compensate for the event_map(), otherwise
   1616 		 * we are looking up the wrong event in the map
   1617 		 */
   1618 		offset++;
   1619 	}
   1620 }
   1621 
   1622 /* Merge two pointer arrays */
   1623 __init struct attribute **merge_attr(struct attribute **a, struct attribute **b)
   1624 {
   1625 	struct attribute **new;
   1626 	int j, i;
   1627 
   1628 	for (j = 0; a && a[j]; j++)
   1629 		;
   1630 	for (i = 0; b && b[i]; i++)
   1631 		j++;
   1632 	j++;
   1633 
   1634 	new = kmalloc_array(j, sizeof(struct attribute *), GFP_KERNEL);
   1635 	if (!new)
   1636 		return NULL;
   1637 
   1638 	j = 0;
   1639 	for (i = 0; a && a[i]; i++)
   1640 		new[j++] = a[i];
   1641 	for (i = 0; b && b[i]; i++)
   1642 		new[j++] = b[i];
   1643 	new[j] = NULL;
   1644 
   1645 	return new;
   1646 }
   1647 
   1648 ssize_t events_sysfs_show(struct device *dev, struct device_attribute *attr, char *page)
   1649 {
   1650 	struct perf_pmu_events_attr *pmu_attr = \
   1651 		container_of(attr, struct perf_pmu_events_attr, attr);
   1652 	u64 config = x86_pmu.event_map(pmu_attr->id);
   1653 
   1654 	/* string trumps id */
   1655 	if (pmu_attr->event_str)
   1656 		return sprintf(page, "%s", pmu_attr->event_str);
   1657 
   1658 	return x86_pmu.events_sysfs_show(page, config);
   1659 }
   1660 EXPORT_SYMBOL_GPL(events_sysfs_show);
   1661 
   1662 ssize_t events_ht_sysfs_show(struct device *dev, struct device_attribute *attr,
   1663 			  char *page)
   1664 {
   1665 	struct perf_pmu_events_ht_attr *pmu_attr =
   1666 		container_of(attr, struct perf_pmu_events_ht_attr, attr);
   1667 
   1668 	/*
   1669 	 * Report conditional events depending on Hyper-Threading.
   1670 	 *
   1671 	 * This is overly conservative as usually the HT special
   1672 	 * handling is not needed if the other CPU thread is idle.
   1673 	 *
   1674 	 * Note this does not (and cannot) handle the case when thread
   1675 	 * siblings are invisible, for example with virtualization
   1676 	 * if they are owned by some other guest.  The user tool
   1677 	 * has to re-read when a thread sibling gets onlined later.
   1678 	 */
   1679 	return sprintf(page, "%s",
   1680 			topology_max_smt_threads() > 1 ?
   1681 			pmu_attr->event_str_ht :
   1682 			pmu_attr->event_str_noht);
   1683 }
   1684 
   1685 EVENT_ATTR(cpu-cycles,			CPU_CYCLES		);
   1686 EVENT_ATTR(instructions,		INSTRUCTIONS		);
   1687 EVENT_ATTR(cache-references,		CACHE_REFERENCES	);
   1688 EVENT_ATTR(cache-misses, 		CACHE_MISSES		);
   1689 EVENT_ATTR(branch-instructions,		BRANCH_INSTRUCTIONS	);
   1690 EVENT_ATTR(branch-misses,		BRANCH_MISSES		);
   1691 EVENT_ATTR(bus-cycles,			BUS_CYCLES		);
   1692 EVENT_ATTR(stalled-cycles-frontend,	STALLED_CYCLES_FRONTEND	);
   1693 EVENT_ATTR(stalled-cycles-backend,	STALLED_CYCLES_BACKEND	);
   1694 EVENT_ATTR(ref-cycles,			REF_CPU_CYCLES		);
   1695 
   1696 static struct attribute *empty_attrs;
   1697 
   1698 static struct attribute *events_attr[] = {
   1699 	EVENT_PTR(CPU_CYCLES),
   1700 	EVENT_PTR(INSTRUCTIONS),
   1701 	EVENT_PTR(CACHE_REFERENCES),
   1702 	EVENT_PTR(CACHE_MISSES),
   1703 	EVENT_PTR(BRANCH_INSTRUCTIONS),
   1704 	EVENT_PTR(BRANCH_MISSES),
   1705 	EVENT_PTR(BUS_CYCLES),
   1706 	EVENT_PTR(STALLED_CYCLES_FRONTEND),
   1707 	EVENT_PTR(STALLED_CYCLES_BACKEND),
   1708 	EVENT_PTR(REF_CPU_CYCLES),
   1709 	NULL,
   1710 };
   1711 
   1712 static struct attribute_group x86_pmu_events_group __ro_after_init = {
   1713 	.name = "events",
   1714 	.attrs = events_attr,
   1715 };
   1716 
   1717 ssize_t x86_event_sysfs_show(char *page, u64 config, u64 event)
   1718 {
   1719 	u64 umask  = (config & ARCH_PERFMON_EVENTSEL_UMASK) >> 8;
   1720 	u64 cmask  = (config & ARCH_PERFMON_EVENTSEL_CMASK) >> 24;
   1721 	bool edge  = (config & ARCH_PERFMON_EVENTSEL_EDGE);
   1722 	bool pc    = (config & ARCH_PERFMON_EVENTSEL_PIN_CONTROL);
   1723 	bool any   = (config & ARCH_PERFMON_EVENTSEL_ANY);
   1724 	bool inv   = (config & ARCH_PERFMON_EVENTSEL_INV);
   1725 	ssize_t ret;
   1726 
   1727 	/*
   1728 	* We have whole page size to spend and just little data
   1729 	* to write, so we can safely use sprintf.
   1730 	*/
   1731 	ret = sprintf(page, "event=0x%02llx", event);
   1732 
   1733 	if (umask)
   1734 		ret += sprintf(page + ret, ",umask=0x%02llx", umask);
   1735 
   1736 	if (edge)
   1737 		ret += sprintf(page + ret, ",edge");
   1738 
   1739 	if (pc)
   1740 		ret += sprintf(page + ret, ",pc");
   1741 
   1742 	if (any)
   1743 		ret += sprintf(page + ret, ",any");
   1744 
   1745 	if (inv)
   1746 		ret += sprintf(page + ret, ",inv");
   1747 
   1748 	if (cmask)
   1749 		ret += sprintf(page + ret, ",cmask=0x%02llx", cmask);
   1750 
   1751 	ret += sprintf(page + ret, "\n");
   1752 
   1753 	return ret;
   1754 }
   1755 
   1756 static struct attribute_group x86_pmu_attr_group;
   1757 static struct attribute_group x86_pmu_caps_group;
   1758 
   1759 static int __init init_hw_perf_events(void)
   1760 {
   1761 	struct x86_pmu_quirk *quirk;
   1762 	int err;
   1763 
   1764 	pr_info("Performance Events: ");
   1765 
   1766 	switch (boot_cpu_data.x86_vendor) {
   1767 	case X86_VENDOR_INTEL:
   1768 		err = intel_pmu_init();
   1769 		break;
   1770 	case X86_VENDOR_AMD:
   1771 		err = amd_pmu_init();
   1772 		break;
   1773 	case X86_VENDOR_HYGON:
   1774 		err = amd_pmu_init();
   1775 		x86_pmu.name = "HYGON";
   1776 		break;
   1777 	default:
   1778 		err = -ENOTSUPP;
   1779 	}
   1780 	if (err != 0) {
   1781 		pr_cont("no PMU driver, software events only.\n");
   1782 		return 0;
   1783 	}
   1784 
   1785 	pmu_check_apic();
   1786 
   1787 	/* sanity check that the hardware exists or is emulated */
   1788 	if (!check_hw_exists())
   1789 		return 0;
   1790 
   1791 	pr_cont("%s PMU driver.\n", x86_pmu.name);
   1792 
   1793 	x86_pmu.attr_rdpmc = 1; /* enable userspace RDPMC usage by default */
   1794 
   1795 	for (quirk = x86_pmu.quirks; quirk; quirk = quirk->next)
   1796 		quirk->func();
   1797 
   1798 	if (!x86_pmu.intel_ctrl)
   1799 		x86_pmu.intel_ctrl = (1 << x86_pmu.num_counters) - 1;
   1800 
   1801 	perf_events_lapic_init();
   1802 	register_nmi_handler(NMI_LOCAL, perf_event_nmi_handler, 0, "PMI");
   1803 
   1804 	unconstrained = (struct event_constraint)
   1805 		__EVENT_CONSTRAINT(0, (1ULL << x86_pmu.num_counters) - 1,
   1806 				   0, x86_pmu.num_counters, 0, 0);
   1807 
   1808 	x86_pmu_format_group.attrs = x86_pmu.format_attrs;
   1809 
   1810 	if (x86_pmu.caps_attrs) {
   1811 		struct attribute **tmp;
   1812 
   1813 		tmp = merge_attr(x86_pmu_caps_group.attrs, x86_pmu.caps_attrs);
   1814 		if (!WARN_ON(!tmp))
   1815 			x86_pmu_caps_group.attrs = tmp;
   1816 	}
   1817 
   1818 	if (x86_pmu.event_attrs)
   1819 		x86_pmu_events_group.attrs = x86_pmu.event_attrs;
   1820 
   1821 	if (!x86_pmu.events_sysfs_show)
   1822 		x86_pmu_events_group.attrs = &empty_attrs;
   1823 	else
   1824 		filter_events(x86_pmu_events_group.attrs);
   1825 
   1826 	if (x86_pmu.cpu_events) {
   1827 		struct attribute **tmp;
   1828 
   1829 		tmp = merge_attr(x86_pmu_events_group.attrs, x86_pmu.cpu_events);
   1830 		if (!WARN_ON(!tmp))
   1831 			x86_pmu_events_group.attrs = tmp;
   1832 	}
   1833 
   1834 	if (x86_pmu.attrs) {
   1835 		struct attribute **tmp;
   1836 
   1837 		tmp = merge_attr(x86_pmu_attr_group.attrs, x86_pmu.attrs);
   1838 		if (!WARN_ON(!tmp))
   1839 			x86_pmu_attr_group.attrs = tmp;
   1840 	}
   1841 
   1842 	pr_info("... version:                %d\n",     x86_pmu.version);
   1843 	pr_info("... bit width:              %d\n",     x86_pmu.cntval_bits);
   1844 	pr_info("... generic registers:      %d\n",     x86_pmu.num_counters);
   1845 	pr_info("... value mask:             %016Lx\n", x86_pmu.cntval_mask);
   1846 	pr_info("... max period:             %016Lx\n", x86_pmu.max_period);
   1847 	pr_info("... fixed-purpose events:   %d\n",     x86_pmu.num_counters_fixed);
   1848 	pr_info("... event mask:             %016Lx\n", x86_pmu.intel_ctrl);
   1849 
   1850 	/*
   1851 	 * Install callbacks. Core will call them for each online
   1852 	 * cpu.
   1853 	 */
   1854 	err = cpuhp_setup_state(CPUHP_PERF_X86_PREPARE, "perf/x86:prepare",
   1855 				x86_pmu_prepare_cpu, x86_pmu_dead_cpu);
   1856 	if (err)
   1857 		return err;
   1858 
   1859 	err = cpuhp_setup_state(CPUHP_AP_PERF_X86_STARTING,
   1860 				"perf/x86:starting", x86_pmu_starting_cpu,
   1861 				x86_pmu_dying_cpu);
   1862 	if (err)
   1863 		goto out;
   1864 
   1865 	err = cpuhp_setup_state(CPUHP_AP_PERF_X86_ONLINE, "perf/x86:online",
   1866 				x86_pmu_online_cpu, NULL);
   1867 	if (err)
   1868 		goto out1;
   1869 
   1870 	err = perf_pmu_register(&pmu, "cpu", PERF_TYPE_RAW);
   1871 	if (err)
   1872 		goto out2;
   1873 
   1874 	return 0;
   1875 
   1876 out2:
   1877 	cpuhp_remove_state(CPUHP_AP_PERF_X86_ONLINE);
   1878 out1:
   1879 	cpuhp_remove_state(CPUHP_AP_PERF_X86_STARTING);
   1880 out:
   1881 	cpuhp_remove_state(CPUHP_PERF_X86_PREPARE);
   1882 	return err;
   1883 }
   1884 early_initcall(init_hw_perf_events);
   1885 
   1886 static inline void x86_pmu_read(struct perf_event *event)
   1887 {
   1888 	if (x86_pmu.read)
   1889 		return x86_pmu.read(event);
   1890 	x86_perf_event_update(event);
   1891 }
   1892 
   1893 /*
   1894  * Start group events scheduling transaction
   1895  * Set the flag to make pmu::enable() not perform the
   1896  * schedulability test, it will be performed at commit time
   1897  *
   1898  * We only support PERF_PMU_TXN_ADD transactions. Save the
   1899  * transaction flags but otherwise ignore non-PERF_PMU_TXN_ADD
   1900  * transactions.
   1901  */
   1902 static void x86_pmu_start_txn(struct pmu *pmu, unsigned int txn_flags)
   1903 {
   1904 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
   1905 
   1906 	WARN_ON_ONCE(cpuc->txn_flags);		/* txn already in flight */
   1907 
   1908 	cpuc->txn_flags = txn_flags;
   1909 	if (txn_flags & ~PERF_PMU_TXN_ADD)
   1910 		return;
   1911 
   1912 	perf_pmu_disable(pmu);
   1913 	__this_cpu_write(cpu_hw_events.n_txn, 0);
   1914 }
   1915 
   1916 /*
   1917  * Stop group events scheduling transaction
   1918  * Clear the flag and pmu::enable() will perform the
   1919  * schedulability test.
   1920  */
   1921 static void x86_pmu_cancel_txn(struct pmu *pmu)
   1922 {
   1923 	unsigned int txn_flags;
   1924 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
   1925 
   1926 	WARN_ON_ONCE(!cpuc->txn_flags);	/* no txn in flight */
   1927 
   1928 	txn_flags = cpuc->txn_flags;
   1929 	cpuc->txn_flags = 0;
   1930 	if (txn_flags & ~PERF_PMU_TXN_ADD)
   1931 		return;
   1932 
   1933 	/*
   1934 	 * Truncate collected array by the number of events added in this
   1935 	 * transaction. See x86_pmu_add() and x86_pmu_*_txn().
   1936 	 */
   1937 	__this_cpu_sub(cpu_hw_events.n_added, __this_cpu_read(cpu_hw_events.n_txn));
   1938 	__this_cpu_sub(cpu_hw_events.n_events, __this_cpu_read(cpu_hw_events.n_txn));
   1939 	perf_pmu_enable(pmu);
   1940 }
   1941 
   1942 /*
   1943  * Commit group events scheduling transaction
   1944  * Perform the group schedulability test as a whole
   1945  * Return 0 if success
   1946  *
   1947  * Does not cancel the transaction on failure; expects the caller to do this.
   1948  */
   1949 static int x86_pmu_commit_txn(struct pmu *pmu)
   1950 {
   1951 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
   1952 	int assign[X86_PMC_IDX_MAX];
   1953 	int n, ret;
   1954 
   1955 	WARN_ON_ONCE(!cpuc->txn_flags);	/* no txn in flight */
   1956 
   1957 	if (cpuc->txn_flags & ~PERF_PMU_TXN_ADD) {
   1958 		cpuc->txn_flags = 0;
   1959 		return 0;
   1960 	}
   1961 
   1962 	n = cpuc->n_events;
   1963 
   1964 	if (!x86_pmu_initialized())
   1965 		return -EAGAIN;
   1966 
   1967 	ret = x86_pmu.schedule_events(cpuc, n, assign);
   1968 	if (ret)
   1969 		return ret;
   1970 
   1971 	/*
   1972 	 * copy new assignment, now we know it is possible
   1973 	 * will be used by hw_perf_enable()
   1974 	 */
   1975 	memcpy(cpuc->assign, assign, n*sizeof(int));
   1976 
   1977 	cpuc->txn_flags = 0;
   1978 	perf_pmu_enable(pmu);
   1979 	return 0;
   1980 }
   1981 /*
   1982  * a fake_cpuc is used to validate event groups. Due to
   1983  * the extra reg logic, we need to also allocate a fake
   1984  * per_core and per_cpu structure. Otherwise, group events
   1985  * using extra reg may conflict without the kernel being
   1986  * able to catch this when the last event gets added to
   1987  * the group.
   1988  */
   1989 static void free_fake_cpuc(struct cpu_hw_events *cpuc)
   1990 {
   1991 	intel_cpuc_finish(cpuc);
   1992 	kfree(cpuc);
   1993 }
   1994 
   1995 static struct cpu_hw_events *allocate_fake_cpuc(void)
   1996 {
   1997 	struct cpu_hw_events *cpuc;
   1998 	int cpu = raw_smp_processor_id();
   1999 
   2000 	cpuc = kzalloc(sizeof(*cpuc), GFP_KERNEL);
   2001 	if (!cpuc)
   2002 		return ERR_PTR(-ENOMEM);
   2003 	cpuc->is_fake = 1;
   2004 
   2005 	if (intel_cpuc_prepare(cpuc, cpu))
   2006 		goto error;
   2007 
   2008 	return cpuc;
   2009 error:
   2010 	free_fake_cpuc(cpuc);
   2011 	return ERR_PTR(-ENOMEM);
   2012 }
   2013 
   2014 /*
   2015  * validate that we can schedule this event
   2016  */
   2017 static int validate_event(struct perf_event *event)
   2018 {
   2019 	struct cpu_hw_events *fake_cpuc;
   2020 	struct event_constraint *c;
   2021 	int ret = 0;
   2022 
   2023 	fake_cpuc = allocate_fake_cpuc();
   2024 	if (IS_ERR(fake_cpuc))
   2025 		return PTR_ERR(fake_cpuc);
   2026 
   2027 	c = x86_pmu.get_event_constraints(fake_cpuc, -1, event);
   2028 
   2029 	if (!c || !c->weight)
   2030 		ret = -EINVAL;
   2031 
   2032 	if (x86_pmu.put_event_constraints)
   2033 		x86_pmu.put_event_constraints(fake_cpuc, event);
   2034 
   2035 	free_fake_cpuc(fake_cpuc);
   2036 
   2037 	return ret;
   2038 }
   2039 
   2040 /*
   2041  * validate a single event group
   2042  *
   2043  * validation include:
   2044  *	- check events are compatible which each other
   2045  *	- events do not compete for the same counter
   2046  *	- number of events <= number of counters
   2047  *
   2048  * validation ensures the group can be loaded onto the
   2049  * PMU if it was the only group available.
   2050  */
   2051 static int validate_group(struct perf_event *event)
   2052 {
   2053 	struct perf_event *leader = event->group_leader;
   2054 	struct cpu_hw_events *fake_cpuc;
   2055 	int ret = -EINVAL, n;
   2056 
   2057 	fake_cpuc = allocate_fake_cpuc();
   2058 	if (IS_ERR(fake_cpuc))
   2059 		return PTR_ERR(fake_cpuc);
   2060 	/*
   2061 	 * the event is not yet connected with its
   2062 	 * siblings therefore we must first collect
   2063 	 * existing siblings, then add the new event
   2064 	 * before we can simulate the scheduling
   2065 	 */
   2066 	n = collect_events(fake_cpuc, leader, true);
   2067 	if (n < 0)
   2068 		goto out;
   2069 
   2070 	fake_cpuc->n_events = n;
   2071 	n = collect_events(fake_cpuc, event, false);
   2072 	if (n < 0)
   2073 		goto out;
   2074 
   2075 	fake_cpuc->n_events = n;
   2076 
   2077 	ret = x86_pmu.schedule_events(fake_cpuc, n, NULL);
   2078 
   2079 out:
   2080 	free_fake_cpuc(fake_cpuc);
   2081 	return ret;
   2082 }
   2083 
   2084 static int x86_pmu_event_init(struct perf_event *event)
   2085 {
   2086 	struct pmu *tmp;
   2087 	int err;
   2088 
   2089 	switch (event->attr.type) {
   2090 	case PERF_TYPE_RAW:
   2091 	case PERF_TYPE_HARDWARE:
   2092 	case PERF_TYPE_HW_CACHE:
   2093 		break;
   2094 
   2095 	default:
   2096 		return -ENOENT;
   2097 	}
   2098 
   2099 	err = __x86_pmu_event_init(event);
   2100 	if (!err) {
   2101 		/*
   2102 		 * we temporarily connect event to its pmu
   2103 		 * such that validate_group() can classify
   2104 		 * it as an x86 event using is_x86_event()
   2105 		 */
   2106 		tmp = event->pmu;
   2107 		event->pmu = &pmu;
   2108 
   2109 		if (event->group_leader != event)
   2110 			err = validate_group(event);
   2111 		else
   2112 			err = validate_event(event);
   2113 
   2114 		event->pmu = tmp;
   2115 	}
   2116 	if (err) {
   2117 		if (event->destroy)
   2118 			event->destroy(event);
   2119 	}
   2120 
   2121 	if (READ_ONCE(x86_pmu.attr_rdpmc) &&
   2122 	    !(event->hw.flags & PERF_X86_EVENT_LARGE_PEBS))
   2123 		event->hw.flags |= PERF_X86_EVENT_RDPMC_ALLOWED;
   2124 
   2125 	return err;
   2126 }
   2127 
   2128 static void refresh_pce(void *ignored)
   2129 {
   2130 	load_mm_cr4(this_cpu_read(cpu_tlbstate.loaded_mm));
   2131 }
   2132 
   2133 static void x86_pmu_event_mapped(struct perf_event *event, struct mm_struct *mm)
   2134 {
   2135 	if (!(event->hw.flags & PERF_X86_EVENT_RDPMC_ALLOWED))
   2136 		return;
   2137 
   2138 	/*
   2139 	 * This function relies on not being called concurrently in two
   2140 	 * tasks in the same mm.  Otherwise one task could observe
   2141 	 * perf_rdpmc_allowed > 1 and return all the way back to
   2142 	 * userspace with CR4.PCE clear while another task is still
   2143 	 * doing on_each_cpu_mask() to propagate CR4.PCE.
   2144 	 *
   2145 	 * For now, this can't happen because all callers hold mmap_sem
   2146 	 * for write.  If this changes, we'll need a different solution.
   2147 	 */
   2148 	lockdep_assert_held_exclusive(&mm->mmap_sem);
   2149 
   2150 	if (atomic_inc_return(&mm->context.perf_rdpmc_allowed) == 1)
   2151 		on_each_cpu_mask(mm_cpumask(mm), refresh_pce, NULL, 1);
   2152 }
   2153 
   2154 static void x86_pmu_event_unmapped(struct perf_event *event, struct mm_struct *mm)
   2155 {
   2156 
   2157 	if (!(event->hw.flags & PERF_X86_EVENT_RDPMC_ALLOWED))
   2158 		return;
   2159 
   2160 	if (atomic_dec_and_test(&mm->context.perf_rdpmc_allowed))
   2161 		on_each_cpu_mask(mm_cpumask(mm), refresh_pce, NULL, 1);
   2162 }
   2163 
   2164 static int x86_pmu_event_idx(struct perf_event *event)
   2165 {
   2166 	int idx = event->hw.idx;
   2167 
   2168 	if (!(event->hw.flags & PERF_X86_EVENT_RDPMC_ALLOWED))
   2169 		return 0;
   2170 
   2171 	if (x86_pmu.num_counters_fixed && idx >= INTEL_PMC_IDX_FIXED) {
   2172 		idx -= INTEL_PMC_IDX_FIXED;
   2173 		idx |= 1 << 30;
   2174 	}
   2175 
   2176 	return idx + 1;
   2177 }
   2178 
   2179 static ssize_t get_attr_rdpmc(struct device *cdev,
   2180 			      struct device_attribute *attr,
   2181 			      char *buf)
   2182 {
   2183 	return snprintf(buf, 40, "%d\n", x86_pmu.attr_rdpmc);
   2184 }
   2185 
   2186 static ssize_t set_attr_rdpmc(struct device *cdev,
   2187 			      struct device_attribute *attr,
   2188 			      const char *buf, size_t count)
   2189 {
   2190 	unsigned long val;
   2191 	ssize_t ret;
   2192 
   2193 	ret = kstrtoul(buf, 0, &val);
   2194 	if (ret)
   2195 		return ret;
   2196 
   2197 	if (val > 2)
   2198 		return -EINVAL;
   2199 
   2200 	if (x86_pmu.attr_rdpmc_broken)
   2201 		return -ENOTSUPP;
   2202 
   2203 	if ((val == 2) != (x86_pmu.attr_rdpmc == 2)) {
   2204 		/*
   2205 		 * Changing into or out of always available, aka
   2206 		 * perf-event-bypassing mode.  This path is extremely slow,
   2207 		 * but only root can trigger it, so it's okay.
   2208 		 */
   2209 		if (val == 2)
   2210 			static_branch_inc(&rdpmc_always_available_key);
   2211 		else
   2212 			static_branch_dec(&rdpmc_always_available_key);
   2213 		on_each_cpu(refresh_pce, NULL, 1);
   2214 	}
   2215 
   2216 	x86_pmu.attr_rdpmc = val;
   2217 
   2218 	return count;
   2219 }
   2220 
   2221 static DEVICE_ATTR(rdpmc, S_IRUSR | S_IWUSR, get_attr_rdpmc, set_attr_rdpmc);
   2222 
   2223 static struct attribute *x86_pmu_attrs[] = {
   2224 	&dev_attr_rdpmc.attr,
   2225 	NULL,
   2226 };
   2227 
   2228 static struct attribute_group x86_pmu_attr_group __ro_after_init = {
   2229 	.attrs = x86_pmu_attrs,
   2230 };
   2231 
   2232 static ssize_t max_precise_show(struct device *cdev,
   2233 				  struct device_attribute *attr,
   2234 				  char *buf)
   2235 {
   2236 	return snprintf(buf, PAGE_SIZE, "%d\n", x86_pmu_max_precise());
   2237 }
   2238 
   2239 static DEVICE_ATTR_RO(max_precise);
   2240 
   2241 static struct attribute *x86_pmu_caps_attrs[] = {
   2242 	&dev_attr_max_precise.attr,
   2243 	NULL
   2244 };
   2245 
   2246 static struct attribute_group x86_pmu_caps_group __ro_after_init = {
   2247 	.name = "caps",
   2248 	.attrs = x86_pmu_caps_attrs,
   2249 };
   2250 
   2251 static const struct attribute_group *x86_pmu_attr_groups[] = {
   2252 	&x86_pmu_attr_group,
   2253 	&x86_pmu_format_group,
   2254 	&x86_pmu_events_group,
   2255 	&x86_pmu_caps_group,
   2256 	NULL,
   2257 };
   2258 
   2259 static void x86_pmu_sched_task(struct perf_event_context *ctx, bool sched_in)
   2260 {
   2261 	if (x86_pmu.sched_task)
   2262 		x86_pmu.sched_task(ctx, sched_in);
   2263 }
   2264 
   2265 void perf_check_microcode(void)
   2266 {
   2267 	if (x86_pmu.check_microcode)
   2268 		x86_pmu.check_microcode();
   2269 }
   2270 
   2271 static int x86_pmu_check_period(struct perf_event *event, u64 value)
   2272 {
   2273 	if (x86_pmu.check_period && x86_pmu.check_period(event, value))
   2274 		return -EINVAL;
   2275 
   2276 	if (value && x86_pmu.limit_period) {
   2277 		if (x86_pmu.limit_period(event, value) > value)
   2278 			return -EINVAL;
   2279 	}
   2280 
   2281 	return 0;
   2282 }
   2283 
   2284 static struct pmu pmu = {
   2285 	.pmu_enable		= x86_pmu_enable,
   2286 	.pmu_disable		= x86_pmu_disable,
   2287 
   2288 	.attr_groups		= x86_pmu_attr_groups,
   2289 
   2290 	.event_init		= x86_pmu_event_init,
   2291 
   2292 	.event_mapped		= x86_pmu_event_mapped,
   2293 	.event_unmapped		= x86_pmu_event_unmapped,
   2294 
   2295 	.add			= x86_pmu_add,
   2296 	.del			= x86_pmu_del,
   2297 	.start			= x86_pmu_start,
   2298 	.stop			= x86_pmu_stop,
   2299 	.read			= x86_pmu_read,
   2300 
   2301 	.start_txn		= x86_pmu_start_txn,
   2302 	.cancel_txn		= x86_pmu_cancel_txn,
   2303 	.commit_txn		= x86_pmu_commit_txn,
   2304 
   2305 	.event_idx		= x86_pmu_event_idx,
   2306 	.sched_task		= x86_pmu_sched_task,
   2307 	.task_ctx_size          = sizeof(struct x86_perf_task_context),
   2308 	.check_period		= x86_pmu_check_period,
   2309 };
   2310 
   2311 void arch_perf_update_userpage(struct perf_event *event,
   2312 			       struct perf_event_mmap_page *userpg, u64 now)
   2313 {
   2314 	struct cyc2ns_data data;
   2315 	u64 offset;
   2316 
   2317 	userpg->cap_user_time = 0;
   2318 	userpg->cap_user_time_zero = 0;
   2319 	userpg->cap_user_rdpmc =
   2320 		!!(event->hw.flags & PERF_X86_EVENT_RDPMC_ALLOWED);
   2321 	userpg->pmc_width = x86_pmu.cntval_bits;
   2322 
   2323 	if (!using_native_sched_clock() || !sched_clock_stable())
   2324 		return;
   2325 
   2326 	cyc2ns_read_begin(&data);
   2327 
   2328 	offset = data.cyc2ns_offset + __sched_clock_offset;
   2329 
   2330 	/*
   2331 	 * Internal timekeeping for enabled/running/stopped times
   2332 	 * is always in the local_clock domain.
   2333 	 */
   2334 	userpg->cap_user_time = 1;
   2335 	userpg->time_mult = data.cyc2ns_mul;
   2336 	userpg->time_shift = data.cyc2ns_shift;
   2337 	userpg->time_offset = offset - now;
   2338 
   2339 	/*
   2340 	 * cap_user_time_zero doesn't make sense when we're using a different
   2341 	 * time base for the records.
   2342 	 */
   2343 	if (!event->attr.use_clockid) {
   2344 		userpg->cap_user_time_zero = 1;
   2345 		userpg->time_zero = offset;
   2346 	}
   2347 
   2348 	cyc2ns_read_end();
   2349 }
   2350 
   2351 void
   2352 perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs)
   2353 {
   2354 	struct unwind_state state;
   2355 	unsigned long addr;
   2356 
   2357 	if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) {
   2358 		/* TODO: We don't support guest os callchain now */
   2359 		return;
   2360 	}
   2361 
   2362 	if (perf_callchain_store(entry, regs->ip))
   2363 		return;
   2364 
   2365 	for (unwind_start(&state, current, regs, NULL); !unwind_done(&state);
   2366 	     unwind_next_frame(&state)) {
   2367 		addr = unwind_get_return_address(&state);
   2368 		if (!addr || perf_callchain_store(entry, addr))
   2369 			return;
   2370 	}
   2371 }
   2372 
   2373 static inline int
   2374 valid_user_frame(const void __user *fp, unsigned long size)
   2375 {
   2376 	return (__range_not_ok(fp, size, TASK_SIZE) == 0);
   2377 }
   2378 
   2379 static unsigned long get_segment_base(unsigned int segment)
   2380 {
   2381 	struct desc_struct *desc;
   2382 	unsigned int idx = segment >> 3;
   2383 
   2384 	if ((segment & SEGMENT_TI_MASK) == SEGMENT_LDT) {
   2385 #ifdef CONFIG_MODIFY_LDT_SYSCALL
   2386 		struct ldt_struct *ldt;
   2387 
   2388 		/* IRQs are off, so this synchronizes with smp_store_release */
   2389 		ldt = READ_ONCE(current->active_mm->context.ldt);
   2390 		if (!ldt || idx >= ldt->nr_entries)
   2391 			return 0;
   2392 
   2393 		desc = &ldt->entries[idx];
   2394 #else
   2395 		return 0;
   2396 #endif
   2397 	} else {
   2398 		if (idx >= GDT_ENTRIES)
   2399 			return 0;
   2400 
   2401 		desc = raw_cpu_ptr(gdt_page.gdt) + idx;
   2402 	}
   2403 
   2404 	return get_desc_base(desc);
   2405 }
   2406 
   2407 #ifdef CONFIG_IA32_EMULATION
   2408 
   2409 #include <linux/compat.h>
   2410 
   2411 static inline int
   2412 perf_callchain_user32(struct pt_regs *regs, struct perf_callchain_entry_ctx *entry)
   2413 {
   2414 	/* 32-bit process in 64-bit kernel. */
   2415 	unsigned long ss_base, cs_base;
   2416 	struct stack_frame_ia32 frame;
   2417 	const void __user *fp;
   2418 
   2419 	if (!test_thread_flag(TIF_IA32))
   2420 		return 0;
   2421 
   2422 	cs_base = get_segment_base(regs->cs);
   2423 	ss_base = get_segment_base(regs->ss);
   2424 
   2425 	fp = compat_ptr(ss_base + regs->bp);
   2426 	pagefault_disable();
   2427 	while (entry->nr < entry->max_stack) {
   2428 		unsigned long bytes;
   2429 		frame.next_frame     = 0;
   2430 		frame.return_address = 0;
   2431 
   2432 		if (!valid_user_frame(fp, sizeof(frame)))
   2433 			break;
   2434 
   2435 		bytes = __copy_from_user_nmi(&frame.next_frame, fp, 4);
   2436 		if (bytes != 0)
   2437 			break;
   2438 		bytes = __copy_from_user_nmi(&frame.return_address, fp+4, 4);
   2439 		if (bytes != 0)
   2440 			break;
   2441 
   2442 		perf_callchain_store(entry, cs_base + frame.return_address);
   2443 		fp = compat_ptr(ss_base + frame.next_frame);
   2444 	}
   2445 	pagefault_enable();
   2446 	return 1;
   2447 }
   2448 #else
   2449 static inline int
   2450 perf_callchain_user32(struct pt_regs *regs, struct perf_callchain_entry_ctx *entry)
   2451 {
   2452     return 0;
   2453 }
   2454 #endif
   2455 
   2456 void
   2457 perf_callchain_user(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs)
   2458 {
   2459 	struct stack_frame frame;
   2460 	const unsigned long __user *fp;
   2461 
   2462 	if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) {
   2463 		/* TODO: We don't support guest os callchain now */
   2464 		return;
   2465 	}
   2466 
   2467 	/*
   2468 	 * We don't know what to do with VM86 stacks.. ignore them for now.
   2469 	 */
   2470 	if (regs->flags & (X86_VM_MASK | PERF_EFLAGS_VM))
   2471 		return;
   2472 
   2473 	fp = (unsigned long __user *)regs->bp;
   2474 
   2475 	perf_callchain_store(entry, regs->ip);
   2476 
   2477 	if (!nmi_uaccess_okay())
   2478 		return;
   2479 
   2480 	if (perf_callchain_user32(regs, entry))
   2481 		return;
   2482 
   2483 	pagefault_disable();
   2484 	while (entry->nr < entry->max_stack) {
   2485 		unsigned long bytes;
   2486 
   2487 		frame.next_frame	     = NULL;
   2488 		frame.return_address = 0;
   2489 
   2490 		if (!valid_user_frame(fp, sizeof(frame)))
   2491 			break;
   2492 
   2493 		bytes = __copy_from_user_nmi(&frame.next_frame, fp, sizeof(*fp));
   2494 		if (bytes != 0)
   2495 			break;
   2496 		bytes = __copy_from_user_nmi(&frame.return_address, fp + 1, sizeof(*fp));
   2497 		if (bytes != 0)
   2498 			break;
   2499 
   2500 		perf_callchain_store(entry, frame.return_address);
   2501 		fp = (void __user *)frame.next_frame;
   2502 	}
   2503 	pagefault_enable();
   2504 }
   2505 
   2506 /*
   2507  * Deal with code segment offsets for the various execution modes:
   2508  *
   2509  *   VM86 - the good olde 16 bit days, where the linear address is
   2510  *          20 bits and we use regs->ip + 0x10 * regs->cs.
   2511  *
   2512  *   IA32 - Where we need to look at GDT/LDT segment descriptor tables
   2513  *          to figure out what the 32bit base address is.
   2514  *
   2515  *    X32 - has TIF_X32 set, but is running in x86_64
   2516  *
   2517  * X86_64 - CS,DS,SS,ES are all zero based.
   2518  */
   2519 static unsigned long code_segment_base(struct pt_regs *regs)
   2520 {
   2521 	/*
   2522 	 * For IA32 we look at the GDT/LDT segment base to convert the
   2523 	 * effective IP to a linear address.
   2524 	 */
   2525 
   2526 #ifdef CONFIG_X86_32
   2527 	/*
   2528 	 * If we are in VM86 mode, add the segment offset to convert to a
   2529 	 * linear address.
   2530 	 */
   2531 	if (regs->flags & X86_VM_MASK)
   2532 		return 0x10 * regs->cs;
   2533 
   2534 	if (user_mode(regs) && regs->cs != __USER_CS)
   2535 		return get_segment_base(regs->cs);
   2536 #else
   2537 	if (user_mode(regs) && !user_64bit_mode(regs) &&
   2538 	    regs->cs != __USER32_CS)
   2539 		return get_segment_base(regs->cs);
   2540 #endif
   2541 	return 0;
   2542 }
   2543 
   2544 unsigned long perf_instruction_pointer(struct pt_regs *regs)
   2545 {
   2546 	if (perf_guest_cbs && perf_guest_cbs->is_in_guest())
   2547 		return perf_guest_cbs->get_guest_ip();
   2548 
   2549 	return regs->ip + code_segment_base(regs);
   2550 }
   2551 
   2552 unsigned long perf_misc_flags(struct pt_regs *regs)
   2553 {
   2554 	int misc = 0;
   2555 
   2556 	if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) {
   2557 		if (perf_guest_cbs->is_user_mode())
   2558 			misc |= PERF_RECORD_MISC_GUEST_USER;
   2559 		else
   2560 			misc |= PERF_RECORD_MISC_GUEST_KERNEL;
   2561 	} else {
   2562 		if (user_mode(regs))
   2563 			misc |= PERF_RECORD_MISC_USER;
   2564 		else
   2565 			misc |= PERF_RECORD_MISC_KERNEL;
   2566 	}
   2567 
   2568 	if (regs->flags & PERF_EFLAGS_EXACT)
   2569 		misc |= PERF_RECORD_MISC_EXACT_IP;
   2570 
   2571 	return misc;
   2572 }
   2573 
   2574 void perf_get_x86_pmu_capability(struct x86_pmu_capability *cap)
   2575 {
   2576 	cap->version		= x86_pmu.version;
   2577 	cap->num_counters_gp	= x86_pmu.num_counters;
   2578 	cap->num_counters_fixed	= x86_pmu.num_counters_fixed;
   2579 	cap->bit_width_gp	= x86_pmu.cntval_bits;
   2580 	cap->bit_width_fixed	= x86_pmu.cntval_bits;
   2581 	cap->events_mask	= (unsigned int)x86_pmu.events_maskl;
   2582 	cap->events_mask_len	= x86_pmu.events_mask_len;
   2583 }
   2584 EXPORT_SYMBOL_GPL(perf_get_x86_pmu_capability);