whiterose

linux unikernel
Log | Files | Refs | README | LICENSE | git clone https://git.ne02ptzero.me/git/whiterose

bpf_trace.c (35636B)


      1 // SPDX-License-Identifier: GPL-2.0
      2 /* Copyright (c) 2011-2015 PLUMgrid, http://plumgrid.com
      3  * Copyright (c) 2016 Facebook
      4  */
      5 #include <linux/kernel.h>
      6 #include <linux/types.h>
      7 #include <linux/slab.h>
      8 #include <linux/bpf.h>
      9 #include <linux/bpf_perf_event.h>
     10 #include <linux/filter.h>
     11 #include <linux/uaccess.h>
     12 #include <linux/ctype.h>
     13 #include <linux/kprobes.h>
     14 #include <linux/syscalls.h>
     15 #include <linux/error-injection.h>
     16 
     17 #include "trace_probe.h"
     18 #include "trace.h"
     19 
     20 #ifdef CONFIG_MODULES
     21 struct bpf_trace_module {
     22 	struct module *module;
     23 	struct list_head list;
     24 };
     25 
     26 static LIST_HEAD(bpf_trace_modules);
     27 static DEFINE_MUTEX(bpf_module_mutex);
     28 
     29 static struct bpf_raw_event_map *bpf_get_raw_tracepoint_module(const char *name)
     30 {
     31 	struct bpf_raw_event_map *btp, *ret = NULL;
     32 	struct bpf_trace_module *btm;
     33 	unsigned int i;
     34 
     35 	mutex_lock(&bpf_module_mutex);
     36 	list_for_each_entry(btm, &bpf_trace_modules, list) {
     37 		for (i = 0; i < btm->module->num_bpf_raw_events; ++i) {
     38 			btp = &btm->module->bpf_raw_events[i];
     39 			if (!strcmp(btp->tp->name, name)) {
     40 				if (try_module_get(btm->module))
     41 					ret = btp;
     42 				goto out;
     43 			}
     44 		}
     45 	}
     46 out:
     47 	mutex_unlock(&bpf_module_mutex);
     48 	return ret;
     49 }
     50 #else
     51 static struct bpf_raw_event_map *bpf_get_raw_tracepoint_module(const char *name)
     52 {
     53 	return NULL;
     54 }
     55 #endif /* CONFIG_MODULES */
     56 
     57 u64 bpf_get_stackid(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5);
     58 u64 bpf_get_stack(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5);
     59 
     60 /**
     61  * trace_call_bpf - invoke BPF program
     62  * @call: tracepoint event
     63  * @ctx: opaque context pointer
     64  *
     65  * kprobe handlers execute BPF programs via this helper.
     66  * Can be used from static tracepoints in the future.
     67  *
     68  * Return: BPF programs always return an integer which is interpreted by
     69  * kprobe handler as:
     70  * 0 - return from kprobe (event is filtered out)
     71  * 1 - store kprobe event into ring buffer
     72  * Other values are reserved and currently alias to 1
     73  */
     74 unsigned int trace_call_bpf(struct trace_event_call *call, void *ctx)
     75 {
     76 	unsigned int ret;
     77 
     78 	if (in_nmi()) /* not supported yet */
     79 		return 1;
     80 
     81 	preempt_disable();
     82 
     83 	if (unlikely(__this_cpu_inc_return(bpf_prog_active) != 1)) {
     84 		/*
     85 		 * since some bpf program is already running on this cpu,
     86 		 * don't call into another bpf program (same or different)
     87 		 * and don't send kprobe event into ring-buffer,
     88 		 * so return zero here
     89 		 */
     90 		ret = 0;
     91 		goto out;
     92 	}
     93 
     94 	/*
     95 	 * Instead of moving rcu_read_lock/rcu_dereference/rcu_read_unlock
     96 	 * to all call sites, we did a bpf_prog_array_valid() there to check
     97 	 * whether call->prog_array is empty or not, which is
     98 	 * a heurisitc to speed up execution.
     99 	 *
    100 	 * If bpf_prog_array_valid() fetched prog_array was
    101 	 * non-NULL, we go into trace_call_bpf() and do the actual
    102 	 * proper rcu_dereference() under RCU lock.
    103 	 * If it turns out that prog_array is NULL then, we bail out.
    104 	 * For the opposite, if the bpf_prog_array_valid() fetched pointer
    105 	 * was NULL, you'll skip the prog_array with the risk of missing
    106 	 * out of events when it was updated in between this and the
    107 	 * rcu_dereference() which is accepted risk.
    108 	 */
    109 	ret = BPF_PROG_RUN_ARRAY_CHECK(call->prog_array, ctx, BPF_PROG_RUN);
    110 
    111  out:
    112 	__this_cpu_dec(bpf_prog_active);
    113 	preempt_enable();
    114 
    115 	return ret;
    116 }
    117 EXPORT_SYMBOL_GPL(trace_call_bpf);
    118 
    119 #ifdef CONFIG_BPF_KPROBE_OVERRIDE
    120 BPF_CALL_2(bpf_override_return, struct pt_regs *, regs, unsigned long, rc)
    121 {
    122 	regs_set_return_value(regs, rc);
    123 	override_function_with_return(regs);
    124 	return 0;
    125 }
    126 
    127 static const struct bpf_func_proto bpf_override_return_proto = {
    128 	.func		= bpf_override_return,
    129 	.gpl_only	= true,
    130 	.ret_type	= RET_INTEGER,
    131 	.arg1_type	= ARG_PTR_TO_CTX,
    132 	.arg2_type	= ARG_ANYTHING,
    133 };
    134 #endif
    135 
    136 BPF_CALL_3(bpf_probe_read, void *, dst, u32, size, const void *, unsafe_ptr)
    137 {
    138 	int ret;
    139 
    140 	ret = probe_kernel_read(dst, unsafe_ptr, size);
    141 	if (unlikely(ret < 0))
    142 		memset(dst, 0, size);
    143 
    144 	return ret;
    145 }
    146 
    147 static const struct bpf_func_proto bpf_probe_read_proto = {
    148 	.func		= bpf_probe_read,
    149 	.gpl_only	= true,
    150 	.ret_type	= RET_INTEGER,
    151 	.arg1_type	= ARG_PTR_TO_UNINIT_MEM,
    152 	.arg2_type	= ARG_CONST_SIZE_OR_ZERO,
    153 	.arg3_type	= ARG_ANYTHING,
    154 };
    155 
    156 BPF_CALL_3(bpf_probe_write_user, void *, unsafe_ptr, const void *, src,
    157 	   u32, size)
    158 {
    159 	/*
    160 	 * Ensure we're in user context which is safe for the helper to
    161 	 * run. This helper has no business in a kthread.
    162 	 *
    163 	 * access_ok() should prevent writing to non-user memory, but in
    164 	 * some situations (nommu, temporary switch, etc) access_ok() does
    165 	 * not provide enough validation, hence the check on KERNEL_DS.
    166 	 */
    167 
    168 	if (unlikely(in_interrupt() ||
    169 		     current->flags & (PF_KTHREAD | PF_EXITING)))
    170 		return -EPERM;
    171 	if (unlikely(uaccess_kernel()))
    172 		return -EPERM;
    173 	if (!access_ok(unsafe_ptr, size))
    174 		return -EPERM;
    175 
    176 	return probe_kernel_write(unsafe_ptr, src, size);
    177 }
    178 
    179 static const struct bpf_func_proto bpf_probe_write_user_proto = {
    180 	.func		= bpf_probe_write_user,
    181 	.gpl_only	= true,
    182 	.ret_type	= RET_INTEGER,
    183 	.arg1_type	= ARG_ANYTHING,
    184 	.arg2_type	= ARG_PTR_TO_MEM,
    185 	.arg3_type	= ARG_CONST_SIZE,
    186 };
    187 
    188 static const struct bpf_func_proto *bpf_get_probe_write_proto(void)
    189 {
    190 	pr_warn_ratelimited("%s[%d] is installing a program with bpf_probe_write_user helper that may corrupt user memory!",
    191 			    current->comm, task_pid_nr(current));
    192 
    193 	return &bpf_probe_write_user_proto;
    194 }
    195 
    196 /*
    197  * Only limited trace_printk() conversion specifiers allowed:
    198  * %d %i %u %x %ld %li %lu %lx %lld %lli %llu %llx %p %s
    199  */
    200 BPF_CALL_5(bpf_trace_printk, char *, fmt, u32, fmt_size, u64, arg1,
    201 	   u64, arg2, u64, arg3)
    202 {
    203 	bool str_seen = false;
    204 	int mod[3] = {};
    205 	int fmt_cnt = 0;
    206 	u64 unsafe_addr;
    207 	char buf[64];
    208 	int i;
    209 
    210 	/*
    211 	 * bpf_check()->check_func_arg()->check_stack_boundary()
    212 	 * guarantees that fmt points to bpf program stack,
    213 	 * fmt_size bytes of it were initialized and fmt_size > 0
    214 	 */
    215 	if (fmt[--fmt_size] != 0)
    216 		return -EINVAL;
    217 
    218 	/* check format string for allowed specifiers */
    219 	for (i = 0; i < fmt_size; i++) {
    220 		if ((!isprint(fmt[i]) && !isspace(fmt[i])) || !isascii(fmt[i]))
    221 			return -EINVAL;
    222 
    223 		if (fmt[i] != '%')
    224 			continue;
    225 
    226 		if (fmt_cnt >= 3)
    227 			return -EINVAL;
    228 
    229 		/* fmt[i] != 0 && fmt[last] == 0, so we can access fmt[i + 1] */
    230 		i++;
    231 		if (fmt[i] == 'l') {
    232 			mod[fmt_cnt]++;
    233 			i++;
    234 		} else if (fmt[i] == 'p' || fmt[i] == 's') {
    235 			mod[fmt_cnt]++;
    236 			/* disallow any further format extensions */
    237 			if (fmt[i + 1] != 0 &&
    238 			    !isspace(fmt[i + 1]) &&
    239 			    !ispunct(fmt[i + 1]))
    240 				return -EINVAL;
    241 			fmt_cnt++;
    242 			if (fmt[i] == 's') {
    243 				if (str_seen)
    244 					/* allow only one '%s' per fmt string */
    245 					return -EINVAL;
    246 				str_seen = true;
    247 
    248 				switch (fmt_cnt) {
    249 				case 1:
    250 					unsafe_addr = arg1;
    251 					arg1 = (long) buf;
    252 					break;
    253 				case 2:
    254 					unsafe_addr = arg2;
    255 					arg2 = (long) buf;
    256 					break;
    257 				case 3:
    258 					unsafe_addr = arg3;
    259 					arg3 = (long) buf;
    260 					break;
    261 				}
    262 				buf[0] = 0;
    263 				strncpy_from_unsafe(buf,
    264 						    (void *) (long) unsafe_addr,
    265 						    sizeof(buf));
    266 			}
    267 			continue;
    268 		}
    269 
    270 		if (fmt[i] == 'l') {
    271 			mod[fmt_cnt]++;
    272 			i++;
    273 		}
    274 
    275 		if (fmt[i] != 'i' && fmt[i] != 'd' &&
    276 		    fmt[i] != 'u' && fmt[i] != 'x')
    277 			return -EINVAL;
    278 		fmt_cnt++;
    279 	}
    280 
    281 /* Horrid workaround for getting va_list handling working with different
    282  * argument type combinations generically for 32 and 64 bit archs.
    283  */
    284 #define __BPF_TP_EMIT()	__BPF_ARG3_TP()
    285 #define __BPF_TP(...)							\
    286 	__trace_printk(0 /* Fake ip */,					\
    287 		       fmt, ##__VA_ARGS__)
    288 
    289 #define __BPF_ARG1_TP(...)						\
    290 	((mod[0] == 2 || (mod[0] == 1 && __BITS_PER_LONG == 64))	\
    291 	  ? __BPF_TP(arg1, ##__VA_ARGS__)				\
    292 	  : ((mod[0] == 1 || (mod[0] == 0 && __BITS_PER_LONG == 32))	\
    293 	      ? __BPF_TP((long)arg1, ##__VA_ARGS__)			\
    294 	      : __BPF_TP((u32)arg1, ##__VA_ARGS__)))
    295 
    296 #define __BPF_ARG2_TP(...)						\
    297 	((mod[1] == 2 || (mod[1] == 1 && __BITS_PER_LONG == 64))	\
    298 	  ? __BPF_ARG1_TP(arg2, ##__VA_ARGS__)				\
    299 	  : ((mod[1] == 1 || (mod[1] == 0 && __BITS_PER_LONG == 32))	\
    300 	      ? __BPF_ARG1_TP((long)arg2, ##__VA_ARGS__)		\
    301 	      : __BPF_ARG1_TP((u32)arg2, ##__VA_ARGS__)))
    302 
    303 #define __BPF_ARG3_TP(...)						\
    304 	((mod[2] == 2 || (mod[2] == 1 && __BITS_PER_LONG == 64))	\
    305 	  ? __BPF_ARG2_TP(arg3, ##__VA_ARGS__)				\
    306 	  : ((mod[2] == 1 || (mod[2] == 0 && __BITS_PER_LONG == 32))	\
    307 	      ? __BPF_ARG2_TP((long)arg3, ##__VA_ARGS__)		\
    308 	      : __BPF_ARG2_TP((u32)arg3, ##__VA_ARGS__)))
    309 
    310 	return __BPF_TP_EMIT();
    311 }
    312 
    313 static const struct bpf_func_proto bpf_trace_printk_proto = {
    314 	.func		= bpf_trace_printk,
    315 	.gpl_only	= true,
    316 	.ret_type	= RET_INTEGER,
    317 	.arg1_type	= ARG_PTR_TO_MEM,
    318 	.arg2_type	= ARG_CONST_SIZE,
    319 };
    320 
    321 const struct bpf_func_proto *bpf_get_trace_printk_proto(void)
    322 {
    323 	/*
    324 	 * this program might be calling bpf_trace_printk,
    325 	 * so allocate per-cpu printk buffers
    326 	 */
    327 	trace_printk_init_buffers();
    328 
    329 	return &bpf_trace_printk_proto;
    330 }
    331 
    332 static __always_inline int
    333 get_map_perf_counter(struct bpf_map *map, u64 flags,
    334 		     u64 *value, u64 *enabled, u64 *running)
    335 {
    336 	struct bpf_array *array = container_of(map, struct bpf_array, map);
    337 	unsigned int cpu = smp_processor_id();
    338 	u64 index = flags & BPF_F_INDEX_MASK;
    339 	struct bpf_event_entry *ee;
    340 
    341 	if (unlikely(flags & ~(BPF_F_INDEX_MASK)))
    342 		return -EINVAL;
    343 	if (index == BPF_F_CURRENT_CPU)
    344 		index = cpu;
    345 	if (unlikely(index >= array->map.max_entries))
    346 		return -E2BIG;
    347 
    348 	ee = READ_ONCE(array->ptrs[index]);
    349 	if (!ee)
    350 		return -ENOENT;
    351 
    352 	return perf_event_read_local(ee->event, value, enabled, running);
    353 }
    354 
    355 BPF_CALL_2(bpf_perf_event_read, struct bpf_map *, map, u64, flags)
    356 {
    357 	u64 value = 0;
    358 	int err;
    359 
    360 	err = get_map_perf_counter(map, flags, &value, NULL, NULL);
    361 	/*
    362 	 * this api is ugly since we miss [-22..-2] range of valid
    363 	 * counter values, but that's uapi
    364 	 */
    365 	if (err)
    366 		return err;
    367 	return value;
    368 }
    369 
    370 static const struct bpf_func_proto bpf_perf_event_read_proto = {
    371 	.func		= bpf_perf_event_read,
    372 	.gpl_only	= true,
    373 	.ret_type	= RET_INTEGER,
    374 	.arg1_type	= ARG_CONST_MAP_PTR,
    375 	.arg2_type	= ARG_ANYTHING,
    376 };
    377 
    378 BPF_CALL_4(bpf_perf_event_read_value, struct bpf_map *, map, u64, flags,
    379 	   struct bpf_perf_event_value *, buf, u32, size)
    380 {
    381 	int err = -EINVAL;
    382 
    383 	if (unlikely(size != sizeof(struct bpf_perf_event_value)))
    384 		goto clear;
    385 	err = get_map_perf_counter(map, flags, &buf->counter, &buf->enabled,
    386 				   &buf->running);
    387 	if (unlikely(err))
    388 		goto clear;
    389 	return 0;
    390 clear:
    391 	memset(buf, 0, size);
    392 	return err;
    393 }
    394 
    395 static const struct bpf_func_proto bpf_perf_event_read_value_proto = {
    396 	.func		= bpf_perf_event_read_value,
    397 	.gpl_only	= true,
    398 	.ret_type	= RET_INTEGER,
    399 	.arg1_type	= ARG_CONST_MAP_PTR,
    400 	.arg2_type	= ARG_ANYTHING,
    401 	.arg3_type	= ARG_PTR_TO_UNINIT_MEM,
    402 	.arg4_type	= ARG_CONST_SIZE,
    403 };
    404 
    405 static DEFINE_PER_CPU(struct perf_sample_data, bpf_trace_sd);
    406 
    407 static __always_inline u64
    408 __bpf_perf_event_output(struct pt_regs *regs, struct bpf_map *map,
    409 			u64 flags, struct perf_sample_data *sd)
    410 {
    411 	struct bpf_array *array = container_of(map, struct bpf_array, map);
    412 	unsigned int cpu = smp_processor_id();
    413 	u64 index = flags & BPF_F_INDEX_MASK;
    414 	struct bpf_event_entry *ee;
    415 	struct perf_event *event;
    416 
    417 	if (index == BPF_F_CURRENT_CPU)
    418 		index = cpu;
    419 	if (unlikely(index >= array->map.max_entries))
    420 		return -E2BIG;
    421 
    422 	ee = READ_ONCE(array->ptrs[index]);
    423 	if (!ee)
    424 		return -ENOENT;
    425 
    426 	event = ee->event;
    427 	if (unlikely(event->attr.type != PERF_TYPE_SOFTWARE ||
    428 		     event->attr.config != PERF_COUNT_SW_BPF_OUTPUT))
    429 		return -EINVAL;
    430 
    431 	if (unlikely(event->oncpu != cpu))
    432 		return -EOPNOTSUPP;
    433 
    434 	return perf_event_output(event, sd, regs);
    435 }
    436 
    437 BPF_CALL_5(bpf_perf_event_output, struct pt_regs *, regs, struct bpf_map *, map,
    438 	   u64, flags, void *, data, u64, size)
    439 {
    440 	struct perf_sample_data *sd = this_cpu_ptr(&bpf_trace_sd);
    441 	struct perf_raw_record raw = {
    442 		.frag = {
    443 			.size = size,
    444 			.data = data,
    445 		},
    446 	};
    447 
    448 	if (unlikely(flags & ~(BPF_F_INDEX_MASK)))
    449 		return -EINVAL;
    450 
    451 	perf_sample_data_init(sd, 0, 0);
    452 	sd->raw = &raw;
    453 
    454 	return __bpf_perf_event_output(regs, map, flags, sd);
    455 }
    456 
    457 static const struct bpf_func_proto bpf_perf_event_output_proto = {
    458 	.func		= bpf_perf_event_output,
    459 	.gpl_only	= true,
    460 	.ret_type	= RET_INTEGER,
    461 	.arg1_type	= ARG_PTR_TO_CTX,
    462 	.arg2_type	= ARG_CONST_MAP_PTR,
    463 	.arg3_type	= ARG_ANYTHING,
    464 	.arg4_type	= ARG_PTR_TO_MEM,
    465 	.arg5_type	= ARG_CONST_SIZE_OR_ZERO,
    466 };
    467 
    468 static DEFINE_PER_CPU(struct pt_regs, bpf_pt_regs);
    469 static DEFINE_PER_CPU(struct perf_sample_data, bpf_misc_sd);
    470 
    471 u64 bpf_event_output(struct bpf_map *map, u64 flags, void *meta, u64 meta_size,
    472 		     void *ctx, u64 ctx_size, bpf_ctx_copy_t ctx_copy)
    473 {
    474 	struct perf_sample_data *sd = this_cpu_ptr(&bpf_misc_sd);
    475 	struct pt_regs *regs = this_cpu_ptr(&bpf_pt_regs);
    476 	struct perf_raw_frag frag = {
    477 		.copy		= ctx_copy,
    478 		.size		= ctx_size,
    479 		.data		= ctx,
    480 	};
    481 	struct perf_raw_record raw = {
    482 		.frag = {
    483 			{
    484 				.next	= ctx_size ? &frag : NULL,
    485 			},
    486 			.size	= meta_size,
    487 			.data	= meta,
    488 		},
    489 	};
    490 
    491 	perf_fetch_caller_regs(regs);
    492 	perf_sample_data_init(sd, 0, 0);
    493 	sd->raw = &raw;
    494 
    495 	return __bpf_perf_event_output(regs, map, flags, sd);
    496 }
    497 
    498 BPF_CALL_0(bpf_get_current_task)
    499 {
    500 	return (long) current;
    501 }
    502 
    503 static const struct bpf_func_proto bpf_get_current_task_proto = {
    504 	.func		= bpf_get_current_task,
    505 	.gpl_only	= true,
    506 	.ret_type	= RET_INTEGER,
    507 };
    508 
    509 BPF_CALL_2(bpf_current_task_under_cgroup, struct bpf_map *, map, u32, idx)
    510 {
    511 	struct bpf_array *array = container_of(map, struct bpf_array, map);
    512 	struct cgroup *cgrp;
    513 
    514 	if (unlikely(idx >= array->map.max_entries))
    515 		return -E2BIG;
    516 
    517 	cgrp = READ_ONCE(array->ptrs[idx]);
    518 	if (unlikely(!cgrp))
    519 		return -EAGAIN;
    520 
    521 	return task_under_cgroup_hierarchy(current, cgrp);
    522 }
    523 
    524 static const struct bpf_func_proto bpf_current_task_under_cgroup_proto = {
    525 	.func           = bpf_current_task_under_cgroup,
    526 	.gpl_only       = false,
    527 	.ret_type       = RET_INTEGER,
    528 	.arg1_type      = ARG_CONST_MAP_PTR,
    529 	.arg2_type      = ARG_ANYTHING,
    530 };
    531 
    532 BPF_CALL_3(bpf_probe_read_str, void *, dst, u32, size,
    533 	   const void *, unsafe_ptr)
    534 {
    535 	int ret;
    536 
    537 	/*
    538 	 * The strncpy_from_unsafe() call will likely not fill the entire
    539 	 * buffer, but that's okay in this circumstance as we're probing
    540 	 * arbitrary memory anyway similar to bpf_probe_read() and might
    541 	 * as well probe the stack. Thus, memory is explicitly cleared
    542 	 * only in error case, so that improper users ignoring return
    543 	 * code altogether don't copy garbage; otherwise length of string
    544 	 * is returned that can be used for bpf_perf_event_output() et al.
    545 	 */
    546 	ret = strncpy_from_unsafe(dst, unsafe_ptr, size);
    547 	if (unlikely(ret < 0))
    548 		memset(dst, 0, size);
    549 
    550 	return ret;
    551 }
    552 
    553 static const struct bpf_func_proto bpf_probe_read_str_proto = {
    554 	.func		= bpf_probe_read_str,
    555 	.gpl_only	= true,
    556 	.ret_type	= RET_INTEGER,
    557 	.arg1_type	= ARG_PTR_TO_UNINIT_MEM,
    558 	.arg2_type	= ARG_CONST_SIZE_OR_ZERO,
    559 	.arg3_type	= ARG_ANYTHING,
    560 };
    561 
    562 static const struct bpf_func_proto *
    563 tracing_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
    564 {
    565 	switch (func_id) {
    566 	case BPF_FUNC_map_lookup_elem:
    567 		return &bpf_map_lookup_elem_proto;
    568 	case BPF_FUNC_map_update_elem:
    569 		return &bpf_map_update_elem_proto;
    570 	case BPF_FUNC_map_delete_elem:
    571 		return &bpf_map_delete_elem_proto;
    572 	case BPF_FUNC_probe_read:
    573 		return &bpf_probe_read_proto;
    574 	case BPF_FUNC_ktime_get_ns:
    575 		return &bpf_ktime_get_ns_proto;
    576 	case BPF_FUNC_tail_call:
    577 		return &bpf_tail_call_proto;
    578 	case BPF_FUNC_get_current_pid_tgid:
    579 		return &bpf_get_current_pid_tgid_proto;
    580 	case BPF_FUNC_get_current_task:
    581 		return &bpf_get_current_task_proto;
    582 	case BPF_FUNC_get_current_uid_gid:
    583 		return &bpf_get_current_uid_gid_proto;
    584 	case BPF_FUNC_get_current_comm:
    585 		return &bpf_get_current_comm_proto;
    586 	case BPF_FUNC_trace_printk:
    587 		return bpf_get_trace_printk_proto();
    588 	case BPF_FUNC_get_smp_processor_id:
    589 		return &bpf_get_smp_processor_id_proto;
    590 	case BPF_FUNC_get_numa_node_id:
    591 		return &bpf_get_numa_node_id_proto;
    592 	case BPF_FUNC_perf_event_read:
    593 		return &bpf_perf_event_read_proto;
    594 	case BPF_FUNC_probe_write_user:
    595 		return bpf_get_probe_write_proto();
    596 	case BPF_FUNC_current_task_under_cgroup:
    597 		return &bpf_current_task_under_cgroup_proto;
    598 	case BPF_FUNC_get_prandom_u32:
    599 		return &bpf_get_prandom_u32_proto;
    600 	case BPF_FUNC_probe_read_str:
    601 		return &bpf_probe_read_str_proto;
    602 #ifdef CONFIG_CGROUPS
    603 	case BPF_FUNC_get_current_cgroup_id:
    604 		return &bpf_get_current_cgroup_id_proto;
    605 #endif
    606 	default:
    607 		return NULL;
    608 	}
    609 }
    610 
    611 static const struct bpf_func_proto *
    612 kprobe_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
    613 {
    614 	switch (func_id) {
    615 	case BPF_FUNC_perf_event_output:
    616 		return &bpf_perf_event_output_proto;
    617 	case BPF_FUNC_get_stackid:
    618 		return &bpf_get_stackid_proto;
    619 	case BPF_FUNC_get_stack:
    620 		return &bpf_get_stack_proto;
    621 	case BPF_FUNC_perf_event_read_value:
    622 		return &bpf_perf_event_read_value_proto;
    623 #ifdef CONFIG_BPF_KPROBE_OVERRIDE
    624 	case BPF_FUNC_override_return:
    625 		return &bpf_override_return_proto;
    626 #endif
    627 	default:
    628 		return tracing_func_proto(func_id, prog);
    629 	}
    630 }
    631 
    632 /* bpf+kprobe programs can access fields of 'struct pt_regs' */
    633 static bool kprobe_prog_is_valid_access(int off, int size, enum bpf_access_type type,
    634 					const struct bpf_prog *prog,
    635 					struct bpf_insn_access_aux *info)
    636 {
    637 	if (off < 0 || off >= sizeof(struct pt_regs))
    638 		return false;
    639 	if (type != BPF_READ)
    640 		return false;
    641 	if (off % size != 0)
    642 		return false;
    643 	/*
    644 	 * Assertion for 32 bit to make sure last 8 byte access
    645 	 * (BPF_DW) to the last 4 byte member is disallowed.
    646 	 */
    647 	if (off + size > sizeof(struct pt_regs))
    648 		return false;
    649 
    650 	return true;
    651 }
    652 
    653 const struct bpf_verifier_ops kprobe_verifier_ops = {
    654 	.get_func_proto  = kprobe_prog_func_proto,
    655 	.is_valid_access = kprobe_prog_is_valid_access,
    656 };
    657 
    658 const struct bpf_prog_ops kprobe_prog_ops = {
    659 };
    660 
    661 BPF_CALL_5(bpf_perf_event_output_tp, void *, tp_buff, struct bpf_map *, map,
    662 	   u64, flags, void *, data, u64, size)
    663 {
    664 	struct pt_regs *regs = *(struct pt_regs **)tp_buff;
    665 
    666 	/*
    667 	 * r1 points to perf tracepoint buffer where first 8 bytes are hidden
    668 	 * from bpf program and contain a pointer to 'struct pt_regs'. Fetch it
    669 	 * from there and call the same bpf_perf_event_output() helper inline.
    670 	 */
    671 	return ____bpf_perf_event_output(regs, map, flags, data, size);
    672 }
    673 
    674 static const struct bpf_func_proto bpf_perf_event_output_proto_tp = {
    675 	.func		= bpf_perf_event_output_tp,
    676 	.gpl_only	= true,
    677 	.ret_type	= RET_INTEGER,
    678 	.arg1_type	= ARG_PTR_TO_CTX,
    679 	.arg2_type	= ARG_CONST_MAP_PTR,
    680 	.arg3_type	= ARG_ANYTHING,
    681 	.arg4_type	= ARG_PTR_TO_MEM,
    682 	.arg5_type	= ARG_CONST_SIZE_OR_ZERO,
    683 };
    684 
    685 BPF_CALL_3(bpf_get_stackid_tp, void *, tp_buff, struct bpf_map *, map,
    686 	   u64, flags)
    687 {
    688 	struct pt_regs *regs = *(struct pt_regs **)tp_buff;
    689 
    690 	/*
    691 	 * Same comment as in bpf_perf_event_output_tp(), only that this time
    692 	 * the other helper's function body cannot be inlined due to being
    693 	 * external, thus we need to call raw helper function.
    694 	 */
    695 	return bpf_get_stackid((unsigned long) regs, (unsigned long) map,
    696 			       flags, 0, 0);
    697 }
    698 
    699 static const struct bpf_func_proto bpf_get_stackid_proto_tp = {
    700 	.func		= bpf_get_stackid_tp,
    701 	.gpl_only	= true,
    702 	.ret_type	= RET_INTEGER,
    703 	.arg1_type	= ARG_PTR_TO_CTX,
    704 	.arg2_type	= ARG_CONST_MAP_PTR,
    705 	.arg3_type	= ARG_ANYTHING,
    706 };
    707 
    708 BPF_CALL_4(bpf_get_stack_tp, void *, tp_buff, void *, buf, u32, size,
    709 	   u64, flags)
    710 {
    711 	struct pt_regs *regs = *(struct pt_regs **)tp_buff;
    712 
    713 	return bpf_get_stack((unsigned long) regs, (unsigned long) buf,
    714 			     (unsigned long) size, flags, 0);
    715 }
    716 
    717 static const struct bpf_func_proto bpf_get_stack_proto_tp = {
    718 	.func		= bpf_get_stack_tp,
    719 	.gpl_only	= true,
    720 	.ret_type	= RET_INTEGER,
    721 	.arg1_type	= ARG_PTR_TO_CTX,
    722 	.arg2_type	= ARG_PTR_TO_UNINIT_MEM,
    723 	.arg3_type	= ARG_CONST_SIZE_OR_ZERO,
    724 	.arg4_type	= ARG_ANYTHING,
    725 };
    726 
    727 static const struct bpf_func_proto *
    728 tp_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
    729 {
    730 	switch (func_id) {
    731 	case BPF_FUNC_perf_event_output:
    732 		return &bpf_perf_event_output_proto_tp;
    733 	case BPF_FUNC_get_stackid:
    734 		return &bpf_get_stackid_proto_tp;
    735 	case BPF_FUNC_get_stack:
    736 		return &bpf_get_stack_proto_tp;
    737 	default:
    738 		return tracing_func_proto(func_id, prog);
    739 	}
    740 }
    741 
    742 static bool tp_prog_is_valid_access(int off, int size, enum bpf_access_type type,
    743 				    const struct bpf_prog *prog,
    744 				    struct bpf_insn_access_aux *info)
    745 {
    746 	if (off < sizeof(void *) || off >= PERF_MAX_TRACE_SIZE)
    747 		return false;
    748 	if (type != BPF_READ)
    749 		return false;
    750 	if (off % size != 0)
    751 		return false;
    752 
    753 	BUILD_BUG_ON(PERF_MAX_TRACE_SIZE % sizeof(__u64));
    754 	return true;
    755 }
    756 
    757 const struct bpf_verifier_ops tracepoint_verifier_ops = {
    758 	.get_func_proto  = tp_prog_func_proto,
    759 	.is_valid_access = tp_prog_is_valid_access,
    760 };
    761 
    762 const struct bpf_prog_ops tracepoint_prog_ops = {
    763 };
    764 
    765 BPF_CALL_3(bpf_perf_prog_read_value, struct bpf_perf_event_data_kern *, ctx,
    766 	   struct bpf_perf_event_value *, buf, u32, size)
    767 {
    768 	int err = -EINVAL;
    769 
    770 	if (unlikely(size != sizeof(struct bpf_perf_event_value)))
    771 		goto clear;
    772 	err = perf_event_read_local(ctx->event, &buf->counter, &buf->enabled,
    773 				    &buf->running);
    774 	if (unlikely(err))
    775 		goto clear;
    776 	return 0;
    777 clear:
    778 	memset(buf, 0, size);
    779 	return err;
    780 }
    781 
    782 static const struct bpf_func_proto bpf_perf_prog_read_value_proto = {
    783          .func           = bpf_perf_prog_read_value,
    784          .gpl_only       = true,
    785          .ret_type       = RET_INTEGER,
    786          .arg1_type      = ARG_PTR_TO_CTX,
    787          .arg2_type      = ARG_PTR_TO_UNINIT_MEM,
    788          .arg3_type      = ARG_CONST_SIZE,
    789 };
    790 
    791 static const struct bpf_func_proto *
    792 pe_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
    793 {
    794 	switch (func_id) {
    795 	case BPF_FUNC_perf_event_output:
    796 		return &bpf_perf_event_output_proto_tp;
    797 	case BPF_FUNC_get_stackid:
    798 		return &bpf_get_stackid_proto_tp;
    799 	case BPF_FUNC_get_stack:
    800 		return &bpf_get_stack_proto_tp;
    801 	case BPF_FUNC_perf_prog_read_value:
    802 		return &bpf_perf_prog_read_value_proto;
    803 	default:
    804 		return tracing_func_proto(func_id, prog);
    805 	}
    806 }
    807 
    808 /*
    809  * bpf_raw_tp_regs are separate from bpf_pt_regs used from skb/xdp
    810  * to avoid potential recursive reuse issue when/if tracepoints are added
    811  * inside bpf_*_event_output, bpf_get_stackid and/or bpf_get_stack
    812  */
    813 static DEFINE_PER_CPU(struct pt_regs, bpf_raw_tp_regs);
    814 BPF_CALL_5(bpf_perf_event_output_raw_tp, struct bpf_raw_tracepoint_args *, args,
    815 	   struct bpf_map *, map, u64, flags, void *, data, u64, size)
    816 {
    817 	struct pt_regs *regs = this_cpu_ptr(&bpf_raw_tp_regs);
    818 
    819 	perf_fetch_caller_regs(regs);
    820 	return ____bpf_perf_event_output(regs, map, flags, data, size);
    821 }
    822 
    823 static const struct bpf_func_proto bpf_perf_event_output_proto_raw_tp = {
    824 	.func		= bpf_perf_event_output_raw_tp,
    825 	.gpl_only	= true,
    826 	.ret_type	= RET_INTEGER,
    827 	.arg1_type	= ARG_PTR_TO_CTX,
    828 	.arg2_type	= ARG_CONST_MAP_PTR,
    829 	.arg3_type	= ARG_ANYTHING,
    830 	.arg4_type	= ARG_PTR_TO_MEM,
    831 	.arg5_type	= ARG_CONST_SIZE_OR_ZERO,
    832 };
    833 
    834 BPF_CALL_3(bpf_get_stackid_raw_tp, struct bpf_raw_tracepoint_args *, args,
    835 	   struct bpf_map *, map, u64, flags)
    836 {
    837 	struct pt_regs *regs = this_cpu_ptr(&bpf_raw_tp_regs);
    838 
    839 	perf_fetch_caller_regs(regs);
    840 	/* similar to bpf_perf_event_output_tp, but pt_regs fetched differently */
    841 	return bpf_get_stackid((unsigned long) regs, (unsigned long) map,
    842 			       flags, 0, 0);
    843 }
    844 
    845 static const struct bpf_func_proto bpf_get_stackid_proto_raw_tp = {
    846 	.func		= bpf_get_stackid_raw_tp,
    847 	.gpl_only	= true,
    848 	.ret_type	= RET_INTEGER,
    849 	.arg1_type	= ARG_PTR_TO_CTX,
    850 	.arg2_type	= ARG_CONST_MAP_PTR,
    851 	.arg3_type	= ARG_ANYTHING,
    852 };
    853 
    854 BPF_CALL_4(bpf_get_stack_raw_tp, struct bpf_raw_tracepoint_args *, args,
    855 	   void *, buf, u32, size, u64, flags)
    856 {
    857 	struct pt_regs *regs = this_cpu_ptr(&bpf_raw_tp_regs);
    858 
    859 	perf_fetch_caller_regs(regs);
    860 	return bpf_get_stack((unsigned long) regs, (unsigned long) buf,
    861 			     (unsigned long) size, flags, 0);
    862 }
    863 
    864 static const struct bpf_func_proto bpf_get_stack_proto_raw_tp = {
    865 	.func		= bpf_get_stack_raw_tp,
    866 	.gpl_only	= true,
    867 	.ret_type	= RET_INTEGER,
    868 	.arg1_type	= ARG_PTR_TO_CTX,
    869 	.arg2_type	= ARG_PTR_TO_MEM,
    870 	.arg3_type	= ARG_CONST_SIZE_OR_ZERO,
    871 	.arg4_type	= ARG_ANYTHING,
    872 };
    873 
    874 static const struct bpf_func_proto *
    875 raw_tp_prog_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
    876 {
    877 	switch (func_id) {
    878 	case BPF_FUNC_perf_event_output:
    879 		return &bpf_perf_event_output_proto_raw_tp;
    880 	case BPF_FUNC_get_stackid:
    881 		return &bpf_get_stackid_proto_raw_tp;
    882 	case BPF_FUNC_get_stack:
    883 		return &bpf_get_stack_proto_raw_tp;
    884 	default:
    885 		return tracing_func_proto(func_id, prog);
    886 	}
    887 }
    888 
    889 static bool raw_tp_prog_is_valid_access(int off, int size,
    890 					enum bpf_access_type type,
    891 					const struct bpf_prog *prog,
    892 					struct bpf_insn_access_aux *info)
    893 {
    894 	/* largest tracepoint in the kernel has 12 args */
    895 	if (off < 0 || off >= sizeof(__u64) * 12)
    896 		return false;
    897 	if (type != BPF_READ)
    898 		return false;
    899 	if (off % size != 0)
    900 		return false;
    901 	return true;
    902 }
    903 
    904 const struct bpf_verifier_ops raw_tracepoint_verifier_ops = {
    905 	.get_func_proto  = raw_tp_prog_func_proto,
    906 	.is_valid_access = raw_tp_prog_is_valid_access,
    907 };
    908 
    909 const struct bpf_prog_ops raw_tracepoint_prog_ops = {
    910 };
    911 
    912 static bool pe_prog_is_valid_access(int off, int size, enum bpf_access_type type,
    913 				    const struct bpf_prog *prog,
    914 				    struct bpf_insn_access_aux *info)
    915 {
    916 	const int size_u64 = sizeof(u64);
    917 
    918 	if (off < 0 || off >= sizeof(struct bpf_perf_event_data))
    919 		return false;
    920 	if (type != BPF_READ)
    921 		return false;
    922 	if (off % size != 0) {
    923 		if (sizeof(unsigned long) != 4)
    924 			return false;
    925 		if (size != 8)
    926 			return false;
    927 		if (off % size != 4)
    928 			return false;
    929 	}
    930 
    931 	switch (off) {
    932 	case bpf_ctx_range(struct bpf_perf_event_data, sample_period):
    933 		bpf_ctx_record_field_size(info, size_u64);
    934 		if (!bpf_ctx_narrow_access_ok(off, size, size_u64))
    935 			return false;
    936 		break;
    937 	case bpf_ctx_range(struct bpf_perf_event_data, addr):
    938 		bpf_ctx_record_field_size(info, size_u64);
    939 		if (!bpf_ctx_narrow_access_ok(off, size, size_u64))
    940 			return false;
    941 		break;
    942 	default:
    943 		if (size != sizeof(long))
    944 			return false;
    945 	}
    946 
    947 	return true;
    948 }
    949 
    950 static u32 pe_prog_convert_ctx_access(enum bpf_access_type type,
    951 				      const struct bpf_insn *si,
    952 				      struct bpf_insn *insn_buf,
    953 				      struct bpf_prog *prog, u32 *target_size)
    954 {
    955 	struct bpf_insn *insn = insn_buf;
    956 
    957 	switch (si->off) {
    958 	case offsetof(struct bpf_perf_event_data, sample_period):
    959 		*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct bpf_perf_event_data_kern,
    960 						       data), si->dst_reg, si->src_reg,
    961 				      offsetof(struct bpf_perf_event_data_kern, data));
    962 		*insn++ = BPF_LDX_MEM(BPF_DW, si->dst_reg, si->dst_reg,
    963 				      bpf_target_off(struct perf_sample_data, period, 8,
    964 						     target_size));
    965 		break;
    966 	case offsetof(struct bpf_perf_event_data, addr):
    967 		*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct bpf_perf_event_data_kern,
    968 						       data), si->dst_reg, si->src_reg,
    969 				      offsetof(struct bpf_perf_event_data_kern, data));
    970 		*insn++ = BPF_LDX_MEM(BPF_DW, si->dst_reg, si->dst_reg,
    971 				      bpf_target_off(struct perf_sample_data, addr, 8,
    972 						     target_size));
    973 		break;
    974 	default:
    975 		*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct bpf_perf_event_data_kern,
    976 						       regs), si->dst_reg, si->src_reg,
    977 				      offsetof(struct bpf_perf_event_data_kern, regs));
    978 		*insn++ = BPF_LDX_MEM(BPF_SIZEOF(long), si->dst_reg, si->dst_reg,
    979 				      si->off);
    980 		break;
    981 	}
    982 
    983 	return insn - insn_buf;
    984 }
    985 
    986 const struct bpf_verifier_ops perf_event_verifier_ops = {
    987 	.get_func_proto		= pe_prog_func_proto,
    988 	.is_valid_access	= pe_prog_is_valid_access,
    989 	.convert_ctx_access	= pe_prog_convert_ctx_access,
    990 };
    991 
    992 const struct bpf_prog_ops perf_event_prog_ops = {
    993 };
    994 
    995 static DEFINE_MUTEX(bpf_event_mutex);
    996 
    997 #define BPF_TRACE_MAX_PROGS 64
    998 
    999 int perf_event_attach_bpf_prog(struct perf_event *event,
   1000 			       struct bpf_prog *prog)
   1001 {
   1002 	struct bpf_prog_array __rcu *old_array;
   1003 	struct bpf_prog_array *new_array;
   1004 	int ret = -EEXIST;
   1005 
   1006 	/*
   1007 	 * Kprobe override only works if they are on the function entry,
   1008 	 * and only if they are on the opt-in list.
   1009 	 */
   1010 	if (prog->kprobe_override &&
   1011 	    (!trace_kprobe_on_func_entry(event->tp_event) ||
   1012 	     !trace_kprobe_error_injectable(event->tp_event)))
   1013 		return -EINVAL;
   1014 
   1015 	mutex_lock(&bpf_event_mutex);
   1016 
   1017 	if (event->prog)
   1018 		goto unlock;
   1019 
   1020 	old_array = event->tp_event->prog_array;
   1021 	if (old_array &&
   1022 	    bpf_prog_array_length(old_array) >= BPF_TRACE_MAX_PROGS) {
   1023 		ret = -E2BIG;
   1024 		goto unlock;
   1025 	}
   1026 
   1027 	ret = bpf_prog_array_copy(old_array, NULL, prog, &new_array);
   1028 	if (ret < 0)
   1029 		goto unlock;
   1030 
   1031 	/* set the new array to event->tp_event and set event->prog */
   1032 	event->prog = prog;
   1033 	rcu_assign_pointer(event->tp_event->prog_array, new_array);
   1034 	bpf_prog_array_free(old_array);
   1035 
   1036 unlock:
   1037 	mutex_unlock(&bpf_event_mutex);
   1038 	return ret;
   1039 }
   1040 
   1041 void perf_event_detach_bpf_prog(struct perf_event *event)
   1042 {
   1043 	struct bpf_prog_array __rcu *old_array;
   1044 	struct bpf_prog_array *new_array;
   1045 	int ret;
   1046 
   1047 	mutex_lock(&bpf_event_mutex);
   1048 
   1049 	if (!event->prog)
   1050 		goto unlock;
   1051 
   1052 	old_array = event->tp_event->prog_array;
   1053 	ret = bpf_prog_array_copy(old_array, event->prog, NULL, &new_array);
   1054 	if (ret == -ENOENT)
   1055 		goto unlock;
   1056 	if (ret < 0) {
   1057 		bpf_prog_array_delete_safe(old_array, event->prog);
   1058 	} else {
   1059 		rcu_assign_pointer(event->tp_event->prog_array, new_array);
   1060 		bpf_prog_array_free(old_array);
   1061 	}
   1062 
   1063 	bpf_prog_put(event->prog);
   1064 	event->prog = NULL;
   1065 
   1066 unlock:
   1067 	mutex_unlock(&bpf_event_mutex);
   1068 }
   1069 
   1070 int perf_event_query_prog_array(struct perf_event *event, void __user *info)
   1071 {
   1072 	struct perf_event_query_bpf __user *uquery = info;
   1073 	struct perf_event_query_bpf query = {};
   1074 	u32 *ids, prog_cnt, ids_len;
   1075 	int ret;
   1076 
   1077 	if (!capable(CAP_SYS_ADMIN))
   1078 		return -EPERM;
   1079 	if (event->attr.type != PERF_TYPE_TRACEPOINT)
   1080 		return -EINVAL;
   1081 	if (copy_from_user(&query, uquery, sizeof(query)))
   1082 		return -EFAULT;
   1083 
   1084 	ids_len = query.ids_len;
   1085 	if (ids_len > BPF_TRACE_MAX_PROGS)
   1086 		return -E2BIG;
   1087 	ids = kcalloc(ids_len, sizeof(u32), GFP_USER | __GFP_NOWARN);
   1088 	if (!ids)
   1089 		return -ENOMEM;
   1090 	/*
   1091 	 * The above kcalloc returns ZERO_SIZE_PTR when ids_len = 0, which
   1092 	 * is required when user only wants to check for uquery->prog_cnt.
   1093 	 * There is no need to check for it since the case is handled
   1094 	 * gracefully in bpf_prog_array_copy_info.
   1095 	 */
   1096 
   1097 	mutex_lock(&bpf_event_mutex);
   1098 	ret = bpf_prog_array_copy_info(event->tp_event->prog_array,
   1099 				       ids,
   1100 				       ids_len,
   1101 				       &prog_cnt);
   1102 	mutex_unlock(&bpf_event_mutex);
   1103 
   1104 	if (copy_to_user(&uquery->prog_cnt, &prog_cnt, sizeof(prog_cnt)) ||
   1105 	    copy_to_user(uquery->ids, ids, ids_len * sizeof(u32)))
   1106 		ret = -EFAULT;
   1107 
   1108 	kfree(ids);
   1109 	return ret;
   1110 }
   1111 
   1112 extern struct bpf_raw_event_map __start__bpf_raw_tp[];
   1113 extern struct bpf_raw_event_map __stop__bpf_raw_tp[];
   1114 
   1115 struct bpf_raw_event_map *bpf_get_raw_tracepoint(const char *name)
   1116 {
   1117 	struct bpf_raw_event_map *btp = __start__bpf_raw_tp;
   1118 
   1119 	for (; btp < __stop__bpf_raw_tp; btp++) {
   1120 		if (!strcmp(btp->tp->name, name))
   1121 			return btp;
   1122 	}
   1123 
   1124 	return bpf_get_raw_tracepoint_module(name);
   1125 }
   1126 
   1127 void bpf_put_raw_tracepoint(struct bpf_raw_event_map *btp)
   1128 {
   1129 	struct module *mod = __module_address((unsigned long)btp);
   1130 
   1131 	if (mod)
   1132 		module_put(mod);
   1133 }
   1134 
   1135 static __always_inline
   1136 void __bpf_trace_run(struct bpf_prog *prog, u64 *args)
   1137 {
   1138 	rcu_read_lock();
   1139 	preempt_disable();
   1140 	(void) BPF_PROG_RUN(prog, args);
   1141 	preempt_enable();
   1142 	rcu_read_unlock();
   1143 }
   1144 
   1145 #define UNPACK(...)			__VA_ARGS__
   1146 #define REPEAT_1(FN, DL, X, ...)	FN(X)
   1147 #define REPEAT_2(FN, DL, X, ...)	FN(X) UNPACK DL REPEAT_1(FN, DL, __VA_ARGS__)
   1148 #define REPEAT_3(FN, DL, X, ...)	FN(X) UNPACK DL REPEAT_2(FN, DL, __VA_ARGS__)
   1149 #define REPEAT_4(FN, DL, X, ...)	FN(X) UNPACK DL REPEAT_3(FN, DL, __VA_ARGS__)
   1150 #define REPEAT_5(FN, DL, X, ...)	FN(X) UNPACK DL REPEAT_4(FN, DL, __VA_ARGS__)
   1151 #define REPEAT_6(FN, DL, X, ...)	FN(X) UNPACK DL REPEAT_5(FN, DL, __VA_ARGS__)
   1152 #define REPEAT_7(FN, DL, X, ...)	FN(X) UNPACK DL REPEAT_6(FN, DL, __VA_ARGS__)
   1153 #define REPEAT_8(FN, DL, X, ...)	FN(X) UNPACK DL REPEAT_7(FN, DL, __VA_ARGS__)
   1154 #define REPEAT_9(FN, DL, X, ...)	FN(X) UNPACK DL REPEAT_8(FN, DL, __VA_ARGS__)
   1155 #define REPEAT_10(FN, DL, X, ...)	FN(X) UNPACK DL REPEAT_9(FN, DL, __VA_ARGS__)
   1156 #define REPEAT_11(FN, DL, X, ...)	FN(X) UNPACK DL REPEAT_10(FN, DL, __VA_ARGS__)
   1157 #define REPEAT_12(FN, DL, X, ...)	FN(X) UNPACK DL REPEAT_11(FN, DL, __VA_ARGS__)
   1158 #define REPEAT(X, FN, DL, ...)		REPEAT_##X(FN, DL, __VA_ARGS__)
   1159 
   1160 #define SARG(X)		u64 arg##X
   1161 #define COPY(X)		args[X] = arg##X
   1162 
   1163 #define __DL_COM	(,)
   1164 #define __DL_SEM	(;)
   1165 
   1166 #define __SEQ_0_11	0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11
   1167 
   1168 #define BPF_TRACE_DEFN_x(x)						\
   1169 	void bpf_trace_run##x(struct bpf_prog *prog,			\
   1170 			      REPEAT(x, SARG, __DL_COM, __SEQ_0_11))	\
   1171 	{								\
   1172 		u64 args[x];						\
   1173 		REPEAT(x, COPY, __DL_SEM, __SEQ_0_11);			\
   1174 		__bpf_trace_run(prog, args);				\
   1175 	}								\
   1176 	EXPORT_SYMBOL_GPL(bpf_trace_run##x)
   1177 BPF_TRACE_DEFN_x(1);
   1178 BPF_TRACE_DEFN_x(2);
   1179 BPF_TRACE_DEFN_x(3);
   1180 BPF_TRACE_DEFN_x(4);
   1181 BPF_TRACE_DEFN_x(5);
   1182 BPF_TRACE_DEFN_x(6);
   1183 BPF_TRACE_DEFN_x(7);
   1184 BPF_TRACE_DEFN_x(8);
   1185 BPF_TRACE_DEFN_x(9);
   1186 BPF_TRACE_DEFN_x(10);
   1187 BPF_TRACE_DEFN_x(11);
   1188 BPF_TRACE_DEFN_x(12);
   1189 
   1190 static int __bpf_probe_register(struct bpf_raw_event_map *btp, struct bpf_prog *prog)
   1191 {
   1192 	struct tracepoint *tp = btp->tp;
   1193 
   1194 	/*
   1195 	 * check that program doesn't access arguments beyond what's
   1196 	 * available in this tracepoint
   1197 	 */
   1198 	if (prog->aux->max_ctx_offset > btp->num_args * sizeof(u64))
   1199 		return -EINVAL;
   1200 
   1201 	return tracepoint_probe_register(tp, (void *)btp->bpf_func, prog);
   1202 }
   1203 
   1204 int bpf_probe_register(struct bpf_raw_event_map *btp, struct bpf_prog *prog)
   1205 {
   1206 	return __bpf_probe_register(btp, prog);
   1207 }
   1208 
   1209 int bpf_probe_unregister(struct bpf_raw_event_map *btp, struct bpf_prog *prog)
   1210 {
   1211 	return tracepoint_probe_unregister(btp->tp, (void *)btp->bpf_func, prog);
   1212 }
   1213 
   1214 int bpf_get_perf_event_info(const struct perf_event *event, u32 *prog_id,
   1215 			    u32 *fd_type, const char **buf,
   1216 			    u64 *probe_offset, u64 *probe_addr)
   1217 {
   1218 	bool is_tracepoint, is_syscall_tp;
   1219 	struct bpf_prog *prog;
   1220 	int flags, err = 0;
   1221 
   1222 	prog = event->prog;
   1223 	if (!prog)
   1224 		return -ENOENT;
   1225 
   1226 	/* not supporting BPF_PROG_TYPE_PERF_EVENT yet */
   1227 	if (prog->type == BPF_PROG_TYPE_PERF_EVENT)
   1228 		return -EOPNOTSUPP;
   1229 
   1230 	*prog_id = prog->aux->id;
   1231 	flags = event->tp_event->flags;
   1232 	is_tracepoint = flags & TRACE_EVENT_FL_TRACEPOINT;
   1233 	is_syscall_tp = is_syscall_trace_event(event->tp_event);
   1234 
   1235 	if (is_tracepoint || is_syscall_tp) {
   1236 		*buf = is_tracepoint ? event->tp_event->tp->name
   1237 				     : event->tp_event->name;
   1238 		*fd_type = BPF_FD_TYPE_TRACEPOINT;
   1239 		*probe_offset = 0x0;
   1240 		*probe_addr = 0x0;
   1241 	} else {
   1242 		/* kprobe/uprobe */
   1243 		err = -EOPNOTSUPP;
   1244 #ifdef CONFIG_KPROBE_EVENTS
   1245 		if (flags & TRACE_EVENT_FL_KPROBE)
   1246 			err = bpf_get_kprobe_info(event, fd_type, buf,
   1247 						  probe_offset, probe_addr,
   1248 						  event->attr.type == PERF_TYPE_TRACEPOINT);
   1249 #endif
   1250 #ifdef CONFIG_UPROBE_EVENTS
   1251 		if (flags & TRACE_EVENT_FL_UPROBE)
   1252 			err = bpf_get_uprobe_info(event, fd_type, buf,
   1253 						  probe_offset,
   1254 						  event->attr.type == PERF_TYPE_TRACEPOINT);
   1255 #endif
   1256 	}
   1257 
   1258 	return err;
   1259 }
   1260 
   1261 #ifdef CONFIG_MODULES
   1262 int bpf_event_notify(struct notifier_block *nb, unsigned long op, void *module)
   1263 {
   1264 	struct bpf_trace_module *btm, *tmp;
   1265 	struct module *mod = module;
   1266 
   1267 	if (mod->num_bpf_raw_events == 0 ||
   1268 	    (op != MODULE_STATE_COMING && op != MODULE_STATE_GOING))
   1269 		return 0;
   1270 
   1271 	mutex_lock(&bpf_module_mutex);
   1272 
   1273 	switch (op) {
   1274 	case MODULE_STATE_COMING:
   1275 		btm = kzalloc(sizeof(*btm), GFP_KERNEL);
   1276 		if (btm) {
   1277 			btm->module = module;
   1278 			list_add(&btm->list, &bpf_trace_modules);
   1279 		}
   1280 		break;
   1281 	case MODULE_STATE_GOING:
   1282 		list_for_each_entry_safe(btm, tmp, &bpf_trace_modules, list) {
   1283 			if (btm->module == module) {
   1284 				list_del(&btm->list);
   1285 				kfree(btm);
   1286 				break;
   1287 			}
   1288 		}
   1289 		break;
   1290 	}
   1291 
   1292 	mutex_unlock(&bpf_module_mutex);
   1293 
   1294 	return 0;
   1295 }
   1296 
   1297 static struct notifier_block bpf_module_nb = {
   1298 	.notifier_call = bpf_event_notify,
   1299 };
   1300 
   1301 int __init bpf_event_init(void)
   1302 {
   1303 	register_module_notifier(&bpf_module_nb);
   1304 	return 0;
   1305 }
   1306 
   1307 fs_initcall(bpf_event_init);
   1308 #endif /* CONFIG_MODULES */