Blame view
kernel/trace/bpf_trace.c
17.8 KB
2541517c3 tracing, perf: Im... |
1 |
/* Copyright (c) 2011-2015 PLUMgrid, http://plumgrid.com |
0515e5999 bpf: introduce BP... |
2 |
* Copyright (c) 2016 Facebook |
2541517c3 tracing, perf: Im... |
3 4 5 6 7 8 9 10 11 |
* * This program is free software; you can redistribute it and/or * modify it under the terms of version 2 of the GNU General Public * License as published by the Free Software Foundation. */ #include <linux/kernel.h> #include <linux/types.h> #include <linux/slab.h> #include <linux/bpf.h> |
0515e5999 bpf: introduce BP... |
12 |
#include <linux/bpf_perf_event.h> |
2541517c3 tracing, perf: Im... |
13 14 |
#include <linux/filter.h> #include <linux/uaccess.h> |
9c959c863 tracing: Allow BP... |
15 |
#include <linux/ctype.h> |
2541517c3 tracing, perf: Im... |
16 |
#include "trace.h" |
2541517c3 tracing, perf: Im... |
17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 |
/** * trace_call_bpf - invoke BPF program * @prog: BPF program * @ctx: opaque context pointer * * kprobe handlers execute BPF programs via this helper. * Can be used from static tracepoints in the future. * * Return: BPF programs always return an integer which is interpreted by * kprobe handler as: * 0 - return from kprobe (event is filtered out) * 1 - store kprobe event into ring buffer * Other values are reserved and currently alias to 1 */ unsigned int trace_call_bpf(struct bpf_prog *prog, void *ctx) { unsigned int ret; if (in_nmi()) /* not supported yet */ return 1; preempt_disable(); if (unlikely(__this_cpu_inc_return(bpf_prog_active) != 1)) { /* * since some bpf program is already running on this cpu, * don't call into another bpf program (same or different) * and don't send kprobe event into ring-buffer, * so return zero here */ ret = 0; goto out; } rcu_read_lock(); ret = BPF_PROG_RUN(prog, ctx); rcu_read_unlock(); out: __this_cpu_dec(bpf_prog_active); preempt_enable(); return ret; } EXPORT_SYMBOL_GPL(trace_call_bpf); |
f3694e001 bpf: add BPF_CALL... |
62 |
BPF_CALL_3(bpf_probe_read, void *, dst, u32, size, const void *, unsafe_ptr) |
2541517c3 tracing, perf: Im... |
63 |
{ |
f3694e001 bpf: add BPF_CALL... |
64 |
int ret; |
2541517c3 tracing, perf: Im... |
65 |
|
074f528ee bpf: convert rele... |
66 67 68 69 70 |
ret = probe_kernel_read(dst, unsafe_ptr, size); if (unlikely(ret < 0)) memset(dst, 0, size); return ret; |
2541517c3 tracing, perf: Im... |
71 72 73 74 75 76 |
} static const struct bpf_func_proto bpf_probe_read_proto = { .func = bpf_probe_read, .gpl_only = true, .ret_type = RET_INTEGER, |
39f19ebbf bpf: rename ARG_P... |
77 78 |
.arg1_type = ARG_PTR_TO_UNINIT_MEM, .arg2_type = ARG_CONST_SIZE, |
2541517c3 tracing, perf: Im... |
79 80 |
.arg3_type = ARG_ANYTHING, }; |
f3694e001 bpf: add BPF_CALL... |
81 82 |
BPF_CALL_3(bpf_probe_write_user, void *, unsafe_ptr, const void *, src, u32, size) |
96ae52279 bpf: Add bpf_prob... |
83 |
{ |
96ae52279 bpf: Add bpf_prob... |
84 85 86 87 88 89 90 91 92 93 94 95 |
/* * Ensure we're in user context which is safe for the helper to * run. This helper has no business in a kthread. * * access_ok() should prevent writing to non-user memory, but in * some situations (nommu, temporary switch, etc) access_ok() does * not provide enough validation, hence the check on KERNEL_DS. */ if (unlikely(in_interrupt() || current->flags & (PF_KTHREAD | PF_EXITING))) return -EPERM; |
db68ce10c new helper: uacce... |
96 |
if (unlikely(uaccess_kernel())) |
96ae52279 bpf: Add bpf_prob... |
97 98 99 100 101 102 103 104 105 106 107 108 |
return -EPERM; if (!access_ok(VERIFY_WRITE, unsafe_ptr, size)) return -EPERM; return probe_kernel_write(unsafe_ptr, src, size); } static const struct bpf_func_proto bpf_probe_write_user_proto = { .func = bpf_probe_write_user, .gpl_only = true, .ret_type = RET_INTEGER, .arg1_type = ARG_ANYTHING, |
39f19ebbf bpf: rename ARG_P... |
109 110 |
.arg2_type = ARG_PTR_TO_MEM, .arg3_type = ARG_CONST_SIZE, |
96ae52279 bpf: Add bpf_prob... |
111 112 113 114 115 116 117 118 119 |
}; static const struct bpf_func_proto *bpf_get_probe_write_proto(void) { pr_warn_ratelimited("%s[%d] is installing a program with bpf_probe_write_user helper that may corrupt user memory!", current->comm, task_pid_nr(current)); return &bpf_probe_write_user_proto; } |
9c959c863 tracing: Allow BP... |
120 |
/* |
7bda4b40c bpf: extend bpf_t... |
121 122 |
* Only limited trace_printk() conversion specifiers allowed: * %d %i %u %x %ld %li %lu %lx %lld %lli %llu %llx %p %s |
9c959c863 tracing: Allow BP... |
123 |
*/ |
f3694e001 bpf: add BPF_CALL... |
124 125 |
BPF_CALL_5(bpf_trace_printk, char *, fmt, u32, fmt_size, u64, arg1, u64, arg2, u64, arg3) |
9c959c863 tracing: Allow BP... |
126 |
{ |
8d3b7dce8 bpf: add support ... |
127 |
bool str_seen = false; |
9c959c863 tracing: Allow BP... |
128 129 |
int mod[3] = {}; int fmt_cnt = 0; |
8d3b7dce8 bpf: add support ... |
130 131 |
u64 unsafe_addr; char buf[64]; |
9c959c863 tracing: Allow BP... |
132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 |
int i; /* * bpf_check()->check_func_arg()->check_stack_boundary() * guarantees that fmt points to bpf program stack, * fmt_size bytes of it were initialized and fmt_size > 0 */ if (fmt[--fmt_size] != 0) return -EINVAL; /* check format string for allowed specifiers */ for (i = 0; i < fmt_size; i++) { if ((!isprint(fmt[i]) && !isspace(fmt[i])) || !isascii(fmt[i])) return -EINVAL; if (fmt[i] != '%') continue; if (fmt_cnt >= 3) return -EINVAL; /* fmt[i] != 0 && fmt[last] == 0, so we can access fmt[i + 1] */ i++; if (fmt[i] == 'l') { mod[fmt_cnt]++; i++; |
8d3b7dce8 bpf: add support ... |
158 |
} else if (fmt[i] == 'p' || fmt[i] == 's') { |
9c959c863 tracing: Allow BP... |
159 |
mod[fmt_cnt]++; |
9209043b2 bpf: fix check of... |
160 161 162 163 |
/* disallow any further format extensions */ if (fmt[i + 1] != 0 && !isspace(fmt[i + 1]) && !ispunct(fmt[i + 1])) |
9c959c863 tracing: Allow BP... |
164 165 |
return -EINVAL; fmt_cnt++; |
9209043b2 bpf: fix check of... |
166 |
if (fmt[i] == 's') { |
8d3b7dce8 bpf: add support ... |
167 168 169 170 171 172 173 |
if (str_seen) /* allow only one '%s' per fmt string */ return -EINVAL; str_seen = true; switch (fmt_cnt) { case 1: |
f3694e001 bpf: add BPF_CALL... |
174 175 |
unsafe_addr = arg1; arg1 = (long) buf; |
8d3b7dce8 bpf: add support ... |
176 177 |
break; case 2: |
f3694e001 bpf: add BPF_CALL... |
178 179 |
unsafe_addr = arg2; arg2 = (long) buf; |
8d3b7dce8 bpf: add support ... |
180 181 |
break; case 3: |
f3694e001 bpf: add BPF_CALL... |
182 183 |
unsafe_addr = arg3; arg3 = (long) buf; |
8d3b7dce8 bpf: add support ... |
184 185 186 187 188 189 190 |
break; } buf[0] = 0; strncpy_from_unsafe(buf, (void *) (long) unsafe_addr, sizeof(buf)); } |
9c959c863 tracing: Allow BP... |
191 192 193 194 195 196 197 |
continue; } if (fmt[i] == 'l') { mod[fmt_cnt]++; i++; } |
7bda4b40c bpf: extend bpf_t... |
198 199 |
if (fmt[i] != 'i' && fmt[i] != 'd' && fmt[i] != 'u' && fmt[i] != 'x') |
9c959c863 tracing: Allow BP... |
200 201 202 |
return -EINVAL; fmt_cnt++; } |
88a5c690b bpf: fix bpf_trac... |
203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 |
/* Horrid workaround for getting va_list handling working with different * argument type combinations generically for 32 and 64 bit archs. */ #define __BPF_TP_EMIT() __BPF_ARG3_TP() #define __BPF_TP(...) \ __trace_printk(1 /* Fake ip will not be printed. */, \ fmt, ##__VA_ARGS__) #define __BPF_ARG1_TP(...) \ ((mod[0] == 2 || (mod[0] == 1 && __BITS_PER_LONG == 64)) \ ? __BPF_TP(arg1, ##__VA_ARGS__) \ : ((mod[0] == 1 || (mod[0] == 0 && __BITS_PER_LONG == 32)) \ ? __BPF_TP((long)arg1, ##__VA_ARGS__) \ : __BPF_TP((u32)arg1, ##__VA_ARGS__))) #define __BPF_ARG2_TP(...) \ ((mod[1] == 2 || (mod[1] == 1 && __BITS_PER_LONG == 64)) \ ? __BPF_ARG1_TP(arg2, ##__VA_ARGS__) \ : ((mod[1] == 1 || (mod[1] == 0 && __BITS_PER_LONG == 32)) \ ? __BPF_ARG1_TP((long)arg2, ##__VA_ARGS__) \ : __BPF_ARG1_TP((u32)arg2, ##__VA_ARGS__))) #define __BPF_ARG3_TP(...) \ ((mod[2] == 2 || (mod[2] == 1 && __BITS_PER_LONG == 64)) \ ? __BPF_ARG2_TP(arg3, ##__VA_ARGS__) \ : ((mod[2] == 1 || (mod[2] == 0 && __BITS_PER_LONG == 32)) \ ? __BPF_ARG2_TP((long)arg3, ##__VA_ARGS__) \ : __BPF_ARG2_TP((u32)arg3, ##__VA_ARGS__))) return __BPF_TP_EMIT(); |
9c959c863 tracing: Allow BP... |
233 234 235 236 237 238 |
} static const struct bpf_func_proto bpf_trace_printk_proto = { .func = bpf_trace_printk, .gpl_only = true, .ret_type = RET_INTEGER, |
39f19ebbf bpf: rename ARG_P... |
239 240 |
.arg1_type = ARG_PTR_TO_MEM, .arg2_type = ARG_CONST_SIZE, |
9c959c863 tracing: Allow BP... |
241 |
}; |
0756ea3e8 bpf: allow networ... |
242 243 244 245 246 247 248 249 250 251 |
const struct bpf_func_proto *bpf_get_trace_printk_proto(void) { /* * this program might be calling bpf_trace_printk, * so allocate per-cpu printk buffers */ trace_printk_init_buffers(); return &bpf_trace_printk_proto; } |
f3694e001 bpf: add BPF_CALL... |
252 |
BPF_CALL_2(bpf_perf_event_read, struct bpf_map *, map, u64, flags) |
35578d798 bpf: Implement fu... |
253 |
{ |
35578d798 bpf: Implement fu... |
254 |
struct bpf_array *array = container_of(map, struct bpf_array, map); |
6816a7ffc bpf, trace: add B... |
255 256 |
unsigned int cpu = smp_processor_id(); u64 index = flags & BPF_F_INDEX_MASK; |
3b1efb196 bpf, maps: flush ... |
257 |
struct bpf_event_entry *ee; |
f91840a32 perf, bpf: Add BP... |
258 259 |
u64 value = 0; int err; |
35578d798 bpf: Implement fu... |
260 |
|
6816a7ffc bpf, trace: add B... |
261 262 263 264 |
if (unlikely(flags & ~(BPF_F_INDEX_MASK))) return -EINVAL; if (index == BPF_F_CURRENT_CPU) index = cpu; |
35578d798 bpf: Implement fu... |
265 266 |
if (unlikely(index >= array->map.max_entries)) return -E2BIG; |
3b1efb196 bpf, maps: flush ... |
267 |
ee = READ_ONCE(array->ptrs[index]); |
1ca1cc98b bpf: minor cleanu... |
268 |
if (!ee) |
35578d798 bpf: Implement fu... |
269 |
return -ENOENT; |
f91840a32 perf, bpf: Add BP... |
270 |
err = perf_event_read_local(ee->event, &value); |
35578d798 bpf: Implement fu... |
271 |
/* |
f91840a32 perf, bpf: Add BP... |
272 273 |
* this api is ugly since we miss [-22..-2] range of valid * counter values, but that's uapi |
35578d798 bpf: Implement fu... |
274 |
*/ |
f91840a32 perf, bpf: Add BP... |
275 276 277 |
if (err) return err; return value; |
35578d798 bpf: Implement fu... |
278 |
} |
62544ce8e bpf: fix bpf_perf... |
279 |
static const struct bpf_func_proto bpf_perf_event_read_proto = { |
35578d798 bpf: Implement fu... |
280 |
.func = bpf_perf_event_read, |
1075ef595 bpf: make tracing... |
281 |
.gpl_only = true, |
35578d798 bpf: Implement fu... |
282 283 284 285 |
.ret_type = RET_INTEGER, .arg1_type = ARG_CONST_MAP_PTR, .arg2_type = ARG_ANYTHING, }; |
a23244e88 bpf: fix corrupti... |
286 |
static DEFINE_PER_CPU(struct perf_sample_data, bpf_trace_sd); |
20b9d7ac4 bpf: avoid excess... |
287 |
|
8e7a3920a bpf, perf: split ... |
288 289 |
static __always_inline u64 __bpf_perf_event_output(struct pt_regs *regs, struct bpf_map *map, |
a23244e88 bpf: fix corrupti... |
290 |
u64 flags, struct perf_sample_data *sd) |
a43eec304 bpf: introduce bp... |
291 |
{ |
a43eec304 bpf: introduce bp... |
292 |
struct bpf_array *array = container_of(map, struct bpf_array, map); |
d79313303 bpf, trace: fetch... |
293 |
unsigned int cpu = smp_processor_id(); |
1e33759c7 bpf, trace: add B... |
294 |
u64 index = flags & BPF_F_INDEX_MASK; |
3b1efb196 bpf, maps: flush ... |
295 |
struct bpf_event_entry *ee; |
a43eec304 bpf: introduce bp... |
296 |
struct perf_event *event; |
a43eec304 bpf: introduce bp... |
297 |
|
1e33759c7 bpf, trace: add B... |
298 |
if (index == BPF_F_CURRENT_CPU) |
d79313303 bpf, trace: fetch... |
299 |
index = cpu; |
a43eec304 bpf: introduce bp... |
300 301 |
if (unlikely(index >= array->map.max_entries)) return -E2BIG; |
3b1efb196 bpf, maps: flush ... |
302 |
ee = READ_ONCE(array->ptrs[index]); |
1ca1cc98b bpf: minor cleanu... |
303 |
if (!ee) |
a43eec304 bpf: introduce bp... |
304 |
return -ENOENT; |
3b1efb196 bpf, maps: flush ... |
305 |
event = ee->event; |
a43eec304 bpf: introduce bp... |
306 307 308 |
if (unlikely(event->attr.type != PERF_TYPE_SOFTWARE || event->attr.config != PERF_COUNT_SW_BPF_OUTPUT)) return -EINVAL; |
d79313303 bpf, trace: fetch... |
309 |
if (unlikely(event->oncpu != cpu)) |
a43eec304 bpf: introduce bp... |
310 |
return -EOPNOTSUPP; |
20b9d7ac4 bpf: avoid excess... |
311 |
perf_event_output(event, sd, regs); |
a43eec304 bpf: introduce bp... |
312 313 |
return 0; } |
f3694e001 bpf: add BPF_CALL... |
314 315 |
BPF_CALL_5(bpf_perf_event_output, struct pt_regs *, regs, struct bpf_map *, map, u64, flags, void *, data, u64, size) |
8e7a3920a bpf, perf: split ... |
316 |
{ |
a23244e88 bpf: fix corrupti... |
317 |
struct perf_sample_data *sd = this_cpu_ptr(&bpf_trace_sd); |
8e7a3920a bpf, perf: split ... |
318 319 320 321 322 323 324 325 326 |
struct perf_raw_record raw = { .frag = { .size = size, .data = data, }, }; if (unlikely(flags & ~(BPF_F_INDEX_MASK))) return -EINVAL; |
a23244e88 bpf: fix corrupti... |
327 328 329 330 |
perf_sample_data_init(sd, 0, 0); sd->raw = &raw; return __bpf_perf_event_output(regs, map, flags, sd); |
8e7a3920a bpf, perf: split ... |
331 |
} |
a43eec304 bpf: introduce bp... |
332 333 |
static const struct bpf_func_proto bpf_perf_event_output_proto = { .func = bpf_perf_event_output, |
1075ef595 bpf: make tracing... |
334 |
.gpl_only = true, |
a43eec304 bpf: introduce bp... |
335 336 337 338 |
.ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_CONST_MAP_PTR, .arg3_type = ARG_ANYTHING, |
39f19ebbf bpf: rename ARG_P... |
339 340 |
.arg4_type = ARG_PTR_TO_MEM, .arg5_type = ARG_CONST_SIZE, |
a43eec304 bpf: introduce bp... |
341 |
}; |
bd570ff97 bpf: add event ou... |
342 |
static DEFINE_PER_CPU(struct pt_regs, bpf_pt_regs); |
a23244e88 bpf: fix corrupti... |
343 |
static DEFINE_PER_CPU(struct perf_sample_data, bpf_misc_sd); |
bd570ff97 bpf: add event ou... |
344 |
|
555c8a862 bpf: avoid stack ... |
345 346 |
u64 bpf_event_output(struct bpf_map *map, u64 flags, void *meta, u64 meta_size, void *ctx, u64 ctx_size, bpf_ctx_copy_t ctx_copy) |
bd570ff97 bpf: add event ou... |
347 |
{ |
a23244e88 bpf: fix corrupti... |
348 |
struct perf_sample_data *sd = this_cpu_ptr(&bpf_misc_sd); |
bd570ff97 bpf: add event ou... |
349 |
struct pt_regs *regs = this_cpu_ptr(&bpf_pt_regs); |
555c8a862 bpf: avoid stack ... |
350 351 352 353 354 355 356 |
struct perf_raw_frag frag = { .copy = ctx_copy, .size = ctx_size, .data = ctx, }; struct perf_raw_record raw = { .frag = { |
183fc1537 kernel/trace/bpf_... |
357 358 359 |
{ .next = ctx_size ? &frag : NULL, }, |
555c8a862 bpf: avoid stack ... |
360 361 362 363 |
.size = meta_size, .data = meta, }, }; |
bd570ff97 bpf: add event ou... |
364 365 |
perf_fetch_caller_regs(regs); |
a23244e88 bpf: fix corrupti... |
366 367 |
perf_sample_data_init(sd, 0, 0); sd->raw = &raw; |
bd570ff97 bpf: add event ou... |
368 |
|
a23244e88 bpf: fix corrupti... |
369 |
return __bpf_perf_event_output(regs, map, flags, sd); |
bd570ff97 bpf: add event ou... |
370 |
} |
f3694e001 bpf: add BPF_CALL... |
371 |
BPF_CALL_0(bpf_get_current_task) |
606274c5a bpf: introduce bp... |
372 373 374 375 376 377 378 379 380 |
{ return (long) current; } static const struct bpf_func_proto bpf_get_current_task_proto = { .func = bpf_get_current_task, .gpl_only = true, .ret_type = RET_INTEGER, }; |
f3694e001 bpf: add BPF_CALL... |
381 |
BPF_CALL_2(bpf_current_task_under_cgroup, struct bpf_map *, map, u32, idx) |
60d20f919 bpf: Add bpf_curr... |
382 |
{ |
60d20f919 bpf: Add bpf_curr... |
383 384 |
struct bpf_array *array = container_of(map, struct bpf_array, map); struct cgroup *cgrp; |
60d20f919 bpf: Add bpf_curr... |
385 386 387 |
if (unlikely(in_interrupt())) return -EINVAL; |
60d20f919 bpf: Add bpf_curr... |
388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 |
if (unlikely(idx >= array->map.max_entries)) return -E2BIG; cgrp = READ_ONCE(array->ptrs[idx]); if (unlikely(!cgrp)) return -EAGAIN; return task_under_cgroup_hierarchy(current, cgrp); } static const struct bpf_func_proto bpf_current_task_under_cgroup_proto = { .func = bpf_current_task_under_cgroup, .gpl_only = false, .ret_type = RET_INTEGER, .arg1_type = ARG_CONST_MAP_PTR, .arg2_type = ARG_ANYTHING, }; |
a5e8c0705 bpf: add bpf_prob... |
405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 |
BPF_CALL_3(bpf_probe_read_str, void *, dst, u32, size, const void *, unsafe_ptr) { int ret; /* * The strncpy_from_unsafe() call will likely not fill the entire * buffer, but that's okay in this circumstance as we're probing * arbitrary memory anyway similar to bpf_probe_read() and might * as well probe the stack. Thus, memory is explicitly cleared * only in error case, so that improper users ignoring return * code altogether don't copy garbage; otherwise length of string * is returned that can be used for bpf_perf_event_output() et al. */ ret = strncpy_from_unsafe(dst, unsafe_ptr, size); if (unlikely(ret < 0)) memset(dst, 0, size); return ret; } static const struct bpf_func_proto bpf_probe_read_str_proto = { .func = bpf_probe_read_str, .gpl_only = true, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_UNINIT_MEM, .arg2_type = ARG_CONST_SIZE, .arg3_type = ARG_ANYTHING, }; |
9fd82b610 bpf: register BPF... |
434 |
static const struct bpf_func_proto *tracing_func_proto(enum bpf_func_id func_id) |
2541517c3 tracing, perf: Im... |
435 436 437 438 439 440 441 442 443 444 |
{ switch (func_id) { case BPF_FUNC_map_lookup_elem: return &bpf_map_lookup_elem_proto; case BPF_FUNC_map_update_elem: return &bpf_map_update_elem_proto; case BPF_FUNC_map_delete_elem: return &bpf_map_delete_elem_proto; case BPF_FUNC_probe_read: return &bpf_probe_read_proto; |
d9847d310 tracing: Allow BP... |
445 446 |
case BPF_FUNC_ktime_get_ns: return &bpf_ktime_get_ns_proto; |
04fd61ab3 bpf: allow bpf pr... |
447 448 |
case BPF_FUNC_tail_call: return &bpf_tail_call_proto; |
ffeedafbf bpf: introduce cu... |
449 450 |
case BPF_FUNC_get_current_pid_tgid: return &bpf_get_current_pid_tgid_proto; |
606274c5a bpf: introduce bp... |
451 452 |
case BPF_FUNC_get_current_task: return &bpf_get_current_task_proto; |
ffeedafbf bpf: introduce cu... |
453 454 455 456 |
case BPF_FUNC_get_current_uid_gid: return &bpf_get_current_uid_gid_proto; case BPF_FUNC_get_current_comm: return &bpf_get_current_comm_proto; |
9c959c863 tracing: Allow BP... |
457 |
case BPF_FUNC_trace_printk: |
0756ea3e8 bpf: allow networ... |
458 |
return bpf_get_trace_printk_proto(); |
ab1973d32 bpf: let kprobe p... |
459 460 |
case BPF_FUNC_get_smp_processor_id: return &bpf_get_smp_processor_id_proto; |
2d0e30c30 bpf: add helper f... |
461 462 |
case BPF_FUNC_get_numa_node_id: return &bpf_get_numa_node_id_proto; |
35578d798 bpf: Implement fu... |
463 464 |
case BPF_FUNC_perf_event_read: return &bpf_perf_event_read_proto; |
96ae52279 bpf: Add bpf_prob... |
465 466 |
case BPF_FUNC_probe_write_user: return bpf_get_probe_write_proto(); |
60d20f919 bpf: Add bpf_curr... |
467 468 |
case BPF_FUNC_current_task_under_cgroup: return &bpf_current_task_under_cgroup_proto; |
8937bd80f bpf: allow bpf_ge... |
469 470 |
case BPF_FUNC_get_prandom_u32: return &bpf_get_prandom_u32_proto; |
a5e8c0705 bpf: add bpf_prob... |
471 472 |
case BPF_FUNC_probe_read_str: return &bpf_probe_read_str_proto; |
9fd82b610 bpf: register BPF... |
473 474 475 476 477 478 479 480 |
default: return NULL; } } static const struct bpf_func_proto *kprobe_prog_func_proto(enum bpf_func_id func_id) { switch (func_id) { |
a43eec304 bpf: introduce bp... |
481 482 |
case BPF_FUNC_perf_event_output: return &bpf_perf_event_output_proto; |
d5a3b1f69 bpf: introduce BP... |
483 484 |
case BPF_FUNC_get_stackid: return &bpf_get_stackid_proto; |
2541517c3 tracing, perf: Im... |
485 |
default: |
9fd82b610 bpf: register BPF... |
486 |
return tracing_func_proto(func_id); |
2541517c3 tracing, perf: Im... |
487 488 489 490 |
} } /* bpf+kprobe programs can access fields of 'struct pt_regs' */ |
19de99f70 bpf: fix matching... |
491 |
static bool kprobe_prog_is_valid_access(int off, int size, enum bpf_access_type type, |
239946314 bpf: possibly avo... |
492 |
struct bpf_insn_access_aux *info) |
2541517c3 tracing, perf: Im... |
493 |
{ |
2541517c3 tracing, perf: Im... |
494 495 |
if (off < 0 || off >= sizeof(struct pt_regs)) return false; |
2541517c3 tracing, perf: Im... |
496 497 |
if (type != BPF_READ) return false; |
2541517c3 tracing, perf: Im... |
498 499 |
if (off % size != 0) return false; |
2d071c643 bpf, trace: make ... |
500 501 502 503 504 505 |
/* * Assertion for 32 bit to make sure last 8 byte access * (BPF_DW) to the last 4 byte member is disallowed. */ if (off + size > sizeof(struct pt_regs)) return false; |
2541517c3 tracing, perf: Im... |
506 507 |
return true; } |
be9370a7d bpf: remove struc... |
508 |
const struct bpf_verifier_ops kprobe_prog_ops = { |
2541517c3 tracing, perf: Im... |
509 510 511 |
.get_func_proto = kprobe_prog_func_proto, .is_valid_access = kprobe_prog_is_valid_access, }; |
f3694e001 bpf: add BPF_CALL... |
512 513 |
BPF_CALL_5(bpf_perf_event_output_tp, void *, tp_buff, struct bpf_map *, map, u64, flags, void *, data, u64, size) |
9940d67c9 bpf: support bpf_... |
514 |
{ |
f3694e001 bpf: add BPF_CALL... |
515 |
struct pt_regs *regs = *(struct pt_regs **)tp_buff; |
9940d67c9 bpf: support bpf_... |
516 517 518 |
/* * r1 points to perf tracepoint buffer where first 8 bytes are hidden * from bpf program and contain a pointer to 'struct pt_regs'. Fetch it |
f3694e001 bpf: add BPF_CALL... |
519 |
* from there and call the same bpf_perf_event_output() helper inline. |
9940d67c9 bpf: support bpf_... |
520 |
*/ |
f3694e001 bpf: add BPF_CALL... |
521 |
return ____bpf_perf_event_output(regs, map, flags, data, size); |
9940d67c9 bpf: support bpf_... |
522 523 524 525 526 527 528 529 530 |
} static const struct bpf_func_proto bpf_perf_event_output_proto_tp = { .func = bpf_perf_event_output_tp, .gpl_only = true, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_CONST_MAP_PTR, .arg3_type = ARG_ANYTHING, |
39f19ebbf bpf: rename ARG_P... |
531 532 |
.arg4_type = ARG_PTR_TO_MEM, .arg5_type = ARG_CONST_SIZE, |
9940d67c9 bpf: support bpf_... |
533 |
}; |
f3694e001 bpf: add BPF_CALL... |
534 535 |
BPF_CALL_3(bpf_get_stackid_tp, void *, tp_buff, struct bpf_map *, map, u64, flags) |
9940d67c9 bpf: support bpf_... |
536 |
{ |
f3694e001 bpf: add BPF_CALL... |
537 |
struct pt_regs *regs = *(struct pt_regs **)tp_buff; |
9940d67c9 bpf: support bpf_... |
538 |
|
f3694e001 bpf: add BPF_CALL... |
539 540 541 542 543 544 545 |
/* * Same comment as in bpf_perf_event_output_tp(), only that this time * the other helper's function body cannot be inlined due to being * external, thus we need to call raw helper function. */ return bpf_get_stackid((unsigned long) regs, (unsigned long) map, flags, 0, 0); |
9940d67c9 bpf: support bpf_... |
546 547 548 549 550 551 552 553 554 555 |
} static const struct bpf_func_proto bpf_get_stackid_proto_tp = { .func = bpf_get_stackid_tp, .gpl_only = true, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_CONST_MAP_PTR, .arg3_type = ARG_ANYTHING, }; |
9fd82b610 bpf: register BPF... |
556 557 558 559 |
static const struct bpf_func_proto *tp_prog_func_proto(enum bpf_func_id func_id) { switch (func_id) { case BPF_FUNC_perf_event_output: |
9940d67c9 bpf: support bpf_... |
560 |
return &bpf_perf_event_output_proto_tp; |
9fd82b610 bpf: register BPF... |
561 |
case BPF_FUNC_get_stackid: |
9940d67c9 bpf: support bpf_... |
562 |
return &bpf_get_stackid_proto_tp; |
9fd82b610 bpf: register BPF... |
563 564 565 566 |
default: return tracing_func_proto(func_id); } } |
19de99f70 bpf: fix matching... |
567 |
static bool tp_prog_is_valid_access(int off, int size, enum bpf_access_type type, |
239946314 bpf: possibly avo... |
568 |
struct bpf_insn_access_aux *info) |
9fd82b610 bpf: register BPF... |
569 570 571 572 573 574 575 |
{ if (off < sizeof(void *) || off >= PERF_MAX_TRACE_SIZE) return false; if (type != BPF_READ) return false; if (off % size != 0) return false; |
2d071c643 bpf, trace: make ... |
576 577 |
BUILD_BUG_ON(PERF_MAX_TRACE_SIZE % sizeof(__u64)); |
9fd82b610 bpf: register BPF... |
578 579 |
return true; } |
be9370a7d bpf: remove struc... |
580 |
const struct bpf_verifier_ops tracepoint_prog_ops = { |
9fd82b610 bpf: register BPF... |
581 582 583 |
.get_func_proto = tp_prog_func_proto, .is_valid_access = tp_prog_is_valid_access, }; |
0515e5999 bpf: introduce BP... |
584 |
static bool pe_prog_is_valid_access(int off, int size, enum bpf_access_type type, |
239946314 bpf: possibly avo... |
585 |
struct bpf_insn_access_aux *info) |
0515e5999 bpf: introduce BP... |
586 |
{ |
f96da0947 bpf: simplify nar... |
587 588 |
const int size_sp = FIELD_SIZEOF(struct bpf_perf_event_data, sample_period); |
31fd85816 bpf: permits narr... |
589 |
|
0515e5999 bpf: introduce BP... |
590 591 592 593 594 595 |
if (off < 0 || off >= sizeof(struct bpf_perf_event_data)) return false; if (type != BPF_READ) return false; if (off % size != 0) return false; |
31fd85816 bpf: permits narr... |
596 |
|
f96da0947 bpf: simplify nar... |
597 598 599 600 |
switch (off) { case bpf_ctx_range(struct bpf_perf_event_data, sample_period): bpf_ctx_record_field_size(info, size_sp); if (!bpf_ctx_narrow_access_ok(off, size, size_sp)) |
239946314 bpf: possibly avo... |
601 |
return false; |
f96da0947 bpf: simplify nar... |
602 603 |
break; default: |
0515e5999 bpf: introduce BP... |
604 605 606 |
if (size != sizeof(long)) return false; } |
f96da0947 bpf: simplify nar... |
607 |
|
0515e5999 bpf: introduce BP... |
608 609 |
return true; } |
6b8cc1d11 bpf: pass origina... |
610 611 |
static u32 pe_prog_convert_ctx_access(enum bpf_access_type type, const struct bpf_insn *si, |
0515e5999 bpf: introduce BP... |
612 |
struct bpf_insn *insn_buf, |
f96da0947 bpf: simplify nar... |
613 |
struct bpf_prog *prog, u32 *target_size) |
0515e5999 bpf: introduce BP... |
614 615 |
{ struct bpf_insn *insn = insn_buf; |
6b8cc1d11 bpf: pass origina... |
616 |
switch (si->off) { |
0515e5999 bpf: introduce BP... |
617 |
case offsetof(struct bpf_perf_event_data, sample_period): |
f035a5153 bpf: add BPF_SIZE... |
618 |
*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct bpf_perf_event_data_kern, |
6b8cc1d11 bpf: pass origina... |
619 |
data), si->dst_reg, si->src_reg, |
0515e5999 bpf: introduce BP... |
620 |
offsetof(struct bpf_perf_event_data_kern, data)); |
6b8cc1d11 bpf: pass origina... |
621 |
*insn++ = BPF_LDX_MEM(BPF_DW, si->dst_reg, si->dst_reg, |
f96da0947 bpf: simplify nar... |
622 623 |
bpf_target_off(struct perf_sample_data, period, 8, target_size)); |
0515e5999 bpf: introduce BP... |
624 625 |
break; default: |
f035a5153 bpf: add BPF_SIZE... |
626 |
*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct bpf_perf_event_data_kern, |
6b8cc1d11 bpf: pass origina... |
627 |
regs), si->dst_reg, si->src_reg, |
0515e5999 bpf: introduce BP... |
628 |
offsetof(struct bpf_perf_event_data_kern, regs)); |
6b8cc1d11 bpf: pass origina... |
629 630 |
*insn++ = BPF_LDX_MEM(BPF_SIZEOF(long), si->dst_reg, si->dst_reg, si->off); |
0515e5999 bpf: introduce BP... |
631 632 633 634 635 |
break; } return insn - insn_buf; } |
be9370a7d bpf: remove struc... |
636 |
const struct bpf_verifier_ops perf_event_prog_ops = { |
0515e5999 bpf: introduce BP... |
637 638 639 640 |
.get_func_proto = tp_prog_func_proto, .is_valid_access = pe_prog_is_valid_access, .convert_ctx_access = pe_prog_convert_ctx_access, }; |