Blame view
kernel/bpf/core.c
54.1 KB
2874c5fd2 treewide: Replace... |
1 |
// SPDX-License-Identifier: GPL-2.0-or-later |
f5bffecda net: filter: spli... |
2 3 4 5 6 7 8 9 10 11 12 13 14 15 |
/* * Linux Socket Filter - Kernel level socket filtering * * Based on the design of the Berkeley Packet Filter. The new * internal format has been designed by PLUMgrid: * * Copyright (c) 2011 - 2014 PLUMgrid, http://plumgrid.com * * Authors: * * Jay Schulist <jschlst@samba.org> * Alexei Starovoitov <ast@plumgrid.com> * Daniel Borkmann <dborkman@redhat.com> * |
f5bffecda net: filter: spli... |
16 |
* Andi Kleen - Fix a few bad bugs and races. |
4df95ff48 net: filter: rena... |
17 |
* Kris Katterjohn - Added many additional checks in bpf_check_classic() |
f5bffecda net: filter: spli... |
18 |
*/ |
738cbe72a net: bpf: consoli... |
19 |
|
838e96904 bpf: Introduce bp... |
20 |
#include <uapi/linux/btf.h> |
f5bffecda net: filter: spli... |
21 22 |
#include <linux/filter.h> #include <linux/skbuff.h> |
60a3b2253 net: bpf: make eB... |
23 |
#include <linux/vmalloc.h> |
738cbe72a net: bpf: consoli... |
24 25 |
#include <linux/random.h> #include <linux/moduleloader.h> |
09756af46 bpf: expand BPF s... |
26 |
#include <linux/bpf.h> |
838e96904 bpf: Introduce bp... |
27 |
#include <linux/btf.h> |
39853cc0c bpf: Mark __bpf_p... |
28 |
#include <linux/frame.h> |
74451e66d bpf: make jited p... |
29 30 31 |
#include <linux/rbtree_latch.h> #include <linux/kallsyms.h> #include <linux/rcupdate.h> |
c195651e5 bpf: add bpf_get_... |
32 |
#include <linux/perf_event.h> |
f5bffecda net: filter: spli... |
33 |
|
3324b584b ebpf: misc core c... |
34 |
#include <asm/unaligned.h> |
f5bffecda net: filter: spli... |
35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 |
/* Registers */ #define BPF_R0 regs[BPF_REG_0] #define BPF_R1 regs[BPF_REG_1] #define BPF_R2 regs[BPF_REG_2] #define BPF_R3 regs[BPF_REG_3] #define BPF_R4 regs[BPF_REG_4] #define BPF_R5 regs[BPF_REG_5] #define BPF_R6 regs[BPF_REG_6] #define BPF_R7 regs[BPF_REG_7] #define BPF_R8 regs[BPF_REG_8] #define BPF_R9 regs[BPF_REG_9] #define BPF_R10 regs[BPF_REG_10] /* Named registers */ #define DST regs[insn->dst_reg] #define SRC regs[insn->src_reg] #define FP regs[BPF_REG_FP] |
144cd91c4 bpf: move tmp var... |
52 |
#define AX regs[BPF_REG_AX] |
f5bffecda net: filter: spli... |
53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 |
#define ARG1 regs[BPF_REG_ARG1] #define CTX regs[BPF_REG_CTX] #define IMM insn->imm /* No hurry in this branch * * Exported for the bpf jit load helper. */ void *bpf_internal_load_pointer_neg_helper(const struct sk_buff *skb, int k, unsigned int size) { u8 *ptr = NULL; if (k >= SKF_NET_OFF) ptr = skb_network_header(skb) + k - SKF_NET_OFF; else if (k >= SKF_LL_OFF) ptr = skb_mac_header(skb) + k - SKF_LL_OFF; |
3324b584b ebpf: misc core c... |
69 |
|
f5bffecda net: filter: spli... |
70 71 72 73 74 |
if (ptr >= skb->head && ptr + size <= skb_tail_pointer(skb)) return ptr; return NULL; } |
492ecee89 bpf: enable progr... |
75 |
struct bpf_prog *bpf_prog_alloc_no_stats(unsigned int size, gfp_t gfp_extra_flags) |
60a3b2253 net: bpf: make eB... |
76 |
{ |
19809c2da mm, vmalloc: use ... |
77 |
gfp_t gfp_flags = GFP_KERNEL | __GFP_ZERO | gfp_extra_flags; |
09756af46 bpf: expand BPF s... |
78 |
struct bpf_prog_aux *aux; |
60a3b2253 net: bpf: make eB... |
79 80 81 82 83 84 |
struct bpf_prog *fp; size = round_up(size, PAGE_SIZE); fp = __vmalloc(size, gfp_flags, PAGE_KERNEL); if (fp == NULL) return NULL; |
09756af46 bpf: expand BPF s... |
85 86 |
aux = kzalloc(sizeof(*aux), GFP_KERNEL | gfp_extra_flags); if (aux == NULL) { |
60a3b2253 net: bpf: make eB... |
87 88 89 90 91 |
vfree(fp); return NULL; } fp->pages = size / PAGE_SIZE; |
09756af46 bpf: expand BPF s... |
92 |
fp->aux = aux; |
e9d8afa90 bpf: consolidate ... |
93 |
fp->aux->prog = fp; |
60b58afc9 bpf: fix net.core... |
94 |
fp->jit_requested = ebpf_jit_enabled(); |
60a3b2253 net: bpf: make eB... |
95 |
|
74451e66d bpf: make jited p... |
96 |
INIT_LIST_HEAD_RCU(&fp->aux->ksym_lnode); |
60a3b2253 net: bpf: make eB... |
97 98 |
return fp; } |
492ecee89 bpf: enable progr... |
99 100 101 102 103 |
struct bpf_prog *bpf_prog_alloc(unsigned int size, gfp_t gfp_extra_flags) { gfp_t gfp_flags = GFP_KERNEL | __GFP_ZERO | gfp_extra_flags; struct bpf_prog *prog; |
4b9113045 bpf: fix u64_stat... |
104 |
int cpu; |
492ecee89 bpf: enable progr... |
105 106 107 108 109 110 111 112 113 114 115 |
prog = bpf_prog_alloc_no_stats(size, gfp_extra_flags); if (!prog) return NULL; prog->aux->stats = alloc_percpu_gfp(struct bpf_prog_stats, gfp_flags); if (!prog->aux->stats) { kfree(prog->aux); vfree(prog); return NULL; } |
4b9113045 bpf: fix u64_stat... |
116 117 118 119 120 121 |
for_each_possible_cpu(cpu) { struct bpf_prog_stats *pstats; pstats = per_cpu_ptr(prog->aux->stats, cpu); u64_stats_init(&pstats->syncp); } |
492ecee89 bpf: enable progr... |
122 123 |
return prog; } |
60a3b2253 net: bpf: make eB... |
124 |
EXPORT_SYMBOL_GPL(bpf_prog_alloc); |
c454a46b5 bpf: Add bpf_line... |
125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 |
int bpf_prog_alloc_jited_linfo(struct bpf_prog *prog) { if (!prog->aux->nr_linfo || !prog->jit_requested) return 0; prog->aux->jited_linfo = kcalloc(prog->aux->nr_linfo, sizeof(*prog->aux->jited_linfo), GFP_KERNEL | __GFP_NOWARN); if (!prog->aux->jited_linfo) return -ENOMEM; return 0; } void bpf_prog_free_jited_linfo(struct bpf_prog *prog) { kfree(prog->aux->jited_linfo); prog->aux->jited_linfo = NULL; } void bpf_prog_free_unused_jited_linfo(struct bpf_prog *prog) { if (prog->aux->jited_linfo && !prog->aux->jited_linfo[0]) bpf_prog_free_jited_linfo(prog); } /* The jit engine is responsible to provide an array * for insn_off to the jited_off mapping (insn_to_jit_off). * * The idx to this array is the insn_off. Hence, the insn_off * here is relative to the prog itself instead of the main prog. * This array has one entry for each xlated bpf insn. * * jited_off is the byte off to the last byte of the jited insn. * * Hence, with * insn_start: * The first bpf insn off of the prog. The insn off * here is relative to the main prog. * e.g. if prog is a subprog, insn_start > 0 * linfo_idx: * The prog's idx to prog->aux->linfo and jited_linfo * * jited_linfo[linfo_idx] = prog->bpf_func * * For i > linfo_idx, * * jited_linfo[i] = prog->bpf_func + * insn_to_jit_off[linfo[i].insn_off - insn_start - 1] */ void bpf_prog_fill_jited_linfo(struct bpf_prog *prog, const u32 *insn_to_jit_off) { u32 linfo_idx, insn_start, insn_end, nr_linfo, i; const struct bpf_line_info *linfo; void **jited_linfo; if (!prog->aux->jited_linfo) /* Userspace did not provide linfo */ return; linfo_idx = prog->aux->linfo_idx; linfo = &prog->aux->linfo[linfo_idx]; insn_start = linfo[0].insn_off; insn_end = insn_start + prog->len; jited_linfo = &prog->aux->jited_linfo[linfo_idx]; jited_linfo[0] = prog->bpf_func; nr_linfo = prog->aux->nr_linfo - linfo_idx; for (i = 1; i < nr_linfo && linfo[i].insn_off < insn_end; i++) /* The verifier ensures that linfo[i].insn_off is * strictly increasing */ jited_linfo[i] = prog->bpf_func + insn_to_jit_off[linfo[i].insn_off - insn_start - 1]; } void bpf_prog_free_linfo(struct bpf_prog *prog) { bpf_prog_free_jited_linfo(prog); kvfree(prog->aux->linfo); } |
60a3b2253 net: bpf: make eB... |
209 210 211 |
struct bpf_prog *bpf_prog_realloc(struct bpf_prog *fp_old, unsigned int size, gfp_t gfp_extra_flags) { |
19809c2da mm, vmalloc: use ... |
212 |
gfp_t gfp_flags = GFP_KERNEL | __GFP_ZERO | gfp_extra_flags; |
60a3b2253 net: bpf: make eB... |
213 |
struct bpf_prog *fp; |
5ccb071e9 bpf: fix overflow... |
214 215 |
u32 pages, delta; int ret; |
60a3b2253 net: bpf: make eB... |
216 217 218 219 |
BUG_ON(fp_old == NULL); size = round_up(size, PAGE_SIZE); |
5ccb071e9 bpf: fix overflow... |
220 221 |
pages = size / PAGE_SIZE; if (pages <= fp_old->pages) |
60a3b2253 net: bpf: make eB... |
222 |
return fp_old; |
5ccb071e9 bpf: fix overflow... |
223 224 225 226 |
delta = pages - fp_old->pages; ret = __bpf_prog_charge(fp_old->aux->user, delta); if (ret) return NULL; |
60a3b2253 net: bpf: make eB... |
227 |
fp = __vmalloc(size, gfp_flags, PAGE_KERNEL); |
5ccb071e9 bpf: fix overflow... |
228 229 230 |
if (fp == NULL) { __bpf_prog_uncharge(fp_old->aux->user, delta); } else { |
60a3b2253 net: bpf: make eB... |
231 |
memcpy(fp, fp_old, fp_old->pages * PAGE_SIZE); |
5ccb071e9 bpf: fix overflow... |
232 |
fp->pages = pages; |
e9d8afa90 bpf: consolidate ... |
233 |
fp->aux->prog = fp; |
60a3b2253 net: bpf: make eB... |
234 |
|
09756af46 bpf: expand BPF s... |
235 |
/* We keep fp->aux from fp_old around in the new |
60a3b2253 net: bpf: make eB... |
236 237 |
* reallocated structure. */ |
09756af46 bpf: expand BPF s... |
238 |
fp_old->aux = NULL; |
60a3b2253 net: bpf: make eB... |
239 240 241 242 243 |
__bpf_prog_free(fp_old); } return fp; } |
60a3b2253 net: bpf: make eB... |
244 245 246 |
void __bpf_prog_free(struct bpf_prog *fp) { |
492ecee89 bpf: enable progr... |
247 248 249 250 |
if (fp->aux) { free_percpu(fp->aux->stats); kfree(fp->aux); } |
60a3b2253 net: bpf: make eB... |
251 252 |
vfree(fp); } |
60a3b2253 net: bpf: make eB... |
253 |
|
f1f7714ea bpf: rework prog_... |
254 |
int bpf_prog_calc_tag(struct bpf_prog *fp) |
7bd509e31 bpf: add prog_dig... |
255 256 |
{ const u32 bits_offset = SHA_MESSAGE_BYTES - sizeof(__be64); |
f1f7714ea bpf: rework prog_... |
257 258 |
u32 raw_size = bpf_prog_tag_scratch_size(fp); u32 digest[SHA_DIGEST_WORDS]; |
aafe6ae9c bpf: dynamically ... |
259 |
u32 ws[SHA_WORKSPACE_WORDS]; |
7bd509e31 bpf: add prog_dig... |
260 |
u32 i, bsize, psize, blocks; |
aafe6ae9c bpf: dynamically ... |
261 |
struct bpf_insn *dst; |
7bd509e31 bpf: add prog_dig... |
262 |
bool was_ld_map; |
aafe6ae9c bpf: dynamically ... |
263 |
u8 *raw, *todo; |
7bd509e31 bpf: add prog_dig... |
264 265 |
__be32 *result; __be64 *bits; |
aafe6ae9c bpf: dynamically ... |
266 267 268 |
raw = vmalloc(raw_size); if (!raw) return -ENOMEM; |
f1f7714ea bpf: rework prog_... |
269 |
sha_init(digest); |
7bd509e31 bpf: add prog_dig... |
270 271 272 273 274 |
memset(ws, 0, sizeof(ws)); /* We need to take out the map fd for the digest calculation * since they are unstable from user space side. */ |
aafe6ae9c bpf: dynamically ... |
275 |
dst = (void *)raw; |
7bd509e31 bpf: add prog_dig... |
276 277 278 279 |
for (i = 0, was_ld_map = false; i < fp->len; i++) { dst[i] = fp->insnsi[i]; if (!was_ld_map && dst[i].code == (BPF_LD | BPF_IMM | BPF_DW) && |
d8eca5bbb bpf: implement lo... |
280 281 |
(dst[i].src_reg == BPF_PSEUDO_MAP_FD || dst[i].src_reg == BPF_PSEUDO_MAP_VALUE)) { |
7bd509e31 bpf: add prog_dig... |
282 283 284 285 286 287 288 289 290 291 292 293 294 |
was_ld_map = true; dst[i].imm = 0; } else if (was_ld_map && dst[i].code == 0 && dst[i].dst_reg == 0 && dst[i].src_reg == 0 && dst[i].off == 0) { was_ld_map = false; dst[i].imm = 0; } else { was_ld_map = false; } } |
aafe6ae9c bpf: dynamically ... |
295 296 |
psize = bpf_prog_insn_size(fp); memset(&raw[psize], 0, raw_size - psize); |
7bd509e31 bpf: add prog_dig... |
297 298 299 300 |
raw[psize++] = 0x80; bsize = round_up(psize, SHA_MESSAGE_BYTES); blocks = bsize / SHA_MESSAGE_BYTES; |
aafe6ae9c bpf: dynamically ... |
301 |
todo = raw; |
7bd509e31 bpf: add prog_dig... |
302 303 304 305 306 307 308 309 310 |
if (bsize - psize >= sizeof(__be64)) { bits = (__be64 *)(todo + bsize - sizeof(__be64)); } else { bits = (__be64 *)(todo + bsize + bits_offset); blocks++; } *bits = cpu_to_be64((psize - 1) << 3); while (blocks--) { |
f1f7714ea bpf: rework prog_... |
311 |
sha_transform(digest, todo, ws); |
7bd509e31 bpf: add prog_dig... |
312 313 |
todo += SHA_MESSAGE_BYTES; } |
f1f7714ea bpf: rework prog_... |
314 |
result = (__force __be32 *)digest; |
7bd509e31 bpf: add prog_dig... |
315 |
for (i = 0; i < SHA_DIGEST_WORDS; i++) |
f1f7714ea bpf: rework prog_... |
316 317 |
result[i] = cpu_to_be32(digest[i]); memcpy(fp->tag, result, sizeof(fp->tag)); |
aafe6ae9c bpf: dynamically ... |
318 319 320 |
vfree(raw); return 0; |
7bd509e31 bpf: add prog_dig... |
321 |
} |
2cbd95a5c bpf: change param... |
322 |
static int bpf_adj_delta_to_imm(struct bpf_insn *insn, u32 pos, s32 end_old, |
af959b18f bpf: fix out of b... |
323 |
s32 end_new, s32 curr, const bool probe_pass) |
c237ee5eb bpf: add bpf_patc... |
324 |
{ |
050fad7c4 bpf: fix truncate... |
325 |
const s64 imm_min = S32_MIN, imm_max = S32_MAX; |
2cbd95a5c bpf: change param... |
326 |
s32 delta = end_new - end_old; |
050fad7c4 bpf: fix truncate... |
327 |
s64 imm = insn->imm; |
2cbd95a5c bpf: change param... |
328 |
if (curr < pos && curr + imm + 1 >= end_old) |
050fad7c4 bpf: fix truncate... |
329 |
imm += delta; |
2cbd95a5c bpf: change param... |
330 |
else if (curr >= end_new && curr + imm + 1 < end_new) |
050fad7c4 bpf: fix truncate... |
331 332 333 334 335 336 337 |
imm -= delta; if (imm < imm_min || imm > imm_max) return -ERANGE; if (!probe_pass) insn->imm = imm; return 0; } |
2cbd95a5c bpf: change param... |
338 |
static int bpf_adj_delta_to_off(struct bpf_insn *insn, u32 pos, s32 end_old, |
af959b18f bpf: fix out of b... |
339 |
s32 end_new, s32 curr, const bool probe_pass) |
050fad7c4 bpf: fix truncate... |
340 341 |
{ const s32 off_min = S16_MIN, off_max = S16_MAX; |
2cbd95a5c bpf: change param... |
342 |
s32 delta = end_new - end_old; |
050fad7c4 bpf: fix truncate... |
343 |
s32 off = insn->off; |
2cbd95a5c bpf: change param... |
344 |
if (curr < pos && curr + off + 1 >= end_old) |
050fad7c4 bpf: fix truncate... |
345 |
off += delta; |
2cbd95a5c bpf: change param... |
346 |
else if (curr >= end_new && curr + off + 1 < end_new) |
050fad7c4 bpf: fix truncate... |
347 348 349 350 351 352 353 |
off -= delta; if (off < off_min || off > off_max) return -ERANGE; if (!probe_pass) insn->off = off; return 0; } |
2cbd95a5c bpf: change param... |
354 355 |
static int bpf_adj_branches(struct bpf_prog *prog, u32 pos, s32 end_old, s32 end_new, const bool probe_pass) |
050fad7c4 bpf: fix truncate... |
356 |
{ |
2cbd95a5c bpf: change param... |
357 |
u32 i, insn_cnt = prog->len + (probe_pass ? end_new - end_old : 0); |
c237ee5eb bpf: add bpf_patc... |
358 |
struct bpf_insn *insn = prog->insnsi; |
050fad7c4 bpf: fix truncate... |
359 |
int ret = 0; |
c237ee5eb bpf: add bpf_patc... |
360 361 |
for (i = 0; i < insn_cnt; i++, insn++) { |
050fad7c4 bpf: fix truncate... |
362 363 364 365 366 367 368 |
u8 code; /* In the probing pass we still operate on the original, * unpatched image in order to check overflows before we * do any other adjustments. Therefore skip the patchlet. */ if (probe_pass && i == pos) { |
2cbd95a5c bpf: change param... |
369 370 |
i = end_new; insn = prog->insnsi + end_old; |
050fad7c4 bpf: fix truncate... |
371 |
} |
1ea47e01a bpf: add support ... |
372 |
code = insn->code; |
092ed0968 bpf: verifier sup... |
373 374 |
if ((BPF_CLASS(code) != BPF_JMP && BPF_CLASS(code) != BPF_JMP32) || |
050fad7c4 bpf: fix truncate... |
375 |
BPF_OP(code) == BPF_EXIT) |
1ea47e01a bpf: add support ... |
376 |
continue; |
050fad7c4 bpf: fix truncate... |
377 |
/* Adjust offset of jmps if we cross patch boundaries. */ |
1ea47e01a bpf: add support ... |
378 |
if (BPF_OP(code) == BPF_CALL) { |
050fad7c4 bpf: fix truncate... |
379 |
if (insn->src_reg != BPF_PSEUDO_CALL) |
1ea47e01a bpf: add support ... |
380 |
continue; |
2cbd95a5c bpf: change param... |
381 382 |
ret = bpf_adj_delta_to_imm(insn, pos, end_old, end_new, i, probe_pass); |
1ea47e01a bpf: add support ... |
383 |
} else { |
2cbd95a5c bpf: change param... |
384 385 |
ret = bpf_adj_delta_to_off(insn, pos, end_old, end_new, i, probe_pass); |
1ea47e01a bpf: add support ... |
386 |
} |
050fad7c4 bpf: fix truncate... |
387 388 |
if (ret) break; |
c237ee5eb bpf: add bpf_patc... |
389 |
} |
050fad7c4 bpf: fix truncate... |
390 391 |
return ret; |
c237ee5eb bpf: add bpf_patc... |
392 |
} |
c454a46b5 bpf: Add bpf_line... |
393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 |
static void bpf_adj_linfo(struct bpf_prog *prog, u32 off, u32 delta) { struct bpf_line_info *linfo; u32 i, nr_linfo; nr_linfo = prog->aux->nr_linfo; if (!nr_linfo || !delta) return; linfo = prog->aux->linfo; for (i = 0; i < nr_linfo; i++) if (off < linfo[i].insn_off) break; /* Push all off < linfo[i].insn_off by delta */ for (; i < nr_linfo; i++) linfo[i].insn_off += delta; } |
c237ee5eb bpf: add bpf_patc... |
412 413 414 415 |
struct bpf_prog *bpf_patch_insn_single(struct bpf_prog *prog, u32 off, const struct bpf_insn *patch, u32 len) { u32 insn_adj_cnt, insn_rest, insn_delta = len - 1; |
050fad7c4 bpf: fix truncate... |
416 |
const u32 cnt_max = S16_MAX; |
c237ee5eb bpf: add bpf_patc... |
417 |
struct bpf_prog *prog_adj; |
4f73379ec bpf: verbose jump... |
418 |
int err; |
c237ee5eb bpf: add bpf_patc... |
419 420 421 422 423 424 425 426 |
/* Since our patchlet doesn't expand the image, we're done. */ if (insn_delta == 0) { memcpy(prog->insnsi + off, patch, sizeof(*patch)); return prog; } insn_adj_cnt = prog->len + insn_delta; |
050fad7c4 bpf: fix truncate... |
427 428 429 430 431 432 |
/* Reject anything that would potentially let the insn->off * target overflow when we have excessive program expansions. * We need to probe here before we do any reallocation where * we afterwards may not fail anymore. */ if (insn_adj_cnt > cnt_max && |
4f73379ec bpf: verbose jump... |
433 434 |
(err = bpf_adj_branches(prog, off, off + 1, off + len, true))) return ERR_PTR(err); |
050fad7c4 bpf: fix truncate... |
435 |
|
c237ee5eb bpf: add bpf_patc... |
436 437 438 439 440 441 442 |
/* Several new instructions need to be inserted. Make room * for them. Likely, there's no need for a new allocation as * last page could have large enough tailroom. */ prog_adj = bpf_prog_realloc(prog, bpf_prog_size(insn_adj_cnt), GFP_USER); if (!prog_adj) |
4f73379ec bpf: verbose jump... |
443 |
return ERR_PTR(-ENOMEM); |
c237ee5eb bpf: add bpf_patc... |
444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 |
prog_adj->len = insn_adj_cnt; /* Patching happens in 3 steps: * * 1) Move over tail of insnsi from next instruction onwards, * so we can patch the single target insn with one or more * new ones (patching is always from 1 to n insns, n > 0). * 2) Inject new instructions at the target location. * 3) Adjust branch offsets if necessary. */ insn_rest = insn_adj_cnt - off - len; memmove(prog_adj->insnsi + off + len, prog_adj->insnsi + off + 1, sizeof(*patch) * insn_rest); memcpy(prog_adj->insnsi + off, patch, sizeof(*patch) * len); |
050fad7c4 bpf: fix truncate... |
460 461 462 463 |
/* We are guaranteed to not fail at this point, otherwise * the ship has sailed to reverse to the original state. An * overflow cannot happen at this point. */ |
2cbd95a5c bpf: change param... |
464 |
BUG_ON(bpf_adj_branches(prog_adj, off, off + 1, off + len, false)); |
c237ee5eb bpf: add bpf_patc... |
465 |
|
c454a46b5 bpf: Add bpf_line... |
466 |
bpf_adj_linfo(prog_adj, off, insn_delta); |
c237ee5eb bpf: add bpf_patc... |
467 468 |
return prog_adj; } |
52875a04f bpf: verifier: re... |
469 470 471 472 473 474 475 476 477 478 479 |
int bpf_remove_insns(struct bpf_prog *prog, u32 off, u32 cnt) { /* Branch offsets can't overflow when program is shrinking, no need * to call bpf_adj_branches(..., true) here */ memmove(prog->insnsi + off, prog->insnsi + off + cnt, sizeof(struct bpf_insn) * (prog->len - off - cnt)); prog->len -= cnt; return WARN_ON_ONCE(bpf_adj_branches(prog, off, off + cnt, off, false)); } |
cd7455f10 bpf: Fix use afte... |
480 |
static void bpf_prog_kallsyms_del_subprogs(struct bpf_prog *fp) |
7d1982b4e bpf: fix panic in... |
481 482 483 484 485 486 487 488 489 490 491 492 |
{ int i; for (i = 0; i < fp->aux->func_cnt; i++) bpf_prog_kallsyms_del(fp->aux->func[i]); } void bpf_prog_kallsyms_del_all(struct bpf_prog *fp) { bpf_prog_kallsyms_del_subprogs(fp); bpf_prog_kallsyms_del(fp); } |
b954d8342 net: bpf: only bu... |
493 |
#ifdef CONFIG_BPF_JIT |
fa9dd599b bpf: get rid of p... |
494 495 496 497 |
/* All BPF JIT sysctl knobs here. */ int bpf_jit_enable __read_mostly = IS_BUILTIN(CONFIG_BPF_JIT_ALWAYS_ON); int bpf_jit_harden __read_mostly; int bpf_jit_kallsyms __read_mostly; |
fdadd0493 bpf: fix bpf_jit_... |
498 |
long bpf_jit_limit __read_mostly; |
fa9dd599b bpf: get rid of p... |
499 |
|
74451e66d bpf: make jited p... |
500 501 502 503 504 505 506 507 508 509 510 511 512 |
static __always_inline void bpf_get_prog_addr_region(const struct bpf_prog *prog, unsigned long *symbol_start, unsigned long *symbol_end) { const struct bpf_binary_header *hdr = bpf_jit_binary_hdr(prog); unsigned long addr = (unsigned long)hdr; WARN_ON_ONCE(!bpf_prog_ebpf_jited(prog)); *symbol_start = addr; *symbol_end = addr + hdr->pages * PAGE_SIZE; } |
6ee52e2a3 perf, bpf: Introd... |
513 |
void bpf_get_prog_name(const struct bpf_prog *prog, char *sym) |
74451e66d bpf: make jited p... |
514 |
{ |
368211fb9 bpf: Append prog-... |
515 |
const char *end = sym + KSYM_NAME_LEN; |
838e96904 bpf: Introduce bp... |
516 517 |
const struct btf_type *type; const char *func_name; |
368211fb9 bpf: Append prog-... |
518 |
|
74451e66d bpf: make jited p... |
519 |
BUILD_BUG_ON(sizeof("bpf_prog_") + |
368211fb9 bpf: Append prog-... |
520 521 522 523 524 525 526 527 528 |
sizeof(prog->tag) * 2 + /* name has been null terminated. * We should need +1 for the '_' preceding * the name. However, the null character * is double counted between the name and the * sizeof("bpf_prog_") above, so we omit * the +1 here. */ sizeof(prog->aux->name) > KSYM_NAME_LEN); |
74451e66d bpf: make jited p... |
529 530 531 |
sym += snprintf(sym, KSYM_NAME_LEN, "bpf_prog_"); sym = bin2hex(sym, prog->tag, sizeof(prog->tag)); |
838e96904 bpf: Introduce bp... |
532 533 |
/* prog->aux->name will be ignored if full btf name is available */ |
7337224fc bpf: Improve the ... |
534 |
if (prog->aux->func_info_cnt) { |
ba64e7d85 bpf: btf: support... |
535 536 |
type = btf_type_by_id(prog->aux->btf, prog->aux->func_info[prog->aux->func_idx].type_id); |
838e96904 bpf: Introduce bp... |
537 538 539 540 |
func_name = btf_name_by_offset(prog->aux->btf, type->name_off); snprintf(sym, (size_t)(end - sym), "_%s", func_name); return; } |
368211fb9 bpf: Append prog-... |
541 542 543 544 |
if (prog->aux->name[0]) snprintf(sym, (size_t)(end - sym), "_%s", prog->aux->name); else *sym = 0; |
74451e66d bpf: make jited p... |
545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 |
} static __always_inline unsigned long bpf_get_prog_addr_start(struct latch_tree_node *n) { unsigned long symbol_start, symbol_end; const struct bpf_prog_aux *aux; aux = container_of(n, struct bpf_prog_aux, ksym_tnode); bpf_get_prog_addr_region(aux->prog, &symbol_start, &symbol_end); return symbol_start; } static __always_inline bool bpf_tree_less(struct latch_tree_node *a, struct latch_tree_node *b) { return bpf_get_prog_addr_start(a) < bpf_get_prog_addr_start(b); } static __always_inline int bpf_tree_comp(void *key, struct latch_tree_node *n) { unsigned long val = (unsigned long)key; unsigned long symbol_start, symbol_end; const struct bpf_prog_aux *aux; aux = container_of(n, struct bpf_prog_aux, ksym_tnode); bpf_get_prog_addr_region(aux->prog, &symbol_start, &symbol_end); if (val < symbol_start) return -1; if (val >= symbol_end) return 1; return 0; } static const struct latch_tree_ops bpf_tree_ops = { .less = bpf_tree_less, .comp = bpf_tree_comp, }; static DEFINE_SPINLOCK(bpf_lock); static LIST_HEAD(bpf_kallsyms); static struct latch_tree_root bpf_tree __cacheline_aligned; |
74451e66d bpf: make jited p... |
590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 |
static void bpf_prog_ksym_node_add(struct bpf_prog_aux *aux) { WARN_ON_ONCE(!list_empty(&aux->ksym_lnode)); list_add_tail_rcu(&aux->ksym_lnode, &bpf_kallsyms); latch_tree_insert(&aux->ksym_tnode, &bpf_tree, &bpf_tree_ops); } static void bpf_prog_ksym_node_del(struct bpf_prog_aux *aux) { if (list_empty(&aux->ksym_lnode)) return; latch_tree_erase(&aux->ksym_tnode, &bpf_tree, &bpf_tree_ops); list_del_rcu(&aux->ksym_lnode); } static bool bpf_prog_kallsyms_candidate(const struct bpf_prog *fp) { return fp->jited && !bpf_prog_was_classic(fp); } static bool bpf_prog_kallsyms_verify_off(const struct bpf_prog *fp) { return list_empty(&fp->aux->ksym_lnode) || fp->aux->ksym_lnode.prev == LIST_POISON2; } void bpf_prog_kallsyms_add(struct bpf_prog *fp) { |
74451e66d bpf: make jited p... |
619 620 621 |
if (!bpf_prog_kallsyms_candidate(fp) || !capable(CAP_SYS_ADMIN)) return; |
d24f7c7fb bpf: bpf_lock on ... |
622 |
spin_lock_bh(&bpf_lock); |
74451e66d bpf: make jited p... |
623 |
bpf_prog_ksym_node_add(fp->aux); |
d24f7c7fb bpf: bpf_lock on ... |
624 |
spin_unlock_bh(&bpf_lock); |
74451e66d bpf: make jited p... |
625 626 627 628 |
} void bpf_prog_kallsyms_del(struct bpf_prog *fp) { |
74451e66d bpf: make jited p... |
629 630 |
if (!bpf_prog_kallsyms_candidate(fp)) return; |
d24f7c7fb bpf: bpf_lock on ... |
631 |
spin_lock_bh(&bpf_lock); |
74451e66d bpf: make jited p... |
632 |
bpf_prog_ksym_node_del(fp->aux); |
d24f7c7fb bpf: bpf_lock on ... |
633 |
spin_unlock_bh(&bpf_lock); |
74451e66d bpf: make jited p... |
634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 |
} static struct bpf_prog *bpf_prog_kallsyms_find(unsigned long addr) { struct latch_tree_node *n; if (!bpf_jit_kallsyms_enabled()) return NULL; n = latch_tree_find((void *)addr, &bpf_tree, &bpf_tree_ops); return n ? container_of(n, struct bpf_prog_aux, ksym_tnode)->prog : NULL; } const char *__bpf_address_lookup(unsigned long addr, unsigned long *size, unsigned long *off, char *sym) { unsigned long symbol_start, symbol_end; struct bpf_prog *prog; char *ret = NULL; rcu_read_lock(); prog = bpf_prog_kallsyms_find(addr); if (prog) { bpf_get_prog_addr_region(prog, &symbol_start, &symbol_end); bpf_get_prog_name(prog, sym); ret = sym; if (size) *size = symbol_end - symbol_start; if (off) *off = addr - symbol_start; } rcu_read_unlock(); return ret; } bool is_bpf_text_address(unsigned long addr) { bool ret; rcu_read_lock(); ret = bpf_prog_kallsyms_find(addr) != NULL; rcu_read_unlock(); return ret; } int bpf_get_kallsym(unsigned int symnum, unsigned long *value, char *type, char *sym) { |
74451e66d bpf: make jited p... |
687 688 689 690 691 692 693 694 695 696 697 |
struct bpf_prog_aux *aux; unsigned int it = 0; int ret = -ERANGE; if (!bpf_jit_kallsyms_enabled()) return ret; rcu_read_lock(); list_for_each_entry_rcu(aux, &bpf_kallsyms, ksym_lnode) { if (it++ != symnum) continue; |
74451e66d bpf: make jited p... |
698 |
bpf_get_prog_name(aux->prog, sym); |
df0734702 bpf: show real ji... |
699 |
*value = (unsigned long)aux->prog->bpf_func; |
74451e66d bpf: make jited p... |
700 701 702 703 704 705 706 707 708 |
*type = BPF_SYM_ELF_TYPE; ret = 0; break; } rcu_read_unlock(); return ret; } |
ede95a63b bpf: add bpf_jit_... |
709 |
static atomic_long_t bpf_jit_current; |
fdadd0493 bpf: fix bpf_jit_... |
710 711 712 713 714 715 |
/* Can be overridden by an arch's JIT compiler if it has a custom, * dedicated BPF backend memory area, or if neither of the two * below apply. */ u64 __weak bpf_jit_alloc_exec_limit(void) { |
ede95a63b bpf: add bpf_jit_... |
716 |
#if defined(MODULES_VADDR) |
fdadd0493 bpf: fix bpf_jit_... |
717 718 719 720 721 |
return MODULES_END - MODULES_VADDR; #else return VMALLOC_END - VMALLOC_START; #endif } |
ede95a63b bpf: add bpf_jit_... |
722 723 724 |
static int __init bpf_jit_charge_init(void) { /* Only used as heuristic here to derive limit. */ |
fdadd0493 bpf: fix bpf_jit_... |
725 726 |
bpf_jit_limit = min_t(u64, round_up(bpf_jit_alloc_exec_limit() >> 2, PAGE_SIZE), LONG_MAX); |
ede95a63b bpf: add bpf_jit_... |
727 728 729 |
return 0; } pure_initcall(bpf_jit_charge_init); |
ede95a63b bpf: add bpf_jit_... |
730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 |
static int bpf_jit_charge_modmem(u32 pages) { if (atomic_long_add_return(pages, &bpf_jit_current) > (bpf_jit_limit >> PAGE_SHIFT)) { if (!capable(CAP_SYS_ADMIN)) { atomic_long_sub(pages, &bpf_jit_current); return -EPERM; } } return 0; } static void bpf_jit_uncharge_modmem(u32 pages) { atomic_long_sub(pages, &bpf_jit_current); } |
dc002bb62 bpf: add __weak h... |
748 749 750 751 752 753 754 755 756 |
void *__weak bpf_jit_alloc_exec(unsigned long size) { return module_alloc(size); } void __weak bpf_jit_free_exec(void *addr) { module_memfree(addr); } |
738cbe72a net: bpf: consoli... |
757 758 759 760 761 762 |
struct bpf_binary_header * bpf_jit_binary_alloc(unsigned int proglen, u8 **image_ptr, unsigned int alignment, bpf_jit_fill_hole_t bpf_fill_ill_insns) { struct bpf_binary_header *hdr; |
ede95a63b bpf: add bpf_jit_... |
763 |
u32 size, hole, start, pages; |
738cbe72a net: bpf: consoli... |
764 765 766 767 768 769 |
/* Most of BPF filters are really small, but if some of them * fill a page, allow at least 128 extra bytes to insert a * random section of illegal instructions. */ size = round_up(proglen + sizeof(*hdr) + 128, PAGE_SIZE); |
ede95a63b bpf: add bpf_jit_... |
770 771 772 773 |
pages = size / PAGE_SIZE; if (bpf_jit_charge_modmem(pages)) return NULL; |
dc002bb62 bpf: add __weak h... |
774 |
hdr = bpf_jit_alloc_exec(size); |
ede95a63b bpf: add bpf_jit_... |
775 776 |
if (!hdr) { bpf_jit_uncharge_modmem(pages); |
738cbe72a net: bpf: consoli... |
777 |
return NULL; |
ede95a63b bpf: add bpf_jit_... |
778 |
} |
738cbe72a net: bpf: consoli... |
779 780 781 |
/* Fill space with illegal/arch-dep instructions. */ bpf_fill_ill_insns(hdr, size); |
ede95a63b bpf: add bpf_jit_... |
782 |
hdr->pages = pages; |
738cbe72a net: bpf: consoli... |
783 784 |
hole = min_t(unsigned int, size - (proglen + sizeof(*hdr)), PAGE_SIZE - sizeof(*hdr)); |
b7552e1bc bpf: rather use g... |
785 |
start = (get_random_int() % hole) & ~(alignment - 1); |
738cbe72a net: bpf: consoli... |
786 787 788 789 790 791 792 793 794 |
/* Leave a random number of instructions before BPF code. */ *image_ptr = &hdr->image[start]; return hdr; } void bpf_jit_binary_free(struct bpf_binary_header *hdr) { |
ede95a63b bpf: add bpf_jit_... |
795 |
u32 pages = hdr->pages; |
dc002bb62 bpf: add __weak h... |
796 |
bpf_jit_free_exec(hdr); |
ede95a63b bpf: add bpf_jit_... |
797 |
bpf_jit_uncharge_modmem(pages); |
738cbe72a net: bpf: consoli... |
798 |
} |
4f3446bb8 bpf: add generic ... |
799 |
|
74451e66d bpf: make jited p... |
800 801 802 803 804 805 806 807 |
/* This symbol is only overridden by archs that have different * requirements than the usual eBPF JITs, f.e. when they only * implement cBPF JIT, do not set images read-only, etc. */ void __weak bpf_jit_free(struct bpf_prog *fp) { if (fp->jited) { struct bpf_binary_header *hdr = bpf_jit_binary_hdr(fp); |
74451e66d bpf: make jited p... |
808 809 810 811 812 813 814 |
bpf_jit_binary_free(hdr); WARN_ON_ONCE(!bpf_prog_kallsyms_verify_off(fp)); } bpf_prog_unlock_free(fp); } |
e2c95a616 bpf, ppc64: gener... |
815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 |
int bpf_jit_get_func_addr(const struct bpf_prog *prog, const struct bpf_insn *insn, bool extra_pass, u64 *func_addr, bool *func_addr_fixed) { s16 off = insn->off; s32 imm = insn->imm; u8 *addr; *func_addr_fixed = insn->src_reg != BPF_PSEUDO_CALL; if (!*func_addr_fixed) { /* Place-holder address till the last pass has collected * all addresses for JITed subprograms in which case we * can pick them up from prog->aux. */ if (!extra_pass) addr = NULL; else if (prog->aux->func && off >= 0 && off < prog->aux->func_cnt) addr = (u8 *)prog->aux->func[off]->bpf_func; else return -EINVAL; } else { /* Address of a BPF helper call. Since part of the core * kernel, it's always at a fixed location. __bpf_call_base * and the helper with imm relative to it are both in core * kernel. */ addr = (u8 *)__bpf_call_base + imm; } *func_addr = (unsigned long)addr; return 0; } |
4f3446bb8 bpf: add generic ... |
848 849 |
static int bpf_jit_blind_insn(const struct bpf_insn *from, const struct bpf_insn *aux, |
ede7c460b bpf: handle 32-bi... |
850 851 |
struct bpf_insn *to_buff, bool emit_zext) |
4f3446bb8 bpf: add generic ... |
852 853 |
{ struct bpf_insn *to = to_buff; |
b7552e1bc bpf: rather use g... |
854 |
u32 imm_rnd = get_random_int(); |
4f3446bb8 bpf: add generic ... |
855 856 857 858 |
s16 off; BUILD_BUG_ON(BPF_REG_AX + 1 != MAX_BPF_JIT_REG); BUILD_BUG_ON(MAX_BPF_REG + 1 != MAX_BPF_JIT_REG); |
9b73bfdd0 bpf: enable acces... |
859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 |
/* Constraints on AX register: * * AX register is inaccessible from user space. It is mapped in * all JITs, and used here for constant blinding rewrites. It is * typically "stateless" meaning its contents are only valid within * the executed instruction, but not across several instructions. * There are a few exceptions however which are further detailed * below. * * Constant blinding is only used by JITs, not in the interpreter. * The interpreter uses AX in some occasions as a local temporary * register e.g. in DIV or MOD instructions. * * In restricted circumstances, the verifier can also use the AX * register for rewrites as long as they do not interfere with * the above cases! */ if (from->dst_reg == BPF_REG_AX || from->src_reg == BPF_REG_AX) goto out; |
4f3446bb8 bpf: add generic ... |
878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 |
if (from->imm == 0 && (from->code == (BPF_ALU | BPF_MOV | BPF_K) || from->code == (BPF_ALU64 | BPF_MOV | BPF_K))) { *to++ = BPF_ALU64_REG(BPF_XOR, from->dst_reg, from->dst_reg); goto out; } switch (from->code) { case BPF_ALU | BPF_ADD | BPF_K: case BPF_ALU | BPF_SUB | BPF_K: case BPF_ALU | BPF_AND | BPF_K: case BPF_ALU | BPF_OR | BPF_K: case BPF_ALU | BPF_XOR | BPF_K: case BPF_ALU | BPF_MUL | BPF_K: case BPF_ALU | BPF_MOV | BPF_K: case BPF_ALU | BPF_DIV | BPF_K: case BPF_ALU | BPF_MOD | BPF_K: *to++ = BPF_ALU32_IMM(BPF_MOV, BPF_REG_AX, imm_rnd ^ from->imm); *to++ = BPF_ALU32_IMM(BPF_XOR, BPF_REG_AX, imm_rnd); *to++ = BPF_ALU32_REG(from->code, from->dst_reg, BPF_REG_AX); break; case BPF_ALU64 | BPF_ADD | BPF_K: case BPF_ALU64 | BPF_SUB | BPF_K: case BPF_ALU64 | BPF_AND | BPF_K: case BPF_ALU64 | BPF_OR | BPF_K: case BPF_ALU64 | BPF_XOR | BPF_K: case BPF_ALU64 | BPF_MUL | BPF_K: case BPF_ALU64 | BPF_MOV | BPF_K: case BPF_ALU64 | BPF_DIV | BPF_K: case BPF_ALU64 | BPF_MOD | BPF_K: *to++ = BPF_ALU64_IMM(BPF_MOV, BPF_REG_AX, imm_rnd ^ from->imm); *to++ = BPF_ALU64_IMM(BPF_XOR, BPF_REG_AX, imm_rnd); *to++ = BPF_ALU64_REG(from->code, from->dst_reg, BPF_REG_AX); break; case BPF_JMP | BPF_JEQ | BPF_K: case BPF_JMP | BPF_JNE | BPF_K: case BPF_JMP | BPF_JGT | BPF_K: |
92b31a9af bpf: add BPF_J{LT... |
917 |
case BPF_JMP | BPF_JLT | BPF_K: |
4f3446bb8 bpf: add generic ... |
918 |
case BPF_JMP | BPF_JGE | BPF_K: |
92b31a9af bpf: add BPF_J{LT... |
919 |
case BPF_JMP | BPF_JLE | BPF_K: |
4f3446bb8 bpf: add generic ... |
920 |
case BPF_JMP | BPF_JSGT | BPF_K: |
92b31a9af bpf: add BPF_J{LT... |
921 |
case BPF_JMP | BPF_JSLT | BPF_K: |
4f3446bb8 bpf: add generic ... |
922 |
case BPF_JMP | BPF_JSGE | BPF_K: |
92b31a9af bpf: add BPF_J{LT... |
923 |
case BPF_JMP | BPF_JSLE | BPF_K: |
4f3446bb8 bpf: add generic ... |
924 925 926 927 928 929 930 931 932 |
case BPF_JMP | BPF_JSET | BPF_K: /* Accommodate for extra offset in case of a backjump. */ off = from->off; if (off < 0) off -= 2; *to++ = BPF_ALU64_IMM(BPF_MOV, BPF_REG_AX, imm_rnd ^ from->imm); *to++ = BPF_ALU64_IMM(BPF_XOR, BPF_REG_AX, imm_rnd); *to++ = BPF_JMP_REG(from->code, from->dst_reg, BPF_REG_AX, off); break; |
a7b76c885 bpf: JIT blinds s... |
933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 |
case BPF_JMP32 | BPF_JEQ | BPF_K: case BPF_JMP32 | BPF_JNE | BPF_K: case BPF_JMP32 | BPF_JGT | BPF_K: case BPF_JMP32 | BPF_JLT | BPF_K: case BPF_JMP32 | BPF_JGE | BPF_K: case BPF_JMP32 | BPF_JLE | BPF_K: case BPF_JMP32 | BPF_JSGT | BPF_K: case BPF_JMP32 | BPF_JSLT | BPF_K: case BPF_JMP32 | BPF_JSGE | BPF_K: case BPF_JMP32 | BPF_JSLE | BPF_K: case BPF_JMP32 | BPF_JSET | BPF_K: /* Accommodate for extra offset in case of a backjump. */ off = from->off; if (off < 0) off -= 2; *to++ = BPF_ALU32_IMM(BPF_MOV, BPF_REG_AX, imm_rnd ^ from->imm); *to++ = BPF_ALU32_IMM(BPF_XOR, BPF_REG_AX, imm_rnd); *to++ = BPF_JMP32_REG(from->code, from->dst_reg, BPF_REG_AX, off); break; |
4f3446bb8 bpf: add generic ... |
953 954 955 956 957 958 959 960 961 |
case BPF_LD | BPF_IMM | BPF_DW: *to++ = BPF_ALU64_IMM(BPF_MOV, BPF_REG_AX, imm_rnd ^ aux[1].imm); *to++ = BPF_ALU64_IMM(BPF_XOR, BPF_REG_AX, imm_rnd); *to++ = BPF_ALU64_IMM(BPF_LSH, BPF_REG_AX, 32); *to++ = BPF_ALU64_REG(BPF_MOV, aux[0].dst_reg, BPF_REG_AX); break; case 0: /* Part 2 of BPF_LD | BPF_IMM | BPF_DW. */ *to++ = BPF_ALU32_IMM(BPF_MOV, BPF_REG_AX, imm_rnd ^ aux[0].imm); *to++ = BPF_ALU32_IMM(BPF_XOR, BPF_REG_AX, imm_rnd); |
ede7c460b bpf: handle 32-bi... |
962 963 |
if (emit_zext) *to++ = BPF_ZEXT_REG(BPF_REG_AX); |
4f3446bb8 bpf: add generic ... |
964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 |
*to++ = BPF_ALU64_REG(BPF_OR, aux[0].dst_reg, BPF_REG_AX); break; case BPF_ST | BPF_MEM | BPF_DW: case BPF_ST | BPF_MEM | BPF_W: case BPF_ST | BPF_MEM | BPF_H: case BPF_ST | BPF_MEM | BPF_B: *to++ = BPF_ALU64_IMM(BPF_MOV, BPF_REG_AX, imm_rnd ^ from->imm); *to++ = BPF_ALU64_IMM(BPF_XOR, BPF_REG_AX, imm_rnd); *to++ = BPF_STX_MEM(from->code, from->dst_reg, BPF_REG_AX, from->off); break; } out: return to - to_buff; } static struct bpf_prog *bpf_prog_clone_create(struct bpf_prog *fp_other, gfp_t gfp_extra_flags) { |
19809c2da mm, vmalloc: use ... |
983 |
gfp_t gfp_flags = GFP_KERNEL | __GFP_ZERO | gfp_extra_flags; |
4f3446bb8 bpf: add generic ... |
984 985 986 987 |
struct bpf_prog *fp; fp = __vmalloc(fp_other->pages * PAGE_SIZE, gfp_flags, PAGE_KERNEL); if (fp != NULL) { |
4f3446bb8 bpf: add generic ... |
988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 |
/* aux->prog still points to the fp_other one, so * when promoting the clone to the real program, * this still needs to be adapted. */ memcpy(fp, fp_other, fp_other->pages * PAGE_SIZE); } return fp; } static void bpf_prog_clone_free(struct bpf_prog *fp) { /* aux was stolen by the other clone, so we cannot free * it from this path! It will be freed eventually by the * other program on release. * * At this point, we don't need a deferred release since * clone is guaranteed to not be locked. */ fp->aux = NULL; __bpf_prog_free(fp); } void bpf_jit_prog_release_other(struct bpf_prog *fp, struct bpf_prog *fp_other) { /* We have to repoint aux->prog to self, as we don't * know whether fp here is the clone or the original. */ fp->aux->prog = fp; bpf_prog_clone_free(fp_other); } struct bpf_prog *bpf_jit_blind_constants(struct bpf_prog *prog) { struct bpf_insn insn_buff[16], aux[2]; struct bpf_prog *clone, *tmp; int insn_delta, insn_cnt; struct bpf_insn *insn; int i, rewritten; |
1c2a088a6 bpf: x64: add JIT... |
1027 |
if (!bpf_jit_blinding_enabled(prog) || prog->blinded) |
4f3446bb8 bpf: add generic ... |
1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 |
return prog; clone = bpf_prog_clone_create(prog, GFP_USER); if (!clone) return ERR_PTR(-ENOMEM); insn_cnt = clone->len; insn = clone->insnsi; for (i = 0; i < insn_cnt; i++, insn++) { /* We temporarily need to hold the original ld64 insn * so that we can still access the first part in the * second blinding run. */ if (insn[0].code == (BPF_LD | BPF_IMM | BPF_DW) && insn[1].code == 0) memcpy(aux, insn, sizeof(aux)); |
ede7c460b bpf: handle 32-bi... |
1045 1046 |
rewritten = bpf_jit_blind_insn(insn, aux, insn_buff, clone->aux->verifier_zext); |
4f3446bb8 bpf: add generic ... |
1047 1048 1049 1050 |
if (!rewritten) continue; tmp = bpf_patch_insn_single(clone, i, insn_buff, rewritten); |
4f73379ec bpf: verbose jump... |
1051 |
if (IS_ERR(tmp)) { |
4f3446bb8 bpf: add generic ... |
1052 1053 1054 1055 1056 |
/* Patching may have repointed aux->prog during * realloc from the original one, so we need to * fix it up here on error. */ bpf_jit_prog_release_other(prog, clone); |
4f73379ec bpf: verbose jump... |
1057 |
return tmp; |
4f3446bb8 bpf: add generic ... |
1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 |
} clone = tmp; insn_delta = rewritten - 1; /* Walk new program and skip insns we just inserted. */ insn = clone->insnsi + i + insn_delta; insn_cnt += insn_delta; i += insn_delta; } |
1c2a088a6 bpf: x64: add JIT... |
1068 |
clone->blinded = 1; |
4f3446bb8 bpf: add generic ... |
1069 1070 |
return clone; } |
b954d8342 net: bpf: only bu... |
1071 |
#endif /* CONFIG_BPF_JIT */ |
738cbe72a net: bpf: consoli... |
1072 |
|
f5bffecda net: filter: spli... |
1073 1074 |
/* Base function for offset calculation. Needs to go into .text section, * therefore keeping it non-static as well; will also be used by JITs |
7105e828c bpf: allow for co... |
1075 1076 1077 |
* anyway later on, so do not let the compiler omit it. This also needs * to go into kallsyms for correlation from e.g. bpftool, so naming * must not change. |
f5bffecda net: filter: spli... |
1078 1079 1080 1081 1082 |
*/ noinline u64 __bpf_call_base(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5) { return 0; } |
4d9c5c53a test_bpf: add bpf... |
1083 |
EXPORT_SYMBOL_GPL(__bpf_call_base); |
f5bffecda net: filter: spli... |
1084 |
|
5e581dad4 bpf: make unknown... |
1085 1086 1087 1088 |
/* All UAPI available opcodes. */ #define BPF_INSN_MAP(INSN_2, INSN_3) \ /* 32 bit ALU operations. */ \ /* Register based. */ \ |
2dc6b100f bpf: interpreter ... |
1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 |
INSN_3(ALU, ADD, X), \ INSN_3(ALU, SUB, X), \ INSN_3(ALU, AND, X), \ INSN_3(ALU, OR, X), \ INSN_3(ALU, LSH, X), \ INSN_3(ALU, RSH, X), \ INSN_3(ALU, XOR, X), \ INSN_3(ALU, MUL, X), \ INSN_3(ALU, MOV, X), \ INSN_3(ALU, ARSH, X), \ INSN_3(ALU, DIV, X), \ INSN_3(ALU, MOD, X), \ |
5e581dad4 bpf: make unknown... |
1101 1102 1103 1104 |
INSN_2(ALU, NEG), \ INSN_3(ALU, END, TO_BE), \ INSN_3(ALU, END, TO_LE), \ /* Immediate based. */ \ |
2dc6b100f bpf: interpreter ... |
1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 |
INSN_3(ALU, ADD, K), \ INSN_3(ALU, SUB, K), \ INSN_3(ALU, AND, K), \ INSN_3(ALU, OR, K), \ INSN_3(ALU, LSH, K), \ INSN_3(ALU, RSH, K), \ INSN_3(ALU, XOR, K), \ INSN_3(ALU, MUL, K), \ INSN_3(ALU, MOV, K), \ INSN_3(ALU, ARSH, K), \ INSN_3(ALU, DIV, K), \ INSN_3(ALU, MOD, K), \ |
5e581dad4 bpf: make unknown... |
1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 |
/* 64 bit ALU operations. */ \ /* Register based. */ \ INSN_3(ALU64, ADD, X), \ INSN_3(ALU64, SUB, X), \ INSN_3(ALU64, AND, X), \ INSN_3(ALU64, OR, X), \ INSN_3(ALU64, LSH, X), \ INSN_3(ALU64, RSH, X), \ INSN_3(ALU64, XOR, X), \ INSN_3(ALU64, MUL, X), \ INSN_3(ALU64, MOV, X), \ INSN_3(ALU64, ARSH, X), \ INSN_3(ALU64, DIV, X), \ INSN_3(ALU64, MOD, X), \ INSN_2(ALU64, NEG), \ /* Immediate based. */ \ INSN_3(ALU64, ADD, K), \ INSN_3(ALU64, SUB, K), \ INSN_3(ALU64, AND, K), \ INSN_3(ALU64, OR, K), \ INSN_3(ALU64, LSH, K), \ INSN_3(ALU64, RSH, K), \ INSN_3(ALU64, XOR, K), \ INSN_3(ALU64, MUL, K), \ INSN_3(ALU64, MOV, K), \ INSN_3(ALU64, ARSH, K), \ INSN_3(ALU64, DIV, K), \ INSN_3(ALU64, MOD, K), \ /* Call instruction. */ \ INSN_2(JMP, CALL), \ /* Exit instruction. */ \ INSN_2(JMP, EXIT), \ |
503a8865a bpf: interpreter ... |
1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 |
/* 32-bit Jump instructions. */ \ /* Register based. */ \ INSN_3(JMP32, JEQ, X), \ INSN_3(JMP32, JNE, X), \ INSN_3(JMP32, JGT, X), \ INSN_3(JMP32, JLT, X), \ INSN_3(JMP32, JGE, X), \ INSN_3(JMP32, JLE, X), \ INSN_3(JMP32, JSGT, X), \ INSN_3(JMP32, JSLT, X), \ INSN_3(JMP32, JSGE, X), \ INSN_3(JMP32, JSLE, X), \ INSN_3(JMP32, JSET, X), \ /* Immediate based. */ \ INSN_3(JMP32, JEQ, K), \ INSN_3(JMP32, JNE, K), \ INSN_3(JMP32, JGT, K), \ INSN_3(JMP32, JLT, K), \ INSN_3(JMP32, JGE, K), \ INSN_3(JMP32, JLE, K), \ INSN_3(JMP32, JSGT, K), \ INSN_3(JMP32, JSLT, K), \ INSN_3(JMP32, JSGE, K), \ INSN_3(JMP32, JSLE, K), \ INSN_3(JMP32, JSET, K), \ |
5e581dad4 bpf: make unknown... |
1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 |
/* Jump instructions. */ \ /* Register based. */ \ INSN_3(JMP, JEQ, X), \ INSN_3(JMP, JNE, X), \ INSN_3(JMP, JGT, X), \ INSN_3(JMP, JLT, X), \ INSN_3(JMP, JGE, X), \ INSN_3(JMP, JLE, X), \ INSN_3(JMP, JSGT, X), \ INSN_3(JMP, JSLT, X), \ INSN_3(JMP, JSGE, X), \ INSN_3(JMP, JSLE, X), \ INSN_3(JMP, JSET, X), \ /* Immediate based. */ \ INSN_3(JMP, JEQ, K), \ INSN_3(JMP, JNE, K), \ INSN_3(JMP, JGT, K), \ INSN_3(JMP, JLT, K), \ INSN_3(JMP, JGE, K), \ INSN_3(JMP, JLE, K), \ INSN_3(JMP, JSGT, K), \ INSN_3(JMP, JSLT, K), \ INSN_3(JMP, JSGE, K), \ INSN_3(JMP, JSLE, K), \ INSN_3(JMP, JSET, K), \ INSN_2(JMP, JA), \ /* Store instructions. */ \ /* Register based. */ \ INSN_3(STX, MEM, B), \ INSN_3(STX, MEM, H), \ INSN_3(STX, MEM, W), \ INSN_3(STX, MEM, DW), \ INSN_3(STX, XADD, W), \ INSN_3(STX, XADD, DW), \ /* Immediate based. */ \ INSN_3(ST, MEM, B), \ INSN_3(ST, MEM, H), \ INSN_3(ST, MEM, W), \ INSN_3(ST, MEM, DW), \ /* Load instructions. */ \ /* Register based. */ \ INSN_3(LDX, MEM, B), \ INSN_3(LDX, MEM, H), \ INSN_3(LDX, MEM, W), \ INSN_3(LDX, MEM, DW), \ /* Immediate based. */ \ |
e0cea7ce9 bpf: implement ld... |
1220 |
INSN_3(LD, IMM, DW) |
5e581dad4 bpf: make unknown... |
1221 1222 1223 1224 1225 1226 1227 1228 1229 |
bool bpf_opcode_in_insntable(u8 code) { #define BPF_INSN_2_TBL(x, y) [BPF_##x | BPF_##y] = true #define BPF_INSN_3_TBL(x, y, z) [BPF_##x | BPF_##y | BPF_##z] = true static const bool public_insntable[256] = { [0 ... 255] = false, /* Now overwrite non-defaults ... */ BPF_INSN_MAP(BPF_INSN_2_TBL, BPF_INSN_3_TBL), |
e0cea7ce9 bpf: implement ld... |
1230 1231 1232 1233 1234 1235 1236 |
/* UAPI exposed, but rewritten opcodes. cBPF carry-over. */ [BPF_LD | BPF_ABS | BPF_B] = true, [BPF_LD | BPF_ABS | BPF_H] = true, [BPF_LD | BPF_ABS | BPF_W] = true, [BPF_LD | BPF_IND | BPF_B] = true, [BPF_LD | BPF_IND | BPF_H] = true, [BPF_LD | BPF_IND | BPF_W] = true, |
5e581dad4 bpf: make unknown... |
1237 1238 1239 1240 1241 |
}; #undef BPF_INSN_3_TBL #undef BPF_INSN_2_TBL return public_insntable[code]; } |
290af8662 bpf: introduce BP... |
1242 |
#ifndef CONFIG_BPF_JIT_ALWAYS_ON |
f5bffecda net: filter: spli... |
1243 |
/** |
7ae457c1e net: filter: spli... |
1244 |
* __bpf_prog_run - run eBPF program on a given context |
de1da68d9 bpf: fix bitrotte... |
1245 |
* @regs: is the array of MAX_BPF_EXT_REG eBPF pseudo-registers |
7ae457c1e net: filter: spli... |
1246 |
* @insn: is the array of eBPF instructions |
de1da68d9 bpf: fix bitrotte... |
1247 |
* @stack: is the eBPF storage stack |
f5bffecda net: filter: spli... |
1248 |
* |
7ae457c1e net: filter: spli... |
1249 |
* Decode and execute eBPF instructions. |
f5bffecda net: filter: spli... |
1250 |
*/ |
3193c0836 bpf: Disable GCC ... |
1251 |
static u64 __no_fgcse ___bpf_prog_run(u64 *regs, const struct bpf_insn *insn, u64 *stack) |
f5bffecda net: filter: spli... |
1252 |
{ |
5e581dad4 bpf: make unknown... |
1253 1254 |
#define BPF_INSN_2_LBL(x, y) [BPF_##x | BPF_##y] = &&x##_##y #define BPF_INSN_3_LBL(x, y, z) [BPF_##x | BPF_##y | BPF_##z] = &&x##_##y##_##z |
e55a73251 bpf: Fix ORC unwi... |
1255 |
static const void * const jumptable[256] __annotate_jump_table = { |
f5bffecda net: filter: spli... |
1256 1257 |
[0 ... 255] = &&default_label, /* Now overwrite non-defaults ... */ |
5e581dad4 bpf: make unknown... |
1258 1259 |
BPF_INSN_MAP(BPF_INSN_2_LBL, BPF_INSN_3_LBL), /* Non-UAPI available opcodes. */ |
1ea47e01a bpf: add support ... |
1260 |
[BPF_JMP | BPF_CALL_ARGS] = &&JMP_CALL_ARGS, |
71189fa9b bpf: free up BPF_... |
1261 |
[BPF_JMP | BPF_TAIL_CALL] = &&JMP_TAIL_CALL, |
f5bffecda net: filter: spli... |
1262 |
}; |
5e581dad4 bpf: make unknown... |
1263 1264 |
#undef BPF_INSN_3_LBL #undef BPF_INSN_2_LBL |
04fd61ab3 bpf: allow bpf pr... |
1265 |
u32 tail_call_cnt = 0; |
f5bffecda net: filter: spli... |
1266 1267 1268 |
#define CONT ({ insn++; goto select_insn; }) #define CONT_JMP ({ insn++; goto select_insn; }) |
f5bffecda net: filter: spli... |
1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 |
select_insn: goto *jumptable[insn->code]; /* ALU */ #define ALU(OPCODE, OP) \ ALU64_##OPCODE##_X: \ DST = DST OP SRC; \ CONT; \ ALU_##OPCODE##_X: \ DST = (u32) DST OP (u32) SRC; \ CONT; \ ALU64_##OPCODE##_K: \ DST = DST OP IMM; \ CONT; \ ALU_##OPCODE##_K: \ DST = (u32) DST OP (u32) IMM; \ CONT; ALU(ADD, +) ALU(SUB, -) ALU(AND, &) ALU(OR, |) ALU(LSH, <<) ALU(RSH, >>) ALU(XOR, ^) ALU(MUL, *) #undef ALU ALU_NEG: DST = (u32) -DST; CONT; ALU64_NEG: DST = -DST; CONT; ALU_MOV_X: DST = (u32) SRC; CONT; ALU_MOV_K: DST = (u32) IMM; CONT; ALU64_MOV_X: DST = SRC; CONT; ALU64_MOV_K: DST = IMM; CONT; |
02ab695bb net: filter: add ... |
1314 1315 1316 1317 |
LD_IMM_DW: DST = (u64) (u32) insn[0].imm | ((u64) (u32) insn[1].imm) << 32; insn++; CONT; |
2dc6b100f bpf: interpreter ... |
1318 |
ALU_ARSH_X: |
75672dda2 bpf: fix BPF_ALU3... |
1319 |
DST = (u64) (u32) (((s32) DST) >> SRC); |
2dc6b100f bpf: interpreter ... |
1320 1321 |
CONT; ALU_ARSH_K: |
75672dda2 bpf: fix BPF_ALU3... |
1322 |
DST = (u64) (u32) (((s32) DST) >> IMM); |
2dc6b100f bpf: interpreter ... |
1323 |
CONT; |
f5bffecda net: filter: spli... |
1324 1325 1326 1327 1328 1329 1330 |
ALU64_ARSH_X: (*(s64 *) &DST) >>= SRC; CONT; ALU64_ARSH_K: (*(s64 *) &DST) >>= IMM; CONT; ALU64_MOD_X: |
144cd91c4 bpf: move tmp var... |
1331 1332 |
div64_u64_rem(DST, SRC, &AX); DST = AX; |
f5bffecda net: filter: spli... |
1333 1334 |
CONT; ALU_MOD_X: |
144cd91c4 bpf: move tmp var... |
1335 1336 |
AX = (u32) DST; DST = do_div(AX, (u32) SRC); |
f5bffecda net: filter: spli... |
1337 1338 |
CONT; ALU64_MOD_K: |
144cd91c4 bpf: move tmp var... |
1339 1340 |
div64_u64_rem(DST, IMM, &AX); DST = AX; |
f5bffecda net: filter: spli... |
1341 1342 |
CONT; ALU_MOD_K: |
144cd91c4 bpf: move tmp var... |
1343 1344 |
AX = (u32) DST; DST = do_div(AX, (u32) IMM); |
f5bffecda net: filter: spli... |
1345 1346 |
CONT; ALU64_DIV_X: |
876a7ae65 bpf: fix 64-bit d... |
1347 |
DST = div64_u64(DST, SRC); |
f5bffecda net: filter: spli... |
1348 1349 |
CONT; ALU_DIV_X: |
144cd91c4 bpf: move tmp var... |
1350 1351 1352 |
AX = (u32) DST; do_div(AX, (u32) SRC); DST = (u32) AX; |
f5bffecda net: filter: spli... |
1353 1354 |
CONT; ALU64_DIV_K: |
876a7ae65 bpf: fix 64-bit d... |
1355 |
DST = div64_u64(DST, IMM); |
f5bffecda net: filter: spli... |
1356 1357 |
CONT; ALU_DIV_K: |
144cd91c4 bpf: move tmp var... |
1358 1359 1360 |
AX = (u32) DST; do_div(AX, (u32) IMM); DST = (u32) AX; |
f5bffecda net: filter: spli... |
1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 |
CONT; ALU_END_TO_BE: switch (IMM) { case 16: DST = (__force u16) cpu_to_be16(DST); break; case 32: DST = (__force u32) cpu_to_be32(DST); break; case 64: DST = (__force u64) cpu_to_be64(DST); break; } CONT; ALU_END_TO_LE: switch (IMM) { case 16: DST = (__force u16) cpu_to_le16(DST); break; case 32: DST = (__force u32) cpu_to_le32(DST); break; case 64: DST = (__force u64) cpu_to_le64(DST); break; } CONT; /* CALL */ JMP_CALL: /* Function call scratches BPF_R1-BPF_R5 registers, * preserves BPF_R6-BPF_R9, and stores return value * into BPF_R0. */ BPF_R0 = (__bpf_call_base + insn->imm)(BPF_R1, BPF_R2, BPF_R3, BPF_R4, BPF_R5); CONT; |
1ea47e01a bpf: add support ... |
1398 1399 1400 1401 1402 1403 |
JMP_CALL_ARGS: BPF_R0 = (__bpf_call_base_args + insn->imm)(BPF_R1, BPF_R2, BPF_R3, BPF_R4, BPF_R5, insn + insn->off + 1); CONT; |
04fd61ab3 bpf: allow bpf pr... |
1404 1405 1406 1407 |
JMP_TAIL_CALL: { struct bpf_map *map = (struct bpf_map *) (unsigned long) BPF_R2; struct bpf_array *array = container_of(map, struct bpf_array, map); struct bpf_prog *prog; |
90caccdd8 bpf: fix bpf_tail... |
1408 |
u32 index = BPF_R3; |
04fd61ab3 bpf: allow bpf pr... |
1409 1410 1411 |
if (unlikely(index >= array->map.max_entries)) goto out; |
04fd61ab3 bpf: allow bpf pr... |
1412 1413 1414 1415 |
if (unlikely(tail_call_cnt > MAX_TAIL_CALL_CNT)) goto out; tail_call_cnt++; |
2a36f0b92 bpf: Make the bpf... |
1416 |
prog = READ_ONCE(array->ptrs[index]); |
1ca1cc98b bpf: minor cleanu... |
1417 |
if (!prog) |
04fd61ab3 bpf: allow bpf pr... |
1418 |
goto out; |
c4675f935 ebpf: remove self... |
1419 1420 1421 1422 1423 |
/* ARG1 at this point is guaranteed to point to CTX from * the verifier side due to the fact that the tail call is * handeled like a helper, that is, bpf_tail_call_proto, * where arg1_type is ARG_PTR_TO_CTX. */ |
04fd61ab3 bpf: allow bpf pr... |
1424 1425 1426 1427 1428 |
insn = prog->insnsi; goto select_insn; out: CONT; } |
f5bffecda net: filter: spli... |
1429 1430 1431 |
JMP_JA: insn += insn->off; CONT; |
f5bffecda net: filter: spli... |
1432 1433 |
JMP_EXIT: return BPF_R0; |
503a8865a bpf: interpreter ... |
1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 |
/* JMP */ #define COND_JMP(SIGN, OPCODE, CMP_OP) \ JMP_##OPCODE##_X: \ if ((SIGN##64) DST CMP_OP (SIGN##64) SRC) { \ insn += insn->off; \ CONT_JMP; \ } \ CONT; \ JMP32_##OPCODE##_X: \ if ((SIGN##32) DST CMP_OP (SIGN##32) SRC) { \ insn += insn->off; \ CONT_JMP; \ } \ CONT; \ JMP_##OPCODE##_K: \ if ((SIGN##64) DST CMP_OP (SIGN##64) IMM) { \ insn += insn->off; \ CONT_JMP; \ } \ CONT; \ JMP32_##OPCODE##_K: \ if ((SIGN##32) DST CMP_OP (SIGN##32) IMM) { \ insn += insn->off; \ CONT_JMP; \ } \ CONT; COND_JMP(u, JEQ, ==) COND_JMP(u, JNE, !=) COND_JMP(u, JGT, >) COND_JMP(u, JLT, <) COND_JMP(u, JGE, >=) COND_JMP(u, JLE, <=) COND_JMP(u, JSET, &) COND_JMP(s, JSGT, >) COND_JMP(s, JSLT, <) COND_JMP(s, JSGE, >=) COND_JMP(s, JSLE, <=) #undef COND_JMP |
f5bffecda net: filter: spli... |
1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 |
/* STX and ST and LDX*/ #define LDST(SIZEOP, SIZE) \ STX_MEM_##SIZEOP: \ *(SIZE *)(unsigned long) (DST + insn->off) = SRC; \ CONT; \ ST_MEM_##SIZEOP: \ *(SIZE *)(unsigned long) (DST + insn->off) = IMM; \ CONT; \ LDX_MEM_##SIZEOP: \ DST = *(SIZE *)(unsigned long) (SRC + insn->off); \ CONT; LDST(B, u8) LDST(H, u16) LDST(W, u32) LDST(DW, u64) #undef LDST STX_XADD_W: /* lock xadd *(u32 *)(dst_reg + off16) += src_reg */ atomic_add((u32) SRC, (atomic_t *)(unsigned long) (DST + insn->off)); CONT; STX_XADD_DW: /* lock xadd *(u64 *)(dst_reg + off16) += src_reg */ atomic64_add((u64) SRC, (atomic64_t *)(unsigned long) (DST + insn->off)); CONT; |
f5bffecda net: filter: spli... |
1497 1498 |
default_label: |
5e581dad4 bpf: make unknown... |
1499 1500 1501 1502 1503 1504 1505 1506 1507 |
/* If we ever reach this, we have a bug somewhere. Die hard here * instead of just returning 0; we could be somewhere in a subprog, * so execution could continue otherwise which we do /not/ want. * * Note, verifier whitelists all opcodes in bpf_opcode_in_insntable(). */ pr_warn("BPF interpreter: unknown opcode %02x ", insn->code); BUG_ON(1); |
f5bffecda net: filter: spli... |
1508 1509 |
return 0; } |
f696b8f47 bpf: split bpf co... |
1510 |
|
b870aa901 bpf: use differen... |
1511 1512 1513 1514 1515 |
#define PROG_NAME(stack_size) __bpf_prog_run##stack_size #define DEFINE_BPF_PROG_RUN(stack_size) \ static unsigned int PROG_NAME(stack_size)(const void *ctx, const struct bpf_insn *insn) \ { \ u64 stack[stack_size / sizeof(u64)]; \ |
144cd91c4 bpf: move tmp var... |
1516 |
u64 regs[MAX_BPF_EXT_REG]; \ |
b870aa901 bpf: use differen... |
1517 1518 1519 1520 |
\ FP = (u64) (unsigned long) &stack[ARRAY_SIZE(stack)]; \ ARG1 = (u64) (unsigned long) ctx; \ return ___bpf_prog_run(regs, insn, stack); \ |
f696b8f47 bpf: split bpf co... |
1521 |
} |
f5bffecda net: filter: spli... |
1522 |
|
1ea47e01a bpf: add support ... |
1523 1524 1525 1526 1527 1528 |
#define PROG_NAME_ARGS(stack_size) __bpf_prog_run_args##stack_size #define DEFINE_BPF_PROG_RUN_ARGS(stack_size) \ static u64 PROG_NAME_ARGS(stack_size)(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5, \ const struct bpf_insn *insn) \ { \ u64 stack[stack_size / sizeof(u64)]; \ |
144cd91c4 bpf: move tmp var... |
1529 |
u64 regs[MAX_BPF_EXT_REG]; \ |
1ea47e01a bpf: add support ... |
1530 1531 1532 1533 1534 1535 1536 1537 1538 |
\ FP = (u64) (unsigned long) &stack[ARRAY_SIZE(stack)]; \ BPF_R1 = r1; \ BPF_R2 = r2; \ BPF_R3 = r3; \ BPF_R4 = r4; \ BPF_R5 = r5; \ return ___bpf_prog_run(regs, insn, stack); \ } |
b870aa901 bpf: use differen... |
1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 |
#define EVAL1(FN, X) FN(X) #define EVAL2(FN, X, Y...) FN(X) EVAL1(FN, Y) #define EVAL3(FN, X, Y...) FN(X) EVAL2(FN, Y) #define EVAL4(FN, X, Y...) FN(X) EVAL3(FN, Y) #define EVAL5(FN, X, Y...) FN(X) EVAL4(FN, Y) #define EVAL6(FN, X, Y...) FN(X) EVAL5(FN, Y) EVAL6(DEFINE_BPF_PROG_RUN, 32, 64, 96, 128, 160, 192); EVAL6(DEFINE_BPF_PROG_RUN, 224, 256, 288, 320, 352, 384); EVAL4(DEFINE_BPF_PROG_RUN, 416, 448, 480, 512); |
1ea47e01a bpf: add support ... |
1549 1550 1551 |
EVAL6(DEFINE_BPF_PROG_RUN_ARGS, 32, 64, 96, 128, 160, 192); EVAL6(DEFINE_BPF_PROG_RUN_ARGS, 224, 256, 288, 320, 352, 384); EVAL4(DEFINE_BPF_PROG_RUN_ARGS, 416, 448, 480, 512); |
b870aa901 bpf: use differen... |
1552 1553 1554 1555 1556 1557 1558 1559 |
#define PROG_NAME_LIST(stack_size) PROG_NAME(stack_size), static unsigned int (*interpreters[])(const void *ctx, const struct bpf_insn *insn) = { EVAL6(PROG_NAME_LIST, 32, 64, 96, 128, 160, 192) EVAL6(PROG_NAME_LIST, 224, 256, 288, 320, 352, 384) EVAL4(PROG_NAME_LIST, 416, 448, 480, 512) }; |
1ea47e01a bpf: add support ... |
1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 |
#undef PROG_NAME_LIST #define PROG_NAME_LIST(stack_size) PROG_NAME_ARGS(stack_size), static u64 (*interpreters_args[])(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5, const struct bpf_insn *insn) = { EVAL6(PROG_NAME_LIST, 32, 64, 96, 128, 160, 192) EVAL6(PROG_NAME_LIST, 224, 256, 288, 320, 352, 384) EVAL4(PROG_NAME_LIST, 416, 448, 480, 512) }; #undef PROG_NAME_LIST void bpf_patch_call_args(struct bpf_insn *insn, u32 stack_depth) { stack_depth = max_t(u32, stack_depth, 1); insn->off = (s16) insn->imm; insn->imm = interpreters_args[(round_up(stack_depth, 32) / 32) - 1] - __bpf_call_base_args; insn->code = BPF_JMP | BPF_CALL_ARGS; } |
b870aa901 bpf: use differen... |
1578 |
|
290af8662 bpf: introduce BP... |
1579 |
#else |
fa9dd599b bpf: get rid of p... |
1580 1581 |
static unsigned int __bpf_prog_ret0_warn(const void *ctx, const struct bpf_insn *insn) |
290af8662 bpf: introduce BP... |
1582 |
{ |
fa9dd599b bpf: get rid of p... |
1583 1584 1585 1586 |
/* If this handler ever gets executed, then BPF_JIT_ALWAYS_ON * is not working properly, so warn about it! */ WARN_ON_ONCE(1); |
290af8662 bpf: introduce BP... |
1587 1588 1589 |
return 0; } #endif |
3324b584b ebpf: misc core c... |
1590 1591 |
bool bpf_prog_array_compatible(struct bpf_array *array, const struct bpf_prog *fp) |
04fd61ab3 bpf: allow bpf pr... |
1592 |
{ |
9802d8658 bpf: add a bpf_ov... |
1593 1594 |
if (fp->kprobe_override) return false; |
3324b584b ebpf: misc core c... |
1595 1596 1597 1598 |
if (!array->owner_prog_type) { /* There's no owner yet where we could check for * compatibility. */ |
04fd61ab3 bpf: allow bpf pr... |
1599 1600 |
array->owner_prog_type = fp->type; array->owner_jited = fp->jited; |
3324b584b ebpf: misc core c... |
1601 1602 |
return true; |
04fd61ab3 bpf: allow bpf pr... |
1603 |
} |
3324b584b ebpf: misc core c... |
1604 1605 1606 |
return array->owner_prog_type == fp->type && array->owner_jited == fp->jited; |
04fd61ab3 bpf: allow bpf pr... |
1607 |
} |
3324b584b ebpf: misc core c... |
1608 |
static int bpf_check_tail_call(const struct bpf_prog *fp) |
04fd61ab3 bpf: allow bpf pr... |
1609 1610 1611 1612 1613 |
{ struct bpf_prog_aux *aux = fp->aux; int i; for (i = 0; i < aux->used_map_cnt; i++) { |
3324b584b ebpf: misc core c... |
1614 |
struct bpf_map *map = aux->used_maps[i]; |
04fd61ab3 bpf: allow bpf pr... |
1615 |
struct bpf_array *array; |
04fd61ab3 bpf: allow bpf pr... |
1616 |
|
04fd61ab3 bpf: allow bpf pr... |
1617 1618 |
if (map->map_type != BPF_MAP_TYPE_PROG_ARRAY) continue; |
3324b584b ebpf: misc core c... |
1619 |
|
04fd61ab3 bpf: allow bpf pr... |
1620 1621 1622 1623 1624 1625 1626 |
array = container_of(map, struct bpf_array, map); if (!bpf_prog_array_compatible(array, fp)) return -EINVAL; } return 0; } |
9facc3368 bpf: reject any p... |
1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 |
static void bpf_prog_select_func(struct bpf_prog *fp) { #ifndef CONFIG_BPF_JIT_ALWAYS_ON u32 stack_depth = max_t(u32, fp->aux->stack_depth, 1); fp->bpf_func = interpreters[(round_up(stack_depth, 32) / 32) - 1]; #else fp->bpf_func = __bpf_prog_ret0_warn; #endif } |
f5bffecda net: filter: spli... |
1637 |
/** |
3324b584b ebpf: misc core c... |
1638 |
* bpf_prog_select_runtime - select exec runtime for BPF program |
7ae457c1e net: filter: spli... |
1639 |
* @fp: bpf_prog populated with internal BPF program |
d1c55ab5e bpf: prepare bpf_... |
1640 |
* @err: pointer to error variable |
f5bffecda net: filter: spli... |
1641 |
* |
3324b584b ebpf: misc core c... |
1642 1643 |
* Try to JIT eBPF program, if JIT is not available, use interpreter. * The BPF program will be executed via BPF_PROG_RUN() macro. |
f5bffecda net: filter: spli... |
1644 |
*/ |
d1c55ab5e bpf: prepare bpf_... |
1645 |
struct bpf_prog *bpf_prog_select_runtime(struct bpf_prog *fp, int *err) |
f5bffecda net: filter: spli... |
1646 |
{ |
9facc3368 bpf: reject any p... |
1647 1648 1649 1650 1651 |
/* In case of BPF to BPF calls, verifier did all the prep * work with regards to JITing, etc. */ if (fp->bpf_func) goto finalize; |
8007e40a2 bpf: Fix out-of-b... |
1652 |
|
9facc3368 bpf: reject any p... |
1653 |
bpf_prog_select_func(fp); |
f5bffecda net: filter: spli... |
1654 |
|
d1c55ab5e bpf: prepare bpf_... |
1655 1656 1657 1658 1659 1660 |
/* eBPF JITs can rewrite the program in case constant * blinding is active. However, in case of error during * blinding, bpf_int_jit_compile() must always return a * valid program, which in this case would simply not * be JITed, but falls back to the interpreter. */ |
ab3f0063c bpf: offload: add... |
1661 |
if (!bpf_prog_is_dev_bound(fp->aux)) { |
c454a46b5 bpf: Add bpf_line... |
1662 1663 1664 |
*err = bpf_prog_alloc_jited_linfo(fp); if (*err) return fp; |
ab3f0063c bpf: offload: add... |
1665 |
fp = bpf_int_jit_compile(fp); |
290af8662 bpf: introduce BP... |
1666 |
if (!fp->jited) { |
c454a46b5 bpf: Add bpf_line... |
1667 1668 |
bpf_prog_free_jited_linfo(fp); #ifdef CONFIG_BPF_JIT_ALWAYS_ON |
290af8662 bpf: introduce BP... |
1669 1670 |
*err = -ENOTSUPP; return fp; |
290af8662 bpf: introduce BP... |
1671 |
#endif |
c454a46b5 bpf: Add bpf_line... |
1672 1673 1674 |
} else { bpf_prog_free_unused_jited_linfo(fp); } |
ab3f0063c bpf: offload: add... |
1675 1676 1677 1678 1679 |
} else { *err = bpf_prog_offload_compile(fp); if (*err) return fp; } |
9facc3368 bpf: reject any p... |
1680 1681 |
finalize: |
60a3b2253 net: bpf: make eB... |
1682 |
bpf_prog_lock_ro(fp); |
04fd61ab3 bpf: allow bpf pr... |
1683 |
|
3324b584b ebpf: misc core c... |
1684 1685 1686 1687 1688 |
/* The tail call compatibility check can only be done at * this late stage as we need to determine, if we deal * with JITed or non JITed program concatenations and not * all eBPF JITs might immediately support all features. */ |
d1c55ab5e bpf: prepare bpf_... |
1689 |
*err = bpf_check_tail_call(fp); |
85782e037 bpf: undo prog re... |
1690 |
|
d1c55ab5e bpf: prepare bpf_... |
1691 |
return fp; |
f5bffecda net: filter: spli... |
1692 |
} |
7ae457c1e net: filter: spli... |
1693 |
EXPORT_SYMBOL_GPL(bpf_prog_select_runtime); |
f5bffecda net: filter: spli... |
1694 |
|
e87c6bc38 bpf: permit multi... |
1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 |
static unsigned int __bpf_prog_ret1(const void *ctx, const struct bpf_insn *insn) { return 1; } static struct bpf_prog_dummy { struct bpf_prog prog; } dummy_bpf_prog = { .prog = { .bpf_func = __bpf_prog_ret1, }, }; |
324bda9e6 bpf: multi progra... |
1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 |
/* to avoid allocating empty bpf_prog_array for cgroups that * don't have bpf program attached use one global 'empty_prog_array' * It will not be modified the caller of bpf_prog_array_alloc() * (since caller requested prog_cnt == 0) * that pointer should be 'freed' by bpf_prog_array_free() */ static struct { struct bpf_prog_array hdr; struct bpf_prog *null_prog; } empty_prog_array = { .null_prog = NULL, }; |
d29ab6e1f bpf: bpf_prog_arr... |
1720 |
struct bpf_prog_array *bpf_prog_array_alloc(u32 prog_cnt, gfp_t flags) |
324bda9e6 bpf: multi progra... |
1721 1722 1723 |
{ if (prog_cnt) return kzalloc(sizeof(struct bpf_prog_array) + |
394e40a29 bpf: extend bpf_p... |
1724 1725 |
sizeof(struct bpf_prog_array_item) * (prog_cnt + 1), |
324bda9e6 bpf: multi progra... |
1726 1727 1728 1729 |
flags); return &empty_prog_array.hdr; } |
54e9c9d4b bpf: remove __rcu... |
1730 |
void bpf_prog_array_free(struct bpf_prog_array *progs) |
324bda9e6 bpf: multi progra... |
1731 |
{ |
54e9c9d4b bpf: remove __rcu... |
1732 |
if (!progs || progs == &empty_prog_array.hdr) |
324bda9e6 bpf: multi progra... |
1733 1734 1735 |
return; kfree_rcu(progs, rcu); } |
54e9c9d4b bpf: remove __rcu... |
1736 |
int bpf_prog_array_length(struct bpf_prog_array *array) |
468e2f64d bpf: introduce BP... |
1737 |
{ |
394e40a29 bpf: extend bpf_p... |
1738 |
struct bpf_prog_array_item *item; |
468e2f64d bpf: introduce BP... |
1739 |
u32 cnt = 0; |
54e9c9d4b bpf: remove __rcu... |
1740 |
for (item = array->items; item->prog; item++) |
394e40a29 bpf: extend bpf_p... |
1741 |
if (item->prog != &dummy_bpf_prog.prog) |
c8c088ba0 bpf: set maximum ... |
1742 |
cnt++; |
468e2f64d bpf: introduce BP... |
1743 1744 |
return cnt; } |
0d01da6af bpf: implement ge... |
1745 1746 1747 1748 1749 1750 1751 1752 1753 |
bool bpf_prog_array_is_empty(struct bpf_prog_array *array) { struct bpf_prog_array_item *item; for (item = array->items; item->prog; item++) if (item->prog != &dummy_bpf_prog.prog) return false; return true; } |
394e40a29 bpf: extend bpf_p... |
1754 |
|
54e9c9d4b bpf: remove __rcu... |
1755 |
static bool bpf_prog_array_copy_core(struct bpf_prog_array *array, |
3a38bb98d bpf/tracing: fix ... |
1756 1757 1758 |
u32 *prog_ids, u32 request_cnt) { |
394e40a29 bpf: extend bpf_p... |
1759 |
struct bpf_prog_array_item *item; |
3a38bb98d bpf/tracing: fix ... |
1760 |
int i = 0; |
54e9c9d4b bpf: remove __rcu... |
1761 |
for (item = array->items; item->prog; item++) { |
394e40a29 bpf: extend bpf_p... |
1762 |
if (item->prog == &dummy_bpf_prog.prog) |
3a38bb98d bpf/tracing: fix ... |
1763 |
continue; |
394e40a29 bpf: extend bpf_p... |
1764 |
prog_ids[i] = item->prog->aux->id; |
3a38bb98d bpf/tracing: fix ... |
1765 |
if (++i == request_cnt) { |
394e40a29 bpf: extend bpf_p... |
1766 |
item++; |
3a38bb98d bpf/tracing: fix ... |
1767 1768 1769 |
break; } } |
394e40a29 bpf: extend bpf_p... |
1770 |
return !!(item->prog); |
3a38bb98d bpf/tracing: fix ... |
1771 |
} |
54e9c9d4b bpf: remove __rcu... |
1772 |
int bpf_prog_array_copy_to_user(struct bpf_prog_array *array, |
468e2f64d bpf: introduce BP... |
1773 1774 |
__u32 __user *prog_ids, u32 cnt) { |
0911287ce bpf: fix bpf_prog... |
1775 |
unsigned long err = 0; |
0911287ce bpf: fix bpf_prog... |
1776 |
bool nospc; |
3a38bb98d bpf/tracing: fix ... |
1777 |
u32 *ids; |
0911287ce bpf: fix bpf_prog... |
1778 1779 1780 1781 1782 |
/* users of this function are doing: * cnt = bpf_prog_array_length(); * if (cnt > 0) * bpf_prog_array_copy_to_user(..., cnt); |
54e9c9d4b bpf: remove __rcu... |
1783 |
* so below kcalloc doesn't need extra cnt > 0 check. |
0911287ce bpf: fix bpf_prog... |
1784 |
*/ |
9c481b908 bpf: fix bpf_prog... |
1785 |
ids = kcalloc(cnt, sizeof(u32), GFP_USER | __GFP_NOWARN); |
0911287ce bpf: fix bpf_prog... |
1786 1787 |
if (!ids) return -ENOMEM; |
394e40a29 bpf: extend bpf_p... |
1788 |
nospc = bpf_prog_array_copy_core(array, ids, cnt); |
0911287ce bpf: fix bpf_prog... |
1789 1790 1791 1792 1793 |
err = copy_to_user(prog_ids, ids, cnt * sizeof(u32)); kfree(ids); if (err) return -EFAULT; if (nospc) |
468e2f64d bpf: introduce BP... |
1794 1795 1796 |
return -ENOSPC; return 0; } |
54e9c9d4b bpf: remove __rcu... |
1797 |
void bpf_prog_array_delete_safe(struct bpf_prog_array *array, |
e87c6bc38 bpf: permit multi... |
1798 1799 |
struct bpf_prog *old_prog) { |
54e9c9d4b bpf: remove __rcu... |
1800 |
struct bpf_prog_array_item *item; |
e87c6bc38 bpf: permit multi... |
1801 |
|
54e9c9d4b bpf: remove __rcu... |
1802 |
for (item = array->items; item->prog; item++) |
394e40a29 bpf: extend bpf_p... |
1803 1804 |
if (item->prog == old_prog) { WRITE_ONCE(item->prog, &dummy_bpf_prog.prog); |
e87c6bc38 bpf: permit multi... |
1805 1806 1807 |
break; } } |
54e9c9d4b bpf: remove __rcu... |
1808 |
int bpf_prog_array_copy(struct bpf_prog_array *old_array, |
e87c6bc38 bpf: permit multi... |
1809 1810 1811 1812 1813 |
struct bpf_prog *exclude_prog, struct bpf_prog *include_prog, struct bpf_prog_array **new_array) { int new_prog_cnt, carry_prog_cnt = 0; |
394e40a29 bpf: extend bpf_p... |
1814 |
struct bpf_prog_array_item *existing; |
e87c6bc38 bpf: permit multi... |
1815 |
struct bpf_prog_array *array; |
170a7e3ea bpf: bpf_prog_arr... |
1816 |
bool found_exclude = false; |
e87c6bc38 bpf: permit multi... |
1817 1818 1819 1820 1821 1822 |
int new_prog_idx = 0; /* Figure out how many existing progs we need to carry over to * the new array. */ if (old_array) { |
394e40a29 bpf: extend bpf_p... |
1823 1824 1825 |
existing = old_array->items; for (; existing->prog; existing++) { if (existing->prog == exclude_prog) { |
170a7e3ea bpf: bpf_prog_arr... |
1826 1827 1828 |
found_exclude = true; continue; } |
394e40a29 bpf: extend bpf_p... |
1829 |
if (existing->prog != &dummy_bpf_prog.prog) |
e87c6bc38 bpf: permit multi... |
1830 |
carry_prog_cnt++; |
394e40a29 bpf: extend bpf_p... |
1831 |
if (existing->prog == include_prog) |
e87c6bc38 bpf: permit multi... |
1832 1833 1834 |
return -EEXIST; } } |
170a7e3ea bpf: bpf_prog_arr... |
1835 1836 |
if (exclude_prog && !found_exclude) return -ENOENT; |
e87c6bc38 bpf: permit multi... |
1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 |
/* How many progs (not NULL) will be in the new array? */ new_prog_cnt = carry_prog_cnt; if (include_prog) new_prog_cnt += 1; /* Do we have any prog (not NULL) in the new array? */ if (!new_prog_cnt) { *new_array = NULL; return 0; } /* +1 as the end of prog_array is marked with NULL */ array = bpf_prog_array_alloc(new_prog_cnt + 1, GFP_KERNEL); if (!array) return -ENOMEM; /* Fill in the new prog array */ if (carry_prog_cnt) { |
394e40a29 bpf: extend bpf_p... |
1855 1856 1857 1858 1859 1860 1861 |
existing = old_array->items; for (; existing->prog; existing++) if (existing->prog != exclude_prog && existing->prog != &dummy_bpf_prog.prog) { array->items[new_prog_idx++].prog = existing->prog; } |
e87c6bc38 bpf: permit multi... |
1862 1863 |
} if (include_prog) |
394e40a29 bpf: extend bpf_p... |
1864 1865 |
array->items[new_prog_idx++].prog = include_prog; array->items[new_prog_idx].prog = NULL; |
e87c6bc38 bpf: permit multi... |
1866 1867 1868 |
*new_array = array; return 0; } |
54e9c9d4b bpf: remove __rcu... |
1869 |
int bpf_prog_array_copy_info(struct bpf_prog_array *array, |
3a38bb98d bpf/tracing: fix ... |
1870 1871 |
u32 *prog_ids, u32 request_cnt, u32 *prog_cnt) |
f371b304f bpf/tracing: allo... |
1872 1873 1874 1875 1876 |
{ u32 cnt = 0; if (array) cnt = bpf_prog_array_length(array); |
3a38bb98d bpf/tracing: fix ... |
1877 |
*prog_cnt = cnt; |
f371b304f bpf/tracing: allo... |
1878 1879 1880 1881 |
/* return early if user requested only program count or nothing to copy */ if (!request_cnt || !cnt) return 0; |
3a38bb98d bpf/tracing: fix ... |
1882 |
/* this function is called under trace/bpf_trace.c: bpf_event_mutex */ |
394e40a29 bpf: extend bpf_p... |
1883 |
return bpf_prog_array_copy_core(array, prog_ids, request_cnt) ? -ENOSPC |
3a38bb98d bpf/tracing: fix ... |
1884 |
: 0; |
f371b304f bpf/tracing: allo... |
1885 |
} |
60a3b2253 net: bpf: make eB... |
1886 1887 |
static void bpf_prog_free_deferred(struct work_struct *work) { |
09756af46 bpf: expand BPF s... |
1888 |
struct bpf_prog_aux *aux; |
1c2a088a6 bpf: x64: add JIT... |
1889 |
int i; |
60a3b2253 net: bpf: make eB... |
1890 |
|
09756af46 bpf: expand BPF s... |
1891 |
aux = container_of(work, struct bpf_prog_aux, work); |
ab3f0063c bpf: offload: add... |
1892 1893 |
if (bpf_prog_is_dev_bound(aux)) bpf_prog_offload_destroy(aux->prog); |
c195651e5 bpf: add bpf_get_... |
1894 1895 1896 1897 |
#ifdef CONFIG_PERF_EVENTS if (aux->prog->has_callchain_buf) put_callchain_buffers(); #endif |
1c2a088a6 bpf: x64: add JIT... |
1898 1899 1900 1901 1902 1903 1904 1905 |
for (i = 0; i < aux->func_cnt; i++) bpf_jit_free(aux->func[i]); if (aux->func_cnt) { kfree(aux->func); bpf_prog_unlock_free(aux->prog); } else { bpf_jit_free(aux->prog); } |
60a3b2253 net: bpf: make eB... |
1906 1907 1908 |
} /* Free internal BPF program */ |
7ae457c1e net: filter: spli... |
1909 |
void bpf_prog_free(struct bpf_prog *fp) |
f5bffecda net: filter: spli... |
1910 |
{ |
09756af46 bpf: expand BPF s... |
1911 |
struct bpf_prog_aux *aux = fp->aux; |
60a3b2253 net: bpf: make eB... |
1912 |
|
09756af46 bpf: expand BPF s... |
1913 |
INIT_WORK(&aux->work, bpf_prog_free_deferred); |
09756af46 bpf: expand BPF s... |
1914 |
schedule_work(&aux->work); |
f5bffecda net: filter: spli... |
1915 |
} |
7ae457c1e net: filter: spli... |
1916 |
EXPORT_SYMBOL_GPL(bpf_prog_free); |
f89b7755f bpf: split eBPF o... |
1917 |
|
3ad004057 bpf: split state ... |
1918 1919 1920 1921 1922 1923 1924 |
/* RNG for unpriviledged user space with separated state from prandom_u32(). */ static DEFINE_PER_CPU(struct rnd_state, bpf_user_rnd_state); void bpf_user_rnd_init_once(void) { prandom_init_once(&bpf_user_rnd_state); } |
f3694e001 bpf: add BPF_CALL... |
1925 |
BPF_CALL_0(bpf_user_rnd_u32) |
3ad004057 bpf: split state ... |
1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 |
{ /* Should someone ever have the rather unwise idea to use some * of the registers passed into this function, then note that * this function is called from native eBPF and classic-to-eBPF * transformations. Register assignments from both sides are * different, f.e. classic always sets fn(ctx, A, X) here. */ struct rnd_state *state; u32 res; state = &get_cpu_var(bpf_user_rnd_state); res = prandom_u32_state(state); |
b761fe226 bpf: clean up put... |
1938 |
put_cpu_var(bpf_user_rnd_state); |
3ad004057 bpf: split state ... |
1939 1940 1941 |
return res; } |
3ba67daba ebpf: bpf_map_*: ... |
1942 1943 1944 1945 |
/* Weak definitions of helper functions in case we don't have bpf syscall. */ const struct bpf_func_proto bpf_map_lookup_elem_proto __weak; const struct bpf_func_proto bpf_map_update_elem_proto __weak; const struct bpf_func_proto bpf_map_delete_elem_proto __weak; |
f1a2e44a3 bpf: add queue an... |
1946 1947 1948 |
const struct bpf_func_proto bpf_map_push_elem_proto __weak; const struct bpf_func_proto bpf_map_pop_elem_proto __weak; const struct bpf_func_proto bpf_map_peek_elem_proto __weak; |
d83525ca6 bpf: introduce bp... |
1949 1950 |
const struct bpf_func_proto bpf_spin_lock_proto __weak; const struct bpf_func_proto bpf_spin_unlock_proto __weak; |
3ba67daba ebpf: bpf_map_*: ... |
1951 |
|
03e69b508 ebpf: add prandom... |
1952 |
const struct bpf_func_proto bpf_get_prandom_u32_proto __weak; |
c04167ce2 ebpf: add helper ... |
1953 |
const struct bpf_func_proto bpf_get_smp_processor_id_proto __weak; |
2d0e30c30 bpf: add helper f... |
1954 |
const struct bpf_func_proto bpf_get_numa_node_id_proto __weak; |
17ca8cbf4 ebpf: allow bpf_k... |
1955 |
const struct bpf_func_proto bpf_ktime_get_ns_proto __weak; |
bd570ff97 bpf: add event ou... |
1956 |
|
ffeedafbf bpf: introduce cu... |
1957 1958 1959 |
const struct bpf_func_proto bpf_get_current_pid_tgid_proto __weak; const struct bpf_func_proto bpf_get_current_uid_gid_proto __weak; const struct bpf_func_proto bpf_get_current_comm_proto __weak; |
bf6fa2c89 bpf: implement bp... |
1960 |
const struct bpf_func_proto bpf_get_current_cgroup_id_proto __weak; |
cd3394317 bpf: introduce th... |
1961 |
const struct bpf_func_proto bpf_get_local_storage_proto __weak; |
bd570ff97 bpf: add event ou... |
1962 |
|
0756ea3e8 bpf: allow networ... |
1963 1964 1965 1966 |
const struct bpf_func_proto * __weak bpf_get_trace_printk_proto(void) { return NULL; } |
03e69b508 ebpf: add prandom... |
1967 |
|
555c8a862 bpf: avoid stack ... |
1968 1969 1970 |
u64 __weak bpf_event_output(struct bpf_map *map, u64 flags, void *meta, u64 meta_size, void *ctx, u64 ctx_size, bpf_ctx_copy_t ctx_copy) |
bd570ff97 bpf: add event ou... |
1971 |
{ |
555c8a862 bpf: avoid stack ... |
1972 |
return -ENOTSUPP; |
bd570ff97 bpf: add event ou... |
1973 |
} |
6cb5fb389 bpf: export bpf_e... |
1974 |
EXPORT_SYMBOL_GPL(bpf_event_output); |
bd570ff97 bpf: add event ou... |
1975 |
|
3324b584b ebpf: misc core c... |
1976 1977 1978 1979 1980 1981 1982 1983 1984 |
/* Always built-in helper functions. */ const struct bpf_func_proto bpf_tail_call_proto = { .func = NULL, .gpl_only = false, .ret_type = RET_VOID, .arg1_type = ARG_PTR_TO_CTX, .arg2_type = ARG_CONST_MAP_PTR, .arg3_type = ARG_ANYTHING, }; |
9383191da bpf: remove stubs... |
1985 1986 1987 1988 |
/* Stub for JITs that only support cBPF. eBPF programs are interpreted. * It is encouraged to implement bpf_int_jit_compile() instead, so that * eBPF and implicitly also cBPF can get JITed! */ |
d1c55ab5e bpf: prepare bpf_... |
1989 |
struct bpf_prog * __weak bpf_int_jit_compile(struct bpf_prog *prog) |
3324b584b ebpf: misc core c... |
1990 |
{ |
d1c55ab5e bpf: prepare bpf_... |
1991 |
return prog; |
3324b584b ebpf: misc core c... |
1992 |
} |
9383191da bpf: remove stubs... |
1993 1994 1995 1996 1997 1998 |
/* Stub for JITs that support eBPF. All cBPF code gets transformed into * eBPF by the kernel and is later compiled by bpf_int_jit_compile(). */ void __weak bpf_jit_compile(struct bpf_prog *prog) { } |
17bedab27 bpf: xdp: Allow h... |
1999 |
bool __weak bpf_helper_changes_pkt_data(void *func) |
969bf05eb bpf: direct packe... |
2000 2001 2002 |
{ return false; } |
a4b1d3c1d bpf: verifier: in... |
2003 2004 2005 2006 2007 2008 2009 2010 |
/* Return TRUE if the JIT backend wants verifier to enable sub-register usage * analysis code and wants explicit zero extension inserted by verifier. * Otherwise, return FALSE. */ bool __weak bpf_jit_needs_zext(void) { return false; } |
f89b7755f bpf: split eBPF o... |
2011 2012 2013 2014 2015 2016 2017 2018 |
/* To execute LD_ABS/LD_IND instructions __bpf_prog_run() may call * skb_copy_bits(), so provide a weak definition of it for NET-less config. */ int __weak skb_copy_bits(const struct sk_buff *skb, int offset, void *to, int len) { return -EFAULT; } |
a67edbf4f bpf: add initial ... |
2019 |
|
492ecee89 bpf: enable progr... |
2020 2021 |
DEFINE_STATIC_KEY_FALSE(bpf_stats_enabled_key); EXPORT_SYMBOL(bpf_stats_enabled_key); |
492ecee89 bpf: enable progr... |
2022 |
|
a67edbf4f bpf: add initial ... |
2023 2024 2025 2026 2027 |
/* All definitions of tracepoints related to BPF. */ #define CREATE_TRACE_POINTS #include <linux/bpf_trace.h> EXPORT_TRACEPOINT_SYMBOL_GPL(xdp_exception); |
e7d479896 xdp: Add tracepoi... |
2028 |
EXPORT_TRACEPOINT_SYMBOL_GPL(xdp_bulk_tx); |