Blame view
net/sched/act_bpf.c
9.7 KB
d23b8ad8a tc: add BPF based... |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 |
/* * Copyright (c) 2015 Jiri Pirko <jiri@resnulli.us> * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or * (at your option) any later version. */ #include <linux/module.h> #include <linux/init.h> #include <linux/kernel.h> #include <linux/skbuff.h> #include <linux/rtnetlink.h> #include <linux/filter.h> |
a8cb5f556 act_bpf: add init... |
16 |
#include <linux/bpf.h> |
d23b8ad8a tc: add BPF based... |
17 18 19 20 21 |
#include <net/netlink.h> #include <net/pkt_sched.h> #include <linux/tc_act/tc_bpf.h> #include <net/tc_act/tc_bpf.h> |
a8cb5f556 act_bpf: add init... |
22 23 24 25 26 27 |
#define BPF_TAB_MASK 15 #define ACT_BPF_NAME_LEN 256 struct tcf_bpf_cfg { struct bpf_prog *filter; struct sock_filter *bpf_ops; |
f4eaed28c act_bpf: fix memo... |
28 |
const char *bpf_name; |
a8cb5f556 act_bpf: add init... |
29 30 |
u32 bpf_fd; u16 bpf_num_ops; |
f4eaed28c act_bpf: fix memo... |
31 |
bool is_ebpf; |
a8cb5f556 act_bpf: add init... |
32 |
}; |
d23b8ad8a tc: add BPF based... |
33 |
|
ddf97ccdd net_sched: add ne... |
34 |
static int bpf_net_id; |
a85a970af net_sched: move t... |
35 |
static struct tc_action_ops act_bpf_ops; |
ddf97ccdd net_sched: add ne... |
36 |
|
a8cb5f556 act_bpf: add init... |
37 |
static int tcf_bpf(struct sk_buff *skb, const struct tc_action *act, |
d23b8ad8a tc: add BPF based... |
38 39 |
struct tcf_result *res) { |
f53d8c7b1 bpf: use skb_at_t... |
40 |
bool at_ingress = skb_at_tc_ingress(skb); |
a85a970af net_sched: move t... |
41 |
struct tcf_bpf *prog = to_bpf(act); |
cff82457c net_sched: act_bp... |
42 |
struct bpf_prog *filter; |
ced585c83 act_bpf: allow no... |
43 |
int action, filter_res; |
d23b8ad8a tc: add BPF based... |
44 |
|
cff82457c net_sched: act_bp... |
45 46 |
tcf_lastuse_update(&prog->tcf_tm); bstats_cpu_update(this_cpu_ptr(prog->common.cpu_bstats), skb); |
d23b8ad8a tc: add BPF based... |
47 |
|
a8cb5f556 act_bpf: add init... |
48 |
rcu_read_lock(); |
cff82457c net_sched: act_bp... |
49 |
filter = rcu_dereference(prog->filter); |
3431205e0 bpf: make program... |
50 51 |
if (at_ingress) { __skb_push(skb, skb->mac_len); |
db58ba459 bpf: wire in data... |
52 |
bpf_compute_data_end(skb); |
cff82457c net_sched: act_bp... |
53 |
filter_res = BPF_PROG_RUN(filter, skb); |
3431205e0 bpf: make program... |
54 55 |
__skb_pull(skb, skb->mac_len); } else { |
db58ba459 bpf: wire in data... |
56 |
bpf_compute_data_end(skb); |
cff82457c net_sched: act_bp... |
57 |
filter_res = BPF_PROG_RUN(filter, skb); |
3431205e0 bpf: make program... |
58 |
} |
a8cb5f556 act_bpf: add init... |
59 |
rcu_read_unlock(); |
ced585c83 act_bpf: allow no... |
60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 |
/* A BPF program may overwrite the default action opcode. * Similarly as in cls_bpf, if filter_res == -1 we use the * default action specified from tc. * * In case a different well-known TC_ACT opcode has been * returned, it will overwrite the default one. * * For everything else that is unkown, TC_ACT_UNSPEC is * returned. */ switch (filter_res) { case TC_ACT_PIPE: case TC_ACT_RECLASSIFY: case TC_ACT_OK: |
27b29f630 bpf: add bpf_redi... |
75 |
case TC_ACT_REDIRECT: |
ced585c83 act_bpf: allow no... |
76 77 78 79 |
action = filter_res; break; case TC_ACT_SHOT: action = filter_res; |
cff82457c net_sched: act_bp... |
80 |
qstats_drop_inc(this_cpu_ptr(prog->common.cpu_qstats)); |
ced585c83 act_bpf: allow no... |
81 82 |
break; case TC_ACT_UNSPEC: |
a8cb5f556 act_bpf: add init... |
83 |
action = prog->tcf_action; |
ced585c83 act_bpf: allow no... |
84 85 86 87 |
break; default: action = TC_ACT_UNSPEC; break; |
d23b8ad8a tc: add BPF based... |
88 |
} |
d23b8ad8a tc: add BPF based... |
89 90 |
return action; } |
a8cb5f556 act_bpf: add init... |
91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 |
static bool tcf_bpf_is_ebpf(const struct tcf_bpf *prog) { return !prog->bpf_ops; } static int tcf_bpf_dump_bpf_info(const struct tcf_bpf *prog, struct sk_buff *skb) { struct nlattr *nla; if (nla_put_u16(skb, TCA_ACT_BPF_OPS_LEN, prog->bpf_num_ops)) return -EMSGSIZE; nla = nla_reserve(skb, TCA_ACT_BPF_OPS, prog->bpf_num_ops * sizeof(struct sock_filter)); if (nla == NULL) return -EMSGSIZE; memcpy(nla_data(nla), prog->bpf_ops, nla_len(nla)); return 0; } static int tcf_bpf_dump_ebpf_info(const struct tcf_bpf *prog, struct sk_buff *skb) { if (nla_put_u32(skb, TCA_ACT_BPF_FD, prog->bpf_fd)) return -EMSGSIZE; if (prog->bpf_name && nla_put_string(skb, TCA_ACT_BPF_NAME, prog->bpf_name)) return -EMSGSIZE; return 0; } static int tcf_bpf_dump(struct sk_buff *skb, struct tc_action *act, |
d23b8ad8a tc: add BPF based... |
128 129 130 |
int bind, int ref) { unsigned char *tp = skb_tail_pointer(skb); |
a85a970af net_sched: move t... |
131 |
struct tcf_bpf *prog = to_bpf(act); |
d23b8ad8a tc: add BPF based... |
132 |
struct tc_act_bpf opt = { |
a8cb5f556 act_bpf: add init... |
133 134 135 136 |
.index = prog->tcf_index, .refcnt = prog->tcf_refcnt - ref, .bindcnt = prog->tcf_bindcnt - bind, .action = prog->tcf_action, |
d23b8ad8a tc: add BPF based... |
137 |
}; |
a8cb5f556 act_bpf: add init... |
138 139 |
struct tcf_t tm; int ret; |
d23b8ad8a tc: add BPF based... |
140 141 142 |
if (nla_put(skb, TCA_ACT_BPF_PARMS, sizeof(opt), &opt)) goto nla_put_failure; |
a8cb5f556 act_bpf: add init... |
143 144 145 146 147 |
if (tcf_bpf_is_ebpf(prog)) ret = tcf_bpf_dump_ebpf_info(prog, skb); else ret = tcf_bpf_dump_bpf_info(prog, skb); if (ret) |
d23b8ad8a tc: add BPF based... |
148 |
goto nla_put_failure; |
48d8ee169 net sched actions... |
149 |
tcf_tm_dump(&tm, &prog->tcf_tm); |
9854518ea sched: align nlat... |
150 151 |
if (nla_put_64bit(skb, TCA_ACT_BPF_TM, sizeof(tm), &tm, TCA_ACT_BPF_PAD)) |
d23b8ad8a tc: add BPF based... |
152 |
goto nla_put_failure; |
a8cb5f556 act_bpf: add init... |
153 |
|
d23b8ad8a tc: add BPF based... |
154 155 156 157 158 159 160 161 162 |
return skb->len; nla_put_failure: nlmsg_trim(skb, tp); return -1; } static const struct nla_policy act_bpf_policy[TCA_ACT_BPF_MAX + 1] = { [TCA_ACT_BPF_PARMS] = { .len = sizeof(struct tc_act_bpf) }, |
a8cb5f556 act_bpf: add init... |
163 |
[TCA_ACT_BPF_FD] = { .type = NLA_U32 }, |
0b0f43fe2 net sched: indent... |
164 165 |
[TCA_ACT_BPF_NAME] = { .type = NLA_NUL_STRING, .len = ACT_BPF_NAME_LEN }, |
d23b8ad8a tc: add BPF based... |
166 167 168 169 |
[TCA_ACT_BPF_OPS_LEN] = { .type = NLA_U16 }, [TCA_ACT_BPF_OPS] = { .type = NLA_BINARY, .len = sizeof(struct sock_filter) * BPF_MAXINSNS }, }; |
a8cb5f556 act_bpf: add init... |
170 |
static int tcf_bpf_init_from_ops(struct nlattr **tb, struct tcf_bpf_cfg *cfg) |
d23b8ad8a tc: add BPF based... |
171 |
{ |
d23b8ad8a tc: add BPF based... |
172 |
struct sock_filter *bpf_ops; |
a8cb5f556 act_bpf: add init... |
173 |
struct sock_fprog_kern fprog_tmp; |
d23b8ad8a tc: add BPF based... |
174 |
struct bpf_prog *fp; |
a8cb5f556 act_bpf: add init... |
175 |
u16 bpf_size, bpf_num_ops; |
d23b8ad8a tc: add BPF based... |
176 |
int ret; |
d23b8ad8a tc: add BPF based... |
177 178 179 180 181 |
bpf_num_ops = nla_get_u16(tb[TCA_ACT_BPF_OPS_LEN]); if (bpf_num_ops > BPF_MAXINSNS || bpf_num_ops == 0) return -EINVAL; bpf_size = bpf_num_ops * sizeof(*bpf_ops); |
fd3e646c8 net: act_bpf: fix... |
182 183 |
if (bpf_size != nla_len(tb[TCA_ACT_BPF_OPS])) return -EINVAL; |
d23b8ad8a tc: add BPF based... |
184 |
bpf_ops = kzalloc(bpf_size, GFP_KERNEL); |
a8cb5f556 act_bpf: add init... |
185 |
if (bpf_ops == NULL) |
d23b8ad8a tc: add BPF based... |
186 187 188 |
return -ENOMEM; memcpy(bpf_ops, nla_data(tb[TCA_ACT_BPF_OPS]), bpf_size); |
a8cb5f556 act_bpf: add init... |
189 190 |
fprog_tmp.len = bpf_num_ops; fprog_tmp.filter = bpf_ops; |
d23b8ad8a tc: add BPF based... |
191 |
|
a8cb5f556 act_bpf: add init... |
192 193 194 195 196 |
ret = bpf_prog_create(&fp, &fprog_tmp); if (ret < 0) { kfree(bpf_ops); return ret; } |
d23b8ad8a tc: add BPF based... |
197 |
|
a8cb5f556 act_bpf: add init... |
198 199 200 |
cfg->bpf_ops = bpf_ops; cfg->bpf_num_ops = bpf_num_ops; cfg->filter = fp; |
f4eaed28c act_bpf: fix memo... |
201 |
cfg->is_ebpf = false; |
a8cb5f556 act_bpf: add init... |
202 203 204 205 206 207 208 209 210 211 212 |
return 0; } static int tcf_bpf_init_from_efd(struct nlattr **tb, struct tcf_bpf_cfg *cfg) { struct bpf_prog *fp; char *name = NULL; u32 bpf_fd; bpf_fd = nla_get_u32(tb[TCA_ACT_BPF_FD]); |
113214be7 bpf: refactor bpf... |
213 |
fp = bpf_prog_get_type(bpf_fd, BPF_PROG_TYPE_SCHED_ACT); |
a8cb5f556 act_bpf: add init... |
214 215 |
if (IS_ERR(fp)) return PTR_ERR(fp); |
a8cb5f556 act_bpf: add init... |
216 217 218 219 220 221 222 223 224 225 226 227 228 |
if (tb[TCA_ACT_BPF_NAME]) { name = kmemdup(nla_data(tb[TCA_ACT_BPF_NAME]), nla_len(tb[TCA_ACT_BPF_NAME]), GFP_KERNEL); if (!name) { bpf_prog_put(fp); return -ENOMEM; } } cfg->bpf_fd = bpf_fd; cfg->bpf_name = name; cfg->filter = fp; |
f4eaed28c act_bpf: fix memo... |
229 |
cfg->is_ebpf = true; |
a8cb5f556 act_bpf: add init... |
230 231 232 |
return 0; } |
f4eaed28c act_bpf: fix memo... |
233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 |
static void tcf_bpf_cfg_cleanup(const struct tcf_bpf_cfg *cfg) { if (cfg->is_ebpf) bpf_prog_put(cfg->filter); else bpf_prog_destroy(cfg->filter); kfree(cfg->bpf_ops); kfree(cfg->bpf_name); } static void tcf_bpf_prog_fill_cfg(const struct tcf_bpf *prog, struct tcf_bpf_cfg *cfg) { cfg->is_ebpf = tcf_bpf_is_ebpf(prog); |
cff82457c net_sched: act_bp... |
248 249 250 251 |
/* updates to prog->filter are prevented, since it's called either * with rtnl lock or during final cleanup in rcu callback */ cfg->filter = rcu_dereference_protected(prog->filter, 1); |
f4eaed28c act_bpf: fix memo... |
252 253 254 255 |
cfg->bpf_ops = prog->bpf_ops; cfg->bpf_name = prog->bpf_name; } |
a8cb5f556 act_bpf: add init... |
256 |
static int tcf_bpf_init(struct net *net, struct nlattr *nla, |
a85a970af net_sched: move t... |
257 |
struct nlattr *est, struct tc_action **act, |
a8cb5f556 act_bpf: add init... |
258 259 |
int replace, int bind) { |
ddf97ccdd net_sched: add ne... |
260 |
struct tc_action_net *tn = net_generic(net, bpf_net_id); |
a8cb5f556 act_bpf: add init... |
261 |
struct nlattr *tb[TCA_ACT_BPF_MAX + 1]; |
f4eaed28c act_bpf: fix memo... |
262 |
struct tcf_bpf_cfg cfg, old; |
a8cb5f556 act_bpf: add init... |
263 264 |
struct tc_act_bpf *parm; struct tcf_bpf *prog; |
a8cb5f556 act_bpf: add init... |
265 |
bool is_bpf, is_ebpf; |
a5c90b29e act_bpf: properly... |
266 |
int ret, res = 0; |
a8cb5f556 act_bpf: add init... |
267 268 269 270 271 272 273 |
if (!nla) return -EINVAL; ret = nla_parse_nested(tb, TCA_ACT_BPF_MAX, nla, act_bpf_policy); if (ret < 0) return ret; |
a5c90b29e act_bpf: properly... |
274 |
if (!tb[TCA_ACT_BPF_PARMS]) |
a8cb5f556 act_bpf: add init... |
275 276 277 |
return -EINVAL; parm = nla_data(tb[TCA_ACT_BPF_PARMS]); |
ddf97ccdd net_sched: add ne... |
278 279 |
if (!tcf_hash_check(tn, parm->index, act, bind)) { ret = tcf_hash_create(tn, parm->index, est, act, |
a85a970af net_sched: move t... |
280 |
&act_bpf_ops, bind, true); |
a8cb5f556 act_bpf: add init... |
281 |
if (ret < 0) |
a5c90b29e act_bpf: properly... |
282 |
return ret; |
d23b8ad8a tc: add BPF based... |
283 |
|
a5c90b29e act_bpf: properly... |
284 |
res = ACT_P_CREATED; |
d23b8ad8a tc: add BPF based... |
285 |
} else { |
a8cb5f556 act_bpf: add init... |
286 |
/* Don't override defaults. */ |
d23b8ad8a tc: add BPF based... |
287 |
if (bind) |
a5c90b29e act_bpf: properly... |
288 |
return 0; |
a8cb5f556 act_bpf: add init... |
289 |
|
a85a970af net_sched: move t... |
290 |
tcf_hash_release(*act, bind); |
a5c90b29e act_bpf: properly... |
291 292 |
if (!replace) return -EEXIST; |
d23b8ad8a tc: add BPF based... |
293 |
} |
a5c90b29e act_bpf: properly... |
294 295 296 297 298 299 300 301 302 303 304 305 306 307 |
is_bpf = tb[TCA_ACT_BPF_OPS_LEN] && tb[TCA_ACT_BPF_OPS]; is_ebpf = tb[TCA_ACT_BPF_FD]; if ((!is_bpf && !is_ebpf) || (is_bpf && is_ebpf)) { ret = -EINVAL; goto out; } memset(&cfg, 0, sizeof(cfg)); ret = is_bpf ? tcf_bpf_init_from_ops(tb, &cfg) : tcf_bpf_init_from_efd(tb, &cfg); if (ret < 0) goto out; |
a85a970af net_sched: move t... |
308 |
prog = to_bpf(*act); |
cff82457c net_sched: act_bp... |
309 |
ASSERT_RTNL(); |
a8cb5f556 act_bpf: add init... |
310 |
|
faa54be4c net_sched: act_bp... |
311 |
if (res != ACT_P_CREATED) |
f4eaed28c act_bpf: fix memo... |
312 |
tcf_bpf_prog_fill_cfg(prog, &old); |
a8cb5f556 act_bpf: add init... |
313 314 315 316 317 318 319 320 321 |
prog->bpf_ops = cfg.bpf_ops; prog->bpf_name = cfg.bpf_name; if (cfg.bpf_num_ops) prog->bpf_num_ops = cfg.bpf_num_ops; if (cfg.bpf_fd) prog->bpf_fd = cfg.bpf_fd; prog->tcf_action = parm->action; |
cff82457c net_sched: act_bp... |
322 |
rcu_assign_pointer(prog->filter, cfg.filter); |
d23b8ad8a tc: add BPF based... |
323 |
|
cff82457c net_sched: act_bp... |
324 |
if (res == ACT_P_CREATED) { |
a85a970af net_sched: move t... |
325 |
tcf_hash_insert(tn, *act); |
cff82457c net_sched: act_bp... |
326 327 328 |
} else { /* make sure the program being replaced is no longer executing */ synchronize_rcu(); |
f4eaed28c act_bpf: fix memo... |
329 |
tcf_bpf_cfg_cleanup(&old); |
cff82457c net_sched: act_bp... |
330 |
} |
a8cb5f556 act_bpf: add init... |
331 |
|
a5c90b29e act_bpf: properly... |
332 333 334 |
return res; out: if (res == ACT_P_CREATED) |
a85a970af net_sched: move t... |
335 |
tcf_hash_cleanup(*act, est); |
d23b8ad8a tc: add BPF based... |
336 |
|
d23b8ad8a tc: add BPF based... |
337 338 |
return ret; } |
a8cb5f556 act_bpf: add init... |
339 |
static void tcf_bpf_cleanup(struct tc_action *act, int bind) |
d23b8ad8a tc: add BPF based... |
340 |
{ |
f4eaed28c act_bpf: fix memo... |
341 |
struct tcf_bpf_cfg tmp; |
ddf06c1e5 tc: act_bpf: fix ... |
342 |
|
a85a970af net_sched: move t... |
343 |
tcf_bpf_prog_fill_cfg(to_bpf(act), &tmp); |
f4eaed28c act_bpf: fix memo... |
344 |
tcf_bpf_cfg_cleanup(&tmp); |
d23b8ad8a tc: add BPF based... |
345 |
} |
ddf97ccdd net_sched: add ne... |
346 347 |
static int tcf_bpf_walker(struct net *net, struct sk_buff *skb, struct netlink_callback *cb, int type, |
a85a970af net_sched: move t... |
348 |
const struct tc_action_ops *ops) |
ddf97ccdd net_sched: add ne... |
349 350 |
{ struct tc_action_net *tn = net_generic(net, bpf_net_id); |
a85a970af net_sched: move t... |
351 |
return tcf_generic_walker(tn, skb, cb, type, ops); |
ddf97ccdd net_sched: add ne... |
352 |
} |
a85a970af net_sched: move t... |
353 |
static int tcf_bpf_search(struct net *net, struct tc_action **a, u32 index) |
ddf97ccdd net_sched: add ne... |
354 355 356 357 358 |
{ struct tc_action_net *tn = net_generic(net, bpf_net_id); return tcf_hash_search(tn, a, index); } |
a8cb5f556 act_bpf: add init... |
359 360 361 362 363 364 365 366 |
static struct tc_action_ops act_bpf_ops __read_mostly = { .kind = "bpf", .type = TCA_ACT_BPF, .owner = THIS_MODULE, .act = tcf_bpf, .dump = tcf_bpf_dump, .cleanup = tcf_bpf_cleanup, .init = tcf_bpf_init, |
ddf97ccdd net_sched: add ne... |
367 368 |
.walk = tcf_bpf_walker, .lookup = tcf_bpf_search, |
a85a970af net_sched: move t... |
369 |
.size = sizeof(struct tcf_bpf), |
ddf97ccdd net_sched: add ne... |
370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 |
}; static __net_init int bpf_init_net(struct net *net) { struct tc_action_net *tn = net_generic(net, bpf_net_id); return tc_action_net_init(tn, &act_bpf_ops, BPF_TAB_MASK); } static void __net_exit bpf_exit_net(struct net *net) { struct tc_action_net *tn = net_generic(net, bpf_net_id); tc_action_net_exit(tn); } static struct pernet_operations bpf_net_ops = { .init = bpf_init_net, .exit = bpf_exit_net, .id = &bpf_net_id, .size = sizeof(struct tc_action_net), |
d23b8ad8a tc: add BPF based... |
391 392 393 394 |
}; static int __init bpf_init_module(void) { |
ddf97ccdd net_sched: add ne... |
395 |
return tcf_register_action(&act_bpf_ops, &bpf_net_ops); |
d23b8ad8a tc: add BPF based... |
396 397 398 399 |
} static void __exit bpf_cleanup_module(void) { |
ddf97ccdd net_sched: add ne... |
400 |
tcf_unregister_action(&act_bpf_ops, &bpf_net_ops); |
d23b8ad8a tc: add BPF based... |
401 402 403 404 405 406 407 408 |
} module_init(bpf_init_module); module_exit(bpf_cleanup_module); MODULE_AUTHOR("Jiri Pirko <jiri@resnulli.us>"); MODULE_DESCRIPTION("TC BPF based action"); MODULE_LICENSE("GPL v2"); |