Blame view
net/sched/sch_fq_codel.c
18.7 KB
2874c5fd2 treewide: Replace... |
1 |
// SPDX-License-Identifier: GPL-2.0-or-later |
4b549a2ef fq_codel: Fair Qu... |
2 3 4 |
/* * Fair Queue CoDel discipline * |
80ba92fa1 codel: add ce_thr... |
5 |
* Copyright (C) 2012,2015 Eric Dumazet <edumazet@google.com> |
4b549a2ef fq_codel: Fair Qu... |
6 7 8 9 10 11 12 13 14 15 16 |
*/ #include <linux/module.h> #include <linux/types.h> #include <linux/kernel.h> #include <linux/jiffies.h> #include <linux/string.h> #include <linux/in.h> #include <linux/errno.h> #include <linux/init.h> #include <linux/skbuff.h> |
4b549a2ef fq_codel: Fair Qu... |
17 18 19 20 |
#include <linux/slab.h> #include <linux/vmalloc.h> #include <net/netlink.h> #include <net/pkt_sched.h> |
cf1facda2 sched: move tcf_p... |
21 |
#include <net/pkt_cls.h> |
4b549a2ef fq_codel: Fair Qu... |
22 |
#include <net/codel.h> |
d068ca2ae codel: split into... |
23 24 |
#include <net/codel_impl.h> #include <net/codel_qdisc.h> |
4b549a2ef fq_codel: Fair Qu... |
25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 |
/* Fair Queue CoDel. * * Principles : * Packets are classified (internal classifier or external) on flows. * This is a Stochastic model (as we use a hash, several flows * might be hashed on same slot) * Each flow has a CoDel managed queue. * Flows are linked onto two (Round Robin) lists, * so that new flows have priority on old ones. * * For a given flow, packets are not reordered (CoDel uses a FIFO) * head drops only. * ECN capability is on by default. * Low memory footprint (64 bytes per flow) */ struct fq_codel_flow { struct sk_buff *head; struct sk_buff *tail; struct list_head flowchain; int deficit; |
4b549a2ef fq_codel: Fair Qu... |
47 48 49 50 |
struct codel_vars cvars; }; /* please try to keep this structure <= 64 bytes */ struct fq_codel_sched_data { |
25d8c0d55 net: rcu-ify tcf_... |
51 |
struct tcf_proto __rcu *filter_list; /* optional external classifier */ |
6529eaba3 net: sched: intro... |
52 |
struct tcf_block *block; |
4b549a2ef fq_codel: Fair Qu... |
53 54 55 |
struct fq_codel_flow *flows; /* Flows table [flows_cnt] */ u32 *backlogs; /* backlog table [flows_cnt] */ u32 flows_cnt; /* number of flows */ |
4b549a2ef fq_codel: Fair Qu... |
56 |
u32 quantum; /* psched_mtu(qdisc_dev(sch)); */ |
9d18562a2 fq_codel: add bat... |
57 |
u32 drop_batch_size; |
95b58430a fq_codel: add mem... |
58 |
u32 memory_limit; |
4b549a2ef fq_codel: Fair Qu... |
59 60 |
struct codel_params cparams; struct codel_stats cstats; |
95b58430a fq_codel: add mem... |
61 62 |
u32 memory_usage; u32 drop_overmemory; |
4b549a2ef fq_codel: Fair Qu... |
63 64 65 66 67 68 69 70 |
u32 drop_overlimit; u32 new_flow_count; struct list_head new_flows; /* list of new flows */ struct list_head old_flows; /* list of old flows */ }; static unsigned int fq_codel_hash(const struct fq_codel_sched_data *q, |
342db2218 sched: Call skb_g... |
71 |
struct sk_buff *skb) |
4b549a2ef fq_codel: Fair Qu... |
72 |
{ |
264b87fa6 fq_codel: Avoid r... |
73 |
return reciprocal_scale(skb_get_hash(skb), q->flows_cnt); |
4b549a2ef fq_codel: Fair Qu... |
74 75 76 77 78 79 |
} static unsigned int fq_codel_classify(struct sk_buff *skb, struct Qdisc *sch, int *qerr) { struct fq_codel_sched_data *q = qdisc_priv(sch); |
25d8c0d55 net: rcu-ify tcf_... |
80 |
struct tcf_proto *filter; |
4b549a2ef fq_codel: Fair Qu... |
81 82 83 84 85 86 87 |
struct tcf_result res; int result; if (TC_H_MAJ(skb->priority) == sch->handle && TC_H_MIN(skb->priority) > 0 && TC_H_MIN(skb->priority) <= q->flows_cnt) return TC_H_MIN(skb->priority); |
69204cf7e net: fix suspicio... |
88 |
filter = rcu_dereference_bh(q->filter_list); |
25d8c0d55 net: rcu-ify tcf_... |
89 |
if (!filter) |
4b549a2ef fq_codel: Fair Qu... |
90 91 92 |
return fq_codel_hash(q, skb) + 1; *qerr = NET_XMIT_SUCCESS | __NET_XMIT_BYPASS; |
87d83093b net: sched: move ... |
93 |
result = tcf_classify(skb, filter, &res, false); |
4b549a2ef fq_codel: Fair Qu... |
94 95 96 97 98 |
if (result >= 0) { #ifdef CONFIG_NET_CLS_ACT switch (result) { case TC_ACT_STOLEN: case TC_ACT_QUEUED: |
e25ea21ff net: sched: intro... |
99 |
case TC_ACT_TRAP: |
4b549a2ef fq_codel: Fair Qu... |
100 |
*qerr = NET_XMIT_SUCCESS | __NET_XMIT_STOLEN; |
964201de6 net/sched: Use fa... |
101 |
fallthrough; |
4b549a2ef fq_codel: Fair Qu... |
102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 |
case TC_ACT_SHOT: return 0; } #endif if (TC_H_MIN(res.classid) <= q->flows_cnt) return TC_H_MIN(res.classid); } return 0; } /* helper functions : might be changed when/if skb use a standard list_head */ /* remove one skb from head of slot queue */ static inline struct sk_buff *dequeue_head(struct fq_codel_flow *flow) { struct sk_buff *skb = flow->head; flow->head = skb->next; |
a8305bff6 net: Add and use ... |
120 |
skb_mark_not_on_list(skb); |
4b549a2ef fq_codel: Fair Qu... |
121 122 123 124 125 126 127 128 129 130 131 132 133 134 |
return skb; } /* add skb to flow queue (tail add) */ static inline void flow_queue_add(struct fq_codel_flow *flow, struct sk_buff *skb) { if (flow->head == NULL) flow->head = skb; else flow->tail->next = skb; flow->tail = skb; skb->next = NULL; } |
520ac30f4 net_sched: drop p... |
135 136 |
static unsigned int fq_codel_drop(struct Qdisc *sch, unsigned int max_packets, struct sk_buff **to_free) |
4b549a2ef fq_codel: Fair Qu... |
137 138 139 140 141 |
{ struct fq_codel_sched_data *q = qdisc_priv(sch); struct sk_buff *skb; unsigned int maxbacklog = 0, idx = 0, i, len; struct fq_codel_flow *flow; |
9d18562a2 fq_codel: add bat... |
142 |
unsigned int threshold; |
95b58430a fq_codel: add mem... |
143 |
unsigned int mem = 0; |
4b549a2ef fq_codel: Fair Qu... |
144 |
|
9d18562a2 fq_codel: add bat... |
145 |
/* Queue is full! Find the fat flow and drop packet(s) from it. |
4b549a2ef fq_codel: Fair Qu... |
146 147 148 |
* This might sound expensive, but with 1024 flows, we scan * 4KB of memory, and we dont need to handle a complex tree * in fast path (packet queue/enqueue) with many cache misses. |
9d18562a2 fq_codel: add bat... |
149 150 |
* In stress mode, we'll try to drop 64 packets from the flow, * amortizing this linear lookup to one cache line per drop. |
4b549a2ef fq_codel: Fair Qu... |
151 152 153 154 155 156 157 |
*/ for (i = 0; i < q->flows_cnt; i++) { if (q->backlogs[i] > maxbacklog) { maxbacklog = q->backlogs[i]; idx = i; } } |
9d18562a2 fq_codel: add bat... |
158 159 160 |
/* Our goal is to drop half of this fat flow backlog */ threshold = maxbacklog >> 1; |
4b549a2ef fq_codel: Fair Qu... |
161 |
flow = &q->flows[idx]; |
9d18562a2 fq_codel: add bat... |
162 163 164 165 166 |
len = 0; i = 0; do { skb = dequeue_head(flow); len += qdisc_pkt_len(skb); |
008830bc3 net_sched: fq_cod... |
167 |
mem += get_codel_cb(skb)->mem_usage; |
520ac30f4 net_sched: drop p... |
168 |
__qdisc_drop(skb, to_free); |
9d18562a2 fq_codel: add bat... |
169 |
} while (++i < max_packets && len < threshold); |
ae697f3bf Increase fq_codel... |
170 171 |
/* Tell codel to increase its signal strength also */ flow->cvars.count += i; |
4b549a2ef fq_codel: Fair Qu... |
172 |
q->backlogs[idx] -= len; |
95b58430a fq_codel: add mem... |
173 |
q->memory_usage -= mem; |
9d18562a2 fq_codel: add bat... |
174 175 176 |
sch->qstats.drops += i; sch->qstats.backlog -= len; sch->q.qlen -= i; |
4b549a2ef fq_codel: Fair Qu... |
177 178 |
return idx; } |
520ac30f4 net_sched: drop p... |
179 180 |
static int fq_codel_enqueue(struct sk_buff *skb, struct Qdisc *sch, struct sk_buff **to_free) |
4b549a2ef fq_codel: Fair Qu... |
181 182 |
{ struct fq_codel_sched_data *q = qdisc_priv(sch); |
9d18562a2 fq_codel: add bat... |
183 |
unsigned int idx, prev_backlog, prev_qlen; |
4b549a2ef fq_codel: Fair Qu... |
184 |
struct fq_codel_flow *flow; |
3f649ab72 treewide: Remove ... |
185 |
int ret; |
80e509db5 fq_codel: fix NET... |
186 |
unsigned int pkt_len; |
95b58430a fq_codel: add mem... |
187 |
bool memory_limited; |
4b549a2ef fq_codel: Fair Qu... |
188 189 190 191 |
idx = fq_codel_classify(skb, sch, &ret); if (idx == 0) { if (ret & __NET_XMIT_BYPASS) |
25331d6ce net: sched: imple... |
192 |
qdisc_qstats_drop(sch); |
520ac30f4 net_sched: drop p... |
193 |
__qdisc_drop(skb, to_free); |
4b549a2ef fq_codel: Fair Qu... |
194 195 196 197 198 199 200 201 |
return ret; } idx--; codel_set_enqueue_time(skb); flow = &q->flows[idx]; flow_queue_add(flow, skb); q->backlogs[idx] += qdisc_pkt_len(skb); |
25331d6ce net: sched: imple... |
202 |
qdisc_qstats_backlog_inc(sch, skb); |
4b549a2ef fq_codel: Fair Qu... |
203 204 205 |
if (list_empty(&flow->flowchain)) { list_add_tail(&flow->flowchain, &q->new_flows); |
4b549a2ef fq_codel: Fair Qu... |
206 207 |
q->new_flow_count++; flow->deficit = q->quantum; |
4b549a2ef fq_codel: Fair Qu... |
208 |
} |
008830bc3 net_sched: fq_cod... |
209 210 |
get_codel_cb(skb)->mem_usage = skb->truesize; q->memory_usage += get_codel_cb(skb)->mem_usage; |
95b58430a fq_codel: add mem... |
211 212 |
memory_limited = q->memory_usage > q->memory_limit; if (++sch->q.qlen <= sch->limit && !memory_limited) |
4b549a2ef fq_codel: Fair Qu... |
213 |
return NET_XMIT_SUCCESS; |
2ccccf5fb net_sched: update... |
214 |
prev_backlog = sch->qstats.backlog; |
9d18562a2 fq_codel: add bat... |
215 |
prev_qlen = sch->q.qlen; |
80e509db5 fq_codel: fix NET... |
216 217 |
/* save this packet length as it might be dropped by fq_codel_drop() */ pkt_len = qdisc_pkt_len(skb); |
9d18562a2 fq_codel: add bat... |
218 219 220 221 |
/* fq_codel_drop() is quite expensive, as it performs a linear search * in q->backlogs[] to find a fat flow. * So instead of dropping a single packet, drop half of its backlog * with a 64 packets limit to not add a too big cpu spike here. |
4b549a2ef fq_codel: Fair Qu... |
222 |
*/ |
520ac30f4 net_sched: drop p... |
223 |
ret = fq_codel_drop(sch, q->drop_batch_size, to_free); |
9d18562a2 fq_codel: add bat... |
224 |
|
80e509db5 fq_codel: fix NET... |
225 226 227 |
prev_qlen -= sch->q.qlen; prev_backlog -= sch->qstats.backlog; q->drop_overlimit += prev_qlen; |
95b58430a fq_codel: add mem... |
228 |
if (memory_limited) |
80e509db5 fq_codel: fix NET... |
229 |
q->drop_overmemory += prev_qlen; |
9d18562a2 fq_codel: add bat... |
230 |
|
80e509db5 fq_codel: fix NET... |
231 232 233 234 235 236 237 238 239 240 241 |
/* As we dropped packet(s), better let upper stack know this. * If we dropped a packet for this flow, return NET_XMIT_CN, * but in this case, our parents wont increase their backlogs. */ if (ret == idx) { qdisc_tree_reduce_backlog(sch, prev_qlen - 1, prev_backlog - pkt_len); return NET_XMIT_CN; } qdisc_tree_reduce_backlog(sch, prev_qlen, prev_backlog); return NET_XMIT_SUCCESS; |
4b549a2ef fq_codel: Fair Qu... |
242 243 244 245 246 247 |
} /* This is the specific function called from codel_dequeue() * to dequeue a packet from queue. Note: backlog is handled in * codel, we dont need to reduce it here. */ |
79bdc4c86 codel: generalize... |
248 |
static struct sk_buff *dequeue_func(struct codel_vars *vars, void *ctx) |
4b549a2ef fq_codel: Fair Qu... |
249 |
{ |
79bdc4c86 codel: generalize... |
250 |
struct Qdisc *sch = ctx; |
865ec5523 fq_codel: should ... |
251 |
struct fq_codel_sched_data *q = qdisc_priv(sch); |
4b549a2ef fq_codel: Fair Qu... |
252 253 254 255 256 257 |
struct fq_codel_flow *flow; struct sk_buff *skb = NULL; flow = container_of(vars, struct fq_codel_flow, cvars); if (flow->head) { skb = dequeue_head(flow); |
865ec5523 fq_codel: should ... |
258 |
q->backlogs[flow - q->flows] -= qdisc_pkt_len(skb); |
008830bc3 net_sched: fq_cod... |
259 |
q->memory_usage -= get_codel_cb(skb)->mem_usage; |
4b549a2ef fq_codel: Fair Qu... |
260 |
sch->q.qlen--; |
79bdc4c86 codel: generalize... |
261 |
sch->qstats.backlog -= qdisc_pkt_len(skb); |
4b549a2ef fq_codel: Fair Qu... |
262 263 264 |
} return skb; } |
79bdc4c86 codel: generalize... |
265 266 267 |
static void drop_func(struct sk_buff *skb, void *ctx) { struct Qdisc *sch = ctx; |
520ac30f4 net_sched: drop p... |
268 269 |
kfree_skb(skb); qdisc_qstats_drop(sch); |
79bdc4c86 codel: generalize... |
270 |
} |
4b549a2ef fq_codel: Fair Qu... |
271 272 273 274 275 276 |
static struct sk_buff *fq_codel_dequeue(struct Qdisc *sch) { struct fq_codel_sched_data *q = qdisc_priv(sch); struct sk_buff *skb; struct fq_codel_flow *flow; struct list_head *head; |
4b549a2ef fq_codel: Fair Qu... |
277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 |
begin: head = &q->new_flows; if (list_empty(head)) { head = &q->old_flows; if (list_empty(head)) return NULL; } flow = list_first_entry(head, struct fq_codel_flow, flowchain); if (flow->deficit <= 0) { flow->deficit += q->quantum; list_move_tail(&flow->flowchain, &q->old_flows); goto begin; } |
79bdc4c86 codel: generalize... |
292 293 294 |
skb = codel_dequeue(sch, &sch->qstats.backlog, &q->cparams, &flow->cvars, &q->cstats, qdisc_pkt_len, codel_get_enqueue_time, drop_func, dequeue_func); |
4b549a2ef fq_codel: Fair Qu... |
295 |
|
4b549a2ef fq_codel: Fair Qu... |
296 297 298 299 300 301 302 303 304 305 |
if (!skb) { /* force a pass through old_flows to prevent starvation */ if ((head == &q->new_flows) && !list_empty(&q->old_flows)) list_move_tail(&flow->flowchain, &q->old_flows); else list_del_init(&flow->flowchain); goto begin; } qdisc_bstats_update(sch, skb); flow->deficit -= qdisc_pkt_len(skb); |
2ccccf5fb net_sched: update... |
306 |
/* We cant call qdisc_tree_reduce_backlog() if our qlen is 0, |
4b549a2ef fq_codel: Fair Qu... |
307 308 309 |
* or HTB crashes. Defer it for next round. */ if (q->cstats.drop_count && sch->q.qlen) { |
2ccccf5fb net_sched: update... |
310 311 |
qdisc_tree_reduce_backlog(sch, q->cstats.drop_count, q->cstats.drop_len); |
4b549a2ef fq_codel: Fair Qu... |
312 |
q->cstats.drop_count = 0; |
2ccccf5fb net_sched: update... |
313 |
q->cstats.drop_len = 0; |
4b549a2ef fq_codel: Fair Qu... |
314 315 316 |
} return skb; } |
ece5d4c72 net_sched: fq_cod... |
317 318 319 320 321 |
static void fq_codel_flow_purge(struct fq_codel_flow *flow) { rtnl_kfree_skbs(flow->head, flow->tail); flow->head = NULL; } |
4b549a2ef fq_codel: Fair Qu... |
322 323 |
static void fq_codel_reset(struct Qdisc *sch) { |
3d0e0af40 fq_codel: explici... |
324 325 |
struct fq_codel_sched_data *q = qdisc_priv(sch); int i; |
4b549a2ef fq_codel: Fair Qu... |
326 |
|
3d0e0af40 fq_codel: explici... |
327 328 329 330 |
INIT_LIST_HEAD(&q->new_flows); INIT_LIST_HEAD(&q->old_flows); for (i = 0; i < q->flows_cnt; i++) { struct fq_codel_flow *flow = q->flows + i; |
ece5d4c72 net_sched: fq_cod... |
331 |
fq_codel_flow_purge(flow); |
3d0e0af40 fq_codel: explici... |
332 333 334 335 336 |
INIT_LIST_HEAD(&flow->flowchain); codel_vars_init(&flow->cvars); } memset(q->backlogs, 0, q->flows_cnt * sizeof(u32)); sch->q.qlen = 0; |
ece5d4c72 net_sched: fq_cod... |
337 |
sch->qstats.backlog = 0; |
77f577614 fq_codel: fix mem... |
338 |
q->memory_usage = 0; |
4b549a2ef fq_codel: Fair Qu... |
339 340 341 342 343 344 345 346 347 |
} static const struct nla_policy fq_codel_policy[TCA_FQ_CODEL_MAX + 1] = { [TCA_FQ_CODEL_TARGET] = { .type = NLA_U32 }, [TCA_FQ_CODEL_LIMIT] = { .type = NLA_U32 }, [TCA_FQ_CODEL_INTERVAL] = { .type = NLA_U32 }, [TCA_FQ_CODEL_ECN] = { .type = NLA_U32 }, [TCA_FQ_CODEL_FLOWS] = { .type = NLA_U32 }, [TCA_FQ_CODEL_QUANTUM] = { .type = NLA_U32 }, |
80ba92fa1 codel: add ce_thr... |
348 |
[TCA_FQ_CODEL_CE_THRESHOLD] = { .type = NLA_U32 }, |
9d18562a2 fq_codel: add bat... |
349 |
[TCA_FQ_CODEL_DROP_BATCH_SIZE] = { .type = NLA_U32 }, |
95b58430a fq_codel: add mem... |
350 |
[TCA_FQ_CODEL_MEMORY_LIMIT] = { .type = NLA_U32 }, |
4b549a2ef fq_codel: Fair Qu... |
351 |
}; |
2030721cc net: sched: sch: ... |
352 353 |
static int fq_codel_change(struct Qdisc *sch, struct nlattr *opt, struct netlink_ext_ack *extack) |
4b549a2ef fq_codel: Fair Qu... |
354 355 356 357 358 359 360 |
{ struct fq_codel_sched_data *q = qdisc_priv(sch); struct nlattr *tb[TCA_FQ_CODEL_MAX + 1]; int err; if (!opt) return -EINVAL; |
8cb081746 netlink: make val... |
361 362 |
err = nla_parse_nested_deprecated(tb, TCA_FQ_CODEL_MAX, opt, fq_codel_policy, NULL); |
4b549a2ef fq_codel: Fair Qu... |
363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 |
if (err < 0) return err; if (tb[TCA_FQ_CODEL_FLOWS]) { if (q->flows) return -EINVAL; q->flows_cnt = nla_get_u32(tb[TCA_FQ_CODEL_FLOWS]); if (!q->flows_cnt || q->flows_cnt > 65536) return -EINVAL; } sch_tree_lock(sch); if (tb[TCA_FQ_CODEL_TARGET]) { u64 target = nla_get_u32(tb[TCA_FQ_CODEL_TARGET]); q->cparams.target = (target * NSEC_PER_USEC) >> CODEL_SHIFT; } |
80ba92fa1 codel: add ce_thr... |
380 381 382 383 384 |
if (tb[TCA_FQ_CODEL_CE_THRESHOLD]) { u64 val = nla_get_u32(tb[TCA_FQ_CODEL_CE_THRESHOLD]); q->cparams.ce_threshold = (val * NSEC_PER_USEC) >> CODEL_SHIFT; } |
4b549a2ef fq_codel: Fair Qu... |
385 386 387 388 389 390 391 392 393 394 395 396 397 398 |
if (tb[TCA_FQ_CODEL_INTERVAL]) { u64 interval = nla_get_u32(tb[TCA_FQ_CODEL_INTERVAL]); q->cparams.interval = (interval * NSEC_PER_USEC) >> CODEL_SHIFT; } if (tb[TCA_FQ_CODEL_LIMIT]) sch->limit = nla_get_u32(tb[TCA_FQ_CODEL_LIMIT]); if (tb[TCA_FQ_CODEL_ECN]) q->cparams.ecn = !!nla_get_u32(tb[TCA_FQ_CODEL_ECN]); if (tb[TCA_FQ_CODEL_QUANTUM]) q->quantum = max(256U, nla_get_u32(tb[TCA_FQ_CODEL_QUANTUM])); |
9d18562a2 fq_codel: add bat... |
399 |
if (tb[TCA_FQ_CODEL_DROP_BATCH_SIZE]) |
14695212d fq_codel: fix TCA... |
400 |
q->drop_batch_size = max(1U, nla_get_u32(tb[TCA_FQ_CODEL_DROP_BATCH_SIZE])); |
9d18562a2 fq_codel: add bat... |
401 |
|
95b58430a fq_codel: add mem... |
402 403 404 405 406 |
if (tb[TCA_FQ_CODEL_MEMORY_LIMIT]) q->memory_limit = min(1U << 31, nla_get_u32(tb[TCA_FQ_CODEL_MEMORY_LIMIT])); while (sch->q.qlen > sch->limit || q->memory_usage > q->memory_limit) { |
4b549a2ef fq_codel: Fair Qu... |
407 |
struct sk_buff *skb = fq_codel_dequeue(sch); |
2ccccf5fb net_sched: update... |
408 |
q->cstats.drop_len += qdisc_pkt_len(skb); |
ece5d4c72 net_sched: fq_cod... |
409 |
rtnl_kfree_skbs(skb, skb); |
4b549a2ef fq_codel: Fair Qu... |
410 411 |
q->cstats.drop_count++; } |
2ccccf5fb net_sched: update... |
412 |
qdisc_tree_reduce_backlog(sch, q->cstats.drop_count, q->cstats.drop_len); |
4b549a2ef fq_codel: Fair Qu... |
413 |
q->cstats.drop_count = 0; |
2ccccf5fb net_sched: update... |
414 |
q->cstats.drop_len = 0; |
4b549a2ef fq_codel: Fair Qu... |
415 416 417 418 |
sch_tree_unlock(sch); return 0; } |
4b549a2ef fq_codel: Fair Qu... |
419 420 421 |
static void fq_codel_destroy(struct Qdisc *sch) { struct fq_codel_sched_data *q = qdisc_priv(sch); |
6529eaba3 net: sched: intro... |
422 |
tcf_block_put(q->block); |
752ade68c treewide: use kv[... |
423 424 |
kvfree(q->backlogs); kvfree(q->flows); |
4b549a2ef fq_codel: Fair Qu... |
425 |
} |
e63d7dfd2 net: sched: sch: ... |
426 427 |
static int fq_codel_init(struct Qdisc *sch, struct nlattr *opt, struct netlink_ext_ack *extack) |
4b549a2ef fq_codel: Fair Qu... |
428 429 430 |
{ struct fq_codel_sched_data *q = qdisc_priv(sch); int i; |
6529eaba3 net: sched: intro... |
431 |
int err; |
4b549a2ef fq_codel: Fair Qu... |
432 433 434 |
sch->limit = 10*1024; q->flows_cnt = 1024; |
95b58430a fq_codel: add mem... |
435 |
q->memory_limit = 32 << 20; /* 32 MBytes */ |
9d18562a2 fq_codel: add bat... |
436 |
q->drop_batch_size = 64; |
4b549a2ef fq_codel: Fair Qu... |
437 |
q->quantum = psched_mtu(qdisc_dev(sch)); |
4b549a2ef fq_codel: Fair Qu... |
438 439 |
INIT_LIST_HEAD(&q->new_flows); INIT_LIST_HEAD(&q->old_flows); |
79bdc4c86 codel: generalize... |
440 |
codel_params_init(&q->cparams); |
4b549a2ef fq_codel: Fair Qu... |
441 442 |
codel_stats_init(&q->cstats); q->cparams.ecn = true; |
79bdc4c86 codel: generalize... |
443 |
q->cparams.mtu = psched_mtu(qdisc_dev(sch)); |
4b549a2ef fq_codel: Fair Qu... |
444 445 |
if (opt) { |
83fe6b870 sch_fq_codel: zer... |
446 |
err = fq_codel_change(sch, opt, extack); |
4b549a2ef fq_codel: Fair Qu... |
447 |
if (err) |
83fe6b870 sch_fq_codel: zer... |
448 |
goto init_failure; |
4b549a2ef fq_codel: Fair Qu... |
449 |
} |
8d1a77f97 net: sch: api: ad... |
450 |
err = tcf_block_get(&q->block, &q->filter_list, sch, extack); |
6529eaba3 net: sched: intro... |
451 |
if (err) |
83fe6b870 sch_fq_codel: zer... |
452 |
goto init_failure; |
6529eaba3 net: sched: intro... |
453 |
|
4b549a2ef fq_codel: Fair Qu... |
454 |
if (!q->flows) { |
778e1cdd8 treewide: kvzallo... |
455 456 457 |
q->flows = kvcalloc(q->flows_cnt, sizeof(struct fq_codel_flow), GFP_KERNEL); |
83fe6b870 sch_fq_codel: zer... |
458 459 460 461 |
if (!q->flows) { err = -ENOMEM; goto init_failure; } |
778e1cdd8 treewide: kvzallo... |
462 |
q->backlogs = kvcalloc(q->flows_cnt, sizeof(u32), GFP_KERNEL); |
83fe6b870 sch_fq_codel: zer... |
463 464 465 466 |
if (!q->backlogs) { err = -ENOMEM; goto alloc_failure; } |
4b549a2ef fq_codel: Fair Qu... |
467 468 469 470 |
for (i = 0; i < q->flows_cnt; i++) { struct fq_codel_flow *flow = q->flows + i; INIT_LIST_HEAD(&flow->flowchain); |
b379135c4 fq_codel: dont re... |
471 |
codel_vars_init(&flow->cvars); |
4b549a2ef fq_codel: Fair Qu... |
472 473 474 475 476 477 478 |
} } if (sch->limit >= 1) sch->flags |= TCQ_F_CAN_BYPASS; else sch->flags &= ~TCQ_F_CAN_BYPASS; return 0; |
83fe6b870 sch_fq_codel: zer... |
479 480 481 482 483 484 485 |
alloc_failure: kvfree(q->flows); q->flows = NULL; init_failure: q->flows_cnt = 0; return err; |
4b549a2ef fq_codel: Fair Qu... |
486 487 488 489 490 491 |
} static int fq_codel_dump(struct Qdisc *sch, struct sk_buff *skb) { struct fq_codel_sched_data *q = qdisc_priv(sch); struct nlattr *opts; |
ae0be8de9 netlink: make nla... |
492 |
opts = nla_nest_start_noflag(skb, TCA_OPTIONS); |
4b549a2ef fq_codel: Fair Qu... |
493 494 495 496 497 498 499 500 501 502 503 504 505 |
if (opts == NULL) goto nla_put_failure; if (nla_put_u32(skb, TCA_FQ_CODEL_TARGET, codel_time_to_us(q->cparams.target)) || nla_put_u32(skb, TCA_FQ_CODEL_LIMIT, sch->limit) || nla_put_u32(skb, TCA_FQ_CODEL_INTERVAL, codel_time_to_us(q->cparams.interval)) || nla_put_u32(skb, TCA_FQ_CODEL_ECN, q->cparams.ecn) || nla_put_u32(skb, TCA_FQ_CODEL_QUANTUM, q->quantum) || |
9d18562a2 fq_codel: add bat... |
506 507 |
nla_put_u32(skb, TCA_FQ_CODEL_DROP_BATCH_SIZE, q->drop_batch_size) || |
95b58430a fq_codel: add mem... |
508 509 |
nla_put_u32(skb, TCA_FQ_CODEL_MEMORY_LIMIT, q->memory_limit) || |
4b549a2ef fq_codel: Fair Qu... |
510 511 512 |
nla_put_u32(skb, TCA_FQ_CODEL_FLOWS, q->flows_cnt)) goto nla_put_failure; |
80ba92fa1 codel: add ce_thr... |
513 514 515 516 |
if (q->cparams.ce_threshold != CODEL_DISABLED_THRESHOLD && nla_put_u32(skb, TCA_FQ_CODEL_CE_THRESHOLD, codel_time_to_us(q->cparams.ce_threshold))) goto nla_put_failure; |
d59b7d805 net_sched: return... |
517 |
return nla_nest_end(skb, opts); |
4b549a2ef fq_codel: Fair Qu... |
518 519 520 521 522 523 524 525 526 527 |
nla_put_failure: return -1; } static int fq_codel_dump_stats(struct Qdisc *sch, struct gnet_dump *d) { struct fq_codel_sched_data *q = qdisc_priv(sch); struct tc_fq_codel_xstats st = { .type = TCA_FQ_CODEL_XSTATS_QDISC, |
4b549a2ef fq_codel: Fair Qu... |
528 529 |
}; struct list_head *pos; |
669d67bf7 net: codel: fix b... |
530 531 532 533 |
st.qdisc_stats.maxpacket = q->cstats.maxpacket; st.qdisc_stats.drop_overlimit = q->drop_overlimit; st.qdisc_stats.ecn_mark = q->cstats.ecn_mark; st.qdisc_stats.new_flow_count = q->new_flow_count; |
80ba92fa1 codel: add ce_thr... |
534 |
st.qdisc_stats.ce_mark = q->cstats.ce_mark; |
95b58430a fq_codel: add mem... |
535 536 |
st.qdisc_stats.memory_usage = q->memory_usage; st.qdisc_stats.drop_overmemory = q->drop_overmemory; |
669d67bf7 net: codel: fix b... |
537 |
|
edb09eb17 net: sched: do no... |
538 |
sch_tree_lock(sch); |
4b549a2ef fq_codel: Fair Qu... |
539 540 541 542 543 |
list_for_each(pos, &q->new_flows) st.qdisc_stats.new_flows_len++; list_for_each(pos, &q->old_flows) st.qdisc_stats.old_flows_len++; |
edb09eb17 net: sched: do no... |
544 |
sch_tree_unlock(sch); |
4b549a2ef fq_codel: Fair Qu... |
545 546 547 548 549 550 551 552 |
return gnet_stats_copy_app(d, &st, sizeof(st)); } static struct Qdisc *fq_codel_leaf(struct Qdisc *sch, unsigned long arg) { return NULL; } |
143976ce9 net_sched: remove... |
553 |
static unsigned long fq_codel_find(struct Qdisc *sch, u32 classid) |
4b549a2ef fq_codel: Fair Qu... |
554 555 556 557 558 559 560 |
{ return 0; } static unsigned long fq_codel_bind(struct Qdisc *sch, unsigned long parent, u32 classid) { |
4b549a2ef fq_codel: Fair Qu... |
561 562 |
return 0; } |
143976ce9 net_sched: remove... |
563 |
static void fq_codel_unbind(struct Qdisc *q, unsigned long cl) |
4b549a2ef fq_codel: Fair Qu... |
564 565 |
{ } |
cbaacc4e8 net: sched: sch: ... |
566 567 |
static struct tcf_block *fq_codel_tcf_block(struct Qdisc *sch, unsigned long cl, struct netlink_ext_ack *extack) |
4b549a2ef fq_codel: Fair Qu... |
568 569 570 571 572 |
{ struct fq_codel_sched_data *q = qdisc_priv(sch); if (cl) return NULL; |
6529eaba3 net: sched: intro... |
573 |
return q->block; |
4b549a2ef fq_codel: Fair Qu... |
574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 |
} static int fq_codel_dump_class(struct Qdisc *sch, unsigned long cl, struct sk_buff *skb, struct tcmsg *tcm) { tcm->tcm_handle |= TC_H_MIN(cl); return 0; } static int fq_codel_dump_class_stats(struct Qdisc *sch, unsigned long cl, struct gnet_dump *d) { struct fq_codel_sched_data *q = qdisc_priv(sch); u32 idx = cl - 1; struct gnet_stats_queue qs = { 0 }; struct tc_fq_codel_xstats xstats; if (idx < q->flows_cnt) { const struct fq_codel_flow *flow = &q->flows[idx]; |
edb09eb17 net: sched: do no... |
593 |
const struct sk_buff *skb; |
4b549a2ef fq_codel: Fair Qu... |
594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 |
memset(&xstats, 0, sizeof(xstats)); xstats.type = TCA_FQ_CODEL_XSTATS_CLASS; xstats.class_stats.deficit = flow->deficit; xstats.class_stats.ldelay = codel_time_to_us(flow->cvars.ldelay); xstats.class_stats.count = flow->cvars.count; xstats.class_stats.lastcount = flow->cvars.lastcount; xstats.class_stats.dropping = flow->cvars.dropping; if (flow->cvars.dropping) { codel_tdiff_t delta = flow->cvars.drop_next - codel_get_time(); xstats.class_stats.drop_next = (delta >= 0) ? codel_time_to_us(delta) : -codel_time_to_us(-delta); } |
edb09eb17 net: sched: do no... |
611 612 613 614 615 616 617 618 |
if (flow->head) { sch_tree_lock(sch); skb = flow->head; while (skb) { qs.qlen++; skb = skb->next; } sch_tree_unlock(sch); |
4b549a2ef fq_codel: Fair Qu... |
619 620 |
} qs.backlog = q->backlogs[idx]; |
77ddaff21 fq_codel: Kill us... |
621 |
qs.drops = 0; |
4b549a2ef fq_codel: Fair Qu... |
622 |
} |
aafddbf0c fq_codel: return ... |
623 |
if (gnet_stats_copy_queue(d, NULL, &qs, qs.qlen) < 0) |
4b549a2ef fq_codel: Fair Qu... |
624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 |
return -1; if (idx < q->flows_cnt) return gnet_stats_copy_app(d, &xstats, sizeof(xstats)); return 0; } static void fq_codel_walk(struct Qdisc *sch, struct qdisc_walker *arg) { struct fq_codel_sched_data *q = qdisc_priv(sch); unsigned int i; if (arg->stop) return; for (i = 0; i < q->flows_cnt; i++) { if (list_empty(&q->flows[i].flowchain) || arg->count < arg->skip) { arg->count++; continue; } if (arg->fn(sch, i + 1, arg) < 0) { arg->stop = 1; break; } arg->count++; } } static const struct Qdisc_class_ops fq_codel_class_ops = { .leaf = fq_codel_leaf, |
143976ce9 net_sched: remove... |
654 |
.find = fq_codel_find, |
6529eaba3 net: sched: intro... |
655 |
.tcf_block = fq_codel_tcf_block, |
4b549a2ef fq_codel: Fair Qu... |
656 |
.bind_tcf = fq_codel_bind, |
143976ce9 net_sched: remove... |
657 |
.unbind_tcf = fq_codel_unbind, |
4b549a2ef fq_codel: Fair Qu... |
658 659 660 661 662 663 664 665 666 667 668 669 |
.dump = fq_codel_dump_class, .dump_stats = fq_codel_dump_class_stats, .walk = fq_codel_walk, }; static struct Qdisc_ops fq_codel_qdisc_ops __read_mostly = { .cl_ops = &fq_codel_class_ops, .id = "fq_codel", .priv_size = sizeof(struct fq_codel_sched_data), .enqueue = fq_codel_enqueue, .dequeue = fq_codel_dequeue, .peek = qdisc_peek_dequeued, |
4b549a2ef fq_codel: Fair Qu... |
670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 |
.init = fq_codel_init, .reset = fq_codel_reset, .destroy = fq_codel_destroy, .change = fq_codel_change, .dump = fq_codel_dump, .dump_stats = fq_codel_dump_stats, .owner = THIS_MODULE, }; static int __init fq_codel_module_init(void) { return register_qdisc(&fq_codel_qdisc_ops); } static void __exit fq_codel_module_exit(void) { unregister_qdisc(&fq_codel_qdisc_ops); } module_init(fq_codel_module_init) module_exit(fq_codel_module_exit) MODULE_AUTHOR("Eric Dumazet"); MODULE_LICENSE("GPL"); |
67c20de35 net: Add MODULE_D... |
693 |
MODULE_DESCRIPTION("Fair Queue CoDel discipline"); |