Blame view
lib/dynamic_queue_limits.c
4.33 KB
b24413180 License cleanup: ... |
1 |
// SPDX-License-Identifier: GPL-2.0 |
75957ba36 dql: Dynamic queu... |
2 3 4 5 6 |
/* * Dynamic byte queue limits. See include/linux/dynamic_queue_limits.h * * Copyright (c) 2011, Tom Herbert <therbert@google.com> */ |
75957ba36 dql: Dynamic queu... |
7 |
#include <linux/types.h> |
75957ba36 dql: Dynamic queu... |
8 |
#include <linux/kernel.h> |
930c514f6 dql: Fix undefine... |
9 |
#include <linux/jiffies.h> |
75957ba36 dql: Dynamic queu... |
10 |
#include <linux/dynamic_queue_limits.h> |
565ac23b8 lib/dynamic_queue... |
11 12 |
#include <linux/compiler.h> #include <linux/export.h> |
75957ba36 dql: Dynamic queu... |
13 |
|
0cfd32b73 bql: Fix POSDIFF(... |
14 |
#define POSDIFF(A, B) ((int)((A) - (B)) > 0 ? (A) - (B) : 0) |
25426b794 bql: Avoid unneed... |
15 |
#define AFTER_EQ(A, B) ((int)((A) - (B)) >= 0) |
75957ba36 dql: Dynamic queu... |
16 17 18 19 20 |
/* Records completed count and recalculates the queue limit */ void dql_completed(struct dql *dql, unsigned int count) { unsigned int inprogress, prev_inprogress, limit; |
914bec101 bql: Avoid possib... |
21 |
unsigned int ovlimit, completed, num_queued; |
25426b794 bql: Avoid unneed... |
22 |
bool all_prev_completed; |
75957ba36 dql: Dynamic queu... |
23 |
|
6aa7de059 locking/atomics: ... |
24 |
num_queued = READ_ONCE(dql->num_queued); |
914bec101 bql: Avoid possib... |
25 |
|
75957ba36 dql: Dynamic queu... |
26 |
/* Can't complete more than what's in queue */ |
914bec101 bql: Avoid possib... |
27 |
BUG_ON(count > num_queued - dql->num_completed); |
75957ba36 dql: Dynamic queu... |
28 29 30 |
completed = dql->num_completed + count; limit = dql->limit; |
914bec101 bql: Avoid possib... |
31 32 |
ovlimit = POSDIFF(num_queued - dql->num_completed, limit); inprogress = num_queued - completed; |
75957ba36 dql: Dynamic queu... |
33 |
prev_inprogress = dql->prev_num_queued - dql->num_completed; |
25426b794 bql: Avoid unneed... |
34 |
all_prev_completed = AFTER_EQ(completed, dql->prev_num_queued); |
75957ba36 dql: Dynamic queu... |
35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 |
if ((ovlimit && !inprogress) || (dql->prev_ovlimit && all_prev_completed)) { /* * Queue considered starved if: * - The queue was over-limit in the last interval, * and there is no more data in the queue. * OR * - The queue was over-limit in the previous interval and * when enqueuing it was possible that all queued data * had been consumed. This covers the case when queue * may have becomes starved between completion processing * running and next time enqueue was scheduled. * * When queue is starved increase the limit by the amount * of bytes both sent and completed in the last interval, * plus any previous over-limit. */ limit += POSDIFF(completed, dql->prev_num_queued) + dql->prev_ovlimit; dql->slack_start_time = jiffies; dql->lowest_slack = UINT_MAX; } else if (inprogress && prev_inprogress && !all_prev_completed) { /* * Queue was not starved, check if the limit can be decreased. * A decrease is only considered if the queue has been busy in * the whole interval (the check above). * * If there is slack, the amount of execess data queued above * the the amount needed to prevent starvation, the queue limit * can be decreased. To avoid hysteresis we consider the * minimum amount of slack found over several iterations of the * completion routine. */ unsigned int slack, slack_last_objs; /* * Slack is the maximum of * - The queue limit plus previous over-limit minus twice * the number of objects completed. Note that two times * number of completed bytes is a basis for an upper bound * of the limit. * - Portion of objects in the last queuing operation that * was not part of non-zero previous over-limit. That is * "round down" by non-overlimit portion of the last * queueing operation. */ slack = POSDIFF(limit + dql->prev_ovlimit, 2 * (completed - dql->num_completed)); slack_last_objs = dql->prev_ovlimit ? POSDIFF(dql->prev_last_obj_cnt, dql->prev_ovlimit) : 0; slack = max(slack, slack_last_objs); if (slack < dql->lowest_slack) dql->lowest_slack = slack; if (time_after(jiffies, dql->slack_start_time + dql->slack_hold_time)) { limit = POSDIFF(limit, dql->lowest_slack); dql->slack_start_time = jiffies; dql->lowest_slack = UINT_MAX; } } /* Enforce bounds on limit */ limit = clamp(limit, dql->min_limit, dql->max_limit); if (limit != dql->limit) { dql->limit = limit; ovlimit = 0; } dql->adj_limit = limit + completed; dql->prev_ovlimit = ovlimit; dql->prev_last_obj_cnt = dql->last_obj_cnt; dql->num_completed = completed; |
914bec101 bql: Avoid possib... |
112 |
dql->prev_num_queued = num_queued; |
75957ba36 dql: Dynamic queu... |
113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 |
} EXPORT_SYMBOL(dql_completed); void dql_reset(struct dql *dql) { /* Reset all dynamic values */ dql->limit = 0; dql->num_queued = 0; dql->num_completed = 0; dql->last_obj_cnt = 0; dql->prev_num_queued = 0; dql->prev_last_obj_cnt = 0; dql->prev_ovlimit = 0; dql->lowest_slack = UINT_MAX; dql->slack_start_time = jiffies; } EXPORT_SYMBOL(dql_reset); |
7a0947e75 dql: make dql_ini... |
130 |
void dql_init(struct dql *dql, unsigned int hold_time) |
75957ba36 dql: Dynamic queu... |
131 132 133 134 135 |
{ dql->max_limit = DQL_MAX_LIMIT; dql->min_limit = 0; dql->slack_hold_time = hold_time; dql_reset(dql); |
75957ba36 dql: Dynamic queu... |
136 137 |
} EXPORT_SYMBOL(dql_init); |