Blame view
mm/page_counter.c
6.56 KB
b24413180 License cleanup: ... |
1 |
// SPDX-License-Identifier: GPL-2.0 |
3e32cb2e0 mm: memcontrol: l... |
2 3 4 5 6 7 8 9 10 11 12 13 14 |
/* * Lockless hierarchical page accounting & limiting * * Copyright (C) 2014 Red Hat, Inc., Johannes Weiner */ #include <linux/page_counter.h> #include <linux/atomic.h> #include <linux/kernel.h> #include <linux/string.h> #include <linux/sched.h> #include <linux/bug.h> #include <asm/page.h> |
bf8d5d52f memcg: introduce ... |
15 16 |
static void propagate_protected_usage(struct page_counter *c, unsigned long usage) |
230671533 mm: memory.low hi... |
17 |
{ |
bf8d5d52f memcg: introduce ... |
18 |
unsigned long protected, old_protected; |
230671533 mm: memory.low hi... |
19 20 21 22 |
long delta; if (!c->parent) return; |
bf8d5d52f memcg: introduce ... |
23 24 25 26 27 28 29 30 31 32 33 |
if (c->min || atomic_long_read(&c->min_usage)) { if (usage <= c->min) protected = usage; else protected = 0; old_protected = atomic_long_xchg(&c->min_usage, protected); delta = protected - old_protected; if (delta) atomic_long_add(delta, &c->parent->children_min_usage); } |
230671533 mm: memory.low hi... |
34 |
|
bf8d5d52f memcg: introduce ... |
35 36 37 38 39 |
if (c->low || atomic_long_read(&c->low_usage)) { if (usage <= c->low) protected = usage; else protected = 0; |
230671533 mm: memory.low hi... |
40 |
|
bf8d5d52f memcg: introduce ... |
41 42 43 44 45 |
old_protected = atomic_long_xchg(&c->low_usage, protected); delta = protected - old_protected; if (delta) atomic_long_add(delta, &c->parent->children_low_usage); } |
230671533 mm: memory.low hi... |
46 |
} |
3e32cb2e0 mm: memcontrol: l... |
47 48 49 50 |
/** * page_counter_cancel - take pages out of the local counter * @counter: counter * @nr_pages: number of pages to cancel |
3e32cb2e0 mm: memcontrol: l... |
51 |
*/ |
64f219938 mm: memcontrol: r... |
52 |
void page_counter_cancel(struct page_counter *counter, unsigned long nr_pages) |
3e32cb2e0 mm: memcontrol: l... |
53 54 |
{ long new; |
bbec2e151 mm: rename page_c... |
55 |
new = atomic_long_sub_return(nr_pages, &counter->usage); |
bf8d5d52f memcg: introduce ... |
56 |
propagate_protected_usage(counter, new); |
3e32cb2e0 mm: memcontrol: l... |
57 58 |
/* More uncharges than charges? */ WARN_ON_ONCE(new < 0); |
3e32cb2e0 mm: memcontrol: l... |
59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 |
} /** * page_counter_charge - hierarchically charge pages * @counter: counter * @nr_pages: number of pages to charge * * NOTE: This does not consider any configured counter limits. */ void page_counter_charge(struct page_counter *counter, unsigned long nr_pages) { struct page_counter *c; for (c = counter; c; c = c->parent) { long new; |
bbec2e151 mm: rename page_c... |
74 |
new = atomic_long_add_return(nr_pages, &c->usage); |
bf8d5d52f memcg: introduce ... |
75 |
propagate_protected_usage(counter, new); |
3e32cb2e0 mm: memcontrol: l... |
76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 |
/* * This is indeed racy, but we can live with some * inaccuracy in the watermark. */ if (new > c->watermark) c->watermark = new; } } /** * page_counter_try_charge - try to hierarchically charge pages * @counter: counter * @nr_pages: number of pages to charge * @fail: points first counter to hit its limit, if any * |
6071ca520 mm: page_counter:... |
91 92 |
* Returns %true on success, or %false and @fail if the counter or one * of its ancestors has hit its configured limit. |
3e32cb2e0 mm: memcontrol: l... |
93 |
*/ |
6071ca520 mm: page_counter:... |
94 95 96 |
bool page_counter_try_charge(struct page_counter *counter, unsigned long nr_pages, struct page_counter **fail) |
3e32cb2e0 mm: memcontrol: l... |
97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 |
{ struct page_counter *c; for (c = counter; c; c = c->parent) { long new; /* * Charge speculatively to avoid an expensive CAS. If * a bigger charge fails, it might falsely lock out a * racing smaller charge and send it into reclaim * early, but the error is limited to the difference * between the two sizes, which is less than 2M/4M in * case of a THP locking out a regular page charge. * * The atomic_long_add_return() implies a full memory * barrier between incrementing the count and reading * the limit. When racing with page_counter_limit(), * we either see the new limit or the setter sees the * counter has changed and retries. */ |
bbec2e151 mm: rename page_c... |
116 117 118 |
new = atomic_long_add_return(nr_pages, &c->usage); if (new > c->max) { atomic_long_sub(nr_pages, &c->usage); |
bf8d5d52f memcg: introduce ... |
119 |
propagate_protected_usage(counter, new); |
3e32cb2e0 mm: memcontrol: l... |
120 121 122 123 124 125 126 127 |
/* * This is racy, but we can live with some * inaccuracy in the failcnt. */ c->failcnt++; *fail = c; goto failed; } |
bf8d5d52f memcg: introduce ... |
128 |
propagate_protected_usage(counter, new); |
3e32cb2e0 mm: memcontrol: l... |
129 130 131 132 133 134 135 |
/* * Just like with failcnt, we can live with some * inaccuracy in the watermark. */ if (new > c->watermark) c->watermark = new; } |
6071ca520 mm: page_counter:... |
136 |
return true; |
3e32cb2e0 mm: memcontrol: l... |
137 138 139 140 |
failed: for (c = counter; c != *fail; c = c->parent) page_counter_cancel(c, nr_pages); |
6071ca520 mm: page_counter:... |
141 |
return false; |
3e32cb2e0 mm: memcontrol: l... |
142 143 144 145 146 147 |
} /** * page_counter_uncharge - hierarchically uncharge pages * @counter: counter * @nr_pages: number of pages to uncharge |
3e32cb2e0 mm: memcontrol: l... |
148 |
*/ |
64f219938 mm: memcontrol: r... |
149 |
void page_counter_uncharge(struct page_counter *counter, unsigned long nr_pages) |
3e32cb2e0 mm: memcontrol: l... |
150 151 |
{ struct page_counter *c; |
3e32cb2e0 mm: memcontrol: l... |
152 |
|
64f219938 mm: memcontrol: r... |
153 154 |
for (c = counter; c; c = c->parent) page_counter_cancel(c, nr_pages); |
3e32cb2e0 mm: memcontrol: l... |
155 156 157 |
} /** |
bbec2e151 mm: rename page_c... |
158 |
* page_counter_set_max - set the maximum number of pages allowed |
3e32cb2e0 mm: memcontrol: l... |
159 |
* @counter: counter |
bbec2e151 mm: rename page_c... |
160 |
* @nr_pages: limit to set |
3e32cb2e0 mm: memcontrol: l... |
161 162 163 164 165 166 |
* * Returns 0 on success, -EBUSY if the current number of pages on the * counter already exceeds the specified limit. * * The caller must serialize invocations on the same counter. */ |
bbec2e151 mm: rename page_c... |
167 |
int page_counter_set_max(struct page_counter *counter, unsigned long nr_pages) |
3e32cb2e0 mm: memcontrol: l... |
168 169 170 |
{ for (;;) { unsigned long old; |
bbec2e151 mm: rename page_c... |
171 |
long usage; |
3e32cb2e0 mm: memcontrol: l... |
172 173 174 175 176 177 178 179 180 181 182 183 |
/* * Update the limit while making sure that it's not * below the concurrently-changing counter value. * * The xchg implies two full memory barriers before * and after, so the read-swap-read is ordered and * ensures coherency with page_counter_try_charge(): * that function modifies the count before checking * the limit, so if it sees the old limit, we see the * modified counter and retry. */ |
bbec2e151 mm: rename page_c... |
184 |
usage = atomic_long_read(&counter->usage); |
3e32cb2e0 mm: memcontrol: l... |
185 |
|
bbec2e151 mm: rename page_c... |
186 |
if (usage > nr_pages) |
3e32cb2e0 mm: memcontrol: l... |
187 |
return -EBUSY; |
bbec2e151 mm: rename page_c... |
188 |
old = xchg(&counter->max, nr_pages); |
3e32cb2e0 mm: memcontrol: l... |
189 |
|
bbec2e151 mm: rename page_c... |
190 |
if (atomic_long_read(&counter->usage) <= usage) |
3e32cb2e0 mm: memcontrol: l... |
191 |
return 0; |
bbec2e151 mm: rename page_c... |
192 |
counter->max = old; |
3e32cb2e0 mm: memcontrol: l... |
193 194 195 196 197 |
cond_resched(); } } /** |
bf8d5d52f memcg: introduce ... |
198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 |
* page_counter_set_min - set the amount of protected memory * @counter: counter * @nr_pages: value to set * * The caller must serialize invocations on the same counter. */ void page_counter_set_min(struct page_counter *counter, unsigned long nr_pages) { struct page_counter *c; counter->min = nr_pages; for (c = counter; c; c = c->parent) propagate_protected_usage(c, atomic_long_read(&c->usage)); } /** |
230671533 mm: memory.low hi... |
215 216 217 218 219 220 221 222 223 224 225 226 227 |
* page_counter_set_low - set the amount of protected memory * @counter: counter * @nr_pages: value to set * * The caller must serialize invocations on the same counter. */ void page_counter_set_low(struct page_counter *counter, unsigned long nr_pages) { struct page_counter *c; counter->low = nr_pages; for (c = counter; c; c = c->parent) |
bf8d5d52f memcg: introduce ... |
228 |
propagate_protected_usage(c, atomic_long_read(&c->usage)); |
230671533 mm: memory.low hi... |
229 230 231 |
} /** |
3e32cb2e0 mm: memcontrol: l... |
232 233 |
* page_counter_memparse - memparse() for page counter limits * @buf: string to parse |
650c5e565 mm: page_counter:... |
234 |
* @max: string meaning maximum possible value |
3e32cb2e0 mm: memcontrol: l... |
235 236 237 238 239 |
* @nr_pages: returns the result in number of pages * * Returns -EINVAL, or 0 and @nr_pages on success. @nr_pages will be * limited to %PAGE_COUNTER_MAX. */ |
650c5e565 mm: page_counter:... |
240 241 |
int page_counter_memparse(const char *buf, const char *max, unsigned long *nr_pages) |
3e32cb2e0 mm: memcontrol: l... |
242 |
{ |
3e32cb2e0 mm: memcontrol: l... |
243 244 |
char *end; u64 bytes; |
650c5e565 mm: page_counter:... |
245 |
if (!strcmp(buf, max)) { |
3e32cb2e0 mm: memcontrol: l... |
246 247 248 249 250 251 252 253 254 255 256 257 |
*nr_pages = PAGE_COUNTER_MAX; return 0; } bytes = memparse(buf, &end); if (*end != '\0') return -EINVAL; *nr_pages = min(bytes / PAGE_SIZE, (u64)PAGE_COUNTER_MAX); return 0; } |