Blame view
mm/page_counter.c
6.65 KB
b24413180 License cleanup: ... |
1 |
// SPDX-License-Identifier: GPL-2.0 |
3e32cb2e0 mm: memcontrol: l... |
2 3 4 5 6 7 8 9 10 11 12 13 14 |
/* * Lockless hierarchical page accounting & limiting * * Copyright (C) 2014 Red Hat, Inc., Johannes Weiner */ #include <linux/page_counter.h> #include <linux/atomic.h> #include <linux/kernel.h> #include <linux/string.h> #include <linux/sched.h> #include <linux/bug.h> #include <asm/page.h> |
bf8d5d52f memcg: introduce ... |
15 16 |
static void propagate_protected_usage(struct page_counter *c, unsigned long usage) |
230671533 mm: memory.low hi... |
17 |
{ |
bf8d5d52f memcg: introduce ... |
18 |
unsigned long protected, old_protected; |
c3d532008 mm, memcg: preven... |
19 |
unsigned long low, min; |
230671533 mm: memory.low hi... |
20 21 22 23 |
long delta; if (!c->parent) return; |
c3d532008 mm, memcg: preven... |
24 25 26 |
min = READ_ONCE(c->min); if (min || atomic_long_read(&c->min_usage)) { protected = min(usage, min); |
bf8d5d52f memcg: introduce ... |
27 28 29 30 31 |
old_protected = atomic_long_xchg(&c->min_usage, protected); delta = protected - old_protected; if (delta) atomic_long_add(delta, &c->parent->children_min_usage); } |
230671533 mm: memory.low hi... |
32 |
|
f86b810c2 mm, memcg: preven... |
33 34 35 |
low = READ_ONCE(c->low); if (low || atomic_long_read(&c->low_usage)) { protected = min(usage, low); |
bf8d5d52f memcg: introduce ... |
36 37 38 39 40 |
old_protected = atomic_long_xchg(&c->low_usage, protected); delta = protected - old_protected; if (delta) atomic_long_add(delta, &c->parent->children_low_usage); } |
230671533 mm: memory.low hi... |
41 |
} |
3e32cb2e0 mm: memcontrol: l... |
42 43 44 45 |
/** * page_counter_cancel - take pages out of the local counter * @counter: counter * @nr_pages: number of pages to cancel |
3e32cb2e0 mm: memcontrol: l... |
46 |
*/ |
64f219938 mm: memcontrol: r... |
47 |
void page_counter_cancel(struct page_counter *counter, unsigned long nr_pages) |
3e32cb2e0 mm: memcontrol: l... |
48 49 |
{ long new; |
bbec2e151 mm: rename page_c... |
50 |
new = atomic_long_sub_return(nr_pages, &counter->usage); |
bf8d5d52f memcg: introduce ... |
51 |
propagate_protected_usage(counter, new); |
3e32cb2e0 mm: memcontrol: l... |
52 53 |
/* More uncharges than charges? */ WARN_ON_ONCE(new < 0); |
3e32cb2e0 mm: memcontrol: l... |
54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 |
} /** * page_counter_charge - hierarchically charge pages * @counter: counter * @nr_pages: number of pages to charge * * NOTE: This does not consider any configured counter limits. */ void page_counter_charge(struct page_counter *counter, unsigned long nr_pages) { struct page_counter *c; for (c = counter; c; c = c->parent) { long new; |
bbec2e151 mm: rename page_c... |
69 |
new = atomic_long_add_return(nr_pages, &c->usage); |
a6f23d14e mm/page_counter.c... |
70 |
propagate_protected_usage(c, new); |
3e32cb2e0 mm: memcontrol: l... |
71 72 73 74 |
/* * This is indeed racy, but we can live with some * inaccuracy in the watermark. */ |
6e4bd50f3 mm/page_counter: ... |
75 76 |
if (new > READ_ONCE(c->watermark)) WRITE_ONCE(c->watermark, new); |
3e32cb2e0 mm: memcontrol: l... |
77 78 79 80 81 82 83 84 85 |
} } /** * page_counter_try_charge - try to hierarchically charge pages * @counter: counter * @nr_pages: number of pages to charge * @fail: points first counter to hit its limit, if any * |
6071ca520 mm: page_counter:... |
86 87 |
* Returns %true on success, or %false and @fail if the counter or one * of its ancestors has hit its configured limit. |
3e32cb2e0 mm: memcontrol: l... |
88 |
*/ |
6071ca520 mm: page_counter:... |
89 90 91 |
bool page_counter_try_charge(struct page_counter *counter, unsigned long nr_pages, struct page_counter **fail) |
3e32cb2e0 mm: memcontrol: l... |
92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 |
{ struct page_counter *c; for (c = counter; c; c = c->parent) { long new; /* * Charge speculatively to avoid an expensive CAS. If * a bigger charge fails, it might falsely lock out a * racing smaller charge and send it into reclaim * early, but the error is limited to the difference * between the two sizes, which is less than 2M/4M in * case of a THP locking out a regular page charge. * * The atomic_long_add_return() implies a full memory * barrier between incrementing the count and reading |
d437024e6 mm/page_counter: ... |
107 |
* the limit. When racing with page_counter_set_max(), |
3e32cb2e0 mm: memcontrol: l... |
108 109 110 |
* we either see the new limit or the setter sees the * counter has changed and retries. */ |
bbec2e151 mm: rename page_c... |
111 112 113 |
new = atomic_long_add_return(nr_pages, &c->usage); if (new > c->max) { atomic_long_sub(nr_pages, &c->usage); |
a6f23d14e mm/page_counter.c... |
114 |
propagate_protected_usage(c, new); |
3e32cb2e0 mm: memcontrol: l... |
115 116 |
/* * This is racy, but we can live with some |
6e4bd50f3 mm/page_counter: ... |
117 118 |
* inaccuracy in the failcnt which is only used * to report stats. |
3e32cb2e0 mm: memcontrol: l... |
119 |
*/ |
6e4bd50f3 mm/page_counter: ... |
120 |
data_race(c->failcnt++); |
3e32cb2e0 mm: memcontrol: l... |
121 122 123 |
*fail = c; goto failed; } |
a6f23d14e mm/page_counter.c... |
124 |
propagate_protected_usage(c, new); |
3e32cb2e0 mm: memcontrol: l... |
125 126 127 128 |
/* * Just like with failcnt, we can live with some * inaccuracy in the watermark. */ |
6e4bd50f3 mm/page_counter: ... |
129 130 |
if (new > READ_ONCE(c->watermark)) WRITE_ONCE(c->watermark, new); |
3e32cb2e0 mm: memcontrol: l... |
131 |
} |
6071ca520 mm: page_counter:... |
132 |
return true; |
3e32cb2e0 mm: memcontrol: l... |
133 134 135 136 |
failed: for (c = counter; c != *fail; c = c->parent) page_counter_cancel(c, nr_pages); |
6071ca520 mm: page_counter:... |
137 |
return false; |
3e32cb2e0 mm: memcontrol: l... |
138 139 140 141 142 143 |
} /** * page_counter_uncharge - hierarchically uncharge pages * @counter: counter * @nr_pages: number of pages to uncharge |
3e32cb2e0 mm: memcontrol: l... |
144 |
*/ |
64f219938 mm: memcontrol: r... |
145 |
void page_counter_uncharge(struct page_counter *counter, unsigned long nr_pages) |
3e32cb2e0 mm: memcontrol: l... |
146 147 |
{ struct page_counter *c; |
3e32cb2e0 mm: memcontrol: l... |
148 |
|
64f219938 mm: memcontrol: r... |
149 150 |
for (c = counter; c; c = c->parent) page_counter_cancel(c, nr_pages); |
3e32cb2e0 mm: memcontrol: l... |
151 152 153 |
} /** |
bbec2e151 mm: rename page_c... |
154 |
* page_counter_set_max - set the maximum number of pages allowed |
3e32cb2e0 mm: memcontrol: l... |
155 |
* @counter: counter |
bbec2e151 mm: rename page_c... |
156 |
* @nr_pages: limit to set |
3e32cb2e0 mm: memcontrol: l... |
157 158 159 160 161 162 |
* * Returns 0 on success, -EBUSY if the current number of pages on the * counter already exceeds the specified limit. * * The caller must serialize invocations on the same counter. */ |
bbec2e151 mm: rename page_c... |
163 |
int page_counter_set_max(struct page_counter *counter, unsigned long nr_pages) |
3e32cb2e0 mm: memcontrol: l... |
164 165 166 |
{ for (;;) { unsigned long old; |
bbec2e151 mm: rename page_c... |
167 |
long usage; |
3e32cb2e0 mm: memcontrol: l... |
168 169 170 171 172 173 174 175 176 177 178 179 |
/* * Update the limit while making sure that it's not * below the concurrently-changing counter value. * * The xchg implies two full memory barriers before * and after, so the read-swap-read is ordered and * ensures coherency with page_counter_try_charge(): * that function modifies the count before checking * the limit, so if it sees the old limit, we see the * modified counter and retry. */ |
bbec2e151 mm: rename page_c... |
180 |
usage = atomic_long_read(&counter->usage); |
3e32cb2e0 mm: memcontrol: l... |
181 |
|
bbec2e151 mm: rename page_c... |
182 |
if (usage > nr_pages) |
3e32cb2e0 mm: memcontrol: l... |
183 |
return -EBUSY; |
bbec2e151 mm: rename page_c... |
184 |
old = xchg(&counter->max, nr_pages); |
3e32cb2e0 mm: memcontrol: l... |
185 |
|
bbec2e151 mm: rename page_c... |
186 |
if (atomic_long_read(&counter->usage) <= usage) |
3e32cb2e0 mm: memcontrol: l... |
187 |
return 0; |
bbec2e151 mm: rename page_c... |
188 |
counter->max = old; |
3e32cb2e0 mm: memcontrol: l... |
189 190 191 192 193 |
cond_resched(); } } /** |
bf8d5d52f memcg: introduce ... |
194 195 196 197 198 199 200 201 202 |
* page_counter_set_min - set the amount of protected memory * @counter: counter * @nr_pages: value to set * * The caller must serialize invocations on the same counter. */ void page_counter_set_min(struct page_counter *counter, unsigned long nr_pages) { struct page_counter *c; |
c3d532008 mm, memcg: preven... |
203 |
WRITE_ONCE(counter->min, nr_pages); |
bf8d5d52f memcg: introduce ... |
204 205 206 207 208 209 |
for (c = counter; c; c = c->parent) propagate_protected_usage(c, atomic_long_read(&c->usage)); } /** |
230671533 mm: memory.low hi... |
210 211 212 213 214 215 216 217 218 |
* page_counter_set_low - set the amount of protected memory * @counter: counter * @nr_pages: value to set * * The caller must serialize invocations on the same counter. */ void page_counter_set_low(struct page_counter *counter, unsigned long nr_pages) { struct page_counter *c; |
f86b810c2 mm, memcg: preven... |
219 |
WRITE_ONCE(counter->low, nr_pages); |
230671533 mm: memory.low hi... |
220 221 |
for (c = counter; c; c = c->parent) |
bf8d5d52f memcg: introduce ... |
222 |
propagate_protected_usage(c, atomic_long_read(&c->usage)); |
230671533 mm: memory.low hi... |
223 224 225 |
} /** |
3e32cb2e0 mm: memcontrol: l... |
226 227 |
* page_counter_memparse - memparse() for page counter limits * @buf: string to parse |
650c5e565 mm: page_counter:... |
228 |
* @max: string meaning maximum possible value |
3e32cb2e0 mm: memcontrol: l... |
229 230 231 232 233 |
* @nr_pages: returns the result in number of pages * * Returns -EINVAL, or 0 and @nr_pages on success. @nr_pages will be * limited to %PAGE_COUNTER_MAX. */ |
650c5e565 mm: page_counter:... |
234 235 |
int page_counter_memparse(const char *buf, const char *max, unsigned long *nr_pages) |
3e32cb2e0 mm: memcontrol: l... |
236 |
{ |
3e32cb2e0 mm: memcontrol: l... |
237 238 |
char *end; u64 bytes; |
650c5e565 mm: page_counter:... |
239 |
if (!strcmp(buf, max)) { |
3e32cb2e0 mm: memcontrol: l... |
240 241 242 243 244 245 246 247 248 249 250 251 |
*nr_pages = PAGE_COUNTER_MAX; return 0; } bytes = memparse(buf, &end); if (*end != '\0') return -EINVAL; *nr_pages = min(bytes / PAGE_SIZE, (u64)PAGE_COUNTER_MAX); return 0; } |