Blame view
mm/page_counter.c
4.77 KB
3e32cb2e0 mm: memcontrol: l... |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 |
/* * Lockless hierarchical page accounting & limiting * * Copyright (C) 2014 Red Hat, Inc., Johannes Weiner */ #include <linux/page_counter.h> #include <linux/atomic.h> #include <linux/kernel.h> #include <linux/string.h> #include <linux/sched.h> #include <linux/bug.h> #include <asm/page.h> /** * page_counter_cancel - take pages out of the local counter * @counter: counter * @nr_pages: number of pages to cancel |
3e32cb2e0 mm: memcontrol: l... |
19 |
*/ |
64f219938 mm: memcontrol: r... |
20 |
void page_counter_cancel(struct page_counter *counter, unsigned long nr_pages) |
3e32cb2e0 mm: memcontrol: l... |
21 22 23 24 |
{ long new; new = atomic_long_sub_return(nr_pages, &counter->count); |
3e32cb2e0 mm: memcontrol: l... |
25 26 |
/* More uncharges than charges? */ WARN_ON_ONCE(new < 0); |
3e32cb2e0 mm: memcontrol: l... |
27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 |
} /** * page_counter_charge - hierarchically charge pages * @counter: counter * @nr_pages: number of pages to charge * * NOTE: This does not consider any configured counter limits. */ void page_counter_charge(struct page_counter *counter, unsigned long nr_pages) { struct page_counter *c; for (c = counter; c; c = c->parent) { long new; new = atomic_long_add_return(nr_pages, &c->count); /* * This is indeed racy, but we can live with some * inaccuracy in the watermark. */ if (new > c->watermark) c->watermark = new; } } /** * page_counter_try_charge - try to hierarchically charge pages * @counter: counter * @nr_pages: number of pages to charge * @fail: points first counter to hit its limit, if any * |
6071ca520 mm: page_counter:... |
59 60 |
* Returns %true on success, or %false and @fail if the counter or one * of its ancestors has hit its configured limit. |
3e32cb2e0 mm: memcontrol: l... |
61 |
*/ |
6071ca520 mm: page_counter:... |
62 63 64 |
bool page_counter_try_charge(struct page_counter *counter, unsigned long nr_pages, struct page_counter **fail) |
3e32cb2e0 mm: memcontrol: l... |
65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 |
{ struct page_counter *c; for (c = counter; c; c = c->parent) { long new; /* * Charge speculatively to avoid an expensive CAS. If * a bigger charge fails, it might falsely lock out a * racing smaller charge and send it into reclaim * early, but the error is limited to the difference * between the two sizes, which is less than 2M/4M in * case of a THP locking out a regular page charge. * * The atomic_long_add_return() implies a full memory * barrier between incrementing the count and reading * the limit. When racing with page_counter_limit(), * we either see the new limit or the setter sees the * counter has changed and retries. */ new = atomic_long_add_return(nr_pages, &c->count); if (new > c->limit) { atomic_long_sub(nr_pages, &c->count); /* * This is racy, but we can live with some * inaccuracy in the failcnt. */ c->failcnt++; *fail = c; goto failed; } /* * Just like with failcnt, we can live with some * inaccuracy in the watermark. */ if (new > c->watermark) c->watermark = new; } |
6071ca520 mm: page_counter:... |
102 |
return true; |
3e32cb2e0 mm: memcontrol: l... |
103 104 105 106 |
failed: for (c = counter; c != *fail; c = c->parent) page_counter_cancel(c, nr_pages); |
6071ca520 mm: page_counter:... |
107 |
return false; |
3e32cb2e0 mm: memcontrol: l... |
108 109 110 111 112 113 |
} /** * page_counter_uncharge - hierarchically uncharge pages * @counter: counter * @nr_pages: number of pages to uncharge |
3e32cb2e0 mm: memcontrol: l... |
114 |
*/ |
64f219938 mm: memcontrol: r... |
115 |
void page_counter_uncharge(struct page_counter *counter, unsigned long nr_pages) |
3e32cb2e0 mm: memcontrol: l... |
116 117 |
{ struct page_counter *c; |
3e32cb2e0 mm: memcontrol: l... |
118 |
|
64f219938 mm: memcontrol: r... |
119 120 |
for (c = counter; c; c = c->parent) page_counter_cancel(c, nr_pages); |
3e32cb2e0 mm: memcontrol: l... |
121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 |
} /** * page_counter_limit - limit the number of pages allowed * @counter: counter * @limit: limit to set * * Returns 0 on success, -EBUSY if the current number of pages on the * counter already exceeds the specified limit. * * The caller must serialize invocations on the same counter. */ int page_counter_limit(struct page_counter *counter, unsigned long limit) { for (;;) { unsigned long old; long count; /* * Update the limit while making sure that it's not * below the concurrently-changing counter value. * * The xchg implies two full memory barriers before * and after, so the read-swap-read is ordered and * ensures coherency with page_counter_try_charge(): * that function modifies the count before checking * the limit, so if it sees the old limit, we see the * modified counter and retry. */ count = atomic_long_read(&counter->count); if (count > limit) return -EBUSY; old = xchg(&counter->limit, limit); if (atomic_long_read(&counter->count) <= count) return 0; counter->limit = old; cond_resched(); } } /** * page_counter_memparse - memparse() for page counter limits * @buf: string to parse |
650c5e565 mm: page_counter:... |
168 |
* @max: string meaning maximum possible value |
3e32cb2e0 mm: memcontrol: l... |
169 170 171 172 173 |
* @nr_pages: returns the result in number of pages * * Returns -EINVAL, or 0 and @nr_pages on success. @nr_pages will be * limited to %PAGE_COUNTER_MAX. */ |
650c5e565 mm: page_counter:... |
174 175 |
int page_counter_memparse(const char *buf, const char *max, unsigned long *nr_pages) |
3e32cb2e0 mm: memcontrol: l... |
176 |
{ |
3e32cb2e0 mm: memcontrol: l... |
177 178 |
char *end; u64 bytes; |
650c5e565 mm: page_counter:... |
179 |
if (!strcmp(buf, max)) { |
3e32cb2e0 mm: memcontrol: l... |
180 181 182 183 184 185 186 187 188 189 190 191 |
*nr_pages = PAGE_COUNTER_MAX; return 0; } bytes = memparse(buf, &end); if (*end != '\0') return -EINVAL; *nr_pages = min(bytes / PAGE_SIZE, (u64)PAGE_COUNTER_MAX); return 0; } |