Blame view
mm/hugetlb_cgroup.c
10.4 KB
2bc64a204 mm/hugetlb: add n... |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 |
/* * * Copyright IBM Corporation, 2012 * Author Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com> * * This program is free software; you can redistribute it and/or modify it * under the terms of version 2.1 of the GNU Lesser General Public License * as published by the Free Software Foundation. * * This program is distributed in the hope that it would be useful, but * WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. * */ #include <linux/cgroup.h> |
71f87bee3 mm: hugetlb_cgrou... |
17 |
#include <linux/page_counter.h> |
2bc64a204 mm/hugetlb: add n... |
18 19 20 21 22 23 24 25 26 |
#include <linux/slab.h> #include <linux/hugetlb.h> #include <linux/hugetlb_cgroup.h> struct hugetlb_cgroup { struct cgroup_subsys_state css; /* * the counter to account for hugepages from hugetlb. */ |
71f87bee3 mm: hugetlb_cgrou... |
27 |
struct page_counter hugepage[HUGE_MAX_HSTATE]; |
2bc64a204 mm/hugetlb: add n... |
28 |
}; |
abb8206cb hugetlb/cgroup: a... |
29 30 31 |
#define MEMFILE_PRIVATE(x, val) (((x) << 16) | (val)) #define MEMFILE_IDX(val) (((val) >> 16) & 0xffff) #define MEMFILE_ATTR(val) ((val) & 0xffff) |
2bc64a204 mm/hugetlb: add n... |
32 33 34 35 36 |
static struct hugetlb_cgroup *root_h_cgroup __read_mostly; static inline struct hugetlb_cgroup *hugetlb_cgroup_from_css(struct cgroup_subsys_state *s) { |
a7c6d554a cgroup: add/updat... |
37 |
return s ? container_of(s, struct hugetlb_cgroup, css) : NULL; |
2bc64a204 mm/hugetlb: add n... |
38 39 40 |
} static inline |
2bc64a204 mm/hugetlb: add n... |
41 42 |
struct hugetlb_cgroup *hugetlb_cgroup_from_task(struct task_struct *task) { |
073219e99 cgroup: clean up ... |
43 |
return hugetlb_cgroup_from_css(task_css(task, hugetlb_cgrp_id)); |
2bc64a204 mm/hugetlb: add n... |
44 45 46 47 48 49 |
} static inline bool hugetlb_cgroup_is_root(struct hugetlb_cgroup *h_cg) { return (h_cg == root_h_cgroup); } |
3f7985183 hugetlb_cgroup: p... |
50 51 |
static inline struct hugetlb_cgroup * parent_hugetlb_cgroup(struct hugetlb_cgroup *h_cg) |
2bc64a204 mm/hugetlb: add n... |
52 |
{ |
5c9d535b8 cgroup: remove cs... |
53 |
return hugetlb_cgroup_from_css(h_cg->css.parent); |
2bc64a204 mm/hugetlb: add n... |
54 |
} |
3f7985183 hugetlb_cgroup: p... |
55 |
static inline bool hugetlb_cgroup_have_usage(struct hugetlb_cgroup *h_cg) |
2bc64a204 mm/hugetlb: add n... |
56 57 |
{ int idx; |
2bc64a204 mm/hugetlb: add n... |
58 59 |
for (idx = 0; idx < hugetlb_max_hstate; idx++) { |
71f87bee3 mm: hugetlb_cgrou... |
60 |
if (page_counter_read(&h_cg->hugepage[idx])) |
2bc64a204 mm/hugetlb: add n... |
61 62 63 64 |
return true; } return false; } |
eb95419b0 cgroup: pass arou... |
65 66 |
static struct cgroup_subsys_state * hugetlb_cgroup_css_alloc(struct cgroup_subsys_state *parent_css) |
2bc64a204 mm/hugetlb: add n... |
67 |
{ |
eb95419b0 cgroup: pass arou... |
68 69 |
struct hugetlb_cgroup *parent_h_cgroup = hugetlb_cgroup_from_css(parent_css); struct hugetlb_cgroup *h_cgroup; |
2bc64a204 mm/hugetlb: add n... |
70 |
int idx; |
2bc64a204 mm/hugetlb: add n... |
71 72 73 74 |
h_cgroup = kzalloc(sizeof(*h_cgroup), GFP_KERNEL); if (!h_cgroup) return ERR_PTR(-ENOMEM); |
eb95419b0 cgroup: pass arou... |
75 |
if (parent_h_cgroup) { |
2bc64a204 mm/hugetlb: add n... |
76 |
for (idx = 0; idx < HUGE_MAX_HSTATE; idx++) |
71f87bee3 mm: hugetlb_cgrou... |
77 78 |
page_counter_init(&h_cgroup->hugepage[idx], &parent_h_cgroup->hugepage[idx]); |
2bc64a204 mm/hugetlb: add n... |
79 80 81 |
} else { root_h_cgroup = h_cgroup; for (idx = 0; idx < HUGE_MAX_HSTATE; idx++) |
71f87bee3 mm: hugetlb_cgrou... |
82 |
page_counter_init(&h_cgroup->hugepage[idx], NULL); |
2bc64a204 mm/hugetlb: add n... |
83 84 85 |
} return &h_cgroup->css; } |
eb95419b0 cgroup: pass arou... |
86 |
static void hugetlb_cgroup_css_free(struct cgroup_subsys_state *css) |
2bc64a204 mm/hugetlb: add n... |
87 88 |
{ struct hugetlb_cgroup *h_cgroup; |
eb95419b0 cgroup: pass arou... |
89 |
h_cgroup = hugetlb_cgroup_from_css(css); |
2bc64a204 mm/hugetlb: add n... |
90 91 |
kfree(h_cgroup); } |
da1def559 hugetlb/cgroup: a... |
92 93 94 95 96 97 98 99 |
/* * Should be called with hugetlb_lock held. * Since we are holding hugetlb_lock, pages cannot get moved from * active list or uncharged from the cgroup, So no need to get * page reference and test for page active here. This function * cannot fail. */ |
3f7985183 hugetlb_cgroup: p... |
100 |
static void hugetlb_cgroup_move_parent(int idx, struct hugetlb_cgroup *h_cg, |
da1def559 hugetlb/cgroup: a... |
101 102 |
struct page *page) { |
71f87bee3 mm: hugetlb_cgrou... |
103 104 |
unsigned int nr_pages; struct page_counter *counter; |
da1def559 hugetlb/cgroup: a... |
105 |
struct hugetlb_cgroup *page_hcg; |
3f7985183 hugetlb_cgroup: p... |
106 |
struct hugetlb_cgroup *parent = parent_hugetlb_cgroup(h_cg); |
da1def559 hugetlb/cgroup: a... |
107 108 109 110 111 112 113 114 115 |
page_hcg = hugetlb_cgroup_from_page(page); /* * We can have pages in active list without any cgroup * ie, hugepage with less than 3 pages. We can safely * ignore those pages. */ if (!page_hcg || page_hcg != h_cg) goto out; |
71f87bee3 mm: hugetlb_cgrou... |
116 |
nr_pages = 1 << compound_order(page); |
da1def559 hugetlb/cgroup: a... |
117 118 119 |
if (!parent) { parent = root_h_cgroup; /* root has no limit */ |
71f87bee3 mm: hugetlb_cgrou... |
120 |
page_counter_charge(&parent->hugepage[idx], nr_pages); |
da1def559 hugetlb/cgroup: a... |
121 122 |
} counter = &h_cg->hugepage[idx]; |
71f87bee3 mm: hugetlb_cgrou... |
123 124 |
/* Take the pages off the local counter */ page_counter_cancel(counter, nr_pages); |
da1def559 hugetlb/cgroup: a... |
125 126 127 128 129 130 131 132 133 134 |
set_hugetlb_cgroup(page, parent); out: return; } /* * Force the hugetlb cgroup to empty the hugetlb resources by moving them to * the parent cgroup. */ |
eb95419b0 cgroup: pass arou... |
135 |
static void hugetlb_cgroup_css_offline(struct cgroup_subsys_state *css) |
2bc64a204 mm/hugetlb: add n... |
136 |
{ |
eb95419b0 cgroup: pass arou... |
137 |
struct hugetlb_cgroup *h_cg = hugetlb_cgroup_from_css(css); |
da1def559 hugetlb/cgroup: a... |
138 139 |
struct hstate *h; struct page *page; |
9d093cb10 hugetlb: do not f... |
140 |
int idx = 0; |
da1def559 hugetlb/cgroup: a... |
141 142 |
do { |
da1def559 hugetlb/cgroup: a... |
143 144 145 |
for_each_hstate(h) { spin_lock(&hugetlb_lock); list_for_each_entry(page, &h->hugepage_activelist, lru) |
3f7985183 hugetlb_cgroup: p... |
146 |
hugetlb_cgroup_move_parent(idx, h_cg, page); |
da1def559 hugetlb/cgroup: a... |
147 148 149 150 151 |
spin_unlock(&hugetlb_lock); idx++; } cond_resched(); |
3f7985183 hugetlb_cgroup: p... |
152 |
} while (hugetlb_cgroup_have_usage(h_cg)); |
2bc64a204 mm/hugetlb: add n... |
153 |
} |
6d76dcf40 hugetlb/cgroup: a... |
154 155 156 157 |
int hugetlb_cgroup_charge_cgroup(int idx, unsigned long nr_pages, struct hugetlb_cgroup **ptr) { int ret = 0; |
71f87bee3 mm: hugetlb_cgrou... |
158 |
struct page_counter *counter; |
6d76dcf40 hugetlb/cgroup: a... |
159 |
struct hugetlb_cgroup *h_cg = NULL; |
6d76dcf40 hugetlb/cgroup: a... |
160 161 162 163 164 165 166 167 168 169 170 171 |
if (hugetlb_cgroup_disabled()) goto done; /* * We don't charge any cgroup if the compound page have less * than 3 pages. */ if (huge_page_order(&hstates[idx]) < HUGETLB_CGROUP_MIN_ORDER) goto done; again: rcu_read_lock(); h_cg = hugetlb_cgroup_from_task(current); |
ec903c0c8 cgroup: rename cs... |
172 |
if (!css_tryget_online(&h_cg->css)) { |
6d76dcf40 hugetlb/cgroup: a... |
173 174 175 176 |
rcu_read_unlock(); goto again; } rcu_read_unlock(); |
71f87bee3 mm: hugetlb_cgrou... |
177 |
ret = page_counter_try_charge(&h_cg->hugepage[idx], nr_pages, &counter); |
6d76dcf40 hugetlb/cgroup: a... |
178 179 180 181 182 |
css_put(&h_cg->css); done: *ptr = h_cg; return ret; } |
94ae8ba71 hugetlb/cgroup: a... |
183 |
/* Should be called with hugetlb_lock held */ |
6d76dcf40 hugetlb/cgroup: a... |
184 185 186 187 188 189 |
void hugetlb_cgroup_commit_charge(int idx, unsigned long nr_pages, struct hugetlb_cgroup *h_cg, struct page *page) { if (hugetlb_cgroup_disabled() || !h_cg) return; |
6d76dcf40 hugetlb/cgroup: a... |
190 |
set_hugetlb_cgroup(page, h_cg); |
6d76dcf40 hugetlb/cgroup: a... |
191 192 193 194 195 196 197 198 199 200 |
return; } /* * Should be called with hugetlb_lock held */ void hugetlb_cgroup_uncharge_page(int idx, unsigned long nr_pages, struct page *page) { struct hugetlb_cgroup *h_cg; |
6d76dcf40 hugetlb/cgroup: a... |
201 202 203 |
if (hugetlb_cgroup_disabled()) return; |
7ea8574e5 hugetlb_cgroup: u... |
204 |
lockdep_assert_held(&hugetlb_lock); |
6d76dcf40 hugetlb/cgroup: a... |
205 206 207 208 |
h_cg = hugetlb_cgroup_from_page(page); if (unlikely(!h_cg)) return; set_hugetlb_cgroup(page, NULL); |
71f87bee3 mm: hugetlb_cgrou... |
209 |
page_counter_uncharge(&h_cg->hugepage[idx], nr_pages); |
6d76dcf40 hugetlb/cgroup: a... |
210 211 212 213 214 215 |
return; } void hugetlb_cgroup_uncharge_cgroup(int idx, unsigned long nr_pages, struct hugetlb_cgroup *h_cg) { |
6d76dcf40 hugetlb/cgroup: a... |
216 217 218 219 220 |
if (hugetlb_cgroup_disabled() || !h_cg) return; if (huge_page_order(&hstates[idx]) < HUGETLB_CGROUP_MIN_ORDER) return; |
71f87bee3 mm: hugetlb_cgrou... |
221 |
page_counter_uncharge(&h_cg->hugepage[idx], nr_pages); |
6d76dcf40 hugetlb/cgroup: a... |
222 223 |
return; } |
71f87bee3 mm: hugetlb_cgrou... |
224 225 226 227 228 229 |
enum { RES_USAGE, RES_LIMIT, RES_MAX_USAGE, RES_FAILCNT, }; |
716f479d2 hugetlb_cgroup: c... |
230 231 |
static u64 hugetlb_cgroup_read_u64(struct cgroup_subsys_state *css, struct cftype *cft) |
abb8206cb hugetlb/cgroup: a... |
232 |
{ |
71f87bee3 mm: hugetlb_cgrou... |
233 |
struct page_counter *counter; |
182446d08 cgroup: pass arou... |
234 |
struct hugetlb_cgroup *h_cg = hugetlb_cgroup_from_css(css); |
abb8206cb hugetlb/cgroup: a... |
235 |
|
71f87bee3 mm: hugetlb_cgrou... |
236 |
counter = &h_cg->hugepage[MEMFILE_IDX(cft->private)]; |
abb8206cb hugetlb/cgroup: a... |
237 |
|
71f87bee3 mm: hugetlb_cgrou... |
238 239 240 241 242 243 244 245 246 247 248 249 |
switch (MEMFILE_ATTR(cft->private)) { case RES_USAGE: return (u64)page_counter_read(counter) * PAGE_SIZE; case RES_LIMIT: return (u64)counter->limit * PAGE_SIZE; case RES_MAX_USAGE: return (u64)counter->watermark * PAGE_SIZE; case RES_FAILCNT: return counter->failcnt; default: BUG(); } |
abb8206cb hugetlb/cgroup: a... |
250 |
} |
71f87bee3 mm: hugetlb_cgrou... |
251 |
static DEFINE_MUTEX(hugetlb_limit_mutex); |
451af504d cgroup: replace c... |
252 253 |
static ssize_t hugetlb_cgroup_write(struct kernfs_open_file *of, char *buf, size_t nbytes, loff_t off) |
abb8206cb hugetlb/cgroup: a... |
254 |
{ |
71f87bee3 mm: hugetlb_cgrou... |
255 256 |
int ret, idx; unsigned long nr_pages; |
451af504d cgroup: replace c... |
257 |
struct hugetlb_cgroup *h_cg = hugetlb_cgroup_from_css(of_css(of)); |
abb8206cb hugetlb/cgroup: a... |
258 |
|
71f87bee3 mm: hugetlb_cgrou... |
259 260 |
if (hugetlb_cgroup_is_root(h_cg)) /* Can't set limit on root */ return -EINVAL; |
451af504d cgroup: replace c... |
261 |
buf = strstrip(buf); |
650c5e565 mm: page_counter:... |
262 |
ret = page_counter_memparse(buf, "-1", &nr_pages); |
71f87bee3 mm: hugetlb_cgrou... |
263 264 |
if (ret) return ret; |
451af504d cgroup: replace c... |
265 |
idx = MEMFILE_IDX(of_cft(of)->private); |
abb8206cb hugetlb/cgroup: a... |
266 |
|
71f87bee3 mm: hugetlb_cgrou... |
267 |
switch (MEMFILE_ATTR(of_cft(of)->private)) { |
abb8206cb hugetlb/cgroup: a... |
268 |
case RES_LIMIT: |
71f87bee3 mm: hugetlb_cgrou... |
269 270 271 |
mutex_lock(&hugetlb_limit_mutex); ret = page_counter_limit(&h_cg->hugepage[idx], nr_pages); mutex_unlock(&hugetlb_limit_mutex); |
abb8206cb hugetlb/cgroup: a... |
272 273 274 275 276 |
break; default: ret = -EINVAL; break; } |
451af504d cgroup: replace c... |
277 |
return ret ?: nbytes; |
abb8206cb hugetlb/cgroup: a... |
278 |
} |
6770c64e5 cgroup: replace c... |
279 280 |
static ssize_t hugetlb_cgroup_reset(struct kernfs_open_file *of, char *buf, size_t nbytes, loff_t off) |
abb8206cb hugetlb/cgroup: a... |
281 |
{ |
71f87bee3 mm: hugetlb_cgrou... |
282 283 |
int ret = 0; struct page_counter *counter; |
6770c64e5 cgroup: replace c... |
284 |
struct hugetlb_cgroup *h_cg = hugetlb_cgroup_from_css(of_css(of)); |
abb8206cb hugetlb/cgroup: a... |
285 |
|
71f87bee3 mm: hugetlb_cgrou... |
286 |
counter = &h_cg->hugepage[MEMFILE_IDX(of_cft(of)->private)]; |
abb8206cb hugetlb/cgroup: a... |
287 |
|
71f87bee3 mm: hugetlb_cgrou... |
288 |
switch (MEMFILE_ATTR(of_cft(of)->private)) { |
abb8206cb hugetlb/cgroup: a... |
289 |
case RES_MAX_USAGE: |
71f87bee3 mm: hugetlb_cgrou... |
290 |
page_counter_reset_watermark(counter); |
abb8206cb hugetlb/cgroup: a... |
291 292 |
break; case RES_FAILCNT: |
71f87bee3 mm: hugetlb_cgrou... |
293 |
counter->failcnt = 0; |
abb8206cb hugetlb/cgroup: a... |
294 295 296 297 298 |
break; default: ret = -EINVAL; break; } |
6770c64e5 cgroup: replace c... |
299 |
return ret ?: nbytes; |
abb8206cb hugetlb/cgroup: a... |
300 301 302 303 304 305 306 307 308 309 310 311 |
} static char *mem_fmt(char *buf, int size, unsigned long hsize) { if (hsize >= (1UL << 30)) snprintf(buf, size, "%luGB", hsize >> 30); else if (hsize >= (1UL << 20)) snprintf(buf, size, "%luMB", hsize >> 20); else snprintf(buf, size, "%luKB", hsize >> 10); return buf; } |
7179e7bf4 mm/hugetlb: creat... |
312 |
static void __init __hugetlb_cgroup_file_init(int idx) |
abb8206cb hugetlb/cgroup: a... |
313 314 315 316 317 318 319 320 321 322 323 324 |
{ char buf[32]; struct cftype *cft; struct hstate *h = &hstates[idx]; /* format the size */ mem_fmt(buf, 32, huge_page_size(h)); /* Add the limit file */ cft = &h->cgroup_files[0]; snprintf(cft->name, MAX_CFTYPE_NAME, "%s.limit_in_bytes", buf); cft->private = MEMFILE_PRIVATE(idx, RES_LIMIT); |
716f479d2 hugetlb_cgroup: c... |
325 |
cft->read_u64 = hugetlb_cgroup_read_u64; |
451af504d cgroup: replace c... |
326 |
cft->write = hugetlb_cgroup_write; |
abb8206cb hugetlb/cgroup: a... |
327 328 329 330 331 |
/* Add the usage file */ cft = &h->cgroup_files[1]; snprintf(cft->name, MAX_CFTYPE_NAME, "%s.usage_in_bytes", buf); cft->private = MEMFILE_PRIVATE(idx, RES_USAGE); |
716f479d2 hugetlb_cgroup: c... |
332 |
cft->read_u64 = hugetlb_cgroup_read_u64; |
abb8206cb hugetlb/cgroup: a... |
333 334 335 336 337 |
/* Add the MAX usage file */ cft = &h->cgroup_files[2]; snprintf(cft->name, MAX_CFTYPE_NAME, "%s.max_usage_in_bytes", buf); cft->private = MEMFILE_PRIVATE(idx, RES_MAX_USAGE); |
6770c64e5 cgroup: replace c... |
338 |
cft->write = hugetlb_cgroup_reset; |
716f479d2 hugetlb_cgroup: c... |
339 |
cft->read_u64 = hugetlb_cgroup_read_u64; |
abb8206cb hugetlb/cgroup: a... |
340 341 342 343 344 |
/* Add the failcntfile */ cft = &h->cgroup_files[3]; snprintf(cft->name, MAX_CFTYPE_NAME, "%s.failcnt", buf); cft->private = MEMFILE_PRIVATE(idx, RES_FAILCNT); |
6770c64e5 cgroup: replace c... |
345 |
cft->write = hugetlb_cgroup_reset; |
716f479d2 hugetlb_cgroup: c... |
346 |
cft->read_u64 = hugetlb_cgroup_read_u64; |
abb8206cb hugetlb/cgroup: a... |
347 348 349 350 |
/* NULL terminate the last cft */ cft = &h->cgroup_files[4]; memset(cft, 0, sizeof(*cft)); |
2cf669a58 cgroup: replace c... |
351 352 |
WARN_ON(cgroup_add_legacy_cftypes(&hugetlb_cgrp_subsys, h->cgroup_files)); |
7179e7bf4 mm/hugetlb: creat... |
353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 |
} void __init hugetlb_cgroup_file_init(void) { struct hstate *h; for_each_hstate(h) { /* * Add cgroup control files only if the huge page consists * of more than two normal pages. This is because we use * page[2].lru.next for storing cgroup details. */ if (huge_page_order(h) >= HUGETLB_CGROUP_MIN_ORDER) __hugetlb_cgroup_file_init(hstate_index(h)); } |
abb8206cb hugetlb/cgroup: a... |
368 |
} |
75754681f hugetlb/cgroup: r... |
369 370 371 372 |
/* * hugetlb_lock will make sure a parallel cgroup rmdir won't happen * when we migrate hugepages */ |
8e6ac7fab hugetlb/cgroup: m... |
373 374 375 |
void hugetlb_cgroup_migrate(struct page *oldhpage, struct page *newhpage) { struct hugetlb_cgroup *h_cg; |
94ae8ba71 hugetlb/cgroup: a... |
376 |
struct hstate *h = page_hstate(oldhpage); |
8e6ac7fab hugetlb/cgroup: m... |
377 378 379 |
if (hugetlb_cgroup_disabled()) return; |
309381fea mm: dump page whe... |
380 |
VM_BUG_ON_PAGE(!PageHuge(oldhpage), oldhpage); |
8e6ac7fab hugetlb/cgroup: m... |
381 382 383 |
spin_lock(&hugetlb_lock); h_cg = hugetlb_cgroup_from_page(oldhpage); set_hugetlb_cgroup(oldhpage, NULL); |
8e6ac7fab hugetlb/cgroup: m... |
384 385 386 |
/* move the h_cg details to new cgroup */ set_hugetlb_cgroup(newhpage, h_cg); |
94ae8ba71 hugetlb/cgroup: a... |
387 |
list_move(&newhpage->lru, &h->hugepage_activelist); |
8e6ac7fab hugetlb/cgroup: m... |
388 |
spin_unlock(&hugetlb_lock); |
8e6ac7fab hugetlb/cgroup: m... |
389 390 |
return; } |
073219e99 cgroup: clean up ... |
391 |
struct cgroup_subsys hugetlb_cgrp_subsys = { |
92fb97487 cgroup: rename ->... |
392 393 394 |
.css_alloc = hugetlb_cgroup_css_alloc, .css_offline = hugetlb_cgroup_css_offline, .css_free = hugetlb_cgroup_css_free, |
2bc64a204 mm/hugetlb: add n... |
395 |
}; |